From 18d666788658c4ba6b5691babc0f019519999390 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 20 Jan 2026 10:19:20 -0800 Subject: [PATCH 01/44] fix: module updates --- .../k8s/bootstrap/acme-cloudflare-ssl/main.tf | 127 --- modules/k8s/bootstrap/cert-manager/main.tf | 110 ++- modules/k8s/bootstrap/coder-server/main.tf | 550 +++++++++---- modules/k8s/bootstrap/ebs-controller/main.tf | 8 +- modules/k8s/bootstrap/efs-controller/main.tf | 106 +++ modules/k8s/bootstrap/external-dns/main.tf | 153 ++++ modules/k8s/bootstrap/external-dns/policy.tf | 20 + .../k8s/bootstrap/external-secrets/main.tf | 123 +++ .../k8s/bootstrap/external-secrets/policy.tf | 42 + modules/k8s/bootstrap/karpenter/main.tf | 71 +- modules/k8s/bootstrap/lb-controller/main.tf | 31 +- .../bootstrap/litellm-generate-key/main.tf | 296 ------- .../litellm-generate-key/scripts/rotate.sh | 27 - modules/k8s/bootstrap/litellm/README.md | 156 ++++ modules/k8s/bootstrap/litellm/main.tf | 747 +++++++++--------- modules/k8s/bootstrap/litellm/policy.tf | 16 - .../k8s/bootstrap/litellm/scripts/config.yaml | 220 ------ .../scripts/strip_header_middleware.py | 33 - modules/k8s/bootstrap/metrics-server/main.tf | 6 +- modules/k8s/objects/ec2nodeclass/main.tf | 6 + 20 files changed, 1540 insertions(+), 1308 deletions(-) delete mode 100644 modules/k8s/bootstrap/acme-cloudflare-ssl/main.tf create mode 100644 modules/k8s/bootstrap/efs-controller/main.tf create mode 100644 modules/k8s/bootstrap/external-dns/main.tf create mode 100644 modules/k8s/bootstrap/external-dns/policy.tf create mode 100644 modules/k8s/bootstrap/external-secrets/main.tf create mode 100644 modules/k8s/bootstrap/external-secrets/policy.tf delete mode 100644 modules/k8s/bootstrap/litellm-generate-key/main.tf delete mode 100644 modules/k8s/bootstrap/litellm-generate-key/scripts/rotate.sh create mode 100644 modules/k8s/bootstrap/litellm/README.md delete mode 100644 modules/k8s/bootstrap/litellm/policy.tf delete mode 100644 modules/k8s/bootstrap/litellm/scripts/config.yaml delete mode 100644 modules/k8s/bootstrap/litellm/scripts/strip_header_middleware.py diff --git a/modules/k8s/bootstrap/acme-cloudflare-ssl/main.tf b/modules/k8s/bootstrap/acme-cloudflare-ssl/main.tf deleted file mode 100644 index d74b0fb..0000000 --- a/modules/k8s/bootstrap/acme-cloudflare-ssl/main.tf +++ /dev/null @@ -1,127 +0,0 @@ -terraform { - required_providers { - kubernetes = { - source = "hashicorp/kubernetes" - } - coderd = { - source = "coder/coderd" - } - acme = { - source = "vancluever/acme" - } - tls = { - source = "hashicorp/tls" - } - null = { - source = "hashicorp/null" - } - } -} - -variable "dns_names" { - type = list(string) -} - -variable "common_name" { - type = string -} - -variable "acme_registration_email" { - type = string -} - -variable "acme_days_until_renewal" { - type = number - default = 30 -} - -variable "acme_revoke_certificate" { - type = bool - default = true -} - -variable "cloudflare_api_token" { - type = string - sensitive = true -} - -variable "kubernetes_secret_name" { - type = string -} - -variable "kubernetes_namespace" { - type = string - default = "default" -} - -resource "null_resource" "registration-email" { - triggers = { - registration_email = var.acme_registration_email - } -} - -resource "acme_registration" "this" { - email_address = var.acme_registration_email - - lifecycle { - replace_triggered_by = [null_resource.registration-email] - } -} - -resource "tls_private_key" "this" { - algorithm = "RSA" - rsa_bits = 4096 -} - -resource "tls_cert_request" "this" { - private_key_pem = tls_private_key.this.private_key_pem - dns_names = var.dns_names - subject { - common_name = var.common_name - } -} - -resource "acme_certificate" "this" { - account_key_pem = acme_registration.this.account_key_pem - certificate_request_pem = tls_cert_request.this.cert_request_pem - min_days_remaining = var.acme_days_until_renewal - revoke_certificate_on_destroy = var.acme_revoke_certificate - revoke_certificate_reason = "cessation-of-operation" - dns_challenge { - provider = "cloudflare" - config = { - CF_DNS_API_TOKEN = var.cloudflare_api_token - } - } -} - -locals { - full_chain = "${acme_certificate.this.certificate_pem}${acme_certificate.this.issuer_pem}" -} - -resource "kubernetes_secret" "coder-proxy-tls" { - metadata { - name = var.kubernetes_secret_name - namespace = var.kubernetes_namespace - } - data = { - "tls.key" = tls_private_key.this.private_key_pem - "tls.crt" = local.full_chain - } - type = "kubernetes_namespace.io/tls" - -} - -output "private_key_pem" { - value = tls_private_key.this.private_key_pem - sensitive = true -} - -output "full_chain_certificate_pem" { - value = local.full_chain - sensitive = true -} - -output "kubernetes_secret_name" { - value = kubernetes_secret.coder-proxy-tls.metadata[0].name -} \ No newline at end of file diff --git a/modules/k8s/bootstrap/cert-manager/main.tf b/modules/k8s/bootstrap/cert-manager/main.tf index 919a2cf..d622b2e 100644 --- a/modules/k8s/bootstrap/cert-manager/main.tf +++ b/modules/k8s/bootstrap/cert-manager/main.tf @@ -13,7 +13,6 @@ terraform { } } - variable "cluster_name" { type = string } @@ -62,28 +61,17 @@ variable "helm_version" { } ## -# ACME Certificate Inputs +# Create Default Resources? ## -variable "cloudflare_token_secret_name" { - type = string - default = "cloudflare-token" -} - -variable "cloudflare_token_secret_key" { - type = string - default = "token.key" -} - -variable "cloudflare_token_secret" { - type = string - sensitive = true +variable "create_default_cluster_issuer" { + type = bool + default = true } -variable "cloudflare_token_secret_email" { - type = string - sensitive = true -} +## +# ACME Certificate Inputs +## variable "issuer_private_key_secret_name" { type = string @@ -161,6 +149,25 @@ resource "helm_release" "cert-manager" { })] } +## +# Use Route53 for the DNS01 Challenge Provider +## + +variable "use_route53" { + type = bool + default = false +} + +variable "route53_region" { + type = string + default = "us-east-2" +} + +variable "route53_sa_role" { + type = string + default = "" +} + resource "kubernetes_service_account" "route53" { metadata { name = "cert-manager-acme-dns01-route53" @@ -201,6 +208,37 @@ resource "kubernetes_role_binding" "route53" { } } +## +# Use CloudFlare for the DNS01 Challenge Provider +## + +variable "use_cloudflare" { + type = bool + default = true +} + +variable "cloudflare_token_secret_name" { + type = string + default = "cloudflare-token" +} + +variable "cloudflare_token_secret_key" { + type = string + default = "token.key" +} + +variable "cloudflare_token_secret" { + type = string + sensitive = true + default = "" +} + +variable "cloudflare_token_secret_email" { + type = string + sensitive = true + default = "" +} + resource "kubernetes_secret" "cloudflare" { metadata { name = var.cloudflare_token_secret_name @@ -211,13 +249,45 @@ resource "kubernetes_secret" "cloudflare" { } } -resource "kubernetes_manifest" "default-cluster-issuer" { +# ---------------- + +locals { + dns01_cf = { + cloudflare = { + apiTokenSecretRef = { + key = var.cloudflare_token_secret_key + name = kubernetes_secret.cloudflare.metadata[0].name + } + email = var.cloudflare_token_secret_email + } + } + dns01_r53 = { + route53 = { + region = var.route53_region + auth = { + kubernetes = { + serviceAccountRef = { + name = kubernetes_service_account.route53.metadata[0].name + } + } + } + } + } +} + +## +# Setup the default Cluster Issuer +## + +resource "kubernetes_manifest" "default-issuer" { depends_on = [ helm_release.cert-manager, kubernetes_secret.cloudflare ] + count = var.create_default_cluster_issuer ? 1 : 0 + field_manager { force_conflicts = true } diff --git a/modules/k8s/bootstrap/coder-server/main.tf b/modules/k8s/bootstrap/coder-server/main.tf index a3e80ad..bbb4b14 100644 --- a/modules/k8s/bootstrap/coder-server/main.tf +++ b/modules/k8s/bootstrap/coder-server/main.tf @@ -51,28 +51,12 @@ variable "role_name" { # TLS/SSL Inputs ## -variable "acme_registration_email" { - type = string - default = "" -} - -variable "acme_days_until_renewal" { - type = number - default = 30 -} - -variable "acme_revoke_certificate" { - type = bool - default = true -} - variable "cloudflare_api_token" { type = string default = "" sensitive = true } - ## # Kubernetes Inputs ## @@ -83,7 +67,7 @@ variable "namespace" { variable "helm_timeout" { type = number - default = 120 # In Seconds + default = 300 # In Seconds } variable "helm_version" { @@ -215,10 +199,14 @@ variable "termination_grace_period_seconds" { variable "ssl_cert_config" { type = object({ name = string + caissuer = optional(string, "issuer") + secretissuer = optional(string, "issuer") create_secret = optional(bool, true) }) default = { name = "coder-tls" + caissuer = "issuer" + secretissuer = "issuer" create_secret = true } } @@ -238,6 +226,11 @@ variable "db_secret_url" { sensitive = true } +variable "enable_oidc" { + type = bool + default = true +} + variable "oidc_config" { type = object({ sign_in_text = string @@ -245,6 +238,12 @@ variable "oidc_config" { scopes = list(string) email_domain = string }) + default = { + sign_in_text = "" + icon_url = "" + scopes = [] + email_domain = "" + } } variable "oidc_secret_name" { @@ -260,6 +259,7 @@ variable "oidc_secret_issuer_url_key" { variable "oidc_secret_issuer_url" { type = string sensitive = true + default = "" } variable "oidc_secret_client_id_key" { @@ -270,6 +270,7 @@ variable "oidc_secret_client_id_key" { variable "oidc_secret_client_id" { type = string sensitive = true + default = "" } variable "oidc_secret_client_secret_key" { @@ -280,6 +281,12 @@ variable "oidc_secret_client_secret_key" { variable "oidc_secret_client_secret" { type = string sensitive = true + default = "" +} + +variable "enable_oauth" { + type = bool + default = true } variable "oauth_secret_name" { @@ -295,6 +302,7 @@ variable "oauth_secret_client_id_key" { variable "oauth_secret_client_id" { type = string sensitive = true + default = "" } variable "oauth_secret_client_secret_key" { @@ -305,6 +313,12 @@ variable "oauth_secret_client_secret_key" { variable "oauth_secret_client_secret" { type = string sensitive = true + default = "" +} + +variable "enable_github_external_auth" { + type = bool + default = true } variable "github_external_auth_config" { @@ -331,6 +345,7 @@ variable "github_external_auth_secret_client_id_key" { variable "github_external_auth_secret_client_id" { type = string sensitive = true + default = "" } variable "github_external_auth_secret_client_secret_key" { @@ -341,6 +356,7 @@ variable "github_external_auth_secret_client_secret_key" { variable "github_external_auth_secret_client_secret" { type = string sensitive = true + default = "" } variable "coder_builtin_provisioner_count" { @@ -406,27 +422,80 @@ data "aws_region" "this" {} data "aws_caller_identity" "this" {} +locals { + oidc_secret_issuer_url = var.enable_oidc ? { + name = "CODER_OIDC_ISSUER_URL" + valueFrom = { + secretKeyRef = { + name = var.oidc_secret_name + key = var.oidc_secret_issuer_url_key + } + } + } : null + oidc_client_id = var.enable_oidc ? { + name = "CODER_OIDC_CLIENT_ID" + valueFrom = { + secretKeyRef = { + name = var.oidc_secret_name + key = var.oidc_secret_client_id_key + } + } + } : null + oidc_client_secret = var.enable_oidc ? { + name = "CODER_OIDC_CLIENT_SECRET" + valueFrom = { + secretKeyRef = { + name = var.oidc_secret_name + key = var.oidc_secret_client_secret_key + } + } + } : null + oauth2_github_client_id = var.enable_oauth ? { + name = "CODER_OAUTH2_GITHUB_CLIENT_ID" + valueFrom = { + secretKeyRef = { + name = var.oauth_secret_name + key = var.oauth_secret_client_id_key + } + } + } : null + oauth2_github_client_secret = var.enable_oauth ? { + name = "CODER_OAUTH2_GITHUB_CLIENT_SECRET" + valueFrom = { + secretKeyRef = { + name = var.oauth_secret_name + key = var.oauth_secret_client_secret_key + } + } + } : null + external_auth_github_client_id = var.enable_github_external_auth ? { + name = "CODER_EXTERNAL_AUTH_0_CLIENT_ID" + valueFrom = { + secretKeyRef = { + name = var.oauth_secret_name + key = var.oauth_secret_client_id_key + } + } + } : null + external_auth_github_client_secret = var.enable_github_external_auth ? { + name = "CODER_EXTERNAL_AUTH_0_CLIENT_SECRET" + valueFrom = { + secretKeyRef = { + name = var.oauth_secret_name + key = var.oauth_secret_client_secret_key + } + } + } : null +} + locals { github_allow_everyone = length(var.coder_github_allowed_orgs) == 0 - primary_env_vars = { + primary_env_vars = merge({ CODER_ACCESS_URL = var.primary_access_url CODER_WILDCARD_ACCESS_URL = var.wildcard_access_url CODER_REDIRECT_TO_ACCESS_URL = true CODER_PG_AUTH = "password" - CODER_OIDC_SIGN_IN_TEXT = var.oidc_config.sign_in_text - CODER_OIDC_ICON_URL = var.oidc_config.icon_url - CODER_OIDC_SCOPES = join(",", var.oidc_config.scopes) - CODER_OIDC_EMAIL_DOMAIN = var.oidc_config.email_domain - - CODER_OAUTH2_GITHUB_DEFAULT_PROVIDER_ENABLE = false - CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS = true - CODER_OAUTH2_GITHUB_DEVICE_FLOW = false - "${local.github_allow_everyone ? "CODER_OAUTH2_GITHUB_ALLOW_EVERYONE" : "CODER_OAUTH2_GITHUB_ALLOWED_ORGS"}" = "${local.github_allow_everyone ? "true" : join(",", var.coder_github_allowed_orgs)}" - - CODER_EXTERNAL_AUTH_0_ID = "primary-github" - CODER_EXTERNAL_AUTH_0_TYPE = "github" - CODER_ENABLE_TERRAFORM_DEBUG_MODE = true CODER_TRACE_LOGS = true CODER_LOG_FILTER = ".*" @@ -447,10 +516,23 @@ locals { CODER_AIBRIDGE_ENABLED = var.openai_llm_endpoint != "" || var.anthropic_llm_endpoint != "" # Experimental Coder Features - CODER_EXPERIMENTS = join(",", var.coder_experiments) + # CODER_EXPERIMENTS = join(",", var.coder_experiments) # Needed by the ai-tasks experiment to embed workspace apps running on subdomains in iframes CODER_ADDITIONAL_CSP_POLICY = "frame-src ${var.primary_access_url}" - } + }, merge(var.enable_oidc ? { + CODER_OIDC_SIGN_IN_TEXT = var.oidc_config.sign_in_text + CODER_OIDC_ICON_URL = var.oidc_config.icon_url + CODER_OIDC_SCOPES = join(",", var.oidc_config.scopes) + CODER_OIDC_EMAIL_DOMAIN = var.oidc_config.email_domain + } : {}, merge(var.enable_oauth ? { + CODER_OAUTH2_GITHUB_DEFAULT_PROVIDER_ENABLE = false + CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS = true + CODER_OAUTH2_GITHUB_DEVICE_FLOW = false + "${local.github_allow_everyone ? "CODER_OAUTH2_GITHUB_ALLOW_EVERYONE" : "CODER_OAUTH2_GITHUB_ALLOWED_ORGS"}" = "${local.github_allow_everyone ? "true" : join(",", var.coder_github_allowed_orgs)}" + } : {}, var.enable_github_external_auth ? { + CODER_EXTERNAL_AUTH_0_ID = "primary-github" + CODER_EXTERNAL_AUTH_0_TYPE = "github" + } : {}))) env_vars = concat([ for k, v in merge(local.primary_env_vars, var.env_vars) : { name = k, value = tostring(v) } ], concat([{ @@ -461,68 +543,19 @@ locals { key = var.db_secret_key } } - }, { - name = "CODER_OIDC_ISSUER_URL" - valueFrom = { - secretKeyRef = { - name = var.oidc_secret_name - key = var.oidc_secret_issuer_url_key - } - } - }, { - name = "CODER_OIDC_CLIENT_ID" - valueFrom = { - secretKeyRef = { - name = var.oidc_secret_name - key = var.oidc_secret_client_id_key - } - } - }, { - name = "CODER_OIDC_CLIENT_SECRET" - valueFrom = { - secretKeyRef = { - name = var.oidc_secret_name - key = var.oidc_secret_client_secret_key - } - } - }, { - name = "CODER_OAUTH2_GITHUB_CLIENT_ID" - valueFrom = { - secretKeyRef = { - name = var.oauth_secret_name - key = var.oauth_secret_client_id_key - } - } - }, { - name = "CODER_OAUTH2_GITHUB_CLIENT_SECRET" - valueFrom = { - secretKeyRef = { - name = var.oauth_secret_name - key = var.oauth_secret_client_secret_key - } - } - }, { - name = "CODER_EXTERNAL_AUTH_0_CLIENT_ID" - valueFrom = { - secretKeyRef = { - name = var.oauth_secret_name - key = var.oauth_secret_client_id_key - } - } - }, { - name = "CODER_EXTERNAL_AUTH_0_CLIENT_SECRET" - valueFrom = { - secretKeyRef = { - name = var.oauth_secret_name - key = var.oauth_secret_client_secret_key - } - } - }, + }, + # local.oidc_secret_issuer_url, + # local.oidc_client_id, + # local.oidc_client_secret, + # local.oauth2_github_client_id, + # local.oauth2_github_client_secret, + # local.external_auth_github_client_id, + # local.external_auth_github_client_secret, ], concat(var.anthropic_llm_endpoint != "" ? [{ name = "CODER_AIBRIDGE_ANTHROPIC_KEY" valueFrom = { secretKeyRef = { - name = var.anthropic_llm_secret_name + name = kubernetes_secret.anthropic-llm-secret[0].metadata[0].name key = "key" } } @@ -530,7 +563,7 @@ locals { name = "CODER_AIBRIDGE_ANTHROPIC_BASE_URL" valueFrom = { secretKeyRef = { - name = var.anthropic_llm_secret_name + name = kubernetes_secret.anthropic-llm-secret[0].metadata[0].name key = "base_url" } } @@ -539,7 +572,7 @@ locals { name = "CODER_AIBRIDGE_OPENAI_KEY" valueFrom = { secretKeyRef = { - name = var.openai_llm_secret_name + name = kubernetes_secret.openai-llm-secret[0].metadata[0].name key = "key" } } @@ -547,7 +580,7 @@ locals { name = "CODER_AIBRIDGE_OPENAI_BASE_URL" valueFrom = { secretKeyRef = { - name = var.openai_llm_secret_name + name = kubernetes_secret.openai-llm-secret[0].metadata[0].name key = "base_url" } } @@ -614,64 +647,8 @@ resource "kubernetes_namespace" "this" { } } -resource "helm_release" "coder-server" { - name = "coder" - namespace = kubernetes_namespace.this.metadata[0].name - chart = "coder" - repository = "https://helm.coder.com/v2" - create_namespace = false - upgrade_install = true - skip_crds = false - wait = true - wait_for_jobs = true - version = var.helm_version - timeout = var.helm_timeout - - values = [yamlencode({ - coder = { - image = { - repo = var.image_repo - tag = var.image_tag - pullPolicy = var.image_pull_policy - pullSecrets = var.image_pull_secrets - } - env = local.env_vars - tls = { - secretNames = [var.ssl_cert_config.name] - } - podAnnotations = { - "prometheus.io/scrape" = "true" - "prometheus.io/port" = "2112" - } - service = { - enable = true - type = "LoadBalancer" - sessionAffinity = "None" - externalTrafficPolicy = "Cluster" - loadBalancerClass = var.load_balancer_class - annotations = var.service_annotations - } - replicaCount = var.replica_count - resources = { - requests = var.resource_request - limits = var.resource_limit - } - serviceAccount = { - annotations = var.coder_builtin_provisioner_count == 0 ? var.service_account_annotations : merge({ - "eks.amazonaws.com/role-arn" : module.provisioner-oidc-role[0].role_arn - }, var.service_account_annotations) - } - nodeSelector = var.node_selector - tolerations = var.tolerations - topologySpreadConstraints = local.topology_spread_constraints - affinity = { - podAntiAffinity = { - preferredDuringSchedulingIgnoredDuringExecution = local.pod_anti_affinity_preferred_during_scheduling_ignored_during_execution - } - } - terminationGracePeriodSeconds = var.termination_grace_period_seconds - } - })] +locals { + ssl_vol_friendly_name = replace(var.ssl_cert_config.name, ".", "-") } resource "kubernetes_secret" "pg-connection" { @@ -751,20 +728,204 @@ resource "kubernetes_secret" "anthropic-llm-secret" { locals { common_name = trimprefix(trimprefix(var.primary_access_url, "https://"), "http://") wildcard_name = trimprefix(trimprefix(var.wildcard_access_url, "https://"), "http://") + cert_refresh_interval = "2160h" # 90 days + cert_renew_before = "360h" # 15 days + secret_refresh_interval = "1812h0m0s" # 75.5 days + tls_secret_key = "tls.key" + tls_secret_crt = "tls.crt" + tls_remote_key = "tls5.key" + tls_remote_crt = "tls5.crt" +} + +resource "kubernetes_manifest" "pull" { + + field_manager { + force_conflicts = true + } + + wait { + fields = { + "status.conditions[0].type" = "Ready" + } + } + + timeouts { + create = "1m" + update = "1m" + delete = "30s" + } + + manifest = { + apiVersion = "external-secrets.io/v1" + kind = "ExternalSecret" + metadata = { + name = var.ssl_cert_config.name + namespace = kubernetes_namespace.this.metadata[0].name + } + spec = { + secretStoreRef = { + kind = "ClusterSecretStore" + name = var.ssl_cert_config.secretissuer + } + refreshPolicy = "Periodic" + refreshInterval = local.secret_refresh_interval + target = { + name = local.ssl_vol_friendly_name + creationPolicy = "Orphan" + deletionPolicy = "Retain" + template = { + type = "kubernetes.io/tls" + metadata = { + labels = { + "controller.cert-manager.io/fao" = "true" + } + annotations = { + "cert-manager.io/alt-names" = "${local.wildcard_name},${local.common_name}" + "cert-manager.io/certificate-name" = var.ssl_cert_config.name + "cert-manager.io/common-name" = local.common_name + "cert-manager.io/ip-sans" = "" + "cert-manager.io/issuer-group" = "" + "cert-manager.io/issuer-kind" = "ClusterIssuer" + "cert-manager.io/issuer-name" = var.ssl_cert_config.caissuer + "cert-manager.io/uri-sans" = "" + } + } + } + } + data = [{ + secretKey = local.tls_secret_crt + remoteRef = { + key = local.tls_remote_crt + } + },{ + secretKey = local.tls_secret_key + remoteRef = { + key = local.tls_remote_key + } + }] + } + } } -module "acme-cloudflare-ssl" { - source = "../acme-cloudflare-ssl" - count = var.ssl_cert_config.create_secret ? 1 : 0 +resource "time_sleep" "wait" { + # Let the secret create first if it exists in AWS Secrets Manager. + depends_on = [ kubernetes_manifest.pull ] + create_duration = "30s" +} + +## +# Requires the cert-manager +## + +resource "kubernetes_manifest" "certificate" { + + depends_on = [ time_sleep.wait ] - dns_names = [local.common_name, local.wildcard_name] - common_name = local.common_name - kubernetes_secret_name = var.ssl_cert_config.name - kubernetes_namespace = kubernetes_namespace.this.metadata[0].name - acme_registration_email = var.acme_registration_email - acme_days_until_renewal = var.acme_days_until_renewal - acme_revoke_certificate = var.acme_revoke_certificate - cloudflare_api_token = var.cloudflare_api_token + field_manager { + force_conflicts = true + } + + wait { + condition { + type = "Ready" + status = "True" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "Certificate" + metadata = { + name = var.ssl_cert_config.name + namespace = kubernetes_namespace.this.metadata[0].name + } + spec = { + commonName = local.common_name + dnsNames = [ + local.common_name, + local.wildcard_name + ] + duration = local.cert_refresh_interval + renewBefore = local.cert_renew_before + issuerRef = { + kind = "ClusterIssuer" + name = var.ssl_cert_config.caissuer + } + secretName = local.ssl_vol_friendly_name + privateKey = { + rotationPolicy = "Never" + algorithm = "RSA" + encoding = "PKCS1" + size = "2048" + } + } + } +} + +resource "kubernetes_manifest" "push" { + + depends_on = [ kubernetes_manifest.certificate ] + + field_manager { + force_conflicts = true + } + + wait { + condition { + type = "Ready" + status = "True" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "external-secrets.io/v1alpha1" + kind = "PushSecret" + metadata = { + name = var.ssl_cert_config.name + namespace = kubernetes_namespace.this.metadata[0].name + } + spec = { + updatePolicy = "Replace" + deletionPolicy = "None" + refreshInterval = local.secret_refresh_interval + secretStoreRefs = [{ + kind = "ClusterSecretStore" + name = var.ssl_cert_config.secretissuer + }] + selector = { + secret = { + name = kubernetes_manifest.certificate.manifest.spec.secretName + } + } + data = [{ + match = { + secretKey = local.tls_secret_crt + remoteRef = { + remoteKey = local.tls_remote_crt + } + } + },{ + match = { + secretKey = local.tls_secret_key + remoteRef = { + remoteKey = local.tls_remote_key + } + } + }] + } + } } resource "kubernetes_service" "prometheus" { @@ -787,4 +948,67 @@ resource "kubernetes_service" "prometheus" { "app.kubernetes.io/name" = "coder" } } +} + +resource "helm_release" "coder-server" { + + depends_on = [ kubernetes_manifest.push ] + + name = "coder" + namespace = kubernetes_namespace.this.metadata[0].name + chart = "coder" + repository = "https://helm.coder.com/v2" + create_namespace = false + upgrade_install = true + skip_crds = false + wait = true + wait_for_jobs = true + version = var.helm_version + timeout = var.helm_timeout + + values = [yamlencode({ + coder = { + image = { + repo = var.image_repo + tag = var.image_tag + pullPolicy = var.image_pull_policy + pullSecrets = var.image_pull_secrets + } + env = local.env_vars + tls = { + secretNames = [ local.ssl_vol_friendly_name ] + } + podAnnotations = { + "prometheus.io/scrape" = "true" + "prometheus.io/port" = "2112" + } + service = { + enable = true + type = "LoadBalancer" + sessionAffinity = "None" + externalTrafficPolicy = "Cluster" + loadBalancerClass = var.load_balancer_class + annotations = var.service_annotations + } + replicaCount = var.replica_count + resources = { + requests = var.resource_request + limits = var.resource_limit + } + serviceAccount = { + annotations = var.coder_builtin_provisioner_count == 0 ? var.service_account_annotations : merge({ + "eks.amazonaws.com/role-arn" : module.provisioner-oidc-role[0].role_arn + }, var.service_account_annotations) + } + nodeSelector = var.node_selector + tolerations = var.tolerations + topologySpreadConstraints = local.topology_spread_constraints + affinity = { + podAntiAffinity = { + preferredDuringSchedulingIgnoredDuringExecution = local.pod_anti_affinity_preferred_during_scheduling_ignored_during_execution + } + } + terminationGracePeriodSeconds = var.termination_grace_period_seconds + } + })] } \ No newline at end of file diff --git a/modules/k8s/bootstrap/ebs-controller/main.tf b/modules/k8s/bootstrap/ebs-controller/main.tf index 84e6eef..aac1255 100644 --- a/modules/k8s/bootstrap/ebs-controller/main.tf +++ b/modules/k8s/bootstrap/ebs-controller/main.tf @@ -5,7 +5,7 @@ terraform { } helm = { source = "hashicorp/helm" - version = "2.17.0" + version = ">= 2.17.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -48,6 +48,11 @@ variable "service_account_annotations" { default = {} } +variable "node_selector" { + type = map(string) + default = {} +} + variable "replace" { type = bool default = false @@ -97,6 +102,7 @@ resource "helm_release" "ebs-controller" { "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn }, var.service_account_annotations) } + nodeSelector = {} } })] } diff --git a/modules/k8s/bootstrap/efs-controller/main.tf b/modules/k8s/bootstrap/efs-controller/main.tf new file mode 100644 index 0000000..a18eeed --- /dev/null +++ b/modules/k8s/bootstrap/efs-controller/main.tf @@ -0,0 +1,106 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.17.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} + +## +# https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/install.md +## + +variable "cluster_name" { + type = string +} + +variable "role_name" { + type = string + default = "" +} + +variable "cluster_oidc_provider_arn" { + type = string +} + +variable "tags" { + type = map(string) + default = {} +} + +variable "namespace" { + type = string +} + +variable "chart_version" { + type = string +} + +variable "service_account_annotations" { + type = map(string) + default = {} +} + +variable "replace" { + type = bool + default = false +} + +data "aws_region" "this" {} + +locals { + role_name = var.role_name == "" ? "efs-controller-${data.aws_region.this.region}" : var.role_name +} + +module "oidc-role" { + source = "../../../security/role/access-entry" + name = local.role_name + cluster_name = var.cluster_name + policy_arns = { + "AmazonEFSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" + } + cluster_policy_arns = { + "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" + } + oidc_principals = { + "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] + } + tags = var.tags +} + +resource "helm_release" "efs-controller" { + name = "aws-efs-csi-driver" + namespace = var.namespace + chart = "aws-efs-csi-driver" + repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver/" + create_namespace = true + upgrade_install = true + skip_crds = false + replace = var.replace + wait = true + wait_for_jobs = true + version = var.chart_version + timeout = 120 # in seconds + + values = [yamlencode({ + controller = { + serviceAccount = { + # https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/install.md + annotations = merge({ + "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn + }, var.service_account_annotations) + } + } + })] +} + +output "oidc_role_arn" { + value = module.oidc-role.role_arn +} \ No newline at end of file diff --git a/modules/k8s/bootstrap/external-dns/main.tf b/modules/k8s/bootstrap/external-dns/main.tf new file mode 100644 index 0000000..d5dbf3a --- /dev/null +++ b/modules/k8s/bootstrap/external-dns/main.tf @@ -0,0 +1,153 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.17.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} + +## +# https://kubernetes-sigs.github.io/external-dns/latest/ +# https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.13.2/docs/install/iam_policy.json +## + +variable "cluster_name" { + type = string +} + +variable "cluster_oidc_provider_arn" { + type = string +} + +variable "role_name" { + type = string + default = "" +} + +variable "policy_name" { + type = string + default = "" +} + +variable "policy_resource_region" { + type = string + default = "" +} + +variable "policy_resource_account" { + type = string + default = "" +} + +variable "tags" { + type = map(string) + default = {} +} + +variable "namespace" { + type = string +} + +variable "chart_version" { + type = string + default = "1.20.0" +} + +variable "node_selector" { + type = map(string) + default = {} +} + +data "aws_region" "this" {} + +data "aws_caller_identity" "this" {} + +locals { + region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region + account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account + policy_name = var.policy_name == "" ? "ExternalDNS-${data.aws_region.this.region}" : var.policy_name + role_name = var.role_name == "" ? "externaldns-${data.aws_region.this.region}" : var.role_name +} + +module "policy" { + source = "../../../security/policy" + name = local.policy_name + path = "/" + description = "External DNS Policy." + policy_json = data.aws_iam_policy_document.this.json +} + +module "oidc-role" { + source = "../../../security/role/access-entry" + name = local.role_name + cluster_name = var.cluster_name + policy_arns = { + "ExternalDNS" = module.policy.policy_arn + } + cluster_policy_arns = { + "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", + } + oidc_principals = { + "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] + } + tags = var.tags +} + +variable "domain_name" { + type = string +} + +variable "is_private_domain" { + type = bool + default = false +} + +data "aws_route53_zone" "this" { + name = "${var.domain_name}." + private_zone = var.is_private_domain +} + +resource "helm_release" "chart" { + name = "external-dns" + namespace = var.namespace + chart = "external-dns" + repository = "https://kubernetes-sigs.github.io/external-dns/" + create_namespace = true + upgrade_install = true + skip_crds = false + wait = true + wait_for_jobs = true + version = var.chart_version + timeout = 120 # in seconds + + values = [yamlencode({ + provider = { + name = "aws" + } + sources = [ + "ingress", + "service" + ] + registry = "txt" + txtPrefix = "%%{record_type}-txt-." + txtOwnerId = "coder" + policy = "sync" # Force cleanup + insertion of record. + env = [{ + name = "AWS_DEFAULT_REGION" + value = data.aws_region.this.region + }] + serviceAccount = { + annotations = { + "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn + } + } + nodeSelector = var.node_selector + })] +} \ No newline at end of file diff --git a/modules/k8s/bootstrap/external-dns/policy.tf b/modules/k8s/bootstrap/external-dns/policy.tf new file mode 100644 index 0000000..c6d376a --- /dev/null +++ b/modules/k8s/bootstrap/external-dns/policy.tf @@ -0,0 +1,20 @@ +data "aws_iam_policy_document" "this" { + + statement { + effect = "Allow" + actions = ["route53:ListHostedZones"] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:ListTagsForResources" + ] + resources = [ + "arn:aws:route53:::hostedzone/*" + ] + } +} \ No newline at end of file diff --git a/modules/k8s/bootstrap/external-secrets/main.tf b/modules/k8s/bootstrap/external-secrets/main.tf new file mode 100644 index 0000000..d6d5671 --- /dev/null +++ b/modules/k8s/bootstrap/external-secrets/main.tf @@ -0,0 +1,123 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.17.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} + +## +# https://kubernetes-sigs.github.io/external-dns/latest/ +# https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.13.2/docs/install/iam_policy.json +## + +variable "cluster_name" { + type = string +} + +variable "cluster_oidc_provider_arn" { + type = string +} + +variable "role_name" { + type = string + default = "" +} + +variable "policy_name" { + type = string + default = "" +} + +variable "policy_resource_region" { + type = string + default = "" +} + +variable "policy_resource_account" { + type = string + default = "" +} + +variable "tags" { + type = map(string) + default = {} +} + +variable "namespace" { + type = string +} + +variable "chart_version" { + type = string + default = "1.20.0" +} + +variable "node_selector" { + type = map(string) + default = {} +} + +data "aws_region" "this" {} + +data "aws_caller_identity" "this" {} + +locals { + region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region + account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account + policy_name = var.policy_name == "" ? "ExternalSec-${data.aws_region.this.region}" : var.policy_name + role_name = var.role_name == "" ? "externalsec-${data.aws_region.this.region}" : var.role_name +} + +module "policy" { + source = "../../../security/policy" + name = local.policy_name + path = "/" + description = "External Secrets Policy." + policy_json = data.aws_iam_policy_document.this.json +} + +module "oidc-role" { + source = "../../../security/role/access-entry" + name = local.role_name + cluster_name = var.cluster_name + policy_arns = { + "ExternalSecrets" = module.policy.policy_arn + } + cluster_policy_arns = { + "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", + } + oidc_principals = { + "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] + } + tags = var.tags +} + +resource "helm_release" "chart" { + name = "external-secrets" + namespace = var.namespace + chart = "external-secrets" + repository = "https://charts.external-secrets.io" + create_namespace = true + upgrade_install = true + skip_crds = false + wait = true + wait_for_jobs = true + version = var.chart_version + timeout = 120 # in seconds + + values = [yamlencode({ + serviceAccount = { + annotations = { + "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn + } + } + })] +} \ No newline at end of file diff --git a/modules/k8s/bootstrap/external-secrets/policy.tf b/modules/k8s/bootstrap/external-secrets/policy.tf new file mode 100644 index 0000000..f699ba7 --- /dev/null +++ b/modules/k8s/bootstrap/external-secrets/policy.tf @@ -0,0 +1,42 @@ +data "aws_iam_policy_document" "this" { + + statement { + effect = "Allow" + actions = [ + "secretsmanager:ListSecrets", + "secretsmanager:BatchGetSecretValue" + ] + resources = ["*"] + } + + # Batch get secrets + statement { + effect = "Allow" + actions = [ + "secretsmanager:GetResourcePolicy", + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + "secretsmanager:ListSecretVersionIds" + ] + resources = [ + "arn:aws:secretsmanager:${local.region}:${local.account_id}:secret:*" + ] + } + + # Batch get secrets + statement { + effect = "Allow" + actions = [ + "secretsmanager:CreateSecret", + "secretsmanager:PutSecretValue", + "secretsmanager:TagResource", + "secretsmanager:DeleteSecret", + "secretsmanager:GetResourcePolicy", + "secretsmanager:PutResourcePolicy", + "secretsmanager:DeleteResourcePolicy" + ] + resources = [ + "arn:aws:secretsmanager:${local.region}:${local.account_id}:secret:*" + ] + } +} \ No newline at end of file diff --git a/modules/k8s/bootstrap/karpenter/main.tf b/modules/k8s/bootstrap/karpenter/main.tf index 429be43..dfe91b0 100644 --- a/modules/k8s/bootstrap/karpenter/main.tf +++ b/modules/k8s/bootstrap/karpenter/main.tf @@ -21,6 +21,10 @@ variable "cluster_oidc_provider_arn" { type = string } +variable "cluster_oidc_provider" { + type = string +} + variable "namespace" { type = string default = "karpenter" @@ -89,6 +93,16 @@ variable "karpenter_node_role_tags" { default = {} } +variable "iam_role_use_name_prefix" { + type = bool + default = true +} + +variable "node_iam_role_use_name_prefix" { + type = bool + default = true +} + variable "ec2nodeclass_configs" { type = list(object({ name = string @@ -143,9 +157,9 @@ locals { std_karpenter_format = "${var.cluster_name}-kptr-${data.aws_region.this.region}" karpenter_queue_name = var.karpenter_queue_name == "" ? "${var.cluster_name}-kptr" : var.karpenter_queue_name karpenter_queue_rule_name = var.karpenter_queue_rule_name == "" ? "${var.cluster_name}-kptr" : var.karpenter_queue_rule_name - karpenter_controller_role_name = var.karpenter_controller_role_name == "" ? local.std_karpenter_format : var.karpenter_controller_role_name + karpenter_controller_role_name = var.karpenter_controller_role_name == "" ? "${local.std_karpenter_format}-ctrl" : var.karpenter_controller_role_name karpenter_controller_policy_name = var.karpenter_controller_policy_name == "" ? local.std_karpenter_format : var.karpenter_controller_policy_name - karpenter_node_role_name = var.karpenter_node_role_name == "" ? local.std_karpenter_format : var.karpenter_node_role_name + karpenter_node_role_name = var.karpenter_node_role_name == "" ? "${local.std_karpenter_format}-node" : var.karpenter_node_role_name } data "aws_iam_policy_document" "sts" { @@ -163,9 +177,36 @@ resource "aws_iam_policy" "sts" { policy = data.aws_iam_policy_document.sts.json } +data "aws_iam_policy_document" "kptr_ctrl_assume_role_policy" { + statement { + effect = "Allow" + + principals { + type = "Federated" + identifiers = [var.cluster_oidc_provider_arn] + } + + condition { + test = "StringEquals" + variable = "${var.cluster_oidc_provider}:sub" + values = ["system:serviceaccount:karpenter:karpenter"] + } + + # https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-oidc-and-irsa/?nc1=h_ls + condition { + test = "StringEquals" + variable = "${var.cluster_oidc_provider}:aud" + values = ["sts.amazonaws.com"] + } + + actions = ["sts:AssumeRoleWithWebIdentity"] + } +} + module "karpenter" { source = "terraform-aws-modules/eks/aws//modules/karpenter" - version = "20.34.0" # "21.3.1" + # version = "20.34.0" # "21.3.1" + version = "21.14.0" cluster_name = var.cluster_name queue_name = local.karpenter_queue_name @@ -174,8 +215,11 @@ module "karpenter" { # Karpenter Controller Role create_iam_role = true iam_role_name = local.karpenter_controller_role_name - iam_role_use_name_prefix = true + iam_role_use_name_prefix = var.iam_role_use_name_prefix iam_role_policies = var.karpenter_controller_role_policies + iam_role_source_assume_policy_documents = [ + data.aws_iam_policy_document.kptr_ctrl_assume_role_policy.json, + ] # Karpenter Controller Policies iam_policy_use_name_prefix = true @@ -189,17 +233,20 @@ module "karpenter" { # Karpenter Node Role create_node_iam_role = true node_iam_role_name = local.karpenter_node_role_name - node_iam_role_use_name_prefix = true + node_iam_role_use_name_prefix = var.node_iam_role_use_name_prefix node_iam_role_additional_policies = merge({ AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" STSAssumeRole = aws_iam_policy.sts.arn }, var.karpenter_node_role_policies) - enable_irsa = true - enable_pod_identity = false + create_pod_identity_association = false enable_spot_termination = true - irsa_oidc_provider_arn = var.cluster_oidc_provider_arn + ### DEPRECATED in v21. + # enable_irsa = true + # irsa_oidc_provider_arn = var.cluster_oidc_provider_arn + # enable_pod_identity = false + # tags = merge(var.tags, var.karpenter_tags) # iam_role_tags = merge(var.tags, var.karpenter_role_tags) @@ -224,12 +271,12 @@ resource "helm_release" "karpenter" { controller = { resources = { limits = { - cpu = "500m" - memory = "512Mi" + cpu = "1000m" + memory = "2Gi" } requests = { - cpu = "100m" - memory = "256Mi" + cpu = "500m" + memory = "1Gi" } } } diff --git a/modules/k8s/bootstrap/lb-controller/main.tf b/modules/k8s/bootstrap/lb-controller/main.tf index 55a3118..a560656 100644 --- a/modules/k8s/bootstrap/lb-controller/main.tf +++ b/modules/k8s/bootstrap/lb-controller/main.tf @@ -5,7 +5,7 @@ terraform { } helm = { source = "hashicorp/helm" - version = "2.17.0" + version = ">= 2.17.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -79,6 +79,22 @@ variable "cluster_asg_node_labels" { default = {} } +variable "vpc_id" { + description = "(Optional). Set this when your pods can't use the IMDS to auto-determine this" + type = string + default = "" +} + +variable "node_selector" { + type = map(string) + default = {} +} + +variable "create_alb_class" { + type = bool + default = true +} + data "aws_region" "this" {} data "aws_caller_identity" "this" {} @@ -133,7 +149,7 @@ resource "helm_release" "lb-controller" { wait = true wait_for_jobs = true version = var.chart_version - timeout = 120 # in seconds + timeout = 300 # in seconds values = [yamlencode({ clusterName = var.cluster_name @@ -145,13 +161,19 @@ resource "helm_release" "lb-controller" { automountServiceAccountToken = true imagePullSecrets = [] } + vpcId = var.vpc_id enableCertManager = var.enable_cert_manager - nodeSelector = var.cluster_asg_node_labels + nodeSelector = var.node_selector serviceTargetENISGTags = local.service_target_eni_sg_tags + serviceMutatorWebhookConfig = { + # Ref - https://github.com/awslabs/data-on-eks/issues/458 + failurePolicy = "Ignore" + } })] } resource "kubernetes_manifest" "alb-class-params" { + count = var.create_alb_class ? 1 : 0 depends_on = [helm_release.lb-controller] manifest = { apiVersion = "elbv2.k8s.aws/v1beta1" @@ -166,7 +188,8 @@ resource "kubernetes_manifest" "alb-class-params" { } resource "kubernetes_manifest" "alb-class" { - depends_on = [helm_release.lb-controller, kubernetes_manifest.alb-class-params] + count = var.create_alb_class ? 1 : 0 + depends_on = [helm_release.lb-controller, kubernetes_manifest.alb-class-params[0]] manifest = { apiVersion = "networking.k8s.io/v1" kind = "IngressClass" diff --git a/modules/k8s/bootstrap/litellm-generate-key/main.tf b/modules/k8s/bootstrap/litellm-generate-key/main.tf deleted file mode 100644 index 4b14834..0000000 --- a/modules/k8s/bootstrap/litellm-generate-key/main.tf +++ /dev/null @@ -1,296 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } -} - -variable "cluster_name" { - type = string -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "role_name" { - type = string - default = "" -} - -variable "policy_name" { - type = string - default = "" -} - -variable "namespace" { - type = string - default = "litellm" -} - -variable "name" { - type = string - default = "rotate-key" -} - -variable "image_repo" { - type = string -} - -variable "image_tag" { - type = string -} - -variable "rotate_key_script_file_name" { - type = string - default = "rotate.sh" -} - -variable "litellm_key_secret_name" { - type = string - default = "litellm.env" -} - -variable "litellm_key_master_secret_key" { - type = string - default = "master" -} - -variable "litellm_key_salt_secret_key" { - type = string - default = "salt" -} - -variable "litellm_master_key" { - type = string - sensitive = true - default = "" -} - -variable "litellm_salt_key" { - type = string - sensitive = true - default = "" -} - -variable "litellm_create_secret" { - type = bool - default = false -} - -variable "litellm_url" { - type = string -} - -variable "tags" { - type = map(string) - default = {} -} - -variable "secret_id" { - type = string - sensitive = true -} - -variable "secret_region" { - type = string - sensitive = true -} - -data "aws_region" "this" {} - -data "aws_caller_identity" "this" {} - -locals { - app_labels = { - "app.kubernetes.io/name" : var.name - "app.kubernetes.io/part-of" : var.name - } - policy_name = var.policy_name == "" ? "LiteLLM-Create-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "litellm-create-${data.aws_region.this.region}" : var.role_name -} - -module "oidc-role" { - source = "../../../security/role/access-entry" - name = local.role_name - cluster_name = var.cluster_name - policy_arns = { - "SecretsManagerReadWrite" = "arn:aws:iam::aws:policy/SecretsManagerReadWrite" - } - cluster_policy_arns = { - "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", - } - oidc_principals = { - "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] - } - tags = var.tags -} - -resource "kubernetes_role" "this" { - metadata { - name = var.name - namespace = var.namespace - labels = local.app_labels - } - rule { - api_groups = [""] - resources = ["secrets"] - resource_names = [var.litellm_key_secret_name] - verbs = ["get", "create", "update", "patch"] - } - rule { - api_groups = ["apps", "extensions"] - resources = ["deployments"] - resource_names = [var.name] - verbs = ["get", "patch"] - } -} - -resource "kubernetes_service_account" "this" { - metadata { - name = var.name - namespace = var.namespace - annotations = { - "eks.amazonaws.com/role-arn" : module.oidc-role.role_arn - } - labels = local.app_labels - } -} - -resource "kubernetes_role_binding" "this" { - metadata { - name = var.name - namespace = var.namespace - labels = local.app_labels - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "Role" - name = kubernetes_role.this.metadata[0].name - } - subject { - kind = "ServiceAccount" - name = kubernetes_service_account.this.metadata[0].name - namespace = var.namespace - } -} - -resource "kubernetes_config_map" "this" { - metadata { - name = var.name - namespace = var.namespace - labels = local.app_labels - } - data = { - "${var.rotate_key_script_file_name}" = templatefile("${path.module}/scripts/${var.rotate_key_script_file_name}", {}) - } -} - -locals { - primary_env_vars = { - LITELLM_URL = var.litellm_url - AWS_SECRETS_MANAGER_ID = var.secret_id - AWS_SECRET_REGION = var.secret_region - KEY_NAME = "litellm-tmp" - KEY_DURATION = "8h" - USERNAME = "robot_user.ai" - USER_EMAIL = "robot_user@gmail.com" - } - secret_env_vars = { - LITELLM_MASTER_KEY = { - name = var.litellm_key_secret_name - key = var.litellm_key_master_secret_key - } - LITELLM_SALT_KEY = { - name = var.litellm_key_secret_name - key = var.litellm_key_salt_secret_key - } - } -} - -resource "kubernetes_secret" "key" { - count = var.litellm_create_secret ? 1 : 0 - metadata { - name = var.litellm_key_secret_name - namespace = var.namespace - labels = local.app_labels - } - data = { - "${var.litellm_key_master_secret_key}" = var.litellm_master_key - "${var.litellm_key_salt_secret_key}" = var.litellm_salt_key - } -} - - -resource "kubernetes_cron_job_v1" "this" { - metadata { - name = var.name - namespace = var.namespace - labels = local.app_labels - } - spec { - timezone = "America/Vancouver" - successful_jobs_history_limit = 0 - failed_jobs_history_limit = 1 - concurrency_policy = "Replace" - schedule = "*/5 * * * *" - job_template { - metadata { - labels = local.app_labels - } - spec { - parallelism = 1 - template { - metadata { - labels = local.app_labels - } - spec { - service_account_name = kubernetes_service_account.this.metadata[0].name - restart_policy = "OnFailure" - container { - name = var.name - image = "${var.image_repo}:${var.image_tag}" - image_pull_policy = "IfNotPresent" - command = split(" ", "/bin/bash -c /tmp/${var.rotate_key_script_file_name}") - dynamic "env" { - for_each = local.primary_env_vars - content { - name = env.key - value = tostring(env.value) - } - } - dynamic "env" { - for_each = local.secret_env_vars - content { - name = env.key - value_from { - secret_key_ref { - name = env.value.name - key = env.value.key - } - } - } - } - volume_mount { - name = kubernetes_config_map.this.metadata[0].name - mount_path = "/tmp/${var.rotate_key_script_file_name}" - sub_path = var.rotate_key_script_file_name - } - } - volume { - name = kubernetes_config_map.this.metadata[0].name - config_map { - name = kubernetes_config_map.this.metadata[0].name - default_mode = "0777" - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/modules/k8s/bootstrap/litellm-generate-key/scripts/rotate.sh b/modules/k8s/bootstrap/litellm-generate-key/scripts/rotate.sh deleted file mode 100644 index 83a35cf..0000000 --- a/modules/k8s/bootstrap/litellm-generate-key/scripts/rotate.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -curl -L -X POST "$LITELLM_URL/user/new" \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ - -H 'Content-Type: application/json' \ - -d "{\"username\":\"$USERNAME\",\"user_id\":\"$USERNAME\",\"email\":\"$USER_EMAIL\",\"key_alias\":\"$KEY_NAME\",\"duration\":\"$KEY_DURATION\"}" || true ; - -curl -L -X POST "$LITELLM_URL/key/delete" \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ - -H 'Content-Type: application/json' \ - -d "{\"key_aliases\":[\"$KEY_NAME\"]}" || true ; - -NEW_LITELLM_USER_KEY=$(curl -L -X POST $LITELLM_URL/key/generate \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ - -H 'Content-Type: application/json' \ - -d "{\"key_alias\":\"$KEY_NAME\",\"duration\":\"$KEY_DURATION\",\"metadata\":{\"user_id\":\"$USERNAME\"}}" | jq -r '.key') ; - -aws secretsmanager put-secret-value \ - --region $AWS_SECRET_REGION \ - --secret-id $AWS_SECRETS_MANAGER_ID \ - --secret-string "{\"LITELLM_MASTER_KEY\":\"$NEW_LITELLM_USER_KEY\"}" || \ -aws secretsmanager create-secret \ - --region $AWS_SECRET_REGION \ - --name $AWS_SECRETS_MANAGER_ID \ - --secret-string "{\"LITELLM_MASTER_KEY\":\"$NEW_LITELLM_USER_KEY\"}" \ No newline at end of file diff --git a/modules/k8s/bootstrap/litellm/README.md b/modules/k8s/bootstrap/litellm/README.md new file mode 100644 index 0000000..4560108 --- /dev/null +++ b/modules/k8s/bootstrap/litellm/README.md @@ -0,0 +1,156 @@ +# LiteLLM Helm Chart Terraform Module + +This Terraform module deploys [LiteLLM](https://github.com/BerriAI/litellm) as a Kubernetes Helm chart on AWS EKS. LiteLLM provides a unified proxy interface to multiple LLM providers, configured here to route requests to AWS Bedrock models (Claude Sonnet and Haiku) across multiple AWS regions. + +## Overview + +This module: +- Deploys LiteLLM proxy to an EKS cluster using the official Helm chart + - https://artifacthub.io/packages/helm/litellm/litellm-helm +- Configures AWS IAM roles for service accounts (IRSA) for Bedrock access +- Sets up load balancer exposure via AWS Network Load Balancer +- Configures multi-region failover for Claude models +- Includes a PostgreSQL database for LiteLLM state management + +## Architecture + +``` +┌─────────────┐ +│ Client │ +└──────┬──────┘ + │ + ▼ +┌─────────────────────────┐ +│ AWS NLB (Port 4000) │ +└──────────┬──────────────┘ + │ + ▼ + ┌──────────────┐ + │ LiteLLM │ + │ Proxy │ + └──────┬───────┘ + │ + ├──► AWS Bedrock (us-east-2) + ├──► AWS Bedrock (us-east-1) + └──► AWS Bedrock (us-west-2) +``` + +## Prerequisites + +- AWS EKS cluster (default: `aidemo-eks`) +- AWS CLI configured with appropriate profile +- Terraform >= 1.0 +- Helm provider >= 3.1.1 +- Kubernetes provider >= 3.0.1 + +## Variables + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `cluster_name` | string | `aidemo-eks` | Name of the EKS cluster | +| `cluster_region` | string | `us-east-2` | AWS region where the cluster is located | +| `cluster_profile` | string | `demo-coder` | AWS CLI profile to use | +| `namespace` | string | `litellm-tmp` | Kubernetes namespace for LiteLLM | +| `chart_version` | string | `0.1.830` | Version of the LiteLLM Helm chart | +| `cluster_oidc_provider_arn` | string | (see main.tf) | ARN of the EKS OIDC provider | + +## Configured Models + +The proxy is configured with the following AWS Bedrock models across multiple regions for high availability: + +### Claude Sonnet 4.5 +- Model: `anthropic.claude-sonnet-4-5-20250929-v1:0` +- Regions: `us-east-2`, `us-east-1`, `us-west-2` + +### Claude 3 Haiku +- Model: `anthropic.claude-3-haiku-20240307-v1:0` +- Regions: `us-east-2`, `us-east-1`, `us-west-2` + +## Usage + +### Deploy the Module + +```bash +terraform init +terraform plan +terraform apply +``` + +### Testing the Deployment + +Use the included test script to verify the proxy is working: + +```bash +./test.sh +``` + +This script configures Claude Code to use the LiteLLM proxy as the API endpoint. + +### Get the Load Balancer URL + +```bash +kubectl get svc -n litellm-tmp litellm +``` + +### Making API Requests + +```bash +curl http://:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-2Uw00oiTMSxwEtHA19" \ + -d '{ + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + +## Configuration Details + +### IAM Role for Service Account (IRSA) + +The module creates an IAM role with: +- AWS Bedrock Limited Access policy +- EKS Cluster Admin policy +- Trust relationship with the EKS OIDC provider + +The service account annotation automatically injects AWS credentials into the pod. + +### Service Configuration + +- **Type**: LoadBalancer (AWS Network Load Balancer) +- **Port**: 4000 +- **Scheme**: Internet-facing +- **Target Type**: Instance + +### Database + +- **Type**: PostgreSQL (standalone via Bitnami chart) +- **Purpose**: Stores LiteLLM configuration and state +- **Migration**: Automatic schema migration on deployment + +### Security + +- Master key authentication required for API access +- AWS credentials managed via IRSA (no static credentials) +- Secrets managed via Kubernetes secrets + +## Files + +- `main.tf` - Main Terraform configuration +- `values.yaml` - Helm chart values template +- `test.sh` - Quick test script for Claude Code integration + +## Notes + +- The load balancer is internet-facing by default +- Multi-region configuration provides automatic failover +- PostgreSQL credentials default to weak passwords and should be overridden in production +- The test script includes a hardcoded API key for testing purposes + +## Cleanup + +```bash +terraform destroy +``` + +This will remove the Helm release, namespace, and associated IAM resources. \ No newline at end of file diff --git a/modules/k8s/bootstrap/litellm/main.tf b/modules/k8s/bootstrap/litellm/main.tf index 8b54c7c..c4ad5ae 100644 --- a/modules/k8s/bootstrap/litellm/main.tf +++ b/modules/k8s/bootstrap/litellm/main.tf @@ -3,38 +3,30 @@ terraform { aws = { source = "hashicorp/aws" } + helm = { + source = "hashicorp/helm" + version = "3.1.1" + } kubernetes = { source = "hashicorp/kubernetes" + version = "3.0.1" } } } variable "cluster_name" { type = string + default = "aidemo-eks" } -variable "cluster_oidc_provider_arn" { +variable "cluster_region" { type = string + default = "us-east-2" } -variable "role_name" { - type = string - default = "" -} - -variable "policy_name" { - type = string - default = "" -} - -variable "policy_resource_region" { - type = string - default = "" -} - -variable "policy_resource_account" { +variable "cluster_profile" { type = string - default = "" + default = "demo-coder" } variable "namespace" { @@ -42,491 +34,476 @@ variable "namespace" { default = "litellm" } -variable "name" { +variable "chart_version" { type = string - default = "litellm" + default = "0.1.830" } -variable "replicas" { - type = number - default = 0 -} - -variable "image_repo" { - type = string - default = "ghcr.io/berriai/litellm" +variable "cluster_oidc_provider_arn" { + type = string } -variable "image_tag" { - type = string - default = "v1.72.6-stable" +provider "aws" { + region = var.cluster_region + profile = var.cluster_profile } -variable "app_container_port" { - type = number - default = 4000 +data "aws_eks_cluster" "this" { + name = var.cluster_name } -variable "host_name" { - type = string +data "aws_eks_cluster_auth" "this" { + name = var.cluster_name } -variable "resource_requests" { +variable "registry_config" { type = object({ - cpu = string - memory = string + url = optional(string, "oci://ghcr.io") + username = string + password = string }) - default = { - cpu = "250m" - memory = "512Mi" - } + sensitive = true } -variable "resource_limits" { - type = object({ - cpu = string - memory = string - }) - default = { - cpu = "500m" - memory = "1Gi" +provider "helm" { + kubernetes = { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.this.token } + registries = [{ + url = nonsensitive(var.registry_config.url) + username = nonsensitive(var.registry_config.username) + password = var.registry_config.password + }] } -variable "aws_ingress_certificate_arn" { - type = string - sensitive = true -} - -variable "aws_bedrock_region" { - type = string - default = "us-east-2" +variable "tags" { + type = map(string) + default = {} } -variable "db_url_secret_name" { - type = string - default = "url" +variable "replicas" { + type = number + default = 1 } -variable "db_url_secret_key" { - type = string - default = "postgres.env" +variable "image_config" { + type = object({ + repo = optional(string, "ghcr.io/berriai/litellm-database") + pull_policy = optional(string, "IfNotPresent") + tag = optional(string, "main-latest") + }) + default = {} } -variable "db_url" { - type = string +variable "litellm_master_key" { + type = string sensitive = true -} -variable "redis_host_secret_name" { - type = string - default = "redis.env" -} - -variable "redis_host_secret_key" { - type = string - default = "host" + validation { + condition = startswith(var.litellm_master_key, "sk-") + error_message = "The LiteLLM master key must start with 'sk-'." + } } -variable "redis_password_secret_key" { - type = string - default = "password" +variable "litellm_master_secret_name" { + type = string + default = "masterkey" } -variable "redis_host" { - type = string +variable "db_config" { + type = object({ + use_existing = optional(bool, false) + secret_name = optional(string, "postgres") + db_name = optional(string, "litellm") + username = optional(string, "litellm") + admin_password = optional(string, "NoTaGrEaTpAsSwOrD") + user_password = optional(string, "NoTaGrEaTpAsSwOrD") + endpoint = optional(string, "localhost") + }) + default = { + use_existing = false + secret_name = "postgres" + db_name = "litellm" + username = "litellm" + admin_password = "NoTaGrEaTpAsSwOrD" + user_password = "NoTaGrEaTpAsSwOrD" + endpoint = "localhost" + } sensitive = true } -variable "redis_password" { - type = string - sensitive = true +variable "service_account_annotations" { + type = map(string) + default = {} } -variable "litellm_key_secret_name" { - type = string - default = "litellm.env" +variable "service_lb_class" { + type = string + default = "service.k8s.aws/nlb" } -variable "litellm_key_master_secret_key" { - type = string - default = "master" +variable "service_annotations" { + type = map(string) + default = {} } -variable "litellm_key_salt_secret_key" { - type = string - default = "salt" +variable "service_port" { + type = number + default = 80 } -variable "litellm_salt_key" { - type = string - sensitive = true +variable "health_port" { + type = number + default = 8081 } -variable "litellm_master_key" { - type = string - sensitive = true +variable "proxy_config" { + type = any + default = {} } -variable "gcloud_auth_secret_name" { - type = string - default = "gcloud-auth" +variable "env_vars" { + type = map(string) + default = {} } -variable "gcloud_auth_secret_key" { - type = string - default = "service_account.json" +variable "volumes" { + type = list(any) + default = [] } -variable "gcloud_auth_file_path" { - type = string - default = "/tmp" +variable "volume_mounts" { + type = list(any) + default = [] } -variable "gcloud_auth" { - type = string - sensitive = true +variable "autoscaling_min_replicas" { + type = number + default = 1 } -variable "litellm_config_name" { - type = string - default = "config-yaml" +variable "autoscaling_max_replicas" { + type = number + default = 5 } -variable "litellm_config_key" { - type = string - default = "config.yaml" +variable "autoscaling_target_cpu_use" { + type = number + description = "The target CPU utilization percentage for autoscaling." + default = 80 } -variable "litellm_config_middleware_name" { - type = string - default = "strip-header-middleware-py" +variable "autoscaling_target_memory_use" { + type = number + description = "The target memory utilization percentage for autoscaling." + default = 80 } -variable "litellm_config_middleware_key" { - type = string - default = "strip_header_middleware.py" +variable "node_selector" { + type = map(string) + default = {} } -variable "tags" { - type = map(string) +variable "tolerations" { + type = any default = {} } -data "aws_region" "this" {} - -data "aws_caller_identity" "this" {} +variable "affinity" { + type = any + default = {} +} -locals { - app_labels = { - "app.kubernetes.io/name" : var.name - "app.kubernetes.io/part-of" : var.name +module "oidc-role" { + source = "../../../security/role/access-entry" + name = "LiteLLM-Bedrock" + cluster_name = var.cluster_name + policy_arns = { + "AmazonBedrockLimitedAccess" = "arn:aws:iam::aws:policy/AmazonBedrockLimitedAccess" + } + cluster_policy_arns = { + "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" } + oidc_principals = { + "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] + } + tags = var.tags } -resource "kubernetes_namespace" "this" { +resource "kubernetes_namespace_v1" "litellm" { metadata { name = var.namespace } } -resource "kubernetes_ingress_v1" "this" { +resource "kubernetes_secret_v1" "master-key" { metadata { - name = var.name - namespace = kubernetes_namespace.this.metadata[0].name - annotations = { - "alb.ingress.kubernetes.io/certificate-arn" = var.aws_ingress_certificate_arn - "alb.ingress.kubernetes.io/group.order" = 10 - "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":443}]" - "alb.ingress.kubernetes.io/scheme" = "internet-facing" - "alb.ingress.kubernetes.io/unhealthy-threshold-count" = 3 - } - labels = local.app_labels + name = var.litellm_master_secret_name + namespace = kubernetes_namespace_v1.litellm.metadata[0].name } - spec { - ingress_class_name = "alb" - rule { - host = var.host_name - http { - path { - backend { - service { - name = var.name - port { - number = 80 - } - } - } - path = "/" - path_type = "Prefix" - } - } - } + data = { + password = var.litellm_master_key } + type = "Opaque" } -resource "kubernetes_service" "this" { +resource "kubernetes_secret_v1" "db-auth" { metadata { - name = var.name - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels + name = nonsensitive(var.db_config.secret_name) + namespace = kubernetes_namespace_v1.litellm.metadata[0].name } - spec { - type = "NodePort" - internal_traffic_policy = "Cluster" - ip_families = ["IPv4"] - ip_family_policy = "SingleStack" - port { - name = "http" - protocol = "TCP" - port = 80 - target_port = "http" - } - selector = { - app = var.name - } + data = { + username = nonsensitive(var.db_config.username) + postgres-password = var.db_config.admin_password + password = var.db_config.user_password + endpoint = var.db_config.endpoint } + type = "Opaque" } -locals { - policy_name = var.policy_name == "" ? "LiteLLM-BR-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "litellm-br-${data.aws_region.this.region}" : var.role_name -} - -module "bedrock-policy" { - source = "../../../security/policy" - name = local.policy_name - path = "/" - description = "LiteLLM Bedrock IAM Policy" - policy_json = data.aws_iam_policy_document.bedrock-policy.json +variable "access_url" { + type = string + default = "" } -module "bedrock-oidc-role" { - source = "../../../security/role/access-entry" - name = local.role_name - policy_arns = { - "BedrockPolicy" = module.bedrock-policy.policy_arn - } - cluster_name = var.cluster_name - cluster_policy_arns = {} - oidc_principals = { - "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] +variable "ssl_cert_config" { + type = object({ + create_secret = optional(bool, true) + name = optional(string, "ssl-certs") + days_until_renewal = optional(number, 30) + }) + default = { + create_secret = true + name = "ssl-certs" + days_until_renewal = 30 } - tags = var.tags } -resource "kubernetes_service_account" "litellm" { - metadata { - name = var.name - namespace = kubernetes_namespace.this.metadata[0].name - annotations = { - "eks.amazonaws.com/role-arn" : module.bedrock-oidc-role.role_arn - } - labels = local.app_labels - } +locals { + common_name = replace(replace(var.access_url, "https://", ""), "http://", "") } -resource "kubernetes_secret" "postgres" { - metadata { - name = var.db_url_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels - } - data = { - "${var.db_url_secret_key}" = var.db_url - } -} +resource "kubernetes_manifest" "cert" { -resource "kubernetes_secret" "redis" { - metadata { - name = var.redis_host_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels - } - data = { - "${var.redis_host_secret_key}" = var.redis_host - "${var.redis_password_secret_key}" = var.redis_password - } -} + count = var.ssl_cert_config.create_secret ? 1 : 0 -resource "kubernetes_secret" "key" { - metadata { - name = var.litellm_key_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels + field_manager { + force_conflicts = true } - data = { - "${var.litellm_key_master_secret_key}" = var.litellm_master_key - "${var.litellm_key_salt_secret_key}" = var.litellm_salt_key + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "Certificate" + metadata = { + labels = {} # var.cert_labels + name = var.ssl_cert_config.name + namespace = kubernetes_namespace_v1.litellm.metadata[0].name + } + spec = { + secretName = var.ssl_cert_config.name + commonName = local.common_name + dnsNames = [local.common_name] + duration = "${var.ssl_cert_config.days_until_renewal * 24}h" + renewBefore = "8h" + additionalOutputFormats = [{ + type = "CombinedPEM" + },{ + type = "DER" + }] + issuerRef = { + kind = "ClusterIssuer" + name = "issuer" + } + } } } -resource "kubernetes_secret" "gcloud" { - metadata { - name = var.gcloud_auth_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels - } - data = { - "${var.gcloud_auth_secret_key}" = var.gcloud_auth - } +locals { + ssl_volume = var.ssl_cert_config.create_secret ? {} : {} } -resource "kubernetes_config_map" "config" { - metadata { - name = var.litellm_config_name - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels - } - data = { - "${var.litellm_config_key}" = templatefile("${path.module}/scripts/${var.litellm_config_key}", { - GCP_CRED_PATH = "${var.gcloud_auth_file_path}/${var.gcloud_auth_secret_key}" - }) - } +variable "gcloud_auth" { + type = string + sensitive = true } -resource "kubernetes_config_map" "middleware" { +resource "kubernetes_secret_v1" "gcloud" { metadata { - name = var.litellm_config_middleware_name - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels + name = "gcloud-auth" + namespace = kubernetes_namespace_v1.litellm.metadata[0].name + labels = {} } data = { - "${var.litellm_config_middleware_key}" = templatefile("${path.module}/scripts/${var.litellm_config_middleware_key}", {}) + "service_account.json" = var.gcloud_auth } } -locals { - primary_env_vars = { - AWS_REGION_NAME = var.aws_bedrock_region - DOCS_URL = "/swagger" - LITELLM_LOG = "ERROR" - LITELLM_LOG_LEVEL = "ERROR" - LITELLM_MODE = "PRODUCTION" - REDIS_PORT = "6379" - REDIS_SSL = "True" - } - secret_env_vars = { - DATABASE_URL = { - name = var.db_url_secret_name - key = var.db_url_secret_key - } - LITELLM_MASTER_KEY = { - name = var.litellm_key_secret_name - key = var.litellm_key_master_secret_key +resource "helm_release" "litellm" { + name = "litellm" + namespace = var.namespace + chart = "litellm-helm" + repository = "oci://ghcr.io/berriai" + create_namespace = true + upgrade_install = true + skip_crds = false + replace = true + wait = true + wait_for_jobs = true + reuse_values = false + version = var.chart_version + timeout = 120 # in seconds + + values = [yamlencode({ + replicaCount = var.replicas + image = { + repository = var.image_config.repo + pullPolicy = var.image_config.pull_policy + tag = var.image_config.tag } - LITELLM_SALT_KEY = { - name = var.litellm_key_secret_name - key = var.litellm_key_salt_secret_key + imagePullSecrets = [] + nameOverride = "litellm" + fullnameOverride = "" + serviceAccount = { + create = true + automount = true + annotations = merge({ + "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn + }, var.service_account_annotations) } - REDIS_HOST = { - name = var.redis_host_secret_name - key = var.redis_host_secret_key + service = { + type = "LoadBalancer" + loadBalancerClass = var.service_lb_class + port = var.service_port + annotations = var.service_annotations } - REDIS_PASSWORD = { - name = var.redis_host_secret_name - key = var.redis_password_secret_key + separateHealthApp = true + separateHealthPort = var.health_port + + masterkeySecretName = kubernetes_secret_v1.master-key.metadata[0].name + masterkeySecretKey = "password" + + proxyConfigMap = { + create = true } - } -} -resource "kubernetes_deployment" "litellm" { - metadata { - name = var.name - namespace = kubernetes_namespace.this.metadata[0].name - labels = merge(local.app_labels, { - app = var.name - }) - } - spec { - replicas = var.replicas - strategy { - type = "RollingUpdate" + proxy_config = var.proxy_config + + autoscaling = { + enabled = true + minReplicas = var.autoscaling_min_replicas + maxReplicas = var.autoscaling_max_replicas + targetCPUUtilizationPercentage = var.autoscaling_target_cpu_use + targetMemoryUtilizationPercentage = var.autoscaling_target_memory_use } - selector { - match_labels = { - app = var.name + + nodeSelector = var.node_selector + tolerations = var.tolerations + affinity = var.affinity + + db = { + deployStandalone = var.db_config.endpoint == "localhost" + useExisting = nonsensitive(var.db_config.use_existing) + database = nonsensitive(var.db_config.db_name) + url = "postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)" + secret = { + name = kubernetes_secret_v1.db-auth.metadata[0].name + usernameKey = "username" + passwordKey = "password" + # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint + endpointKey = "endpoint" } + useStackgresOperator = false } - template { - metadata { - annotations = {} - labels = { - app = var.name + + # Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored otherwise) + postgresql = { + architecture = "standalone" + auth = { + username = var.db_config.username + database = "litellm" + enablePostgresUser = true + + # A secret is created by this chart (litellm-helm) with the credentials that the new Postgres instance should use. + existingSecret = kubernetes_secret_v1.db-auth.metadata[0].name + secretKeys = { + adminPasswordKey = "postgres-password" + userPasswordKey = "password" } } - spec { - service_account_name = kubernetes_service_account.litellm.metadata[0].name - container { - name = var.name - image = "${var.image_repo}:${var.image_tag}" - command = split(" ", "litellm --port ${var.app_container_port} --config /app/${var.litellm_config_key} --detailed_debug") - dynamic "env" { - for_each = local.primary_env_vars - content { - name = env.key - value = tostring(env.value) - } - } - dynamic "env" { - for_each = local.secret_env_vars - content { - name = env.key - value_from { - secret_key_ref { - name = env.value.name - key = env.value.key - } - } - } - } - port { - container_port = var.app_container_port - name = "http" - protocol = "TCP" - } - resources { - limits = var.resource_limits - requests = var.resource_requests - } - volume_mount { - mount_path = "/app/${var.litellm_config_key}" - name = kubernetes_config_map.config.metadata[0].name - read_only = false - sub_path = var.litellm_config_key - } - volume_mount { - mount_path = "/app/${var.litellm_config_middleware_key}" - name = kubernetes_config_map.middleware.metadata[0].name - read_only = false - sub_path = var.litellm_config_middleware_key - } - volume_mount { - mount_path = var.gcloud_auth_file_path - name = kubernetes_secret.gcloud.metadata[0].name - read_only = true - sub_path = "" - } - } - volume { - name = kubernetes_config_map.config.metadata[0].name - config_map { - name = kubernetes_config_map.config.metadata[0].name - } - } - volume { - name = kubernetes_config_map.middleware.metadata[0].name - config_map { - name = kubernetes_config_map.middleware.metadata[0].name - } + } + + redis = { + enabled = false + architecture = "standalone" + } + + migrationJob = { + enabled = true # Enable or disable the schema migration Job + retries = 3 # Number of retries for the Job in case of failure + backoffLimit = 4 # Backoff limit for Job restarts + disableSchemaUpdate = false # Skip schema migrations for specific environments. When True, the job will exit with code 0. + annotations = {} + ttlSecondsAfterFinished = 120 + resources = {} + # requests: + # cpu: 100m + # memory: 100Mi + extraContainers = [] + + # Hook configuration + hooks = { + argocd = { + enabled = false } - volume { - name = kubernetes_secret.gcloud.metadata[0].name - secret { - secret_name = kubernetes_secret.gcloud.metadata[0].name - } + helm = { + enabled = false } } } - } + + + envVars = merge({ + NO_DOCS = "False" + }, merge(var.access_url != "" ? { + # PROXY_BASE_URL = "${var.access_url}" + } : {}, var.ssl_cert_config.create_secret ? { + SSL_CERT_FILE = "/tmp/ssl/${local.common_name}/tls-combined.pem" + SSL_KEYFILE_PATH = "/tmp/ssl/${local.common_name}/tls.key" + SSL_CERTFILE_PATH = "/tmp/ssl/${local.common_name}/tls.crt" + SSL_VERIFY = "False" + } : {})) + + extraEnvVars = {} + + # Additional volumes on the output Deployment definition. + volumes = [{ + name = kubernetes_manifest.cert[0].manifest.metadata.name + secret = { + secretName = kubernetes_manifest.cert[0].manifest.metadata.name + optional = false + } + },{ + name = kubernetes_secret_v1.gcloud.metadata[0].name + secret = { + secretName = kubernetes_secret_v1.gcloud.metadata[0].name + optional = false + } + }] + + # Additional volumeMounts on the output Deployment definition. + volumeMounts = [{ + name = kubernetes_manifest.cert[0].manifest.metadata.name + mountPath = "/tmp/ssl/${local.common_name}" + readOnly = true + },{ + name = kubernetes_secret_v1.gcloud.metadata[0].name + mountPath = "/tmp/gcloud/" + readOnly = true + },] + })] +} + +output "namespace" { + value = kubernetes_namespace_v1.litellm.metadata[0].name } \ No newline at end of file diff --git a/modules/k8s/bootstrap/litellm/policy.tf b/modules/k8s/bootstrap/litellm/policy.tf deleted file mode 100644 index 3238223..0000000 --- a/modules/k8s/bootstrap/litellm/policy.tf +++ /dev/null @@ -1,16 +0,0 @@ -data "aws_iam_policy_document" "bedrock-policy" { - statement { - sid = "AllowModelInvocation" - effect = "Allow" - actions = [ - "bedrock:InvokeModel", - "bedrock:InvokeModelWithResponseStream", - "bedrock:ListInferenceProfiles" - ] - resources = [ - "arn:aws:bedrock:*:*:*", - "arn:aws:bedrock:*:*:*/*", - "arn:aws:bedrock:*:*:*:*", - ] - } -} \ No newline at end of file diff --git a/modules/k8s/bootstrap/litellm/scripts/config.yaml b/modules/k8s/bootstrap/litellm/scripts/config.yaml deleted file mode 100644 index cb34afe..0000000 --- a/modules/k8s/bootstrap/litellm/scripts/config.yaml +++ /dev/null @@ -1,220 +0,0 @@ -model_list: - # AWS Bedrock - ## - # Ohio Models - ## - - model_name: anthropic.claude.haiku - model_info: - base_model: bedrock/us.anthropic.claude-3-haiku-20240307-v1:0 - litellm_params: - model: bedrock/us.anthropic.claude-3-haiku-20240307-v1:0 - aws_region_name: us-east-2 - rpm: 800 - tpm: 600000 - - model_name: anthropic.claude.sonnet - model_info: - base_model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0 - litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0 - aws_region_name: us-east-2 - rpm: 100 - tpm: 800000 - - model_name: anthropic.claude.haiku - model_info: - base_model: bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0 - litellm_params: - model: bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0 - aws_region_name: us-east-2 - rpm: 10 - tpm: 50000 - - model_name: anthropic.claude.sonnet - model_info: - base_model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0 - litellm_params: - model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0 - aws_region_name: us-east-2 - rpm: 4 - tpm: 10000 - - ## - # Oregon Models - ## - - model_name: anthropic.claude.haiku - model_info: - base_model: bedrock/us.anthropic.claude-3-haiku-20240307-v1:0 - litellm_params: - model: bedrock/us.anthropic.claude-3-haiku-20240307-v1:0 - aws_region_name: us-west-2 - rpm: 20 - tpm: 40000 - - model_name: anthropic.claude.haiku - model_info: - base_model: bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0 - litellm_params: - model: bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0 - aws_region_name: us-west-2 - rpm: 20 - tpm: 40000 - - model_name: anthropic.claude.sonnet - litellm_params: - model: bedrock/us.anthropic.claude-3-sonnet-20240229-v1:0 - aws_region_name: us-west-2 - rpm: 10 - tpm: 20000 - - model_name: anthropic.claude.sonnet - model_info: - base_model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0 - litellm_params: - model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0 - aws_region_name: us-west-2 - rpm: 4 - tpm: 10000 - - ## - # Virginia Models - ## - - model_name: anthropic.claude.haiku - model_info: - base_model: bedrock/us.anthropic.claude-3-haiku-20240307-v1:0 - litellm_params: - model: bedrock/us.anthropic.claude-3-haiku-20240307-v1:0 - aws_region_name: us-east-1 - rpm: 20 - tpm: 40000 - - model_name: anthropic.claude.haiku - model_info: - base_model: bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0 - litellm_params: - model: bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0 - aws_region_name: us-east-1 - rpm: 20 - tpm: 40000 - - model_name: anthropic.claude.sonnet - model_info: - base_model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0 - litellm_params: - model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0 - aws_region_name: us-east-1 - rpm: 4 - tpm: 10000 - - # GCP Vertex - - model_name: anthropic.claude.sonnet - model_info: - base_model: vertex_ai/claude-3-7-sonnet@20250219 - litellm_params: - model: vertex_ai/claude-3-7-sonnet@20250219 - vertex_project: coder-vertex-demos - vertex_location: europe-west1 - vertex_credentials: ${GCP_CRED_PATH} - rpm: 280 - tpm: 1500000 - - model_name: anthropic.claude.sonnet - model_info: - base_model: vertex_ai/claude-3-7-sonnet@20250219 - litellm_params: - model: vertex_ai/claude-3-7-sonnet@20250219 - vertex_project: coder-vertex-demos - vertex_location: us-east5 - vertex_credentials: ${GCP_CRED_PATH} - rpm: 385 - tpm: 2500000 - - model_name: anthropic.claude.haiku - model_info: - base_model: vertex_ai/claude-3-haiku@20240307 - litellm_params: - model: vertex_ai/claude-3-haiku@20240307 - vertex_project: coder-vertex-demos - vertex_location: europe-west1 - vertex_credentials: ${GCP_CRED_PATH} - rpm: 370 - tpm: 907000 - - model_name: anthropic.claude.haiku - model_info: - base_model: vertex_ai/claude-3-haiku@20240307 - litellm_params: - model: vertex_ai/claude-3-haiku@20240307 - vertex_project: coder-vertex-demos - vertex_location: us-east5 - vertex_credentials: ${GCP_CRED_PATH} - rpm: 1220 - tpm: 3000000 - - model_name: anthropic.claude.haiku - model_info: - base_model: vertex_ai/claude-3-haiku@20240307 - litellm_params: - model: vertex_ai/claude-3-haiku@20240307 - vertex_project: coder-vertex-demos - vertex_location: us-central1 - vertex_credentials: ${GCP_CRED_PATH} - rpm: 1220 - tpm: 3000000 - - model_name: anthropic.claude.sonnet - model_info: - base_model: vertex_ai/claude-3-5-sonnet@20240620 - litellm_params: - model: vertex_ai/claude-3-5-sonnet@20240620 - vertex_project: coder-vertex-demos - vertex_location: europe-west1 - vertex_credentials: ${GCP_CRED_PATH} - rpm: 650 - tpm: 3000000 - - model_name: anthropic.claude.sonnet - model_info: - base_model: vertex_ai/claude-3-5-sonnet@20240620 - litellm_params: - model: vertex_ai/claude-3-5-sonnet@20240620 - vertex_project: coder-vertex-demos - vertex_location: us-east5 - vertex_credentials: ${GCP_CRED_PATH} - rpm: 600 - tpm: 2775000 - - model_name: anthropic.claude.sonnet - model_info: - base_model: vertex_ai/claude-3-5-sonnet-v2@20241022 - litellm_params: - model: vertex_ai/claude-3-5-sonnet-v2@20241022 - vertex_project: coder-vertex-demos - vertex_location: europe-west1 - vertex_credentials: ${GCP_CRED_PATH} - rpm: 275 - tpm: 1670000 - - model_name: anthropic.claude.sonnet - model_info: - base_model: vertex_ai/claude-3-5-sonnet-v2@20241022 - litellm_params: - model: vertex_ai/claude-3-5-sonnet-v2@20241022 - vertex_project: coder-vertex-demos - vertex_location: us-east5 - vertex_credentials: ${GCP_CRED_PATH} - rpm: 450 - tpm: 2720000 - -litellm_settings: - num_retries: 2 - request_timeout: 45 - allowed_fails: 3 - cooldown_time: 30 - set_verbose: true - json_logs: false - cache: true - callbacks: - - strip_header_middleware.strip_header_callback - -general_settings: - store_model_in_db: true - store_prompts_in_spend_logs: true - proxy_batch_write_at: 60 - database_connection_pool_limit: 10 - - disable_error_logs: true - allow_requests_on_db_unavailable: true - -router_settings: - routing_strategy: usage-based-routing-v2 - num_retries: 2 - redis_host: os.environ/REDIS_HOST - redis_password: os.environ/REDIS_PASSWORD - redis_port: os.environ/REDIS_PORT - redis_ssl: os.environ/REDIS_SSL - timeout: 30 diff --git a/modules/k8s/bootstrap/litellm/scripts/strip_header_middleware.py b/modules/k8s/bootstrap/litellm/scripts/strip_header_middleware.py deleted file mode 100644 index 39bdff8..0000000 --- a/modules/k8s/bootstrap/litellm/scripts/strip_header_middleware.py +++ /dev/null @@ -1,33 +0,0 @@ -from litellm.integrations.custom_logger import CustomLogger -import litellm -from litellm.proxy.proxy_server import UserAPIKeyAuth, DualCache -from typing import Optional, Literal - -class HeaderHandler(CustomLogger): - def __init__(self): - pass - - async def async_pre_call_hook(self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict, call_type: Literal[ - "completion", - "text_completion", - "embeddings", - "image_generation", - "moderation", - "audio_transcription", - ]): - - v = data["proxy_server_request"]["headers"].pop("anthropic-beta", None) - if v not in [None, "claude-code-20250219"]: - data["proxy_server_request"]["headers"]["anthropic-beta"] = v - - v = data.get("provider_specific_header", {}).get("extra_headers", {}).pop("anthropic-beta", None) - if v not in [None, "claude-code-20250219"]: - data["provider_specific_header"]["extra_headers"]["anthropic-beta"] = v - - v = data.get("litellm_metadata", {}).get("headers", {}).pop("anthropic-beta", None) - if v not in [None, "claude-code-20250219"]: - data["litellm_metadata"]["headers"]["anthropic-beta"] = v - print(str(data)) - return data - -strip_header_callback = HeaderHandler() diff --git a/modules/k8s/bootstrap/metrics-server/main.tf b/modules/k8s/bootstrap/metrics-server/main.tf index 1ec1567..a7f5d70 100644 --- a/modules/k8s/bootstrap/metrics-server/main.tf +++ b/modules/k8s/bootstrap/metrics-server/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { helm = { source = "hashicorp/helm" - version = "2.17.0" + version = ">= 2.17.0" } } } @@ -40,8 +40,6 @@ resource "helm_release" "metrics-server" { timeout = 120 # in seconds values = [yamlencode({ - nodeSelector = { - "node.amazonaws.io/managed-by" : "asg" - } + nodeSelector = var.node_selector })] } \ No newline at end of file diff --git a/modules/k8s/objects/ec2nodeclass/main.tf b/modules/k8s/objects/ec2nodeclass/main.tf index dd56139..2b3b6bc 100644 --- a/modules/k8s/objects/ec2nodeclass/main.tf +++ b/modules/k8s/objects/ec2nodeclass/main.tf @@ -23,6 +23,11 @@ variable "sg_selector_tags" { default = {} } +variable "user_data" { + type = string + default = "" +} + variable "block_device_mappings" { type = list(object({ device_name = string @@ -69,6 +74,7 @@ output "manifest" { tags = var.sg_selector_tags }] blockDeviceMappings = local.block_device_mappings + userData = var.user_data } }) } \ No newline at end of file From 47ac0ce363fbe51525ee6f84fe5972b85c2ca0a0 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 20 Jan 2026 10:20:18 -0800 Subject: [PATCH 02/44] chore: remove efs module --- modules/k8s/bootstrap/efs-controller/main.tf | 106 ------------------- 1 file changed, 106 deletions(-) delete mode 100644 modules/k8s/bootstrap/efs-controller/main.tf diff --git a/modules/k8s/bootstrap/efs-controller/main.tf b/modules/k8s/bootstrap/efs-controller/main.tf deleted file mode 100644 index a18eeed..0000000 --- a/modules/k8s/bootstrap/efs-controller/main.tf +++ /dev/null @@ -1,106 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.17.0" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } -} - -## -# https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/install.md -## - -variable "cluster_name" { - type = string -} - -variable "role_name" { - type = string - default = "" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "tags" { - type = map(string) - default = {} -} - -variable "namespace" { - type = string -} - -variable "chart_version" { - type = string -} - -variable "service_account_annotations" { - type = map(string) - default = {} -} - -variable "replace" { - type = bool - default = false -} - -data "aws_region" "this" {} - -locals { - role_name = var.role_name == "" ? "efs-controller-${data.aws_region.this.region}" : var.role_name -} - -module "oidc-role" { - source = "../../../security/role/access-entry" - name = local.role_name - cluster_name = var.cluster_name - policy_arns = { - "AmazonEFSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" - } - cluster_policy_arns = { - "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" - } - oidc_principals = { - "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] - } - tags = var.tags -} - -resource "helm_release" "efs-controller" { - name = "aws-efs-csi-driver" - namespace = var.namespace - chart = "aws-efs-csi-driver" - repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver/" - create_namespace = true - upgrade_install = true - skip_crds = false - replace = var.replace - wait = true - wait_for_jobs = true - version = var.chart_version - timeout = 120 # in seconds - - values = [yamlencode({ - controller = { - serviceAccount = { - # https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/install.md - annotations = merge({ - "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn - }, var.service_account_annotations) - } - } - })] -} - -output "oidc_role_arn" { - value = module.oidc-role.role_arn -} \ No newline at end of file From b91ec6413a76942835e3a3da430047f648b9959a Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 20 Jan 2026 12:30:13 -0800 Subject: [PATCH 03/44] feat: adds 1-click solution --- infra/1-click/0-infra/0-vpc.tf | 156 ++ infra/1-click/0-infra/1-db.tf | 150 ++ infra/1-click/0-infra/2-s3.tf | 48 + infra/1-click/0-infra/3-eks.tf | 112 + infra/1-click/0-infra/4-bootstrap.tf | 110 + infra/1-click/0-infra/main.tf | 65 + infra/1-click/0-initialize.sh | 18 + infra/1-click/1-plan-n-deploy.sh | 36 + infra/1-click/1-setup/5-manifests.tf | 300 +++ infra/1-click/1-setup/6-coder-server.tf | 345 +++ .../1-setup/dashboards/aibridge-demoable.json | 1382 +++++++++++ .../1-setup/dashboards/aibridge-example.json | 1411 ++++++++++++ .../1-click/1-setup/dashboards/aibridge.json | 1080 +++++++++ infra/1-click/1-setup/dashboards/coderd.json | 1472 ++++++++++++ .../1-click/1-setup/dashboards/prebuilds.json | 1450 ++++++++++++ .../1-setup/dashboards/provisionerd.json | 1019 +++++++++ infra/1-click/1-setup/dashboards/status.json | 2038 +++++++++++++++++ .../1-setup/dashboards/workspace_detail.json | 1342 +++++++++++ .../1-setup/dashboards/workspaces.json | 1624 +++++++++++++ infra/1-click/1-setup/main.tf | 104 + infra/1-click/1-setup/scripts/add-license.sh | 32 + infra/1-click/1-setup/scripts/first-user.sh | 33 + infra/1-click/2-clean.sh | 28 + infra/1-click/2-coder/7-coder.tf | 91 + infra/1-click/2-coder/fetch.sh | 24 + infra/1-click/2-coder/main.tf | 87 + infra/1-click/README.md | 73 + infra/1-click/coder.env | 3 + 28 files changed, 14633 insertions(+) create mode 100644 infra/1-click/0-infra/0-vpc.tf create mode 100644 infra/1-click/0-infra/1-db.tf create mode 100644 infra/1-click/0-infra/2-s3.tf create mode 100644 infra/1-click/0-infra/3-eks.tf create mode 100644 infra/1-click/0-infra/4-bootstrap.tf create mode 100644 infra/1-click/0-infra/main.tf create mode 100755 infra/1-click/0-initialize.sh create mode 100755 infra/1-click/1-plan-n-deploy.sh create mode 100644 infra/1-click/1-setup/5-manifests.tf create mode 100644 infra/1-click/1-setup/6-coder-server.tf create mode 100644 infra/1-click/1-setup/dashboards/aibridge-demoable.json create mode 100644 infra/1-click/1-setup/dashboards/aibridge-example.json create mode 100644 infra/1-click/1-setup/dashboards/aibridge.json create mode 100644 infra/1-click/1-setup/dashboards/coderd.json create mode 100644 infra/1-click/1-setup/dashboards/prebuilds.json create mode 100644 infra/1-click/1-setup/dashboards/provisionerd.json create mode 100644 infra/1-click/1-setup/dashboards/status.json create mode 100644 infra/1-click/1-setup/dashboards/workspace_detail.json create mode 100644 infra/1-click/1-setup/dashboards/workspaces.json create mode 100644 infra/1-click/1-setup/main.tf create mode 100644 infra/1-click/1-setup/scripts/add-license.sh create mode 100755 infra/1-click/1-setup/scripts/first-user.sh create mode 100755 infra/1-click/2-clean.sh create mode 100644 infra/1-click/2-coder/7-coder.tf create mode 100755 infra/1-click/2-coder/fetch.sh create mode 100644 infra/1-click/2-coder/main.tf create mode 100644 infra/1-click/README.md create mode 100644 infra/1-click/coder.env diff --git a/infra/1-click/0-infra/0-vpc.tf b/infra/1-click/0-infra/0-vpc.tf new file mode 100644 index 0000000..9fbf7a3 --- /dev/null +++ b/infra/1-click/0-infra/0-vpc.tf @@ -0,0 +1,156 @@ +## +# Networking Infrastructure +## + +locals { + # Subnet Discovery - https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/subnet_discovery/#subnet-filtering + tags_lb_subnet_discovery = { + "kubernetes.io/cluster/${var.name}" = "owned" + } + # Internet-Facing LB - https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/subnet_discovery/#subnet-role-tag + tags_public_lb = { + "kubernetes.io/role/elb" = 1 + } + tags_public_subnet = merge( + local.tags_lb_subnet_discovery, + local.tags_public_lb + ) +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 6.6.0" + + name = var.name + + enable_nat_gateway = false + enable_dns_hostnames = true + + cidr = "10.0.0.0/16" + azs = ["${var.region}a", "${var.region}b", "${var.region}c"] + + ## + # Handle public subnets via the VPC module. + ## + public_subnets = ["10.0.8.0/21", "10.0.4.0/22", "10.0.0.0/22"] + public_subnet_tags = local.tags_public_subnet + + tags = local.tags_global +} + +module "nat-instance" { + source = "RaJiska/fck-nat/aws" + + name = var.name + + vpc_id = module.vpc.vpc_id + subnet_id = module.vpc.public_subnets[0] + instance_type = "c6gn.medium" + ha_mode = true # Enables high-availability mode + # eip_allocation_ids = ["eipalloc-abc1234"] # Allocation ID of an existing EIP + use_cloudwatch_agent = true # Enables Cloudwatch agent and have metrics reported + + update_route_tables = true + route_tables_ids = {} + + tags = local.tags_global +} + +## +# Coder Subnets +## + +locals { + # Karpenter Subnet Discovery - https://karpenter.sh/v1.0/concepts/nodeclasses/#specsubnetselectorterms + tags_kptr_subnet_discovery = { + "karpenter.sh/discovery" = var.name + } + tags_private_subnet = merge( + local.tags_lb_subnet_discovery, + local.tags_kptr_subnet_discovery + ) +} + +# 4096 - 5 (reserved by AWS) IPs each + +module "general-subnet-az1" { + source = "../../../modules/network/subnet/private" + name = var.name + vpc_id = module.vpc.vpc_id + eni_id = module.nat-instance.eni_id + cidr_block = "10.0.64.0/20" + availability_zone = "${var.region}a" + subnet_tags = local.tags_private_subnet + tags = local.tags_global +} + +module "general-subnet-az2" { + source = "../../../modules/network/subnet/private" + name = var.name + vpc_id = module.vpc.vpc_id + eni_id = module.nat-instance.eni_id + cidr_block = "10.0.80.0/20" + availability_zone = "${var.region}b" + subnet_tags = local.tags_private_subnet + tags = local.tags_global +} + +module "general-subnet-az3" { + source = "../../../modules/network/subnet/private" + name = var.name + vpc_id = module.vpc.vpc_id + eni_id = module.nat-instance.eni_id + cidr_block = "10.0.96.0/20" + availability_zone = "${var.region}c" + subnet_tags = local.tags_private_subnet + tags = local.tags_global +} + +## +# Kubernetes System Subnets +## + +module "system-subnet-az1" { + source = "../../../modules/network/subnet/private" + name = var.name + vpc_id = module.vpc.vpc_id + eni_id = module.nat-instance.eni_id + cidr_block = "10.0.16.0/20" + availability_zone = "${var.region}a" + subnet_tags = local.tags_private_subnet + tags = local.tags_global +} + +module "system-subnet-az2" { + source = "../../../modules/network/subnet/private" + name = var.name + vpc_id = module.vpc.vpc_id + eni_id = module.nat-instance.eni_id + cidr_block = "10.0.32.0/20" + availability_zone = "${var.region}b" + subnet_tags = local.tags_private_subnet + tags = local.tags_global +} + +module "system-subnet-az3" { + source = "../../../modules/network/subnet/private" + name = var.name + vpc_id = module.vpc.vpc_id + eni_id = module.nat-instance.eni_id + cidr_block = "10.0.48.0/20" + availability_zone = "${var.region}c" + subnet_tags = local.tags_private_subnet + tags = local.tags_global +} + +locals { + private_subnet_ids = [ + module.general-subnet-az1.subnet_id, + module.general-subnet-az2.subnet_id, + module.general-subnet-az3.subnet_id, + module.system-subnet-az1.subnet_id, + module.system-subnet-az2.subnet_id, + module.system-subnet-az3.subnet_id + ] + public_subnet_ids = concat([], module.vpc.public_subnets) +} \ No newline at end of file diff --git a/infra/1-click/0-infra/1-db.tf b/infra/1-click/0-infra/1-db.tf new file mode 100644 index 0000000..917ef7f --- /dev/null +++ b/infra/1-click/0-infra/1-db.tf @@ -0,0 +1,150 @@ +## +# Database Infrastructure +## + +variable "coder_username" { + description = "Coder DB's username." + type = string + default = "coder" +} + +variable "coder_password" { + description = "Coder DB's password." + type = string + sensitive = true + default = "th1s1sn0tas3cur3pass0wrd" +} + +variable "litellm_username" { + description = "LiteLLM DB's username." + type = string + default = "litellm" +} + +variable "litellm_password" { + description = "LiteLLM DB's password." + type = string + sensitive = true + default = "th1s1sn0tas3cur3pass0wrd" +} + +variable "grafana_username" { + description = "Grafana DB's username." + type = string + default = "grafana" +} + +variable "grafana_password" { + description = "Grafana DB's password." + type = string + sensitive = true + default = "th1s1sn0tas3cur3pass0wrd" +} + +resource "aws_security_group" "postgres" { + vpc_id = module.vpc.vpc_id + name = "postgres" + description = "security group for postgres all egress traffic" + tags = { + Name = "PostgreSQL" + } +} + +resource "aws_vpc_security_group_ingress_rule" "postgres" { + security_group_id = aws_security_group.postgres.id + cidr_ipv4 = module.vpc.vpc_cidr_block + ip_protocol = "tcp" + from_port = 5432 + to_port = 5432 +} + +resource "aws_vpc_security_group_egress_rule" "postgres" { + security_group_id = aws_security_group.postgres.id + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = -1 +} + +resource "aws_db_subnet_group" "coder" { + name = "coder" + subnet_ids = local.private_subnet_ids + + tags = { + Name = "coder" + } +} + +resource "aws_db_instance" "coder" { + identifier = "coder" + instance_class = "db.t4g.large" + allocated_storage = 50 + engine = "postgres" + engine_version = "15.12" + # backup_retention_period = 7 + username = var.coder_username + password = var.coder_password + db_name = "coder" + db_subnet_group_name = aws_db_subnet_group.coder.name + vpc_security_group_ids = [aws_security_group.postgres.id] + publicly_accessible = false + skip_final_snapshot = true + # final_snapshot_identifier = "coder-final" + + tags = { + Name = "coder" + } + lifecycle { + ignore_changes = [ + snapshot_identifier + ] + } +} + +resource "aws_db_instance" "litellm" { + identifier = "litellm" + instance_class = "db.t4g.medium" + allocated_storage = 50 + engine = "postgres" + engine_version = "15.12" + username = var.litellm_username + password = var.litellm_password + db_name = "litellm" + db_subnet_group_name = aws_db_subnet_group.coder.name + vpc_security_group_ids = [aws_security_group.postgres.id] + publicly_accessible = false + skip_final_snapshot = true + # final_snapshot_identifier = "litellm-final" + + tags = { + Name = "litellm" + } + lifecycle { + ignore_changes = [ + snapshot_identifier + ] + } +} + +resource "aws_db_instance" "grafana" { + identifier = "grafana" + instance_class = "db.t4g.large" + allocated_storage = 50 + engine = "postgres" + engine_version = "15.12" + username = var.grafana_username + password = var.grafana_password + db_name = "grafana" + db_subnet_group_name = aws_db_subnet_group.coder.name + vpc_security_group_ids = [aws_security_group.postgres.id] + publicly_accessible = false + skip_final_snapshot = true + # final_snapshot_identifier = "grafana-final" + + tags = { + Name = "grafana" + } + lifecycle { + ignore_changes = [ + snapshot_identifier + ] + } +} \ No newline at end of file diff --git a/infra/1-click/0-infra/2-s3.tf b/infra/1-click/0-infra/2-s3.tf new file mode 100644 index 0000000..e385a1c --- /dev/null +++ b/infra/1-click/0-infra/2-s3.tf @@ -0,0 +1,48 @@ +## +# Bucket Infrastructure +## + +## +# Loki Inputs +## + +variable "loki_s3_bucket_name" { + type = string + default = "" +} + +variable "loki_s3_bucket_tags" { + type = map(string) + default = {} +} + +resource "random_string" "bucket" { + keepers = { + static = "true" + } + length = 16 + special = false + upper = false + lower = true +} + +resource "aws_s3_bucket" "loki" { + bucket = var.loki_s3_bucket_name == "" ? "granafa-logs-${random_string.bucket.result}" : var.loki_s3_bucket_name + tags = var.loki_s3_bucket_tags +} + +resource "aws_s3_bucket_lifecycle_configuration" "loki" { + bucket = aws_s3_bucket.loki.id + + rule { + id = "logs" + + filter {} # Target all objects in bucket. + + expiration { + days = 365 + } + + status = "Enabled" + } +} \ No newline at end of file diff --git a/infra/1-click/0-infra/3-eks.tf b/infra/1-click/0-infra/3-eks.tf new file mode 100644 index 0000000..961931b --- /dev/null +++ b/infra/1-click/0-infra/3-eks.tf @@ -0,0 +1,112 @@ +## +# Kubernetes Infrastructure +## + +data "aws_iam_policy_document" "sts" { + statement { + effect = "Allow" + actions = ["sts:*"] + resources = ["*"] + } +} + +resource "aws_iam_policy" "sts" { + name_prefix = "sts" + path = "/" + description = "Assume Role Policy" + policy = data.aws_iam_policy_document.sts.json + tags = local.tags_global +} + +locals { + # Karpenter Security Group Discovery - https://karpenter.sh/v1.0/concepts/nodeclasses/#specsecuritygroupselectorterms + tags_kptr_sg_discovery = { + "karpenter.sh/discovery" = var.name + } + labels_system_node = { + "scheduling.coder.com/pool" = "system" + } +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 21.14.0" + + vpc_id = module.vpc.vpc_id + + # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster#vpc_config-1 + # https://docs.aws.amazon.com/eks/latest/userguide/network-reqs.html#network-requirements-subnets + subnet_ids = toset(concat( + # local.public_subnet_ids, + local.private_subnet_ids + )) + + name = var.name + + kubernetes_version = "1.34" + endpoint_public_access = true + endpoint_private_access = true + + create_security_group = true + create_node_security_group = true + create_iam_role = true + node_security_group_tags = local.tags_kptr_sg_discovery + + compute_config = { + # Disables EKS Auto Mode. Manually handle scaling via Karpenter + enabled = false + } + + attach_encryption_policy = false + create_kms_key = false # Enable unless needed + encryption_config = null + enable_cluster_creator_admin_permissions = true + enable_irsa = true + + eks_managed_node_groups = { + # Initial nodes are dedicated to system-level processes + system = { + min_size = 0 + max_size = 10 + desired_size = 3 # Ignored after creation. Override from AWS Console as needed. + + # K8s Labels - https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group#labels-1 + labels = local.labels_system_node + + instance_types = ["t3.xlarge"] + capacity_type = "ON_DEMAND" + iam_role_additional_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + STSAssumeRole = aws_iam_policy.sts.arn + } + metadata_options = { + http_endpoint = "enabled" + http_put_response_hop_limit = 2 + http_tokens = "required" + } + + # System Nodes should not be public + subnet_ids = local.private_subnet_ids + } + } + + addons = { + coredns = { + before_compute = true + } + kube-proxy = { + before_compute = true + } + vpc-cni = { + before_compute = true + configuration_values = jsonencode({ + enableNetworkPolicy = "true" + nodeAgent = { + enablePolicyEventLogs = "true" + } + }) + } + } + + tags = local.tags_global +} \ No newline at end of file diff --git a/infra/1-click/0-infra/4-bootstrap.tf b/infra/1-click/0-infra/4-bootstrap.tf new file mode 100644 index 0000000..0ba5c42 --- /dev/null +++ b/infra/1-click/0-infra/4-bootstrap.tf @@ -0,0 +1,110 @@ +variable "domain_name" { + type = string +} + +## +# System-Level Addons +## + +module "karpenter" { + source = "../../../modules/k8s/bootstrap/karpenter" + + cluster_name = module.eks.cluster_name + cluster_oidc_provider_arn = module.eks.oidc_provider_arn + cluster_oidc_provider = module.eks.oidc_provider + + namespace = "karpenter" + chart_version = "1.8.4" + node_selector = local.labels_system_node + + iam_role_use_name_prefix = false + node_iam_role_use_name_prefix = false + karpenter_controller_role_policies = { + "AmazonEFSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" + } +} + +module "metrics-server" { + + depends_on = [ module.karpenter ] + + source = "../../../modules/k8s/bootstrap/metrics-server" + + namespace = "metrics-server" + chart_version = "3.13.0" + node_selector = local.labels_system_node +} + +module "cert-manager" { + + depends_on = [ module.karpenter ] + + source = "../../../modules/k8s/bootstrap/cert-manager" + cluster_name = module.eks.cluster_name + cluster_oidc_provider_arn = module.eks.oidc_provider_arn + + namespace = "cert-manager" + helm_version = "v1.18.2" + use_cloudflare = false + use_route53 = true + create_default_cluster_issuer = false +} + +module "lb-controller" { + + depends_on = [ module.cert-manager, module.karpenter ] + + source = "../../../modules/k8s/bootstrap/lb-controller" + cluster_name = module.eks.cluster_name + cluster_oidc_provider_arn = module.eks.oidc_provider_arn + + namespace = "lb-crtl" + chart_version = "1.13.2" + enable_cert_manager = true + vpc_id = module.vpc.vpc_id + service_target_eni_sg_tags = { + Name = module.eks.cluster_name + } + create_alb_class = false + node_selector = local.labels_system_node +} + +module "ebs-controller" { + + depends_on = [ module.cert-manager, module.karpenter ] + + source = "../../../modules/k8s/bootstrap/ebs-controller" + cluster_name = module.eks.cluster_name + cluster_oidc_provider_arn = module.eks.oidc_provider_arn + + namespace = "ebs-crtl" + chart_version = "2.22.1" + node_selector = local.labels_system_node + replace = true +} + +module "external-dns" { + + depends_on = [ module.cert-manager, module.karpenter ] + + source = "../../../modules/k8s/bootstrap/external-dns" + cluster_name = module.eks.cluster_name + cluster_oidc_provider_arn = module.eks.oidc_provider_arn + + namespace = "external-dns" + chart_version = "1.20.0" + domain_name = var.domain_name + node_selector = local.labels_system_node +} + +module "external-secrets" { + + depends_on = [ module.cert-manager ] + + source = "../../../modules/k8s/bootstrap/external-secrets" + cluster_name = module.eks.cluster_name + cluster_oidc_provider_arn = module.eks.oidc_provider_arn + + namespace = "external-secrets" + chart_version = "1.2.1" +} \ No newline at end of file diff --git a/infra/1-click/0-infra/main.tf b/infra/1-click/0-infra/main.tf new file mode 100644 index 0000000..94b5e95 --- /dev/null +++ b/infra/1-click/0-infra/main.tf @@ -0,0 +1,65 @@ +terraform { + required_version = ">= 1.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.46" + } + helm = { + source = "hashicorp/helm" + version = ">= 3.1.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + # backend "s3" {} +} + +## +# Global Inputs + Providers +## + +variable "region" { + description = "The AWS region" + type = string + default = "us-east-2" +} + +variable "name" { + description = "Name for created resources and tag prefix" + type = string + default = "coder" +} + +variable "profile" { + type = string + default = "default" +} + +data "aws_eks_cluster_auth" "coder" { + name = module.eks.cluster_name +} + +provider "aws" { + region = var.region + profile = var.profile +} + +provider "helm" { + kubernetes = { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.coder.token + } +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.coder.token +} + +locals { + tags_global = {} +} \ No newline at end of file diff --git a/infra/1-click/0-initialize.sh b/infra/1-click/0-initialize.sh new file mode 100755 index 0000000..5441342 --- /dev/null +++ b/infra/1-click/0-initialize.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +echo "Change directory into '0-infra'." +cd 0-infra +terraform init +cd ../ + +echo "Change directory into '1-setup'." +cd 1-setup +terraform init +cd ../ + +echo "Change directory into '2-coder'." +cd 2-coder +terraform init +cd ../ \ No newline at end of file diff --git a/infra/1-click/1-plan-n-deploy.sh b/infra/1-click/1-plan-n-deploy.sh new file mode 100755 index 0000000..dd9ec8b --- /dev/null +++ b/infra/1-click/1-plan-n-deploy.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -ae -o pipefail + +AWS_PROFILE="${CODER_AWS_PROFILE:-default}" +DOMAIN_NAME="${CODER_DOMAIN_NAME:-}" +LICENSE="${CODER_LICENSE:-}" + +if [ -z "${DOMAIN_NAME}" ]; then + echo "A domain name is required! Be sure to register or use an existing one from Route53!" + exit 1; +fi + +echo "Change directory into '0-infra'." +cd 0-infra +terraform plan -out=tf.plan \ + -var profile=$AWS_PROFILE \ + -var domain_name=$DOMAIN_NAME +terraform apply tf.plan +cd ../ + +echo "Change directory into '1-setup'." +cd 1-setup +terraform plan -out=tf.plan \ + -var profile=$AWS_PROFILE \ + -var domain_name=$DOMAIN_NAME \ + -var coder_license=$LICENSE +terraform apply tf.plan +cd ../ + +# echo "Change directory into '2-coder'." +# cd 2-coder +# terraform plan -out=tf.plan \ +# -var profile="one-click" \ +# -var domain_name="oneclick-jullian.click" +# cd ../ \ No newline at end of file diff --git a/infra/1-click/1-setup/5-manifests.tf b/infra/1-click/1-setup/5-manifests.tf new file mode 100644 index 0000000..b9a8cc2 --- /dev/null +++ b/infra/1-click/1-setup/5-manifests.tf @@ -0,0 +1,300 @@ +## +# Manifest Setup Post Addon-Deployment +# Includes auxiliary resources depending on CRDs +## + +## +# NodeClass + NodePool for Coder Server, Provisioner, & Workspaces +## + +locals { + global_node_labels = { + "node.coder.io/instance" = "coder-v2" + "node.coder.io/managed-by" = "karpenter" + } + global_node_reqs = [{ + key = "kubernetes.io/arch" + operator = "In" + values = ["amd64"] + }, { + key = "kubernetes.io/os" + operator = "In" + values = ["linux"] + }, { + key = "kubernetes.sh/capacity-type" + operator = "In" + values = ["spot", "on-demand"] + }] + sg_tags = { + "karpenter.sh/discovery" = var.name + } + subnet_tags = { + "karpenter.sh/discovery" = var.name + } +} + +locals { + node_role_name = data.aws_iam_role.kptr-node-role.name + nodeclass_configs = { + "coder-server" = { + user_data = "" + block_device_mappings = [] + } + "coder-ws" = { + user_data = <<-EOF + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + config: + registryPullQPS: 30 + EOF + block_device_mappings = [{ + deviceName = "/dev/xvda" + ebs = { + volumeSize = "500Gi" + volumeType = "gp3" + encrypted = false + deleteOnTermination = true + } + }] + } + "coder-provisioner" = { + user_data = "" + block_device_mappings = [] + } + } +} + +resource "kubernetes_manifest" "nodeclass" { + + for_each = local.nodeclass_configs + + manifest = { + apiVersion = "karpenter.k8s.aws/v1" + kind = "EC2NodeClass" + metadata = { + name = each.key + } + spec = { + role = local.node_role_name + amiSelectorTerms = [{ + alias = "al2023@latest" + }] + subnetSelectorTerms = [{ + tags = local.subnet_tags + }] + securityGroupSelectorTerms = [{ + tags = local.sg_tags + }] + blockDeviceMappings = each.value.block_device_mappings + userData = each.value.user_data + } + } +} + +locals { + nodepool_configs = { + "coder-server" = { + node_expires_after = "Never" + disruption_consolidation_policy = "WhenEmpty" + disruption_consolidate_after = "1m" + } + "coder-provisioner" = { + node_expires_after = "Never" + disruption_consolidation_policy = "WhenEmpty" + disruption_consolidate_after = "1m" + } + "coder-ws" = { + node_expires_after = "Never" + disruption_consolidation_policy = "WhenEmpty" + disruption_consolidate_after = "30m" + } + } +} + +resource "kubernetes_manifest" "nodepool" { + + depends_on = [ kubernetes_manifest.nodeclass ] + + for_each = local.nodepool_configs + + field_manager { + force_conflicts = true + } + + wait { + condition { + type = "Ready" + status = "True" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "karpenter.sh/v1" + kind = "NodePool" + metadata = { + name = each.key + } + spec = { + template = { + metadata = { + labels = merge(local.global_node_labels, { + "node.coder.io/name" = "coder" + "node.coder.io/part-of" = "coder" + "node.coder.io/used-for" = each.key + }) + } + spec = { + taints = [{ + key = "dedicated" + value = each.key + effect = "NoSchedule" + }] + requirements =concat(local.global_node_reqs, [{ + key = "node.kubernetes.io/instance-type" + operator = "In" + values = ["t3a.xlarge"] + }]) + nodeClassRef = { + group = "karpenter.k8s.aws" + kind = "EC2NodeClass" + name = each.key + } + expireAfter = each.value.node_expires_after + } + } + disruption = { + consolidationPolicy = each.value.disruption_consolidation_policy + consolidateAfter = each.value.disruption_consolidate_after + } + } + } +} + +## +# Cert-Manager ClusterIssuer for CA +## + +data "kubernetes_service_account_v1" "r53" { + metadata { + name = "cert-manager-acme-dns01-route53" + namespace = "cert-manager" + } +} + +resource "kubernetes_manifest" "default-issuer" { + + field_manager { + force_conflicts = true + } + + wait { + condition { + type = "Ready" + status = "True" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "ClusterIssuer" + metadata = { + labels = {} + name = "issuer" + } + spec = { + acme = { + privateKeySecretRef = { + name = "issuer-account-key" + } + server = "https://acme-v02.api.letsencrypt.org/directory" + solvers = [ + { + dns01 = { + route53 = { + region = var.region + role = data.kubernetes_service_account_v1.r53.metadata[0].annotations["eks.amazonaws.com/role-arn"] + auth = { + kubernetes = { + serviceAccountRef = { + name = data.kubernetes_service_account_v1.r53.metadata[0].name + } + } + } + } + } + } + ] + } + } + } +} + +## +# External-Secrets ClusterStore for syncing TLS/SSL +## + +data "kubernetes_service_account_v1" "sm" { + metadata { + name = "external-secrets" + namespace = "external-secrets" + } +} + +resource "kubernetes_manifest" "secret-store" { + + field_manager { + force_conflicts = true + } + + wait { + condition { + type = "Ready" + status = "True" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "external-secrets.io/v1" + kind = "ClusterSecretStore" + metadata = { + labels = {} + name = "issuer" + } + spec = { + provider = { + aws = { + service = "SecretsManager" + region = "us-east-2" + auth = { + jwt = { + serviceAccountRef = { + name = data.kubernetes_service_account_v1.sm.metadata[0].name + namespace = data.kubernetes_service_account_v1.sm.metadata[0].namespace + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/infra/1-click/1-setup/6-coder-server.tf b/infra/1-click/1-setup/6-coder-server.tf new file mode 100644 index 0000000..a0fe762 --- /dev/null +++ b/infra/1-click/1-setup/6-coder-server.tf @@ -0,0 +1,345 @@ +## +# Coder Helm Chart Installation w/ auxillary dependcies on: +# - cert-manager +# - karpenter +# - external-dns +# - external-secrets +## + +variable "domain_name" { + type = string +} + +variable "coder_username" { + description = "Coder DB's username." + type = string + default = "coder" +} + +variable "coder_password" { + description = "Coder DB's password." + type = string + sensitive = true + default = "th1s1sn0tas3cur3pass0wrd" +} + +variable "coder_license" { + type = string + sensitive = true + default = "" +} + +variable "coder_admin_email" { + type = string + default = "admin@coder.com" +} + +variable "coder_admin_username" { + type = string + default = "admin" +} + +variable "coder_admin_password" { + type = string + sensitive = true + default = "Th1s1sN0TS3CuR3!!" +} + +module "coder-server" { + + depends_on = [ kubernetes_manifest.nodepool ] + + source = "../../../modules/k8s/bootstrap/coder-server" + + cluster_name = var.name + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn + + namespace = "coder" + + replica_count = 2 + helm_version = "2.29.1" + image_repo = "ghcr.io/coder/coder" + image_tag = "v2.29.1" + primary_access_url = "https://${var.domain_name}" + wildcard_access_url = "*.${var.domain_name}" + db_secret_url = "postgresql://${var.coder_username}:${var.coder_password}@${data.aws_db_instance.coder.endpoint}/coder" + + coder_builtin_provisioner_count = 0 + # coder_github_allowed_orgs = var.coder_github_allowed_orgs + + ssl_cert_config = { + name = var.domain_name + caissuer = kubernetes_manifest.default-issuer.manifest.metadata.name + secretissuer = kubernetes_manifest.secret-store.manifest.metadata.name + create_secret = true + } + + enable_oidc = false + enable_oauth = false + enable_github_external_auth = false + + tags = {} + + service_annotations = { + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" + "external-dns.alpha.kubernetes.io/hostname" = "${var.domain_name}" + "external-dns.alpha.kubernetes.io/ttl" = 30 + } + + node_selector = kubernetes_manifest.nodepool["coder-server"].manifest.spec.template.metadata.labels + tolerations = [ for toleration in kubernetes_manifest.nodepool["coder-server"].manifest.spec.template.spec.taints : { + key = toleration.key + operator = "Equal" + value = toleration.value + effect = toleration.effect + }] + + topology_spread_constraints = [{ + max_skew = 1 + topology_key = "kubernetes.io/hostname" + when_unsatisfiable = "ScheduleAnyway" + label_selector = { + match_labels = { + "app.kubernetes.io/name" = "coder" + "app.kubernetes.io/part-of" = "coder" + } + } + match_label_keys = [ + "app.kubernetes.io/instance" + ] + }] + pod_anti_affinity_preferred_during_scheduling_ignored_during_execution = [{ + weight = 100 + pod_affinity_term = { + label_selector = { + match_labels = { + "app.kubernetes.io/instance" = "coder-v2" + "app.kubernetes.io/name" = "coder" + "app.kubernetes.io/part-of" = "coder" + } + } + topology_key = "kubernetes.io/hostname" + } + }] +} + +# Wait for DNS propagation. May require multiple redeploys +resource "time_sleep" "wait_for_dns" { + create_duration = "300s" + depends_on = [ module.coder-server ] +} + +data "external" "first-user" { + + depends_on = [ time_sleep.wait_for_dns ] + + program = ["bash", "${path.module}/scripts/first-user.sh"] + + query = { + access_url = "https://${var.domain_name}" + admin_email = var.coder_admin_email + admin_username = var.coder_admin_username + admin_password = var.coder_admin_password + } + +} + +output "coder_session_token" { + value = data.external.first-user.result.session_token +} + +data "external" "add-license" { + + count = var.coder_license != "" ? 1 : 0 + + depends_on = [ time_sleep.wait_for_dns ] + + program = ["bash", "${path.module}/scripts/add-license.sh"] + + query = { + access_url = "https://${var.domain_name}" + license_key = var.coder_license + session_token = data.external.first-user.result.session_token + } + +} + +# locals { +# dashboards-path = "${path.module}/dashboards" +# # coderd_selector = "pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`${local.coderd_namespace}`" +# coderd_selector = "pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=~`(coder)`" + +# provisionerd_selector = "pod=~`coder-provisioner.*`, namespace=~`(coder-ws|coder-ws-experiment|coder-ws-demo)`" + +# # workspaces_selector = "namespace=`coder-ws*`" +# workspaces_selector = "pod!~`coder.*`, namespace=~`(coder-ws|coder-ws-experiment|coder-ws-demo)`" +# non_workspaces_selector = "namespace=~`(coder|coder-ws|coder-ws-experiment|coder-ws-demo)`" + +# dashboard_timerange = "12h" +# dashboard_refresh = "30s" +# } + +# module "monitoring" { +# source = "../../../modules/k8s/bootstrap/monitoring" + +# namespace = var.addon_namespace +# helm_coder_observability_version = var.helm_coder_observability_version +# helm_coder_observability_timeout = var.helm_coder_observability_timeout +# helm_prometheus_operator_version = var.helm_prometheus_operator_version +# helm_prometheus_operator_timeout = var.helm_prometheus_operator_timeout +# cluster_name = var.cluster_name +# cluster_oidc_provider_arn = var.cluster_oidc_provider_arn + +# coder_db_username = var.coder_db_username +# coder_db_password = var.coder_db_password +# coder_db_host = var.coder_db_host +# coder_db_port = var.coder_db_port + +# loki_s3_chunk_bucket_name = var.loki_s3_chunk_bucket_name +# loki_s3_ruler_bucket_name = var.loki_s3_ruler_bucket_name +# loki_s3_bucket_region = var.loki_s3_bucket_region +# loki_iam_role_name = var.loki_iam_role_name +# loki_replicas = var.loki_replicas + +# grafana_security_key = var.grafana_security_key +# grafana_auth_username = var.grafana_auth_username +# grafana_auth_password = var.grafana_auth_password +# grafana_db_user = var.grafana_db_user +# grafana_db_name = var.grafana_db_name +# grafana_db_password = var.grafana_db_password +# grafana_db_host = var.grafana_db_host +# grafana_admin_username = var.grafana_admin_username +# grafana_admin_password = var.grafana_admin_password +# grafana_root_domain = var.grafana_root_domain +# grafana_subdomain = var.grafana_subdomain +# grafana_cert_name = var.grafana_cert_name +# grafana_cert_mnt_path = var.grafana_cert_mnt_path +# grafana_replicas = 2 +# grafana_service_annotations = { +# "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" +# "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" +# "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=true" +# } + +# dashboards = [ +# { +# name = "coder-dashboard-status" +# localPath = "${local.dashboards-path}/status.json" +# args = { +# HELM_NAMESPACE = var.addon_namespace +# CODERD_SELECTOR = local.coderd_selector +# PROVISIONERD_SELECTOR = local.provisionerd_selector +# WORKSPACES_SELECTOR = local.workspaces_selector +# PROMETHEUS_JOB = "${var.addon_namespace}/prometheus/server" +# LOKI_JOB = "${var.addon_namespace}/loki" +# GRAFANA_AGENT_JOB = "${var.addon_namespace}/grafana-agent/grafana-agent" +# } +# }, +# { +# name = "coder-dashboard-coderd" +# localPath = "${local.dashboards-path}/coderd.json" +# args = { +# DASHBOARD_TIMERANGE = local.dashboard_timerange +# DASHBOARD_REFRESH = local.dashboard_refresh +# CODERD_SELECTOR = local.coderd_selector +# } +# }, +# { +# name = "coder-dashboard-provisionerd" +# localPath = "${local.dashboards-path}/provisionerd.json" +# args = { +# DASHBOARD_TIMERANGE = local.dashboard_timerange +# DASHBOARD_REFRESH = local.dashboard_refresh +# PROVISIONERD_SELECTOR = local.provisionerd_selector +# NON_WORKSPACES_SELECTOR = local.non_workspaces_selector +# } +# }, +# { +# name = "coder-dashboard-workspaces" +# localPath = "${local.dashboards-path}/workspaces.json" +# args = { +# DASHBOARD_TIMERANGE = local.dashboard_timerange +# DASHBOARD_REFRESH = local.dashboard_refresh +# WORKSPACES_SELECTOR = local.workspaces_selector +# NON_WORKSPACES_SELECTOR = local.non_workspaces_selector +# } +# }, +# { +# name = "coder-dashboard-workspace-detail" +# localPath = "${local.dashboards-path}/workspace_detail.json" +# args = { +# DASHBOARD_TIMERANGE = local.dashboard_timerange +# DASHBOARD_REFRESH = local.dashboard_refresh +# WORKSPACES_SELECTOR = local.workspaces_selector +# NON_WORKSPACES_SELECTOR = local.non_workspaces_selector +# } +# }, +# { +# name = "coder-dashboard-prebuilds" +# localPath = "${local.dashboards-path}/prebuilds.json" +# args = { +# DASHBOARD_TIMERANGE = local.dashboard_timerange +# DASHBOARD_REFRESH = local.dashboard_refresh +# } +# }, +# { +# name = "coder-dashboard-aibridge" +# localPath = "${local.dashboards-path}/aibridge.json" +# args = {} +# }, +# # { +# # name = "coder-dashboard-proxyd" +# # localPath = "${local.dashboards-path}/proxyd.json" +# # args = { +# # DASHBOARD_TIMERANGE = local.dashboard_timerange +# # DASHBOARD_REFRESH = local.dashboard_refresh +# # CODERD_SELECTOR = local.coderd_selector +# # } +# # } +# ] + +# # service_annotations = { +# # "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" +# # "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" +# # "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=true" +# # } +# # node_selector = { +# # "node.coder.io/managed-by" = "karpenter" +# # "node.coder.io/used-for" = "coder" +# # } +# # tolerations = [{ +# # key = "dedicated" +# # operator = "Equal" +# # value = "coder" +# # effect = "NoSchedule" +# # }] +# # topology_spread_constraints = [{ +# # max_skew = 1 +# # topology_key = "kubernetes.io/hostname" +# # when_unsatisfiable = "ScheduleAnyway" +# # label_selector = { +# # match_labels = { +# # "app.kubernetes.io/name" = "coder" +# # "app.kubernetes.io/part-of" = "coder" +# # } +# # } +# # match_label_keys = [ +# # "app.kubernetes.io/instance" +# # ] +# # }] +# # pod_anti_affinity_preferred_during_scheduling_ignored_during_execution = [{ +# # weight = 100 +# # pod_affinity_term = { +# # label_selector = { +# # match_labels = { +# # "app.kubernetes.io/instance" = "coder-v2" +# # "app.kubernetes.io/name" = "coder" +# # "app.kubernetes.io/part-of" = "coder" +# # } +# # } +# # topology_key = "kubernetes.io/hostname" +# # } +# # }] +# } \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/aibridge-demoable.json b/infra/1-click/1-setup/dashboards/aibridge-demoable.json new file mode 100644 index 0000000..70d570f --- /dev/null +++ b/infra/1-click/1-setup/dashboards/aibridge-demoable.json @@ -0,0 +1,1382 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 138, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Usage leaderboards", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\norder by input desc", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total usage for $username", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "cache_creation_input": 3, + "cache_read_input": 2, + "input": 0, + "output": 1 + }, + "renameByName": { + "cache_creation_input": "Cache Write", + "cache_read_input": "Cache Read", + "input": "Input", + "output": "Ouput" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "output" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache Read" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Input" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache Write" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 20, + "x": 4, + "y": 1 + }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select u.username, sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\ngroup by u.username\norder by input desc", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Leaderboard per user", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "cache_creation_input": "Cache Write", + "cache_read_input": "Cache Read", + "input": "Input", + "output": "Output", + "username": "" + } + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Cost.*/" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + }, + { + "id": "decimals", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Total Cost" + } + ] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "columns": [], + "computed_columns": [], + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "dezthk9i9m48we" + }, + "filterExpression": "", + "filters": [], + "format": "table", + "global_query_id": "", + "hide": false, + "pagination_mode": "none", + "parser": "backend", + "refId": "A", + "root_selector": "$ ~> $each(function($v, $k) {\n {\n \"model\": $k,\n \"input_cost_per_token\": $v.input_cost_per_token ? $v.input_cost_per_token : 0,\n \"output_cost_per_token\": $v.output_cost_per_token ? $v.output_cost_per_token : 0,\n \"cache_creation_input_token_cost\": $v.cache_creation_input_token_cost ? $v.cache_creation_input_token_cost : 0,\n \"cache_read_input_token_cost\": $v.cache_read_input_token_cost ? $v.cache_read_input_token_cost : 0\n }\n})", + "source": "url", + "type": "json", + "url": "", + "url_options": { + "data": "", + "method": "GET" + } + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "select i.provider, i.model,\nsum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\ngroup by i.provider, i.model\norder by input desc", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Approximate Cost", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "model", + "mode": "inner" + } + }, + { + "id": "calculateField", + "options": { + "alias": "Input Cost", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "input_cost_per_token A" + } + }, + "operator": "*", + "right": { + "matcher": { + "id": "byName", + "options": "input" + } + } + }, + "mode": "binary", + "reduce": { + "include": [ + "input_cost_per_token A", + "input" + ], + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Output Cost", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "output_cost_per_token A" + } + }, + "operator": "*", + "right": { + "matcher": { + "id": "byName", + "options": "output" + } + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Cache Read Cost", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "cache_read_input_token_cost A" + } + }, + "operator": "*", + "right": { + "matcher": { + "id": "byName", + "options": "cache_read_input" + } + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Cache Write Cost", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "cache_creation_input_token_cost A" + } + }, + "operator": "*", + "right": { + "matcher": { + "id": "byName", + "options": "cache_creation_input" + } + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Total Cost", + "binary": { + "left": { + "matcher": { + "id": "byType", + "options": "number" + } + }, + "right": { + "fixed": "" + } + }, + "cumulative": { + "field": "Input Cost", + "reducer": "sum" + }, + "mode": "reduceRow", + "reduce": { + "include": [ + "Input Cost", + "Output Cost", + "Cache Read Cost", + "Cache Write Cost" + ], + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "cache_creation_input": false, + "cache_creation_input_token_cost A": true, + "cache_read_input": false, + "cache_read_input_token_cost A": true, + "input": false, + "input_cost_per_token A": true, + "output": false, + "output_cost_per_token A": true + }, + "includeByName": {}, + "indexByName": { + "Cache Read Cost": 13, + "Cache Write Cost": 14, + "Input Cost": 11, + "Output Cost": 12, + "Total Cost": 2, + "cache_creation_input": 10, + "cache_creation_input_token_cost A": 3, + "cache_read_input": 9, + "cache_read_input_token_cost A": 4, + "input": 7, + "input_cost_per_token A": 5, + "model": 1, + "output": 8, + "output_cost_per_token A": 6, + "provider": 0 + }, + "renameByName": { + "cache_creation_input": "Cache Write", + "cache_read_input": "Cache Read", + "input": "Input", + "model": "Model", + "output": "Output", + "provider": "Provider" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 10, + "panels": [], + "title": "Interceptions", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "output" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 4, + "x": 0, + "y": 23 + }, + "id": 5, + "interval": "1m", + "maxDataPoints": 500, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select count(*) from aibridge_interceptions\nWHERE started_at > $__timeFrom() AND started_at <= $__timeTo()\nAND provider ~ '${provider:regex}'\nAND model ~ '${model:regex}'", + "refId": "A", + "sql": { + "columns": [ + { + "name": "COUNT", + "parameters": [ + { + "name": "id", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "started_at", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "aibridge_interceptions" + } + ], + "title": "Total interceptions", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "output" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 20, + "x": 4, + "y": 23 + }, + "id": 4, + "maxDataPoints": 30, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 10 + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": -45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n$__timeGroupAlias(i.started_at, $__interval, NULL),\ncount(i.id) AS value,\nu.username AS metric\nFROM aibridge_interceptions i\njoin users u ON i.initiator_id = u.id\nWHERE\n$__timeFilter(i.started_at)\nAND u.username ~ '${username:regex}'\nAND i.provider ~ '${provider:regex}'\nAND i.model ~ '${model:regex}'\nGROUP BY u.username, $__timeGroup(i.started_at, $__interval)\nORDER BY $__timeGroup(i.started_at, $__interval)", + "refId": "A", + "sql": { + "columns": [ + { + "name": "COUNT", + "parameters": [ + { + "name": "id", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "started_at", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "aibridge_interceptions" + } + ], + "title": "Interceptions over time by user", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 9, + "panels": [], + "title": "Prompts & tool calls details", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Interception ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 357 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Model" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Provider" + }, + "properties": [ + { + "id": "custom.width", + "value": 157 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Username" + }, + "properties": [ + { + "id": "custom.width", + "value": 188 + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Created At" + } + ] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT i.id,\n u.username,\n i.provider,\n i.model,\n 'REDACTED' as prompt,\n p.created_at\nFROM aibridge_user_prompts p\nJOIN aibridge_interceptions i ON p.interception_id = i.id\nJOIN users u ON i.initiator_id = u.id\nWHERE $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\nORDER BY p.created_at DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "User Prompts", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "created_at": "Created At", + "id": "Interception ID", + "input": "Tool Input", + "invocation_error": "Tool Error", + "model": "Model", + "prompt": "Prompt", + "provider": "Provider", + "server_url": "MCP Server", + "tool": "Tool Name", + "username": "Username" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Tool Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 342 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "invocation_error" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "applyToRow": true, + "type": "color-background", + "wrapText": false + } + }, + { + "id": "noValue" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "match": "null", + "result": { + "color": "green", + "index": 0 + } + }, + "type": "special" + }, + { + "options": { + "pattern": ".+", + "result": { + "color": "red", + "index": 1 + } + }, + "type": "regex" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Tool Input" + }, + "properties": [ + { + "id": "custom.width", + "value": 309 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Interception ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 357 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Model" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Created At" + } + ] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select i.id, u.username, i.provider, i.model, t.server_url, t.tool, 'REDACTED' as input, CASE WHEN t.invocation_error IS NULL THEN '' ELSE 'REDACTED' END as invocation_error, t.created_at FROM aibridge_tool_usages t\njoin aibridge_interceptions i ON t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\nAND u.username ~ '${username:regex}'\nAND i.provider ~ '${provider:regex}'\nAND i.model ~ '${model:regex}'\norder by t.created_at desc", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Tool Calls", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "created_at": "Created At", + "id": "Interception ID", + "input": "Tool Input", + "invocation_error": "Tool Error", + "model": "Model", + "provider": "Provider", + "server_url": "MCP Server", + "tool": "Tool Name", + "username": "Username" + } + } + } + ], + "type": "table" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "allValue": ".+", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "definition": "select username from users where deleted=false;", + "description": "", + "includeAll": true, + "multi": true, + "name": "username", + "options": [], + "query": "select username from users where deleted=false;", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "definition": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", + "description": "", + "includeAll": true, + "multi": true, + "name": "provider", + "options": [], + "query": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "aezivxsm5pnggc" + }, + "definition": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '${provider:regex}' ORDER BY 1;", + "description": "", + "includeAll": true, + "multi": true, + "name": "model", + "options": [], + "query": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '${provider:regex}' ORDER BY 1;", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "aibridge [DEMOABLE]", + "uid": "0c61d33f-c809-4184-9e88-cb27e2d9d223", + "version": 2 +} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/aibridge-example.json b/infra/1-click/1-setup/dashboards/aibridge-example.json new file mode 100644 index 0000000..304d49a --- /dev/null +++ b/infra/1-click/1-setup/dashboards/aibridge-example.json @@ -0,0 +1,1411 @@ +{ + "__inputs": [ + { + "name": "DS_CODER-OBSERVABILITY-RO", + "label": "coder-observability-ro", + "description": "", + "type": "datasource", + "pluginId": "grafana-postgresql-datasource", + "pluginName": "PostgreSQL" + }, + { + "name": "DS_LITELLM-PRICING-DATA", + "label": "litellm-pricing-data", + "description": "", + "type": "datasource", + "pluginId": "yesoreyeram-infinity-datasource", + "pluginName": "Infinity" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.1.0" + }, + { + "type": "datasource", + "id": "grafana-postgresql-datasource", + "name": "PostgreSQL", + "version": "12.1.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "datasource", + "id": "yesoreyeram-infinity-datasource", + "name": "Infinity", + "version": "3.6.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Usage leaderboards", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "output" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache Read" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Input" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache Write" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 20, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select u.username, sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\ngroup by u.username\norder by input desc", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Leaderboard per user", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "cache_creation_input": "Cache Write", + "cache_read_input": "Cache Read", + "input": "Input", + "output": "Output", + "username": "" + } + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\norder by input desc", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total usage for $username", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "cache_creation_input": 3, + "cache_read_input": 2, + "input": 0, + "output": 1 + }, + "renameByName": { + "cache_creation_input": "Cache Write", + "cache_read_input": "Cache Read", + "input": "Input", + "output": "Output" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Cost.*/" + }, + "properties": [ + { + "id": "unit", + "value": "currencyUSD" + }, + { + "id": "decimals", + "value": 2 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": true + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Total Cost" + } + ] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "columns": [], + "computed_columns": [], + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "${DS_LITELLM-PRICING-DATA}" + }, + "filterExpression": "", + "filters": [], + "format": "table", + "global_query_id": "", + "hide": false, + "pagination_mode": "none", + "parser": "backend", + "refId": "A", + "root_selector": "$ ~> $each(function($v, $k) {\n {\n \"model\": $k,\n \"input_cost_per_token\": $v.input_cost_per_token ? $v.input_cost_per_token : 0,\n \"output_cost_per_token\": $v.output_cost_per_token ? $v.output_cost_per_token : 0,\n \"cache_creation_input_token_cost\": $v.cache_creation_input_token_cost ? $v.cache_creation_input_token_cost : 0,\n \"cache_read_input_token_cost\": $v.cache_read_input_token_cost ? $v.cache_read_input_token_cost : 0\n }\n})", + "source": "url", + "type": "json", + "url": "", + "url_options": { + "data": "", + "method": "GET" + } + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "select i.provider, i.model,\nsum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\ngroup by i.provider, i.model\norder by input desc", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Approximate Cost", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "model", + "mode": "inner" + } + }, + { + "id": "calculateField", + "options": { + "alias": "Input Cost", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "input_cost_per_token A" + } + }, + "operator": "*", + "right": { + "matcher": { + "id": "byName", + "options": "input" + } + } + }, + "mode": "binary", + "reduce": { + "include": ["input_cost_per_token A", "input"], + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Output Cost", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "output_cost_per_token A" + } + }, + "operator": "*", + "right": { + "matcher": { + "id": "byName", + "options": "output" + } + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Cache Read Cost", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "cache_read_input_token_cost A" + } + }, + "operator": "*", + "right": { + "matcher": { + "id": "byName", + "options": "cache_read_input" + } + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Cache Write Cost", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "cache_creation_input_token_cost A" + } + }, + "operator": "*", + "right": { + "matcher": { + "id": "byName", + "options": "cache_creation_input" + } + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Total Cost", + "binary": { + "left": { + "matcher": { + "id": "byType", + "options": "number" + } + }, + "right": { + "fixed": "" + } + }, + "cumulative": { + "field": "Input Cost", + "reducer": "sum" + }, + "mode": "reduceRow", + "reduce": { + "include": [ + "Input Cost", + "Output Cost", + "Cache Read Cost", + "Cache Write Cost" + ], + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "cache_creation_input": false, + "cache_creation_input_token_cost A": true, + "cache_read_input": false, + "cache_read_input_token_cost A": true, + "input": false, + "input_cost_per_token A": true, + "output": false, + "output_cost_per_token A": true + }, + "includeByName": {}, + "indexByName": { + "Cache Read Cost": 12, + "Cache Write Cost": 13, + "Input Cost": 10, + "Output Cost": 11, + "Total Cost": 14, + "cache_creation_input": 9, + "cache_creation_input_token_cost A": 2, + "cache_read_input": 8, + "cache_read_input_token_cost A": 3, + "input": 6, + "input_cost_per_token A": 4, + "model": 1, + "output": 7, + "output_cost_per_token A": 5, + "provider": 0 + }, + "renameByName": { + "cache_creation_input": "Cache Write", + "cache_read_input": "Cache Read", + "input": "Input", + "model": "Model", + "output": "Output", + "provider": "Provider" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 10, + "panels": [], + "title": "Interceptions", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "output" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 20, + "x": 0, + "y": 23 + }, + "id": 4, + "maxDataPoints": 30, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 10 + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": -45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n$__timeGroupAlias(i.started_at, $__interval, NULL),\ncount(i.id) AS value,\nu.username AS metric\nFROM aibridge_interceptions i\njoin users u ON i.initiator_id = u.id\nWHERE\n$__timeFilter(i.started_at)\nAND u.username ~ '${username:regex}'\nAND i.provider ~ '${provider:regex}'\nAND i.model ~ '${model:regex}'\nGROUP BY u.username, $__timeGroup(i.started_at, $__interval)\nORDER BY $__timeGroup(i.started_at, $__interval)", + "refId": "A", + "sql": { + "columns": [ + { + "name": "COUNT", + "parameters": [ + { + "name": "id", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "started_at", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "aibridge_interceptions" + } + ], + "title": "Interceptions over time by user", + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "output" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 4, + "x": 20, + "y": 23 + }, + "id": 5, + "interval": "1m", + "maxDataPoints": 500, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select count(*) from aibridge_interceptions\nWHERE started_at > $__timeFrom() AND started_at <= $__timeTo()\nAND provider ~ '${provider:regex}'\nAND model ~ '${model:regex}'", + "refId": "A", + "sql": { + "columns": [ + { + "name": "COUNT", + "parameters": [ + { + "name": "id", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "started_at", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "aibridge_interceptions" + } + ], + "title": "Total interceptions", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 9, + "panels": [], + "title": "Prompts & tool calls details", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Interception ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 357 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Model" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Provider" + }, + "properties": [ + { + "id": "custom.width", + "value": 157 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Username" + }, + "properties": [ + { + "id": "custom.width", + "value": 188 + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Created At" + } + ] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT i.id,\n u.username,\n i.provider,\n i.model,\n p.prompt,\n p.created_at\nFROM aibridge_user_prompts p\nJOIN aibridge_interceptions i ON p.interception_id = i.id\nJOIN users u ON i.initiator_id = u.id\nWHERE $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\nORDER BY p.created_at DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "User Prompts", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "created_at": "Created At", + "id": "Interception ID", + "input": "Tool Input", + "invocation_error": "Tool Error", + "model": "Model", + "prompt": "Prompt", + "provider": "Provider", + "server_url": "MCP Server", + "tool": "Tool Name", + "username": "Username" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Tool Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 342 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "invocation_error" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "applyToRow": true, + "type": "color-background", + "wrapText": false + } + }, + { + "id": "noValue" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "match": "null", + "result": { + "color": "green", + "index": 0 + } + }, + "type": "special" + }, + { + "options": { + "pattern": ".+", + "result": { + "color": "red", + "index": 1 + } + }, + "type": "regex" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Tool Input" + }, + "properties": [ + { + "id": "custom.width", + "value": 309 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Interception ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 357 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Model" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Created At" + } + ] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select i.id, u.username, i.provider, i.model, t.server_url, t.tool, t.input, t.invocation_error, t.created_at FROM aibridge_tool_usages t\njoin aibridge_interceptions i ON t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\nAND u.username ~ '${username:regex}'\nAND i.provider ~ '${provider:regex}'\nAND i.model ~ '${model:regex}'\norder by t.created_at desc", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Tool Calls", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "created_at": "Created At", + "id": "Interception ID", + "input": "Tool Input", + "invocation_error": "Tool Error", + "model": "Model", + "provider": "Provider", + "server_url": "MCP Server", + "tool": "Tool Name", + "username": "Username" + } + } + } + ], + "type": "table" + } + ], + "refresh": "1m", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "allValue": ".+", + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "definition": "select username from users where deleted=false;", + "description": "", + "includeAll": true, + "multi": true, + "name": "username", + "options": [], + "query": "select username from users where deleted=false;", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "definition": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", + "description": "", + "includeAll": true, + "multi": true, + "name": "provider", + "options": [], + "query": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "postgres" + }, + "definition": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '${provider:regex}' ORDER BY 1;", + "description": "", + "includeAll": true, + "multi": true, + "name": "model", + "options": [], + "query": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '${provider:regex}' ORDER BY 1;", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "aibridge", + "uid": "0c61d33f-c809-4184-9e88-cb27e2d9d224", + "version": 43, + "weekStart": "" +} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/aibridge.json b/infra/1-click/1-setup/dashboards/aibridge.json new file mode 100644 index 0000000..0d1fb72 --- /dev/null +++ b/infra/1-click/1-setup/dashboards/aibridge.json @@ -0,0 +1,1080 @@ +{ + "__inputs": [ + { + "name": "DS_CODER-OBSERVABILITY-RO", + "label": "coder-observability-ro", + "description": "", + "type": "datasource", + "pluginId": "grafana-postgresql-datasource", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.1.0" + }, + { + "type": "datasource", + "id": "grafana-postgresql-datasource", + "name": "PostgreSQL", + "version": "12.1.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Usage leaderboards", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "output" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache Read" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Input" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache Write" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 20, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select u.username, sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '$${username:regex}'\n AND i.provider ~ '$${provider:regex}'\n AND i.model ~ '$${model:regex}'\ngroup by u.username\norder by input desc", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Leaderboard per user", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "cache_creation_input": "Cache Write", + "cache_read_input": "Cache Read", + "input": "Input", + "output": "Output", + "username": "" + } + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '$${username:regex}'\n AND i.provider ~ '$${provider:regex}'\n AND i.model ~ '$${model:regex}'\norder by input desc", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total usage for $username", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "cache_creation_input": 3, + "cache_read_input": 2, + "input": 0, + "output": 1 + }, + "renameByName": { + "cache_creation_input": "Cache Write", + "cache_read_input": "Cache Read", + "input": "Input", + "output": "Output" + } + } + } + ], + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 10, + "panels": [], + "title": "Interceptions", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "output" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 20, + "x": 0, + "y": 23 + }, + "id": 4, + "maxDataPoints": 30, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 10 + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": -45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n$__timeGroupAlias(i.started_at, $__interval, NULL),\ncount(i.id) AS value,\nu.username AS metric\nFROM aibridge_interceptions i\njoin users u ON i.initiator_id = u.id\nWHERE\n$__timeFilter(i.started_at)\nAND u.username ~ '$${username:regex}'\nAND i.provider ~ '$${provider:regex}'\nAND i.model ~ '$${model:regex}'\nGROUP BY u.username, $__timeGroup(i.started_at, $__interval)\nORDER BY $__timeGroup(i.started_at, $__interval)", + "refId": "A", + "sql": { + "columns": [ + { + "name": "COUNT", + "parameters": [ + { + "name": "id", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "started_at", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "aibridge_interceptions" + } + ], + "title": "Interceptions over time by user", + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "output" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 4, + "x": 20, + "y": 23 + }, + "id": 5, + "interval": "1m", + "maxDataPoints": 500, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select count(*) from aibridge_interceptions\nWHERE started_at > $__timeFrom() AND started_at <= $__timeTo()\nAND provider ~ '$${provider:regex}'\nAND model ~ '$${model:regex}'", + "refId": "A", + "sql": { + "columns": [ + { + "name": "COUNT", + "parameters": [ + { + "name": "id", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "started_at", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "aibridge_interceptions" + } + ], + "title": "Total interceptions", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 9, + "panels": [], + "title": "Prompts & tool calls details", + "type": "row" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Interception ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 357 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Model" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Provider" + }, + "properties": [ + { + "id": "custom.width", + "value": 157 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Username" + }, + "properties": [ + { + "id": "custom.width", + "value": 188 + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Created At" + } + ] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT i.id,\n u.username,\n i.provider,\n i.model,\n p.prompt,\n p.created_at\nFROM aibridge_user_prompts p\nJOIN aibridge_interceptions i ON p.interception_id = i.id\nJOIN users u ON i.initiator_id = u.id\nWHERE $__timeFilter(i.started_at)\n AND u.username ~ '$${username:regex}'\n AND i.provider ~ '$${provider:regex}'\n AND i.model ~ '$${model:regex}'\nORDER BY p.created_at DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "User Prompts", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "created_at": "Created At", + "id": "Interception ID", + "input": "Tool Input", + "invocation_error": "Tool Error", + "model": "Model", + "prompt": "Prompt", + "provider": "Provider", + "server_url": "MCP Server", + "tool": "Tool Name", + "username": "Username" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Tool Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 342 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "invocation_error" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "applyToRow": true, + "type": "color-background", + "wrapText": false + } + }, + { + "id": "noValue" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "match": "null", + "result": { + "color": "green", + "index": 0 + } + }, + "type": "special" + }, + { + "options": { + "pattern": ".+", + "result": { + "color": "red", + "index": 1 + } + }, + "type": "regex" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Tool Input" + }, + "properties": [ + { + "id": "custom.width", + "value": 309 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Interception ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 357 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Model" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Created At" + } + ] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select i.id, u.username, i.provider, i.model, t.server_url, t.tool, t.input, t.invocation_error, t.created_at FROM aibridge_tool_usages t\njoin aibridge_interceptions i ON t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\nAND u.username ~ '$${username:regex}'\nAND i.provider ~ '$${provider:regex}'\nAND i.model ~ '$${model:regex}'\norder by t.created_at desc", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Tool Calls", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "created_at": "Created At", + "id": "Interception ID", + "input": "Tool Input", + "invocation_error": "Tool Error", + "model": "Model", + "provider": "Provider", + "server_url": "MCP Server", + "tool": "Tool Name", + "username": "Username" + } + } + } + ], + "type": "table" + } + ], + "refresh": "1m", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "allValue": ".+", + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "definition": "select username from users where deleted=false;", + "description": "", + "includeAll": true, + "multi": true, + "name": "username", + "options": [], + "query": "select username from users where deleted=false;", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "definition": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", + "description": "", + "includeAll": true, + "multi": true, + "name": "provider", + "options": [], + "query": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".+", + "current": {}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "grafana-postgresql-datasource" + }, + "definition": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '$${provider:regex}' ORDER BY 1;", + "description": "", + "includeAll": true, + "multi": true, + "name": "model", + "options": [], + "query": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '$${provider:regex}' ORDER BY 1;", + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Coder AI Bridge", + "uid": "0c61d33f-c809-4184-9e88-cb27e2d9d224", + "version": 43, + "weekStart": "" +} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/coderd.json b/infra/1-click/1-setup/dashboards/coderd.json new file mode 100644 index 0000000..b64f5ca --- /dev/null +++ b/infra/1-click/1-setup/dashboards/coderd.json @@ -0,0 +1,1472 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Down" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(up{ ${CODERD_SELECTOR} } == 1) or vector(0)", + "instant": true, + "legendFormat": "Up", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(count(up{ ${CODERD_SELECTOR} } == 0) or vector(0)) > 0", + "hide": false, + "instant": true, + "legendFormat": "Down", + "range": false, + "refId": "B" + } + ], + "title": "Replicas", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 18, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "One or more replicas are required to be running in order to serve the control-plane.\n\nSee [High Availability](https://coder.com/docs/v2/latest/admin/high-availability) for details on how to\nrun multiple `coderd` replicas.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.9 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Enabled" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 1, + "text": "No" + }, + "1": { + "index": 0, + "text": "Yes" + } + }, + "type": "value" + }, + { + "options": { + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 32, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(coderd_license_user_limit_enabled)", + "instant": true, + "legendFormat": "Enabled", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(\n max(coderd_license_active_users) / max(coderd_license_limit_users)\n) > 0", + "hide": false, + "instant": false, + "legendFormat": "Usage", + "range": true, + "refId": "B" + } + ], + "title": "Enterprise License", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 33, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "If you would like to try Coder's [Enterprise features](https://coder.com/docs/v2/latest/enterprise), you can [request a trial license](https://coder.com/docs/v2/latest/faqs#how-do-i-add-an-enterprise-license).", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(Requested|Limit)/" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + }, + { + "id": "custom.fillOpacity", + "value": 5 + }, + { + "id": "custom.drawStyle", + "value": "line" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Requested" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{ ${CODERD_SELECTOR} }[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(kube_pod_container_resource_limits{ ${CODERD_SELECTOR}, resource=\"cpu\"})", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(kube_pod_container_resource_requests{ ${CODERD_SELECTOR}, resource=\"cpu\"})", + "hide": false, + "instant": false, + "legendFormat": "Requested", + "range": true, + "refId": "B" + } + ], + "title": "CPU Usage Seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 6 + }, + "id": 26, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "The cumulative CPU used per core-second. If `coderd` was using a full CPU core, that would be represented as 1 second.\n\nRequests & limits are shown if set.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Requested" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (reason) (\n count_over_time(kube_pod_container_status_terminated_reason{ ${CODERD_SELECTOR} }[$__interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "{{reason}}", + "range": true, + "refId": "C" + } + ], + "title": "Terminations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.0001 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 16, + "y": 6 + }, + "id": 34, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(kube_pod_container_status_restarts_total{ ${CODERD_SELECTOR} }[$__range]))", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 31, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Pods can be terminated for several reasons:\n- `OOMKilled`: pod exceeded its defined memory limit or was terminated by the OS for using excessive memory (if no limit defined)\n- `Error`: usually attributeable to a configuration problem\n- `Evicted`: pod has been evicted from node for overusing resources and will be rescheduled on another node is possible", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(Requested|Limit)/" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + }, + { + "id": "custom.fillOpacity", + "value": 5 + }, + { + "id": "custom.drawStyle", + "value": "line" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Requested" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (pod) (container_memory_working_set_bytes{ ${CODERD_SELECTOR} })", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(kube_pod_container_resource_limits{ ${CODERD_SELECTOR}, resource=\"memory\"})", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(kube_pod_container_resource_requests{ ${CODERD_SELECTOR}, resource=\"memory\"})", + "hide": false, + "instant": false, + "legendFormat": "Requested", + "range": true, + "refId": "B" + } + ], + "title": "RAM Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 28, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This shows the total memory used by each `coderd` container; it is the same metric which the [OOM killer](https://www.kernel.org/doc/gorman/html/understand/understand016.html) uses.\n\nRequests & limits are shown if set.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 100 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Errors" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 12 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantile(0.5, coder_pubsub_send_latency_seconds)", + "instant": false, + "legendFormat": "Send", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantile(0.5, coder_pubsub_receive_latency_seconds)", + "hide": false, + "instant": false, + "legendFormat": "Receive", + "range": true, + "refId": "B" + } + ], + "title": "Pubsub Latency (Median)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Errors" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 16, + "y": 12 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(\n sum(increase(coder_pubsub_latency_measure_errs_total[$__range]))\n / count(coder_pubsub_latency_measure_errs_total)\n) or vector(0)", + "hide": false, + "instant": false, + "legendFormat": "Errors", + "range": true, + "refId": "B" + } + ], + "title": "Pubsub Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 19, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "`coderd` uses Postgres for passing messages between subcomponents for coordination and signalling;\nthis is called \"pubsub\" (or publish-subscribe).\n\nWe measure the time for messages to be sent and received. Latencies higher than 500ms will likely lead to\nyour Coder deployment feeling sluggish. High latency is usually an indication that your Postgres server is under-resourced on CPU.\n\nHigh values for median should be concerning,\nwhile the 90th percentile shows the outliers.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 100 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Errors" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 15 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantile(0.9, coder_pubsub_send_latency_seconds)", + "instant": false, + "legendFormat": "Send", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "quantile(0.9, coder_pubsub_receive_latency_seconds)", + "hide": false, + "instant": false, + "legendFormat": "Receive", + "range": true, + "refId": "B" + } + ], + "title": "Pubsub Latency (P90)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(pod) (rate(coderd_api_requests_processed_total{ ${CODERD_SELECTOR} }[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "API Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 36, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This shows the number of requests per second each `coderd` replica is handling.\n\nHeavy skewing towards a single `coderd` replica indicates faulty loadbalancing.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + } + ], + "refresh": "${DASHBOARD_REFRESH}", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-${DASHBOARD_TIMERANGE}", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Coder Control Plane", + "uid": "coderd", + "version": 6, + "weekStart": "" +} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/prebuilds.json b/infra/1-click/1-setup/dashboards/prebuilds.json new file mode 100644 index 0000000..4c06a15 --- /dev/null +++ b/infra/1-click/1-setup/dashboards/prebuilds.json @@ -0,0 +1,1450 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 132, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 49, + "interval": "30s", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(max(coderd_prebuilt_workspaces_desired) by (template_name, preset_name)) or vector(0)", + "instant": true, + "interval": "", + "legendFormat": "Desired", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(max(coderd_prebuilt_workspaces_running) by (template_name, preset_name)) or vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Running", + "range": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(max(coderd_prebuilt_workspaces_eligible) by (template_name, preset_name)) or vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Eligible", + "range": false, + "refId": "E" + } + ], + "title": "Current: Global", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 48, + "interval": "30s", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_created_total)) or vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Created", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_failed_total)) or vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Failed", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_claimed_total)) or vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Claimed", + "range": false, + "refId": "A" + } + ], + "title": "All Time: Global", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 2, + "panels": [], + "repeat": "preset", + "title": "$preset", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 1, + "interval": "30s", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(coderd_prebuilt_workspaces_created_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Created", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(coderd_prebuilt_workspaces_failed_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Failed", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(coderd_prebuilt_workspaces_claimed_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Claimed", + "range": false, + "refId": "A" + } + ], + "title": "All Time: $preset", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 10, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 13, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Created" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Desired" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Eligible" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Claimed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 6, + "y": 5 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "floor(max(increase(coderd_prebuilt_workspaces_created_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Created", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "floor(max(increase(coderd_prebuilt_workspaces_failed_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Failed", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "floor(max(increase(coderd_prebuilt_workspaces_claimed_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Claimed", + "range": true, + "refId": "F" + } + ], + "title": "Pool Operations: $preset", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 10, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 18, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Desired" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.fillOpacity", + "value": 85 + }, + { + "id": "custom.fillBelowTo", + "value": "Running" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Eligible" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Eligible" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 15, + "y": 5 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(coderd_prebuilt_workspaces_desired{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", + "instant": false, + "interval": "", + "legendFormat": "Desired", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(coderd_prebuilt_workspaces_running{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Running", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(coderd_prebuilt_workspaces_eligible{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Eligible", + "range": true, + "refId": "E" + } + ], + "title": "Pool Capacity: $preset", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 8 + }, + "id": 31, + "interval": "30s", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(coderd_prebuilt_workspaces_desired{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", + "instant": true, + "interval": "", + "legendFormat": "Desired", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(coderd_prebuilt_workspaces_running{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Running", + "range": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(coderd_prebuilt_workspaces_eligible{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Eligible", + "range": false, + "refId": "E" + } + ], + "title": "Current: $preset", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Compares the total number of regular workspace creations to prebuilt workspace claims to date.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 11 + }, + "id": 51, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(max by (template_name, preset_name) (\n coderd_workspace_creation_total{\n template_name=~\"$template\", preset_name=~\"$preset\"\n }\n)) or vector(0)", + "instant": false, + "legendFormat": "Regular workspaces created", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (template_name, preset_name) (\n coderd_prebuilt_workspaces_claimed_total{\n template_name=~\"$template\", preset_name=~\"$preset\"\n }\n)) or vector(0)\n", + "hide": false, + "instant": false, + "legendFormat": "Prebuilt workspaces claimed", + "range": true, + "refId": "B" + } + ], + "title": "All Time: Regular vs Prebuilt", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Median (p50) build time in seconds for Regular Workspace Creation, Prebuilt Workspace Creation, and Prebuilt Workspace Claim", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Regular Creation" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Prebuild Creation" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Prebuild Claim" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 6, + "y": 11 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.5,\n sum(\n coderd_workspace_creation_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\", type=\"regular\"\n }\n )\n)\nor vector(0)", + "legendFormat": "Regular Creation", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,\n sum(\n coderd_workspace_creation_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\", type=\"prebuild\"\n }\n )\n)\nor vector(0)", + "hide": false, + "instant": false, + "legendFormat": "Prebuild Creation", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5,\n sum(\n coderd_prebuilt_workspace_claim_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\"\n }\n )\n)\nor vector(0)", + "hide": false, + "instant": false, + "legendFormat": "Prebuild Claim", + "range": true, + "refId": "C" + } + ], + "title": "Workspace Build Latency p50", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "95th-percentile (p95) build time in seconds for Regular Workspace Creation, Prebuilt Workspace Creation, and Prebuilt Workspace Claim.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Regular Creation" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Prebuild Creation" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Prebuild Claim" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 15, + "y": 11 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.95,\n sum(\n coderd_workspace_creation_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\", type=\"regular\"\n }\n )\n)\nor vector(0)", + "legendFormat": "Regular Creation", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95,\n sum(\n coderd_workspace_creation_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\", type=\"prebuild\"\n }\n )\n)\nor vector(0)", + "hide": false, + "instant": false, + "legendFormat": "Prebuild Creation", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95,\n sum(\n coderd_prebuilt_workspace_claim_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\"\n }\n )\n)\nor vector(0)", + "hide": false, + "instant": false, + "legendFormat": "Prebuild Claim", + "range": true, + "refId": "C" + } + ], + "title": "Workspace Build Latency p95", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "green", + "value": 75 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 54, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "clamp_max(\n 100 *\n (\n sum(\n coderd_prebuilt_workspaces_claimed_total{\n template_name=\"$template\", preset_name=\"$preset\"\n }\n ) or vector(0)\n )\n /\n clamp_min(\n ( \n sum(\n coderd_prebuilt_workspaces_claimed_total{\n template_name=\"$template\", preset_name=\"$preset\"\n }\n ) or vector(0))\n +\n (\n sum(\n coderd_workspace_creation_total{\n template_name=\"$template\", preset_name=\"$preset\"\n }\n ) or vector(0)),\n 1\n ),\n 100\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "All Time: Prebuilds Usage %", + "type": "stat" + } + ], + "preload": false, + "refresh": "${DASHBOARD_REFRESH}", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "coder", + "value": "coder" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(coderd_prebuilt_workspaces_desired,template_name)", + "includeAll": false, + "label": "Template", + "name": "template", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(coderd_prebuilt_workspaces_desired,template_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(coderd_prebuilt_workspaces_desired{template_name=~\"$template\"},preset_name)", + "includeAll": true, + "label": "Preset", + "multi": true, + "name": "preset", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(coderd_prebuilt_workspaces_desired{template_name=~\"$template\"},preset_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-${DASHBOARD_TIMERANGE}", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Coder Prebuilds", + "uid": "cej6jysyme22oa", + "version": 5 +} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/provisionerd.json b/infra/1-click/1-setup/dashboards/provisionerd.json new file mode 100644 index 0000000..f41f277 --- /dev/null +++ b/infra/1-click/1-setup/dashboards/provisionerd.json @@ -0,0 +1,1019 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(coderd_provisionerd_num_daemons{pod=~`coder.*`, pod!~`.*provisioner.*`})", + "instant": true, + "legendFormat": "Built-in", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(coderd_provisionerd_num_daemons{ ${PROVISIONERD_SELECTOR} })", + "hide": false, + "instant": true, + "legendFormat": "External", + "range": false, + "refId": "B" + } + ], + "title": "Provisioners", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 20, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Provisioners are responsible for building workspaces.\n\n`coderd` runs built-in provisioners by default. Control this with the `CODER_PROVISIONER_DAEMONS` environment variable or `--provisioner-daemons` flag.\n\nYou can also consider [External Provisioners](https://coder.com/docs/v2/latest/admin/provisioners). Running both built-in and external provisioners is perfectly valid,\nalthough dedicated (external) provisioners will generally give the best build performance.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(sum(coderd_provisionerd_jobs_current) > 0) or vector(0)", + "instant": false, + "legendFormat": "Current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(coderd_provisionerd_num_daemons)", + "hide": false, + "instant": true, + "legendFormat": "Capacity", + "range": false, + "refId": "B" + } + ], + "title": "Builds", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 22, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "The maximum number of simultaneous builds is equivalent to the number of `provisionerd` daemons running.\n\nThe \"Capacity\" panel shows the how many simultaneous builds are possible.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 23, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.5, sum by(le) (rate(coderd_provisionerd_job_timings_seconds_bucket[$__range])))", + "hide": false, + "instant": true, + "legendFormat": "Median", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.9, sum by(le) (rate(coderd_provisionerd_job_timings_seconds_bucket[$__range])))", + "hide": false, + "instant": true, + "legendFormat": "90th Percentile", + "range": false, + "refId": "A" + } + ], + "title": "Build Times", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 24, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This shows the median and 90th percentile workspace build times.\n\nLong build times can impede developers' productivity while they wait for workspaces to start or be created.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "displayName", + "value": "Failure" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "displayName", + "value": "Success" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 25, + "interval": "1h", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (status) (increase(coderd_provisionerd_job_timings_seconds_count[$__interval]))", + "hide": false, + "instant": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Build Count Per Hour", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 26, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "_NOTE: this will not show the current hour._", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(Limit|Requested)/" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.fillOpacity", + "value": 5 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Requested" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{ ${PROVISIONERD_SELECTOR} }[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(kube_pod_container_resource_limits{ ${PROVISIONERD_SELECTOR} , resource=\"cpu\"})", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(kube_pod_container_resource_requests{ ${PROVISIONERD_SELECTOR} , resource=\"cpu\"})", + "hide": false, + "instant": false, + "legendFormat": "Requested", + "range": true, + "refId": "C" + } + ], + "title": "CPU Usage Seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 14 + }, + "id": 30, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "The cumulative CPU used per core-second. If the process was using a full CPU core, that would be represented as 1 second.\n\nRequests & limits are shown if set.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(Limit|Requested)/" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.fillOpacity", + "value": 5 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Requested" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 14 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (pod) (container_memory_working_set_bytes{ ${PROVISIONERD_SELECTOR} })", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(kube_pod_container_resource_limits{ ${PROVISIONERD_SELECTOR} , resource=\"memory\"})", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(kube_pod_container_resource_requests{ ${PROVISIONERD_SELECTOR} , resource=\"memory\"})", + "hide": false, + "instant": false, + "legendFormat": "Requested", + "range": true, + "refId": "C" + } + ], + "title": "RAM Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 31, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This shows the total memory used by each container; it is the same metric which the [OOM killer](https://www.kernel.org/doc/gorman/html/understand/understand016.html) uses.\n\nRequests & limits are shown if set.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 18, + "w": 18, + "x": 0, + "y": 21 + }, + "id": 27, + "options": { + "dedupStrategy": "exact", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{ ${NON_WORKSPACES_SELECTOR}, logger=~\"(.*runner|terraform|provisioner.*)\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs", + "type": "logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 21 + }, + "id": 32, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This panel shows all logs across built-in and [external provisioners](https://coder.com/docs/v2/latest/admin/provisioners).", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + } + ], + "refresh": "${DASHBOARD_REFRESH}", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-${DASHBOARD_TIMERANGE}", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Coder Provisioners", + "uid": "provisionerd", + "version": 10, + "weekStart": "" +} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/status.json b/infra/1-click/1-setup/dashboards/status.json new file mode 100644 index 0000000..082998f --- /dev/null +++ b/infra/1-click/1-setup/dashboards/status.json @@ -0,0 +1,2038 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": false, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "title": "Application", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Down" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(up{ ${CODERD_SELECTOR} } == 1) or vector(0) > 0", + "instant": true, + "legendFormat": "Up", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(up{ ${CODERD_SELECTOR} } == 0) or vector(0) > 0", + "hide": false, + "instant": true, + "legendFormat": "Down", + "range": false, + "refId": "B" + } + ], + "title": "Coder Replicas", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(coderd_provisionerd_num_daemons{ ${CODERD_SELECTOR} })", + "instant": true, + "legendFormat": "Built-in", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(coderd_provisionerd_num_daemons{ ${PROVISIONERD_SELECTOR} })", + "hide": false, + "instant": true, + "legendFormat": "External", + "range": false, + "refId": "B" + } + ], + "title": "Provisioners", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "displayName", + "value": "Failed" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "displayName", + "value": "Success" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 17, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "round(sum by (status) (increase(coderd_provisionerd_job_timings_seconds_count{pod!=``}[$__range])))", + "instant": true, + "legendFormat": "{{status}}", + "range": false, + "refId": "A" + } + ], + "title": "Workspace Builds", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(kube_pod_status_ready{condition=\"true\", ${WORKSPACES_SELECTOR}} == 1)\nor\nsum(max by (workspace_owner, template_name, template_version) (coderd_workspace_latest_build_status{status=\"succeeded\", workspace_transition=\"start\"}))\nor\nvector(0)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Running Workspaces", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Down" + }, + "1": { + "color": "green", + "index": 0, + "text": "Up" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "empty", + "result": { + "color": "orange", + "index": 3, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null+nan", + "result": { + "index": 4, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*RAM/" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n max_over_time(\n rate(container_cpu_usage_seconds_total{ ${CODERD_SELECTOR} }[1h:1m])\n [$__range:]\n )\n)", + "instant": true, + "legendFormat": "Control Plane CPU", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n max_over_time(\n rate(container_cpu_usage_seconds_total{ ${PROVISIONERD_SELECTOR} }[1h:1m])\n [$__range:]\n )\n)", + "hide": false, + "instant": true, + "legendFormat": "Provisioner CPU", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n max_over_time(\n container_memory_working_set_bytes{ ${CODERD_SELECTOR} }\n [$__range:]\n )\n)", + "hide": false, + "instant": true, + "legendFormat": "Control Plane RAM", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n max_over_time(\n container_memory_working_set_bytes{ ${PROVISIONERD_SELECTOR} }\n [$__range:]\n )\n)", + "hide": false, + "instant": true, + "legendFormat": "Provisioner RAM", + "range": false, + "refId": "D" + } + ], + "title": "Resource Usage High Watermark (Cumulative)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Down" + }, + "1": { + "color": "green", + "index": 0, + "text": "Up" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "empty", + "result": { + "color": "orange", + "index": 3, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null+nan", + "result": { + "index": 4, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(pg_up) or vector(0)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Postgres", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "panels": [], + "title": "Observability Tools", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Down" + }, + "1": { + "color": "green", + "index": 0, + "text": "Up" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "empty", + "result": { + "color": "orange", + "index": 3, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null+nan", + "result": { + "index": 4, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(up{job=\"${PROMETHEUS_JOB}\"}) or vector(0)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Prometheus", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Down" + }, + "1": { + "color": "green", + "index": 0, + "text": "Up" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "empty", + "result": { + "color": "orange", + "index": 3, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null+nan", + "result": { + "index": 4, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 9 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(up{job=\"${LOKI_JOB}/write\"}) or vector(0)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Loki Write Path", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Down" + }, + "1": { + "color": "green", + "index": 0, + "text": "Up" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "empty", + "result": { + "color": "orange", + "index": 3, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null+nan", + "result": { + "index": 4, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 9 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(up{job=\"${LOKI_JOB}/read\"}) or vector(0)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Loki Read Path", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Down" + }, + "1": { + "color": "green", + "index": 0, + "text": "Up" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "empty", + "result": { + "color": "orange", + "index": 3, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null+nan", + "result": { + "index": 4, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 9 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(up{job=\"${LOKI_JOB}/backend\", container=\"loki\"}) or vector(0)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Loki Backend", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Down" + }, + "1": { + "color": "green", + "index": 0, + "text": "Up" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "empty", + "result": { + "color": "orange", + "index": 3, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null+nan", + "result": { + "index": 4, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 9 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(up{job=\"${LOKI_JOB}/canary\"}) or vector(0)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Loki Canary", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 9 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(up{job=\"${GRAFANA_AGENT_JOB}\"}) or vector(0)", + "instant": true, + "legendFormat": "Current", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(up{job=\"${GRAFANA_AGENT_JOB}\"})", + "instant": true, + "legendFormat": "Capacity", + "range": false, + "refId": "B" + } + ], + "title": "Grafana Agent", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Unhealthy" + }, + "1": { + "color": "green", + "index": 0, + "text": "Healthy" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "empty", + "result": { + "color": "orange", + "index": 3, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null+nan", + "result": { + "index": 4, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 14 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "prometheus_config_last_reload_successful{job=\"${PROMETHEUS_JOB}\"}", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Prometheus Config", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Unhealthy" + }, + "1": { + "color": "green", + "index": 0, + "text": "Healthy" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "empty", + "result": { + "color": "orange", + "index": 3, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null+nan", + "result": { + "index": 4, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 14 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(loki_runtime_config_last_reload_successful) or vector(0)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Loki Config", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Unhealthy" + }, + "1": { + "color": "green", + "index": 0, + "text": "Healthy" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "orange", + "index": 2, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "empty", + "result": { + "color": "orange", + "index": 3, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null+nan", + "result": { + "index": 4, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 14 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(agent_config_last_load_successful{job=\"${GRAFANA_AGENT_JOB}\"}) or vector(0)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Grafana Agent Config", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Retention Limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Write-Ahead Log" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Storage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#f9f9fb", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 14 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(\n prometheus_tsdb_wal_storage_size_bytes{job=\"${PROMETHEUS_JOB}\"} +\n prometheus_tsdb_storage_blocks_bytes{job=\"${PROMETHEUS_JOB}\"} +\n prometheus_tsdb_symbol_table_size_bytes{job=\"${PROMETHEUS_JOB}\"}\n)\n/\nprometheus_tsdb_retention_limit_bytes{job=\"${PROMETHEUS_JOB}\"}", + "instant": false, + "legendFormat": "Retention limit used", + "range": true, + "refId": "A" + } + ], + "title": "Prometheus Storage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 14 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 20, + "valueSize": 35 + }, + "textMode": "auto", + "wideLayout": false + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{namespace=\"${HELM_NAMESPACE}\", resource=\"cpu\"})", + "hide": false, + "instant": true, + "legendFormat": "Requested", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n max_over_time(\n rate(container_cpu_usage_seconds_total{namespace=\"${HELM_NAMESPACE}\"}[$__rate_interval])\n [$__range:]\n )\n)", + "hide": false, + "instant": true, + "legendFormat": "High Watermark", + "range": false, + "refId": "D" + } + ], + "title": "CPU", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 14 + }, + "id": 21, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 20, + "valueSize": 35 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{namespace=\"${HELM_NAMESPACE}\", resource=\"memory\"})", + "hide": false, + "instant": true, + "legendFormat": "Requested", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n max_over_time(container_memory_working_set_bytes{namespace=\"${HELM_NAMESPACE}\"}[$__range])\n)", + "instant": true, + "legendFormat": "High Watermark", + "range": false, + "refId": "A" + } + ], + "title": "RAM", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Coder Status", + "uid": "coder-status", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/workspace_detail.json b/infra/1-click/1-setup/dashboards/workspace_detail.json new file mode 100644 index 0000000..a5cc8b3 --- /dev/null +++ b/infra/1-click/1-setup/dashboards/workspace_detail.json @@ -0,0 +1,1342 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "", + "gridPos": { + "h": 1.2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 28, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "**HINT**: use the dropdowns above to filter by specific workspace(s).", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CPUs Requested" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Requested" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PVC Capacity" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 20, + "x": 0, + "y": 1.2 + }, + "id": 29, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 20, + "valueSize": 40 + }, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "group by (template_name) (coderd_agents_up{workspace_name=~\"$workspace_name\"})", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Template Name", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "group by (template_version) (coderd_agents_up{workspace_name=~\"$workspace_name\"})", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Template Version", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "group by (username) (coderd_agents_up{workspace_name=~\"$workspace_name\"})", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Owner", + "range": false, + "refId": "C" + } + ], + "title": "Details", + "transformations": [ + { + "id": "concatenate", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value #A": true, + "Value #B": true, + "Value #C": true, + "Value #D": true + }, + "includeByName": {}, + "indexByName": { + "CPUs Requested": 7, + "PVC Capacity": 9, + "RAM Requested": 8, + "Time": 0, + "Value #A": 5, + "Value #B": 3, + "Value #C": 6, + "template_name": 2, + "template_version": 4, + "username": 1 + }, + "renameByName": { + "Value #C": "", + "lifecycle_state": "Agent State", + "template_name": "Template", + "template_version": "Template Version", + "username": "Owner" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 1.2 + }, + "id": 38, + "links": [ + { + "title": "Provisioners Dashboard", + "url": "/d/provisionerd/provisioners?$${__url_time_range}" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Essential information about the selected workspace.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CPUs Requested" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Requested" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PVC Capacity" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 20, + "x": 0, + "y": 5.2 + }, + "id": 36, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/" + }, + "orientation": "vertical", + "textMode": "value_and_name", + "wideLayout": false, + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "showPercentChange": false, + "text": { + "titleSize": 20, + "valueSize": 40 + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{pod=~\".*$workspace_name.*\", ${WORKSPACES_SELECTOR}, resource=\"cpu\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": "CPUs Requested", + "range": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(kube_pod_container_resource_requests{pod=~\".*$workspace_name.*\", ${WORKSPACES_SELECTOR}, resource=\"memory\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": "RAM Requested", + "range": false, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n kube_pod_spec_volumes_persistentvolumeclaims_info{pod=~\".*$workspace_name.*\", ${WORKSPACES_SELECTOR} }\n * on(persistentvolumeclaim) group_right\n group by (persistentvolumeclaim, persistentvolume) (\n label_replace(\n kube_persistentvolume_claim_ref,\n \"persistentvolumeclaim\",\n \"$1\",\n \"name\",\n \"(.+)\"\n )\n )\n * on (persistentvolume)\n kube_persistentvolume_capacity_bytes\n)", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": "PVC Capacity", + "range": false, + "refId": "F" + } + ], + "title": "Resources", + "transformations": [ + { + "id": "concatenate", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value #A": true, + "Value #B": true, + "Value #C": true, + "Value #D": true + }, + "includeByName": {}, + "indexByName": { + "CPUs Requested": 7, + "PVC Capacity": 9, + "RAM Requested": 8, + "Time": 0, + "Value #A": 5, + "Value #B": 3, + "Value #C": 6, + "template_name": 2, + "template_version": 4, + "username": 1 + }, + "renameByName": { + "Value #C": "", + "lifecycle_state": "Agent State", + "template_name": "Template", + "template_version": "Template Version", + "username": "Owner" + } + } + } + ], + "type": "stat", + "description": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "created": { + "color": "light-blue", + "index": 1, + "text": "Created" + }, + "off": { + "color": "text", + "index": 8, + "text": "Off" + }, + "ready": { + "color": "green", + "index": 0, + "text": "Ready" + }, + "shutdown_error": { + "color": "red", + "index": 7, + "text": "Shutdown Error" + }, + "shutdown_timeout": { + "color": "purple", + "index": 6, + "text": "Shutdown Timeout" + }, + "shutting_down": { + "color": "light-purple", + "index": 5, + "text": "Shutting Down" + }, + "start_error": { + "color": "red", + "index": 4, + "text": "Start Error" + }, + "start_timeout": { + "color": "orange", + "index": 3, + "text": "Start Timeout" + }, + "starting": { + "color": "super-light-green", + "index": 2, + "text": "Starting" + } + }, + "type": "value" + }, + { + "options": { + "match": "empty", + "result": { + "color": "text", + "index": 9, + "text": "Unknown" + } + }, + "type": "special" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 10, + "text": "Unknown" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 9.2 + }, + "id": 35, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^lifecycle_state$/", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 50 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (lifecycle_state) (coderd_agents_connections{workspace_name=~\"$workspace_name\"})", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "D" + } + ], + "title": "Agent Lifecycle State", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "-1": { + "color": "light-orange", + "index": 0, + "text": "Not completed yet" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 60 + }, + { + "color": "red", + "value": 120 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 4, + "y": 9.2 + }, + "id": 33, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Value$/", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 50 + }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(coderd_agentstats_startup_script_seconds{workspace_name=~\"$workspace_name\"}) or vector(-1)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + } + ], + "title": "Agent Startup Script Execution Time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 7, + "y": 9.2 + }, + "id": 39, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 20, + "valueSize": 50 + }, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (app) (\n label_replace(\n {workspace_name=~\"$workspace_name\", __name__=~\"coderd_agentstats_session_count_.*\"},\n \"app\",\n \"$1\",\n \"__name__\",\n \"coderd_agentstats_session_count_(.*)\"\n )\n)>0", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": "{{app}}", + "range": false, + "refId": "C" + } + ], + "title": "App Session Counts", + "transformations": [ + { + "id": "concatenate", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Bytes/" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 9.2 + }, + "id": 34, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 20, + "valueSize": 50 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(coderd_agents_connection_latencies_seconds{workspace_name=~\"$workspace_name\"})", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Connection Latency", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(sum by (pod) (sum_over_time(coderd_agentstats_rx_bytes{workspace_name=~\"$workspace_name\"}[$__range])))", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Received Bytes", + "range": false, + "refId": "rx" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(sum by (pod) (sum_over_time(coderd_agentstats_tx_bytes{workspace_name=~\"$workspace_name\"}[$__range])))", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Transmitted Bytes", + "range": false, + "refId": "tx" + } + ], + "title": "Networking", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value #A": "Received Bytes", + "Value #B": "Transmitted Bytes", + "Value #C": "Connection Latency", + "Value #rx": "Received Bytes", + "Value #tx": "Transmitted Bytes" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 9.2 + }, + "id": 40, + "links": [ + { + "title": "Provisioners Dashboard", + "url": "/d/provisionerd/provisioners?$${__url_time_range}" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Essential information about this workspace's agent.\n\nRead more about the agent [here](https://coder.com/docs/v2/latest/about/architecture#agents).", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "failed": { + "color": "orange", + "index": 1, + "text": "Failure" + }, + "success": { + "color": "green", + "index": 0, + "text": "Success" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Workspace Transition" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "DESTROY": { + "color": "red", + "index": 0 + }, + "START": { + "color": "blue", + "index": 1 + }, + "STOP": { + "color": "purple", + "index": 2 + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 20, + "x": 0, + "y": 15.2 + }, + "id": 6, + "interval": "", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": [], + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Time" + } + ] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (workspace_name, workspace_owner, status, template_name, template_version, workspace_transition) (\n # Since new series are created and are initially set to a value of 1, we cannot use \"increase\" (because an increase from to 1 does not yield 1).\n # So we compare the current series to an interval ago to see if we have any new series and then sum the series we find. \n ((\n coderd_workspace_builds_total{workspace_name=~\"$workspace_name\"} - \n coderd_workspace_builds_total{workspace_name=~\"$workspace_name\"} offset $__interval\n ) >= 0) \n or coderd_workspace_builds_total{workspace_name=~\"$workspace_name\"}\n) > 0", + "format": "table", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Build Log", + "transformations": [ + { + "disabled": true, + "id": "groupBy", + "options": { + "fields": { + "Count": { + "aggregations": [ + "sum" + ], + "operation": "aggregate" + }, + "Status": { + "aggregations": [], + "operation": "groupby" + }, + "Template Name": { + "aggregations": [], + "operation": "groupby" + }, + "Template Version": { + "aggregations": [], + "operation": "groupby" + }, + "Total": { + "aggregations": [ + "sum" + ], + "operation": "aggregate" + }, + "Value": { + "aggregations": [ + "sum" + ], + "operation": "aggregate" + }, + "Workspace Name": { + "aggregations": [], + "operation": "groupby" + }, + "Workspace Ownert": { + "aggregations": [], + "operation": "groupby" + }, + "Workspace Transition": { + "aggregations": [], + "operation": "groupby" + }, + "status": { + "aggregations": [], + "operation": "groupby" + }, + "template_name": { + "aggregations": [], + "operation": "groupby" + }, + "template_version": { + "aggregations": [], + "operation": "groupby" + }, + "workspace_name": { + "aggregations": [], + "operation": "groupby" + }, + "workspace_owner": { + "aggregations": [], + "operation": "groupby" + }, + "workspace_transition": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Value" + } + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": false + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Count", + "Value (sum)": "Total", + "status": "Status", + "template_name": "Template Name", + "template_version": "Template Version", + "workspace_name": "Workspace Name", + "workspace_owner": "Workspace Owner", + "workspace_transition": "Workspace Transition" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 15.2 + }, + "id": 37, + "links": [ + { + "title": "Provisioners Dashboard", + "url": "/d/provisionerd/provisioners?$${__url_time_range}" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This table shows a reverse-chronological log of all workspace builds.\n\nThe \"Count\" field shows the count of events which occurred within a minute, grouped by all columns.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 10, + "w": 20, + "x": 0, + "y": 22.2 + }, + "id": 7, + "options": { + "dedupStrategy": "exact", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{${NON_WORKSPACES_SELECTOR}, logger=~\"(.*runner|terraform|provisioner.*)\"} |~ \"$workspace_name\" | line_format `{{ printf \"[\\033[35m\" }}{{.pod}}{{ printf \"\\033[0m]\\t\" }}{{ __line__ }}`", + "hide": false, + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{${WORKSPACES_SELECTOR}, pod=~\".*($workspace_name).*\"} | line_format `{{ printf \"[\\033[32m\" }}{{.pod}}{{ printf \"\\033[0m]\\t\" }}{{ __line__ }}`", + "hide": false, + "queryType": "range", + "refId": "B" + } + ], + "title": "Logs", + "type": "logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 10, + "w": 4, + "x": 20, + "y": 22.2 + }, + "id": 24, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "The logs to the left come both from provisioners and workspace logs.\n\nProvisioner logs matching the name filter are highlighted in magenta, while\nworkspace logs matching the name filter are highlighted in green.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + } + ], + "refresh": "${DASHBOARD_REFRESH}", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(coderd_agents_up,workspace_name)", + "hide": 0, + "includeAll": false, + "label": "Workspace Name Filter", + "multi": false, + "name": "workspace_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(coderd_agents_up,workspace_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-${DASHBOARD_TIMERANGE}", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Coder Workspace Detail", + "uid": "workspace-detail", + "version": 9, + "weekStart": "" +} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/workspaces.json b/infra/1-click/1-setup/dashboards/workspaces.json new file mode 100644 index 0000000..62e2cfc --- /dev/null +++ b/infra/1-click/1-setup/dashboards/workspaces.json @@ -0,0 +1,1624 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "", + "gridPos": { + "h": 1.2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 28, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "**HINT**: use the dropdowns above to filter by specific workspaces and/or templates.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1.2 + }, + "id": 31, + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 2.2 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "mean", + "stdDev", + "min", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{ ${WORKSPACES_SELECTOR} }[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 10, + "y": 2.2 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "mean", + "stdDev", + "min", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (pod) (container_memory_working_set_bytes{ ${WORKSPACES_SELECTOR} })", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "RAM Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 2.2 + }, + "id": 36, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "The cumulative CPU used per core-second. If a workspace was using a full CPU core, that would be represented as 1 second.\n\nSee the Kubernetes [documentation](https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units) for more details.\n\nThe total memory used by each workspace container is represented; it is the same metric which the [OOM killer](https://www.kernel.org/doc/gorman/html/understand/understand016.html) uses.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 10.2 + }, + "id": 38, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n round(increase(kube_pod_container_status_restarts_total{ ${WORKSPACES_SELECTOR} }[$__interval]))\n) > 0", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Pod Restarts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 10, + "y": 10.2 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod, reason) (\n count_over_time(kube_pod_container_status_terminated_reason{ ${WORKSPACES_SELECTOR} }[$__interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "{{pod}}:{{reason}}", + "range": true, + "refId": "B" + } + ], + "title": "Terminations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 10.2 + }, + "id": 40, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Pods can be terminated for several reasons:\n- `OOMKilled`: pod exceeded its defined memory limit or was terminated by the OS for using excessive memory (if no limit defined)\n- `Error`: usually attributeable to a configuration problem\n- `Evicted`: pod has been evicted from node for overusing resources and will be rescheduled on another node is possible\n\nPod restarts are not necessarily problematic, but they are worth noting.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18.2 + }, + "id": 30, + "panels": [], + "title": "Builds", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "DESTROY" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "STOP" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "START" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 19.2 + }, + "id": 2, + "interval": "5m", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workspace_transition) (\n (\n # Since new series are created and are initially set to a value of 1, we cannot use \"increase\" (because an increase from to 1 does not yield 1).\n # So we compare the current series to an interval ago to see if we have any new series and then sum the series we find. \n (\n coderd_workspace_builds_total{status=\"success\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} - \n coderd_workspace_builds_total{status=\"success\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} offset $__interval\n ) >= 0) \n or coderd_workspace_builds_total{status=\"success\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"}\n) > 0", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Successful Builds by State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "DESTROY" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "STOP" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "START" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 10, + "y": 19.2 + }, + "id": 1, + "interval": "5m", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workspace_transition) (\n (\n # Since new series are created and are initially set to a value of 1, we cannot use \"increase\" (because an increase from to 1 does not yield 1).\n # So we compare the current series to an interval ago to see if we have any new series and then sum the series we find. \n (\n coderd_workspace_builds_total{status=\"failed\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} - \n coderd_workspace_builds_total{status=\"failed\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} offset $__interval\n ) >= 0) \n or coderd_workspace_builds_total{status=\"failed\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"}\n) > 0", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Unsuccessful Builds by State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 19.2 + }, + "id": 34, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Workspaces \"transition\" between `STOP`, `START`, and `DESTROY` states.\n\nWorkspaces transition between states when a \"build\" is initiated, which is an execution of `terraform` against the chosen template.\n\nUse the \"Build Count\" table to identify workspace owners which may be struggling with template builds, in order to proactively reach out to them with assistance.\n\nConsult the [Template documentation](https://coder.com/docs/v2/latest/templates) for more information.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "failed": { + "color": "orange", + "index": 1, + "text": "Failure" + }, + "success": { + "color": "green", + "index": 0, + "text": "Success" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Workspace Transition" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "DESTROY": { + "color": "red", + "index": 0 + }, + "START": { + "color": "blue", + "index": 1 + }, + "STOP": { + "color": "purple", + "index": 2 + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 20, + "x": 0, + "y": 27.2 + }, + "id": 6, + "interval": "", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": [], + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Time" + } + ] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (workspace_name, workspace_owner, status, template_name, template_version, workspace_transition) (\n # Since new series are created and are initially set to a value of 1, we cannot use \"increase\" (because an increase from to 1 does not yield 1).\n # So we compare the current series to an interval ago to see if we have any new series and then sum the series we find. \n ((\n coderd_workspace_builds_total{workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} - \n coderd_workspace_builds_total{workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} offset $__interval\n ) >= 0) \n or coderd_workspace_builds_total{workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"}\n) > 0", + "format": "table", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Build Log", + "transformations": [ + { + "disabled": true, + "id": "groupBy", + "options": { + "fields": { + "Count": { + "aggregations": [ + "sum" + ], + "operation": "aggregate" + }, + "Status": { + "aggregations": [], + "operation": "groupby" + }, + "Template Name": { + "aggregations": [], + "operation": "groupby" + }, + "Template Version": { + "aggregations": [], + "operation": "groupby" + }, + "Total": { + "aggregations": [ + "sum" + ], + "operation": "aggregate" + }, + "Value": { + "aggregations": [ + "sum" + ], + "operation": "aggregate" + }, + "Workspace Name": { + "aggregations": [], + "operation": "groupby" + }, + "Workspace Ownert": { + "aggregations": [], + "operation": "groupby" + }, + "Workspace Transition": { + "aggregations": [], + "operation": "groupby" + }, + "status": { + "aggregations": [], + "operation": "groupby" + }, + "template_name": { + "aggregations": [], + "operation": "groupby" + }, + "template_version": { + "aggregations": [], + "operation": "groupby" + }, + "workspace_name": { + "aggregations": [], + "operation": "groupby" + }, + "workspace_owner": { + "aggregations": [], + "operation": "groupby" + }, + "workspace_transition": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Value" + } + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": false + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Count", + "Value (sum)": "Total", + "status": "Status", + "template_name": "Template Name", + "template_version": "Template Version", + "workspace_name": "Workspace Name", + "workspace_owner": "Workspace Owner", + "workspace_transition": "Workspace Transition" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 10, + "w": 4, + "x": 20, + "y": 27.2 + }, + "id": 29, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This table shows a reverse-chronological log of all workspace builds.\n\nThe \"Count\" field shows the count of events which occurred within a minute, grouped by all columns.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 37.2 + }, + "id": 8, + "interval": "1h", + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count by (workspace_owner) (coderd_workspace_latest_build_status{template_name=~\"$template_name\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Workspace by User", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 5, + "y": 37.2 + }, + "id": 9, + "interval": "1h", + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count by (workspace_owner, template_name) (coderd_workspace_latest_build_status{template_name=~\"$template_name\"})", + "instant": true, + "legendFormat": "{{workspace_owner}}:{{template_name}}", + "range": false, + "refId": "A" + } + ], + "title": "Workspace by User/Template", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 10, + "y": 37.2 + }, + "id": 4, + "interval": "1h", + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count by (template_name) (coderd_workspace_latest_build_status{template_name=~\"$template_name\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Template Usage", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 15, + "y": 37.2 + }, + "id": 5, + "interval": "1h", + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count by (template_name, template_version) (coderd_workspace_latest_build_status{template_name=~\"$template_name\"})", + "instant": true, + "legendFormat": "{{template_name}}:{{template_version}}", + "range": false, + "refId": "A" + } + ], + "title": "Template Version Usage", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 37.2 + }, + "id": 24, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "These charts show the distribution of workspaces and templates.\n\nUse these charts to identify which users have outdated templates, and which templates are the most/least popular in your organisation.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44.2 + }, + "id": 32, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 10, + "w": 20, + "x": 0, + "y": 45.2 + }, + "id": 7, + "options": { + "dedupStrategy": "exact", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{ ${NON_WORKSPACES_SELECTOR}, logger=~\"(.*runner|terraform|provisioner.*)\"} |~ \"$workspace_name\" or \"$template_name\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs", + "type": "logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 10, + "w": 4, + "x": 20, + "y": 45.2 + }, + "id": 22, + "links": [ + { + "title": "Provisioners Dashboard", + "url": "/d/provisionerd/provisioners?$${__url_time_range}" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "These are the logs produced by the [Provisioners](/d/provisionerd/provisioners?$${__url_time_range}).\n\nUse the dropdowns at the top to filter the logs down to a specific workspace and/or template.", + "mode": "markdown" + }, + "pluginVersion": "10.4.0", + "transparent": true, + "type": "text" + } + ], + "refresh": "${DASHBOARD_REFRESH}", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(coderd_workspace_builds_total,workspace_name)", + "hide": 0, + "includeAll": true, + "label": "Workspace Name Filter", + "multi": true, + "name": "workspace_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(coderd_workspace_builds_total,workspace_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(coderd_workspace_builds_total,template_name)", + "hide": 0, + "includeAll": true, + "label": "Template Name Filter", + "multi": true, + "name": "template_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(coderd_workspace_builds_total,template_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-${DASHBOARD_TIMERANGE}", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Coder Workspaces", + "uid": "workspaces", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/infra/1-click/1-setup/main.tf b/infra/1-click/1-setup/main.tf new file mode 100644 index 0000000..dbd774b --- /dev/null +++ b/infra/1-click/1-setup/main.tf @@ -0,0 +1,104 @@ +terraform { + required_version = ">= 1.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.46" + } + helm = { + source = "hashicorp/helm" + version = ">= 3.1.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + external = { + source = "hashicorp/external" + version = ">= 2.3.5" + } + } + # backend "s3" {} +} + +## +# Remote State Resources +## + +data "aws_vpc" "this" { + tags = { + "Name" = var.name + } +} + +data "aws_db_instance" "coder" { + db_instance_identifier = var.name +} + +data "aws_db_instance" "litellm" { + db_instance_identifier = "litellm" +} + +data "aws_db_instance" "grafana" { + db_instance_identifier = "grafana" +} + +data "aws_security_group" "coder" { + name = var.name + vpc_id = data.aws_vpc.this.id +} + +data "aws_eks_cluster" "coder" { + name = var.name +} + +data "aws_eks_cluster_auth" "coder" { + name = var.name +} + +data "aws_iam_openid_connect_provider" "coder" { + url = data.aws_eks_cluster.coder.identity[0].oidc[0].issuer +} + +data "aws_iam_role" "kptr-node-role" { + name = "${var.name}-kptr-${var.region}-node" +} + +## +# Global Inputs + Providers +## + +variable "region" { + description = "The AWS region of the deployment." + type = string + default = "us-east-2" +} + +variable "name" { + description = "Name for created resources and tag prefix." + type = string + default = "coder" +} + +variable "profile" { + type = string + default = "default" +} + +provider "aws" { + region = var.region + profile = var.profile +} + +provider "helm" { + kubernetes = { + host = data.aws_eks_cluster.coder.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.coder.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.coder.token + } +} + +provider "kubernetes" { + host = data.aws_eks_cluster.coder.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.coder.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.coder.token +} diff --git a/infra/1-click/1-setup/scripts/add-license.sh b/infra/1-click/1-setup/scripts/add-license.sh new file mode 100644 index 0000000..b730201 --- /dev/null +++ b/infra/1-click/1-setup/scripts/add-license.sh @@ -0,0 +1,32 @@ + +#!/usr/bin/env bash + +eval "$(jq -r '@sh "CODER_URL=\(.access_url) CODER_LICENSE=\(.license_key) CODER_SESSION_TOKEN=\(.session_token)"')" + +# URL might not be available still. Retry request until available, or fail if max attempts reached. + +IDX=0 +until curl -s -o /dev/null -kL "$CODER_URL"; do + if (( IDX >= 6 )); then + >&2 echo "Error: Unable to run 'curl -s -o /dev/null -kL \"$CODER_URL\"'." + exit 1; + fi + sleep 10 + ((IDX++)) +done + + +RESPONSE=$(curl -ks -X POST "$CODER_URL/api/v2/licenses" \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json' \ + -H "Coder-Session-Token: $CODER_SESSION_TOKEN" \ + -d "{\"license\": \"$CODER_LICENSE\"}") + +if [[ $? -ne 0 ]]; then + >&2 echo "Error: Unable to run 'curl -ks -X POST \"$CODER_URL/api/v2/licenses\"'." + exit 1; +fi + +jq -n '{"success":"true"}' + +exit 0; \ No newline at end of file diff --git a/infra/1-click/1-setup/scripts/first-user.sh b/infra/1-click/1-setup/scripts/first-user.sh new file mode 100755 index 0000000..7da1824 --- /dev/null +++ b/infra/1-click/1-setup/scripts/first-user.sh @@ -0,0 +1,33 @@ + +#!/usr/bin/env bash + +eval "$(jq -r '@sh "CODER_URL=\(.access_url) ADMIN_EMAIL=\(.admin_email) ADMIN_USERNAME=\(.admin_username) ADMIN_PASSWORD=\(.admin_password)"')" + +# URL might not be available still. Retry request until available, or fail if max attempts reached. + +IDX=0 +until curl -s -o /dev/null -kL "$CODER_URL"; do + if (( IDX >= 6 )); then + >&2 echo "Error: Unable to run 'curl -s -o /dev/null -kL \"$CODER_URL\"'." + exit 1; + fi + sleep 10 + ((IDX++)) +done + +RESPONSE=$(curl -ks -X POST "$CODER_URL/api/v2/users/first" \ + -H "Content-Type: application/json" \ + -d "{ + \"email\": \"$ADMIN_EMAIL\", + \"username\": \"$ADMIN_USERNAME\", + \"password\": \"$ADMIN_PASSWORD\", + \"trial\": false + }") + +LOGIN_RESPONSE=$(curl -ks -X POST "$CODER_URL/api/v2/users/login" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"$ADMIN_EMAIL\",\"password\":\"$ADMIN_PASSWORD\"}" | jq -r '.session_token') + +jq -n --arg session_token "$LOGIN_RESPONSE" '{"session_token":$session_token}' + +exit 0; \ No newline at end of file diff --git a/infra/1-click/2-clean.sh b/infra/1-click/2-clean.sh new file mode 100755 index 0000000..8cd72f2 --- /dev/null +++ b/infra/1-click/2-clean.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -ae -o pipefail + +## +# Set this block to 2-coder for cleaning when ready +## + +AWS_PROFILE="${CODER_AWS_PROFILE:-default}" +DOMAIN_NAME="${CODER_DOMAIN_NAME:-}" +LICENSE="${CODER_LICENSE:-}" + +echo "Change directory into '1-setup'." +cd 1-setup +terraform plan -destroy -out=tf.plan \ + -var profile=$AWS_PROFILE \ + -var domain_name=$DOMAIN_NAME \ + -var coder_license=$LICENSE +terraform apply tf.plan +cd ../ + +echo "Change directory into '0-infra'." +cd 0-infra +terraform plan -destroy -out=tf.plan \ + -var profile=$AWS_PROFILE \ + -var domain_name=$DOMAIN_NAME +terraform apply tf.plan +cd ../ \ No newline at end of file diff --git a/infra/1-click/2-coder/7-coder.tf b/infra/1-click/2-coder/7-coder.tf new file mode 100644 index 0000000..90b4b8d --- /dev/null +++ b/infra/1-click/2-coder/7-coder.tf @@ -0,0 +1,91 @@ +variable "domain_name" { + type = string +} + +variable "coder_admin_email" { + type = string + default = "admin@coder.com" +} + +variable "coder_admin_password" { + type = string + sensitive = true + default = "Th1s1sN0TS3CuR3!!" +} + +## +# Coder MUST be in a reachable state by now +## + +data "external" "fetch-token" { + + program = ["bash", "${path.module}/fetch.sh"] + + query = { + coder_url = "https://${var.domain_name}" + admin_email = var.coder_admin_email + admin_password = var.coder_admin_password + } + +} + +provider "coderd" { + url = "https://${var.domain_name}" + token = data.external.fetch-token.result.session_token +} + +output "coder_session_token" { + value = data.external.fetch-token.result.session_token +} + +locals { + image_repo = "ghcr.io/coder/coder" + image_tag = "v2.29.1" + service_account_labels = { + "app.kubernetes.io/instance" = "coder-provisioner" + "app.kubernetes.io/name" = "coder-provisioner" + "app.kubernetes.io/part-of" = "coder-provisioner" + } + node_selector = { + "node.coder.io/managed-by" = "karpenter" + "node.coder.io/used-for" = "coder-provisioner" + } + tolerations = [{ + key = "dedicated" + operator = "Equal" + value = "coder-provisioner" + effect = "NoSchedule" + }] +} + +module "default-ws" { + + depends_on = [ data.external.fetch-token ] + + source = "../../../modules/k8s/bootstrap/coder-provisioner" + + cluster_name = var.name + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn + + coder_organization_name = "coder" + + namespace = "coder-ws" + image_repo = local.image_repo + image_tag = local.image_tag + ws_service_account_name = "coder-ws" + ws_service_account_labels = { + "app.kubernetes.io/instance" = "coder-provisioner" + "app.kubernetes.io/name" = "coder-provisioner" + "app.kubernetes.io/part-of" = "coder-provisioner" + } + provisioner_service_account_name = "coder" + replica_count = 2 + primary_access_url = "https://${var.domain_name}" + env_vars = { + CODER_PROMETHEUS_ENABLE = "true" + CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" + CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" + } + node_selector = local.node_selector + tolerations = local.tolerations +} \ No newline at end of file diff --git a/infra/1-click/2-coder/fetch.sh b/infra/1-click/2-coder/fetch.sh new file mode 100755 index 0000000..38825ed --- /dev/null +++ b/infra/1-click/2-coder/fetch.sh @@ -0,0 +1,24 @@ + +#!/usr/bin/env bash + +eval "$(jq -r '@sh "CODER_URL=\(.coder_url) ADMIN_EMAIL=\(.admin_email) ADMIN_PASSWORD=\(.admin_password)"')" + +# URL might not be available still. Retry request until available, or fail if max attempts reached. + +IDX=0 +until curl -s -o /dev/null -kL "$CODER_URL"; do + if (( IDX >= 6 )); then + >&2 echo "Error: Unable to run 'curl -s -o /dev/null -kL \"$CODER_URL\"'." + exit 1; + fi + sleep 10 + ((IDX++)) +done + +LOGIN_RESPONSE=$(curl -ks -X POST "$CODER_URL/api/v2/users/login" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"$ADMIN_EMAIL\",\"password\":\"$ADMIN_PASSWORD\"}" | jq -r '.session_token') + +jq -n --arg session_token "$LOGIN_RESPONSE" '{"session_token":$session_token}' + +exit 0; \ No newline at end of file diff --git a/infra/1-click/2-coder/main.tf b/infra/1-click/2-coder/main.tf new file mode 100644 index 0000000..41db57f --- /dev/null +++ b/infra/1-click/2-coder/main.tf @@ -0,0 +1,87 @@ +terraform { + required_version = ">= 1.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.46" + } + helm = { + source = "hashicorp/helm" + version = ">= 3.1.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + external = { + source = "hashicorp/external" + version = ">= 2.3.5" + } + coderd = { + source = "coder/coderd" + version = "0.0.12" + } + } + # backend "s3" {} +} + +## +# Remote State Resources +## + +data "aws_vpc" "this" { + tags = { + "Name" = var.name + } +} + +data "aws_eks_cluster" "coder" { + name = var.name +} + +data "aws_eks_cluster_auth" "coder" { + name = var.name +} + +data "aws_iam_openid_connect_provider" "coder" { + url = data.aws_eks_cluster.coder.identity[0].oidc[0].issuer +} + +## +# Global Inputs + Providers +## + +variable "region" { + description = "The AWS region of the deployment." + type = string + default = "us-east-2" +} + +variable "name" { + description = "Name for created resources and tag prefix." + type = string + default = "coder" +} + +variable "profile" { + type = string + default = "default" +} + +provider "aws" { + region = var.region + profile = var.profile +} + +provider "helm" { + kubernetes = { + host = data.aws_eks_cluster.coder.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.coder.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.coder.token + } +} + +provider "kubernetes" { + host = data.aws_eks_cluster.coder.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.coder.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.coder.token +} diff --git a/infra/1-click/README.md b/infra/1-click/README.md new file mode 100644 index 0000000..f7a82aa --- /dev/null +++ b/infra/1-click/README.md @@ -0,0 +1,73 @@ +>[!IMPORTANT] This deployment will take around 30min ~ 1hr to setup everything (AWS resources, K8s Addons, and Coder) + +# Getting Started + +1. Create a new AWS account +2. Login as a user with AdministratorAccess +3. [Purchase/Register](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) a domain in Route53 +4. Fill the `coder.env` file with the following environment variables: + +```.env +CODER_AWS_PROFILE= +CODER_DOMAIN_NAME= +CODER_LICENSE= (Optional) +``` + +4. Run "./0-initialize.sh" +5. Run "set -a && source coder.env && set +a && ./1-plan-n-deploy.sh" +6. Run "set -a && source coder.env && set +a && ./2-clean.sh" + +# Logging In + +1. On your browser, go to "https://" +2. Enter the following login details (if you didn't override it): + +- Email: admin@coder.com +- Password: Th1s1sN0TS3CuR3!! + +3. You're done! + +# Troubleshooting + +- To verify the state of the infrastructure, visit the AWS Console and look at the pages of following AWS services that this solution deploys/manages: + - EC2 + - VPC + - RDS + - EKS + - Secrets Manager + - Route53 + +- To verify the state of the Cluster, you can run `aws eks update-kubeconfig --name --region --profile ` and execute kubectl commands. + +- To verify the state of Cluster addons, you can inspect the following and review their status/logs within the cluster: + - vpc-cni + - coredns + - metrics-server + - cert-manager + - karpenter + - aws-load-balancer-controller (lb-ctrl) + - aws-ebs-csi (ebs-ctrl) + - external-dns + - external-secrets + +- To verify the status of resources tied to CRDs, you can inspect the following: + - Service (check the aws-load-balancer-controller deployment for logs) + - Certificate + - CertificateRequests + - Order + - ExternalSecrets + - PushSecrets + - NodePool + - EC2NodeClass + + +- Be careful with running this deployment multiple times in succession. You may be getting rate-limited, either by: + - AWS (for spamming resource creation) + - Let's Encrypt's CA (for spamming certificate signing requests) + +- If the deployment fails on "data.external.first-user", then you might be running into: + - DNS hasn't propagated yet (you may need to re-run this after waiting a few more minutes) + - Domain name records not set properly. Check Route53 for your domain to see if the K8s "external-dns" addon had properly set the domain to your LoadBalancer + - Local DNS may not also be refreshed yet either (i.e. curl may fail, but browser still accessible) + +- EKS Addons may fail installation due to "taking too long". Try re-running the deployment script to verify if changes have finished applying diff --git a/infra/1-click/coder.env b/infra/1-click/coder.env new file mode 100644 index 0000000..1beb75f --- /dev/null +++ b/infra/1-click/coder.env @@ -0,0 +1,3 @@ +CODER_AWS_PROFILE= +CODER_DOMAIN_NAME= +CODER_LICENSE= \ No newline at end of file From 1bb58ddcd9013d52531f8845efd1547131ea38f6 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 20 Jan 2026 12:32:17 -0800 Subject: [PATCH 04/44] fix: README updates --- infra/1-click/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/infra/1-click/README.md b/infra/1-click/README.md index f7a82aa..36d7778 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -13,17 +13,19 @@ CODER_DOMAIN_NAME= CODER_LICENSE= (Optional) ``` -4. Run "./0-initialize.sh" -5. Run "set -a && source coder.env && set +a && ./1-plan-n-deploy.sh" -6. Run "set -a && source coder.env && set +a && ./2-clean.sh" +4. Run `./0-initialize.sh` +5. Run `set -a && source coder.env && set +a && ./1-plan-n-deploy.sh` +6. Run `set -a && source coder.env && set +a && ./2-clean.sh` # Logging In 1. On your browser, go to "https://" 2. Enter the following login details (if you didn't override it): -- Email: admin@coder.com -- Password: Th1s1sN0TS3CuR3!! +``` +Email: admin@coder.com +Password: Th1s1sN0TS3CuR3!! +``` 3. You're done! From 49277d180e810794c0309695aa6d972a122bcd73 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 20 Jan 2026 12:35:22 -0800 Subject: [PATCH 05/44] fix: README updates --- infra/1-click/README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/infra/1-click/README.md b/infra/1-click/README.md index 36d7778..abe56df 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -4,8 +4,12 @@ 1. Create a new AWS account 2. Login as a user with AdministratorAccess -3. [Purchase/Register](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) a domain in Route53 -4. Fill the `coder.env` file with the following environment variables: +3. Setup your local machine to have an [AWS profile](https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-files.html) + +>[!NOTE] When running `aws login` or `aws configure`, this will automatically setup a `default` profile for you. + +4. [Purchase/Register](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) a domain in Route53 +5. Fill the `coder.env` file with the following environment variables: ```.env CODER_AWS_PROFILE= @@ -13,9 +17,9 @@ CODER_DOMAIN_NAME= CODER_LICENSE= (Optional) ``` -4. Run `./0-initialize.sh` -5. Run `set -a && source coder.env && set +a && ./1-plan-n-deploy.sh` -6. Run `set -a && source coder.env && set +a && ./2-clean.sh` +6. Run `./0-initialize.sh` +7. Run `set -a && source coder.env && set +a && ./1-plan-n-deploy.sh` +8. Run `set -a && source coder.env && set +a && ./2-clean.sh` # Logging In From 6284799b11f649c7cbd195195b8f207190be2780 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 20 Jan 2026 14:50:34 -0800 Subject: [PATCH 06/44] fix: README updates --- infra/1-click/README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/infra/1-click/README.md b/infra/1-click/README.md index abe56df..c236f13 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -1,4 +1,5 @@ ->[!IMPORTANT] This deployment will take around 30min ~ 1hr to setup everything (AWS resources, K8s Addons, and Coder) +> [!IMPORTANT] +> This deployment will take around 30min ~ 1hr to setup everything (AWS resources, K8s Addons, and Coder) # Getting Started @@ -6,9 +7,15 @@ 2. Login as a user with AdministratorAccess 3. Setup your local machine to have an [AWS profile](https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-files.html) ->[!NOTE] When running `aws login` or `aws configure`, this will automatically setup a `default` profile for you. +>[!NOTE] +> When running `aws login` or `aws configure`, this will automatically setup a `default` profile for you. + +4. [Purchase/Register](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) a domain in Route53 and tie it to a public hosted zone. + +> [!IMPORTANT] +> When purchasing, it will create an associated hosted zone in a few minutes. Make sure to wait until this is available. +> If using an existing domain + hosted zone, cleanup any A/AAAA/CNAME/TXT records. -4. [Purchase/Register](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) a domain in Route53 5. Fill the `coder.env` file with the following environment variables: ```.env @@ -58,9 +65,11 @@ Password: Th1s1sN0TS3CuR3!! - To verify the status of resources tied to CRDs, you can inspect the following: - Service (check the aws-load-balancer-controller deployment for logs) + - ClusterIssuer - Certificate - CertificateRequests - Order + - ClusterSecretStore - ExternalSecrets - PushSecrets - NodePool From ea9f850479f8d557641288200af9e5add20569d3 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 20 Jan 2026 14:53:43 -0800 Subject: [PATCH 07/44] fix: README updates --- infra/1-click/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/infra/1-click/README.md b/infra/1-click/README.md index c236f13..1b3636a 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -13,8 +13,7 @@ 4. [Purchase/Register](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) a domain in Route53 and tie it to a public hosted zone. > [!IMPORTANT] -> When purchasing, it will create an associated hosted zone in a few minutes. Make sure to wait until this is available. -> If using an existing domain + hosted zone, cleanup any A/AAAA/CNAME/TXT records. +> When purchasing, it creates a hosted zone in a few minutes, wait for this. AND, if using an existing domain/zone, cleanup any A/AAAA/CNAME/TXT records. 5. Fill the `coder.env` file with the following environment variables: From c627580b765a593d326a65e0f412c22f64b95ed7 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Wed, 21 Jan 2026 01:41:31 -0800 Subject: [PATCH 08/44] fix: Updates module naming conventions, README descriptions, and removes unused modules --- infra/1-click/0-infra/3-eks.tf | 2 +- infra/1-click/0-infra/4-bootstrap.tf | 8 +- infra/1-click/1-setup/5-manifests.tf | 66 ++-- infra/1-click/1-setup/main.tf | 4 - infra/1-click/README.md | 18 +- modules/k8s/bootstrap/cert-manager/main.tf | 4 +- modules/k8s/bootstrap/coder-server/main.tf | 4 +- modules/k8s/bootstrap/ebs-controller/main.tf | 2 +- modules/k8s/bootstrap/external-dns/main.tf | 4 +- .../k8s/bootstrap/external-secrets/main.tf | 5 +- modules/k8s/bootstrap/fetch-and-store/main.tf | 303 ------------------ .../k8s/bootstrap/fetch-and-store/policy.tf | 31 -- .../scripts/fetch-and-store.py | 87 ----- .../fetch-and-store/scripts/requirements.txt | 15 - modules/k8s/bootstrap/karpenter/main.tf | 39 ++- modules/k8s/bootstrap/lb-controller/main.tf | 4 +- .../k8s/bootstrap/litellm-rotate-key/main.tf | 212 ------------ .../litellm-rotate-key/scripts/rotate.sh | 10 - 18 files changed, 87 insertions(+), 731 deletions(-) delete mode 100644 modules/k8s/bootstrap/fetch-and-store/main.tf delete mode 100644 modules/k8s/bootstrap/fetch-and-store/policy.tf delete mode 100644 modules/k8s/bootstrap/fetch-and-store/scripts/fetch-and-store.py delete mode 100644 modules/k8s/bootstrap/fetch-and-store/scripts/requirements.txt delete mode 100644 modules/k8s/bootstrap/litellm-rotate-key/main.tf delete mode 100644 modules/k8s/bootstrap/litellm-rotate-key/scripts/rotate.sh diff --git a/infra/1-click/0-infra/3-eks.tf b/infra/1-click/0-infra/3-eks.tf index 961931b..a65d6c1 100644 --- a/infra/1-click/0-infra/3-eks.tf +++ b/infra/1-click/0-infra/3-eks.tf @@ -11,7 +11,7 @@ data "aws_iam_policy_document" "sts" { } resource "aws_iam_policy" "sts" { - name_prefix = "sts" + name_prefix = "${var.name}-sts-" path = "/" description = "Assume Role Policy" policy = data.aws_iam_policy_document.sts.json diff --git a/infra/1-click/0-infra/4-bootstrap.tf b/infra/1-click/0-infra/4-bootstrap.tf index 0ba5c42..a5749db 100644 --- a/infra/1-click/0-infra/4-bootstrap.tf +++ b/infra/1-click/0-infra/4-bootstrap.tf @@ -17,8 +17,8 @@ module "karpenter" { chart_version = "1.8.4" node_selector = local.labels_system_node - iam_role_use_name_prefix = false - node_iam_role_use_name_prefix = false + iam_role_use_name_prefix = true + node_iam_role_use_name_prefix = true karpenter_controller_role_policies = { "AmazonEFSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" } @@ -58,7 +58,7 @@ module "lb-controller" { cluster_name = module.eks.cluster_name cluster_oidc_provider_arn = module.eks.oidc_provider_arn - namespace = "lb-crtl" + namespace = "lb-ctrl" chart_version = "1.13.2" enable_cert_manager = true vpc_id = module.vpc.vpc_id @@ -77,7 +77,7 @@ module "ebs-controller" { cluster_name = module.eks.cluster_name cluster_oidc_provider_arn = module.eks.oidc_provider_arn - namespace = "ebs-crtl" + namespace = "ebs-ctrl" chart_version = "2.22.1" node_selector = local.labels_system_node replace = true diff --git a/infra/1-click/1-setup/5-manifests.tf b/infra/1-click/1-setup/5-manifests.tf index b9a8cc2..1a0b0fb 100644 --- a/infra/1-click/1-setup/5-manifests.tf +++ b/infra/1-click/1-setup/5-manifests.tf @@ -7,34 +7,14 @@ # NodeClass + NodePool for Coder Server, Provisioner, & Workspaces ## -locals { - global_node_labels = { - "node.coder.io/instance" = "coder-v2" - "node.coder.io/managed-by" = "karpenter" - } - global_node_reqs = [{ - key = "kubernetes.io/arch" - operator = "In" - values = ["amd64"] - }, { - key = "kubernetes.io/os" - operator = "In" - values = ["linux"] - }, { - key = "kubernetes.sh/capacity-type" - operator = "In" - values = ["spot", "on-demand"] - }] - sg_tags = { - "karpenter.sh/discovery" = var.name - } - subnet_tags = { - "karpenter.sh/discovery" = var.name +data "kubernetes_service_account_v1" "kptr" { + metadata { + name = "node-role" + namespace = "karpenter" } } locals { - node_role_name = data.aws_iam_role.kptr-node-role.name nodeclass_configs = { "coder-server" = { user_data = "" @@ -77,15 +57,19 @@ resource "kubernetes_manifest" "nodeclass" { name = each.key } spec = { - role = local.node_role_name + role = data.kubernetes_service_account_v1.kptr.metadata[0].annotations["eks.amazonaws.com/role-arn"] amiSelectorTerms = [{ alias = "al2023@latest" }] subnetSelectorTerms = [{ - tags = local.subnet_tags + tags = { + "karpenter.sh/discovery" = var.name + } }] securityGroupSelectorTerms = [{ - tags = local.sg_tags + tags = { + "karpenter.sh/discovery" = var.name + } }] blockDeviceMappings = each.value.block_device_mappings userData = each.value.user_data @@ -145,11 +129,13 @@ resource "kubernetes_manifest" "nodepool" { spec = { template = { metadata = { - labels = merge(local.global_node_labels, { - "node.coder.io/name" = "coder" - "node.coder.io/part-of" = "coder" - "node.coder.io/used-for" = each.key - }) + labels = { + "node.coder.io/instance" = "coder-v2" + "node.coder.io/managed-by" = "karpenter" + "node.coder.io/name" = "coder" + "node.coder.io/part-of" = "coder" + "node.coder.io/used-for" = each.key + } } spec = { taints = [{ @@ -157,11 +143,23 @@ resource "kubernetes_manifest" "nodepool" { value = each.key effect = "NoSchedule" }] - requirements =concat(local.global_node_reqs, [{ + requirements = [{ + key = "kubernetes.io/arch" + operator = "In" + values = ["amd64"] + }, { + key = "kubernetes.io/os" + operator = "In" + values = ["linux"] + }, { + key = "kubernetes.sh/capacity-type" + operator = "In" + values = ["spot", "on-demand"] + },{ key = "node.kubernetes.io/instance-type" operator = "In" values = ["t3a.xlarge"] - }]) + }] nodeClassRef = { group = "karpenter.k8s.aws" kind = "EC2NodeClass" diff --git a/infra/1-click/1-setup/main.tf b/infra/1-click/1-setup/main.tf index dbd774b..fb337a1 100644 --- a/infra/1-click/1-setup/main.tf +++ b/infra/1-click/1-setup/main.tf @@ -59,10 +59,6 @@ data "aws_iam_openid_connect_provider" "coder" { url = data.aws_eks_cluster.coder.identity[0].oidc[0].issuer } -data "aws_iam_role" "kptr-node-role" { - name = "${var.name}-kptr-${var.region}-node" -} - ## # Global Inputs + Providers ## diff --git a/infra/1-click/README.md b/infra/1-click/README.md index 1b3636a..c8ffc65 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -1,9 +1,15 @@ +# Requirements + +- Terraform +- AWS Account +- Domain in Route53 + > [!IMPORTANT] > This deployment will take around 30min ~ 1hr to setup everything (AWS resources, K8s Addons, and Coder) # Getting Started -1. Create a new AWS account +1. Create a new AWS account or access an existing one 2. Login as a user with AdministratorAccess 3. Setup your local machine to have an [AWS profile](https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-files.html) @@ -29,7 +35,7 @@ CODER_LICENSE= (Optional) # Logging In -1. On your browser, go to "https://" +1. On your browser, go to "https://put.your.domain.here.com" 2. Enter the following login details (if you didn't override it): ``` @@ -44,14 +50,16 @@ Password: Th1s1sN0TS3CuR3!! - To verify the state of the infrastructure, visit the AWS Console and look at the pages of following AWS services that this solution deploys/manages: - EC2 - VPC - - RDS + - RDS (Database, Snapshots, and SubnetGroups) - EKS - Secrets Manager - Route53 + - SSM Parameters + - CloudWatch Logs -- To verify the state of the Cluster, you can run `aws eks update-kubeconfig --name --region --profile ` and execute kubectl commands. +- To verify the cluster state, run `aws eks update-kubeconfig --name --region --profile ` and execute kubectl commands. -- To verify the state of Cluster addons, you can inspect the following and review their status/logs within the cluster: +- To verify the cluster addons, inspect the status/logs of the following cluster addons: - vpc-cni - coredns - metrics-server diff --git a/modules/k8s/bootstrap/cert-manager/main.tf b/modules/k8s/bootstrap/cert-manager/main.tf index d622b2e..1966650 100644 --- a/modules/k8s/bootstrap/cert-manager/main.tf +++ b/modules/k8s/bootstrap/cert-manager/main.tf @@ -95,8 +95,8 @@ data "aws_caller_identity" "this" {} locals { region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "CertManager-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "cert-manager-${data.aws_region.this.region}" : var.role_name + policy_name = var.policy_name == "" ? "${var.cluster_name}-crt-mgr-${data.aws_region.this.region}" : var.policy_name + role_name = var.role_name == "" ? "${var.cluster_name}-crt-mgr-${data.aws_region.this.region}" : var.role_name } module "policy" { diff --git a/modules/k8s/bootstrap/coder-server/main.tf b/modules/k8s/bootstrap/coder-server/main.tf index bbb4b14..b1e990c 100644 --- a/modules/k8s/bootstrap/coder-server/main.tf +++ b/modules/k8s/bootstrap/coder-server/main.tf @@ -612,8 +612,8 @@ locals { locals { region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "Server-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "server-${data.aws_region.this.region}" : var.role_name + policy_name = var.policy_name == "" ? "${var.cluster_name}-coder-srv-${data.aws_region.this.region}" : var.policy_name + role_name = var.role_name == "" ? "${var.cluster_name}-coder-srv-${data.aws_region.this.region}" : var.role_name } module "provisioner-policy" { diff --git a/modules/k8s/bootstrap/ebs-controller/main.tf b/modules/k8s/bootstrap/ebs-controller/main.tf index aac1255..0191fc9 100644 --- a/modules/k8s/bootstrap/ebs-controller/main.tf +++ b/modules/k8s/bootstrap/ebs-controller/main.tf @@ -61,7 +61,7 @@ variable "replace" { data "aws_region" "this" {} locals { - role_name = var.role_name == "" ? "ebs-controller-${data.aws_region.this.region}" : var.role_name + role_name = var.role_name == "" ? "${var.cluster_name}-ebs-ctrl-${data.aws_region.this.region}" : var.role_name } module "oidc-role" { diff --git a/modules/k8s/bootstrap/external-dns/main.tf b/modules/k8s/bootstrap/external-dns/main.tf index d5dbf3a..c518037 100644 --- a/modules/k8s/bootstrap/external-dns/main.tf +++ b/modules/k8s/bootstrap/external-dns/main.tf @@ -72,8 +72,8 @@ data "aws_caller_identity" "this" {} locals { region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "ExternalDNS-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "externaldns-${data.aws_region.this.region}" : var.role_name + policy_name = var.policy_name == "" ? "${var.cluster_name}-ext-dns-${data.aws_region.this.region}" : var.policy_name + role_name = var.role_name == "" ? "${var.cluster_name}-ext-dns-${data.aws_region.this.region}" : var.role_name } module "policy" { diff --git a/modules/k8s/bootstrap/external-secrets/main.tf b/modules/k8s/bootstrap/external-secrets/main.tf index d6d5671..75a62f4 100644 --- a/modules/k8s/bootstrap/external-secrets/main.tf +++ b/modules/k8s/bootstrap/external-secrets/main.tf @@ -72,8 +72,8 @@ data "aws_caller_identity" "this" {} locals { region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "ExternalSec-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "externalsec-${data.aws_region.this.region}" : var.role_name + policy_name = var.policy_name == "" ? "${var.cluster_name}-ext-sec-${data.aws_region.this.region}" : var.policy_name + role_name = var.role_name == "" ? "${var.cluster_name}-ext-sec-${data.aws_region.this.region}" : var.role_name } module "policy" { @@ -114,6 +114,7 @@ resource "helm_release" "chart" { timeout = 120 # in seconds values = [yamlencode({ + nodeSelector = var.node_selector serviceAccount = { annotations = { "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn diff --git a/modules/k8s/bootstrap/fetch-and-store/main.tf b/modules/k8s/bootstrap/fetch-and-store/main.tf deleted file mode 100644 index f4b50ef..0000000 --- a/modules/k8s/bootstrap/fetch-and-store/main.tf +++ /dev/null @@ -1,303 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } -} - -variable "cluster_name" { - type = string -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "policy_name" { - type = string - default = "" -} - -variable "role_name" { - type = string - default = "" -} - -variable "namespace" { - type = string -} - -variable "name" { - type = string - default = "fetch-and-store" -} - -variable "image_repo" { - type = string -} - -variable "image_tag" { - type = string -} - -variable "fetch_and_store_script_file_name" { - type = string - default = "fetch-and-store.py" -} - -variable "fetch_and_store_script_pip_file_name" { - type = string - default = "requirements.txt" -} - -variable "tags" { - type = map(string) - default = {} -} - -data "aws_region" "this" {} - -data "aws_caller_identity" "this" {} - -locals { - app_labels = { - "app.kubernetes.io/name" : var.name - "app.kubernetes.io/part-of" : var.name - } - policy_name = var.policy_name == "" ? "FetchAndStore-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "fetch-and-store-${data.aws_region.this.region}" : var.role_name -} - -module "policy" { - source = "../../../security/policy" - name = local.policy_name - path = "/" - description = "Fetch-and-Store Image Policy" - policy_json = data.aws_iam_policy_document.this.json -} - -module "oidc-role" { - source = "../../../security/role/access-entry" - name = local.role_name - cluster_name = var.cluster_name - policy_arns = { - "FetchAndStore" = module.policy.policy_arn - } - cluster_policy_arns = { - "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", - } - oidc_principals = { - "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] - } - tags = var.tags -} - -resource "kubernetes_namespace" "this" { - metadata { - name = var.namespace - } -} - -resource "kubernetes_role" "this" { - metadata { - name = var.name - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels - } - rule { - api_groups = [""] - resources = ["configmaps"] - resource_names = ["fetch-and-store"] - verbs = ["get", "create", "update", "patch"] - } -} - -resource "kubernetes_service_account" "this" { - metadata { - name = var.name - namespace = kubernetes_namespace.this.metadata[0].name - annotations = { - "eks.amazonaws.com/role-arn" : module.oidc-role.role_arn - } - labels = local.app_labels - } -} - -resource "kubernetes_role_binding" "this" { - metadata { - name = var.name - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "Role" - name = kubernetes_role.this.metadata[0].name - } - subject { - kind = "ServiceAccount" - name = kubernetes_service_account.this.metadata[0].name - namespace = kubernetes_namespace.this.metadata[0].name - } -} - -resource "kubernetes_config_map" "script" { - metadata { - name = "python-fetch-and-store" - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels - } - data = { - "${var.fetch_and_store_script_file_name}" = file("${path.module}/scripts/${var.fetch_and_store_script_file_name}") - } -} - -resource "kubernetes_config_map" "pip" { - metadata { - name = "python-pip-requirements" - namespace = kubernetes_namespace.this.metadata[0].name - labels = local.app_labels - } - data = { - "${var.fetch_and_store_script_pip_file_name}" = file("${path.module}/scripts/${var.fetch_and_store_script_pip_file_name}") - } -} - -resource "kubernetes_manifest" "this" { - field_manager { - force_conflicts = true - } - manifest = { - apiVersion = "batch/v1" - kind = "CronJob" - metadata = { - labels = local.app_labels - name = var.name - namespace = kubernetes_namespace.this.metadata[0].name - } - spec = { - schedule = "0 0 * * 1" - successfulJobsHistoryLimit = 0 - failedJobsHistoryLimit = 1 - suspend = false - timeZone = "America/Vancouver" - concurrencyPolicy = "Replace" - jobTemplate = { - metadata = { - labels = local.app_labels - } - spec = { - parallelism = 1 - template = { - metadata = { - labels = local.app_labels - } - spec = { - serviceAccountName = kubernetes_service_account.this.metadata[0].name - restartPolicy = "OnFailure" - initContainers = [{ - name = "fetch" - image = "ghcr.io/coder/coder-preview:latest" - imagePullPolicy = "IfNotPresent" - command = split(" ", "/bin/sh -c exit 0") - }, { - name = "docker-sidecar" - image = "docker:dind" - restartPolicy = "Always" - imagePullPolicy = "IfNotPresent" - command = split(" ", "dockerd -H tcp://127.0.0.1:2375") - env = [{ - name = "DOCKER_HOST" - value = "tcp://localhost:2375" - }] - resources = { - limits = { - cpu = "1" - ephemeral-storage = "10Gi" - memory = "2Gi" - } - requests = { - ephemeral-storage = "5Gi" - } - } - securityContext = { - allowPrivilegeEscalation = true - privileged = true - runAsUser = 0 - } - }] - containers = [{ - name = "store" - image = "${var.image_repo}:${var.image_tag}" - imagePullPolicy = "IfNotPresent" - command = ["bash", "-c", "pip install -r /tmp/${var.fetch_and_store_script_pip_file_name} && python /tmp/${var.fetch_and_store_script_file_name}"] - resources = { - limits = { - cpu = "2" - ephemeral-storage = "20Gi" - memory = "9Gi" - } - requests = { - cpu = "1" - ephemeral-storage = "10Gi" - memory = "1Gi" - } - } - env = [{ - name = "DOCKER_HOST" - value = "tcp://localhost:2375" - }, { - name = "DESIRED_TAG" - value = "latest" - }, { - name = "GIT_URL" - value = "https://github.com/coder/coder" - }, { - name = "AWS_ACCOUNT_ID" - value = data.aws_caller_identity.this.account_id - }, { - name = "AWS_REGION" - value = data.aws_region.this.region - }] - volumeMounts = [{ - name = kubernetes_config_map.script.metadata[0].name - mountPath = "/tmp/${var.fetch_and_store_script_file_name}" - subPath = var.fetch_and_store_script_file_name - }, { - name = kubernetes_config_map.pip.metadata[0].name - mountPath = "/tmp/${var.fetch_and_store_script_pip_file_name}" - subPath = var.fetch_and_store_script_pip_file_name - }] - }] - volumes = [{ - name = kubernetes_config_map.script.metadata[0].name - configMap = { - name = kubernetes_config_map.script.metadata[0].name - defaultMode = 511 # Equivalent to 777 - items = [{ - key = var.fetch_and_store_script_file_name - path = var.fetch_and_store_script_file_name - }] - } - }, { - name = kubernetes_config_map.pip.metadata[0].name - configMap = { - name = kubernetes_config_map.pip.metadata[0].name - defaultMode = 511 # Equivalent to 777 - items = [{ - key = var.fetch_and_store_script_pip_file_name - path = var.fetch_and_store_script_pip_file_name - }] - } - }] - } - } - } - } - } - } -} \ No newline at end of file diff --git a/modules/k8s/bootstrap/fetch-and-store/policy.tf b/modules/k8s/bootstrap/fetch-and-store/policy.tf deleted file mode 100644 index 441d6b8..0000000 --- a/modules/k8s/bootstrap/fetch-and-store/policy.tf +++ /dev/null @@ -1,31 +0,0 @@ -data "aws_iam_policy_document" "this" { - statement { - effect = "Allow" - actions = ["ecr:GetAuthorizationToken"] - resources = ["*"] - } - - statement { - effect = "Allow" - actions = [ - "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", - "ecr:GetDownloadUrlForLayer" - ] - resources = ["*"] - } - - statement { - effect = "Allow" - actions = [ - "ecr:CompleteLayerUpload", - "ecr:UploadLayerPart", - "ecr:InitiateLayerUpload", - "ecr:BatchCheckLayerAvailability", - "ecr:PutImage", - "ecr:BatchGetImage", - "ecr:DescribeRepositories", - ] - resources = ["arn:aws:ecr:${data.aws_region.this.region}:${data.aws_caller_identity.this.account_id}:repository/*"] - } -} \ No newline at end of file diff --git a/modules/k8s/bootstrap/fetch-and-store/scripts/fetch-and-store.py b/modules/k8s/bootstrap/fetch-and-store/scripts/fetch-and-store.py deleted file mode 100644 index 5a6da52..0000000 --- a/modules/k8s/bootstrap/fetch-and-store/scripts/fetch-and-store.py +++ /dev/null @@ -1,87 +0,0 @@ -from git import Repo -from os import environ -import tempfile, boto3, docker, base64, logging - -class FetchAndStoreFromECR: - def __init__(self, regionName, profileName=""): - self.logger = logging.getLogger(type(self).__name__) - - kwargs = { - 'region_name': regionName - } - if profileName: - kwargs['profile_name'] = profileName - - session = boto3.Session(**kwargs) - ecr = session.client('ecr') - - auth = ecr.get_authorization_token()['authorizationData'][0] - - username, token = base64.b64decode(auth['authorizationToken']).decode().split(":") - self.auth = { - 'username': username, - 'password': token, - 'registry': auth['proxyEndpoint'] - } - - self.client = docker.from_env() - self.client.login(**self.auth) - - def getLatestCommitSha(self, url, branch="main"): - with tempfile.TemporaryDirectory() as tmpDir: - self.logger.info(f"Temporary Dir: {tmpDir}") - workingTree = Repo.clone_from(url, tmpDir) - commitPrefix = workingTree.commit(branch).hexsha[:9] - - return commitPrefix - - def getDigest(self, image): - tag = f"{image.attrs['Id'].replace(":", "-")}.att" - self.logger.info(f"Image Digest: {tag}") - return tag - - def fetch(self, repo, tag="latest", auth={}): - res = self.client.images.pull(repo, tag=tag, auth_config=auth) - self.logger.info(res) - return res - - def tag(self, image, repo: str, tag: str): - res = image.tag(repo, tag=tag) - self.logger.info(res) - return res - - def store(self, repo, tag="latest", auth={}): - res = self.client.images.push(repo, tag=tag, auth_config=auth, stream=True, decode=True) - for line in res: - self.logger.info(line) - return res - - def fetch_and_store(self, desired, desiredTag, target, targetTag=""): - image = self.fetch(desired, desiredTag) - if not targetTag: - targetTag = self.getDigest(image) - self.tag(image, target, targetTag) - self.store(target, targetTag, self.auth) - -profileName = environ.get("AWS_PROFILE_NAME", "") -accountId = environ.get("AWS_ACCOUNT_ID") -regionName = environ.get("AWS_REGION", "us-east-2") -desiredImg = environ.get("DESIRED_IMAGE", "ghcr.io/coder/coder-preview") -desiredTag = environ.get("DESIRED_TAG", "latest") -targetImg = environ.get("TARGET_IMAGE", f"{accountId}.dkr.ecr.{regionName}.amazonaws.com/coder-preview") -targetTag = environ.get("TARGET_TAG", "") -gitUrl = environ.get("GIT_URL", "https://github.com/coder/coder") - -logging.basicConfig(level=logging.INFO) - -client = FetchAndStoreFromECR(regionName, profileName) - -client.fetch_and_store( - desiredImg, desiredTag, - targetImg, targetTag -) - -client.fetch_and_store( - desiredImg, desiredTag, - targetImg, "latest" -) \ No newline at end of file diff --git a/modules/k8s/bootstrap/fetch-and-store/scripts/requirements.txt b/modules/k8s/bootstrap/fetch-and-store/scripts/requirements.txt deleted file mode 100644 index db4ab11..0000000 --- a/modules/k8s/bootstrap/fetch-and-store/scripts/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -boto3==1.40.46 -botocore==1.40.46 -certifi==2025.10.5 -charset-normalizer==3.4.3 -docker==7.1.0 -gitdb==4.0.12 -GitPython==3.1.45 -idna==3.10 -jmespath==1.0.1 -python-dateutil==2.9.0.post0 -requests==2.32.5 -s3transfer==0.14.0 -six==1.17.0 -smmap==5.0.2 -urllib3==2.5.0 diff --git a/modules/k8s/bootstrap/karpenter/main.tf b/modules/k8s/bootstrap/karpenter/main.tf index dfe91b0..776c89e 100644 --- a/modules/k8s/bootstrap/karpenter/main.tf +++ b/modules/k8s/bootstrap/karpenter/main.tf @@ -171,7 +171,7 @@ data "aws_iam_policy_document" "sts" { } resource "aws_iam_policy" "sts" { - name_prefix = "sts" + name_prefix = "${var.cluster_name}-sts-" path = "/" description = "Assume Role Policy" policy = data.aws_iam_policy_document.sts.json @@ -205,7 +205,6 @@ data "aws_iam_policy_document" "kptr_ctrl_assume_role_policy" { module "karpenter" { source = "terraform-aws-modules/eks/aws//modules/karpenter" - # version = "20.34.0" # "21.3.1" version = "21.14.0" cluster_name = var.cluster_name @@ -316,16 +315,28 @@ resource "kubernetes_manifest" "ec2nodeclass" { manifest = yamldecode(module.ec2nodeclass[count.index].manifest) } -# module "nodepool" { -# count = length(local.nodepool_configs) -# source = "../objects/nodepool" -# name = local.nodepool_configs[count.index].name -# node_labels = local.nodepool_configs[count.index].node_labels -# node_taints = local.nodepool_configs[count.index].node_taints -# node_requirements = local.nodepool_configs[count.index].node_requirements -# node_class_ref_name = local.nodepool_configs[count.index].node_class_ref_name -# node_expires_after = local.nodepool_configs[count.index].node_expires_after -# disruption_consolidation_policy = local.nodepool_configs[count.index].disruption_consolidation_policy -# disruption_consolidate_after = local.nodepool_configs[count.index].disruption_consolidate_after -# } +resource "kubernetes_service_account_v1" "ctrl-role" { + depends_on = [helm_release.karpenter] + + metadata { + name = "ctrl-role" + namespace = var.namespace + annotations = { + "eks.amazonaws.com/role-arn" = module.karpenter.iam_role_arn + } + } +} + +resource "kubernetes_service_account_v1" "node-role" { + + depends_on = [helm_release.karpenter] + + metadata { + name = "node-role" + namespace = var.namespace + annotations = { + "eks.amazonaws.com/role-arn" = module.karpenter.node_iam_role_arn + } + } +} \ No newline at end of file diff --git a/modules/k8s/bootstrap/lb-controller/main.tf b/modules/k8s/bootstrap/lb-controller/main.tf index a560656..d928c48 100644 --- a/modules/k8s/bootstrap/lb-controller/main.tf +++ b/modules/k8s/bootstrap/lb-controller/main.tf @@ -102,8 +102,8 @@ data "aws_caller_identity" "this" {} locals { region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "LBController-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "lb-controller-${data.aws_region.this.region}" : var.role_name + policy_name = var.policy_name == "" ? "${var.cluster_name}-lb-ctrl-${data.aws_region.this.region}" : var.policy_name + role_name = var.role_name == "" ? "${var.cluster_name}-lb-ctrl-${data.aws_region.this.region}" : var.role_name } module "policy" { diff --git a/modules/k8s/bootstrap/litellm-rotate-key/main.tf b/modules/k8s/bootstrap/litellm-rotate-key/main.tf deleted file mode 100644 index 69246c9..0000000 --- a/modules/k8s/bootstrap/litellm-rotate-key/main.tf +++ /dev/null @@ -1,212 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } -} - -variable "cluster_name" { - type = string -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "policy_name" { - type = string - default = "" -} - -variable "role_name" { - type = string - default = "" -} - -variable "namespace" { - type = string -} - -variable "name" { - type = string - default = "rotate-key" -} - -variable "image_repo" { - type = string -} - -variable "image_tag" { - type = string -} - -variable "secret_id" { - type = string - sensitive = true -} - -variable "secret_region" { - type = string - sensitive = true -} - -variable "rotate_key_script_file_name" { - type = string - default = "rotate.sh" -} - -variable "tags" { - type = map(string) - default = {} -} - -data "aws_region" "this" {} - -data "aws_caller_identity" "this" {} - -locals { - app_labels = { - "app.kubernetes.io/name" : var.name - "app.kubernetes.io/part-of" : var.name - } - policy_name = var.policy_name == "" ? "LiteLLM-Swap-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "litellm-swap-${data.aws_region.this.region}" : var.role_name -} - -module "oidc-role" { - source = "../../../security/role/access-entry" - name = local.role_name - cluster_name = var.cluster_name - policy_arns = { - "SecretsManagerReadWrite" = "arn:aws:iam::aws:policy/SecretsManagerReadWrite" - } - cluster_policy_arns = { - "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", - } - oidc_principals = { - "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] - } - tags = var.tags -} - -resource "kubernetes_role" "this" { - metadata { - name = var.name - namespace = var.namespace - labels = local.app_labels - } - rule { - api_groups = [""] - resources = ["secrets"] - verbs = ["get", "create", "update", "patch", "list"] - } -} - -resource "kubernetes_service_account" "this" { - metadata { - name = var.name - namespace = var.namespace - annotations = { - "eks.amazonaws.com/role-arn" : module.oidc-role.role_arn - } - labels = local.app_labels - } -} - -resource "kubernetes_role_binding" "this" { - metadata { - name = var.name - namespace = var.namespace - labels = local.app_labels - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "Role" - name = kubernetes_role.this.metadata[0].name - } - subject { - kind = "ServiceAccount" - name = kubernetes_service_account.this.metadata[0].name - namespace = var.namespace - } -} - -resource "kubernetes_config_map" "this" { - metadata { - name = var.name - namespace = var.namespace - labels = local.app_labels - } - data = { - "${var.rotate_key_script_file_name}" = templatefile("${path.module}/scripts/${var.rotate_key_script_file_name}", {}) - } -} - -locals { - primary_env_vars = { - AWS_SECRETS_MANAGER_ID = var.secret_id - AWS_SECRET_REGION = var.secret_region - K8S_NAMESPACE = var.namespace - } -} - -resource "kubernetes_cron_job_v1" "this" { - metadata { - name = var.name - namespace = var.namespace - labels = local.app_labels - } - spec { - timezone = "America/Vancouver" - successful_jobs_history_limit = 0 - failed_jobs_history_limit = 1 - concurrency_policy = "Replace" - schedule = "0 */4 * * *" - job_template { - metadata { - labels = local.app_labels - } - spec { - parallelism = 1 - template { - metadata { - labels = local.app_labels - } - spec { - service_account_name = kubernetes_service_account.this.metadata[0].name - restart_policy = "OnFailure" - container { - name = var.name - image = "${var.image_repo}:${var.image_tag}" - image_pull_policy = "IfNotPresent" - command = split(" ", "/bin/bash -c /tmp/${var.rotate_key_script_file_name}") - dynamic "env" { - for_each = local.primary_env_vars - content { - name = env.key - value = tostring(env.value) - } - } - volume_mount { - name = kubernetes_config_map.this.metadata[0].name - mount_path = "/tmp/${var.rotate_key_script_file_name}" - sub_path = var.rotate_key_script_file_name - } - } - volume { - name = kubernetes_config_map.this.metadata[0].name - config_map { - name = kubernetes_config_map.this.metadata[0].name - default_mode = "0777" - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/modules/k8s/bootstrap/litellm-rotate-key/scripts/rotate.sh b/modules/k8s/bootstrap/litellm-rotate-key/scripts/rotate.sh deleted file mode 100644 index 542e363..0000000 --- a/modules/k8s/bootstrap/litellm-rotate-key/scripts/rotate.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -LITELLM_MASTER_KEY=$(aws secretsmanager get-secret-value \ - --region $AWS_SECRET_REGION \ - --secret-id $AWS_SECRETS_MANAGER_ID | jq -r '.SecretString' | jq -r '.LITELLM_MASTER_KEY') - -kubectl create secret generic litellm -n $K8S_NAMESPACE -o yaml --dry-run=client \ - --from-literal=token=$LITELLM_MASTER_KEY | kubectl apply -f - \ No newline at end of file From 49ddb34443b8646b46b84dd1f55df1b013c0cc3c Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Thu, 22 Jan 2026 10:29:00 -0800 Subject: [PATCH 09/44] fix: change naming conventions + use path-based notation for shorter names --- modules/k8s/bootstrap/cert-manager/main.tf | 7 ++++--- modules/k8s/bootstrap/coder-server/main.tf | 14 ++++++-------- modules/k8s/bootstrap/ebs-controller/main.tf | 3 ++- modules/k8s/bootstrap/external-dns/main.tf | 10 +++++++--- modules/k8s/bootstrap/external-secrets/main.tf | 7 ++++--- modules/k8s/bootstrap/karpenter/main.tf | 10 ++++++---- modules/k8s/bootstrap/lb-controller/main.tf | 7 ++++--- 7 files changed, 33 insertions(+), 25 deletions(-) diff --git a/modules/k8s/bootstrap/cert-manager/main.tf b/modules/k8s/bootstrap/cert-manager/main.tf index 1966650..be5ee01 100644 --- a/modules/k8s/bootstrap/cert-manager/main.tf +++ b/modules/k8s/bootstrap/cert-manager/main.tf @@ -95,14 +95,14 @@ data "aws_caller_identity" "this" {} locals { region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "${var.cluster_name}-crt-mgr-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "${var.cluster_name}-crt-mgr-${data.aws_region.this.region}" : var.role_name + policy_name = var.policy_name == "" ? "crt-mgr" : var.policy_name + role_name = var.role_name == "" ? "crt-mgr" : var.role_name } module "policy" { source = "../../../security/policy" name = local.policy_name - path = "/" + path = "/${var.cluster_name}/${data.aws_region.this.region}/" description = "CertManager for Route53 Policy" policy_json = data.aws_iam_policy_document.route53.json } @@ -110,6 +110,7 @@ module "policy" { module "oidc-role" { source = "../../../security/role/access-entry" name = local.role_name + path = "/${var.cluster_name}/${data.aws_region.this.region}/" cluster_name = var.cluster_name policy_arns = { "CertManagerRoute53" = module.policy.policy_arn diff --git a/modules/k8s/bootstrap/coder-server/main.tf b/modules/k8s/bootstrap/coder-server/main.tf index b1e990c..ac84504 100644 --- a/modules/k8s/bootstrap/coder-server/main.tf +++ b/modules/k8s/bootstrap/coder-server/main.tf @@ -10,9 +10,6 @@ terraform { kubernetes = { source = "hashicorp/kubernetes" } - acme = { - source = "vancluever/acme" - } tls = { source = "hashicorp/tls" } @@ -612,15 +609,15 @@ locals { locals { region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "${var.cluster_name}-coder-srv-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "${var.cluster_name}-coder-srv-${data.aws_region.this.region}" : var.role_name + policy_name = var.policy_name == "" ? "coder-srv" : var.policy_name + role_name = var.role_name == "" ? "coder-srv" : var.role_name } module "provisioner-policy" { count = var.coder_builtin_provisioner_count == 0 ? 0 : 1 source = "../../../security/policy" name = local.policy_name - path = "/" + path = "/${var.cluster_name}/${data.aws_region.this.region}/" description = "Coder Terraform External Provisioner Policy" policy_json = data.aws_iam_policy_document.provisioner-policy.json } @@ -629,6 +626,7 @@ module "provisioner-oidc-role" { count = var.coder_builtin_provisioner_count == 0 ? 0 : 1 source = "../../../security/role/access-entry" name = local.role_name + path = "/${var.cluster_name}/${data.aws_region.this.region}/" cluster_name = var.cluster_name policy_arns = { "AmazonEC2ReadOnlyAccess" = "arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess" @@ -733,8 +731,8 @@ locals { secret_refresh_interval = "1812h0m0s" # 75.5 days tls_secret_key = "tls.key" tls_secret_crt = "tls.crt" - tls_remote_key = "tls5.key" - tls_remote_crt = "tls5.crt" + tls_remote_key = "tls-${local.common_name}.key" + tls_remote_crt = "tls-${local.common_name}.crt" } resource "kubernetes_manifest" "pull" { diff --git a/modules/k8s/bootstrap/ebs-controller/main.tf b/modules/k8s/bootstrap/ebs-controller/main.tf index 0191fc9..ec23c7f 100644 --- a/modules/k8s/bootstrap/ebs-controller/main.tf +++ b/modules/k8s/bootstrap/ebs-controller/main.tf @@ -61,13 +61,14 @@ variable "replace" { data "aws_region" "this" {} locals { - role_name = var.role_name == "" ? "${var.cluster_name}-ebs-ctrl-${data.aws_region.this.region}" : var.role_name + role_name = var.role_name == "" ? "ebs-ctrl" : var.role_name } module "oidc-role" { source = "../../../security/role/access-entry" name = local.role_name cluster_name = var.cluster_name + path = "/${var.cluster_name}/${data.aws_region.this.region}/" policy_arns = { "AmazonEBSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy" } diff --git a/modules/k8s/bootstrap/external-dns/main.tf b/modules/k8s/bootstrap/external-dns/main.tf index c518037..b6fe0fd 100644 --- a/modules/k8s/bootstrap/external-dns/main.tf +++ b/modules/k8s/bootstrap/external-dns/main.tf @@ -72,14 +72,14 @@ data "aws_caller_identity" "this" {} locals { region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "${var.cluster_name}-ext-dns-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "${var.cluster_name}-ext-dns-${data.aws_region.this.region}" : var.role_name + policy_name = var.policy_name == "" ? "ext-dns" : var.policy_name + role_name = var.role_name == "" ? "ext-dns" : var.role_name } module "policy" { source = "../../../security/policy" name = local.policy_name - path = "/" + path = "/${var.cluster_name}/${data.aws_region.this.region}/" description = "External DNS Policy." policy_json = data.aws_iam_policy_document.this.json } @@ -87,6 +87,7 @@ module "policy" { module "oidc-role" { source = "../../../security/role/access-entry" name = local.role_name + path = "/${var.cluster_name}/${data.aws_region.this.region}/" cluster_name = var.cluster_name policy_arns = { "ExternalDNS" = module.policy.policy_arn @@ -149,5 +150,8 @@ resource "helm_release" "chart" { } } nodeSelector = var.node_selector + # extraArgs = [ + # "--aws-prefer-cname" + # ] })] } \ No newline at end of file diff --git a/modules/k8s/bootstrap/external-secrets/main.tf b/modules/k8s/bootstrap/external-secrets/main.tf index 75a62f4..82db376 100644 --- a/modules/k8s/bootstrap/external-secrets/main.tf +++ b/modules/k8s/bootstrap/external-secrets/main.tf @@ -72,14 +72,14 @@ data "aws_caller_identity" "this" {} locals { region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "${var.cluster_name}-ext-sec-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "${var.cluster_name}-ext-sec-${data.aws_region.this.region}" : var.role_name + policy_name = var.policy_name == "" ? "ext-sec" : var.policy_name + role_name = var.role_name == "" ? "ext-sec" : var.role_name } module "policy" { source = "../../../security/policy" name = local.policy_name - path = "/" + path = "/${var.cluster_name}/${data.aws_region.this.region}/" description = "External Secrets Policy." policy_json = data.aws_iam_policy_document.this.json } @@ -87,6 +87,7 @@ module "policy" { module "oidc-role" { source = "../../../security/role/access-entry" name = local.role_name + path = "/${var.cluster_name}/${data.aws_region.this.region}/" cluster_name = var.cluster_name policy_arns = { "ExternalSecrets" = module.policy.policy_arn diff --git a/modules/k8s/bootstrap/karpenter/main.tf b/modules/k8s/bootstrap/karpenter/main.tf index 776c89e..4e977e3 100644 --- a/modules/k8s/bootstrap/karpenter/main.tf +++ b/modules/k8s/bootstrap/karpenter/main.tf @@ -154,9 +154,9 @@ variable "node_selector" { data "aws_region" "this" {} locals { - std_karpenter_format = "${var.cluster_name}-kptr-${data.aws_region.this.region}" + std_karpenter_format = "kptr" karpenter_queue_name = var.karpenter_queue_name == "" ? "${var.cluster_name}-kptr" : var.karpenter_queue_name - karpenter_queue_rule_name = var.karpenter_queue_rule_name == "" ? "${var.cluster_name}-kptr" : var.karpenter_queue_rule_name + # karpenter_queue_rule_name = var.karpenter_queue_rule_name == "" ? "${var.cluster_name}-kptr" : var.karpenter_queue_rule_name karpenter_controller_role_name = var.karpenter_controller_role_name == "" ? "${local.std_karpenter_format}-ctrl" : var.karpenter_controller_role_name karpenter_controller_policy_name = var.karpenter_controller_policy_name == "" ? local.std_karpenter_format : var.karpenter_controller_policy_name karpenter_node_role_name = var.karpenter_node_role_name == "" ? "${local.std_karpenter_format}-node" : var.karpenter_node_role_name @@ -172,7 +172,7 @@ data "aws_iam_policy_document" "sts" { resource "aws_iam_policy" "sts" { name_prefix = "${var.cluster_name}-sts-" - path = "/" + path = "/${var.cluster_name}/${data.aws_region.this.region}/${local.std_karpenter_format}/" description = "Assume Role Policy" policy = data.aws_iam_policy_document.sts.json } @@ -209,12 +209,13 @@ module "karpenter" { cluster_name = var.cluster_name queue_name = local.karpenter_queue_name - rule_name_prefix = "${local.karpenter_queue_rule_name}-" + rule_name_prefix = "" # Karpenter Controller Role create_iam_role = true iam_role_name = local.karpenter_controller_role_name iam_role_use_name_prefix = var.iam_role_use_name_prefix + iam_role_path = "/${var.cluster_name}/${data.aws_region.this.region}/" iam_role_policies = var.karpenter_controller_role_policies iam_role_source_assume_policy_documents = [ data.aws_iam_policy_document.kptr_ctrl_assume_role_policy.json, @@ -233,6 +234,7 @@ module "karpenter" { create_node_iam_role = true node_iam_role_name = local.karpenter_node_role_name node_iam_role_use_name_prefix = var.node_iam_role_use_name_prefix + node_iam_role_path = "/${var.cluster_name}/${data.aws_region.this.region}/" node_iam_role_additional_policies = merge({ AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" STSAssumeRole = aws_iam_policy.sts.arn diff --git a/modules/k8s/bootstrap/lb-controller/main.tf b/modules/k8s/bootstrap/lb-controller/main.tf index d928c48..0e97ba3 100644 --- a/modules/k8s/bootstrap/lb-controller/main.tf +++ b/modules/k8s/bootstrap/lb-controller/main.tf @@ -102,14 +102,14 @@ data "aws_caller_identity" "this" {} locals { region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "${var.cluster_name}-lb-ctrl-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "${var.cluster_name}-lb-ctrl-${data.aws_region.this.region}" : var.role_name + policy_name = var.policy_name == "" ? "lb-ctrl" : var.policy_name + role_name = var.role_name == "" ? "lb-ctrl" : var.role_name } module "policy" { source = "../../../security/policy" name = local.policy_name - path = "/" + path = "/${var.cluster_name}/${data.aws_region.this.region}/" description = "AWS Load Balancer Controller Policy" policy_json = data.aws_iam_policy_document.this.json } @@ -117,6 +117,7 @@ module "policy" { module "oidc-role" { source = "../../../security/role/access-entry" name = local.role_name + path = "/${var.cluster_name}/${data.aws_region.this.region}/" cluster_name = var.cluster_name policy_arns = { "AmazonEKSLoadBalancingPolicy" = "arn:aws:iam::aws:policy/AmazonEKSLoadBalancingPolicy", From 4282cf113844b0b6f29346f00ddf19a15126cdcc Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Thu, 22 Jan 2026 10:30:53 -0800 Subject: [PATCH 10/44] fix: match new naming conventions on top-level resources --- infra/1-click/0-infra/0-vpc.tf | 93 +++++++++---------------- infra/1-click/0-infra/1-db.tf | 12 ++-- infra/1-click/0-infra/2-s3.tf | 12 +--- infra/1-click/0-infra/3-eks.tf | 6 +- infra/1-click/0-infra/4-bootstrap.tf | 4 -- infra/1-click/0-infra/main.tf | 6 ++ infra/1-click/1-setup/5-manifests.tf | 6 +- infra/1-click/1-setup/6-coder-server.tf | 12 ++-- 8 files changed, 54 insertions(+), 97 deletions(-) diff --git a/infra/1-click/0-infra/0-vpc.tf b/infra/1-click/0-infra/0-vpc.tf index 9fbf7a3..2ab42d7 100644 --- a/infra/1-click/0-infra/0-vpc.tf +++ b/infra/1-click/0-infra/0-vpc.tf @@ -2,10 +2,15 @@ # Networking Infrastructure ## +variable "azs" { + type = list(string) + default = ["a", "b", "c"] +} + locals { # Subnet Discovery - https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/subnet_discovery/#subnet-filtering tags_lb_subnet_discovery = { - "kubernetes.io/cluster/${var.name}" = "owned" + "kubernetes.io/cluster/${var.name}-${local.normalized_domain_name}" = "owned" } # Internet-Facing LB - https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/subnet_discovery/#subnet-role-tag tags_public_lb = { @@ -15,24 +20,30 @@ locals { local.tags_lb_subnet_discovery, local.tags_public_lb ) + # Use variable to configure AZ just in case 1 AZ isn't accessible + # https://repost.aws/questions/QUgdQev4KETKG_Bwev9tMtRQ/is-it-possible-to-enable-3rd-availability-zone-in-us-west-1#AN9eAH55FwTC-NSSp-FjIfTQ + availability_zones = [ for az in var.azs : "${var.region}${az}" ] + public_subnet_cidrs = ["10.0.8.0/21", "10.0.4.0/22", "10.0.0.0/22"] + general_subnet_cidrs = ["10.0.64.0/20", "10.0.80.0/20", "10.0.96.0/20"] + system_subnet_cidrs = ["10.0.16.0/20", "10.0.32.0/20", "10.0.48.0/20"] } module "vpc" { source = "terraform-aws-modules/vpc/aws" version = "~> 6.6.0" - name = var.name + name = "${var.name}-${local.normalized_domain_name}" enable_nat_gateway = false enable_dns_hostnames = true cidr = "10.0.0.0/16" - azs = ["${var.region}a", "${var.region}b", "${var.region}c"] + azs = local.availability_zones ## # Handle public subnets via the VPC module. ## - public_subnets = ["10.0.8.0/21", "10.0.4.0/22", "10.0.0.0/22"] + public_subnets = [ for index, _ in var.azs : local.public_subnet_cidrs[index] ] public_subnet_tags = local.tags_public_subnet tags = local.tags_global @@ -41,7 +52,7 @@ module "vpc" { module "nat-instance" { source = "RaJiska/fck-nat/aws" - name = var.name + name = "${var.name}-${local.normalized_domain_name}" vpc_id = module.vpc.vpc_id subnet_id = module.vpc.public_subnets[0] @@ -63,7 +74,7 @@ module "nat-instance" { locals { # Karpenter Subnet Discovery - https://karpenter.sh/v1.0/concepts/nodeclasses/#specsubnetselectorterms tags_kptr_subnet_discovery = { - "karpenter.sh/discovery" = var.name + "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" } tags_private_subnet = merge( local.tags_lb_subnet_discovery, @@ -73,35 +84,16 @@ locals { # 4096 - 5 (reserved by AWS) IPs each -module "general-subnet-az1" { - source = "../../../modules/network/subnet/private" - name = var.name - vpc_id = module.vpc.vpc_id - eni_id = module.nat-instance.eni_id - cidr_block = "10.0.64.0/20" - availability_zone = "${var.region}a" - subnet_tags = local.tags_private_subnet - tags = local.tags_global -} +module "general-subnet" { -module "general-subnet-az2" { - source = "../../../modules/network/subnet/private" - name = var.name - vpc_id = module.vpc.vpc_id - eni_id = module.nat-instance.eni_id - cidr_block = "10.0.80.0/20" - availability_zone = "${var.region}b" - subnet_tags = local.tags_private_subnet - tags = local.tags_global -} + count = length(var.azs) -module "general-subnet-az3" { source = "../../../modules/network/subnet/private" - name = var.name + name = "${var.name}-${local.normalized_domain_name}" vpc_id = module.vpc.vpc_id eni_id = module.nat-instance.eni_id - cidr_block = "10.0.96.0/20" - availability_zone = "${var.region}c" + cidr_block = local.general_subnet_cidrs[count.index] + availability_zone = local.availability_zones[count.index] subnet_tags = local.tags_private_subnet tags = local.tags_global } @@ -110,47 +102,24 @@ module "general-subnet-az3" { # Kubernetes System Subnets ## -module "system-subnet-az1" { - source = "../../../modules/network/subnet/private" - name = var.name - vpc_id = module.vpc.vpc_id - eni_id = module.nat-instance.eni_id - cidr_block = "10.0.16.0/20" - availability_zone = "${var.region}a" - subnet_tags = local.tags_private_subnet - tags = local.tags_global -} +module "system-subnet" { -module "system-subnet-az2" { - source = "../../../modules/network/subnet/private" - name = var.name - vpc_id = module.vpc.vpc_id - eni_id = module.nat-instance.eni_id - cidr_block = "10.0.32.0/20" - availability_zone = "${var.region}b" - subnet_tags = local.tags_private_subnet - tags = local.tags_global -} + count = length(var.azs) -module "system-subnet-az3" { source = "../../../modules/network/subnet/private" - name = var.name + name = "${var.name}-${local.normalized_domain_name}" vpc_id = module.vpc.vpc_id eni_id = module.nat-instance.eni_id - cidr_block = "10.0.48.0/20" - availability_zone = "${var.region}c" + cidr_block = local.system_subnet_cidrs[count.index] + availability_zone = local.availability_zones[count.index] subnet_tags = local.tags_private_subnet tags = local.tags_global } locals { - private_subnet_ids = [ - module.general-subnet-az1.subnet_id, - module.general-subnet-az2.subnet_id, - module.general-subnet-az3.subnet_id, - module.system-subnet-az1.subnet_id, - module.system-subnet-az2.subnet_id, - module.system-subnet-az3.subnet_id - ] + private_subnet_ids = concat( + module.general-subnet.*.subnet_id, + module.system-subnet.*.subnet_id + ) public_subnet_ids = concat([], module.vpc.public_subnets) } \ No newline at end of file diff --git a/infra/1-click/0-infra/1-db.tf b/infra/1-click/0-infra/1-db.tf index 917ef7f..479f2d3 100644 --- a/infra/1-click/0-infra/1-db.tf +++ b/infra/1-click/0-infra/1-db.tf @@ -43,7 +43,7 @@ variable "grafana_password" { resource "aws_security_group" "postgres" { vpc_id = module.vpc.vpc_id - name = "postgres" + name = "${var.name}-${local.normalized_domain_name}-pgsql" description = "security group for postgres all egress traffic" tags = { Name = "PostgreSQL" @@ -65,16 +65,16 @@ resource "aws_vpc_security_group_egress_rule" "postgres" { } resource "aws_db_subnet_group" "coder" { - name = "coder" + name = "${var.name}-${local.normalized_domain_name}-coder" subnet_ids = local.private_subnet_ids tags = { - Name = "coder" + Name = "${var.name}-${local.normalized_domain_name}-coder" } } resource "aws_db_instance" "coder" { - identifier = "coder" + identifier = "${var.name}-${local.normalized_domain_name}-coder" instance_class = "db.t4g.large" allocated_storage = 50 engine = "postgres" @@ -100,7 +100,7 @@ resource "aws_db_instance" "coder" { } resource "aws_db_instance" "litellm" { - identifier = "litellm" + identifier = "${var.name}-${local.normalized_domain_name}-litellm" instance_class = "db.t4g.medium" allocated_storage = 50 engine = "postgres" @@ -125,7 +125,7 @@ resource "aws_db_instance" "litellm" { } resource "aws_db_instance" "grafana" { - identifier = "grafana" + identifier = "${var.name}-${local.normalized_domain_name}-grafana" instance_class = "db.t4g.large" allocated_storage = 50 engine = "postgres" diff --git a/infra/1-click/0-infra/2-s3.tf b/infra/1-click/0-infra/2-s3.tf index e385a1c..f0aa222 100644 --- a/infra/1-click/0-infra/2-s3.tf +++ b/infra/1-click/0-infra/2-s3.tf @@ -16,18 +16,8 @@ variable "loki_s3_bucket_tags" { default = {} } -resource "random_string" "bucket" { - keepers = { - static = "true" - } - length = 16 - special = false - upper = false - lower = true -} - resource "aws_s3_bucket" "loki" { - bucket = var.loki_s3_bucket_name == "" ? "granafa-logs-${random_string.bucket.result}" : var.loki_s3_bucket_name + bucket = var.loki_s3_bucket_name == "" ? "${var.name}-${local.normalized_domain_name}-granafa-logs" : var.loki_s3_bucket_name tags = var.loki_s3_bucket_tags } diff --git a/infra/1-click/0-infra/3-eks.tf b/infra/1-click/0-infra/3-eks.tf index a65d6c1..a0ea3da 100644 --- a/infra/1-click/0-infra/3-eks.tf +++ b/infra/1-click/0-infra/3-eks.tf @@ -11,7 +11,7 @@ data "aws_iam_policy_document" "sts" { } resource "aws_iam_policy" "sts" { - name_prefix = "${var.name}-sts-" + name_prefix = "${var.name}-${local.normalized_domain_name}-sts-" path = "/" description = "Assume Role Policy" policy = data.aws_iam_policy_document.sts.json @@ -21,7 +21,7 @@ resource "aws_iam_policy" "sts" { locals { # Karpenter Security Group Discovery - https://karpenter.sh/v1.0/concepts/nodeclasses/#specsecuritygroupselectorterms tags_kptr_sg_discovery = { - "karpenter.sh/discovery" = var.name + "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" } labels_system_node = { "scheduling.coder.com/pool" = "system" @@ -41,7 +41,7 @@ module "eks" { local.private_subnet_ids )) - name = var.name + name = "${var.name}-${local.normalized_domain_name}" kubernetes_version = "1.34" endpoint_public_access = true diff --git a/infra/1-click/0-infra/4-bootstrap.tf b/infra/1-click/0-infra/4-bootstrap.tf index a5749db..a96997c 100644 --- a/infra/1-click/0-infra/4-bootstrap.tf +++ b/infra/1-click/0-infra/4-bootstrap.tf @@ -1,7 +1,3 @@ -variable "domain_name" { - type = string -} - ## # System-Level Addons ## diff --git a/infra/1-click/0-infra/main.tf b/infra/1-click/0-infra/main.tf index 94b5e95..34d569d 100644 --- a/infra/1-click/0-infra/main.tf +++ b/infra/1-click/0-infra/main.tf @@ -37,6 +37,11 @@ variable "profile" { default = "default" } +variable "domain_name" { + description = "Your Coder domain name (i.e. coder-example.com)" + type = string +} + data "aws_eks_cluster_auth" "coder" { name = module.eks.cluster_name } @@ -61,5 +66,6 @@ provider "kubernetes" { } locals { + normalized_domain_name = split(".", var.domain_name)[0] tags_global = {} } \ No newline at end of file diff --git a/infra/1-click/1-setup/5-manifests.tf b/infra/1-click/1-setup/5-manifests.tf index 1a0b0fb..920094a 100644 --- a/infra/1-click/1-setup/5-manifests.tf +++ b/infra/1-click/1-setup/5-manifests.tf @@ -63,12 +63,12 @@ resource "kubernetes_manifest" "nodeclass" { }] subnetSelectorTerms = [{ tags = { - "karpenter.sh/discovery" = var.name + "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" } }] securityGroupSelectorTerms = [{ tags = { - "karpenter.sh/discovery" = var.name + "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" } }] blockDeviceMappings = each.value.block_device_mappings @@ -282,7 +282,7 @@ resource "kubernetes_manifest" "secret-store" { provider = { aws = { service = "SecretsManager" - region = "us-east-2" + region = var.region auth = { jwt = { serviceAccountRef = { diff --git a/infra/1-click/1-setup/6-coder-server.tf b/infra/1-click/1-setup/6-coder-server.tf index a0fe762..7c51238 100644 --- a/infra/1-click/1-setup/6-coder-server.tf +++ b/infra/1-click/1-setup/6-coder-server.tf @@ -6,10 +6,6 @@ # - external-secrets ## -variable "domain_name" { - type = string -} - variable "coder_username" { description = "Coder DB's username." type = string @@ -51,7 +47,7 @@ module "coder-server" { source = "../../../modules/k8s/bootstrap/coder-server" - cluster_name = var.name + cluster_name = "${var.name}-${local.normalized_domain_name}" cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn namespace = "coder" @@ -127,7 +123,7 @@ module "coder-server" { # Wait for DNS propagation. May require multiple redeploys resource "time_sleep" "wait_for_dns" { - create_duration = "300s" + create_duration = "120s" depends_on = [ module.coder-server ] } @@ -138,7 +134,7 @@ data "external" "first-user" { program = ["bash", "${path.module}/scripts/first-user.sh"] query = { - access_url = "https://${var.domain_name}" + domain = var.domain_name admin_email = var.coder_admin_email admin_username = var.coder_admin_username admin_password = var.coder_admin_password @@ -159,7 +155,7 @@ data "external" "add-license" { program = ["bash", "${path.module}/scripts/add-license.sh"] query = { - access_url = "https://${var.domain_name}" + domain = var.domain_name license_key = var.coder_license session_token = data.external.first-user.result.session_token } From 587de1cc8922777afc1c9439dca38cb253f3c5bd Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Thu, 22 Jan 2026 10:31:47 -0800 Subject: [PATCH 11/44] fix: Add --resolve to quickly resolve DNS rather than depending on local DNS cache --- infra/1-click/1-setup/scripts/add-license.sh | 15 ++++++++++----- infra/1-click/1-setup/scripts/first-user.sh | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/infra/1-click/1-setup/scripts/add-license.sh b/infra/1-click/1-setup/scripts/add-license.sh index b730201..8de8559 100644 --- a/infra/1-click/1-setup/scripts/add-license.sh +++ b/infra/1-click/1-setup/scripts/add-license.sh @@ -1,22 +1,27 @@ #!/usr/bin/env bash -eval "$(jq -r '@sh "CODER_URL=\(.access_url) CODER_LICENSE=\(.license_key) CODER_SESSION_TOKEN=\(.session_token)"')" +eval "$(jq -r '@sh "CODER_DOMAIN=\(.domain) CODER_LICENSE=\(.license_key) CODER_SESSION_TOKEN=\(.session_token)"')" # URL might not be available still. Retry request until available, or fail if max attempts reached. IDX=0 -until curl -s -o /dev/null -kL "$CODER_URL"; do +IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) +while [[ -z "$IP_ADDR" ]]; do + ((IDX++)) if (( IDX >= 6 )); then - >&2 echo "Error: Unable to run 'curl -s -o /dev/null -kL \"$CODER_URL\"'." + >&2 echo "Error: Failed to run \"dig +short $CODER_DOMAIN | head -n1\". Unable to discover IP for \"$CODER_DOMAIN\"." exit 1; fi sleep 10 - ((IDX++)) + IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) done +RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$IP_ADDR" + +CODER_URL=https://$CODER_DOMAIN -RESPONSE=$(curl -ks -X POST "$CODER_URL/api/v2/licenses" \ +RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/licenses" \ -H 'Content-Type: application/json' \ -H 'Accept: application/json' \ -H "Coder-Session-Token: $CODER_SESSION_TOKEN" \ diff --git a/infra/1-click/1-setup/scripts/first-user.sh b/infra/1-click/1-setup/scripts/first-user.sh index 7da1824..9844e9f 100755 --- a/infra/1-click/1-setup/scripts/first-user.sh +++ b/infra/1-click/1-setup/scripts/first-user.sh @@ -1,21 +1,27 @@ #!/usr/bin/env bash -eval "$(jq -r '@sh "CODER_URL=\(.access_url) ADMIN_EMAIL=\(.admin_email) ADMIN_USERNAME=\(.admin_username) ADMIN_PASSWORD=\(.admin_password)"')" +eval "$(jq -r '@sh "CODER_DOMAIN=\(.domain) ADMIN_EMAIL=\(.admin_email) ADMIN_USERNAME=\(.admin_username) ADMIN_PASSWORD=\(.admin_password)"')" # URL might not be available still. Retry request until available, or fail if max attempts reached. IDX=0 -until curl -s -o /dev/null -kL "$CODER_URL"; do +IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) +while [[ -z "$IP_ADDR" ]]; do + ((IDX++)) if (( IDX >= 6 )); then - >&2 echo "Error: Unable to run 'curl -s -o /dev/null -kL \"$CODER_URL\"'." + >&2 echo "Error: Failed to run \"dig +short $CODER_DOMAIN | head -n1\". Unable to discover IP for \"$CODER_DOMAIN\"." exit 1; fi sleep 10 - ((IDX++)) + IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) done -RESPONSE=$(curl -ks -X POST "$CODER_URL/api/v2/users/first" \ +RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$IP_ADDR" + +CODER_URL=https://$CODER_DOMAIN + +RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/users/first" \ -H "Content-Type: application/json" \ -d "{ \"email\": \"$ADMIN_EMAIL\", @@ -24,7 +30,7 @@ RESPONSE=$(curl -ks -X POST "$CODER_URL/api/v2/users/first" \ \"trial\": false }") -LOGIN_RESPONSE=$(curl -ks -X POST "$CODER_URL/api/v2/users/login" \ +LOGIN_RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/users/login" \ -H "Content-Type: application/json" \ -d "{\"email\":\"$ADMIN_EMAIL\",\"password\":\"$ADMIN_PASSWORD\"}" | jq -r '.session_token') From 7831a9dc18492537a435baf61d3a5d9d462dfa98 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Thu, 22 Jan 2026 10:32:22 -0800 Subject: [PATCH 12/44] fix: adds new name formatting --- infra/1-click/1-setup/main.tf | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/infra/1-click/1-setup/main.tf b/infra/1-click/1-setup/main.tf index fb337a1..21cfb1e 100644 --- a/infra/1-click/1-setup/main.tf +++ b/infra/1-click/1-setup/main.tf @@ -26,33 +26,33 @@ terraform { data "aws_vpc" "this" { tags = { - "Name" = var.name + "Name" = "${var.name}-${local.normalized_domain_name}" } } data "aws_db_instance" "coder" { - db_instance_identifier = var.name + db_instance_identifier = "${var.name}-${local.normalized_domain_name}-coder" } data "aws_db_instance" "litellm" { - db_instance_identifier = "litellm" + db_instance_identifier = "${var.name}-${local.normalized_domain_name}-litellm" } data "aws_db_instance" "grafana" { - db_instance_identifier = "grafana" + db_instance_identifier = "${var.name}-${local.normalized_domain_name}-grafana" } data "aws_security_group" "coder" { - name = var.name + name = "${var.name}-${local.normalized_domain_name}-pgsql" vpc_id = data.aws_vpc.this.id } data "aws_eks_cluster" "coder" { - name = var.name + name = "${var.name}-${local.normalized_domain_name}" } data "aws_eks_cluster_auth" "coder" { - name = var.name + name = "${var.name}-${local.normalized_domain_name}" } data "aws_iam_openid_connect_provider" "coder" { @@ -80,6 +80,10 @@ variable "profile" { default = "default" } +variable "domain_name" { + type = string +} + provider "aws" { region = var.region profile = var.profile @@ -98,3 +102,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.coder.certificate_authority[0].data) token = data.aws_eks_cluster_auth.coder.token } + +locals { + normalized_domain_name = split(".", var.domain_name)[0] +} \ No newline at end of file From 73f8c4b4b9ff58cd8b765487ceaa2df7d64bc1fc Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Thu, 22 Jan 2026 10:33:02 -0800 Subject: [PATCH 13/44] fix: exposes region and azs as variables to pass in --- infra/1-click/1-plan-n-deploy.sh | 4 ++++ infra/1-click/2-clean.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/infra/1-click/1-plan-n-deploy.sh b/infra/1-click/1-plan-n-deploy.sh index dd9ec8b..0b8da29 100755 --- a/infra/1-click/1-plan-n-deploy.sh +++ b/infra/1-click/1-plan-n-deploy.sh @@ -3,6 +3,7 @@ set -ae -o pipefail AWS_PROFILE="${CODER_AWS_PROFILE:-default}" +AWS_REGION="${CODER_AWS_REGION:-us-east-2}" DOMAIN_NAME="${CODER_DOMAIN_NAME:-}" LICENSE="${CODER_LICENSE:-}" @@ -15,6 +16,8 @@ echo "Change directory into '0-infra'." cd 0-infra terraform plan -out=tf.plan \ -var profile=$AWS_PROFILE \ + -var region=$AWS_REGION \ + -var azs='["a","c"]' \ -var domain_name=$DOMAIN_NAME terraform apply tf.plan cd ../ @@ -23,6 +26,7 @@ echo "Change directory into '1-setup'." cd 1-setup terraform plan -out=tf.plan \ -var profile=$AWS_PROFILE \ + -var region=$AWS_REGION \ -var domain_name=$DOMAIN_NAME \ -var coder_license=$LICENSE terraform apply tf.plan diff --git a/infra/1-click/2-clean.sh b/infra/1-click/2-clean.sh index 8cd72f2..847da9d 100755 --- a/infra/1-click/2-clean.sh +++ b/infra/1-click/2-clean.sh @@ -7,6 +7,7 @@ set -ae -o pipefail ## AWS_PROFILE="${CODER_AWS_PROFILE:-default}" +AWS_REGION="${CODER_AWS_REGION:-us-east-2}" DOMAIN_NAME="${CODER_DOMAIN_NAME:-}" LICENSE="${CODER_LICENSE:-}" @@ -14,6 +15,7 @@ echo "Change directory into '1-setup'." cd 1-setup terraform plan -destroy -out=tf.plan \ -var profile=$AWS_PROFILE \ + -var region=$AWS_REGION \ -var domain_name=$DOMAIN_NAME \ -var coder_license=$LICENSE terraform apply tf.plan @@ -23,6 +25,8 @@ echo "Change directory into '0-infra'." cd 0-infra terraform plan -destroy -out=tf.plan \ -var profile=$AWS_PROFILE \ + -var region=$AWS_REGION \ + -var azs='["a","c"]' \ -var domain_name=$DOMAIN_NAME terraform apply tf.plan cd ../ \ No newline at end of file From 49588b70f4ab2c9c37ac05bc71fd79bd0e8413e3 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 26 Jan 2026 11:33:29 -0800 Subject: [PATCH 14/44] fix: made new changes --- infra/1-click/0-infra/3-eks.tf | 9 +- infra/1-click/0-infra/4-bootstrap.tf | 11 +- infra/1-click/1-plan-n-deploy.sh | 28 +- infra/1-click/1-setup/5-manifests.tf | 34 +- infra/1-click/1-setup/6-coder-server.tf | 31 +- infra/1-click/1-setup/main.tf | 2 + infra/1-click/1-setup/scripts/add-license.sh | 18 +- infra/1-click/1-setup/scripts/first-user.sh | 25 +- infra/1-click/2-clean.sh | 27 +- infra/1-click/2-coder/7-coder.tf | 91 ----- infra/1-click/2-coder/7-orgs.tf | 108 ++++++ infra/1-click/2-coder/8-templates.tf | 49 +++ infra/1-click/2-coder/fetch.sh | 24 -- infra/1-click/2-coder/main.tf | 17 +- infra/1-click/2-coder/scripts/login.sh | 35 ++ .../2-coder/templates/kubernetes/README.md | 38 ++ .../2-coder/templates/kubernetes/main.tf | 344 ++++++++++++++++++ infra/1-click/README.md | 6 + .../k8s/bootstrap/coder-provisioner/main.tf | 33 +- modules/k8s/bootstrap/coder-server/main.tf | 12 +- modules/k8s/bootstrap/external-dns/main.tf | 12 +- modules/k8s/bootstrap/karpenter/main.tf | 17 +- 22 files changed, 778 insertions(+), 193 deletions(-) delete mode 100644 infra/1-click/2-coder/7-coder.tf create mode 100644 infra/1-click/2-coder/7-orgs.tf create mode 100644 infra/1-click/2-coder/8-templates.tf delete mode 100755 infra/1-click/2-coder/fetch.sh create mode 100755 infra/1-click/2-coder/scripts/login.sh create mode 100644 infra/1-click/2-coder/templates/kubernetes/README.md create mode 100644 infra/1-click/2-coder/templates/kubernetes/main.tf diff --git a/infra/1-click/0-infra/3-eks.tf b/infra/1-click/0-infra/3-eks.tf index a0ea3da..7f02f3f 100644 --- a/infra/1-click/0-infra/3-eks.tf +++ b/infra/1-click/0-infra/3-eks.tf @@ -68,13 +68,18 @@ module "eks" { system = { min_size = 0 max_size = 10 - desired_size = 3 # Ignored after creation. Override from AWS Console as needed. + desired_size = 2 # Ignored after creation. Override from AWS Console as needed. # K8s Labels - https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group#labels-1 labels = local.labels_system_node + # taints = { + # "CriticalAddonsOnly" + # } - instance_types = ["t3.xlarge"] + instance_types = ["t3.large"] capacity_type = "ON_DEMAND" + volume_size = 50 + iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" STSAssumeRole = aws_iam_policy.sts.arn diff --git a/infra/1-click/0-infra/4-bootstrap.tf b/infra/1-click/0-infra/4-bootstrap.tf index a96997c..8baca5b 100644 --- a/infra/1-click/0-infra/4-bootstrap.tf +++ b/infra/1-click/0-infra/4-bootstrap.tf @@ -15,6 +15,7 @@ module "karpenter" { iam_role_use_name_prefix = true node_iam_role_use_name_prefix = true + replicas = 2 karpenter_controller_role_policies = { "AmazonEFSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" } @@ -22,7 +23,7 @@ module "karpenter" { module "metrics-server" { - depends_on = [ module.karpenter ] + # depends_on = [ module.karpenter ] source = "../../../modules/k8s/bootstrap/metrics-server" @@ -33,7 +34,7 @@ module "metrics-server" { module "cert-manager" { - depends_on = [ module.karpenter ] + # depends_on = [ module.karpenter ] source = "../../../modules/k8s/bootstrap/cert-manager" cluster_name = module.eks.cluster_name @@ -48,7 +49,7 @@ module "cert-manager" { module "lb-controller" { - depends_on = [ module.cert-manager, module.karpenter ] + depends_on = [ module.cert-manager ] source = "../../../modules/k8s/bootstrap/lb-controller" cluster_name = module.eks.cluster_name @@ -67,7 +68,7 @@ module "lb-controller" { module "ebs-controller" { - depends_on = [ module.cert-manager, module.karpenter ] + # depends_on = [ module.cert-manager ] source = "../../../modules/k8s/bootstrap/ebs-controller" cluster_name = module.eks.cluster_name @@ -81,7 +82,7 @@ module "ebs-controller" { module "external-dns" { - depends_on = [ module.cert-manager, module.karpenter ] + depends_on = [ module.cert-manager ] source = "../../../modules/k8s/bootstrap/external-dns" cluster_name = module.eks.cluster_name diff --git a/infra/1-click/1-plan-n-deploy.sh b/infra/1-click/1-plan-n-deploy.sh index 0b8da29..82cf1bb 100755 --- a/infra/1-click/1-plan-n-deploy.sh +++ b/infra/1-click/1-plan-n-deploy.sh @@ -14,27 +14,29 @@ fi echo "Change directory into '0-infra'." cd 0-infra -terraform plan -out=tf.plan \ - -var profile=$AWS_PROFILE \ - -var region=$AWS_REGION \ - -var azs='["a","c"]' \ - -var domain_name=$DOMAIN_NAME -terraform apply tf.plan -cd ../ - -echo "Change directory into '1-setup'." -cd 1-setup terraform plan -out=tf.plan \ -var profile=$AWS_PROFILE \ -var region=$AWS_REGION \ -var domain_name=$DOMAIN_NAME \ - -var coder_license=$LICENSE + -var azs='["a","c"]' terraform apply tf.plan cd ../ +# echo "Change directory into '1-setup'." +# cd 1-setup +# terraform plan -out=tf.plan \ +# -var profile=$AWS_PROFILE \ +# -var region=$AWS_REGION \ +# -var domain_name=$DOMAIN_NAME \ +# -var coder_license=$LICENSE +# terraform apply tf.plan +# cd ../ + # echo "Change directory into '2-coder'." # cd 2-coder # terraform plan -out=tf.plan \ -# -var profile="one-click" \ -# -var domain_name="oneclick-jullian.click" +# -var profile=$AWS_PROFILE \ +# -var region=$AWS_REGION \ +# -var domain_name=$DOMAIN_NAME +# terraform apply tf.plan # cd ../ \ No newline at end of file diff --git a/infra/1-click/1-setup/5-manifests.tf b/infra/1-click/1-setup/5-manifests.tf index 920094a..8f0e8cb 100644 --- a/infra/1-click/1-setup/5-manifests.tf +++ b/infra/1-click/1-setup/5-manifests.tf @@ -20,7 +20,7 @@ locals { user_data = "" block_device_mappings = [] } - "coder-ws" = { + "coder-workspace" = { user_data = <<-EOF apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig @@ -83,16 +83,19 @@ locals { node_expires_after = "Never" disruption_consolidation_policy = "WhenEmpty" disruption_consolidate_after = "1m" + taints = null } "coder-provisioner" = { node_expires_after = "Never" disruption_consolidation_policy = "WhenEmpty" disruption_consolidate_after = "1m" + taints = null } - "coder-ws" = { + "coder-workspace" = { node_expires_after = "Never" disruption_consolidation_policy = "WhenEmpty" disruption_consolidate_after = "30m" + taints = null } } } @@ -138,11 +141,11 @@ resource "kubernetes_manifest" "nodepool" { } } spec = { - taints = [{ + taints = each.value.taints == null ? [{ key = "dedicated" value = each.key effect = "NoSchedule" - }] + }] : each.value.taints requirements = [{ key = "kubernetes.io/arch" operator = "In" @@ -176,6 +179,29 @@ resource "kubernetes_manifest" "nodepool" { } } +## +# EBS CSI Default StorageClass +## + +resource "kubernetes_manifest" "default-sc" { + manifest = { + apiVersion = "storage.k8s.io/v1" + kind = "StorageClass" + metadata = { + name = "default" + annotations = { + "storageclass.kubernetes.io/is-default-class" = "true" + } + } + provisioner = "ebs.csi.eks.amazonaws.com" + volumeBindingMode = "WaitForFirstConsumer" + parameters = { + type = "gp3" + encrypted = "true" + } + } +} + ## # Cert-Manager ClusterIssuer for CA ## diff --git a/infra/1-click/1-setup/6-coder-server.tf b/infra/1-click/1-setup/6-coder-server.tf index 7c51238..c14ae01 100644 --- a/infra/1-click/1-setup/6-coder-server.tf +++ b/infra/1-click/1-setup/6-coder-server.tf @@ -41,6 +41,21 @@ variable "coder_admin_password" { default = "Th1s1sN0TS3CuR3!!" } +resource "aws_iam_user" "bedrock" { + name = "bedrock-access" + path = "/${var.cluster_name}/${data.aws_region.this.region}/" +} + +resource "aws_iam_user_policy_attachment" "bedrock" { + user = aws_iam_user.bedrock.name + # https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonBedrockLimitedAccess.html + policy_arn = "arn:aws:iam::aws:policy/AmazonBedrockLimitedAccess" +} + +resource "aws_iam_access_key" "bedrock" { + user = aws_iam_user.bedrock.name +} + module "coder-server" { depends_on = [ kubernetes_manifest.nodepool ] @@ -53,14 +68,22 @@ module "coder-server" { namespace = "coder" replica_count = 2 - helm_version = "2.29.1" + helm_version = "2.29.2" image_repo = "ghcr.io/coder/coder" - image_tag = "v2.29.1" + image_tag = "v2.29.2" primary_access_url = "https://${var.domain_name}" wildcard_access_url = "*.${var.domain_name}" db_secret_url = "postgresql://${var.coder_username}:${var.coder_password}@${data.aws_db_instance.coder.endpoint}/coder" + + env_vars = { + CODER_AIBRIDGE_ENABLED = true + CODER_AIBRIDGE_BEDROCK_REGION = data.aws_region.this.region + CODER_AIBRIDGE_BEDROCK_ACCESS_KEY = aws_iam_access_key.bedrock.id + CODER_AIBRIDGE_BEDROCK_ACCESS_KEY_SECRET = aws_iam_access_key.bedrock.secret + } - coder_builtin_provisioner_count = 0 + # Use this instead of external provisioners. DNS might not propagate fast enough for Coder to be "reachable". + coder_builtin_provisioner_count = 4 # coder_github_allowed_orgs = var.coder_github_allowed_orgs ssl_cert_config = { @@ -139,7 +162,6 @@ data "external" "first-user" { admin_username = var.coder_admin_username admin_password = var.coder_admin_password } - } output "coder_session_token" { @@ -159,7 +181,6 @@ data "external" "add-license" { license_key = var.coder_license session_token = data.external.first-user.result.session_token } - } # locals { diff --git a/infra/1-click/1-setup/main.tf b/infra/1-click/1-setup/main.tf index 21cfb1e..1bf78a6 100644 --- a/infra/1-click/1-setup/main.tf +++ b/infra/1-click/1-setup/main.tf @@ -103,6 +103,8 @@ provider "kubernetes" { token = data.aws_eks_cluster_auth.coder.token } +data "aws_region" "this" {} + locals { normalized_domain_name = split(".", var.domain_name)[0] } \ No newline at end of file diff --git a/infra/1-click/1-setup/scripts/add-license.sh b/infra/1-click/1-setup/scripts/add-license.sh index 8de8559..ee444f1 100644 --- a/infra/1-click/1-setup/scripts/add-license.sh +++ b/infra/1-click/1-setup/scripts/add-license.sh @@ -5,30 +5,30 @@ eval "$(jq -r '@sh "CODER_DOMAIN=\(.domain) CODER_LICENSE=\(.license_key) CODER_ # URL might not be available still. Retry request until available, or fail if max attempts reached. -IDX=0 IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) +RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$IP_ADDR" +CODER_URL=https://$CODER_DOMAIN + +IDX=0 while [[ -z "$IP_ADDR" ]]; do - ((IDX++)) - if (( IDX >= 6 )); then + if (( IDX > 6 )); then >&2 echo "Error: Failed to run \"dig +short $CODER_DOMAIN | head -n1\". Unable to discover IP for \"$CODER_DOMAIN\"." exit 1; fi sleep 10 + ((IDX++)) IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) done -RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$IP_ADDR" - -CODER_URL=https://$CODER_DOMAIN - RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/licenses" \ -H 'Content-Type: application/json' \ -H 'Accept: application/json' \ -H "Coder-Session-Token: $CODER_SESSION_TOKEN" \ -d "{\"license\": \"$CODER_LICENSE\"}") -if [[ $? -ne 0 ]]; then - >&2 echo "Error: Unable to run 'curl -ks -X POST \"$CODER_URL/api/v2/licenses\"'." +if [ $? -ne 0 ]; then + >&2 echo "Error: Unable to run 'curl -ks -X POST \"$CODER_URL/api/v2/licenses\"'. Can't add license." + jq -n '{"success":"false"}' exit 1; fi diff --git a/infra/1-click/1-setup/scripts/first-user.sh b/infra/1-click/1-setup/scripts/first-user.sh index 9844e9f..be15942 100755 --- a/infra/1-click/1-setup/scripts/first-user.sh +++ b/infra/1-click/1-setup/scripts/first-user.sh @@ -5,22 +5,21 @@ eval "$(jq -r '@sh "CODER_DOMAIN=\(.domain) ADMIN_EMAIL=\(.admin_email) ADMIN_US # URL might not be available still. Retry request until available, or fail if max attempts reached. -IDX=0 IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) +RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$IP_ADDR" +CODER_URL=https://$CODER_DOMAIN + +IDX=0 while [[ -z "$IP_ADDR" ]]; do - ((IDX++)) - if (( IDX >= 6 )); then + if (( IDX > 6 )); then >&2 echo "Error: Failed to run \"dig +short $CODER_DOMAIN | head -n1\". Unable to discover IP for \"$CODER_DOMAIN\"." exit 1; fi sleep 10 + ((IDX++)) IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) done -RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$IP_ADDR" - -CODER_URL=https://$CODER_DOMAIN - RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/users/first" \ -H "Content-Type: application/json" \ -d "{ @@ -30,10 +29,22 @@ RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/users/first" \ \"trial\": false }") +if [[ $? -ne 0 ]]; then + >&2 echo "Error: Failed on 'curl -ks $RESOLVE_ARG -X POST \"$CODER_URL/api/v2/users/first'\". Can't create first user." + jq -n '{"session_token":""}' + exit 1; +fi + LOGIN_RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/users/login" \ -H "Content-Type: application/json" \ -d "{\"email\":\"$ADMIN_EMAIL\",\"password\":\"$ADMIN_PASSWORD\"}" | jq -r '.session_token') +if [ $? -ne 0 ]; then + >&2 echo "Error: Failed on 'curl -ks $RESOLVE_ARG -X POST \"$CODER_URL/api/v2/users/login\"'. Can't login." + jq -n '{"session_token":""}' + exit 1; +fi + jq -n --arg session_token "$LOGIN_RESPONSE" '{"session_token":$session_token}' exit 0; \ No newline at end of file diff --git a/infra/1-click/2-clean.sh b/infra/1-click/2-clean.sh index 847da9d..4a8a9df 100755 --- a/infra/1-click/2-clean.sh +++ b/infra/1-click/2-clean.sh @@ -11,22 +11,31 @@ AWS_REGION="${CODER_AWS_REGION:-us-east-2}" DOMAIN_NAME="${CODER_DOMAIN_NAME:-}" LICENSE="${CODER_LICENSE:-}" -echo "Change directory into '1-setup'." -cd 1-setup +echo "Change directory into '2-coder'." +cd 2-coder terraform plan -destroy -out=tf.plan \ -var profile=$AWS_PROFILE \ -var region=$AWS_REGION \ - -var domain_name=$DOMAIN_NAME \ - -var coder_license=$LICENSE + -var domain_name=$DOMAIN_NAME terraform apply tf.plan cd ../ -echo "Change directory into '0-infra'." -cd 0-infra +echo "Change directory into '1-setup'." +cd 1-setup terraform plan -destroy -out=tf.plan \ -var profile=$AWS_PROFILE \ -var region=$AWS_REGION \ - -var azs='["a","c"]' \ - -var domain_name=$DOMAIN_NAME + -var domain_name=$DOMAIN_NAME \ + -var coder_license=$LICENSE terraform apply tf.plan -cd ../ \ No newline at end of file +cd ../ + +# echo "Change directory into '0-infra'." +# cd 0-infra +# terraform plan -destroy -out=tf.plan \ +# -var profile=$AWS_PROFILE \ +# -var region=$AWS_REGION \ +# -var azs='["a","c"]' \ +# -var domain_name=$DOMAIN_NAME +# terraform apply tf.plan +# cd ../ \ No newline at end of file diff --git a/infra/1-click/2-coder/7-coder.tf b/infra/1-click/2-coder/7-coder.tf deleted file mode 100644 index 90b4b8d..0000000 --- a/infra/1-click/2-coder/7-coder.tf +++ /dev/null @@ -1,91 +0,0 @@ -variable "domain_name" { - type = string -} - -variable "coder_admin_email" { - type = string - default = "admin@coder.com" -} - -variable "coder_admin_password" { - type = string - sensitive = true - default = "Th1s1sN0TS3CuR3!!" -} - -## -# Coder MUST be in a reachable state by now -## - -data "external" "fetch-token" { - - program = ["bash", "${path.module}/fetch.sh"] - - query = { - coder_url = "https://${var.domain_name}" - admin_email = var.coder_admin_email - admin_password = var.coder_admin_password - } - -} - -provider "coderd" { - url = "https://${var.domain_name}" - token = data.external.fetch-token.result.session_token -} - -output "coder_session_token" { - value = data.external.fetch-token.result.session_token -} - -locals { - image_repo = "ghcr.io/coder/coder" - image_tag = "v2.29.1" - service_account_labels = { - "app.kubernetes.io/instance" = "coder-provisioner" - "app.kubernetes.io/name" = "coder-provisioner" - "app.kubernetes.io/part-of" = "coder-provisioner" - } - node_selector = { - "node.coder.io/managed-by" = "karpenter" - "node.coder.io/used-for" = "coder-provisioner" - } - tolerations = [{ - key = "dedicated" - operator = "Equal" - value = "coder-provisioner" - effect = "NoSchedule" - }] -} - -module "default-ws" { - - depends_on = [ data.external.fetch-token ] - - source = "../../../modules/k8s/bootstrap/coder-provisioner" - - cluster_name = var.name - cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn - - coder_organization_name = "coder" - - namespace = "coder-ws" - image_repo = local.image_repo - image_tag = local.image_tag - ws_service_account_name = "coder-ws" - ws_service_account_labels = { - "app.kubernetes.io/instance" = "coder-provisioner" - "app.kubernetes.io/name" = "coder-provisioner" - "app.kubernetes.io/part-of" = "coder-provisioner" - } - provisioner_service_account_name = "coder" - replica_count = 2 - primary_access_url = "https://${var.domain_name}" - env_vars = { - CODER_PROMETHEUS_ENABLE = "true" - CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" - CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" - } - node_selector = local.node_selector - tolerations = local.tolerations -} \ No newline at end of file diff --git a/infra/1-click/2-coder/7-orgs.tf b/infra/1-click/2-coder/7-orgs.tf new file mode 100644 index 0000000..14c160d --- /dev/null +++ b/infra/1-click/2-coder/7-orgs.tf @@ -0,0 +1,108 @@ +variable "domain_name" { + type = string +} + +variable "coder_admin_email" { + type = string + default = "admin@coder.com" +} + +variable "coder_admin_password" { + type = string + sensitive = true + default = "Th1s1sN0TS3CuR3!!" +} + +## +# Coder MUST be in a reachable state by now +## + +data "external" "login" { + + program = ["bash", "${path.module}/scripts/login.sh"] + + query = { + domain = var.domain_name + admin_email = var.coder_admin_email + admin_password = var.coder_admin_password + } +} + +provider "coderd" { + url = "https://${var.domain_name}" + token = data.external.login.result.session_token +} + +output "coder_session_token" { + value = data.external.login.result.session_token +} + +locals { + image_repo = "ghcr.io/coder/coder" + image_tag = "v2.29.1" + service_account_labels = { + "app.kubernetes.io/instance" = "coder-provisioner" + "app.kubernetes.io/name" = "coder-provisioner" + "app.kubernetes.io/part-of" = "coder-provisioner" + } + node_selector = { + "node.coder.io/instance" = "coder-v2" + "node.coder.io/managed-by" = "karpenter" + "node.coder.io/name" = "coder" + "node.coder.io/part-of" = "coder" + "node.coder.io/used-for" = "coder-provisioner" + } + tolerations = [{ + key = "dedicated" + operator = "Equal" + value = "coder-provisioner" + effect = "NoSchedule" + }] +} + +data "coderd_organization" "default" { + is_default = true +} + +locals { + default_ns = "coder-ws" +} + +# resource "kubernetes_namespace" "coder-ws" { +# metadata { +# name = local.default_ns +# } +# } + +# module "default-ws" { + +# depends_on = [ data.external.login ] +# source = "../../../modules/k8s/bootstrap/coder-provisioner" + +# cluster_name = "${var.name}-${local.normalized_domain_name}" +# cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn + +# coder_organization_name = data.coderd_organization.default.name +# coder_provisioner_tags = {} + +# image_repo = local.image_repo +# image_tag = local.image_tag + +# namespace = local.default_ns +# ws_service_account_name = "coder-ws" +# ws_service_account_labels = { +# "app.kubernetes.io/instance" = "coder-provisioner" +# "app.kubernetes.io/name" = "coder-provisioner" +# "app.kubernetes.io/part-of" = "coder-provisioner" +# } +# provisioner_service_account_name = "coder" +# replica_count = 2 +# primary_access_url = "https://${var.domain_name}" +# env_vars = { +# CODER_PROMETHEUS_ENABLE = "true" +# CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" +# CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" +# } +# node_selector = local.node_selector +# tolerations = local.tolerations +# } \ No newline at end of file diff --git a/infra/1-click/2-coder/8-templates.tf b/infra/1-click/2-coder/8-templates.tf new file mode 100644 index 0000000..8885f9b --- /dev/null +++ b/infra/1-click/2-coder/8-templates.tf @@ -0,0 +1,49 @@ +data "coderd_group" "everyone" { + organization_id = data.coderd_organization.default.id + name = "Everyone" +} + +data "archive_file" "k8s-default" { + type = "zip" + excludes = ["${path.module}/templates/kubernetes/.terraform"] + source_dir = "${path.module}/templates/kubernetes" + output_path = "/tmp/kubernetes.zip" +} + +resource "time_static" "k8s-default" { + triggers = { + run_on_checksum = "${data.archive_file.k8s-default.id}" + } +} + +resource "coderd_template" "k8s-default" { + name = "kubernetes" + # organization_id = module.default-ws.coderd_organization_id + organization_id = data.coderd_organization.default.id + display_name = "Kubernetes (Deployment)" + description = "Provision Kubernetes Deployments as Coder workspaces" + icon = "https://${var.domain_name}/icon/k8s.png" + versions = [ + { + name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.k8s-default.rfc3339)}" + description = "The stable version of the template." + directory = "${path.module}/templates/kubernetes" + active = true + tf_vars = [{ + name = "namespace" + # value = kubernetes_namespace.coder-ws.metadata[0].name + value = "coder" + },{ + name = "use_kubeconfig" + value = tostring(false) + }] + } + ] + acl = { + users = [] + groups = [{ + id = data.coderd_group.everyone.id + role = "use" + }] + } +} \ No newline at end of file diff --git a/infra/1-click/2-coder/fetch.sh b/infra/1-click/2-coder/fetch.sh deleted file mode 100755 index 38825ed..0000000 --- a/infra/1-click/2-coder/fetch.sh +++ /dev/null @@ -1,24 +0,0 @@ - -#!/usr/bin/env bash - -eval "$(jq -r '@sh "CODER_URL=\(.coder_url) ADMIN_EMAIL=\(.admin_email) ADMIN_PASSWORD=\(.admin_password)"')" - -# URL might not be available still. Retry request until available, or fail if max attempts reached. - -IDX=0 -until curl -s -o /dev/null -kL "$CODER_URL"; do - if (( IDX >= 6 )); then - >&2 echo "Error: Unable to run 'curl -s -o /dev/null -kL \"$CODER_URL\"'." - exit 1; - fi - sleep 10 - ((IDX++)) -done - -LOGIN_RESPONSE=$(curl -ks -X POST "$CODER_URL/api/v2/users/login" \ - -H "Content-Type: application/json" \ - -d "{\"email\":\"$ADMIN_EMAIL\",\"password\":\"$ADMIN_PASSWORD\"}" | jq -r '.session_token') - -jq -n --arg session_token "$LOGIN_RESPONSE" '{"session_token":$session_token}' - -exit 0; \ No newline at end of file diff --git a/infra/1-click/2-coder/main.tf b/infra/1-click/2-coder/main.tf index 41db57f..2d44586 100644 --- a/infra/1-click/2-coder/main.tf +++ b/infra/1-click/2-coder/main.tf @@ -20,8 +20,13 @@ terraform { source = "coder/coderd" version = "0.0.12" } + null = { + source = "hashicorp/null" + } + time = { + source = "hashicorp/time" + } } - # backend "s3" {} } ## @@ -30,16 +35,16 @@ terraform { data "aws_vpc" "this" { tags = { - "Name" = var.name + "Name" = "${var.name}-${local.normalized_domain_name}" } } data "aws_eks_cluster" "coder" { - name = var.name + name = "${var.name}-${local.normalized_domain_name}" } data "aws_eks_cluster_auth" "coder" { - name = var.name + name = "${var.name}-${local.normalized_domain_name}" } data "aws_iam_openid_connect_provider" "coder" { @@ -85,3 +90,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.coder.certificate_authority[0].data) token = data.aws_eks_cluster_auth.coder.token } + +locals { + normalized_domain_name = split(".", var.domain_name)[0] +} \ No newline at end of file diff --git a/infra/1-click/2-coder/scripts/login.sh b/infra/1-click/2-coder/scripts/login.sh new file mode 100755 index 0000000..35aafa9 --- /dev/null +++ b/infra/1-click/2-coder/scripts/login.sh @@ -0,0 +1,35 @@ + +#!/usr/bin/env bash + +eval "$(jq -r '@sh "CODER_DOMAIN=\(.domain) ADMIN_EMAIL=\(.admin_email) ADMIN_PASSWORD=\(.admin_password)"')" + +# URL might not be available still. Retry request until available, or fail if max attempts reached. + +IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) +RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$IP_ADDR" +CODER_URL=https://$CODER_DOMAIN + +IDX=0 +while [[ -z "$IP_ADDR" ]]; do + ((IDX++)) + if (( IDX >= 6 )); then + >&2 echo "Error: Failed to run \"dig +short $CODER_DOMAIN | head -n1\". Unable to discover IP for \"$CODER_DOMAIN\"." + exit 1; + fi + sleep 10 + IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) +done + +LOGIN_RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/users/login" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"$ADMIN_EMAIL\",\"password\":\"$ADMIN_PASSWORD\"}" | jq -r '.session_token') + +if [ $? -ne 0 ]; then + >&2 echo "Error: Failed on 'curl -ks $RESOLVE_ARG -X POST \"$CODER_URL/api/v2/users/login\"'. Can't login." + jq -n '{"session_token":""}' + exit 1; +fi + +jq -n --arg session_token "$LOGIN_RESPONSE" '{"session_token":$session_token}' + +exit 0; \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/kubernetes/README.md b/infra/1-click/2-coder/templates/kubernetes/README.md new file mode 100644 index 0000000..44121f2 --- /dev/null +++ b/infra/1-click/2-coder/templates/kubernetes/README.md @@ -0,0 +1,38 @@ +--- +display_name: Kubernetes (Deployment) +description: Provision Kubernetes Deployments as Coder workspaces +icon: ../../../site/static/icon/k8s.png +maintainer_github: coder +verified: true +tags: [kubernetes, container] +--- + +# Remote Development on Kubernetes Pods + +Provision Kubernetes Pods as [Coder workspaces](https://coder.com/docs/workspaces) with this example template. + + + +## Prerequisites + +### Infrastructure + +**Cluster**: This template requires an existing Kubernetes cluster + +**Container Image**: This template uses the [codercom/enterprise-base:ubuntu image](https://github.com/coder/enterprise-images/tree/main/images/base) with some dev tools preinstalled. To add additional tools, extend this image or build it yourself. + +### Authentication + +This template authenticates using a `~/.kube/config`, if present on the server, or via built-in authentication if the Coder provisioner is running on Kubernetes with an authorized ServiceAccount. To use another [authentication method](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs#authentication), edit the template. + +## Architecture + +This template provisions the following resources: + +- Kubernetes pod (ephemeral) +- Kubernetes persistent volume claim (persistent on `/home/coder`) + +This means, when the workspace restarts, any tools or files outside of the home directory are not persisted. To pre-bake tools into the workspace (e.g. `python3`), modify the container image. Alternatively, individual developers can [personalize](https://coder.com/docs/dotfiles) their workspaces with dotfiles. + +> **Note** +> This template is designed to be a starting point! Edit the Terraform to extend the template to support your use case. \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/kubernetes/main.tf b/infra/1-click/2-coder/templates/kubernetes/main.tf new file mode 100644 index 0000000..38b206c --- /dev/null +++ b/infra/1-click/2-coder/templates/kubernetes/main.tf @@ -0,0 +1,344 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} + +provider "coder" {} + +variable "use_kubeconfig" { + type = bool + description = <<-EOF + Use host kubeconfig? (true/false) + + Set this to false if the Coder host is itself running as a Pod on the same + Kubernetes cluster as you are deploying workspaces to. + + Set this to true if the Coder host is running outside the Kubernetes cluster + for workspaces. A valid "~/.kube/config" must be present on the Coder host. + EOF + default = false +} + +variable "namespace" { + type = string + description = "The Kubernetes namespace to create workspaces in (must exist prior to creating workspaces). If the Coder host is itself running as a Pod on the same Kubernetes cluster as you are deploying workspaces to, set this to the same namespace." +} + +data "coder_parameter" "cpu" { + name = "cpu" + display_name = "CPU" + description = "The number of CPU cores" + default = "2" + icon = "/icon/memory.svg" + mutable = true + option { + name = "2 Cores" + value = "2" + } + option { + name = "4 Cores" + value = "4" + } + option { + name = "6 Cores" + value = "6" + } + option { + name = "8 Cores" + value = "8" + } +} + +data "coder_parameter" "memory" { + name = "memory" + display_name = "Memory" + description = "The amount of memory in GB" + default = "2" + icon = "/icon/memory.svg" + mutable = true + option { + name = "2 GB" + value = "2" + } + option { + name = "4 GB" + value = "4" + } + option { + name = "6 GB" + value = "6" + } + option { + name = "8 GB" + value = "8" + } +} + +data "coder_parameter" "home_disk_size" { + name = "home_disk_size" + display_name = "Home disk size" + description = "The size of the home disk in GB" + default = "10" + type = "number" + icon = "/emojis/1f4be.png" + mutable = false + validation { + min = 1 + max = 99999 + } +} + +provider "kubernetes" { + # Authenticate via ~/.kube/config or a Coder-specific ServiceAccount, depending on admin preferences + config_path = var.use_kubeconfig == true ? "~/.kube/config" : null +} + +data "coder_workspace" "me" {} +data "coder_workspace_owner" "me" {} + +resource "coder_agent" "main" { + os = "linux" + arch = "amd64" + startup_script = <<-EOT + set -e + + # Install the latest code-server. + # Append "--version x.x.x" to install a specific version of code-server. + curl -fsSL https://code-server.dev/install.sh | sh -s -- --method=standalone --prefix=/tmp/code-server + + # Start code-server in the background. + /tmp/code-server/bin/code-server --auth none --port 13337 >/tmp/code-server.log 2>&1 & + EOT + + # The following metadata blocks are optional. They are used to display + # information about your workspace in the dashboard. You can remove them + # if you don't want to display any information. + # For basic resources, you can use the `coder stat` command. + # If you need more control, you can write your own script. + metadata { + display_name = "CPU Usage" + key = "0_cpu_usage" + script = "coder stat cpu" + interval = 10 + timeout = 1 + } + + metadata { + display_name = "RAM Usage" + key = "1_ram_usage" + script = "coder stat mem" + interval = 10 + timeout = 1 + } + + metadata { + display_name = "Home Disk" + key = "3_home_disk" + script = "coder stat disk --path $${HOME}" + interval = 60 + timeout = 1 + } + + metadata { + display_name = "CPU Usage (Host)" + key = "4_cpu_usage_host" + script = "coder stat cpu --host" + interval = 10 + timeout = 1 + } + + metadata { + display_name = "Memory Usage (Host)" + key = "5_mem_usage_host" + script = "coder stat mem --host" + interval = 10 + timeout = 1 + } + + metadata { + display_name = "Load Average (Host)" + key = "6_load_host" + # get load avg scaled by number of cores + script = < [!IMPORTANT] > This deployment will take around 30min ~ 1hr to setup everything (AWS resources, K8s Addons, and Coder) +./1-plan-n-deploy.sh 60.39s user 9.01s system 4% cpu 23:20.70 total + +> Cleaning up will take ~20 minutes + +./2-clean.sh 39.48s user 7.67s system 5% cpu 13:59.86 total + # Getting Started 1. Create a new AWS account or access an existing one diff --git a/modules/k8s/bootstrap/coder-provisioner/main.tf b/modules/k8s/bootstrap/coder-provisioner/main.tf index 6f3d36b..699d5b9 100644 --- a/modules/k8s/bootstrap/coder-provisioner/main.tf +++ b/modules/k8s/bootstrap/coder-provisioner/main.tf @@ -205,6 +205,11 @@ variable "coder_provisioner_key_name" { default = "" } +variable "coder_provisioner_tags" { + type = map(string) + default = null +} + ## # Coder External Provisioner ## @@ -254,6 +259,9 @@ locals { env_vars = [ for k, v in merge(local.primary_env_vars, var.env_vars) : { name = k, value = v } ] + provisioner_tags = var.coder_provisioner_tags == null ? { + region = data.aws_region.this.region + } : var.coder_provisioner_tags topology_spread_constraints = [ for v in var.topology_spread_constraints : { maxSkew = v.max_skew @@ -277,9 +285,7 @@ locals { module "coder-provisioner-key" { source = "../../../coder/provisioner" organization_id = data.coderd_organization.this.id - provisioner_tags = { - region = data.aws_region.this.region - } + provisioner_tags = local.provisioner_tags } resource "kubernetes_namespace" "this" { @@ -434,4 +440,25 @@ resource "kubernetes_service_account" "ws" { output "oidc_role_arn" { value = module.provisioner-oidc-role.role_arn +} + +output "coderd_organization_id" { + + depends_on = [ helm_release.coder-provisioner ] + + value = data.coderd_organization.this.id +} + +output "k8s_namespace" { + + depends_on = [ helm_release.coder-provisioner ] + + value = kubernetes_namespace.this.metadata[0].name +} + +output "k8s_ws_service_account_name" { + + depends_on = [ helm_release.coder-provisioner ] + + value = kubernetes_service_account.ws.metadata[0].name } \ No newline at end of file diff --git a/modules/k8s/bootstrap/coder-server/main.tf b/modules/k8s/bootstrap/coder-server/main.tf index ac84504..fca8092 100644 --- a/modules/k8s/bootstrap/coder-server/main.tf +++ b/modules/k8s/bootstrap/coder-server/main.tf @@ -371,11 +371,6 @@ variable "coder_github_allowed_orgs" { default = [] } -variable "prometheus_port" { - type = number - default = 2112 -} - variable "openai_llm_endpoint" { type = string sensitive = true @@ -508,9 +503,10 @@ locals { CODER_PROMETHEUS_ENABLE = true CODER_PROMETHEUS_COLLECT_AGENT_STATS = true CODER_PROMETHEUS_COLLECT_DB_METRICS = true - CODER_PROMETHEUS_ADDRESS = "127.0.0.1:${var.prometheus_port}" + # CODER_PROMETHEUS_ADDRESS = "127.0.0.1:${var.prometheus_port}" - CODER_AIBRIDGE_ENABLED = var.openai_llm_endpoint != "" || var.anthropic_llm_endpoint != "" + # CODER_AIBRIDGE_BEDROCK_MODEL = "global.anthropic.claude-sonnet-4-5-20250929-v1:0" + # CODER_AIBRIDGE_BEDROCK_SMALL_FAST_MODEL = "global.anthropic.claude-haiku-4-5-20251001-v1:0" # Experimental Coder Features # CODER_EXPERIMENTS = join(",", var.coder_experiments) @@ -939,7 +935,7 @@ resource "kubernetes_service" "prometheus" { name = "prom-http" protocol = "TCP" port = 2112 - target_port = var.prometheus_port + target_port = 2112 } selector = { "app.kubernetes.io/instance" = "coder-v2" diff --git a/modules/k8s/bootstrap/external-dns/main.tf b/modules/k8s/bootstrap/external-dns/main.tf index b6fe0fd..1b89d96 100644 --- a/modules/k8s/bootstrap/external-dns/main.tf +++ b/modules/k8s/bootstrap/external-dns/main.tf @@ -110,11 +110,6 @@ variable "is_private_domain" { default = false } -data "aws_route53_zone" "this" { - name = "${var.domain_name}." - private_zone = var.is_private_domain -} - resource "helm_release" "chart" { name = "external-dns" namespace = var.namespace @@ -150,8 +145,9 @@ resource "helm_release" "chart" { } } nodeSelector = var.node_selector - # extraArgs = [ - # "--aws-prefer-cname" - # ] + extraArgs = [ + "--aws-zone-match-parent" + # "--aws-prefer-cname" + ] })] } \ No newline at end of file diff --git a/modules/k8s/bootstrap/karpenter/main.tf b/modules/k8s/bootstrap/karpenter/main.tf index 4e977e3..9980c2f 100644 --- a/modules/k8s/bootstrap/karpenter/main.tf +++ b/modules/k8s/bootstrap/karpenter/main.tf @@ -103,6 +103,15 @@ variable "node_iam_role_use_name_prefix" { default = true } +variable "topology_spread_constraints" { + type = list(map(string)) + default = [{ + maxSkew = 1 + topologyKey = "topology.kubernetes.io/zone" + whenUnsatisfiable = "DoNotSchedule" + }] +} + variable "ec2nodeclass_configs" { type = list(object({ name = string @@ -151,6 +160,11 @@ variable "node_selector" { default = {} } +variable "replicas" { + type = number + default = 1 +} + data "aws_region" "this" {} locals { @@ -283,12 +297,13 @@ resource "helm_release" "karpenter" { } dnsPolicy = "ClusterFirst" nodeSelector = var.node_selector - replicas = 2 + replicas = var.replicas serviceAccount = { annotations = { "eks.amazonaws.com/role-arn" = module.karpenter.iam_role_arn } } + topologySpreadConstraints = var.topology_spread_constraints settings = { clusterName = var.cluster_name featureGates = { From cb96780e69b9ef219c7c228416ce472c8b28773e Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 26 Jan 2026 12:09:12 -0800 Subject: [PATCH 15/44] fix: made new changes --- infra/1-click/coder.env | 7 ++++--- modules/k8s/bootstrap/karpenter/main.tf | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/infra/1-click/coder.env b/infra/1-click/coder.env index 1beb75f..4493b64 100644 --- a/infra/1-click/coder.env +++ b/infra/1-click/coder.env @@ -1,3 +1,4 @@ -CODER_AWS_PROFILE= -CODER_DOMAIN_NAME= -CODER_LICENSE= \ No newline at end of file +CODER_AWS_PROFILE=default +CODER_AWS_REGION=us-east-2 +CODER_DOMAIN_NAME= +CODER_LICENSE= (Optional) \ No newline at end of file diff --git a/modules/k8s/bootstrap/karpenter/main.tf b/modules/k8s/bootstrap/karpenter/main.tf index 9980c2f..6e0b224 100644 --- a/modules/k8s/bootstrap/karpenter/main.tf +++ b/modules/k8s/bootstrap/karpenter/main.tf @@ -106,7 +106,7 @@ variable "node_iam_role_use_name_prefix" { variable "topology_spread_constraints" { type = list(map(string)) default = [{ - maxSkew = 1 + maxSkew = "1" topologyKey = "topology.kubernetes.io/zone" whenUnsatisfiable = "DoNotSchedule" }] From 5329b5e11d02c245bc7cf143b3e2b74bad77b13c Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 26 Jan 2026 18:22:00 -0800 Subject: [PATCH 16/44] fix: exposes tolerations --- modules/k8s/bootstrap/cert-manager/main.tf | 25 +++++++++++++++++++ modules/k8s/bootstrap/ebs-controller/main.tf | 6 ++++- modules/k8s/bootstrap/external-dns/main.tf | 5 +++- .../k8s/bootstrap/external-secrets/main.tf | 14 +++++++++++ modules/k8s/bootstrap/karpenter/main.tf | 7 ++++-- modules/k8s/bootstrap/lb-controller/main.tf | 4 +++ modules/k8s/bootstrap/metrics-server/main.tf | 4 +++ 7 files changed, 61 insertions(+), 4 deletions(-) diff --git a/modules/k8s/bootstrap/cert-manager/main.tf b/modules/k8s/bootstrap/cert-manager/main.tf index be5ee01..02c0781 100644 --- a/modules/k8s/bootstrap/cert-manager/main.tf +++ b/modules/k8s/bootstrap/cert-manager/main.tf @@ -60,6 +60,13 @@ variable "helm_version" { default = "v1.18.2" } +variable "node_selector" { + type = map(string) + default = { + "kubernetes.io/os" = "linux" + } +} + ## # Create Default Resources? ## @@ -130,6 +137,13 @@ resource "kubernetes_namespace" "this" { } } +locals { + global_tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] +} + resource "helm_release" "cert-manager" { name = "cert-manager" namespace = kubernetes_namespace.this.metadata[0].name @@ -147,6 +161,17 @@ resource "helm_release" "cert-manager" { crds = { enabled = true } + nodeSelector = var.node_selector + tolerations = local.global_tolerations + webhook = { + tolerations = local.global_tolerations + } + cainjector = { + tolerations = local.global_tolerations + } + startupapicheck = { + tolerations = local.global_tolerations + } })] } diff --git a/modules/k8s/bootstrap/ebs-controller/main.tf b/modules/k8s/bootstrap/ebs-controller/main.tf index ec23c7f..1269e6b 100644 --- a/modules/k8s/bootstrap/ebs-controller/main.tf +++ b/modules/k8s/bootstrap/ebs-controller/main.tf @@ -103,7 +103,11 @@ resource "helm_release" "ebs-controller" { "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn }, var.service_account_annotations) } - nodeSelector = {} + nodeSelector = var.node_selector + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] } })] } diff --git a/modules/k8s/bootstrap/external-dns/main.tf b/modules/k8s/bootstrap/external-dns/main.tf index 1b89d96..793cb94 100644 --- a/modules/k8s/bootstrap/external-dns/main.tf +++ b/modules/k8s/bootstrap/external-dns/main.tf @@ -145,9 +145,12 @@ resource "helm_release" "chart" { } } nodeSelector = var.node_selector + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] extraArgs = [ "--aws-zone-match-parent" - # "--aws-prefer-cname" ] })] } \ No newline at end of file diff --git a/modules/k8s/bootstrap/external-secrets/main.tf b/modules/k8s/bootstrap/external-secrets/main.tf index 82db376..2eb3c2c 100644 --- a/modules/k8s/bootstrap/external-secrets/main.tf +++ b/modules/k8s/bootstrap/external-secrets/main.tf @@ -101,6 +101,13 @@ module "oidc-role" { tags = var.tags } +locals { + global_tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] +} + resource "helm_release" "chart" { name = "external-secrets" namespace = var.namespace @@ -116,6 +123,13 @@ resource "helm_release" "chart" { values = [yamlencode({ nodeSelector = var.node_selector + tolerations = local.global_tolerations + webhook = { + tolerations = local.global_tolerations + } + certController = { + tolerations = local.global_tolerations + } serviceAccount = { annotations = { "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn diff --git a/modules/k8s/bootstrap/karpenter/main.tf b/modules/k8s/bootstrap/karpenter/main.tf index 6e0b224..5f37c43 100644 --- a/modules/k8s/bootstrap/karpenter/main.tf +++ b/modules/k8s/bootstrap/karpenter/main.tf @@ -162,7 +162,7 @@ variable "node_selector" { variable "replicas" { type = number - default = 1 + default = 2 } data "aws_region" "this" {} @@ -303,7 +303,10 @@ resource "helm_release" "karpenter" { "eks.amazonaws.com/role-arn" = module.karpenter.iam_role_arn } } - topologySpreadConstraints = var.topology_spread_constraints + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] settings = { clusterName = var.cluster_name featureGates = { diff --git a/modules/k8s/bootstrap/lb-controller/main.tf b/modules/k8s/bootstrap/lb-controller/main.tf index 0e97ba3..a27d3f1 100644 --- a/modules/k8s/bootstrap/lb-controller/main.tf +++ b/modules/k8s/bootstrap/lb-controller/main.tf @@ -165,6 +165,10 @@ resource "helm_release" "lb-controller" { vpcId = var.vpc_id enableCertManager = var.enable_cert_manager nodeSelector = var.node_selector + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] serviceTargetENISGTags = local.service_target_eni_sg_tags serviceMutatorWebhookConfig = { # Ref - https://github.com/awslabs/data-on-eks/issues/458 diff --git a/modules/k8s/bootstrap/metrics-server/main.tf b/modules/k8s/bootstrap/metrics-server/main.tf index a7f5d70..9c3ad1f 100644 --- a/modules/k8s/bootstrap/metrics-server/main.tf +++ b/modules/k8s/bootstrap/metrics-server/main.tf @@ -41,5 +41,9 @@ resource "helm_release" "metrics-server" { values = [yamlencode({ nodeSelector = var.node_selector + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] })] } \ No newline at end of file From 730bf29af629c0ca9e9269c73be5397dff7a193b Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 26 Jan 2026 18:22:34 -0800 Subject: [PATCH 17/44] fix: adjust README for clearer instructions --- infra/1-click/README.md | 56 +++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/infra/1-click/README.md b/infra/1-click/README.md index 5437b77..fa594dc 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -1,43 +1,48 @@ # Requirements -- Terraform -- AWS Account -- Domain in Route53 +- [Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) +- [AWS Account](https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-creating.html) + [CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) +- [Domain in Route53]((https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section)) +- [AWS Bedrock + Anthropic Agreement Completed](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) > [!IMPORTANT] -> This deployment will take around 30min ~ 1hr to setup everything (AWS resources, K8s Addons, and Coder) - -./1-plan-n-deploy.sh 60.39s user 9.01s system 4% cpu 23:20.70 total - -> Cleaning up will take ~20 minutes - -./2-clean.sh 39.48s user 7.67s system 5% cpu 13:59.86 total +> Deploying will take around ~30min to setup everything (AWS resources, K8s Addons, DNS resolution, and Coder). +> Cleaning up will take ~20 minutes. # Getting Started -1. Create a new AWS account or access an existing one +1. Create or use an AWS account 2. Login as a user with AdministratorAccess -3. Setup your local machine to have an [AWS profile](https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-files.html) +3. Setup your local machine to have an [AWS profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) >[!NOTE] > When running `aws login` or `aws configure`, this will automatically setup a `default` profile for you. -4. [Purchase/Register](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) a domain in Route53 and tie it to a public hosted zone. +4. Purchase/Register a domain in Route53 and tie it to a public hosted zone. > [!IMPORTANT] -> When purchasing, it creates a hosted zone in a few minutes, wait for this. AND, if using an existing domain/zone, cleanup any A/AAAA/CNAME/TXT records. +> When purchasing, it creates a hosted zone in a few minutes, wait for this. AND, if using an existing domain/zone, cleanup any conflicting A/AAAA/CNAME/TXT records. 5. Fill the `coder.env` file with the following environment variables: -```.env -CODER_AWS_PROFILE= -CODER_DOMAIN_NAME= -CODER_LICENSE= (Optional) +```env +CODER_AWS_PROFILE=default (Change profile if needed) +CODER_AWS_PROFILE=us-east-2 (Change region if needed) +CODER_DOMAIN_NAME=put.your.domain.here.com +CODER_LICENSE=abcde1234..... (Optional) +``` + +6. Initialize the project by running: + +```bash +./0-initialize.sh ``` -6. Run `./0-initialize.sh` -7. Run `set -a && source coder.env && set +a && ./1-plan-n-deploy.sh` -8. Run `set -a && source coder.env && set +a && ./2-clean.sh` +7. Finally, deploy by running: + +```bash +set -a && source coder.env && set +a && ./1-plan-n-deploy.sh +``` # Logging In @@ -51,6 +56,14 @@ Password: Th1s1sN0TS3CuR3!! 3. You're done! +# Cleaning Up + +1. Just run: + +```bash +set -a && source coder.env && set +a && ./2-clean.sh +``` + # Troubleshooting - To verify the state of the infrastructure, visit the AWS Console and look at the pages of following AWS services that this solution deploys/manages: @@ -62,6 +75,7 @@ Password: Th1s1sN0TS3CuR3!! - Route53 - SSM Parameters - CloudWatch Logs + - Bedrock - To verify the cluster state, run `aws eks update-kubeconfig --name --region --profile ` and execute kubectl commands. From b04004fdb87b5cd46774560888cf8d950b844daa Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 26 Jan 2026 18:23:22 -0800 Subject: [PATCH 18/44] feat: adds claude-based template --- infra/1-click/2-coder/8-templates.tf | 45 +++ .../templates/kubernetes-claude/README.md | 105 ++++++ .../templates/kubernetes-claude/main.tf | 345 ++++++++++++++++++ .../2-coder/templates/kubernetes/README.md | 115 +++++- 4 files changed, 600 insertions(+), 10 deletions(-) create mode 100644 infra/1-click/2-coder/templates/kubernetes-claude/README.md create mode 100644 infra/1-click/2-coder/templates/kubernetes-claude/main.tf diff --git a/infra/1-click/2-coder/8-templates.tf b/infra/1-click/2-coder/8-templates.tf index 8885f9b..819c716 100644 --- a/infra/1-click/2-coder/8-templates.tf +++ b/infra/1-click/2-coder/8-templates.tf @@ -46,4 +46,49 @@ resource "coderd_template" "k8s-default" { role = "use" }] } +} + +data "archive_file" "k8s-claude" { + type = "zip" + excludes = ["${path.module}/templates/kubernetes-claude/.terraform"] + source_dir = "${path.module}/templates/kubernetes-claude" + output_path = "/tmp/kubernetes-claude.zip" +} + +resource "time_static" "k8s-claude" { + triggers = { + run_on_checksum = "${data.archive_file.k8s-claude.id}" + } +} + +resource "coderd_template" "k8s-claude" { + name = "kubernetes-claude" + # organization_id = module.default-ws.coderd_organization_id + organization_id = data.coderd_organization.default.id + display_name = "Kubernetes w/ Claude (Deployment)" + description = "Provision a Kubernetes Deployments with Claude installed." + icon = "https://${var.domain_name}/icon/claude.svg" + versions = [ + { + name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.k8s-claude.rfc3339)}" + description = "The stable version of the template." + directory = "${path.module}/templates/kubernetes-claude" + active = true + tf_vars = [{ + name = "namespace" + # value = kubernetes_namespace.coder-ws.metadata[0].name + value = "coder" + },{ + name = "use_kubeconfig" + value = tostring(false) + }] + } + ] + acl = { + users = [] + groups = [{ + id = data.coderd_group.everyone.id + role = "use" + }] + } } \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/kubernetes-claude/README.md b/infra/1-click/2-coder/templates/kubernetes-claude/README.md new file mode 100644 index 0000000..df936dc --- /dev/null +++ b/infra/1-click/2-coder/templates/kubernetes-claude/README.md @@ -0,0 +1,105 @@ +--- +display_name: Kubernetes w/ Claude Code +description: Provision Kubernetes workspaces with Claude Code and code-server +icon: ../../../site/static/icon/claude.svg +maintainer_github: coder +verified: true +tags: [kubernetes, container, claude, ai] +--- + +# Kubernetes Workspaces with Claude Code + +Provision Kubernetes-based [Coder workspaces](https://coder.com/docs/workspaces) with Claude Code and code-server pre-configured. + +## Features + +- **Claude Code**: AI-powered coding assistant via the [claude-code module](https://registry.coder.com/modules/claude-code) +- **code-server**: Browser-based VS Code experience +- **Persistent Storage**: Home directory persists across workspace restarts +- **Resource Control**: Configurable CPU, memory, and disk allocation +- **Workspace Metrics**: Real-time CPU, memory, and disk usage displayed in the dashboard + +## Prerequisites + +### Infrastructure + +- **Kubernetes Cluster**: An existing Kubernetes cluster with sufficient resources +- **Namespace**: A pre-existing namespace for workspace deployments +- **Storage Class**: A default storage class for persistent volume claims + +### Authentication + +This template supports two authentication methods: + +1. **In-cluster authentication** (default): When Coder runs as a pod in the same cluster, it uses the ServiceAccount for authentication. Set `use_kubeconfig = false`. + +2. **Kubeconfig authentication**: When Coder runs outside the cluster, provide a valid `~/.kube/config` on the Coder host. Set `use_kubeconfig = true`. + +## Configuration + +### Template Variables + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `use_kubeconfig` | bool | `false` | Use host kubeconfig for authentication | +| `namespace` | string | required | Kubernetes namespace for workspaces | + +### Workspace Parameters + +Users can customize these settings when creating a workspace: + +| Parameter | Options | Default | Description | +|-----------|---------|---------|-------------| +| CPU | 2, 4, 6, 8 cores | 2 | CPU cores allocated to the workspace | +| Memory | 2, 4, 6, 8 GB | 2 | Memory allocated to the workspace | +| Home Disk Size | 1-99999 GB | 10 | Persistent storage for `/home/coder` | + +## Architecture + +This template provisions the following Kubernetes resources: + +- **Deployment**: A single-replica deployment running the workspace container +- **Persistent Volume Claim**: Storage for the `/home/coder` directory + +### Container Image + +Uses `codercom/enterprise-base:ubuntu` which includes common development tools. To customize: + +- Extend the image with additional tools +- Build your own image based on the [enterprise-images repository](https://github.com/coder/enterprise-images) + +### Included Applications + +| Application | Access | Description | +|-------------|--------|-------------| +| Claude Code | Subdomain | AI coding assistant with AI Bridge enabled | +| code-server | Subdomain | Browser-based VS Code | + +### Workspace Metadata + +The dashboard displays real-time metrics: + +- CPU Usage (container and host) +- RAM Usage (container and host) +- Home Disk Usage +- Load Average (host) + +## Pod Scheduling + +Workspaces use pod anti-affinity to distribute pods across nodes, improving cluster utilization and resilience. + +## Security + +- Containers run as non-root user (UID 1000) +- File system group set to 1000 +- Resource limits enforced based on user-selected parameters + +## Customization + +This template is designed as a starting point. Common modifications include: + +- Adding environment variables +- Mounting secrets or configmaps +- Changing the container image +- Adding init containers +- Configuring node selectors or tolerations diff --git a/infra/1-click/2-coder/templates/kubernetes-claude/main.tf b/infra/1-click/2-coder/templates/kubernetes-claude/main.tf new file mode 100644 index 0000000..eaba9d3 --- /dev/null +++ b/infra/1-click/2-coder/templates/kubernetes-claude/main.tf @@ -0,0 +1,345 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} + +provider "coder" {} + +variable "use_kubeconfig" { + type = bool + description = <<-EOF + Use host kubeconfig? (true/false) + + Set this to false if the Coder host is itself running as a Pod on the same + Kubernetes cluster as you are deploying workspaces to. + + Set this to true if the Coder host is running outside the Kubernetes cluster + for workspaces. A valid "~/.kube/config" must be present on the Coder host. + EOF + default = false +} + +variable "namespace" { + type = string + description = "The Kubernetes namespace to create workspaces in (must exist prior to creating workspaces). If the Coder host is itself running as a Pod on the same Kubernetes cluster as you are deploying workspaces to, set this to the same namespace." +} + +data "coder_parameter" "cpu" { + name = "cpu" + display_name = "CPU" + description = "The number of CPU cores" + default = "2" + icon = "/icon/memory.svg" + mutable = true + option { + name = "2 Cores" + value = "2" + } + option { + name = "4 Cores" + value = "4" + } + option { + name = "6 Cores" + value = "6" + } + option { + name = "8 Cores" + value = "8" + } +} + +data "coder_parameter" "memory" { + name = "memory" + display_name = "Memory" + description = "The amount of memory in GB" + default = "2" + icon = "/icon/memory.svg" + mutable = true + option { + name = "2 GB" + value = "2" + } + option { + name = "4 GB" + value = "4" + } + option { + name = "6 GB" + value = "6" + } + option { + name = "8 GB" + value = "8" + } +} + +data "coder_parameter" "home_disk_size" { + name = "home_disk_size" + display_name = "Home disk size" + description = "The size of the home disk in GB" + default = "10" + type = "number" + icon = "/emojis/1f4be.png" + mutable = false + validation { + min = 1 + max = 99999 + } +} + +provider "kubernetes" { + # Authenticate via ~/.kube/config or a Coder-specific ServiceAccount, depending on admin preferences + config_path = var.use_kubeconfig == true ? "~/.kube/config" : null +} + +data "coder_workspace" "me" {} +data "coder_workspace_owner" "me" {} + +resource "coder_agent" "main" { + os = "linux" + arch = "amd64" + + # The following metadata blocks are optional. They are used to display + # information about your workspace in the dashboard. You can remove them + # if you don't want to display any information. + # For basic resources, you can use the `coder stat` command. + # If you need more control, you can write your own script. + metadata { + display_name = "CPU Usage" + key = "0_cpu_usage" + script = "coder stat cpu" + interval = 10 + timeout = 1 + } + + metadata { + display_name = "RAM Usage" + key = "1_ram_usage" + script = "coder stat mem" + interval = 10 + timeout = 1 + } + + metadata { + display_name = "Home Disk" + key = "3_home_disk" + script = "coder stat disk --path $${HOME}" + interval = 60 + timeout = 1 + } + + metadata { + display_name = "CPU Usage (Host)" + key = "4_cpu_usage_host" + script = "coder stat cpu --host" + interval = 10 + timeout = 1 + } + + metadata { + display_name = "Memory Usage (Host)" + key = "5_mem_usage_host" + script = "coder stat mem --host" + interval = 10 + timeout = 1 + } + + metadata { + display_name = "Load Average (Host)" + key = "6_load_host" + # get load avg scaled by number of cores + script = < - ## Prerequisites ### Infrastructure -**Cluster**: This template requires an existing Kubernetes cluster +**Cluster**: This template requires an existing Kubernetes cluster. + +**Container Image**: This template uses the [codercom/enterprise-base:ubuntu](https://github.com/coder/enterprise-images/tree/main/images/base) image with dev tools preinstalled. To add additional tools, extend this image or build your own. -**Container Image**: This template uses the [codercom/enterprise-base:ubuntu image](https://github.com/coder/enterprise-images/tree/main/images/base) with some dev tools preinstalled. To add additional tools, extend this image or build it yourself. +**Storage**: A StorageClass must be available in the cluster to provision persistent volumes for workspace home directories. ### Authentication -This template authenticates using a `~/.kube/config`, if present on the server, or via built-in authentication if the Coder provisioner is running on Kubernetes with an authorized ServiceAccount. To use another [authentication method](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs#authentication), edit the template. +This template authenticates using one of two methods: + +1. **In-cluster authentication** (default): If Coder is running as a Pod on the same Kubernetes cluster, it uses the built-in ServiceAccount authentication. +2. **Kubeconfig authentication**: If Coder is running outside the cluster, set `use_kubeconfig = true` and ensure a valid `~/.kube/config` exists on the Coder host. + +To use another [authentication method](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs#authentication), edit the template. ## Architecture This template provisions the following resources: -- Kubernetes pod (ephemeral) -- Kubernetes persistent volume claim (persistent on `/home/coder`) +| Resource | Type | Persistence | +|----------|------|-------------| +| Kubernetes Deployment | Compute | Ephemeral (recreated on restart) | +| Persistent Volume Claim | Storage | Persistent (mounted at `/home/coder`) | + +When the workspace restarts, any tools or files outside the home directory are not persisted. To pre-bake tools into the workspace (e.g., `python3`), modify the container image. Individual developers can also [personalize](https://coder.com/docs/dotfiles) their workspaces with dotfiles. + +## Configuration + +### Template Variables + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `use_kubeconfig` | bool | `false` | Set to `true` if Coder runs outside the Kubernetes cluster | +| `namespace` | string | (required) | Kubernetes namespace for workspaces (must exist) | + +### Workspace Parameters + +Users can configure these options when creating a workspace: + +| Parameter | Options | Default | Mutable | +|-----------|---------|---------|---------| +| CPU | 2, 4, 6, 8 cores | 2 | Yes | +| Memory | 2, 4, 6, 8 GB | 2 GB | Yes | +| Home Disk Size | 1-99999 GB | 10 GB | No | + +## Features + +### Included Applications + +- **code-server**: VS Code in the browser, accessible via the Coder dashboard + +### Workspace Metrics + +The template collects and displays the following metrics in the Coder dashboard: + +- CPU Usage (container and host) +- RAM Usage (container and host) +- Home Disk Usage +- Load Average (host) + +### Resource Management + +- **Requests**: 250m CPU, 512Mi memory (guaranteed minimum) +- **Limits**: Configurable via workspace parameters +- **Pod Anti-Affinity**: Workspaces are spread across nodes when possible + +### Security + +- Runs as non-root user (UID 1000) +- Uses fsGroup 1000 for volume permissions + +## Usage + +1. Create the template in Coder: + ```bash + coder templates create kubernetes --directory . + ``` + +2. Create a workspace from the template: + ```bash + coder create my-workspace --template kubernetes + ``` + +3. Access your workspace via the Coder dashboard or CLI: + ```bash + coder ssh my-workspace + ``` + +## Customization + +This template is designed as a starting point. Common customizations include: + +- **Different base image**: Change the `image` in the container spec +- **Additional environment variables**: Add more `env` blocks +- **GPU support**: Add resource requests/limits for GPUs +- **Init containers**: Add setup containers that run before the main workspace +- **Sidecars**: Add additional containers (databases, proxies, etc.) +- **Node selection**: Add `node_selector` or modify `affinity` rules + +## Troubleshooting + +### Workspace stuck in "Starting" state + +- Check if the namespace exists: `kubectl get ns ` +- Check pod events: `kubectl describe pod -n coder-` +- Verify StorageClass is available: `kubectl get sc` + +### Cannot connect to workspace + +- Ensure the Coder agent is running: check pod logs +- Verify network policies allow traffic to the workspace -This means, when the workspace restarts, any tools or files outside of the home directory are not persisted. To pre-bake tools into the workspace (e.g. `python3`), modify the container image. Alternatively, individual developers can [personalize](https://coder.com/docs/dotfiles) their workspaces with dotfiles. +### Persistent volume not binding -> **Note** -> This template is designed to be a starting point! Edit the Terraform to extend the template to support your use case. \ No newline at end of file +- Check PVC status: `kubectl get pvc -n ` +- Verify StorageClass supports dynamic provisioning From e11dfb038cd7ca703ae038a65fb6a1fb13120062 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 26 Jan 2026 18:23:57 -0800 Subject: [PATCH 19/44] fix: uncomments deploy sequences --- infra/1-click/1-plan-n-deploy.sh | 34 ++++++++++++++++---------------- infra/1-click/2-clean.sh | 18 ++++++++--------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/infra/1-click/1-plan-n-deploy.sh b/infra/1-click/1-plan-n-deploy.sh index 82cf1bb..40fa5c7 100755 --- a/infra/1-click/1-plan-n-deploy.sh +++ b/infra/1-click/1-plan-n-deploy.sh @@ -22,21 +22,21 @@ terraform plan -out=tf.plan \ terraform apply tf.plan cd ../ -# echo "Change directory into '1-setup'." -# cd 1-setup -# terraform plan -out=tf.plan \ -# -var profile=$AWS_PROFILE \ -# -var region=$AWS_REGION \ -# -var domain_name=$DOMAIN_NAME \ -# -var coder_license=$LICENSE -# terraform apply tf.plan -# cd ../ +echo "Change directory into '1-setup'." +cd 1-setup +terraform plan -out=tf.plan \ + -var profile=$AWS_PROFILE \ + -var region=$AWS_REGION \ + -var domain_name=$DOMAIN_NAME \ + -var coder_license=$LICENSE +terraform apply tf.plan +cd ../ -# echo "Change directory into '2-coder'." -# cd 2-coder -# terraform plan -out=tf.plan \ -# -var profile=$AWS_PROFILE \ -# -var region=$AWS_REGION \ -# -var domain_name=$DOMAIN_NAME -# terraform apply tf.plan -# cd ../ \ No newline at end of file +echo "Change directory into '2-coder'." +cd 2-coder +terraform plan -out=tf.plan \ + -var profile=$AWS_PROFILE \ + -var region=$AWS_REGION \ + -var domain_name=$DOMAIN_NAME +terraform apply tf.plan +cd ../ \ No newline at end of file diff --git a/infra/1-click/2-clean.sh b/infra/1-click/2-clean.sh index 4a8a9df..5fae18b 100755 --- a/infra/1-click/2-clean.sh +++ b/infra/1-click/2-clean.sh @@ -30,12 +30,12 @@ terraform plan -destroy -out=tf.plan \ terraform apply tf.plan cd ../ -# echo "Change directory into '0-infra'." -# cd 0-infra -# terraform plan -destroy -out=tf.plan \ -# -var profile=$AWS_PROFILE \ -# -var region=$AWS_REGION \ -# -var azs='["a","c"]' \ -# -var domain_name=$DOMAIN_NAME -# terraform apply tf.plan -# cd ../ \ No newline at end of file +echo "Change directory into '0-infra'." +cd 0-infra +terraform plan -destroy -out=tf.plan \ + -var profile=$AWS_PROFILE \ + -var region=$AWS_REGION \ + -var azs='["a","c"]' \ + -var domain_name=$DOMAIN_NAME +terraform apply tf.plan +cd ../ \ No newline at end of file From bb9aef770aa4c636fcf38be34cef8ae1ba848314 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 26 Jan 2026 18:25:08 -0800 Subject: [PATCH 20/44] fix: quick-fix to deployment --- infra/1-click/0-infra/3-eks.tf | 10 +++++++--- infra/1-click/0-infra/4-bootstrap.tf | 2 +- infra/1-click/1-setup/5-manifests.tf | 6 +++--- infra/1-click/1-setup/6-coder-server.tf | 14 ++++++++++++-- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/infra/1-click/0-infra/3-eks.tf b/infra/1-click/0-infra/3-eks.tf index 7f02f3f..320191b 100644 --- a/infra/1-click/0-infra/3-eks.tf +++ b/infra/1-click/0-infra/3-eks.tf @@ -72,9 +72,13 @@ module "eks" { # K8s Labels - https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group#labels-1 labels = local.labels_system_node - # taints = { - # "CriticalAddonsOnly" - # } + taints = { + "system-only" = { + key = "CriticalAddonsOnly" + value = "true" + effect = "NO_EXECUTE" + } + } instance_types = ["t3.large"] capacity_type = "ON_DEMAND" diff --git a/infra/1-click/0-infra/4-bootstrap.tf b/infra/1-click/0-infra/4-bootstrap.tf index 8baca5b..25557c8 100644 --- a/infra/1-click/0-infra/4-bootstrap.tf +++ b/infra/1-click/0-infra/4-bootstrap.tf @@ -12,7 +12,7 @@ module "karpenter" { namespace = "karpenter" chart_version = "1.8.4" node_selector = local.labels_system_node - + iam_role_use_name_prefix = true node_iam_role_use_name_prefix = true replicas = 2 diff --git a/infra/1-click/1-setup/5-manifests.tf b/infra/1-click/1-setup/5-manifests.tf index 8f0e8cb..582dc6b 100644 --- a/infra/1-click/1-setup/5-manifests.tf +++ b/infra/1-click/1-setup/5-manifests.tf @@ -95,7 +95,7 @@ locals { node_expires_after = "Never" disruption_consolidation_policy = "WhenEmpty" disruption_consolidate_after = "30m" - taints = null + taints = [] } } } @@ -161,7 +161,7 @@ resource "kubernetes_manifest" "nodepool" { },{ key = "node.kubernetes.io/instance-type" operator = "In" - values = ["t3a.xlarge"] + values = ["t3a.medium"] }] nodeClassRef = { group = "karpenter.k8s.aws" @@ -193,7 +193,7 @@ resource "kubernetes_manifest" "default-sc" { "storageclass.kubernetes.io/is-default-class" = "true" } } - provisioner = "ebs.csi.eks.amazonaws.com" + provisioner = "ebs.csi.aws.com" volumeBindingMode = "WaitForFirstConsumer" parameters = { type = "gp3" diff --git a/infra/1-click/1-setup/6-coder-server.tf b/infra/1-click/1-setup/6-coder-server.tf index c14ae01..03f69d1 100644 --- a/infra/1-click/1-setup/6-coder-server.tf +++ b/infra/1-click/1-setup/6-coder-server.tf @@ -43,7 +43,7 @@ variable "coder_admin_password" { resource "aws_iam_user" "bedrock" { name = "bedrock-access" - path = "/${var.cluster_name}/${data.aws_region.this.region}/" + path = "/${local.normalized_domain_name}/${data.aws_region.this.region}/" } resource "aws_iam_user_policy_attachment" "bedrock" { @@ -80,12 +80,22 @@ module "coder-server" { CODER_AIBRIDGE_BEDROCK_REGION = data.aws_region.this.region CODER_AIBRIDGE_BEDROCK_ACCESS_KEY = aws_iam_access_key.bedrock.id CODER_AIBRIDGE_BEDROCK_ACCESS_KEY_SECRET = aws_iam_access_key.bedrock.secret + CODER_EXPERIMENTS = "oauth2,mcp-server-http" } # Use this instead of external provisioners. DNS might not propagate fast enough for Coder to be "reachable". coder_builtin_provisioner_count = 4 # coder_github_allowed_orgs = var.coder_github_allowed_orgs + resource_request = { + cpu = "1000m" + memory = "2Gi" + } + resource_limit = { + cpu = "1000m" + memory = "2Gi" + } + ssl_cert_config = { name = var.domain_name caissuer = kubernetes_manifest.default-issuer.manifest.metadata.name @@ -103,7 +113,7 @@ module "coder-server" { "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" - "external-dns.alpha.kubernetes.io/hostname" = "${var.domain_name}" + "external-dns.alpha.kubernetes.io/hostname" = "${var.domain_name},*.${var.domain_name}" "external-dns.alpha.kubernetes.io/ttl" = 30 } From d67aa9684e949632e5b90e3a2626b1eeed99be91 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 26 Jan 2026 18:26:36 -0800 Subject: [PATCH 21/44] fix: README adjustment --- infra/1-click/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/1-click/README.md b/infra/1-click/README.md index fa594dc..adb8913 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -2,7 +2,7 @@ - [Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) - [AWS Account](https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-creating.html) + [CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) -- [Domain in Route53]((https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section)) +- [Domain in Route53](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) - [AWS Bedrock + Anthropic Agreement Completed](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) > [!IMPORTANT] From af454e27472a7f0f302d5215ef5561a3954c137f Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 26 Jan 2026 21:20:36 -0800 Subject: [PATCH 22/44] fix: adjust region env var --- infra/1-click/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/1-click/README.md b/infra/1-click/README.md index adb8913..48ee36d 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -27,7 +27,7 @@ ```env CODER_AWS_PROFILE=default (Change profile if needed) -CODER_AWS_PROFILE=us-east-2 (Change region if needed) +CODER_AWS_REGION=us-east-2 (Change region if needed) CODER_DOMAIN_NAME=put.your.domain.here.com CODER_LICENSE=abcde1234..... (Optional) ``` From f7c2f0bf83bd987b6b7f9e0032de22123164c6b0 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Wed, 4 Feb 2026 06:10:54 -0800 Subject: [PATCH 23/44] feat: lots of a changes --- infra/1-click/0-infra/0-vpc.tf | 34 +- infra/1-click/0-infra/1-db.tf | 97 +- infra/1-click/0-infra/2-s3.tf | 7 +- infra/1-click/0-infra/3-eks.tf | 42 +- infra/1-click/0-infra/4-bootstrap.tf | 70 +- infra/1-click/0-infra/main.tf | 40 +- infra/1-click/1-plan-n-deploy.sh | 50 +- infra/1-click/1-setup/5-manifests.tf | 425 +++++---- infra/1-click/1-setup/6-coder-server.tf | 580 +++++++----- infra/1-click/1-setup/main.tf | 171 +++- infra/1-click/1-setup/scripts/add-license.sh | 22 +- infra/1-click/1-setup/scripts/first-user.sh | 50 -- infra/1-click/1-setup/scripts/nodeconfig.yaml | 18 + infra/1-click/2-clean.sh | 54 +- infra/1-click/2-coder/7-license.tf | 26 + infra/1-click/2-coder/7-orgs.tf | 108 --- infra/1-click/2-coder/8-templates.tf | 111 ++- infra/1-click/2-coder/main.tf | 71 +- infra/1-click/2-coder/scripts/login.sh | 35 - .../templates/kubernetes-claude/README.md | 2 +- .../kubernetes-claude/boundary-config.yaml | 243 +++++ .../templates/kubernetes-claude/main.tf | 126 ++- .../2-coder/templates/kubernetes/main.tf | 56 +- infra/1-click/README.md | 39 +- modules/k8s/bootstrap/cert-manager/main.tf | 289 ++---- modules/k8s/bootstrap/coder-server/main.tf | 845 +++++++----------- modules/k8s/bootstrap/ebs-controller/main.tf | 12 +- modules/k8s/bootstrap/external-dns/main.tf | 127 ++- .../k8s/bootstrap/external-secrets/main.tf | 18 +- modules/k8s/bootstrap/karpenter/main.tf | 10 +- modules/k8s/bootstrap/lb-controller/main.tf | 10 +- modules/k8s/bootstrap/metrics-server/main.tf | 12 +- modules/k8s/bootstrap/monitoring/main.tf | 596 ++++++++++++ 33 files changed, 2767 insertions(+), 1629 deletions(-) delete mode 100755 infra/1-click/1-setup/scripts/first-user.sh create mode 100644 infra/1-click/1-setup/scripts/nodeconfig.yaml create mode 100644 infra/1-click/2-coder/7-license.tf delete mode 100644 infra/1-click/2-coder/7-orgs.tf delete mode 100755 infra/1-click/2-coder/scripts/login.sh create mode 100644 infra/1-click/2-coder/templates/kubernetes-claude/boundary-config.yaml create mode 100644 modules/k8s/bootstrap/monitoring/main.tf diff --git a/infra/1-click/0-infra/0-vpc.tf b/infra/1-click/0-infra/0-vpc.tf index 2ab42d7..efab265 100644 --- a/infra/1-click/0-infra/0-vpc.tf +++ b/infra/1-click/0-infra/0-vpc.tf @@ -3,7 +3,7 @@ ## variable "azs" { - type = list(string) + type = list(string) default = ["a", "b", "c"] } @@ -14,7 +14,7 @@ locals { } # Internet-Facing LB - https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/subnet_discovery/#subnet-role-tag tags_public_lb = { - "kubernetes.io/role/elb" = 1 + "kubernetes.io/role/elb" = 1 } tags_public_subnet = merge( local.tags_lb_subnet_discovery, @@ -22,28 +22,29 @@ locals { ) # Use variable to configure AZ just in case 1 AZ isn't accessible # https://repost.aws/questions/QUgdQev4KETKG_Bwev9tMtRQ/is-it-possible-to-enable-3rd-availability-zone-in-us-west-1#AN9eAH55FwTC-NSSp-FjIfTQ - availability_zones = [ for az in var.azs : "${var.region}${az}" ] - public_subnet_cidrs = ["10.0.8.0/21", "10.0.4.0/22", "10.0.0.0/22"] + availability_zones = [for az in var.azs : "${var.region}${az}"] + public_subnet_cidrs = ["10.0.8.0/21", "10.0.4.0/22", "10.0.0.0/22"] general_subnet_cidrs = ["10.0.64.0/20", "10.0.80.0/20", "10.0.96.0/20"] - system_subnet_cidrs = ["10.0.16.0/20", "10.0.32.0/20", "10.0.48.0/20"] + system_subnet_cidrs = ["10.0.16.0/20", "10.0.32.0/20", "10.0.48.0/20"] } module "vpc" { - source = "terraform-aws-modules/vpc/aws" + source = "terraform-aws-modules/vpc/aws" version = "~> 6.6.0" - name = "${var.name}-${local.normalized_domain_name}" + name = "${var.name}-${local.normalized_domain_name}" enable_nat_gateway = false enable_dns_hostnames = true - cidr = "10.0.0.0/16" - azs = local.availability_zones - + cidr = "10.0.0.0/16" + azs = local.availability_zones + ## # Handle public subnets via the VPC module. ## - public_subnets = [ for index, _ in var.azs : local.public_subnet_cidrs[index] ] + public_subnet_suffix = "public" + public_subnets = [for index, _ in var.azs : local.public_subnet_cidrs[index]] public_subnet_tags = local.tags_public_subnet tags = local.tags_global @@ -52,10 +53,11 @@ module "vpc" { module "nat-instance" { source = "RaJiska/fck-nat/aws" - name = "${var.name}-${local.normalized_domain_name}" + name = "${var.name}-${local.normalized_domain_name}" vpc_id = module.vpc.vpc_id subnet_id = module.vpc.public_subnets[0] + # https://fck-nat.dev/stable/choosing_an_instance_size/ instance_type = "c6gn.medium" ha_mode = true # Enables high-availability mode # eip_allocation_ids = ["eipalloc-abc1234"] # Allocation ID of an existing EIP @@ -74,7 +76,7 @@ module "nat-instance" { locals { # Karpenter Subnet Discovery - https://karpenter.sh/v1.0/concepts/nodeclasses/#specsubnetselectorterms tags_kptr_subnet_discovery = { - "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" + "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" } tags_private_subnet = merge( local.tags_lb_subnet_discovery, @@ -95,7 +97,7 @@ module "general-subnet" { cidr_block = local.general_subnet_cidrs[count.index] availability_zone = local.availability_zones[count.index] subnet_tags = local.tags_private_subnet - tags = local.tags_global + tags = local.tags_global } ## @@ -113,12 +115,12 @@ module "system-subnet" { cidr_block = local.system_subnet_cidrs[count.index] availability_zone = local.availability_zones[count.index] subnet_tags = local.tags_private_subnet - tags = local.tags_global + tags = local.tags_global } locals { private_subnet_ids = concat( - module.general-subnet.*.subnet_id, + module.general-subnet.*.subnet_id, module.system-subnet.*.subnet_id ) public_subnet_ids = concat([], module.vpc.public_subnets) diff --git a/infra/1-click/0-infra/1-db.tf b/infra/1-click/0-infra/1-db.tf index 479f2d3..18a1c58 100644 --- a/infra/1-click/0-infra/1-db.tf +++ b/infra/1-click/0-infra/1-db.tf @@ -15,23 +15,10 @@ variable "coder_password" { default = "th1s1sn0tas3cur3pass0wrd" } -variable "litellm_username" { - description = "LiteLLM DB's username." - type = string - default = "litellm" -} - -variable "litellm_password" { - description = "LiteLLM DB's password." - type = string - sensitive = true - default = "th1s1sn0tas3cur3pass0wrd" -} - variable "grafana_username" { description = "Grafana DB's username." type = string - default = "grafana" + default = "grafana" } variable "grafana_password" { @@ -73,24 +60,30 @@ resource "aws_db_subnet_group" "coder" { } } +resource "time_static" "coder_snapshot" { + triggers = { + run_on_ip_change = "${var.name}-${local.normalized_domain_name}-coder" + } +} + resource "aws_db_instance" "coder" { identifier = "${var.name}-${local.normalized_domain_name}-coder" - instance_class = "db.t4g.large" + instance_class = "db.t4g.medium" allocated_storage = 50 engine = "postgres" engine_version = "15.12" - # backup_retention_period = 7 - username = var.coder_username - password = var.coder_password - db_name = "coder" - db_subnet_group_name = aws_db_subnet_group.coder.name - vpc_security_group_ids = [aws_security_group.postgres.id] - publicly_accessible = false - skip_final_snapshot = true - # final_snapshot_identifier = "coder-final" + username = var.coder_username + password = var.coder_password + db_name = "coder" + db_subnet_group_name = aws_db_subnet_group.coder.name + vpc_security_group_ids = [aws_security_group.postgres.id] + publicly_accessible = false + snapshot_identifier = null + skip_final_snapshot = false + final_snapshot_identifier = "coder-${replace(time_static.coder_snapshot.rfc3339, ":", "-")}" tags = { - Name = "coder" + Name = "${var.name}-${local.normalized_domain_name}-coder" } lifecycle { ignore_changes = [ @@ -99,48 +92,30 @@ resource "aws_db_instance" "coder" { } } -resource "aws_db_instance" "litellm" { - identifier = "${var.name}-${local.normalized_domain_name}-litellm" - instance_class = "db.t4g.medium" - allocated_storage = 50 - engine = "postgres" - engine_version = "15.12" - username = var.litellm_username - password = var.litellm_password - db_name = "litellm" - db_subnet_group_name = aws_db_subnet_group.coder.name - vpc_security_group_ids = [aws_security_group.postgres.id] - publicly_accessible = false - skip_final_snapshot = true - # final_snapshot_identifier = "litellm-final" - - tags = { - Name = "litellm" - } - lifecycle { - ignore_changes = [ - snapshot_identifier - ] +resource "time_static" "grafana_snapshot" { + triggers = { + run_on_ip_change = "${var.name}-${local.normalized_domain_name}-grafana" } } resource "aws_db_instance" "grafana" { - identifier = "${var.name}-${local.normalized_domain_name}-grafana" - instance_class = "db.t4g.large" - allocated_storage = 50 - engine = "postgres" - engine_version = "15.12" - username = var.grafana_username - password = var.grafana_password - db_name = "grafana" - db_subnet_group_name = aws_db_subnet_group.coder.name - vpc_security_group_ids = [aws_security_group.postgres.id] - publicly_accessible = false - skip_final_snapshot = true - # final_snapshot_identifier = "grafana-final" + identifier = "${var.name}-${local.normalized_domain_name}-grafana" + instance_class = "db.t4g.medium" + allocated_storage = 50 + engine = "postgres" + engine_version = "15.12" + username = var.grafana_username + password = var.grafana_password + db_name = "grafana" + db_subnet_group_name = aws_db_subnet_group.coder.name + vpc_security_group_ids = [aws_security_group.postgres.id] + publicly_accessible = false + snapshot_identifier = null + skip_final_snapshot = false + final_snapshot_identifier = "grafana-${replace(time_static.coder_snapshot.rfc3339, ":", "-")}" tags = { - Name = "grafana" + Name = "${var.name}-${local.normalized_domain_name}-grafana" } lifecycle { ignore_changes = [ diff --git a/infra/1-click/0-infra/2-s3.tf b/infra/1-click/0-infra/2-s3.tf index f0aa222..2798bc1 100644 --- a/infra/1-click/0-infra/2-s3.tf +++ b/infra/1-click/0-infra/2-s3.tf @@ -6,18 +6,13 @@ # Loki Inputs ## -variable "loki_s3_bucket_name" { - type = string - default = "" -} - variable "loki_s3_bucket_tags" { type = map(string) default = {} } resource "aws_s3_bucket" "loki" { - bucket = var.loki_s3_bucket_name == "" ? "${var.name}-${local.normalized_domain_name}-granafa-logs" : var.loki_s3_bucket_name + bucket = "${var.name}-${local.normalized_domain_name}-grafana" tags = var.loki_s3_bucket_tags } diff --git a/infra/1-click/0-infra/3-eks.tf b/infra/1-click/0-infra/3-eks.tf index 320191b..7ec6bf8 100644 --- a/infra/1-click/0-infra/3-eks.tf +++ b/infra/1-click/0-infra/3-eks.tf @@ -10,6 +10,10 @@ data "aws_iam_policy_document" "sts" { } } +## +# Use for troubleshooting K8s nodes in case of issues. +## + resource "aws_iam_policy" "sts" { name_prefix = "${var.name}-${local.normalized_domain_name}-sts-" path = "/" @@ -26,6 +30,15 @@ locals { labels_system_node = { "scheduling.coder.com/pool" = "system" } + taints_system = { + key = "CriticalAddonsOnly" + value = "true" + effect = "NO_EXECUTE" + } + tolerations_system = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] } module "eks" { @@ -37,11 +50,10 @@ module "eks" { # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster#vpc_config-1 # https://docs.aws.amazon.com/eks/latest/userguide/network-reqs.html#network-requirements-subnets subnet_ids = toset(concat( - # local.public_subnet_ids, local.private_subnet_ids )) - name = "${var.name}-${local.normalized_domain_name}" + name = "${var.name}-${local.normalized_domain_name}" kubernetes_version = "1.34" endpoint_public_access = true @@ -50,11 +62,11 @@ module "eks" { create_security_group = true create_node_security_group = true create_iam_role = true - node_security_group_tags = local.tags_kptr_sg_discovery + node_security_group_tags = local.tags_kptr_sg_discovery compute_config = { # Disables EKS Auto Mode. Manually handle scaling via Karpenter - enabled = false + enabled = false } attach_encryption_policy = false @@ -69,29 +81,25 @@ module "eks" { min_size = 0 max_size = 10 desired_size = 2 # Ignored after creation. Override from AWS Console as needed. - + # K8s Labels - https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group#labels-1 labels = local.labels_system_node taints = { - "system-only" = { - key = "CriticalAddonsOnly" - value = "true" - effect = "NO_EXECUTE" - } + "system-only" = local.taints_system } instance_types = ["t3.large"] capacity_type = "ON_DEMAND" - volume_size = 50 - + volume_size = 50 + iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" STSAssumeRole = aws_iam_policy.sts.arn } metadata_options = { - http_endpoint = "enabled" + http_endpoint = "enabled" http_put_response_hop_limit = 2 - http_tokens = "required" + http_tokens = "required" } # System Nodes should not be public @@ -113,6 +121,12 @@ module "eks" { nodeAgent = { enablePolicyEventLogs = "true" } + env = { + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + WARM_IP_TARGET = "0" + AWS_VPC_K8S_CNI_LOGLEVEL = "DEBUG" + } }) } } diff --git a/infra/1-click/0-infra/4-bootstrap.tf b/infra/1-click/0-infra/4-bootstrap.tf index 25557c8..e65df15 100644 --- a/infra/1-click/0-infra/4-bootstrap.tf +++ b/infra/1-click/0-infra/4-bootstrap.tf @@ -3,19 +3,21 @@ ## module "karpenter" { - source = "../../../modules/k8s/bootstrap/karpenter" + + source = "../../../modules/k8s/bootstrap/karpenter" cluster_name = module.eks.cluster_name cluster_oidc_provider_arn = module.eks.oidc_provider_arn - cluster_oidc_provider = module.eks.oidc_provider + cluster_oidc_provider = module.eks.oidc_provider namespace = "karpenter" chart_version = "1.8.4" node_selector = local.labels_system_node - - iam_role_use_name_prefix = true + tolerations = local.tolerations_system + + iam_role_use_name_prefix = true node_iam_role_use_name_prefix = true - replicas = 2 + replicas = 2 karpenter_controller_role_policies = { "AmazonEFSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" } @@ -23,31 +25,37 @@ module "karpenter" { module "metrics-server" { - # depends_on = [ module.karpenter ] - source = "../../../modules/k8s/bootstrap/metrics-server" namespace = "metrics-server" chart_version = "3.13.0" node_selector = local.labels_system_node + tolerations = local.tolerations_system } module "cert-manager" { - # depends_on = [ module.karpenter ] - source = "../../../modules/k8s/bootstrap/cert-manager" cluster_name = module.eks.cluster_name cluster_oidc_provider_arn = module.eks.oidc_provider_arn namespace = "cert-manager" helm_version = "v1.18.2" - use_cloudflare = false - use_route53 = true - create_default_cluster_issuer = false + tolerations = local.tolerations_system + + cf_config = { + enabled = var.cf_config.enabled + email = var.cf_config.email + token = var.cf_config.token + } + r53_config = { + enabled = var.r53_config.enabled + role_name = "crt-mgr" + policy_name = "crt-mgr" + } } -module "lb-controller" { +module "lb-ctrl" { depends_on = [ module.cert-manager ] @@ -57,18 +65,18 @@ module "lb-controller" { namespace = "lb-ctrl" chart_version = "1.13.2" + node_selector = local.labels_system_node + tolerations = local.tolerations_system + enable_cert_manager = true - vpc_id = module.vpc.vpc_id + vpc_id = module.vpc.vpc_id service_target_eni_sg_tags = { Name = module.eks.cluster_name } create_alb_class = false - node_selector = local.labels_system_node } -module "ebs-controller" { - - # depends_on = [ module.cert-manager ] +module "ebs-ctrl" { source = "../../../modules/k8s/bootstrap/ebs-controller" cluster_name = module.eks.cluster_name @@ -77,24 +85,37 @@ module "ebs-controller" { namespace = "ebs-ctrl" chart_version = "2.22.1" node_selector = local.labels_system_node + tolerations = local.tolerations_system replace = true } -module "external-dns" { +module "ext-dns" { + count = var.use_ext_dns ? 1 : 0 depends_on = [ module.cert-manager ] source = "../../../modules/k8s/bootstrap/external-dns" cluster_name = module.eks.cluster_name cluster_oidc_provider_arn = module.eks.oidc_provider_arn - namespace = "external-dns" + namespace = "ext-dns" chart_version = "1.20.0" - domain_name = var.domain_name node_selector = local.labels_system_node + tolerations = local.tolerations_system + + cf_config = { + enabled = var.cf_config.enabled + email = var.cf_config.email + token = var.cf_config.token + } + r53_config = { + enabled = var.r53_config.enabled + role_name = "crt-mgr" + policy_name = "crt-mgr" + } } -module "external-secrets" { +module "ext-sec" { depends_on = [ module.cert-manager ] @@ -102,6 +123,7 @@ module "external-secrets" { cluster_name = module.eks.cluster_name cluster_oidc_provider_arn = module.eks.oidc_provider_arn - namespace = "external-secrets" - chart_version = "1.2.1" + namespace = "ext-sec" + chart_version = "1.2.1" + tolerations = local.tolerations_system } \ No newline at end of file diff --git a/infra/1-click/0-infra/main.tf b/infra/1-click/0-infra/main.tf index 34d569d..f70965e 100644 --- a/infra/1-click/0-infra/main.tf +++ b/infra/1-click/0-infra/main.tf @@ -13,7 +13,6 @@ terraform { source = "hashicorp/kubernetes" } } - # backend "s3" {} } ## @@ -33,13 +32,13 @@ variable "name" { } variable "profile" { - type = string + type = string default = "default" } variable "domain_name" { description = "Your Coder domain name (i.e. coder-example.com)" - type = string + type = string } data "aws_eks_cluster_auth" "coder" { @@ -67,5 +66,38 @@ provider "kubernetes" { locals { normalized_domain_name = split(".", var.domain_name)[0] - tags_global = {} + tags_global = {} +} + +# If R53 enabled, then fetch service account from cert-manager for IAM Role +variable "r53_config" { + description = "Enable to use Route53 as a DNS01 provider for ACME challenges." + type = object({ + enabled = bool + }) + default = { + enabled = false + } +} + +# If CF enabled, then fetch secret from cert-manager for token +variable "cf_config" { + description = "Enable to use CloudFlare as a DNS01 provider for ACME challenges." + type = object({ + enabled = bool + email = string + token = string + }) + default = { + enabled = false + email = "" + token = "" + } + sensitive = true +} + +variable "use_ext_dns" { + description = "Toggle the K8s 'external-dns' addon. Disable in-case you want to manage DNS records yourself." + type = bool + default = true } \ No newline at end of file diff --git a/infra/1-click/1-plan-n-deploy.sh b/infra/1-click/1-plan-n-deploy.sh index 40fa5c7..c629165 100755 --- a/infra/1-click/1-plan-n-deploy.sh +++ b/infra/1-click/1-plan-n-deploy.sh @@ -2,11 +2,36 @@ set -ae -o pipefail +source coder.env + +AWS_AZS="${CODER_AWS_AZS:-[\"a\",\"c\"]}" + AWS_PROFILE="${CODER_AWS_PROFILE:-default}" AWS_REGION="${CODER_AWS_REGION:-us-east-2}" DOMAIN_NAME="${CODER_DOMAIN_NAME:-}" LICENSE="${CODER_LICENSE:-}" +USE_EXTERN_DNS="${CODER_USE_EXTERN_DNS:-true}" + +USE_R53="${CODER_USE_R53:-true}" + +USE_CF="${CODER_USE_CF:-false}" +CF_TOKEN="${CODER_CF_TOKEN:-}" +CF_EMAIL="${CODER_CF_EMAIL:-}" + +SET_REC_USE_R53=false; ! $USE_EXTERN_DNS && $USE_R53 && SET_REC_USE_R53=true +SET_REC_USE_CF=false; ! $USE_EXTERN_DNS && $USE_CF && SET_REC_USE_CF=true + +CODER_DB_USERNAME="${CODER_DB_USERNAME:-coder}" +CODER_DB_PASSWORD="${CODER_DB_PASSWORD:-th1s1sn0tas3cur3pass0wrd}" + +GRAFANA_DB_USERNAME="${CODER_GRAFANA_DB_PASSWORD:-grafana}" +GRAFANA_DB_PASSWORD="${CODER_GRAFANA_DB_PASSWORD:-th1s1sn0tas3cur3pass0wrd}" + +CODER_USERNAME="${CODER_USERNAME:-admin}" +CODER_EMAIL="${CODER_EMAIL:-admin@coder.com}" +CODER_PASSWORD="${CODER_PASSWORD:-Th1s1sN0TS3CuR3!!}" + if [ -z "${DOMAIN_NAME}" ]; then echo "A domain name is required! Be sure to register or use an existing one from Route53!" exit 1; @@ -18,7 +43,14 @@ terraform plan -out=tf.plan \ -var profile=$AWS_PROFILE \ -var region=$AWS_REGION \ -var domain_name=$DOMAIN_NAME \ - -var azs='["a","c"]' + -var azs="$AWS_AZS" \ + -var coder_username=$CODER_DB_USERNAME \ + -var coder_password=$CODER_DB_PASSWORD \ + -var grafana_username=$GRAFANA_DB_USERNAME \ + -var grafana_password=$GRAFANA_DB_PASSWORD \ + -var use_ext_dns=$USE_EXTERN_DNS \ + -var cf_config="{\"enabled\":\"$USE_CF\",\"email\":\"$CF_EMAIL\",\"token\":\"$CF_TOKEN\"}" \ + -var r53_config="{\"enabled\":\"$USE_R53\"}" terraform apply tf.plan cd ../ @@ -28,7 +60,16 @@ terraform plan -out=tf.plan \ -var profile=$AWS_PROFILE \ -var region=$AWS_REGION \ -var domain_name=$DOMAIN_NAME \ - -var coder_license=$LICENSE + -var azs="$AWS_AZS" \ + -var coder_license=$LICENSE \ + -var coder_username=$CODER_DB_USERNAME \ + -var coder_password=$CODER_DB_PASSWORD \ + -var coder_admin_email=$CODER_EMAIL \ + -var coder_admin_username=$CODER_USERNAME \ + -var coder_admin_password=$CODER_PASSWORD \ + -var auto_set_record="{\"use_cf\":\"$SET_REC_USE_CF\",\"cf_token\":\"$CF_TOKEN\",\"use_r53\":\"$SET_REC_USE_R53\"}" \ + -var cf_config="{\"enabled\":\"$USE_CF\",\"email\":\"$CF_EMAIL\"}" \ + -var r53_config="{\"enabled\":\"$USE_R53\"}" terraform apply tf.plan cd ../ @@ -37,6 +78,9 @@ cd 2-coder terraform plan -out=tf.plan \ -var profile=$AWS_PROFILE \ -var region=$AWS_REGION \ - -var domain_name=$DOMAIN_NAME + -var domain_name=$DOMAIN_NAME \ + -var coder_license=$LICENSE \ + -var coder_admin_email=$CODER_EMAIL \ + -var coder_admin_password=$CODER_PASSWORD terraform apply tf.plan cd ../ \ No newline at end of file diff --git a/infra/1-click/1-setup/5-manifests.tf b/infra/1-click/1-setup/5-manifests.tf index 582dc6b..6bbca54 100644 --- a/infra/1-click/1-setup/5-manifests.tf +++ b/infra/1-click/1-setup/5-manifests.tf @@ -9,174 +9,232 @@ data "kubernetes_service_account_v1" "kptr" { metadata { - name = "node-role" + name = "node-role" namespace = "karpenter" } } + locals { - nodeclass_configs = { - "coder-server" = { - user_data = "" - block_device_mappings = [] - } - "coder-workspace" = { - user_data = <<-EOF - apiVersion: node.eks.aws/v1alpha1 - kind: NodeConfig - spec: - kubelet: - config: - registryPullQPS: 30 - EOF - block_device_mappings = [{ - deviceName = "/dev/xvda" - ebs = { - volumeSize = "500Gi" - volumeType = "gp3" - encrypted = false - deleteOnTermination = true - } - }] - } - "coder-provisioner" = { - user_data = "" - block_device_mappings = [] + prefetch-script = templatefile("${path.module}/scripts/prefetch.sh.tftpl", { + IMAGES = join(" ", ["docker.io/codercom/enterprise-base:ubuntu"]) + PRE_SCRIPT = "" + POST_SCRIPT = "" + }) + nodeclass_configs = { + "coder" = { + user_data = <<-EOF + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="//" + + --// + Content-Type: application/node.eks.aws + + ${file("${path.module}/scripts/nodeconfig.yaml")} + + --//-- + EOF + block_device_mappings = [{ + deviceName = "/dev/xvda" + ebs = { + volumeSize = "500Gi" + volumeType = "gp3" + encrypted = false + deleteOnTermination = true } + }] } + } } resource "kubernetes_manifest" "nodeclass" { - for_each = local.nodeclass_configs + for_each = local.nodeclass_configs - manifest = { - apiVersion = "karpenter.k8s.aws/v1" - kind = "EC2NodeClass" - metadata = { - name = each.key + manifest = { + apiVersion = "karpenter.k8s.aws/v1" + kind = "EC2NodeClass" + metadata = { + name = each.key + } + spec = { + role = data.kubernetes_service_account_v1.kptr.metadata[0].annotations["eks.amazonaws.com/role-arn"] + amiSelectorTerms = [{ + alias = "al2023@latest" + }] + subnetSelectorTerms = [{ + tags = { + "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" } - spec = { - role = data.kubernetes_service_account_v1.kptr.metadata[0].annotations["eks.amazonaws.com/role-arn"] - amiSelectorTerms = [{ - alias = "al2023@latest" - }] - subnetSelectorTerms = [{ - tags = { - "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" - } - }] - securityGroupSelectorTerms = [{ - tags = { - "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" - } - }] - blockDeviceMappings = each.value.block_device_mappings - userData = each.value.user_data + }] + securityGroupSelectorTerms = [{ + tags = { + "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" } + }] + blockDeviceMappings = each.value.block_device_mappings + userData = each.value.user_data } + } } locals { - nodepool_configs = { - "coder-server" = { - node_expires_after = "Never" - disruption_consolidation_policy = "WhenEmpty" - disruption_consolidate_after = "1m" - taints = null - } - "coder-provisioner" = { - node_expires_after = "Never" - disruption_consolidation_policy = "WhenEmpty" - disruption_consolidate_after = "1m" - taints = null - } - "coder-workspace" = { - node_expires_after = "Never" - disruption_consolidation_policy = "WhenEmpty" - disruption_consolidate_after = "30m" - taints = [] - } + nodepool_configs = { + "coder" = { + node_expires_after = "Never" + disruption_consolidation_policy = "WhenEmpty" + disruption_consolidate_after = "1m" + instance_type = "t3a.medium" + taints = [] } + } } resource "kubernetes_manifest" "nodepool" { - depends_on = [ kubernetes_manifest.nodeclass ] + depends_on = [kubernetes_manifest.nodeclass] + for_each = local.nodepool_configs - for_each = local.nodepool_configs + field_manager { + force_conflicts = true + } - field_manager { - force_conflicts = true + wait { + condition { + type = "Ready" + status = "True" } + } - wait { - condition { - type = "Ready" - status = "True" + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "karpenter.sh/v1" + kind = "NodePool" + metadata = { + name = each.key + } + spec = { + template = { + metadata = { + labels = { + "node.coder.io/instance" = "coder-v2" + "node.coder.io/managed-by" = "karpenter" + "node.coder.io/name" = "coder" + "node.coder.io/part-of" = "coder" + "node.coder.io/used-for" = each.key + } + } + spec = { + taints = each.value.taints == null ? [{ + key = "dedicated" + value = each.key + effect = "NoSchedule" + }] : each.value.taints + requirements = [{ + key = "kubernetes.io/arch" + operator = "In" + values = ["amd64"] + }, { + key = "kubernetes.io/os" + operator = "In" + values = ["linux"] + }, { + key = "kubernetes.sh/capacity-type" + operator = "In" + values = ["spot", "on-demand"] + }, { + key = "node.kubernetes.io/instance-type" + operator = "In" + values = [ each.value.instance_type ] + }] + nodeClassRef = { + group = "karpenter.k8s.aws" + kind = "EC2NodeClass" + name = each.key + } + expireAfter = each.value.node_expires_after } + } + disruption = { + consolidationPolicy = each.value.disruption_consolidation_policy + consolidateAfter = each.value.disruption_consolidate_after + } + } + } +} + +## +# Image Prefetch DaemonSet. Add images to warm new Coder nodes with workspace image. +## + +resource "kubernetes_daemon_set_v1" "img-fetch" { + + depends_on = [kubernetes_manifest.nodepool] + for_each = local.nodepool_configs + + metadata { + name = "imgs-for-${each.key}" + namespace = "kube-system" + labels = { + "app.kubernetes.io/name" = "img-fetch" + "app.kubernetes.io/part-of" = "coder-workspaces" } + } - timeouts { - create = "10m" - update = "10m" - delete = "30s" + spec { + selector { + match_labels = { + "app.kubernetes.io/name" = "img-fetch" + } } - manifest = { - apiVersion = "karpenter.sh/v1" - kind = "NodePool" - metadata = { - name = each.key + template { + metadata { + labels = { + "app.kubernetes.io/name" = "img-fetch" } - spec = { - template = { - metadata = { - labels = { - "node.coder.io/instance" = "coder-v2" - "node.coder.io/managed-by" = "karpenter" - "node.coder.io/name" = "coder" - "node.coder.io/part-of" = "coder" - "node.coder.io/used-for" = each.key - } - } - spec = { - taints = each.value.taints == null ? [{ - key = "dedicated" - value = each.key - effect = "NoSchedule" - }] : each.value.taints - requirements = [{ - key = "kubernetes.io/arch" - operator = "In" - values = ["amd64"] - }, { - key = "kubernetes.io/os" - operator = "In" - values = ["linux"] - }, { - key = "kubernetes.sh/capacity-type" - operator = "In" - values = ["spot", "on-demand"] - },{ - key = "node.kubernetes.io/instance-type" - operator = "In" - values = ["t3a.medium"] - }] - nodeClassRef = { - group = "karpenter.k8s.aws" - kind = "EC2NodeClass" - name = each.key - } - expireAfter = each.value.node_expires_after - } + } + + spec { + node_selector = kubernetes_manifest.nodepool[each.key].manifest.spec.template.metadata.labels + + toleration { + key = "dedicated" + value = each.key + effect = "NoSchedule" + } + + termination_grace_period_seconds = 5 + + init_container { + name = "enterprise-base" + image = "docker.io/codercom/enterprise-base:ubuntu" + command = [] + } + + container { + name = "pause" + image = "registry.k8s.io/pause:3.9" + + resources { + requests = { + cpu = "1m" + memory = "1Mi" } - disruption = { - consolidationPolicy = each.value.disruption_consolidation_policy - consolidateAfter = each.value.disruption_consolidate_after + limits = { + cpu = "10m" + memory = "10Mi" } + } } + } } + } } ## @@ -186,17 +244,17 @@ resource "kubernetes_manifest" "nodepool" { resource "kubernetes_manifest" "default-sc" { manifest = { apiVersion = "storage.k8s.io/v1" - kind = "StorageClass" + kind = "StorageClass" metadata = { name = "default" annotations = { "storageclass.kubernetes.io/is-default-class" = "true" } } - provisioner = "ebs.csi.aws.com" + provisioner = "ebs.csi.aws.com" volumeBindingMode = "WaitForFirstConsumer" parameters = { - type = "gp3" + type = "gp3" encrypted = "true" } } @@ -206,14 +264,80 @@ resource "kubernetes_manifest" "default-sc" { # Cert-Manager ClusterIssuer for CA ## +# If R53 enabled, then fetch service account from cert-manager for IAM Role +variable "r53_config" { + description = "Enable to use Route53 as a DNS01 provider for ACME challenges." + type = object({ + enabled = bool + }) + default = { + enabled = true + } +} + data "kubernetes_service_account_v1" "r53" { + + count = var.r53_config.enabled ? 1 : 0 + metadata { - name = "cert-manager-acme-dns01-route53" + name = "crt-mgr" namespace = "cert-manager" } } -resource "kubernetes_manifest" "default-issuer" { +# If CF enabled, then fetch secret from cert-manager for token +variable "cf_config" { + description = "Enable to use CloudFlare as a DNS01 provider for ACME challenges." + type = object({ + enabled = bool + email = string + name = optional(string, "cloudflare") + namespace = optional(string, "cert-manager") + }) + default = { + enabled = false + email = "" + name = "" + namespace = "" + } +} + +data "kubernetes_secret_v1" "cf" { + + count = var.cf_config.enabled ? 1 : 0 + + metadata { + name = var.cf_config.name + namespace = var.cf_config.namespace + } +} + +locals { + dns01_r53 = ! var.r53_config.enabled ? null : { + route53 = { + region = var.region + role = data.kubernetes_service_account_v1.r53[0].metadata[0].annotations["eks.amazonaws.com/role-arn"] + auth = { + kubernetes = { + serviceAccountRef = { + name = data.kubernetes_service_account_v1.r53[0].metadata[0].name + } + } + } + } + } + dns01_cf = ! var.cf_config.enabled ? null : { + cloudflare = { + apiTokenSecretRef = { + key = data.kubernetes_secret_v1.cf[0].metadata[0].annotations["custom.kubernetes.secret/key"] + name = data.kubernetes_secret_v1.cf[0].metadata[0].name + } + email = data.kubernetes_secret_v1.cf[0].metadata[0].annotations["custom.kubernetes.secret/email"] + } + } +} + +resource "kubernetes_manifest" "issuer" { field_manager { force_conflicts = true @@ -225,7 +349,7 @@ resource "kubernetes_manifest" "default-issuer" { status = "True" } } - + timeouts { create = "10m" update = "10m" @@ -247,19 +371,10 @@ resource "kubernetes_manifest" "default-issuer" { server = "https://acme-v02.api.letsencrypt.org/directory" solvers = [ { - dns01 = { - route53 = { - region = var.region - role = data.kubernetes_service_account_v1.r53.metadata[0].annotations["eks.amazonaws.com/role-arn"] - auth = { - kubernetes = { - serviceAccountRef = { - name = data.kubernetes_service_account_v1.r53.metadata[0].name - } - } - } - } - } + dns01 = merge( + local.dns01_r53, + local.dns01_cf + ) } ] } @@ -273,8 +388,8 @@ resource "kubernetes_manifest" "default-issuer" { data "kubernetes_service_account_v1" "sm" { metadata { - name = "external-secrets" - namespace = "external-secrets" + name = "external-secrets" + namespace = "ext-sec" } } @@ -308,13 +423,13 @@ resource "kubernetes_manifest" "secret-store" { provider = { aws = { service = "SecretsManager" - region = var.region + region = var.region auth = { jwt = { serviceAccountRef = { - name = data.kubernetes_service_account_v1.sm.metadata[0].name + name = data.kubernetes_service_account_v1.sm.metadata[0].name namespace = data.kubernetes_service_account_v1.sm.metadata[0].namespace - } + } } } } diff --git a/infra/1-click/1-setup/6-coder-server.tf b/infra/1-click/1-setup/6-coder-server.tf index 03f69d1..38017b5 100644 --- a/infra/1-click/1-setup/6-coder-server.tf +++ b/infra/1-click/1-setup/6-coder-server.tf @@ -15,77 +15,144 @@ variable "coder_username" { variable "coder_password" { description = "Coder DB's password." type = string - sensitive = true default = "th1s1sn0tas3cur3pass0wrd" + sensitive = true } variable "coder_license" { - type = string + type = string + default = "" sensitive = true - default = "" } variable "coder_admin_email" { - type = string + type = string default = "admin@coder.com" } variable "coder_admin_username" { - type = string + type = string default = "admin" } variable "coder_admin_password" { - type = string + type = string + default = "Th1s1sN0TS3CuR3!!" + sensitive = true +} + +variable "grafana_username" { + description = "Grafana DB's username." + type = string + default = "grafana" +} + +variable "grafana_password" { + description = "Grafana DB's password." + type = string + default = "th1s1sn0tas3cur3pass0wrd" sensitive = true - default = "Th1s1sN0TS3CuR3!!" +} + +variable "grafana_admin_username" { + type = string + default = "admin" +} + +variable "grafana_admin_password" { + type = string + default = "Th1s1sN0TS3CuR3!!" + sensitive = true +} + +variable "use_ext_dns" { + description = "Toggle the K8s 'external-dns' addon. Disable in-case you want to manage DNS records yourself." + type = bool + default = true +} + +variable "azs" { + type = list(string) + default = ["a", "b", "c"] } resource "aws_iam_user" "bedrock" { + + count = var.coder_license != "" ? 1 : 0 + name = "bedrock-access" path = "/${local.normalized_domain_name}/${data.aws_region.this.region}/" } +resource "aws_iam_access_key" "bedrock" { + + count = var.coder_license != "" ? 1 : 0 + + user = aws_iam_user.bedrock[0].name +} + resource "aws_iam_user_policy_attachment" "bedrock" { - user = aws_iam_user.bedrock.name + + count = var.coder_license != "" ? 1 : 0 + + user = aws_iam_user.bedrock[0].name # https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonBedrockLimitedAccess.html policy_arn = "arn:aws:iam::aws:policy/AmazonBedrockLimitedAccess" } -resource "aws_iam_access_key" "bedrock" { - user = aws_iam_user.bedrock.name +resource "aws_eip" "coder" { + count = length(var.azs) + domain = "vpc" + public_ipv4_pool = "amazon" + tags = { + Name = "${var.name}-${local.normalized_domain_name}-coder-${count.index}" + } +} + +locals { + pub_subs = [ for az in var.azs : "${var.name}-${local.normalized_domain_name}-public-${data.aws_region.this.region}${az}"] } module "coder-server" { - depends_on = [ kubernetes_manifest.nodepool ] + depends_on = [kubernetes_manifest.nodepool] source = "../../../modules/k8s/bootstrap/coder-server" cluster_name = "${var.name}-${local.normalized_domain_name}" cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn - namespace = "coder" - - replica_count = 2 - helm_version = "2.29.2" - image_repo = "ghcr.io/coder/coder" - image_tag = "v2.29.2" - primary_access_url = "https://${var.domain_name}" - wildcard_access_url = "*.${var.domain_name}" - db_secret_url = "postgresql://${var.coder_username}:${var.coder_password}@${data.aws_db_instance.coder.endpoint}/coder" - - env_vars = { - CODER_AIBRIDGE_ENABLED = true - CODER_AIBRIDGE_BEDROCK_REGION = data.aws_region.this.region - CODER_AIBRIDGE_BEDROCK_ACCESS_KEY = aws_iam_access_key.bedrock.id - CODER_AIBRIDGE_BEDROCK_ACCESS_KEY_SECRET = aws_iam_access_key.bedrock.secret - CODER_EXPERIMENTS = "oauth2,mcp-server-http" + namespace = "coder" + + helm_version = "2.29.4" + + coder = { + access_url = "https://${var.domain_name}" + wildcard_url = "*.${var.domain_name}" + pub_ips = aws_eip.coder.*.public_ip + image_repo = "ghcr.io/coder/coder" + image_tag = "v2.29.4" + rep_cnt = 1 + # Use this instead of external provisioners. DNS might not propagate fast enough for Coder to be "reachable". + prov_rep_cnt = 4 + # csp_policy = "frame-src https://${var.domain_name}" + } + + db = { + url = data.aws_db_instance.coder.endpoint + username = var.coder_username + password = var.coder_password + } + + aibridge = var.coder_license == "" ? null : { + enabled = true + bedrock = { + region = data.aws_region.this.region + model = "global.anthropic.claude-opus-4-5-20251101-v1:0" + access_id = aws_iam_access_key.bedrock[0].id + secret_id = aws_iam_access_key.bedrock[0].secret + } } - - # Use this instead of external provisioners. DNS might not propagate fast enough for Coder to be "reachable". - coder_builtin_provisioner_count = 4 - # coder_github_allowed_orgs = var.coder_github_allowed_orgs resource_request = { cpu = "1000m" @@ -96,36 +163,36 @@ module "coder-server" { memory = "2Gi" } - ssl_cert_config = { + cert_config = { name = var.domain_name - caissuer = kubernetes_manifest.default-issuer.manifest.metadata.name - secretissuer = kubernetes_manifest.secret-store.manifest.metadata.name + kind = kubernetes_manifest.issuer.manifest.kind + issuer = kubernetes_manifest.issuer.manifest.metadata.name + store = kubernetes_manifest.secret-store.manifest.metadata.name create_secret = true } - enable_oidc = false - enable_oauth = false - enable_github_external_auth = false - - tags = {} + tags = {} - service_annotations = { + svc_annot = merge({ "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" + "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" = join(",", aws_eip.coder.*.allocation_id) + "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", local.pub_subs) + }, !var.use_ext_dns ? null : { "external-dns.alpha.kubernetes.io/hostname" = "${var.domain_name},*.${var.domain_name}" - "external-dns.alpha.kubernetes.io/ttl" = 30 - } - - node_selector = kubernetes_manifest.nodepool["coder-server"].manifest.spec.template.metadata.labels - tolerations = [ for toleration in kubernetes_manifest.nodepool["coder-server"].manifest.spec.template.spec.taints : { - key = toleration.key - operator = "Equal" - value = toleration.value - effect = toleration.effect + "external-dns.alpha.kubernetes.io/ttl" = 60 + }) + + node_selector = kubernetes_manifest.nodepool["coder"].manifest.spec.template.metadata.labels + tolerations = [for toleration in kubernetes_manifest.nodepool["coder"].manifest.spec.template.spec.taints : { + key = toleration.key + operator = "Equal" + value = toleration.value + effect = toleration.effect }] - topology_spread_constraints = [{ + topology_spread = [{ max_skew = 1 topology_key = "kubernetes.io/hostname" when_unsatisfiable = "ScheduleAnyway" @@ -139,7 +206,7 @@ module "coder-server" { "app.kubernetes.io/instance" ] }] - pod_anti_affinity_preferred_during_scheduling_ignored_during_execution = [{ + pod_aaf_pref_sched_ie = [{ weight = 100 pod_affinity_term = { label_selector = { @@ -155,218 +222,245 @@ module "coder-server" { } # Wait for DNS propagation. May require multiple redeploys -resource "time_sleep" "wait_for_dns" { - create_duration = "120s" - depends_on = [ module.coder-server ] -} +# resource "time_sleep" "wait_for_dns" { +# depends_on = [ module.coder-server ] +# create_duration = "120s" +# } -data "external" "first-user" { - - depends_on = [ time_sleep.wait_for_dns ] +data "http" "first-user" { - program = ["bash", "${path.module}/scripts/first-user.sh"] + # depends_on = [ time_sleep.wait_for_dns ] + depends_on = [ module.coder-server ] - query = { - domain = var.domain_name - admin_email = var.coder_admin_email - admin_username = var.coder_admin_username - admin_password = var.coder_admin_password + url = "https://${aws_eip.coder[0].public_ip}/api/v2/users/first" + method = "POST" + insecure = true + request_headers = { + Host = var.domain_name + Accept = "application/json" + } + request_body = jsonencode({ + email = var.coder_admin_email + username = var.coder_admin_username + password = var.coder_admin_password + trial = false + }) + retry { + attempts = 60+1 + max_delay_ms = 5000 + min_delay_ms = 5000 } } -output "coder_session_token" { - value = data.external.first-user.result.session_token +data "http" "login" { + + depends_on = [data.http.first-user] + + url = "https://${aws_eip.coder[0].public_ip}/api/v2/users/login" + insecure = true + method = "POST" + request_headers = { + Host = var.domain_name + Accept = "application/json" + } + request_body = jsonencode({ + email = var.coder_admin_email + password = var.coder_admin_password + }) } +## +# Adding a license crashes Coder temporarily. +# Use 'external' to allow custom handling, and then wait before proceeding. +## + data "external" "add-license" { - - count = var.coder_license != "" ? 1 : 0 - depends_on = [ time_sleep.wait_for_dns ] + count = var.coder_license != "" ? 1 : 0 program = ["bash", "${path.module}/scripts/add-license.sh"] query = { + ip_addr = aws_eip.coder[0].public_ip domain = var.domain_name license_key = var.coder_license - session_token = data.external.first-user.result.session_token + session_token = jsondecode(data.http.login.response_body).session_token } } -# locals { -# dashboards-path = "${path.module}/dashboards" -# # coderd_selector = "pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=`${local.coderd_namespace}`" -# coderd_selector = "pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=~`(coder)`" +resource "time_sleep" "wait_for_coder" { -# provisionerd_selector = "pod=~`coder-provisioner.*`, namespace=~`(coder-ws|coder-ws-experiment|coder-ws-demo)`" + count = var.coder_license != "" ? 1 : 0 -# # workspaces_selector = "namespace=`coder-ws*`" -# workspaces_selector = "pod!~`coder.*`, namespace=~`(coder-ws|coder-ws-experiment|coder-ws-demo)`" -# non_workspaces_selector = "namespace=~`(coder|coder-ws|coder-ws-experiment|coder-ws-demo)`" + depends_on = [ data.external.add-license[0] ] + create_duration = "30s" +} -# dashboard_timerange = "12h" -# dashboard_refresh = "30s" -# } +resource "aws_eip" "grafana" { + count = length(var.azs) + domain = "vpc" + public_ipv4_pool = "amazon" + tags = { + Name = "${var.name}-${local.normalized_domain_name}-grafana-${count.index}" + } +} + +module "monitoring" { + source = "../../../modules/k8s/bootstrap/monitoring" + + chart_version = "0.7.0-rc.1" + cluster_name = "${var.name}-${local.normalized_domain_name}" + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn + + domain_name = var.domain_name + tolerations = concat([{ + key = "CriticalAddonsOnly" + operator = "Exists" + }], [for toleration in kubernetes_manifest.nodepool["coder"].manifest.spec.template.spec.taints : { + key = toleration.key + operator = "Equal" + value = toleration.value + effect = toleration.effect + }]) + + coder = { + db = { + host = data.aws_db_instance.coder.address + password = var.coder_password + username = var.coder_username + database = data.aws_db_instance.coder.db_name + } + selector = { + coderd = "pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=~`(coder)`" + provisionerd = "pod=~`coder-provisioner.*`, namespace=~`(coder)`" + workspaces = "pod!~`coder.*`, namespace=~`(coder)`" + ctrl_plane_ns = "coder" + ext_prov_ns = "coder" + } + } + grafana = { + admin = { + username = var.grafana_admin_username + password = var.grafana_admin_password + } + db = { + host = data.aws_db_instance.grafana.address + password = var.grafana_password + username = var.grafana_username + database = data.aws_db_instance.grafana.db_name + } + svc = { + annots = merge({ + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol" = "https" + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-path" = "/api/health" + "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" = join(",", aws_eip.grafana.*.allocation_id) + "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", local.pub_subs) + }, !var.use_ext_dns ? null : { + "external-dns.alpha.kubernetes.io/hostname" = "grafana.${var.domain_name},*.grafana.${var.domain_name}" + "external-dns.alpha.kubernetes.io/ttl" = 60 + }) + } + } + loki = { + s3 = { + chunks_bucket = data.aws_s3_bucket.loki.id + ruler_bucket = data.aws_s3_bucket.loki.id + region = data.aws_s3_bucket.loki.bucket_region + } + } +} + +## +# Coder Binary Prefetch +## -# module "monitoring" { -# source = "../../../modules/k8s/bootstrap/monitoring" - -# namespace = var.addon_namespace -# helm_coder_observability_version = var.helm_coder_observability_version -# helm_coder_observability_timeout = var.helm_coder_observability_timeout -# helm_prometheus_operator_version = var.helm_prometheus_operator_version -# helm_prometheus_operator_timeout = var.helm_prometheus_operator_timeout -# cluster_name = var.cluster_name -# cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - -# coder_db_username = var.coder_db_username -# coder_db_password = var.coder_db_password -# coder_db_host = var.coder_db_host -# coder_db_port = var.coder_db_port - -# loki_s3_chunk_bucket_name = var.loki_s3_chunk_bucket_name -# loki_s3_ruler_bucket_name = var.loki_s3_ruler_bucket_name -# loki_s3_bucket_region = var.loki_s3_bucket_region -# loki_iam_role_name = var.loki_iam_role_name -# loki_replicas = var.loki_replicas - -# grafana_security_key = var.grafana_security_key -# grafana_auth_username = var.grafana_auth_username -# grafana_auth_password = var.grafana_auth_password -# grafana_db_user = var.grafana_db_user -# grafana_db_name = var.grafana_db_name -# grafana_db_password = var.grafana_db_password -# grafana_db_host = var.grafana_db_host -# grafana_admin_username = var.grafana_admin_username -# grafana_admin_password = var.grafana_admin_password -# grafana_root_domain = var.grafana_root_domain -# grafana_subdomain = var.grafana_subdomain -# grafana_cert_name = var.grafana_cert_name -# grafana_cert_mnt_path = var.grafana_cert_mnt_path -# grafana_replicas = 2 -# grafana_service_annotations = { -# "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" -# "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" -# "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=true" -# } - -# dashboards = [ -# { -# name = "coder-dashboard-status" -# localPath = "${local.dashboards-path}/status.json" -# args = { -# HELM_NAMESPACE = var.addon_namespace -# CODERD_SELECTOR = local.coderd_selector -# PROVISIONERD_SELECTOR = local.provisionerd_selector -# WORKSPACES_SELECTOR = local.workspaces_selector -# PROMETHEUS_JOB = "${var.addon_namespace}/prometheus/server" -# LOKI_JOB = "${var.addon_namespace}/loki" -# GRAFANA_AGENT_JOB = "${var.addon_namespace}/grafana-agent/grafana-agent" -# } -# }, -# { -# name = "coder-dashboard-coderd" -# localPath = "${local.dashboards-path}/coderd.json" -# args = { -# DASHBOARD_TIMERANGE = local.dashboard_timerange -# DASHBOARD_REFRESH = local.dashboard_refresh -# CODERD_SELECTOR = local.coderd_selector -# } -# }, -# { -# name = "coder-dashboard-provisionerd" -# localPath = "${local.dashboards-path}/provisionerd.json" -# args = { -# DASHBOARD_TIMERANGE = local.dashboard_timerange -# DASHBOARD_REFRESH = local.dashboard_refresh -# PROVISIONERD_SELECTOR = local.provisionerd_selector -# NON_WORKSPACES_SELECTOR = local.non_workspaces_selector -# } -# }, -# { -# name = "coder-dashboard-workspaces" -# localPath = "${local.dashboards-path}/workspaces.json" -# args = { -# DASHBOARD_TIMERANGE = local.dashboard_timerange -# DASHBOARD_REFRESH = local.dashboard_refresh -# WORKSPACES_SELECTOR = local.workspaces_selector -# NON_WORKSPACES_SELECTOR = local.non_workspaces_selector -# } -# }, -# { -# name = "coder-dashboard-workspace-detail" -# localPath = "${local.dashboards-path}/workspace_detail.json" -# args = { -# DASHBOARD_TIMERANGE = local.dashboard_timerange -# DASHBOARD_REFRESH = local.dashboard_refresh -# WORKSPACES_SELECTOR = local.workspaces_selector -# NON_WORKSPACES_SELECTOR = local.non_workspaces_selector -# } -# }, -# { -# name = "coder-dashboard-prebuilds" -# localPath = "${local.dashboards-path}/prebuilds.json" -# args = { -# DASHBOARD_TIMERANGE = local.dashboard_timerange -# DASHBOARD_REFRESH = local.dashboard_refresh -# } -# }, -# { -# name = "coder-dashboard-aibridge" -# localPath = "${local.dashboards-path}/aibridge.json" -# args = {} -# }, -# # { -# # name = "coder-dashboard-proxyd" -# # localPath = "${local.dashboards-path}/proxyd.json" -# # args = { -# # DASHBOARD_TIMERANGE = local.dashboard_timerange -# # DASHBOARD_REFRESH = local.dashboard_refresh -# # CODERD_SELECTOR = local.coderd_selector -# # } -# # } -# ] - -# # service_annotations = { -# # "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" -# # "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" -# # "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=true" -# # } -# # node_selector = { -# # "node.coder.io/managed-by" = "karpenter" -# # "node.coder.io/used-for" = "coder" -# # } -# # tolerations = [{ -# # key = "dedicated" -# # operator = "Equal" -# # value = "coder" -# # effect = "NoSchedule" -# # }] -# # topology_spread_constraints = [{ -# # max_skew = 1 -# # topology_key = "kubernetes.io/hostname" -# # when_unsatisfiable = "ScheduleAnyway" -# # label_selector = { -# # match_labels = { -# # "app.kubernetes.io/name" = "coder" -# # "app.kubernetes.io/part-of" = "coder" -# # } -# # } -# # match_label_keys = [ -# # "app.kubernetes.io/instance" -# # ] -# # }] -# # pod_anti_affinity_preferred_during_scheduling_ignored_during_execution = [{ -# # weight = 100 -# # pod_affinity_term = { -# # label_selector = { -# # match_labels = { -# # "app.kubernetes.io/instance" = "coder-v2" -# # "app.kubernetes.io/name" = "coder" -# # "app.kubernetes.io/part-of" = "coder" -# # } -# # } -# # topology_key = "kubernetes.io/hostname" -# # } -# # }] -# } \ No newline at end of file +locals { + coder_path = "/opt/coder/bin" + bin_fetch_script = <<-EOF + if [ ! -f ${local.coder_path}/coder ]; then + curl -L https://${var.domain_name}/bin/coder-linux-amd64 -o ${local.coder_path}/coder + chmod +x ${local.coder_path}/coder + fi + EOF +} + +resource "kubernetes_daemon_set_v1" "bin-fetch" { + metadata { + name = "coder-bin-fetch" + namespace = module.coder-server.namespace + labels = { + "app.kubernetes.io/name" = "bin-fetch" + "app.kubernetes.io/part-of" = "coder-workspaces" + } + } + + spec { + + selector { + match_labels = { + "app.kubernetes.io/name" = "bin-fetch" + } + } + + template { + + metadata { + labels = { + "app.kubernetes.io/name" = "bin-fetch" + } + } + + spec { + + host_aliases { + hostnames = [var.domain_name] + ip = aws_eip.coder[0].private_ip + } + + security_context { + run_as_user = "0" + } + + init_container { + name = "fetch-binary" + image = "curlimages/curl:latest" + command = ["sh", "-c", "${local.bin_fetch_script}"] + + volume_mount { + name = "coder-bin" + mount_path = local.coder_path + read_only = false + } + + } + + container { + name = "pause" + image = "registry.k8s.io/pause:3.9" + + resources { + requests = { + cpu = "1m" + memory = "1Mi" + } + limits = { + cpu = "10m" + memory = "10Mi" + } + } + } + + volume { + name = "coder-bin" + host_path { + path = local.coder_path + type = "DirectoryOrCreate" + } + } + } + } + } +} \ No newline at end of file diff --git a/infra/1-click/1-setup/main.tf b/infra/1-click/1-setup/main.tf index 1bf78a6..70df511 100644 --- a/infra/1-click/1-setup/main.tf +++ b/infra/1-click/1-setup/main.tf @@ -14,10 +14,17 @@ terraform { } external = { source = "hashicorp/external" - version = ">= 2.3.5" + } + http = { + source = "hashicorp/http" + } + dns = { + source = "hashicorp/dns" + } + cloudflare = { + source = "cloudflare/cloudflare" } } - # backend "s3" {} } ## @@ -30,12 +37,12 @@ data "aws_vpc" "this" { } } -data "aws_db_instance" "coder" { - db_instance_identifier = "${var.name}-${local.normalized_domain_name}-coder" +data "aws_s3_bucket" "loki" { + bucket = "${var.name}-${local.normalized_domain_name}-grafana" } -data "aws_db_instance" "litellm" { - db_instance_identifier = "${var.name}-${local.normalized_domain_name}-litellm" +data "aws_db_instance" "coder" { + db_instance_identifier = "${var.name}-${local.normalized_domain_name}-coder" } data "aws_db_instance" "grafana" { @@ -43,7 +50,7 @@ data "aws_db_instance" "grafana" { } data "aws_security_group" "coder" { - name = "${var.name}-${local.normalized_domain_name}-pgsql" + name = "${var.name}-${local.normalized_domain_name}-pgsql" vpc_id = data.aws_vpc.this.id } @@ -66,17 +73,17 @@ data "aws_iam_openid_connect_provider" "coder" { variable "region" { description = "The AWS region of the deployment." type = string - default = "us-east-2" + default = "us-east-2" } variable "name" { description = "Name for created resources and tag prefix." type = string - default = "coder" + default = "coder" } variable "profile" { - type = string + type = string default = "default" } @@ -103,8 +110,150 @@ provider "kubernetes" { token = data.aws_eks_cluster_auth.coder.token } +variable "auto_set_record" { + description = "Set if you don't want to use external-dns, but still want to automatically set the domain name." + type = object({ + use_cf = bool + cf_token = optional(string, "") + use_r53 = bool + }) + default = { + use_cf = false + cf_token = "" + use_r53 = true + } + sensitive = true + + validation { + condition = !(var.auto_set_record.use_cf && var.auto_set_record.use_r53) + error_message = "'use_cf' and 'use_r53' cannot both be true." + } + + validation { + condition = !(var.auto_set_record.use_cf && var.auto_set_record.cf_token == "") + error_message = "'cf_token' cannot be unset when 'use_cf' is true." + } +} + +provider "cloudflare" { + api_token = var.auto_set_record.cf_token +} + data "aws_region" "this" {} locals { normalized_domain_name = split(".", var.domain_name)[0] -} \ No newline at end of file + apex_domain = join(".", slice(split(".", var.domain_name), length(split(".", var.domain_name))-2, length(split(".", var.domain_name)))) +} + +## +# Fetch DNS Zone +## + +data "aws_route53_zone" "coder" { + count = nonsensitive(var.auto_set_record.use_r53) ? 1 : 0 + name = local.apex_domain +} + +data "cloudflare_zone" "coder" { + count = nonsensitive(var.auto_set_record.use_cf) ? 1 : 0 + filter = { + name = local.apex_domain + } +} + +## +# Coder DNS Record Setup +## + +resource "aws_route53_record" "coder-primary" { + count = nonsensitive(var.auto_set_record.use_r53) ? 1 : 0 + zone_id = data.aws_route53_zone.coder[0].zone_id + name = var.domain_name + type = "A" + ttl = "30" + records = aws_eip.coder.*.public_ip +} + +resource "aws_route53_record" "coder-wildcard" { + count = nonsensitive(var.auto_set_record.use_r53) ? 1 : 0 + zone_id = data.aws_route53_zone.coder[0].zone_id + name = "*.${var.domain_name}" + type = "A" + ttl = "60" + records = aws_eip.coder.*.public_ip +} + +resource "cloudflare_dns_record" "coder-primary" { + + for_each = nonsensitive(var.auto_set_record.use_cf) ? toset(aws_eip.coder.*.public_ip) : toset([]) + + zone_id = data.cloudflare_zone.coder[0].id + name = var.domain_name + ttl = 1 + type = "A" + comment = "" + content = each.value + proxied = true +} + +resource "cloudflare_dns_record" "coder-wildcard" { + + for_each = nonsensitive(var.auto_set_record.use_cf) ? toset(aws_eip.coder.*.public_ip) : toset([]) + + zone_id = data.cloudflare_zone.coder[0].id + name = "*.${var.domain_name}" + ttl = 1 + type = "A" + comment = "" + content = each.value + proxied = true +} + +## +# Grafana DNS Record Setup +## + +resource "aws_route53_record" "grafana-primary" { + count = nonsensitive(var.auto_set_record.use_r53) ? 1 : 0 + zone_id = data.aws_route53_zone.coder[0].zone_id + name = "grafana.${var.domain_name}" + type = "A" + ttl = "30" + records = aws_eip.grafana.*.public_ip +} + +resource "aws_route53_record" "grafana-wildcard" { + count = nonsensitive(var.auto_set_record.use_r53) ? 1 : 0 + zone_id = data.aws_route53_zone.coder[0].zone_id + name = "*.grafana.${var.domain_name}" + type = "A" + ttl = "60" + records = aws_eip.grafana.*.public_ip +} + +resource "cloudflare_dns_record" "grafana-primary" { + + for_each = nonsensitive(var.auto_set_record.use_cf) ? toset(aws_eip.grafana.*.public_ip) : toset([]) + + zone_id = data.cloudflare_zone.coder[0].id + name = "grafana.${var.domain_name}" + ttl = 1 + type = "A" + comment = "" + content = each.value + proxied = true +} + +resource "cloudflare_dns_record" "grafana-wildcard" { + + for_each = nonsensitive(var.auto_set_record.use_cf) ? toset(aws_eip.grafana.*.public_ip) : toset([]) + + zone_id = data.cloudflare_zone.coder[0].id + name = "*.grafana.${var.domain_name}" + ttl = 1 + type = "A" + comment = "" + content = each.value + proxied = true +} diff --git a/infra/1-click/1-setup/scripts/add-license.sh b/infra/1-click/1-setup/scripts/add-license.sh index ee444f1..9a74482 100644 --- a/infra/1-click/1-setup/scripts/add-license.sh +++ b/infra/1-click/1-setup/scripts/add-license.sh @@ -1,25 +1,11 @@ #!/usr/bin/env bash -eval "$(jq -r '@sh "CODER_DOMAIN=\(.domain) CODER_LICENSE=\(.license_key) CODER_SESSION_TOKEN=\(.session_token)"')" +eval "$(jq -r '@sh "CODER_IP_ADDR=\(.ip_addr) CODER_DOMAIN=\(.domain) CODER_LICENSE=\(.license_key) CODER_SESSION_TOKEN=\(.session_token)"')" -# URL might not be available still. Retry request until available, or fail if max attempts reached. - -IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) -RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$IP_ADDR" +RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$CODER_IP_ADDR" CODER_URL=https://$CODER_DOMAIN -IDX=0 -while [[ -z "$IP_ADDR" ]]; do - if (( IDX > 6 )); then - >&2 echo "Error: Failed to run \"dig +short $CODER_DOMAIN | head -n1\". Unable to discover IP for \"$CODER_DOMAIN\"." - exit 1; - fi - sleep 10 - ((IDX++)) - IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) -done - RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/licenses" \ -H 'Content-Type: application/json' \ -H 'Accept: application/json' \ @@ -34,4 +20,6 @@ fi jq -n '{"success":"true"}' -exit 0; \ No newline at end of file +exit 0; + +echo "Hello!" \ No newline at end of file diff --git a/infra/1-click/1-setup/scripts/first-user.sh b/infra/1-click/1-setup/scripts/first-user.sh deleted file mode 100755 index be15942..0000000 --- a/infra/1-click/1-setup/scripts/first-user.sh +++ /dev/null @@ -1,50 +0,0 @@ - -#!/usr/bin/env bash - -eval "$(jq -r '@sh "CODER_DOMAIN=\(.domain) ADMIN_EMAIL=\(.admin_email) ADMIN_USERNAME=\(.admin_username) ADMIN_PASSWORD=\(.admin_password)"')" - -# URL might not be available still. Retry request until available, or fail if max attempts reached. - -IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) -RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$IP_ADDR" -CODER_URL=https://$CODER_DOMAIN - -IDX=0 -while [[ -z "$IP_ADDR" ]]; do - if (( IDX > 6 )); then - >&2 echo "Error: Failed to run \"dig +short $CODER_DOMAIN | head -n1\". Unable to discover IP for \"$CODER_DOMAIN\"." - exit 1; - fi - sleep 10 - ((IDX++)) - IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) -done - -RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/users/first" \ - -H "Content-Type: application/json" \ - -d "{ - \"email\": \"$ADMIN_EMAIL\", - \"username\": \"$ADMIN_USERNAME\", - \"password\": \"$ADMIN_PASSWORD\", - \"trial\": false - }") - -if [[ $? -ne 0 ]]; then - >&2 echo "Error: Failed on 'curl -ks $RESOLVE_ARG -X POST \"$CODER_URL/api/v2/users/first'\". Can't create first user." - jq -n '{"session_token":""}' - exit 1; -fi - -LOGIN_RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/users/login" \ - -H "Content-Type: application/json" \ - -d "{\"email\":\"$ADMIN_EMAIL\",\"password\":\"$ADMIN_PASSWORD\"}" | jq -r '.session_token') - -if [ $? -ne 0 ]; then - >&2 echo "Error: Failed on 'curl -ks $RESOLVE_ARG -X POST \"$CODER_URL/api/v2/users/login\"'. Can't login." - jq -n '{"session_token":""}' - exit 1; -fi - -jq -n --arg session_token "$LOGIN_RESPONSE" '{"session_token":$session_token}' - -exit 0; \ No newline at end of file diff --git a/infra/1-click/1-setup/scripts/nodeconfig.yaml b/infra/1-click/1-setup/scripts/nodeconfig.yaml new file mode 100644 index 0000000..a5bdf87 --- /dev/null +++ b/infra/1-click/1-setup/scripts/nodeconfig.yaml @@ -0,0 +1,18 @@ +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + registryPullQPS: 30 + # imageServiceEndpoint: unix:///run/soci-snapshotter-grpc/soci-snapshotter-grpc.sock + # containerd: + # config: | + # [proxy_plugins.soci] + # type = "snapshot" + # address = "/run/soci-snapshotter-grpc/soci-snapshotter-grpc.sock" + # [proxy_plugins.soci.exports] + # root = "/var/lib/soci-snapshotter-grpc" + # [plugins."io.containerd.grpc.v1.cri".containerd] + # snapshotter = "soci" + # # This line is required for containerd to send information about how to lazily load the image to the snapshotter + # disable_snapshot_annotations = false \ No newline at end of file diff --git a/infra/1-click/2-clean.sh b/infra/1-click/2-clean.sh index 5fae18b..f1ac942 100755 --- a/infra/1-click/2-clean.sh +++ b/infra/1-click/2-clean.sh @@ -2,21 +2,45 @@ set -ae -o pipefail -## -# Set this block to 2-coder for cleaning when ready -## +source coder.env + +AWS_AZS="${CODER_AWS_AZS:-[\"a\",\"c\"]}" AWS_PROFILE="${CODER_AWS_PROFILE:-default}" AWS_REGION="${CODER_AWS_REGION:-us-east-2}" DOMAIN_NAME="${CODER_DOMAIN_NAME:-}" LICENSE="${CODER_LICENSE:-}" +USE_EXTERN_DNS="${CODER_USE_EXTERN_DNS:-true}" + +USE_R53="${CODER_USE_R53:-true}" + +USE_CF="${CODER_USE_CF:-false}" +CF_TOKEN="${CODER_CF_TOKEN:-}" +CF_EMAIL="${CODER_CF_EMAIL:-}" + +SET_REC_USE_R53=false; ! $USE_EXTERN_DNS && $USE_R53 && SET_REC_USE_R53=true +SET_REC_USE_CF=false; ! $USE_EXTERN_DNS && $USE_CF && SET_REC_USE_CF=true + +CODER_DB_USERNAME="${CODER_DB_USERNAME:-coder}" +CODER_DB_PASSWORD="${CODER_DB_PASSWORD:-th1s1sn0tas3cur3pass0wrd}" + +GRAFANA_DB_USERNAME="${CODER_GRAFANA_DB_PASSWORD:-grafana}" +GRAFANA_DB_PASSWORD="${CODER_GRAFANA_DB_PASSWORD:-th1s1sn0tas3cur3pass0wrd}" + +CODER_USERNAME="${CODER_USERNAME:-admin}" +CODER_EMAIL="${CODER_EMAIL:-admin@coder.com}" +CODER_PASSWORD="${CODER_PASSWORD:-Th1s1sN0TS3CuR3!!}" + echo "Change directory into '2-coder'." cd 2-coder terraform plan -destroy -out=tf.plan \ -var profile=$AWS_PROFILE \ -var region=$AWS_REGION \ - -var domain_name=$DOMAIN_NAME + -var domain_name=$DOMAIN_NAME \ + -var coder_license=$LICENSE \ + -var coder_admin_email=$CODER_EMAIL \ + -var coder_admin_password=$CODER_PASSWORD terraform apply tf.plan cd ../ @@ -26,7 +50,16 @@ terraform plan -destroy -out=tf.plan \ -var profile=$AWS_PROFILE \ -var region=$AWS_REGION \ -var domain_name=$DOMAIN_NAME \ - -var coder_license=$LICENSE + -var azs="$AWS_AZS" \ + -var coder_license=$LICENSE \ + -var coder_username=$CODER_DB_USERNAME \ + -var coder_password=$CODER_DB_PASSWORD \ + -var coder_admin_email=$CODER_EMAIL \ + -var coder_admin_username=$CODER_USERNAME \ + -var coder_admin_password=$CODER_PASSWORD \ + -var auto_set_record="{\"use_cf\":\"$SET_REC_USE_CF\",\"cf_token\":\"$CF_TOKEN\",\"use_r53\":\"$SET_REC_USE_R53\"}" \ + -var cf_config="{\"enabled\":\"$USE_CF\",\"email\":\"$CF_EMAIL\"}" \ + -var r53_config="{\"enabled\":\"$USE_R53\"}" terraform apply tf.plan cd ../ @@ -35,7 +68,14 @@ cd 0-infra terraform plan -destroy -out=tf.plan \ -var profile=$AWS_PROFILE \ -var region=$AWS_REGION \ - -var azs='["a","c"]' \ - -var domain_name=$DOMAIN_NAME + -var domain_name=$DOMAIN_NAME \ + -var azs="$AWS_AZS" \ + -var coder_username=$CODER_DB_USERNAME \ + -var coder_password=$CODER_DB_PASSWORD \ + -var grafana_username=$GRAFANA_DB_USERNAME \ + -var grafana_password=$GRAFANA_DB_PASSWORD \ + -var use_ext_dns=$USE_EXTERN_DNS \ + -var cf_config="{\"enabled\":\"$USE_CF\",\"email\":\"$CF_EMAIL\",\"token\":\"$CF_TOKEN\"}" \ + -var r53_config="{\"enabled\":\"$USE_R53\"}" terraform apply tf.plan cd ../ \ No newline at end of file diff --git a/infra/1-click/2-coder/7-license.tf b/infra/1-click/2-coder/7-license.tf new file mode 100644 index 0000000..80398f9 --- /dev/null +++ b/infra/1-click/2-coder/7-license.tf @@ -0,0 +1,26 @@ +variable "coder_license" { + type = string + default = "" + sensitive = true +} + +# resource "coderd_license" "enterprise" { +# license = var.coder_license +# } + +# data "http" "get-license" { + +# depends_on = [data.http.first-user] + +# url = "https://${data.dns_a_record_set.coder.addrs[0]}/api/v2/users/login" +# insecure = true +# method = "POST" +# request_headers = { +# Host = var.domain_name +# Accept = "application/json" +# } +# request_body = jsonencode({ +# email = var.coder_admin_email +# password = var.coder_admin_password +# }) +# } \ No newline at end of file diff --git a/infra/1-click/2-coder/7-orgs.tf b/infra/1-click/2-coder/7-orgs.tf deleted file mode 100644 index 14c160d..0000000 --- a/infra/1-click/2-coder/7-orgs.tf +++ /dev/null @@ -1,108 +0,0 @@ -variable "domain_name" { - type = string -} - -variable "coder_admin_email" { - type = string - default = "admin@coder.com" -} - -variable "coder_admin_password" { - type = string - sensitive = true - default = "Th1s1sN0TS3CuR3!!" -} - -## -# Coder MUST be in a reachable state by now -## - -data "external" "login" { - - program = ["bash", "${path.module}/scripts/login.sh"] - - query = { - domain = var.domain_name - admin_email = var.coder_admin_email - admin_password = var.coder_admin_password - } -} - -provider "coderd" { - url = "https://${var.domain_name}" - token = data.external.login.result.session_token -} - -output "coder_session_token" { - value = data.external.login.result.session_token -} - -locals { - image_repo = "ghcr.io/coder/coder" - image_tag = "v2.29.1" - service_account_labels = { - "app.kubernetes.io/instance" = "coder-provisioner" - "app.kubernetes.io/name" = "coder-provisioner" - "app.kubernetes.io/part-of" = "coder-provisioner" - } - node_selector = { - "node.coder.io/instance" = "coder-v2" - "node.coder.io/managed-by" = "karpenter" - "node.coder.io/name" = "coder" - "node.coder.io/part-of" = "coder" - "node.coder.io/used-for" = "coder-provisioner" - } - tolerations = [{ - key = "dedicated" - operator = "Equal" - value = "coder-provisioner" - effect = "NoSchedule" - }] -} - -data "coderd_organization" "default" { - is_default = true -} - -locals { - default_ns = "coder-ws" -} - -# resource "kubernetes_namespace" "coder-ws" { -# metadata { -# name = local.default_ns -# } -# } - -# module "default-ws" { - -# depends_on = [ data.external.login ] -# source = "../../../modules/k8s/bootstrap/coder-provisioner" - -# cluster_name = "${var.name}-${local.normalized_domain_name}" -# cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn - -# coder_organization_name = data.coderd_organization.default.name -# coder_provisioner_tags = {} - -# image_repo = local.image_repo -# image_tag = local.image_tag - -# namespace = local.default_ns -# ws_service_account_name = "coder-ws" -# ws_service_account_labels = { -# "app.kubernetes.io/instance" = "coder-provisioner" -# "app.kubernetes.io/name" = "coder-provisioner" -# "app.kubernetes.io/part-of" = "coder-provisioner" -# } -# provisioner_service_account_name = "coder" -# replica_count = 2 -# primary_access_url = "https://${var.domain_name}" -# env_vars = { -# CODER_PROMETHEUS_ENABLE = "true" -# CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" -# CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" -# } -# node_selector = local.node_selector -# tolerations = local.tolerations -# } \ No newline at end of file diff --git a/infra/1-click/2-coder/8-templates.tf b/infra/1-click/2-coder/8-templates.tf index 819c716..c11399b 100644 --- a/infra/1-click/2-coder/8-templates.tf +++ b/infra/1-click/2-coder/8-templates.tf @@ -1,94 +1,131 @@ +## +# Coder Template Push +## + +data "coderd_organization" "default" { + + # depends_on = [ coderd_license.enterprise ] + + is_default = true +} + data "coderd_group" "everyone" { + + # depends_on = [ coderd_license.enterprise ] + organization_id = data.coderd_organization.default.id - name = "Everyone" + name = "Everyone" } data "archive_file" "k8s-default" { type = "zip" excludes = ["${path.module}/templates/kubernetes/.terraform"] - source_dir = "${path.module}/templates/kubernetes" + source_dir = "${path.module}/templates/kubernetes" output_path = "/tmp/kubernetes.zip" } resource "time_static" "k8s-default" { - triggers = { - run_on_checksum = "${data.archive_file.k8s-default.id}" - } + triggers = { + run_on_ip_changes = data.aws_eip.coder.private_ip + run_on_checksum = "${data.archive_file.k8s-default.id}" + } } resource "coderd_template" "k8s-default" { - name = "kubernetes" - # organization_id = module.default-ws.coderd_organization_id + + # depends_on = [ coderd_license.enterprise ] + + name = "kubernetes" organization_id = data.coderd_organization.default.id - display_name = "Kubernetes (Deployment)" - description = "Provision Kubernetes Deployments as Coder workspaces" - icon = "https://${var.domain_name}/icon/k8s.png" + display_name = "Kubernetes (Deployment)" + description = "Provision Kubernetes Deployments as Coder workspaces" + icon = "https://${var.domain_name}/icon/k8s.png" versions = [ { name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.k8s-default.rfc3339)}" description = "The stable version of the template." directory = "${path.module}/templates/kubernetes" - active = true + active = true tf_vars = [{ - name = "namespace" - # value = kubernetes_namespace.coder-ws.metadata[0].name + name = "namespace" value = "coder" - },{ - name = "use_kubeconfig" + }, { + name = "use_kubeconfig" value = tostring(false) - }] + }, { + name = "host_ip" + value = data.aws_eip.coder.private_ip + }] } ] - acl = { + acl = var.coder_license == "" ? null :{ users = [] groups = [{ - id = data.coderd_group.everyone.id - role = "use" + id = data.coderd_group.everyone.id + role = "use" }] } } data "archive_file" "k8s-claude" { + + count = var.coder_license != "" ? 1 : 0 + type = "zip" excludes = ["${path.module}/templates/kubernetes-claude/.terraform"] - source_dir = "${path.module}/templates/kubernetes-claude" + source_dir = "${path.module}/templates/kubernetes-claude" output_path = "/tmp/kubernetes-claude.zip" } resource "time_static" "k8s-claude" { - triggers = { - run_on_checksum = "${data.archive_file.k8s-claude.id}" - } + + count = var.coder_license != "" ? 1 : 0 + + triggers = { + run_on_ip_changes = data.aws_eip.coder.private_ip + run_on_checksum = "${data.archive_file.k8s-claude[0].id}" + } } +## +# NOTICE: This template requires a Premium license as it includes the following: +# - AI Bridge +# - Agent Boundaries +## + resource "coderd_template" "k8s-claude" { - name = "kubernetes-claude" - # organization_id = module.default-ws.coderd_organization_id + + # depends_on = [ coderd_license.enterprise ] + count = var.coder_license != "" ? 1 : 0 + + name = "kubernetes-claude" organization_id = data.coderd_organization.default.id - display_name = "Kubernetes w/ Claude (Deployment)" - description = "Provision a Kubernetes Deployments with Claude installed." - icon = "https://${var.domain_name}/icon/claude.svg" + display_name = "Claude Code on Coder" + description = "Provision a Kubernetes Deployments with Claude installed." + icon = "https://${var.domain_name}/icon/claude.svg" versions = [ { - name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.k8s-claude.rfc3339)}" + name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.k8s-claude[0].rfc3339)}" description = "The stable version of the template." directory = "${path.module}/templates/kubernetes-claude" - active = true + active = true tf_vars = [{ - name = "namespace" - # value = kubernetes_namespace.coder-ws.metadata[0].name + name = "namespace" value = "coder" - },{ - name = "use_kubeconfig" + }, { + name = "use_kubeconfig" value = tostring(false) - }] + }, { + name = "host_ip" + value = data.aws_eip.coder.private_ip + }] } ] acl = { users = [] groups = [{ - id = data.coderd_group.everyone.id - role = "use" + id = data.coderd_group.everyone.id + role = "use" }] } } \ No newline at end of file diff --git a/infra/1-click/2-coder/main.tf b/infra/1-click/2-coder/main.tf index 2d44586..aada9e7 100644 --- a/infra/1-click/2-coder/main.tf +++ b/infra/1-click/2-coder/main.tf @@ -13,18 +13,21 @@ terraform { source = "hashicorp/kubernetes" } external = { - source = "hashicorp/external" + source = "hashicorp/external" version = ">= 2.3.5" } + http = { + source = "hashicorp/http" + } coderd = { - source = "coder/coderd" + source = "coder/coderd" version = "0.0.12" } - null = { - source = "hashicorp/null" - } time = { - source = "hashicorp/time" + source = "hashicorp/time" + } + dns = { + source = "hashicorp/dns" } } } @@ -58,17 +61,17 @@ data "aws_iam_openid_connect_provider" "coder" { variable "region" { description = "The AWS region of the deployment." type = string - default = "us-east-2" + default = "us-east-2" } variable "name" { description = "Name for created resources and tag prefix." type = string - default = "coder" + default = "coder" } variable "profile" { - type = string + type = string default = "default" } @@ -91,6 +94,56 @@ provider "kubernetes" { token = data.aws_eks_cluster_auth.coder.token } +provider "dns" {} + locals { normalized_domain_name = split(".", var.domain_name)[0] +} + +## +# Login and Fetch Authentication Token +## + +variable "domain_name" { + type = string +} + +variable "coder_admin_email" { + type = string + default = "admin@coder.com" +} + +variable "coder_admin_password" { + type = string + default = "Th1s1sN0TS3CuR3!!" + sensitive = true +} + +## +# Coder MUST be in a reachable state by now +## + +data "aws_eip" "coder" { + tags = { + Name = "${var.name}-${local.normalized_domain_name}-coder-0" + } +} + +data "http" "login" { + url = "https://${data.aws_eip.coder.public_ip}/api/v2/users/login" + insecure = true + method = "POST" + request_headers = { + Host = var.domain_name + Accept = "application/json" + } + request_body = jsonencode({ + email = var.coder_admin_email + password = var.coder_admin_password + }) +} + +provider "coderd" { + url = "https://${var.domain_name}" + token = jsondecode(data.http.login.response_body).session_token } \ No newline at end of file diff --git a/infra/1-click/2-coder/scripts/login.sh b/infra/1-click/2-coder/scripts/login.sh deleted file mode 100755 index 35aafa9..0000000 --- a/infra/1-click/2-coder/scripts/login.sh +++ /dev/null @@ -1,35 +0,0 @@ - -#!/usr/bin/env bash - -eval "$(jq -r '@sh "CODER_DOMAIN=\(.domain) ADMIN_EMAIL=\(.admin_email) ADMIN_PASSWORD=\(.admin_password)"')" - -# URL might not be available still. Retry request until available, or fail if max attempts reached. - -IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) -RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$IP_ADDR" -CODER_URL=https://$CODER_DOMAIN - -IDX=0 -while [[ -z "$IP_ADDR" ]]; do - ((IDX++)) - if (( IDX >= 6 )); then - >&2 echo "Error: Failed to run \"dig +short $CODER_DOMAIN | head -n1\". Unable to discover IP for \"$CODER_DOMAIN\"." - exit 1; - fi - sleep 10 - IP_ADDR=$(dig +short $CODER_DOMAIN | head -n1) -done - -LOGIN_RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/users/login" \ - -H "Content-Type: application/json" \ - -d "{\"email\":\"$ADMIN_EMAIL\",\"password\":\"$ADMIN_PASSWORD\"}" | jq -r '.session_token') - -if [ $? -ne 0 ]; then - >&2 echo "Error: Failed on 'curl -ks $RESOLVE_ARG -X POST \"$CODER_URL/api/v2/users/login\"'. Can't login." - jq -n '{"session_token":""}' - exit 1; -fi - -jq -n --arg session_token "$LOGIN_RESPONSE" '{"session_token":$session_token}' - -exit 0; \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/kubernetes-claude/README.md b/infra/1-click/2-coder/templates/kubernetes-claude/README.md index df936dc..e687f92 100644 --- a/infra/1-click/2-coder/templates/kubernetes-claude/README.md +++ b/infra/1-click/2-coder/templates/kubernetes-claude/README.md @@ -1,5 +1,5 @@ --- -display_name: Kubernetes w/ Claude Code +display_name: Claude Code on Coder description: Provision Kubernetes workspaces with Claude Code and code-server icon: ../../../site/static/icon/claude.svg maintainer_github: coder diff --git a/infra/1-click/2-coder/templates/kubernetes-claude/boundary-config.yaml b/infra/1-click/2-coder/templates/kubernetes-claude/boundary-config.yaml new file mode 100644 index 0000000..b3cb52c --- /dev/null +++ b/infra/1-click/2-coder/templates/kubernetes-claude/boundary-config.yaml @@ -0,0 +1,243 @@ +allowlist: + # Test domains + - method=GET domain=typicode.com + - method=GET domain=*.typicode.com + + # Coder + - domain=${CODER_DOMAIN} + + # Domain used in coder workspaces + - method=POST domain=http-intake.logs.datadoghq.com + - method=POST domain=http-intake.logs.us5.datadoghq.com + + # Default allowed domains from Claude Code on the web + # Source: https://code.claude.com/docs/en/claude-code-on-the-web#default-allowed-domains + # Anthropic Services + - domain=api.anthropic.com + - domain=statsig.anthropic.com + - domain=claude.ai + + # Version Control + - domain=github.com path=/coder-contrib/* + - domain=github.com path=/coder/* + - domain=www.github.com + - domain=api.github.com + - domain=raw.githubusercontent.com + - domain=objects.githubusercontent.com + - domain=codeload.github.com + - domain=avatars.githubusercontent.com + - domain=camo.githubusercontent.com + - domain=gist.github.com + - domain=gitlab.com + - domain=www.gitlab.com + - domain=registry.gitlab.com + - domain=bitbucket.org + - domain=www.bitbucket.org + - domain=api.bitbucket.org + + # Container Registries + - domain=registry-1.docker.io + - domain=auth.docker.io + - domain=index.docker.io + - domain=hub.docker.com + - domain=www.docker.com + - domain=production.cloudflare.docker.com + - domain=download.docker.com + - domain=*.gcr.io + - domain=ghcr.io + - domain=mcr.microsoft.com + - domain=*.data.mcr.microsoft.com + + # Cloud Platforms + - domain=cloud.google.com + - domain=accounts.google.com + - domain=gcloud.google.com + - domain=*.googleapis.com + - domain=storage.googleapis.com + - domain=compute.googleapis.com + - domain=container.googleapis.com + - domain=azure.com + - domain=portal.azure.com + - domain=microsoft.com + - domain=www.microsoft.com + - domain=*.microsoftonline.com + - domain=packages.microsoft.com + - domain=dotnet.microsoft.com + - domain=dot.net + - domain=visualstudio.com + - domain=dev.azure.com + - domain=oracle.com + - domain=www.oracle.com + - domain=java.com + - domain=www.java.com + - domain=java.net + - domain=www.java.net + - domain=download.oracle.com + - domain=yum.oracle.com + + # Package Managers - JavaScript/Node + - domain=registry.npmjs.org + - domain=www.npmjs.com + - domain=www.npmjs.org + - domain=npmjs.com + - domain=npmjs.org + - domain=yarnpkg.com + - domain=registry.yarnpkg.com + + # Package Managers - Python + - domain=pypi.org + - domain=www.pypi.org + - domain=files.pythonhosted.org + - domain=pythonhosted.org + - domain=test.pypi.org + - domain=pypi.python.org + - domain=pypa.io + - domain=www.pypa.io + + # Package Managers - Ruby + - domain=rubygems.org + - domain=www.rubygems.org + - domain=api.rubygems.org + - domain=index.rubygems.org + - domain=ruby-lang.org + - domain=www.ruby-lang.org + - domain=rubyforge.org + - domain=www.rubyforge.org + - domain=rubyonrails.org + - domain=www.rubyonrails.org + - domain=rvm.io + - domain=get.rvm.io + + # Package Managers - Rust + - domain=crates.io + - domain=www.crates.io + - domain=static.crates.io + - domain=rustup.rs + - domain=static.rust-lang.org + - domain=www.rust-lang.org + + # Package Managers - Go + - domain=proxy.golang.org + - domain=sum.golang.org + - domain=index.golang.org + - domain=golang.org + - domain=www.golang.org + - domain=go.dev + - domain=dl.google.com + - domain=goproxy.io + - domain=pkg.go.dev + + # Go Module Domains (from go.mod) + - domain=cdr.dev + - domain=cel.dev + - domain=dario.cat + - domain=git.sr.ht + - domain=go.mozilla.org + - domain=go.nhat.io + - domain=go.opentelemetry.io + - domain=go.uber.org + - domain=go.yaml.in + - domain=go4.org + - domain=golang.zx2c4.com + - domain=gonum.org + - domain=gopkg.in + - domain=gvisor.dev + - domain=howett.net + - domain=kernel.org + - domain=mvdan.cc + - domain=sigs.k8s.io + - domain=storj.io + + # Package Managers - JVM + - domain=maven.org + - domain=repo.maven.org + - domain=central.maven.org + - domain=repo1.maven.org + - domain=jcenter.bintray.com + - domain=gradle.org + - domain=www.gradle.org + - domain=services.gradle.org + - domain=spring.io + - domain=repo.spring.io + + # Package Managers - Other Languages + - domain=packagist.org + - domain=www.packagist.org + - domain=repo.packagist.org + - domain=nuget.org + - domain=www.nuget.org + - domain=api.nuget.org + - domain=pub.dev + - domain=api.pub.dev + - domain=hex.pm + - domain=www.hex.pm + - domain=cpan.org + - domain=www.cpan.org + - domain=metacpan.org + - domain=www.metacpan.org + - domain=api.metacpan.org + - domain=cocoapods.org + - domain=www.cocoapods.org + - domain=cdn.cocoapods.org + - domain=haskell.org + - domain=www.haskell.org + - domain=hackage.haskell.org + - domain=swift.org + - domain=www.swift.org + + # Linux Distributions + - domain=archive.ubuntu.com + - domain=security.ubuntu.com + - domain=ubuntu.com + - domain=www.ubuntu.com + - domain=*.ubuntu.com + - domain=ppa.launchpad.net + - domain=launchpad.net + - domain=www.launchpad.net + + # Development Tools & Platforms + - domain=dl.k8s.io + - domain=pkgs.k8s.io + - domain=k8s.io + - domain=www.k8s.io + - domain=releases.hashicorp.com + - domain=apt.releases.hashicorp.com + - domain=rpm.releases.hashicorp.com + - domain=archive.releases.hashicorp.com + - domain=hashicorp.com + - domain=www.hashicorp.com + - domain=repo.anaconda.com + - domain=conda.anaconda.org + - domain=anaconda.org + - domain=www.anaconda.com + - domain=anaconda.com + - domain=continuum.io + - domain=apache.org + - domain=www.apache.org + - domain=archive.apache.org + - domain=downloads.apache.org + - domain=eclipse.org + - domain=www.eclipse.org + - domain=download.eclipse.org + - domain=nodejs.org + - domain=www.nodejs.org + + # Cloud Services & Monitoring + - domain=statsig.com + - domain=www.statsig.com + - domain=api.statsig.com + - domain=*.sentry.io + + # Content Delivery & Mirrors + - domain=*.sourceforge.net + - domain=packagecloud.io + - domain=*.packagecloud.io + + # Schema & Configuration + - domain=json-schema.org + - domain=www.json-schema.org + - domain=json.schemastore.org + - domain=www.schemastore.org +log_dir: /tmp/boundary_logs +log_level: warn +proxy_port: 8087 \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/kubernetes-claude/main.tf b/infra/1-click/2-coder/templates/kubernetes-claude/main.tf index eaba9d3..acf863d 100644 --- a/infra/1-click/2-coder/templates/kubernetes-claude/main.tf +++ b/infra/1-click/2-coder/templates/kubernetes-claude/main.tf @@ -30,6 +30,12 @@ variable "namespace" { description = "The Kubernetes namespace to create workspaces in (must exist prior to creating workspaces). If the Coder host is itself running as a Pod on the same Kubernetes cluster as you are deploying workspaces to, set this to the same namespace." } +variable "host_ip" { + type = string + description = "The IP address to use for hostAliases in workspace pods. If empty, no hostAliases will be added." + default = "" +} + data "coder_parameter" "cpu" { name = "cpu" display_name = "CPU" @@ -103,8 +109,8 @@ data "coder_workspace" "me" {} data "coder_workspace_owner" "me" {} resource "coder_agent" "main" { - os = "linux" - arch = "amd64" + os = "linux" + arch = "amd64" # The following metadata blocks are optional. They are used to display # information about your workspace in the dashboard. You can remove them @@ -164,17 +170,36 @@ resource "coder_agent" "main" { } # Claude Code + data "coder_task" "me" {} +locals { + coder_host = replace(replace(data.coder_workspace.me.access_url, "https://", ""), "http://", "") + boundary_config = templatefile("${path.module}/boundary-config.yaml", { + CODER_DOMAIN = local.coder_host + }) +} + module "claude-code" { - source = "registry.coder.com/coder/claude-code/coder" - version = "4.6.0" - agent_id = coder_agent.main.id - workdir = "/home/coder" - subdomain = true - ai_prompt = data.coder_task.me.prompt + source = "registry.coder.com/coder/claude-code/coder" + version = "4.5.0" + agent_id = coder_agent.main.id + workdir = "/home/coder" + subdomain = true + ai_prompt = data.coder_task.me.prompt enable_aibridge = true - + enable_boundary = true + boundary_version = "v0.6.0" + + post_install_script = <<-EOF + #!/usr/bin/env bash + + # Coder Boundary Setup + + mkdir -p ~/.config/coder_boundary + echo '${base64encode(local.boundary_config)}' | base64 -d > ~/.config/coder_boundary/config.yaml + chmod 600 ~/.config/coder_boundary/config.yaml + EOF } resource "coder_ai_task" "task" { @@ -184,11 +209,11 @@ resource "coder_ai_task" "task" { # code-server module "code-server" { - count = data.coder_workspace.me.start_count - source = "registry.coder.com/coder/code-server/coder" - version = "1.4.2" + count = data.coder_workspace.me.start_count + source = "registry.coder.com/coder/code-server/coder" + version = "1.4.2" subdomain = true - agent_id = coder_agent.main.id + agent_id = coder_agent.main.id } resource "kubernetes_persistent_volume_claim_v1" "home" { @@ -221,11 +246,34 @@ resource "kubernetes_persistent_volume_claim_v1" "home" { } } +data "coder_workspace_preset" "standard" { + name = "Standard" + description = "A workspace preset." + icon = "/icon/standard.svg" + default = true + parameters = { + (data.coder_parameter.cpu.name) = "2" + (data.coder_parameter.memory.name) = "2" + (data.coder_parameter.home_disk_size.name) = "10" + } + prebuilds { + instances = 1 + } +} + +locals { + coder_bin = "/opt/coder/bin" + init_script = <<-EOF + if [ -x ${local.coder_bin}/coder ]; then + exec ${local.coder_bin}/coder agent + else + ${coder_agent.main.init_script} + fi + EOF +} + resource "kubernetes_deployment_v1" "main" { count = data.coder_workspace.me.start_count - depends_on = [ - kubernetes_persistent_volume_claim_v1.home - ] wait_for_rollout = false metadata { name = "coder-${data.coder_workspace.me.id}" @@ -236,12 +284,12 @@ resource "kubernetes_deployment_v1" "main" { "app.kubernetes.io/part-of" = "coder" "com.coder.resource" = "true" "com.coder.workspace.id" = data.coder_workspace.me.id - "com.coder.workspace.name" = data.coder_workspace.me.name - "com.coder.user.id" = data.coder_workspace_owner.me.id - "com.coder.user.username" = data.coder_workspace_owner.me.name } annotations = { "com.coder.user.email" = data.coder_workspace_owner.me.email + "com.coder.workspace.name" = data.coder_workspace.me.name + "com.coder.user.id" = data.coder_workspace_owner.me.id + "com.coder.user.username" = data.coder_workspace_owner.me.name } } @@ -254,9 +302,6 @@ resource "kubernetes_deployment_v1" "main" { "app.kubernetes.io/part-of" = "coder" "com.coder.resource" = "true" "com.coder.workspace.id" = data.coder_workspace.me.id - "com.coder.workspace.name" = data.coder_workspace.me.name - "com.coder.user.id" = data.coder_workspace_owner.me.id - "com.coder.user.username" = data.coder_workspace_owner.me.name } } strategy { @@ -283,13 +328,31 @@ resource "kubernetes_deployment_v1" "main" { run_as_non_root = true } + dynamic "host_aliases" { + for_each = var.host_ip != "" ? [1] : [] + content { + hostnames = [local.coder_host] + ip = var.host_ip + } + } + container { name = "dev" image = "codercom/enterprise-base:ubuntu" - image_pull_policy = "Always" - command = ["sh", "-c", coder_agent.main.init_script] + image_pull_policy = "IfNotPresent" + command = ["sh", "-c", local.init_script] security_context { run_as_user = "1000" + allow_privilege_escalation = true + privileged = true + } + env { + name = "CODER_AGENT_AUTH" + value = "token" + } + env { + name = "CODER_AGENT_URL" + value = data.coder_workspace.me.access_url } env { name = "CODER_AGENT_TOKEN" @@ -305,11 +368,18 @@ resource "kubernetes_deployment_v1" "main" { "memory" = "${data.coder_parameter.memory.value}Gi" } } + volume_mount { mount_path = "/home/coder" name = "home" read_only = false } + + volume_mount { + name = "coder-bin" + mount_path = local.coder_bin + read_only = true + } } volume { @@ -320,6 +390,14 @@ resource "kubernetes_deployment_v1" "main" { } } + volume { + name = "coder-bin" + host_path { + path = local.coder_bin + type = "Directory" + } + } + affinity { // This affinity attempts to spread out all workspace pods evenly across // nodes. diff --git a/infra/1-click/2-coder/templates/kubernetes/main.tf b/infra/1-click/2-coder/templates/kubernetes/main.tf index 38b206c..398a61f 100644 --- a/infra/1-click/2-coder/templates/kubernetes/main.tf +++ b/infra/1-click/2-coder/templates/kubernetes/main.tf @@ -30,6 +30,12 @@ variable "namespace" { description = "The Kubernetes namespace to create workspaces in (must exist prior to creating workspaces). If the Coder host is itself running as a Pod on the same Kubernetes cluster as you are deploying workspaces to, set this to the same namespace." } +variable "host_ip" { + type = string + description = "The IP address to use for hostAliases in workspace pods. If empty, no hostAliases will be added." + default = "" +} + data "coder_parameter" "cpu" { name = "cpu" display_name = "CPU" @@ -220,11 +226,19 @@ resource "kubernetes_persistent_volume_claim_v1" "home" { } } +locals { + coder_bin = "/opt/coder/bin" + init_script = <<-EOF + if [ -x ${local.coder_bin}/coder ]; then + exec ${local.coder_bin}/coder agent + else + ${coder_agent.main.init_script} + fi + EOF +} + resource "kubernetes_deployment_v1" "main" { count = data.coder_workspace.me.start_count - depends_on = [ - kubernetes_persistent_volume_claim_v1.home - ] wait_for_rollout = false metadata { name = "coder-${data.coder_workspace.me.id}" @@ -282,14 +296,30 @@ resource "kubernetes_deployment_v1" "main" { run_as_non_root = true } + dynamic "host_aliases" { + for_each = var.host_ip != "" ? [1] : [] + content { + hostnames = [replace(replace(data.coder_workspace.me.access_url, "https://", ""), "http://", "")] + ip = var.host_ip + } + } + container { name = "dev" image = "codercom/enterprise-base:ubuntu" - image_pull_policy = "Always" - command = ["sh", "-c", coder_agent.main.init_script] + image_pull_policy = "IfNotPresent" + command = ["sh", "-c", local.init_script] security_context { run_as_user = "1000" } + env { + name = "CODER_AGENT_AUTH" + value = "token" + } + env { + name = "CODER_AGENT_URL" + value = data.coder_workspace.me.access_url + } env { name = "CODER_AGENT_TOKEN" value = coder_agent.main.token @@ -304,11 +334,19 @@ resource "kubernetes_deployment_v1" "main" { "memory" = "${data.coder_parameter.memory.value}Gi" } } + volume_mount { mount_path = "/home/coder" name = "home" read_only = false } + + volume_mount { + name = "coder-bin" + mount_path = local.coder_bin + read_only = true + } + } volume { @@ -319,6 +357,14 @@ resource "kubernetes_deployment_v1" "main" { } } + volume { + name = "coder-bin" + host_path { + path = local.coder_bin + type = "Directory" + } + } + affinity { // This affinity attempts to spread out all workspace pods evenly across // nodes. diff --git a/infra/1-click/README.md b/infra/1-click/README.md index 48ee36d..4d8f7d5 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -1,27 +1,24 @@ # Requirements +Deploying this will take around ~30 min to setup everything (AWS resources, K8s Addons, DNS resolution, and Coder). Cleaning is ~20 min. The below items are required to run this: + +## OSS - [Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) - [AWS Account](https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-creating.html) + [CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) -- [Domain in Route53](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) -- [AWS Bedrock + Anthropic Agreement Completed](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) +- A Domain ([Route53](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) or [CloudFlare](https://www.cloudflare.com/learning/dns/how-to-buy-a-domain-name/)) -> [!IMPORTANT] -> Deploying will take around ~30min to setup everything (AWS resources, K8s Addons, DNS resolution, and Coder). -> Cleaning up will take ~20 minutes. +## Enterprise +- Same OSS requirements. +- An [Enterprise Coder License](https://coder.com/docs/admin/licensing). +- [AWS Bedrock/Anthropic Agreement Completed](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html). # Getting Started -1. Create or use an AWS account +1. Create or acquire an AWS account 2. Login as a user with AdministratorAccess -3. Setup your local machine to have an [AWS profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) - ->[!NOTE] -> When running `aws login` or `aws configure`, this will automatically setup a `default` profile for you. +3. Setup your local machine to have an [AWS profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). When running `aws login` or `aws configure`, this will automatically setup a `default` profile for you. -4. Purchase/Register a domain in Route53 and tie it to a public hosted zone. - -> [!IMPORTANT] -> When purchasing, it creates a hosted zone in a few minutes, wait for this. AND, if using an existing domain/zone, cleanup any conflicting A/AAAA/CNAME/TXT records. +4. Purchase/Register a domain in Route53 or CloudFlare and tie it to a public zone. When purchasing, it'll create a zone in a few minutes, wait for this. If using an existing domain/zone, cleanup any conflicting A/AAAA/CNAME/TXT records. 5. Fill the `coder.env` file with the following environment variables: @@ -32,6 +29,9 @@ CODER_DOMAIN_NAME=put.your.domain.here.com CODER_LICENSE=abcde1234..... (Optional) ``` +> [!IMPORTANT] +> Upon creating the initial deployment, DO NOT CHANGE THE DOMAIN. Clean up first, then recreate it. The entire infrastructure depends on the name. + 6. Initialize the project by running: ```bash @@ -41,7 +41,7 @@ CODER_LICENSE=abcde1234..... (Optional) 7. Finally, deploy by running: ```bash -set -a && source coder.env && set +a && ./1-plan-n-deploy.sh +./1-plan-n-deploy.sh ``` # Logging In @@ -58,10 +58,14 @@ Password: Th1s1sN0TS3CuR3!! # Cleaning Up -1. Just run: +1. [Delete all Coder workspaces](https://coder.com/docs/user-guides/workspace-lifecycle#deleting-workspaces). + +2. (Optional) If you've added a license to the .env file, the "Claude Code on Coder" template will be deployed. Update it's ["coder_workspace_preset.prebuilds.instances"](https://registry.terraform.io/providers/coder/coder/latest/docs/data-sources/workspace_preset#instances-1) attribute and set it to 0. + +3. Run the following script to tear down the deployment: ```bash -set -a && source coder.env && set +a && ./2-clean.sh +./2-clean.sh ``` # Troubleshooting @@ -96,6 +100,7 @@ set -a && source coder.env && set +a && ./2-clean.sh - Certificate - CertificateRequests - Order + - Challenge - ClusterSecretStore - ExternalSecrets - PushSecrets diff --git a/modules/k8s/bootstrap/cert-manager/main.tf b/modules/k8s/bootstrap/cert-manager/main.tf index 02c0781..b2de974 100644 --- a/modules/k8s/bootstrap/cert-manager/main.tf +++ b/modules/k8s/bootstrap/cert-manager/main.tf @@ -17,26 +17,6 @@ variable "cluster_name" { type = string } -variable "role_name" { - type = string - default = "" -} - -variable "policy_name" { - type = string - default = "" -} - -variable "policy_resource_region" { - type = string - default = "" -} - -variable "policy_resource_account" { - type = string - default = "" -} - variable "cluster_oidc_provider_arn" { type = string } @@ -67,24 +47,15 @@ variable "node_selector" { } } -## -# Create Default Resources? -## - -variable "create_default_cluster_issuer" { - type = bool - default = true +variable "tolerations" { + type = list(map(any)) + default = [] } ## # ACME Certificate Inputs ## -variable "issuer_private_key_secret_name" { - type = string - default = "issuer-account-key" -} - variable "acme_server_url" { type = string default = "https://acme-v02.api.letsencrypt.org/directory" @@ -99,51 +70,12 @@ data "aws_region" "this" {} data "aws_caller_identity" "this" {} -locals { - region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region - account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "crt-mgr" : var.policy_name - role_name = var.role_name == "" ? "crt-mgr" : var.role_name -} - -module "policy" { - source = "../../../security/policy" - name = local.policy_name - path = "/${var.cluster_name}/${data.aws_region.this.region}/" - description = "CertManager for Route53 Policy" - policy_json = data.aws_iam_policy_document.route53.json -} - -module "oidc-role" { - source = "../../../security/role/access-entry" - name = local.role_name - path = "/${var.cluster_name}/${data.aws_region.this.region}/" - cluster_name = var.cluster_name - policy_arns = { - "CertManagerRoute53" = module.policy.policy_arn - } - cluster_policy_arns = { - "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", - } - oidc_principals = { - "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] - } - tags = var.tags -} - resource "kubernetes_namespace" "this" { metadata { name = var.namespace } } -locals { - global_tolerations = [{ - key = "CriticalAddonsOnly" - operator = "Exists" - }] -} - resource "helm_release" "cert-manager" { name = "cert-manager" namespace = kubernetes_namespace.this.metadata[0].name @@ -162,15 +94,15 @@ resource "helm_release" "cert-manager" { enabled = true } nodeSelector = var.node_selector - tolerations = local.global_tolerations + tolerations = var.tolerations webhook = { - tolerations = local.global_tolerations + tolerations = var.tolerations } cainjector = { - tolerations = local.global_tolerations + tolerations = var.tolerations } startupapicheck = { - tolerations = local.global_tolerations + tolerations = var.tolerations } })] } @@ -179,47 +111,94 @@ resource "helm_release" "cert-manager" { # Use Route53 for the DNS01 Challenge Provider ## -variable "use_route53" { - type = bool - default = false +variable "r53_config" { + type = object({ + enabled = bool + region = optional(string, "") + account = optional(string, "") + role_name = optional(string, "crt-mgr") + policy_name = optional(string, "crt-mgr") + }) + default = { + enabled = false + region = "" + account = "" + role_name = "crt-mgr" + policy_name = "crt-mgr" + } } -variable "route53_region" { - type = string - default = "us-east-2" +locals { + region = var.r53_config.region == "" ? data.aws_region.this.region : var.r53_config.region + account_id = var.r53_config.account == "" ? data.aws_caller_identity.this.account_id : var.r53_config.account } -variable "route53_sa_role" { - type = string - default = "" +module "policy" { + + count = var.r53_config.enabled ? 1 : 0 + + source = "../../../security/policy" + name = var.r53_config.policy_name + path = "/${var.cluster_name}/${local.region}/" + description = "Cert-Manager for R53 Policy" + policy_json = data.aws_iam_policy_document.route53.json } -resource "kubernetes_service_account" "route53" { +module "oidc-role" { + + count = var.r53_config.enabled ? 1 : 0 + + source = "../../../security/role/access-entry" + name = var.r53_config.role_name + path = "/${var.cluster_name}/${local.region}/" + cluster_name = var.cluster_name + policy_arns = { + "CertManagerR53" = module.policy[0].policy_arn + } + cluster_policy_arns = { + "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", + } + oidc_principals = { + "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] + } + tags = var.tags +} + +resource "kubernetes_service_account" "r53" { + + count = var.r53_config.enabled ? 1 : 0 + metadata { - name = "cert-manager-acme-dns01-route53" + name = "${var.r53_config.role_name}" namespace = kubernetes_namespace.this.metadata[0].name annotations = { - "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn + "eks.amazonaws.com/role-arn" = module.oidc-role[0].role_arn } } } -resource "kubernetes_role" "route53" { +resource "kubernetes_role" "r53" { + + count = var.r53_config.enabled ? 1 : 0 + metadata { - name = "cert-manager-acme-dns01-route53-tokenrequest" + name = "${var.r53_config.role_name}-tokenrequest" namespace = kubernetes_namespace.this.metadata[0].name } rule { api_groups = [""] resources = ["serviceaccounts/token"] - resource_names = [kubernetes_service_account.route53.metadata[0].name] + resource_names = [kubernetes_service_account.r53[0].metadata[0].name] verbs = ["create"] } } -resource "kubernetes_role_binding" "route53" { +resource "kubernetes_role_binding" "r53" { + + count = var.r53_config.enabled ? 1 : 0 + metadata { - name = "cert-manager-acme-dns01-route53-tokenrequest" + name = "${var.r53_config.role_name}-tokenrequest" namespace = kubernetes_namespace.this.metadata[0].name } subject { @@ -230,7 +209,7 @@ resource "kubernetes_role_binding" "route53" { role_ref { api_group = "rbac.authorization.k8s.io" kind = "Role" - name = kubernetes_role.route53.metadata[0].name + name = kubernetes_role.r53[0].metadata[0].name } } @@ -238,112 +217,42 @@ resource "kubernetes_role_binding" "route53" { # Use CloudFlare for the DNS01 Challenge Provider ## -variable "use_cloudflare" { - type = bool - default = true -} - -variable "cloudflare_token_secret_name" { - type = string - default = "cloudflare-token" -} - -variable "cloudflare_token_secret_key" { - type = string - default = "token.key" -} - -variable "cloudflare_token_secret" { - type = string +variable "cf_config" { + type = object({ + enabled = bool + name = optional(string, "cloudflare") + key = optional(string, "token.key") + token = string + email = string + }) + default = { + enabled = false + name = "cloudflare" + key = "token.key" + token = "" + email = "" + } sensitive = true - default = "" } -variable "cloudflare_token_secret_email" { - type = string - sensitive = true - default = "" +locals { + cf_annot_sec_key = "custom.kubernetes.secret/key" + cf_annot_email_key = "custom.kubernetes.secret/email" } resource "kubernetes_secret" "cloudflare" { - metadata { - name = var.cloudflare_token_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - } - data = { - "${var.cloudflare_token_secret_key}" = var.cloudflare_token_secret - } -} -# ---------------- + count = var.cf_config.enabled ? 1 : 0 -locals { - dns01_cf = { - cloudflare = { - apiTokenSecretRef = { - key = var.cloudflare_token_secret_key - name = kubernetes_secret.cloudflare.metadata[0].name - } - email = var.cloudflare_token_secret_email - } - } - dns01_r53 = { - route53 = { - region = var.route53_region - auth = { - kubernetes = { - serviceAccountRef = { - name = kubernetes_service_account.route53.metadata[0].name - } - } - } + metadata { + name = var.cf_config.name + namespace = kubernetes_namespace.this.metadata[0].name + annotations = { + "${local.cf_annot_sec_key}" = var.cf_config.key + "${local.cf_annot_email_key}" = var.cf_config.email } } -} - -## -# Setup the default Cluster Issuer -## - -resource "kubernetes_manifest" "default-issuer" { - - depends_on = [ - helm_release.cert-manager, - kubernetes_secret.cloudflare - ] - - count = var.create_default_cluster_issuer ? 1 : 0 - - field_manager { - force_conflicts = true - } - manifest = { - apiVersion = "cert-manager.io/v1" - kind = "ClusterIssuer" - metadata = { - labels = {} - name = "issuer" - } - spec = { - acme = { - privateKeySecretRef = { - name = var.issuer_private_key_secret_name - } - server = var.acme_server_url - solvers = [ - { - dns01 = { - cloudflare = { - apiTokenSecretRef = { - key = var.cloudflare_token_secret_key - name = kubernetes_secret.cloudflare.metadata[0].name - } - email = var.cloudflare_token_secret_email - } - } - } - ] - } - } + data = { + "${var.cf_config.key}" = var.cf_config.token } } \ No newline at end of file diff --git a/modules/k8s/bootstrap/coder-server/main.tf b/modules/k8s/bootstrap/coder-server/main.tf index fca8092..51b4a22 100644 --- a/modules/k8s/bootstrap/coder-server/main.tf +++ b/modules/k8s/bootstrap/coder-server/main.tf @@ -10,9 +10,6 @@ terraform { kubernetes = { source = "hashicorp/kubernetes" } - tls = { - source = "hashicorp/tls" - } } } @@ -26,32 +23,22 @@ variable "cluster_oidc_provider_arn" { variable "policy_resource_region" { type = string - default = "" + default = null } variable "policy_resource_account" { type = string - default = "" + default = null } variable "policy_name" { type = string - default = "" + default = null } variable "role_name" { type = string - default = "" -} - -## -# TLS/SSL Inputs -## - -variable "cloudflare_api_token" { - type = string - default = "" - sensitive = true + default = null } ## @@ -72,39 +59,48 @@ variable "helm_version" { default = "2.25.1" } -variable "image_repo" { - type = string - default = "ghcr.io/coder/coder" -} - -variable "image_tag" { - type = string - default = "latest" -} - -variable "image_pull_policy" { - type = string - default = "IfNotPresent" -} - -variable "image_pull_secrets" { - type = list(string) - default = [] -} - -variable "replica_count" { - type = number - default = 0 +variable "coder" { + type = object({ + access_url = string + wildcard_url = string + redirect = optional(bool, true) + image_repo = optional(string, "ghcr.io/coder/coder") + image_tag = optional(string, "latest") + image_pull_policy = optional(string, "IfNotPresent") + image_pull_secrets = optional(list(string), null) + experiments = optional(list(string), null) + csp_policy = optional(string, null) + env_vars = optional(map(string), {}) + rep_cnt = optional(number, 1) + prov_rep_cnt = optional(number, 2) + prov_force_cancel_interval = optional(string, "10m0s") + quiet_hours = optional(string, "CRON_TZ=America/Los_Angeles 50 23 * * *") + allow_custom_quiet = optional(bool, true) + tf_debug_mode = optional(bool, true) + trace_logs = optional(bool, true) + swagger_enable = optional(bool, true) + update_check = optional(bool, true) + cli_upgr_msg = optional(bool, true) + log_filter = optional(string, ".*") + }) } -variable "env_vars" { - type = map(string) - default = {} +variable "prom" { + type = object({ + enable = optional(bool, true) + collect_agent_status = optional(bool, true) + collect_db_metrics = optional(bool, true) + }) + default = { + enable = false + collect_agent_status = false + collect_db_metrics = false + } } -variable "load_balancer_class" { - type = string - default = "service.k8s.aws/nlb" +variable "termination_grace_period" { + type = number + default = 600 } variable "resource_request" { @@ -118,6 +114,11 @@ variable "resource_request" { } } +variable "tags" { + type = map(string) + default = {} +} + variable "resource_limit" { type = object({ cpu = string @@ -129,12 +130,17 @@ variable "resource_limit" { } } -variable "service_annotations" { +variable "svc_annot" { type = map(string) default = {} } -variable "service_account_annotations" { +variable "lb_class" { + type = string + default = "service.k8s.aws/nlb" +} + +variable "svc_acc_annot" { type = map(string) default = {} } @@ -154,7 +160,7 @@ variable "tolerations" { default = [] } -variable "topology_spread_constraints" { +variable "topology_spread" { type = list(object({ max_skew = number topology_key = string @@ -167,7 +173,7 @@ variable "topology_spread_constraints" { default = [] } -variable "pod_anti_affinity_preferred_during_scheduling_ignored_during_execution" { +variable "pod_aaf_pref_sched_ie" { type = list(object({ weight = number pod_affinity_term = object({ @@ -180,234 +186,256 @@ variable "pod_anti_affinity_preferred_during_scheduling_ignored_during_execution default = [] } -variable "primary_access_url" { - type = string -} - -variable "wildcard_access_url" { - type = string -} - -variable "termination_grace_period_seconds" { - type = number - default = 600 -} - -variable "ssl_cert_config" { +variable "cert_config" { type = object({ + create_secret = bool name = string - caissuer = optional(string, "issuer") - secretissuer = optional(string, "issuer") - create_secret = optional(bool, true) + kind = optional(string, "ClusterIssuer") + issuer = optional(string, "issuer") + store = optional(string, "issuer") }) default = { - name = "coder-tls" - caissuer = "issuer" - secretissuer = "issuer" create_secret = true + name = "coder-tls" + kind = "ClusterIssuer" + issuer = "issuer" + store = "issuer" } } -variable "db_secret_name" { - type = string - default = "postgres" -} - -variable "db_secret_key" { - type = string - default = "url" -} - -variable "db_secret_url" { - type = string - sensitive = true -} - -variable "enable_oidc" { - type = bool - default = true +variable "db" { + type = object({ + url = string + username = string + password = string + db = optional(string, "coder") + pg_auth = optional(string, "password") + }) } -variable "oidc_config" { +variable "oidc" { type = object({ + enable = bool sign_in_text = string icon_url = string - scopes = list(string) + scopes = optional(list(string), null) email_domain = string + issuer_url = optional(string, null) + client_id = optional(string, null) + client_secret = optional(string, null) }) default = { - sign_in_text = "" - icon_url = "" - scopes = [] - email_domain = "" + enable = false + sign_in_text = null + icon_url = null + scopes = null + email_domain = null + issuer_url = null + client_id = null + client_secret = null } } -variable "oidc_secret_name" { - type = string - default = "oidc" -} - -variable "oidc_secret_issuer_url_key" { - type = string - default = "issuer-url" -} - -variable "oidc_secret_issuer_url" { - type = string - sensitive = true - default = "" -} - -variable "oidc_secret_client_id_key" { - type = string - default = "client-id" -} - -variable "oidc_secret_client_id" { - type = string - sensitive = true - default = "" -} - -variable "oidc_secret_client_secret_key" { - type = string - default = "client-secret" -} - -variable "oidc_secret_client_secret" { - type = string - sensitive = true - default = "" -} - -variable "enable_oauth" { - type = bool - default = true -} - -variable "oauth_secret_name" { - type = string - default = "oauth" -} - -variable "oauth_secret_client_id_key" { - type = string - default = "client-id" -} - -variable "oauth_secret_client_id" { - type = string - sensitive = true - default = "" -} - -variable "oauth_secret_client_secret_key" { - type = string - default = "client-secret" -} - -variable "oauth_secret_client_secret" { - type = string - sensitive = true - default = "" -} - -variable "enable_github_external_auth" { - type = bool - default = true -} - -variable "github_external_auth_config" { +variable "oauth2" { type = object({ - id = string - type = optional(string, "github") + enable = bool + default_provider_enable = bool + allow_signups = bool + device_flow = bool + allowed_orgs = list(string) # Empty list means allow everyone + client_id = string + client_secret = string + use_extern_auth = bool }) default = { - id = "primary-github" - type = "github" + enable = false + default_provider_enable = false + allow_signups = true + device_flow = false + allowed_orgs = [] + client_id = null + client_secret = null + use_extern_auth = false } } -variable "github_external_auth_secret_name" { - type = string - default = "github-external-auth" -} - -variable "github_external_auth_secret_client_id_key" { - type = string - default = "client-id" -} - -variable "github_external_auth_secret_client_id" { - type = string - sensitive = true - default = "" -} - -variable "github_external_auth_secret_client_secret_key" { - type = string - default = "client-secret" -} - -variable "github_external_auth_secret_client_secret" { - type = string - sensitive = true - default = "" -} - -variable "coder_builtin_provisioner_count" { - type = number - default = 3 -} - -variable "coder_experiments" { - type = list(string) - default = [] -} - -variable "coder_github_allowed_orgs" { - type = list(string) +variable "extern_auth" { + type = list(object({ + id = string + type = string + client_id = string + client_secret = string + auth_url = optional(string, null) + token_url = optional(string, null) + revoke_url = optional(string, null) + validate_url = optional(string, null) + regex = optional(string, null) + })) default = [] } -variable "openai_llm_endpoint" { - type = string - sensitive = true - default = "" -} - -variable "openai_llm_secret_name" { - type = string - default = "coder-openai-llm-key" +variable "aibridge" { + type = object({ + enabled = bool + anthropic = optional(object({ + url = string + key = string + }), null) + openai = optional(object({ + url = string + key = string + }), null) + bedrock = optional(object({ + url = optional(string, null) + region = optional(string, null) + access_id = string + secret_id = string + model = optional(string, "global.anthropic.claude-sonnet-4-5-20250929-v1:0") + fast_model = optional(string, "global.anthropic.claude-haiku-4-5-20251001-v1:0") + }), null) + }) + default = { + enabled = false + anthropic = null + openai = null + bedrock = null + } } -variable "openai_llm_key" { - type = string - sensitive = true - default = "" +locals { + coder = { + CODER_ACCESS_URL = var.coder.access_url + CODER_WILDCARD_ACCESS_URL = var.coder.wildcard_url + CODER_REDIRECT_TO_ACCESS_URL = var.coder.redirect + + CODER_ENABLE_TERRAFORM_DEBUG_MODE = var.coder.tf_debug_mode + CODER_TRACE_LOGS = var.coder.trace_logs + CODER_LOG_FILTER = var.coder.log_filter + CODER_SWAGGER_ENABLE = var.coder.swagger_enable + CODER_UPDATE_CHECK = var.coder.update_check + CODER_CLI_UPGRADE_MESSAGE = var.coder.cli_upgr_msg + + CODER_PROVISIONER_DAEMONS = var.coder.prov_rep_cnt + CODER_PROVISIONER_FORCE_CANCEL_INTERVAL = var.coder.prov_force_cancel_interval + CODER_QUIET_HOURS_DEFAULT_SCHEDULE = var.coder.quiet_hours + CODER_ALLOW_CUSTOM_QUIET_HOURS = var.coder.allow_custom_quiet + } + prom = { + CODER_PROMETHEUS_ENABLE = var.prom.enable + CODER_PROMETHEUS_COLLECT_AGENT_STATS = var.prom.collect_agent_status + CODER_PROMETHEUS_COLLECT_DB_METRICS = var.prom.collect_db_metrics + } + db = { + CODER_PG_CONNECTION_URL = "postgresql://${var.db.username}:${var.db.password}@${var.db.url}/${var.db.db}" + CODER_PG_AUTH = var.db.pg_auth + } + oidc = !var.oidc.enable ? {} : { + CODER_OIDC_ISSUER_URL = var.oidc.issuer_url + CODER_OIDC_CLIENT_ID = var.oidc.client_id + CODER_OIDC_CLIENT_SECRET = var.oidc.client_secret + CODER_OIDC_SIGN_IN_TEXT = var.oidc.sign_in_text + CODER_OIDC_ICON_URL = var.oidc.icon_url + CODER_OIDC_SCOPES = join(",", var.oidc.scopes) + CODER_OIDC_EMAIL_DOMAIN = var.oidc.email_domain + } + oauth2 = !var.oauth2.enable ? { + CODER_OAUTH2_GITHUB_DEFAULT_PROVIDER_ENABLE = false + } : { + CODER_OAUTH2_GITHUB_DEFAULT_PROVIDER_ENABLE = var.oauth2.default_provider_enable + CODER_OAUTH2_GITHUB_CLIENT_ID = var.oauth2.client_id + CODER_OAUTH2_GITHUB_CLIENT_SECRET = var.oauth2.client_secret + CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS = var.oauth2.allow_signups + CODER_OAUTH2_GITHUB_DEVICE_FLOW = var.oauth2.device_flow + CODER_OAUTH2_GITHUB_ALLOW_EVERYONE = "${length(var.oauth2.allowed_orgs) == 0}" + CODER_OAUTH2_GITHUB_ALLOWED_ORGS = join(",", var.oauth2.allowed_orgs) + } + extern_auth = merge([for index, obj in var.extern_auth : { + "CODER_EXTERNAL_AUTH_${index}_ID" = obj.id + "CODER_EXTERNAL_AUTH_${index}_TYPE" = obj.type + "CODER_EXTERNAL_AUTH_${index}_CLIENT_ID" = obj.client_id + "CODER_EXTERNAL_AUTH_${index}_CLIENT_SECRET" = obj.client_secret + "CODER_EXTERNAL_AUTH_${index}_AUTH_URL" = obj.auth_url + "CODER_EXTERNAL_AUTH_${index}_TOKEN_URL" = obj.token_url + "CODER_EXTERNAL_AUTH_${index}_REVOKE_URL" = obj.revoke_url + "CODER_EXTERNAL_AUTH_${index}_VALIDATE_URL" = obj.validate_url + "CODER_EXTERNAL_AUTH_${index}_REGEX" = obj.regex + }]...) + anthropic = var.aibridge.anthropic == null ? {} : { + CODER_AIBRIDGE_ANTHROPIC_BASE_URL = var.aibridge.anthropic.url + CODER_AIBRIDGE_ANTHROPIC_KEY = var.aibridge.anthropic.key + } + openai = var.aibridge.openai == null ? {} : { + CODER_AIBRIDGE_OPENAI_BASE_URL = var.aibridge.openai.url + CODER_AIBRIDGE_OPENAI_KEY = var.aibridge.openai.key + } + bedrock = var.aibridge.bedrock == null ? {} : { + CODER_AIBRIDGE_BEDROCK_BASE_URL = var.aibridge.bedrock.url + CODER_AIBRIDGE_BEDROCK_REGION = var.aibridge.bedrock.region + CODER_AIBRIDGE_BEDROCK_ACCESS_KEY = var.aibridge.bedrock.access_id + CODER_AIBRIDGE_BEDROCK_ACCESS_KEY_SECRET = var.aibridge.bedrock.secret_id + CODER_AIBRIDGE_BEDROCK_MODEL = var.aibridge.bedrock.model + CODER_AIBRIDGE_BEDROCK_SMALL_FAST_MODEL = var.aibridge.bedrock.fast_model + } + aibridge = merge( + local.anthropic, + local.openai, + local.bedrock, + { CODER_AIBRIDGE_ENABLED = var.aibridge.enabled } + ) + secrets = merge({ + CODER_AIBRIDGE_ANTHROPIC_KEY = try(local.anthropic["CODER_AIBRIDGE_ANTHROPIC_KEY"], null) + CODER_AIBRIDGE_OPENAI_KEY = try(local.openai["CODER_AIBRIDGE_OPENAI_KEY"], null) + CODER_AIBRIDGE_BEDROCK_ACCESS_KEY_SECRET = try(local.bedrock["CODER_AIBRIDGE_BEDROCK_ACCESS_KEY_SECRET"], null) + CODER_OAUTH2_GITHUB_CLIENT_SECRET = try(local.oauth2["CODER_OAUTH2_GITHUB_CLIENT_SECRET"], null) + CODER_OIDC_CLIENT_SECRET = try(local.oidc["CODER_OIDC_CLIENT_SECRET"], null) + CODER_PG_CONNECTION_URL = try(local.db["CODER_PG_CONNECTION_URL"], null) + }, { for index, obj in var.extern_auth : + "CODER_EXTERNAL_AUTH_${index}_CLIENT_SECRET" => obj.client_secret + }) + secret_key = "key" + secret_keys = keys(local.secrets) + env = concat([ for k,v in merge( + local.coder, + local.prom, + local.db, + local.oidc, + local.oauth2, + local.extern_auth, + local.aibridge + ) : { + name = k, + value = tostring(v) + } if lookup(local.secrets, k, null) == null ], [ + for k,v in local.secrets : { + name = k, + valueFrom = { + secretKeyRef = { + name = replace(lower(k), "_", "-"), + key = local.secret_key + } + } + } if v != null + ]) } -variable "anthropic_llm_endpoint" { - type = string - sensitive = true - default = "" +output "env" { + value = local.env } -variable "anthropic_llm_secret_name" { - type = string - default = "coder-anthropic-llm-key" -} +resource "kubernetes_secret_v1" "coder" { -variable "anthropic_llm_key" { - type = string - sensitive = true - default = "" -} + for_each = toset(local.secret_keys) -variable "tags" { - type = map(string) - default = {} + metadata { + name = replace(lower(each.key), "_", "-") + namespace = kubernetes_namespace_v1.this.metadata[0].name + annotations = { + "custom.kubernetes.secret/key" = local.secret_key + } + } + data = { + "${local.secret_key}" = sensitive(local.secrets[each.key]) + } } data "aws_region" "this" {} @@ -415,171 +443,8 @@ data "aws_region" "this" {} data "aws_caller_identity" "this" {} locals { - oidc_secret_issuer_url = var.enable_oidc ? { - name = "CODER_OIDC_ISSUER_URL" - valueFrom = { - secretKeyRef = { - name = var.oidc_secret_name - key = var.oidc_secret_issuer_url_key - } - } - } : null - oidc_client_id = var.enable_oidc ? { - name = "CODER_OIDC_CLIENT_ID" - valueFrom = { - secretKeyRef = { - name = var.oidc_secret_name - key = var.oidc_secret_client_id_key - } - } - } : null - oidc_client_secret = var.enable_oidc ? { - name = "CODER_OIDC_CLIENT_SECRET" - valueFrom = { - secretKeyRef = { - name = var.oidc_secret_name - key = var.oidc_secret_client_secret_key - } - } - } : null - oauth2_github_client_id = var.enable_oauth ? { - name = "CODER_OAUTH2_GITHUB_CLIENT_ID" - valueFrom = { - secretKeyRef = { - name = var.oauth_secret_name - key = var.oauth_secret_client_id_key - } - } - } : null - oauth2_github_client_secret = var.enable_oauth ? { - name = "CODER_OAUTH2_GITHUB_CLIENT_SECRET" - valueFrom = { - secretKeyRef = { - name = var.oauth_secret_name - key = var.oauth_secret_client_secret_key - } - } - } : null - external_auth_github_client_id = var.enable_github_external_auth ? { - name = "CODER_EXTERNAL_AUTH_0_CLIENT_ID" - valueFrom = { - secretKeyRef = { - name = var.oauth_secret_name - key = var.oauth_secret_client_id_key - } - } - } : null - external_auth_github_client_secret = var.enable_github_external_auth ? { - name = "CODER_EXTERNAL_AUTH_0_CLIENT_SECRET" - valueFrom = { - secretKeyRef = { - name = var.oauth_secret_name - key = var.oauth_secret_client_secret_key - } - } - } : null -} - -locals { - github_allow_everyone = length(var.coder_github_allowed_orgs) == 0 - primary_env_vars = merge({ - CODER_ACCESS_URL = var.primary_access_url - CODER_WILDCARD_ACCESS_URL = var.wildcard_access_url - CODER_REDIRECT_TO_ACCESS_URL = true - CODER_PG_AUTH = "password" - - CODER_ENABLE_TERRAFORM_DEBUG_MODE = true - CODER_TRACE_LOGS = true - CODER_LOG_FILTER = ".*" - CODER_SWAGGER_ENABLE = true - CODER_UPDATE_CHECK = true - CODER_CLI_UPGRADE_MESSAGE = true - - CODER_PROVISIONER_DAEMONS = var.coder_builtin_provisioner_count - CODER_PROVISIONER_FORCE_CANCEL_INTERVAL = "10m0s" - CODER_QUIET_HOURS_DEFAULT_SCHEDULE = "CRON_TZ=America/Los_Angeles 50 23 * * *" - CODER_ALLOW_CUSTOM_QUIET_HOURS = true - - CODER_PROMETHEUS_ENABLE = true - CODER_PROMETHEUS_COLLECT_AGENT_STATS = true - CODER_PROMETHEUS_COLLECT_DB_METRICS = true - # CODER_PROMETHEUS_ADDRESS = "127.0.0.1:${var.prometheus_port}" - - # CODER_AIBRIDGE_BEDROCK_MODEL = "global.anthropic.claude-sonnet-4-5-20250929-v1:0" - # CODER_AIBRIDGE_BEDROCK_SMALL_FAST_MODEL = "global.anthropic.claude-haiku-4-5-20251001-v1:0" - - # Experimental Coder Features - # CODER_EXPERIMENTS = join(",", var.coder_experiments) - # Needed by the ai-tasks experiment to embed workspace apps running on subdomains in iframes - CODER_ADDITIONAL_CSP_POLICY = "frame-src ${var.primary_access_url}" - }, merge(var.enable_oidc ? { - CODER_OIDC_SIGN_IN_TEXT = var.oidc_config.sign_in_text - CODER_OIDC_ICON_URL = var.oidc_config.icon_url - CODER_OIDC_SCOPES = join(",", var.oidc_config.scopes) - CODER_OIDC_EMAIL_DOMAIN = var.oidc_config.email_domain - } : {}, merge(var.enable_oauth ? { - CODER_OAUTH2_GITHUB_DEFAULT_PROVIDER_ENABLE = false - CODER_OAUTH2_GITHUB_ALLOW_SIGNUPS = true - CODER_OAUTH2_GITHUB_DEVICE_FLOW = false - "${local.github_allow_everyone ? "CODER_OAUTH2_GITHUB_ALLOW_EVERYONE" : "CODER_OAUTH2_GITHUB_ALLOWED_ORGS"}" = "${local.github_allow_everyone ? "true" : join(",", var.coder_github_allowed_orgs)}" - } : {}, var.enable_github_external_auth ? { - CODER_EXTERNAL_AUTH_0_ID = "primary-github" - CODER_EXTERNAL_AUTH_0_TYPE = "github" - } : {}))) - env_vars = concat([ - for k, v in merge(local.primary_env_vars, var.env_vars) : { name = k, value = tostring(v) } - ], concat([{ - name = "CODER_PG_CONNECTION_URL" - valueFrom = { - secretKeyRef = { - name = var.db_secret_name - key = var.db_secret_key - } - } - }, - # local.oidc_secret_issuer_url, - # local.oidc_client_id, - # local.oidc_client_secret, - # local.oauth2_github_client_id, - # local.oauth2_github_client_secret, - # local.external_auth_github_client_id, - # local.external_auth_github_client_secret, - ], concat(var.anthropic_llm_endpoint != "" ? [{ - name = "CODER_AIBRIDGE_ANTHROPIC_KEY" - valueFrom = { - secretKeyRef = { - name = kubernetes_secret.anthropic-llm-secret[0].metadata[0].name - key = "key" - } - } - }, { - name = "CODER_AIBRIDGE_ANTHROPIC_BASE_URL" - valueFrom = { - secretKeyRef = { - name = kubernetes_secret.anthropic-llm-secret[0].metadata[0].name - key = "base_url" - } - } - }] : [], - var.openai_llm_endpoint != "" ? [{ - name = "CODER_AIBRIDGE_OPENAI_KEY" - valueFrom = { - secretKeyRef = { - name = kubernetes_secret.openai-llm-secret[0].metadata[0].name - key = "key" - } - } - }, { - name = "CODER_AIBRIDGE_OPENAI_BASE_URL" - valueFrom = { - secretKeyRef = { - name = kubernetes_secret.openai-llm-secret[0].metadata[0].name - key = "base_url" - } - } - }] : []))) - pod_anti_affinity_preferred_during_scheduling_ignored_during_execution = [ - for k, v in var.pod_anti_affinity_preferred_during_scheduling_ignored_during_execution : { + pod_aaf_pref_sched_ie = [ + for k, v in var.pod_aaf_pref_sched_ie : { weight = v.weight podAffinityTerm = { labelSelector = { @@ -589,8 +454,8 @@ locals { } } ] - topology_spread_constraints = [ - for k, v in var.topology_spread_constraints : { + topology_spread = [ + for k, v in var.topology_spread : { maxSkew = v.max_skew topologyKey = v.topology_key whenUnsatisfiable = v.when_unsatisfiable @@ -603,14 +468,16 @@ locals { } locals { - region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region - account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "coder-srv" : var.policy_name - role_name = var.role_name == "" ? "coder-srv" : var.role_name + region = var.policy_resource_region == null ? data.aws_region.this.region : var.policy_resource_region + account_id = var.policy_resource_account == null ? data.aws_caller_identity.this.account_id : var.policy_resource_account + policy_name = var.policy_name == null ? "coder-srv" : var.policy_name + role_name = var.role_name == null ? "coder-srv" : var.role_name } module "provisioner-policy" { - count = var.coder_builtin_provisioner_count == 0 ? 0 : 1 + + count = var.coder.prov_rep_cnt == 0 ? 0 : 1 + source = "../../../security/policy" name = local.policy_name path = "/${var.cluster_name}/${data.aws_region.this.region}/" @@ -619,7 +486,9 @@ module "provisioner-policy" { } module "provisioner-oidc-role" { - count = var.coder_builtin_provisioner_count == 0 ? 0 : 1 + + count = var.coder.prov_rep_cnt == 0 ? 0 : 1 + source = "../../../security/role/access-entry" name = local.role_name path = "/${var.cluster_name}/${data.aws_region.this.region}/" @@ -635,93 +504,25 @@ module "provisioner-oidc-role" { tags = var.tags } -resource "kubernetes_namespace" "this" { +resource "kubernetes_namespace_v1" "this" { metadata { name = var.namespace } } locals { - ssl_vol_friendly_name = replace(var.ssl_cert_config.name, ".", "-") -} - -resource "kubernetes_secret" "pg-connection" { - metadata { - name = var.db_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - } - data = { - "${var.db_secret_key}" = var.db_secret_url - } - type = "Opaque" + ssl_vol_friendly_name = replace(var.cert_config.name, ".", "-") } -resource "kubernetes_secret" "oidc" { - metadata { - name = var.oidc_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - } - data = { - "${var.oidc_secret_issuer_url_key}" = var.oidc_secret_issuer_url - "${var.oidc_secret_client_id_key}" = var.oidc_secret_client_id - "${var.oidc_secret_client_secret_key}" = var.oidc_secret_client_secret - } - type = "Opaque" -} - -resource "kubernetes_secret" "oauth" { - metadata { - name = var.oauth_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - } - data = { - "${var.oauth_secret_client_id_key}" = var.oauth_secret_client_id - "${var.oauth_secret_client_secret_key}" = var.oauth_secret_client_secret - } - type = "Opaque" -} - -resource "kubernetes_secret" "external_auth" { - metadata { - name = var.github_external_auth_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - } - data = { - "${var.github_external_auth_secret_client_id_key}" = var.github_external_auth_secret_client_id - "${var.github_external_auth_secret_client_secret_key}" = var.github_external_auth_secret_client_secret - } - type = "Opaque" -} - -resource "kubernetes_secret" "openai-llm-secret" { - count = var.openai_llm_endpoint != "" ? 1 : 0 - metadata { - name = var.openai_llm_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - } - data = { - base_url = var.openai_llm_endpoint - key = var.openai_llm_key - } - type = "Opaque" -} - -resource "kubernetes_secret" "anthropic-llm-secret" { - count = var.anthropic_llm_endpoint != "" ? 1 : 0 - metadata { - name = var.anthropic_llm_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - } - data = { - base_url = var.anthropic_llm_endpoint - key = var.anthropic_llm_key - } - type = "Opaque" -} +## +# Store SSL/TLS certs in Secrets Manager. +# Used to avoid throttling Let's Encrypt: +# https://letsencrypt.org/docs/rate-limits/ +## locals { - common_name = trimprefix(trimprefix(var.primary_access_url, "https://"), "http://") - wildcard_name = trimprefix(trimprefix(var.wildcard_access_url, "https://"), "http://") + common_name = trimprefix(trimprefix(var.coder.access_url, "https://"), "http://") + wildcard_name = trimprefix(trimprefix(var.coder.wildcard_url, "https://"), "http://") cert_refresh_interval = "2160h" # 90 days cert_renew_before = "360h" # 15 days secret_refresh_interval = "1812h0m0s" # 75.5 days @@ -753,13 +554,13 @@ resource "kubernetes_manifest" "pull" { apiVersion = "external-secrets.io/v1" kind = "ExternalSecret" metadata = { - name = var.ssl_cert_config.name - namespace = kubernetes_namespace.this.metadata[0].name + name = var.cert_config.name + namespace = kubernetes_namespace_v1.this.metadata[0].name } spec = { secretStoreRef = { kind = "ClusterSecretStore" - name = var.ssl_cert_config.secretissuer + name = var.cert_config.store } refreshPolicy = "Periodic" refreshInterval = local.secret_refresh_interval @@ -775,12 +576,12 @@ resource "kubernetes_manifest" "pull" { } annotations = { "cert-manager.io/alt-names" = "${local.wildcard_name},${local.common_name}" - "cert-manager.io/certificate-name" = var.ssl_cert_config.name + "cert-manager.io/certificate-name" = var.cert_config.name "cert-manager.io/common-name" = local.common_name "cert-manager.io/ip-sans" = "" "cert-manager.io/issuer-group" = "" "cert-manager.io/issuer-kind" = "ClusterIssuer" - "cert-manager.io/issuer-name" = var.ssl_cert_config.caissuer + "cert-manager.io/issuer-name" = var.cert_config.issuer "cert-manager.io/uri-sans" = "" } } @@ -836,8 +637,8 @@ resource "kubernetes_manifest" "certificate" { apiVersion = "cert-manager.io/v1" kind = "Certificate" metadata = { - name = var.ssl_cert_config.name - namespace = kubernetes_namespace.this.metadata[0].name + name = var.cert_config.name + namespace = kubernetes_namespace_v1.this.metadata[0].name } spec = { commonName = local.common_name @@ -848,8 +649,8 @@ resource "kubernetes_manifest" "certificate" { duration = local.cert_refresh_interval renewBefore = local.cert_renew_before issuerRef = { - kind = "ClusterIssuer" - name = var.ssl_cert_config.caissuer + kind = var.cert_config.kind + name = var.cert_config.issuer } secretName = local.ssl_vol_friendly_name privateKey = { @@ -887,8 +688,8 @@ resource "kubernetes_manifest" "push" { apiVersion = "external-secrets.io/v1alpha1" kind = "PushSecret" metadata = { - name = var.ssl_cert_config.name - namespace = kubernetes_namespace.this.metadata[0].name + name = var.cert_config.name + namespace = kubernetes_namespace_v1.this.metadata[0].name } spec = { updatePolicy = "Replace" @@ -896,7 +697,7 @@ resource "kubernetes_manifest" "push" { refreshInterval = local.secret_refresh_interval secretStoreRefs = [{ kind = "ClusterSecretStore" - name = var.ssl_cert_config.secretissuer + name = var.cert_config.store }] selector = { secret = { @@ -922,10 +723,10 @@ resource "kubernetes_manifest" "push" { } } -resource "kubernetes_service" "prometheus" { +resource "kubernetes_service_v1" "prometheus" { metadata { name = "coder-prometheus" - namespace = kubernetes_namespace.this.metadata[0].name + namespace = kubernetes_namespace_v1.this.metadata[0].name # labels = local.app_labels } spec { @@ -949,7 +750,7 @@ resource "helm_release" "coder-server" { depends_on = [ kubernetes_manifest.push ] name = "coder" - namespace = kubernetes_namespace.this.metadata[0].name + namespace = kubernetes_namespace_v1.this.metadata[0].name chart = "coder" repository = "https://helm.coder.com/v2" create_namespace = false @@ -963,12 +764,12 @@ resource "helm_release" "coder-server" { values = [yamlencode({ coder = { image = { - repo = var.image_repo - tag = var.image_tag - pullPolicy = var.image_pull_policy - pullSecrets = var.image_pull_secrets + repo = var.coder.image_repo + tag = var.coder.image_tag + pullPolicy = var.coder.image_pull_policy + pullSecrets = var.coder.image_pull_secrets } - env = local.env_vars + env = local.env tls = { secretNames = [ local.ssl_vol_friendly_name ] } @@ -981,28 +782,32 @@ resource "helm_release" "coder-server" { type = "LoadBalancer" sessionAffinity = "None" externalTrafficPolicy = "Cluster" - loadBalancerClass = var.load_balancer_class - annotations = var.service_annotations + loadBalancerClass = var.lb_class + annotations = var.svc_annot } - replicaCount = var.replica_count + replicaCount = var.coder.rep_cnt resources = { requests = var.resource_request limits = var.resource_limit } serviceAccount = { - annotations = var.coder_builtin_provisioner_count == 0 ? var.service_account_annotations : merge({ + annotations = var.coder.prov_rep_cnt == 0 ? var.svc_acc_annot : merge({ "eks.amazonaws.com/role-arn" : module.provisioner-oidc-role[0].role_arn - }, var.service_account_annotations) + }, var.svc_acc_annot) } nodeSelector = var.node_selector tolerations = var.tolerations - topologySpreadConstraints = local.topology_spread_constraints + topologySpreadConstraints = local.topology_spread affinity = { podAntiAffinity = { - preferredDuringSchedulingIgnoredDuringExecution = local.pod_anti_affinity_preferred_during_scheduling_ignored_during_execution + preferredDuringSchedulingIgnoredDuringExecution = local.pod_aaf_pref_sched_ie } } - terminationGracePeriodSeconds = var.termination_grace_period_seconds + terminationGracePeriodSeconds = var.termination_grace_period } })] +} + +output "namespace" { + value = kubernetes_namespace_v1.this.metadata[0].name } \ No newline at end of file diff --git a/modules/k8s/bootstrap/ebs-controller/main.tf b/modules/k8s/bootstrap/ebs-controller/main.tf index 1269e6b..6337270 100644 --- a/modules/k8s/bootstrap/ebs-controller/main.tf +++ b/modules/k8s/bootstrap/ebs-controller/main.tf @@ -53,6 +53,11 @@ variable "node_selector" { default = {} } +variable "tolerations" { + type = list(map(any)) + default = [] +} + variable "replace" { type = bool default = false @@ -93,7 +98,7 @@ resource "helm_release" "ebs-controller" { wait = true wait_for_jobs = true version = var.chart_version - timeout = 120 # in seconds + timeout = 300 # in seconds values = [yamlencode({ controller = { @@ -104,10 +109,7 @@ resource "helm_release" "ebs-controller" { }, var.service_account_annotations) } nodeSelector = var.node_selector - tolerations = [{ - key = "CriticalAddonsOnly" - operator = "Exists" - }] + tolerations = var.tolerations } })] } diff --git a/modules/k8s/bootstrap/external-dns/main.tf b/modules/k8s/bootstrap/external-dns/main.tf index 793cb94..94949f3 100644 --- a/modules/k8s/bootstrap/external-dns/main.tf +++ b/modules/k8s/bootstrap/external-dns/main.tf @@ -65,32 +65,58 @@ variable "node_selector" { default = {} } +variable "tolerations" { + type = list(map(any)) + default = [] +} + data "aws_region" "this" {} data "aws_caller_identity" "this" {} +variable "r53_config" { + type = object({ + enabled = bool + region = optional(string, "") + account = optional(string, "") + role_name = optional(string, "ext-dns") + policy_name = optional(string, "ext-dns") + }) + default = { + enabled = false + region = "" + account = "" + role_name = "ext-dns" + policy_name = "ext-dns" + } +} + locals { - region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region - account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "ext-dns" : var.policy_name - role_name = var.role_name == "" ? "ext-dns" : var.role_name + region = var.r53_config.region == "" ? data.aws_region.this.region : var.r53_config.region + account_id = var.r53_config.account == "" ? data.aws_caller_identity.this.account_id : var.r53_config.account } module "policy" { + + count = var.r53_config.enabled ? 1 : 0 + source = "../../../security/policy" - name = local.policy_name - path = "/${var.cluster_name}/${data.aws_region.this.region}/" + name = var.r53_config.policy_name + path = "/${var.cluster_name}/${local.region}/" description = "External DNS Policy." policy_json = data.aws_iam_policy_document.this.json } module "oidc-role" { + + count = var.r53_config.enabled ? 1 : 0 + source = "../../../security/role/access-entry" - name = local.role_name - path = "/${var.cluster_name}/${data.aws_region.this.region}/" + name = var.r53_config.role_name + path = "/${var.cluster_name}/${local.region}/" cluster_name = var.cluster_name policy_arns = { - "ExternalDNS" = module.policy.policy_arn + "ExternalDNS" = module.policy[0].policy_arn } cluster_policy_arns = { "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", @@ -101,13 +127,55 @@ module "oidc-role" { tags = var.tags } -variable "domain_name" { - type = string +variable "cf_config" { + type = object({ + enabled = bool + token = string + email = string + }) + default = { + enabled = false + token = "" + email = "" + } + sensitive = true } -variable "is_private_domain" { - type = bool - default = false +locals { + # https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md + provider_aws = !var.r53_config.enabled ? null : { + provider = { name = "aws" } + env = [{ + name = "AWS_DEFAULT_REGION" + value = data.aws_region.this.region + }] + serviceAccount = { + annotations = { + "eks.amazonaws.com/role-arn" = module.oidc-role[0].role_arn + } + } + sources = [ + "ingress", + "service" + ] + extraArgs = [ "--aws-zone-match-parent" ] + } + # https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/cloudflare.md + provider_cf = !var.cf_config.enabled ? null : { + provider = { name = "cloudflare" } + env = [{ + name = "CF_API_TOKEN" + value = var.cf_config.token + }] + } + values = merge(local.provider_cf, merge(local.provider_aws, { + registry = "txt" + txtPrefix = "%%{record_type}-txt-." + txtOwnerId = "coder" + policy = "sync" # Force cleanup + insertion of record. + nodeSelector = var.node_selector + tolerations = var.tolerations + })) } resource "helm_release" "chart" { @@ -123,34 +191,5 @@ resource "helm_release" "chart" { version = var.chart_version timeout = 120 # in seconds - values = [yamlencode({ - provider = { - name = "aws" - } - sources = [ - "ingress", - "service" - ] - registry = "txt" - txtPrefix = "%%{record_type}-txt-." - txtOwnerId = "coder" - policy = "sync" # Force cleanup + insertion of record. - env = [{ - name = "AWS_DEFAULT_REGION" - value = data.aws_region.this.region - }] - serviceAccount = { - annotations = { - "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn - } - } - nodeSelector = var.node_selector - tolerations = [{ - key = "CriticalAddonsOnly" - operator = "Exists" - }] - extraArgs = [ - "--aws-zone-match-parent" - ] - })] + values = [yamlencode(local.values)] } \ No newline at end of file diff --git a/modules/k8s/bootstrap/external-secrets/main.tf b/modules/k8s/bootstrap/external-secrets/main.tf index 2eb3c2c..c6a43c0 100644 --- a/modules/k8s/bootstrap/external-secrets/main.tf +++ b/modules/k8s/bootstrap/external-secrets/main.tf @@ -65,6 +65,11 @@ variable "node_selector" { default = {} } +variable "tolerations" { + type = list(map(any)) + default = [] +} + data "aws_region" "this" {} data "aws_caller_identity" "this" {} @@ -101,13 +106,6 @@ module "oidc-role" { tags = var.tags } -locals { - global_tolerations = [{ - key = "CriticalAddonsOnly" - operator = "Exists" - }] -} - resource "helm_release" "chart" { name = "external-secrets" namespace = var.namespace @@ -123,12 +121,12 @@ resource "helm_release" "chart" { values = [yamlencode({ nodeSelector = var.node_selector - tolerations = local.global_tolerations + tolerations = var.tolerations webhook = { - tolerations = local.global_tolerations + tolerations = var.tolerations } certController = { - tolerations = local.global_tolerations + tolerations = var.tolerations } serviceAccount = { annotations = { diff --git a/modules/k8s/bootstrap/karpenter/main.tf b/modules/k8s/bootstrap/karpenter/main.tf index 5f37c43..e12dfa9 100644 --- a/modules/k8s/bootstrap/karpenter/main.tf +++ b/modules/k8s/bootstrap/karpenter/main.tf @@ -160,6 +160,11 @@ variable "node_selector" { default = {} } +variable "tolerations" { + type = list(map(any)) + default = [] +} + variable "replicas" { type = number default = 2 @@ -303,10 +308,7 @@ resource "helm_release" "karpenter" { "eks.amazonaws.com/role-arn" = module.karpenter.iam_role_arn } } - tolerations = [{ - key = "CriticalAddonsOnly" - operator = "Exists" - }] + tolerations = var.tolerations settings = { clusterName = var.cluster_name featureGates = { diff --git a/modules/k8s/bootstrap/lb-controller/main.tf b/modules/k8s/bootstrap/lb-controller/main.tf index a27d3f1..990db13 100644 --- a/modules/k8s/bootstrap/lb-controller/main.tf +++ b/modules/k8s/bootstrap/lb-controller/main.tf @@ -90,6 +90,11 @@ variable "node_selector" { default = {} } +variable "tolerations" { + type = list(map(any)) + default = [] +} + variable "create_alb_class" { type = bool default = true @@ -165,10 +170,7 @@ resource "helm_release" "lb-controller" { vpcId = var.vpc_id enableCertManager = var.enable_cert_manager nodeSelector = var.node_selector - tolerations = [{ - key = "CriticalAddonsOnly" - operator = "Exists" - }] + tolerations = var.tolerations serviceTargetENISGTags = local.service_target_eni_sg_tags serviceMutatorWebhookConfig = { # Ref - https://github.com/awslabs/data-on-eks/issues/458 diff --git a/modules/k8s/bootstrap/metrics-server/main.tf b/modules/k8s/bootstrap/metrics-server/main.tf index 9c3ad1f..136294c 100644 --- a/modules/k8s/bootstrap/metrics-server/main.tf +++ b/modules/k8s/bootstrap/metrics-server/main.tf @@ -22,6 +22,11 @@ variable "node_selector" { default = {} } +variable "tolerations" { + type = list(map(any)) + default = [] +} + data "aws_region" "this" {} data "aws_caller_identity" "this" {} @@ -37,13 +42,10 @@ resource "helm_release" "metrics-server" { wait = true wait_for_jobs = true version = var.chart_version - timeout = 120 # in seconds + timeout = 300 # in seconds values = [yamlencode({ nodeSelector = var.node_selector - tolerations = [{ - key = "CriticalAddonsOnly" - operator = "Exists" - }] + tolerations = var.tolerations })] } \ No newline at end of file diff --git a/modules/k8s/bootstrap/monitoring/main.tf b/modules/k8s/bootstrap/monitoring/main.tf new file mode 100644 index 0000000..32c7ca1 --- /dev/null +++ b/modules/k8s/bootstrap/monitoring/main.tf @@ -0,0 +1,596 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.17.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} + +variable "chart_version" { + type = string + default = "0.6.2" +} + +variable "chart_timeout" { + type = number + default = 300 +} + +variable "cluster_name" { + type = string +} + +variable "cluster_oidc_provider_arn" { + type = string +} + +variable "coder" { + type = object({ + db = object({ + host = string + port = optional(number, 5432) + password = string + username = string + database = string + sslmode = optional(string, "require") + }) + selector = object({ + coderd = string + provisionerd = string + workspaces = string + ctrl_plane_ns = string + ext_prov_ns = string + }) + }) + sensitive = true +} + +variable "grafana" { + type = object({ + admin = object({ + username = string + password = string + }) + db = object({ + host = string + port = optional(number, 5432) + password = string + username = string + database = string + sslmode = optional(string, "require") + }) + svc = object({ + annots = map(string) + }) + }) + sensitive = true +} + +variable "loki" { + type = object({ + s3 = object({ + chunks_bucket = string + ruler_bucket = string + region = string + }) + }) +} + +variable "domain_name" { + type = string +} + +variable "cert_config" { + type = object({ + create_secret = bool + name = string + kind = optional(string, "ClusterIssuer") + issuer = optional(string, "issuer") + store = optional(string, "issuer") + }) + default = { + create_secret = true + name = "grafana-tls" + kind = "ClusterIssuer" + issuer = "issuer" + store = "issuer" + } +} + +variable "tolerations" { + type = list(map(any)) + default = [] +} + +locals { + normalized_domain_name = split(".", var.domain_name)[0] + apex_domain = join(".", slice(split(".", var.domain_name), length(split(".", var.domain_name))-2, length(split(".", var.domain_name)))) + ssl_vol_friendly_name = replace(var.cert_config.name, ".", "-") + daemonset_tolerations = [{ + effect = "NoSchedule" + operator = "Exists" + }] + system_tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] +} + +data "aws_s3_bucket" "loki" { + bucket = "${var.cluster_name}-grafana" +} + +data "aws_region" "this" {} + +data "aws_caller_identity" "this" {} + +locals { + region = data.aws_region.this.region + account_id = data.aws_caller_identity.this.account_id + policy_name = "loki-s3-access" + role_name = "loki-s3-access" +} + +module "oidc-role" { + source = "../../../security/role/access-entry" + name = local.role_name + path = "/${var.cluster_name}/${data.aws_region.this.region}/" + cluster_name = var.cluster_name + policy_arns = { + "AmazonEKSLoadBalancingPolicy" = "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess", + } + cluster_policy_arns = { + "AmazonEKSClusterAdminPolicy" = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", + } + oidc_principals = { + "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] + } + tags = {} +} + +resource "kubernetes_namespace_v1" "this" { + metadata { + name = "coder-observe" + } +} + +locals { + common_name = "grafana.${trimprefix(trimprefix(var.domain_name, "https://"), "http://")}" + wildcard_name = "*.${local.common_name}" + cert_refresh_interval = "2160h" # 90 days + cert_renew_before = "360h" # 15 days + secret_refresh_interval = "1812h0m0s" # 75.5 days + tls_secret_key = "tls.key" + tls_secret_crt = "tls.crt" + tls_remote_key = "tls-${local.common_name}.key" + tls_remote_crt = "tls-${local.common_name}.crt" +} + +resource "kubernetes_manifest" "pull" { + + field_manager { + force_conflicts = true + } + + wait { + fields = { + "status.conditions[0].type" = "Ready" + } + } + + timeouts { + create = "1m" + update = "1m" + delete = "30s" + } + + manifest = { + apiVersion = "external-secrets.io/v1" + kind = "ExternalSecret" + metadata = { + name = var.cert_config.name + namespace = kubernetes_namespace_v1.this.metadata[0].name + } + spec = { + secretStoreRef = { + kind = "ClusterSecretStore" + name = var.cert_config.store + } + refreshPolicy = "Periodic" + refreshInterval = local.secret_refresh_interval + target = { + name = local.ssl_vol_friendly_name + creationPolicy = "Orphan" + deletionPolicy = "Retain" + template = { + type = "kubernetes.io/tls" + metadata = { + labels = { + "controller.cert-manager.io/fao" = "true" + } + annotations = { + "cert-manager.io/alt-names" = "${local.wildcard_name},${local.common_name}" + "cert-manager.io/certificate-name" = var.cert_config.name + "cert-manager.io/common-name" = local.common_name + "cert-manager.io/ip-sans" = "" + "cert-manager.io/issuer-group" = "" + "cert-manager.io/issuer-kind" = "ClusterIssuer" + "cert-manager.io/issuer-name" = var.cert_config.issuer + "cert-manager.io/uri-sans" = "" + } + } + } + } + data = [{ + secretKey = local.tls_secret_crt + remoteRef = { + key = local.tls_remote_crt + } + },{ + secretKey = local.tls_secret_key + remoteRef = { + key = local.tls_remote_key + } + }] + } + } +} + +resource "time_sleep" "wait" { + # Let the secret create first if it exists in AWS Secrets Manager. + depends_on = [ kubernetes_manifest.pull ] + create_duration = "30s" +} + +## +# Requires the cert-manager +## + +resource "kubernetes_manifest" "certificate" { + + depends_on = [ time_sleep.wait ] + + field_manager { + force_conflicts = true + } + + wait { + condition { + type = "Ready" + status = "True" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "Certificate" + metadata = { + name = var.cert_config.name + namespace = kubernetes_namespace_v1.this.metadata[0].name + } + spec = { + commonName = local.common_name + dnsNames = [ + local.common_name, + local.wildcard_name + ] + duration = local.cert_refresh_interval + renewBefore = local.cert_renew_before + issuerRef = { + kind = var.cert_config.kind + name = var.cert_config.issuer + } + secretName = local.ssl_vol_friendly_name + privateKey = { + rotationPolicy = "Never" + algorithm = "RSA" + encoding = "PKCS1" + size = "2048" + } + } + } +} + +resource "kubernetes_manifest" "push" { + + depends_on = [ kubernetes_manifest.certificate ] + + field_manager { + force_conflicts = true + } + + wait { + condition { + type = "Ready" + status = "True" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "external-secrets.io/v1alpha1" + kind = "PushSecret" + metadata = { + name = var.cert_config.name + namespace = kubernetes_namespace_v1.this.metadata[0].name + } + spec = { + updatePolicy = "Replace" + deletionPolicy = "None" + refreshInterval = local.secret_refresh_interval + secretStoreRefs = [{ + kind = "ClusterSecretStore" + name = var.cert_config.store + }] + selector = { + secret = { + name = kubernetes_manifest.certificate.manifest.spec.secretName + } + } + data = [{ + match = { + secretKey = local.tls_secret_crt + remoteRef = { + remoteKey = local.tls_remote_crt + } + } + },{ + match = { + secretKey = local.tls_secret_key + remoteRef = { + remoteKey = local.tls_remote_key + } + } + }] + } + } +} + +resource "helm_release" "coder-observe" { + name = "coder-observe" + namespace = kubernetes_namespace_v1.this.metadata[0].name + chart = "coder-observability" + repository = "https://helm.coder.com/observability" + create_namespace = false + upgrade_install = true + skip_crds = false + wait = true + wait_for_jobs = true + version = var.chart_version + timeout = var.chart_timeout + + values = [yamlencode({ + global = { + coder = { + coderdSelector = var.coder.selector.coderd + provisionerdSelector = var.coder.selector.provisionerd + workspacesSelector = var.coder.selector.workspaces + controlPlaneNamespace = var.coder.selector.ctrl_plane_ns + externalProvisionersNamespace = var.coder.selector.ext_prov_ns + } + postgres = { + hostname = var.coder.db.host + port = var.coder.db.port + password = var.coder.db.password + username = var.coder.db.username + database = var.coder.db.database + sslmode = var.coder.db.sslmode + mountSecret = "" + } + } + prometheus = { + server = { + tolerations = local.system_tolerations + } + alertmanager = { + tolerations = local.system_tolerations + } + } + grafana = { + adminUser = var.grafana.admin.username + adminPassword = var.grafana.admin.password + env = { + GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION = false + } + "grafana.ini" = { + app_mode = "production" + "auth.anonymous" = { + enabled = false + } + instance_name = "Coder Environment" + database = { + url = "postgres://${var.grafana.db.username}:${var.grafana.db.password}@${var.grafana.db.host}:${var.grafana.db.port}/${var.grafana.db.database}" + ssl_mode = var.grafana.db.sslmode + } + server = { + root_url = "https://${local.common_name}" + domain = local.common_name + enforce_domain = true + http_port = 3000 + protocol = "https" + cert_file = "/mnt/grafana-tls/tls.crt" + cert_key = "/mnt/grafana-tls/tls.key" + } + users = { + allow_sign_up = false + } + } + replicas = 2 + useStatefulSet = true + readinessProbe = { + httpGet = { + scheme = "HTTPS" + } + } + livenessProbe = { + httpGet = { + scheme = "HTTPS" + } + } + persistence = { + enabled = false + } + podAnnotations = { + "prometheus.io/port" = "3000" + "prometheus.io/scheme" = "http" + "prometheus.io/scrape" = "true" + } + service = { + annotations = var.grafana.svc.annots + enabled = true + externalTrafficPolicy = "Cluster" + internalTrafficPolicy = "Cluster" + loadBalancerClass = "service.k8s.aws/nlb" + port = 443 + targetPort = 3000 + type = "LoadBalancer" + } + extraSecretMounts = [{ + name = kubernetes_manifest.certificate.manifest.spec.secretName + mountPath = "/mnt/grafana-tls" + secretName = kubernetes_manifest.certificate.manifest.spec.secretName + readOnly = true + optional = false + subPath = "" + }] + } + grafana-agent = { + enabled = true + controller = { + tolerations = local.daemonset_tolerations + } + discovery = <<-EOF + // Discover k8s nodes + discovery.kubernetes "nodes" { + role = "node" + } + + // Discover k8s pods + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + } + } + EOF + commonRelabellings = <<-EOF + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + // coalesce the following labels and pick the first value; we'll use this to define the "job" label + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_component", "app", "__meta_kubernetes_pod_container_name"] + separator = "/" + target_label = "__meta_app" + action = "replace" + regex = "^/*([^/]+?)(?:/.*)?$" // split by the delimiter if it exists, we only want the first one + replacement = "$${1}" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_app"] + separator = "/" + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + regex = "__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)" + action = "labeldrop" + } + rule { + regex = "pod_template_generation" + action = "labeldrop" + } + rule { + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Pending|Succeeded|Failed|Completed" + action = "drop" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + action = "replace" + target_label = "node" + } + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)" + replacement = "__param_$1" + } + EOF + controller = { + type = "daemonset" + podAnnotations = { + "prometheus.io/scheme" = "http" + "prometheus.io/scrape" = "true" + } + } + } + loki = { + loki = { + storage = { + bucketNames = { + chunks = var.loki.s3.chunks_bucket + ruler = var.loki.s3.ruler_bucket + } + s3 = { + region = var.loki.s3.region + } + type = "s3" + } + } + lokiCanary = { + tolerations = local.daemonset_tolerations + } + backend = { + tolerations = local.system_tolerations + } + resultsCache = { + tolerations = local.system_tolerations + } + chunksCache = { + tolerations = local.system_tolerations + } + storage = { + tolerations = local.system_tolerations + } + write = { + tolerations = local.system_tolerations + } + serviceAccount = { + create = true + annotations = { + "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn + } + } + } + })] +} + +output "namespace" { + value = kubernetes_namespace_v1.this.metadata[0].name +} \ No newline at end of file From 35212eca3b1da18e279cdee0f0c2792d1f56de4e Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Wed, 4 Feb 2026 10:43:39 -0800 Subject: [PATCH 24/44] fix: resolves newline issue from downloading binary --- .gitignore | 3 +- infra/1-click/0-infra/3-eks.tf | 2 +- infra/1-click/1-plan-n-deploy.sh | 32 +++++++++---------- infra/1-click/1-setup/5-manifests.tf | 7 +--- infra/1-click/1-setup/6-coder-server.tf | 11 +++---- .../templates/kubernetes-claude/main.tf | 7 ++-- .../2-coder/templates/kubernetes/main.tf | 5 ++- infra/1-click/tmp.coder.env | 17 ++++++++++ 8 files changed, 46 insertions(+), 38 deletions(-) create mode 100644 infra/1-click/tmp.coder.env diff --git a/.gitignore b/.gitignore index 839afa9..7db87ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Terraform -.terraform/ -.terraform.lock.hcl +.terraform** terraform.tfstate* **.tfvars** tf.plan diff --git a/infra/1-click/0-infra/3-eks.tf b/infra/1-click/0-infra/3-eks.tf index 7ec6bf8..b4b857d 100644 --- a/infra/1-click/0-infra/3-eks.tf +++ b/infra/1-click/0-infra/3-eks.tf @@ -88,7 +88,7 @@ module "eks" { "system-only" = local.taints_system } - instance_types = ["t3.large"] + instance_types = ["t3a.large"] capacity_type = "ON_DEMAND" volume_size = 50 diff --git a/infra/1-click/1-plan-n-deploy.sh b/infra/1-click/1-plan-n-deploy.sh index c629165..956c2c0 100755 --- a/infra/1-click/1-plan-n-deploy.sh +++ b/infra/1-click/1-plan-n-deploy.sh @@ -37,22 +37,22 @@ if [ -z "${DOMAIN_NAME}" ]; then exit 1; fi -echo "Change directory into '0-infra'." -cd 0-infra -terraform plan -out=tf.plan \ - -var profile=$AWS_PROFILE \ - -var region=$AWS_REGION \ - -var domain_name=$DOMAIN_NAME \ - -var azs="$AWS_AZS" \ - -var coder_username=$CODER_DB_USERNAME \ - -var coder_password=$CODER_DB_PASSWORD \ - -var grafana_username=$GRAFANA_DB_USERNAME \ - -var grafana_password=$GRAFANA_DB_PASSWORD \ - -var use_ext_dns=$USE_EXTERN_DNS \ - -var cf_config="{\"enabled\":\"$USE_CF\",\"email\":\"$CF_EMAIL\",\"token\":\"$CF_TOKEN\"}" \ - -var r53_config="{\"enabled\":\"$USE_R53\"}" -terraform apply tf.plan -cd ../ +# echo "Change directory into '0-infra'." +# cd 0-infra +# terraform plan -out=tf.plan \ +# -var profile=$AWS_PROFILE \ +# -var region=$AWS_REGION \ +# -var domain_name=$DOMAIN_NAME \ +# -var azs="$AWS_AZS" \ +# -var coder_username=$CODER_DB_USERNAME \ +# -var coder_password=$CODER_DB_PASSWORD \ +# -var grafana_username=$GRAFANA_DB_USERNAME \ +# -var grafana_password=$GRAFANA_DB_PASSWORD \ +# -var use_ext_dns=$USE_EXTERN_DNS \ +# -var cf_config="{\"enabled\":\"$USE_CF\",\"email\":\"$CF_EMAIL\",\"token\":\"$CF_TOKEN\"}" \ +# -var r53_config="{\"enabled\":\"$USE_R53\"}" +# terraform apply tf.plan +# cd ../ echo "Change directory into '1-setup'." cd 1-setup diff --git a/infra/1-click/1-setup/5-manifests.tf b/infra/1-click/1-setup/5-manifests.tf index 6bbca54..b293bb9 100644 --- a/infra/1-click/1-setup/5-manifests.tf +++ b/infra/1-click/1-setup/5-manifests.tf @@ -16,11 +16,6 @@ data "kubernetes_service_account_v1" "kptr" { locals { - prefetch-script = templatefile("${path.module}/scripts/prefetch.sh.tftpl", { - IMAGES = join(" ", ["docker.io/codercom/enterprise-base:ubuntu"]) - PRE_SCRIPT = "" - POST_SCRIPT = "" - }) nodeclass_configs = { "coder" = { user_data = <<-EOF @@ -84,7 +79,7 @@ locals { node_expires_after = "Never" disruption_consolidation_policy = "WhenEmpty" disruption_consolidate_after = "1m" - instance_type = "t3a.medium" + instance_type = "t3a.large" taints = [] } } diff --git a/infra/1-click/1-setup/6-coder-server.tf b/infra/1-click/1-setup/6-coder-server.tf index 38017b5..265de97 100644 --- a/infra/1-click/1-setup/6-coder-server.tf +++ b/infra/1-click/1-setup/6-coder-server.tf @@ -232,7 +232,7 @@ data "http" "first-user" { # depends_on = [ time_sleep.wait_for_dns ] depends_on = [ module.coder-server ] - url = "https://${aws_eip.coder[0].public_ip}/api/v2/users/first" + url = "https://${aws_eip.coder.0.public_ip}/api/v2/users/first" method = "POST" insecure = true request_headers = { @@ -380,14 +380,13 @@ module "monitoring" { locals { coder_path = "/opt/coder/bin" bin_fetch_script = <<-EOF - if [ ! -f ${local.coder_path}/coder ]; then - curl -L https://${var.domain_name}/bin/coder-linux-amd64 -o ${local.coder_path}/coder - chmod +x ${local.coder_path}/coder - fi + curl -kfsSL --retry 10 --retry-delay 5 -H "Host: ${var.domain_name}" https://coder.coder.svc.cluster.local/bin/coder-linux-amd64 -o ${local.coder_path}/coder + chmod +x ${local.coder_path}/coder EOF } resource "kubernetes_daemon_set_v1" "bin-fetch" { + metadata { name = "coder-bin-fetch" namespace = module.coder-server.namespace @@ -417,7 +416,7 @@ resource "kubernetes_daemon_set_v1" "bin-fetch" { host_aliases { hostnames = [var.domain_name] - ip = aws_eip.coder[0].private_ip + ip = aws_eip.coder[0].public_ip } security_context { diff --git a/infra/1-click/2-coder/templates/kubernetes-claude/main.tf b/infra/1-click/2-coder/templates/kubernetes-claude/main.tf index acf863d..ae2e6e6 100644 --- a/infra/1-click/2-coder/templates/kubernetes-claude/main.tf +++ b/infra/1-click/2-coder/templates/kubernetes-claude/main.tf @@ -257,7 +257,7 @@ data "coder_workspace_preset" "standard" { (data.coder_parameter.home_disk_size.name) = "10" } prebuilds { - instances = 1 + instances = 0 } } @@ -265,10 +265,9 @@ locals { coder_bin = "/opt/coder/bin" init_script = <<-EOF if [ -x ${local.coder_bin}/coder ]; then - exec ${local.coder_bin}/coder agent - else - ${coder_agent.main.init_script} + exec ${local.coder_bin}/coder agent ; fi + ${coder_agent.main.init_script} EOF } diff --git a/infra/1-click/2-coder/templates/kubernetes/main.tf b/infra/1-click/2-coder/templates/kubernetes/main.tf index 398a61f..487a94e 100644 --- a/infra/1-click/2-coder/templates/kubernetes/main.tf +++ b/infra/1-click/2-coder/templates/kubernetes/main.tf @@ -230,10 +230,9 @@ locals { coder_bin = "/opt/coder/bin" init_script = <<-EOF if [ -x ${local.coder_bin}/coder ]; then - exec ${local.coder_bin}/coder agent - else - ${coder_agent.main.init_script} + exec ${local.coder_bin}/coder agent ; fi + ${coder_agent.main.init_script} EOF } diff --git a/infra/1-click/tmp.coder.env b/infra/1-click/tmp.coder.env new file mode 100644 index 0000000..f7e88cb --- /dev/null +++ b/infra/1-click/tmp.coder.env @@ -0,0 +1,17 @@ +CODER_AWS_PROFILE=default +CODER_AWS_REGION=us-east-2 +# CODER_LICENSE="..." + +CODER_USERNAME=admin +CODER_EMAIL=admin@coder.com +CODER_PASSWORD=Th1s1sN0TS3CuR3!! + +CODER_USE_EXTERN_DNS=false + +CODER_DOMAIN_NAME= + +CODER_USE_R53=true + +CODER_USE_CF=false +CODER_CF_EMAIL="..." +CODER_CF_TOKEN="..." \ No newline at end of file From 71cc2ecf9135ca937112496d4e07acad7a0c6648 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 16 Feb 2026 23:09:59 -0800 Subject: [PATCH 25/44] feat: rework for pure AWS + Terragrunt deployment --- infra/1-click/0-infra/0-vpc.tf | 106 +- infra/1-click/0-infra/1-db.tf | 96 +- infra/1-click/0-infra/2-s3.tf | 7 +- infra/1-click/0-infra/3-eks.tf | 109 +- infra/1-click/0-infra/4-bootstrap.tf | 116 +- infra/1-click/0-infra/main.tf | 98 +- infra/1-click/0-infra/terragrunt.hcl | 19 + infra/1-click/0-infra/variables.tf | 58 + infra/1-click/1-setup/5-manifests.tf | 356 +-- infra/1-click/1-setup/6-coder-server.tf | 514 ++--- infra/1-click/1-setup/7-coder-init.tf | 111 + .../1-setup/dashboards/aibridge-demoable.json | 1382 ----------- .../1-setup/dashboards/aibridge-example.json | 1411 ------------ .../1-click/1-setup/dashboards/aibridge.json | 1080 --------- infra/1-click/1-setup/dashboards/coderd.json | 1472 ------------ .../1-click/1-setup/dashboards/prebuilds.json | 1450 ------------ .../1-setup/dashboards/provisionerd.json | 1019 --------- infra/1-click/1-setup/dashboards/status.json | 2038 ----------------- .../1-setup/dashboards/workspace_detail.json | 1342 ----------- .../1-setup/dashboards/workspaces.json | 1624 ------------- infra/1-click/1-setup/main.tf | 238 +- infra/1-click/1-setup/scripts/add-license.sh | 25 - infra/1-click/1-setup/scripts/nodeconfig.yaml | 18 - infra/1-click/1-setup/terragrunt.hcl | 23 + infra/1-click/1-setup/variables.tf | 98 + infra/1-click/2-coder/7-license.tf | 26 - infra/1-click/2-coder/8-backend.tf | 150 ++ infra/1-click/2-coder/8-templates.tf | 131 -- infra/1-click/2-coder/9-templates.tf | 222 ++ infra/1-click/2-coder/main.tf | 139 +- .../templates/aws-devcontainer/README.md | 111 + .../aws-devcontainer/architecture.svg | 8 + .../cloud-init/cloud-config.yaml.tftpl | 15 + .../cloud-init/userdata.sh.tftpl | 37 + .../templates/aws-devcontainer/main.tf | 331 +++ .../2-coder/templates/aws-linux/README.md | 94 + .../cloud-init/cloud-config.yaml.tftpl | 8 + .../aws-linux/cloud-init/userdata.sh.tftpl | 2 + .../2-coder/templates/aws-linux/main.tf | 286 +++ .../2-coder/templates/aws-windows/README.md | 96 + .../2-coder/templates/aws-windows/main.tf | 214 ++ .../kubernetes-claude/boundary-config.yaml | 243 -- .../templates/kubernetes-claude/main.tf | 69 +- .../2-coder/templates/kubernetes/main.tf | 34 +- infra/1-click/2-coder/terragrunt.hcl | 20 + infra/1-click/2-coder/variables.tf | 47 + infra/1-click/root.hcl | 99 + modules/coder/provisioner/main.tf | 16 +- modules/coder/template/main.tf | 74 + modules/k8s/bootstrap/coder-logstream/main.tf | 104 + .../k8s/bootstrap/coder-provisioner/main.tf | 388 +--- .../k8s/bootstrap/coder-provisioner/policy.tf | 185 +- modules/k8s/bootstrap/coder-proxy/main.tf | 345 ++- modules/k8s/bootstrap/coder-server/main.tf | 335 +-- modules/k8s/bootstrap/coder-server/policy.tf | 182 +- .../{ebs-controller => ebs-csi}/main.tf | 0 modules/k8s/bootstrap/karpenter/main.tf | 155 +- modules/k8s/bootstrap/lb-controller/main.tf | 104 +- modules/k8s/bootstrap/monitoring/main.tf | 311 +-- 59 files changed, 3292 insertions(+), 16099 deletions(-) create mode 100644 infra/1-click/0-infra/terragrunt.hcl create mode 100644 infra/1-click/0-infra/variables.tf create mode 100644 infra/1-click/1-setup/7-coder-init.tf delete mode 100644 infra/1-click/1-setup/dashboards/aibridge-demoable.json delete mode 100644 infra/1-click/1-setup/dashboards/aibridge-example.json delete mode 100644 infra/1-click/1-setup/dashboards/aibridge.json delete mode 100644 infra/1-click/1-setup/dashboards/coderd.json delete mode 100644 infra/1-click/1-setup/dashboards/prebuilds.json delete mode 100644 infra/1-click/1-setup/dashboards/provisionerd.json delete mode 100644 infra/1-click/1-setup/dashboards/status.json delete mode 100644 infra/1-click/1-setup/dashboards/workspace_detail.json delete mode 100644 infra/1-click/1-setup/dashboards/workspaces.json delete mode 100644 infra/1-click/1-setup/scripts/add-license.sh delete mode 100644 infra/1-click/1-setup/scripts/nodeconfig.yaml create mode 100644 infra/1-click/1-setup/terragrunt.hcl create mode 100644 infra/1-click/1-setup/variables.tf delete mode 100644 infra/1-click/2-coder/7-license.tf create mode 100644 infra/1-click/2-coder/8-backend.tf delete mode 100644 infra/1-click/2-coder/8-templates.tf create mode 100644 infra/1-click/2-coder/9-templates.tf create mode 100644 infra/1-click/2-coder/templates/aws-devcontainer/README.md create mode 100644 infra/1-click/2-coder/templates/aws-devcontainer/architecture.svg create mode 100644 infra/1-click/2-coder/templates/aws-devcontainer/cloud-init/cloud-config.yaml.tftpl create mode 100644 infra/1-click/2-coder/templates/aws-devcontainer/cloud-init/userdata.sh.tftpl create mode 100644 infra/1-click/2-coder/templates/aws-devcontainer/main.tf create mode 100644 infra/1-click/2-coder/templates/aws-linux/README.md create mode 100644 infra/1-click/2-coder/templates/aws-linux/cloud-init/cloud-config.yaml.tftpl create mode 100644 infra/1-click/2-coder/templates/aws-linux/cloud-init/userdata.sh.tftpl create mode 100644 infra/1-click/2-coder/templates/aws-linux/main.tf create mode 100644 infra/1-click/2-coder/templates/aws-windows/README.md create mode 100644 infra/1-click/2-coder/templates/aws-windows/main.tf delete mode 100644 infra/1-click/2-coder/templates/kubernetes-claude/boundary-config.yaml create mode 100644 infra/1-click/2-coder/terragrunt.hcl create mode 100644 infra/1-click/2-coder/variables.tf create mode 100644 infra/1-click/root.hcl create mode 100644 modules/coder/template/main.tf create mode 100644 modules/k8s/bootstrap/coder-logstream/main.tf rename modules/k8s/bootstrap/{ebs-controller => ebs-csi}/main.tf (100%) diff --git a/infra/1-click/0-infra/0-vpc.tf b/infra/1-click/0-infra/0-vpc.tf index efab265..056f2aa 100644 --- a/infra/1-click/0-infra/0-vpc.tf +++ b/infra/1-click/0-infra/0-vpc.tf @@ -1,16 +1,11 @@ ## # Networking Infrastructure -## - -variable "azs" { - type = list(string) - default = ["a", "b", "c"] -} +## locals { # Subnet Discovery - https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/subnet_discovery/#subnet-filtering tags_lb_subnet_discovery = { - "kubernetes.io/cluster/${var.name}-${local.normalized_domain_name}" = "owned" + "kubernetes.io/cluster/${local.formatted_name}" = "owned" } # Internet-Facing LB - https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/subnet_discovery/#subnet-role-tag tags_public_lb = { @@ -20,108 +15,35 @@ locals { local.tags_lb_subnet_discovery, local.tags_public_lb ) + tags_private_subnet = { + "karpenter.sh/discovery" = "${local.formatted_name}" + } # Use variable to configure AZ just in case 1 AZ isn't accessible # https://repost.aws/questions/QUgdQev4KETKG_Bwev9tMtRQ/is-it-possible-to-enable-3rd-availability-zone-in-us-west-1#AN9eAH55FwTC-NSSp-FjIfTQ availability_zones = [for az in var.azs : "${var.region}${az}"] public_subnet_cidrs = ["10.0.8.0/21", "10.0.4.0/22", "10.0.0.0/22"] - general_subnet_cidrs = ["10.0.64.0/20", "10.0.80.0/20", "10.0.96.0/20"] - system_subnet_cidrs = ["10.0.16.0/20", "10.0.32.0/20", "10.0.48.0/20"] + private_subnet_cidrs = ["10.0.64.0/20", "10.0.80.0/20", "10.0.96.0/20"] } module "vpc" { source = "terraform-aws-modules/vpc/aws" version = "~> 6.6.0" - name = "${var.name}-${local.normalized_domain_name}" + name = local.formatted_name - enable_nat_gateway = false + enable_nat_gateway = true enable_dns_hostnames = true cidr = "10.0.0.0/16" azs = local.availability_zones - ## - # Handle public subnets via the VPC module. - ## public_subnet_suffix = "public" - public_subnets = [for index, _ in var.azs : local.public_subnet_cidrs[index]] - public_subnet_tags = local.tags_public_subnet - - tags = local.tags_global -} - -module "nat-instance" { - source = "RaJiska/fck-nat/aws" - - name = "${var.name}-${local.normalized_domain_name}" - - vpc_id = module.vpc.vpc_id - subnet_id = module.vpc.public_subnets[0] - # https://fck-nat.dev/stable/choosing_an_instance_size/ - instance_type = "c6gn.medium" - ha_mode = true # Enables high-availability mode - # eip_allocation_ids = ["eipalloc-abc1234"] # Allocation ID of an existing EIP - use_cloudwatch_agent = true # Enables Cloudwatch agent and have metrics reported - - update_route_tables = true - route_tables_ids = {} - - tags = local.tags_global -} - -## -# Coder Subnets -## - -locals { - # Karpenter Subnet Discovery - https://karpenter.sh/v1.0/concepts/nodeclasses/#specsubnetselectorterms - tags_kptr_subnet_discovery = { - "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" - } - tags_private_subnet = merge( - local.tags_lb_subnet_discovery, - local.tags_kptr_subnet_discovery - ) -} - -# 4096 - 5 (reserved by AWS) IPs each - -module "general-subnet" { - - count = length(var.azs) - - source = "../../../modules/network/subnet/private" - name = "${var.name}-${local.normalized_domain_name}" - vpc_id = module.vpc.vpc_id - eni_id = module.nat-instance.eni_id - cidr_block = local.general_subnet_cidrs[count.index] - availability_zone = local.availability_zones[count.index] - subnet_tags = local.tags_private_subnet - tags = local.tags_global -} - -## -# Kubernetes System Subnets -## - -module "system-subnet" { + public_subnets = [for index, _ in var.azs : local.public_subnet_cidrs[index]] + public_subnet_tags = local.tags_public_subnet - count = length(var.azs) + private_subnet_suffix = "private" + private_subnets = [for index, _ in var.azs : local.private_subnet_cidrs[index]] + private_subnet_tags = local.tags_private_subnet - source = "../../../modules/network/subnet/private" - name = "${var.name}-${local.normalized_domain_name}" - vpc_id = module.vpc.vpc_id - eni_id = module.nat-instance.eni_id - cidr_block = local.system_subnet_cidrs[count.index] - availability_zone = local.availability_zones[count.index] - subnet_tags = local.tags_private_subnet - tags = local.tags_global -} - -locals { - private_subnet_ids = concat( - module.general-subnet.*.subnet_id, - module.system-subnet.*.subnet_id - ) - public_subnet_ids = concat([], module.vpc.public_subnets) + tags = {} } \ No newline at end of file diff --git a/infra/1-click/0-infra/1-db.tf b/infra/1-click/0-infra/1-db.tf index 18a1c58..fafff67 100644 --- a/infra/1-click/0-infra/1-db.tf +++ b/infra/1-click/0-infra/1-db.tf @@ -1,36 +1,10 @@ ## # Database Infrastructure -## - -variable "coder_username" { - description = "Coder DB's username." - type = string - default = "coder" -} - -variable "coder_password" { - description = "Coder DB's password." - type = string - sensitive = true - default = "th1s1sn0tas3cur3pass0wrd" -} - -variable "grafana_username" { - description = "Grafana DB's username." - type = string - default = "grafana" -} - -variable "grafana_password" { - description = "Grafana DB's password." - type = string - sensitive = true - default = "th1s1sn0tas3cur3pass0wrd" -} +## resource "aws_security_group" "postgres" { vpc_id = module.vpc.vpc_id - name = "${var.name}-${local.normalized_domain_name}-pgsql" + name = "${local.formatted_name}-pgsql" description = "security group for postgres all egress traffic" tags = { Name = "PostgreSQL" @@ -52,38 +26,38 @@ resource "aws_vpc_security_group_egress_rule" "postgres" { } resource "aws_db_subnet_group" "coder" { - name = "${var.name}-${local.normalized_domain_name}-coder" - subnet_ids = local.private_subnet_ids + name = "${local.formatted_name}-coder" + subnet_ids = module.vpc.private_subnets tags = { - Name = "${var.name}-${local.normalized_domain_name}-coder" + Name = "${local.formatted_name}-coder" } } resource "time_static" "coder_snapshot" { triggers = { - run_on_ip_change = "${var.name}-${local.normalized_domain_name}-coder" + run_on_ip_change = "${local.formatted_name}-coder" } } resource "aws_db_instance" "coder" { - identifier = "${var.name}-${local.normalized_domain_name}-coder" - instance_class = "db.t4g.medium" - allocated_storage = 50 - engine = "postgres" - engine_version = "15.12" - username = var.coder_username - password = var.coder_password - db_name = "coder" - db_subnet_group_name = aws_db_subnet_group.coder.name - vpc_security_group_ids = [aws_security_group.postgres.id] - publicly_accessible = false - snapshot_identifier = null - skip_final_snapshot = false + identifier = "${local.formatted_name}-coder" + instance_class = "db.t4g.medium" + allocated_storage = 50 + engine = "postgres" + engine_version = "15.12" + username = var.coder_username + password = var.coder_password + db_name = "coder" + db_subnet_group_name = aws_db_subnet_group.coder.name + vpc_security_group_ids = [aws_security_group.postgres.id] + publicly_accessible = false + snapshot_identifier = null + skip_final_snapshot = false final_snapshot_identifier = "coder-${replace(time_static.coder_snapshot.rfc3339, ":", "-")}" tags = { - Name = "${var.name}-${local.normalized_domain_name}-coder" + Name = "${local.formatted_name}-coder" } lifecycle { ignore_changes = [ @@ -94,28 +68,28 @@ resource "aws_db_instance" "coder" { resource "time_static" "grafana_snapshot" { triggers = { - run_on_ip_change = "${var.name}-${local.normalized_domain_name}-grafana" + run_on_ip_change = "${local.formatted_name}-grafana" } } resource "aws_db_instance" "grafana" { - identifier = "${var.name}-${local.normalized_domain_name}-grafana" - instance_class = "db.t4g.medium" - allocated_storage = 50 - engine = "postgres" - engine_version = "15.12" - username = var.grafana_username - password = var.grafana_password - db_name = "grafana" - db_subnet_group_name = aws_db_subnet_group.coder.name - vpc_security_group_ids = [aws_security_group.postgres.id] - publicly_accessible = false - snapshot_identifier = null - skip_final_snapshot = false + identifier = "${local.formatted_name}-grafana" + instance_class = "db.t4g.medium" + allocated_storage = 50 + engine = "postgres" + engine_version = "15.12" + username = var.grafana_username + password = var.grafana_password + db_name = "grafana" + db_subnet_group_name = aws_db_subnet_group.coder.name + vpc_security_group_ids = [aws_security_group.postgres.id] + publicly_accessible = false + snapshot_identifier = null + skip_final_snapshot = false final_snapshot_identifier = "grafana-${replace(time_static.coder_snapshot.rfc3339, ":", "-")}" tags = { - Name = "${var.name}-${local.normalized_domain_name}-grafana" + Name = "${local.formatted_name}-grafana" } lifecycle { ignore_changes = [ diff --git a/infra/1-click/0-infra/2-s3.tf b/infra/1-click/0-infra/2-s3.tf index 2798bc1..67b73c6 100644 --- a/infra/1-click/0-infra/2-s3.tf +++ b/infra/1-click/0-infra/2-s3.tf @@ -6,13 +6,8 @@ # Loki Inputs ## -variable "loki_s3_bucket_tags" { - type = map(string) - default = {} -} - resource "aws_s3_bucket" "loki" { - bucket = "${var.name}-${local.normalized_domain_name}-grafana" + bucket = "${local.formatted_name}-grafana" tags = var.loki_s3_bucket_tags } diff --git a/infra/1-click/0-infra/3-eks.tf b/infra/1-click/0-infra/3-eks.tf index b4b857d..e3cd839 100644 --- a/infra/1-click/0-infra/3-eks.tf +++ b/infra/1-click/0-infra/3-eks.tf @@ -2,38 +2,17 @@ # Kubernetes Infrastructure ## -data "aws_iam_policy_document" "sts" { - statement { - effect = "Allow" - actions = ["sts:*"] - resources = ["*"] - } -} - -## -# Use for troubleshooting K8s nodes in case of issues. -## - -resource "aws_iam_policy" "sts" { - name_prefix = "${var.name}-${local.normalized_domain_name}-sts-" - path = "/" - description = "Assume Role Policy" - policy = data.aws_iam_policy_document.sts.json - tags = local.tags_global -} - locals { # Karpenter Security Group Discovery - https://karpenter.sh/v1.0/concepts/nodeclasses/#specsecuritygroupselectorterms tags_kptr_sg_discovery = { - "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" + "karpenter.sh/discovery" = "${local.formatted_name}-karpenter" } labels_system_node = { "scheduling.coder.com/pool" = "system" } taints_system = { key = "CriticalAddonsOnly" - value = "true" - effect = "NO_EXECUTE" + effect = "NO_SCHEDULE" } tolerations_system = [{ key = "CriticalAddonsOnly" @@ -43,30 +22,32 @@ locals { module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 21.14.0" + version = "~> 21.15.1" vpc_id = module.vpc.vpc_id # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster#vpc_config-1 # https://docs.aws.amazon.com/eks/latest/userguide/network-reqs.html#network-requirements-subnets subnet_ids = toset(concat( - local.private_subnet_ids + module.vpc.private_subnets )) - name = "${var.name}-${local.normalized_domain_name}" + name = local.formatted_name kubernetes_version = "1.34" endpoint_public_access = true endpoint_private_access = true - create_security_group = true - create_node_security_group = true - create_iam_role = true - node_security_group_tags = local.tags_kptr_sg_discovery + create_security_group = true + create_node_security_group = true + create_iam_role = true + node_security_group_tags = local.tags_kptr_sg_discovery + create_node_iam_role = true + node_iam_role_use_name_prefix = true compute_config = { - # Disables EKS Auto Mode. Manually handle scaling via Karpenter - enabled = false + enabled = true + node_pools = ["system"] } attach_encryption_policy = false @@ -75,38 +56,6 @@ module "eks" { enable_cluster_creator_admin_permissions = true enable_irsa = true - eks_managed_node_groups = { - # Initial nodes are dedicated to system-level processes - system = { - min_size = 0 - max_size = 10 - desired_size = 2 # Ignored after creation. Override from AWS Console as needed. - - # K8s Labels - https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group#labels-1 - labels = local.labels_system_node - taints = { - "system-only" = local.taints_system - } - - instance_types = ["t3a.large"] - capacity_type = "ON_DEMAND" - volume_size = 50 - - iam_role_additional_policies = { - AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" - STSAssumeRole = aws_iam_policy.sts.arn - } - metadata_options = { - http_endpoint = "enabled" - http_put_response_hop_limit = 2 - http_tokens = "required" - } - - # System Nodes should not be public - subnet_ids = local.private_subnet_ids - } - } - addons = { coredns = { before_compute = true @@ -123,13 +72,39 @@ module "eks" { } env = { ENABLE_PREFIX_DELEGATION = "true" - WARM_PREFIX_TARGET = "1" - WARM_IP_TARGET = "0" + WARM_PREFIX_TARGET = "1" + WARM_IP_TARGET = "0" AWS_VPC_K8S_CNI_LOGLEVEL = "DEBUG" } }) } } - tags = local.tags_global + tags = {} +} + +module "karpenter" { + + source = "../../../modules/k8s/bootstrap/karpenter" + + cluster_name = module.eks.cluster_name + cluster_oidc_provider_arn = module.eks.oidc_provider_arn + cluster_oidc_provider = module.eks.oidc_provider + + namespace = "karpenter" + chart_version = "1.9.0" + + node_selector = { + "beta.kubernetes.io/os" = "linux" + "kubernetes.io/os" = null + } + tolerations = local.tolerations_system + topology_spread = [] + + iam_role_use_name_prefix = true + node_iam_role_use_name_prefix = true + replicas = 2 + karpenter_controller_role_policies = { + "AmazonEFSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" + } } \ No newline at end of file diff --git a/infra/1-click/0-infra/4-bootstrap.tf b/infra/1-click/0-infra/4-bootstrap.tf index e65df15..077fa95 100644 --- a/infra/1-click/0-infra/4-bootstrap.tf +++ b/infra/1-click/0-infra/4-bootstrap.tf @@ -2,128 +2,36 @@ # System-Level Addons ## -module "karpenter" { - - source = "../../../modules/k8s/bootstrap/karpenter" - - cluster_name = module.eks.cluster_name - cluster_oidc_provider_arn = module.eks.oidc_provider_arn - cluster_oidc_provider = module.eks.oidc_provider - - namespace = "karpenter" - chart_version = "1.8.4" - node_selector = local.labels_system_node - tolerations = local.tolerations_system - - iam_role_use_name_prefix = true - node_iam_role_use_name_prefix = true - replicas = 2 - karpenter_controller_role_policies = { - "AmazonEFSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" - } -} - module "metrics-server" { - source = "../../../modules/k8s/bootstrap/metrics-server" + depends_on = [module.eks] + source = "../../../modules/k8s/bootstrap/metrics-server" namespace = "metrics-server" chart_version = "3.13.0" - node_selector = local.labels_system_node - tolerations = local.tolerations_system -} - -module "cert-manager" { - - source = "../../../modules/k8s/bootstrap/cert-manager" - cluster_name = module.eks.cluster_name - cluster_oidc_provider_arn = module.eks.oidc_provider_arn - - namespace = "cert-manager" - helm_version = "v1.18.2" - tolerations = local.tolerations_system - - cf_config = { - enabled = var.cf_config.enabled - email = var.cf_config.email - token = var.cf_config.token - } - r53_config = { - enabled = var.r53_config.enabled - role_name = "crt-mgr" - policy_name = "crt-mgr" - } + tolerations = local.tolerations_system } module "lb-ctrl" { - depends_on = [ module.cert-manager ] - source = "../../../modules/k8s/bootstrap/lb-controller" cluster_name = module.eks.cluster_name cluster_oidc_provider_arn = module.eks.oidc_provider_arn - namespace = "lb-ctrl" - chart_version = "1.13.2" - node_selector = local.labels_system_node - tolerations = local.tolerations_system - - enable_cert_manager = true - vpc_id = module.vpc.vpc_id - service_target_eni_sg_tags = { - Name = module.eks.cluster_name - } - create_alb_class = false + namespace = "lb-ctrl" + chart_version = "3.0.0" + vpc_id = module.vpc.vpc_id + tolerations = local.tolerations_system } -module "ebs-ctrl" { +module "ebs-csi" { - source = "../../../modules/k8s/bootstrap/ebs-controller" + source = "../../../modules/k8s/bootstrap/ebs-csi" cluster_name = module.eks.cluster_name cluster_oidc_provider_arn = module.eks.oidc_provider_arn - namespace = "ebs-ctrl" - chart_version = "2.22.1" - node_selector = local.labels_system_node - tolerations = local.tolerations_system + namespace = "ebs-csi" + chart_version = "2.55.0" + tolerations = local.tolerations_system replace = true -} - -module "ext-dns" { - - count = var.use_ext_dns ? 1 : 0 - depends_on = [ module.cert-manager ] - - source = "../../../modules/k8s/bootstrap/external-dns" - cluster_name = module.eks.cluster_name - cluster_oidc_provider_arn = module.eks.oidc_provider_arn - - namespace = "ext-dns" - chart_version = "1.20.0" - node_selector = local.labels_system_node - tolerations = local.tolerations_system - - cf_config = { - enabled = var.cf_config.enabled - email = var.cf_config.email - token = var.cf_config.token - } - r53_config = { - enabled = var.r53_config.enabled - role_name = "crt-mgr" - policy_name = "crt-mgr" - } -} - -module "ext-sec" { - - depends_on = [ module.cert-manager ] - - source = "../../../modules/k8s/bootstrap/external-secrets" - cluster_name = module.eks.cluster_name - cluster_oidc_provider_arn = module.eks.oidc_provider_arn - - namespace = "ext-sec" - chart_version = "1.2.1" - tolerations = local.tolerations_system } \ No newline at end of file diff --git a/infra/1-click/0-infra/main.tf b/infra/1-click/0-infra/main.tf index f70965e..e2b5a0d 100644 --- a/infra/1-click/0-infra/main.tf +++ b/infra/1-click/0-infra/main.tf @@ -1,44 +1,10 @@ -terraform { - required_version = ">= 1.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.46" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } -} - ## # Global Inputs + Providers ## -variable "region" { - description = "The AWS region" - type = string - default = "us-east-2" -} - -variable "name" { - description = "Name for created resources and tag prefix" - type = string - default = "coder" -} - -variable "profile" { - type = string - default = "default" -} - -variable "domain_name" { - description = "Your Coder domain name (i.e. coder-example.com)" - type = string +locals { + normalized_domain_name = split(".", var.domain_name)[0] + formatted_name = "${var.name}-${local.normalized_domain_name}" } data "aws_eks_cluster_auth" "coder" { @@ -54,50 +20,30 @@ provider "helm" { kubernetes = { host = module.eks.cluster_endpoint cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - token = data.aws_eks_cluster_auth.coder.token + exec = { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = [ + "eks", "get-token", + "--cluster-name", module.eks.cluster_name, + "--profile", var.profile, + "--region", var.region + ] + } } } provider "kubernetes" { host = module.eks.cluster_endpoint cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - token = data.aws_eks_cluster_auth.coder.token -} - -locals { - normalized_domain_name = split(".", var.domain_name)[0] - tags_global = {} -} - -# If R53 enabled, then fetch service account from cert-manager for IAM Role -variable "r53_config" { - description = "Enable to use Route53 as a DNS01 provider for ACME challenges." - type = object({ - enabled = bool - }) - default = { - enabled = false - } -} - -# If CF enabled, then fetch secret from cert-manager for token -variable "cf_config" { - description = "Enable to use CloudFlare as a DNS01 provider for ACME challenges." - type = object({ - enabled = bool - email = string - token = string - }) - default = { - enabled = false - email = "" - token = "" + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = [ + "eks", "get-token", + "--cluster-name", module.eks.cluster_name, + "--profile", var.profile, + "--region", var.region + ] } - sensitive = true -} - -variable "use_ext_dns" { - description = "Toggle the K8s 'external-dns' addon. Disable in-case you want to manage DNS records yourself." - type = bool - default = true } \ No newline at end of file diff --git a/infra/1-click/0-infra/terragrunt.hcl b/infra/1-click/0-infra/terragrunt.hcl new file mode 100644 index 0000000..6f1ca3b --- /dev/null +++ b/infra/1-click/0-infra/terragrunt.hcl @@ -0,0 +1,19 @@ +## +# Base Module +## + +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +inputs = { + profile = include.root.locals.CODER_AWS_PROFILE + region = include.root.locals.CODER_AWS_REGION + domain_name = include.root.locals.CODER_DOMAIN_NAME + azs = jsondecode(include.root.locals.CODER_AWS_AZS) + coder_username = include.root.locals.CODER_DB_USERNAME + coder_password = include.root.locals.CODER_DB_PASSWORD + grafana_username = include.root.locals.GRAFANA_DB_USERNAME + grafana_password = include.root.locals.GRAFANA_DB_PASSWORD +} \ No newline at end of file diff --git a/infra/1-click/0-infra/variables.tf b/infra/1-click/0-infra/variables.tf new file mode 100644 index 0000000..be3922c --- /dev/null +++ b/infra/1-click/0-infra/variables.tf @@ -0,0 +1,58 @@ +variable "region" { + description = "The AWS region to deploy AWS-resources to." + type = string + default = "us-east-2" +} + +variable "name" { + description = "Name for created resources and it's tag prefix" + type = string + default = "coder" +} + +variable "profile" { + description = "The local AWS profile to use." + type = string + default = "default" +} + +variable "domain_name" { + description = "Your domain name (i.e. coder-example.com)." + type = string +} + +variable "azs" { + type = list(string) + default = ["a", "b", "c"] +} + +variable "coder_username" { + description = "Coder DB's username." + type = string + default = "coder" +} + +variable "coder_password" { + description = "Coder DB's password." + type = string + sensitive = true + default = "th1s1sn0tas3cur3pass0wrd" +} + +variable "grafana_username" { + description = "Grafana DB's username." + type = string + default = "grafana" +} + +variable "grafana_password" { + description = "Grafana DB's password." + type = string + sensitive = true + default = "th1s1sn0tas3cur3pass0wrd" +} + +variable "loki_s3_bucket_tags" { + type = map(string) + default = {} +} \ No newline at end of file diff --git a/infra/1-click/1-setup/5-manifests.tf b/infra/1-click/1-setup/5-manifests.tf index b293bb9..0561540 100644 --- a/infra/1-click/1-setup/5-manifests.tf +++ b/infra/1-click/1-setup/5-manifests.tf @@ -4,7 +4,58 @@ ## ## -# NodeClass + NodePool for Coder Server, Provisioner, & Workspaces +# EBS CSI StorageClasses +## + +resource "kubernetes_manifest" "default-sc" { + manifest = { + apiVersion = "storage.k8s.io/v1" + kind = "StorageClass" + metadata = { + name = "default" + annotations = { + "storageclass.kubernetes.io/is-default-class" = "true" + } + } + provisioner = "ebs.csi.aws.com" + volumeBindingMode = "WaitForFirstConsumer" + allowedTopologies = [{ + matchLabelExpressions = [{ + key = "topology.ebs.csi.aws.com/zone" + values = [for az in var.azs : "${data.aws_region.this.region}${az}"] + }] + }] + parameters = { + type = "gp3" + encrypted = "true" + } + } +} + +resource "kubernetes_manifest" "automode-sc" { + manifest = { + apiVersion = "storage.k8s.io/v1" + kind = "StorageClass" + metadata = { + name = "automode" + } + provisioner = "ebs.csi.eks.amazonaws.com" + volumeBindingMode = "WaitForFirstConsumer" + allowedTopologies = [{ + matchLabelExpressions = [{ + key = "eks.amazonaws.com/compute-type" + values = ["auto"] + }] + }] + parameters = { + type = "gp3" + encrypted = "true" + } + } +} + +## +# NodeClass(es) for Coder (Server, Provisioner, & Workspaces) ## data "kubernetes_service_account_v1" "kptr" { @@ -14,21 +65,40 @@ data "kubernetes_service_account_v1" "kptr" { } } - locals { nodeclass_configs = { "coder" = { - user_data = <<-EOF + api_version = "karpenter.k8s.aws/v1" + kind = "EC2NodeClass" + user_data = <<-EOT MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="//" --// Content-Type: application/node.eks.aws - ${file("${path.module}/scripts/nodeconfig.yaml")} - + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + config: + registryPullQPS: 30 + --//-- - EOF + EOT + subnet_selector = [{ + tags = { + "karpenter.sh/discovery" = "${local.formatted_name}" + } + }] + sg_selector = [{ + # Use for EKS AutoMode. AWS manages this, not TF: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster#cluster_security_group_id-1 + id = data.aws_eks_cluster.coder.vpc_config[0].cluster_security_group_id + # tags = { + # # If AutoMode is enabled, THIS BREAKS. Communication to CoreDNS locked behind system nodes. Kptr SG cant talk to AutoMode's SGs (only trusts itself and another AWS-managed SG) + # "karpenter.sh/discovery" = "${local.formatted_name}-karpenter" + # } + }] block_device_mappings = [{ deviceName = "/dev/xvda" ebs = { @@ -47,8 +117,8 @@ resource "kubernetes_manifest" "nodeclass" { for_each = local.nodeclass_configs manifest = { - apiVersion = "karpenter.k8s.aws/v1" - kind = "EC2NodeClass" + apiVersion = each.value.api_version + kind = each.value.kind metadata = { name = each.key } @@ -57,30 +127,43 @@ resource "kubernetes_manifest" "nodeclass" { amiSelectorTerms = [{ alias = "al2023@latest" }] - subnetSelectorTerms = [{ - tags = { - "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" - } - }] - securityGroupSelectorTerms = [{ - tags = { - "karpenter.sh/discovery" = "${var.name}-${local.normalized_domain_name}" - } - }] - blockDeviceMappings = each.value.block_device_mappings - userData = each.value.user_data + subnetSelectorTerms = each.value.subnet_selector + securityGroupSelectorTerms = each.value.sg_selector + blockDeviceMappings = each.value.block_device_mappings + userData = each.value.user_data } } } +## +# NodePool(s) for Coder (Server, Provisioner, & Workspaces) +## + locals { nodepool_configs = { - "coder" = { - node_expires_after = "Never" + # "automode" = { + # node_expires_after = "24h" + # disruption_consolidation_policy = "WhenEmpty" + # disruption_consolidate_after = "1m" + # instance_type = "t3a.large" + # node_class_ref = { + # group = "eks.amazonaws.com" + # kind = "NodeClass" + # name = "default" + # } + # taints = [] + # } + "karpenter" = { + node_expires_after = "24h" disruption_consolidation_policy = "WhenEmpty" disruption_consolidate_after = "1m" - instance_type = "t3a.large" - taints = [] + instance_type = "t3a.large" + node_class_ref = { + group = "karpenter.k8s.aws" + kind = "EC2NodeClass" + name = "coder" + } + taints = [] } } } @@ -88,7 +171,7 @@ locals { resource "kubernetes_manifest" "nodepool" { depends_on = [kubernetes_manifest.nodeclass] - for_each = local.nodepool_configs + for_each = local.nodepool_configs field_manager { force_conflicts = true @@ -121,7 +204,8 @@ resource "kubernetes_manifest" "nodepool" { "node.coder.io/managed-by" = "karpenter" "node.coder.io/name" = "coder" "node.coder.io/part-of" = "coder" - "node.coder.io/used-for" = each.key + # "eks.amazonaws.com/compute-type" = "auto" + "node.coder.io/used-for" = each.key } } spec = { @@ -145,14 +229,10 @@ resource "kubernetes_manifest" "nodepool" { }, { key = "node.kubernetes.io/instance-type" operator = "In" - values = [ each.value.instance_type ] + values = [each.value.instance_type] }] - nodeClassRef = { - group = "karpenter.k8s.aws" - kind = "EC2NodeClass" - name = each.key - } - expireAfter = each.value.node_expires_after + nodeClassRef = each.value.node_class_ref + expireAfter = each.value.node_expires_after } } disruption = { @@ -169,7 +249,6 @@ resource "kubernetes_manifest" "nodepool" { resource "kubernetes_daemon_set_v1" "img-fetch" { - depends_on = [kubernetes_manifest.nodepool] for_each = local.nodepool_configs metadata { @@ -196,12 +275,14 @@ resource "kubernetes_daemon_set_v1" "img-fetch" { } spec { + + # Select's Coder-specific nodes. Do not pull on system nodes. node_selector = kubernetes_manifest.nodepool[each.key].manifest.spec.template.metadata.labels toleration { - key = "dedicated" - value = each.key - effect = "NoSchedule" + key = "dedicated" + value = each.key + effect = "NoSchedule" } termination_grace_period_seconds = 5 @@ -230,205 +311,4 @@ resource "kubernetes_daemon_set_v1" "img-fetch" { } } } -} - -## -# EBS CSI Default StorageClass -## - -resource "kubernetes_manifest" "default-sc" { - manifest = { - apiVersion = "storage.k8s.io/v1" - kind = "StorageClass" - metadata = { - name = "default" - annotations = { - "storageclass.kubernetes.io/is-default-class" = "true" - } - } - provisioner = "ebs.csi.aws.com" - volumeBindingMode = "WaitForFirstConsumer" - parameters = { - type = "gp3" - encrypted = "true" - } - } -} - -## -# Cert-Manager ClusterIssuer for CA -## - -# If R53 enabled, then fetch service account from cert-manager for IAM Role -variable "r53_config" { - description = "Enable to use Route53 as a DNS01 provider for ACME challenges." - type = object({ - enabled = bool - }) - default = { - enabled = true - } -} - -data "kubernetes_service_account_v1" "r53" { - - count = var.r53_config.enabled ? 1 : 0 - - metadata { - name = "crt-mgr" - namespace = "cert-manager" - } -} - -# If CF enabled, then fetch secret from cert-manager for token -variable "cf_config" { - description = "Enable to use CloudFlare as a DNS01 provider for ACME challenges." - type = object({ - enabled = bool - email = string - name = optional(string, "cloudflare") - namespace = optional(string, "cert-manager") - }) - default = { - enabled = false - email = "" - name = "" - namespace = "" - } -} - -data "kubernetes_secret_v1" "cf" { - - count = var.cf_config.enabled ? 1 : 0 - - metadata { - name = var.cf_config.name - namespace = var.cf_config.namespace - } -} - -locals { - dns01_r53 = ! var.r53_config.enabled ? null : { - route53 = { - region = var.region - role = data.kubernetes_service_account_v1.r53[0].metadata[0].annotations["eks.amazonaws.com/role-arn"] - auth = { - kubernetes = { - serviceAccountRef = { - name = data.kubernetes_service_account_v1.r53[0].metadata[0].name - } - } - } - } - } - dns01_cf = ! var.cf_config.enabled ? null : { - cloudflare = { - apiTokenSecretRef = { - key = data.kubernetes_secret_v1.cf[0].metadata[0].annotations["custom.kubernetes.secret/key"] - name = data.kubernetes_secret_v1.cf[0].metadata[0].name - } - email = data.kubernetes_secret_v1.cf[0].metadata[0].annotations["custom.kubernetes.secret/email"] - } - } -} - -resource "kubernetes_manifest" "issuer" { - - field_manager { - force_conflicts = true - } - - wait { - condition { - type = "Ready" - status = "True" - } - } - - timeouts { - create = "10m" - update = "10m" - delete = "30s" - } - - manifest = { - apiVersion = "cert-manager.io/v1" - kind = "ClusterIssuer" - metadata = { - labels = {} - name = "issuer" - } - spec = { - acme = { - privateKeySecretRef = { - name = "issuer-account-key" - } - server = "https://acme-v02.api.letsencrypt.org/directory" - solvers = [ - { - dns01 = merge( - local.dns01_r53, - local.dns01_cf - ) - } - ] - } - } - } -} - -## -# External-Secrets ClusterStore for syncing TLS/SSL -## - -data "kubernetes_service_account_v1" "sm" { - metadata { - name = "external-secrets" - namespace = "ext-sec" - } -} - -resource "kubernetes_manifest" "secret-store" { - - field_manager { - force_conflicts = true - } - - wait { - condition { - type = "Ready" - status = "True" - } - } - - timeouts { - create = "10m" - update = "10m" - delete = "30s" - } - - manifest = { - apiVersion = "external-secrets.io/v1" - kind = "ClusterSecretStore" - metadata = { - labels = {} - name = "issuer" - } - spec = { - provider = { - aws = { - service = "SecretsManager" - region = var.region - auth = { - jwt = { - serviceAccountRef = { - name = data.kubernetes_service_account_v1.sm.metadata[0].name - namespace = data.kubernetes_service_account_v1.sm.metadata[0].namespace - } - } - } - } - } - } - } } \ No newline at end of file diff --git a/infra/1-click/1-setup/6-coder-server.tf b/infra/1-click/1-setup/6-coder-server.tf index 265de97..b5fcef8 100644 --- a/infra/1-click/1-setup/6-coder-server.tf +++ b/infra/1-click/1-setup/6-coder-server.tf @@ -1,157 +1,208 @@ ## -# Coder Helm Chart Installation w/ auxillary dependcies on: -# - cert-manager -# - karpenter -# - external-dns -# - external-secrets +# Fetch DNS Zone ## -variable "coder_username" { - description = "Coder DB's username." - type = string - default = "coder" +data "aws_route53_zone" "coder" { + name = local.apex_domain } -variable "coder_password" { - description = "Coder DB's password." - type = string - default = "th1s1sn0tas3cur3pass0wrd" - sensitive = true -} +## +# SSL Certificate +## -variable "coder_license" { - type = string - default = "" - sensitive = true +resource "aws_acm_certificate" "coder" { + domain_name = var.domain_name + subject_alternative_names = ["*.${var.domain_name}"] + validation_method = "DNS" } -variable "coder_admin_email" { - type = string - default = "admin@coder.com" -} +resource "aws_route53_record" "coder_validation" { -variable "coder_admin_username" { - type = string - default = "admin" -} + for_each = { + for rec in aws_acm_certificate.coder.domain_validation_options : rec.domain_name => { + name = rec.resource_record_name + record = rec.resource_record_value + type = rec.resource_record_type + } + } -variable "coder_admin_password" { - type = string - default = "Th1s1sN0TS3CuR3!!" - sensitive = true + allow_overwrite = true + name = each.value.name + records = [each.value.record] + ttl = 60 + type = each.value.type + zone_id = data.aws_route53_zone.coder.zone_id } -variable "grafana_username" { - description = "Grafana DB's username." - type = string - default = "grafana" -} +resource "aws_acm_certificate_validation" "coder" { + certificate_arn = aws_acm_certificate.coder.arn + validation_record_fqdns = [for record in aws_route53_record.coder_validation : record.fqdn] -variable "grafana_password" { - description = "Grafana DB's password." - type = string - default = "th1s1sn0tas3cur3pass0wrd" - sensitive = true + timeouts { + create = "30m" + } } -variable "grafana_admin_username" { - type = string - default = "admin" +resource "aws_acm_certificate" "grafana" { + domain_name = "grafana.${var.domain_name}" + subject_alternative_names = ["*.grafana.${var.domain_name}"] + validation_method = "DNS" } -variable "grafana_admin_password" { - type = string - default = "Th1s1sN0TS3CuR3!!" - sensitive = true -} +resource "aws_route53_record" "grafana_validation" { -variable "use_ext_dns" { - description = "Toggle the K8s 'external-dns' addon. Disable in-case you want to manage DNS records yourself." - type = bool - default = true -} + for_each = { + for rec in aws_acm_certificate.grafana.domain_validation_options : rec.domain_name => { + name = rec.resource_record_name + record = rec.resource_record_value + type = rec.resource_record_type + } + } -variable "azs" { - type = list(string) - default = ["a", "b", "c"] + allow_overwrite = true + name = each.value.name + records = [each.value.record] + ttl = 60 + type = each.value.type + zone_id = data.aws_route53_zone.coder.zone_id } -resource "aws_iam_user" "bedrock" { +resource "aws_acm_certificate_validation" "grafana" { + certificate_arn = aws_acm_certificate.grafana.arn + validation_record_fqdns = [for record in aws_route53_record.grafana_validation : record.fqdn] - count = var.coder_license != "" ? 1 : 0 + timeouts { + create = "30m" + } +} + +## +# ------ +## +resource "aws_iam_user" "bedrock" { name = "bedrock-access" path = "/${local.normalized_domain_name}/${data.aws_region.this.region}/" } resource "aws_iam_access_key" "bedrock" { - - count = var.coder_license != "" ? 1 : 0 - - user = aws_iam_user.bedrock[0].name + user = aws_iam_user.bedrock.name } resource "aws_iam_user_policy_attachment" "bedrock" { - - count = var.coder_license != "" ? 1 : 0 - - user = aws_iam_user.bedrock[0].name + user = aws_iam_user.bedrock.name # https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonBedrockLimitedAccess.html policy_arn = "arn:aws:iam::aws:policy/AmazonBedrockLimitedAccess" } +locals { + pub_subs = [for az in var.azs : "${local.formatted_name}-public-${data.aws_region.this.region}${az}"] +} + +## +# EIP Mapping +## + resource "aws_eip" "coder" { - count = length(var.azs) - domain = "vpc" + count = length(var.azs) + domain = "vpc" + public_ipv4_pool = "amazon" + tags = { + Name = "${local.formatted_name}-coder-${count.index}" + } +} + +resource "aws_eip" "grafana" { + count = 1 + domain = "vpc" public_ipv4_pool = "amazon" tags = { - Name = "${var.name}-${local.normalized_domain_name}-coder-${count.index}" + Name = "${local.formatted_name}-grafana-${count.index}" } } +resource "aws_route53_record" "coder-primary" { + name = var.domain_name + records = aws_eip.coder.*.public_ip + ttl = 60 + type = "A" + zone_id = data.aws_route53_zone.coder.zone_id +} + +resource "aws_route53_record" "coder-wildcard" { + name = "*.${var.domain_name}" + records = aws_eip.coder.*.public_ip + ttl = 60 + type = "A" + zone_id = data.aws_route53_zone.coder.zone_id +} + +resource "aws_route53_record" "grafana-primary" { + name = "grafana.${var.domain_name}" + records = aws_eip.grafana.*.public_ip + ttl = 60 + type = "A" + zone_id = data.aws_route53_zone.coder.zone_id +} + +resource "aws_route53_record" "grafana-wildcard" { + name = "*.grafana.${var.domain_name}" + records = aws_eip.grafana.*.public_ip + ttl = 60 + type = "A" + zone_id = data.aws_route53_zone.coder.zone_id +} + +## +# Helm Installs +## + locals { - pub_subs = [ for az in var.azs : "${var.name}-${local.normalized_domain_name}-public-${data.aws_region.this.region}${az}"] + coder_release_name = "coder" + coder_ns = "coder" } module "coder-server" { - depends_on = [kubernetes_manifest.nodepool] - source = "../../../modules/k8s/bootstrap/coder-server" - cluster_name = "${var.name}-${local.normalized_domain_name}" + release_name = local.coder_release_name + chart_version = var.coder_version + cluster_name = data.aws_eks_cluster.coder.id cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn - namespace = "coder" - - helm_version = "2.29.4" + namespace = local.coder_ns + # lb_class = "eks.amazonaws.com/nlb" + lb_class = "service.k8s.aws/nlb" coder = { - access_url = "https://${var.domain_name}" + access_url = "https://${var.domain_name}" wildcard_url = "*.${var.domain_name}" - pub_ips = aws_eip.coder.*.public_ip - image_repo = "ghcr.io/coder/coder" - image_tag = "v2.29.4" - rep_cnt = 1 - # Use this instead of external provisioners. DNS might not propagate fast enough for Coder to be "reachable". - prov_rep_cnt = 4 - # csp_policy = "frame-src https://${var.domain_name}" + image_repo = "ghcr.io/coder/coder" + image_tag = "v${var.coder_version}" + rep_cnt = 1 + # External Provisioners will be used + prov_rep_cnt = var.coder_license != "" ? 0 : 5 + } + + prometheus = { + enable = true } db = { - url = data.aws_db_instance.coder.endpoint + url = data.aws_db_instance.coder.endpoint username = var.coder_username password = var.coder_password } - aibridge = var.coder_license == "" ? null : { - enabled = true - bedrock = { - region = data.aws_region.this.region - model = "global.anthropic.claude-opus-4-5-20251101-v1:0" - access_id = aws_iam_access_key.bedrock[0].id - secret_id = aws_iam_access_key.bedrock[0].secret - } + aibridge = { + enabled = var.coder_license != "" + bedrock = var.coder_license != "" ? { + region = data.aws_region.this.region + model = "global.anthropic.claude-opus-4-5-20251101-v1:0" + access_id = aws_iam_access_key.bedrock.id + secret_id = aws_iam_access_key.bedrock.secret + } : null } resource_request = { @@ -163,29 +214,20 @@ module "coder-server" { memory = "2Gi" } - cert_config = { - name = var.domain_name - kind = kubernetes_manifest.issuer.manifest.kind - issuer = kubernetes_manifest.issuer.manifest.metadata.name - store = kubernetes_manifest.secret-store.manifest.metadata.name - create_secret = true - } - tags = {} - svc_annot = merge({ + svc_annot = { "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" = join(",", aws_eip.coder.*.allocation_id) - "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", local.pub_subs) - }, !var.use_ext_dns ? null : { - "external-dns.alpha.kubernetes.io/hostname" = "${var.domain_name},*.${var.domain_name}" - "external-dns.alpha.kubernetes.io/ttl" = 60 - }) - - node_selector = kubernetes_manifest.nodepool["coder"].manifest.spec.template.metadata.labels - tolerations = [for toleration in kubernetes_manifest.nodepool["coder"].manifest.spec.template.spec.taints : { + "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", local.pub_subs) + "service.beta.kubernetes.io/aws-load-balancer-ssl-cert" = aws_acm_certificate.coder.arn + "service.beta.kubernetes.io/aws-load-balancer-ssl-ports" = 443 + } + + node_selector = kubernetes_manifest.nodepool["karpenter"].manifest.spec.template.metadata.labels + tolerations = [for toleration in kubernetes_manifest.nodepool["karpenter"].manifest.spec.template.spec.taints : { key = toleration.key operator = "Equal" value = toleration.value @@ -198,7 +240,7 @@ module "coder-server" { when_unsatisfiable = "ScheduleAnyway" label_selector = { match_labels = { - "app.kubernetes.io/name" = "coder" + "app.kubernetes.io/name" = local.coder_release_name "app.kubernetes.io/part-of" = "coder" } } @@ -206,260 +248,18 @@ module "coder-server" { "app.kubernetes.io/instance" ] }] + pod_aaf_pref_sched_ie = [{ weight = 100 pod_affinity_term = { label_selector = { match_labels = { - "app.kubernetes.io/instance" = "coder-v2" - "app.kubernetes.io/name" = "coder" + "app.kubernetes.io/instance" = "coder" + "app.kubernetes.io/name" = local.coder_release_name "app.kubernetes.io/part-of" = "coder" } } topology_key = "kubernetes.io/hostname" } }] -} - -# Wait for DNS propagation. May require multiple redeploys -# resource "time_sleep" "wait_for_dns" { -# depends_on = [ module.coder-server ] -# create_duration = "120s" -# } - -data "http" "first-user" { - - # depends_on = [ time_sleep.wait_for_dns ] - depends_on = [ module.coder-server ] - - url = "https://${aws_eip.coder.0.public_ip}/api/v2/users/first" - method = "POST" - insecure = true - request_headers = { - Host = var.domain_name - Accept = "application/json" - } - request_body = jsonencode({ - email = var.coder_admin_email - username = var.coder_admin_username - password = var.coder_admin_password - trial = false - }) - retry { - attempts = 60+1 - max_delay_ms = 5000 - min_delay_ms = 5000 - } -} - -data "http" "login" { - - depends_on = [data.http.first-user] - - url = "https://${aws_eip.coder[0].public_ip}/api/v2/users/login" - insecure = true - method = "POST" - request_headers = { - Host = var.domain_name - Accept = "application/json" - } - request_body = jsonencode({ - email = var.coder_admin_email - password = var.coder_admin_password - }) -} - -## -# Adding a license crashes Coder temporarily. -# Use 'external' to allow custom handling, and then wait before proceeding. -## - -data "external" "add-license" { - - count = var.coder_license != "" ? 1 : 0 - - program = ["bash", "${path.module}/scripts/add-license.sh"] - - query = { - ip_addr = aws_eip.coder[0].public_ip - domain = var.domain_name - license_key = var.coder_license - session_token = jsondecode(data.http.login.response_body).session_token - } -} - -resource "time_sleep" "wait_for_coder" { - - count = var.coder_license != "" ? 1 : 0 - - depends_on = [ data.external.add-license[0] ] - create_duration = "30s" -} - -resource "aws_eip" "grafana" { - count = length(var.azs) - domain = "vpc" - public_ipv4_pool = "amazon" - tags = { - Name = "${var.name}-${local.normalized_domain_name}-grafana-${count.index}" - } -} - -module "monitoring" { - source = "../../../modules/k8s/bootstrap/monitoring" - - chart_version = "0.7.0-rc.1" - cluster_name = "${var.name}-${local.normalized_domain_name}" - cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn - - domain_name = var.domain_name - tolerations = concat([{ - key = "CriticalAddonsOnly" - operator = "Exists" - }], [for toleration in kubernetes_manifest.nodepool["coder"].manifest.spec.template.spec.taints : { - key = toleration.key - operator = "Equal" - value = toleration.value - effect = toleration.effect - }]) - - coder = { - db = { - host = data.aws_db_instance.coder.address - password = var.coder_password - username = var.coder_username - database = data.aws_db_instance.coder.db_name - } - selector = { - coderd = "pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=~`(coder)`" - provisionerd = "pod=~`coder-provisioner.*`, namespace=~`(coder)`" - workspaces = "pod!~`coder.*`, namespace=~`(coder)`" - ctrl_plane_ns = "coder" - ext_prov_ns = "coder" - } - } - grafana = { - admin = { - username = var.grafana_admin_username - password = var.grafana_admin_password - } - db = { - host = data.aws_db_instance.grafana.address - password = var.grafana_password - username = var.grafana_username - database = data.aws_db_instance.grafana.db_name - } - svc = { - annots = merge({ - "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" - "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" - "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" - "service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol" = "https" - "service.beta.kubernetes.io/aws-load-balancer-healthcheck-path" = "/api/health" - "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" = join(",", aws_eip.grafana.*.allocation_id) - "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", local.pub_subs) - }, !var.use_ext_dns ? null : { - "external-dns.alpha.kubernetes.io/hostname" = "grafana.${var.domain_name},*.grafana.${var.domain_name}" - "external-dns.alpha.kubernetes.io/ttl" = 60 - }) - } - } - loki = { - s3 = { - chunks_bucket = data.aws_s3_bucket.loki.id - ruler_bucket = data.aws_s3_bucket.loki.id - region = data.aws_s3_bucket.loki.bucket_region - } - } -} - -## -# Coder Binary Prefetch -## - -locals { - coder_path = "/opt/coder/bin" - bin_fetch_script = <<-EOF - curl -kfsSL --retry 10 --retry-delay 5 -H "Host: ${var.domain_name}" https://coder.coder.svc.cluster.local/bin/coder-linux-amd64 -o ${local.coder_path}/coder - chmod +x ${local.coder_path}/coder - EOF -} - -resource "kubernetes_daemon_set_v1" "bin-fetch" { - - metadata { - name = "coder-bin-fetch" - namespace = module.coder-server.namespace - labels = { - "app.kubernetes.io/name" = "bin-fetch" - "app.kubernetes.io/part-of" = "coder-workspaces" - } - } - - spec { - - selector { - match_labels = { - "app.kubernetes.io/name" = "bin-fetch" - } - } - - template { - - metadata { - labels = { - "app.kubernetes.io/name" = "bin-fetch" - } - } - - spec { - - host_aliases { - hostnames = [var.domain_name] - ip = aws_eip.coder[0].public_ip - } - - security_context { - run_as_user = "0" - } - - init_container { - name = "fetch-binary" - image = "curlimages/curl:latest" - command = ["sh", "-c", "${local.bin_fetch_script}"] - - volume_mount { - name = "coder-bin" - mount_path = local.coder_path - read_only = false - } - - } - - container { - name = "pause" - image = "registry.k8s.io/pause:3.9" - - resources { - requests = { - cpu = "1m" - memory = "1Mi" - } - limits = { - cpu = "10m" - memory = "10Mi" - } - } - } - - volume { - name = "coder-bin" - host_path { - path = local.coder_path - type = "DirectoryOrCreate" - } - } - } - } - } } \ No newline at end of file diff --git a/infra/1-click/1-setup/7-coder-init.tf b/infra/1-click/1-setup/7-coder-init.tf new file mode 100644 index 0000000..03be15e --- /dev/null +++ b/infra/1-click/1-setup/7-coder-init.tf @@ -0,0 +1,111 @@ +# Wait for DNS propagation. May require multiple requests + +data "http" "first-user" { + + depends_on = [module.coder-server] + + url = "http://${aws_eip.coder.0.public_ip}/api/v2/users/first" + method = "POST" + request_headers = { + Host = var.domain_name + Accept = "application/json" + } + request_body = jsonencode({ + email = var.coder_admin_email + username = var.coder_admin_username + password = var.coder_admin_password + trial = false + }) + retry { + attempts = 60 + 1 + max_delay_ms = 5000 + min_delay_ms = 5000 + } +} + +locals { + coder_svc_url = "http://${local.coder_release_name}.${local.coder_ns}.svc.cluster.local" +} + +module "coder-kube-logstream" { + + depends_on = [module.coder-server] + source = "../../../modules/k8s/bootstrap/coder-logstream" + + coder = { + access_url = local.coder_svc_url + ws_ns = [local.coder_ns] + } +} + +module "monitoring" { + + source = "../../../modules/k8s/bootstrap/monitoring" + + chart_version = "0.7.0-rc.1" + chart_timeout = 360 + cluster_name = "${var.name}-${local.normalized_domain_name}" + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn + + lb_class = "service.k8s.aws/nlb" + storage_class = kubernetes_manifest.automode-sc.manifest.metadata.name + + domain_name = "grafana.${var.domain_name}" + tolerations = concat([{ + key = "CriticalAddonsOnly" + operator = "Exists" + }], [for toleration in kubernetes_manifest.nodepool["karpenter"].manifest.spec.template.spec.taints : { + key = toleration.key + operator = "Equal" + value = toleration.value + effect = toleration.effect + }]) + + coder = { + db = { + host = data.aws_db_instance.coder.address + password = var.coder_password + username = var.coder_username + database = data.aws_db_instance.coder.db_name + } + selector = { + coderd = "pod=~`coder.*`, pod!~`.*provisioner.*`, namespace=~`(coder)`" + provisionerd = "pod=~`coder-provisioner.*`, namespace=~`(coder)`" + workspaces = "pod!~`coder.*`, namespace=~`(coder)`" + ctrl_plane_ns = "coder" + ext_prov_ns = "coder" + } + } + grafana = { + admin = { + username = var.grafana_admin_username + password = var.grafana_admin_password + } + db = { + host = data.aws_db_instance.grafana.address + password = var.grafana_password + username = var.grafana_username + database = data.aws_db_instance.grafana.db_name + } + svc = { + annots = { + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol" = "tcp" + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-path" = "/api/health" + "service.beta.kubernetes.io/aws-load-balancer-ssl-cert" = aws_acm_certificate.grafana.arn + "service.beta.kubernetes.io/aws-load-balancer-ssl-ports" = 443 + "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" = join(",", aws_eip.grafana.*.allocation_id) + "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", [local.pub_subs[0]]) + } + } + } + loki = { + s3 = { + chunks_bucket = data.aws_s3_bucket.loki.id + ruler_bucket = data.aws_s3_bucket.loki.id + region = data.aws_s3_bucket.loki.bucket_region + } + } +} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/aibridge-demoable.json b/infra/1-click/1-setup/dashboards/aibridge-demoable.json deleted file mode 100644 index 70d570f..0000000 --- a/infra/1-click/1-setup/dashboards/aibridge-demoable.json +++ /dev/null @@ -1,1382 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 138, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 11, - "panels": [], - "title": "Usage leaderboards", - "type": "row" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\norder by input desc", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Total usage for $username", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": { - "cache_creation_input": 3, - "cache_read_input": 2, - "input": 0, - "output": 1 - }, - "renameByName": { - "cache_creation_input": "Cache Write", - "cache_read_input": "Cache Read", - "input": "Input", - "output": "Ouput" - } - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "output" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache Read" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Input" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache Write" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 20, - "x": 4, - "y": 1 - }, - "id": 1, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select u.username, sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\ngroup by u.username\norder by input desc", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Leaderboard per user", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "cache_creation_input": "Cache Write", - "cache_read_input": "Cache Read", - "input": "Input", - "output": "Output", - "username": "" - } - } - } - ], - "type": "barchart" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Cost.*/" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - }, - { - "id": "decimals", - "value": 2 - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 12, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": true - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Total Cost" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "columns": [], - "computed_columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "dezthk9i9m48we" - }, - "filterExpression": "", - "filters": [], - "format": "table", - "global_query_id": "", - "hide": false, - "pagination_mode": "none", - "parser": "backend", - "refId": "A", - "root_selector": "$ ~> $each(function($v, $k) {\n {\n \"model\": $k,\n \"input_cost_per_token\": $v.input_cost_per_token ? $v.input_cost_per_token : 0,\n \"output_cost_per_token\": $v.output_cost_per_token ? $v.output_cost_per_token : 0,\n \"cache_creation_input_token_cost\": $v.cache_creation_input_token_cost ? $v.cache_creation_input_token_cost : 0,\n \"cache_read_input_token_cost\": $v.cache_read_input_token_cost ? $v.cache_read_input_token_cost : 0\n }\n})", - "source": "url", - "type": "json", - "url": "", - "url_options": { - "data": "", - "method": "GET" - } - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "select i.provider, i.model,\nsum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\ngroup by i.provider, i.model\norder by input desc", - "refId": "B", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Approximate Cost", - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "model", - "mode": "inner" - } - }, - { - "id": "calculateField", - "options": { - "alias": "Input Cost", - "binary": { - "left": { - "matcher": { - "id": "byName", - "options": "input_cost_per_token A" - } - }, - "operator": "*", - "right": { - "matcher": { - "id": "byName", - "options": "input" - } - } - }, - "mode": "binary", - "reduce": { - "include": [ - "input_cost_per_token A", - "input" - ], - "reducer": "sum" - } - } - }, - { - "id": "calculateField", - "options": { - "alias": "Output Cost", - "binary": { - "left": { - "matcher": { - "id": "byName", - "options": "output_cost_per_token A" - } - }, - "operator": "*", - "right": { - "matcher": { - "id": "byName", - "options": "output" - } - } - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "calculateField", - "options": { - "alias": "Cache Read Cost", - "binary": { - "left": { - "matcher": { - "id": "byName", - "options": "cache_read_input_token_cost A" - } - }, - "operator": "*", - "right": { - "matcher": { - "id": "byName", - "options": "cache_read_input" - } - } - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "calculateField", - "options": { - "alias": "Cache Write Cost", - "binary": { - "left": { - "matcher": { - "id": "byName", - "options": "cache_creation_input_token_cost A" - } - }, - "operator": "*", - "right": { - "matcher": { - "id": "byName", - "options": "cache_creation_input" - } - } - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "calculateField", - "options": { - "alias": "Total Cost", - "binary": { - "left": { - "matcher": { - "id": "byType", - "options": "number" - } - }, - "right": { - "fixed": "" - } - }, - "cumulative": { - "field": "Input Cost", - "reducer": "sum" - }, - "mode": "reduceRow", - "reduce": { - "include": [ - "Input Cost", - "Output Cost", - "Cache Read Cost", - "Cache Write Cost" - ], - "reducer": "sum" - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "cache_creation_input": false, - "cache_creation_input_token_cost A": true, - "cache_read_input": false, - "cache_read_input_token_cost A": true, - "input": false, - "input_cost_per_token A": true, - "output": false, - "output_cost_per_token A": true - }, - "includeByName": {}, - "indexByName": { - "Cache Read Cost": 13, - "Cache Write Cost": 14, - "Input Cost": 11, - "Output Cost": 12, - "Total Cost": 2, - "cache_creation_input": 10, - "cache_creation_input_token_cost A": 3, - "cache_read_input": 9, - "cache_read_input_token_cost A": 4, - "input": 7, - "input_cost_per_token A": 5, - "model": 1, - "output": 8, - "output_cost_per_token A": 6, - "provider": 0 - }, - "renameByName": { - "cache_creation_input": "Cache Write", - "cache_read_input": "Cache Read", - "input": "Input", - "model": "Model", - "output": "Output", - "provider": "Provider" - } - } - } - ], - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 10, - "panels": [], - "title": "Interceptions", - "type": "row" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "output" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 4, - "x": 0, - "y": 23 - }, - "id": 5, - "interval": "1m", - "maxDataPoints": 500, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select count(*) from aibridge_interceptions\nWHERE started_at > $__timeFrom() AND started_at <= $__timeTo()\nAND provider ~ '${provider:regex}'\nAND model ~ '${model:regex}'", - "refId": "A", - "sql": { - "columns": [ - { - "name": "COUNT", - "parameters": [ - { - "name": "id", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "name": "started_at", - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "aibridge_interceptions" - } - ], - "title": "Total interceptions", - "type": "stat" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "output" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 20, - "x": 4, - "y": 23 - }, - "id": 4, - "maxDataPoints": 30, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "normal", - "text": { - "valueSize": 10 - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": -45, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "editorMode": "code", - "format": "time_series", - "rawQuery": true, - "rawSql": "SELECT\n$__timeGroupAlias(i.started_at, $__interval, NULL),\ncount(i.id) AS value,\nu.username AS metric\nFROM aibridge_interceptions i\njoin users u ON i.initiator_id = u.id\nWHERE\n$__timeFilter(i.started_at)\nAND u.username ~ '${username:regex}'\nAND i.provider ~ '${provider:regex}'\nAND i.model ~ '${model:regex}'\nGROUP BY u.username, $__timeGroup(i.started_at, $__interval)\nORDER BY $__timeGroup(i.started_at, $__interval)", - "refId": "A", - "sql": { - "columns": [ - { - "name": "COUNT", - "parameters": [ - { - "name": "id", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "name": "started_at", - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "aibridge_interceptions" - } - ], - "title": "Interceptions over time by user", - "type": "barchart" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 9, - "panels": [], - "title": "Prompts & tool calls details", - "type": "row" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto", - "wrapText": false - }, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Interception ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 357 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Model" - }, - "properties": [ - { - "id": "custom.width", - "value": 240 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Provider" - }, - "properties": [ - { - "id": "custom.width", - "value": 157 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Username" - }, - "properties": [ - { - "id": "custom.width", - "value": 188 - } - ] - } - ] - }, - "gridPos": { - "h": 14, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 7, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Created At" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT i.id,\n u.username,\n i.provider,\n i.model,\n 'REDACTED' as prompt,\n p.created_at\nFROM aibridge_user_prompts p\nJOIN aibridge_interceptions i ON p.interception_id = i.id\nJOIN users u ON i.initiator_id = u.id\nWHERE $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\nORDER BY p.created_at DESC;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "User Prompts", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "created_at": "Created At", - "id": "Interception ID", - "input": "Tool Input", - "invocation_error": "Tool Error", - "model": "Model", - "prompt": "Prompt", - "provider": "Provider", - "server_url": "MCP Server", - "tool": "Tool Name", - "username": "Username" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto", - "wrapText": false - }, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Tool Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 342 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "invocation_error" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "applyToRow": true, - "type": "color-background", - "wrapText": false - } - }, - { - "id": "noValue" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "match": "null", - "result": { - "color": "green", - "index": 0 - } - }, - "type": "special" - }, - { - "options": { - "pattern": ".+", - "result": { - "color": "red", - "index": 1 - } - }, - "type": "regex" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Tool Input" - }, - "properties": [ - { - "id": "custom.width", - "value": 309 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Interception ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 357 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Model" - }, - "properties": [ - { - "id": "custom.width", - "value": 240 - } - ] - } - ] - }, - "gridPos": { - "h": 14, - "w": 24, - "x": 0, - "y": 50 - }, - "id": 6, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Created At" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select i.id, u.username, i.provider, i.model, t.server_url, t.tool, 'REDACTED' as input, CASE WHEN t.invocation_error IS NULL THEN '' ELSE 'REDACTED' END as invocation_error, t.created_at FROM aibridge_tool_usages t\njoin aibridge_interceptions i ON t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\nAND u.username ~ '${username:regex}'\nAND i.provider ~ '${provider:regex}'\nAND i.model ~ '${model:regex}'\norder by t.created_at desc", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Tool Calls", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "created_at": "Created At", - "id": "Interception ID", - "input": "Tool Input", - "invocation_error": "Tool Error", - "model": "Model", - "provider": "Provider", - "server_url": "MCP Server", - "tool": "Tool Name", - "username": "Username" - } - } - } - ], - "type": "table" - } - ], - "preload": false, - "refresh": "1m", - "schemaVersion": 41, - "tags": [], - "templating": { - "list": [ - { - "allValue": ".+", - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "definition": "select username from users where deleted=false;", - "description": "", - "includeAll": true, - "multi": true, - "name": "username", - "options": [], - "query": "select username from users where deleted=false;", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "definition": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", - "description": "", - "includeAll": true, - "multi": true, - "name": "provider", - "options": [], - "query": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "aezivxsm5pnggc" - }, - "definition": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '${provider:regex}' ORDER BY 1;", - "description": "", - "includeAll": true, - "multi": true, - "name": "model", - "options": [], - "query": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '${provider:regex}' ORDER BY 1;", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-7d", - "to": "now" - }, - "timepicker": {}, - "timezone": "utc", - "title": "aibridge [DEMOABLE]", - "uid": "0c61d33f-c809-4184-9e88-cb27e2d9d223", - "version": 2 -} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/aibridge-example.json b/infra/1-click/1-setup/dashboards/aibridge-example.json deleted file mode 100644 index 304d49a..0000000 --- a/infra/1-click/1-setup/dashboards/aibridge-example.json +++ /dev/null @@ -1,1411 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_CODER-OBSERVABILITY-RO", - "label": "coder-observability-ro", - "description": "", - "type": "datasource", - "pluginId": "grafana-postgresql-datasource", - "pluginName": "PostgreSQL" - }, - { - "name": "DS_LITELLM-PRICING-DATA", - "label": "litellm-pricing-data", - "description": "", - "type": "datasource", - "pluginId": "yesoreyeram-infinity-datasource", - "pluginName": "Infinity" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "panel", - "id": "barchart", - "name": "Bar chart", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "12.1.0" - }, - { - "type": "datasource", - "id": "grafana-postgresql-datasource", - "name": "PostgreSQL", - "version": "12.1.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "datasource", - "id": "yesoreyeram-infinity-datasource", - "name": "Infinity", - "version": "3.6.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 11, - "panels": [], - "title": "Usage leaderboards", - "type": "row" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "output" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache Read" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Input" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache Write" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 20, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select u.username, sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\ngroup by u.username\norder by input desc", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Leaderboard per user", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "cache_creation_input": "Cache Write", - "cache_read_input": "Cache Read", - "input": "Input", - "output": "Output", - "username": "" - } - } - } - ], - "type": "barchart" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\norder by input desc", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Total usage for $username", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": { - "cache_creation_input": 3, - "cache_read_input": 2, - "input": 0, - "output": 1 - }, - "renameByName": { - "cache_creation_input": "Cache Write", - "cache_read_input": "Cache Read", - "input": "Input", - "output": "Output" - } - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "datasource", - "uid": "-- Mixed --" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Cost.*/" - }, - "properties": [ - { - "id": "unit", - "value": "currencyUSD" - }, - { - "id": "decimals", - "value": 2 - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 12, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": true - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Total Cost" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "columns": [], - "computed_columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "${DS_LITELLM-PRICING-DATA}" - }, - "filterExpression": "", - "filters": [], - "format": "table", - "global_query_id": "", - "hide": false, - "pagination_mode": "none", - "parser": "backend", - "refId": "A", - "root_selector": "$ ~> $each(function($v, $k) {\n {\n \"model\": $k,\n \"input_cost_per_token\": $v.input_cost_per_token ? $v.input_cost_per_token : 0,\n \"output_cost_per_token\": $v.output_cost_per_token ? $v.output_cost_per_token : 0,\n \"cache_creation_input_token_cost\": $v.cache_creation_input_token_cost ? $v.cache_creation_input_token_cost : 0,\n \"cache_read_input_token_cost\": $v.cache_read_input_token_cost ? $v.cache_read_input_token_cost : 0\n }\n})", - "source": "url", - "type": "json", - "url": "", - "url_options": { - "data": "", - "method": "GET" - } - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "select i.provider, i.model,\nsum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\ngroup by i.provider, i.model\norder by input desc", - "refId": "B", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Approximate Cost", - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "model", - "mode": "inner" - } - }, - { - "id": "calculateField", - "options": { - "alias": "Input Cost", - "binary": { - "left": { - "matcher": { - "id": "byName", - "options": "input_cost_per_token A" - } - }, - "operator": "*", - "right": { - "matcher": { - "id": "byName", - "options": "input" - } - } - }, - "mode": "binary", - "reduce": { - "include": ["input_cost_per_token A", "input"], - "reducer": "sum" - } - } - }, - { - "id": "calculateField", - "options": { - "alias": "Output Cost", - "binary": { - "left": { - "matcher": { - "id": "byName", - "options": "output_cost_per_token A" - } - }, - "operator": "*", - "right": { - "matcher": { - "id": "byName", - "options": "output" - } - } - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "calculateField", - "options": { - "alias": "Cache Read Cost", - "binary": { - "left": { - "matcher": { - "id": "byName", - "options": "cache_read_input_token_cost A" - } - }, - "operator": "*", - "right": { - "matcher": { - "id": "byName", - "options": "cache_read_input" - } - } - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "calculateField", - "options": { - "alias": "Cache Write Cost", - "binary": { - "left": { - "matcher": { - "id": "byName", - "options": "cache_creation_input_token_cost A" - } - }, - "operator": "*", - "right": { - "matcher": { - "id": "byName", - "options": "cache_creation_input" - } - } - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "calculateField", - "options": { - "alias": "Total Cost", - "binary": { - "left": { - "matcher": { - "id": "byType", - "options": "number" - } - }, - "right": { - "fixed": "" - } - }, - "cumulative": { - "field": "Input Cost", - "reducer": "sum" - }, - "mode": "reduceRow", - "reduce": { - "include": [ - "Input Cost", - "Output Cost", - "Cache Read Cost", - "Cache Write Cost" - ], - "reducer": "sum" - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "cache_creation_input": false, - "cache_creation_input_token_cost A": true, - "cache_read_input": false, - "cache_read_input_token_cost A": true, - "input": false, - "input_cost_per_token A": true, - "output": false, - "output_cost_per_token A": true - }, - "includeByName": {}, - "indexByName": { - "Cache Read Cost": 12, - "Cache Write Cost": 13, - "Input Cost": 10, - "Output Cost": 11, - "Total Cost": 14, - "cache_creation_input": 9, - "cache_creation_input_token_cost A": 2, - "cache_read_input": 8, - "cache_read_input_token_cost A": 3, - "input": 6, - "input_cost_per_token A": 4, - "model": 1, - "output": 7, - "output_cost_per_token A": 5, - "provider": 0 - }, - "renameByName": { - "cache_creation_input": "Cache Write", - "cache_read_input": "Cache Read", - "input": "Input", - "model": "Model", - "output": "Output", - "provider": "Provider" - } - } - } - ], - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 10, - "panels": [], - "title": "Interceptions", - "type": "row" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "output" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 20, - "x": 0, - "y": 23 - }, - "id": 4, - "maxDataPoints": 30, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "normal", - "text": { - "valueSize": 10 - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": -45, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "editorMode": "code", - "format": "time_series", - "rawQuery": true, - "rawSql": "SELECT\n$__timeGroupAlias(i.started_at, $__interval, NULL),\ncount(i.id) AS value,\nu.username AS metric\nFROM aibridge_interceptions i\njoin users u ON i.initiator_id = u.id\nWHERE\n$__timeFilter(i.started_at)\nAND u.username ~ '${username:regex}'\nAND i.provider ~ '${provider:regex}'\nAND i.model ~ '${model:regex}'\nGROUP BY u.username, $__timeGroup(i.started_at, $__interval)\nORDER BY $__timeGroup(i.started_at, $__interval)", - "refId": "A", - "sql": { - "columns": [ - { - "name": "COUNT", - "parameters": [ - { - "name": "id", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "name": "started_at", - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "aibridge_interceptions" - } - ], - "title": "Interceptions over time by user", - "type": "barchart" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "output" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 4, - "x": 20, - "y": 23 - }, - "id": 5, - "interval": "1m", - "maxDataPoints": 500, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select count(*) from aibridge_interceptions\nWHERE started_at > $__timeFrom() AND started_at <= $__timeTo()\nAND provider ~ '${provider:regex}'\nAND model ~ '${model:regex}'", - "refId": "A", - "sql": { - "columns": [ - { - "name": "COUNT", - "parameters": [ - { - "name": "id", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "name": "started_at", - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "aibridge_interceptions" - } - ], - "title": "Total interceptions", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 9, - "panels": [], - "title": "Prompts & tool calls details", - "type": "row" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto", - "wrapText": false - }, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Interception ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 357 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Model" - }, - "properties": [ - { - "id": "custom.width", - "value": 240 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Provider" - }, - "properties": [ - { - "id": "custom.width", - "value": 157 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Username" - }, - "properties": [ - { - "id": "custom.width", - "value": 188 - } - ] - } - ] - }, - "gridPos": { - "h": 14, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 7, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Created At" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT i.id,\n u.username,\n i.provider,\n i.model,\n p.prompt,\n p.created_at\nFROM aibridge_user_prompts p\nJOIN aibridge_interceptions i ON p.interception_id = i.id\nJOIN users u ON i.initiator_id = u.id\nWHERE $__timeFilter(i.started_at)\n AND u.username ~ '${username:regex}'\n AND i.provider ~ '${provider:regex}'\n AND i.model ~ '${model:regex}'\nORDER BY p.created_at DESC;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "User Prompts", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "created_at": "Created At", - "id": "Interception ID", - "input": "Tool Input", - "invocation_error": "Tool Error", - "model": "Model", - "prompt": "Prompt", - "provider": "Provider", - "server_url": "MCP Server", - "tool": "Tool Name", - "username": "Username" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto", - "wrapText": false - }, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Tool Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 342 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "invocation_error" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "applyToRow": true, - "type": "color-background", - "wrapText": false - } - }, - { - "id": "noValue" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "match": "null", - "result": { - "color": "green", - "index": 0 - } - }, - "type": "special" - }, - { - "options": { - "pattern": ".+", - "result": { - "color": "red", - "index": 1 - } - }, - "type": "regex" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Tool Input" - }, - "properties": [ - { - "id": "custom.width", - "value": 309 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Interception ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 357 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Model" - }, - "properties": [ - { - "id": "custom.width", - "value": 240 - } - ] - } - ] - }, - "gridPos": { - "h": 14, - "w": 24, - "x": 0, - "y": 50 - }, - "id": 6, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Created At" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select i.id, u.username, i.provider, i.model, t.server_url, t.tool, t.input, t.invocation_error, t.created_at FROM aibridge_tool_usages t\njoin aibridge_interceptions i ON t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\nAND u.username ~ '${username:regex}'\nAND i.provider ~ '${provider:regex}'\nAND i.model ~ '${model:regex}'\norder by t.created_at desc", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Tool Calls", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "created_at": "Created At", - "id": "Interception ID", - "input": "Tool Input", - "invocation_error": "Tool Error", - "model": "Model", - "provider": "Provider", - "server_url": "MCP Server", - "tool": "Tool Name", - "username": "Username" - } - } - } - ], - "type": "table" - } - ], - "refresh": "1m", - "schemaVersion": 41, - "tags": [], - "templating": { - "list": [ - { - "allValue": ".+", - "current": {}, - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "definition": "select username from users where deleted=false;", - "description": "", - "includeAll": true, - "multi": true, - "name": "username", - "options": [], - "query": "select username from users where deleted=false;", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": {}, - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "definition": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", - "description": "", - "includeAll": true, - "multi": true, - "name": "provider", - "options": [], - "query": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": {}, - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "postgres" - }, - "definition": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '${provider:regex}' ORDER BY 1;", - "description": "", - "includeAll": true, - "multi": true, - "name": "model", - "options": [], - "query": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '${provider:regex}' ORDER BY 1;", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-7d", - "to": "now" - }, - "timepicker": {}, - "timezone": "utc", - "title": "aibridge", - "uid": "0c61d33f-c809-4184-9e88-cb27e2d9d224", - "version": 43, - "weekStart": "" -} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/aibridge.json b/infra/1-click/1-setup/dashboards/aibridge.json deleted file mode 100644 index 0d1fb72..0000000 --- a/infra/1-click/1-setup/dashboards/aibridge.json +++ /dev/null @@ -1,1080 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_CODER-OBSERVABILITY-RO", - "label": "coder-observability-ro", - "description": "", - "type": "datasource", - "pluginId": "grafana-postgresql-datasource", - "pluginName": "PostgreSQL" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "panel", - "id": "barchart", - "name": "Bar chart", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "12.1.0" - }, - { - "type": "datasource", - "id": "grafana-postgresql-datasource", - "name": "PostgreSQL", - "version": "12.1.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 11, - "panels": [], - "title": "Usage leaderboards", - "type": "row" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "output" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache Read" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Input" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache Write" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 20, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select u.username, sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '$${username:regex}'\n AND i.provider ~ '$${provider:regex}'\n AND i.model ~ '$${model:regex}'\ngroup by u.username\norder by input desc", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Leaderboard per user", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "cache_creation_input": "Cache Write", - "cache_read_input": "Cache Read", - "input": "Input", - "output": "Output", - "username": "" - } - } - } - ], - "type": "barchart" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select sum(t.input_tokens) as input,\nsum(t.output_tokens) as output,\nsum(\n COALESCE(\n t.metadata->>'cache_read_input', -- Anthropic\n t.metadata->>'prompt_cached' -- OpenAI\n )::int\n) AS cache_read_input,\nsum((t.metadata->>'cache_creation_input')::int) AS cache_creation_input -- Anthropic\nfrom aibridge_token_usages t\njoin aibridge_interceptions i on t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\n AND u.username ~ '$${username:regex}'\n AND i.provider ~ '$${provider:regex}'\n AND i.model ~ '$${model:regex}'\norder by input desc", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Total usage for $username", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": { - "cache_creation_input": 3, - "cache_read_input": 2, - "input": 0, - "output": 1 - }, - "renameByName": { - "cache_creation_input": "Cache Write", - "cache_read_input": "Cache Read", - "input": "Input", - "output": "Output" - } - } - } - ], - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 10, - "panels": [], - "title": "Interceptions", - "type": "row" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "output" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 20, - "x": 0, - "y": 23 - }, - "id": 4, - "maxDataPoints": 30, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "auto", - "stacking": "normal", - "text": { - "valueSize": 10 - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": -45, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "editorMode": "code", - "format": "time_series", - "rawQuery": true, - "rawSql": "SELECT\n$__timeGroupAlias(i.started_at, $__interval, NULL),\ncount(i.id) AS value,\nu.username AS metric\nFROM aibridge_interceptions i\njoin users u ON i.initiator_id = u.id\nWHERE\n$__timeFilter(i.started_at)\nAND u.username ~ '$${username:regex}'\nAND i.provider ~ '$${provider:regex}'\nAND i.model ~ '$${model:regex}'\nGROUP BY u.username, $__timeGroup(i.started_at, $__interval)\nORDER BY $__timeGroup(i.started_at, $__interval)", - "refId": "A", - "sql": { - "columns": [ - { - "name": "COUNT", - "parameters": [ - { - "name": "id", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "name": "started_at", - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "aibridge_interceptions" - } - ], - "title": "Interceptions over time by user", - "type": "barchart" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "output" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 4, - "x": 20, - "y": 23 - }, - "id": 5, - "interval": "1m", - "maxDataPoints": 500, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select count(*) from aibridge_interceptions\nWHERE started_at > $__timeFrom() AND started_at <= $__timeTo()\nAND provider ~ '$${provider:regex}'\nAND model ~ '$${model:regex}'", - "refId": "A", - "sql": { - "columns": [ - { - "name": "COUNT", - "parameters": [ - { - "name": "id", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "name": "started_at", - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "aibridge_interceptions" - } - ], - "title": "Total interceptions", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 9, - "panels": [], - "title": "Prompts & tool calls details", - "type": "row" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto", - "wrapText": false - }, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Interception ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 357 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Model" - }, - "properties": [ - { - "id": "custom.width", - "value": 240 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Provider" - }, - "properties": [ - { - "id": "custom.width", - "value": 157 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Username" - }, - "properties": [ - { - "id": "custom.width", - "value": 188 - } - ] - } - ] - }, - "gridPos": { - "h": 14, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 7, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Created At" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT i.id,\n u.username,\n i.provider,\n i.model,\n p.prompt,\n p.created_at\nFROM aibridge_user_prompts p\nJOIN aibridge_interceptions i ON p.interception_id = i.id\nJOIN users u ON i.initiator_id = u.id\nWHERE $__timeFilter(i.started_at)\n AND u.username ~ '$${username:regex}'\n AND i.provider ~ '$${provider:regex}'\n AND i.model ~ '$${model:regex}'\nORDER BY p.created_at DESC;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "User Prompts", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "created_at": "Created At", - "id": "Interception ID", - "input": "Tool Input", - "invocation_error": "Tool Error", - "model": "Model", - "prompt": "Prompt", - "provider": "Provider", - "server_url": "MCP Server", - "tool": "Tool Name", - "username": "Username" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto", - "wrapText": false - }, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Tool Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 342 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "invocation_error" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "applyToRow": true, - "type": "color-background", - "wrapText": false - } - }, - { - "id": "noValue" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "match": "null", - "result": { - "color": "green", - "index": 0 - } - }, - "type": "special" - }, - { - "options": { - "pattern": ".+", - "result": { - "color": "red", - "index": 1 - } - }, - "type": "regex" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Tool Input" - }, - "properties": [ - { - "id": "custom.width", - "value": 309 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Interception ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 357 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Model" - }, - "properties": [ - { - "id": "custom.width", - "value": 240 - } - ] - } - ] - }, - "gridPos": { - "h": 14, - "w": 24, - "x": 0, - "y": 50 - }, - "id": 6, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": ["sum"], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Created At" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select i.id, u.username, i.provider, i.model, t.server_url, t.tool, t.input, t.invocation_error, t.created_at FROM aibridge_tool_usages t\njoin aibridge_interceptions i ON t.interception_id = i.id\njoin users u on i.initiator_id = u.id\nwhere $__timeFilter(i.started_at)\nAND u.username ~ '$${username:regex}'\nAND i.provider ~ '$${provider:regex}'\nAND i.model ~ '$${model:regex}'\norder by t.created_at desc", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Tool Calls", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "created_at": "Created At", - "id": "Interception ID", - "input": "Tool Input", - "invocation_error": "Tool Error", - "model": "Model", - "provider": "Provider", - "server_url": "MCP Server", - "tool": "Tool Name", - "username": "Username" - } - } - } - ], - "type": "table" - } - ], - "refresh": "1m", - "schemaVersion": 41, - "tags": [], - "templating": { - "list": [ - { - "allValue": ".+", - "current": {}, - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "definition": "select username from users where deleted=false;", - "description": "", - "includeAll": true, - "multi": true, - "name": "username", - "options": [], - "query": "select username from users where deleted=false;", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": {}, - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "definition": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", - "description": "", - "includeAll": true, - "multi": true, - "name": "provider", - "options": [], - "query": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": {}, - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" - }, - "definition": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '$${provider:regex}' ORDER BY 1;", - "description": "", - "includeAll": true, - "multi": true, - "name": "model", - "options": [], - "query": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '$${provider:regex}' ORDER BY 1;", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-7d", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Coder AI Bridge", - "uid": "0c61d33f-c809-4184-9e88-cb27e2d9d224", - "version": 43, - "weekStart": "" -} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/coderd.json b/infra/1-click/1-setup/dashboards/coderd.json deleted file mode 100644 index b64f5ca..0000000 --- a/infra/1-click/1-setup/dashboards/coderd.json +++ /dev/null @@ -1,1472 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Down" - }, - "properties": [ - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 0 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": false - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count(up{ ${CODERD_SELECTOR} } == 1) or vector(0)", - "instant": true, - "legendFormat": "Up", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(count(up{ ${CODERD_SELECTOR} } == 0) or vector(0)) > 0", - "hide": false, - "instant": true, - "legendFormat": "Down", - "range": false, - "refId": "B" - } - ], - "title": "Replicas", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 18, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "One or more replicas are required to be running in order to serve the control-plane.\n\nSee [High Availability](https://coder.com/docs/v2/latest/admin/high-availability) for details on how to\nrun multiple `coderd` replicas.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 0.9 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Enabled" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 1, - "text": "No" - }, - "1": { - "index": 0, - "text": "Yes" - } - }, - "type": "value" - }, - { - "options": { - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - } - ] - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 0 - }, - "id": 32, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": false - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(coderd_license_user_limit_enabled)", - "instant": true, - "legendFormat": "Enabled", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(\n max(coderd_license_active_users) / max(coderd_license_limit_users)\n) > 0", - "hide": false, - "instant": false, - "legendFormat": "Usage", - "range": true, - "refId": "B" - } - ], - "title": "Enterprise License", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 33, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "If you would like to try Coder's [Enterprise features](https://coder.com/docs/v2/latest/enterprise), you can [request a trial license](https://coder.com/docs/v2/latest/faqs#how-do-i-add-an-enterprise-license).", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/(Requested|Limit)/" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 0, - 10 - ], - "fill": "dot" - } - }, - { - "id": "custom.fillOpacity", - "value": 5 - }, - { - "id": "custom.drawStyle", - "value": "line" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Requested" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 6 - }, - "id": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{ ${CODERD_SELECTOR} }[$__rate_interval]))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(kube_pod_container_resource_limits{ ${CODERD_SELECTOR}, resource=\"cpu\"})", - "hide": false, - "instant": false, - "legendFormat": "Limit", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(kube_pod_container_resource_requests{ ${CODERD_SELECTOR}, resource=\"cpu\"})", - "hide": false, - "instant": false, - "legendFormat": "Requested", - "range": true, - "refId": "B" - } - ], - "title": "CPU Usage Seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 6 - }, - "id": 26, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "The cumulative CPU used per core-second. If `coderd` was using a full CPU core, that would be represented as 1 second.\n\nRequests & limits are shown if set.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Requested" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 6 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (reason) (\n count_over_time(kube_pod_container_status_terminated_reason{ ${CODERD_SELECTOR} }[$__interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "{{reason}}", - "range": true, - "refId": "C" - } - ], - "title": "Terminations", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0001 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 16, - "y": 6 - }, - "id": 34, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(kube_pod_container_status_restarts_total{ ${CODERD_SELECTOR} }[$__range]))", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "B" - } - ], - "title": "Restarts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 6 - }, - "id": 31, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Pods can be terminated for several reasons:\n- `OOMKilled`: pod exceeded its defined memory limit or was terminated by the OS for using excessive memory (if no limit defined)\n- `Error`: usually attributeable to a configuration problem\n- `Evicted`: pod has been evicted from node for overusing resources and will be rescheduled on another node is possible", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/(Requested|Limit)/" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 0, - 10 - ], - "fill": "dot" - } - }, - { - "id": "custom.fillOpacity", - "value": 5 - }, - { - "id": "custom.drawStyle", - "value": "line" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Requested" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 12 - }, - "id": 29, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max by (pod) (container_memory_working_set_bytes{ ${CODERD_SELECTOR} })", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(kube_pod_container_resource_limits{ ${CODERD_SELECTOR}, resource=\"memory\"})", - "hide": false, - "instant": false, - "legendFormat": "Limit", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(kube_pod_container_resource_requests{ ${CODERD_SELECTOR}, resource=\"memory\"})", - "hide": false, - "instant": false, - "legendFormat": "Requested", - "range": true, - "refId": "B" - } - ], - "title": "RAM Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 12 - }, - "id": 28, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "This shows the total memory used by each `coderd` container; it is the same metric which the [OOM killer](https://www.kernel.org/doc/gorman/html/understand/understand016.html) uses.\n\nRequests & limits are shown if set.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 100 - }, - { - "color": "red", - "value": 500 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Errors" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 12 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "quantile(0.5, coder_pubsub_send_latency_seconds)", - "instant": false, - "legendFormat": "Send", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "quantile(0.5, coder_pubsub_receive_latency_seconds)", - "hide": false, - "instant": false, - "legendFormat": "Receive", - "range": true, - "refId": "B" - } - ], - "title": "Pubsub Latency (Median)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Errors" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 2, - "x": 16, - "y": 12 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "(\n sum(increase(coder_pubsub_latency_measure_errs_total[$__range]))\n / count(coder_pubsub_latency_measure_errs_total)\n) or vector(0)", - "hide": false, - "instant": false, - "legendFormat": "Errors", - "range": true, - "refId": "B" - } - ], - "title": "Pubsub Errors", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 12 - }, - "id": 19, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "`coderd` uses Postgres for passing messages between subcomponents for coordination and signalling;\nthis is called \"pubsub\" (or publish-subscribe).\n\nWe measure the time for messages to be sent and received. Latencies higher than 500ms will likely lead to\nyour Coder deployment feeling sluggish. High latency is usually an indication that your Postgres server is under-resourced on CPU.\n\nHigh values for median should be concerning,\nwhile the 90th percentile shows the outliers.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 100 - }, - { - "color": "red", - "value": 500 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Errors" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 15 - }, - "id": 21, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "quantile(0.9, coder_pubsub_send_latency_seconds)", - "instant": false, - "legendFormat": "Send", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "quantile(0.9, coder_pubsub_receive_latency_seconds)", - "hide": false, - "instant": false, - "legendFormat": "Receive", - "range": true, - "refId": "B" - } - ], - "title": "Pubsub Latency (P90)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 18 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by(pod) (rate(coderd_api_requests_processed_total{ ${CODERD_SELECTOR} }[$__rate_interval]))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "API Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 18 - }, - "id": 36, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "This shows the number of requests per second each `coderd` replica is handling.\n\nHeavy skewing towards a single `coderd` replica indicates faulty loadbalancing.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - } - ], - "refresh": "${DASHBOARD_REFRESH}", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-${DASHBOARD_TIMERANGE}", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Coder Control Plane", - "uid": "coderd", - "version": 6, - "weekStart": "" -} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/prebuilds.json b/infra/1-click/1-setup/dashboards/prebuilds.json deleted file mode 100644 index 4c06a15..0000000 --- a/infra/1-click/1-setup/dashboards/prebuilds.json +++ /dev/null @@ -1,1450 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 132, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "text", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 49, - "interval": "30s", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "vertical", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(max(coderd_prebuilt_workspaces_desired) by (template_name, preset_name)) or vector(0)", - "instant": true, - "interval": "", - "legendFormat": "Desired", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(max(coderd_prebuilt_workspaces_running) by (template_name, preset_name)) or vector(0)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Running", - "range": false, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(max(coderd_prebuilt_workspaces_eligible) by (template_name, preset_name)) or vector(0)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Eligible", - "range": false, - "refId": "E" - } - ], - "title": "Current: Global", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "text", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 48, - "interval": "30s", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "vertical", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_created_total)) or vector(0)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Created", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_failed_total)) or vector(0)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Failed", - "range": false, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(max by (template_name, preset_name) (coderd_prebuilt_workspaces_claimed_total)) or vector(0)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Claimed", - "range": false, - "refId": "A" - } - ], - "title": "All Time: Global", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 2, - "panels": [], - "repeat": "preset", - "title": "$preset", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "text", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 5 - }, - "id": 1, - "interval": "30s", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "vertical", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(coderd_prebuilt_workspaces_created_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Created", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(coderd_prebuilt_workspaces_failed_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Failed", - "range": false, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(coderd_prebuilt_workspaces_claimed_total{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Claimed", - "range": false, - "refId": "A" - } - ], - "title": "All Time: $preset", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 10, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 13, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Failed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Created" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Desired" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Running" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Eligible" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Claimed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 6, - "y": 5 - }, - "id": 38, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "floor(max(increase(coderd_prebuilt_workspaces_created_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Created", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "floor(max(increase(coderd_prebuilt_workspaces_failed_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Failed", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "floor(max(increase(coderd_prebuilt_workspaces_claimed_total{template_name=~\"$template\", preset_name=~\"$preset\"}[$__rate_interval]))) or vector(0)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Claimed", - "range": true, - "refId": "F" - } - ], - "title": "Pool Operations: $preset", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 10, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 18, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Desired" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - }, - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 10, - 10 - ], - "fill": "dash" - } - }, - { - "id": "custom.fillOpacity", - "value": 85 - }, - { - "id": "custom.fillBelowTo", - "value": "Running" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Running" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - }, - { - "id": "custom.fillBelowTo", - "value": "Eligible" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Eligible" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 15, - "y": 5 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(coderd_prebuilt_workspaces_desired{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", - "instant": false, - "interval": "", - "legendFormat": "Desired", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(coderd_prebuilt_workspaces_running{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Running", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(coderd_prebuilt_workspaces_eligible{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Eligible", - "range": true, - "refId": "E" - } - ], - "title": "Pool Capacity: $preset", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "text", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 31, - "interval": "30s", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "vertical", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(coderd_prebuilt_workspaces_desired{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", - "instant": true, - "interval": "", - "legendFormat": "Desired", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(coderd_prebuilt_workspaces_running{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Running", - "range": false, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(coderd_prebuilt_workspaces_eligible{template_name=~\"$template\", preset_name=~\"$preset\"}) or vector(0)", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Eligible", - "range": false, - "refId": "E" - } - ], - "title": "Current: $preset", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Compares the total number of regular workspace creations to prebuilt workspace claims to date.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 11 - }, - "id": 51, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "sum(max by (template_name, preset_name) (\n coderd_workspace_creation_total{\n template_name=~\"$template\", preset_name=~\"$preset\"\n }\n)) or vector(0)", - "instant": false, - "legendFormat": "Regular workspaces created", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(max by (template_name, preset_name) (\n coderd_prebuilt_workspaces_claimed_total{\n template_name=~\"$template\", preset_name=~\"$preset\"\n }\n)) or vector(0)\n", - "hide": false, - "instant": false, - "legendFormat": "Prebuilt workspaces claimed", - "range": true, - "refId": "B" - } - ], - "title": "All Time: Regular vs Prebuilt", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Median (p50) build time in seconds for Regular Workspace Creation, Prebuilt Workspace Creation, and Prebuilt Workspace Claim", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineStyle": { - "dash": [ - 10, - 10 - ], - "fill": "dash" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Regular Creation" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Prebuild Creation" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Prebuild Claim" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 6, - "y": 11 - }, - "id": 50, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "editorMode": "code", - "expr": "histogram_quantile(0.5,\n sum(\n coderd_workspace_creation_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\", type=\"regular\"\n }\n )\n)\nor vector(0)", - "legendFormat": "Regular Creation", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5,\n sum(\n coderd_workspace_creation_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\", type=\"prebuild\"\n }\n )\n)\nor vector(0)", - "hide": false, - "instant": false, - "legendFormat": "Prebuild Creation", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5,\n sum(\n coderd_prebuilt_workspace_claim_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\"\n }\n )\n)\nor vector(0)", - "hide": false, - "instant": false, - "legendFormat": "Prebuild Claim", - "range": true, - "refId": "C" - } - ], - "title": "Workspace Build Latency p50", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "95th-percentile (p95) build time in seconds for Regular Workspace Creation, Prebuilt Workspace Creation, and Prebuilt Workspace Claim.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineStyle": { - "dash": [ - 10, - 10 - ], - "fill": "dash" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Regular Creation" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Prebuild Creation" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Prebuild Claim" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 15, - "y": 11 - }, - "id": 53, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "editorMode": "code", - "expr": "histogram_quantile(0.95,\n sum(\n coderd_workspace_creation_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\", type=\"regular\"\n }\n )\n)\nor vector(0)", - "legendFormat": "Regular Creation", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95,\n sum(\n coderd_workspace_creation_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\", type=\"prebuild\"\n }\n )\n)\nor vector(0)", - "hide": false, - "instant": false, - "legendFormat": "Prebuild Creation", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95,\n sum(\n coderd_prebuilt_workspace_claim_duration_seconds{\n template_name=~\"$template\", preset_name=~\"$preset\"\n }\n )\n)\nor vector(0)", - "hide": false, - "instant": false, - "legendFormat": "Prebuild Claim", - "range": true, - "refId": "C" - } - ], - "title": "Workspace Build Latency p95", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "#EAB839", - "value": 50 - }, - { - "color": "green", - "value": 75 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 14 - }, - "id": 54, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "editorMode": "code", - "expr": "clamp_max(\n 100 *\n (\n sum(\n coderd_prebuilt_workspaces_claimed_total{\n template_name=\"$template\", preset_name=\"$preset\"\n }\n ) or vector(0)\n )\n /\n clamp_min(\n ( \n sum(\n coderd_prebuilt_workspaces_claimed_total{\n template_name=\"$template\", preset_name=\"$preset\"\n }\n ) or vector(0))\n +\n (\n sum(\n coderd_workspace_creation_total{\n template_name=\"$template\", preset_name=\"$preset\"\n }\n ) or vector(0)),\n 1\n ),\n 100\n)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "All Time: Prebuilds Usage %", - "type": "stat" - } - ], - "preload": false, - "refresh": "${DASHBOARD_REFRESH}", - "schemaVersion": 41, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "coder", - "value": "coder" - }, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "definition": "label_values(coderd_prebuilt_workspaces_desired,template_name)", - "includeAll": false, - "label": "Template", - "name": "template", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(coderd_prebuilt_workspaces_desired,template_name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "definition": "label_values(coderd_prebuilt_workspaces_desired{template_name=~\"$template\"},preset_name)", - "includeAll": true, - "label": "Preset", - "multi": true, - "name": "preset", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(coderd_prebuilt_workspaces_desired{template_name=~\"$template\"},preset_name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-${DASHBOARD_TIMERANGE}", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Coder Prebuilds", - "uid": "cej6jysyme22oa", - "version": 5 -} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/provisionerd.json b/infra/1-click/1-setup/dashboards/provisionerd.json deleted file mode 100644 index f41f277..0000000 --- a/infra/1-click/1-setup/dashboards/provisionerd.json +++ /dev/null @@ -1,1019 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 0 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": false - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(coderd_provisionerd_num_daemons{pod=~`coder.*`, pod!~`.*provisioner.*`})", - "instant": true, - "legendFormat": "Built-in", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(coderd_provisionerd_num_daemons{ ${PROVISIONERD_SELECTOR} })", - "hide": false, - "instant": true, - "legendFormat": "External", - "range": false, - "refId": "B" - } - ], - "title": "Provisioners", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 20, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Provisioners are responsible for building workspaces.\n\n`coderd` runs built-in provisioners by default. Control this with the `CODER_PROVISIONER_DAEMONS` environment variable or `--provisioner-daemons` flag.\n\nYou can also consider [External Provisioners](https://coder.com/docs/v2/latest/admin/provisioners). Running both built-in and external provisioners is perfectly valid,\nalthough dedicated (external) provisioners will generally give the best build performance.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 0 - }, - "id": 21, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(sum(coderd_provisionerd_jobs_current) > 0) or vector(0)", - "instant": false, - "legendFormat": "Current", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(coderd_provisionerd_num_daemons)", - "hide": false, - "instant": true, - "legendFormat": "Capacity", - "range": false, - "refId": "B" - } - ], - "title": "Builds", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 22, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "The maximum number of simultaneous builds is equivalent to the number of `provisionerd` daemons running.\n\nThe \"Capacity\" panel shows the how many simultaneous builds are possible.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 7 - }, - "id": 23, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.5, sum by(le) (rate(coderd_provisionerd_job_timings_seconds_bucket[$__range])))", - "hide": false, - "instant": true, - "legendFormat": "Median", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.9, sum by(le) (rate(coderd_provisionerd_job_timings_seconds_bucket[$__range])))", - "hide": false, - "instant": true, - "legendFormat": "90th Percentile", - "range": false, - "refId": "A" - } - ], - "title": "Build Times", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 7 - }, - "id": 24, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "This shows the median and 90th percentile workspace build times.\n\nLong build times can impede developers' productivity while they wait for workspaces to start or be created.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "failed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - }, - { - "id": "displayName", - "value": "Failure" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "success" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - }, - { - "id": "displayName", - "value": "Success" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 7 - }, - "id": 25, - "interval": "1h", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (status) (increase(coderd_provisionerd_job_timings_seconds_count[$__interval]))", - "hide": false, - "instant": false, - "interval": "1h", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Build Count Per Hour", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 7 - }, - "id": 26, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "_NOTE: this will not show the current hour._", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/(Limit|Requested)/" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.fillOpacity", - "value": 5 - }, - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 0, - 10 - ], - "fill": "dot" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Requested" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 14 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{ ${PROVISIONERD_SELECTOR} }[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(kube_pod_container_resource_limits{ ${PROVISIONERD_SELECTOR} , resource=\"cpu\"})", - "hide": false, - "instant": false, - "legendFormat": "Limit", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(kube_pod_container_resource_requests{ ${PROVISIONERD_SELECTOR} , resource=\"cpu\"})", - "hide": false, - "instant": false, - "legendFormat": "Requested", - "range": true, - "refId": "C" - } - ], - "title": "CPU Usage Seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 14 - }, - "id": 30, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "The cumulative CPU used per core-second. If the process was using a full CPU core, that would be represented as 1 second.\n\nRequests & limits are shown if set.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/(Limit|Requested)/" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.fillOpacity", - "value": 5 - }, - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 0, - 10 - ], - "fill": "dot" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Requested" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 14 - }, - "id": 29, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max by (pod) (container_memory_working_set_bytes{ ${PROVISIONERD_SELECTOR} })", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(kube_pod_container_resource_limits{ ${PROVISIONERD_SELECTOR} , resource=\"memory\"})", - "hide": false, - "instant": false, - "legendFormat": "Limit", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(kube_pod_container_resource_requests{ ${PROVISIONERD_SELECTOR} , resource=\"memory\"})", - "hide": false, - "instant": false, - "legendFormat": "Requested", - "range": true, - "refId": "C" - } - ], - "title": "RAM Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 14 - }, - "id": 31, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "This shows the total memory used by each container; it is the same metric which the [OOM killer](https://www.kernel.org/doc/gorman/html/understand/understand016.html) uses.\n\nRequests & limits are shown if set.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 18, - "w": 18, - "x": 0, - "y": 21 - }, - "id": 27, - "options": { - "dedupStrategy": "exact", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "code", - "expr": "{ ${NON_WORKSPACES_SELECTOR}, logger=~\"(.*runner|terraform|provisioner.*)\"}", - "queryType": "range", - "refId": "A" - } - ], - "title": "Logs", - "type": "logs" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 21 - }, - "id": 32, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "This panel shows all logs across built-in and [external provisioners](https://coder.com/docs/v2/latest/admin/provisioners).", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - } - ], - "refresh": "${DASHBOARD_REFRESH}", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-${DASHBOARD_TIMERANGE}", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Coder Provisioners", - "uid": "provisionerd", - "version": 10, - "weekStart": "" -} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/status.json b/infra/1-click/1-setup/dashboards/status.json deleted file mode 100644 index 082998f..0000000 --- a/infra/1-click/1-setup/dashboards/status.json +++ /dev/null @@ -1,2038 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": false, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 9, - "title": "Application", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Down" - }, - "properties": [ - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": false - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count(up{ ${CODERD_SELECTOR} } == 1) or vector(0) > 0", - "instant": true, - "legendFormat": "Up", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count(up{ ${CODERD_SELECTOR} } == 0) or vector(0) > 0", - "hide": false, - "instant": true, - "legendFormat": "Down", - "range": false, - "refId": "B" - } - ], - "title": "Coder Replicas", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": false - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(coderd_provisionerd_num_daemons{ ${CODERD_SELECTOR} })", - "instant": true, - "legendFormat": "Built-in", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(coderd_provisionerd_num_daemons{ ${PROVISIONERD_SELECTOR} })", - "hide": false, - "instant": true, - "legendFormat": "External", - "range": false, - "refId": "B" - } - ], - "title": "Provisioners", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "failed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - }, - { - "id": "displayName", - "value": "Failed" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "success" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - }, - { - "id": "displayName", - "value": "Success" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 17, - "options": { - "displayLabels": [ - "name", - "value" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "values": [ - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "round(sum by (status) (increase(coderd_provisionerd_job_timings_seconds_count{pod!=``}[$__range])))", - "instant": true, - "legendFormat": "{{status}}", - "range": false, - "refId": "A" - } - ], - "title": "Workspace Builds", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count(kube_pod_status_ready{condition=\"true\", ${WORKSPACES_SELECTOR}} == 1)\nor\nsum(max by (workspace_owner, template_name, template_version) (coderd_workspace_latest_build_status{status=\"succeeded\", workspace_transition=\"start\"}))\nor\nvector(0)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Running Workspaces", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Down" - }, - "1": { - "color": "green", - "index": 0, - "text": "Up" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "orange", - "index": 3, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null+nan", - "result": { - "index": 4, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*RAM/" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(\n max_over_time(\n rate(container_cpu_usage_seconds_total{ ${CODERD_SELECTOR} }[1h:1m])\n [$__range:]\n )\n)", - "instant": true, - "legendFormat": "Control Plane CPU", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(\n max_over_time(\n rate(container_cpu_usage_seconds_total{ ${PROVISIONERD_SELECTOR} }[1h:1m])\n [$__range:]\n )\n)", - "hide": false, - "instant": true, - "legendFormat": "Provisioner CPU", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(\n max_over_time(\n container_memory_working_set_bytes{ ${CODERD_SELECTOR} }\n [$__range:]\n )\n)", - "hide": false, - "instant": true, - "legendFormat": "Control Plane RAM", - "range": false, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(\n max_over_time(\n container_memory_working_set_bytes{ ${PROVISIONERD_SELECTOR} }\n [$__range:]\n )\n)", - "hide": false, - "instant": true, - "legendFormat": "Provisioner RAM", - "range": false, - "refId": "D" - } - ], - "title": "Resource Usage High Watermark (Cumulative)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Down" - }, - "1": { - "color": "green", - "index": 0, - "text": "Up" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "orange", - "index": 3, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null+nan", - "result": { - "index": 4, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "min(pg_up) or vector(0)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Postgres", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 3, - "panels": [], - "title": "Observability Tools", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Down" - }, - "1": { - "color": "green", - "index": 0, - "text": "Up" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "orange", - "index": 3, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null+nan", - "result": { - "index": 4, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 9 - }, - "id": 1, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "min(up{job=\"${PROMETHEUS_JOB}\"}) or vector(0)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Prometheus", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Down" - }, - "1": { - "color": "green", - "index": 0, - "text": "Up" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "orange", - "index": 3, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null+nan", - "result": { - "index": 4, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 9 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "min(up{job=\"${LOKI_JOB}/write\"}) or vector(0)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Loki Write Path", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Down" - }, - "1": { - "color": "green", - "index": 0, - "text": "Up" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "orange", - "index": 3, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null+nan", - "result": { - "index": 4, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 9 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "min(up{job=\"${LOKI_JOB}/read\"}) or vector(0)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Loki Read Path", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Down" - }, - "1": { - "color": "green", - "index": 0, - "text": "Up" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "orange", - "index": 3, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null+nan", - "result": { - "index": 4, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 9 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "min(up{job=\"${LOKI_JOB}/backend\", container=\"loki\"}) or vector(0)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Loki Backend", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Down" - }, - "1": { - "color": "green", - "index": 0, - "text": "Up" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "orange", - "index": 3, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null+nan", - "result": { - "index": 4, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 9 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "min(up{job=\"${LOKI_JOB}/canary\"}) or vector(0)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Loki Canary", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 9 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(up{job=\"${GRAFANA_AGENT_JOB}\"}) or vector(0)", - "instant": true, - "legendFormat": "Current", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count(up{job=\"${GRAFANA_AGENT_JOB}\"})", - "instant": true, - "legendFormat": "Capacity", - "range": false, - "refId": "B" - } - ], - "title": "Grafana Agent", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Unhealthy" - }, - "1": { - "color": "green", - "index": 0, - "text": "Healthy" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "orange", - "index": 3, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null+nan", - "result": { - "index": 4, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 14 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "prometheus_config_last_reload_successful{job=\"${PROMETHEUS_JOB}\"}", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Prometheus Config", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Unhealthy" - }, - "1": { - "color": "green", - "index": 0, - "text": "Healthy" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "orange", - "index": 3, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null+nan", - "result": { - "index": 4, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 14 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "min(loki_runtime_config_last_reload_successful) or vector(0)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Loki Config", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Unhealthy" - }, - "1": { - "color": "green", - "index": 0, - "text": "Healthy" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "orange", - "index": 3, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null+nan", - "result": { - "index": 4, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 14 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "min(agent_config_last_load_successful{job=\"${GRAFANA_AGENT_JOB}\"}) or vector(0)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Grafana Agent Config", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Retention Limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Write-Ahead Log" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Storage" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#f9f9fb", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 14 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "(\n prometheus_tsdb_wal_storage_size_bytes{job=\"${PROMETHEUS_JOB}\"} +\n prometheus_tsdb_storage_blocks_bytes{job=\"${PROMETHEUS_JOB}\"} +\n prometheus_tsdb_symbol_table_size_bytes{job=\"${PROMETHEUS_JOB}\"}\n)\n/\nprometheus_tsdb_retention_limit_bytes{job=\"${PROMETHEUS_JOB}\"}", - "instant": false, - "legendFormat": "Retention limit used", - "range": true, - "refId": "A" - } - ], - "title": "Prometheus Storage", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 14 - }, - "id": 20, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 20, - "valueSize": 35 - }, - "textMode": "auto", - "wideLayout": false - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{namespace=\"${HELM_NAMESPACE}\", resource=\"cpu\"})", - "hide": false, - "instant": true, - "legendFormat": "Requested", - "range": false, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(\n max_over_time(\n rate(container_cpu_usage_seconds_total{namespace=\"${HELM_NAMESPACE}\"}[$__rate_interval])\n [$__range:]\n )\n)", - "hide": false, - "instant": true, - "legendFormat": "High Watermark", - "range": false, - "refId": "D" - } - ], - "title": "CPU", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 14 - }, - "id": 21, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "center", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 20, - "valueSize": 35 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{namespace=\"${HELM_NAMESPACE}\", resource=\"memory\"})", - "hide": false, - "instant": true, - "legendFormat": "Requested", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(\n max_over_time(container_memory_working_set_bytes{namespace=\"${HELM_NAMESPACE}\"}[$__range])\n)", - "instant": true, - "legendFormat": "High Watermark", - "range": false, - "refId": "A" - } - ], - "title": "RAM", - "type": "stat" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Coder Status", - "uid": "coder-status", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/workspace_detail.json b/infra/1-click/1-setup/dashboards/workspace_detail.json deleted file mode 100644 index a5cc8b3..0000000 --- a/infra/1-click/1-setup/dashboards/workspace_detail.json +++ /dev/null @@ -1,1342 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "", - "gridPos": { - "h": 1.2, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 28, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "**HINT**: use the dropdowns above to filter by specific workspace(s).", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "CPUs Requested" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "decimals", - "value": 2 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Requested" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PVC Capacity" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 20, - "x": 0, - "y": 1.2 - }, - "id": 29, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "center", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 20, - "valueSize": 40 - }, - "textMode": "value_and_name", - "wideLayout": false - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "group by (template_name) (coderd_agents_up{workspace_name=~\"$workspace_name\"})", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Template Name", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "group by (template_version) (coderd_agents_up{workspace_name=~\"$workspace_name\"})", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Template Version", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "group by (username) (coderd_agents_up{workspace_name=~\"$workspace_name\"})", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Owner", - "range": false, - "refId": "C" - } - ], - "title": "Details", - "transformations": [ - { - "id": "concatenate", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value #A": true, - "Value #B": true, - "Value #C": true, - "Value #D": true - }, - "includeByName": {}, - "indexByName": { - "CPUs Requested": 7, - "PVC Capacity": 9, - "RAM Requested": 8, - "Time": 0, - "Value #A": 5, - "Value #B": 3, - "Value #C": 6, - "template_name": 2, - "template_version": 4, - "username": 1 - }, - "renameByName": { - "Value #C": "", - "lifecycle_state": "Agent State", - "template_name": "Template", - "template_version": "Template Version", - "username": "Owner" - } - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 8, - "w": 4, - "x": 20, - "y": 1.2 - }, - "id": 38, - "links": [ - { - "title": "Provisioners Dashboard", - "url": "/d/provisionerd/provisioners?$${__url_time_range}" - } - ], - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Essential information about the selected workspace.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "CPUs Requested" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "decimals", - "value": 2 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Requested" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PVC Capacity" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 20, - "x": 0, - "y": 5.2 - }, - "id": 36, - "options": { - "reduceOptions": { - "values": false, - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/" - }, - "orientation": "vertical", - "textMode": "value_and_name", - "wideLayout": false, - "colorMode": "none", - "graphMode": "none", - "justifyMode": "center", - "showPercentChange": false, - "text": { - "titleSize": 20, - "valueSize": 40 - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{pod=~\".*$workspace_name.*\", ${WORKSPACES_SELECTOR}, resource=\"cpu\"})", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": "CPUs Requested", - "range": false, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{pod=~\".*$workspace_name.*\", ${WORKSPACES_SELECTOR}, resource=\"memory\"})", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": "RAM Requested", - "range": false, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(\n kube_pod_spec_volumes_persistentvolumeclaims_info{pod=~\".*$workspace_name.*\", ${WORKSPACES_SELECTOR} }\n * on(persistentvolumeclaim) group_right\n group by (persistentvolumeclaim, persistentvolume) (\n label_replace(\n kube_persistentvolume_claim_ref,\n \"persistentvolumeclaim\",\n \"$1\",\n \"name\",\n \"(.+)\"\n )\n )\n * on (persistentvolume)\n kube_persistentvolume_capacity_bytes\n)", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": "PVC Capacity", - "range": false, - "refId": "F" - } - ], - "title": "Resources", - "transformations": [ - { - "id": "concatenate", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value #A": true, - "Value #B": true, - "Value #C": true, - "Value #D": true - }, - "includeByName": {}, - "indexByName": { - "CPUs Requested": 7, - "PVC Capacity": 9, - "RAM Requested": 8, - "Time": 0, - "Value #A": 5, - "Value #B": 3, - "Value #C": 6, - "template_name": 2, - "template_version": 4, - "username": 1 - }, - "renameByName": { - "Value #C": "", - "lifecycle_state": "Agent State", - "template_name": "Template", - "template_version": "Template Version", - "username": "Owner" - } - } - } - ], - "type": "stat", - "description": "" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "created": { - "color": "light-blue", - "index": 1, - "text": "Created" - }, - "off": { - "color": "text", - "index": 8, - "text": "Off" - }, - "ready": { - "color": "green", - "index": 0, - "text": "Ready" - }, - "shutdown_error": { - "color": "red", - "index": 7, - "text": "Shutdown Error" - }, - "shutdown_timeout": { - "color": "purple", - "index": 6, - "text": "Shutdown Timeout" - }, - "shutting_down": { - "color": "light-purple", - "index": 5, - "text": "Shutting Down" - }, - "start_error": { - "color": "red", - "index": 4, - "text": "Start Error" - }, - "start_timeout": { - "color": "orange", - "index": 3, - "text": "Start Timeout" - }, - "starting": { - "color": "super-light-green", - "index": 2, - "text": "Starting" - } - }, - "type": "value" - }, - { - "options": { - "match": "empty", - "result": { - "color": "text", - "index": 9, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "null", - "result": { - "color": "text", - "index": 10, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 9.2 - }, - "id": 35, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^lifecycle_state$/", - "values": false - }, - "showPercentChange": false, - "text": { - "valueSize": 50 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max by (lifecycle_state) (coderd_agents_connections{workspace_name=~\"$workspace_name\"})", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "D" - } - ], - "title": "Agent Lifecycle State", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "-1": { - "color": "light-orange", - "index": 0, - "text": "Not completed yet" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 60 - }, - { - "color": "red", - "value": 120 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 4, - "y": 9.2 - }, - "id": 33, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^Value$/", - "values": false - }, - "showPercentChange": false, - "text": { - "valueSize": 50 - }, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(coderd_agentstats_startup_script_seconds{workspace_name=~\"$workspace_name\"}) or vector(-1)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "C" - } - ], - "title": "Agent Startup Script Execution Time", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 7, - "y": 9.2 - }, - "id": 39, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 20, - "valueSize": 50 - }, - "textMode": "value_and_name", - "wideLayout": false - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max by (app) (\n label_replace(\n {workspace_name=~\"$workspace_name\", __name__=~\"coderd_agentstats_session_count_.*\"},\n \"app\",\n \"$1\",\n \"__name__\",\n \"coderd_agentstats_session_count_(.*)\"\n )\n)>0", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": "{{app}}", - "range": false, - "refId": "C" - } - ], - "title": "App Session Counts", - "transformations": [ - { - "id": "concatenate", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "includeByName": {}, - "indexByName": {}, - "renameByName": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Bytes/" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 9.2 - }, - "id": 34, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "center", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 20, - "valueSize": 50 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(coderd_agents_connection_latencies_seconds{workspace_name=~\"$workspace_name\"})", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Connection Latency", - "range": false, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(sum by (pod) (sum_over_time(coderd_agentstats_rx_bytes{workspace_name=~\"$workspace_name\"}[$__range])))", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Received Bytes", - "range": false, - "refId": "rx" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(sum by (pod) (sum_over_time(coderd_agentstats_tx_bytes{workspace_name=~\"$workspace_name\"}[$__range])))", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "Transmitted Bytes", - "range": false, - "refId": "tx" - } - ], - "title": "Networking", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "Value #A": "Received Bytes", - "Value #B": "Transmitted Bytes", - "Value #C": "Connection Latency", - "Value #rx": "Received Bytes", - "Value #tx": "Transmitted Bytes" - } - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 9.2 - }, - "id": 40, - "links": [ - { - "title": "Provisioners Dashboard", - "url": "/d/provisionerd/provisioners?$${__url_time_range}" - } - ], - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Essential information about this workspace's agent.\n\nRead more about the agent [here](https://coder.com/docs/v2/latest/about/architecture#agents).", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "status" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "failed": { - "color": "orange", - "index": 1, - "text": "Failure" - }, - "success": { - "color": "green", - "index": 0, - "text": "Success" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Workspace Transition" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "DESTROY": { - "color": "red", - "index": 0 - }, - "START": { - "color": "blue", - "index": 1 - }, - "STOP": { - "color": "purple", - "index": 2 - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 20, - "x": 0, - "y": 15.2 - }, - "id": 6, - "interval": "", - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": [], - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Time" - } - ] - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (workspace_name, workspace_owner, status, template_name, template_version, workspace_transition) (\n # Since new series are created and are initially set to a value of 1, we cannot use \"increase\" (because an increase from to 1 does not yield 1).\n # So we compare the current series to an interval ago to see if we have any new series and then sum the series we find. \n ((\n coderd_workspace_builds_total{workspace_name=~\"$workspace_name\"} - \n coderd_workspace_builds_total{workspace_name=~\"$workspace_name\"} offset $__interval\n ) >= 0) \n or coderd_workspace_builds_total{workspace_name=~\"$workspace_name\"}\n) > 0", - "format": "table", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Build Log", - "transformations": [ - { - "disabled": true, - "id": "groupBy", - "options": { - "fields": { - "Count": { - "aggregations": [ - "sum" - ], - "operation": "aggregate" - }, - "Status": { - "aggregations": [], - "operation": "groupby" - }, - "Template Name": { - "aggregations": [], - "operation": "groupby" - }, - "Template Version": { - "aggregations": [], - "operation": "groupby" - }, - "Total": { - "aggregations": [ - "sum" - ], - "operation": "aggregate" - }, - "Value": { - "aggregations": [ - "sum" - ], - "operation": "aggregate" - }, - "Workspace Name": { - "aggregations": [], - "operation": "groupby" - }, - "Workspace Ownert": { - "aggregations": [], - "operation": "groupby" - }, - "Workspace Transition": { - "aggregations": [], - "operation": "groupby" - }, - "status": { - "aggregations": [], - "operation": "groupby" - }, - "template_name": { - "aggregations": [], - "operation": "groupby" - }, - "template_version": { - "aggregations": [], - "operation": "groupby" - }, - "workspace_name": { - "aggregations": [], - "operation": "groupby" - }, - "workspace_owner": { - "aggregations": [], - "operation": "groupby" - }, - "workspace_transition": { - "aggregations": [], - "operation": "groupby" - } - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Value" - } - ] - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": false - }, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "Value": "Count", - "Value (sum)": "Total", - "status": "Status", - "template_name": "Template Name", - "template_version": "Template Version", - "workspace_name": "Workspace Name", - "workspace_owner": "Workspace Owner", - "workspace_transition": "Workspace Transition" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 4, - "x": 20, - "y": 15.2 - }, - "id": 37, - "links": [ - { - "title": "Provisioners Dashboard", - "url": "/d/provisionerd/provisioners?$${__url_time_range}" - } - ], - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "This table shows a reverse-chronological log of all workspace builds.\n\nThe \"Count\" field shows the count of events which occurred within a minute, grouped by all columns.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 10, - "w": 20, - "x": 0, - "y": 22.2 - }, - "id": 7, - "options": { - "dedupStrategy": "exact", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "code", - "expr": "{${NON_WORKSPACES_SELECTOR}, logger=~\"(.*runner|terraform|provisioner.*)\"} |~ \"$workspace_name\" | line_format `{{ printf \"[\\033[35m\" }}{{.pod}}{{ printf \"\\033[0m]\\t\" }}{{ __line__ }}`", - "hide": false, - "queryType": "range", - "refId": "A" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "code", - "expr": "{${WORKSPACES_SELECTOR}, pod=~\".*($workspace_name).*\"} | line_format `{{ printf \"[\\033[32m\" }}{{.pod}}{{ printf \"\\033[0m]\\t\" }}{{ __line__ }}`", - "hide": false, - "queryType": "range", - "refId": "B" - } - ], - "title": "Logs", - "type": "logs" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 10, - "w": 4, - "x": 20, - "y": 22.2 - }, - "id": 24, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "The logs to the left come both from provisioners and workspace logs.\n\nProvisioner logs matching the name filter are highlighted in magenta, while\nworkspace logs matching the name filter are highlighted in green.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - } - ], - "refresh": "${DASHBOARD_REFRESH}", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [ - { - "allValue": "", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "definition": "label_values(coderd_agents_up,workspace_name)", - "hide": 0, - "includeAll": false, - "label": "Workspace Name Filter", - "multi": false, - "name": "workspace_name", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(coderd_agents_up,workspace_name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-${DASHBOARD_TIMERANGE}", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Coder Workspace Detail", - "uid": "workspace-detail", - "version": 9, - "weekStart": "" -} \ No newline at end of file diff --git a/infra/1-click/1-setup/dashboards/workspaces.json b/infra/1-click/1-setup/dashboards/workspaces.json deleted file mode 100644 index 62e2cfc..0000000 --- a/infra/1-click/1-setup/dashboards/workspaces.json +++ /dev/null @@ -1,1624 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "", - "gridPos": { - "h": 1.2, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 28, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "**HINT**: use the dropdowns above to filter by specific workspaces and/or templates.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1.2 - }, - "id": 31, - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 10, - "x": 0, - "y": 2.2 - }, - "id": 33, - "options": { - "legend": { - "calcs": [ - "mean", - "stdDev", - "min", - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{ ${WORKSPACES_SELECTOR} }[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 10, - "x": 10, - "y": 2.2 - }, - "id": 37, - "options": { - "legend": { - "calcs": [ - "mean", - "stdDev", - "min", - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (pod) (container_memory_working_set_bytes{ ${WORKSPACES_SELECTOR} })", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "RAM Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 8, - "w": 4, - "x": 20, - "y": 2.2 - }, - "id": 36, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "The cumulative CPU used per core-second. If a workspace was using a full CPU core, that would be represented as 1 second.\n\nSee the Kubernetes [documentation](https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units) for more details.\n\nThe total memory used by each workspace container is represented; it is the same metric which the [OOM killer](https://www.kernel.org/doc/gorman/html/understand/understand016.html) uses.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 10, - "x": 0, - "y": 10.2 - }, - "id": 38, - "options": { - "legend": { - "calcs": [ - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (pod) (\n round(increase(kube_pod_container_status_restarts_total{ ${WORKSPACES_SELECTOR} }[$__interval]))\n) > 0", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "Pod Restarts", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 10, - "x": 10, - "y": 10.2 - }, - "id": 39, - "options": { - "legend": { - "calcs": [ - "sum" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (pod, reason) (\n count_over_time(kube_pod_container_status_terminated_reason{ ${WORKSPACES_SELECTOR} }[$__interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "{{pod}}:{{reason}}", - "range": true, - "refId": "B" - } - ], - "title": "Terminations", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 8, - "w": 4, - "x": 20, - "y": 10.2 - }, - "id": 40, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Pods can be terminated for several reasons:\n- `OOMKilled`: pod exceeded its defined memory limit or was terminated by the OS for using excessive memory (if no limit defined)\n- `Error`: usually attributeable to a configuration problem\n- `Evicted`: pod has been evicted from node for overusing resources and will be rescheduled on another node is possible\n\nPod restarts are not necessarily problematic, but they are worth noting.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18.2 - }, - "id": 30, - "panels": [], - "title": "Builds", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "DESTROY" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "STOP" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "START" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 10, - "x": 0, - "y": 19.2 - }, - "id": 2, - "interval": "5m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (workspace_transition) (\n (\n # Since new series are created and are initially set to a value of 1, we cannot use \"increase\" (because an increase from to 1 does not yield 1).\n # So we compare the current series to an interval ago to see if we have any new series and then sum the series we find. \n (\n coderd_workspace_builds_total{status=\"success\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} - \n coderd_workspace_builds_total{status=\"success\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} offset $__interval\n ) >= 0) \n or coderd_workspace_builds_total{status=\"success\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"}\n) > 0", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "Successful Builds by State", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "DESTROY" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "STOP" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "START" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 10, - "x": 10, - "y": 19.2 - }, - "id": 1, - "interval": "5m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (workspace_transition) (\n (\n # Since new series are created and are initially set to a value of 1, we cannot use \"increase\" (because an increase from to 1 does not yield 1).\n # So we compare the current series to an interval ago to see if we have any new series and then sum the series we find. \n (\n coderd_workspace_builds_total{status=\"failed\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} - \n coderd_workspace_builds_total{status=\"failed\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} offset $__interval\n ) >= 0) \n or coderd_workspace_builds_total{status=\"failed\", workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"}\n) > 0", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Unsuccessful Builds by State", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 8, - "w": 4, - "x": 20, - "y": 19.2 - }, - "id": 34, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Workspaces \"transition\" between `STOP`, `START`, and `DESTROY` states.\n\nWorkspaces transition between states when a \"build\" is initiated, which is an execution of `terraform` against the chosen template.\n\nUse the \"Build Count\" table to identify workspace owners which may be struggling with template builds, in order to proactively reach out to them with assistance.\n\nConsult the [Template documentation](https://coder.com/docs/v2/latest/templates) for more information.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "status" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "failed": { - "color": "orange", - "index": 1, - "text": "Failure" - }, - "success": { - "color": "green", - "index": 0, - "text": "Success" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Workspace Transition" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "DESTROY": { - "color": "red", - "index": 0 - }, - "START": { - "color": "blue", - "index": 1 - }, - "STOP": { - "color": "purple", - "index": 2 - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 20, - "x": 0, - "y": 27.2 - }, - "id": 6, - "interval": "", - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": [], - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Time" - } - ] - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (workspace_name, workspace_owner, status, template_name, template_version, workspace_transition) (\n # Since new series are created and are initially set to a value of 1, we cannot use \"increase\" (because an increase from to 1 does not yield 1).\n # So we compare the current series to an interval ago to see if we have any new series and then sum the series we find. \n ((\n coderd_workspace_builds_total{workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} - \n coderd_workspace_builds_total{workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"} offset $__interval\n ) >= 0) \n or coderd_workspace_builds_total{workspace_name=~\"$workspace_name\", template_name=~\"$template_name\"}\n) > 0", - "format": "table", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Build Log", - "transformations": [ - { - "disabled": true, - "id": "groupBy", - "options": { - "fields": { - "Count": { - "aggregations": [ - "sum" - ], - "operation": "aggregate" - }, - "Status": { - "aggregations": [], - "operation": "groupby" - }, - "Template Name": { - "aggregations": [], - "operation": "groupby" - }, - "Template Version": { - "aggregations": [], - "operation": "groupby" - }, - "Total": { - "aggregations": [ - "sum" - ], - "operation": "aggregate" - }, - "Value": { - "aggregations": [ - "sum" - ], - "operation": "aggregate" - }, - "Workspace Name": { - "aggregations": [], - "operation": "groupby" - }, - "Workspace Ownert": { - "aggregations": [], - "operation": "groupby" - }, - "Workspace Transition": { - "aggregations": [], - "operation": "groupby" - }, - "status": { - "aggregations": [], - "operation": "groupby" - }, - "template_name": { - "aggregations": [], - "operation": "groupby" - }, - "template_version": { - "aggregations": [], - "operation": "groupby" - }, - "workspace_name": { - "aggregations": [], - "operation": "groupby" - }, - "workspace_owner": { - "aggregations": [], - "operation": "groupby" - }, - "workspace_transition": { - "aggregations": [], - "operation": "groupby" - } - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Value" - } - ] - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": false - }, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "Value": "Count", - "Value (sum)": "Total", - "status": "Status", - "template_name": "Template Name", - "template_version": "Template Version", - "workspace_name": "Workspace Name", - "workspace_owner": "Workspace Owner", - "workspace_transition": "Workspace Transition" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 10, - "w": 4, - "x": 20, - "y": 27.2 - }, - "id": 29, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "This table shows a reverse-chronological log of all workspace builds.\n\nThe \"Count\" field shows the count of events which occurred within a minute, grouped by all columns.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 5, - "x": 0, - "y": 37.2 - }, - "id": 8, - "interval": "1h", - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "values": [ - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by (workspace_owner) (coderd_workspace_latest_build_status{template_name=~\"$template_name\"})", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Workspace by User", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 5, - "x": 5, - "y": 37.2 - }, - "id": 9, - "interval": "1h", - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "values": [ - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by (workspace_owner, template_name) (coderd_workspace_latest_build_status{template_name=~\"$template_name\"})", - "instant": true, - "legendFormat": "{{workspace_owner}}:{{template_name}}", - "range": false, - "refId": "A" - } - ], - "title": "Workspace by User/Template", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 5, - "x": 10, - "y": 37.2 - }, - "id": 4, - "interval": "1h", - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "values": [ - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by (template_name) (coderd_workspace_latest_build_status{template_name=~\"$template_name\"})", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Template Usage", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 5, - "x": 15, - "y": 37.2 - }, - "id": 5, - "interval": "1h", - "options": { - "displayLabels": [], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "values": [ - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by (template_name, template_version) (coderd_workspace_latest_build_status{template_name=~\"$template_name\"})", - "instant": true, - "legendFormat": "{{template_name}}:{{template_version}}", - "range": false, - "refId": "A" - } - ], - "title": "Template Version Usage", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 4, - "x": 20, - "y": 37.2 - }, - "id": 24, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "These charts show the distribution of workspaces and templates.\n\nUse these charts to identify which users have outdated templates, and which templates are the most/least popular in your organisation.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 44.2 - }, - "id": 32, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 10, - "w": 20, - "x": 0, - "y": 45.2 - }, - "id": 7, - "options": { - "dedupStrategy": "exact", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "editorMode": "code", - "expr": "{ ${NON_WORKSPACES_SELECTOR}, logger=~\"(.*runner|terraform|provisioner.*)\"} |~ \"$workspace_name\" or \"$template_name\"", - "queryType": "range", - "refId": "A" - } - ], - "title": "Logs", - "type": "logs" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "gridPos": { - "h": 10, - "w": 4, - "x": 20, - "y": 45.2 - }, - "id": 22, - "links": [ - { - "title": "Provisioners Dashboard", - "url": "/d/provisionerd/provisioners?$${__url_time_range}" - } - ], - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "These are the logs produced by the [Provisioners](/d/provisionerd/provisioners?$${__url_time_range}).\n\nUse the dropdowns at the top to filter the logs down to a specific workspace and/or template.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "transparent": true, - "type": "text" - } - ], - "refresh": "${DASHBOARD_REFRESH}", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [ - { - "allValue": "", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "definition": "label_values(coderd_workspace_builds_total,workspace_name)", - "hide": 0, - "includeAll": true, - "label": "Workspace Name Filter", - "multi": true, - "name": "workspace_name", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(coderd_workspace_builds_total,workspace_name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": "", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "definition": "label_values(coderd_workspace_builds_total,template_name)", - "hide": 0, - "includeAll": true, - "label": "Template Name Filter", - "multi": true, - "name": "template_name", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(coderd_workspace_builds_total,template_name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-${DASHBOARD_TIMERANGE}", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Coder Workspaces", - "uid": "workspaces", - "version": 2, - "weekStart": "" -} \ No newline at end of file diff --git a/infra/1-click/1-setup/main.tf b/infra/1-click/1-setup/main.tf index 70df511..3e960e9 100644 --- a/infra/1-click/1-setup/main.tf +++ b/infra/1-click/1-setup/main.tf @@ -1,96 +1,7 @@ -terraform { - required_version = ">= 1.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.46" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - external = { - source = "hashicorp/external" - } - http = { - source = "hashicorp/http" - } - dns = { - source = "hashicorp/dns" - } - cloudflare = { - source = "cloudflare/cloudflare" - } - } -} - ## # Remote State Resources ## -data "aws_vpc" "this" { - tags = { - "Name" = "${var.name}-${local.normalized_domain_name}" - } -} - -data "aws_s3_bucket" "loki" { - bucket = "${var.name}-${local.normalized_domain_name}-grafana" -} - -data "aws_db_instance" "coder" { - db_instance_identifier = "${var.name}-${local.normalized_domain_name}-coder" -} - -data "aws_db_instance" "grafana" { - db_instance_identifier = "${var.name}-${local.normalized_domain_name}-grafana" -} - -data "aws_security_group" "coder" { - name = "${var.name}-${local.normalized_domain_name}-pgsql" - vpc_id = data.aws_vpc.this.id -} - -data "aws_eks_cluster" "coder" { - name = "${var.name}-${local.normalized_domain_name}" -} - -data "aws_eks_cluster_auth" "coder" { - name = "${var.name}-${local.normalized_domain_name}" -} - -data "aws_iam_openid_connect_provider" "coder" { - url = data.aws_eks_cluster.coder.identity[0].oidc[0].issuer -} - -## -# Global Inputs + Providers -## - -variable "region" { - description = "The AWS region of the deployment." - type = string - default = "us-east-2" -} - -variable "name" { - description = "Name for created resources and tag prefix." - type = string - default = "coder" -} - -variable "profile" { - type = string - default = "default" -} - -variable "domain_name" { - type = string -} - provider "aws" { region = var.region profile = var.profile @@ -110,150 +21,47 @@ provider "kubernetes" { token = data.aws_eks_cluster_auth.coder.token } -variable "auto_set_record" { - description = "Set if you don't want to use external-dns, but still want to automatically set the domain name." - type = object({ - use_cf = bool - cf_token = optional(string, "") - use_r53 = bool - }) - default = { - use_cf = false - cf_token = "" - use_r53 = true - } - sensitive = true - - validation { - condition = !(var.auto_set_record.use_cf && var.auto_set_record.use_r53) - error_message = "'use_cf' and 'use_r53' cannot both be true." - } - - validation { - condition = !(var.auto_set_record.use_cf && var.auto_set_record.cf_token == "") - error_message = "'cf_token' cannot be unset when 'use_cf' is true." - } -} - -provider "cloudflare" { - api_token = var.auto_set_record.cf_token -} - data "aws_region" "this" {} locals { + apex_domain = join(".", slice(split(".", var.domain_name), length(split(".", var.domain_name)) - 2, length(split(".", var.domain_name)))) normalized_domain_name = split(".", var.domain_name)[0] - apex_domain = join(".", slice(split(".", var.domain_name), length(split(".", var.domain_name))-2, length(split(".", var.domain_name)))) + formatted_name = "${var.name}-${local.normalized_domain_name}" } -## -# Fetch DNS Zone -## - -data "aws_route53_zone" "coder" { - count = nonsensitive(var.auto_set_record.use_r53) ? 1 : 0 - name = local.apex_domain -} - -data "cloudflare_zone" "coder" { - count = nonsensitive(var.auto_set_record.use_cf) ? 1 : 0 - filter = { - name = local.apex_domain +data "aws_vpc" "this" { + tags = { + "Name" = local.formatted_name } } -## -# Coder DNS Record Setup -## - -resource "aws_route53_record" "coder-primary" { - count = nonsensitive(var.auto_set_record.use_r53) ? 1 : 0 - zone_id = data.aws_route53_zone.coder[0].zone_id - name = var.domain_name - type = "A" - ttl = "30" - records = aws_eip.coder.*.public_ip -} - -resource "aws_route53_record" "coder-wildcard" { - count = nonsensitive(var.auto_set_record.use_r53) ? 1 : 0 - zone_id = data.aws_route53_zone.coder[0].zone_id - name = "*.${var.domain_name}" - type = "A" - ttl = "60" - records = aws_eip.coder.*.public_ip +data "aws_s3_bucket" "loki" { + bucket = "${local.formatted_name}-grafana" } -resource "cloudflare_dns_record" "coder-primary" { - - for_each = nonsensitive(var.auto_set_record.use_cf) ? toset(aws_eip.coder.*.public_ip) : toset([]) - - zone_id = data.cloudflare_zone.coder[0].id - name = var.domain_name - ttl = 1 - type = "A" - comment = "" - content = each.value - proxied = true +data "aws_db_instance" "coder" { + db_instance_identifier = "${local.formatted_name}-coder" } -resource "cloudflare_dns_record" "coder-wildcard" { - - for_each = nonsensitive(var.auto_set_record.use_cf) ? toset(aws_eip.coder.*.public_ip) : toset([]) - - zone_id = data.cloudflare_zone.coder[0].id - name = "*.${var.domain_name}" - ttl = 1 - type = "A" - comment = "" - content = each.value - proxied = true +data "aws_db_instance" "grafana" { + db_instance_identifier = "${local.formatted_name}-grafana" } -## -# Grafana DNS Record Setup -## - -resource "aws_route53_record" "grafana-primary" { - count = nonsensitive(var.auto_set_record.use_r53) ? 1 : 0 - zone_id = data.aws_route53_zone.coder[0].zone_id - name = "grafana.${var.domain_name}" - type = "A" - ttl = "30" - records = aws_eip.grafana.*.public_ip +data "aws_security_group" "coder" { + name = "${local.formatted_name}-pgsql" + vpc_id = data.aws_vpc.this.id } -resource "aws_route53_record" "grafana-wildcard" { - count = nonsensitive(var.auto_set_record.use_r53) ? 1 : 0 - zone_id = data.aws_route53_zone.coder[0].zone_id - name = "*.grafana.${var.domain_name}" - type = "A" - ttl = "60" - records = aws_eip.grafana.*.public_ip +data "aws_eks_cluster" "coder" { + name = local.formatted_name + region = var.region } -resource "cloudflare_dns_record" "grafana-primary" { - - for_each = nonsensitive(var.auto_set_record.use_cf) ? toset(aws_eip.grafana.*.public_ip) : toset([]) - - zone_id = data.cloudflare_zone.coder[0].id - name = "grafana.${var.domain_name}" - ttl = 1 - type = "A" - comment = "" - content = each.value - proxied = true +data "aws_eks_cluster_auth" "coder" { + name = local.formatted_name + region = var.region } -resource "cloudflare_dns_record" "grafana-wildcard" { - - for_each = nonsensitive(var.auto_set_record.use_cf) ? toset(aws_eip.grafana.*.public_ip) : toset([]) - - zone_id = data.cloudflare_zone.coder[0].id - name = "*.grafana.${var.domain_name}" - ttl = 1 - type = "A" - comment = "" - content = each.value - proxied = true -} +data "aws_iam_openid_connect_provider" "coder" { + url = data.aws_eks_cluster.coder.identity[0].oidc[0].issuer +} \ No newline at end of file diff --git a/infra/1-click/1-setup/scripts/add-license.sh b/infra/1-click/1-setup/scripts/add-license.sh deleted file mode 100644 index 9a74482..0000000 --- a/infra/1-click/1-setup/scripts/add-license.sh +++ /dev/null @@ -1,25 +0,0 @@ - -#!/usr/bin/env bash - -eval "$(jq -r '@sh "CODER_IP_ADDR=\(.ip_addr) CODER_DOMAIN=\(.domain) CODER_LICENSE=\(.license_key) CODER_SESSION_TOKEN=\(.session_token)"')" - -RESOLVE_ARG="--resolve $CODER_DOMAIN:443:$CODER_IP_ADDR" -CODER_URL=https://$CODER_DOMAIN - -RESPONSE=$(curl -ks $RESOLVE_ARG -X POST "$CODER_URL/api/v2/licenses" \ - -H 'Content-Type: application/json' \ - -H 'Accept: application/json' \ - -H "Coder-Session-Token: $CODER_SESSION_TOKEN" \ - -d "{\"license\": \"$CODER_LICENSE\"}") - -if [ $? -ne 0 ]; then - >&2 echo "Error: Unable to run 'curl -ks -X POST \"$CODER_URL/api/v2/licenses\"'. Can't add license." - jq -n '{"success":"false"}' - exit 1; -fi - -jq -n '{"success":"true"}' - -exit 0; - -echo "Hello!" \ No newline at end of file diff --git a/infra/1-click/1-setup/scripts/nodeconfig.yaml b/infra/1-click/1-setup/scripts/nodeconfig.yaml deleted file mode 100644 index a5bdf87..0000000 --- a/infra/1-click/1-setup/scripts/nodeconfig.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: node.eks.aws/v1alpha1 -kind: NodeConfig -spec: - kubelet: - config: - registryPullQPS: 30 - # imageServiceEndpoint: unix:///run/soci-snapshotter-grpc/soci-snapshotter-grpc.sock - # containerd: - # config: | - # [proxy_plugins.soci] - # type = "snapshot" - # address = "/run/soci-snapshotter-grpc/soci-snapshotter-grpc.sock" - # [proxy_plugins.soci.exports] - # root = "/var/lib/soci-snapshotter-grpc" - # [plugins."io.containerd.grpc.v1.cri".containerd] - # snapshotter = "soci" - # # This line is required for containerd to send information about how to lazily load the image to the snapshotter - # disable_snapshot_annotations = false \ No newline at end of file diff --git a/infra/1-click/1-setup/terragrunt.hcl b/infra/1-click/1-setup/terragrunt.hcl new file mode 100644 index 0000000..e633e43 --- /dev/null +++ b/infra/1-click/1-setup/terragrunt.hcl @@ -0,0 +1,23 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = ["../0-infra"] +} + +inputs = { + # Optional. Set if using Terragrunt. + profile = include.root.locals.CODER_AWS_PROFILE + region = include.root.locals.CODER_AWS_REGION + domain_name = include.root.locals.CODER_DOMAIN_NAME + azs = jsondecode(include.root.locals.CODER_AWS_AZS) + coder_version = include.root.locals.CODER_VERSION + coder_username = include.root.locals.CODER_DB_USERNAME + coder_password = include.root.locals.CODER_DB_PASSWORD + coder_license = include.root.locals.CODER_LICENSE + coder_admin_email = include.root.locals.CODER_EMAIL + coder_admin_username = include.root.locals.CODER_USERNAME + coder_admin_password = include.root.locals.CODER_PASSWORD +} \ No newline at end of file diff --git a/infra/1-click/1-setup/variables.tf b/infra/1-click/1-setup/variables.tf new file mode 100644 index 0000000..0cac0e0 --- /dev/null +++ b/infra/1-click/1-setup/variables.tf @@ -0,0 +1,98 @@ + +## +# Global Inputs + Providers +## + +variable "region" { + description = "The AWS region of the deployment." + type = string + default = "us-east-2" +} + +variable "name" { + description = "Name for created resources and tag prefix." + type = string + default = "coder" +} + +variable "profile" { + type = string + default = "default" +} + +variable "domain_name" { + type = string +} + +## +# Coder K8s Inputs +## + +variable "coder_username" { + description = "Coder DB's username." + type = string + default = "coder" +} + +variable "coder_password" { + description = "Coder DB's password." + type = string + default = "th1s1sn0tas3cur3pass0wrd" + sensitive = true +} + +variable "coder_version" { + type = string + default = "2.30.0" +} + +variable "coder_license" { + type = string + default = "" + sensitive = true +} + +variable "coder_admin_email" { + type = string + default = "admin@coder.com" +} + +variable "coder_admin_username" { + type = string + default = "admin" +} + +variable "coder_admin_password" { + type = string + default = "Th1s1sN0TS3CuR3!!" + sensitive = true +} + +variable "grafana_username" { + description = "Grafana DB's username." + type = string + default = "grafana" +} + +variable "grafana_password" { + description = "Grafana DB's password." + type = string + default = "th1s1sn0tas3cur3pass0wrd" + sensitive = true +} + +variable "grafana_admin_username" { + type = string + default = "admin" +} + +variable "grafana_admin_password" { + type = string + default = "Th1s1sN0TS3CuR3!!" + sensitive = true +} + +variable "azs" { + type = list(string) + default = ["a", "b", "c"] +} \ No newline at end of file diff --git a/infra/1-click/2-coder/7-license.tf b/infra/1-click/2-coder/7-license.tf deleted file mode 100644 index 80398f9..0000000 --- a/infra/1-click/2-coder/7-license.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "coder_license" { - type = string - default = "" - sensitive = true -} - -# resource "coderd_license" "enterprise" { -# license = var.coder_license -# } - -# data "http" "get-license" { - -# depends_on = [data.http.first-user] - -# url = "https://${data.dns_a_record_set.coder.addrs[0]}/api/v2/users/login" -# insecure = true -# method = "POST" -# request_headers = { -# Host = var.domain_name -# Accept = "application/json" -# } -# request_body = jsonencode({ -# email = var.coder_admin_email -# password = var.coder_admin_password -# }) -# } \ No newline at end of file diff --git a/infra/1-click/2-coder/8-backend.tf b/infra/1-click/2-coder/8-backend.tf new file mode 100644 index 0000000..a23d278 --- /dev/null +++ b/infra/1-click/2-coder/8-backend.tf @@ -0,0 +1,150 @@ +## +# Setup main provisioner + proxy if license is added. +## + +## +# External Provisioner Setup +## + +resource "coderd_license" "enterprise" { + count = var.coder_license != "" ? 1 : 0 + license = var.coder_license +} + +locals { + coder_ns = "coder" + coder_release_name = "coder" + coder_svc_url = "http://${local.coder_release_name}.${local.coder_ns}.svc.cluster.local" +} + +module "coder-ext-prov" { + + count = length(coderd_license.enterprise) + depends_on = [ coderd_license.enterprise ] + source = "../../../modules/k8s/bootstrap/coder-provisioner" + + release_name = "coder-provisioner" + namespace = "coder-provisioner" + chart_name = "coder-provisioner" + chart_version = var.coder_version + + cluster_name = "${local.formatted_name}" + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.coder.arn + + coder = { + access_url = local.coder_svc_url + ws_ns = [local.coder_ns] + image_tag = "v${var.coder_version}" + rep_cnt = 5 + } +} + +## +# Proxy Setup +## + +locals { + apex_domain = join(".", slice(split(".", var.domain_name), length(split(".", var.domain_name)) - 2, length(split(".", var.domain_name)))) + pub_subs = [for az in var.azs : "${local.formatted_name}-public-${data.aws_region.this.region}${az}"] + proxy_access_url = "proxy.${var.domain_name}" + proxy_wildcard_url = "*.proxy.${var.domain_name}" +} + +## +# Fetch AWS Hosted Zone +## + +data "aws_route53_zone" "coder" { + name = local.apex_domain +} + +resource "aws_eip" "proxy" { + count = 1 + domain = "vpc" + public_ipv4_pool = "amazon" + tags = { + Name = "${local.formatted_name}-proxy-${count.index}" + } +} + +resource "aws_route53_record" "proxy-primary" { + name = local.proxy_access_url + records = aws_eip.proxy.*.public_ip + ttl = 60 + type = "A" + zone_id = data.aws_route53_zone.coder.zone_id +} + +resource "aws_route53_record" "proxy-wildcard" { + name = local.proxy_wildcard_url + records = aws_eip.proxy.*.public_ip + ttl = 60 + type = "A" + zone_id = data.aws_route53_zone.coder.zone_id +} + +resource "aws_acm_certificate" "proxy" { + domain_name = local.proxy_access_url + subject_alternative_names = [local.proxy_wildcard_url] + validation_method = "DNS" +} + +resource "aws_route53_record" "proxy_validation" { + + for_each = { + for rec in aws_acm_certificate.proxy.domain_validation_options : rec.domain_name => { + name = rec.resource_record_name + record = rec.resource_record_value + type = rec.resource_record_type + } + } + + allow_overwrite = true + name = each.value.name + records = [each.value.record] + ttl = 60 + type = each.value.type + zone_id = data.aws_route53_zone.coder.zone_id +} + +resource "aws_acm_certificate_validation" "proxy" { + certificate_arn = aws_acm_certificate.proxy.arn + validation_record_fqdns = [for record in aws_route53_record.proxy_validation : record.fqdn] + + timeouts { + create = "30m" + } +} + +# module "coder-proxy" { + +# count = length(coderd_license.enterprise) +# depends_on = [ coderd_license.enterprise, module.coder-ext-prov ] +# source = "../../../modules/k8s/bootstrap/coder-proxy" + +# release_name = "coder-proxy" +# namespace = "coder-proxy" +# chart_name = "coder" +# chart_version = var.coder_version + +# proxy = { +# name = "proxy" +# display_name = "Coder Proxy" +# access_url = "https://${local.proxy_access_url}" +# icon = "/emojis/1f4a1.png" # 💡 +# wildcard_url = local.proxy_wildcard_url +# coder_access_url = "https://${var.domain_name}" +# image_tag = "v${var.coder_version}" +# rep_cnt = 1 +# } + +# svc_annot = { +# "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" +# "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" +# "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" +# "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" = join(",", aws_eip.proxy.*.allocation_id) +# "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", [local.pub_subs[0]]) +# "service.beta.kubernetes.io/aws-load-balancer-ssl-cert" = aws_acm_certificate.proxy.arn +# "service.beta.kubernetes.io/aws-load-balancer-ssl-ports" = 443 +# } +# } \ No newline at end of file diff --git a/infra/1-click/2-coder/8-templates.tf b/infra/1-click/2-coder/8-templates.tf deleted file mode 100644 index c11399b..0000000 --- a/infra/1-click/2-coder/8-templates.tf +++ /dev/null @@ -1,131 +0,0 @@ -## -# Coder Template Push -## - -data "coderd_organization" "default" { - - # depends_on = [ coderd_license.enterprise ] - - is_default = true -} - -data "coderd_group" "everyone" { - - # depends_on = [ coderd_license.enterprise ] - - organization_id = data.coderd_organization.default.id - name = "Everyone" -} - -data "archive_file" "k8s-default" { - type = "zip" - excludes = ["${path.module}/templates/kubernetes/.terraform"] - source_dir = "${path.module}/templates/kubernetes" - output_path = "/tmp/kubernetes.zip" -} - -resource "time_static" "k8s-default" { - triggers = { - run_on_ip_changes = data.aws_eip.coder.private_ip - run_on_checksum = "${data.archive_file.k8s-default.id}" - } -} - -resource "coderd_template" "k8s-default" { - - # depends_on = [ coderd_license.enterprise ] - - name = "kubernetes" - organization_id = data.coderd_organization.default.id - display_name = "Kubernetes (Deployment)" - description = "Provision Kubernetes Deployments as Coder workspaces" - icon = "https://${var.domain_name}/icon/k8s.png" - versions = [ - { - name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.k8s-default.rfc3339)}" - description = "The stable version of the template." - directory = "${path.module}/templates/kubernetes" - active = true - tf_vars = [{ - name = "namespace" - value = "coder" - }, { - name = "use_kubeconfig" - value = tostring(false) - }, { - name = "host_ip" - value = data.aws_eip.coder.private_ip - }] - } - ] - acl = var.coder_license == "" ? null :{ - users = [] - groups = [{ - id = data.coderd_group.everyone.id - role = "use" - }] - } -} - -data "archive_file" "k8s-claude" { - - count = var.coder_license != "" ? 1 : 0 - - type = "zip" - excludes = ["${path.module}/templates/kubernetes-claude/.terraform"] - source_dir = "${path.module}/templates/kubernetes-claude" - output_path = "/tmp/kubernetes-claude.zip" -} - -resource "time_static" "k8s-claude" { - - count = var.coder_license != "" ? 1 : 0 - - triggers = { - run_on_ip_changes = data.aws_eip.coder.private_ip - run_on_checksum = "${data.archive_file.k8s-claude[0].id}" - } -} - -## -# NOTICE: This template requires a Premium license as it includes the following: -# - AI Bridge -# - Agent Boundaries -## - -resource "coderd_template" "k8s-claude" { - - # depends_on = [ coderd_license.enterprise ] - count = var.coder_license != "" ? 1 : 0 - - name = "kubernetes-claude" - organization_id = data.coderd_organization.default.id - display_name = "Claude Code on Coder" - description = "Provision a Kubernetes Deployments with Claude installed." - icon = "https://${var.domain_name}/icon/claude.svg" - versions = [ - { - name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.k8s-claude[0].rfc3339)}" - description = "The stable version of the template." - directory = "${path.module}/templates/kubernetes-claude" - active = true - tf_vars = [{ - name = "namespace" - value = "coder" - }, { - name = "use_kubeconfig" - value = tostring(false) - }, { - name = "host_ip" - value = data.aws_eip.coder.private_ip - }] - } - ] - acl = { - users = [] - groups = [{ - id = data.coderd_group.everyone.id - role = "use" - }] - } -} \ No newline at end of file diff --git a/infra/1-click/2-coder/9-templates.tf b/infra/1-click/2-coder/9-templates.tf new file mode 100644 index 0000000..77a81e9 --- /dev/null +++ b/infra/1-click/2-coder/9-templates.tf @@ -0,0 +1,222 @@ +## +# Coder Template Push +## + +data "coderd_organization" "default" { + is_default = true +} + +## +# Standalone Kubernetes Template +## + +data "archive_file" "k8s-default" { + type = "zip" + excludes = ["${path.module}/templates/kubernetes/.terraform"] + source_dir = "${path.module}/templates/kubernetes" + output_path = "${path.module}/templates/kubernetes.zip" +} + +resource "time_static" "k8s-default" { + triggers = { + run_on_ip_changes = data.aws_eip.coder.private_ip + run_on_checksum = "${data.archive_file.k8s-default.id}" + } +} + +resource "coderd_template" "k8s-default" { + + depends_on = [module.coder-ext-prov] + + name = "kubernetes" + organization_id = data.coderd_organization.default.id + display_name = "Kubernetes (Deployment)" + description = "Provision Kubernetes Deployments as Coder workspaces" + icon = "https://${var.domain_name}/icon/k8s.png" + versions = [ + { + name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.k8s-default.rfc3339)}" + description = "The stable version of the template." + directory = "${path.module}/templates/kubernetes" + active = true + tf_vars = [{ + name = "namespace" + value = "coder" + }, { + name = "use_kubeconfig" + value = tostring(false) + }, { + name = "host_ip" + value = data.aws_eip.coder.private_ip + }] + } + ] +} + +## +# Claude-Code Kubernetes Template +## + +## +# NOTICE: This template requires a Premium license as it includes the following: +# - AI Bridge +# +# NOTE: AI Boundary's will not work in EKS Auto Mode. +## + +data "archive_file" "k8s-claude" { + + count = var.coder_license != "" && length(module.coder-ext-prov) > 0 ? 1 : 0 + + type = "zip" + excludes = ["${path.module}/templates/kubernetes-claude/.terraform"] + source_dir = "${path.module}/templates/kubernetes-claude" + output_path = "${path.module}/templates/kubernetes-claude.zip" +} + +resource "time_static" "k8s-claude" { + + count = var.coder_license != "" && length(module.coder-ext-prov) > 0 ? 1 : 0 + + triggers = { + run_on_ip_changes = data.aws_eip.coder.private_ip + run_on_checksum = "${data.archive_file.k8s-claude[0].id}" + } +} + +resource "coderd_template" "k8s-claude" { + + count = var.coder_license != "" && length(module.coder-ext-prov) > 0 ? 1 : 0 + + name = "kubernetes-claude" + organization_id = data.coderd_organization.default.id + display_name = "Claude Code on Coder" + description = "Provision a Kubernetes Deployments with Claude installed." + icon = "https://${var.domain_name}/icon/claude.svg" + versions = [ + { + name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.k8s-claude[0].rfc3339)}" + description = "The stable version of the template." + directory = "${path.module}/templates/kubernetes-claude" + active = true + tf_vars = [{ + name = "namespace" + value = "coder" + }, { + name = "use_kubeconfig" + value = tostring(false) + }, { + name = "host_ip" + value = data.aws_eip.coder.private_ip + }] + } + ] +} + +## +# AWS EC2 Linux +## + +data "archive_file" "aws-linux" { + type = "zip" + excludes = ["${path.module}/templates/aws-linux/.terraform"] + source_dir = "${path.module}/templates/aws-linux" + output_path = "${path.module}/templates/aws-linux.zip" +} + +resource "time_static" "aws-linux" { + triggers = { + run_on_checksum = "${data.archive_file.aws-linux.id}" + } +} + +resource "coderd_template" "aws-linux" { + + depends_on = [module.coder-ext-prov] + + name = "aws-linux" + organization_id = data.coderd_organization.default.id + display_name = "AWS EC2 (Linux)" + description = "Provision an AWS EC2 Linux VM." + icon = "https://${var.domain_name}/icon/aws.svg" + versions = [ + { + name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.aws-linux.rfc3339)}" + description = "The stable version of the template." + directory = "${path.module}/templates/aws-linux" + active = true + } + ] +} + +## +# AWS EC2 Windows +## + +data "archive_file" "aws-windows" { + type = "zip" + excludes = ["${path.module}/templates/aws-windows/.terraform"] + source_dir = "${path.module}/templates/aws-windows" + output_path = "${path.module}/templates/aws-windows.zip" +} + +resource "time_static" "aws-windows" { + triggers = { + run_on_checksum = "${data.archive_file.aws-windows.id}" + } +} + +resource "coderd_template" "aws-windows" { + + depends_on = [module.coder-ext-prov] + + name = "aws-windows" + organization_id = data.coderd_organization.default.id + display_name = "AWS EC2 (Windows)" + description = "Provision an AWS EC2 Windows VM." + icon = "https://${var.domain_name}/icon/windows.svg" + versions = [ + { + name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.aws-windows.rfc3339)}" + description = "The stable version of the template." + directory = "${path.module}/templates/aws-windows" + active = true + } + ] +} + +## +# AWS EC2 DevContainer +## + +data "archive_file" "aws-devcontainer" { + type = "zip" + excludes = ["${path.module}/templates/aws-devcontainer/.terraform"] + source_dir = "${path.module}/templates/aws-devcontainer" + output_path = "${path.module}/templates/aws-devcontainer.zip" +} + +resource "time_static" "aws-devcontainer" { + triggers = { + run_on_checksum = "${data.archive_file.aws-devcontainer.id}" + } +} + +resource "coderd_template" "aws-devcontainer" { + + depends_on = [module.coder-ext-prov] + + name = "aws-devcontainer" + organization_id = data.coderd_organization.default.id + display_name = "AWS EC2 (Linux with Devcontainer)" + description = "Provision an AWS EC2 Linux VM running DevContainers." + icon = "https://${var.domain_name}/icon/aws.svg" + versions = [ + { + name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.aws-devcontainer.rfc3339)}" + description = "The stable version of the template." + directory = "${path.module}/templates/aws-devcontainer" + active = true + } + ] +} \ No newline at end of file diff --git a/infra/1-click/2-coder/main.tf b/infra/1-click/2-coder/main.tf index aada9e7..be172b6 100644 --- a/infra/1-click/2-coder/main.tf +++ b/infra/1-click/2-coder/main.tf @@ -1,78 +1,10 @@ -terraform { - required_version = ">= 1.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.46" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - external = { - source = "hashicorp/external" - version = ">= 2.3.5" - } - http = { - source = "hashicorp/http" - } - coderd = { - source = "coder/coderd" - version = "0.0.12" - } - time = { - source = "hashicorp/time" - } - dns = { - source = "hashicorp/dns" - } - } -} - -## -# Remote State Resources -## - -data "aws_vpc" "this" { - tags = { - "Name" = "${var.name}-${local.normalized_domain_name}" - } -} - -data "aws_eks_cluster" "coder" { - name = "${var.name}-${local.normalized_domain_name}" -} - -data "aws_eks_cluster_auth" "coder" { - name = "${var.name}-${local.normalized_domain_name}" -} - -data "aws_iam_openid_connect_provider" "coder" { - url = data.aws_eks_cluster.coder.identity[0].oidc[0].issuer -} - ## # Global Inputs + Providers ## -variable "region" { - description = "The AWS region of the deployment." - type = string - default = "us-east-2" -} - -variable "name" { - description = "Name for created resources and tag prefix." - type = string - default = "coder" -} - -variable "profile" { - type = string - default = "default" +locals { + formatted_name = "${var.name}-${local.normalized_domain_name}" + normalized_domain_name = split(".", var.domain_name)[0] } provider "aws" { @@ -80,43 +12,20 @@ provider "aws" { profile = var.profile } -provider "helm" { - kubernetes = { - host = data.aws_eks_cluster.coder.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.coder.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.coder.token - } -} +data "aws_region" "this" {} -provider "kubernetes" { - host = data.aws_eks_cluster.coder.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.coder.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.coder.token -} - -provider "dns" {} - -locals { - normalized_domain_name = split(".", var.domain_name)[0] -} - -## -# Login and Fetch Authentication Token -## - -variable "domain_name" { - type = string +data "aws_eks_cluster" "coder" { + name = local.formatted_name + region = var.region } -variable "coder_admin_email" { - type = string - default = "admin@coder.com" +data "aws_eks_cluster_auth" "coder" { + name = local.formatted_name + region = var.region } -variable "coder_admin_password" { - type = string - default = "Th1s1sN0TS3CuR3!!" - sensitive = true +data "aws_iam_openid_connect_provider" "coder" { + url = data.aws_eks_cluster.coder.identity[0].oidc[0].issuer } ## @@ -124,17 +33,17 @@ variable "coder_admin_password" { ## data "aws_eip" "coder" { + region = var.region tags = { - Name = "${var.name}-${local.normalized_domain_name}-coder-0" + Name = "${local.formatted_name}-coder-0" } } data "http" "login" { - url = "https://${data.aws_eip.coder.public_ip}/api/v2/users/login" - insecure = true + url = "http://${data.aws_eip.coder.public_ip}/api/v2/users/login" method = "POST" request_headers = { - Host = var.domain_name + Host = var.domain_name Accept = "application/json" } request_body = jsonencode({ @@ -143,7 +52,21 @@ data "http" "login" { }) } +provider "helm" { + kubernetes = { + host = data.aws_eks_cluster.coder.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.coder.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.coder.token + } +} + +provider "kubernetes" { + host = data.aws_eks_cluster.coder.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.coder.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.coder.token +} + provider "coderd" { - url = "https://${var.domain_name}" + url = "http://${data.aws_eip.coder.public_ip}" token = jsondecode(data.http.login.response_body).session_token } \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-devcontainer/README.md b/infra/1-click/2-coder/templates/aws-devcontainer/README.md new file mode 100644 index 0000000..3b58d73 --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-devcontainer/README.md @@ -0,0 +1,111 @@ +--- +display_name: AWS EC2 (Devcontainer) +description: Provision AWS EC2 VMs with a devcontainer as Coder workspaces +icon: ../../../site/static/icon/aws.svg +maintainer_github: coder +verified: true +tags: [vm, linux, aws, persistent, devcontainer] +--- + +# Remote Development on AWS EC2 VMs using a Devcontainer + +Provision AWS EC2 VMs as [Coder workspaces](https://coder.com/docs) with this example template. +![Architecture Diagram](./architecture.svg) + + + +## Prerequisites + +### Authentication + +By default, this template authenticates to AWS using the provider's default [authentication methods](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration). + +The simplest way (without making changes to the template) is via environment variables (e.g. `AWS_ACCESS_KEY_ID`) or a [credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format). If you are running Coder on a VM, this file must be in `/home/coder/aws/credentials`. + +To use another [authentication method](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication), edit the template. + +## Required permissions / policy + +The following sample policy allows Coder to create EC2 instances and modify +instances provisioned by Coder: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:GetDefaultCreditSpecification", + "ec2:DescribeIamInstanceProfileAssociations", + "ec2:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceStatus", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:DescribeInstanceCreditSpecifications", + "ec2:DescribeImages", + "ec2:ModifyDefaultCreditSpecification", + "ec2:DescribeVolumes" + ], + "Resource": "*" + }, + { + "Sid": "CoderResources", + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstanceAttribute", + "ec2:UnmonitorInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances", + "ec2:DeleteTags", + "ec2:MonitorInstances", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyInstanceCreditSpecification" + ], + "Resource": "arn:aws:ec2:*:*:instance/*", + "Condition": { + "StringEquals": { + "aws:ResourceTag/Coder_Provisioned": "true" + } + } + } + ] +} +``` + +## Architecture + +This template provisions the following resources: + +- AWS Instance + +Coder uses `aws_ec2_instance_state` to start and stop the VM. This example template is fully persistent, meaning the full filesystem is preserved when the workspace restarts. See this [community example](https://github.com/bpmct/coder-templates/tree/main/aws-linux-ephemeral) of an ephemeral AWS instance. + +> **Note** +> This template is designed to be a starting point! Edit the Terraform to extend the template to support your use case. + +## Caching + +To speed up your builds, you can use a container registry as a cache. +When creating the template, set the parameter `cache_repo` to a valid Docker repository in the form `host.tld/path/to/repo`. + +See the [Envbuilder Terraform Provider Examples](https://github.com/coder/terraform-provider-envbuilder/blob/main/examples/resources/envbuilder_cached_image/envbuilder_cached_image_resource.tf/) for a more complete example of how the provider works. + +> [!NOTE] +> We recommend using a registry cache with authentication enabled. +> To allow Envbuilder to authenticate with a registry cache hosted on ECR, specify an IAM instance +> profile that has read and write access to the given registry. For more information, see the +> [AWS documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html). +> +> Alternatively, you can specify the variable `cache_repo_docker_config_path` +> with the path to a Docker config `.json` on disk containing valid credentials for the registry. + +## code-server + +`code-server` is installed via the [`code-server`](https://registry.coder.com/modules/code-server) registry module. For a list of all modules and templates pplease check [Coder Registry](https://registry.coder.com). \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-devcontainer/architecture.svg b/infra/1-click/2-coder/templates/aws-devcontainer/architecture.svg new file mode 100644 index 0000000..737db74 --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-devcontainer/architecture.svg @@ -0,0 +1,8 @@ +AWSAWSHostingHostingVirtual MachineVirtual MachineLinux HardwareLinux HardwareCoder WorkspaceCoder WorkspaceDevcontainerDevcontainerenvbuilder created filesytemenvbuilder created filesytemA Clone of your repoA Clone of your repoSource codeSource codeLanguagesLanguagesPython. Go, etcPython. Go, etcToolingToolingExtensions, linting, formatting, etcExtensions, linting, formatting, etcCPUsCPUsDisk StorageDisk StorageCode EditorCode EditorVS Code DesktopVS Code DesktopLocal InstallationLocal InstallationVS Code DesktopVS Code DesktopLocal InstallationLocal Installationcode-servercode-serverA web IDEA web IDEJetBrains GatewayJetBrains GatewayLocal InstallationLocal InstallationCommand LineCommand LineSSH via Coder CLISSH via Coder CLI \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-devcontainer/cloud-init/cloud-config.yaml.tftpl b/infra/1-click/2-coder/templates/aws-devcontainer/cloud-init/cloud-config.yaml.tftpl new file mode 100644 index 0000000..3aaefb4 --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-devcontainer/cloud-init/cloud-config.yaml.tftpl @@ -0,0 +1,15 @@ +#cloud-config +cloud_final_modules: + - [scripts-user, always] +hostname: ${hostname} +users: + - name: ${linux_user} + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + - "${ssh_pubkey}" +# Automatically grow the partition +growpart: + mode: auto + devices: ['/'] + ignore_growroot_disabled: false \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-devcontainer/cloud-init/userdata.sh.tftpl b/infra/1-click/2-coder/templates/aws-devcontainer/cloud-init/userdata.sh.tftpl new file mode 100644 index 0000000..9fb8117 --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-devcontainer/cloud-init/userdata.sh.tftpl @@ -0,0 +1,37 @@ +#!/bin/bash +# Install Docker +if ! command -v docker &> /dev/null +then + echo "Docker not found, installing..." + curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh 2>&1 >/dev/null + usermod -aG docker ${linux_user} + newgrp docker +else + echo "Docker is already installed." +fi + +# Set up Docker credentials +mkdir -p "/home/${linux_user}/.docker" + +if [ -n "${docker_config_json_base64}" ]; then + # Write the Docker config JSON to disk if it is provided. + printf "%s" "${docker_config_json_base64}" | base64 -d | tee "/home/${linux_user}/.docker/config.json" +else + # Assume that we're going to use the instance IAM role to pull from the cache repo if we need to. + # Set up the ecr credential helper. + apt-get update -y && apt-get install -y amazon-ecr-credential-helper + mkdir -p .docker + printf '{"credsStore": "ecr-login"}' | tee "/home/${linux_user}/.docker/config.json" +fi +chown -R ${linux_user}:${linux_user} "/home/${linux_user}/.docker" + +# Start envbuilder +sudo -u coder docker run \ + --rm \ + --net=host \ + -h ${hostname} \ + -v /home/${linux_user}/envbuilder:/workspaces \ + %{ for key, value in environment ~} + -e ${key}="${value}" \ + %{ endfor ~} + ${builder_image} \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-devcontainer/main.tf b/infra/1-click/2-coder/templates/aws-devcontainer/main.tf new file mode 100644 index 0000000..703c1b3 --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-devcontainer/main.tf @@ -0,0 +1,331 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + aws = { + source = "hashicorp/aws" + } + cloudinit = { + source = "hashicorp/cloudinit" + } + envbuilder = { + source = "coder/envbuilder" + } + } +} + +module "aws_region" { + source = "https://registry.coder.com/modules/aws-region" + default = "us-east-1" +} + +provider "aws" { + region = module.aws_region.value +} + +variable "cache_repo" { + default = "" + description = "(Optional) Use a container registry as a cache to speed up builds. Example: host.tld/path/to/repo." + type = string +} + +variable "cache_repo_docker_config_path" { + default = "" + description = "(Optional) Path to a docker config.json containing credentials to the provided cache repo, if required. This will depend on your Coder setup. Example: `/home/coder/.docker/config.json`." + sensitive = true + type = string +} + +variable "iam_instance_profile" { + default = "" + description = "(Optional) Name of an IAM instance profile to assign to the instance." + type = string +} + +data "coder_workspace" "me" {} +data "coder_workspace_owner" "me" {} + +data "aws_ami" "ubuntu" { + most_recent = true + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } + owners = ["099720109477"] # Canonical +} + +data "coder_parameter" "instance_type" { + name = "instance_type" + display_name = "Instance type" + description = "What instance type should your workspace use?" + default = "t3.micro" + mutable = false + option { + name = "2 vCPU, 1 GiB RAM" + value = "t3.micro" + } + option { + name = "2 vCPU, 2 GiB RAM" + value = "t3.small" + } + option { + name = "2 vCPU, 4 GiB RAM" + value = "t3.medium" + } + option { + name = "2 vCPU, 8 GiB RAM" + value = "t3.large" + } + option { + name = "4 vCPU, 16 GiB RAM" + value = "t3.xlarge" + } + option { + name = "8 vCPU, 32 GiB RAM" + value = "t3.2xlarge" + } +} + +data "coder_parameter" "root_volume_size_gb" { + name = "root_volume_size_gb" + display_name = "Root Volume Size (GB)" + description = "How large should the root volume for the instance be?" + default = 30 + type = "number" + mutable = true + validation { + min = 1 + monotonic = "increasing" + } +} + +data "coder_parameter" "fallback_image" { + default = "codercom/enterprise-base:ubuntu" + description = "This image runs if the devcontainer fails to build." + display_name = "Fallback Image" + mutable = true + name = "fallback_image" + order = 3 +} + +data "coder_parameter" "devcontainer_builder" { + description = <<-EOF +Image that will build the devcontainer. +Find the latest version of Envbuilder here: https://ghcr.io/coder/envbuilder +Be aware that using the `:latest` tag may expose you to breaking changes. +EOF + display_name = "Devcontainer Builder" + mutable = true + name = "devcontainer_builder" + default = "ghcr.io/coder/envbuilder:latest" + order = 4 +} + +data "coder_parameter" "repo_url" { + name = "repo_url" + display_name = "Repository URL" + default = "https://github.com/coder/envbuilder-starter-devcontainer" + description = "Repository URL" + mutable = true +} + +data "coder_parameter" "ssh_pubkey" { + name = "ssh_pubkey" + display_name = "SSH Public Key" + default = "" + description = "(Optional) Add an SSH public key to the `coder` user's authorized_keys. Useful for troubleshooting. You may need to add a security group to the instance." + mutable = false +} + +data "local_sensitive_file" "cache_repo_dockerconfigjson" { + count = var.cache_repo_docker_config_path == "" ? 0 : 1 + filename = var.cache_repo_docker_config_path +} + +data "aws_iam_instance_profile" "vm_instance_profile" { + count = var.iam_instance_profile == "" ? 0 : 1 + name = var.iam_instance_profile +} + +# Be careful when modifying the below locals! +locals { + # TODO: provide a way to pick the availability zone. + aws_availability_zone = "${module.aws_region.value}a" + + hostname = lower(data.coder_workspace.me.name) + linux_user = "coder" + + # The devcontainer builder image is the image that will build the devcontainer. + devcontainer_builder_image = data.coder_parameter.devcontainer_builder.value + + # We may need to authenticate with a registry. If so, the user will provide a path to a docker config.json. + docker_config_json_base64 = try(data.local_sensitive_file.cache_repo_dockerconfigjson[0].content_base64, "") + + # The envbuilder provider requires a key-value map of environment variables. Build this here. + envbuilder_env = { + # ENVBUILDER_GIT_URL and ENVBUILDER_CACHE_REPO will be overridden by the provider + # if the cache repo is enabled. + "ENVBUILDER_GIT_URL" : data.coder_parameter.repo_url.value, + # The agent token is required for the agent to connect to the Coder platform. + "CODER_AGENT_TOKEN" : try(coder_agent.dev.0.token, ""), + # The agent URL is required for the agent to connect to the Coder platform. + "CODER_AGENT_URL" : data.coder_workspace.me.access_url, + # The agent init script is required for the agent to start up. We base64 encode it here + # to avoid quoting issues. + "ENVBUILDER_INIT_SCRIPT" : "echo ${base64encode(try(coder_agent.dev[0].init_script, ""))} | base64 -d | sh", + "ENVBUILDER_DOCKER_CONFIG_BASE64" : local.docker_config_json_base64, + # The fallback image is the image that will run if the devcontainer fails to build. + "ENVBUILDER_FALLBACK_IMAGE" : data.coder_parameter.fallback_image.value, + # The following are used to push the image to the cache repo, if defined. + "ENVBUILDER_CACHE_REPO" : var.cache_repo, + "ENVBUILDER_PUSH_IMAGE" : var.cache_repo == "" ? "" : "true", + # You can add other required environment variables here. + # See: https://github.com/coder/envbuilder/?tab=readme-ov-file#environment-variables + } +} + +# Check for the presence of a prebuilt image in the cache repo +# that we can use instead. +resource "envbuilder_cached_image" "cached" { + count = var.cache_repo == "" ? 0 : data.coder_workspace.me.start_count + builder_image = local.devcontainer_builder_image + git_url = data.coder_parameter.repo_url.value + cache_repo = var.cache_repo + extra_env = local.envbuilder_env +} + +data "cloudinit_config" "user_data" { + gzip = false + base64_encode = false + + boundary = "//" + + part { + filename = "cloud-config.yaml" + content_type = "text/cloud-config" + + content = templatefile("${path.module}/cloud-init/cloud-config.yaml.tftpl", { + hostname = local.hostname + linux_user = local.linux_user + + ssh_pubkey = data.coder_parameter.ssh_pubkey.value + }) + } + + part { + filename = "userdata.sh" + content_type = "text/x-shellscript" + + content = templatefile("${path.module}/cloud-init/userdata.sh.tftpl", { + hostname = local.hostname + linux_user = local.linux_user + + # If we have a cached image, use the cached image's environment variables. + # Otherwise, just use the environment variables we've defined in locals. + environment = try(envbuilder_cached_image.cached[0].env_map, local.envbuilder_env) + + # Builder image will either be the builder image parameter, or the cached image, if cache is provided. + builder_image = try(envbuilder_cached_image.cached[0].image, data.coder_parameter.devcontainer_builder.value) + + docker_config_json_base64 = local.docker_config_json_base64 + }) + } +} + +# This is useful for debugging the startup script. Left here for reference. +# resource local_file "startup_script" { +# content = data.cloudinit_config.user_data.rendered +# filename = "${path.module}/user_data.txt" +# } + +resource "aws_instance" "vm" { + ami = data.aws_ami.ubuntu.id + availability_zone = local.aws_availability_zone + instance_type = data.coder_parameter.instance_type.value + iam_instance_profile = try(data.aws_iam_instance_profile.vm_instance_profile[0].name, null) + root_block_device { + volume_size = data.coder_parameter.root_volume_size_gb.value + } + + user_data = data.cloudinit_config.user_data.rendered + tags = { + Name = "coder-${data.coder_workspace_owner.me.name}-${lower(data.coder_workspace.me.name)}" + # Required if you are using our example policy, see template README + Coder_Provisioned = "true" + } + lifecycle { + ignore_changes = [ami] + } +} + +resource "aws_ec2_instance_state" "vm" { + instance_id = aws_instance.vm.id + state = data.coder_workspace.me.transition == "start" ? "running" : "stopped" +} + +resource "coder_agent" "dev" { + count = data.coder_workspace.me.start_count + arch = "amd64" + auth = "token" + os = "linux" + dir = "/workspaces/${trimsuffix(basename(data.coder_parameter.repo_url.value), ".git")}" + connection_timeout = 0 + + metadata { + key = "cpu" + display_name = "CPU Usage" + interval = 5 + timeout = 5 + script = "coder stat cpu" + } + metadata { + key = "memory" + display_name = "Memory Usage" + interval = 5 + timeout = 5 + script = "coder stat mem" + } +} + +resource "coder_metadata" "info" { + count = data.coder_workspace.me.start_count + resource_id = coder_agent.dev[0].id + item { + key = "ami" + value = aws_instance.vm.ami + } + item { + key = "availability_zone" + value = local.aws_availability_zone + } + item { + key = "instance_type" + value = data.coder_parameter.instance_type.value + } + item { + key = "ssh_pubkey" + value = data.coder_parameter.ssh_pubkey.value + } + item { + key = "repo_url" + value = data.coder_parameter.repo_url.value + } + item { + key = "devcontainer_builder" + value = data.coder_parameter.devcontainer_builder.value + } +} + +# See https://registry.coder.com/modules/coder/code-server +module "code-server" { + count = data.coder_workspace.me.start_count + source = "registry.coder.com/coder/code-server/coder" + # This ensures that the latest non-breaking version of the module gets downloaded, you can also pin the module version to prevent breaking changes in production. + version = "~> 1.0" + agent_id = coder_agent.dev[0].id +} \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-linux/README.md b/infra/1-click/2-coder/templates/aws-linux/README.md new file mode 100644 index 0000000..3575640 --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-linux/README.md @@ -0,0 +1,94 @@ +--- +display_name: AWS EC2 (Linux) +description: Provision AWS EC2 VMs as Coder workspaces +icon: ../../../site/static/icon/aws.svg +maintainer_github: coder +verified: true +tags: [vm, linux, aws, persistent-vm] +--- + +# Remote Development on AWS EC2 VMs (Linux) + +Provision AWS EC2 VMs as [Coder workspaces](https://coder.com/docs/workspaces) with this example template. + +## Prerequisites + +### Authentication + +By default, this template authenticates to AWS using the provider's default [authentication methods](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration). + +The simplest way (without making changes to the template) is via environment variables (e.g. `AWS_ACCESS_KEY_ID`) or a [credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format). If you are running Coder on a VM, this file must be in `/home/coder/aws/credentials`. + +To use another [authentication method](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication), edit the template. + +## Required permissions / policy + +The following sample policy allows Coder to create EC2 instances and modify +instances provisioned by Coder: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:GetDefaultCreditSpecification", + "ec2:DescribeIamInstanceProfileAssociations", + "ec2:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceStatus", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:DescribeInstanceCreditSpecifications", + "ec2:DescribeImages", + "ec2:ModifyDefaultCreditSpecification", + "ec2:DescribeVolumes" + ], + "Resource": "*" + }, + { + "Sid": "CoderResources", + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstanceAttribute", + "ec2:UnmonitorInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances", + "ec2:DeleteTags", + "ec2:MonitorInstances", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyInstanceCreditSpecification" + ], + "Resource": "arn:aws:ec2:*:*:instance/*", + "Condition": { + "StringEquals": { + "aws:ResourceTag/Coder_Provisioned": "true" + } + } + } + ] +} +``` + +## Architecture + +This template provisions the following resources: + +- AWS Instance + +Coder uses `aws_ec2_instance_state` to start and stop the VM. This example template is fully persistent, meaning the full filesystem is preserved when the workspace restarts. See this [community example](https://github.com/bpmct/coder-templates/tree/main/aws-linux-ephemeral) of an ephemeral AWS instance. + +> **Note** +> This template is designed to be a starting point! Edit the Terraform to extend the template to support your use case. + +## code-server + +`code-server` is installed via the `startup_script` argument in the `coder_agent` +resource block. The `coder_app` resource is defined to access `code-server` through +the dashboard UI over `localhost:13337`. \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-linux/cloud-init/cloud-config.yaml.tftpl b/infra/1-click/2-coder/templates/aws-linux/cloud-init/cloud-config.yaml.tftpl new file mode 100644 index 0000000..b0ae9b2 --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-linux/cloud-init/cloud-config.yaml.tftpl @@ -0,0 +1,8 @@ +#cloud-config +cloud_final_modules: + - [scripts-user, always] +hostname: ${hostname} +users: + - name: ${linux_user} + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-linux/cloud-init/userdata.sh.tftpl b/infra/1-click/2-coder/templates/aws-linux/cloud-init/userdata.sh.tftpl new file mode 100644 index 0000000..0045120 --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-linux/cloud-init/userdata.sh.tftpl @@ -0,0 +1,2 @@ +#!/bin/bash +sudo -u '${linux_user}' sh -c '${init_script}' \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-linux/main.tf b/infra/1-click/2-coder/templates/aws-linux/main.tf new file mode 100644 index 0000000..ba22558 --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-linux/main.tf @@ -0,0 +1,286 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + cloudinit = { + source = "hashicorp/cloudinit" + } + aws = { + source = "hashicorp/aws" + } + } +} + +# Last updated 2023-03-14 +# aws ec2 describe-regions | jq -r '[.Regions[].RegionName] | sort' +data "coder_parameter" "region" { + name = "region" + display_name = "Region" + description = "The region to deploy the workspace in." + default = "us-east-1" + mutable = false + option { + name = "Asia Pacific (Tokyo)" + value = "ap-northeast-1" + icon = "/emojis/1f1ef-1f1f5.png" + } + option { + name = "Asia Pacific (Seoul)" + value = "ap-northeast-2" + icon = "/emojis/1f1f0-1f1f7.png" + } + option { + name = "Asia Pacific (Osaka)" + value = "ap-northeast-3" + icon = "/emojis/1f1ef-1f1f5.png" + } + option { + name = "Asia Pacific (Mumbai)" + value = "ap-south-1" + icon = "/emojis/1f1ee-1f1f3.png" + } + option { + name = "Asia Pacific (Singapore)" + value = "ap-southeast-1" + icon = "/emojis/1f1f8-1f1ec.png" + } + option { + name = "Asia Pacific (Sydney)" + value = "ap-southeast-2" + icon = "/emojis/1f1e6-1f1fa.png" + } + option { + name = "Canada (Central)" + value = "ca-central-1" + icon = "/emojis/1f1e8-1f1e6.png" + } + option { + name = "EU (Frankfurt)" + value = "eu-central-1" + icon = "/emojis/1f1ea-1f1fa.png" + } + option { + name = "EU (Stockholm)" + value = "eu-north-1" + icon = "/emojis/1f1ea-1f1fa.png" + } + option { + name = "EU (Ireland)" + value = "eu-west-1" + icon = "/emojis/1f1ea-1f1fa.png" + } + option { + name = "EU (London)" + value = "eu-west-2" + icon = "/emojis/1f1ea-1f1fa.png" + } + option { + name = "EU (Paris)" + value = "eu-west-3" + icon = "/emojis/1f1ea-1f1fa.png" + } + option { + name = "South America (São Paulo)" + value = "sa-east-1" + icon = "/emojis/1f1e7-1f1f7.png" + } + option { + name = "US East (N. Virginia)" + value = "us-east-1" + icon = "/emojis/1f1fa-1f1f8.png" + } + option { + name = "US East (Ohio)" + value = "us-east-2" + icon = "/emojis/1f1fa-1f1f8.png" + } + option { + name = "US West (N. California)" + value = "us-west-1" + icon = "/emojis/1f1fa-1f1f8.png" + } + option { + name = "US West (Oregon)" + value = "us-west-2" + icon = "/emojis/1f1fa-1f1f8.png" + } +} + +data "coder_parameter" "instance_type" { + name = "instance_type" + display_name = "Instance type" + description = "What instance type should your workspace use?" + default = "t3.micro" + mutable = false + option { + name = "2 vCPU, 1 GiB RAM" + value = "t3.micro" + } + option { + name = "2 vCPU, 2 GiB RAM" + value = "t3.small" + } + option { + name = "2 vCPU, 4 GiB RAM" + value = "t3.medium" + } + option { + name = "2 vCPU, 8 GiB RAM" + value = "t3.large" + } + option { + name = "4 vCPU, 16 GiB RAM" + value = "t3.xlarge" + } + option { + name = "8 vCPU, 32 GiB RAM" + value = "t3.2xlarge" + } +} + +provider "aws" { + region = data.coder_parameter.region.value +} + +data "coder_workspace" "me" {} +data "coder_workspace_owner" "me" {} + +data "aws_ami" "ubuntu" { + most_recent = true + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } + owners = ["099720109477"] # Canonical +} + +resource "coder_agent" "dev" { + count = data.coder_workspace.me.start_count + arch = "amd64" + auth = "aws-instance-identity" + os = "linux" + startup_script = <<-EOT + set -e + + # Add any commands that should be executed at workspace startup (e.g install requirements, start a program, etc) here + EOT + + metadata { + key = "cpu" + display_name = "CPU Usage" + interval = 5 + timeout = 5 + script = "coder stat cpu" + } + metadata { + key = "memory" + display_name = "Memory Usage" + interval = 5 + timeout = 5 + script = "coder stat mem" + } + metadata { + key = "disk" + display_name = "Disk Usage" + interval = 600 # every 10 minutes + timeout = 30 # df can take a while on large filesystems + script = "coder stat disk --path $HOME" + } +} + +# See https://registry.coder.com/modules/coder/code-server +module "code-server" { + count = data.coder_workspace.me.start_count + source = "registry.coder.com/modules/code-server/coder" + + # This ensures that the latest non-breaking version of the module gets downloaded, you can also pin the module version to prevent breaking changes in production. + version = "~> 1.0" + + agent_id = coder_agent.dev[0].id + order = 1 +} + +# See https://registry.coder.com/modules/coder/jetbrains +module "jetbrains" { + count = data.coder_workspace.me.start_count + source = "registry.coder.com/coder/jetbrains/coder" + version = "~> 1.0" + agent_id = coder_agent.dev[0].id + agent_name = "dev" + folder = "/home/coder" +} + +locals { + hostname = lower(data.coder_workspace.me.name) + linux_user = "coder" +} + +data "cloudinit_config" "user_data" { + gzip = false + base64_encode = false + + boundary = "//" + + part { + filename = "cloud-config.yaml" + content_type = "text/cloud-config" + + content = templatefile("${path.module}/cloud-init/cloud-config.yaml.tftpl", { + hostname = local.hostname + linux_user = local.linux_user + }) + } + + part { + filename = "userdata.sh" + content_type = "text/x-shellscript" + + content = templatefile("${path.module}/cloud-init/userdata.sh.tftpl", { + linux_user = local.linux_user + + init_script = try(coder_agent.dev[0].init_script, "") + }) + } +} + +resource "aws_instance" "dev" { + ami = data.aws_ami.ubuntu.id + availability_zone = "${data.coder_parameter.region.value}a" + instance_type = data.coder_parameter.instance_type.value + + user_data = data.cloudinit_config.user_data.rendered + tags = { + Name = "coder-${data.coder_workspace_owner.me.name}-${data.coder_workspace.me.name}" + # Required if you are using our example policy, see template README + Coder_Provisioned = "true" + } + lifecycle { + ignore_changes = [ami] + } +} + +resource "coder_metadata" "workspace_info" { + resource_id = aws_instance.dev.id + item { + key = "region" + value = data.coder_parameter.region.value + } + item { + key = "instance type" + value = aws_instance.dev.instance_type + } + item { + key = "disk" + value = "${aws_instance.dev.root_block_device[0].volume_size} GiB" + } +} + +resource "aws_ec2_instance_state" "dev" { + instance_id = aws_instance.dev.id + state = data.coder_workspace.me.transition == "start" ? "running" : "stopped" +} \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-windows/README.md b/infra/1-click/2-coder/templates/aws-windows/README.md new file mode 100644 index 0000000..b5af8cb --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-windows/README.md @@ -0,0 +1,96 @@ +--- +display_name: AWS EC2 (Windows) +description: Provision AWS EC2 VMs as Coder workspaces +icon: ../../../site/static/icon/aws.svg +maintainer_github: coder +verified: true +tags: [vm, windows, aws] +--- + +# Remote Development on AWS EC2 VMs (Windows) + +Provision AWS EC2 Windows VMs as [Coder workspaces](https://coder.com/docs/workspaces) with this example template. + + + +## Prerequisites + +### Authentication + +By default, this template authenticates to AWS with using the provider's default [authentication methods](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration). + +The simplest way (without making changes to the template) is via environment variables (e.g. `AWS_ACCESS_KEY_ID`) or a [credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-format). If you are running Coder on a VM, this file must be in `/home/coder/aws/credentials`. + +To use another [authentication method](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication), edit the template. + +## Required permissions / policy + +The following sample policy allows Coder to create EC2 instances and modify +instances provisioned by Coder: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:GetDefaultCreditSpecification", + "ec2:DescribeIamInstanceProfileAssociations", + "ec2:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceStatus", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:DescribeInstanceCreditSpecifications", + "ec2:DescribeImages", + "ec2:ModifyDefaultCreditSpecification", + "ec2:DescribeVolumes" + ], + "Resource": "*" + }, + { + "Sid": "CoderResources", + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstanceAttribute", + "ec2:UnmonitorInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances", + "ec2:DeleteTags", + "ec2:MonitorInstances", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyInstanceCreditSpecification" + ], + "Resource": "arn:aws:ec2:*:*:instance/*", + "Condition": { + "StringEquals": { + "aws:ResourceTag/Coder_Provisioned": "true" + } + } + } + ] +} +``` + +## Architecture + +This template provisions the following resources: + +- AWS Instance + +Coder uses `aws_ec2_instance_state` to start and stop the VM. This example template is fully persistent, meaning the full filesystem is preserved when the workspace restarts. See this [community example](https://github.com/bpmct/coder-templates/tree/main/aws-linux-ephemeral) of an ephemeral AWS instance. + +> **Note** +> This template is designed to be a starting point! Edit the Terraform to extend the template to support your use case. + +## code-server + +`code-server` is installed via the `startup_script` argument in the `coder_agent` +resource block. The `coder_app` resource is defined to access `code-server` through +the dashboard UI over `localhost:13337`. \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/aws-windows/main.tf b/infra/1-click/2-coder/templates/aws-windows/main.tf new file mode 100644 index 0000000..d157b24 --- /dev/null +++ b/infra/1-click/2-coder/templates/aws-windows/main.tf @@ -0,0 +1,214 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + aws = { + source = "hashicorp/aws" + } + } +} + +# Last updated 2023-03-14 +# aws ec2 describe-regions | jq -r '[.Regions[].RegionName] | sort' +data "coder_parameter" "region" { + name = "region" + display_name = "Region" + description = "The region to deploy the workspace in." + default = "us-east-1" + mutable = false + option { + name = "Asia Pacific (Tokyo)" + value = "ap-northeast-1" + icon = "/emojis/1f1ef-1f1f5.png" + } + option { + name = "Asia Pacific (Seoul)" + value = "ap-northeast-2" + icon = "/emojis/1f1f0-1f1f7.png" + } + option { + name = "Asia Pacific (Osaka-Local)" + value = "ap-northeast-3" + icon = "/emojis/1f1f0-1f1f7.png" + } + option { + name = "Asia Pacific (Mumbai)" + value = "ap-south-1" + icon = "/emojis/1f1f0-1f1f7.png" + } + option { + name = "Asia Pacific (Singapore)" + value = "ap-southeast-1" + icon = "/emojis/1f1f0-1f1f7.png" + } + option { + name = "Asia Pacific (Sydney)" + value = "ap-southeast-2" + icon = "/emojis/1f1f0-1f1f7.png" + } + option { + name = "Canada (Central)" + value = "ca-central-1" + icon = "/emojis/1f1e8-1f1e6.png" + } + option { + name = "EU (Frankfurt)" + value = "eu-central-1" + icon = "/emojis/1f1ea-1f1fa.png" + } + option { + name = "EU (Stockholm)" + value = "eu-north-1" + icon = "/emojis/1f1ea-1f1fa.png" + } + option { + name = "EU (Ireland)" + value = "eu-west-1" + icon = "/emojis/1f1ea-1f1fa.png" + } + option { + name = "EU (London)" + value = "eu-west-2" + icon = "/emojis/1f1ea-1f1fa.png" + } + option { + name = "EU (Paris)" + value = "eu-west-3" + icon = "/emojis/1f1ea-1f1fa.png" + } + option { + name = "South America (São Paulo)" + value = "sa-east-1" + icon = "/emojis/1f1e7-1f1f7.png" + } + option { + name = "US East (N. Virginia)" + value = "us-east-1" + icon = "/emojis/1f1fa-1f1f8.png" + } + option { + name = "US East (Ohio)" + value = "us-east-2" + icon = "/emojis/1f1fa-1f1f8.png" + } + option { + name = "US West (N. California)" + value = "us-west-1" + icon = "/emojis/1f1fa-1f1f8.png" + } + option { + name = "US West (Oregon)" + value = "us-west-2" + icon = "/emojis/1f1fa-1f1f8.png" + } +} + +data "coder_parameter" "instance_type" { + name = "instance_type" + display_name = "Instance type" + description = "What instance type should your workspace use?" + default = "t3.micro" + mutable = false + option { + name = "2 vCPU, 1 GiB RAM" + value = "t3.micro" + } + option { + name = "2 vCPU, 2 GiB RAM" + value = "t3.small" + } + option { + name = "2 vCPU, 4 GiB RAM" + value = "t3.medium" + } + option { + name = "2 vCPU, 8 GiB RAM" + value = "t3.large" + } + option { + name = "4 vCPU, 16 GiB RAM" + value = "t3.xlarge" + } + option { + name = "8 vCPU, 32 GiB RAM" + value = "t3.2xlarge" + } +} + +provider "aws" { + region = data.coder_parameter.region.value +} + +data "coder_workspace" "me" { +} +data "coder_workspace_owner" "me" {} + +data "aws_ami" "windows" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["Windows_Server-2019-English-Full-Base-*"] + } +} + +resource "coder_agent" "main" { + arch = "amd64" + auth = "aws-instance-identity" + os = "windows" +} + +locals { + + # User data is used to stop/start AWS instances. See: + # https://github.com/hashicorp/terraform-provider-aws/issues/22 + + user_data_start = < +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +${coder_agent.main.init_script} + +true +EOT + + user_data_end = < +shutdown /s + +true +EOT +} + +resource "aws_instance" "dev" { + ami = data.aws_ami.windows.id + availability_zone = "${data.coder_parameter.region.value}a" + instance_type = data.coder_parameter.instance_type.value + + user_data = data.coder_workspace.me.transition == "start" ? local.user_data_start : local.user_data_end + tags = { + Name = "coder-${data.coder_workspace_owner.me.name}-${data.coder_workspace.me.name}" + # Required if you are using our example policy, see template README + Coder_Provisioned = "true" + } + lifecycle { + ignore_changes = [ami] + } +} + +resource "coder_metadata" "workspace_info" { + resource_id = aws_instance.dev.id + item { + key = "region" + value = data.coder_parameter.region.value + } + item { + key = "instance type" + value = aws_instance.dev.instance_type + } + item { + key = "disk" + value = "${aws_instance.dev.root_block_device[0].volume_size} GiB" + } +} \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/kubernetes-claude/boundary-config.yaml b/infra/1-click/2-coder/templates/kubernetes-claude/boundary-config.yaml deleted file mode 100644 index b3cb52c..0000000 --- a/infra/1-click/2-coder/templates/kubernetes-claude/boundary-config.yaml +++ /dev/null @@ -1,243 +0,0 @@ -allowlist: - # Test domains - - method=GET domain=typicode.com - - method=GET domain=*.typicode.com - - # Coder - - domain=${CODER_DOMAIN} - - # Domain used in coder workspaces - - method=POST domain=http-intake.logs.datadoghq.com - - method=POST domain=http-intake.logs.us5.datadoghq.com - - # Default allowed domains from Claude Code on the web - # Source: https://code.claude.com/docs/en/claude-code-on-the-web#default-allowed-domains - # Anthropic Services - - domain=api.anthropic.com - - domain=statsig.anthropic.com - - domain=claude.ai - - # Version Control - - domain=github.com path=/coder-contrib/* - - domain=github.com path=/coder/* - - domain=www.github.com - - domain=api.github.com - - domain=raw.githubusercontent.com - - domain=objects.githubusercontent.com - - domain=codeload.github.com - - domain=avatars.githubusercontent.com - - domain=camo.githubusercontent.com - - domain=gist.github.com - - domain=gitlab.com - - domain=www.gitlab.com - - domain=registry.gitlab.com - - domain=bitbucket.org - - domain=www.bitbucket.org - - domain=api.bitbucket.org - - # Container Registries - - domain=registry-1.docker.io - - domain=auth.docker.io - - domain=index.docker.io - - domain=hub.docker.com - - domain=www.docker.com - - domain=production.cloudflare.docker.com - - domain=download.docker.com - - domain=*.gcr.io - - domain=ghcr.io - - domain=mcr.microsoft.com - - domain=*.data.mcr.microsoft.com - - # Cloud Platforms - - domain=cloud.google.com - - domain=accounts.google.com - - domain=gcloud.google.com - - domain=*.googleapis.com - - domain=storage.googleapis.com - - domain=compute.googleapis.com - - domain=container.googleapis.com - - domain=azure.com - - domain=portal.azure.com - - domain=microsoft.com - - domain=www.microsoft.com - - domain=*.microsoftonline.com - - domain=packages.microsoft.com - - domain=dotnet.microsoft.com - - domain=dot.net - - domain=visualstudio.com - - domain=dev.azure.com - - domain=oracle.com - - domain=www.oracle.com - - domain=java.com - - domain=www.java.com - - domain=java.net - - domain=www.java.net - - domain=download.oracle.com - - domain=yum.oracle.com - - # Package Managers - JavaScript/Node - - domain=registry.npmjs.org - - domain=www.npmjs.com - - domain=www.npmjs.org - - domain=npmjs.com - - domain=npmjs.org - - domain=yarnpkg.com - - domain=registry.yarnpkg.com - - # Package Managers - Python - - domain=pypi.org - - domain=www.pypi.org - - domain=files.pythonhosted.org - - domain=pythonhosted.org - - domain=test.pypi.org - - domain=pypi.python.org - - domain=pypa.io - - domain=www.pypa.io - - # Package Managers - Ruby - - domain=rubygems.org - - domain=www.rubygems.org - - domain=api.rubygems.org - - domain=index.rubygems.org - - domain=ruby-lang.org - - domain=www.ruby-lang.org - - domain=rubyforge.org - - domain=www.rubyforge.org - - domain=rubyonrails.org - - domain=www.rubyonrails.org - - domain=rvm.io - - domain=get.rvm.io - - # Package Managers - Rust - - domain=crates.io - - domain=www.crates.io - - domain=static.crates.io - - domain=rustup.rs - - domain=static.rust-lang.org - - domain=www.rust-lang.org - - # Package Managers - Go - - domain=proxy.golang.org - - domain=sum.golang.org - - domain=index.golang.org - - domain=golang.org - - domain=www.golang.org - - domain=go.dev - - domain=dl.google.com - - domain=goproxy.io - - domain=pkg.go.dev - - # Go Module Domains (from go.mod) - - domain=cdr.dev - - domain=cel.dev - - domain=dario.cat - - domain=git.sr.ht - - domain=go.mozilla.org - - domain=go.nhat.io - - domain=go.opentelemetry.io - - domain=go.uber.org - - domain=go.yaml.in - - domain=go4.org - - domain=golang.zx2c4.com - - domain=gonum.org - - domain=gopkg.in - - domain=gvisor.dev - - domain=howett.net - - domain=kernel.org - - domain=mvdan.cc - - domain=sigs.k8s.io - - domain=storj.io - - # Package Managers - JVM - - domain=maven.org - - domain=repo.maven.org - - domain=central.maven.org - - domain=repo1.maven.org - - domain=jcenter.bintray.com - - domain=gradle.org - - domain=www.gradle.org - - domain=services.gradle.org - - domain=spring.io - - domain=repo.spring.io - - # Package Managers - Other Languages - - domain=packagist.org - - domain=www.packagist.org - - domain=repo.packagist.org - - domain=nuget.org - - domain=www.nuget.org - - domain=api.nuget.org - - domain=pub.dev - - domain=api.pub.dev - - domain=hex.pm - - domain=www.hex.pm - - domain=cpan.org - - domain=www.cpan.org - - domain=metacpan.org - - domain=www.metacpan.org - - domain=api.metacpan.org - - domain=cocoapods.org - - domain=www.cocoapods.org - - domain=cdn.cocoapods.org - - domain=haskell.org - - domain=www.haskell.org - - domain=hackage.haskell.org - - domain=swift.org - - domain=www.swift.org - - # Linux Distributions - - domain=archive.ubuntu.com - - domain=security.ubuntu.com - - domain=ubuntu.com - - domain=www.ubuntu.com - - domain=*.ubuntu.com - - domain=ppa.launchpad.net - - domain=launchpad.net - - domain=www.launchpad.net - - # Development Tools & Platforms - - domain=dl.k8s.io - - domain=pkgs.k8s.io - - domain=k8s.io - - domain=www.k8s.io - - domain=releases.hashicorp.com - - domain=apt.releases.hashicorp.com - - domain=rpm.releases.hashicorp.com - - domain=archive.releases.hashicorp.com - - domain=hashicorp.com - - domain=www.hashicorp.com - - domain=repo.anaconda.com - - domain=conda.anaconda.org - - domain=anaconda.org - - domain=www.anaconda.com - - domain=anaconda.com - - domain=continuum.io - - domain=apache.org - - domain=www.apache.org - - domain=archive.apache.org - - domain=downloads.apache.org - - domain=eclipse.org - - domain=www.eclipse.org - - domain=download.eclipse.org - - domain=nodejs.org - - domain=www.nodejs.org - - # Cloud Services & Monitoring - - domain=statsig.com - - domain=www.statsig.com - - domain=api.statsig.com - - domain=*.sentry.io - - # Content Delivery & Mirrors - - domain=*.sourceforge.net - - domain=packagecloud.io - - domain=*.packagecloud.io - - # Schema & Configuration - - domain=json-schema.org - - domain=www.json-schema.org - - domain=json.schemastore.org - - domain=www.schemastore.org -log_dir: /tmp/boundary_logs -log_level: warn -proxy_port: 8087 \ No newline at end of file diff --git a/infra/1-click/2-coder/templates/kubernetes-claude/main.tf b/infra/1-click/2-coder/templates/kubernetes-claude/main.tf index ae2e6e6..28817a2 100644 --- a/infra/1-click/2-coder/templates/kubernetes-claude/main.tf +++ b/infra/1-click/2-coder/templates/kubernetes-claude/main.tf @@ -175,31 +175,18 @@ data "coder_task" "me" {} locals { coder_host = replace(replace(data.coder_workspace.me.access_url, "https://", ""), "http://", "") - boundary_config = templatefile("${path.module}/boundary-config.yaml", { - CODER_DOMAIN = local.coder_host - }) } module "claude-code" { source = "registry.coder.com/coder/claude-code/coder" - version = "4.5.0" + version = "4.5.0" agent_id = coder_agent.main.id workdir = "/home/coder" subdomain = true ai_prompt = data.coder_task.me.prompt enable_aibridge = true - enable_boundary = true - boundary_version = "v0.6.0" - - post_install_script = <<-EOF - #!/usr/bin/env bash - - # Coder Boundary Setup - - mkdir -p ~/.config/coder_boundary - echo '${base64encode(local.boundary_config)}' | base64 -d > ~/.config/coder_boundary/config.yaml - chmod 600 ~/.config/coder_boundary/config.yaml - EOF + # Does not work on EKS Auto-Mode + enable_boundary = false } resource "coder_ai_task" "task" { @@ -252,27 +239,17 @@ data "coder_workspace_preset" "standard" { icon = "/icon/standard.svg" default = true parameters = { - (data.coder_parameter.cpu.name) = "2" - (data.coder_parameter.memory.name) = "2" - (data.coder_parameter.home_disk_size.name) = "10" + (data.coder_parameter.cpu.name) = "2" + (data.coder_parameter.memory.name) = "2" + (data.coder_parameter.home_disk_size.name) = "10" } prebuilds { instances = 0 } } -locals { - coder_bin = "/opt/coder/bin" - init_script = <<-EOF - if [ -x ${local.coder_bin}/coder ]; then - exec ${local.coder_bin}/coder agent ; - fi - ${coder_agent.main.init_script} - EOF -} - resource "kubernetes_deployment_v1" "main" { - count = data.coder_workspace.me.start_count + count = data.coder_workspace.me.start_count wait_for_rollout = false metadata { name = "coder-${data.coder_workspace.me.id}" @@ -285,10 +262,10 @@ resource "kubernetes_deployment_v1" "main" { "com.coder.workspace.id" = data.coder_workspace.me.id } annotations = { - "com.coder.user.email" = data.coder_workspace_owner.me.email - "com.coder.workspace.name" = data.coder_workspace.me.name - "com.coder.user.id" = data.coder_workspace_owner.me.id - "com.coder.user.username" = data.coder_workspace_owner.me.name + "com.coder.user.email" = data.coder_workspace_owner.me.email + "com.coder.workspace.name" = data.coder_workspace.me.name + "com.coder.user.id" = data.coder_workspace_owner.me.id + "com.coder.user.username" = data.coder_workspace_owner.me.name } } @@ -331,7 +308,7 @@ resource "kubernetes_deployment_v1" "main" { for_each = var.host_ip != "" ? [1] : [] content { hostnames = [local.coder_host] - ip = var.host_ip + ip = var.host_ip } } @@ -339,18 +316,16 @@ resource "kubernetes_deployment_v1" "main" { name = "dev" image = "codercom/enterprise-base:ubuntu" image_pull_policy = "IfNotPresent" - command = ["sh", "-c", local.init_script] + command = ["sh", "-c", coder_agent.main.init_script] security_context { run_as_user = "1000" - allow_privilege_escalation = true - privileged = true } env { - name = "CODER_AGENT_AUTH" + name = "CODER_AGENT_AUTH" value = "token" } env { - name = "CODER_AGENT_URL" + name = "CODER_AGENT_URL" value = data.coder_workspace.me.access_url } env { @@ -373,12 +348,6 @@ resource "kubernetes_deployment_v1" "main" { name = "home" read_only = false } - - volume_mount { - name = "coder-bin" - mount_path = local.coder_bin - read_only = true - } } volume { @@ -389,14 +358,6 @@ resource "kubernetes_deployment_v1" "main" { } } - volume { - name = "coder-bin" - host_path { - path = local.coder_bin - type = "Directory" - } - } - affinity { // This affinity attempts to spread out all workspace pods evenly across // nodes. diff --git a/infra/1-click/2-coder/templates/kubernetes/main.tf b/infra/1-click/2-coder/templates/kubernetes/main.tf index 487a94e..e63cd73 100644 --- a/infra/1-click/2-coder/templates/kubernetes/main.tf +++ b/infra/1-click/2-coder/templates/kubernetes/main.tf @@ -226,18 +226,8 @@ resource "kubernetes_persistent_volume_claim_v1" "home" { } } -locals { - coder_bin = "/opt/coder/bin" - init_script = <<-EOF - if [ -x ${local.coder_bin}/coder ]; then - exec ${local.coder_bin}/coder agent ; - fi - ${coder_agent.main.init_script} - EOF -} - resource "kubernetes_deployment_v1" "main" { - count = data.coder_workspace.me.start_count + count = data.coder_workspace.me.start_count wait_for_rollout = false metadata { name = "coder-${data.coder_workspace.me.id}" @@ -299,7 +289,7 @@ resource "kubernetes_deployment_v1" "main" { for_each = var.host_ip != "" ? [1] : [] content { hostnames = [replace(replace(data.coder_workspace.me.access_url, "https://", ""), "http://", "")] - ip = var.host_ip + ip = var.host_ip } } @@ -307,16 +297,16 @@ resource "kubernetes_deployment_v1" "main" { name = "dev" image = "codercom/enterprise-base:ubuntu" image_pull_policy = "IfNotPresent" - command = ["sh", "-c", local.init_script] + command = ["sh", "-c", coder_agent.main.init_script] security_context { run_as_user = "1000" } env { - name = "CODER_AGENT_AUTH" + name = "CODER_AGENT_AUTH" value = "token" } env { - name = "CODER_AGENT_URL" + name = "CODER_AGENT_URL" value = data.coder_workspace.me.access_url } env { @@ -340,12 +330,6 @@ resource "kubernetes_deployment_v1" "main" { read_only = false } - volume_mount { - name = "coder-bin" - mount_path = local.coder_bin - read_only = true - } - } volume { @@ -356,14 +340,6 @@ resource "kubernetes_deployment_v1" "main" { } } - volume { - name = "coder-bin" - host_path { - path = local.coder_bin - type = "Directory" - } - } - affinity { // This affinity attempts to spread out all workspace pods evenly across // nodes. diff --git a/infra/1-click/2-coder/terragrunt.hcl b/infra/1-click/2-coder/terragrunt.hcl new file mode 100644 index 0000000..0f8e0da --- /dev/null +++ b/infra/1-click/2-coder/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = ["../0-infra", "../1-setup"] +} + +inputs = { + # Optional. Set if using Terragrunt. + profile = include.root.locals.CODER_AWS_PROFILE + region = include.root.locals.CODER_AWS_REGION + azs = jsondecode(include.root.locals.CODER_AWS_AZS) + domain_name = include.root.locals.CODER_DOMAIN_NAME + coder_version = include.root.locals.CODER_VERSION + coder_license = include.root.locals.CODER_LICENSE + coder_admin_email = include.root.locals.CODER_EMAIL + coder_admin_password = include.root.locals.CODER_PASSWORD +} \ No newline at end of file diff --git a/infra/1-click/2-coder/variables.tf b/infra/1-click/2-coder/variables.tf new file mode 100644 index 0000000..165407d --- /dev/null +++ b/infra/1-click/2-coder/variables.tf @@ -0,0 +1,47 @@ +variable "region" { + description = "The AWS region of the deployment." + type = string + default = "us-east-2" +} + +variable "azs" { + type = list(string) + default = ["a", "b", "c"] +} + +variable "name" { + description = "Name for created resources and tag prefix." + type = string + default = "coder" +} + +variable "profile" { + type = string + default = "default" +} + +variable "domain_name" { + type = string +} + +variable "coder_admin_email" { + type = string + default = "admin@coder.com" +} + +variable "coder_admin_password" { + type = string + default = "Th1s1sN0TS3CuR3!!" + sensitive = true +} + +variable "coder_license" { + type = string + default = "" + sensitive = true +} + +variable "coder_version" { + type = string + default = "2.30.0" +} \ No newline at end of file diff --git a/infra/1-click/root.hcl b/infra/1-click/root.hcl new file mode 100644 index 0000000..4d1168f --- /dev/null +++ b/infra/1-click/root.hcl @@ -0,0 +1,99 @@ +locals { + CODER_TF_BACKEND_AWS_BUCKET_NAME = get_env("CODER_TF_BACKEND_AWS_BUCKET_NAME") + CODER_TF_BACKEND_AWS_REGION = get_env("CODER_TF_BACKEND_AWS_REGION", "us-east-2") + CODER_TF_BACKEND_AWS_PROFILE = get_env("CODER_TF_BACKEND_AWS_PROFILE", "default") + CODER_TF_BACKEND_ENCRYPT = get_env("CODER_TF_BACKEND_ENCRYPT", "false") + CODER_TF_USE_REMOTE_STATE = get_env("CODER_TF_USE_REMOTE_STATE", "false") == "true" + + CODER_AWS_PROFILE = get_env("CODER_AWS_PROFILE", "default") + CODER_AWS_REGION = get_env("CODER_AWS_REGION", "us-east-2") + CODER_AWS_AZS = get_env("CODER_AWS_AZS", jsonencode(["a", "c"])) + + CODER_DB_USERNAME = get_env("CODER_DB_USERNAME", "coder") + CODER_DB_PASSWORD = get_env("CODER_DB_PASSWORD", "th1s1sn0tas3cur3pass0wrd") + GRAFANA_DB_USERNAME = get_env("GRAFANA_DB_USERNAME", "grafana") + GRAFANA_DB_PASSWORD = get_env("GRAFANA_DB_PASSWORD", "th1s1sn0tas3cur3pass0wrd") + + CODER_DOMAIN_NAME = get_env("CODER_DOMAIN_NAME") + CODER_LICENSE = get_env("CODER_LICENSE", "") + CODER_VERSION = get_env("CODER_VERSION", "2.30.0") + + CODER_USERNAME = get_env("CODER_USERNAME", "admin") + CODER_EMAIL = get_env("CODER_EMAIL", "admin@coder.com") + CODER_PASSWORD = get_env("CODER_PASSWORD", "Th1s1sN0TS3CuR3!!") + + GRAFANA_USERNAME = get_env("GRAFANA_ADMIN_USERNAME", "admin") + GRAFANA_PASSWORD = get_env("GRAFANA_ADMIN_PASSWORD", "Th1s1sN0TS3CuR3!!") +} + +generate "backend" { + path = "backend.tf" + if_exists = "overwrite" + disable = !local.CODER_TF_USE_REMOTE_STATE + if_disabled = "remove_terragrunt" + + contents = <<-EOF + terraform { + backend "s3" { + bucket = "${local.CODER_TF_BACKEND_AWS_BUCKET_NAME}" + key = "${path_relative_to_include()}/terraform.tfstate" + region = "${local.CODER_TF_BACKEND_AWS_REGION}" + profile = "${local.CODER_TF_BACKEND_AWS_PROFILE}" + encrypt = "${local.CODER_TF_BACKEND_ENCRYPT}" + } + } + EOF +} + +generate "provider" { + path = "provider.tf" + if_exists = "overwrite" + + contents = <<-EOF + terraform { + required_version = ">= 1.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.32.1" + } + helm = { + source = "hashicorp/helm" + version = "~> 3.1.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 3.0.1" + } + external = { + source = "hashicorp/external" + version = "~> 2.3.5" + } + time = { + source = "hashicorp/time" + version = "~> 0.13.1" + } + http = { + source = "hashicorp/http" + version = "~> 3.5.0" + } + dns = { + source = "hashicorp/dns" + version = "~> 3.5.0" + } + archive = { + source = "hashicorp/archive" + version = "~> 2.7.1" + } + random = { + source = "hashicorp/random" + version = "~> 3.8.1" + } + coderd = { + source = "coder/coderd" + version = "~> 0.0.12" + } + } + } + EOF +} \ No newline at end of file diff --git a/modules/coder/provisioner/main.tf b/modules/coder/provisioner/main.tf index 1381722..0287f7e 100644 --- a/modules/coder/provisioner/main.tf +++ b/modules/coder/provisioner/main.tf @@ -32,16 +32,24 @@ variable "provisioner_tags" { # Resources ## -resource "random_id" "provisioner_key_name" { +resource "random_string" "provisioner_key_name" { keepers = { # Generate a new ID only when a key is defined provisioner_key_name = "${var.provisioner_key_name}" } - byte_length = 8 + length = 8 + special = true + numeric = true + lower = true + override_special = "-" +} + +locals { + provisioner_key_name = var.provisioner_key_name == "" ? lower(random_string.provisioner_key_name.result) : var.provisioner_key_name } resource "coderd_provisioner_key" "key" { - name = var.provisioner_key_name == "" ? random_id.provisioner_key_name.id : var.provisioner_key_name + name = local.provisioner_key_name organization_id = var.organization_id tags = var.provisioner_tags } @@ -52,7 +60,7 @@ resource "coderd_provisioner_key" "key" { output "provisioner_key_name" { description = "Coder Provisioner Key Name" - value = var.provisioner_key_name == "" ? random_id.provisioner_key_name.id : var.provisioner_key_name + value = local.provisioner_key_name } output "provisioner_key_secret" { diff --git a/modules/coder/template/main.tf b/modules/coder/template/main.tf new file mode 100644 index 0000000..c629570 --- /dev/null +++ b/modules/coder/template/main.tf @@ -0,0 +1,74 @@ +terraform { + required_version = ">= 1.0" + required_providers { + coderd = { + source = "coder/coderd" + version = "0.0.12" + } + archive = { + source = "hashicorp/archive" + } + time = { + source = "hashicorp/time" + } + } +} + +variable "template_config" { + type = object({ + name = string + display_name = string + dir = string + description = string + icon = string + org_id = string + tf_vars = optional(list(object({ + name = string + value = string + })), []) + }) +} + +variable "archive_config" { + type = object({ + type = optional(string, "zip") + excludes = optional(list(string), []) + output_path = string + }) +} + +variable "time_static_triggers" { + type = map(string) + default = {} +} + +data "archive_file" "this" { + type = var.archive_config.type + excludes = var.archive_config.excludes + source_dir = var.template_config.dir + output_path = var.archive_config.output_path +} + +resource "time_static" "this" { + triggers = merge({ + run_on_checksum = "${data.archive_file.this.id}" + run_on_tf_vars = jsonencode(var.template_config.tf_vars) + }, var.time_static_triggers) +} + +resource "coderd_template" "this" { + name = var.template_config.name + organization_id = var.template_config.org_id + display_name = var.template_config.display_name + description = var.template_config.description + icon = var.template_config.icon + versions = [ + { + name = "stable-${formatdate("YYYY-MM-DD_hh-mm-ss", time_static.this.rfc3339)}" + description = "The stable version of the template." + directory = var.template_config.dir + active = true + tf_vars = toset(var.template_config.tf_vars) + } + ] +} \ No newline at end of file diff --git a/modules/k8s/bootstrap/coder-logstream/main.tf b/modules/k8s/bootstrap/coder-logstream/main.tf new file mode 100644 index 0000000..8fc7fd9 --- /dev/null +++ b/modules/k8s/bootstrap/coder-logstream/main.tf @@ -0,0 +1,104 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.17.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} + +variable "release_name" { + description = "The release name of the installed Helm app." + type = string + default = "coder-logstream-kube" +} + +variable "chart_name" { + description = "The chart name of the installed Helm app." + type = string + default = "coder-logstream-kube" +} + +variable "helm_timeout" { + type = number + default = 300 # In Seconds +} + +variable "chart_version" { + type = string + default = "v0.0.14" +} + +variable "namespace" { + type = string + default = "coder-logstream-kube" +} + +variable "image_repo" { + type = string + default = "ghcr.io/coder/coder-logstream-kube" +} + +variable "image_tag" { + type = string + default = "v0.0.14" +} + +variable "image_pull_policy" { + type = string + default = "IfNotPresent" +} + +variable "coder" { + type = object({ + access_url = string + ws_ns = optional(list(string), []) + }) +} + +resource "kubernetes_namespace_v1" "logstream" { + metadata { + name = var.namespace + } +} + +resource "helm_release" "coder-logstream" { + + name = var.release_name + namespace = kubernetes_namespace_v1.logstream.metadata[0].name + chart = var.chart_name + repository = "https://helm.coder.com/logstream-kube" + create_namespace = false + upgrade_install = true + skip_crds = false + wait = true + wait_for_jobs = true + version = var.chart_version + timeout = var.helm_timeout + + values = [yamlencode({ + url = var.coder.access_url + namespaces = var.coder.ws_ns + image = { + repo = var.image_repo + tag = var.image_tag + pullPolicy = var.image_pull_policy + } + # nodeSelector = {} + # affinity = {} + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] + })] +} + +output "namespace" { + value = kubernetes_namespace_v1.logstream.metadata[0].name +} \ No newline at end of file diff --git a/modules/k8s/bootstrap/coder-provisioner/main.tf b/modules/k8s/bootstrap/coder-provisioner/main.tf index 699d5b9..4a2ec51 100644 --- a/modules/k8s/bootstrap/coder-provisioner/main.tf +++ b/modules/k8s/bootstrap/coder-provisioner/main.tf @@ -16,110 +16,77 @@ terraform { } } -variable "cluster_name" { - type = string -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "role_name" { - type = string - default = "" -} - -variable "policy_name" { - type = string - default = "" -} - -variable "policy_resource_region" { - type = string - default = "" -} - -variable "policy_resource_account" { - type = string - default = "" -} - -variable "tags" { - type = map(string) - default = {} -} - -variable "namespace" { - type = string -} - -variable "image_repo" { - type = string -} - -variable "image_tag" { +variable "chart_name" { type = string + default = "coder-provisioner" } -variable "image_pull_policy" { +variable "chart_version" { type = string - default = "IfNotPresent" -} - -variable "image_pull_secrets" { - type = list(string) - default = [] + default = "2.30.0" } -variable "provisioner_chart_version" { - type = string - default = "2.23.0" +variable "chart_timeout" { + type = number + default = 300 } -variable "logstream_chart_version" { +variable "release_name" { type = string - default = "0.0.11" + default = "coder-provisioner" } -variable "primary_access_url" { +variable "cluster_name" { type = string } -variable "ws_service_account_name" { +variable "cluster_oidc_provider_arn" { type = string } -variable "ws_service_account_labels" { - type = map(string) - default = {} -} - -variable "ws_service_account_annotations" { +variable "tags" { type = map(string) default = {} } -variable "provisioner_service_account_name" { - type = string - default = "coder" -} - -variable "provisioner_service_account_annotations" { - type = map(string) - default = {} +variable "coder" { + type = object({ + access_url = string + prov_tags = optional(map(string), {}) + prov_secret_key = optional(string, "key") + org_name = optional(string, "coder") + ws_ns = optional(list(string), []) + image_repo = optional(string, "ghcr.io/coder/coder") + image_tag = optional(string, "latest") + image_pull_policy = optional(string, "IfNotPresent") + image_pull_secrets = optional(list(string), null) + env_vars = optional(map(string), {}) + rep_cnt = optional(number, 1) + tf_debug_mode = optional(bool, true) + trace_logs = optional(bool, true) + }) + # sensitive = true } -variable "replica_count" { - type = number - default = 0 +variable "namespace" { + type = string + default = "coder-provisioner" } -variable "env_vars" { - type = map(string) - default = {} +variable "svc_acc" { + type = object({ + create = optional(bool, true) + name = optional(string, "coder-provisioner") + annots = optional(map(string), {}) + }) + default = { + create = true + name = "coder-provisioner" + annots = {} + } } -variable "resource_requests" { +variable "rsrc_req" { type = object({ cpu = string memory = string @@ -130,7 +97,7 @@ variable "resource_requests" { } } -variable "resource_limits" { +variable "rsrc_lim" { type = object({ cpu = string memory = string @@ -156,7 +123,7 @@ variable "tolerations" { default = [] } -variable "topology_spread_constraints" { +variable "topology_spread" { type = list(object({ max_skew = number topology_key = string @@ -169,7 +136,7 @@ variable "topology_spread_constraints" { default = [] } -variable "pod_anti_affinity_preferred_during_scheduling_ignored_during_execution" { +variable "pod_aaf_pref_sched_ie" { type = list(object({ weight = number pod_affinity_term = object({ @@ -182,68 +149,39 @@ variable "pod_anti_affinity_preferred_during_scheduling_ignored_during_execution default = [] } -variable "provisioner_secret" { - type = object({ - key_secret_name = optional(string, "psk") - key_secret_key = optional(string, "token.key") - termination_grace_period_seconds = optional(number, 600) - }) - default = { - key_secret_name = "psk" - key_secret_key = "token.key" - termination_grace_period_seconds = 600 - } -} - -variable "coder_organization_name" { - type = string - default = "coder" -} - -variable "coder_provisioner_key_name" { - type = string - default = "" -} - -variable "coder_provisioner_tags" { - type = map(string) - default = null -} - ## # Coder External Provisioner ## data "aws_region" "this" {} - data "aws_caller_identity" "this" {} data "coderd_organization" "this" { - name = var.coder_organization_name + name = var.coder.org_name } locals { - region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region - account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == "" ? "Provisioner-${data.aws_region.this.region}" : var.policy_name - role_name = var.role_name == "" ? "provisioner-${data.aws_region.this.region}" : var.role_name + region = data.aws_region.this.region + account_id = data.aws_caller_identity.this.account_id + policy_name = "Provisioner-${data.aws_region.this.region}" + role_name = "provisioner-${data.aws_region.this.region}" } -module "provisioner-policy" { +module "iam-policy" { source = "../../../security/policy" name = local.policy_name path = "/" - description = "Coder Terraform External Provisioner Policy" - policy_json = data.aws_iam_policy_document.provisioner-policy.json + description = "Coder External Provisioner Policy" + policy_json = data.aws_iam_policy_document.ext-prov.json } -module "provisioner-oidc-role" { +module "oidc-role" { source = "../../../security/role/access-entry" name = local.role_name cluster_name = var.cluster_name policy_arns = { "AmazonEC2ReadOnlyAccess" = "arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess" - "TFProvisionerPolicy" = module.provisioner-policy.policy_arn + "TFProvisionerPolicy" = module.iam-policy.policy_arn } cluster_policy_arns = {} oidc_principals = { @@ -252,83 +190,97 @@ module "provisioner-oidc-role" { tags = var.tags } -locals { - primary_env_vars = { - CODER_URL = var.primary_access_url +module "ext-prov" { + source = "../../../coder/provisioner" + organization_id = data.coderd_organization.this.id + provisioner_tags = var.coder.prov_tags +} + +resource "kubernetes_namespace_v1" "this" { + metadata { + name = var.namespace + } +} + +resource "kubernetes_secret_v1" "ext-prov" { + metadata { + name = module.ext-prov.provisioner_key_name + namespace = kubernetes_namespace_v1.this.metadata[0].name + annotations = { + "custom.kubernetes.secret/key" = var.coder.prov_secret_key + } + } + type = "Opaque" + data = { + "${var.coder.prov_secret_key}" = module.ext-prov.provisioner_key_secret } - env_vars = [ - for k, v in merge(local.primary_env_vars, var.env_vars) : { name = k, value = v } +} + +locals { + pod_aaf_pref_sched_ie = [ + for k, v in var.pod_aaf_pref_sched_ie : { + weight = v.weight + podAffinityTerm = { + labelSelector = { + matchLabels = try(v.pod_affinity_term.label_selector.match_labels, {}) + } + topologyKey = try(v.pod_affinity_term.topology_key, {}) + } + } ] - provisioner_tags = var.coder_provisioner_tags == null ? { - region = data.aws_region.this.region - } : var.coder_provisioner_tags - topology_spread_constraints = [ - for v in var.topology_spread_constraints : { + topology_spread = [ + for k, v in var.topology_spread : { maxSkew = v.max_skew topologyKey = v.topology_key whenUnsatisfiable = v.when_unsatisfiable - labelSelector = v.label_selector - matchLabelKeys = v.match_label_keys - } - ] - pod_anti_affinity_preferred_during_scheduling_ignored_during_execution = [ - for v in var.pod_anti_affinity_preferred_during_scheduling_ignored_during_execution : { - weight = v.weight - podAffinityTerm = { - labelSelector = v.pod_affinity_term.label_selector - topologyKey = v.pod_affinity_term.topology_key + labelSelector = { + matchLabels = try(v.label_selector.match_labels, {}) } + matchLabelKeys = v.match_label_keys } ] } -module "coder-provisioner-key" { - source = "../../../coder/provisioner" - organization_id = data.coderd_organization.this.id - provisioner_tags = local.provisioner_tags -} - -resource "kubernetes_namespace" "this" { - metadata { - name = var.namespace - } -} - resource "helm_release" "coder-provisioner" { - name = "coder-provisioner" - namespace = kubernetes_namespace.this.metadata[0].name - chart = "coder-provisioner" + name = var.release_name + namespace = kubernetes_namespace_v1.this.metadata[0].name + chart = var.chart_name repository = "https://helm.coder.com/v2" create_namespace = false upgrade_install = true skip_crds = false wait = true wait_for_jobs = true - version = var.provisioner_chart_version - timeout = 120 # in seconds + version = var.chart_version + timeout = var.chart_timeout values = [yamlencode({ coder = { image = { - repo = var.image_repo - tag = var.image_tag - pullPolicy = var.image_pull_policy - pullSecrets = var.image_pull_secrets + repo = var.coder.image_repo + tag = var.coder.image_tag + pullPolicy = var.coder.image_pull_policy + pullSecrets = var.coder.image_pull_secrets } serviceAccount = { workspacePerms = true enableDeployments = true - name = var.provisioner_service_account_name - disableCreate = false + name = var.svc_acc.name + disableCreate = !var.svc_acc.create + workspaceNamespaces = [ for v in var.coder.ws_ns : { name = v } ] annotations = merge({ - "eks.amazonaws.com/role-arn" = module.provisioner-oidc-role.role_arn - }, var.provisioner_service_account_annotations) + "eks.amazonaws.com/role-arn" = module.oidc-role.role_arn + }, var.svc_acc.annots) } podAnnotations = { "prometheus.io/scrape" = "true" "prometheus.io/port" = "2112" } - env = local.env_vars + env = [ + for k, v in merge({ + CODER_URL = var.coder.access_url + }, var.coder.env_vars) : { name = k, value = v } + ] securityContext = { runAsNonRoot = true runAsUser = 1000 @@ -340,108 +292,27 @@ resource "helm_release" "coder-provisioner" { allowPrivilegeEscalation = false } resources = { - requests = var.resource_requests - limits = var.resource_limits + requests = var.rsrc_req + limits = var.rsrc_lim } nodeSelector = var.node_selector - replicaCount = var.replica_count + replicaCount = var.coder.rep_cnt tolerations = var.tolerations - topologySpreadConstraints = local.topology_spread_constraints + topologySpreadConstraints = local.topology_spread affinity = { podAntiAffinity = { - preferredDuringSchedulingIgnoredDuringExecution = local.pod_anti_affinity_preferred_during_scheduling_ignored_during_execution + preferredDuringSchedulingIgnoredDuringExecution = local.pod_aaf_pref_sched_ie } } } provisionerDaemon = { - keySecretKey = var.provisioner_secret.key_secret_key - keySecretName = var.provisioner_secret.key_secret_name - terminationGracePeriodSeconds = var.provisioner_secret.termination_grace_period_seconds + keySecretKey = kubernetes_secret_v1.ext-prov.metadata[0].annotations["custom.kubernetes.secret/key"] + keySecretName = kubernetes_secret_v1.ext-prov.metadata[0].name + terminationGracePeriodSeconds = 600 } })] } -resource "kubernetes_secret" "coder-provisioner-key" { - metadata { - name = var.provisioner_secret.key_secret_name - namespace = kubernetes_namespace.this.metadata[0].name - } - data = { - "${var.provisioner_secret.key_secret_key}" = module.coder-provisioner-key.provisioner_key_secret - } - type = "Opaque" -} - -resource "helm_release" "coder-logstream" { - name = "coder-logstream-kube" - namespace = kubernetes_namespace.this.metadata[0].name - chart = "coder-logstream-kube" - repository = "https://helm.coder.com/logstream-kube" - create_namespace = false - upgrade_install = true - skip_crds = false - wait = true - wait_for_jobs = true - version = var.logstream_chart_version - timeout = 120 # in seconds - - values = [yamlencode({ - url = var.primary_access_url - })] -} - -module "ws-policy" { - source = "../../../security/policy" - name = local.policy_name - path = "/" - description = "Coder Workspace IAM Policy" - policy_json = data.aws_iam_policy_document.ws-policy.json -} - -module "ws-service-role" { - source = "../../../security/role/service" - name = "ws-service-role" - policy_arns = { - "AmazonEC2ContainerServiceforEC2Role" = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role", - "AmazonSSMManagedInstanceCore" = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore", - "WorkspacePolicy" = module.ws-policy.policy_arn - } - service_actions = ["sts:AssumeRole"] - service_principals = [ - "ec2.amazonaws.com" - ] - tags = var.tags -} - -module "ws-oidc-role" { - source = "../../../security/role/access-entry" - name = "ws-container-role" - policy_arns = { - "WorkspacePolicy" = module.ws-policy.policy_arn - } - cluster_name = var.cluster_name - cluster_policy_arns = {} - oidc_principals = { - "${var.cluster_oidc_provider_arn}" = ["system:serviceaccount:*:*"] - } - tags = var.tags -} - -resource "kubernetes_service_account" "ws" { - metadata { - name = var.ws_service_account_name - namespace = kubernetes_namespace.this.metadata[0].name - labels = var.ws_service_account_labels - annotations = merge({ - "eks.amazonaws.com/role-arn" = module.ws-oidc-role.role_arn - }, var.ws_service_account_annotations) - } -} - -output "oidc_role_arn" { - value = module.provisioner-oidc-role.role_arn -} - output "coderd_organization_id" { depends_on = [ helm_release.coder-provisioner ] @@ -453,12 +324,5 @@ output "k8s_namespace" { depends_on = [ helm_release.coder-provisioner ] - value = kubernetes_namespace.this.metadata[0].name -} - -output "k8s_ws_service_account_name" { - - depends_on = [ helm_release.coder-provisioner ] - - value = kubernetes_service_account.ws.metadata[0].name + value = kubernetes_namespace_v1.this.metadata[0].name } \ No newline at end of file diff --git a/modules/k8s/bootstrap/coder-provisioner/policy.tf b/modules/k8s/bootstrap/coder-provisioner/policy.tf index 1457d62..75b8155 100644 --- a/modules/k8s/bootstrap/coder-provisioner/policy.tf +++ b/modules/k8s/bootstrap/coder-provisioner/policy.tf @@ -1,177 +1,38 @@ -data "aws_iam_policy_document" "provisioner-policy" { +data "aws_iam_policy_document" "ext-prov" { statement { - sid = "EC2InstanceLifecycle" effect = "Allow" actions = [ - "ec2:RunInstances", - "ec2:StartInstances", - "ec2:StopInstances", - "ec2:TerminateInstances", + "ec2:GetDefaultCreditSpecification", + "ec2:DescribeIamInstanceProfileAssociations", + "ec2:DescribeTags", "ec2:DescribeInstances", - "ec2:RebootInstances", - "ec2:ModifyInstanceAttribute", - "ec2:DescribeInstanceAttribute", - "ec2:MonitorInstances" - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - "arn:aws:ec2:${local.region}::image/*" - ] - } - - statement { - sid = "EC2ManageHostLifecycle" - effect = "Allow" - actions = [ - "ec2:AllocateHosts", - "ec2:ModifyHosts", - "ec2:ReleaseHosts" - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - "arn:aws:ec2:${local.region}::image/*" - ] - } - - statement { - sid = "EBSVolumeLifecycle" - effect = "Allow" - actions = [ - "ec2:CreateVolume", - "ec2:AttachVolume", - "ec2:DetachVolume", - "ec2:DeleteVolume", - "ec2:DescribeVolumes", - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - ] - } - - statement { - sid = "SecurityGroupLifecycle" - effect = "Allow" - actions = [ - "ec2:CreateSecurityGroup", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:AuthorizeSecurityGroupEgress", - "ec2:RevokeSecurityGroupIngress", - "ec2:RevokeSecurityGroupEgress", - "ec2:DeleteSecurityGroup", - "ec2:DescribeSecurityGroups", - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - ] - } - - statement { - sid = "TagLifecycle" - effect = "Allow" - actions = [ + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceStatus", "ec2:CreateTags", - "ec2:DeleteTags", - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - ] - } - - statement { - sid = "NetworkInterfaceLifecycle" - effect = "Allow" - actions = [ - "ec2:CreateNetworkInterface", - "ec2:AttachNetworkInterface", - "ec2:DetachNetworkInterface", - "ec2:DeleteNetworkInterface", - "ec2:DescribeNetworkInterfaces", - "ec2:ModifyNetworkInterfaceAttribute", - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - ] - } - - statement { - sid = "ECRAuth" - effect = "Allow" - actions = [ - "ecr:GetAuthorizationToken" - ] - resources = ["*"] - } - - statement { - sid = "ECRDownloadImages" - effect = "Allow" - actions = [ - "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", - "ecr:GetDownloadUrlForLayer" + "ec2:RunInstances", + "ec2:DescribeInstanceCreditSpecifications", + "ec2:DescribeImages", + "ec2:ModifyDefaultCreditSpecification", + "ec2:DescribeVolumes" ] resources = ["*"] } statement { - sid = "ECRUploadImages" - effect = "Allow" - actions = [ - "ecr:CompleteLayerUpload", - "ecr:UploadLayerPart", - "ecr:InitiateLayerUpload", - "ecr:BatchCheckLayerAvailability", - "ecr:PutImage", - "ecr:BatchGetImage" - ] - resources = ["arn:aws:ecr:${local.region}:${local.account_id}:repository/*"] - } - - statement { - sid = "IAMReadOnly" effect = "Allow" actions = [ - "iam:Get*", - "iam:List*" - ] - resources = ["arn:aws:iam::${local.account_id}:*"] - } - - statement { - sid = "IAMPassRole" - effect = "Allow" - actions = [ - "iam:PassRole", - ] - resources = ["arn:aws:iam::${local.account_id}:*"] - } -} - -data "aws_iam_policy_document" "ws-policy" { - statement { - sid = "AllowModelInvocation" - effect = "Allow" - actions = [ - "bedrock:InvokeModel", - "bedrock:InvokeModelWithResponseStream", - "bedrock:ListInferenceProfiles" - ] - resources = [ - "arn:aws:bedrock:*:*:*", - "arn:aws:bedrock:*:*:*/*", - "arn:aws:bedrock:*:*:*:*", + "ec2:DescribeInstanceAttribute", + "ec2:UnmonitorInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances", + "ec2:DeleteTags", + "ec2:MonitorInstances", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyInstanceCreditSpecification" ] + resources = ["arn:aws:ec2:*:*:instance/*"] } } \ No newline at end of file diff --git a/modules/k8s/bootstrap/coder-proxy/main.tf b/modules/k8s/bootstrap/coder-proxy/main.tf index 034bd5a..7184b78 100644 --- a/modules/k8s/bootstrap/coder-proxy/main.tf +++ b/modules/k8s/bootstrap/coder-proxy/main.tf @@ -26,85 +26,54 @@ terraform { # Coderd Inputs ## -variable "coder_proxy_name" { - type = string -} - -variable "coder_proxy_display_name" { - type = string -} - -variable "coder_proxy_icon" { - type = string +variable "proxy" { + type = object({ + access_url = string + wildcard_url = string + coder_access_url = string + mount_ssl = optional(bool, false) + mount_ssl_name = optional(string, "cert") + name = string + display_name = string + icon = optional(string, "") + rep_cnt = optional(number, 1) + image_repo = optional(string, "ghcr.io/coder/coder") + image_tag = optional(string, "latest") + image_pull_policy = optional(string, "IfNotPresent") + image_pull_secrets = optional(list(string), null) + trace_logs = optional(bool, true) + log_filter = optional(string, ".*") + }) } ## -# TLS/SSL Inputs +# Kubernetes Inputs ## -variable "acme_registration_email" { - type = string - default = "" -} - -variable "acme_days_until_renewal" { - type = number - default = 30 -} - -variable "acme_revoke_certificate" { - type = bool - default = true +variable "release_name" { + description = "The release name of the installed Helm app." + type = string + default = "coder-proxy" } -variable "cloudflare_api_token" { - type = string - default = "" - sensitive = true +variable "chart_name" { + description = "The chart name of the installed Helm app." + type = string + default = "coder" } - -## -# Kubernetes Inputs -## - variable "namespace" { type = string } -variable "helm_timeout" { +variable "chart_timeout" { type = number - default = 120 # In Seconds + default = 300 # In Seconds } -variable "helm_version" { +variable "chart_version" { type = string - default = "2.25.1" -} - -variable "image_repo" { - type = string - default = "ghcr.io/coder/coder" -} - -variable "image_tag" { - type = string - default = "latest" -} - -variable "image_pull_policy" { - type = string - default = "IfNotPresent" -} - -variable "image_pull_secrets" { - type = list(string) - default = [] -} - -variable "replica_count" { - type = number - default = 0 + default = "2.30.0" } variable "env_vars" { @@ -112,7 +81,7 @@ variable "env_vars" { default = {} } -variable "load_balancer_class" { +variable "lb_class" { type = string default = "service.k8s.aws/nlb" } @@ -139,12 +108,12 @@ variable "resource_limit" { } } -variable "service_annotations" { +variable "svc_annot" { type = map(string) default = {} } -variable "service_account_annotations" { +variable "svc_acc_annot" { type = map(string) default = {} } @@ -164,7 +133,7 @@ variable "tolerations" { default = [] } -variable "topology_spread_constraints" { +variable "topology_spread" { type = list(object({ max_skew = number topology_key = string @@ -177,7 +146,7 @@ variable "topology_spread_constraints" { default = [] } -variable "pod_anti_affinity_preferred_during_scheduling_ignored_during_execution" { +variable "pod_aaf_pref_sched_ie" { type = list(object({ weight = number pod_affinity_term = object({ @@ -190,122 +159,33 @@ variable "pod_anti_affinity_preferred_during_scheduling_ignored_during_execution default = [] } -variable "primary_access_url" { - type = string -} - -variable "proxy_access_url" { - type = string -} - -variable "proxy_wildcard_url" { - type = string -} - -variable "termination_grace_period_seconds" { +variable "termination_grace_period" { type = number default = 600 } -variable "ssl_cert_config" { - type = object({ - name = string - create_secret = optional(bool, true) - }) - default = { - name = "coder-proxy-tls" - create_secret = true - } -} - -variable "proxy_token_config" { - type = object({ - name = optional(string, "proxy-token") - key = optional(string, "proxy.key") - }) - default = { - name = "proxy-token" - key = "proxy.key" - } -} - - resource "coderd_workspace_proxy" "this" { - name = var.coder_proxy_name - display_name = var.coder_proxy_display_name - icon = var.coder_proxy_icon -} - -resource "kubernetes_namespace" "this" { - metadata { - name = var.namespace - } -} - -resource "kubernetes_secret" "coder-proxy-key" { - metadata { - name = var.proxy_token_config.name - namespace = kubernetes_namespace.this.metadata[0].name - } - data = { - "${var.proxy_token_config.key}" = coderd_workspace_proxy.this.session_token - } - type = "Opaque" + name = var.proxy.name + display_name = var.proxy.display_name + icon = var.proxy.icon } locals { - common_name = trimprefix(trimprefix(var.proxy_access_url, "https://"), "http://") - wildcard_name = trimprefix(trimprefix(var.proxy_wildcard_url, "https://"), "http://") -} - -resource "kubernetes_manifest" "certificate" { - - count = var.ssl_cert_config.create_secret ? 1 : 0 - - field_manager { - force_conflicts = true - } - manifest = { - apiVersion = "cert-manager.io/v1" - kind = "Certificate" - metadata = { - labels = {} # var.cert_labels - name = var.ssl_cert_config.name - namespace = kubernetes_namespace.this.metadata[0].name - } - spec = { - secretName = var.ssl_cert_config.name - commonName = local.common_name - dnsNames = [local.common_name, local.wildcard_name] - duration = "${var.acme_days_until_renewal * 24}h" - renewBefore = "8h" - issuerRef = { - kind = "ClusterIssuer" - name = "issuer" - } - } + proxy = { + CODER_ACCESS_URL = var.proxy.access_url + CODER_WILDCARD_ACCESS_URL = var.proxy.wildcard_url + CODER_PRIMARY_ACCESS_URL = var.proxy.coder_access_url + CODER_PROXY_SESSION_TOKEN = coderd_workspace_proxy.this.session_token + CODER_TRACE_LOGS = var.proxy.trace_logs + CODER_LOG_FILTER = var.proxy.log_filter } -} - -locals { - primary_env_vars = { - CODER_PRIMARY_ACCESS_URL = var.primary_access_url - CODER_ACCESS_URL = var.proxy_access_url - CODER_WILDCARD_ACCESS_URL = var.proxy_wildcard_url + secrets = { + CODER_PROXY_SESSION_TOKEN = local.proxy["CODER_PROXY_SESSION_TOKEN"] } - env_vars = concat([ - for k, v in merge(local.primary_env_vars, var.env_vars) : { name = k, value = v } - ], [{ - name = "CODER_PROXY_SESSION_TOKEN" - valueFrom = { - secretKeyRef = { - name = kubernetes_secret.coder-proxy-key.metadata[0].name - key = var.proxy_token_config.key - } - } - }]) - pod_anti_affinity_preferred_during_scheduling_ignored_during_execution = [ - for k, v in var.pod_anti_affinity_preferred_during_scheduling_ignored_during_execution : { + secret_key = "key" + secret_keys = keys(local.secrets) + pod_aaf_pref_sched_ie = [ + for k, v in var.pod_aaf_pref_sched_ie : { weight = v.weight podAffinityTerm = { labelSelector = { @@ -315,8 +195,8 @@ locals { } } ] - topology_spread_constraints = [ - for k, v in var.topology_spread_constraints : { + topology_spread = [ + for k, v in var.topology_spread : { maxSkew = v.max_skew topologyKey = v.topology_key whenUnsatisfiable = v.when_unsatisfiable @@ -324,62 +204,127 @@ locals { matchLabels = try(v.label_selector.match_labels, {}) } matchLabelKeys = v.match_label_keys - } ] + env = concat([ for k,v in merge( + local.proxy + ) : { + name = k, + value = tostring(v) + } if lookup(local.secrets, k, null) == null ], [ + for k,v in local.secrets : { + name = k, + valueFrom = { + secretKeyRef = { + name = replace(lower(k), "_", "-"), + key = local.secret_key + } + } + } if v != null + ]) +} + +resource "kubernetes_namespace_v1" "this" { + metadata { + name = var.namespace + } +} + +resource "kubernetes_secret_v1" "coder" { + + for_each = toset(local.secret_keys) + + metadata { + name = replace(lower(each.key), "_", "-") + namespace = kubernetes_namespace_v1.this.metadata[0].name + annotations = { + "custom.kubernetes.secret/key" = local.secret_key + } + } + data = { + "${local.secret_key}" = sensitive(local.secrets[each.key]) + } +} + +resource "kubernetes_service_v1" "coder" { + + wait_for_load_balancer = true + + metadata { + name = var.release_name + namespace = kubernetes_namespace_v1.this.metadata[0].name + labels = {} + annotations = var.svc_annot + } + spec { + type = "LoadBalancer" + load_balancer_class = var.lb_class + port { + name = "http" + port = 80 + protocol = "TCP" + target_port = "http" + } + port { + name = "https" + port = 443 + protocol = "TCP" + target_port = "http" + } + selector = { + "app.kubernetes.io/instance" = var.release_name + "app.kubernetes.io/name" = var.chart_name + "app.kubernetes.io/part-of" = var.chart_name + } + } } resource "helm_release" "coder-proxy" { - name = "coder-v2" - namespace = kubernetes_namespace.this.metadata[0].name - chart = "coder" + name = var.release_name + namespace = kubernetes_namespace_v1.this.metadata[0].name + chart = var.chart_name repository = "https://helm.coder.com/v2" create_namespace = false upgrade_install = true skip_crds = false wait = true wait_for_jobs = true - version = var.helm_version - timeout = var.helm_timeout + version = var.chart_version + timeout = var.chart_timeout values = [yamlencode({ coder = { image = { - repo = var.image_repo - tag = var.image_tag - pullPolicy = var.image_pull_policy - pullSecrets = var.image_pull_secrets + repo = var.proxy.image_repo + tag = var.proxy.image_tag + pullPolicy = var.proxy.image_pull_policy + pullSecrets = var.proxy.image_pull_secrets } workspaceProxy = true - env = local.env_vars - tls = { - secretNames = [var.ssl_cert_config.name] - } + env = local.env service = { - enable = true - type = "LoadBalancer" - sessionAffinity = "None" - externalTrafficPolicy = "Cluster" - loadBalancerClass = var.load_balancer_class - annotations = var.service_annotations + enable = false + } + tls = { + secretNames = var.coder.mount_ssl ? [ var.coder.mount_ssl_name ] : [] } - replicaCount = var.replica_count + replicaCount = var.proxy.rep_cnt resources = { requests = var.resource_request limits = var.resource_limit } serviceAccount = { - annotations = var.service_account_annotations + annotations = var.svc_acc_annot } nodeSelector = var.node_selector tolerations = var.tolerations - topologySpreadConstraints = local.topology_spread_constraints + topologySpreadConstraints = local.topology_spread affinity = { podAntiAffinity = { - preferredDuringSchedulingIgnoredDuringExecution = local.pod_anti_affinity_preferred_during_scheduling_ignored_during_execution + preferredDuringSchedulingIgnoredDuringExecution = local.pod_aaf_pref_sched_ie } } - terminationGracePeriodSeconds = var.termination_grace_period_seconds + terminationGracePeriodSeconds = var.termination_grace_period } })] } \ No newline at end of file diff --git a/modules/k8s/bootstrap/coder-server/main.tf b/modules/k8s/bootstrap/coder-server/main.tf index 51b4a22..95459a4 100644 --- a/modules/k8s/bootstrap/coder-server/main.tf +++ b/modules/k8s/bootstrap/coder-server/main.tf @@ -13,6 +13,18 @@ terraform { } } +variable "release_name" { + description = "The release name of the installed Helm app." + type = string + default = "coder" +} + +variable "chart_name" { + description = "The chart name of the installed Helm app." + type = string + default = "coder" +} + variable "cluster_name" { type = string } @@ -33,12 +45,12 @@ variable "policy_resource_account" { variable "policy_name" { type = string - default = null + default = "coder-srv" } variable "role_name" { type = string - default = null + default = "coder-srv" } ## @@ -54,7 +66,7 @@ variable "helm_timeout" { default = 300 # In Seconds } -variable "helm_version" { +variable "chart_version" { type = string default = "2.25.1" } @@ -64,6 +76,8 @@ variable "coder" { access_url = string wildcard_url = string redirect = optional(bool, true) + mount_ssl = optional(bool, false) + mount_ssl_name = optional(string, "cert") image_repo = optional(string, "ghcr.io/coder/coder") image_tag = optional(string, "latest") image_pull_policy = optional(string, "IfNotPresent") @@ -85,7 +99,7 @@ variable "coder" { }) } -variable "prom" { +variable "prometheus" { type = object({ enable = optional(bool, true) collect_agent_status = optional(bool, true) @@ -186,23 +200,6 @@ variable "pod_aaf_pref_sched_ie" { default = [] } -variable "cert_config" { - type = object({ - create_secret = bool - name = string - kind = optional(string, "ClusterIssuer") - issuer = optional(string, "issuer") - store = optional(string, "issuer") - }) - default = { - create_secret = true - name = "coder-tls" - kind = "ClusterIssuer" - issuer = "issuer" - store = "issuer" - } -} - variable "db" { type = object({ url = string @@ -306,7 +303,10 @@ locals { coder = { CODER_ACCESS_URL = var.coder.access_url CODER_WILDCARD_ACCESS_URL = var.coder.wildcard_url - CODER_REDIRECT_TO_ACCESS_URL = var.coder.redirect + + # TLS Termination handled on the LB + CODER_REDIRECT_TO_ACCESS_URL = var.coder.mount_ssl + CODER_TLS_ENABLE = var.coder.mount_ssl CODER_ENABLE_TERRAFORM_DEBUG_MODE = var.coder.tf_debug_mode CODER_TRACE_LOGS = var.coder.trace_logs @@ -321,9 +321,9 @@ locals { CODER_ALLOW_CUSTOM_QUIET_HOURS = var.coder.allow_custom_quiet } prom = { - CODER_PROMETHEUS_ENABLE = var.prom.enable - CODER_PROMETHEUS_COLLECT_AGENT_STATS = var.prom.collect_agent_status - CODER_PROMETHEUS_COLLECT_DB_METRICS = var.prom.collect_db_metrics + CODER_PROMETHEUS_ENABLE = var.prometheus.enable + CODER_PROMETHEUS_COLLECT_AGENT_STATS = var.prometheus.collect_agent_status + CODER_PROMETHEUS_COLLECT_DB_METRICS = var.prometheus.collect_db_metrics } db = { CODER_PG_CONNECTION_URL = "postgresql://${var.db.username}:${var.db.password}@${var.db.url}/${var.db.db}" @@ -404,7 +404,7 @@ locals { local.aibridge ) : { name = k, - value = tostring(v) + value = tostring(v) } if lookup(local.secrets, k, null) == null ], [ for k,v in local.secrets : { name = k, @@ -418,10 +418,6 @@ locals { ]) } -output "env" { - value = local.env -} - resource "kubernetes_secret_v1" "coder" { for_each = toset(local.secret_keys) @@ -468,10 +464,8 @@ locals { } locals { - region = var.policy_resource_region == null ? data.aws_region.this.region : var.policy_resource_region - account_id = var.policy_resource_account == null ? data.aws_caller_identity.this.account_id : var.policy_resource_account - policy_name = var.policy_name == null ? "coder-srv" : var.policy_name - role_name = var.role_name == null ? "coder-srv" : var.role_name + region = data.aws_region.this.region + account_id = data.aws_caller_identity.this.account_id } module "provisioner-policy" { @@ -479,8 +473,8 @@ module "provisioner-policy" { count = var.coder.prov_rep_cnt == 0 ? 0 : 1 source = "../../../security/policy" - name = local.policy_name - path = "/${var.cluster_name}/${data.aws_region.this.region}/" + name = var.policy_name + path = "/${var.cluster_name}/${local.region}/" description = "Coder Terraform External Provisioner Policy" policy_json = data.aws_iam_policy_document.provisioner-policy.json } @@ -490,8 +484,8 @@ module "provisioner-oidc-role" { count = var.coder.prov_rep_cnt == 0 ? 0 : 1 source = "../../../security/role/access-entry" - name = local.role_name - path = "/${var.cluster_name}/${data.aws_region.this.region}/" + name = var.role_name + path = "/${var.cluster_name}/${local.region}/" cluster_name = var.cluster_name policy_arns = { "AmazonEC2ReadOnlyAccess" = "arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess" @@ -510,255 +504,73 @@ resource "kubernetes_namespace_v1" "this" { } } -locals { - ssl_vol_friendly_name = replace(var.cert_config.name, ".", "-") -} - -## -# Store SSL/TLS certs in Secrets Manager. -# Used to avoid throttling Let's Encrypt: -# https://letsencrypt.org/docs/rate-limits/ -## - -locals { - common_name = trimprefix(trimprefix(var.coder.access_url, "https://"), "http://") - wildcard_name = trimprefix(trimprefix(var.coder.wildcard_url, "https://"), "http://") - cert_refresh_interval = "2160h" # 90 days - cert_renew_before = "360h" # 15 days - secret_refresh_interval = "1812h0m0s" # 75.5 days - tls_secret_key = "tls.key" - tls_secret_crt = "tls.crt" - tls_remote_key = "tls-${local.common_name}.key" - tls_remote_crt = "tls-${local.common_name}.crt" -} - -resource "kubernetes_manifest" "pull" { - - field_manager { - force_conflicts = true - } - - wait { - fields = { - "status.conditions[0].type" = "Ready" - } - } +resource "kubernetes_service_v1" "coder" { - timeouts { - create = "1m" - update = "1m" - delete = "30s" - } - - manifest = { - apiVersion = "external-secrets.io/v1" - kind = "ExternalSecret" - metadata = { - name = var.cert_config.name - namespace = kubernetes_namespace_v1.this.metadata[0].name - } - spec = { - secretStoreRef = { - kind = "ClusterSecretStore" - name = var.cert_config.store - } - refreshPolicy = "Periodic" - refreshInterval = local.secret_refresh_interval - target = { - name = local.ssl_vol_friendly_name - creationPolicy = "Orphan" - deletionPolicy = "Retain" - template = { - type = "kubernetes.io/tls" - metadata = { - labels = { - "controller.cert-manager.io/fao" = "true" - } - annotations = { - "cert-manager.io/alt-names" = "${local.wildcard_name},${local.common_name}" - "cert-manager.io/certificate-name" = var.cert_config.name - "cert-manager.io/common-name" = local.common_name - "cert-manager.io/ip-sans" = "" - "cert-manager.io/issuer-group" = "" - "cert-manager.io/issuer-kind" = "ClusterIssuer" - "cert-manager.io/issuer-name" = var.cert_config.issuer - "cert-manager.io/uri-sans" = "" - } - } - } - } - data = [{ - secretKey = local.tls_secret_crt - remoteRef = { - key = local.tls_remote_crt - } - },{ - secretKey = local.tls_secret_key - remoteRef = { - key = local.tls_remote_key - } - }] - } - } -} + wait_for_load_balancer = true -resource "time_sleep" "wait" { - # Let the secret create first if it exists in AWS Secrets Manager. - depends_on = [ kubernetes_manifest.pull ] - create_duration = "30s" -} - -## -# Requires the cert-manager -## - -resource "kubernetes_manifest" "certificate" { - - depends_on = [ time_sleep.wait ] - - field_manager { - force_conflicts = true - } - - wait { - condition { - type = "Ready" - status = "True" - } - } - - timeouts { - create = "10m" - update = "10m" - delete = "30s" - } - - manifest = { - apiVersion = "cert-manager.io/v1" - kind = "Certificate" - metadata = { - name = var.cert_config.name - namespace = kubernetes_namespace_v1.this.metadata[0].name - } - spec = { - commonName = local.common_name - dnsNames = [ - local.common_name, - local.wildcard_name - ] - duration = local.cert_refresh_interval - renewBefore = local.cert_renew_before - issuerRef = { - kind = var.cert_config.kind - name = var.cert_config.issuer - } - secretName = local.ssl_vol_friendly_name - privateKey = { - rotationPolicy = "Never" - algorithm = "RSA" - encoding = "PKCS1" - size = "2048" - } - } - } -} - -resource "kubernetes_manifest" "push" { - - depends_on = [ kubernetes_manifest.certificate ] - - field_manager { - force_conflicts = true + metadata { + name = var.release_name + namespace = kubernetes_namespace_v1.this.metadata[0].name + labels = {} + annotations = var.svc_annot } - - wait { - condition { - type = "Ready" - status = "True" + spec { + type = "LoadBalancer" + load_balancer_class = var.lb_class + port { + name = "http" + port = 80 + protocol = "TCP" + target_port = "http" } - } - - timeouts { - create = "10m" - update = "10m" - delete = "30s" - } - - manifest = { - apiVersion = "external-secrets.io/v1alpha1" - kind = "PushSecret" - metadata = { - name = var.cert_config.name - namespace = kubernetes_namespace_v1.this.metadata[0].name + port { + name = "https" + port = 443 + protocol = "TCP" + target_port = "http" } - spec = { - updatePolicy = "Replace" - deletionPolicy = "None" - refreshInterval = local.secret_refresh_interval - secretStoreRefs = [{ - kind = "ClusterSecretStore" - name = var.cert_config.store - }] - selector = { - secret = { - name = kubernetes_manifest.certificate.manifest.spec.secretName - } - } - data = [{ - match = { - secretKey = local.tls_secret_crt - remoteRef = { - remoteKey = local.tls_remote_crt - } - } - },{ - match = { - secretKey = local.tls_secret_key - remoteRef = { - remoteKey = local.tls_remote_key - } - } - }] + selector = { + "app.kubernetes.io/instance" = var.chart_name + "app.kubernetes.io/name" = var.release_name } } } resource "kubernetes_service_v1" "prometheus" { + count = var.prometheus.enable ? 1 : 0 metadata { - name = "coder-prometheus" + name = "${var.release_name}-prometheus" namespace = kubernetes_namespace_v1.this.metadata[0].name - # labels = local.app_labels + labels = {} } spec { type = "ClusterIP" cluster_ip = "None" port { - name = "prom-http" + name = "http" protocol = "TCP" port = 2112 target_port = 2112 } selector = { - "app.kubernetes.io/instance" = "coder-v2" - "app.kubernetes.io/name" = "coder" + "app.kubernetes.io/instance" = var.chart_name + "app.kubernetes.io/name" = var.release_name } } } resource "helm_release" "coder-server" { - depends_on = [ kubernetes_manifest.push ] - - name = "coder" + name = var.release_name namespace = kubernetes_namespace_v1.this.metadata[0].name - chart = "coder" + chart = var.chart_name repository = "https://helm.coder.com/v2" create_namespace = false upgrade_install = true skip_crds = false wait = true wait_for_jobs = true - version = var.helm_version + version = var.chart_version timeout = var.helm_timeout values = [yamlencode({ @@ -770,20 +582,15 @@ resource "helm_release" "coder-server" { pullSecrets = var.coder.image_pull_secrets } env = local.env - tls = { - secretNames = [ local.ssl_vol_friendly_name ] - } - podAnnotations = { + podAnnotations = var.prometheus.enable ? { "prometheus.io/scrape" = "true" - "prometheus.io/port" = "2112" - } + "prometheus.io/port" = kubernetes_service_v1.prometheus[0].spec[0].port[0].port + } : {} service = { - enable = true - type = "LoadBalancer" - sessionAffinity = "None" - externalTrafficPolicy = "Cluster" - loadBalancerClass = var.lb_class - annotations = var.svc_annot + enable = false + } + tls = { + secretNames = var.coder.mount_ssl ? [ var.coder.mount_ssl_name ] : [] } replicaCount = var.coder.rep_cnt resources = { diff --git a/modules/k8s/bootstrap/coder-server/policy.tf b/modules/k8s/bootstrap/coder-server/policy.tf index 8caf963..f9d9392 100644 --- a/modules/k8s/bootstrap/coder-server/policy.tf +++ b/modules/k8s/bootstrap/coder-server/policy.tf @@ -1,176 +1,38 @@ data "aws_iam_policy_document" "provisioner-policy" { statement { - sid = "EC2InstanceLifecycle" effect = "Allow" actions = [ - "ec2:RunInstances", - "ec2:StartInstances", - "ec2:StopInstances", - "ec2:TerminateInstances", + "ec2:GetDefaultCreditSpecification", + "ec2:DescribeIamInstanceProfileAssociations", + "ec2:DescribeTags", "ec2:DescribeInstances", - "ec2:RebootInstances", - "ec2:ModifyInstanceAttribute", - "ec2:DescribeInstanceAttribute" - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - "arn:aws:ec2:${local.region}::image/*" - ] - } - - statement { - sid = "EC2ManageHostLifecycle" - effect = "Allow" - actions = [ - "ec2:AllocateHosts", - "ec2:ModifyHosts", - "ec2:ReleaseHosts" - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - "arn:aws:ec2:${local.region}::image/*" - ] - } - - statement { - sid = "EBSVolumeLifecycle" - effect = "Allow" - actions = [ - "ec2:CreateVolume", - "ec2:AttachVolume", - "ec2:DetachVolume", - "ec2:DeleteVolume", - "ec2:DescribeVolumes", - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - ] - } - - statement { - sid = "SecurityGroupLifecycle" - effect = "Allow" - actions = [ - "ec2:CreateSecurityGroup", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:AuthorizeSecurityGroupEgress", - "ec2:RevokeSecurityGroupIngress", - "ec2:RevokeSecurityGroupEgress", - "ec2:DeleteSecurityGroup", - "ec2:DescribeSecurityGroups", - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - ] - } - - statement { - sid = "TagLifecycle" - effect = "Allow" - actions = [ + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceStatus", "ec2:CreateTags", - "ec2:DeleteTags", - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - ] - } - - statement { - sid = "NetworkInterfaceLifecycle" - effect = "Allow" - actions = [ - "ec2:CreateNetworkInterface", - "ec2:AttachNetworkInterface", - "ec2:DetachNetworkInterface", - "ec2:DeleteNetworkInterface", - "ec2:DescribeNetworkInterfaces", - "ec2:ModifyNetworkInterfaceAttribute", - ] - resources = [ - "arn:aws:ec2:${local.region}:${local.account_id}:*", - "arn:aws:ec2:${local.region}:${local.account_id}:*/*", - "arn:aws:ec2:${local.region}:${local.account_id}:*:*", - ] - } - - statement { - sid = "ECRAuth" - effect = "Allow" - actions = [ - "ecr:GetAuthorizationToken" - ] - resources = ["*"] - } - - statement { - sid = "ECRDownloadImages" - effect = "Allow" - actions = [ - "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", - "ecr:GetDownloadUrlForLayer" + "ec2:RunInstances", + "ec2:DescribeInstanceCreditSpecifications", + "ec2:DescribeImages", + "ec2:ModifyDefaultCreditSpecification", + "ec2:DescribeVolumes" ] resources = ["*"] } statement { - sid = "ECRUploadImages" - effect = "Allow" - actions = [ - "ecr:CompleteLayerUpload", - "ecr:UploadLayerPart", - "ecr:InitiateLayerUpload", - "ecr:BatchCheckLayerAvailability", - "ecr:PutImage", - "ecr:BatchGetImage" - ] - resources = ["arn:aws:ecr:${local.region}:${local.account_id}:repository/*"] - } - - statement { - sid = "IAMReadOnly" effect = "Allow" actions = [ - "iam:Get*", - "iam:List*" - ] - resources = ["arn:aws:iam::${local.account_id}:*"] - } - - statement { - sid = "IAMPassRole" - effect = "Allow" - actions = [ - "iam:PassRole", - ] - resources = ["arn:aws:iam::${local.account_id}:*"] - } -} - -data "aws_iam_policy_document" "ws-policy" { - statement { - sid = "AllowModelInvocation" - effect = "Allow" - actions = [ - "bedrock:InvokeModel", - "bedrock:InvokeModelWithResponseStream", - "bedrock:ListInferenceProfiles" - ] - resources = [ - "arn:aws:bedrock:*:*:*", - "arn:aws:bedrock:*:*:*/*", - "arn:aws:bedrock:*:*:*:*", + "ec2:DescribeInstanceAttribute", + "ec2:UnmonitorInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances", + "ec2:DeleteTags", + "ec2:MonitorInstances", + "ec2:CreateTags", + "ec2:RunInstances", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyInstanceCreditSpecification" ] + resources = ["arn:aws:ec2:*:*:instance/*"] } } \ No newline at end of file diff --git a/modules/k8s/bootstrap/ebs-controller/main.tf b/modules/k8s/bootstrap/ebs-csi/main.tf similarity index 100% rename from modules/k8s/bootstrap/ebs-controller/main.tf rename to modules/k8s/bootstrap/ebs-csi/main.tf diff --git a/modules/k8s/bootstrap/karpenter/main.tf b/modules/k8s/bootstrap/karpenter/main.tf index e12dfa9..a2cf7da 100644 --- a/modules/k8s/bootstrap/karpenter/main.tf +++ b/modules/k8s/bootstrap/karpenter/main.tf @@ -103,56 +103,40 @@ variable "node_iam_role_use_name_prefix" { default = true } -variable "topology_spread_constraints" { - type = list(map(string)) - default = [{ - maxSkew = "1" - topologyKey = "topology.kubernetes.io/zone" - whenUnsatisfiable = "DoNotSchedule" - }] -} - -variable "ec2nodeclass_configs" { +variable "pod_aaf_pref_sched_ie" { type = list(object({ - name = string - node_role_name = optional(string, "") - ami_alias = optional(string, "al2023@latest") - subnet_selector_tags = map(string) - sg_selector_tags = map(string) - user_data = optional(string, "") - block_device_mappings = optional(list(object({ - device_name = string - ebs = object({ - volume_size = string - volume_type = string - encrypted = optional(bool, false) - delete_on_termination = optional(bool, true) + weight = number + pod_affinity_term = object({ + label_selector = object({ + match_labels = map(string) }) - })), []) + topology_key = string + }) })) default = [] } -variable "nodepool_configs" { +variable "topology_spread" { type = list(object({ - name = string - node_labels = map(string) - node_taints = optional(list(object({ - key = string - value = string - effect = string - })), []) - node_requirements = optional(list(object({ - key = string - operator = string - values = list(string) - })), []) - node_class_ref_name = string - node_expires_after = optional(string, "Never") - disruption_consolidation_policy = optional(string, "WhenEmpty") - disruption_consolidate_after = optional(string, "1m") + max_skew = number + topology_key = string + when_unsatisfiable = optional(string, "DoNotSchedule") + label_selector = object({ + match_labels = map(string) + }) + match_label_keys = optional(list(string), []) })) - default = [] + default = [{ + max_skew = 1 + topology_key = "topology.kubernetes.io/zone" + when_unsatisfiable = "DoNotSchedule" + label_selector = { + match_labels = { + "app.kubernetes.io/instance" = "karpenter" + "app.kubernetes.io/name" = "karpenter" + } + } + }] } variable "node_selector" { @@ -273,13 +257,44 @@ module "karpenter" { # node_iam_role_tags = merge(var.tags, var.karpenter_node_role_tags) } +resource "kubernetes_namespace_v1" "this" { + metadata { + name = var.namespace + } +} + +locals { + pod_aaf_pref_sched_ie = [ + for k, v in var.pod_aaf_pref_sched_ie : { + weight = v.weight + podAffinityTerm = { + labelSelector = { + matchLabels = try(v.pod_affinity_term.label_selector.match_labels, {}) + } + topologyKey = try(v.pod_affinity_term.topology_key, {}) + } + } + ] + topology_spread = [ + for k, v in var.topology_spread : { + maxSkew = v.max_skew + topologyKey = v.topology_key + whenUnsatisfiable = v.when_unsatisfiable + labelSelector = { + matchLabels = try(v.label_selector.match_labels, {}) + } + matchLabelKeys = v.match_label_keys + } + ] +} + resource "helm_release" "karpenter" { - depends_on = [module.karpenter] + depends_on = [ module.karpenter ] name = "karpenter" - namespace = var.namespace + namespace = kubernetes_namespace_v1.this.metadata[0].name chart = "karpenter" repository = "oci://public.ecr.aws/karpenter" - create_namespace = true + create_namespace = false upgrade_install = true skip_crds = false wait = true @@ -290,14 +305,8 @@ resource "helm_release" "karpenter" { values = [yamlencode({ controller = { resources = { - limits = { - cpu = "1000m" - memory = "2Gi" - } - requests = { - cpu = "500m" - memory = "1Gi" - } + limits = null + requests = null } } dnsPolicy = "ClusterFirst" @@ -309,6 +318,24 @@ resource "helm_release" "karpenter" { } } tolerations = var.tolerations + topologySpreadConstraints = local.topology_spread + affinity = { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [{ + matchExpressions = [{ + # Prevent Karpenter from scheduling Karpenter Ctrl pods onto itself. + key = "karpenter.sh/nodepool" + operator = "In" + values = ["system"] + }] + }] + } + } + podAntiAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = [] + } + } settings = { clusterName = var.cluster_name featureGates = { @@ -319,31 +346,13 @@ resource "helm_release" "karpenter" { })] } -module "ec2nodeclass" { - count = length(var.ec2nodeclass_configs) - source = "../../objects/ec2nodeclass" - name = var.ec2nodeclass_configs[count.index].name - node_role_name = var.ec2nodeclass_configs[count.index].node_role_name == "" ? module.karpenter.node_iam_role_name : var.ec2nodeclass_configs[count.index].node_role_name - ami_alias = var.ec2nodeclass_configs[count.index].ami_alias - subnet_selector_tags = var.ec2nodeclass_configs[count.index].subnet_selector_tags - sg_selector_tags = var.ec2nodeclass_configs[count.index].sg_selector_tags - block_device_mappings = var.ec2nodeclass_configs[count.index].block_device_mappings - user_data = var.ec2nodeclass_configs[count.index].user_data -} - -resource "kubernetes_manifest" "ec2nodeclass" { - depends_on = [helm_release.karpenter] - count = length(var.ec2nodeclass_configs) - manifest = yamldecode(module.ec2nodeclass[count.index].manifest) -} - resource "kubernetes_service_account_v1" "ctrl-role" { depends_on = [helm_release.karpenter] metadata { name = "ctrl-role" - namespace = var.namespace + namespace = kubernetes_namespace_v1.this.metadata[0].name annotations = { "eks.amazonaws.com/role-arn" = module.karpenter.iam_role_arn } @@ -356,7 +365,7 @@ resource "kubernetes_service_account_v1" "node-role" { metadata { name = "node-role" - namespace = var.namespace + namespace = kubernetes_namespace_v1.this.metadata[0].name annotations = { "eks.amazonaws.com/role-arn" = module.karpenter.node_iam_role_arn } diff --git a/modules/k8s/bootstrap/lb-controller/main.tf b/modules/k8s/bootstrap/lb-controller/main.tf index 990db13..95e49d2 100644 --- a/modules/k8s/bootstrap/lb-controller/main.tf +++ b/modules/k8s/bootstrap/lb-controller/main.tf @@ -18,6 +18,11 @@ terraform { # https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.13.2/docs/install/iam_policy.json ## +variable "release_name" { + type = string + default = "aws-load-balancer-controller" +} + variable "cluster_name" { type = string } @@ -36,16 +41,6 @@ variable "policy_name" { default = "" } -variable "policy_resource_region" { - type = string - default = "" -} - -variable "policy_resource_account" { - type = string - default = "" -} - variable "tags" { type = map(string) default = {} @@ -57,6 +52,7 @@ variable "namespace" { variable "chart_version" { type = string + default = "3.0.0" } variable "enable_cert_manager" { @@ -105,8 +101,8 @@ data "aws_region" "this" {} data "aws_caller_identity" "this" {} locals { - region = var.policy_resource_region == "" ? data.aws_region.this.region : var.policy_resource_region - account_id = var.policy_resource_account == "" ? data.aws_caller_identity.this.account_id : var.policy_resource_account + account_id = data.aws_caller_identity.this.account_id + region = data.aws_region.this.region policy_name = var.policy_name == "" ? "lb-ctrl" : var.policy_name role_name = var.role_name == "" ? "lb-ctrl" : var.role_name } @@ -145,7 +141,7 @@ locals { } resource "helm_release" "lb-controller" { - name = "aws-load-balancer-controller" + name = var.release_name namespace = var.namespace chart = "aws-load-balancer-controller" repository = "https://aws.github.io/eks-charts" @@ -179,44 +175,44 @@ resource "helm_release" "lb-controller" { })] } -resource "kubernetes_manifest" "alb-class-params" { - count = var.create_alb_class ? 1 : 0 - depends_on = [helm_release.lb-controller] - manifest = { - apiVersion = "elbv2.k8s.aws/v1beta1" - kind = "IngressClassParams" - metadata = { - labels = { - "app.kubernetes.io/name" : "aws-load-balancer-controller" - } - name = "alb" - } - } -} - -resource "kubernetes_manifest" "alb-class" { - count = var.create_alb_class ? 1 : 0 - depends_on = [helm_release.lb-controller, kubernetes_manifest.alb-class-params[0]] - manifest = { - apiVersion = "networking.k8s.io/v1" - kind = "IngressClass" - metadata = { - labels = { - "app.kubernetes.io/name" : "aws-load-balancer-controller" - } - name = "alb" - } - spec = { - controller = "ingress.k8s.aws/alb" - parameters = { - apiGroup = "elbv2.k8s.aws" - kind = "IngressClassParams" - name = "alb" - } - } - } -} - -output "oidc_role_arn" { - value = module.oidc-role.role_arn -} \ No newline at end of file +# resource "kubernetes_manifest" "alb-class-params" { +# count = var.create_alb_class ? 1 : 0 +# depends_on = [helm_release.lb-controller] +# manifest = { +# apiVersion = "elbv2.k8s.aws/v1beta1" +# kind = "IngressClassParams" +# metadata = { +# labels = { +# "app.kubernetes.io/name" : var.release_name +# } +# name = "alb" +# } +# } +# } + +# resource "kubernetes_manifest" "alb-class" { +# count = var.create_alb_class ? 1 : 0 +# depends_on = [helm_release.lb-controller, kubernetes_manifest.alb-class-params[0]] +# manifest = { +# apiVersion = "networking.k8s.io/v1" +# kind = "IngressClass" +# metadata = { +# labels = { +# "app.kubernetes.io/name" : var.release_name +# } +# name = "alb" +# } +# spec = { +# controller = "ingress.k8s.aws/alb" +# parameters = { +# apiGroup = "elbv2.k8s.aws" +# kind = "IngressClassParams" +# name = "alb" +# } +# } +# } +# } + +# output "oidc_role_arn" { +# value = module.oidc-role.role_arn +# } \ No newline at end of file diff --git a/modules/k8s/bootstrap/monitoring/main.tf b/modules/k8s/bootstrap/monitoring/main.tf index 32c7ca1..ebc8f70 100644 --- a/modules/k8s/bootstrap/monitoring/main.tf +++ b/modules/k8s/bootstrap/monitoring/main.tf @@ -31,6 +31,11 @@ variable "cluster_oidc_provider_arn" { type = string } +variable "storage_class" { + type = string + default = "" +} + variable "coder" { type = object({ db = object({ @@ -87,21 +92,9 @@ variable "domain_name" { type = string } -variable "cert_config" { - type = object({ - create_secret = bool - name = string - kind = optional(string, "ClusterIssuer") - issuer = optional(string, "issuer") - store = optional(string, "issuer") - }) - default = { - create_secret = true - name = "grafana-tls" - kind = "ClusterIssuer" - issuer = "issuer" - store = "issuer" - } +variable "lb_class" { + type = string + default = "service.k8s.aws/nlb" } variable "tolerations" { @@ -109,16 +102,20 @@ variable "tolerations" { default = [] } -locals { - normalized_domain_name = split(".", var.domain_name)[0] - apex_domain = join(".", slice(split(".", var.domain_name), length(split(".", var.domain_name))-2, length(split(".", var.domain_name)))) - ssl_vol_friendly_name = replace(var.cert_config.name, ".", "-") - daemonset_tolerations = [{ - effect = "NoSchedule" +variable "system_tolerations" { + description = "(Optional) Override if you need to adjust where critical monitoring addons need to be moved." + type = list(map(any)) + default = [{ + key = "CriticalAddonsOnly" operator = "Exists" }] - system_tolerations = [{ - key = "CriticalAddonsOnly" +} + +variable "daemonset_tolerations" { + description = "(Optional) Override if you need to adjust where monitoring DaemonSets need to be placed." + type = list(map(any)) + default = [{ + effect = "NoSchedule" operator = "Exists" }] } @@ -161,209 +158,6 @@ resource "kubernetes_namespace_v1" "this" { } } -locals { - common_name = "grafana.${trimprefix(trimprefix(var.domain_name, "https://"), "http://")}" - wildcard_name = "*.${local.common_name}" - cert_refresh_interval = "2160h" # 90 days - cert_renew_before = "360h" # 15 days - secret_refresh_interval = "1812h0m0s" # 75.5 days - tls_secret_key = "tls.key" - tls_secret_crt = "tls.crt" - tls_remote_key = "tls-${local.common_name}.key" - tls_remote_crt = "tls-${local.common_name}.crt" -} - -resource "kubernetes_manifest" "pull" { - - field_manager { - force_conflicts = true - } - - wait { - fields = { - "status.conditions[0].type" = "Ready" - } - } - - timeouts { - create = "1m" - update = "1m" - delete = "30s" - } - - manifest = { - apiVersion = "external-secrets.io/v1" - kind = "ExternalSecret" - metadata = { - name = var.cert_config.name - namespace = kubernetes_namespace_v1.this.metadata[0].name - } - spec = { - secretStoreRef = { - kind = "ClusterSecretStore" - name = var.cert_config.store - } - refreshPolicy = "Periodic" - refreshInterval = local.secret_refresh_interval - target = { - name = local.ssl_vol_friendly_name - creationPolicy = "Orphan" - deletionPolicy = "Retain" - template = { - type = "kubernetes.io/tls" - metadata = { - labels = { - "controller.cert-manager.io/fao" = "true" - } - annotations = { - "cert-manager.io/alt-names" = "${local.wildcard_name},${local.common_name}" - "cert-manager.io/certificate-name" = var.cert_config.name - "cert-manager.io/common-name" = local.common_name - "cert-manager.io/ip-sans" = "" - "cert-manager.io/issuer-group" = "" - "cert-manager.io/issuer-kind" = "ClusterIssuer" - "cert-manager.io/issuer-name" = var.cert_config.issuer - "cert-manager.io/uri-sans" = "" - } - } - } - } - data = [{ - secretKey = local.tls_secret_crt - remoteRef = { - key = local.tls_remote_crt - } - },{ - secretKey = local.tls_secret_key - remoteRef = { - key = local.tls_remote_key - } - }] - } - } -} - -resource "time_sleep" "wait" { - # Let the secret create first if it exists in AWS Secrets Manager. - depends_on = [ kubernetes_manifest.pull ] - create_duration = "30s" -} - -## -# Requires the cert-manager -## - -resource "kubernetes_manifest" "certificate" { - - depends_on = [ time_sleep.wait ] - - field_manager { - force_conflicts = true - } - - wait { - condition { - type = "Ready" - status = "True" - } - } - - timeouts { - create = "10m" - update = "10m" - delete = "30s" - } - - manifest = { - apiVersion = "cert-manager.io/v1" - kind = "Certificate" - metadata = { - name = var.cert_config.name - namespace = kubernetes_namespace_v1.this.metadata[0].name - } - spec = { - commonName = local.common_name - dnsNames = [ - local.common_name, - local.wildcard_name - ] - duration = local.cert_refresh_interval - renewBefore = local.cert_renew_before - issuerRef = { - kind = var.cert_config.kind - name = var.cert_config.issuer - } - secretName = local.ssl_vol_friendly_name - privateKey = { - rotationPolicy = "Never" - algorithm = "RSA" - encoding = "PKCS1" - size = "2048" - } - } - } -} - -resource "kubernetes_manifest" "push" { - - depends_on = [ kubernetes_manifest.certificate ] - - field_manager { - force_conflicts = true - } - - wait { - condition { - type = "Ready" - status = "True" - } - } - - timeouts { - create = "10m" - update = "10m" - delete = "30s" - } - - manifest = { - apiVersion = "external-secrets.io/v1alpha1" - kind = "PushSecret" - metadata = { - name = var.cert_config.name - namespace = kubernetes_namespace_v1.this.metadata[0].name - } - spec = { - updatePolicy = "Replace" - deletionPolicy = "None" - refreshInterval = local.secret_refresh_interval - secretStoreRefs = [{ - kind = "ClusterSecretStore" - name = var.cert_config.store - }] - selector = { - secret = { - name = kubernetes_manifest.certificate.manifest.spec.secretName - } - } - data = [{ - match = { - secretKey = local.tls_secret_crt - remoteRef = { - remoteKey = local.tls_remote_crt - } - } - },{ - match = { - secretKey = local.tls_secret_key - remoteRef = { - remoteKey = local.tls_remote_key - } - } - }] - } - } -} - resource "helm_release" "coder-observe" { name = "coder-observe" namespace = kubernetes_namespace_v1.this.metadata[0].name @@ -398,10 +192,19 @@ resource "helm_release" "coder-observe" { } prometheus = { server = { - tolerations = local.system_tolerations + tolerations = var.system_tolerations + persistentVolume = { + enabled = true + storageClassName = var.storage_class + } } alertmanager = { - tolerations = local.system_tolerations + enabled = true + tolerations = var.system_tolerations + persistence = { + enabled = true + storageClass = var.storage_class + } } } grafana = { @@ -421,13 +224,11 @@ resource "helm_release" "coder-observe" { ssl_mode = var.grafana.db.sslmode } server = { - root_url = "https://${local.common_name}" - domain = local.common_name + root_url = "https://${var.domain_name}" + domain = var.domain_name enforce_domain = true http_port = 3000 - protocol = "https" - cert_file = "/mnt/grafana-tls/tls.crt" - cert_key = "/mnt/grafana-tls/tls.key" + protocol = "http" } users = { allow_sign_up = false @@ -437,12 +238,12 @@ resource "helm_release" "coder-observe" { useStatefulSet = true readinessProbe = { httpGet = { - scheme = "HTTPS" + scheme = "HTTP" } } livenessProbe = { httpGet = { - scheme = "HTTPS" + scheme = "HTTP" } } persistence = { @@ -458,24 +259,17 @@ resource "helm_release" "coder-observe" { enabled = true externalTrafficPolicy = "Cluster" internalTrafficPolicy = "Cluster" - loadBalancerClass = "service.k8s.aws/nlb" + loadBalancerClass = var.lb_class port = 443 targetPort = 3000 type = "LoadBalancer" } - extraSecretMounts = [{ - name = kubernetes_manifest.certificate.manifest.spec.secretName - mountPath = "/mnt/grafana-tls" - secretName = kubernetes_manifest.certificate.manifest.spec.secretName - readOnly = true - optional = false - subPath = "" - }] + tolerations = var.system_tolerations } grafana-agent = { enabled = true controller = { - tolerations = local.daemonset_tolerations + tolerations = var.daemonset_tolerations } discovery = <<-EOF // Discover k8s nodes @@ -564,22 +358,35 @@ resource "helm_release" "coder-observe" { } } lokiCanary = { - tolerations = local.daemonset_tolerations + tolerations = var.daemonset_tolerations } backend = { - tolerations = local.system_tolerations + tolerations = var.system_tolerations + persistence = { + volumeClaimsEnabled = false + # storageClass = var.storage_class + } } resultsCache = { - tolerations = local.system_tolerations + tolerations = var.system_tolerations } chunksCache = { - tolerations = local.system_tolerations + tolerations = var.system_tolerations + persistence = { + enabled = true + storageClass = var.storage_class + } } - storage = { - tolerations = local.system_tolerations + minio = { + enable = false + # tolerations = var.system_tolerations } write = { - tolerations = local.system_tolerations + tolerations = var.system_tolerations + persistence = { + volumeClaimsEnabled = false + # storageClass = var.storage_class + } } serviceAccount = { create = true From 279e1c115f3e2afa464da7cdc7189123893f457b Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 16 Feb 2026 23:12:11 -0800 Subject: [PATCH 26/44] fix: update coder.env to reflect bare-minimum env vars --- infra/1-click/coder.env | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/infra/1-click/coder.env b/infra/1-click/coder.env index 4493b64..c0849e2 100644 --- a/infra/1-click/coder.env +++ b/infra/1-click/coder.env @@ -1,4 +1,14 @@ -CODER_AWS_PROFILE=default +CODER_TF_USE_REMOTE_STATE=false +CODER_TF_BACKEND_AWS_BUCKET_NAME= +CODER_TF_BACKEND_AWS_REGION= + CODER_AWS_REGION=us-east-2 -CODER_DOMAIN_NAME= -CODER_LICENSE= (Optional) \ No newline at end of file + +CODER_VERSION=2.30.0 +CODER_LICENSE= + +CODER_USERNAME=admin +CODER_EMAIL=admin@coder.com +CODER_PASSWORD=Th1s1sN0TS3CuR3!! + +CODER_DOMAIN_NAME= \ No newline at end of file From 608caa455c0b0c3e0ea071875d5de3c5694c442e Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 16 Feb 2026 23:13:12 -0800 Subject: [PATCH 27/44] fix: rework scripts for Terragrunt --- infra/1-click/0-init.sh | 7 +++ infra/1-click/0-initialize.sh | 18 ------- infra/1-click/1-apply.sh | 7 +++ infra/1-click/1-plan-n-deploy.sh | 86 -------------------------------- infra/1-click/2-clean.sh | 76 +--------------------------- 5 files changed, 15 insertions(+), 179 deletions(-) create mode 100755 infra/1-click/0-init.sh delete mode 100755 infra/1-click/0-initialize.sh create mode 100755 infra/1-click/1-apply.sh delete mode 100755 infra/1-click/1-plan-n-deploy.sh diff --git a/infra/1-click/0-init.sh b/infra/1-click/0-init.sh new file mode 100755 index 0000000..369a3ad --- /dev/null +++ b/infra/1-click/0-init.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -ae -o pipefail + +source coder.env + +terragrunt run --all --non-interactive init -- -migrate-state -upgrade \ No newline at end of file diff --git a/infra/1-click/0-initialize.sh b/infra/1-click/0-initialize.sh deleted file mode 100755 index 5441342..0000000 --- a/infra/1-click/0-initialize.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -e -o pipefail - -echo "Change directory into '0-infra'." -cd 0-infra -terraform init -cd ../ - -echo "Change directory into '1-setup'." -cd 1-setup -terraform init -cd ../ - -echo "Change directory into '2-coder'." -cd 2-coder -terraform init -cd ../ \ No newline at end of file diff --git a/infra/1-click/1-apply.sh b/infra/1-click/1-apply.sh new file mode 100755 index 0000000..8b5cf7b --- /dev/null +++ b/infra/1-click/1-apply.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -ae -o pipefail + +source coder.env + +terragrunt run --all --non-interactive apply \ No newline at end of file diff --git a/infra/1-click/1-plan-n-deploy.sh b/infra/1-click/1-plan-n-deploy.sh deleted file mode 100755 index 956c2c0..0000000 --- a/infra/1-click/1-plan-n-deploy.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -set -ae -o pipefail - -source coder.env - -AWS_AZS="${CODER_AWS_AZS:-[\"a\",\"c\"]}" - -AWS_PROFILE="${CODER_AWS_PROFILE:-default}" -AWS_REGION="${CODER_AWS_REGION:-us-east-2}" -DOMAIN_NAME="${CODER_DOMAIN_NAME:-}" -LICENSE="${CODER_LICENSE:-}" - -USE_EXTERN_DNS="${CODER_USE_EXTERN_DNS:-true}" - -USE_R53="${CODER_USE_R53:-true}" - -USE_CF="${CODER_USE_CF:-false}" -CF_TOKEN="${CODER_CF_TOKEN:-}" -CF_EMAIL="${CODER_CF_EMAIL:-}" - -SET_REC_USE_R53=false; ! $USE_EXTERN_DNS && $USE_R53 && SET_REC_USE_R53=true -SET_REC_USE_CF=false; ! $USE_EXTERN_DNS && $USE_CF && SET_REC_USE_CF=true - -CODER_DB_USERNAME="${CODER_DB_USERNAME:-coder}" -CODER_DB_PASSWORD="${CODER_DB_PASSWORD:-th1s1sn0tas3cur3pass0wrd}" - -GRAFANA_DB_USERNAME="${CODER_GRAFANA_DB_PASSWORD:-grafana}" -GRAFANA_DB_PASSWORD="${CODER_GRAFANA_DB_PASSWORD:-th1s1sn0tas3cur3pass0wrd}" - -CODER_USERNAME="${CODER_USERNAME:-admin}" -CODER_EMAIL="${CODER_EMAIL:-admin@coder.com}" -CODER_PASSWORD="${CODER_PASSWORD:-Th1s1sN0TS3CuR3!!}" - -if [ -z "${DOMAIN_NAME}" ]; then - echo "A domain name is required! Be sure to register or use an existing one from Route53!" - exit 1; -fi - -# echo "Change directory into '0-infra'." -# cd 0-infra -# terraform plan -out=tf.plan \ -# -var profile=$AWS_PROFILE \ -# -var region=$AWS_REGION \ -# -var domain_name=$DOMAIN_NAME \ -# -var azs="$AWS_AZS" \ -# -var coder_username=$CODER_DB_USERNAME \ -# -var coder_password=$CODER_DB_PASSWORD \ -# -var grafana_username=$GRAFANA_DB_USERNAME \ -# -var grafana_password=$GRAFANA_DB_PASSWORD \ -# -var use_ext_dns=$USE_EXTERN_DNS \ -# -var cf_config="{\"enabled\":\"$USE_CF\",\"email\":\"$CF_EMAIL\",\"token\":\"$CF_TOKEN\"}" \ -# -var r53_config="{\"enabled\":\"$USE_R53\"}" -# terraform apply tf.plan -# cd ../ - -echo "Change directory into '1-setup'." -cd 1-setup -terraform plan -out=tf.plan \ - -var profile=$AWS_PROFILE \ - -var region=$AWS_REGION \ - -var domain_name=$DOMAIN_NAME \ - -var azs="$AWS_AZS" \ - -var coder_license=$LICENSE \ - -var coder_username=$CODER_DB_USERNAME \ - -var coder_password=$CODER_DB_PASSWORD \ - -var coder_admin_email=$CODER_EMAIL \ - -var coder_admin_username=$CODER_USERNAME \ - -var coder_admin_password=$CODER_PASSWORD \ - -var auto_set_record="{\"use_cf\":\"$SET_REC_USE_CF\",\"cf_token\":\"$CF_TOKEN\",\"use_r53\":\"$SET_REC_USE_R53\"}" \ - -var cf_config="{\"enabled\":\"$USE_CF\",\"email\":\"$CF_EMAIL\"}" \ - -var r53_config="{\"enabled\":\"$USE_R53\"}" -terraform apply tf.plan -cd ../ - -echo "Change directory into '2-coder'." -cd 2-coder -terraform plan -out=tf.plan \ - -var profile=$AWS_PROFILE \ - -var region=$AWS_REGION \ - -var domain_name=$DOMAIN_NAME \ - -var coder_license=$LICENSE \ - -var coder_admin_email=$CODER_EMAIL \ - -var coder_admin_password=$CODER_PASSWORD -terraform apply tf.plan -cd ../ \ No newline at end of file diff --git a/infra/1-click/2-clean.sh b/infra/1-click/2-clean.sh index f1ac942..adb2b91 100755 --- a/infra/1-click/2-clean.sh +++ b/infra/1-click/2-clean.sh @@ -4,78 +4,4 @@ set -ae -o pipefail source coder.env -AWS_AZS="${CODER_AWS_AZS:-[\"a\",\"c\"]}" - -AWS_PROFILE="${CODER_AWS_PROFILE:-default}" -AWS_REGION="${CODER_AWS_REGION:-us-east-2}" -DOMAIN_NAME="${CODER_DOMAIN_NAME:-}" -LICENSE="${CODER_LICENSE:-}" - -USE_EXTERN_DNS="${CODER_USE_EXTERN_DNS:-true}" - -USE_R53="${CODER_USE_R53:-true}" - -USE_CF="${CODER_USE_CF:-false}" -CF_TOKEN="${CODER_CF_TOKEN:-}" -CF_EMAIL="${CODER_CF_EMAIL:-}" - -SET_REC_USE_R53=false; ! $USE_EXTERN_DNS && $USE_R53 && SET_REC_USE_R53=true -SET_REC_USE_CF=false; ! $USE_EXTERN_DNS && $USE_CF && SET_REC_USE_CF=true - -CODER_DB_USERNAME="${CODER_DB_USERNAME:-coder}" -CODER_DB_PASSWORD="${CODER_DB_PASSWORD:-th1s1sn0tas3cur3pass0wrd}" - -GRAFANA_DB_USERNAME="${CODER_GRAFANA_DB_PASSWORD:-grafana}" -GRAFANA_DB_PASSWORD="${CODER_GRAFANA_DB_PASSWORD:-th1s1sn0tas3cur3pass0wrd}" - -CODER_USERNAME="${CODER_USERNAME:-admin}" -CODER_EMAIL="${CODER_EMAIL:-admin@coder.com}" -CODER_PASSWORD="${CODER_PASSWORD:-Th1s1sN0TS3CuR3!!}" - -echo "Change directory into '2-coder'." -cd 2-coder -terraform plan -destroy -out=tf.plan \ - -var profile=$AWS_PROFILE \ - -var region=$AWS_REGION \ - -var domain_name=$DOMAIN_NAME \ - -var coder_license=$LICENSE \ - -var coder_admin_email=$CODER_EMAIL \ - -var coder_admin_password=$CODER_PASSWORD -terraform apply tf.plan -cd ../ - -echo "Change directory into '1-setup'." -cd 1-setup -terraform plan -destroy -out=tf.plan \ - -var profile=$AWS_PROFILE \ - -var region=$AWS_REGION \ - -var domain_name=$DOMAIN_NAME \ - -var azs="$AWS_AZS" \ - -var coder_license=$LICENSE \ - -var coder_username=$CODER_DB_USERNAME \ - -var coder_password=$CODER_DB_PASSWORD \ - -var coder_admin_email=$CODER_EMAIL \ - -var coder_admin_username=$CODER_USERNAME \ - -var coder_admin_password=$CODER_PASSWORD \ - -var auto_set_record="{\"use_cf\":\"$SET_REC_USE_CF\",\"cf_token\":\"$CF_TOKEN\",\"use_r53\":\"$SET_REC_USE_R53\"}" \ - -var cf_config="{\"enabled\":\"$USE_CF\",\"email\":\"$CF_EMAIL\"}" \ - -var r53_config="{\"enabled\":\"$USE_R53\"}" -terraform apply tf.plan -cd ../ - -echo "Change directory into '0-infra'." -cd 0-infra -terraform plan -destroy -out=tf.plan \ - -var profile=$AWS_PROFILE \ - -var region=$AWS_REGION \ - -var domain_name=$DOMAIN_NAME \ - -var azs="$AWS_AZS" \ - -var coder_username=$CODER_DB_USERNAME \ - -var coder_password=$CODER_DB_PASSWORD \ - -var grafana_username=$GRAFANA_DB_USERNAME \ - -var grafana_password=$GRAFANA_DB_PASSWORD \ - -var use_ext_dns=$USE_EXTERN_DNS \ - -var cf_config="{\"enabled\":\"$USE_CF\",\"email\":\"$CF_EMAIL\",\"token\":\"$CF_TOKEN\"}" \ - -var r53_config="{\"enabled\":\"$USE_R53\"}" -terraform apply tf.plan -cd ../ \ No newline at end of file +terragrunt run --all --non-interactive destroy \ No newline at end of file From 45bf47d2a1a14a9e5c98dbe0cfca6081d7161869 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 16 Feb 2026 23:14:05 -0800 Subject: [PATCH 28/44] chore: remove unnecessary tmp.coder.env file --- infra/1-click/tmp.coder.env | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 infra/1-click/tmp.coder.env diff --git a/infra/1-click/tmp.coder.env b/infra/1-click/tmp.coder.env deleted file mode 100644 index f7e88cb..0000000 --- a/infra/1-click/tmp.coder.env +++ /dev/null @@ -1,17 +0,0 @@ -CODER_AWS_PROFILE=default -CODER_AWS_REGION=us-east-2 -# CODER_LICENSE="..." - -CODER_USERNAME=admin -CODER_EMAIL=admin@coder.com -CODER_PASSWORD=Th1s1sN0TS3CuR3!! - -CODER_USE_EXTERN_DNS=false - -CODER_DOMAIN_NAME= - -CODER_USE_R53=true - -CODER_USE_CF=false -CODER_CF_EMAIL="..." -CODER_CF_TOKEN="..." \ No newline at end of file From 137a13900b48450af3f19d86ca48c178928a88d6 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Mon, 16 Feb 2026 23:14:18 -0800 Subject: [PATCH 29/44] feat: README updates --- infra/1-click/README.md | 177 ++++++++++++++++++++++++++-------------- 1 file changed, 117 insertions(+), 60 deletions(-) diff --git a/infra/1-click/README.md b/infra/1-click/README.md index 4d8f7d5..7748e50 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -1,16 +1,55 @@ -# Requirements - -Deploying this will take around ~30 min to setup everything (AWS resources, K8s Addons, DNS resolution, and Coder). Cleaning is ~20 min. The below items are required to run this: - -## OSS +# Coder 1-Click + +The following project an opinonated deployment of Coder on AWS EKS. It is not intended to be used in production, but to quickly try out any new Coder features. + +This will create the following resources: + +- Networking + - 1 AWS VPC with public and private subnets + - AWS NAT Gateway + - Security Groups + - Elastic IPs (for Grafana, Coder Server, and Proxy load balancers) +- Compute + - AWS EKS Cluster (with AutoMode enabled) + - Karpenter for dynamic node provisioning + - EC2 nodes (managed by Karpenter NodePools) +- Storage + - AWS S3 Bucket (for Loki logs) + - 2 AWS RDS PostgreSQL Databases (Coder and Grafana) + - AWS EBS volumes (via EBS CSI Driver) +- DNS + - AWS Route53 DNS Records (A records for Grafana, Coder Server, and Proxy) + - AWS ACM SSL Certificates (for Grafana, Coder Server, and Proxy with wildcard support) +- IAM + - IAM User with Bedrock access (for AI features) + - IAM Roles for EKS, Karpenter, and cluster addons +- K8s Addons + - CoreDNS, VPC CNI, Kube-proxy (EKS managed) + - Metrics Server + - AWS Load Balancer Controller + - AWS EBS CSI Driver + - Karpenter +- K8s Apps + - Coder Server + - Coder Proxy (only when Coder license is added) + - Coder Provisioner (only when Coder license is added) + - Coder Logstream + - Monitoring Stack for Coder (Grafana, Loki, Prometheus) + +## Requirements + +Deploying this will take 20-30 minutes to setup everything (AWS resources, K8s Addons, DNS resolution, and Coder). Cleaning is ~20 min. The below items are required to run this: + +### OSS - [Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) +- [Terragrunt](https://terragrunt.gruntwork.io/docs/getting-started/install/) - [AWS Account](https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-creating.html) + [CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) -- A Domain ([Route53](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) or [CloudFlare](https://www.cloudflare.com/learning/dns/how-to-buy-a-domain-name/)) +- [AWS Bedrock/Anthropic Agreement Completed](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html). +- [AWS Route53 Domain](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) -## Enterprise +### Enterprise - Same OSS requirements. - An [Enterprise Coder License](https://coder.com/docs/admin/licensing). -- [AWS Bedrock/Anthropic Agreement Completed](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html). # Getting Started @@ -18,36 +57,36 @@ Deploying this will take around ~30 min to setup everything (AWS resources, K8s 2. Login as a user with AdministratorAccess 3. Setup your local machine to have an [AWS profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). When running `aws login` or `aws configure`, this will automatically setup a `default` profile for you. -4. Purchase/Register a domain in Route53 or CloudFlare and tie it to a public zone. When purchasing, it'll create a zone in a few minutes, wait for this. If using an existing domain/zone, cleanup any conflicting A/AAAA/CNAME/TXT records. +4. Purchase/Register a domain in Route53 and tie it to a public zone. When purchasing, it'll create a zone in a few minutes, wait for this. If using an existing domain, cleanup any conflicting A/AAAA/CNAME/TXT records. 5. Fill the `coder.env` file with the following environment variables: ```env -CODER_AWS_PROFILE=default (Change profile if needed) -CODER_AWS_REGION=us-east-2 (Change region if needed) +CODER_AWS_PROFILE=default # (Change profile if needed) +CODER_AWS_REGION=us-east-2 # (Change region if needed) CODER_DOMAIN_NAME=put.your.domain.here.com -CODER_LICENSE=abcde1234..... (Optional) +CODER_LICENSE=abcde1234..... # (Optional) ``` > [!IMPORTANT] > Upon creating the initial deployment, DO NOT CHANGE THE DOMAIN. Clean up first, then recreate it. The entire infrastructure depends on the name. -6. Initialize the project by running: +6. Initialize the project: ```bash -./0-initialize.sh +./0-init.sh ``` -7. Finally, deploy by running: +7. Finally, deploy: ```bash -./1-plan-n-deploy.sh +./1-apply.sh ``` -# Logging In +## Logging In 1. On your browser, go to "https://put.your.domain.here.com" -2. Enter the following login details (if you didn't override it): +2. Enter the below login details (if you didn't override it): ``` Email: admin@coder.com @@ -56,65 +95,83 @@ Password: Th1s1sN0TS3CuR3!! 3. You're done! -# Cleaning Up +## Cleaning Up 1. [Delete all Coder workspaces](https://coder.com/docs/user-guides/workspace-lifecycle#deleting-workspaces). -2. (Optional) If you've added a license to the .env file, the "Claude Code on Coder" template will be deployed. Update it's ["coder_workspace_preset.prebuilds.instances"](https://registry.terraform.io/providers/coder/coder/latest/docs/data-sources/workspace_preset#instances-1) attribute and set it to 0. - -3. Run the following script to tear down the deployment: +2. Run this script to tear down the deployment: ```bash ./2-clean.sh ``` +## Advanced Configuration + +### Using a Remote State + +You can enable the deployment to store your terraform state remotely if you need to interact with this outside your local desktop. + +This is useful if you plan on running this deployment within an ephemeral environment and need to persist it's state. + +Otherwise, if you're running this on your laptop or on a machine that persists state, then feel free to keep the state local. + +If using a remote state, then follow this to setup the S3 backend: https://spacelift.io/blog/terraform-s3-backend#how-to-create-a-terraform-s3-backend + +Follow this link if you need more details on Terraform's S3 backend: https://developer.hashicorp.com/terraform/language/backend/s3 + +Once the AWS S3 Bucket is setup, set the following environment variables on your `coder.env` file: + +``` +CODER_TF_USE_REMOTE_STATE=true +CODER_TF_BACKEND_AWS_BUCKET_NAME= +CODER_TF_BACKEND_AWS_REGION= +``` + +### Changing the Coder Version + +There's no rolling back in Coder, only updating forward. If you need to change the Coder version, change the following environment variable: + +``` +CODER_VERSION= +``` + +Make sure that the version is ALWAYS greater than the previous. + # Troubleshooting - To verify the state of the infrastructure, visit the AWS Console and look at the pages of following AWS services that this solution deploys/manages: - - EC2 - - VPC - - RDS (Database, Snapshots, and SubnetGroups) - - EKS - - Secrets Manager - - Route53 - - SSM Parameters - - CloudWatch Logs - - Bedrock + - EC2 + - VPC (NAT Gateway, Subnets, Security Groups, EIPs) + - RDS (Database, Snapshots, and SubnetGroups) + - EKS + - Route53 + - CloudWatch Logs + - Bedrock - To verify the cluster state, run `aws eks update-kubeconfig --name --region --profile ` and execute kubectl commands. - To verify the cluster addons, inspect the status/logs of the following cluster addons: - - vpc-cni - - coredns - - metrics-server - - cert-manager - - karpenter - - aws-load-balancer-controller (lb-ctrl) - - aws-ebs-csi (ebs-ctrl) - - external-dns - - external-secrets + - vpc-cni + - coredns + - metrics-server + - karpenter + - aws-load-balancer-controller (lb-ctrl) + - aws-ebs-csi (ebs-ctrl) - To verify the status of resources tied to CRDs, you can inspect the following: - - Service (check the aws-load-balancer-controller deployment for logs) - - ClusterIssuer - - Certificate - - CertificateRequests - - Order - - Challenge - - ClusterSecretStore - - ExternalSecrets - - PushSecrets - - NodePool - - EC2NodeClass - - -- Be careful with running this deployment multiple times in succession. You may be getting rate-limited, either by: - - AWS (for spamming resource creation) - - Let's Encrypt's CA (for spamming certificate signing requests) + - Service (check the aws-load-balancer-controller deployment for logs) + - PersistentVolumeClaim (check the ebs-csi deployment for logs) + - NodePool (EKS AutoMode + Karpenter) + - EC2NodeClass (Karpenter) + - NodeClass (EKS AutoMode) + + +- Be careful with running this deployment multiple times in succession. You may be getting rate-limited by AWS. - If the deployment fails on "data.external.first-user", then you might be running into: - - DNS hasn't propagated yet (you may need to re-run this after waiting a few more minutes) - - Domain name records not set properly. Check Route53 for your domain to see if the K8s "external-dns" addon had properly set the domain to your LoadBalancer - - Local DNS may not also be refreshed yet either (i.e. curl may fail, but browser still accessible) + - DNS hasn't propagated yet (you may need to re-run this after waiting a few more minutes) + - Domain name records not set properly. Check Route53 for your domain to see if the A record was properly set to the correct IP. + - Local DNS may not also be refreshed yet either (i.e. curl may fail, but browser still accessible) + - Load Balancer health checks may be failing (i.e. Availability Zone is temporarily unavailable, or Coder failed to start.) -- EKS Addons may fail installation due to "taking too long". Try re-running the deployment script to verify if changes have finished applying +- EKS Addons may fail installation due to "taking too long". Try re-running the deployment script to verify if changes have finished applying \ No newline at end of file From 4d197dc0baca242537e33b284338669ac5962b7c Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 17 Feb 2026 21:51:07 +0000 Subject: [PATCH 30/44] chore: update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 7db87ab..69fa847 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Other +*.zip + # Terraform .terraform** terraform.tfstate* From 5f8c9551bdb474a53c7bbc351316531875e1cec5 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 17 Feb 2026 21:51:45 +0000 Subject: [PATCH 31/44] chore: adds config to point to root.hcl --- infra/1-click/0-init.sh | 2 +- infra/1-click/1-apply.sh | 2 +- infra/1-click/2-clean.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/1-click/0-init.sh b/infra/1-click/0-init.sh index 369a3ad..83cb72f 100755 --- a/infra/1-click/0-init.sh +++ b/infra/1-click/0-init.sh @@ -4,4 +4,4 @@ set -ae -o pipefail source coder.env -terragrunt run --all --non-interactive init -- -migrate-state -upgrade \ No newline at end of file +terragrunt run --all --non-interactive --config root.hcl init -- -migrate-state -upgrade \ No newline at end of file diff --git a/infra/1-click/1-apply.sh b/infra/1-click/1-apply.sh index 8b5cf7b..a36c589 100755 --- a/infra/1-click/1-apply.sh +++ b/infra/1-click/1-apply.sh @@ -4,4 +4,4 @@ set -ae -o pipefail source coder.env -terragrunt run --all --non-interactive apply \ No newline at end of file +terragrunt run --all --non-interactive --config root.hcl apply \ No newline at end of file diff --git a/infra/1-click/2-clean.sh b/infra/1-click/2-clean.sh index adb2b91..73c1463 100755 --- a/infra/1-click/2-clean.sh +++ b/infra/1-click/2-clean.sh @@ -4,4 +4,4 @@ set -ae -o pipefail source coder.env -terragrunt run --all --non-interactive destroy \ No newline at end of file +terragrunt run --all --non-interactive --config root.hcl destroy \ No newline at end of file From 387c06608a44651423fd780075f49c3b42dc8599 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 17 Feb 2026 21:52:07 +0000 Subject: [PATCH 32/44] chore: update README --- infra/1-click/README.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/infra/1-click/README.md b/infra/1-click/README.md index 7748e50..dabcc29 100644 --- a/infra/1-click/README.md +++ b/infra/1-click/README.md @@ -41,9 +41,10 @@ This will create the following resources: Deploying this will take 20-30 minutes to setup everything (AWS resources, K8s Addons, DNS resolution, and Coder). Cleaning is ~20 min. The below items are required to run this: ### OSS -- [Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) -- [Terragrunt](https://terragrunt.gruntwork.io/docs/getting-started/install/) -- [AWS Account](https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-creating.html) + [CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) +- [Terraform >= v1.14.1](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) +- [Terragrunt (>= v0.78.0)](https://terragrunt.gruntwork.io/docs/getting-started/install/) +- [AWS CLI (>= v2.33.23)](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) +- [AWS Account](https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-creating.html) - [AWS Bedrock/Anthropic Agreement Completed](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html). - [AWS Route53 Domain](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/domain-register.html#domain-register-procedure-section) @@ -55,7 +56,10 @@ Deploying this will take 20-30 minutes to setup everything (AWS resources, K8s A 1. Create or acquire an AWS account 2. Login as a user with AdministratorAccess -3. Setup your local machine to have an [AWS profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). When running `aws login` or `aws configure`, this will automatically setup a `default` profile for you. +3. Setup your local machine to have an [AWS profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). + - When running `aws configure`, this will automatically setup a `default` profile for you. + - If using `aws configure sso` you must manually set the profile name to `default`. Afterwards, you can login with `aws sso login` + - If using `aws login` follow this [thread](https://github.com/hashicorp/terraform-provider-aws/issues/45316) to see the fix. 4. Purchase/Register a domain in Route53 and tie it to a public zone. When purchasing, it'll create a zone in a few minutes, wait for this. If using an existing domain, cleanup any conflicting A/AAAA/CNAME/TXT records. @@ -139,6 +143,14 @@ Make sure that the version is ALWAYS greater than the previous. # Troubleshooting +- When logging in to AWS you might run into issues such as: `An error occurred (InvalidRequestException)`. Make sure that: + - Your spelling is correct. + - The correct region for your SSO Start URL is set. + - You're consuming the correct SSO URL. + +- When initializing the project, and if you ran `aws login`, you might run into: `Error: failed to refresh cached credentials, no EC2 IMDS role found`. + - View this [thread]((https://github.com/hashicorp/terraform-provider-aws/issues/45316)) to see how to workaround this. Otherwise, try the other options to login (manually setting the profile/credentials, or use AWS SSO). + - To verify the state of the infrastructure, visit the AWS Console and look at the pages of following AWS services that this solution deploys/manages: - EC2 - VPC (NAT Gateway, Subnets, Security Groups, EIPs) From f740501aa8014e48372bb8d18e1df1bb36bf4c2c Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 24 Feb 2026 21:54:41 +0000 Subject: [PATCH 33/44] feat: refactor modules --- modules/k8s/bootstrap/cert-manager/main.tf | 26 +-- .../k8s/bootstrap/coder-provisioner/main.tf | 38 +--- modules/k8s/bootstrap/coder-proxy/main.tf | 59 ++--- modules/k8s/bootstrap/coder-server/main.tf | 57 ++--- modules/k8s/bootstrap/ebs-csi/main.tf | 6 + modules/k8s/bootstrap/litellm/main.tf | 202 +++++++----------- modules/k8s/bootstrap/monitoring/main.tf | 196 +++++++++++++---- 7 files changed, 287 insertions(+), 297 deletions(-) diff --git a/modules/k8s/bootstrap/cert-manager/main.tf b/modules/k8s/bootstrap/cert-manager/main.tf index b2de974..ff445c0 100644 --- a/modules/k8s/bootstrap/cert-manager/main.tf +++ b/modules/k8s/bootstrap/cert-manager/main.tf @@ -70,7 +70,7 @@ data "aws_region" "this" {} data "aws_caller_identity" "this" {} -resource "kubernetes_namespace" "this" { +resource "kubernetes_namespace_v1" "this" { metadata { name = var.namespace } @@ -78,7 +78,7 @@ resource "kubernetes_namespace" "this" { resource "helm_release" "cert-manager" { name = "cert-manager" - namespace = kubernetes_namespace.this.metadata[0].name + namespace = kubernetes_namespace_v1.this.metadata[0].name chart = "cert-manager" repository = "oci://quay.io/jetstack/charts" create_namespace = false @@ -164,52 +164,52 @@ module "oidc-role" { tags = var.tags } -resource "kubernetes_service_account" "r53" { +resource "kubernetes_service_account_v1" "r53" { count = var.r53_config.enabled ? 1 : 0 metadata { name = "${var.r53_config.role_name}" - namespace = kubernetes_namespace.this.metadata[0].name + namespace = kubernetes_namespace_v1.this.metadata[0].name annotations = { "eks.amazonaws.com/role-arn" = module.oidc-role[0].role_arn } } } -resource "kubernetes_role" "r53" { +resource "kubernetes_role_v1" "r53" { count = var.r53_config.enabled ? 1 : 0 metadata { name = "${var.r53_config.role_name}-tokenrequest" - namespace = kubernetes_namespace.this.metadata[0].name + namespace = kubernetes_namespace_v1.this.metadata[0].name } rule { api_groups = [""] resources = ["serviceaccounts/token"] - resource_names = [kubernetes_service_account.r53[0].metadata[0].name] + resource_names = [kubernetes_service_account_v1.r53[0].metadata[0].name] verbs = ["create"] } } -resource "kubernetes_role_binding" "r53" { +resource "kubernetes_role_binding_v1" "r53" { count = var.r53_config.enabled ? 1 : 0 metadata { name = "${var.r53_config.role_name}-tokenrequest" - namespace = kubernetes_namespace.this.metadata[0].name + namespace = kubernetes_namespace_v1.this.metadata[0].name } subject { kind = "ServiceAccount" name = "cert-manager" - namespace = kubernetes_namespace.this.metadata[0].name + namespace = kubernetes_namespace_v1.this.metadata[0].name } role_ref { api_group = "rbac.authorization.k8s.io" kind = "Role" - name = kubernetes_role.r53[0].metadata[0].name + name = kubernetes_role_v1.r53[0].metadata[0].name } } @@ -240,13 +240,13 @@ locals { cf_annot_email_key = "custom.kubernetes.secret/email" } -resource "kubernetes_secret" "cloudflare" { +resource "kubernetes_secret_v1" "cloudflare" { count = var.cf_config.enabled ? 1 : 0 metadata { name = var.cf_config.name - namespace = kubernetes_namespace.this.metadata[0].name + namespace = kubernetes_namespace_v1.this.metadata[0].name annotations = { "${local.cf_annot_sec_key}" = var.cf_config.key "${local.cf_annot_email_key}" = var.cf_config.email diff --git a/modules/k8s/bootstrap/coder-provisioner/main.tf b/modules/k8s/bootstrap/coder-provisioner/main.tf index 4a2ec51..1f9a59c 100644 --- a/modules/k8s/bootstrap/coder-provisioner/main.tf +++ b/modules/k8s/bootstrap/coder-provisioner/main.tf @@ -114,12 +114,7 @@ variable "node_selector" { } variable "tolerations" { - type = list(object({ - key = string - operator = optional(string, "Equal") - value = string - effect = optional(string, "NoSchedule") - })) + type = any default = [] } @@ -136,17 +131,9 @@ variable "topology_spread" { default = [] } -variable "pod_aaf_pref_sched_ie" { - type = list(object({ - weight = number - pod_affinity_term = object({ - label_selector = object({ - match_labels = map(string) - }) - topology_key = string - }) - })) - default = [] +variable "affinity" { + type = any + default = {} } ## @@ -217,17 +204,6 @@ resource "kubernetes_secret_v1" "ext-prov" { } locals { - pod_aaf_pref_sched_ie = [ - for k, v in var.pod_aaf_pref_sched_ie : { - weight = v.weight - podAffinityTerm = { - labelSelector = { - matchLabels = try(v.pod_affinity_term.label_selector.match_labels, {}) - } - topologyKey = try(v.pod_affinity_term.topology_key, {}) - } - } - ] topology_spread = [ for k, v in var.topology_spread : { maxSkew = v.max_skew @@ -299,11 +275,7 @@ resource "helm_release" "coder-provisioner" { replicaCount = var.coder.rep_cnt tolerations = var.tolerations topologySpreadConstraints = local.topology_spread - affinity = { - podAntiAffinity = { - preferredDuringSchedulingIgnoredDuringExecution = local.pod_aaf_pref_sched_ie - } - } + affinity = var.affinity } provisionerDaemon = { keySecretKey = kubernetes_secret_v1.ext-prov.metadata[0].annotations["custom.kubernetes.secret/key"] diff --git a/modules/k8s/bootstrap/coder-proxy/main.tf b/modules/k8s/bootstrap/coder-proxy/main.tf index 7184b78..bdb2865 100644 --- a/modules/k8s/bootstrap/coder-proxy/main.tf +++ b/modules/k8s/bootstrap/coder-proxy/main.tf @@ -13,9 +13,6 @@ terraform { coderd = { source = "coder/coderd" } - acme = { - source = "vancluever/acme" - } tls = { source = "hashicorp/tls" } @@ -98,14 +95,8 @@ variable "resource_request" { } variable "resource_limit" { - type = object({ - cpu = string - memory = string - }) - default = { - cpu = "500m" - memory = "1Gi" - } + type = map(string) + default = {} } variable "svc_annot" { @@ -124,12 +115,7 @@ variable "node_selector" { } variable "tolerations" { - type = list(object({ - key = string - operator = optional(string, "Equal") - value = string - effect = optional(string, "NoSchedule") - })) + type = list(any) default = [] } @@ -146,17 +132,9 @@ variable "topology_spread" { default = [] } -variable "pod_aaf_pref_sched_ie" { - type = list(object({ - weight = number - pod_affinity_term = object({ - label_selector = object({ - match_labels = map(string) - }) - topology_key = string - }) - })) - default = [] +variable "affinity" { + type = any + default = {} } variable "termination_grace_period" { @@ -184,17 +162,6 @@ locals { } secret_key = "key" secret_keys = keys(local.secrets) - pod_aaf_pref_sched_ie = [ - for k, v in var.pod_aaf_pref_sched_ie : { - weight = v.weight - podAffinityTerm = { - labelSelector = { - matchLabels = try(v.pod_affinity_term.label_selector.match_labels, {}) - } - topologyKey = try(v.pod_affinity_term.topology_key, {}) - } - } - ] topology_spread = [ for k, v in var.topology_spread : { maxSkew = v.max_skew @@ -269,7 +236,7 @@ resource "kubernetes_service_v1" "coder" { name = "https" port = 443 protocol = "TCP" - target_port = "http" + target_port = var.proxy.mount_ssl ? "https" : "http" } selector = { "app.kubernetes.io/instance" = var.release_name @@ -306,7 +273,7 @@ resource "helm_release" "coder-proxy" { enable = false } tls = { - secretNames = var.coder.mount_ssl ? [ var.coder.mount_ssl_name ] : [] + secretNames = var.proxy.mount_ssl ? [ var.proxy.mount_ssl_name ] : [] } replicaCount = var.proxy.rep_cnt resources = { @@ -319,12 +286,12 @@ resource "helm_release" "coder-proxy" { nodeSelector = var.node_selector tolerations = var.tolerations topologySpreadConstraints = local.topology_spread - affinity = { - podAntiAffinity = { - preferredDuringSchedulingIgnoredDuringExecution = local.pod_aaf_pref_sched_ie - } - } + affinity = var.affinity terminationGracePeriodSeconds = var.termination_grace_period } })] +} + +output "namespace" { + value = kubernetes_namespace_v1.this.metadata[0].name } \ No newline at end of file diff --git a/modules/k8s/bootstrap/coder-server/main.tf b/modules/k8s/bootstrap/coder-server/main.tf index 95459a4..af28b77 100644 --- a/modules/k8s/bootstrap/coder-server/main.tf +++ b/modules/k8s/bootstrap/coder-server/main.tf @@ -134,14 +134,8 @@ variable "tags" { } variable "resource_limit" { - type = object({ - cpu = string - memory = string - }) - default = { - cpu = "4000m" - memory = "8Gi" - } + type = map(any) + default = {} } variable "svc_annot" { @@ -165,12 +159,7 @@ variable "node_selector" { } variable "tolerations" { - type = list(object({ - key = string - operator = optional(string, "Equal") - value = string - effect = optional(string, "NoSchedule") - })) + type = list(any) default = [] } @@ -187,17 +176,9 @@ variable "topology_spread" { default = [] } -variable "pod_aaf_pref_sched_ie" { - type = list(object({ - weight = number - pod_affinity_term = object({ - label_selector = object({ - match_labels = map(string) - }) - topology_key = string - }) - })) - default = [] +variable "affinity" { + type = any + default = {} } variable "db" { @@ -401,7 +382,8 @@ locals { local.oidc, local.oauth2, local.extern_auth, - local.aibridge + local.aibridge, + var.coder.env_vars ) : { name = k, value = tostring(v) @@ -439,17 +421,6 @@ data "aws_region" "this" {} data "aws_caller_identity" "this" {} locals { - pod_aaf_pref_sched_ie = [ - for k, v in var.pod_aaf_pref_sched_ie : { - weight = v.weight - podAffinityTerm = { - labelSelector = { - matchLabels = try(v.pod_affinity_term.label_selector.match_labels, {}) - } - topologyKey = try(v.pod_affinity_term.topology_key, {}) - } - } - ] topology_spread = [ for k, v in var.topology_spread : { maxSkew = v.max_skew @@ -527,7 +498,7 @@ resource "kubernetes_service_v1" "coder" { name = "https" port = 443 protocol = "TCP" - target_port = "http" + target_port = var.coder.mount_ssl ? "https" : "http" } selector = { "app.kubernetes.io/instance" = var.chart_name @@ -582,6 +553,10 @@ resource "helm_release" "coder-server" { pullSecrets = var.coder.image_pull_secrets } env = local.env + annotations = var.prometheus.enable ? { + "prometheus.io/scrape" = "true" + "prometheus.io/port" = kubernetes_service_v1.prometheus[0].spec[0].port[0].port + } : {} podAnnotations = var.prometheus.enable ? { "prometheus.io/scrape" = "true" "prometheus.io/port" = kubernetes_service_v1.prometheus[0].spec[0].port[0].port @@ -605,11 +580,7 @@ resource "helm_release" "coder-server" { nodeSelector = var.node_selector tolerations = var.tolerations topologySpreadConstraints = local.topology_spread - affinity = { - podAntiAffinity = { - preferredDuringSchedulingIgnoredDuringExecution = local.pod_aaf_pref_sched_ie - } - } + affinity = var.affinity terminationGracePeriodSeconds = var.termination_grace_period } })] diff --git a/modules/k8s/bootstrap/ebs-csi/main.tf b/modules/k8s/bootstrap/ebs-csi/main.tf index 6337270..f1578c9 100644 --- a/modules/k8s/bootstrap/ebs-csi/main.tf +++ b/modules/k8s/bootstrap/ebs-csi/main.tf @@ -53,6 +53,11 @@ variable "node_selector" { default = {} } +variable "affinity" { + type = map(any) + default = {} +} + variable "tolerations" { type = list(map(any)) default = [] @@ -110,6 +115,7 @@ resource "helm_release" "ebs-controller" { } nodeSelector = var.node_selector tolerations = var.tolerations + affinity = var.affinity } })] } diff --git a/modules/k8s/bootstrap/litellm/main.tf b/modules/k8s/bootstrap/litellm/main.tf index c4ad5ae..7cf9cb0 100644 --- a/modules/k8s/bootstrap/litellm/main.tf +++ b/modules/k8s/bootstrap/litellm/main.tf @@ -37,6 +37,7 @@ variable "namespace" { variable "chart_version" { type = string default = "0.1.830" + # 1.81.13 } variable "cluster_oidc_provider_arn" { @@ -56,28 +57,6 @@ data "aws_eks_cluster_auth" "this" { name = var.cluster_name } -variable "registry_config" { - type = object({ - url = optional(string, "oci://ghcr.io") - username = string - password = string - }) - sensitive = true -} - -provider "helm" { - kubernetes = { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token - } - registries = [{ - url = nonsensitive(var.registry_config.url) - username = nonsensitive(var.registry_config.username) - password = var.registry_config.password - }] -} - variable "tags" { type = map(string) default = {} @@ -112,7 +91,7 @@ variable "litellm_master_secret_name" { default = "masterkey" } -variable "db_config" { +variable "db" { type = object({ use_existing = optional(bool, false) secret_name = optional(string, "postgres") @@ -144,12 +123,12 @@ variable "service_lb_class" { default = "service.k8s.aws/nlb" } -variable "service_annotations" { +variable "svc_annots" { type = map(string) default = {} } -variable "service_port" { +variable "svc_port" { type = number default = 80 } @@ -169,13 +148,12 @@ variable "env_vars" { default = {} } -variable "volumes" { - type = list(any) - default = [] -} - -variable "volume_mounts" { - type = list(any) +variable "mounts" { + type = list(object({ + secret_name = optional(string, "") + path = optional(string, "") + read_only = optional(bool, false) + })) default = [] } @@ -251,14 +229,14 @@ resource "kubernetes_secret_v1" "master-key" { resource "kubernetes_secret_v1" "db-auth" { metadata { - name = nonsensitive(var.db_config.secret_name) + name = nonsensitive(var.db.secret_name) namespace = kubernetes_namespace_v1.litellm.metadata[0].name } data = { - username = nonsensitive(var.db_config.username) - postgres-password = var.db_config.admin_password - password = var.db_config.user_password - endpoint = var.db_config.endpoint + username = nonsensitive(var.db.username) + postgres-password = var.db.admin_password + password = var.db.user_password + endpoint = var.db.endpoint } type = "Opaque" } @@ -281,62 +259,38 @@ variable "ssl_cert_config" { } } -locals { - common_name = replace(replace(var.access_url, "https://", ""), "http://", "") -} - -resource "kubernetes_manifest" "cert" { - - count = var.ssl_cert_config.create_secret ? 1 : 0 - - field_manager { - force_conflicts = true - } - manifest = { - apiVersion = "cert-manager.io/v1" - kind = "Certificate" - metadata = { - labels = {} # var.cert_labels - name = var.ssl_cert_config.name - namespace = kubernetes_namespace_v1.litellm.metadata[0].name - } - spec = { - secretName = var.ssl_cert_config.name - commonName = local.common_name - dnsNames = [local.common_name] - duration = "${var.ssl_cert_config.days_until_renewal * 24}h" - renewBefore = "8h" - additionalOutputFormats = [{ - type = "CombinedPEM" - },{ - type = "DER" - }] - issuerRef = { - kind = "ClusterIssuer" - name = "issuer" - } - } +variable "mount_ssl" { + type = object({ + enable = optional(bool, true) + secret_name = optional(string, "ssl-cert") + path = optional(string, "") + key_name = optional(string, "tls.key") + crt_name = optional(string, "tls.crt") + pem_name = optional(string, "tls-combined.pem") + }) + default = { + enable = false + name = "ssl-cert" + path = "/tmp/ssl/ssl-cert" + key_name = "tls.key" + crt_name = "tls.crt" + pem_name = "tls-combined.pem" } } locals { - ssl_volume = var.ssl_cert_config.create_secret ? {} : {} -} - -variable "gcloud_auth" { - type = string - sensitive = true -} - -resource "kubernetes_secret_v1" "gcloud" { - metadata { - name = "gcloud-auth" - namespace = kubernetes_namespace_v1.litellm.metadata[0].name - labels = {} - } - data = { - "service_account.json" = var.gcloud_auth - } + volumes = [ for v in var.mounts : merge(v.secret_name != "" ? { + name = v.secret_name + secret = { + secretName = v.secret_name + optional = false + } + } : null) ] + volumeMounts = [ for v in var.mounts : merge(v.secret_name != "" ? { + name = v.secret_name + mountPath = v.path + readOnly = v.read_only + } : null) ] } resource "helm_release" "litellm" { @@ -374,8 +328,8 @@ resource "helm_release" "litellm" { service = { type = "LoadBalancer" loadBalancerClass = var.service_lb_class - port = var.service_port - annotations = var.service_annotations + port = var.svc_port + annotations = var.svc_annots } separateHealthApp = true separateHealthPort = var.health_port @@ -402,9 +356,9 @@ resource "helm_release" "litellm" { affinity = var.affinity db = { - deployStandalone = var.db_config.endpoint == "localhost" - useExisting = nonsensitive(var.db_config.use_existing) - database = nonsensitive(var.db_config.db_name) + deployStandalone = var.db.endpoint == "localhost" + useExisting = nonsensitive(var.db.use_existing) + database = nonsensitive(var.db.db_name) url = "postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)" secret = { name = kubernetes_secret_v1.db-auth.metadata[0].name @@ -420,7 +374,7 @@ resource "helm_release" "litellm" { postgresql = { architecture = "standalone" auth = { - username = var.db_config.username + username = var.db.username database = "litellm" enablePostgresUser = true @@ -465,42 +419,38 @@ resource "helm_release" "litellm" { envVars = merge({ NO_DOCS = "False" - }, merge(var.access_url != "" ? { - # PROXY_BASE_URL = "${var.access_url}" - } : {}, var.ssl_cert_config.create_secret ? { - SSL_CERT_FILE = "/tmp/ssl/${local.common_name}/tls-combined.pem" - SSL_KEYFILE_PATH = "/tmp/ssl/${local.common_name}/tls.key" - SSL_CERTFILE_PATH = "/tmp/ssl/${local.common_name}/tls.crt" + }, var.mount_ssl.enable ? { + SSL_CERT_FILE = "${var.mount_ssl.path}/${var.mount_ssl.pem_name}" + SSL_KEYFILE_PATH = "${var.mount_ssl.path}/${var.mount_ssl.key_name}" + SSL_CERTFILE_PATH = "${var.mount_ssl.path}/${var.mount_ssl.crt_name}" SSL_VERIFY = "False" - } : {})) + } : {}, + var.env_vars + ) extraEnvVars = {} # Additional volumes on the output Deployment definition. - volumes = [{ - name = kubernetes_manifest.cert[0].manifest.metadata.name - secret = { - secretName = kubernetes_manifest.cert[0].manifest.metadata.name - optional = false - } - },{ - name = kubernetes_secret_v1.gcloud.metadata[0].name - secret = { - secretName = kubernetes_secret_v1.gcloud.metadata[0].name - optional = false - } - }] - - # Additional volumeMounts on the output Deployment definition. - volumeMounts = [{ - name = kubernetes_manifest.cert[0].manifest.metadata.name - mountPath = "/tmp/ssl/${local.common_name}" - readOnly = true - },{ - name = kubernetes_secret_v1.gcloud.metadata[0].name - mountPath = "/tmp/gcloud/" - readOnly = true - },] + + volumes = concat( + var.mount_ssl.enable ? [{ + name = var.mount_ssl.secret_name + secret = { + secretName = var.mount_ssl.secret_name + optional = false + } + }] : [], + local.volumes + ) + + volumeMounts = concat( + var.mount_ssl.enable ? [{ + name = var.mount_ssl.secret_name + mountPath = var.mount_ssl.path + readOnly = true + }] : [], + local.volumeMounts + ) })] } diff --git a/modules/k8s/bootstrap/monitoring/main.tf b/modules/k8s/bootstrap/monitoring/main.tf index ebc8f70..ada12f6 100644 --- a/modules/k8s/bootstrap/monitoring/main.tf +++ b/modules/k8s/bootstrap/monitoring/main.tf @@ -36,11 +36,33 @@ variable "storage_class" { default = "" } +variable "namespace" { + type = string + default = "coder-observe" +} + +variable "dashboards" { + type = object({ + use_builtins = optional(bool, true) + default_home_path = optional(string, "") + config_maps = optional(map(object({ + mount_path = optional(string, "") + local_path = optional(string, "") + args = optional(map(string), {}) + read_only = optional(bool, false) + optional = optional(bool, true) + }))) + }) + default = { + use_builtins = true + config_maps = {} + } +} + variable "coder" { type = object({ db = object({ host = string - port = optional(number, 5432) password = string username = string database = string @@ -59,21 +81,23 @@ variable "coder" { variable "grafana" { type = object({ + instance_name = optional(string, "Coder Environment") admin = object({ username = string password = string }) db = object({ host = string - port = optional(number, 5432) password = string username = string database = string sslmode = optional(string, "require") }) svc = object({ - annots = map(string) + port = optional(number, 80) + annots = optional(map(string), {}) }) + affinity = optional(map(any), {}) }) sensitive = true } @@ -88,6 +112,23 @@ variable "loki" { }) } +variable "mount_ssl" { + type = object({ + enable = optional(bool, true) + secret_name = optional(string, "ssl-cert") + mount_path = optional(string, "") + key_name = optional(string, "tls.key") + crt_name = optional(string, "tls.crt") + }) + default = { + enable = false + secret_name = "ssl-cert" + mount_path = "" + key_name = "tls.key" + crt_name = "tls.crt" + } +} + variable "domain_name" { type = string } @@ -98,7 +139,7 @@ variable "lb_class" { } variable "tolerations" { - type = list(map(any)) + type = list(any) default = [] } @@ -111,17 +152,25 @@ variable "system_tolerations" { }] } +variable "system_affinity" { + description = "(Optional) Override if you need to adjust where critical monitoring addons need to be moved." + type = any + default = {} +} + variable "daemonset_tolerations" { description = "(Optional) Override if you need to adjust where monitoring DaemonSets need to be placed." - type = list(map(any)) + type = list(any) default = [{ effect = "NoSchedule" operator = "Exists" }] } -data "aws_s3_bucket" "loki" { - bucket = "${var.cluster_name}-grafana" +variable "daemonset_node_selector" { + description = "(Optional) Override if you need to adjust where monitoring DaemonSets need to be placed." + type = map(string) + default = {} } data "aws_region" "this" {} @@ -129,9 +178,6 @@ data "aws_region" "this" {} data "aws_caller_identity" "this" {} locals { - region = data.aws_region.this.region - account_id = data.aws_caller_identity.this.account_id - policy_name = "loki-s3-access" role_name = "loki-s3-access" } @@ -154,10 +200,39 @@ module "oidc-role" { resource "kubernetes_namespace_v1" "this" { metadata { - name = "coder-observe" + name = var.namespace } } +resource "kubernetes_config_map_v1" "dashboard" { + + for_each = var.dashboards.config_maps + + metadata { + name = each.key + namespace = kubernetes_namespace_v1.this.metadata[0].name + } + data = { + (element(split("/", each.value.local_path), -1)) = templatefile(each.value.local_path, each.value.args) + } +} + +resource "random_id" "grafana_server_secret" { + keepers = { + # Generate a new secret if the admin password changes + grafana_admin_username = var.grafana.admin.username + grafana_admin_password = var.grafana.admin.password + } + byte_length = 16 +} + +locals { + coder_db_host = split(":", var.coder.db.host)[0] + coder_db_port = split(":", var.coder.db.host)[1] + grafana_db_host = split(":", var.grafana.db.host)[0] + grafana_db_port = split(":", var.grafana.db.host)[1] +} + resource "helm_release" "coder-observe" { name = "coder-observe" namespace = kubernetes_namespace_v1.this.metadata[0].name @@ -181,18 +256,26 @@ resource "helm_release" "coder-observe" { externalProvisionersNamespace = var.coder.selector.ext_prov_ns } postgres = { - hostname = var.coder.db.host - port = var.coder.db.port + hostname = local.coder_db_host + port = local.coder_db_port password = var.coder.db.password username = var.coder.db.username database = var.coder.db.database sslmode = var.coder.db.sslmode mountSecret = "" } + dashboards = { + enabled = var.dashboards.use_builtins + } } prometheus = { + # prometheus-node-exporter = { + # tolerations = var.daemonset_tolerations + # nodeSelector = var.daemonset_node_selector + # } server = { tolerations = var.system_tolerations + affinity = var.system_affinity persistentVolume = { enabled = true storageClassName = var.storage_class @@ -201,6 +284,7 @@ resource "helm_release" "coder-observe" { alertmanager = { enabled = true tolerations = var.system_tolerations + affinity = var.system_affinity persistence = { enabled = true storageClass = var.storage_class @@ -208,6 +292,8 @@ resource "helm_release" "coder-observe" { } } grafana = { + # https://github.com/grafana/helm-charts/blob/grafana-7.3.7/charts/grafana/values.yaml#L1313-L1321 + assertNoLeakedSecrets = false adminUser = var.grafana.admin.username adminPassword = var.grafana.admin.password env = { @@ -218,32 +304,52 @@ resource "helm_release" "coder-observe" { "auth.anonymous" = { enabled = false } - instance_name = "Coder Environment" + dashboards = { + default_home_dashboard_path = var.dashboards.default_home_path + } + instance_name = var.grafana.instance_name database = { - url = "postgres://${var.grafana.db.username}:${var.grafana.db.password}@${var.grafana.db.host}:${var.grafana.db.port}/${var.grafana.db.database}" - ssl_mode = var.grafana.db.sslmode + host = local.grafana_db_host + port = local.grafana_db_port + name = var.grafana.db.database + username = var.grafana.db.username + password = "\"\"\"${var.grafana.db.password}\"\"\"" + ssl_mode = var.grafana.db.sslmode } - server = { - root_url = "https://${var.domain_name}" - domain = var.domain_name - enforce_domain = true - http_port = 3000 - protocol = "http" + security = { + secret_key = random_id.grafana_server_secret.hex + cookie_secure = true + cookie_samesite = "lax" + cookie_domain = var.domain_name } + server = merge(var.mount_ssl.enable ? { + cert_key = "${trimsuffix(var.mount_ssl.mount_path, "/")}/${var.mount_ssl.key_name}" + cert_file = "${trimsuffix(var.mount_ssl.mount_path, "/")}/${var.mount_ssl.crt_name}" + protocol = "https" + root_url = "https://${var.domain_name}" + } : { + protocol = "http" + root_url = "http://${var.domain_name}" + }, { + domain = var.domain_name + enforce_domain = false + http_port = 3000 + }) users = { allow_sign_up = false } } - replicas = 2 + affinity = var.system_affinity + replicas = 1 useStatefulSet = true readinessProbe = { httpGet = { - scheme = "HTTP" + scheme = var.mount_ssl.enable ? "HTTPS" : "HTTP" } } livenessProbe = { httpGet = { - scheme = "HTTP" + scheme = var.mount_ssl.enable ? "HTTPS" : "HTTP" } } persistence = { @@ -251,25 +357,45 @@ resource "helm_release" "coder-observe" { } podAnnotations = { "prometheus.io/port" = "3000" - "prometheus.io/scheme" = "http" + "prometheus.io/scheme" = var.mount_ssl.enable ? "https" : "http" "prometheus.io/scrape" = "true" } service = { - annotations = var.grafana.svc.annots enabled = true externalTrafficPolicy = "Cluster" internalTrafficPolicy = "Cluster" loadBalancerClass = var.lb_class - port = 443 + port = var.grafana.svc.port targetPort = 3000 type = "LoadBalancer" + annotations = var.grafana.svc.annots } tolerations = var.system_tolerations + affinity = var.system_affinity + extraConfigmapMounts = [ for k, v in var.dashboards.config_maps : { + name = k + configMap = kubernetes_config_map_v1.dashboard[k].metadata[0].name + mountPath = v.mount_path + readOnly = v.read_only + optional = v.optional + } ] + extraSecretMounts = var.mount_ssl.enable ? [{ + name = var.mount_ssl.secret_name + mountPath = var.mount_ssl.mount_path + secretName = var.mount_ssl.secret_name + readOnly = true + }] : [] } grafana-agent = { enabled = true controller = { - tolerations = var.daemonset_tolerations + type = "daemonset" + podAnnotations = { + "prometheus.io/scheme" = "http" + "prometheus.io/scrape" = "true" + } + tolerations = var.daemonset_tolerations + nodeSelector = var.daemonset_node_selector } discovery = <<-EOF // Discover k8s nodes @@ -336,13 +462,6 @@ resource "helm_release" "coder-observe" { replacement = "__param_$1" } EOF - controller = { - type = "daemonset" - podAnnotations = { - "prometheus.io/scheme" = "http" - "prometheus.io/scrape" = "true" - } - } } loki = { loki = { @@ -359,9 +478,11 @@ resource "helm_release" "coder-observe" { } lokiCanary = { tolerations = var.daemonset_tolerations + nodeSelector = var.daemonset_node_selector } backend = { tolerations = var.system_tolerations + affinity = var.system_affinity persistence = { volumeClaimsEnabled = false # storageClass = var.storage_class @@ -369,9 +490,11 @@ resource "helm_release" "coder-observe" { } resultsCache = { tolerations = var.system_tolerations + affinity = var.system_affinity } chunksCache = { tolerations = var.system_tolerations + affinity = var.system_affinity persistence = { enabled = true storageClass = var.storage_class @@ -383,6 +506,7 @@ resource "helm_release" "coder-observe" { } write = { tolerations = var.system_tolerations + affinity = var.system_affinity persistence = { volumeClaimsEnabled = false # storageClass = var.storage_class From c14ccfb6aae6721e91869f35f83080600310ace7 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 24 Feb 2026 21:55:37 +0000 Subject: [PATCH 34/44] feat: transition to using terragrunt --- infra/aws/eu-west-2/eks/terragrunt.hcl | 20 ++ .../eu-west-2/k8s/cert-manager/terragrunt.hcl | 20 ++ .../eu-west-2/k8s/coder-proxy/terragrunt.hcl | 40 ++++ .../k8s/lb-controller/terragrunt.hcl | 23 +++ .../k8s/metrics-server/terragrunt.hcl | 20 ++ infra/aws/eu-west-2/k8s/other/terragrunt.hcl | 25 +++ infra/aws/eu-west-2/vpc/terragrunt.hcl | 21 ++ infra/aws/root.hcl | 179 ++++++++++++++++++ infra/aws/us-east-2/eks/terragrunt.hcl | 20 ++ .../us-east-2/k8s/cert-manager/terragrunt.hcl | 20 ++ .../us-east-2/k8s/coder-server/terragrunt.hcl | 58 ++++++ .../aws/us-east-2/k8s/coder-ws/terragrunt.hcl | 29 +++ .../aws/us-east-2/k8s/ebs-csi/terragrunt.hcl | 21 ++ .../us-east-2/k8s/karpenter/terragrunt.hcl | 20 ++ .../k8s/lb-controller/terragrunt.hcl | 22 +++ .../aws/us-east-2/k8s/litellm/terragrunt.hcl | 35 ++++ .../k8s/metrics-server/terragrunt.hcl | 20 ++ .../us-east-2/k8s/monitoring/terragrunt.hcl | 48 +++++ infra/aws/us-east-2/k8s/other/terragrunt.hcl | 23 +++ infra/aws/us-east-2/rds/terragrunt.hcl | 35 ++++ infra/aws/us-east-2/s3/terragrunt.hcl | 11 ++ infra/aws/us-east-2/vpc/terragrunt.hcl | 17 ++ infra/aws/us-west-2/eks/terragrunt.hcl | 20 ++ .../us-west-2/k8s/cert-manager/terragrunt.hcl | 20 ++ .../us-west-2/k8s/coder-proxy/terragrunt.hcl | 40 ++++ .../k8s/lb-controller/terragrunt.hcl | 23 +++ .../k8s/metrics-server/terragrunt.hcl | 20 ++ infra/aws/us-west-2/k8s/other/terragrunt.hcl | 25 +++ infra/aws/us-west-2/vpc/terragrunt.hcl | 21 ++ 29 files changed, 896 insertions(+) create mode 100644 infra/aws/eu-west-2/eks/terragrunt.hcl create mode 100644 infra/aws/eu-west-2/k8s/cert-manager/terragrunt.hcl create mode 100644 infra/aws/eu-west-2/k8s/coder-proxy/terragrunt.hcl create mode 100644 infra/aws/eu-west-2/k8s/lb-controller/terragrunt.hcl create mode 100644 infra/aws/eu-west-2/k8s/metrics-server/terragrunt.hcl create mode 100644 infra/aws/eu-west-2/k8s/other/terragrunt.hcl create mode 100644 infra/aws/eu-west-2/vpc/terragrunt.hcl create mode 100644 infra/aws/root.hcl create mode 100644 infra/aws/us-east-2/eks/terragrunt.hcl create mode 100644 infra/aws/us-east-2/k8s/cert-manager/terragrunt.hcl create mode 100644 infra/aws/us-east-2/k8s/coder-server/terragrunt.hcl create mode 100644 infra/aws/us-east-2/k8s/coder-ws/terragrunt.hcl create mode 100644 infra/aws/us-east-2/k8s/ebs-csi/terragrunt.hcl create mode 100644 infra/aws/us-east-2/k8s/karpenter/terragrunt.hcl create mode 100644 infra/aws/us-east-2/k8s/lb-controller/terragrunt.hcl create mode 100644 infra/aws/us-east-2/k8s/litellm/terragrunt.hcl create mode 100644 infra/aws/us-east-2/k8s/metrics-server/terragrunt.hcl create mode 100644 infra/aws/us-east-2/k8s/monitoring/terragrunt.hcl create mode 100644 infra/aws/us-east-2/k8s/other/terragrunt.hcl create mode 100644 infra/aws/us-east-2/rds/terragrunt.hcl create mode 100644 infra/aws/us-east-2/s3/terragrunt.hcl create mode 100644 infra/aws/us-east-2/vpc/terragrunt.hcl create mode 100644 infra/aws/us-west-2/eks/terragrunt.hcl create mode 100644 infra/aws/us-west-2/k8s/cert-manager/terragrunt.hcl create mode 100644 infra/aws/us-west-2/k8s/coder-proxy/terragrunt.hcl create mode 100644 infra/aws/us-west-2/k8s/lb-controller/terragrunt.hcl create mode 100644 infra/aws/us-west-2/k8s/metrics-server/terragrunt.hcl create mode 100644 infra/aws/us-west-2/k8s/other/terragrunt.hcl create mode 100644 infra/aws/us-west-2/vpc/terragrunt.hcl diff --git a/infra/aws/eu-west-2/eks/terragrunt.hcl b/infra/aws/eu-west-2/eks/terragrunt.hcl new file mode 100644 index 0000000..3d508ad --- /dev/null +++ b/infra/aws/eu-west-2/eks/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = ["../vpc"] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="eu-west-2" + + name=include.root.locals.CODER_CLUSTER_NAME + vpc_name=include.root.locals.CODER_VPC_NAME + eks_version=include.root.locals.CODER_CLUSTER_VERSION + instance_type=include.root.locals.CODER_CLUSTER_INSTANCE_TYPE + public_subnet_suffix=include.root.locals.CODER_PUBLIC_SUBNET_SUFFIX + private_subnet_suffix=include.root.locals.CODER_PRIVATE_SUBNET_SUFFIX +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/cert-manager/terragrunt.hcl b/infra/aws/eu-west-2/k8s/cert-manager/terragrunt.hcl new file mode 100644 index 0000000..dc8cb8e --- /dev/null +++ b/infra/aws/eu-west-2/k8s/cert-manager/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="eu-west-2" + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.CRTMGR_ADDON_NAMESPACE + addon_version=include.root.locals.CRTMGR_ADDON_VERSION +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/coder-proxy/terragrunt.hcl b/infra/aws/eu-west-2/k8s/coder-proxy/terragrunt.hcl new file mode 100644 index 0000000..a661dbb --- /dev/null +++ b/infra/aws/eu-west-2/k8s/coder-proxy/terragrunt.hcl @@ -0,0 +1,40 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks", + "../lb-controller", + "../cert-manager", + "../other", + "../../../us-east-2/k8s/coder-server" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="eu-west-2" + + azs=jsondecode(include.root.locals.CODER_AWS_AZS) + vpc_name=include.root.locals.CODER_VPC_NAME + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + coder_access_url=include.root.locals.CODER_DOMAIN_NAME + coder_wildcard_access_url=include.root.locals.CODER_WILDCARD_URL + + coder_proxy_url="https://emea-proxy.ai.coder.com" + coder_proxy_wildcard_url="*.emea-proxy.ai.coder.com" + coder_proxy_name="eu-west-2" + coder_proxy_display_name="Europe (London)" + coder_proxy_icon="/emojis/1f1ec-1f1e7.png" + + image_repo=include.root.locals.CODER_IMAGE_REPO + image_tag=include.root.locals.CODER_IMAGE_TAG + addon_version=include.root.locals.CODER_ADDON_VERSION + + coder_admin_email=include.root.locals.CODER_EMAIL + coder_admin_password=include.root.locals.CODER_PASSWORD +} + diff --git a/infra/aws/eu-west-2/k8s/lb-controller/terragrunt.hcl b/infra/aws/eu-west-2/k8s/lb-controller/terragrunt.hcl new file mode 100644 index 0000000..670a7d7 --- /dev/null +++ b/infra/aws/eu-west-2/k8s/lb-controller/terragrunt.hcl @@ -0,0 +1,23 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks", + "../cert-manager" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="eu-west-2" + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.LB_ADDON_NAMESPACE + addon_version=include.root.locals.LB_ADDON_VERSION + vpc_name=include.root.locals.CODER_VPC_NAME + use_cert_manager=include.root.locals.LB_ADDON_USE_CERTMGR +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/metrics-server/terragrunt.hcl b/infra/aws/eu-west-2/k8s/metrics-server/terragrunt.hcl new file mode 100644 index 0000000..6c0c0d2 --- /dev/null +++ b/infra/aws/eu-west-2/k8s/metrics-server/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="eu-west-2" + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.METRICS_SRV_ADDON_NAMESPACE + addon_version=include.root.locals.METRICS_SRV_ADDON_VERSION +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/other/terragrunt.hcl b/infra/aws/eu-west-2/k8s/other/terragrunt.hcl new file mode 100644 index 0000000..3268609 --- /dev/null +++ b/infra/aws/eu-west-2/k8s/other/terragrunt.hcl @@ -0,0 +1,25 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks", + "../lb-controller", + "../cert-manager", + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="eu-west-2" + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + azs=include.root.locals.CODER_VPC_AZS + + cloudflare_api_token=include.root.locals.CF_TOKEN + cloudflare_secret_namespace=include.root.locals.CRTMGR_ADDON_NAMESPACE + cloudflare_email=include.root.locals.CF_EMAIL +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/vpc/terragrunt.hcl b/infra/aws/eu-west-2/vpc/terragrunt.hcl new file mode 100644 index 0000000..7858497 --- /dev/null +++ b/infra/aws/eu-west-2/vpc/terragrunt.hcl @@ -0,0 +1,21 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="eu-west-2" + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + vpc_name=include.root.locals.CODER_VPC_NAME + vpc_cidr=include.root.locals.CODER_VPC_CIDR + azs=include.root.locals.CODER_VPC_AZS + nat_name=include.root.locals.CODER_VPC_NAT_NAME + public_subnet_suffix=include.root.locals.CODER_PUBLIC_SUBNET_SUFFIX + private_subnet_suffix=include.root.locals.CODER_PRIVATE_SUBNET_SUFFIX +} \ No newline at end of file diff --git a/infra/aws/root.hcl b/infra/aws/root.hcl new file mode 100644 index 0000000..29c77ef --- /dev/null +++ b/infra/aws/root.hcl @@ -0,0 +1,179 @@ +locals { + CODER_TF_BACKEND_AWS_BUCKET_NAME = get_env("CODER_TF_BACKEND_AWS_BUCKET_NAME") + CODER_TF_BACKEND_AWS_REGION = get_env("CODER_TF_BACKEND_AWS_REGION", "us-east-2") + CODER_TF_BACKEND_AWS_PROFILE = get_env("CODER_TF_BACKEND_AWS_PROFILE", "default") + CODER_TF_BACKEND_ENCRYPT = get_env("CODER_TF_BACKEND_ENCRYPT", "false") + + CODER_AWS_PROFILE = get_env("CODER_AWS_PROFILE", "default") + CODER_AWS_REGION = get_env("CODER_AWS_REGION", "us-east-2") + CODER_AWS_AZS = get_env("CODER_AWS_AZS", jsonencode(["a", "c"])) + + CODER_CLUSTER_NAME = get_env("CODER_CLUSTER_NAME", "coder") + CODER_CLUSTER_VERSION = get_env("CODER_CLUSTER_VERSION", "1.35") + CODER_CLUSTER_INSTANCE_TYPE = get_env("CODER_CLUSTER_INSTANCE_TYPE", "c6a.large") + CODER_VPC_NAME = get_env("CODER_VPC_NAME", "coder") + CODER_VPC_CIDR = get_env("CODER_VPC_CIDR", "10.0.0.0/16") + CODER_VPC_AZS = get_env("CODER_VPC_AZS", jsonencode(["a", "b", "c"])) + CODER_VPC_NAT_NAME = get_env("CODER_VPC_NAT_NAME", "nat-instance") + CODER_DB_SUBNET_GROUP_NAME = get_env("CODER_DB_SUBNET_GROUP_NAME", "coder-db-subnet-group") + CODER_PUBLIC_SUBNET_SUFFIX = get_env("CODER_PUBLIC_SUBNET_SUFFIX", "public") + CODER_PRIVATE_SUBNET_SUFFIX = get_env("CODER_PRIVATE_SUBNET_SUFFIX", "private") + + CODER_DB_RDS_ID = get_env("CODER_DB_RDS_ID", "coder") + CODER_DB_USERNAME = get_env("CODER_DB_USERNAME", "coder") + CODER_DB_PASSWORD = get_env("CODER_DB_PASSWORD", "th1s1sn0tas3cur3pass0wrd") + CODER_DB_NAME = get_env("CODER_DB_NAME", "coder") + + LITELLM_DOMAIN_NAME = get_env("LITELLM_DOMAIN_NAME") + LITELLM_ADDON_NAMESPACE = get_env("LITELLM_ADDON_NAMESPACE", "litellm") + LITELLM_ADDON_VERSION = get_env("LITELLM_ADDON_VERSION", "0.1.830") + LITELLM_DB_RDS_ID = get_env("LITELLM_DB_RDS_ID", "litellm") + LITELLM_DB_NAME = get_env("LITELLM_DB_NAME", "litellm") + LITELLM_DB_USERNAME = get_env("LITELLM_DB_USERNAME", "litellm") + LITELLM_DB_USER_PASSWORD = get_env("LITELLM_DB_USER_PASSWORD", "th1s1sn0tas3cur3pass0wrd") + LITELLM_DB_ADMIN_PASSWORD = get_env("LITELLM_DB_ADMIN_PASSWORD", "th1s1sn0tas3cur3pass0wrd") + LITELLM_GCLOUD_AUTH = get_env("LITELLM_GCLOUD_AUTH", "th1s1sn0tas3cur3pass0wrd") + LITELLM_MASTER_KEY = get_env("LITELLM_MASTER_KEY", "th1s1sn0tas3cur3pass0wrd") + + CODER_IMAGE_REPO = get_env("CODER_IMAGE_REPO", "ghcr.io/coder/coder") + CODER_IMAGE_TAG = get_env("CODER_IMAGE_TAG", "v2.28.6") + CODER_ADDON_VERSION = get_env("CODER_ADDON_VERSION", "2.25.1") + CODER_LOGSTREAM_ADDON_VERSION = get_env("CODER_LOGSTREAM_ADDON_VERSION", "0.0.11") + + KPTR_ADDON_NAMESPACE = get_env("KPTR_ADDON_NAMESPACE", "karpenter") + KPTR_ADDON_VERSION = get_env("KPTR_ADDON_VERSION", "1.9.0") + + CRTMGR_ADDON_VERSION = get_env("CRTMGR_ADDON_VERSION", "v1.18.2") + CRTMGR_ADDON_NAMESPACE = get_env("CRTMGR_ADDON_NAMESPACE", "cert-manager") + + EBS_ADDON_NAMESPACE = get_env("EBS_ADDON_NAMESPACE", "ebs-controller") + EBS_ADDON_VERSION = get_env("EBS_ADDON_VERSION", "2.22.1") + EBS_ADDON_REPLACE = get_env("EBS_ADDON_REPLACE", "true") + + LB_ADDON_NAMESPACE = get_env("LB_ADDON_NAMESPACE", "lb-controller") + LB_ADDON_VERSION = get_env("LB_ADDON_VERSION", "1.13.2") + LB_ADDON_USE_CERTMGR = get_env("LB_ADDON_USE_CERTMGR", "true") + + METRICS_SRV_ADDON_NAMESPACE = get_env("METRICS_SRV_ADDON_NAMESPACE", "kube-system") + METRICS_SRV_ADDON_VERSION = get_env("METRICS_SRV_ADDON_VERSION", "3.13.0") + + CODER_OBSRV_CHART_VERSION = get_env("CODER_OBSRV_CHART_VERSION", "0.7.0-rc.1") + CODER_OBSRV_CHART_NAMESPACE = get_env("CODER_OBSRV_CHART_NAMESPACE", "observability") + + LOKI_S3_BUCKET_NAME = get_env("LOKI_S3_BUCKET_NAME", "loki") + LOKI_S3_BUCKET_REGION = get_env("LOKI_S3_BUCKET_REGION", "us-east-2") + + GRAFANA_DOMAIN_NAME = get_env("GRAFANA_DOMAIN_NAME") + + GRAFANA_DB_RDS_ID = get_env("GRAFANA_DB_RDS_ID", "grafana") + GRAFANA_DB_NAME = get_env("GRAFANA_DB_NAME", "grafana") + GRAFANA_DB_USERNAME = get_env("GRAFANA_DB_USERNAME", "grafana") + GRAFANA_DB_PASSWORD = get_env("GRAFANA_DB_PASSWORD", "Th1s1sN0TS3CuR3!!") + + GRAFANA_USERNAME = get_env("GRAFANA_USERNAME", "admin") + GRAFANA_PASSWORD = get_env("GRAFANA_PASSWORD", "Th1s1sN0TS3CuR3!!") + + GRAFANA_ADMIN_USERNAME = get_env("GRAFANA_ADMIN_USERNAME", "admin") + GRAFANA_ADMIN_PASSWORD = get_env("GRAFANA_ADMIN_PASSWORD", "Th1s1sN0TS3CuR3!!") + + CODER_DOMAIN_NAME = get_env("CODER_DOMAIN_NAME") + CODER_WILDCARD_URL = get_env("CODER_WILDCARD_URL") + CODER_LICENSE = get_env("CODER_LICENSE", "") + CODER_EXPERIMENTS = get_env("CODER_EXPERIMENTS", jsonencode([])) + CODER_BUILT_IN_PROVISIONER_COUNT = get_env("CODER_BUILT_IN_PROVISIONER_COUNT", "0") + + CODER_USERNAME = get_env("CODER_USERNAME", "admin") + CODER_EMAIL = get_env("CODER_EMAIL", "admin@coder.com") + CODER_PASSWORD = get_env("CODER_PASSWORD", "Th1s1sN0TS3CuR3!!") + + CF_EMAIL = get_env("CF_EMAIL") + CF_TOKEN = get_env("CF_TOKEN") + + CODER_OIDC_SIGN_IN_TEXT = get_env("CODER_OIDC_SIGN_IN_TEXT") + CODER_OIDC_ICON_URL = get_env("CODER_OIDC_ICON_URL") + CODER_OIDC_SCOPES = get_env("CODER_OIDC_SCOPES", "[]") + CODER_OIDC_EMAIL_DOMAIN = get_env("CODER_OIDC_EMAIL_DOMAIN") + CODER_OIDC_ISSUER_URL = get_env("CODER_OIDC_ISSUER_URL") + CODER_OIDC_CLIENT_ID = get_env("CODER_OIDC_CLIENT_ID") + CODER_OIDC_CLIENT_SECRET = get_env("CODER_OIDC_CLIENT_SECRET") + + CODER_GITHUB_OAUTH_CLIENT_ID = get_env("CODER_GITHUB_OAUTH_CLIENT_ID") + CODER_GITHUB_OAUTH_CLIENT_SECRET = get_env("CODER_GITHUB_OAUTH_CLIENT_SECRET") + CODER_GITHUB_EXTERN_AUTH_CLIENT_ID = get_env("CODER_GITHUB_EXTERN_AUTH_CLIENT_ID") + CODER_GITHUB_EXTERN_AUTH_CLIENT_SECRET = get_env("CODER_GITHUB_EXTERN_AUTH_CLIENT_SECRET") + + CODER_ANTHROPIC_LLM_ENDPOINT = get_env("CODER_ANTHROPIC_LLM_ENDPOINT") + CODER_ANTHROPIC_LLM_KEY = get_env("CODER_ANTHROPIC_LLM_KEY") + CODER_OPENAI_LLM_ENDPOINT = get_env("CODER_OPENAI_LLM_ENDPOINT") + CODER_OPENAI_LLM_KEY = get_env("CODER_OPENAI_LLM_KEY") +} + +generate "backend" { + path = "backend.tf" + if_exists = "overwrite" + + contents = <<-EOF + terraform { + backend "s3" { + bucket = "${local.CODER_TF_BACKEND_AWS_BUCKET_NAME}" + key = "ai.coder.com/infra/aws/${path_relative_to_include()}/terraform.tfstate" + region = "${local.CODER_TF_BACKEND_AWS_REGION}" + profile = "${local.CODER_TF_BACKEND_AWS_PROFILE}" + encrypt = "${local.CODER_TF_BACKEND_ENCRYPT}" + } + } + EOF +} + +generate "provider" { + path = "provider.tf" + if_exists = "overwrite" + + contents = <<-EOF + terraform { + required_version = ">= 1.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.32.1" + } + helm = { + source = "hashicorp/helm" + version = "~> 3.1.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 3.0.1" + } + external = { + source = "hashicorp/external" + version = "~> 2.3.5" + } + time = { + source = "hashicorp/time" + version = "~> 0.13.1" + } + http = { + source = "hashicorp/http" + version = "~> 3.5.0" + } + dns = { + source = "hashicorp/dns" + version = "~> 3.5.0" + } + archive = { + source = "hashicorp/archive" + version = "~> 2.7.1" + } + random = { + source = "hashicorp/random" + version = "~> 3.8.1" + } + coderd = { + source = "coder/coderd" + version = "~> 0.0.12" + } + } + } + EOF +} \ No newline at end of file diff --git a/infra/aws/us-east-2/eks/terragrunt.hcl b/infra/aws/us-east-2/eks/terragrunt.hcl new file mode 100644 index 0000000..5e7fb39 --- /dev/null +++ b/infra/aws/us-east-2/eks/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = ["../vpc"] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + name=include.root.locals.CODER_CLUSTER_NAME + vpc_name=include.root.locals.CODER_VPC_NAME + eks_version=include.root.locals.CODER_CLUSTER_VERSION + instance_type=include.root.locals.CODER_CLUSTER_INSTANCE_TYPE + public_subnet_suffix=include.root.locals.CODER_PUBLIC_SUBNET_SUFFIX + private_subnet_suffix=include.root.locals.CODER_PRIVATE_SUBNET_SUFFIX +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/cert-manager/terragrunt.hcl b/infra/aws/us-east-2/k8s/cert-manager/terragrunt.hcl new file mode 100644 index 0000000..27da9ba --- /dev/null +++ b/infra/aws/us-east-2/k8s/cert-manager/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.CRTMGR_ADDON_NAMESPACE + addon_version=include.root.locals.CRTMGR_ADDON_VERSION +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/coder-server/terragrunt.hcl b/infra/aws/us-east-2/k8s/coder-server/terragrunt.hcl new file mode 100644 index 0000000..6790e89 --- /dev/null +++ b/infra/aws/us-east-2/k8s/coder-server/terragrunt.hcl @@ -0,0 +1,58 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks", + "../../rds", + "../litellm", + "../karpenter", + "../lb-controller", + "../cert-manager", + "../other" # Deploy's auxillary manifests + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + azs=jsondecode(include.root.locals.CODER_AWS_AZS) + vpc_name=include.root.locals.CODER_VPC_NAME + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + coder_access_url=include.root.locals.CODER_DOMAIN_NAME + coder_wildcard_access_url=include.root.locals.CODER_WILDCARD_URL + coder_experiments=jsondecode(include.root.locals.CODER_EXPERIMENTS) + coder_builtin_provisioner_count=include.root.locals.CODER_BUILT_IN_PROVISIONER_COUNT + + coder_db_rds_name=include.root.locals.CODER_DB_RDS_ID + coder_db_username=include.root.locals.CODER_DB_USERNAME + coder_db_password=include.root.locals.CODER_DB_PASSWORD + coder_db_name=include.root.locals.CODER_DB_NAME + + oidc_sign_in_text=include.root.locals.CODER_OIDC_SIGN_IN_TEXT + oidc_icon_url=include.root.locals.CODER_OIDC_ICON_URL + oidc_scopes=include.root.locals.CODER_OIDC_SCOPES + oidc_email_domain=include.root.locals.CODER_OIDC_EMAIL_DOMAIN + coder_oidc_secret_issuer_url=include.root.locals.CODER_OIDC_ISSUER_URL + coder_oidc_secret_client_id=include.root.locals.CODER_OIDC_CLIENT_ID + coder_oidc_secret_client_secret=include.root.locals.CODER_OIDC_CLIENT_SECRET + + coder_oauth_secret_client_id=include.root.locals.CODER_GITHUB_OAUTH_CLIENT_ID + coder_oauth_secret_client_secret=include.root.locals.CODER_GITHUB_OAUTH_CLIENT_SECRET + coder_github_external_auth_secret_client_id=include.root.locals.CODER_GITHUB_EXTERN_AUTH_CLIENT_ID + coder_github_external_auth_secret_client_secret=include.root.locals.CODER_GITHUB_EXTERN_AUTH_CLIENT_SECRET + + image_repo=include.root.locals.CODER_IMAGE_REPO + image_tag=include.root.locals.CODER_IMAGE_TAG + addon_version=include.root.locals.CODER_ADDON_VERSION + + anthropic_llm_endpoint=include.root.locals.CODER_ANTHROPIC_LLM_ENDPOINT + anthropic_llm_key=include.root.locals.CODER_ANTHROPIC_LLM_KEY + + openai_llm_endpoint=include.root.locals.CODER_OPENAI_LLM_ENDPOINT + openai_llm_key=include.root.locals.CODER_OPENAI_LLM_KEY +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/coder-ws/terragrunt.hcl b/infra/aws/us-east-2/k8s/coder-ws/terragrunt.hcl new file mode 100644 index 0000000..1992111 --- /dev/null +++ b/infra/aws/us-east-2/k8s/coder-ws/terragrunt.hcl @@ -0,0 +1,29 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks", + "../litellm", + "../coder-server", + "../other" # Deploy's auxillary manifests + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + coder_access_url=include.root.locals.CODER_DOMAIN_NAME + coder_admin_email=include.root.locals.CODER_EMAIL + coder_admin_password=include.root.locals.CODER_PASSWORD + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + image_repo=include.root.locals.CODER_IMAGE_REPO + image_tag=include.root.locals.CODER_IMAGE_TAG + addon_version=include.root.locals.CODER_ADDON_VERSION + logstream_addon_version=include.root.locals.CODER_LOGSTREAM_ADDON_VERSION +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/ebs-csi/terragrunt.hcl b/infra/aws/us-east-2/k8s/ebs-csi/terragrunt.hcl new file mode 100644 index 0000000..2a314ed --- /dev/null +++ b/infra/aws/us-east-2/k8s/ebs-csi/terragrunt.hcl @@ -0,0 +1,21 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.EBS_ADDON_NAMESPACE + addon_version=include.root.locals.EBS_ADDON_VERSION + addon_replace=include.root.locals.EBS_ADDON_REPLACE +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/karpenter/terragrunt.hcl b/infra/aws/us-east-2/k8s/karpenter/terragrunt.hcl new file mode 100644 index 0000000..2a69c09 --- /dev/null +++ b/infra/aws/us-east-2/k8s/karpenter/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.KPTR_ADDON_NAMESPACE + addon_version=include.root.locals.KPTR_ADDON_VERSION +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/lb-controller/terragrunt.hcl b/infra/aws/us-east-2/k8s/lb-controller/terragrunt.hcl new file mode 100644 index 0000000..ee0e6fb --- /dev/null +++ b/infra/aws/us-east-2/k8s/lb-controller/terragrunt.hcl @@ -0,0 +1,22 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.LB_ADDON_NAMESPACE + addon_version=include.root.locals.LB_ADDON_VERSION + vpc_name=include.root.locals.CODER_VPC_NAME + use_cert_manager=include.root.locals.LB_ADDON_USE_CERTMGR +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/litellm/terragrunt.hcl b/infra/aws/us-east-2/k8s/litellm/terragrunt.hcl new file mode 100644 index 0000000..38f7c15 --- /dev/null +++ b/infra/aws/us-east-2/k8s/litellm/terragrunt.hcl @@ -0,0 +1,35 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks", + "../../rds", + "../lb-controller", + "../cert-manager", + "../other" # Deploy's auxillary manifests + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.LITELLM_ADDON_NAMESPACE + addon_version=include.root.locals.LITELLM_ADDON_VERSION + + host_name=include.root.locals.LITELLM_DOMAIN_NAME + vpc_name=include.root.locals.CODER_VPC_NAME + azs=include.root.locals.CODER_VPC_AZS + + db_rds_id=include.root.locals.LITELLM_DB_RDS_ID + db_admin_password=include.root.locals.LITELLM_DB_ADMIN_PASSWORD + db_user_password=include.root.locals.LITELLM_DB_USER_PASSWORD + + gcloud_auth=include.root.locals.LITELLM_GCLOUD_AUTH + litellm_master_key=include.root.locals.LITELLM_MASTER_KEY +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/metrics-server/terragrunt.hcl b/infra/aws/us-east-2/k8s/metrics-server/terragrunt.hcl new file mode 100644 index 0000000..d7c1154 --- /dev/null +++ b/infra/aws/us-east-2/k8s/metrics-server/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.METRICS_SRV_ADDON_NAMESPACE + addon_version=include.root.locals.METRICS_SRV_ADDON_VERSION +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/monitoring/terragrunt.hcl b/infra/aws/us-east-2/k8s/monitoring/terragrunt.hcl new file mode 100644 index 0000000..80156af --- /dev/null +++ b/infra/aws/us-east-2/k8s/monitoring/terragrunt.hcl @@ -0,0 +1,48 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks", + "../../rds", + "../../s3", + "../cert-manager", + "../lb-controller", + "../litellm", + "../coder-server", + "../other" # Deploy's auxillary manifests + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + chart_version=include.root.locals.CODER_OBSRV_CHART_VERSION + namespace=include.root.locals.CODER_OBSRV_CHART_NAMESPACE + + domain_name=include.root.locals.GRAFANA_DOMAIN_NAME + vpc_name=include.root.locals.CODER_VPC_NAME + azs=include.root.locals.CODER_VPC_AZS + + loki_s3_bucket_name=include.root.locals.LOKI_S3_BUCKET_NAME + loki_s3_bucket_region=include.root.locals.LOKI_S3_BUCKET_REGION + + coder_db_rds_id=include.root.locals.CODER_DB_RDS_ID + coder_db_username=include.root.locals.CODER_DB_USERNAME + coder_db_password=include.root.locals.CODER_DB_PASSWORD + + grafana_db_rds_id=include.root.locals.GRAFANA_DB_RDS_ID + grafana_db_user=include.root.locals.GRAFANA_DB_USERNAME + grafana_db_password=include.root.locals.GRAFANA_DB_PASSWORD + + grafana_auth_username=include.root.locals.GRAFANA_USERNAME + grafana_auth_password=include.root.locals.GRAFANA_PASSWORD + + grafana_admin_username=include.root.locals.GRAFANA_ADMIN_USERNAME + grafana_admin_password=include.root.locals.GRAFANA_ADMIN_PASSWORD +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/other/terragrunt.hcl b/infra/aws/us-east-2/k8s/other/terragrunt.hcl new file mode 100644 index 0000000..fc0fae2 --- /dev/null +++ b/infra/aws/us-east-2/k8s/other/terragrunt.hcl @@ -0,0 +1,23 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + azs=include.root.locals.CODER_VPC_AZS + + cloudflare_api_token=include.root.locals.CF_TOKEN + cloudflare_secret_namespace=include.root.locals.CRTMGR_ADDON_NAMESPACE + cloudflare_email=include.root.locals.CF_EMAIL +} \ No newline at end of file diff --git a/infra/aws/us-east-2/rds/terragrunt.hcl b/infra/aws/us-east-2/rds/terragrunt.hcl new file mode 100644 index 0000000..0c69426 --- /dev/null +++ b/infra/aws/us-east-2/rds/terragrunt.hcl @@ -0,0 +1,35 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = ["../vpc"] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + vpc_name=include.root.locals.CODER_VPC_NAME + db_subnet_group_name=include.root.locals.CODER_DB_SUBNET_GROUP_NAME + private_subnet_suffix=include.root.locals.CODER_PRIVATE_SUBNET_SUFFIX + + coder_db_rds_id=include.root.locals.CODER_DB_RDS_ID + coder_db_name=include.root.locals.CODER_DB_NAME + coder_username=include.root.locals.CODER_DB_USERNAME + coder_password=include.root.locals.CODER_DB_PASSWORD + + + grafana_db_rds_id=include.root.locals.GRAFANA_DB_RDS_ID + grafana_db_name=include.root.locals.GRAFANA_DB_NAME + grafana_username=include.root.locals.GRAFANA_DB_USERNAME + grafana_password=include.root.locals.GRAFANA_DB_PASSWORD + + + litellm_db_rds_id=include.root.locals.LITELLM_DB_RDS_ID + litellm_db_name=include.root.locals.LITELLM_DB_NAME + litellm_username=include.root.locals.LITELLM_DB_USERNAME + litellm_password=include.root.locals.LITELLM_DB_ADMIN_PASSWORD + +} \ No newline at end of file diff --git a/infra/aws/us-east-2/s3/terragrunt.hcl b/infra/aws/us-east-2/s3/terragrunt.hcl new file mode 100644 index 0000000..9c65077 --- /dev/null +++ b/infra/aws/us-east-2/s3/terragrunt.hcl @@ -0,0 +1,11 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + loki_s3_bucket_name=include.root.locals.LOKI_S3_BUCKET_NAME +} \ No newline at end of file diff --git a/infra/aws/us-east-2/vpc/terragrunt.hcl b/infra/aws/us-east-2/vpc/terragrunt.hcl new file mode 100644 index 0000000..235f81a --- /dev/null +++ b/infra/aws/us-east-2/vpc/terragrunt.hcl @@ -0,0 +1,17 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region=include.root.locals.CODER_AWS_REGION + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + vpc_name=include.root.locals.CODER_VPC_NAME + vpc_cidr=include.root.locals.CODER_VPC_CIDR + azs=include.root.locals.CODER_VPC_AZS + nat_name=include.root.locals.CODER_VPC_NAT_NAME + public_subnet_suffix=include.root.locals.CODER_PUBLIC_SUBNET_SUFFIX + private_subnet_suffix=include.root.locals.CODER_PRIVATE_SUBNET_SUFFIX +} \ No newline at end of file diff --git a/infra/aws/us-west-2/eks/terragrunt.hcl b/infra/aws/us-west-2/eks/terragrunt.hcl new file mode 100644 index 0000000..a9d8259 --- /dev/null +++ b/infra/aws/us-west-2/eks/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = ["../vpc"] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="us-west-2" + + name=include.root.locals.CODER_CLUSTER_NAME + vpc_name=include.root.locals.CODER_VPC_NAME + eks_version=include.root.locals.CODER_CLUSTER_VERSION + instance_type=include.root.locals.CODER_CLUSTER_INSTANCE_TYPE + public_subnet_suffix=include.root.locals.CODER_PUBLIC_SUBNET_SUFFIX + private_subnet_suffix=include.root.locals.CODER_PRIVATE_SUBNET_SUFFIX +} \ No newline at end of file diff --git a/infra/aws/us-west-2/k8s/cert-manager/terragrunt.hcl b/infra/aws/us-west-2/k8s/cert-manager/terragrunt.hcl new file mode 100644 index 0000000..88c5370 --- /dev/null +++ b/infra/aws/us-west-2/k8s/cert-manager/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="us-west-2" + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.CRTMGR_ADDON_NAMESPACE + addon_version=include.root.locals.CRTMGR_ADDON_VERSION +} \ No newline at end of file diff --git a/infra/aws/us-west-2/k8s/coder-proxy/terragrunt.hcl b/infra/aws/us-west-2/k8s/coder-proxy/terragrunt.hcl new file mode 100644 index 0000000..3a629c8 --- /dev/null +++ b/infra/aws/us-west-2/k8s/coder-proxy/terragrunt.hcl @@ -0,0 +1,40 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks", + "../lb-controller", + "../cert-manager", + "../other", + "../../../us-east-2/k8s/coder-server" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="us-west-2" + + azs=jsondecode(include.root.locals.CODER_AWS_AZS) + vpc_name=include.root.locals.CODER_VPC_NAME + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + coder_access_url=include.root.locals.CODER_DOMAIN_NAME + coder_wildcard_access_url=include.root.locals.CODER_WILDCARD_URL + + coder_proxy_url="https://oregon-proxy.ai.coder.com" + coder_proxy_wildcard_url="*.oregon-proxy.ai.coder.com" + coder_proxy_name="us-west-2" + coder_proxy_display_name="US West (Oregon)" + coder_proxy_icon="/emojis/1f1fa-1f1f8.png" + + image_repo=include.root.locals.CODER_IMAGE_REPO + image_tag=include.root.locals.CODER_IMAGE_TAG + addon_version=include.root.locals.CODER_ADDON_VERSION + + coder_admin_email=include.root.locals.CODER_EMAIL + coder_admin_password=include.root.locals.CODER_PASSWORD +} + diff --git a/infra/aws/us-west-2/k8s/lb-controller/terragrunt.hcl b/infra/aws/us-west-2/k8s/lb-controller/terragrunt.hcl new file mode 100644 index 0000000..d63c23d --- /dev/null +++ b/infra/aws/us-west-2/k8s/lb-controller/terragrunt.hcl @@ -0,0 +1,23 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks", + "../cert-manager" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="us-west-2" + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.LB_ADDON_NAMESPACE + addon_version=include.root.locals.LB_ADDON_VERSION + vpc_name=include.root.locals.CODER_VPC_NAME + use_cert_manager=include.root.locals.LB_ADDON_USE_CERTMGR +} \ No newline at end of file diff --git a/infra/aws/us-west-2/k8s/metrics-server/terragrunt.hcl b/infra/aws/us-west-2/k8s/metrics-server/terragrunt.hcl new file mode 100644 index 0000000..bd6f671 --- /dev/null +++ b/infra/aws/us-west-2/k8s/metrics-server/terragrunt.hcl @@ -0,0 +1,20 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks" + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="us-west-2" + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + addon_namespace=include.root.locals.METRICS_SRV_ADDON_NAMESPACE + addon_version=include.root.locals.METRICS_SRV_ADDON_VERSION +} \ No newline at end of file diff --git a/infra/aws/us-west-2/k8s/other/terragrunt.hcl b/infra/aws/us-west-2/k8s/other/terragrunt.hcl new file mode 100644 index 0000000..3a8882a --- /dev/null +++ b/infra/aws/us-west-2/k8s/other/terragrunt.hcl @@ -0,0 +1,25 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [ + "../../eks", + "../lb-controller", + "../cert-manager", + ] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="us-west-2" + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + + azs=include.root.locals.CODER_VPC_AZS + + cloudflare_api_token=include.root.locals.CF_TOKEN + cloudflare_secret_namespace=include.root.locals.CRTMGR_ADDON_NAMESPACE + cloudflare_email=include.root.locals.CF_EMAIL +} \ No newline at end of file diff --git a/infra/aws/us-west-2/vpc/terragrunt.hcl b/infra/aws/us-west-2/vpc/terragrunt.hcl new file mode 100644 index 0000000..f68557f --- /dev/null +++ b/infra/aws/us-west-2/vpc/terragrunt.hcl @@ -0,0 +1,21 @@ +include "root" { + path = find_in_parent_folders("root.hcl") + expose = true +} + +dependencies { + paths = [] +} + +inputs = { + profile=include.root.locals.CODER_AWS_PROFILE + region="us-west-2" + + cluster_name=include.root.locals.CODER_CLUSTER_NAME + vpc_name=include.root.locals.CODER_VPC_NAME + vpc_cidr=include.root.locals.CODER_VPC_CIDR + azs=include.root.locals.CODER_VPC_AZS + nat_name=include.root.locals.CODER_VPC_NAT_NAME + public_subnet_suffix=include.root.locals.CODER_PUBLIC_SUBNET_SUFFIX + private_subnet_suffix=include.root.locals.CODER_PRIVATE_SUBNET_SUFFIX +} \ No newline at end of file From fac7ebeaee4840d6e45907766ec321df844a1b32 Mon Sep 17 00:00:00 2001 From: Jullian Pepito Date: Tue, 24 Feb 2026 21:59:53 +0000 Subject: [PATCH 35/44] feat: refactor infrastructure --- infra/aws/eu-west-2/eks/main.tf | 214 ++--- infra/aws/eu-west-2/eks/variables.tf | 38 + infra/aws/eu-west-2/k8s/cert-manager/main.tf | 72 +- .../eu-west-2/k8s/cert-manager/variables.tf | 22 + infra/aws/eu-west-2/k8s/coder-proxy/main.tf | 302 +++--- .../eu-west-2/k8s/coder-proxy/variables.tf | 69 ++ infra/aws/eu-west-2/k8s/coder-ws/main.tf | 209 ----- .../aws/eu-west-2/k8s/ebs-controller/main.tf | 72 -- infra/aws/eu-west-2/k8s/karpenter/main.tf | 203 ---- infra/aws/eu-west-2/k8s/lb-controller/main.tf | 81 +- .../eu-west-2/k8s/lb-controller/variables.tf | 31 + .../aws/eu-west-2/k8s/metrics-server/main.tf | 47 +- .../eu-west-2/k8s/metrics-server/variables.tf | 22 + infra/aws/eu-west-2/k8s/other/main.tf | 135 +++ infra/aws/eu-west-2/k8s/other/variables.tf | 42 + infra/aws/eu-west-2/vpc/main.tf | 67 ++ infra/aws/eu-west-2/vpc/variables.tf | 48 + infra/aws/us-east-2/eks/main.tf | 157 ++-- infra/aws/us-east-2/eks/variables.tf | 38 + infra/aws/us-east-2/k8s/cert-manager/main.tf | 72 +- .../us-east-2/k8s/cert-manager/variables.tf | 22 + infra/aws/us-east-2/k8s/coder-server/main.tf | 446 ++++----- .../us-east-2/k8s/coder-server/variables.tf | 147 +++ infra/aws/us-east-2/k8s/coder-ws/main.tf | 388 +++++--- infra/aws/us-east-2/k8s/coder-ws/variables.tf | 45 + .../aws/us-east-2/k8s/ebs-controller/main.tf | 78 -- infra/aws/us-east-2/k8s/ebs-csi/main.tf | 58 ++ infra/aws/us-east-2/k8s/ebs-csi/variables.tf | 27 + .../aws/us-east-2/k8s/efs-controller/main.tf | 78 -- .../aws/us-east-2/k8s/efs-controller/pv.yaml | 15 - .../k8s/efs-controller/storageclass.yaml | 14 - .../aws/us-east-2/k8s/fetch-and-store/main.tf | 78 -- infra/aws/us-east-2/k8s/karpenter/main.tf | 179 ++-- .../aws/us-east-2/k8s/karpenter/variables.tf | 21 + infra/aws/us-east-2/k8s/lb-controller/main.tf | 73 +- .../us-east-2/k8s/lb-controller/variables.tf | 31 + infra/aws/us-east-2/k8s/litellm/main.tf | 402 +++++--- infra/aws/us-east-2/k8s/litellm/variables.tf | 56 ++ .../aws/us-east-2/k8s/metrics-server/main.tf | 47 +- .../us-east-2/k8s/metrics-server/variables.tf | 22 + .../dashboards/aibridge-demoable.json | 32 +- .../k8s/monitoring/dashboards/aibridge.json | 64 +- .../k8s/monitoring/dashboards/boundary.json | 875 ++++++++++++++++++ infra/aws/us-east-2/k8s/monitoring/main.tf | 576 +++++------- .../aws/us-east-2/k8s/monitoring/variables.tf | 144 +++ infra/aws/us-east-2/k8s/other/main.tf | 423 +++++++++ infra/aws/us-east-2/k8s/other/variables.tf | 42 + infra/aws/us-east-2/rds/main.tf | 181 ++-- infra/aws/us-east-2/rds/variables.tf | 99 ++ infra/aws/us-east-2/s3/main.tf | 37 - infra/aws/us-east-2/s3/variables.tf | 25 + infra/aws/us-east-2/vpc/main.tf | 237 +---- infra/aws/us-east-2/vpc/variables.tf | 48 + infra/aws/us-west-2/eks/main.tf | 208 ++--- infra/aws/us-west-2/eks/variables.tf | 38 + infra/aws/us-west-2/k8s/cert-manager/main.tf | 72 +- .../us-west-2/k8s/cert-manager/variables.tf | 22 + infra/aws/us-west-2/k8s/coder-proxy/main.tf | 287 +++--- .../us-west-2/k8s/coder-proxy/variables.tf | 69 ++ infra/aws/us-west-2/k8s/coder-ws/main.tf | 209 ----- .../aws/us-west-2/k8s/ebs-controller/main.tf | 72 -- infra/aws/us-west-2/k8s/karpenter/main.tf | 199 ---- infra/aws/us-west-2/k8s/lb-controller/main.tf | 81 +- .../us-west-2/k8s/lb-controller/variables.tf | 31 + .../aws/us-west-2/k8s/metrics-server/main.tf | 47 +- .../us-west-2/k8s/metrics-server/variables.tf | 22 + infra/aws/us-west-2/k8s/other/main.tf | 135 +++ infra/aws/us-west-2/k8s/other/variables.tf | 42 + infra/aws/us-west-2/vpc/main.tf | 67 ++ infra/aws/us-west-2/vpc/variables.tf | 48 + 70 files changed, 4887 insertions(+), 3713 deletions(-) create mode 100644 infra/aws/eu-west-2/eks/variables.tf create mode 100644 infra/aws/eu-west-2/k8s/cert-manager/variables.tf create mode 100644 infra/aws/eu-west-2/k8s/coder-proxy/variables.tf delete mode 100644 infra/aws/eu-west-2/k8s/coder-ws/main.tf delete mode 100644 infra/aws/eu-west-2/k8s/ebs-controller/main.tf delete mode 100644 infra/aws/eu-west-2/k8s/karpenter/main.tf create mode 100644 infra/aws/eu-west-2/k8s/lb-controller/variables.tf create mode 100644 infra/aws/eu-west-2/k8s/metrics-server/variables.tf create mode 100644 infra/aws/eu-west-2/k8s/other/main.tf create mode 100644 infra/aws/eu-west-2/k8s/other/variables.tf create mode 100644 infra/aws/eu-west-2/vpc/main.tf create mode 100644 infra/aws/eu-west-2/vpc/variables.tf create mode 100644 infra/aws/us-east-2/eks/variables.tf create mode 100644 infra/aws/us-east-2/k8s/cert-manager/variables.tf create mode 100644 infra/aws/us-east-2/k8s/coder-server/variables.tf create mode 100644 infra/aws/us-east-2/k8s/coder-ws/variables.tf delete mode 100644 infra/aws/us-east-2/k8s/ebs-controller/main.tf create mode 100644 infra/aws/us-east-2/k8s/ebs-csi/main.tf create mode 100644 infra/aws/us-east-2/k8s/ebs-csi/variables.tf delete mode 100644 infra/aws/us-east-2/k8s/efs-controller/main.tf delete mode 100644 infra/aws/us-east-2/k8s/efs-controller/pv.yaml delete mode 100644 infra/aws/us-east-2/k8s/efs-controller/storageclass.yaml delete mode 100644 infra/aws/us-east-2/k8s/fetch-and-store/main.tf create mode 100644 infra/aws/us-east-2/k8s/karpenter/variables.tf create mode 100644 infra/aws/us-east-2/k8s/lb-controller/variables.tf create mode 100644 infra/aws/us-east-2/k8s/litellm/variables.tf create mode 100644 infra/aws/us-east-2/k8s/metrics-server/variables.tf create mode 100644 infra/aws/us-east-2/k8s/monitoring/dashboards/boundary.json create mode 100644 infra/aws/us-east-2/k8s/monitoring/variables.tf create mode 100644 infra/aws/us-east-2/k8s/other/main.tf create mode 100644 infra/aws/us-east-2/k8s/other/variables.tf create mode 100644 infra/aws/us-east-2/rds/variables.tf create mode 100644 infra/aws/us-east-2/s3/variables.tf create mode 100644 infra/aws/us-east-2/vpc/variables.tf create mode 100644 infra/aws/us-west-2/eks/variables.tf create mode 100644 infra/aws/us-west-2/k8s/cert-manager/variables.tf create mode 100644 infra/aws/us-west-2/k8s/coder-proxy/variables.tf delete mode 100644 infra/aws/us-west-2/k8s/coder-ws/main.tf delete mode 100644 infra/aws/us-west-2/k8s/ebs-controller/main.tf delete mode 100644 infra/aws/us-west-2/k8s/karpenter/main.tf create mode 100644 infra/aws/us-west-2/k8s/lb-controller/variables.tf create mode 100644 infra/aws/us-west-2/k8s/metrics-server/variables.tf create mode 100644 infra/aws/us-west-2/k8s/other/main.tf create mode 100644 infra/aws/us-west-2/k8s/other/variables.tf create mode 100644 infra/aws/us-west-2/vpc/main.tf create mode 100644 infra/aws/us-west-2/vpc/variables.tf diff --git a/infra/aws/eu-west-2/eks/main.tf b/infra/aws/eu-west-2/eks/main.tf index 2bffa33..1d3c589 100644 --- a/infra/aws/eu-west-2/eks/main.tf +++ b/infra/aws/eu-west-2/eks/main.tf @@ -1,121 +1,56 @@ -terraform { - required_version = ">= 1.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.100.0" - } - } - backend "s3" {} -} - -variable "name" { - description = "The resource name and tag prefix" - type = string +provider "aws" { + region = var.region + profile = var.profile } -variable "profile" { - type = string +data "aws_vpc" "this" { + tags = { + Name = var.vpc_name + } } -variable "region" { - description = "The aws region to deploy eks cluster" - type = string -} +data "aws_subnets" "public" { + filter { + name = "vpc-id" + values = [data.aws_vpc.this.id] + } -variable "cluster_version" { - description = "The eks version" - type = string + tags = { + Name = "*${var.public_subnet_suffix}*" + } } -variable "cluster_instance_type" { - description = "EKS Instance Size/Type" - default = "t3.xlarge" - type = string -} +data "aws_subnets" "private" { + filter { + name = "vpc-id" + values = [data.aws_vpc.this.id] + } -provider "aws" { - region = var.region - profile = var.profile + tags = { + Name = "*${var.private_subnet_suffix}*" + } } -## -# Cluster Infrastructure -## - locals { - system_subnet_tags = { - "subnet.amazonaws.io/system/owned-by" = var.name + tags = { + Name = var.name + "karpenter.sh/discovery" = var.name } - provisioner_subnet_tags = { - "subnet.amazonaws.io/coder-provisioner/owned-by" = var.name + # Karpenter Security Group Discovery - https://karpenter.sh/v1.0/concepts/nodeclasses/#specsecuritygroupselectorterms + tags_kptr_sg_discovery = { + "karpenter.sh/discovery" = var.name } - ws_all_subnet_tags = { - "subnet.amazonaws.io/coder-ws-all/owned-by" = var.name + labels_system_node = { + "scheduling.coder.com/pool" = "system" } - system_sg_tags = { - "subnet.amazonaws.io/system/owned-by" = var.name - } - provisioner_sg_tags = { - "sg.amazonaws.io/coder-provisioner/owned-by" = var.name - } - ws_all_sg_tags = { - "sg.amazonaws.io/coder-ws-all/owned-by" = var.name - } - cluster_asg_node_labels = { - "node.amazonaws.io/managed-by" = "asg" - } -} - -data "aws_region" "this" {} - -module "eks-network" { - source = "../../../../modules/network/eks-vpc" - - name = var.name - vpc_cidr_block = "10.0.0.0/16" - public_subnets = { - "system0" = { - cidr_block = "10.0.10.0/24" - availability_zone = "${data.aws_region.this.name}a" - map_public_ip_on_launch = true - private_dns_hostname_type_on_launch = "ip-name" - } - "system1" = { - cidr_block = "10.0.11.0/24" - availability_zone = "${data.aws_region.this.name}b" - map_public_ip_on_launch = true - private_dns_hostname_type_on_launch = "ip-name" - } - } - private_subnets = { - "system0" = { - cidr_block = "10.0.20.0/24" - availability_zone = "${data.aws_region.this.name}a" - private_dns_hostname_type_on_launch = "ip-name" - tags = local.system_subnet_tags - } - "system1" = { - cidr_block = "10.0.21.0/24" - availability_zone = "${data.aws_region.this.name}b" - private_dns_hostname_type_on_launch = "ip-name" - tags = local.system_subnet_tags - } - "provisioner" = { - cidr_block = "10.0.22.0/24" - availability_zone = "${data.aws_region.this.name}a" - map_public_ip_on_launch = true - private_dns_hostname_type_on_launch = "ip-name" - tags = local.provisioner_subnet_tags - } - "ws-all" = { - cidr_block = "10.0.16.0/22" - availability_zone = "${data.aws_region.this.name}b" - map_public_ip_on_launch = true - private_dns_hostname_type_on_launch = "ip-name" - tags = local.ws_all_subnet_tags - } + taints_system = { + key = "CriticalAddonsOnly" + effect = "NO_SCHEDULE" } + tolerations_system = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] } data "aws_iam_policy_document" "sts" { @@ -131,47 +66,62 @@ resource "aws_iam_policy" "sts" { path = "/" description = "Assume Role Policy" policy = data.aws_iam_policy_document.sts.json + tags = local.tags } -module "cluster" { +module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" + version = "~> 21.15.1" - vpc_id = module.eks-network.vpc_id - subnet_ids = toset(concat(concat( - module.eks-network.public_subnet_ids, - module.eks-network.private_subnet_ids), - module.eks-network.intra_subnet_ids + vpc_id = data.aws_vpc.this.id + subnet_ids = toset(concat( + data.aws_subnets.private.ids )) - cluster_name = var.name - cluster_version = var.cluster_version - cluster_endpoint_public_access = true - cluster_endpoint_private_access = true + name = var.name + kubernetes_version = var.eks_version + endpoint_public_access = true + endpoint_private_access = true - create_cluster_security_group = true + create_security_group = true create_node_security_group = true - node_security_group_tags = merge( - local.system_sg_tags, - merge(local.provisioner_sg_tags, local.ws_all_sg_tags) - ) - create_iam_role = true - cluster_addons = { + create_iam_role = true + node_security_group_tags = local.tags_kptr_sg_discovery + create_node_iam_role = true + node_iam_role_use_name_prefix = true + + compute_config = { + enabled = true + node_pools = ["system"] + } + + addons = { coredns = { - most_recent = true + before_compute = true } kube-proxy = { - most_recent = true + before_compute = true } vpc-cni = { - most_recent = true + before_compute = true + configuration_values = jsonencode({ + enableNetworkPolicy = "true" + nodeAgent = { + enablePolicyEventLogs = "true" + } + env = { + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + WARM_IP_TARGET = "0" + AWS_VPC_K8S_CNI_LOGLEVEL = "DEBUG" + } + }) } } - # Disable KMS, speed up cluster creation times. Enable if encryption is necessary. - attach_cluster_encryption_policy = false - create_kms_key = false - cluster_encryption_config = {} + attach_encryption_policy = false + create_kms_key = false # Enable unless needed + encryption_config = null enable_cluster_creator_admin_permissions = true enable_irsa = true @@ -180,9 +130,9 @@ module "cluster" { min_size = 0 max_size = 10 desired_size = 0 # Cant be modified after creation. Override from AWS Console - labels = local.cluster_asg_node_labels + labels = local.labels_system_node - instance_types = [var.cluster_instance_type] + instance_types = [var.instance_type] capacity_type = "ON_DEMAND" iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" @@ -190,7 +140,9 @@ module "cluster" { } # System Nodes should not be public - subnet_ids = module.eks-network.private_subnet_ids + subnet_ids = data.aws_subnets.private.ids } } + + tags = local.tags } \ No newline at end of file diff --git a/infra/aws/eu-west-2/eks/variables.tf b/infra/aws/eu-west-2/eks/variables.tf new file mode 100644 index 0000000..5a5d820 --- /dev/null +++ b/infra/aws/eu-west-2/eks/variables.tf @@ -0,0 +1,38 @@ +variable "profile" { + type = string +} + +variable "region" { + description = "The aws region to deploy eks cluster" + type = string +} + +variable "name" { + description = "The resource name and tag prefix" + type = string +} + +variable "eks_version" { + description = "The eks version" + type = string +} + +variable "instance_type" { + description = "EKS Instance Size/Type" + default = "t3.xlarge" + type = string +} + +variable "vpc_name" { + type = string +} + +variable "private_subnet_suffix" { + type = string + default = "private" +} + +variable "public_subnet_suffix" { + type = string + default = "public" +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/cert-manager/main.tf b/infra/aws/eu-west-2/k8s/cert-manager/main.tf index c095843..5b19d13 100644 --- a/infra/aws/eu-west-2/k8s/cert-manager/main.tf +++ b/infra/aws/eu-west-2/k8s/cert-manager/main.tf @@ -1,59 +1,6 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "addon_namespace" { - type = string - default = "cert-manager" -} - -variable "addon_version" { - type = string - default = "v1.18.2" -} - -variable "cloudflare_api_token" { - type = string - sensitive = true -} - -variable "cloudflare_email" { - type = string - sensitive = true -} - provider "aws" { - region = var.cluster_region - profile = var.cluster_profile + region = var.region + profile = var.profile } data "aws_eks_cluster" "this" { @@ -64,6 +11,10 @@ data "aws_eks_cluster_auth" "this" { name = var.cluster_name } +data "aws_iam_openid_connect_provider" "this" { + url = data.aws_eks_cluster.this.identity[0].oidc[0].issuer +} + provider "helm" { kubernetes = { host = data.aws_eks_cluster.this.endpoint @@ -79,12 +30,17 @@ provider "kubernetes" { } module "cert-manager" { + source = "../../../../../modules/k8s/bootstrap/cert-manager" + cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn namespace = var.addon_namespace helm_version = var.addon_version - cloudflare_token_secret = var.cloudflare_api_token - cloudflare_token_secret_email = var.cloudflare_email + + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] } \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/cert-manager/variables.tf b/infra/aws/eu-west-2/k8s/cert-manager/variables.tf new file mode 100644 index 0000000..f93b94c --- /dev/null +++ b/infra/aws/eu-west-2/k8s/cert-manager/variables.tf @@ -0,0 +1,22 @@ +variable "cluster_name" { + type = string +} + +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "addon_namespace" { + type = string + default = "cert-manager" +} + +variable "addon_version" { + type = string + default = "v1.18.2" +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/coder-proxy/main.tf b/infra/aws/eu-west-2/k8s/coder-proxy/main.tf index 3b5cf7b..00ac5af 100644 --- a/infra/aws/eu-west-2/k8s/coder-proxy/main.tf +++ b/infra/aws/eu-west-2/k8s/coder-proxy/main.tf @@ -1,114 +1,6 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - coderd = { - source = "coder/coderd" - } - acme = { - source = "vancluever/acme" - } - tls = { - source = "hashicorp/tls" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "acme_server_url" { - type = string - default = "https://acme-v02.api.letsencrypt.org/directory" -} - -variable "acme_registration_email" { - type = string -} - -variable "addon_version" { - type = string - default = "2.25.1" -} - -variable "coder_proxy_name" { - type = string -} - -variable "coder_proxy_display_name" { - type = string -} - -variable "coder_proxy_icon" { - type = string -} - -variable "coder_access_url" { - type = string - # sensitive = true -} - -variable "coder_proxy_url" { - type = string - # sensitive = true -} - -variable "coder_proxy_wildcard_url" { - type = string - # sensitive = true -} - -variable "coder_token" { - type = string - sensitive = true -} - -variable "image_repo" { - type = string - sensitive = true -} - -variable "image_tag" { - type = string - default = "latest" -} - -variable "kubernetes_ssl_secret_name" { - type = string -} - -variable "kubernetes_create_ssl_secret" { - type = bool - default = true -} - -variable "cloudflare_api_token" { - type = string - sensitive = true -} - provider "aws" { - region = var.cluster_region - profile = var.cluster_profile + region = var.region + profile = var.profile } data "aws_eks_cluster" "this" { @@ -119,6 +11,8 @@ data "aws_eks_cluster_auth" "this" { name = var.cluster_name } +data "aws_region" "this" {} + provider "helm" { kubernetes = { host = data.aws_eks_cluster.this.endpoint @@ -133,79 +27,173 @@ provider "kubernetes" { token = data.aws_eks_cluster_auth.this.token } +data "http" "login" { + url = "${var.coder_access_url}/api/v2/users/login" + method = "POST" + request_headers = { + Accept = "application/json" + } + request_body = jsonencode({ + email = var.coder_admin_email + password = var.coder_admin_password + }) + + retry { + attempts = 5 + min_delay_ms = (5*1000) # 5 seconds + } +} + provider "coderd" { url = var.coder_access_url - token = var.coder_token + token = jsondecode(data.http.login.response_body).session_token } -provider "acme" { - server_url = var.acme_server_url +locals { + azs = slice(var.azs, 0, 1) + pub_subs = [for az in local.azs : "${var.vpc_name}-public-${data.aws_region.this.region}${az}"] + release_name = "coder" + chart_name = "coder" + namespace = "coder" + + common_name = trimprefix(trimprefix(var.coder_proxy_url, "https://"), "http://") + wildcard_name = trimprefix(trimprefix(var.coder_proxy_wildcard_url, "https://"), "http://") + ssl_vol_friendly_name = replace(local.common_name, ".", "-") +} + +resource "kubernetes_manifest" "certificate" { + + field_manager { + force_conflicts = true + } + + wait { + condition { + type = "Ready" + status = "True" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "Certificate" + metadata = { + name = local.ssl_vol_friendly_name + namespace = module.coder-proxy.namespace + } + spec = { + commonName = local.common_name + dnsNames = [ + local.common_name, + local.wildcard_name + ] + duration = "2160h" # 90 days + renewBefore = "360h" # 15 days + issuerRef = { + kind = "ClusterIssuer" + name = "issuer" + } + secretName = local.ssl_vol_friendly_name + privateKey = { + rotationPolicy = "Never" + algorithm = "RSA" + encoding = "PKCS1" + size = "2048" + } + } + } +} + +resource "aws_eip" "coder" { + count = length(local.pub_subs) + domain = "vpc" + public_ipv4_pool = "amazon" + tags = { + Name = "coder-eip-${count.index}" + } } module "coder-proxy" { + source = "../../../../../modules/k8s/bootstrap/coder-proxy" - namespace = "coder-proxy" - acme_registration_email = var.acme_registration_email - acme_days_until_renewal = 90 - replica_count = 2 - helm_version = var.addon_version - image_repo = var.image_repo - image_tag = var.image_tag - primary_access_url = var.coder_access_url - proxy_access_url = var.coder_proxy_url - proxy_wildcard_url = var.coder_proxy_wildcard_url - coder_proxy_name = var.coder_proxy_name - coder_proxy_display_name = var.coder_proxy_display_name - coder_proxy_icon = var.coder_proxy_icon - proxy_token_config = { - name = "coder-proxy" + proxy = { + access_url = var.coder_proxy_url + wildcard_url = var.coder_proxy_wildcard_url + coder_access_url = var.coder_access_url + mount_ssl = true + mount_ssl_name = kubernetes_manifest.certificate.manifest.spec.secretName + name = var.coder_proxy_name + display_name = var.coder_proxy_display_name + icon = var.coder_proxy_icon + rep_cnt = 2 + image_repo = var.image_repo + image_tag = var.image_tag } - cloudflare_api_token = var.cloudflare_api_token - ssl_cert_config = { - name = var.kubernetes_ssl_secret_name - create_secret = var.kubernetes_create_ssl_secret + + namespace = local.namespace + resource_limit = {} + resource_request = { + cpu = "500m" + memory = "1Gi" } - service_annotations = { - "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" + svc_annot = { + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "ip" "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" - "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=true" - } - node_selector = { - "node.coder.io/managed-by" = "karpenter" - "node.coder.io/used-for" = "coder-proxy" + "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" + "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" = join(",", aws_eip.coder.*.allocation_id) + "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", local.pub_subs) } tolerations = [{ - key = "dedicated" - operator = "Equal" - value = "coder-proxy" - effect = "NoSchedule" + key = "CriticalAddonsOnly" + operator = "Exists" }] - topology_spread_constraints = [{ + topology_spread = [{ max_skew = 1 topology_key = "kubernetes.io/hostname" when_unsatisfiable = "ScheduleAnyway" label_selector = { match_labels = { - "app.kubernetes.io/name" = "coder" - "app.kubernetes.io/part-of" = "coder" + "app.kubernetes.io/name" = local.chart_name + "app.kubernetes.io/part-of" = local.chart_name } } match_label_keys = [ "app.kubernetes.io/instance" ] }] - pod_anti_affinity_preferred_during_scheduling_ignored_during_execution = [{ - weight = 100 - pod_affinity_term = { - label_selector = { - match_labels = { - "app.kubernetes.io/instance" = "coder-v2" - "app.kubernetes.io/name" = "coder" - "app.kubernetes.io/part-of" = "coder" - } + affinity = { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [{ + matchExpressions = [{ + key = "topology.kubernetes.io/zone" + operator = "In" + values = [for az in local.azs : "${data.aws_region.this.region}${az}"] + }] + }] } - topology_key = "kubernetes.io/hostname" } - }] + # podAntiAffinity = { + # preferredDuringSchedulingIgnoredDuringExecution = [{ + # weight = 100 + # podAffinityTerm = { + # topologyKey = "kubernetes.io/hostname" + # labelSelector = { + # matchLabels = { + # "app.kubernetes.io/instance" = local.release_name + # "app.kubernetes.io/name" = local.chart_name + # "app.kubernetes.io/part-of" = local.chart_name + # } + # } + # } + # }] + # } + } } \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/coder-proxy/variables.tf b/infra/aws/eu-west-2/k8s/coder-proxy/variables.tf new file mode 100644 index 0000000..489f715 --- /dev/null +++ b/infra/aws/eu-west-2/k8s/coder-proxy/variables.tf @@ -0,0 +1,69 @@ +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "azs" { + type = list(string) + default = ["a","b","c"] +} + +variable "vpc_name" { + type = string +} + +variable "cluster_name" { + type = string +} + +variable "addon_version" { + type = string + default = "2.25.1" +} + +variable "coder_proxy_name" { + type = string +} + +variable "coder_proxy_display_name" { + type = string +} + +variable "coder_proxy_icon" { + type = string +} + +variable "coder_access_url" { + type = string +} + +variable "coder_proxy_url" { + type = string +} + +variable "coder_proxy_wildcard_url" { + type = string +} + +variable "image_repo" { + type = string + sensitive = true +} + +variable "image_tag" { + type = string + default = "latest" +} + +variable "coder_admin_email" { + type = string +} + +variable "coder_admin_password" { + type = string + sensitive = true +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/coder-ws/main.tf b/infra/aws/eu-west-2/k8s/coder-ws/main.tf deleted file mode 100644 index 31c49ac..0000000 --- a/infra/aws/eu-west-2/k8s/coder-ws/main.tf +++ /dev/null @@ -1,209 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - coderd = { - source = "coder/coderd" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "provisioner_addon_version" { - type = string - default = "2.23.0" -} - -variable "logstream_addon_version" { - type = string - default = "0.0.11" -} - -variable "coder_access_url" { - type = string - sensitive = true -} - -variable "coder_token" { - type = string - sensitive = true -} - -variable "image_repo" { - type = string - sensitive = true -} - -variable "image_tag" { - type = string - default = "latest" -} - -variable "rotate_key_image_repo" { - type = string - sensitive = true -} - -variable "rotate_key_image_tag" { - type = string - sensitive = true -} - -variable "aws_secret_id" { - type = string - sensitive = true -} - -variable "aws_secret_region" { - type = string - sensitive = true -} - -provider "aws" { - region = var.cluster_region - profile = var.cluster_profile -} - -data "aws_eks_cluster" "this" { - name = var.cluster_name -} - -data "aws_eks_cluster_auth" "this" { - name = var.cluster_name -} - -provider "helm" { - kubernetes = { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token - } -} - -provider "kubernetes" { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token -} - -provider "coderd" { - url = var.coder_access_url - token = var.coder_token -} - -locals { - service_account_labels = { - "app.kubernetes.io/instance" = "coder-provisioner" - "app.kubernetes.io/name" = "coder-provisioner" - "app.kubernetes.io/part-of" = "coder-provisioner" - } - node_selector = { - "node.coder.io/managed-by" = "karpenter" - "node.coder.io/used-for" = "coder-provisioner" - } - tolerations = [{ - key = "dedicated" - operator = "Equal" - value = "coder-provisioner" - effect = "NoSchedule" - }] -} - -module "default-ws" { - source = "../../../../../modules/k8s/bootstrap/coder-provisioner" - cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - - coder_organization_name = "coder" - - namespace = "coder-ws" - image_repo = var.image_repo - image_tag = var.image_tag - ws_service_account_name = "coder-ws" - ws_service_account_labels = local.service_account_labels - provisioner_service_account_name = "coder" - replica_count = 6 - primary_access_url = var.coder_access_url - env_vars = { - CODER_PROMETHEUS_ENABLE = "false" - CODER_PROMETHEUS_COLLECT_AGENT_STATS = "false" - CODER_PROMETHEUS_COLLECT_DB_METRICS = "false" - } - node_selector = local.node_selector - tolerations = local.tolerations -} - -module "experiment-ws" { - source = "../../../../../modules/k8s/bootstrap/coder-provisioner" - - cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - - coder_organization_name = "experiment" - - namespace = "coder-ws-experiment" - image_repo = var.image_repo - image_tag = var.image_tag - ws_service_account_name = "coder-ws-experiment" - ws_service_account_labels = local.service_account_labels - provisioner_service_account_name = "coder" - replica_count = 2 - primary_access_url = var.coder_access_url - env_vars = { - CODER_PROMETHEUS_ENABLE = "false" - CODER_PROMETHEUS_COLLECT_AGENT_STATS = "false" - CODER_PROMETHEUS_COLLECT_DB_METRICS = "false" - } - node_selector = local.node_selector - tolerations = local.tolerations -} - -module "demo-ws" { - source = "../../../../../modules/k8s/bootstrap/coder-provisioner" - - cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - - coder_organization_name = "demo" - - namespace = "coder-ws-demo" - image_repo = var.image_repo - image_tag = var.image_tag - ws_service_account_name = "coder-ws-demo" - ws_service_account_labels = local.service_account_labels - provisioner_service_account_name = "coder" - replica_count = 2 - primary_access_url = var.coder_access_url - env_vars = { - CODER_PROMETHEUS_ENABLE = "false" - CODER_PROMETHEUS_COLLECT_AGENT_STATS = "false" - CODER_PROMETHEUS_COLLECT_DB_METRICS = "false" - } - node_selector = local.node_selector - tolerations = local.tolerations -} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/ebs-controller/main.tf b/infra/aws/eu-west-2/k8s/ebs-controller/main.tf deleted file mode 100644 index e882e1d..0000000 --- a/infra/aws/eu-west-2/k8s/ebs-controller/main.tf +++ /dev/null @@ -1,72 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "addon_version" { - type = string - default = "2.22.1" -} - -variable "addon_namespace" { - type = string - default = "default" -} - -provider "aws" { - region = var.cluster_region - profile = var.cluster_profile -} - -data "aws_eks_cluster" "this" { - name = var.cluster_name -} - -data "aws_eks_cluster_auth" "this" { - name = var.cluster_name -} - -provider "helm" { - kubernetes = { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token - } -} - -module "ebs-controller" { - source = "../../../../../modules/k8s/bootstrap/ebs-controller" - cluster_name = data.aws_eks_cluster.this.name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - - namespace = var.addon_namespace - chart_version = var.addon_version -} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/karpenter/main.tf b/infra/aws/eu-west-2/k8s/karpenter/main.tf deleted file mode 100644 index a997bdf..0000000 --- a/infra/aws/eu-west-2/k8s/karpenter/main.tf +++ /dev/null @@ -1,203 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "addon_version" { - type = string -} - -variable "addon_namespace" { - type = string - default = "default" -} - -provider "aws" { - region = var.cluster_region - profile = var.cluster_profile -} - -data "aws_eks_cluster" "this" { - name = var.cluster_name -} - -data "aws_eks_cluster_auth" "this" { - name = var.cluster_name -} - -provider "helm" { - kubernetes = { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token - } -} - -provider "kubernetes" { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token -} - -locals { - global_node_labels = { - "node.coder.io/instance" = "coder-v2" - "node.coder.io/managed-by" = "karpenter" - } - global_node_reqs = [{ - key = "kubernetes.io/arch" - operator = "In" - values = ["amd64"] - }, { - key = "kubernetes.io/os" - operator = "In" - values = ["linux"] - }, { - key = "kubernetes.sh/capacity-type" - operator = "In" - values = ["spot", "on-demand"] - }] - provisioner_subnet_tags = { - "subnet.amazonaws.io/coder-provisioner/owned-by" = var.cluster_name - } - ws_all_subnet_tags = { - "subnet.amazonaws.io/coder-ws-all/owned-by" = var.cluster_name - } - provisioner_sg_tags = { - "sg.amazonaws.io/coder-provisioner/owned-by" = var.cluster_name - } - ws_all_sg_tags = { - "sg.amazonaws.io/coder-ws-all/owned-by" = var.cluster_name - } -} - -locals { - nodepool_configs = [{ - name = "coder-proxy" - node_labels = merge(local.global_node_labels, { - "node.coder.io/name" = "coder" - "node.coder.io/part-of" = "coder" - "node.coder.io/used-for" = "coder-proxy" - }) - node_taints = [{ - key = "dedicated" - value = "coder-proxy" - effect = "NoSchedule" - }] - node_requirements = concat(local.global_node_reqs, [{ - key = "node.kubernetes.io/instance-type" - operator = "In" - values = ["m5a.xlarge", "m6a.xlarge"] - }]) - node_class_ref_name = "coder-proxy-class" - }, { - name = "coder-provisioner" - node_labels = merge(local.global_node_labels, { - "node.coder.io/name" = "coder" - "node.coder.io/part-of" = "coder" - "node.coder.io/used-for" = "coder-provisioner" - }) - node_taints = [{ - key = "dedicated" - value = "coder-provisioner" - effect = "NoSchedule" - }] - node_requirements = concat(local.global_node_reqs, [{ - key = "node.kubernetes.io/instance-type" - operator = "In" - values = ["m5a.4xlarge", "m6a.4xlarge"] - }]) - node_class_ref_name = "coder-provisioner-class" - }, { - name = "coder-ws-all" - node_labels = merge(local.global_node_labels, { - "node.coder.io/name" = "coder" - "node.coder.io/part-of" = "coder" - "node.coder.io/used-for" = "coder-ws-all" - }) - node_taints = [{ - key = "dedicated" - value = "coder-ws" - effect = "NoSchedule" - }] - node_requirements = concat(local.global_node_reqs, [{ - key = "node.kubernetes.io/instance-type" - operator = "In" - values = ["c6a.32xlarge", "c5a.32xlarge"] - }]) - node_class_ref_name = "coder-ws-class" - disruption_consolidate_after = "30m" - }] -} - -module "karpenter-addon" { - source = "../../../../../modules/k8s/bootstrap/karpenter" - cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - - namespace = var.addon_namespace - chart_version = var.addon_version - node_selector = { - "node.amazonaws.io/managed-by" : "asg" - } - ec2nodeclass_configs = [{ - name = "coder-proxy-class" - subnet_selector_tags = local.provisioner_subnet_tags - sg_selector_tags = local.provisioner_sg_tags - }, { - name = "coder-ws-class" - subnet_selector_tags = local.ws_all_subnet_tags - # ami_alias = "al2023@latest" # Use /dev/xvda - ami_alias = "bottlerocket@latest" # Use /dev/xvda + /dev/xvdb - sg_selector_tags = local.ws_all_sg_tags - user_data = <<-EOF - [settings.kubernetes] - 'registry-qps' = 20 - EOF - block_device_mappings = [{ - device_name = "/dev/xvda" - ebs = { - volume_size = "1000Gi" - volume_type = "gp3" - } - }, { - device_name = "/dev/xvdb" - ebs = { - volume_size = "1000Gi" - volume_type = "gp3" - } - }] - }, { - name = "coder-provisioner-class" - subnet_selector_tags = local.provisioner_subnet_tags - sg_selector_tags = local.provisioner_sg_tags - }] -} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/lb-controller/main.tf b/infra/aws/eu-west-2/k8s/lb-controller/main.tf index 6bab002..d04ee31 100644 --- a/infra/aws/eu-west-2/k8s/lb-controller/main.tf +++ b/infra/aws/eu-west-2/k8s/lb-controller/main.tf @@ -1,54 +1,6 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "addon_namespace" { - type = string - default = "default" -} - -variable "addon_version" { - type = string - default = "1.13.2" -} - -variable "use_cert_manager" { - type = bool - default = false -} - provider "aws" { - region = var.cluster_region - profile = var.cluster_profile + region = var.region + profile = var.profile } data "aws_eks_cluster" "this" { @@ -59,6 +11,16 @@ data "aws_eks_cluster_auth" "this" { name = var.cluster_name } +data "aws_iam_openid_connect_provider" "this" { + url = data.aws_eks_cluster.this.identity[0].oidc[0].issuer +} + +data "aws_vpc" "this" { + tags = { + Name = var.vpc_name + } +} + provider "helm" { kubernetes = { host = data.aws_eks_cluster.this.endpoint @@ -67,18 +29,27 @@ provider "helm" { } } +provider "kubernetes" { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.this.token +} + module "lb-controller" { source = "../../../../../modules/k8s/bootstrap/lb-controller" cluster_name = data.aws_eks_cluster.this.name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn namespace = var.addon_namespace chart_version = var.addon_version enable_cert_manager = var.use_cert_manager + vpc_id = data.aws_vpc.this.id + + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] service_target_eni_sg_tags = { - Name = "aidemo-node" - } - cluster_asg_node_labels = { # var.karpenter_node_labels - "node.amazonaws.io/managed-by" = "asg" + Name = "aidemo-eks-node" } } \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/lb-controller/variables.tf b/infra/aws/eu-west-2/k8s/lb-controller/variables.tf new file mode 100644 index 0000000..1b505c0 --- /dev/null +++ b/infra/aws/eu-west-2/k8s/lb-controller/variables.tf @@ -0,0 +1,31 @@ +variable "cluster_name" { + type = string +} + +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "addon_namespace" { + type = string + default = "default" +} + +variable "addon_version" { + type = string + default = "1.13.2" +} + +variable "vpc_name" { + type = string +} + +variable "use_cert_manager" { + type = bool + default = false +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/metrics-server/main.tf b/infra/aws/eu-west-2/k8s/metrics-server/main.tf index 8b50a90..ed8c8d8 100644 --- a/infra/aws/eu-west-2/k8s/metrics-server/main.tf +++ b/infra/aws/eu-west-2/k8s/metrics-server/main.tf @@ -1,42 +1,6 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "addon_namespace" { - type = string - default = "kube-system" -} - -variable "addon_version" { - type = string - default = "3.13.0" -} - provider "aws" { - region = var.cluster_region - profile = var.cluster_profile + region = var.region + profile = var.profile } data "aws_eks_cluster" "this" { @@ -60,7 +24,8 @@ module "metrics-server" { namespace = var.addon_namespace chart_version = var.addon_version - node_selector = { - "node.amazonaws.io/managed-by" = "asg" - } + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] } \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/metrics-server/variables.tf b/infra/aws/eu-west-2/k8s/metrics-server/variables.tf new file mode 100644 index 0000000..6e7dfc0 --- /dev/null +++ b/infra/aws/eu-west-2/k8s/metrics-server/variables.tf @@ -0,0 +1,22 @@ +variable "cluster_name" { + type = string +} + +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "addon_namespace" { + type = string + default = "kube-system" +} + +variable "addon_version" { + type = string + default = "3.13.0" +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/other/main.tf b/infra/aws/eu-west-2/k8s/other/main.tf new file mode 100644 index 0000000..0e1b164 --- /dev/null +++ b/infra/aws/eu-west-2/k8s/other/main.tf @@ -0,0 +1,135 @@ +provider "aws" { + region = var.region + profile = var.profile +} + +data "aws_eks_cluster" "this" { + name = var.cluster_name +} + +data "aws_eks_cluster_auth" "this" { + name = var.cluster_name +} + +data "aws_iam_openid_connect_provider" "this" { + url = data.aws_eks_cluster.this.identity[0].oidc[0].issuer +} + +data "aws_region" "this" {} + +provider "helm" { + kubernetes = { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.this.token + } +} + +provider "kubernetes" { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.this.token +} + +## +# Manifest Setup Post Addon-Deployment +# Includes auxiliary resources depending on CRDs +## + +## +# EBS CSI StorageClasses +## + +resource "kubernetes_manifest" "automode-gp3" { + manifest = { + apiVersion = "storage.k8s.io/v1" + kind = "StorageClass" + metadata = { + name = "gp3-automode" + } + provisioner = "ebs.csi.eks.amazonaws.com" + volumeBindingMode = "WaitForFirstConsumer" + allowedTopologies = [{ + matchLabelExpressions = [{ + key = "eks.amazonaws.com/compute-type" + values = ["auto"] + }] + }] + parameters = { + type = "gp3" + encrypted = "true" + } + } +} + +## +# Setup Cert-Manager ClusterIssuer +## + +locals { + cf_secret_key = "key" +} + +resource "kubernetes_secret_v1" "cf" { + metadata { + name = "cloudflare-token" + namespace = var.cloudflare_secret_namespace + annotations = { + "custom.kubernetes.secret/key" = local.cf_secret_key + "custom.kubernetes.secret/email" = var.cloudflare_email + } + } + data = { + (local.cf_secret_key) = var.cloudflare_api_token + } +} + +resource "kubernetes_manifest" "issuer" { + + field_manager { + force_conflicts = true + } + + wait { + condition { + type = "Ready" + status = "True" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "ClusterIssuer" + metadata = { + labels = {} + name = var.cluster_issuer_name + } + spec = { + acme = { + privateKeySecretRef = { + name = var.cluster_issuer_priv_key_ref + } + server = "https://acme-v02.api.letsencrypt.org/directory" + solvers = [ + { + dns01 = { + cloudflare = { + apiTokenSecretRef = { + key = kubernetes_secret_v1.cf.metadata[0].annotations["custom.kubernetes.secret/key"] + name = kubernetes_secret_v1.cf.metadata[0].name + } + email = kubernetes_secret_v1.cf.metadata[0].annotations["custom.kubernetes.secret/email"] + } + } + } + ] + } + } + } +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/k8s/other/variables.tf b/infra/aws/eu-west-2/k8s/other/variables.tf new file mode 100644 index 0000000..d0ba21f --- /dev/null +++ b/infra/aws/eu-west-2/k8s/other/variables.tf @@ -0,0 +1,42 @@ +variable "profile" { + type = string + default = "default" +} + +variable "region" { + type = string + default = "us-east-2" +} + +variable "cluster_name" { + type = string +} + +variable "cluster_issuer_name" { + type = string + default = "issuer" +} + +variable "cluster_issuer_priv_key_ref" { + type = string + default = "issuer-account-key" +} + +variable "azs" { + type = list(string) + default = ["a","b","c"] +} + +variable "cloudflare_api_token" { + type = string + sensitive = true +} + +variable "cloudflare_secret_namespace" { + type = string + default = "cert-manager" +} + +variable "cloudflare_email" { + type = string +} diff --git a/infra/aws/eu-west-2/vpc/main.tf b/infra/aws/eu-west-2/vpc/main.tf new file mode 100644 index 0000000..c0e2e5d --- /dev/null +++ b/infra/aws/eu-west-2/vpc/main.tf @@ -0,0 +1,67 @@ +provider "aws" { + region = var.region + profile = var.profile +} + +locals { + # Subnet Discovery - https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/subnet_discovery/#subnet-filtering + tags_lb_subnet_discovery = { + "kubernetes.io/cluster/${var.cluster_name}" = "owned" + } + # Internet-Facing LB - https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/subnet_discovery/#subnet-role-tag + tags_public_lb = { + "kubernetes.io/role/elb" = 1 + } + tags_public_subnet = merge( + local.tags_lb_subnet_discovery, + local.tags_public_lb + ) + tags_private_subnet = { + "karpenter.sh/discovery" = "${var.cluster_name}" + } + # Use variable to configure AZ just in case 1 AZ isn't accessible + # https://repost.aws/questions/QUgdQev4KETKG_Bwev9tMtRQ/is-it-possible-to-enable-3rd-availability-zone-in-us-west-1#AN9eAH55FwTC-NSSp-FjIfTQ + availability_zones = [for az in var.azs : "${var.region}${az}"] + public_subnet_cidrs = [ for index, _ in var.azs : cidrsubnet(var.vpc_cidr, 8, index+1) ] + private_subnet_cidrs = [ for index, _ in var.azs : cidrsubnet(var.vpc_cidr, 2, index+1) ] +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 6.6.0" + + name = var.vpc_name + + enable_nat_gateway = false + enable_dns_hostnames = true + + cidr = var.vpc_cidr + azs = local.availability_zones + + public_subnet_suffix = var.public_subnet_suffix + public_subnets = [for index, _ in var.azs : local.public_subnet_cidrs[index]] + public_subnet_tags = local.tags_public_subnet + + private_subnet_suffix = var.private_subnet_suffix + private_subnets = [for index, _ in var.azs : local.private_subnet_cidrs[index]] + private_subnet_tags = local.tags_private_subnet + + tags = {} +} + +module "nat" { + + source = "RaJiska/fck-nat/aws" + version = "~> 1.4.0" + + name = "${var.region}-${var.nat_name}" + vpc_id = module.vpc.vpc_id + subnet_id = module.vpc.public_subnets[0] + ha_mode = true # Enables high-availability mode + use_cloudwatch_agent = true # Enables Cloudwatch agent and have metrics reported + + update_route_tables = true + route_tables_ids = { + for index, _ in var.azs : "rtb-${index}" => module.vpc.private_route_table_ids[index] + } +} \ No newline at end of file diff --git a/infra/aws/eu-west-2/vpc/variables.tf b/infra/aws/eu-west-2/vpc/variables.tf new file mode 100644 index 0000000..3cb5964 --- /dev/null +++ b/infra/aws/eu-west-2/vpc/variables.tf @@ -0,0 +1,48 @@ +variable "profile" { + type = string +} + +variable "region" { + description = "The aws region for the vpc" + type = string +} + +variable "cluster_name" { + description = "Cluster name" + type = string +} + +variable "azs" { + type = list(string) + default = ["a", "b", "c"] + + validation { + condition = length(var.azs) <= 3 + error_message = "There should only be at most 3 availability zones specified." + } +} + +variable "vpc_name" { + description = "Name for created resources and tag prefix" + type = string +} + +variable "vpc_cidr" { + type = string + default = "10.0.0.0/16" +} + +variable "private_subnet_suffix" { + type = string + default = "private" +} + +variable "public_subnet_suffix" { + type = string + default = "public" +} + +variable "nat_name" { + description = "Name for created resources and tag prefix" + type = string +} \ No newline at end of file diff --git a/infra/aws/us-east-2/eks/main.tf b/infra/aws/us-east-2/eks/main.tf index e23a5bf..1d3c589 100644 --- a/infra/aws/us-east-2/eks/main.tf +++ b/infra/aws/us-east-2/eks/main.tf @@ -1,88 +1,56 @@ -terraform { - required_version = ">= 1.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.46" - } - } - backend "s3" {} -} - -variable "name" { - description = "The resource name and tag prefix" - type = string -} - -variable "profile" { - type = string -} - -variable "region" { - description = "The aws region to deploy eks cluster" - type = string -} - -variable "cluster_version" { - description = "The eks version" - type = string +provider "aws" { + region = var.region + profile = var.profile } -variable "cluster_instance_type" { - description = "EKS Instance Size/Type" - default = "t3.xlarge" - type = string +data "aws_vpc" "this" { + tags = { + Name = var.vpc_name + } } -variable "vpc_id" { - type = string - sensitive = true -} +data "aws_subnets" "public" { + filter { + name = "vpc-id" + values = [data.aws_vpc.this.id] + } -variable "private_subnet_ids" { - type = list(string) - default = [] - sensitive = true + tags = { + Name = "*${var.public_subnet_suffix}*" + } } -variable "public_subnet_ids" { - type = list(string) - default = [] - sensitive = true -} +data "aws_subnets" "private" { + filter { + name = "vpc-id" + values = [data.aws_vpc.this.id] + } -provider "aws" { - region = var.region - profile = var.profile + tags = { + Name = "*${var.private_subnet_suffix}*" + } } locals { tags = { - Environment = "prod" - Name = "${var.name}-eks-cluster" + Name = var.name "karpenter.sh/discovery" = var.name } - system_subnet_tags = { - "subnet.amazonaws.io/system/owned-by" = var.name - } - provisioner_subnet_tags = { - "subnet.amazonaws.io/coder-provisioner/owned-by" = var.name - } - ws_all_subnet_tags = { - "subnet.amazonaws.io/coder-ws-all/owned-by" = var.name - } - system_sg_tags = { - "subnet.amazonaws.io/system/owned-by" = var.name - } - provisioner_sg_tags = { - "sg.amazonaws.io/coder-provisioner/owned-by" = var.name + # Karpenter Security Group Discovery - https://karpenter.sh/v1.0/concepts/nodeclasses/#specsecuritygroupselectorterms + tags_kptr_sg_discovery = { + "karpenter.sh/discovery" = var.name } - ws_all_sg_tags = { - "sg.amazonaws.io/coder-ws-all/owned-by" = var.name + labels_system_node = { + "scheduling.coder.com/pool" = "system" } - cluster_asg_node_labels = { - "node.amazonaws.io/managed-by" = "asg" + taints_system = { + key = "CriticalAddonsOnly" + effect = "NO_SCHEDULE" } + tolerations_system = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] } data "aws_iam_policy_document" "sts" { @@ -103,40 +71,57 @@ resource "aws_iam_policy" "sts" { module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 20.0" + version = "~> 21.15.1" - vpc_id = var.vpc_id - subnet_ids = toset(concat(var.public_subnet_ids, var.private_subnet_ids)) + vpc_id = data.aws_vpc.this.id + subnet_ids = toset(concat( + data.aws_subnets.private.ids + )) - cluster_name = var.name - cluster_version = var.cluster_version - cluster_endpoint_public_access = true - cluster_endpoint_private_access = true + name = var.name + kubernetes_version = var.eks_version + endpoint_public_access = true + endpoint_private_access = true - create_cluster_security_group = true + create_security_group = true create_node_security_group = true create_iam_role = true - cluster_addons = { + node_security_group_tags = local.tags_kptr_sg_discovery + create_node_iam_role = true + node_iam_role_use_name_prefix = true + + compute_config = { + enabled = true + node_pools = ["system"] + } + + addons = { coredns = { - most_recent = true + before_compute = true } kube-proxy = { - most_recent = true + before_compute = true } vpc-cni = { - most_recent = true + before_compute = true configuration_values = jsonencode({ enableNetworkPolicy = "true" nodeAgent = { enablePolicyEventLogs = "true" } + env = { + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + WARM_IP_TARGET = "0" + AWS_VPC_K8S_CNI_LOGLEVEL = "DEBUG" + } }) } } - attach_cluster_encryption_policy = false - create_kms_key = false - cluster_encryption_config = {} + attach_encryption_policy = false + create_kms_key = false # Enable unless needed + encryption_config = null enable_cluster_creator_admin_permissions = true enable_irsa = true @@ -145,9 +130,9 @@ module "eks" { min_size = 0 max_size = 10 desired_size = 0 # Cant be modified after creation. Override from AWS Console - labels = local.cluster_asg_node_labels + labels = local.labels_system_node - instance_types = [var.cluster_instance_type] + instance_types = [var.instance_type] capacity_type = "ON_DEMAND" iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" @@ -155,7 +140,7 @@ module "eks" { } # System Nodes should not be public - subnet_ids = var.private_subnet_ids + subnet_ids = data.aws_subnets.private.ids } } diff --git a/infra/aws/us-east-2/eks/variables.tf b/infra/aws/us-east-2/eks/variables.tf new file mode 100644 index 0000000..5a5d820 --- /dev/null +++ b/infra/aws/us-east-2/eks/variables.tf @@ -0,0 +1,38 @@ +variable "profile" { + type = string +} + +variable "region" { + description = "The aws region to deploy eks cluster" + type = string +} + +variable "name" { + description = "The resource name and tag prefix" + type = string +} + +variable "eks_version" { + description = "The eks version" + type = string +} + +variable "instance_type" { + description = "EKS Instance Size/Type" + default = "t3.xlarge" + type = string +} + +variable "vpc_name" { + type = string +} + +variable "private_subnet_suffix" { + type = string + default = "private" +} + +variable "public_subnet_suffix" { + type = string + default = "public" +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/cert-manager/main.tf b/infra/aws/us-east-2/k8s/cert-manager/main.tf index c095843..5b19d13 100644 --- a/infra/aws/us-east-2/k8s/cert-manager/main.tf +++ b/infra/aws/us-east-2/k8s/cert-manager/main.tf @@ -1,59 +1,6 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "addon_namespace" { - type = string - default = "cert-manager" -} - -variable "addon_version" { - type = string - default = "v1.18.2" -} - -variable "cloudflare_api_token" { - type = string - sensitive = true -} - -variable "cloudflare_email" { - type = string - sensitive = true -} - provider "aws" { - region = var.cluster_region - profile = var.cluster_profile + region = var.region + profile = var.profile } data "aws_eks_cluster" "this" { @@ -64,6 +11,10 @@ data "aws_eks_cluster_auth" "this" { name = var.cluster_name } +data "aws_iam_openid_connect_provider" "this" { + url = data.aws_eks_cluster.this.identity[0].oidc[0].issuer +} + provider "helm" { kubernetes = { host = data.aws_eks_cluster.this.endpoint @@ -79,12 +30,17 @@ provider "kubernetes" { } module "cert-manager" { + source = "../../../../../modules/k8s/bootstrap/cert-manager" + cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn namespace = var.addon_namespace helm_version = var.addon_version - cloudflare_token_secret = var.cloudflare_api_token - cloudflare_token_secret_email = var.cloudflare_email + + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] } \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/cert-manager/variables.tf b/infra/aws/us-east-2/k8s/cert-manager/variables.tf new file mode 100644 index 0000000..f93b94c --- /dev/null +++ b/infra/aws/us-east-2/k8s/cert-manager/variables.tf @@ -0,0 +1,22 @@ +variable "cluster_name" { + type = string +} + +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "addon_namespace" { + type = string + default = "cert-manager" +} + +variable "addon_version" { + type = string + default = "v1.18.2" +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/coder-server/main.tf b/infra/aws/us-east-2/k8s/coder-server/main.tf index 10da024..e8a3e5c 100644 --- a/infra/aws/us-east-2/k8s/coder-server/main.tf +++ b/infra/aws/us-east-2/k8s/coder-server/main.tf @@ -1,198 +1,30 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - coderd = { - source = "coder/coderd" - } - acme = { - source = "vancluever/acme" - } - tls = { - source = "hashicorp/tls" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "acme_server_url" { - type = string - default = "https://acme-v02.api.letsencrypt.org/directory" -} - -variable "acme_registration_email" { - type = string -} - -variable "addon_version" { - type = string - default = "2.25.1" -} - -variable "coder_access_url" { - type = string -} - -variable "coder_wildcard_access_url" { - type = string -} - -variable "coder_experiments" { - type = list(string) - default = [] -} - -variable "coder_github_allowed_orgs" { - type = list(string) - default = [] -} - -variable "coder_builtin_provisioner_count" { - type = number - default = 0 -} - -variable "coder_github_external_auth_secret_client_secret" { - type = string - sensitive = true -} - -variable "coder_github_external_auth_secret_client_id" { - type = string - sensitive = true -} - -variable "coder_oauth_secret_client_secret" { - type = string - sensitive = true -} - -variable "coder_oauth_secret_client_id" { - type = string - sensitive = true -} - -variable "coder_oidc_secret_client_secret" { - type = string - sensitive = true -} - -variable "coder_oidc_secret_client_id" { - type = string - sensitive = true -} - -variable "coder_oidc_secret_issuer_url" { - type = string - sensitive = true -} - -variable "coder_db_secret_url" { - type = string - sensitive = true -} - -variable "coder_token" { - type = string - sensitive = true -} - -variable "image_repo" { - type = string - sensitive = true -} - -variable "image_tag" { - type = string - default = "latest" -} - -variable "kubernetes_ssl_secret_name" { - type = string -} - -variable "kubernetes_create_ssl_secret" { - type = bool - default = true -} - -variable "cloudflare_api_token" { - type = string - sensitive = true -} - -variable "oidc_sign_in_text" { - type = string -} - -variable "oidc_icon_url" { - type = string -} - -variable "oidc_scopes" { - type = list(string) -} - -variable "oidc_email_domain" { - type = string -} - -variable "anthropic_llm_endpoint" { - type = string - sensitive = true +provider "aws" { + region = var.region + profile = var.profile } -variable "anthropic_llm_key" { - type = string - sensitive = true -} +data "aws_region" "this" {} -variable "openai_llm_endpoint" { - type = string - sensitive = true +data "aws_eks_cluster" "this" { + name = var.cluster_name } -variable "openai_llm_key" { - type = string - sensitive = true +data "aws_eks_cluster_auth" "this" { + name = var.cluster_name } -provider "aws" { - region = var.cluster_region - profile = var.cluster_profile +data "aws_iam_openid_connect_provider" "this" { + url = data.aws_eks_cluster.this.identity[0].oidc[0].issuer } -data "aws_eks_cluster" "this" { - name = var.cluster_name +data "aws_db_instance" "coder" { + db_instance_identifier = var.coder_db_rds_name } -data "aws_eks_cluster_auth" "this" { - name = var.cluster_name +data "aws_vpc" "this" { + tags = { + Name = var.vpc_name + } } provider "helm" { @@ -209,97 +41,213 @@ provider "kubernetes" { token = data.aws_eks_cluster_auth.this.token } -provider "coderd" { - url = var.coder_access_url - token = var.coder_token +locals { + common_name = trimprefix(trimprefix(var.coder_access_url, "https://"), "http://") + wildcard_name = trimprefix(trimprefix(var.coder_wildcard_access_url, "https://"), "http://") + ssl_vol_friendly_name = replace(local.common_name, ".", "-") +} + +resource "kubernetes_manifest" "certificate" { + + field_manager { + force_conflicts = true + } + + wait { + condition { + type = "Ready" + status = "True" + } + } + + timeouts { + create = "10m" + update = "10m" + delete = "30s" + } + + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "Certificate" + metadata = { + name = local.ssl_vol_friendly_name + namespace = module.coder-server.namespace + } + spec = { + commonName = local.common_name + dnsNames = [ + local.common_name, + local.wildcard_name + ] + duration = "2160h" # 90 days + renewBefore = "360h" # 15 days + issuerRef = { + kind = "ClusterIssuer" + name = "issuer" + } + secretName = local.ssl_vol_friendly_name + privateKey = { + rotationPolicy = "Never" + algorithm = "RSA" + encoding = "PKCS1" + size = "2048" + } + } + } +} + +locals { + azs = slice(var.azs, 0, 1) + pub_subs = [for az in local.azs : "${var.vpc_name}-public-${data.aws_region.this.region}${az}"] + release_name = "coder" + chart_name = "coder" + namespace = "coder" } -provider "acme" { - server_url = var.acme_server_url +resource "aws_eip" "coder" { + count = length(local.pub_subs) + domain = "vpc" + public_ipv4_pool = "amazon" + tags = { + Name = "coder-eip-${count.index}" + } } module "coder-server" { + source = "../../../../../modules/k8s/bootstrap/coder-server" - cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn + release_name = local.release_name + chart_version = var.addon_version + chart_name = local.chart_name + cluster_name = data.aws_eks_cluster.this.id + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn + + coder = { + access_url = var.coder_access_url + wildcard_url = var.coder_wildcard_access_url + mount_ssl = true + mount_ssl_name = kubernetes_manifest.certificate.manifest.spec.secretName + + image_repo = var.image_repo + image_tag = var.image_tag + + rep_cnt = 2 + # External Provisioners will be used + prov_rep_cnt = var.coder_builtin_provisioner_count + experiments = var.coder_experiments + } - namespace = "coder" - acme_registration_email = var.acme_registration_email - acme_days_until_renewal = 90 - replica_count = 2 - helm_version = var.addon_version - image_repo = var.image_repo - image_tag = var.image_tag - primary_access_url = var.coder_access_url - wildcard_access_url = var.coder_wildcard_access_url - cloudflare_api_token = var.cloudflare_api_token - coder_experiments = var.coder_experiments - coder_builtin_provisioner_count = var.coder_builtin_provisioner_count - coder_github_allowed_orgs = var.coder_github_allowed_orgs - anthropic_llm_endpoint = var.anthropic_llm_endpoint - anthropic_llm_key = var.anthropic_llm_key - openai_llm_endpoint = var.openai_llm_endpoint - openai_llm_key = var.openai_llm_key - ssl_cert_config = { - name = var.kubernetes_ssl_secret_name - create_secret = var.kubernetes_create_ssl_secret + db = { + url = data.aws_db_instance.coder.endpoint + username = var.coder_db_username + password = var.coder_db_password + db = var.coder_db_name } - oidc_config = { + + prometheus = { + enable = true + } + + oidc = { + enable = true sign_in_text = var.oidc_sign_in_text icon_url = var.oidc_icon_url scopes = var.oidc_scopes email_domain = var.oidc_email_domain + issuer_url = var.coder_oidc_secret_issuer_url + client_id = var.coder_oidc_secret_client_id + client_secret = var.coder_oidc_secret_client_secret } - db_secret_url = var.coder_db_secret_url - oidc_secret_issuer_url = var.coder_oidc_secret_issuer_url - oidc_secret_client_id = var.coder_oidc_secret_client_id - oidc_secret_client_secret = var.coder_oidc_secret_client_secret - oauth_secret_client_id = var.coder_oauth_secret_client_id - oauth_secret_client_secret = var.coder_oauth_secret_client_secret - github_external_auth_secret_client_id = var.coder_github_external_auth_secret_client_id - github_external_auth_secret_client_secret = var.coder_github_external_auth_secret_client_secret - tags = {} - service_annotations = { - "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" - "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" - "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=true" - } - node_selector = { - "node.coder.io/managed-by" = "karpenter" - "node.coder.io/used-for" = "coder-server" + + oauth2 = { + enable = true + default_provider_enable = false + allow_signups = true + device_flow = false + allowed_orgs = var.coder_github_allowed_orgs + client_id = var.coder_oauth_secret_client_id + client_secret = var.coder_oauth_secret_client_secret + use_extern_auth = false } - tolerations = [{ - key = "dedicated" - operator = "Equal" - value = "coder-server" - effect = "NoSchedule" + + extern_auth = [{ + id = "primary-github" + type = "github" + client_id = var.coder_github_external_auth_secret_client_id + client_secret = var.coder_github_external_auth_secret_client_secret }] - topology_spread_constraints = [{ - max_skew = 1 + + aibridge = { + enabled = true + anthropic = { + url = var.anthropic_llm_endpoint + key = var.anthropic_llm_key + } + openai = { + url = var.openai_llm_endpoint + key = var.openai_llm_key + } + } + + namespace = local.namespace + resource_limit = {} + resource_request = { + cpu = "500m" + memory = "1Gi" + } + lb_class = "service.k8s.aws/nlb" + svc_annot = { + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "ip" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" + "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" = join(",", aws_eip.coder.*.allocation_id) + "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", local.pub_subs) + } + topology_spread = [{ + max_skew = 2 topology_key = "kubernetes.io/hostname" when_unsatisfiable = "ScheduleAnyway" label_selector = { match_labels = { - "app.kubernetes.io/name" = "coder" - "app.kubernetes.io/part-of" = "coder" + "app.kubernetes.io/name" = local.chart_name + "app.kubernetes.io/part-of" = local.chart_name } } match_label_keys = [ "app.kubernetes.io/instance" ] }] - pod_anti_affinity_preferred_during_scheduling_ignored_during_execution = [{ - weight = 100 - pod_affinity_term = { - label_selector = { - match_labels = { - "app.kubernetes.io/instance" = "coder-v2" - "app.kubernetes.io/name" = "coder" - "app.kubernetes.io/part-of" = "coder" - } + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + },{ + key = "dedicated" + value = "general" + effect = "NoSchedule" + }] + affinity = { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [{ + matchExpressions = [ + { + key = "topology.kubernetes.io/zone" + operator = "In" + values = [for az in local.azs : "${data.aws_region.this.region}${az}"] + }, + # { + # key = "karpenter.sh/nodepool", + # operator = "Exists" + # }, + { + key = "eks.amazonaws.com/compute-type", + operator = "In", + values = ["auto"] + } + ] + }] } - topology_key = "kubernetes.io/hostname" } - }] + } } \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/coder-server/variables.tf b/infra/aws/us-east-2/k8s/coder-server/variables.tf new file mode 100644 index 0000000..e440871 --- /dev/null +++ b/infra/aws/us-east-2/k8s/coder-server/variables.tf @@ -0,0 +1,147 @@ +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "azs" { + type = list(string) + default = ["a", "c"] +} + +variable "vpc_name" { + type = string +} + +variable "cluster_name" { + type = string +} + +variable "addon_version" { + type = string + default = "2.25.1" +} + +variable "coder_access_url" { + type = string +} + +variable "coder_wildcard_access_url" { + type = string +} + +variable "coder_experiments" { + type = list(string) + default = [] +} + +variable "coder_github_allowed_orgs" { + type = list(string) + default = [] +} + +variable "coder_builtin_provisioner_count" { + type = number + default = 0 +} + +variable "coder_github_external_auth_secret_client_secret" { + type = string + sensitive = true +} + +variable "coder_github_external_auth_secret_client_id" { + type = string + sensitive = true +} + +variable "coder_oauth_secret_client_secret" { + type = string + sensitive = true +} + +variable "coder_oauth_secret_client_id" { + type = string + sensitive = true +} + +variable "coder_oidc_secret_client_secret" { + type = string + sensitive = true +} + +variable "coder_oidc_secret_client_id" { + type = string + sensitive = true +} + +variable "coder_oidc_secret_issuer_url" { + type = string + sensitive = true +} + +variable "coder_db_rds_name" { + type = string +} + +variable "coder_db_username" { + type = string +} + +variable "coder_db_password" { + type = string + sensitive = true +} + +variable "coder_db_name" { + type = string +} + +variable "image_repo" { + type = string + sensitive = true +} + +variable "image_tag" { + type = string + default = "latest" +} + +variable "oidc_sign_in_text" { + type = string +} + +variable "oidc_icon_url" { + type = string +} + +variable "oidc_scopes" { + type = list(string) +} + +variable "oidc_email_domain" { + type = string +} + +variable "anthropic_llm_endpoint" { + type = string + sensitive = true +} + +variable "anthropic_llm_key" { + type = string + sensitive = true +} + +variable "openai_llm_endpoint" { + type = string + sensitive = true +} + +variable "openai_llm_key" { + type = string + sensitive = true +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/coder-ws/main.tf b/infra/aws/us-east-2/k8s/coder-ws/main.tf index 0dc793f..bbda89b 100644 --- a/infra/aws/us-east-2/k8s/coder-ws/main.tf +++ b/infra/aws/us-east-2/k8s/coder-ws/main.tf @@ -1,94 +1,10 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - coderd = { - source = "coder/coderd" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "provisioner_addon_version" { - type = string - default = "2.23.0" -} - -variable "logstream_addon_version" { - type = string - default = "0.0.11" -} - -variable "coder_access_url" { - type = string - sensitive = true -} - -variable "coder_token" { - type = string - sensitive = true -} - -variable "image_repo" { - type = string - sensitive = true -} - -variable "image_tag" { - type = string - default = "latest" -} - -variable "rotate_key_image_repo" { - type = string - sensitive = true -} - -variable "rotate_key_image_tag" { - type = string - sensitive = true -} - -variable "aws_secret_id" { - type = string - sensitive = true -} - -variable "aws_secret_region" { - type = string - sensitive = true -} - provider "aws" { - region = var.cluster_region - profile = var.cluster_profile + region = var.region + profile = var.profile } +data "aws_region" "this" {} + data "aws_eks_cluster" "this" { name = var.cluster_name } @@ -97,6 +13,27 @@ data "aws_eks_cluster_auth" "this" { name = var.cluster_name } +data "aws_iam_openid_connect_provider" "this" { + url = data.aws_eks_cluster.this.identity[0].oidc[0].issuer +} + +data "http" "login" { + url = "${var.coder_access_url}/api/v2/users/login" + method = "POST" + request_headers = { + Accept = "application/json" + } + request_body = jsonencode({ + email = var.coder_admin_email + password = var.coder_admin_password + }) + + retry { + attempts = 5 + min_delay_ms = (5*1000) # 5 seconds + } +} + provider "helm" { kubernetes = { host = data.aws_eks_cluster.this.endpoint @@ -113,97 +50,252 @@ provider "kubernetes" { provider "coderd" { url = var.coder_access_url - token = var.coder_token + token = jsondecode(data.http.login.response_body).session_token } locals { - service_account_labels = { - "app.kubernetes.io/instance" = "coder-provisioner" - "app.kubernetes.io/name" = "coder-provisioner" - "app.kubernetes.io/part-of" = "coder-provisioner" - } - node_selector = { - "node.coder.io/managed-by" = "karpenter" - "node.coder.io/used-for" = "coder-provisioner" - } + node_selector = {} tolerations = [{ - key = "dedicated" - operator = "Equal" - value = "coder-provisioner" - effect = "NoSchedule" + key = "CriticalAddonsOnly" + operator = "Exists" }] + affinity = { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [{ + matchExpressions = [{ + key = "eks.amazonaws.com/compute-type" + operator = "In" + values = ["auto"] + }] + }] + } + } + } + release_name = "coder" + chart_name = "coder-provisioner" + namespace = "coder" } module "default-ws" { + source = "../../../../../modules/k8s/bootstrap/coder-provisioner" - cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - - coder_organization_name = "coder" - - namespace = "coder-ws" - image_repo = var.image_repo - image_tag = var.image_tag - ws_service_account_name = "coder-ws" - ws_service_account_labels = local.service_account_labels - provisioner_service_account_name = "coder" - replica_count = 6 - primary_access_url = var.coder_access_url - env_vars = { - CODER_PROMETHEUS_ENABLE = "true" - CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" - CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" + + release_name = local.release_name + chart_version = var.addon_version + chart_name = local.chart_name + cluster_name = data.aws_eks_cluster.this.id + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn + + namespace = "coder-ws" + + coder = { + access_url = var.coder_access_url + org_name = "coder" + image_repo = var.image_repo + image_tag = var.image_tag + rep_cnt = 4 + env_vars = { + CODER_PROMETHEUS_ENABLE = "true" + CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" + CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" + } + } + + svc_acc = { + create = true + name = "coder" } + node_selector = local.node_selector tolerations = local.tolerations + affinity = local.affinity } module "experiment-ws" { + source = "../../../../../modules/k8s/bootstrap/coder-provisioner" + release_name = local.release_name + chart_version = var.addon_version + chart_name = local.chart_name cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - - coder_organization_name = "experiment" + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn namespace = "coder-ws-experiment" - image_repo = var.image_repo - image_tag = var.image_tag - ws_service_account_name = "coder-ws-experiment" - ws_service_account_labels = local.service_account_labels - provisioner_service_account_name = "coder" - replica_count = 2 - primary_access_url = var.coder_access_url - env_vars = { - CODER_PROMETHEUS_ENABLE = "false" - CODER_PROMETHEUS_COLLECT_AGENT_STATS = "false" - CODER_PROMETHEUS_COLLECT_DB_METRICS = "false" + + coder = { + access_url = var.coder_access_url + org_name = "experiment" + image_repo = var.image_repo + image_tag = var.image_tag + rep_cnt = 4 + env_vars = { + CODER_PROMETHEUS_ENABLE = "true" + CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" + CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" + } } + + svc_acc = { + create = true + name = "coder" + } + node_selector = local.node_selector tolerations = local.tolerations + affinity = local.affinity } module "demo-ws" { + source = "../../../../../modules/k8s/bootstrap/coder-provisioner" + release_name = local.release_name + chart_version = var.addon_version + chart_name = local.chart_name cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - - coder_organization_name = "demo" + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn namespace = "coder-ws-demo" - image_repo = var.image_repo - image_tag = var.image_tag - ws_service_account_name = "coder-ws-demo" - ws_service_account_labels = local.service_account_labels - provisioner_service_account_name = "coder" - replica_count = 2 - primary_access_url = var.coder_access_url - env_vars = { - CODER_PROMETHEUS_ENABLE = "true" - CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" - CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" + + coder = { + access_url = var.coder_access_url + org_name = "demo" + image_repo = var.image_repo + image_tag = var.image_tag + rep_cnt = 4 + env_vars = { + CODER_PROMETHEUS_ENABLE = "true" + CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" + CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" + } + } + + svc_acc = { + create = true + name = "coder" } + node_selector = local.node_selector tolerations = local.tolerations -} \ No newline at end of file + affinity = local.affinity +} + + +# module "default-ws-tagged" { + +# source = "../../../../../modules/k8s/bootstrap/coder-provisioner" + +# release_name = local.release_name +# chart_version = var.addon_version +# chart_name = local.chart_name +# cluster_name = data.aws_eks_cluster.this.id +# cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn + +# namespace = "coder-ws-tagged" + +# coder = { +# access_url = var.coder_access_url +# org_name = "coder" +# image_repo = var.image_repo +# image_tag = var.image_tag +# ws_ns = ["coder-ws"] +# prov_tags = { +# region = "us-east-2" +# } +# rep_cnt = 2 +# env_vars = { +# CODER_PROMETHEUS_ENABLE = "true" +# CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" +# CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" +# } +# } + +# svc_acc = { +# create = true +# name = "coder" +# } + +# node_selector = local.node_selector +# tolerations = local.tolerations +# affinity = local.affinity +# } + +# module "experiment-ws-tagged" { + +# source = "../../../../../modules/k8s/bootstrap/coder-provisioner" + +# release_name = local.release_name +# chart_version = var.addon_version +# chart_name = local.chart_name +# cluster_name = var.cluster_name +# cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn + +# namespace = "coder-ws-experiment-tagged" + +# coder = { +# access_url = var.coder_access_url +# org_name = "experiment" +# image_repo = var.image_repo +# image_tag = var.image_tag +# ws_ns = ["coder-ws-experiment"] +# prov_tags = { +# region = "us-east-2" +# } +# rep_cnt = 2 +# env_vars = { +# CODER_PROMETHEUS_ENABLE = "true" +# CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" +# CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" +# } +# } + +# svc_acc = { +# create = true +# name = "coder" +# } + +# node_selector = local.node_selector +# tolerations = local.tolerations +# affinity = local.affinity +# } + +# module "demo-ws-tagged" { + +# source = "../../../../../modules/k8s/bootstrap/coder-provisioner" + +# release_name = local.release_name +# chart_version = var.addon_version +# chart_name = local.chart_name +# cluster_name = var.cluster_name +# cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn + +# namespace = "coder-ws-demo-tagged" + +# coder = { +# access_url = var.coder_access_url +# org_name = "demo" +# image_repo = var.image_repo +# image_tag = var.image_tag +# ws_ns = ["coder-ws-demo"] +# prov_tags = { +# region = "us-east-2" +# } +# rep_cnt = 2 +# env_vars = { +# CODER_PROMETHEUS_ENABLE = "true" +# CODER_PROMETHEUS_COLLECT_AGENT_STATS = "true" +# CODER_PROMETHEUS_COLLECT_DB_METRICS = "true" +# } +# } + +# svc_acc = { +# create = true +# name = "coder" +# } + +# node_selector = local.node_selector +# tolerations = local.tolerations +# affinity = local.affinity +# } \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/coder-ws/variables.tf b/infra/aws/us-east-2/k8s/coder-ws/variables.tf new file mode 100644 index 0000000..6ecbdcd --- /dev/null +++ b/infra/aws/us-east-2/k8s/coder-ws/variables.tf @@ -0,0 +1,45 @@ +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "cluster_name" { + type = string +} + +variable "addon_version" { + type = string + default = "2.23.0" +} + +variable "logstream_addon_version" { + type = string + default = "0.0.11" +} + +variable "coder_access_url" { + type = string +} + +variable "image_repo" { + type = string + sensitive = true +} + +variable "image_tag" { + type = string + default = "latest" +} + +variable "coder_admin_email" { + type = string +} + +variable "coder_admin_password" { + type = string + sensitive = true +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/ebs-controller/main.tf b/infra/aws/us-east-2/k8s/ebs-controller/main.tf deleted file mode 100644 index df1252c..0000000 --- a/infra/aws/us-east-2/k8s/ebs-controller/main.tf +++ /dev/null @@ -1,78 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "addon_version" { - type = string - default = "2.22.1" -} - -variable "addon_namespace" { - type = string - default = "default" -} - -variable "addon_replace" { - type = bool - default = false -} - -provider "aws" { - region = var.cluster_region - profile = var.cluster_profile -} - -data "aws_eks_cluster" "this" { - name = var.cluster_name -} - -data "aws_eks_cluster_auth" "this" { - name = var.cluster_name -} - -provider "helm" { - kubernetes = { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token - } -} - -module "ebs-controller" { - source = "../../../../../modules/k8s/bootstrap/ebs-controller" - cluster_name = data.aws_eks_cluster.this.name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - - namespace = var.addon_namespace - chart_version = var.addon_version - replace = var.addon_replace -} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/ebs-csi/main.tf b/infra/aws/us-east-2/k8s/ebs-csi/main.tf new file mode 100644 index 0000000..c766920 --- /dev/null +++ b/infra/aws/us-east-2/k8s/ebs-csi/main.tf @@ -0,0 +1,58 @@ +provider "aws" { + region = var.region + profile = var.profile +} + +data "aws_eks_cluster" "this" { + name = var.cluster_name +} + +data "aws_eks_cluster_auth" "this" { + name = var.cluster_name +} + +data "aws_iam_openid_connect_provider" "this" { + url = data.aws_eks_cluster.this.identity[0].oidc[0].issuer +} + +provider "helm" { + kubernetes = { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.this.token + } +} + +module "ebs-controller" { + source = "../../../../../modules/k8s/bootstrap/ebs-csi" + cluster_name = data.aws_eks_cluster.this.name + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn + + namespace = var.addon_namespace + chart_version = var.addon_version + replace = var.addon_replace + + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + },{ + key = "dedicated" + value = "general" + effect = "NoSchedule" + }] + affinity = { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [{ + matchExpressions = [ + { + key = "eks.amazonaws.com/compute-type", + operator = "In", + values = ["auto"] + } + ] + }] + } + } + } +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/ebs-csi/variables.tf b/infra/aws/us-east-2/k8s/ebs-csi/variables.tf new file mode 100644 index 0000000..dd4d169 --- /dev/null +++ b/infra/aws/us-east-2/k8s/ebs-csi/variables.tf @@ -0,0 +1,27 @@ +variable "cluster_name" { + type = string +} + +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "addon_version" { + type = string + default = "2.22.1" +} + +variable "addon_namespace" { + type = string + default = "default" +} + +variable "addon_replace" { + type = bool + default = false +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/efs-controller/main.tf b/infra/aws/us-east-2/k8s/efs-controller/main.tf deleted file mode 100644 index a5e2eb2..0000000 --- a/infra/aws/us-east-2/k8s/efs-controller/main.tf +++ /dev/null @@ -1,78 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "addon_version" { - type = string - default = "3.2.4" -} - -variable "addon_namespace" { - type = string - default = "default" -} - -variable "addon_replace" { - type = bool - default = false -} - -provider "aws" { - region = var.cluster_region - profile = var.cluster_profile -} - -data "aws_eks_cluster" "this" { - name = var.cluster_name -} - -data "aws_eks_cluster_auth" "this" { - name = var.cluster_name -} - -provider "helm" { - kubernetes = { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token - } -} - -module "efs-controller" { - source = "../../../../../modules/k8s/bootstrap/efs-controller" - cluster_name = data.aws_eks_cluster.this.name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - - namespace = var.addon_namespace - chart_version = var.addon_version - replace = var.addon_replace -} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/efs-controller/pv.yaml b/infra/aws/us-east-2/k8s/efs-controller/pv.yaml deleted file mode 100644 index 9b0a19f..0000000 --- a/infra/aws/us-east-2/k8s/efs-controller/pv.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: efs-pv -spec: - capacity: - storage: 10Gi - volumeMode: Filesystem - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - storageClassName: efs-sc - csi: - driver: efs.csi.aws.com - volumeHandle: "fs-0243b71fd52de9c0d" \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/efs-controller/storageclass.yaml b/infra/aws/us-east-2/k8s/efs-controller/storageclass.yaml deleted file mode 100644 index 5878fda..0000000 --- a/infra/aws/us-east-2/k8s/efs-controller/storageclass.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: efs-sc -provisioner: efs.csi.aws.com -parameters: - provisioningMode: efs-ap - fileSystemId: fs-0243b71fd52de9c0d - directoryPerms: "777" - # gidRangeStart: "1000" # optional - # gidRangeEnd: "2000" # optional - uid: "0" - gid: "1000" - basePath: "/dynamic_provisioning" \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/fetch-and-store/main.tf b/infra/aws/us-east-2/k8s/fetch-and-store/main.tf deleted file mode 100644 index 49194c0..0000000 --- a/infra/aws/us-east-2/k8s/fetch-and-store/main.tf +++ /dev/null @@ -1,78 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_oidc_provider_arn" { - type = string - sensitive = true -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "addon_namespace" { - type = string - default = "kube-system" -} - -variable "addon_version" { - type = string - default = "3.13.0" -} - -variable "image_repo" { - type = string - sensitive = true -} - -variable "image_tag" { - type = string - sensitive = true -} - -provider "aws" { - region = var.cluster_region - profile = var.cluster_profile -} - -data "aws_eks_cluster" "this" { - name = var.cluster_name -} - -data "aws_eks_cluster_auth" "this" { - name = var.cluster_name -} - -provider "kubernetes" { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token -} - -module "fetch-and-store" { - source = "../../../../../modules/k8s/bootstrap/fetch-and-store" - - cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - namespace = var.addon_namespace - image_repo = var.image_repo - image_tag = var.image_tag -} diff --git a/infra/aws/us-east-2/k8s/karpenter/main.tf b/infra/aws/us-east-2/k8s/karpenter/main.tf index 5e1a8a7..9451163 100644 --- a/infra/aws/us-east-2/k8s/karpenter/main.tf +++ b/infra/aws/us-east-2/k8s/karpenter/main.tf @@ -1,48 +1,6 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "addon_version" { - type = string -} - -variable "addon_namespace" { - type = string - default = "default" -} - provider "aws" { - region = var.cluster_region - profile = var.cluster_profile + region = var.region + profile = var.profile } data "aws_eks_cluster" "this" { @@ -53,6 +11,10 @@ data "aws_eks_cluster_auth" "this" { name = var.cluster_name } +data "aws_iam_openid_connect_provider" "this" { + url = data.aws_eks_cluster.this.identity[0].oidc[0].issuer +} + provider "helm" { kubernetes = { host = data.aws_eks_cluster.this.endpoint @@ -161,66 +123,85 @@ locals { } module "karpenter-addon" { + source = "../../../../../modules/k8s/bootstrap/karpenter" cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn + cluster_oidc_provider = trimprefix(data.aws_iam_openid_connect_provider.this.url, "https://") + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn namespace = var.addon_namespace chart_version = var.addon_version - node_selector = { - "node.amazonaws.io/managed-by" : "asg" - } + + node_selector = {} + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] + topology_spread = [] + + iam_role_use_name_prefix = true + node_iam_role_use_name_prefix = true + replicas = 2 karpenter_controller_role_policies = { "AmazonEFSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" } - ec2nodeclass_configs = [{ - name = "coder-server-class" - subnet_selector_tags = local.provisioner_subnet_tags - sg_selector_tags = local.provisioner_sg_tags - }, { - name = "coder-ws-class" - subnet_selector_tags = local.ws_all_subnet_tags - sg_selector_tags = local.ws_all_sg_tags - ami_alias = "al2023@latest" # Use /dev/xvda - user_data = <<-EOF - apiVersion: node.eks.aws/v1alpha1 - kind: NodeConfig - spec: - kubelet: - config: - registryPullQPS: 30 - EOF - block_device_mappings = [{ - device_name = "/dev/xvda" - ebs = { - volume_size = "500Gi" - volume_type = "gp3" - } - }] - # # Bottlerocket configs. - # ami_alias = "bottlerocket@latest" # Use /dev/xvda + /dev/xvdb - # user_data = <<-EOF - # [settings.kubernetes] - # 'registry-qps' = 30 - # [settings.kernel.sysctl] - # 'user.max_user_namespaces' = "16384" - # EOF - # block_device_mappings = [{ - # device_name = "/dev/xvda" - # ebs = { - # volume_size = "500Gi" - # volume_type = "gp3" - # } - # }, { - # device_name = "/dev/xvdb" - # ebs = { - # volume_size = "500Gi" - # volume_type = "gp3" - # } - # }] - }, { - name = "coder-provisioner-class" - subnet_selector_tags = local.provisioner_subnet_tags - sg_selector_tags = local.provisioner_sg_tags - }] -} \ No newline at end of file +} + + # namespace = var.addon_namespace + # chart_version = var.addon_version + # node_selector = { + # "node.amazonaws.io/managed-by" : "asg" + # } + # karpenter_controller_role_policies = { + # "AmazonEFSCSIDriverPolicy" = "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" + # } + # ec2nodeclass_configs = [{ + # name = "coder-server-class" + # subnet_selector_tags = local.provisioner_subnet_tags + # sg_selector_tags = local.provisioner_sg_tags + # }, { + # name = "coder-ws-class" + # subnet_selector_tags = local.ws_all_subnet_tags + # sg_selector_tags = local.ws_all_sg_tags + # ami_alias = "al2023@latest" # Use /dev/xvda + # user_data = <<-EOF + # apiVersion: node.eks.aws/v1alpha1 + # kind: NodeConfig + # spec: + # kubelet: + # config: + # registryPullQPS: 30 + # EOF + # block_device_mappings = [{ + # device_name = "/dev/xvda" + # ebs = { + # volume_size = "500Gi" + # volume_type = "gp3" + # } + # }] + # # # Bottlerocket configs. + # # ami_alias = "bottlerocket@latest" # Use /dev/xvda + /dev/xvdb + # # user_data = <<-EOF + # # [settings.kubernetes] + # # 'registry-qps' = 30 + # # [settings.kernel.sysctl] + # # 'user.max_user_namespaces' = "16384" + # # EOF + # # block_device_mappings = [{ + # # device_name = "/dev/xvda" + # # ebs = { + # # volume_size = "500Gi" + # # volume_type = "gp3" + # # } + # # }, { + # # device_name = "/dev/xvdb" + # # ebs = { + # # volume_size = "500Gi" + # # volume_type = "gp3" + # # } + # # }] + # }, { + # name = "coder-provisioner-class" + # subnet_selector_tags = local.provisioner_subnet_tags + # sg_selector_tags = local.provisioner_sg_tags + # }] \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/karpenter/variables.tf b/infra/aws/us-east-2/k8s/karpenter/variables.tf new file mode 100644 index 0000000..442aaa5 --- /dev/null +++ b/infra/aws/us-east-2/k8s/karpenter/variables.tf @@ -0,0 +1,21 @@ +variable "cluster_name" { + type = string +} + +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "addon_version" { + type = string +} + +variable "addon_namespace" { + type = string + default = "default" +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/lb-controller/main.tf b/infra/aws/us-east-2/k8s/lb-controller/main.tf index b0b7e2d..d04ee31 100644 --- a/infra/aws/us-east-2/k8s/lb-controller/main.tf +++ b/infra/aws/us-east-2/k8s/lb-controller/main.tf @@ -1,54 +1,6 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "cluster_oidc_provider_arn" { - type = string -} - -variable "addon_namespace" { - type = string - default = "default" -} - -variable "addon_version" { - type = string - default = "1.13.2" -} - -variable "use_cert_manager" { - type = bool - default = false -} - provider "aws" { - region = var.cluster_region - profile = var.cluster_profile + region = var.region + profile = var.profile } data "aws_eks_cluster" "this" { @@ -59,6 +11,16 @@ data "aws_eks_cluster_auth" "this" { name = var.cluster_name } +data "aws_iam_openid_connect_provider" "this" { + url = data.aws_eks_cluster.this.identity[0].oidc[0].issuer +} + +data "aws_vpc" "this" { + tags = { + Name = var.vpc_name + } +} + provider "helm" { kubernetes = { host = data.aws_eks_cluster.this.endpoint @@ -76,15 +38,18 @@ provider "kubernetes" { module "lb-controller" { source = "../../../../../modules/k8s/bootstrap/lb-controller" cluster_name = data.aws_eks_cluster.this.name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn namespace = var.addon_namespace chart_version = var.addon_version enable_cert_manager = var.use_cert_manager + vpc_id = data.aws_vpc.this.id + + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] service_target_eni_sg_tags = { Name = "aidemo-eks-node" } - cluster_asg_node_labels = { # var.karpenter_node_labels - "node.amazonaws.io/managed-by" = "asg" - } } \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/lb-controller/variables.tf b/infra/aws/us-east-2/k8s/lb-controller/variables.tf new file mode 100644 index 0000000..1b505c0 --- /dev/null +++ b/infra/aws/us-east-2/k8s/lb-controller/variables.tf @@ -0,0 +1,31 @@ +variable "cluster_name" { + type = string +} + +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "addon_namespace" { + type = string + default = "default" +} + +variable "addon_version" { + type = string + default = "1.13.2" +} + +variable "vpc_name" { + type = string +} + +variable "use_cert_manager" { + type = bool + default = false +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/litellm/main.tf b/infra/aws/us-east-2/k8s/litellm/main.tf index a759c7f..751815b 100644 --- a/infra/aws/us-east-2/k8s/litellm/main.tf +++ b/infra/aws/us-east-2/k8s/litellm/main.tf @@ -1,151 +1,329 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 3.1.1" - } - } - backend "s3" {} +provider "aws" { + region = var.region + profile = var.profile } -variable "cluster_name" { - type = string +data "aws_eks_cluster" "this" { + name = var.cluster_name } -variable "cluster_region" { - type = string +data "aws_eks_cluster_auth" "this" { + name = var.cluster_name } -variable "cluster_oidc_provider_arn" { - type = string +data "aws_iam_openid_connect_provider" "this" { + url = data.aws_eks_cluster.this.identity[0].oidc[0].issuer } -variable "cluster_profile" { - type = string - default = "default" +data "aws_db_instance" "litellm" { + db_instance_identifier = var.db_rds_id } -variable "addon_namespace" { - type = string - default = "kube-system" -} +data "aws_region" "this" {} -variable "aws_ingress_certificate_arn" { - type = string - sensitive = true +provider "kubernetes" { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.this.token } -variable "db_url" { - type = string - sensitive = true +provider "helm" { + kubernetes = { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.this.token + } } -variable "litellm_salt_key" { - type = string - sensitive = true +locals { + common_name = replace(replace(var.host_name, "https://", ""), "http://", "") + ssl_vol_friendly_name = replace(local.common_name, ".", "-") } -variable "litellm_master_key" { - type = string - sensitive = true -} +resource "kubernetes_manifest" "cert" { -variable "redis_host" { - type = string - sensitive = true + field_manager { + force_conflicts = true + } + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "Certificate" + metadata = { + labels = {} # var.cert_labels + name = local.ssl_vol_friendly_name + namespace = module.litellm.namespace + } + spec = { + secretName = local.ssl_vol_friendly_name + commonName = local.common_name + dnsNames = [local.common_name] + duration = "${90 * 24}h" + renewBefore = "8h" + additionalOutputFormats = [{ + type = "CombinedPEM" + },{ + type = "DER" + }] + issuerRef = { + kind = "ClusterIssuer" + name = "issuer" + } + } + } } -variable "redis_password" { - type = string - sensitive = true +resource "kubernetes_secret_v1" "gcloud" { + metadata { + name = "gcloud-auth" + namespace = module.litellm.namespace + labels = {} + } + data = { + "service_account.json" = var.gcloud_auth + } } -variable "gcloud_auth" { - type = string - sensitive = true +locals { + azs = slice(var.azs, 0, 1) + pub_subs = [for az in local.azs : "${var.vpc_name}-public-${data.aws_region.this.region}${az}"] + # App port is actually being ignored on the LiteLLM app-level. Statically set to 4000 + app_port = 4000 + # release_name = "coder" + # chart_name = "coder" + # namespace = "coder" } -variable "host_name" { - type = string - sensitive = true +resource "aws_eip" "litellm" { + count = length(local.pub_subs) + domain = "vpc" + public_ipv4_pool = "amazon" + tags = { + Name = "litellm-eip-${count.index}" + } } -variable "rotate_key_image_repo" { - type = string - sensitive = true -} +module "litellm" { + source = "../../../../../modules/k8s/bootstrap/litellm" -variable "rotate_key_image_tag" { - type = string - sensitive = true -} + cluster_name = var.cluster_name + cluster_oidc_provider_arn = data.aws_iam_openid_connect_provider.this.arn + namespace = var.addon_namespace + + access_url = var.host_name + replicas = 8 -variable "aws_secret_id" { - type = string - sensitive = true -} + litellm_master_key = var.litellm_master_key -variable "aws_secret_region" { - type = string - sensitive = true -} + service_lb_class = "service.k8s.aws/nlb" + svc_port = local.app_port + svc_annots = { + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "ip" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + "service.beta.kubernetes.io/aws-load-balancer-attributes" = "deletion_protection.enabled=false" + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol" = "https" + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-port" = "${local.app_port}" + "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" = join(",", aws_eip.litellm.*.allocation_id) + "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", local.pub_subs) + } + node_selector = {} + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + },{ + key = "dedicated" + value = "general" + effect = "NoSchedule" + }] + affinity = { + nodeAffinity = { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [{ + matchExpressions = [ + { + key = "topology.kubernetes.io/zone" + operator = "In" + values = [for az in local.azs : "${data.aws_region.this.region}${az}"] + }, + { + key = "eks.amazonaws.com/compute-type", + operator = "In", + values = ["auto"] + } + ] + }] + } + } + } -provider "aws" { - region = var.cluster_region - profile = var.cluster_profile -} + db = { + use_existing = true + db_name = "litellm" + endpoint = data.aws_db_instance.litellm.endpoint + username = "litellm" + admin_password = var.db_admin_password + user_password = var.db_user_password + } -data "aws_eks_cluster" "this" { - name = var.cluster_name -} + proxy_config = { + general_settings = { + master_key = "os.environ/PROXY_MASTER_KEY" -data "aws_eks_cluster_auth" "this" { - name = var.cluster_name -} + store_model_in_db = true + store_prompts_in_spend_logs = true + proxy_batch_write_at = 60 + database_connection_pool_limit = 10 -provider "kubernetes" { - host = data.aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token -} + disable_error_logs = true + allow_requests_on_db_unavailable = true + } -module "litellm" { - source = "../../../../../modules/k8s/bootstrap/litellm" + litellm_settings = { + allowed_fails = 3 + cooldown_time = 30 + num_retries = 1 + request_timeout = 45 + set_verbose = true + json_logs = false + cache = false + } - cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - namespace = var.addon_namespace - name = "litellm" - host_name = var.host_name - resource_limits = { - cpu = "2" - memory = "4Gi" + model_list = [ + { + model_name = "anthropic.claude-opus-4-5-20251101-v1:0" + litellm_params = { + max_parallel_requests = 50 + model = "vertex_ai/claude-opus-4-5@20251101" + rpm = 450 + tpm = 6000000 + vertex_location = "us-east5" + vertex_project = "coder-vertex-demos" + vertex_credentials = "/tmp/gcloud/service_account.json" + } + }, + { + model_name = "anthropic.claude-opus-4-5-20251101-v1:0" + litellm_params = { + max_parallel_requests = 50 + model = "vertex_ai/claude-opus-4-5@20251101" + rpm = 450 + tpm = 6000000 + vertex_location = "europe-west1" + vertex_project = "coder-vertex-demos" + vertex_credentials = "/tmp/gcloud/service_account.json" + } + }, + { + model_name = "anthropic.claude-haiku-4-5-20251001-v1:0" + litellm_params = { + max_parallel_requests = 50 + model = "vertex_ai/claude-haiku-4-5@20251001" + rpm = 3000 + tpm = 3000000 + vertex_location = "us-east5" + vertex_project = "coder-vertex-demos" + vertex_credentials = "/tmp/gcloud/service_account.json" + } + }, + { + model_name = "anthropic.claude-haiku-4-5-20251001-v1:0" + litellm_params = { + max_parallel_requests = 50 + model = "vertex_ai/claude-haiku-4-5@20251001" + rpm = 3600 + tpm = 3600000 + vertex_location = "europe-west1" + vertex_project = "coder-vertex-demos" + vertex_credentials = "/tmp/gcloud/service_account.json" + } + }, + { + model_name = "claude-opus-4-5" + litellm_params = { + max_parallel_requests = 50 + model = "vertex_ai/claude-opus-4-5@20251101" + rpm = 450 + tpm = 6000000 + vertex_location = "us-east5" + vertex_project = "coder-vertex-demos" + vertex_credentials = "/tmp/gcloud/service_account.json" + } + }, + { + model_name = "claude-opus-4-5" + litellm_params = { + max_parallel_requests = 50 + model = "vertex_ai/claude-opus-4-5@20251101" + rpm = 450 + tpm = 6000000 + vertex_location = "europe-west1" + vertex_project = "coder-vertex-demos" + vertex_credentials = "/tmp/gcloud/service_account.json" + } + }, + { + model_name = "claude-sonnet-4-5" + litellm_params = { + max_parallel_requests = 50 + model = "vertex_ai/claude-sonnet-4-5@20250929" + rpm = 3000 + tpm = 3000000 + vertex_location = "us-east5" + vertex_project = "coder-vertex-demos" + vertex_credentials = "/tmp/gcloud/service_account.json" + } + }, + { + model_name = "claude-sonnet-4-5" + litellm_params = { + max_parallel_requests = 50 + model = "vertex_ai/claude-sonnet-4-5@20250929" + rpm = 3600 + tpm = 3600000 + vertex_location = "europe-west1" + vertex_project = "coder-vertex-demos" + vertex_credentials = "/tmp/gcloud/service_account.json" + } + }, + { + model_name = "claude-haiku-4-5" + litellm_params = { + max_parallel_requests = 50 + model = "vertex_ai/claude-haiku-4-5@20251001" + rpm = 3000 + tpm = 3000000 + vertex_location = "us-east5" + vertex_project = "coder-vertex-demos" + vertex_credentials = "/tmp/gcloud/service_account.json" + } + }, + { + model_name = "claude-haiku-4-5" + litellm_params = { + max_parallel_requests = 50 + model = "vertex_ai/claude-haiku-4-5@20251001" + rpm = 3600 + tpm = 3600000 + vertex_location = "europe-west1" + vertex_project = "coder-vertex-demos" + vertex_credentials = "/tmp/gcloud/service_account.json" + } + } + ] + + router_settings = { + num_retries = 2 + routing_strategy = "usage-based-routing-v2" + } } - replicas = 8 - aws_ingress_certificate_arn = var.aws_ingress_certificate_arn - db_url = var.db_url - litellm_salt_key = var.litellm_salt_key - litellm_master_key = var.litellm_master_key - redis_host = var.redis_host - redis_password = var.redis_password - gcloud_auth = var.gcloud_auth -} -module "litellm-gen-key" { - depends_on = [module.litellm] - source = "../../../../../modules/k8s/bootstrap/litellm-generate-key" + mount_ssl = { + enable = true + secret_name = kubernetes_manifest.cert.manifest.metadata.name + path = "/tmp/ssl" + } - cluster_name = var.cluster_name - cluster_oidc_provider_arn = var.cluster_oidc_provider_arn - namespace = var.addon_namespace - name = "rotate-key" - image_repo = var.rotate_key_image_repo - image_tag = var.rotate_key_image_tag - litellm_create_secret = false - litellm_url = "https://${var.host_name}" - secret_id = var.aws_secret_id - secret_region = var.aws_secret_region + mounts = [{ + secret_name = kubernetes_secret_v1.gcloud.metadata[0].name + path = "/tmp/gcloud/" + }] } \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/litellm/variables.tf b/infra/aws/us-east-2/k8s/litellm/variables.tf new file mode 100644 index 0000000..bc87707 --- /dev/null +++ b/infra/aws/us-east-2/k8s/litellm/variables.tf @@ -0,0 +1,56 @@ +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "cluster_name" { + type = string +} + +variable "addon_namespace" { + type = string + default = "kube-system" +} + +variable "db_rds_id" { + type = string + sensitive = true +} + +variable "db_admin_password" { + type = string + sensitive = true +} + +variable "db_user_password" { + type = string + sensitive = true +} + +variable "litellm_master_key" { + type = string + sensitive = true +} + +variable "gcloud_auth" { + type = string + sensitive = true +} + +variable "host_name" { + type = string + sensitive = true +} + +variable "vpc_name" { + type = string +} + +variable "azs" { + type = list(string) + default = ["a","b","c"] +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/metrics-server/main.tf b/infra/aws/us-east-2/k8s/metrics-server/main.tf index fe2967e..ed8c8d8 100644 --- a/infra/aws/us-east-2/k8s/metrics-server/main.tf +++ b/infra/aws/us-east-2/k8s/metrics-server/main.tf @@ -1,42 +1,6 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.17.0" - } - } - backend "s3" {} -} - -variable "cluster_name" { - type = string -} - -variable "cluster_region" { - type = string -} - -variable "cluster_profile" { - type = string - default = "default" -} - -variable "addon_namespace" { - type = string - default = "kube-system" -} - -variable "addon_version" { - type = string - default = "3.13.0" -} - provider "aws" { - region = var.cluster_region - profile = var.cluster_profile + region = var.region + profile = var.profile } data "aws_eks_cluster" "this" { @@ -60,7 +24,8 @@ module "metrics-server" { namespace = var.addon_namespace chart_version = var.addon_version - node_selector = { - "node.amazonaws.io/managed-by" = "asg" - } + tolerations = [{ + key = "CriticalAddonsOnly" + operator = "Exists" + }] } \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/metrics-server/variables.tf b/infra/aws/us-east-2/k8s/metrics-server/variables.tf new file mode 100644 index 0000000..6e7dfc0 --- /dev/null +++ b/infra/aws/us-east-2/k8s/metrics-server/variables.tf @@ -0,0 +1,22 @@ +variable "cluster_name" { + type = string +} + +variable "region" { + type = string +} + +variable "profile" { + type = string + default = "default" +} + +variable "addon_namespace" { + type = string + default = "kube-system" +} + +variable "addon_version" { + type = string + default = "3.13.0" +} \ No newline at end of file diff --git a/infra/aws/us-east-2/k8s/monitoring/dashboards/aibridge-demoable.json b/infra/aws/us-east-2/k8s/monitoring/dashboards/aibridge-demoable.json index 70d570f..bcfd1d5 100644 --- a/infra/aws/us-east-2/k8s/monitoring/dashboards/aibridge-demoable.json +++ b/infra/aws/us-east-2/k8s/monitoring/dashboards/aibridge-demoable.json @@ -36,7 +36,7 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "fieldConfig": { @@ -86,7 +86,7 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "editorMode": "code", @@ -139,7 +139,7 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "fieldConfig": { @@ -276,7 +276,7 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "editorMode": "code", @@ -429,7 +429,7 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "editorMode": "code", @@ -660,7 +660,7 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "fieldConfig": { @@ -728,7 +728,7 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "editorMode": "code", @@ -768,7 +768,7 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "fieldConfig": { @@ -865,7 +865,7 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "editorMode": "code", @@ -918,7 +918,7 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "fieldConfig": { @@ -1029,7 +1029,7 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "editorMode": "code", @@ -1083,7 +1083,7 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "fieldConfig": { @@ -1238,7 +1238,7 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "editorMode": "code", @@ -1305,7 +1305,7 @@ ] }, "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "definition": "select username from users where deleted=false;", @@ -1329,7 +1329,7 @@ ] }, "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "definition": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", @@ -1353,7 +1353,7 @@ ] }, "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "aezivxsm5pnggc" }, "definition": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '${provider:regex}' ORDER BY 1;", diff --git a/infra/aws/us-east-2/k8s/monitoring/dashboards/aibridge.json b/infra/aws/us-east-2/k8s/monitoring/dashboards/aibridge.json index 0d1fb72..f7a5286 100644 --- a/infra/aws/us-east-2/k8s/monitoring/dashboards/aibridge.json +++ b/infra/aws/us-east-2/k8s/monitoring/dashboards/aibridge.json @@ -5,7 +5,7 @@ "label": "coder-observability-ro", "description": "", "type": "datasource", - "pluginId": "grafana-postgresql-datasource", + "pluginId": "postgres", "pluginName": "PostgreSQL" } ], @@ -25,7 +25,7 @@ }, { "type": "datasource", - "id": "grafana-postgresql-datasource", + "id": "postgres", "name": "PostgreSQL", "version": "12.1.0" }, @@ -79,8 +79,8 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "fieldConfig": { "defaults": { @@ -216,8 +216,8 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "editorMode": "code", "format": "table", @@ -265,8 +265,8 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "fieldConfig": { "defaults": { @@ -313,8 +313,8 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "editorMode": "code", "format": "table", @@ -379,8 +379,8 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "fieldConfig": { "defaults": { @@ -476,8 +476,8 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "editorMode": "code", "format": "time_series", @@ -516,8 +516,8 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "fieldConfig": { "defaults": { @@ -582,8 +582,8 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "editorMode": "code", "format": "table", @@ -635,8 +635,8 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "fieldConfig": { "defaults": { @@ -744,8 +744,8 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "editorMode": "code", "format": "table", @@ -798,8 +798,8 @@ }, { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "fieldConfig": { "defaults": { @@ -951,8 +951,8 @@ "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "editorMode": "code", "format": "table", @@ -1012,8 +1012,8 @@ "allValue": ".+", "current": {}, "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "definition": "select username from users where deleted=false;", "description": "", @@ -1031,8 +1031,8 @@ "allValue": ".+", "current": {}, "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "definition": "SELECT DISTINCT provider FROM aibridge_interceptions WHERE provider IS NOT NULL ORDER BY 1;", "description": "", @@ -1050,8 +1050,8 @@ "allValue": ".+", "current": {}, "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "grafana-postgresql-datasource" + "type": "postgres", + "uid": "postgres" }, "definition": "SELECT DISTINCT model FROM aibridge_interceptions WHERE model IS NOT NULL AND provider ~ '$${provider:regex}' ORDER BY 1;", "description": "", diff --git a/infra/aws/us-east-2/k8s/monitoring/dashboards/boundary.json b/infra/aws/us-east-2/k8s/monitoring/dashboards/boundary.json new file mode 100644 index 0000000..5990323 --- /dev/null +++ b/infra/aws/us-east-2/k8s/monitoring/dashboards/boundary.json @@ -0,0 +1,875 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "This dashboard shows HTTP requests audited by agent boundaries within Coder workspaces to provide visibility into workspace network activity.\n\nWhat it shows:\n - Total count of allowed and denied outbound HTTP requests\n - Top 20 most frequently accessed allowed domains\n - Top 20 most frequently blocked domains\n - Recent allowed requests with details (time, domain, method, path, workspace owner, workspace name, template ID)\n - Recent denied requests with the same details\n\nWho it's for:\n - Platform administrators and template administrators who need to audit workspace network activity and define agent boundary policies\n - Security team members who want to know what HTTP requests AI agents made in Coder workspaces for security incident review\n - Agent Boundaries policy owners who want to refine network access controls/security posture\n\nFilters available:\n - Template ID\n - HTTP request domain\n - Workspace owner", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "Total number of requests allowed/denied", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "{decision=\"allow\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Allowed" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "{decision=\"deny\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Denied" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "sum by (decision) (count_over_time({ ${NON_WORKSPACE_SELECTOR}, logger=`coderd.agentrpc`} |= `boundary_request` | logfmt | decision=~`deny|allow` | owner=~`$owner` | domain=~`$domain` | template_id=~`$template_id` | template_version_id=~`$template_version_id` [$__range]))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Request Totals", + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "Top 20 allowed domains for HTTP requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Domain" + }, + "properties": [ + { + "id": "custom.width", + "value": 340 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Count" + } + ] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "topk(20, sum by (domain) (count_over_time({ ${NON_WORKSPACE_SELECTOR}, logger=`coderd.agentrpc`} |= `boundary_request` | logfmt | decision=`allow` | owner=~`$owner` | template_id=~`$template_id` | template_version_id=~`$template_version_id` | regexp `http_url=(?Phttps?)://(?P[^/:]+)` | domain=~`$domain` [$__auto])))", + "legendFormat": "", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Allowed Domains", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Time": "", + "Value #A": "Count", + "domain": "Domain" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Count" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "Top 20 denied domains for HTTP requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Domain" + }, + "properties": [ + { + "id": "custom.width", + "value": 382 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Count" + } + ] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "topk(20, sum by (domain) (count_over_time({ ${NON_WORKSPACE_SELECTOR}, logger=`coderd.agentrpc`} |= `boundary_request` | logfmt | decision=`deny` | owner=~`$owner` | template_id=~`$template_id` | template_version_id=~`$template_version_id` | regexp `http_url=(?Phttps?)://(?P[^/:]+)` | domain=~`$domain` [$__auto])))", + "legendFormat": "", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top Denied Domains", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Time": "", + "Value #A": "Count", + "domain": "Domain" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Count" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 282 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Domain" + }, + "properties": [ + { + "id": "custom.width", + "value": 185 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "method" + }, + "properties": [ + { + "id": "custom.width", + "value": 73 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "path" + }, + "properties": [ + { + "id": "custom.width", + "value": 397 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Workspace Owner" + }, + "properties": [ + { + "id": "custom.width", + "value": 144 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Template ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 195 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Workspace Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 204 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "{ ${NON_WORKSPACE_SELECTOR}, logger=`coderd.agentrpc`} |= `boundary_request` | logfmt | decision=`allow` | owner=~`$owner` | template_id=~`$template_id` | template_version_id=~`$template_version_id` | regexp `http_url=https?://(?P[^/?# ]+)(?P/[^?# ]*)?` | domain=~`$domain` | line_format `time=\"{{.event_time}}\" method=\"{{.http_method}}\" domain=\"{{.domain}}\" path=\"{{.path}}\" owner=\"{{.owner}}\" workspace_name=\"{{.workspace_name}}\" template_id=\"{{.template_id}}\" template_version_id=\"{{.template_version_id}}\"`", + "queryType": "range", + "refId": "A" + } + ], + "title": "Most recent allowed requests", + "transformations": [ + { + "id": "extractFields", + "options": { + "delimiter": "|", + "format": "kvp", + "keepTime": false, + "replace": true, + "source": "Line" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": { + "domain": true, + "method": true, + "owner": true, + "path": true, + "template_id": true, + "template_version_id": true, + "time": true, + "workspace_name": true + }, + "indexByName": { + "domain": 2, + "method": 1, + "owner": 4, + "path": 3, + "template_id": 6, + "time": 0, + "workspace_name": 5 + }, + "renameByName": { + "domain": "Domain", + "method": "Method", + "owner": "Workspace Owner", + "path": "Path", + "template_id": "Template ID", + "template_version_id": "Template Version ID", + "time": "Time", + "workspace_name": "Workspace Name" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + }, + { + "id": "limit", + "options": { + "limitField": "10" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 282 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Domain" + }, + "properties": [ + { + "id": "custom.width", + "value": 185 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "method" + }, + "properties": [ + { + "id": "custom.width", + "value": 73 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "path" + }, + "properties": [ + { + "id": "custom.width", + "value": 397 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Workspace Owner" + }, + "properties": [ + { + "id": "custom.width", + "value": 152 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "direction": "backward", + "editorMode": "code", + "expr": "{ ${NON_WORKSPACE_SELECTOR}, logger=`coderd.agentrpc`} |= `boundary_request` | logfmt | decision=`deny` | owner=~`$owner` | template_id=~`$template_id` | template_version_id=~`$template_version_id` | regexp `http_url=https?://(?P[^/?# ]+)(?P/[^?# ]*)?` | domain=~`$domain` | line_format `time=\"{{.event_time}}\" method=\"{{.http_method}}\" domain=\"{{.domain}}\" path=\"{{.path}}\" owner=\"{{.owner}}\" workspace_name=\"{{.workspace_name}}\" template_id=\"{{.template_id}}\" template_version_id=\"{{.template_version_id}}\"`", + "queryType": "range", + "refId": "A" + } + ], + "title": "Most recent denied requests", + "transformations": [ + { + "id": "extractFields", + "options": { + "delimiter": "|", + "format": "kvp", + "keepTime": false, + "replace": true, + "source": "Line" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": { + "domain": true, + "method": true, + "owner": true, + "path": true, + "template_id": true, + "template_version_id": true, + "time": true, + "workspace_name": true + }, + "indexByName": { + "domain": 2, + "method": 1, + "owner": 4, + "path": 3, + "template_id": 6, + "time": 0, + "workspace_name": 5 + }, + "renameByName": { + "domain": "Domain", + "method": "Method", + "owner": "Workspace Owner", + "path": "Path", + "template_id": "Template ID", + "template_version_id": "Template Version ID", + "time": "Time", + "workspace_name": "Workspace Name" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + }, + { + "id": "limit", + "options": { + "limitField": "10" + } + } + ], + "type": "table" + } + ], + "preload": false, + "refresh": "${DASHBOARD_REFRESH}", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "description": "Filter by request domain", + "label": "Domain", + "name": "domain", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "type": "textbox" + }, + { + "current": { + "text": "", + "value": "" + }, + "description": "Filter requests by workspace owner", + "label": "Workspace Owner", + "name": "owner", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "type": "textbox" + }, + { + "current": { + "text": "", + "value": "" + }, + "description": "Filter requests by template ID (UUID). Template IDs can be found via the CLI with \"coder templates list\".", + "label": "Template ID", + "name": "template_id", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "type": "textbox" + }, + { + "current": { + "text": "", + "value": "" + }, + "description": "Filter by template version ID (UUID). A templates version IDs can be found via the CLI with \"coder templates versions list