diff --git a/.github/workflows/scan.yaml b/.github/workflows/scan.yaml new file mode 100644 index 0000000..daa14ef --- /dev/null +++ b/.github/workflows/scan.yaml @@ -0,0 +1,20 @@ +name: Kubescape scanning for misconfigurations +on: [push, pull_request] +jobs: + kubescape: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: kubescape/github-action@main + continue-on-error: true + with: + format: sarif + outputFile: results.sarif + # Optional: Specify the Kubescape cloud account ID + # account: ${{secrets.KUBESCAPE_ACCOUNT}} + # # Optional: Scan a specific path. Default will scan the whole repository + # files: "examples/*.yaml" + - name: Upload Kubescape scan results to Github Code Scanning + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results.sarif \ No newline at end of file diff --git a/README.md b/README.md index eb45a7f..e957f62 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The FinOps Stack is the blueprint for a solution to automate FinOps best practic Our goal is to empower organisations with the tools they need to manage, visualise, and optimise their cloud resources in complex, ever-changing environments. -The FinOps Stack is designed to work out-of-the-box seamlessly with GKE standard/autopilot clusters using Google Managed Prometheus, and can be customised for an organisation’s business requirements and/or Kubernetes distribution. +The FinOps Stack is designed to work out-of-the-box seamlessly with Kind, GKE standard/autopilot clusters using Google Managed Prometheus, and EKS and can be customised for an organisation’s business requirements and/or Kubernetes distribution. This repository contains the core components of FinOps Stack, including: diff --git a/charts/opencost-config/templates/opencost/configmap-custom-pricing.yaml b/charts/opencost-config/templates/opencost/configmap-custom-pricing.yaml new file mode 100644 index 0000000..cbd8455 --- /dev/null +++ b/charts/opencost-config/templates/opencost/configmap-custom-pricing.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.opencost.customPricing.createConfigmap .Values.opencost.customPricing.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.opencost.customPricing.configmapName }} +data: + {{ .Values.opencost.configFileName }}.json: |- + { +{{- range $key, $val := .Values.opencost.customPricing.costModel }} +{{ $key | quote | indent 6}}: {{ $val | quote }}, +{{- end}} + "provider" : {{ .Values.opencost.provider | quote }} + } +{{- end }} diff --git a/charts/opencost-config/templates/opencost/dashboards.yaml b/charts/opencost-config/templates/opencost/dashboards.yaml index c91e911..b775af1 100644 --- a/charts/opencost-config/templates/opencost/dashboards.yaml +++ b/charts/opencost-config/templates/opencost/dashboards.yaml @@ -1,3 +1,4 @@ +{{ if .Values.dashboards }} {{- range $path, $_ := .Files.Glob "dashboards/**.json" }} --- apiVersion: v1 @@ -11,3 +12,4 @@ data: {{ base $path }}: |- {{ ($.Files.Get $path) | nindent 6 }} {{ end }} +{{ end }} diff --git a/charts/opencost-config/values-gke.yaml b/charts/opencost-config/values-gke.yaml deleted file mode 100644 index 7d86430..0000000 --- a/charts/opencost-config/values-gke.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Extension of values.yaml for GKE -gke: - podmonitoring: true - # prometheusFrontend: - # enabled: true - # serviceAccount: - # annotations: - # iam.gke.io/gcp-service-account: SERVICEACCOUNT_NAME@PROJECT_ID.iam.gserviceaccount.com - - - - diff --git a/content/architecture.md b/content/architecture.md index 6426de2..eae2a65 100644 --- a/content/architecture.md +++ b/content/architecture.md @@ -7,7 +7,7 @@ FinOps Stack makes use of a wide range of software to provide a highly efficient - **[Goldilocks](https://www.fairwinds.com/goldilocks)**: Fairwinds Goldilocks is designed to provide 'just-right' recommendations for container requests and limits. It does this by using the Vertical Pod Autoscaler in recommendation mode. Additional guidance on interpresting and setting resource requests and limits can be found in our [GKE Autopilot guide](./distribution-gke-autopilot.md). - **[Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/README.md)**: frees users from the necessity of setting up-to-date resource limits and requests for the containers in their pods. When configured, it will set the requests automatically based on usage and thus allow proper scheduling onto nodes so that appropriate resource amount is available for each pod. - **[Grafana](https://grafana.com/grafana/)**: An open-source platform used for monitoring and visualising time-series data from various sources, allowing users to create interactive and real-time dashboards for metrics, logs, and analytics. The FinOps Stack uses Grafana to provide visualisations of cost and efficiency metrics. -- **gmp-proxy**: A proxy designed to simplify the integration between open-source Grafana and Google Managed Prometheus. +- **gmp-proxy**: A proxy designed to simplify the integration between open-source Grafana and Google Managed Prometheus (if using a GKE cluster). - **[Cert-Manager](https://cert-manager.io/)**: An open source tool that provides certificate management for Kubernetes. In the FinOps Stack, cert-manager can be optionally installed to create a tls certificate if using ingress for Grafana. diff --git a/content/install.md b/content/install.md index 38fb5ba..fada58e 100644 --- a/content/install.md +++ b/content/install.md @@ -8,7 +8,9 @@ To simplify installation, the FinOps Stack is installed using a single Helmfile The following instructions are designed to work with a Kind cluster for quick setup. For full instructions, prerequisites and customisations, please see the [installation README](https://github.com/jetstack/finops-stack/blob/main/installation/README.md). -To work with GKE standard and GKE autopilot see the [ GKE installation guide]((https://github.com/jetstack/finops-stack/blob/main/installation/README.md)) +To work with GKE standard and GKE autopilot see the [ GKE installation guide]((https://github.com/jetstack/finops-stack/blob/main/installation/gke.md)) + +To work with EKS cluster see the [ EKS installation guide]((https://github.com/jetstack/finops-stack/blob/main/installation/eks.md)) ### Helmfile diff --git a/content/intro.md b/content/intro.md index f0c58cc..6011208 100644 --- a/content/intro.md +++ b/content/intro.md @@ -6,7 +6,7 @@ The FinOps Stack is the blueprint for a solution to automate FinOps best practic Our goal is to empower organisations with the tools they need to manage, visualise, and optimise their cloud resources in complex, ever-changing environments. -The FinOps Stack is designed to work out-of-the-box seamlessly with GKE standard/autopilot clusters using Google Managed Prometheus, and can be customised for an organisation’s business requirements and/or Kubernetes distribution. Additionally, there is an option to deploy the FinOps Stack on a Kind cluster for a quick setup, allowing you to explore its functionality. +The FinOps Stack is designed to work out-of-the-box seamlessly with GKE standard/autopilot clusters using [Google Managed Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus), or with an EKS cluster using standard Prometheus. It can be customised for an organisation’s business requirements and/or Kubernetes distribution. Additionally, for a quick setup and exploration, the FinOps Stack can be deployed on a Kind cluster. ## Key Features and Capabilities diff --git a/installation/Helmfile_eks.yaml b/installation/Helmfile_eks.yaml new file mode 100644 index 0000000..eae5dd1 --- /dev/null +++ b/installation/Helmfile_eks.yaml @@ -0,0 +1,139 @@ +repositories: +- name: kyverno + url: https://kyverno.github.io/kyverno/ +- name: prometheus-opencost-exporter + url: https://prometheus-community.github.io/helm-charts +- name: prometheus + url: https://prometheus-community.github.io/helm-charts +- name: grafana + url: https://grafana.github.io/helm-charts +- name: vpa + url: https://charts.fairwinds.com/stable +- name: cert-manager + url: https://charts.jetstack.io +- name: fairwinds-stable + url: https://charts.fairwinds.com/stable + +--- +helmDefaults: + wait: true + timeout: 1200 +--- +environments: + default: + values: + - "./config/{{ requiredEnv "HOST_ENV" }}/enabled.yaml" +--- +releases: +- name: kyverno + version: "3.2.6" + chart: kyverno/kyverno + condition: kyverno.enabled + namespace: finops-stack + values: + - "./config/common/kyverno-values.yaml" + - "./config/{{ requiredEnv "HOST_ENV" }}/kyverno-values.yaml" + +- name: finops-policies + version: "0.1.0" + chart: "../charts/finops-policies" + condition: finops-policies.enabled + namespace: finops-stack + disableValidationOnInstall: true + needs: + - kyverno + +- name: cert-manager + version: v1.15.3 + chart: cert-manager/cert-manager + condition: cert-manager.enabled + namespace: cert-manager + values: + - "./config/common/cert-manager-values.yaml" + - "./config/{{ requiredEnv "HOST_ENV" }}/cert-manager-values.yaml" + - global: + leaderElection: + namespace: cert-manager + commonLabels: + cost-center-label: "xyz" + - serviceAccount: + annotations: + {{ env "CERT_MANAGER_SA_ANNOTATION" }} + +- name: cert-manager-config + version: "0.1.0" + chart: "../charts/cert-manager-config" + condition: cert-manager.enabled + namespace: finops-stack + disableValidationOnInstall: true + needs: + - cert-manager/cert-manager + values: + - email: {{ env "CERT_MANAGER_EMAIL" }} + - grafanaTLSCert: + hostname: {{ env "GRAFANA_FQDN" }} + +- name: grafana + version: "8.4.7" + chart: grafana/grafana + condition: grafana.enabled + namespace: finops-stack + values: + - "./config/common/grafana-values.yaml" + - "./config/{{ requiredEnv "HOST_ENV" }}/grafana-values.yaml" + - adminUser: {{ env "GRAFANA_ADMIN_USER" }} + - adminPassword: {{ env "GRAFANA_ADMIN_PW" }} + - serviceAccount: + annotations: + {{ env "GRAFANA_SA_ANNOTATION" }} + + # If you don't require ingress via an FQDN remove this ingress section + - ingress: + enabled: {{ env "GRAFANA_INGRESS" }} + annotations: + kubernetes.io/ingress.global-static-ip-name: {{ env "GRAFANA_PUBLIC_IP_NAME" }} + hosts: + - {{ env "GRAFANA_FQDN" }} + # If you don't require https access to the Grafana dashboard remove this tls section + tls: + - secretName: {{ env "GRAFANA_FQDN" }}-tls + hosts: + - {{ env "GRAFANA_FQDN" }} + needs: + - cert-manager/cert-manager + +- name: vpa + version: "4.5.0" + chart: "vpa/vpa" + condition: vpa.enabled + namespace: finops-stack + values: + - "./config/common/vpa-values.yaml" + +- name: prometheus-opencost-exporter + version: "0.1.1" + chart: "prometheus-opencost-exporter/prometheus-opencost-exporter" + condition: opencost-exporter.enabled + namespace: finops-stack + values: + - "./config/common/prometheus-opencost-exporter-values.yaml" + - "./config/{{ requiredEnv "HOST_ENV" }}/prometheus-opencost-exporter-values.yaml" + +- name: finops-stack-opencost-templates + version: "0.1.0" + chart: "../charts/opencost-config" + condition: opencost-templates.enabled + namespace: finops-stack + disableValidationOnInstall: true + values: + - "./config/{{ requiredEnv "HOST_ENV" }}/opencost-templates-values.yaml" + +- name: goldilocks + version: "9.0.0" + chart: fairwinds-stable/goldilocks + condition: goldilocks.enabled + namespace: finops-stack + values: + - "./config/common/goldilocks-values.yaml" + + diff --git a/installation/Helmfile.yaml b/installation/Helmfile_gke.yaml similarity index 90% rename from installation/Helmfile.yaml rename to installation/Helmfile_gke.yaml index a69bd3e..cfebc50 100644 --- a/installation/Helmfile.yaml +++ b/installation/Helmfile_gke.yaml @@ -1,8 +1,6 @@ repositories: - name: kyverno url: https://kyverno.github.io/kyverno/ -- name: opencost - url: https://opencost.github.io/opencost-helm-chart - name: prometheus-opencost-exporter url: https://prometheus-community.github.io/helm-charts - name: grafana @@ -22,7 +20,7 @@ helmDefaults: environments: default: values: - - ./config/common/enabled.yaml # edit this to specify which of the releases to install + - "./config/common/enabled.yaml" # edit this to specify which of the releases to install --- releases: - name: kyverno @@ -111,17 +109,6 @@ releases: values: - "./config/common/vpa-values.yaml" -- name: opencost - version: "1.42.0" - chart: "opencost/opencost" - condition: opencost.enabled - namespace: finops-stack - values: - - "./config/{{ requiredEnv "HOST_ENV" }}/opencost-values.yaml" - - opencost: - exporter: - cloudProviderApiKey: {{ requiredEnv "CSP_API_KEY" }} - - name: gmp-proxy version: "0.1.0" chart: "../charts/gmp-proxy" diff --git a/installation/Helmfile_kind.yaml b/installation/Helmfile_kind.yaml new file mode 100644 index 0000000..3c6b85a --- /dev/null +++ b/installation/Helmfile_kind.yaml @@ -0,0 +1,95 @@ +repositories: +- name: kyverno + url: https://kyverno.github.io/kyverno/ +- name: prometheus-opencost-exporter + url: https://prometheus-community.github.io/helm-charts +- name: prometheus + url: https://prometheus-community.github.io/helm-charts +- name: grafana + url: https://grafana.github.io/helm-charts +- name: vpa + url: https://charts.fairwinds.com/stable +- name: fairwinds-stable + url: https://charts.fairwinds.com/stable + +--- +helmDefaults: + wait: true + timeout: 1200 +--- +environments: + default: + values: + - ./config/{{ requiredEnv "HOST_ENV" }}/enabled.yaml +--- +releases: +- name: kyverno + version: "3.2.6" + chart: kyverno/kyverno + condition: kyverno.enabled + namespace: finops-stack + values: + - "./config/common/kyverno-values.yaml" + # - "./config/{{ requiredEnv "HOST_ENV" }}/kyverno-values.yaml" + +- name: finops-policies + version: "0.1.0" + chart: "../charts/finops-policies" + condition: finops-policies.enabled + namespace: finops-stack + disableValidationOnInstall: true + needs: + - kyverno + +- name: grafana + version: "8.4.7" + chart: grafana/grafana + condition: grafana.enabled + namespace: finops-stack + values: + - "./config/common/grafana-values.yaml" + - "./config/{{ requiredEnv "HOST_ENV" }}/grafana-values.yaml" + - adminUser: {{ requiredEnv "GRAFANA_ADMIN_USER" }} + - adminPassword: {{ requiredEnv "GRAFANA_ADMIN_PW" }} + +- name: vpa + version: "4.5.0" + chart: "vpa/vpa" + condition: vpa.enabled + namespace: finops-stack + values: + - "./config/common/vpa-values.yaml" + +- name: prometheus + version: "25.27.0" + chart: "prometheus/prometheus" + condition: prometheus.enabled + namespace: finops-stack + values: + - "./config/{{ requiredEnv "HOST_ENV" }}/prometheus-values.yaml" + +- name: prometheus-opencost-exporter + version: "0.1.1" + chart: "prometheus-opencost-exporter/prometheus-opencost-exporter" + condition: opencost-exporter.enabled + namespace: finops-stack + values: + - "./config/common/prometheus-opencost-exporter-values.yaml" + - "./config/{{ requiredEnv "HOST_ENV" }}/prometheus-opencost-exporter-values.yaml" + +- name: finops-stack-opencost-templates + version: "0.1.0" + chart: "../charts/opencost-config" + condition: opencost-templates.enabled + namespace: finops-stack + disableValidationOnInstall: true + values: + - "./config/{{ requiredEnv "HOST_ENV" }}/opencost-templates-values.yaml" + +- name: goldilocks + version: "9.0.0" + chart: fairwinds-stable/goldilocks + condition: goldilocks.enabled + namespace: finops-stack + values: + - "./config/common/goldilocks-values.yaml" diff --git a/installation/Makefile b/installation/Makefile new file mode 100644 index 0000000..f20be81 --- /dev/null +++ b/installation/Makefile @@ -0,0 +1,13 @@ +.DEFAULT_GOAL := help + +cluster: ## Setup cluster + kind create cluster + +finops-stack: ## Deploy FinOps Stack using Helmfile + set -a; source .env; set +a; helmfile apply --file helmfile_kind.yaml --interactive + +cleanup: ## Cleaup + kind delete cluster + +help: ## Display this help + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m \t%s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/installation/README.md b/installation/README.md index 710cf35..3593680 100644 --- a/installation/README.md +++ b/installation/README.md @@ -1,40 +1,44 @@ -# Installation using Helmfile +# FinOps Stack deployment -Installing Helm charts with lots of dependencies and CRDs is challenging; these instructions use Helmfile to mitigate issues with Helm. +This documentation provides instructions for installing the FinOps Stack in Kind cluster for a quick setup. -This documentation focuses on installing the FinOps Stack in GKE standard/autopilot clusters. +For deployment on a GKE cluster, refer to the [GKE docs](./gke.md) and deployment on a EKS cluster refer to the [EKS docs](./eks.md). + +## Using Helmfile + +Installing Helm charts with lots of dependencies and CRDs is challenging; these instructions use Helmfile to mitigate issues with Helm. ## Pre-requisites -- A GKE standard or autopilot cluster with: - - kubectl access - - cluster-admin permissions - - workload identity enabled +- [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) installed on your local machine +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) - [Helmfile](https://helmfile.readthedocs.io/en/latest/#installation) installed on your local machine -- A Google Service Account with the following: - - roles/monitor.viewer and roles/iam.serviceAccountTokenCreator permissions - - workload identity configured for grafana kubernetes service account: `[finops-stack/grafana]`. Have a look at this [blog post](https://venafi.com/blog/gke-workload-identity-federation-for-kubernetes-principals/) to configure workload identity. -- Unless you want to access the Grafana dashboard via `kubectl port-forward` you'll need a domain name - -## Distribution support -### GKE Autopilot +## Installation -- Enable cost allocation +## Create a kind cluster -## Installation +```bash +make cluster +``` ### Configuration changes for your cluster environment 1. To control which Finops Stack components to install, edit the [enabled.yaml](./installation/config/common/enabled.yaml) file -1. Copy `./env.tmpl` to `./.env` and replace the env var values accordingly. As a minimum, you will need to change the `GCP_PROJECT`, `CSP_API_KEY`, `GRAFANA_SA_ANNOTATION` values. +1. Copy env.tmpl file and replace the env var values accordingly (`GRAFANA_FQDN` for example). + +```sh +cp ./env.tmpl ./.env +``` ### Install everything using Helmfile For the first run: ```bash -set -a; source .env; set +a; helmfile apply --interactive +make finops-stack +# FinOps stack is install using Helmfile: +# set -a; source .env; set +a; helmfile apply --file helmfile_kind.yaml --interactive ``` NOTE: it will take several minutes for all workloads to install and start running. Helmfile does display its progress in the terminal. All workloads get installed into the `finops-stack` namespace so you can also view progress using `kubectl`. @@ -61,8 +65,11 @@ General guidance when configuring ingress: ## Enable Goldilocks for namespaces -For Goldilocks to analyse namespaces and add then to its dashboard you need to add this label to the namespace resource: `goldilocks.fairwinds.com/enabled=true`, e.g. -`kubectl label ns finops-stack goldilocks.fairwinds.com/enabled=true` +For Goldilocks to analyse namespaces and add then to its dashboard you need to add this label to the namespace resource: `goldilocks.fairwinds.com/enabled=true`, e.g: + +```bash +kubectl label ns finops-stack goldilocks.fairwinds.com/enabled=true +``` ## Useful commands diff --git a/installation/config/common/enabled.yaml b/installation/config/common/enabled.yaml index 81f356a..e3a2105 100644 --- a/installation/config/common/enabled.yaml +++ b/installation/config/common/enabled.yaml @@ -14,19 +14,19 @@ grafana: # -- Enable / Disable the installation of cert-manager cert-manager: enabled: false -# -- Enable / Disable the installation of OpenCost -opencost: - enabled: false # -- Enable / Disable the installation of OpenCost Exporter opencost-exporter: enabled: true +# -- Enable / Disable the installation of Prometheus (required when using EKS or Kind) +prometheus: + enabled: false opencost-templates: enabled: true # -- Enable / Disable the installation of GMP Proxy (required when using Google Managed Prometheus) gmp-proxy: - enabled: true + enabled: false # -- Enable / Disable the installation of Fairwinds' Goldilocks goldilocks: diff --git a/installation/config/eks/grafana-values.yaml b/installation/config/eks/grafana-values.yaml new file mode 100644 index 0000000..781dc41 --- /dev/null +++ b/installation/config/eks/grafana-values.yaml @@ -0,0 +1,16 @@ +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + uid: "PFB5ABA51A8A585D7" + url: http://prometheus-server + isDefault: true + editable: true +ingress: + enabled: false + annotations: + kubernetes.io/ingress.global-static-ip-name: "ip-name" + hosts: + - grafana.example.com diff --git a/installation/config/eks/kyverno-values.yaml b/installation/config/eks/kyverno-values.yaml new file mode 100644 index 0000000..2d4980c --- /dev/null +++ b/installation/config/eks/kyverno-values.yaml @@ -0,0 +1,13 @@ +features: + backgroundScan: + skipResourceFilters: false # Important so that background Scan's Exclude Namespaces, etc +config: + resourceFiltersIncludeNamespaces: + - kube-system + webhooks: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system diff --git a/installation/config/eks/opencost-templates-values.yaml b/installation/config/eks/opencost-templates-values.yaml new file mode 100644 index 0000000..3a6f68b --- /dev/null +++ b/installation/config/eks/opencost-templates-values.yaml @@ -0,0 +1,5 @@ +dashboards: true +opencost: + customPricing: + # -- This is only to enable custom pricing on on-premises or kind + enabled: false diff --git a/installation/config/eks/opencost-values.yaml b/installation/config/eks/opencost-values.yaml new file mode 100644 index 0000000..bc6dbdf --- /dev/null +++ b/installation/config/eks/opencost-values.yaml @@ -0,0 +1,18 @@ +cloudCost: + enabled: false +opencost: + cloudCost: + enabled: false + prometheus: + external: + enabled: false + # url: http://prometheus-server + internal: + # -- Use in-cluster Prometheus + enabled: true + # -- Service name of in-cluster Prometheus + serviceName: prometheus-server + # -- Namespace of in-cluster Prometheus + namespaceName: finops-stack + # -- Service port of in-cluster Prometheus + port: 80 diff --git a/installation/config/eks/prometheus-opencost-exporter-values.yaml b/installation/config/eks/prometheus-opencost-exporter-values.yaml new file mode 100644 index 0000000..de70cd1 --- /dev/null +++ b/installation/config/eks/prometheus-opencost-exporter-values.yaml @@ -0,0 +1,27 @@ +service: + enabled: true +opencost: + prometheus: + external: + enabled: false + # url: http://prometheus-server + internal: + # -- Use in-cluster Prometheus + enabled: true + # -- Service name of in-cluster Prometheus + serviceName: prometheus-server + # -- Namespace of in-cluster Prometheus + namespaceName: finops-stack + # -- Service port of in-cluster Prometheus + port: 80 + exporter: + persistence: + enabled: false + # -- Annotations for persistent volume + annotations: {} + # -- Access mode for persistent volume + accessMode: "" + # -- Storage class for persistent volume + storageClass: "standard-rwo" + # -- Size for persistent volume + size: "10g" diff --git a/installation/config/eks/prometheus-values.yaml b/installation/config/eks/prometheus-values.yaml new file mode 100644 index 0000000..8c0a0b1 --- /dev/null +++ b/installation/config/eks/prometheus-values.yaml @@ -0,0 +1,17 @@ +extraScrapeConfigs: | + - job_name: opencost + honor_labels: true + scrape_interval: 1m + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + dns_sd_configs: + - names: + - opencost.opencost + type: 'A' + port: 9003 + +alertmanager: + enabled: false +prometheus-pushgateway: + enabled: false diff --git a/installation/config/gke/opencost-templates-values.yaml b/installation/config/gke/opencost-templates-values.yaml index 92397d3..417b1e2 100644 --- a/installation/config/gke/opencost-templates-values.yaml +++ b/installation/config/gke/opencost-templates-values.yaml @@ -1,2 +1,7 @@ +dashboards: true +opencost: + customPricing: + # -- This is only to enable custom pricing on on-premises or kind + enabled: false gke: - podmonitoring: true \ No newline at end of file + podmonitoring: true diff --git a/installation/config/kind/grafana-values.yaml b/installation/config/kind/grafana-values.yaml new file mode 100644 index 0000000..f52a84e --- /dev/null +++ b/installation/config/kind/grafana-values.yaml @@ -0,0 +1,10 @@ +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + uid: "PFB5ABA51A8A585D7" + url: http://prometheus-server + isDefault: true + editable: true diff --git a/installation/config/kind/opencost-templates-values.yaml b/installation/config/kind/opencost-templates-values.yaml new file mode 100644 index 0000000..c9efd85 --- /dev/null +++ b/installation/config/kind/opencost-templates-values.yaml @@ -0,0 +1,25 @@ +dashboards: true +opencost: + configFileName: default + provider: kind + customPricing: + # -- Enables custom pricing for on-premise setup. + enabled: true + configmapName: custom-pricing-model + # -- Path for the pricing configuration. + configPath: /tmp/custom-config + # -- Configures the pricing model provided in the values file. + createConfigmap: true + # -- More information about these values here: https://www.opencost.io/docs/configuration/on-prem#custom-pricing-using-the-opencost-helm-chart + costModel: + description: Modified prices based on your internal pricing + CPU: 1.25 + spotCPU: 0.006655 + RAM: 0.50 + spotRAM: 0.000892 + GPU: 0.95 + storage: 0.25 + zoneNetworkEgress: 0.01 + regionNetworkEgress: 0.01 + internetNetworkEgress: 0.12 + loadBalancer: 0 diff --git a/installation/config/kind/prometheus-opencost-exporter-values.yaml b/installation/config/kind/prometheus-opencost-exporter-values.yaml new file mode 100644 index 0000000..7aa9cca --- /dev/null +++ b/installation/config/kind/prometheus-opencost-exporter-values.yaml @@ -0,0 +1,27 @@ +opencost: + prometheus: + external: + enabled: true + url: http://prometheus-server + internal: + enabled: false + customPricing: + # -- Enables custom pricing for on-premise setup. + enabled: true + configmapName: custom-pricing-model + # -- Path for the pricing configuration. + configPath: /tmp/custom-config + # -- Configures the pricing model provided in the values file. + createConfigmap: true + # -- More information about these values here: https://www.opencost.io/docs/configuration/on-prem#custom-pricing-using-the-opencost-helm-chart + costModel: + description: Modified prices based on your internal pricing + CPU: 1.25 + spotCPU: 0.006655 + RAM: 0.50 + spotRAM: 0.000892 + GPU: 0.95 + storage: 0.25 + zoneNetworkEgress: 0.01 + regionNetworkEgress: 0.01 + internetNetworkEgress: 0.12 diff --git a/installation/config/kind/prometheus-values.yaml b/installation/config/kind/prometheus-values.yaml new file mode 100644 index 0000000..87c2dce --- /dev/null +++ b/installation/config/kind/prometheus-values.yaml @@ -0,0 +1,17 @@ +extraScrapeConfigs: | + - job_name: opencost + honor_labels: true + scrape_interval: 1m + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + dns_sd_configs: + - names: + - opencost.opencost + type: 'A' + port: 9003 + +alertmanager: + enabled: false +prometheus-pushgateway: + enabled: false \ No newline at end of file diff --git a/installation/eks.md b/installation/eks.md new file mode 100644 index 0000000..879a30c --- /dev/null +++ b/installation/eks.md @@ -0,0 +1,81 @@ +# FinOps Stack on a EKS cluster + +This documentation focuses on installing the FinOps Stack in EKS clusters. + +## Using Helmfile + +Installing Helm charts with lots of dependencies and CRDs is challenging; these instructions use Helmfile to mitigate issues with Helm. + +## Pre-requisites + +- A EKS cluster with: + - kubectl access + - (Optional) If your cluster has Spot Instances, EKS Pod Identities need to be configured. See [documentation](https://www.opencost.io/docs/configuration/aws#eks-pod-identities). +- [Helmfile](https://helmfile.readthedocs.io/en/latest/#installation) installed on your local machine +- Unless you want to access the Grafana dashboard via `kubectl port-forward` you'll need a domain name or external public IP. + +## Installation + +### Configuration changes for your cluster environment + +1. To control which Finops Stack components to install, edit the [enabled.yaml](./installation/config/common/enabled.yaml) file +1. Copy `./env_eks.tmpl` to `./.env` and replace the env var values accordingly. + +### Install everything using Helmfile + +For the first run: + +```bash +set -a; source .env; set +a; helmfile apply --file Helmfile_eks.yaml --interactive +``` + +NOTE: it will take several minutes for all workloads to install and start running. Helmfile does display its progress in the terminal. All workloads get installed into the `finops-stack` namespace so you can also view progress using `kubectl`. + +To speed up subsequent runs: + +```bash +set -a; source .env; set +a; helmfile apply --file Helmfile_eks.yaml --interactive --skip-deps +``` + +## Optional: Making Grafana accessible via DNS + +### Pre-requisites + +Already have an FQDN setup and registered with a public IP, e.g. grafana.example.com + +### Grafana Helm values + +These are specified in `config/common/grafana-values.yaml`, `config/gke/grafana-values.yaml` and under the Grafana release in `helmfile.yaml`. Probably all the changes you will want to make can be done by changing the values in `helmfile.yaml`, e.g. the admin user and what type of ingress you require. + +General guidance when configuring ingress: +- Update the `.env` file with the FQDN and public IP for you domain. +- If you wish to enable tls, then ensure that cert-manager.enabled is set to true and update the values in `.env` accordingly. + +## Enable Goldilocks for namespaces + +For Goldilocks to analyse namespaces and add then to its dashboard you need to add this label to the namespace resource: `goldilocks.fairwinds.com/enabled=true`, e.g. +`kubectl label ns finops-stack goldilocks.fairwinds.com/enabled=true` + +## Useful commands + +To port forward to Grafana: + +```bash +kubectl --namespace finops-stack port-forward service/grafana 3000:80 +``` + +Access via http://localhost:3000 + +To port forward to the metrics endpoint of the Opencost Prometheus exporter (to examine what metrics are being scraped): + +```bash +kubectl --namespace finops-stack port-forward service/prometheus-opencost-exporter 9003:9003 +``` + +To access the Goldilocks dashboard (assuming you've enabled it): + +```bash +kubectl -n finops-stack port-forward svc/goldilocks-dashboard 8080:80 +``` + +Then goto http://localhost:8080 \ No newline at end of file diff --git a/installation/env.tmpl b/installation/env.tmpl index 90719c7..93ec26a 100644 --- a/installation/env.tmpl +++ b/installation/env.tmpl @@ -1,24 +1,12 @@ # Sub-dir under ./config that holds your hosting env specific Helm values.files, e.g. gke -HOST_ENV=gke - -# GCP Project - used by gmp-proxy and cert-manager -GCP_PROJECT="my-gcp-project" - -# API Key used by Opencost to call your CSPs billing APIs: https://www.opencost.io/docs/configuration/gcp#configuring-gcp-pricing -CSP_API_KEY=paste_key_here +HOST_ENV=kind # Grafana admin user credentials GRAFANA_ADMIN_USER=finops GRAFANA_ADMIN_PW=s7@ck -# GCP SA for workload identity -GRAFANA_SA_ANNOTATION="iam.gke.io/gcp-service-account: sa-name@my-gcp-project.iam.gserviceaccount.com" - # Grafana ingress settings GRAFANA_INGRESS="false" # GRAFANA_PUBLIC_IP_NAME="name-of-public-ip" # GRAFANA_FQDN="grafana.host.name" -## GCP SA for workload identity for cert-manager (need to be defined but only used if cert-manager is being installed) -CERT_MANAGER_SA_ANNOTATION="iam.gke.io/gcp-service-account: cert-manager-sa@my-gcp-project.iam.gserviceaccount.com" -CERT_MANAGER_EMAIL="issuer@example.com" diff --git a/installation/env_eks.tmpl b/installation/env_eks.tmpl new file mode 100644 index 0000000..5a8b849 --- /dev/null +++ b/installation/env_eks.tmpl @@ -0,0 +1,19 @@ +# Sub-dir under ./config that holds your hosting env specific Helm values.files, e.g. gke +HOST_ENV=eks + +# -- AWS secret access key and access key id +AWS_SECRET_ACCESS_KEY="" +AWS_ACCESS_KEY_ID="" + +# Grafana admin user credentials +GRAFANA_ADMIN_USER=finops +GRAFANA_ADMIN_PW=s7@ck + +# Grafana ingress settings +GRAFANA_INGRESS="false" +# GRAFANA_PUBLIC_IP_NAME="name-of-public-ip" +# GRAFANA_FQDN="grafana.host.name" + +## GCP SA for workload identity for cert-manager (need to be defined but only used if cert-manager is being installed) +CERT_MANAGER_SA_ANNOTATION="" +CERT_MANAGER_EMAIL="issuer@example.com" diff --git a/installation/env_gke.tmpl b/installation/env_gke.tmpl new file mode 100644 index 0000000..90719c7 --- /dev/null +++ b/installation/env_gke.tmpl @@ -0,0 +1,24 @@ +# Sub-dir under ./config that holds your hosting env specific Helm values.files, e.g. gke +HOST_ENV=gke + +# GCP Project - used by gmp-proxy and cert-manager +GCP_PROJECT="my-gcp-project" + +# API Key used by Opencost to call your CSPs billing APIs: https://www.opencost.io/docs/configuration/gcp#configuring-gcp-pricing +CSP_API_KEY=paste_key_here + +# Grafana admin user credentials +GRAFANA_ADMIN_USER=finops +GRAFANA_ADMIN_PW=s7@ck + +# GCP SA for workload identity +GRAFANA_SA_ANNOTATION="iam.gke.io/gcp-service-account: sa-name@my-gcp-project.iam.gserviceaccount.com" + +# Grafana ingress settings +GRAFANA_INGRESS="false" +# GRAFANA_PUBLIC_IP_NAME="name-of-public-ip" +# GRAFANA_FQDN="grafana.host.name" + +## GCP SA for workload identity for cert-manager (need to be defined but only used if cert-manager is being installed) +CERT_MANAGER_SA_ANNOTATION="iam.gke.io/gcp-service-account: cert-manager-sa@my-gcp-project.iam.gserviceaccount.com" +CERT_MANAGER_EMAIL="issuer@example.com" diff --git a/installation/gke.md b/installation/gke.md new file mode 100644 index 0000000..ff3f12d --- /dev/null +++ b/installation/gke.md @@ -0,0 +1,92 @@ +# FinOps Stack on a GKE cluster + +This documentation focuses on installing the FinOps Stack in GKE standard/autopilot clusters. + +## Using Helmfile + +Installing Helm charts with lots of dependencies and CRDs is challenging; these instructions use Helmfile to mitigate issues with Helm. + +## Pre-requisites + +- A GKE standard or autopilot cluster with: + - kubectl access + - cluster-admin permissions + - workload identity enabled + - [Google Managed Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus) enabled. +- [Helmfile](https://helmfile.readthedocs.io/en/latest/#installation) installed on your local machine +- A Google Service Account with the following: + - roles/monitor.viewer and roles/iam.serviceAccountTokenCreator permissions + - workload identity configured for grafana kubernetes service account: `[finops-stack/grafana]`. Have a look at this [blog post](https://venafi.com/blog/gke-workload-identity-federation-for-kubernetes-principals/) to configure workload identity. +- Unless you want to access the Grafana dashboard via `kubectl port-forward` you'll need a domain name + +## Distribution support + +### GKE Autopilot + +- Enable cost allocation + +## Installation + +### Configuration changes for your cluster environment + +1. To control which Finops Stack components to install, edit the [enabled.yaml](./installation/config/common/enabled.yaml) file +1. Copy `./env_gke.tmpl` to `./.env` and replace the env var values accordingly. As a minimum, you will need to change the `GCP_PROJECT`, `CSP_API_KEY`, `GRAFANA_SA_ANNOTATION` values. + +### Install everything using Helmfile + +For the first run: + +```bash +set -a; source .env; set +a; helmfile apply --file Helmfile_gke.yaml --interactive +``` + +NOTE: it will take several minutes for all workloads to install and start running. Helmfile does display its progress in the terminal. All workloads get installed into the `finops-stack` namespace so you can also view progress using `kubectl`. + +To speed up subsequent runs: + +```bash +set -a; source .env; set +a; helmfile apply --interactive --skip-deps +``` + +## Optional: Configure ingress for Grafana + +### Pre-requisites + +Already have an FQDN setup and registered with a public IP, e.g. grafana.example.com + +### Grafana Helm values + +These are specified in `config/common/grafana-values.yaml`, `config/gke/grafana-values.yaml` and under the Grafana release in `helmfile.yaml`. Probably all the changes you will want to make can be done by changing the values in `helmfile.yaml`, e.g. the admin user and what type of ingress you require. + +General guidance when configuring ingress: +- Update the `.env` file with the FQDN and public IP for you domain. +- If you wish to enable tls, then ensure that cert-manager.enabled is set to true and update the values in `.env` accordingly. + +## Enable Goldilocks for namespaces + +For Goldilocks to analyse namespaces and add then to its dashboard you need to add this label to the namespace resource: `goldilocks.fairwinds.com/enabled=true`, e.g. +`kubectl label ns finops-stack goldilocks.fairwinds.com/enabled=true` + +## Useful commands + +To port forward to Grafana: + +```bash +kubectl --namespace finops-stack port-forward service/grafana 3000:80 +``` + +Access via http://localhost:3000 + +To port forward to the metrics endpoint of the Opencost Prometheus exporter (to examine what metrics are being scraped): + +```bash +kubectl --namespace finops-stack port-forward service/prometheus-opencost-exporter 9003:9003 +``` + +To access the Goldilocks dashboard (assuming you've enabled it): + +```bash +kubectl -n finops-stack port-forward svc/goldilocks-dashboard 8080:80 +``` + +Then goto http://localhost:8080 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 47696c4..0c11364 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ mkdocs==1.6.1 mkdocs-get-deps==0.2.0 -mkdocs-git-revision-date-localized-plugin==1.2.9 -mkdocs-include-markdown-plugin==6.2.2 -mkdocs-material==9.5.36 -mkdocs-material[imaging]==9.5.36 +mkdocs-git-revision-date-localized-plugin==1.4.5 +mkdocs-include-markdown-plugin==7.1.5 +mkdocs-material==9.6.12 +mkdocs-material[imaging]==9.6.12 mkdocs-material-extensions==1.3.1 mkdocs-minify-plugin==0.8.0 -pymdown-extensions==10.10.1 +pymdown-extensions==10.15 readtime==3.0.0 paginate==0.5.7 mkdocs-git-revision-date-localized-plugin -diagrams==0.23.4 +diagrams==0.24.4 diff --git a/scripts/chart-dep-updater/go.mod b/scripts/chart-dep-updater/go.mod index 9e35ca8..4c584af 100644 --- a/scripts/chart-dep-updater/go.mod +++ b/scripts/chart-dep-updater/go.mod @@ -89,13 +89,13 @@ require ( go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.starlark.net v0.0.0-20240725214946-42030a7cedce // indirect - golang.org/x/crypto v0.26.0 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a // indirect google.golang.org/grpc v1.65.0 // indirect diff --git a/scripts/chart-dep-updater/go.sum b/scripts/chart-dep-updater/go.sum index 9bc6605..68dc946 100644 --- a/scripts/chart-dep-updater/go.sum +++ b/scripts/chart-dep-updater/go.sum @@ -333,8 +333,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -349,8 +349,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -359,8 +359,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -374,19 +374,19 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=