When running from within a k8s pod, use the k8s API to get the cgroup name (netdata#5576)

cakrit · web-flow · commit b7998ec82d57 · 2019-03-13T12:47:20.000+01:00
* When running from within a k8s pod, use the k8s API to get the pod name

* Check the last part of an underscore-delimited or slash-delimited container id

* Add a docker image builder that adds a single image to a user-specified registry, for use in k8s

* When running in k8s, disable cgroups that the pod API does not return

* Use longer name for k8s containers

* Add reference to build-test.sh to packaging/docker/README.md

* Anonymous statistics should not break when /proc/1/sched is not available
diff --git a/collectors/cgroups.plugin/cgroup-name.sh.in b/collectors/cgroups.plugin/cgroup-name.sh.in
@@ -64,6 +64,28 @@ function docker_get_name_api() {
 	return 0
 }
 
+function k8s_get_name() {
+	# Take the last part of the delimited path identifier (expecting either _ or / as a delimiter).
+	local id="${1##*_}"
+	if [ "${id}" == "${1}" ]; then
+		id="${1##*/}"
+	fi
+	KUBE_TOKEN="$(</var/run/secrets/kubernetes.io/serviceaccount/token)"
+	NAME="$(
+curl -sSk -H "Authorization: Bearer $KUBE_TOKEN"  "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/pods" |
+jq -r '.items[] | "k8s_\(.metadata.namespace)_\(.metadata.name)_\(.metadata.uid)_\(.status.containerStatuses[0].name) \(.status.containerStatuses[0].containerID)"' |
+grep "$id" |
+cut -d' ' -f1
+)"
+	if [ -z "${NAME}" ]; then
+		warning "cannot find the name of k8s pod with containerID '${id}'. Setting name to ${id} and disabling it"
+		NAME="${id}"
+		NAME_NOT_FOUND=3
+	else
+		info "k8s containerID '${id}' has chart name (namespace_podname_poduid_containername) '${NAME}'"
+	fi
+}
+
 function docker_get_name() {
 	local id="${1}"
 	if hash docker 2>/dev/null; then
@@ -73,7 +95,7 @@ function docker_get_name() {
 	fi
 	if [ -z "${NAME}" ]; then
 		warning "cannot find the name of docker container '${id}'"
-		NAME_NOT_FOUND=1
+		NAME_NOT_FOUND=2
 		NAME="${id:0:12}"
 	else
 		info "docker container '${id}' is named '${NAME}'"
@@ -89,6 +111,7 @@ function docker_validate_id() {
 	fi
 }
 
+
 # -----------------------------------------------------------------------------
 
 [ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"
@@ -118,25 +141,22 @@ for CONFIG in "${NETDATA_USER_CONFIG_DIR}/cgroups-names.conf" "${NETDATA_STOCK_C
 	fi
 done
 
+if [ -z "${NAME}" ] && [ -n "${KUBERNETES_SERVICE_HOST}" ] && [ -n "${KUBERNETES_PORT_443_TCP_PORT}" ] && [[ ${CGROUP} =~ ^.*kubepods.* ]]; then
+	k8s_get_name "${CGROUP}"
+fi
+
 if [ -z "${NAME}" ]; then
 	if [[ ${CGROUP} =~ ^.*docker[-_/\.][a-fA-F0-9]+[-_\.]?.*$ ]]; then
 		# docker containers
 		#shellcheck disable=SC1117
 		DOCKERID="$(echo "${CGROUP}" | sed "s|^.*docker[-_/]\([a-fA-F0-9]\+\)[-_\.]\?.*$|\1|")"
 		docker_validate_id "${DOCKERID}"
-
 	elif [[ ${CGROUP} =~ ^.*ecs[-_/\.][a-fA-F0-9]+[-_\.]?.*$ ]]; then
 		# ECS
 		#shellcheck disable=SC1117
 		DOCKERID="$(echo "${CGROUP}" | sed "s|^.*ecs[-_/].*[-_/]\([a-fA-F0-9]\+\)[-_\.]\?.*$|\1|")"
 		docker_validate_id "${DOCKERID}"
 
-	elif [[ ${CGROUP} =~ ^.*kubepods[_/].*[_/]pod[a-fA-F0-9-]+[_/][a-fA-F0-9]+$ ]]; then
-		# kubernetes
-		#shellcheck disable=SC1117
-		DOCKERID="$(echo "${CGROUP}" | sed "s|^.*kubepods[_/].*[_/]pod[a-fA-F0-9-]\+[_/]\([a-fA-F0-9]\+\)$|\1|")"
-		docker_validate_id "${DOCKERID}"
-
 	elif [[ ${CGROUP} =~ machine.slice[_/].*\.service ]]; then
 		# systemd-nspawn
 		NAME="$(echo "${CGROUP}" | sed 's/.*machine.slice[_\/]\(.*\)\.service/\1/g')"
@@ -177,8 +197,5 @@ fi
 info "cgroup '${CGROUP}' is called '${NAME}'"
 echo "${NAME}"
 
-if [ "${NAME_NOT_FOUND}" -eq 1 ]; then
-	exit 2
-else
-	exit 0
-fi
+exit ${NAME_NOT_FOUND}
+
diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c
@@ -893,7 +893,7 @@ static inline void cgroup_get_chart_name(struct cgroup *cg) {
 
     snprintfz(command, CGROUP_CHARTID_LINE_MAX, "exec %s '%s'", cgroups_rename_script, cg->chart_id);
 
-    debug(D_CGROUP, "executing command \"%s\" for cgroup '%s'", command, cg->id);
+    debug(D_CGROUP, "executing command \"%s\" for cgroup '%s'", command, cg->chart_id);
     FILE *fp = mypopen(command, &cgroup_pid);
     if(fp) {
         // debug(D_CGROUP, "reading from command '%s' for cgroup '%s'", command, cg->id);
@@ -904,12 +904,16 @@ static inline void cgroup_get_chart_name(struct cgroup *cg) {
         // debug(D_CGROUP, "closed command for cgroup '%s'", cg->id);
 
         if(s && *s && *s != '\n') {
-            debug(D_CGROUP, "cgroup '%s' should be renamed to '%s'", cg->id, s);
+            debug(D_CGROUP, "cgroup '%s' should be renamed to '%s'", cg->chart_id, s);
 
             s = trim(s);
             if (s) {
-                if(likely(!name_error))
+                if(likely(name_error==0))
                     cg->pending_renames = 0;
+                else if (unlikely(name_error==3)) {
+                    debug(D_CGROUP, "cgroup '%s' disabled based due to rename command output", cg->chart_id);
+                    cg->enabled = 0;
+                }
 
                 if(likely(cg->pending_renames < 2)) {
                     freez(cg->chart_title);
diff --git a/daemon/anonymous-statistics.sh.in b/daemon/anonymous-statistics.sh.in
@@ -85,13 +85,14 @@ fi
 # -------------------------------------------------------------------------------------------------
 # detect containers with heuristics
 
-if [ "${CONTAINER}" = "unknown" ] ; then
-	IFS='(, ' read -r process _ </proc/1/sched
-	if [ "${process}" = "netdata" ]; then
-		CONTAINER="container"
-		CONT_DETECTION="process"
+if [ "${CONTAINER}" = "unknown" ]; then
+	if [ -f /proc/1/sched ] ; then
+		IFS='(, ' read -r process _ </proc/1/sched
+		if [ "${process}" = "netdata" ]; then
+			CONTAINER="container"
+			CONT_DETECTION="process"
+		fi
 	fi
-
 	# ubuntu and debian supply /bin/running-in-container
 	# https://www.apt-browse.org/browse/ubuntu/trusty/main/i386/upstart/1.12.1-0ubuntu4/file/bin/running-in-container
 	if /bin/running-in-container >/dev/null 2>&1; then
diff --git a/packaging/docker/README.md b/packaging/docker/README.md
@@ -124,3 +124,35 @@ services:
 You can restrict access by following [official caddy guide](https://caddyserver.com/docs/basicauth) and adding lines to Caddyfile.
 
 [![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fpackaging%2Fdocker%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)]()
+
+### Publish a test image to your own repository
+
+The script `packaging/docker/build-test.sh` can be used to create an image and upload it to a repository of your choosing. 
+
+```
+Usage: packaging/docker/build-test.sh -r <REPOSITORY> -v <VERSION> -u <DOCKER_USERNAME> -p <DOCKER_PASSWORD> [-s]
+	-s skip build, just push the image
+Builds an amd64 image and pushes it to the docker hub repository REPOSITORY
+```
+
+This is especially useful when testing a Pull Request for Kubernetes, since you can set `image` to an immutable repository and tag, set the `imagePullPolicy` to `Always` and just keep uploading new images.
+
+Example:
+
+We get a local copy of the Helm chart at https://github.com/netdata/helmchart. We modify `values.yaml` to have the following:
+
+```
+image:
+  repository: cakrit/netdata-prs
+  tag: PR5576
+  pullPolicy: Always
+```
+
+We check out PR5576 and run the following:
+```
+./packaging/docker/build-test.sh -r cakrit/netdata-prs -v PR5576 -u cakrit -p 'XXX'
+```
+
+Then we can run `helm install [path to our helmchart clone]`.
+
+If we make changes to the code, we execute the same `build-test.sh` command, followed by `helm upgrade [name] [path to our helmchart clone]`
diff --git a/packaging/docker/build-test.sh b/packaging/docker/build-test.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0-or-later
+# Author  : Chris Akritidis (cakrit)
+# Cross-arch docker build helper script
+
+printhelp() {
+	echo "Usage: packaging/docker/build-test.sh -r <REPOSITORY> -v <VERSION> -u <DOCKER_USERNAME> -p <DOCKER_PASSWORD> [-s]
+	-s skip build, just push the image
+Builds an amd64 image and pushes it to the docker hub repository REPOSITORY"
+}
+
+set -e
+
+if [ ! -f .gitignore ]; then
+	echo "Run as ./packaging/docker/$(basename "$0") from top level directory of git repository"
+	exit 1
+fi
+
+DOBUILD=1
+while getopts :r:v:u:p:s option
+do
+	case "$option" in
+	r)
+		REPOSITORY=$OPTARG
+	 	;;
+	v)
+		VERSION=$OPTARG
+		;;
+	u)
+		DOCKER_USERNAME=$OPTARG
+		;;
+	p) 
+		DOCKER_PASSWORD=$OPTARG
+		;;
+	s)
+		DOBUILD=0
+		;;
+	*)
+		printhelp
+		exit 1
+		;;
+	esac
+done
+
+if [ -n "${REPOSITORY}" ] && [ -n "${VERSION}" ] && [ -n "${DOCKER_USERNAME}" ] && [ -n "${DOCKER_PASSWORD}" ] ; then
+	if [ $DOBUILD -eq 1 ] ; then
+		echo "Building ${VERSION} of ${REPOSITORY} container"
+		docker run --rm --privileged multiarch/qemu-user-static:register --reset
+
+		# Build images using multi-arch Dockerfile.
+		eval docker build --build-arg ARCH="amd64" --tag "${REPOSITORY}:${VERSION}" --file packaging/docker/Dockerfile ./
+
+		# Create temporary docker CLI config with experimental features enabled (manifests v2 need it)
+		mkdir -p /tmp/docker
+		#echo '{"experimental":"enabled"}' > /tmp/docker/config.json
+	fi
+
+	# Login to docker hub to allow futher operations
+	echo "Logging into docker"
+	echo "$DOCKER_PASSWORD" | docker --config /tmp/docker login -u "$DOCKER_USERNAME" --password-stdin
+
+	echo "Pushing ${REPOSITORY}:${VERSION}"
+	docker --config /tmp/docker push "${REPOSITORY}:${VERSION}"
+else
+	echo "Missing parameter. REPOSITORY=${REPOSITORY} VERSION=${VERSION} DOCKER_USERNAME=${DOCKER_USERNAME} DOCKER_PASSWORD=${DOCKER_PASSWORD}"
+	printhelp
+	exit 1
+fi
