From 33a503743383430aa8ca3a9c20b1d71a3fec3f19 Mon Sep 17 00:00:00 2001 From: Ilyes Ben Dlala Date: Fri, 28 Jul 2023 13:52:44 +0200 Subject: [PATCH 1/4] #109 Explicitly install the tools kubectl/helm/kind in the CI tools are first archived as artifacts and downloaded when needed https://github.com/secureCodeBox/internal/issues/109 Signed-off-by: Ilyes Ben Dlala --- .github/workflows/ci.yaml | 173 +++++++++++++++++++++++++++++++++++++- 1 file changed, 172 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e1b02b9063..1928f83d27 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,6 +22,9 @@ env: PYTHON_VERSION: "3.9.16" NVM_VERSION: "0.39.3" # Does not need to be updated as often as the Node.js version or the NPM version KIND_NODE_IMAGE: "kindest/node:v1.27.3@sha256:9dd3392d79af1b084671b05bcf65b21de476256ad1dcc853d9f3b10b4ac52dde" + KIND_BINARY_VERSION: "v0.20.0" + KUBECTL_VERSION: "v1.27.4" + HELM_VERSION: "v3.12.2" jobs: # ---- NVM Setup ---- @@ -52,6 +55,46 @@ jobs: path: ./nvm-node.tar.gz retention-days: 1 + k8s-setup: + name: "Setup Kind" + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Install Kind + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/${{ env.KIND_BINARY_VERSION }}/kind-linux-amd64 + chmod +x ./kind + + - name: Install Kubectl + run: | + curl -Lo ./kubectl curl -LO https://dl.k8s.io/release/${{ env.KUBECTL_VERSION }}/bin/linux/amd64/kubectl + chmod +x ./kubectl + + - name: Install Helm + run: | + curl -Lo ./helm.tar.gz https://get.helm.sh/helm-${{ env.HELM_VERSION }}-linux-amd64.tar.gz + tar -xzf ./helm.tar.gz + chmod +x ./linux-amd64/helm + + - name: Archive Kind + uses: actions/upload-artifact@v2 + with: + name: kind + path: ./kind + + - name: Archive Kubectl + uses: actions/upload-artifact@v2 + with: + name: kubectl + path: ./kubectl + + - name: Archive Helm + uses: actions/upload-artifact@v2 + with: + name: helm + path: ./linux-amd64/helm + # ---- Unit-Test ---- # ---- Unit-Test | Java ---- @@ -156,7 +199,9 @@ jobs: auto-discovery-kubernetes-secret-extraction-container: name: "Autodiscovery | Kubernetes | SecretExtractionInitContainer" runs-on: ubuntu-22.04 - needs: nvm-setup + needs: + - nvm-setup + - k8s-setup steps: - name: Checkout uses: actions/checkout@v3 @@ -179,6 +224,36 @@ jobs: [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" nvm use + - name: Download Kind + uses: actions/download-artifact@v2 + with: + name: kind + path: ./kind + + - name: Download Kubectl + uses: actions/download-artifact@v2 + with: + name: kubectl + path: ./kubectl + + - name: Download Helm + uses: actions/download-artifact@v2 + with: + name: helm + path: ./helm + + - name: Make binaries globally available + run: | + chmod +x ./kind/kind && sudo mv ./kind/kind /usr/local/bin/kind + chmod +x ./kubectl/kubectl && sudo mv ./kubectl/kubectl /usr/local/bin/kubectl + chmod +x ./helm/helm && sudo mv ./helm/helm /usr/local/bin/helm + + - name: Verify tools + run: | + kind version + kubectl version + helm version + - name: Unit Tests working-directory: ./auto-discovery/kubernetes/pull-secret-extractor run: make unit-test @@ -250,6 +325,7 @@ jobs: - sdk - operator - nvm-setup + - k8s-setup runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -294,6 +370,36 @@ jobs: [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" nvm use + - name: Download Kind + uses: actions/download-artifact@v2 + with: + name: kind + path: ./kind + + - name: Download Kubectl + uses: actions/download-artifact@v2 + with: + name: kubectl + path: ./kubectl + + - name: Download Helm + uses: actions/download-artifact@v2 + with: + name: helm + path: ./helm + + - name: Make binaries globally available + run: | + chmod +x ./kind/kind && sudo mv ./kind/kind /usr/local/bin/kind + chmod +x ./kubectl/kubectl && sudo mv ./kubectl/kubectl /usr/local/bin/kubectl + chmod +x ./helm/helm && sudo mv ./helm/helm /usr/local/bin/helm + + - name: Verify tools + run: | + kind version + kubectl version + helm version + - name: Install Dependencies working-directory: ./scanners/${{ matrix.unit }}/ run: make install-deps @@ -408,6 +514,7 @@ jobs: - sdk - operator - nvm-setup + - kind-setup runs-on: ubuntu-22.04 steps: - name: Checkout @@ -426,6 +533,36 @@ jobs: [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" nvm use + - name: Download Kind + uses: actions/download-artifact@v2 + with: + name: kind + path: ./kind + + - name: Download Kubectl + uses: actions/download-artifact@v2 + with: + name: kubectl + path: ./kubectl + + - name: Download Helm + uses: actions/download-artifact@v2 + with: + name: helm + path: ./helm + + - name: Make binaries globally available + run: | + chmod +x ./kind/kind && sudo mv ./kind/kind /usr/local/bin/kind + chmod +x ./kubectl/kubectl && sudo mv ./kubectl/kubectl /usr/local/bin/kubectl + chmod +x ./helm/helm && sudo mv ./helm/helm /usr/local/bin/helm + + - name: Verify tools + run: | + kind version + kubectl version + helm version + - name: Install Dependencies working-directory: ./scanners/zap-advanced/ run: make install-deps @@ -556,6 +693,7 @@ jobs: needs: - operator - nvm-setup + - kind-setup runs-on: ubuntu-22.04 strategy: matrix: @@ -641,6 +779,39 @@ jobs: docker load --input /tmp/hook-sdk.tar docker images | grep sdk + + # ---- K8s Cluster Setup ---- # + + - name: Download Kind + uses: actions/download-artifact@v2 + with: + name: kind + path: ./kind + + - name: Download Kubectl + uses: actions/download-artifact@v2 + with: + name: kubectl + path: ./kubectl + + - name: Download Helm + uses: actions/download-artifact@v2 + with: + name: helm + path: ./helm + + - name: Make binaries globally available + run: | + chmod +x ./kind/kind && sudo mv ./kind/kind /usr/local/bin/kind + chmod +x ./kubectl/kubectl && sudo mv ./kubectl/kubectl /usr/local/bin/kubectl + chmod +x ./helm/helm && sudo mv ./helm/helm /usr/local/bin/helm + + - name: Verify tools + run: | + kind version + kubectl version + helm version + # ----- Build Test-Scan ScanType Image ---- - name: Build Images From e45683a5e1d477d7fe5f0f7026363b085b8f1991 Mon Sep 17 00:00:00 2001 From: Ilyes Ben Dlala Date: Fri, 28 Jul 2023 13:55:03 +0200 Subject: [PATCH 2/4] Fixed in the naming of the job kind-setup -> k8s-setup Signed-off-by: Ilyes Ben Dlala --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1928f83d27..e051d19485 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -56,7 +56,7 @@ jobs: retention-days: 1 k8s-setup: - name: "Setup Kind" + name: "Setup Kind & Kubectl & Helm" runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -514,7 +514,7 @@ jobs: - sdk - operator - nvm-setup - - kind-setup + - k8s-setup runs-on: ubuntu-22.04 steps: - name: Checkout @@ -693,7 +693,7 @@ jobs: needs: - operator - nvm-setup - - kind-setup + - k8s-setup runs-on: ubuntu-22.04 strategy: matrix: From 921f0ff677c616034bb8b2ab749be9dd48ee8fe2 Mon Sep 17 00:00:00 2001 From: Ilyes Ben Dlala Date: Fri, 28 Jul 2023 14:00:20 +0200 Subject: [PATCH 3/4] #109 Ignores kubectl version exit code It it expected since the cluster was not started yet Signed-off-by: Ilyes Ben Dlala --- .github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e051d19485..86d25be64d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -251,7 +251,7 @@ jobs: - name: Verify tools run: | kind version - kubectl version + kubectl version || true helm version - name: Unit Tests @@ -397,7 +397,7 @@ jobs: - name: Verify tools run: | kind version - kubectl version + kubectl version || true helm version - name: Install Dependencies @@ -560,7 +560,7 @@ jobs: - name: Verify tools run: | kind version - kubectl version + kubectl version || true helm version - name: Install Dependencies @@ -809,7 +809,7 @@ jobs: - name: Verify tools run: | kind version - kubectl version + kubectl version || true helm version # ----- Build Test-Scan ScanType Image ---- From e94e2cf8e83a21a8f5cdff27bb8aa299da5ebce6 Mon Sep 17 00:00:00 2001 From: Ilyes Ben Dlala Date: Fri, 28 Jul 2023 14:42:22 +0200 Subject: [PATCH 4/4] internal#109 Updated actions from v2 to v3 to be consistent with the other steps Signed-off-by: Ilyes Ben Dlala --- .github/workflows/ci.yaml | 58 +++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 86d25be64d..86dec57025 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -78,19 +78,19 @@ jobs: chmod +x ./linux-amd64/helm - name: Archive Kind - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: kind path: ./kind - name: Archive Kubectl - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: kubectl path: ./kubectl - name: Archive Helm - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: helm path: ./linux-amd64/helm @@ -212,7 +212,7 @@ jobs: python-version: "${{ env.PYTHON_VERSION }}" - name: Download nvm and Node.js - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: nvm-node path: nvm-node @@ -225,19 +225,19 @@ jobs: nvm use - name: Download Kind - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: kind path: ./kind - name: Download Kubectl - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: kubectl path: ./kubectl - name: Download Helm - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: helm path: ./helm @@ -358,7 +358,7 @@ jobs: uses: actions/checkout@v3 - name: Download nvm and Node.js - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: nvm-node path: nvm-node @@ -371,19 +371,19 @@ jobs: nvm use - name: Download Kind - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: kind path: ./kind - name: Download Kubectl - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: kubectl path: ./kubectl - name: Download Helm - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: helm path: ./helm @@ -409,7 +409,7 @@ jobs: run: make unit-tests - name: Download Parser SDK Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: parser-sdk-image path: /tmp @@ -438,13 +438,13 @@ jobs: kubectl get node - name: Download Operator Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: operator-image path: ./operator - name: Download Lurker Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: lurker-image path: ./operator @@ -521,7 +521,7 @@ jobs: uses: actions/checkout@v3 - name: Download nvm and Node.js - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: nvm-node path: ./nvm-node @@ -534,19 +534,19 @@ jobs: nvm use - name: Download Kind - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: kind path: ./kind - name: Download Kubectl - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: kubectl path: ./kubectl - name: Download Helm - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: helm path: ./helm @@ -572,7 +572,7 @@ jobs: run: make unit-tests - name: Download Parser SDK Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: parser-sdk-image path: /tmp @@ -601,13 +601,13 @@ jobs: kubectl get node - name: Download Operator Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: operator-image path: ./operator - name: Download Lurker Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: lurker-image path: ./operator @@ -720,13 +720,13 @@ jobs: # ---- Install Operator & Create Namespaces ---- - name: Download Operator Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: operator-image path: ./operator - name: Download Lurker Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: lurker-image path: ./operator @@ -756,7 +756,7 @@ jobs: # ---- Import Parser SDK Artifact - name: Download Parser SDK Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: parser-sdk-image path: /tmp @@ -769,7 +769,7 @@ jobs: # ---- Import Hook SDK Artifact - name: Download Hook SDK Image - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: hook-sdk-image path: /tmp @@ -783,19 +783,19 @@ jobs: # ---- K8s Cluster Setup ---- # - name: Download Kind - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: kind path: ./kind - name: Download Kubectl - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: kubectl path: ./kubectl - name: Download Helm - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: helm path: ./helm @@ -845,7 +845,7 @@ jobs: run: make kind-import - name: Download nvm and Node.js - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: nvm-node path: ./nvm-node