test(node-indexer): exercise ClairCore mapping download retries - #23436

export KUBECONFIG=/path/to/test-cluster.kubeconfig

MAIN_IMAGE_TAG="$(make tag)" \
ROX_SCANNER_V4=true \
ROX_NODE_INDEX_ENABLED=true \
LOAD_BALANCER=route \
CLUSTER=node-retry-smoke \
./deploy/openshift/deploy.sh

kubectl -n stackrox rollout status deployment/sensor --timeout=5m
kubectl -n stackrox rollout status daemonset/collector --timeout=5m

SMOKE_DIR="$(mktemp -d)"

curl -fsSL \
  https://security.access.redhat.com/data/metrics/repository-to-cpe.json \
  -o "$SMOKE_DIR/mapping.json"

python3 - "$SMOKE_DIR" <<'PY'
import gzip, pathlib, sys
root = pathlib.Path(sys.argv[1])
(root / "mapping.gz").write_bytes(
    gzip.compress((root / "mapping.json").read_bytes())
)
PY

cat > "$SMOKE_DIR/server.py" <<'PY'
import gzip
import json
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path

mapping = gzip.decompress(Path("/fixture/mapping.gz").read_bytes())
counts = {}
lock = threading.Lock()

class Handler(BaseHTTPRequestHandler):
    def do_GET(self):
        key = (self.path, self.client_address[0])
        with lock:
            counts[key] = counts.get(key, 0) + 1
            attempt = counts[key]
        if self.path == "/exhausted":
            status = 503
        elif self.path == "/recovery" and attempt <= 3:
            status = 503
        elif self.path == "/permanent":
            status = 404
        else:
            status = 200
        body = mapping if status == 200 else b"injected mapping failure"
        print(json.dumps({
            "time": time.time(), "path": self.path,
            "client": self.client_address[0],
            "attempt": attempt, "status": status,
        }), flush=True)
        self.send_response(status)
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self.wfile.write(body)

    def log_message(self, *args):
        pass

ThreadingHTTPServer(("0.0.0.0", 8080), Handler).serve_forever()
PY

kubectl -n stackrox create configmap repo2cpe-smoke \
  --from-file=server.py="$SMOKE_DIR/server.py" \
  --from-file=mapping.gz="$SMOKE_DIR/mapping.gz" \
  --dry-run=client -o yaml | kubectl apply -f -

kubectl apply -f - <<'YAML'
apiVersion: apps/v1
kind: Deployment
metadata:
  name: repo2cpe-smoke
  namespace: stackrox
spec:
  replicas: 1
  selector:
    matchLabels:
      app: repo2cpe-smoke
  template:
    metadata:
      labels:
        app: repo2cpe-smoke
    spec:
      containers:
        - name: http
          image: docker.io/library/python:3.12-alpine
          command: ["python3", "/fixture/server.py"]
          ports:
            - containerPort: 8080
          volumeMounts:
            - name: fixture
              mountPath: /fixture
              readOnly: true
          resources:
            requests:
              cpu: 10m
              memory: 48Mi
            limits:
              memory: 128Mi
      volumes:
        - name: fixture
          configMap:
            name: repo2cpe-smoke
---
apiVersion: v1
kind: Service
metadata:
  name: repo2cpe-smoke
  namespace: stackrox
spec:
  selector:
    app: repo2cpe-smoke
  ports:
    - port: 8080
      targetPort: 8080
YAML

kubectl -n stackrox rollout status deployment/repo2cpe-smoke --timeout=3m

EXPECTED_NODES="$(kubectl -n stackrox get ds collector \
  -o jsonpath='{.status.desiredNumberScheduled}')"

for scenario in recovery exhausted permanent; do
  CASE_START="$(date -u +%Y-%m-%dT%H:%M:%SZ)"

  kubectl -n stackrox set env ds/collector --containers=compliance \
    ROX_NODE_SCANNING_MAX_INITIAL_WAIT=1s \
    ROX_NODE_INDEX_MAPPING_URL="http://repo2cpe-smoke.stackrox.svc:8080/$scenario"

  kubectl -n stackrox rollout status ds/collector --timeout=3m
  sleep 20

  kubectl -n stackrox logs deployment/repo2cpe-smoke \
    > "$SMOKE_DIR/requests.jsonl"

  python3 - "$scenario" "$EXPECTED_NODES" "$SMOKE_DIR/requests.jsonl" <<'PY'
import collections, json, pathlib, sys

scenario, expected_nodes, filename = sys.argv[1:]
groups = collections.defaultdict(list)
for line in pathlib.Path(filename).read_text().splitlines():
    if not line.startswith("{"):
        continue
    row = json.loads(line)
    if row["path"] == "/" + scenario:
        groups[row["client"]].append(row)

expected = {
    "recovery": [503, 503, 503, 200],
    "exhausted": [503, 503, 503, 503],
    "permanent": [404],
}[scenario]

assert len(groups) == int(expected_nodes), (scenario, len(groups))
for client, rows in sorted(groups.items()):
    statuses = [row["status"] for row in rows]
    delays = [
        round(b["time"] - a["time"], 2)
        for a, b in zip(rows, rows[1:])
    ]
    assert statuses == expected, (client, statuses)
    print(scenario, client, "statuses:", statuses, "waits:", delays)
print("PASS:", scenario, len(groups), "nodes")
PY

  if [ "$scenario" = recovery ]; then
    kubectl -n stackrox logs deployment/sensor \
      --since-time="$CASE_START" | grep 'Received index report'
  else
    kubectl -n stackrox logs -l app=collector -c compliance \
      --prefix --tail=100 --max-log-requests=20 \
      | grep 'Error creating node index:'
  fi
done

kubectl -n stackrox set env ds/collector --containers=compliance \
  ROX_NODE_INDEX_MAPPING_URL- \
  ROX_NODE_SCANNING_MAX_INITIAL_WAIT-

kubectl -n stackrox rollout status ds/collector --timeout=3m

kubectl -n stackrox delete \
  deployment/repo2cpe-smoke \
  service/repo2cpe-smoke \
  configmap/repo2cpe-smoke

kubectl -n stackrox get ds collector