diff --git a/.claude/agents/go-expert-dev.md b/.claude/agents/go-expert-dev.md index 18f63a4024ada..d68857fb2d95a 100644 --- a/.claude/agents/go-expert-dev.md +++ b/.claude/agents/go-expert-dev.md @@ -1,8 +1,10 @@ --- name: go-expert-dev -description: Use this agent when working with Go 1.24+ development, including modern language features, concurrency patterns, performance optimization, or any Go-related technical decisions. This agent should be used proactively during Go development workflows. +description: Use when working on Go development tasks - code editing, debugging, testing, refactoring, or dependency analysis. TRIGGER when user edits .go files, asks about Go patterns, needs Go-specific diagnostics, or requests Go code improvements. model: sonnet color: purple +skills: + - use-modern-go --- You are a Go expert specializing in modern Go 1.24+ development with advanced concurrency patterns, performance optimization, and production-ready system design. You know how to use go tools (e.g. gopls) to navigate, search and validate the code. You follow test driven development (TDD) methodology to build reliable solutions based on given requirements. diff --git a/.claude/commands/triage.md b/.claude/commands/triage.md index 188a69fb49de9..5b50228e254a2 100644 --- a/.claude/commands/triage.md +++ b/.claude/commands/triage.md @@ -1 +1,12 @@ -Analyze this CI failure and find a root cause and permanent solution: +Investigate this StackRox CI failure by launching the `stackrox-ci-failure-investigator` +subagent (via the Agent tool). Pass it the input below — a Prow build ID or URL, a failing +job name, a ROX-XXXXX ticket, or pasted logs — and have it download and read the CI +artifacts (Prow step logs plus the collected k8s service logs, pod descriptions, events, and +node capacity) before forming a hypothesis. + +Do not guess at a cause or attribute the failure to flakiness or a version being behind until +the artifacts rule everything else out. Follow the causal chain to its origin — a failed +check is usually a downstream symptom. Do not write code until the root cause is confirmed +with the user. + +$ARGUMENTS diff --git a/.claude/hooks/gofmt-hook.sh b/.claude/hooks/gofmt-hook.sh new file mode 100755 index 0000000000000..c776d4df8bd2e --- /dev/null +++ b/.claude/hooks/gofmt-hook.sh @@ -0,0 +1,4 @@ +#!/bin/bash +file=$(jq -r '.tool_input.file_path') +[[ "$file" == *.go ]] && go fmt "$file" 2>/dev/null +exit 0 diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000000000..b932254f5f704 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Edit|Write", + "hooks": [ + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/.claude/hooks/gofmt-hook.sh" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/.claude/skills/go-dependency-analyzer/SKILL.md b/.claude/skills/go-dependency-analyzer/SKILL.md new file mode 100644 index 0000000000000..bdaa345e8b471 --- /dev/null +++ b/.claude/skills/go-dependency-analyzer/SKILL.md @@ -0,0 +1,713 @@ +--- +name: go-dependency-analyzer +description: Analyzes Go dependencies to determine usage in production code, what functionality is used, and where it's located. Use when user asks "verify where we use [dependency]", "is [dependency] used in production", "analyze dependency [name]", "what uses [package]", "dependency analysis", mentions CVE numbers, security vulnerabilities, or needs to understand dependency impact for triage, upgrades, or removal decisions. +disable-model-invocation: true +--- + +# Go Dependency Analyzer + +Analyzes Go dependencies to determine production impact, used functionality, and code locations. Supports CVE triage, security issue assignment, dependency upgrade decisions, and understanding what would break if a dependency is removed or changed. + +## When to Use This Skill + +**ALWAYS use this skill when the user asks ANY question about Go dependencies:** + +- "Why do we need [dependency]?" +- "What uses [package/module]?" +- "Is [dependency] used in production?" +- "Where is [dependency] imported?" +- "Should we upgrade [dependency]?" +- "What would break if we remove [dependency]?" +- "Analyze [dependency]" +- "Check [dependency] for CVE-XXXXX" +- "Verify dependency usage for [security issue]" +- "Which team owns [dependency]?" + +**Trigger on:** +- Dependency names (e.g., "pgx", "zap", "docker/distribution") +- Module paths (e.g., "github.com/jackc/pgx/v5") +- CVE numbers (e.g., "CVE-2024-12345") +- Security vulnerability discussions +- Dependency upgrade/removal decisions +- Questions about Go packages in go.mod + +If the user mentions a Go dependency name or asks about usage/impact of any package, **invoke this skill immediately** before attempting manual analysis. + +## Key Features + +- **Branching Workflow:** Automatically detects direct vs transitive dependencies and uses appropriate analysis path +- **Wrapper Pattern Detection:** Identifies when dependencies are wrapped by internal packages (e.g., zap via pkg/logging) +- **goda Visualization:** Mandatory dependency tree visualization for understanding import chains +- **Transitive Chain Analysis:** Shows full dependency path: Our code → Intermediate → Dependency +- **Replace Directive Tracking:** Detects StackRox forks and version gaps +- **go mod why Integration:** Includes dependency justification in all reports + +## Prerequisites + +This skill requires the `goda` tool for dependency graph visualization. Install it with: + +```bash +go install github.com/loov/goda@latest +``` + +## Instructions + +### Step 1: Identify the Dependency + +When given a dependency name, CVE, or package: +1. Extract the Go module path (e.g., `github.com/jackc/pgx/v5`) +2. If given a CVE number, search for the affected package name first +3. Normalize the module path (handle version suffixes like `/v2`, `/v5`) +4. Accept partial names (e.g., "pgx" → find full path `github.com/jackc/pgx/v5`) + +### Step 2: Check Dependency Presence & Type + +IMPORTANT: Work in current directory - do not change directories (respects git worktrees). + +**Pre-flight check:** +```bash +# Verify go.mod exists in current directory +test -f go.mod || { echo "ERROR: go.mod not found. Run this skill from the Go module root directory."; exit 1; } +``` + +Run these commands in parallel: + +```bash +# Check if dependency is in go.mod (direct or indirect) +grep "module-name" go.mod + +# Check for replace directives (StackRox forks) +grep "module-name" go.mod | grep "=>" + +# Check if indirect +grep "module-name" go.mod | grep "// indirect" +``` + +**Multi-module Workspace Note:** +StackRox uses a single root `go.mod` (monorepo). This skill analyzes only the root `go.mod`. + +**Tool vs Production Dependencies:** +- Dependencies used only by `/tools/`, `/scripts/`, build infrastructure: Lower CVE priority (not shipped) +- Dependencies in `/central/`, `/sensor/`, `/scanner/`, `/roxctl/`, `/operator/`: Higher CVE priority (production code) +- If dependency appears only in tool build files, include this in status classification + +**If NOT found:** Respond immediately that the dependency is not used - issue can be closed. + +**If replaced:** Note the fork/replacement in analysis (e.g., `go.uber.org/zap => github.com/stackrox/zap`). + +**CRITICAL DECISION POINT:** + +A dependency can be BOTH direct and indirect (imported directly by some packages, pulled transitively by others). + +Check if dependency is marked `// indirect`: +- **If YES (only indirect):** Follow **Step 2A: Transitive Dependency Analysis Path** +- **If NO (direct, or both direct+indirect):** Follow **Step 3: Direct Dependency Analysis Path** + +Note: If `go mod why` shows direct import paths AND the dependency is marked `// indirect`, it means the dependency is used both ways - report this in the analysis. + +### Step 2A: Transitive Dependency Analysis Path + +**ONLY use this path for indirect dependencies.** + +```bash +# Show why this transitive dependency is needed +go mod why -m module-name + +# Show full dependency chain using go mod graph +go mod graph | grep "module-name" + +# Visualize what pulls this dependency +goda graph "reach(./..., module-name)" | head -30 +``` + +**Analysis for transitive dependencies:** + +1. **Identify the chain:** Who pulls this dependency? + - Format: `Our package → Intermediate package → module-name` + - Example: `central/externalbackups → cloud.google.com/go/storage → protoc-gen-validate` + +2. **Check if we use the intermediate package:** + ```bash + # Production usage (excludes test files) + grep -r "intermediate-package" --include="*.go" --exclude-dir=vendor --exclude-dir=test --exclude="*_test.go" | wc -l + + # Test-only usage + grep -r "intermediate-package" --include="*_test.go" --exclude-dir=vendor | wc -l + ``` + + **Test-Only Transitive Dependencies:** + - If intermediate package is only imported in `*_test.go` files, the transitive dependency is test-only + - Test-only transitive dependencies have lower CVE priority (not in shipped binaries) + - Include "TEST INFRASTRUCTURE ONLY" in status if applicable + - For CVE triage: Test-only transitive dependencies can usually be deferred unless critical + +3. **Report format for transitive:** + ``` + ## Dependency Analysis: [Module Name] (TRANSITIVE) + + **Status:** TRANSITIVE ONLY - Not directly used + + **Dependency Chain:** + ``` + Our code + ↓ imports + [intermediate package] + ↓ depends on + [module-name] v[version] + ``` + + **Why it exists:** + - [Explain what intermediate package needs it for] + - Used by: [List our files using intermediate package] + + **Action:** + - No direct maintenance needed under normal circumstances + - Updated automatically when [intermediate package] updates + - Monitor for security issues but typically lower priority (transitive) + + **When to Proactively Bump Indirect Dependencies:** + - **Critical CVE** in transitive dependency AND intermediate package hasn't updated within reasonable timeframe (e.g., 30 days after CVE disclosure) + - **High severity** with known exploits and production impact through intermediate package + - **Workaround:** Can add explicit `require` in go.mod to force minimum version, but this: + - Creates maintenance burden (must track when intermediate updates) + - May conflict with intermediate's version constraints + - Should be temporary until intermediate package updates + - **Preferred approach:** File issue/PR with intermediate package maintainers for faster upstream fix + ``` + +**SKIP to Step 7 (Generate Report)** after transitive analysis. + +### Step 3: Direct Dependency Analysis Path + +**ONLY use this path for direct dependencies (not marked `// indirect`).** + +```bash +# Confirm it's direct +go list -m module-name + +# MANDATORY: Show why dependency is needed +go mod why -m module-name + +# Visualize dependency graph with goda +goda graph "reach(./..., module-name)" | head -30 + +# Check how many packages use it +goda list "reach(./..., module-name)" | cut -d: -f1 | sort -u | wc -l +``` + +**Why Both Tools Are Needed:** + +`go mod why -m module-name` (MANDATORY): +- Shows ONE import path from our code to the dependency +- Limitation: If multiple packages import the dependency, only shows first path found +- Use for: Dependency justification, "why does this exist in go.mod" + +`goda graph "reach(./..., module-name)"` (COMPREHENSIVE): +- Shows ALL import paths and the complete dependency tree +- Identifies all packages that transitively depend on the module +- Use for: Full impact analysis, finding all usage locations +- Complements go mod why by revealing paths go mod why misses + +**Example scenario:** +- go mod why shows: `central/auth → module-name` +- goda reveals additional paths: `sensor/detector → module-name`, `roxctl/cli → module-name` +- Without goda, you'd miss sensor and roxctl usage + +Both tools are required for complete analysis. + +**Handling "does not need" Output:** + +If `go mod why` returns "(main module does not need module)" despite the dependency appearing in go.mod without `// indirect`: +- This indicates stale go.mod (dependency removed from code but not tidied) OR build-tag-excluded imports +- Document this in the report as: "Listed in go.mod but go mod why reports 'does not need' - likely stale entry or build-tag-excluded import" +- Recommend running `go mod tidy` to clean up go.mod +- Still analyze with goda to check for any actual usage + +**Analysis:** +- **Direct dependency**: We explicitly import it +- **go mod why output**: MANDATORY - include in final report +- **goda visualization**: Shows which of our packages import it + +### Step 4: Find Actual Usage Locations (Direct Dependencies Only) + +**Check for wrapper pattern:** Some dependencies are wrapped by `pkg/` packages (e.g., zap via pkg/logging). If wrapped, count BOTH direct imports AND wrapper usage. + +```bash +# Find direct imports +grep -r '"module-name' --include="*.go" --exclude-dir=vendor | cut -d: -f1 | sort -u | head -20 +goda list "reach(./..., module-name)" | head -30 + +# Separate production vs test +grep -r '"module-name' --include="*.go" --exclude="*_test.go" --exclude-dir=vendor --exclude-dir=test | wc -l +grep -r '"module-name' --include="*_test.go" --exclude-dir=vendor | wc -l + +# If wrapper detected in pkg/, count wrapper users too +grep -r 'pkg/wrapper"' --include="*.go" --exclude="*_test.go" --exclude-dir=vendor --exclude-dir=test | wc -l +``` + +### Step 5: Analyze Used Functionality (Direct Dependencies Only) + +Use gopls MCP tools (`mcp__gopls__go_search`, `mcp__gopls__go_symbol_references`) or grep to find specific function usage: + +```bash +# Search for vulnerable functions mentioned in CVE +grep -r "FunctionName" --include="*.go" --exclude-dir=vendor | head -20 +``` + +Note: GitNexus hooks may auto-provide symbol context if available. + +### Step 6: Determine Production Impact + +**Production code if:** +- Used in `*.go` files (not `*_test.go`) +- Not in `/test/`, `/qa-tests-backend/`, `/tools/` directories +- Imported by main application packages (`/central/`, `/sensor/`, `/scanner/`, `/roxctl/`) + +**Non-production if:** +- Only in `*_test.go` files +- Only in `/qa-tests-backend/`, `/tools/`, `/scripts/` +- Only transitive through test dependencies + +### Step 7: Generate Analysis Report + +**For DIRECT dependencies:** + +``` +## Dependency Analysis: [Dependency Name or CVE-ID] + +**Dependency:** module-path v[version] +**Status:** USED IN PRODUCTION | USED IN TESTS ONLY | WRAPPER PATTERN + +**Usage Summary:** +- Direct dependency: YES +- Production code files: [count] files (direct imports) +- Wrapper pattern: [if applicable: "+ [count] files via pkg/wrapper"] +- Test code files: [count] files +- Primary components: [list: central, sensor, scanner, roxctl, etc.] + +**Why Needed (go mod why):** +``` +[MANDATORY: Include go mod why -m output here] +github.com/stackrox/rox/path/to/package + imports github.com/example/dependency +``` + +**Specific Functionality Used:** +- [List actual functions/types imported and used] +- CVE affects: [specific function from CVE if applicable] +- We use: [functions we actually call] + +**Replace Directive:** +- Original: [module path if replaced] +- Fork/Replace: [replacement path from go.mod] +- Version Gap: [if fork behind: "Fork at v1.18, upstream at v1.27"] +- Reason: [if known, note why fork exists - check commit history] + +**Locations:** +[List key files with line numbers where possible] +- central/path/file.go:123 - uses QueryRow() +- sensor/pkg/file.go:456 - uses Connect() + +[If wrapper pattern detected:] +**Wrapper Usage:** +- Wrapped by: pkg/wrapper-name +- Components using wrapper: [list] +- Total indirect users: [count] files + +**Team Assignment:** +[Team name(s) from CODEOWNERS based on component usage] +``` + +**For TRANSITIVE dependencies (from Step 2A):** + +``` +## Dependency Analysis: [Dependency Name] (TRANSITIVE) + +**Dependency:** module-path v[version] +**Status:** TRANSITIVE ONLY - Not directly imported + +**Dependency Chain:** +``` +Our code (StackRox) + ↓ imports +[intermediate-package] v[version] + ↓ depends on +[module-name] v[version] +``` + +**Why It Exists:** +[From go mod why output - explain the chain] + +**Our Usage of Intermediate Package:** +- We import: [intermediate-package] +- Used in: [count] production files +- Components: [list components using intermediate] +- Files: [list key files] + +**What Intermediate Uses It For:** +- [Brief explanation - e.g., "GCS SDK uses it for proto validation"] +- [CVE impact: "If CVE affects this, check if intermediate is vulnerable"] + +**Team Assignment:** +N/A - Transitive dependency managed via [intermediate-package] updates +[If CVE with production impact: assign to team(s) from CODEOWNERS owning files that use the intermediate package] +``` + +## Examples + +### Example 1: Direct production dependency + +**User:** "Verify where we use pgx dependency for CVE-2024-12345" + +**Actions:** +1. Search go.mod: `grep pgx go.mod` → Found: `github.com/jackc/pgx/v5 v5.4.0` +2. Check usage: `grep -r "jackc/pgx" --include="*.go" --exclude-dir=vendor --exclude-dir=test` +3. Use gopls: `mcp__gopls__go_search` with query "pgx" +4. Analyze files: Found in `central/database/postgres/store.go` +5. Check functions: `grep -r "QueryRow\|Query\|Exec" central/database/postgres/` + +**Result:** +``` +## CVE-2024-12345 Analysis: pgx SQL Injection + +**Status:** USED IN PRODUCTION - HIGH PRIORITY + +**Usage Summary:** +- Direct dependency: YES +- Production files: 23 files in central/ +- Test files: 45 files +- Primary components: Central (PostgreSQL storage layer) + +**Functionality Used:** +- CVE affects: pgx.QueryRow() with string concatenation +- We use: pgx.Query(), pgx.Exec(), pgx.QueryRow() - VULNERABLE + +**Locations:** +- central/database/postgres/store.go:234 - uses QueryRow() +- central/database/postgres/migration.go:89 - uses Exec() + +**Team Assignment:** +@stackrox/core-workflows (from CODEOWNERS: `pkg/postgres/**/*` and `migrator/**/*`) +``` + +### Example 2: Test-only dependency + +**User:** "Is testify/assert used in production for CVE-2024-99999?" + +**Actions:** +1. `grep testify go.mod` → Found as indirect +2. `grep -r "testify/assert" --include="*.go" --exclude="*_test.go" --exclude-dir=vendor --exclude-dir=test` → No results +3. `grep -r "testify/assert" --include="*_test.go"` → 500+ files + +**Result:** +``` +## CVE-2024-99999 Analysis: testify + +**Status:** TEST CODE ONLY - LOW PRIORITY + +**Usage Summary:** +- Direct dependency: NO (indirect via test dependencies) +- Production files: 0 files +- Test files: 500+ files +- Primary components: Test infrastructure only + +**Team Assignment:** +N/A - Test-only transitive dependency (not production-critical) +``` + +### Example 3: Multiple dependency analysis + +**User:** "Check pgx, docker, and zap dependencies" + +**Actions:** +1. Run grep searches in parallel for all three in go.mod +2. Detect replace directives: `grep "zap.*=>" go.mod` +3. Count files for each dependency +4. Identify components using each +5. Create summary table + +**Result:** +```markdown +## Multi-Dependency Analysis + +### Summary Table + +| Dependency | Version | Status | Production Files | Components | Priority | +|------------|---------|--------|------------------|------------|----------| +| **pgx** | v5.9.2 | Direct | 155 | Central, Scanner, Migrator | CRITICAL | +| **docker/distribution** | v2.8.3 | Direct | 3 | pkg/registries | Medium | +| **zap** | v1.18.2 (fork) | Direct | 7 | pkg/logging, operator | CRITICAL | +``` + +### Example 4: Transitive dependency (test infrastructure) + +**User:** "Check CVE-2024-11111 in protoc-gen-validate" + +**Actions:** +1. Search go.mod: `grep protoc-gen-validate go.mod` → Found: `github.com/envoyproxy/protoc-gen-validate v0.10.1 // indirect` +2. Run `go mod why`: Shows dependency chain via `cloud.google.com/go/storage` +3. Check intermediate usage: `grep -r "cloud.google.com/go/storage" --include="*.go" --exclude="*_test.go" --exclude-dir=vendor --exclude-dir=test` → 0 results +4. Check test usage: `grep -r "cloud.google.com/go/storage" --include="*_test.go"` → 15 files in `central/externalbackups/` +5. Run `go mod graph | grep protoc-gen-validate` to see full chain + +**Result:** +``` +## CVE-2024-11111 Analysis: protoc-gen-validate (TRANSITIVE) + +**Dependency:** github.com/envoyproxy/protoc-gen-validate v0.10.1 +**Status:** TRANSITIVE ONLY - TEST INFRASTRUCTURE ONLY + +**Dependency Chain:** +``` +StackRox test code + ↓ imports (test files only) +cloud.google.com/go/storage v1.30.1 + ↓ depends on +github.com/envoyproxy/protoc-gen-validate v0.10.1 +``` + +**Why It Exists:** +GCS SDK (cloud.google.com/go/storage) uses protoc-gen-validate for protocol buffer validation. +We import GCS SDK only in test code for external backup testing. + +**Our Usage of Intermediate Package:** +- We import: cloud.google.com/go/storage +- Used in: 0 production files, 15 test files +- Components: central/externalbackups (test infrastructure only) +- Files: central/externalbackups/gcs_test.go, central/externalbackups/s3_test.go + +**What Intermediate Uses It For:** +- GCS SDK uses it for proto validation in API responses +- CVE impact: Only affects test execution, not shipped binaries + +**Action:** +- No direct maintenance needed under normal circumstances +- Test-only transitive dependency - lower CVE priority +- Updated automatically when cloud.google.com/go/storage updates +- For critical CVEs: Consider fixing in upstream GCS SDK or upgrading GCS SDK version + +**Team Assignment:** +N/A - Test-only transitive dependency (low priority) +If fix needed: @stackrox/core-workflows (owns central/externalbackups tests) +``` + +### Example 5: Replace directive (StackRox fork) + +**User:** "Analyze github.com/hashicorp/consul/api dependency for CVE-2024-22222" + +**Actions:** +1. Search go.mod: `grep consul go.mod` → Found with replace directive +2. Check replace: `grep "consul.*=>" go.mod` → `github.com/hashicorp/consul/api => github.com/stackrox/consul v1.15.3-0.20240215` +3. Check versions: Upstream at v1.18.0, fork at v1.15.3 (3 minor versions behind) +4. Find usage: `grep -r "consul/api" --include="*.go" --exclude="*_test.go" --exclude-dir=vendor --exclude-dir=test` → 12 files in `sensor/kubernetes/` +5. Run `go mod why -m github.com/hashicorp/consul/api` +6. Check StackRox fork repo for reason: Security patches backported from v1.18.0 + +**Result:** +``` +## CVE-2024-22222 Analysis: Consul API + +**Dependency:** github.com/hashicorp/consul/api v1.18.0 (replaced) +**Status:** USED IN PRODUCTION - FORK WITH SECURITY PATCHES + +**Usage Summary:** +- Direct dependency: YES (with StackRox fork) +- Production files: 12 files in sensor/kubernetes/ +- Test files: 8 files +- Primary components: Sensor (service discovery) + +**Why Needed (go mod why):** +``` +github.com/stackrox/rox/sensor/kubernetes/localscanner + imports github.com/hashicorp/consul/api +``` + +**Replace Directive:** +- Original: github.com/hashicorp/consul/api v1.18.0 +- Fork/Replace: github.com/stackrox/consul v1.15.3-0.20240215134248 +- Version Gap: Fork at v1.15.3, upstream at v1.18.0 (3 minor versions behind) +- Reason: CVE-2023-XXXXX security patches backported to v1.15.3 (check stackrox/consul repo commits) + +**CVE Analysis:** +- CVE affects: v1.15.0-v1.17.2 in ConsulClient.Query() +- StackRox fork status: Patch applied in fork commit abc123 (2024-02-15) +- **ALREADY PATCHED** - StackRox fork includes fix despite version gap + +**Specific Functionality Used:** +- sensor/kubernetes/localscanner/discovery.go:45 - ConsulClient.Health() +- sensor/kubernetes/localscanner/watcher.go:89 - ConsulClient.Query() +- CVE affects Query() - **WE USE THIS FUNCTION** but fork is patched + +**Locations:** +- sensor/kubernetes/localscanner/discovery.go:45 +- sensor/kubernetes/localscanner/watcher.go:89 +- sensor/kubernetes/localscanner/config.go:23 + +**Team Assignment:** +@stackrox/sensor-team (from CODEOWNERS: `sensor/**/*`) + +**Recommendation:** +No immediate action needed - fork already contains security patches. +Consider syncing fork to v1.18.0+ to reduce version gap and maintenance burden. +``` + +### Example 6: Wrapper pattern (infrastructure library) + +**User:** "Who needs zap logger?" + +**Actions:** +1. Check go.mod: `grep zap go.mod` → Found with replace directive +2. Check indirect: `grep zap go.mod | grep indirect` → NO, it's direct +3. Find direct imports: `grep -r '"go.uber.org/zap"' --include="*.go" --exclude-dir=vendor --exclude-dir=test` → Only 7 files +4. **Detect wrapper:** All 7 files in `pkg/logging/` +5. Count wrapper usage: `grep -r 'pkg/logging"' --include="*.go" --exclude-dir=vendor --exclude-dir=test` → 563 files! +6. Run `go mod why -m go.uber.org/zap` + +**Result:** +``` +## Dependency Analysis: Zap Logger + +**Dependency:** go.uber.org/zap v1.27.1 +**Status:** WRAPPER PATTERN - Used via pkg/logging + +**Usage Summary:** +- Direct dependency: YES (with StackRox fork) +- Direct zap imports: 7 files (all in pkg/logging infrastructure) +- Wrapper pattern: + 563 files via pkg/logging +- Primary components: ALL (Central, Sensor, Scanner, Operator, Migrator, roxctl, Compliance) + +**Why Needed (go mod why):** +``` +github.com/stackrox/rox/pkg/logging + imports go.uber.org/zap +``` + +**Replace Directive:** +- Original: go.uber.org/zap v1.27.1 +- Fork/Replace: github.com/stackrox/zap v1.18.2-0.20240314134248-5f932edd0404 +- Version Gap: Fork at v1.18.2, upstream at v1.27.1 (9 minor versions behind) +- Reason: StackRox-specific customizations (check stackrox/zap repo) + +**Wrapper Usage:** +- Wrapped by: pkg/logging +- Components using wrapper: Central (100+ files), Sensor (80+ files), ALL other components +- Architecture: All platform logging flows through pkg/logging → zap + +**Team Assignment:** +From CODEOWNERS: Multiple teams own components using logging. +Primary: Team owning `pkg/logging` wrapper. +Impact: ALL TEAMS (critical infrastructure) +``` + +## Troubleshooting + +### Error: "goda: command not found" + +**Cause:** goda not installed + +**Solution:** +```bash +go install github.com/loov/goda@latest +``` + +### Error: "gopls not responding" + +**Cause:** Large codebase, gopls indexing + +**Solution:** +- Fall back to grep-based analysis +- Use `go list` and `go mod why` instead +- Wait for gopls initialization to complete + +### Module path variations + +**Problem:** Package imported as `pgx/v5` but searching for `pgx` + +**Solution:** +- Search for base package name: `grep "pgx"` (case-sensitive - Go module paths distinguish github.com/Foo vs github.com/foo) +- Include version suffix: `grep "pgx/v[0-9]"` +- Use go list: `go list -m all | grep pgx` + +### Replace directives (forks) + +**Problem:** go.mod shows one version but code uses a fork + +**Example:** +``` +go.uber.org/zap v1.27.1 +go.uber.org/zap => github.com/stackrox/zap v1.18.2 +``` + +**Solution:** +- Always check for replace: `grep "module-name.*=>" go.mod` +- Report BOTH versions in analysis +- Note version differences (upstream 1.27.1 vs fork 1.18.2) +- Investigate why fork exists (security patch, features, etc.) +- Consider: Should we sync with upstream? + +**Impact:** +- Actual version used: The replacement version (v1.18.2) +- Declared version: What appears without replace (v1.27.1) +- CVEs may reference declared version but fork may have fix + +### Too many results + +**Problem:** Common package name matches everywhere + +**Solution:** +- Use full import path: `grep "github.com/jackc/pgx/v5"` +- Limit to specific directories: `grep -r "pgx" central/ sensor/` +- Use goda with specific scope: `goda list "reach(./central/..., pgx)"` + +### Uncertain about production vs test + +**Quick verification:** +```bash +# Check directory structure +ls -la path/to/file.go # Is it in /test/ or /qa-tests-backend/? + +# Check imports +head -20 path/to/file.go # Does it import testing packages? + +# Check build tags +grep "//go:build" path/to/file.go # Build constraints? +``` + +### Working in git worktrees + +**Symptom:** Commands failing or analyzing wrong repository + +**Cause:** Skill working in different directory than expected + +**Solution:** +- The skill ALWAYS works in your current directory +- Do NOT change directories with `cd` commands +- If you need to analyze the main repo from a worktree, use absolute paths: + ```bash + grep "module-name" /path/to/main/repo/go.mod + ``` +- Use `git worktree list` to see all worktrees +- Use `pwd` to confirm current directory before analysis + +**Best practice:** Stay in current worktree, analyze current go.mod + +## Component to Team Mapping + +When assigning issues, consult `.github/CODEOWNERS` to determine team ownership based on the files/directories where the dependency is used. + +**How to use CODEOWNERS:** +1. Identify which files/directories use the dependency (from your analysis) +2. Read `.github/CODEOWNERS` to find matching patterns +3. The LAST matching line wins (CODEOWNERS rule) +4. Assign to the team(s) listed for those patterns + +**Multi-team dependencies:** If used across multiple components with different owners, list all affected teams. + +## Performance Notes + +- Run grep searches in parallel when possible +- Use `--include="*.go"` to avoid searching binaries +- Use `--exclude-dir=vendor` to skip vendored code +- For large repos, limit scope: `./central/...` instead of `./...` +- Cache `go mod graph` output if analyzing multiple dependencies +- When analyzing multiple dependencies, batch the go.mod greps in one message diff --git a/.claude/skills/proto-writer/SKILL.md b/.claude/skills/proto-writer/SKILL.md new file mode 100644 index 0000000000000..02ac7a8a21094 --- /dev/null +++ b/.claude/skills/proto-writer/SKILL.md @@ -0,0 +1,260 @@ +--- +name: proto-writer +description: Guide the creation or modification of StackRox protobuf definitions and Postgres store generation. Use when creating new storage types, adding fields to existing protos, setting up gen.go files, or making architectural decisions about data modeling (embedded vs FK relationships, search scope, API/storage separation). +disable-model-invocation: true +--- + +# Proto Writer + +Interactive skill for creating and modifying StackRox protobuf definitions with correct tags, gen.go files, and store generation. + +## Tag and Flag Reference + +Read `proto/TAGS.md` for the complete reference on all proto `@gotags:` annotations (sql, search, policy, hash, sensorhash, scrub, validate, crYaml), gen.go generator flags, scenarios, and type mappings. The sections below guide the interactive workflow. + +## Step 1: Determine the Change Type + +Ask the user which scenario applies: + +### A) Adding or modifying fields on an existing proto + +- Identify the proto file and the gen.go file that generates its store. +- The gen.go file may NOT need changes -- only execution. See the decision below. +- **gen.go changes needed when:** adding a new FK relationship (`--references`), adding a new search category (`--search-category`), changing search scope (`--search-scope`), or changing store behavior. +- **gen.go changes NOT needed (but execution IS) when:** adding/removing `search:` tags, `sql:` index tags, non-SQL tags (hash, scrub, policy), or new fields without FK/search-category changes. + +### B) Creating a new storage proto + store + +- Needs a new proto file in `proto/storage/`, a new gen.go, and registration in several files. +- Follow all steps below. + +### C) Creating a new no-serialized store + +- Same as (B) but uses `--no-serialized` flag — all proto fields become individual DB columns, no serialized bytea blob. +- Generates a `NoSerializedStore[T]` with column-based scan/insert and optional child fetch control. +- `sql:"-"` is **disallowed** (walker panics). Use `sql:"strategy(bytea)"` for repeated messages you want in the parent table. +- Follow all steps below, with the no-serialized architectural decision in Step 3c. + +### D) Adding a v2 API for a storage proto + +- Storage protos (`proto/storage/`) are written first. v2 API protos (`proto/api/v2/`) are created afterward as a separate layer. +- v2 API protos MUST have separate message bodies from storage protos — even if the fields look similar. +- A `convert.go` bridges them. See `central/reports/service/v2/convert.go` for the pattern. +- When updating storage protos, ask: "Should I update the v2 API proto and converter too?" + +## Fast Path: Simple Field Addition + +If the user is adding a plain field to an existing proto that already has a gen.go (no FK, no new search category, no secrets, no policy criteria), skip directly to **Step 6**. Only the proto field and possibly a `search:` or `sql:"index"` tag are needed — the gen.go doesn't need changes, just re-execution. + +## Step 2: Tag Decision Tree + +When adding a field, walk through these questions with the user. Refer to `proto/TAGS.md` for full syntax of each tag. + +1. **Is this the primary key?** + - Yes -> `sql:"pk,id,type(uuid)"` (one per table, except singletons) + +2. **Should this field be searchable?** + - Yes -> `search:"Display Name"` and register a `FieldLabel` in `pkg/search/options.go` + - Should it be hidden from UI? -> add `,hidden` + +3. **Does this field reference another table?** + - Yes -> `sql:"fk(TypeName:field)"` and add `--references=storage.TypeName` to gen.go + - Does it need a real SQL FK constraint? If not -> add `,no-fk-constraint` + - Should it allow NULL? -> add `,allow-null` + - Should it cascade delete? (default yes) If restrict -> add `,restrict-delete` (use sparingly) + +4. **Should this field be indexed?** + - Yes -> `sql:"index=btree"` (default), `brin` for time-series, `gin` for arrays. `hash` indexes have been observed to be expensive; prefer btree unless the user explicitly requests a hash index. + - Part of a composite unique index? -> `sql:"index=name:my_index;category:unique"` (same name on all fields) + +5. **Is this a repeated message field? Which storage strategy?** + - Default (no tag) -> creates a child table with parent FK + `idx` column + - `sql:"strategy(bytea)"` -> serializes as bytea in the parent table (no child table, no SQL-level access to individual elements) + - Child table is best for queryable/joinable data. Bytea is best for opaque data that doesn't need SQL access. + +6. **Should this field be a policy criteria?** + - Yes -> `policy:"Display Name"` + +7. **Is this a sensitive credential?** + - Yes -> `scrub:"always"` for secrets, `scrub:"dependent"` for related fields like endpoints + +8. **Should this field be excluded from hash computation?** + - Yes (timestamps, computed scores, the hash field itself) -> `hash:"ignore"` or `sensorhash:"ignore"` + +9. **Is this an endpoint URL that must not be localhost?** + - Yes -> `validate:"nolocalendpoint"` + +## Step 3: Architectural Decisions + +When creating new protos with relationships, present these tradeoffs and ASK the user to choose. + +### 3a. Relationship Modeling: Embedded vs FK + +**Option 1: Embedded repeated field** (child in parent proto) +```protobuf +message Deployment { + repeated Container containers = 11; +} +``` +- Auto-generates a child table (`deployments_containers`) with parent FK + `idx` column. +- Parent's `serialized` column stores the full proto INCLUDING all children (data duplication). +- Each repeated message field = 1 child table. Nested repeated fields cascade further. +- Good when: children always accessed with parent, moderate cardinality, no independent lifecycle. +- Examples: `Deployment.containers`, `ImageV2.layers`. + +**Option 2: Separate protos with FK** (child references parent) +```protobuf +message VirtualMachineScanV2 { + string vm_v2_id = 2; // @gotags: sql:"fk(VirtualMachineV2:id),type(uuid),index=btree" +} +``` +- Each proto has its own store, datastore, gen.go, search category. +- No data duplication in serialized blobs. +- Good when: children are top-level entities, queried independently, high cardinality. +- Examples: `VirtualMachineV2` -> `VirtualMachineScanV2` -> `VirtualMachineComponentV2`. + +**Ask the user:** +> "Your proto has a relationship with `{child_type}`. Should it be an embedded repeated field (child table auto-generated, data duplicated in parent's serialized blob) or a separate top-level proto with its own store? Consider: +> 1. Does `{child_type}` have an independent lifecycle? +> 2. Will it be queried independently from the parent? +> 3. How many child rows per parent? +> 4. How many child tables will be generated?" + +### 3b. Search Scope and BFS Join Concerns + +Without `--search-scope`, the search framework BFS-traverses ALL connected schemas to resolve query fields. + +**Potential issues:** +- If two connected schemas have the same search field name, BFS picks non-deterministically. +- Multiple paths between schemas can cause BFS to choose the wrong join path even with `--search-scope` restrictions, because scope operates at the table level, not field level (ROX-17252). +- Duplicate search fields are sometimes intentional (e.g., Container in Deployment has Image-related search tags that also exist in Image proto). + +**Ask the user:** +> "Are there overlapping search fields with connected tables? Overlapping names are sometimes necessary for business logic, but be aware that `--search-scope` can only exclude entire tables — it cannot resolve ambiguity between tables that are both in scope. If overlapping names are intentional, verify that the BFS join path produces correct results for your queries." + +### 3c. Serialized vs No-Serialized Store + +**Option 1: Serialized (default)** +- A `serialized` bytea column stores the full proto blob alongside indexed columns. +- Reads deserialize from the blob — fast single-row retrieval, no need to scan every column. +- Data is duplicated between the blob and indexed columns. +- `sql:"-"` can exclude fields from columns (they're still in the blob). +- Mature, battle-tested path used by all existing stores. + +**Option 2: No-Serialized (`--no-serialized`)** +- All proto fields become individual DB columns. No serialized blob. +- No data duplication. Every field is directly queryable via SQL. +- Generated store is `NoSerializedStore[T]` with column-based scan/insert. +- Supports child fetch control: `WithChildren()` / `WithoutChildren()` functional options. +- `sql:"-"` is disallowed (walker panics). Use `sql:"strategy(bytea)"` for repeated messages that don't need SQL access. +- Best for new tables where you want full SQL-level access to all fields. + +**Ask the user:** +> "Should this store use the default serialized blob or the no-serialized mode? +> - Serialized: proven, fast single-row reads, allows `sql:\"-\"` exclusions. +> - No-serialized: no data duplication, full SQL access to all fields, child fetch control." + +### 3d. API Proto vs Storage Proto + +For v2 APIs, storage and API protos MUST be separate messages even if they look similar. + +**Pattern:** +- `proto/storage/report_configuration.proto` -> storage representation +- `proto/api/v2/report_service.proto` -> API representation +- `central/reports/service/v2/convert.go` -> converter between them + +**Ask the user:** +> "Is this for a v2 API? I'll create separate API and storage protos with a converter." + +## Step 4: gen.go Creation + +### Creating a New gen.go + +Create the file at `/datastore/[internal/]store/postgres/gen.go`: + +```go +package postgres + +//go:generate pg-table-bindings-wrapper --type=storage.TypeName [flags...] +``` + +Select flags based on the decisions above. Common patterns: + +**Standard searchable store:** +```go +//go:generate pg-table-bindings-wrapper --type=storage.MyResource --search-category MY_RESOURCES +``` + +**With FK references:** +```go +//go:generate pg-table-bindings-wrapper --type=storage.MyResource --search-category MY_RESOURCES --references=storage.OtherType +``` + +**Schema-only (hand-written store):** +```go +//go:generate pg-table-bindings-wrapper --type=storage.MyResource --schema-only --search-category MY_RESOURCES +``` + +**No-serialized store (all fields as columns):** +```go +//go:generate pg-table-bindings-wrapper --type=storage.MyResource --no-serialized --search-category MY_RESOURCES +``` + +**Singleton (config/settings):** +```go +//go:generate pg-table-bindings-wrapper --type=storage.MyConfig --singleton +``` + +See `proto/TAGS.md` for the complete flag reference with all options. + +### Critical Gotcha + +- Every `fk(TypeName:field)` tag in proto REQUIRES `storage.TypeName` in gen.go `--references`. Missing it causes a **panic** at generation time. +- The reverse (`--references` without a matching `fk(...)`) is silently ignored -- it does nothing. + +## Step 5: Registration Checklist + +For new types, these files may need updating: + +1. **`tools/generate-helpers/pg-table-bindings/list.go`** + - Add `storage.TypeName` -> `resources.ResourceHandle` mapping in the `typeRegistry`. + - Keep entries in lexicographic order. + - Required for ALL new types. + +2. **`proto/api/v1/search_service.proto`** + - Add a new `SearchCategory` enum value. + - Required only if the store is searchable (`--search-category`). + +3. **`pkg/search/options.go`** + - Add `FieldLabel` constants for each new `search:` tag value. + - The label string must match the tag value exactly. + +4. **`pkg/sac/resources/list.go`** + - Add a new resource type if none of the existing ones fit. + - Required only for new SAC resource types. + +## Step 6: Generate and Verify + +```bash +# 1. Regenerate proto Go code (if proto files changed) +make proto-generated-srcs + +# 2. Run Postgres bindings generator +gogen -run pg-table-bindings # gogen is from https://github.com/stackrox/workflow + +# 3. Verify generated output +# - pkg/postgres/schema/.go -- schema definition +# - /datastore/store/postgres/store.go -- store interface (if not --schema-only) +# - pkg/search/postgres/mapping/mapping.go -- search registration (if searchable) +# - pkg/postgres/schema/all.go -- table registration +``` + +## Step 7: Wire Compatibility + +All proto changes must be wire-compatible and schema changes must be backwards-compatible: + +- **Never** remove or renumber existing proto fields. Use `reserved` and `[deprecated = true]`. +- **Never** remove SQL columns. Add columns, don't remove. +- New columns tolerate their zero value until normal operation populates them — a migration is not needed immediately if that's acceptable. +- Migrations are needed when existing data must be backfilled or transformed for correct behavior. They can often be added later in the feature development cycle rather than upfront. See `migrator/README.md`. +- When deprecating a field: `string old_field = 5 [deprecated = true];` and add `reserved` for the field number if removing entirely later. diff --git a/.claude/skills/test-plan/SKILL.md b/.claude/skills/test-plan/SKILL.md new file mode 100644 index 0000000000000..bb116c715dbfb --- /dev/null +++ b/.claude/skills/test-plan/SKILL.md @@ -0,0 +1,336 @@ +--- +name: test-plan +description: Generate a manual test plan for the current feature/work following StackRox testing patterns +disable-model-invocation: true +arguments: + - name: feature_description + description: Brief description of what's being tested (e.g., "cluster label scoping in policies") + required: false +--- + +# Test Plan Generator + +Generates a manual test plan following StackRox patterns for manual feature testing. + +## Usage + +```bash +/test-plan "cluster label scoping in SecurityPolicy CRDs" +``` + +Or without arguments to be prompted: +```bash +/test-plan +``` + +--- + +## Instructions + +**Step 1: Gather Context** + +Get current context: +```bash +# Get current branch +git branch --show-current + +# Compute the merge base for comparison (handles repos where master/main may not share history) +BASE=$(git merge-base HEAD "$(git rev-parse --abbrev-ref origin/HEAD 2>/dev/null | sed 's|^origin/||' || echo master)" 2>/dev/null || git rev-parse HEAD~10 2>/dev/null || git rev-parse HEAD) + +# Get recent commits to understand changes +git log "$BASE"..HEAD --oneline | head -10 + +# Get changed files to understand scope +git diff "$BASE"...HEAD --name-only +``` + +{{#if feature_description}} +Feature being tested: {{feature_description}} +{{else}} +Infer the feature being tested from the branch name, commit messages, and changed files. Look for JIRA ticket references (ROX-XXXXX) in the branch name or commits. Check if any feature flags are referenced in the changes. +{{/if}} + +**Step 2: Spawn Test Plan Generation Agent** + +Use the Agent tool to generate the test plan: + +``` +Agent( + description: "Generate manual test plan", + subagent_type: "general-purpose", + prompt: "" +) +``` + +**Agent Prompt:** + +``` +You are generating a manual test plan for StackRox feature testing. + +**Context provided:** +- Feature: {{feature_description}} +- Branch: +- JIRA: +- PR: +- Feature flags: +- Changed files: + +**Your task:** + +Generate a comprehensive manual test plan following this EXACT structure: + +--- + +# Manual Test Plan for + +**Feature:** +**Branch:** +**PR:** # (if exists) +**JIRA:** https://redhat.atlassian.net/browse/ (if exists) +**Feature Flags:** + +--- + +## Setup + +### 1. Set Environment Variables + +```bash +# Get the image tag from rhacs-bot's comment on your PR +# It will look something like: 4.11.x-XXX-gABCDEF +export MAIN_IMAGE_TAG="" +echo "Using image tag: $MAIN_IMAGE_TAG" + +# Set quay.io registry credentials (needed for roxie to pull images) +export REGISTRY_USERNAME="" +export REGISTRY_PASSWORD="" +``` + +### 2. Deploy ACS with roxie + +```bash +cd ~/Workspace/roxie +# Add --features if the feature requires a feature flag +./roxie deploy --single-namespace +``` + +**Verify deployment:** +```bash +kubectl get pods -n stackrox + +# Get Central's external IP for API access +export CENTRAL_IP=$(kubectl get svc -n stackrox central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +echo "Central IP: $CENTRAL_IP" +``` + +### Prerequisites +- kubectl access to the cluster +- jq installed for JSON parsing +- Central admin password (auto-retrieved below) + +### Environment Setup + +```bash +# Get and export Central admin password +export PASSWORD=$(kubectl get secret -n stackrox admin-password -o jsonpath='{.data.password}' | base64 -d) + +# Verify connectivity and authentication work +curl -sk -u "admin:$PASSWORD" https://$CENTRAL_IP/v1/metadata | jq +``` + +--- + +## Test Case 1: + +**Objective:** + +### Steps: + +1. **** +```bash +# Commands to execute the action +``` + +2. **Verify ** +```bash +# Verification commands +``` +**Expected:** + +3. **Check via API** (if applicable) +```bash +# Get resource details via Central API +RESOURCE_ID=$(curl -sk -u "admin:$PASSWORD" https://$CENTRAL_IP/v1/ | jq -r '.[] | select(.name == "") | .id') +curl -sk -u "admin:$PASSWORD" https://$CENTRAL_IP/v1//$RESOURCE_ID | jq '' +``` +**Expected:** + +--- + +## Test Case 2: + +**Objective:** + +### Steps: + +[Follow same pattern as Test Case 1] + +--- + +[Add more test cases as needed based on the feature scope] + +--- + +## Cleanup + +```bash +# Teardown the deployment (removes all resources from stackrox namespace) +cd ~/Workspace/roxie +./roxie teardown --single-namespace +``` + +--- + +## Summary + +**Total Test Cases:** + +| Test Case | Description | Pass/Fail | +|-----------|-------------|-----------| +| 1 | | | +| 2 | | | +| N | | | + +**Result:** _____ / passed + +--- + +## Troubleshooting + +### Common Issues + +**1. Feature flag not enabled error** + +**Symptom:** Resource rejected with message about feature flag required + +**Solution:** Redeploy with feature flag enabled: +```bash +cd ~/Workspace/roxie +./roxie teardown --single-namespace +./roxie deploy --single-namespace --features +``` + +--- + +**2. curl commands return "credentials not found" or null** + +**Symptom:** +- `curl` returns: `"failed to identify user with username \"admin\"` +- `jq` returns: `Cannot iterate over null (null)` + +**Common Causes:** +- **CENTRAL_IP not set**: Run `export CENTRAL_IP=$(kubectl get svc -n stackrox central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}')` +- **Wrong password variable**: Use `$PASSWORD` (not `$ROX_PASSWORD`) +- **Wrong secret name**: Use `admin-password` secret (not `central-htpasswd`) + +**Verification:** +```bash +# Check password is set +echo "Password length: ${#PASSWORD}" + +# Test connectivity +curl -sk https://$CENTRAL_IP/v1/ping + +# Test auth +curl -sk -u "admin:$PASSWORD" https://$CENTRAL_IP/v1/metadata | jq +``` + +--- + +**CRITICAL PATTERNS TO FOLLOW:** + +1. **Authentication Setup:** + - Secret name: `admin-password` (NOT `central-htpasswd`) + - Variable name: `PASSWORD` (NOT `ROX_PASSWORD`) + - Auth format in curl: `"admin:$PASSWORD"` + - Always verify auth works before running tests + +2. **API Command Patterns:** + - Combine ID lookup and details fetch into single command blocks: + ```bash + RESOURCE_ID=$(curl -sk -u "admin:$PASSWORD" https://$CENTRAL_IP/v1/ | jq -r '.[] | select(.name == "") | .id') + curl -sk -u "admin:$PASSWORD" https://$CENTRAL_IP/v1//$RESOURCE_ID | jq '' + ``` + - Use `jq -r` for extracting single values to shell variables + - Use `jq` (without -r) for pretty JSON output + - Don't make users manually copy-paste IDs between commands + +3. **Central API Access:** + - Use the LoadBalancer external IP: `export CENTRAL_IP=$(kubectl get svc -n stackrox central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}')` + - All curl commands use `https://$CENTRAL_IP/v1/...` + +4. **Roxie Deployment:** + - Deploy: `./roxie deploy --single-namespace` + - With feature flags: `./roxie deploy --single-namespace --features ` + - Teardown: `./roxie teardown --single-namespace` (removes resources but keeps cluster) + - Registry credentials required: Set `REGISTRY_USERNAME` and `REGISTRY_PASSWORD` env vars + +5. **Test Case Structure:** + - Start with clear objective statement + - Use numbered steps + - Put all commands in bash code blocks + - Explicitly state expected outputs with "Expected:" prefix + - Include both kubectl verification AND API verification where applicable + +6. **What to Test:** + - Review changed files to understand scope + - Test happy path (feature working correctly) + - Test different variations (e.g., different field types, scopes, configurations) + - Test edge cases if applicable + - Verify via both Kubernetes API (kubectl) and Central API (curl) + - Test both creation/configuration AND runtime behavior if applicable + +7. **Generating Test Cases:** + - Think about natural boundaries in the feature (e.g., different object types, different scopes) + - Each test case should validate one major aspect + - Keep test cases focused and atomic + - Typical count: 3-5 test cases for most features + - Include at least one "combined" test case if feature has multiple components + +8. **Expected Outputs:** + - Be specific - show exact JSON structure expected + - Use `jq` filters to show relevant fields only + - Include sample values that match the test data + - For kubectl commands, describe what you expect to see (e.g., "all conditions True") + +**Generate the complete test plan now, following the structure above exactly.** +``` + +**Step 3: Write Test Plan to File** + +Ensure the output directory exists: +```bash +mkdir -p tmp +``` + +Get the filename: +- If JIRA ticket: `tmp/-test-plan.md` +- Otherwise: `tmp/-test-plan.md` (replace `/` with `-` in branch name) + +Write the test plan returned by the agent to the file. + +Display to user: +``` +Test plan generated! + +Written to: + +Review and customize as needed. The test plan includes: +- Setup instructions with roxie deployment +- Environment configuration (password, port-forward) +- test cases based on the feature scope +- Cleanup instructions +- Troubleshooting guide for common issues + +Run through the test plan and update the summary table with Pass/Fail results. +``` diff --git a/.claude/skills/use-modern-go/SKILL.md b/.claude/skills/use-modern-go/SKILL.md new file mode 100644 index 0000000000000..7c19b734534a2 --- /dev/null +++ b/.claude/skills/use-modern-go/SKILL.md @@ -0,0 +1,310 @@ +--- +name: use-modern-go +description: Apply modern Go syntax guidelines based on project's Go version. Use when user ask for modern Go code guidelines. Source https://github.com/JetBrains/go-modern-guidelines +--- + +# Modern Go Guidelines + +## Detected Go Version + +!`grep -rh "^go " --include="go.mod" . 2>/dev/null | cut -d' ' -f2 | sort | uniq -c | sort -nr | head -1 | xargs | cut -d' ' -f2 | grep . || echo unknown` + +## How to Use This Skill + +DO NOT search for go.mod files or try to detect the version yourself. Use ONLY the version shown above. + +**If version detected (not "unknown"):** +- Say: "This project is using Go X.XX, so I’ll stick to modern Go best practices and freely use language features up to and including this version. If you’d prefer a different target version, just let me know." +- Do NOT list features, do NOT ask for confirmation + +**If version is "unknown":** +- Say: "Could not detect Go version in this repository" +- Use AskUserQuestion: "Which Go version should I target?" → [1.23] / [1.24] / [1.25] / [1.26] + +**When writing Go code**, use ALL features from this document up to the target version: +- Prefer modern built-ins and packages (`slices`, `maps`, `cmp`) over legacy patterns +- Never use features from newer Go versions than the target +- Never use outdated patterns when a modern alternative is available + +--- + +## Features by Go Version + +### Go 1.0+ + +- `time.Since`: `time.Since(start)` instead of `time.Now().Sub(start)` + +### Go 1.8+ + +- `time.Until`: `time.Until(deadline)` instead of `deadline.Sub(time.Now())` + +### Go 1.13+ + +- `errors.Is`: `errors.Is(err, target)` instead of `err == target` (works with wrapped errors) + +### Go 1.18+ + +- `any`: Use `any` instead of `interface{}` +- `bytes.Cut`: `before, after, found := bytes.Cut(b, sep)` instead of Index+slice +- `strings.Cut`: `before, after, found := strings.Cut(s, sep)` + +### Go 1.19+ + +- `fmt.Appendf`: `buf = fmt.Appendf(buf, "x=%d", x)` instead of `[]byte(fmt.Sprintf(...))` +- `atomic.Bool`/`atomic.Int64`/`atomic.Pointer[T]`: Type-safe atomics instead of `atomic.StoreInt32` + +```go +var flag atomic.Bool +flag.Store(true) +if flag.Load() { ... } + +var ptr atomic.Pointer[Config] +ptr.Store(cfg) +``` + +### Go 1.20+ + +- `strings.Clone`: `strings.Clone(s)` to copy string without sharing memory +- `bytes.Clone`: `bytes.Clone(b)` to copy byte slice +- `strings.CutPrefix/CutSuffix`: `if rest, ok := strings.CutPrefix(s, "pre:"); ok { ... }` +- `errors.Join`: `errors.Join(err1, err2)` to combine multiple errors +- `context.WithCancelCause`: `ctx, cancel := context.WithCancelCause(parent)` then `cancel(err)` +- `context.Cause`: `context.Cause(ctx)` to get the error that caused cancellation + +### Go 1.21+ + +**Built-ins:** +- `min`/`max`: `max(a, b)` instead of if/else comparisons +- `clear`: `clear(m)` to delete all map entries, `clear(s)` to zero slice elements + +**slices package:** +- `slices.Contains`: `slices.Contains(items, x)` instead of manual loops +- `slices.Index`: `slices.Index(items, x)` returns index (-1 if not found) +- `slices.IndexFunc`: `slices.IndexFunc(items, func(item T) bool { return item.ID == id })` +- `slices.SortFunc`: `slices.SortFunc(items, func(a, b T) int { return cmp.Compare(a.X, b.X) })` +- `slices.Sort`: `slices.Sort(items)` for ordered types +- `slices.Max`/`slices.Min`: `slices.Max(items)` instead of manual loop +- `slices.Reverse`: `slices.Reverse(items)` instead of manual swap loop +- `slices.Compact`: `slices.Compact(items)` removes consecutive duplicates in-place +- `slices.Clip`: `slices.Clip(s)` removes unused capacity +- `slices.Clone`: `slices.Clone(s)` creates a copy + +**maps package:** +- `maps.Clone`: `maps.Clone(m)` instead of manual map iteration +- `maps.Copy`: `maps.Copy(dst, src)` copies entries from src to dst +- `maps.DeleteFunc`: `maps.DeleteFunc(m, func(k K, v V) bool { return condition })` + +**sync package:** +- `sync.OnceFunc`: `f := sync.OnceFunc(func() { ... })` instead of `sync.Once` + wrapper +- `sync.OnceValue`: `getter := sync.OnceValue(func() T { return computeValue() })` + +**context package:** +- `context.AfterFunc`: `stop := context.AfterFunc(ctx, cleanup)` runs cleanup on cancellation +- `context.WithTimeoutCause`: `ctx, cancel := context.WithTimeoutCause(parent, d, err)` +- `context.WithDeadlineCause`: Similar with deadline instead of duration + +### Go 1.22+ + +**Loops:** +- `for i := range n`: `for i := range len(items)` instead of `for i := 0; i < len(items); i++` +- Loop variables are now safe to capture in goroutines (each iteration has its own copy) + +**cmp package:** +- `cmp.Or`: `cmp.Or(flag, env, config, "default")` returns first non-zero value + +```go +// Instead of: +name := os.Getenv("NAME") +if name == "" { + name = "default" +} +// Use: +name := cmp.Or(os.Getenv("NAME"), "default") +``` + +**reflect package:** +- `reflect.TypeFor`: `reflect.TypeFor[T]()` instead of `reflect.TypeOf((*T)(nil)).Elem()` + +**net/http:** +- Enhanced `http.ServeMux` patterns: `mux.HandleFunc("GET /api/{id}", handler)` with method and path params +- `r.PathValue("id")` to get path parameters + +### Go 1.23+ + +- `maps.Keys(m)` / `maps.Values(m)` return iterators +- `slices.Collect(iter)` not manual loop to build slice from iterator +- `slices.Sorted(iter)` to collect and sort in one step + +```go +keys := slices.Collect(maps.Keys(m)) // not: for k := range m { keys = append(keys, k) } +sortedKeys := slices.Sorted(maps.Keys(m)) // collect + sort +for k := range maps.Keys(m) { process(k) } // iterate directly +``` + +**time package** + +- `time.Tick`: Use `time.Tick` freely — as of Go 1.23, the garbage collector can recover unreferenced tickers, even if they haven't been stopped. The Stop method is no longer necessary to help the garbage collector. There is no longer any reason to prefer NewTicker when Tick will do. + +### Go 1.24+ + +- `t.Context()` not `context.WithCancel(context.Background())` in tests. + ALWAYS use t.Context() when a test function needs a context. + +Before: +```go +func TestFoo(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + result := doSomething(ctx) +} +``` +After: +```go +func TestFoo(t *testing.T) { + ctx := t.Context() + result := doSomething(ctx) +} +``` + +- `omitzero` not `omitempty` in JSON struct tags. + ALWAYS use omitzero for time.Duration, time.Time, structs, slices, maps. + +Before: +```go +type Config struct { + Timeout time.Duration `json:"timeout,omitempty"` // doesn't work for Duration! +} +``` +After: +```go +type Config struct { + Timeout time.Duration `json:"timeout,omitzero"` +} +``` + +- `b.Loop()` not `for i := 0; i < b.N; i++` in benchmarks. + ALWAYS use b.Loop() for the main loop in benchmark functions. + +Before: +```go +func BenchmarkFoo(b *testing.B) { + for i := 0; i < b.N; i++ { + doWork() + } +} +``` +After: +```go +func BenchmarkFoo(b *testing.B) { + for b.Loop() { + doWork() + } +} +``` + +- `strings.SplitSeq` not `strings.Split` when iterating. + ALWAYS use SplitSeq/FieldsSeq when iterating over split results in a for-range loop. + +Before: +```go +for _, part := range strings.Split(s, ",") { + process(part) +} +``` +After: +```go +for part := range strings.SplitSeq(s, ",") { + process(part) +} +``` +Also: `strings.FieldsSeq`, `bytes.SplitSeq`, `bytes.FieldsSeq`. + +**testing/synctest package** + +- Use `testing/synctest` for deterministic testing of concurrent code with goroutines and time. +- `synctest.Run(func())` executes code in a bubble where time and goroutines are controlled. +- Fast, deterministic tests for timeouts, tickers, and race conditions. + +```go +func TestTimeout(t *testing.T) { + synctest.Run(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Simulated time advances only when goroutines block + // Test completes instantly but behaves as if 5 seconds passed + <-ctx.Done() + }) +} +``` + +### Go 1.25+ + +- `wg.Go(fn)` not `wg.Add(1)` + `go func() { defer wg.Done(); ... }()`. + ALWAYS use wg.Go() when spawning goroutines with sync.WaitGroup. + +Before: +```go +var wg sync.WaitGroup +for _, item := range items { + wg.Add(1) + go func() { + defer wg.Done() + process(item) + }() +} +wg.Wait() +``` +After: +```go +var wg sync.WaitGroup +for _, item := range items { + wg.Go(func() { + process(item) + }) +} +wg.Wait() +``` + +### Go 1.26+ + +- `new(val)` not `x := val; &x` — returns pointer to any value. + Go 1.26 extends new() to accept expressions, not just types. + Type is inferred: new(0) → *int, new("s") → *string, new(T{}) → *T. + DO NOT use `x := val; &x` pattern — always use new(val) directly. + DO NOT use redundant casts like new(int(0)) — just write new(0). + Common use case: struct fields with pointer types. + +Before: +```go +timeout := 30 +debug := true +cfg := Config{ + Timeout: &timeout, + Debug: &debug, +} +``` +After: +```go +cfg := Config{ + Timeout: new(30), // *int + Debug: new(true), // *bool +} +``` + +- `errors.AsType[T](err)` not `errors.As(err, &target)`. + ALWAYS use errors.AsType when checking if error matches a specific type. + +Before: +```go +var pathErr *os.PathError +if errors.As(err, &pathErr) { + handle(pathErr) +} +``` +After: +```go +if pathErr, ok := errors.AsType[*os.PathError](err); ok { + handle(pathErr) +} +``` diff --git a/.cursor/rules/README.md b/.cursor/rules/README.md index 5d7c00bda9933..8581c6c8435d6 100644 --- a/.cursor/rules/README.md +++ b/.cursor/rules/README.md @@ -6,5 +6,5 @@ This directory contains a set of [Cursor project rules](https://docs.cursor.com/ The rules in [security/](security/) provide security guidelines for Cursor to generate secure code/suggestions -- The [security/product-security](security/product-security/) directory is a clone of [Product Security's Cursor rules](https://gitlab.cee.redhat.com/product-security/security-cursor-rules). +- The [security/product-security](security/product-security/) directory is a copy of [Product Security's Cursor rules](https://gitlab.cee.redhat.com/product-security/security-cursor-rules) in this repo. - Rules directly in [security/](security/) are Stackrox-specific rules that aren't currently covered by ProdSec rules. diff --git a/.cursor/rules/product-security/security_rules.mdc b/.cursor/rules/product-security/security_rules.mdc deleted file mode 100644 index e33561b7dbebc..0000000000000 --- a/.cursor/rules/product-security/security_rules.mdc +++ /dev/null @@ -1,30 +0,0 @@ ---- -description: Global security rules for all projects 🛡️ -globs: - - "**/*" -alwaysApply: true ---- - -# Security Principles - -- **Complete Mediation:** Ensure every access request is validated and authorized. No bypasses allowed. -- **Compromise Recording:** Implement mechanisms to detect, record, and respond to security breaches. -- **Defense in Depth:** Use multiple, layered security controls to protect systems and data. -- **Economy of Mechanism:** Keep designs simple and easy to understand. Avoid unnecessary complexity. -- **Least Common Mechanism:** Minimize shared resources and isolate components to reduce risk. -- **Least Privilege:** Grant only the minimum permissions necessary for users and systems. -- **Open Design:** Favor transparency and well-understood mechanisms over security by obscurity. -- **Psychological Acceptability:** Make security controls user-friendly and aligned with user expectations. -- **Secure by Design, Default, Deployment (SD3):** Ship with secure defaults, deny by default, and avoid hardcoded credentials. - -# Security Controls - -- **Authentication:** Use strong, standard authentication methods. Require authentication for all non-public areas. Store credentials securely and enforce password policies. Use multi-factor authentication where possible. -- **Authorization:** Enforce least privilege and explicit permissions. Use a single, trusted component for authorization checks. Deny by default and audit permissions regularly. -- **Encryption:** Encrypt all network traffic and data at rest where applicable. Use approved certificates and protocols. -- **Logging:** Log security events centrally. Do not log sensitive data. Restrict log access and monitor for suspicious activity. -- **Networking:** Encrypt all communications. Do not expose unnecessary endpoints or ports. Restrict network access to only what is required. - ---- - -Apply these rules to all code, infrastructure, and processes to maintain a strong security posture across your projects. diff --git a/.cursor/rules/product-security/technology/containers.mdc b/.cursor/rules/product-security/technology/containers.mdc deleted file mode 100644 index 4ea80b15dae76..0000000000000 --- a/.cursor/rules/product-security/technology/containers.mdc +++ /dev/null @@ -1,11 +0,0 @@ ---- -description: Secure coding rules for Containers 🐳 -globs: - - "**/Containerfile" - - "**/Dockerfile" -alwaysApply: false ---- - -- Use minimal, up-to-date base images from trusted sources. -- Remove non-essential software and keep essential ones updated. -- Run containers as non-root and use read-only filesystems when possible. diff --git a/.cursor/rules/product-security/technology/databases.mdc b/.cursor/rules/product-security/technology/databases.mdc deleted file mode 100644 index 71dfcb43f7b5f..0000000000000 --- a/.cursor/rules/product-security/technology/databases.mdc +++ /dev/null @@ -1,12 +0,0 @@ ---- -description: Secure coding rules for Databases 🗄️ -globs: - - "**/database*" - - "**/*.sql" -alwaysApply: false ---- - -- Encrypt communication and data at rest. -- Enable auditing and separate test from production databases. -- Use least privilege for application database access. -- Store connection strings securely and encrypted. diff --git a/.cursor/rules/product-security/technology/go.mdc b/.cursor/rules/product-security/technology/go.mdc deleted file mode 100644 index 53236f1b53977..0000000000000 --- a/.cursor/rules/product-security/technology/go.mdc +++ /dev/null @@ -1,16 +0,0 @@ ---- -description: Secure coding rules for Go projects 🦫 -globs: - - "**/*.go" - - "**/go.mod" - - "**/go.sum" -alwaysApply: false ---- - -- Use Go modules for dependency management. -- Validate all input entries. -- Use html/template for output to prevent XSS. -- Use parameterized queries to prevent SQL injection. -- Avoid using `fmt.Sprintf()` to build SQL or shell commands. -- Encrypt sensitive information and enforce HTTPS. -- Log errors securely, avoiding leaks of sensitive data like credentials or tokens. diff --git a/.cursor/rules/product-security/technology/helm.mdc b/.cursor/rules/product-security/technology/helm.mdc deleted file mode 100644 index f92ab2e4ed69d..0000000000000 --- a/.cursor/rules/product-security/technology/helm.mdc +++ /dev/null @@ -1,11 +0,0 @@ ---- -description: Secure coding rules for Helm charts ⎈ -globs: - - "**/Chart.yaml" - - "**/values.yaml" -alwaysApply: false ---- - -- Check for vulnerable dependencies and validate chart provenance. -- Secure configuration and manage secrets outside of values.yaml. -- Use tools like checkov for security scanning. diff --git a/.cursor/rules/product-security/technology/kafka.mdc b/.cursor/rules/product-security/technology/kafka.mdc deleted file mode 100644 index 0ae4c97e7ef38..0000000000000 --- a/.cursor/rules/product-security/technology/kafka.mdc +++ /dev/null @@ -1,12 +0,0 @@ ---- -description: Secure coding rules for Kafka and AMQ Streams 📨 -globs: - - "**/kafka*" - - "**/amq*" -alwaysApply: false ---- - -- Encrypt data in transit using TLS/SSL. -- Use authentication (SSL/SASL) for all connections. -- Apply access control lists (ACLs) to limit topic access. -- Monitor and alert on abnormal access patterns. diff --git a/.cursor/rules/product-security/technology/llm.mdc b/.cursor/rules/product-security/technology/llm.mdc deleted file mode 100644 index bb852e987616b..0000000000000 --- a/.cursor/rules/product-security/technology/llm.mdc +++ /dev/null @@ -1,11 +0,0 @@ ---- -description: Secure coding rules for Large Language Models (LLM) 🤖 -globs: - - "**/llm*" -alwaysApply: false ---- - -- Enforce strict input validation and privilege control for LLM plugins and APIs. -- Prevent prompt injection and insecure output handling. -- Limit LLM access to sensitive APIs and data sources. -- Apply regularization and differential privacy to protect training data. diff --git a/.cursor/rules/product-security/technology/mcp.mdc b/.cursor/rules/product-security/technology/mcp.mdc deleted file mode 100644 index 35e4fb8d9eaef..0000000000000 --- a/.cursor/rules/product-security/technology/mcp.mdc +++ /dev/null @@ -1,15 +0,0 @@ ---- -description: Secure coding rules for Model Context Protocol (MCP) clients and servers -globs: - - "**/*mcp*" -alwaysApply: false ---- - -- Authenticate and authorize actions using the privileges of the principal (for example, the user) initiating the interaction, via standard authentication protocols (such as OAuth). -- Don't let servers use their own credentials to perform actions on behalf of the user. -- All actions executed by the MCP server must be traceable to the authenticated principal and conform to their permissions and scope. -- Prevent prompt injection. -- Prevent command injection. -- Prevent API call injection. -- Implement logging the tools and the parameters to be called as well as the outputs from the tools. -- MCP clients must request the approval from the user before calling actions from an MCP server. diff --git a/.cursor/rules/product-security/technology/operators.mdc b/.cursor/rules/product-security/technology/operators.mdc deleted file mode 100644 index 8ac15862b9a12..0000000000000 --- a/.cursor/rules/product-security/technology/operators.mdc +++ /dev/null @@ -1,29 +0,0 @@ ---- -description: Secure coding rules for Kubernetes Operators ⚙️ -globs: - - "**/operator*" -alwaysApply: false ---- - -- Restrict access to sensitive resources (e.g., secrets) by using Role-Based Access Control (RBAC) and the Principle of Least Privilege. -- To minimize cluster and namespace permissions, create dedicated roles for each workload with only the necessary permissions and avoid using wildcards. Never use service accounts with cluster-admin or similar broad privileges. -- Isolate operator workloads by deploying them in dedicated namespaces. This prevents a compromise from spreading and affecting other workloads in the cluster. -- Instead of using default service accounts, create dedicated ones with only the minimum necessary permissions for your workload. -- Store all credentials and sensitive information in Secrets instead of hard-coding them in configuration files or environment variables. -- Ensure all communications are encrypted with TLS 1.2 or 1.3 (recommended). -- If using an older TLS version for compatibility, ensure all workloads have updated TLS configurations with weak cipher suites disabled. -- Define CPU and memory resource requests and limits to prevent resource exhaustion. -- Establish network policies with a "deny all" default for ingress and egress traffic. Explicitly define specific allow rules to control flow between pods, namespaces, and external services. -- Use certificates issued by an Openshift trusted CA. -- Avoid outdated dependencies. Replace deprecated and obsolete libraries with modern alternatives. -- Avoid or redact sensitive information in logs and error messages. -- Ensure the audit log captures enough detail for effective incident investigation. -- If the workload exposes any endpoints (like metrics), add authentication unless they are intended for public access. This is a crucial security practice that protects against future data exposure, even if the endpoint isn't currently handling sensitive information. -- Use the most restrictive Security Context Constraints (SCCs), such as restricted or restricted-v2, to enforce policies. This disallows privileged containers, prevents containers from running as root or escalating privileges, and denies host breakouts by disabling features like hostPID, hostIPC, hostNetwork, and allowedHostPath. -- If an operator requires privileged permissions and cannot use restrictive SCCs, consider implementing the following controls as appropriate to mitigate the risk: - - Ensure operator containers are mounted as read-only by setting “readOnlyRootFilesystem: true”. This prevents the container from writing to the root filesystem, which is a powerful defense against attackers who try to install malicious code. - - Ensures the container process runs as a non-root user by setting “runAsNonRoot: true”. - - Configure pods with “allowPrivilegeEscalation: false” to prevent a container from gaining more privileges than its parent process. - - Drop all Linux capabilities by setting “capabilities: drop: [ALL]”. This ensures the container operates with the fewest possible permissions, and you can add specific capabilities back if needed. - - Use seccomp profiles to control system calls and enforce security policies. - - Ensure your workloads are not running privileged. Avoid setting “privileged: true” unless it's an absolute necessity. Instead, consider workloads that can run with lower, more fine-grained permissions. diff --git a/.cursor/rules/product-security/technology/python.mdc b/.cursor/rules/product-security/technology/python.mdc deleted file mode 100644 index 3d9f8133acfcf..0000000000000 --- a/.cursor/rules/product-security/technology/python.mdc +++ /dev/null @@ -1,16 +0,0 @@ ---- -description: Secure coding rules for Python projects 🐍 -globs: - - "**/*.py" - - "**/requirements.txt" - - "**/Pipfile" - - "**/pyproject.toml" -alwaysApply: false ---- - -- Validate and sanitize all input. -- Avoid using `eval()`, `exec()` or `pickle` on untrusted data. -- Do not hardcode secrets in code. -- Pin exact versions in `requirements.txt` or `pyproject.toml`. Avoid using `*` or `latest`. -- Avoid shell execution with untrusted input. Use `subprocess.run([...], check=True)` instead of `os.system()`. -- Handle exceptions securely; avoid exposing debug traces or stack dumps to users. diff --git a/.cursor/rules/product-security/technology/react.mdc b/.cursor/rules/product-security/technology/react.mdc deleted file mode 100644 index bfcf585ca048a..0000000000000 --- a/.cursor/rules/product-security/technology/react.mdc +++ /dev/null @@ -1,12 +0,0 @@ ---- -description: Secure coding rules for React projects ⚛️ -globs: - - "**/*.jsx" - - "**/*.tsx" -alwaysApply: false ---- - -- Use default React data binding to prevent XSS. -- Sanitize HTML before using dangerouslySetInnerHTML. -- Validate URLs before using them in the DOM. -- Secure server-side rendering and avoid JSON injection. diff --git a/.cursor/rules/product-security/technology/rust.mdc b/.cursor/rules/product-security/technology/rust.mdc deleted file mode 100644 index d3fcf5da3c47b..0000000000000 --- a/.cursor/rules/product-security/technology/rust.mdc +++ /dev/null @@ -1,28 +0,0 @@ ---- -description: Secure coding rules for Rust projects 🦀 -globs: - - "**/Cargo.toml" - - "**/*.rs" -alwaysApply: false ---- - -- Avoid using unsafe code -- Always handle `Result` and `Option` using `match`, do not use `unwrap` -- Propagate errors using the `?` handle -- Always display meaningful error messages, which the user can act upon -- Prevent buffer overflows by using safe indexing methods (e.g., `get()`, `get_mut()`) and validating input sizes -- Prevent SQL injection by using parameterized queries and escaping user input -- Prevent XSS by escaping user input when rendering HTML -- Prevent command injection by avoiding the use of `std::process::Command` with user-supplied arguments -- Protect against DoS attacks by limiting resource usage (e.g., memory, CPU, network connections) -- Use the `checked_add`, `checked_sub`, `checked_mul`, etc. methods on integers to prevent overflows -- Avoid data races by using appropriate synchronization primitives (`Mutex`, `RwLock`, channels) -- Be aware of integer overflow and use checked arithmetic methods to prevent it -- Handle Unicode characters correctly to avoid unexpected behavior -- Handle file paths correctly, especially when dealing with different operating systems -- Be careful when writing concurrent code to avoid data races and deadlocks -- Split code into smaller, reusable modules -- Try to use as much the standard library as possible, only use crates if needed -- Prefer using crates with less dependencies, utilize crate features, in order to limit indirect dependencies -- Prefer crates with broader community adoption, which have more downloads, more github stars, are used in major projects -- Prefer crates with more than one maintainers, with recent commits/changes \ No newline at end of file diff --git a/.cursor/rules/tls-certificates.mdc b/.cursor/rules/tls-certificates.mdc new file mode 100644 index 0000000000000..d41aacc500b56 --- /dev/null +++ b/.cursor/rules/tls-certificates.mdc @@ -0,0 +1,13 @@ +--- +description: TLS certificate architecture, caching, rotation, and hot-reload across StackRox components. Read when working on TLS, mTLS, certificates, CA rotation, certwatch, or the TLSChallenge endpoint. +alwaysApply: false +--- + +# TLS certificates in StackRox + +Read `pkg/mtls/README.md` for the full architecture overview covering mTLS, +certificate caching, CA rotation, and hot-reload behavior across all StackRox +components. + +When making changes to TLS-related code, review and update that README if the +change affects the documented architecture, caching behavior, or CA rotation flow. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 512481c525896..1c271f2314e55 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image":"quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1", + "image":"quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11", "containerEnv":{ "CI":"true" }, diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dd9ff7e3d85f5..d318cf685d35c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -19,6 +19,7 @@ migrator/**/* @stackrox/core-workflows pkg/postgres/**/* @stackrox/core-workflows tests/upgrade/* @stackrox/core-workflows pkg/search/**/* @stackrox/core-workflows +tests/policy_as_code_test.go @stackrox/core-workflows # sensor-ecosystem main responsibilities include: # administration events, authN (authproviders), authZ (SAC) @@ -31,6 +32,7 @@ pkg/search/**/* @stackrox/core-workflows */discoveredclusters/**/ @stackrox/sensor-ecosystem */signatureintegration/**/* @stackrox/sensor-ecosystem compliance/virtualmachines/**/* @stackrox/sensor-ecosystem +scripts/ci/add-vms/**/* @stackrox/sensor-ecosystem @vikin91 pkg/features/**/* @stackrox/sensor-ecosystem pkg/images/defaults/**/* @stackrox/sensor-ecosystem pkg/sac/**/* @stackrox/sensor-ecosystem @@ -40,8 +42,13 @@ qa-tests-backend/**/*CloudSource* @stackrox/sensor-ecosystem qa-tests-backend/**/*Signature* @stackrox/sensor-ecosystem roxctl/**/* @stackrox/sensor-ecosystem sensor/**/* @stackrox/sensor-ecosystem +sensor/admission-control/**/* @stackrox/core-workflows +tests/ocp_dynamic_plugin_test.go @stackrox/sensor-ecosystem tests/performance/**/* @stackrox/sensor-ecosystem tests/roxctl/**/* @stackrox/sensor-ecosystem +roxctl/central/migratetooperator/**/* @stackrox/sensor-ecosystem @stackrox/install +roxctl/common/migratetooperator/**/* @stackrox/sensor-ecosystem @stackrox/install +roxctl/sensor/migratetooperator/**/* @stackrox/sensor-ecosystem @stackrox/install qa-tests-backend/**/* @janisz @@ -53,14 +60,17 @@ image/templates/** @stackrox/install operator/** @stackrox/install pkg/crs/** @stackrox/install pkg/helm/** @stackrox/install +pkg/migratetooperator/** @stackrox/install sensor/kubernetes/certrefresh/** @stackrox/install @stackrox/sensor-ecosystem sensor/kubernetes/crs/** @stackrox/install @stackrox/sensor-ecosystem roxctl/central/crs/** @stackrox/install @stackrox/sensor-ecosystem +# Also the e2e tests for legacy-to-operator migration commands in roxctl: +tests/roxctl/bats-tests/cluster/sensor-migrate-to-operator.bats @stackrox/install +tests/roxctl/bats-tests/local/central-migrate-to-operator.bats @stackrox/install # Scanner team's responsibilities include anything related to the scanner itself and scanning utilities # such as vulnerability uploading and image integrations. /.github/workflows/scanner* @stackrox/scanner -/.github/workflows/update_scanner_periodic.yaml @stackrox/scanner /central/image/service/ @stackrox/scanner /central/imageintegration/service/ @stackrox/scanner /central/imageintegration/store/defaults.go @stackrox/scanner @@ -89,6 +99,7 @@ roxctl/central/crs/** @stackrox/install @stackrox/sensor-ecosystem # Konflux maintainers for ACS review all changes related to the Konflux pipelines, Dockerfiles, etc. # Release engineers need to merge MintMaker PRs at the time of the release. # rhacs-bot needs an ability to auto-approve MintMaker PRs for automated task and security updates. +# The list here should be kept in sync with the .github/labeler.yml configuration for the konflux-build label. **/konflux.*Dockerfile @stackrox/konflux-maintainers-no-email @stackrox/release-mgmt-no-email @rhacs-bot /.tekton/ @stackrox/konflux-maintainers-no-email @stackrox/release-mgmt-no-email @rhacs-bot rpms.* @stackrox/konflux-maintainers-no-email @stackrox/release-mgmt-no-email @rhacs-bot diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000000000..5efc527065f62 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,8 @@ +self-hosted-runner: + labels: [] +paths: + .github/workflows/**: + ignore: + # GHA coerces bool to 0/1 for array indexing at runtime. + # This is a well-known idiom: fromJSON('["a","b"]')[cond != 'val'] + - 'index access of array must be type of number but got "bool"' diff --git a/.github/actions/cache-eslint/action.yaml b/.github/actions/cache-eslint/action.yaml new file mode 100644 index 0000000000000..1693a44002ee6 --- /dev/null +++ b/.github/actions/cache-eslint/action.yaml @@ -0,0 +1,20 @@ +name: Cache ESLint Results +description: Cache ESLint results to speed up linting on subsequent runs +runs: + using: composite + steps: + - name: Cache ESLint (save) + if: github.event_name == 'push' && github.ref_name == github.event.repository.default_branch + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # ratchet:actions/cache@v6.1.0 + with: + path: ui/apps/platform/.eslintcache + key: eslint-v1-${{ github.job }}-${{ hashFiles('ui/apps/platform/package-lock.json', 'ui/apps/platform/eslint.config.js', 'ui/apps/platform/eslint-plugins/**') }}-${{ github.sha }} + restore-keys: eslint-v1-${{ github.job }}-${{ hashFiles('ui/apps/platform/package-lock.json', 'ui/apps/platform/eslint.config.js', 'ui/apps/platform/eslint-plugins/**') }}- + + - name: Cache ESLint (restore) + if: ${{ !(github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # ratchet:actions/cache/restore@v6.1.0 + with: + path: ui/apps/platform/.eslintcache + key: eslint-v1-${{ github.job }}-${{ hashFiles('ui/apps/platform/package-lock.json', 'ui/apps/platform/eslint.config.js', 'ui/apps/platform/eslint-plugins/**') }}-${{ github.sha }} + restore-keys: eslint-v1-${{ github.job }}-${{ hashFiles('ui/apps/platform/package-lock.json', 'ui/apps/platform/eslint.config.js', 'ui/apps/platform/eslint-plugins/**') }}- diff --git a/.github/actions/cache-go-dependencies/action.yaml b/.github/actions/cache-go-dependencies/action.yaml index 8592329c7a4f1..b4696c0ecd0b4 100644 --- a/.github/actions/cache-go-dependencies/action.yaml +++ b/.github/actions/cache-go-dependencies/action.yaml @@ -1,30 +1,151 @@ name: Cache Go Dependencies description: Cache Go Dependencies +inputs: + save: + description: Whether this job saves caches on pushes to the default branch. + required: false + default: 'true' + key-suffix: + description: Extra suffix for cache keys (use to distinguish matrix entries that share github.job). + required: false + default: '' runs: using: composite steps: - name: Determine Go cache paths id: cache-paths run: | - echo "GOCACHE=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" - echo "GOMODCACHE=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" - echo "GOARCH=$(go env GOARCH)" >> "$GITHUB_OUTPUT" + # Keep assignments separate from echo so Go failures are not masked. + if ! gocache=$(go env GOCACHE); then + echo "::error::Failed to determine GOCACHE: go env GOCACHE failed. See the Go error above." + exit 1 + fi + if [[ -z "$gocache" ]]; then + echo "::error::go env GOCACHE returned an empty value. Cannot configure the Go build cache." + exit 1 + fi + if ! goarch=$(go env GOARCH); then + echo "::error::Failed to determine GOARCH: go env GOARCH failed. See the Go error above." + exit 1 + fi + if [[ -z "$goarch" ]]; then + echo "::error::go env GOARCH returned an empty value. Cannot configure the Go build cache." + exit 1 + fi + echo "GOCACHE=$gocache" >> "$GITHUB_OUTPUT" + echo "GOARCH=$goarch" >> "$GITHUB_OUTPUT" + echo "TAG=${{ github.sha }}" >> "$GITHUB_OUTPUT" + # Hash go.mod once for use in cache keys. When dependencies change, + # the GOCACHE must be invalidated because Go's module index caches + # parsed imports by (filename, mtime, size) and our fixed mtimes + # can cause stale cached imports when file content changes. + echo "GOMOD_HASH=$(sha256sum go.mod | cut -c1-16)" >> "$GITHUB_OUTPUT" + suffix="${{ inputs.key-suffix }}" + echo "KEY_SUFFIX=${suffix:+-$suffix}" >> "$GITHUB_OUTPUT" # prepends dash if set shell: bash - - name: Cache Go Dependencies - uses: actions/cache@v5 + - name: Cache Go Build (save) + if: inputs.save == 'true' && (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # ratchet:actions/cache@v6.1.0 with: - path: | - ${{ steps.cache-paths.outputs.GOMODCACHE }} - key: go-mod-v1-${{ hashFiles('**/go.sum') }} + path: ${{ steps.cache-paths.outputs.GOCACHE }} + key: go-build-v2-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ steps.cache-paths.outputs.GOMOD_HASH }}-${{ steps.cache-paths.outputs.TAG }} + restore-keys: | + go-build-v2-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ steps.cache-paths.outputs.GOMOD_HASH }}- + go-build-v2-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}- - - name: Cache Go Build - uses: actions/cache@v5 + - name: Cache Go Build (restore) + if: ${{ !(inputs.save == 'true' && (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)) }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # ratchet:actions/cache/restore@v6.1.0 with: - path: | - ${{ steps.cache-paths.outputs.GOCACHE }} - key: go-build-v1-${{ github.job }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ hashFiles('**/go.sum') }} + path: ${{ steps.cache-paths.outputs.GOCACHE }} + key: go-build-v2-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ steps.cache-paths.outputs.GOMOD_HASH }}-${{ steps.cache-paths.outputs.TAG }} + restore-keys: | + go-build-v2-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ steps.cache-paths.outputs.GOMOD_HASH }}- + go-build-v2-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}- - - name: Download Go modules - run: make deps --always-make + - name: Mark GOCACHE entries for stale detection + run: | + gocache="${{ steps.cache-paths.outputs.GOCACHE }}" + if [[ -d "$gocache" ]]; then + # Backdate all cache entries to year 2000. During the build/test, + # Go's markUsed() updates mtimes of accessed entries to "now" + # (it always updates when mtime is >1 hour old). The post-step + # below trims entries still at year 2000 before cache save. + timeout 120 find "$gocache" -type f -exec touch -t 200001010000 {} + || true + # Protect trim.txt: if backdated, Go's built-in Trim() sees + # "last trim was in year 2000" and deletes ALL backdated entries + # before the build starts. Setting it to now prevents this. + echo "$(date +%s)" > "$gocache/trim.txt" + echo "Marked GOCACHE entries for stale detection" + fi + shell: bash + + # Registered AFTER cache steps so the post-step runs BEFORE cache save + # (GHA runs post-steps in reverse registration order). + - name: Trim stale GOCACHE entries (post-step) + uses: gacts/run-and-post-run@fc6503862aba998e2f2123225ad63c09a92ed46e # v1.4.0 + with: + post: > + set +e; + gocache="$(go env GOCACHE)"; + [[ -d "$gocache" ]] && [[ -f "$gocache/trim.txt" ]] || { echo "Skipping GOCACHE trim"; exit 0; }; + before=$(du -sm "$gocache" 2>/dev/null | cut -f1); + cutoff=$(mktemp); touch -t 200001020000 "$cutoff"; + find "$gocache" -type f ! -newer "$cutoff" -delete 2>/dev/null || true; + rm -f "$cutoff"; + find "$gocache" -mindepth 1 -type d -empty -delete 2>/dev/null || true; + after=$(du -sm "$gocache" 2>/dev/null | cut -f1); + echo "GOCACHE trimmed: ${before:-0}MB -> ${after:-0}MB (removed $((${before:-0} - ${after:-0}))MB)" + + - name: Clear test cache on non-PR runs + if: github.event_name != 'pull_request' + run: | + # Non-PR runs (master, release branches, nightlies) always run tests + # from scratch to catch real failures. go clean -testcache removes + # cached test results while preserving the compilation cache. + go clean -testcache + echo "Cleared test cache for fresh run" + shell: bash + + - name: Stabilize file mtimes for Go test cache and module index + run: | + # Go's test cache and module index both validate inputs by + # (path, size, mtime) -- NOT content. Whatever mtime git checkout + # gives a file (which varies between runs) can make either cache + # serve a stale result when content changes but size doesn't (e.g. + # an import path swapped for one of equal length -- the golang-jwt + # v4->v5 incident). Derive each regular file's mtime from its git + # blob hash instead: same content always hashes to the same mtime + # (cache hit), different content to a different one (correctly + # invalidates). Reads the git index (git ls-files -s), so no + # history walk is needed -- works on any checkout depth. + # + # Spreads the hash across one year of seconds from 2001-01-01 -- + # 31,536,000 distinct values. A stale hit needs BOTH an unchanged + # size (filename is fixed -- it's the same file -- already ruling + # out most edits) AND the new content's blob hash landing in the + # same 1-in-31.5M slot as the old one. No real same-size content + # change in this repo's history has ever landed in the same slot, + # including the motivating golang-jwt v4->v5 change itself (same + # length, different slot). + # + # Symlinks (mode 120000) and directories have no usable blob hash + # for this: a symlink's blob hash is its target path string, not + # file content, and utime() on a symlink path dereferences to the + # target anyway, so both get a fixed constant instead -- set + # first, so a symlink that happens to point at a tracked regular + # file doesn't clobber that file's content-derived mtime set + # afterward. + git ls-files -s | perl -lne '/^120000 \S+ \S+\t(.+)$/ and print $1' \ + | xargs -r touch -t 200101010000 + git ls-files | sed 's|/[^/]*$||' | sort -u | xargs touch -t 200101010000 + + git ls-files -s | perl -lne ' + /^100\S+ (\S+) \S+\t(.+)$/ or next; + my $v = 0; + $v = ($v * 16 + hex $_) % 31536000 for split //, $1; + my $ts = 978307200 + $v; # 2001-01-01T00:00:00Z + up to 1 year + utime $ts, $ts, $2; + ' shell: bash diff --git a/.github/actions/cache-gradle-dependencies/action.yaml b/.github/actions/cache-gradle-dependencies/action.yaml index d78810f048325..eeeb7fa5de331 100644 --- a/.github/actions/cache-gradle-dependencies/action.yaml +++ b/.github/actions/cache-gradle-dependencies/action.yaml @@ -4,7 +4,7 @@ runs: using: composite steps: - name: Cache Gradle Dependencies - uses: actions/cache@v5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # ratchet:actions/cache@v6.1.0 with: path: | ~/.gradle/caches diff --git a/.github/actions/cache-ui-dependencies/action.yaml b/.github/actions/cache-ui-dependencies/action.yaml index 908105a981e0b..134e5aa3c92b2 100644 --- a/.github/actions/cache-ui-dependencies/action.yaml +++ b/.github/actions/cache-ui-dependencies/action.yaml @@ -1,18 +1,27 @@ name: Cache UI Dependencies -description: Cache UI Dependencies +description: Cache node_modules and Cypress binary for UI jobs inputs: lockFile: description: Where the package lock is written - required: true + required: false default: "ui/apps/platform/package-lock.json" + working-directory: + description: Directory containing the UI app + required: false + default: "ui/apps/platform" +outputs: + cache-hit: + description: Whether the cache was restored (exact match) + value: ${{ steps.cache.outputs.cache-hit }} runs: using: composite steps: - name: Cache UI Dependencies - uses: actions/cache@v5 + id: cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # ratchet:actions/cache@v6.1.0 with: path: | - /github/home/.npm - /github/home/.cache/Cypress - /usr/local/share/.cache - key: npm-v2-${{ hashFiles(inputs.lockFile) }} + ${{ inputs.working-directory }}/node_modules + ~/.cache/Cypress + key: ui-deps-v1-${{ github.job }}-${{ hashFiles(inputs.lockFile) }} + restore-keys: ui-deps-v1-${{ github.job }}- diff --git a/.github/actions/download-artifact-with-retry/action.yaml b/.github/actions/download-artifact-with-retry/action.yaml index 7ffb4b3947c35..69d9d86ddeac6 100644 --- a/.github/actions/download-artifact-with-retry/action.yaml +++ b/.github/actions/download-artifact-with-retry/action.yaml @@ -14,7 +14,7 @@ runs: using: composite steps: - id: download-artifact-try1 - uses: actions/download-artifact@v7 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # ratchet:actions/download-artifact@v8 with: name: ${{ inputs.name }} path: ${{ inputs.path }} @@ -22,14 +22,14 @@ runs: continue-on-error: true - id: download-artifact-try2 if: steps.download-artifact-try1.outcome == 'failure' - uses: actions/download-artifact@v7 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # ratchet:actions/download-artifact@v8 with: name: ${{ inputs.name }} path: ${{ inputs.path }} pattern: ${{ inputs.pattern }} continue-on-error: true - if: steps.download-artifact-try2.outcome == 'failure' - uses: actions/download-artifact@v7 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # ratchet:actions/download-artifact@v8 with: name: ${{ inputs.name }} path: ${{ inputs.path }} diff --git a/.github/actions/job-preamble/action.yaml b/.github/actions/job-preamble/action.yaml index 099de277da465..8bb2eb47b1154 100644 --- a/.github/actions/job-preamble/action.yaml +++ b/.github/actions/job-preamble/action.yaml @@ -16,7 +16,7 @@ runs: # Skip for external contributions. if: | ( github.event_name == 'push' || !github.event.pull_request.head.repo.fork ) && inputs.gcp-account != '' - uses: 'google-github-actions/auth@v3' + uses: 'google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093' # ratchet:google-github-actions/auth@v3 with: credentials_json: '${{ inputs.gcp-account }}' export_environment_variables: true @@ -32,7 +32,7 @@ runs: # setup-gcloud authenticates gcloud - name: Install gcloud CLI - uses: 'google-github-actions/setup-gcloud@v2' + uses: 'google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db' # ratchet:google-github-actions/setup-gcloud@v3.0.1 with: skip_install: ${{ steps.gcloud-check.outcome != 'failure' }} @@ -139,8 +139,25 @@ runs: git config --global --add safe.directory "$(pwd)" shell: bash + - name: Capture runner fingerprint + id: runner-info + continue-on-error: true + shell: bash + run: | + set +e + cpu_model=$(grep -m1 'model name' /proc/cpuinfo 2>/dev/null | cut -d: -f2 | xargs) + if [[ -z "$cpu_model" ]]; then + cpu_model=$(lscpu 2>/dev/null | awk -F: '/Model name/{gsub(/^ +/,"",$2); print $2}') + fi + echo "cpu_model=${cpu_model:-$(uname -m)}" >> "$GITHUB_OUTPUT" + echo "runner_image=${ImageOS:+${ImageOS}-${ImageVersion:-}}" >> "$GITHUB_OUTPUT" + mem_gb=$(awk '/MemTotal/{printf "%.0f", $2/1048576}' /proc/meminfo 2>/dev/null) + disk_gb=$(df -BG --output=size / 2>/dev/null | tail -1 | tr -d ' G') + echo "runner_resources=${mem_gb:-0}GB ram / ${disk_gb:-0}GB disk" >> "$GITHUB_OUTPUT" + echo "container_runtime=$([[ -f /.dockerenv ]] && echo container || echo None)" >> "$GITHUB_OUTPUT" + - name: Record job info - uses: gacts/run-and-post-run@d803f6920adc9a47eeac4cb6c93dbc2e2890c684 # v1.4.0 + uses: gacts/run-and-post-run@fc6503862aba998e2f2123225ad63c09a92ed46e # v1.4.0 id: record_job_info env: GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ inputs.gcp-account }} @@ -155,7 +172,11 @@ runs: source scripts/ci/lib.sh || true; save_job_record "${{ github.job }}" "gha" \ outcome "${{ job.status }}" \ - started_at "${{ steps.record_job_info.outputs.started_at }}" || true; + started_at "${{ steps.record_job_info.outputs.started_at }}" \ + cut_os_image "${{ steps.runner-info.outputs.runner_image }}" \ + build "${{ steps.runner-info.outputs.cpu_model }}" \ + test_target "${{ steps.runner-info.outputs.runner_resources }}" \ + cut_container_runtime_version "${{ steps.runner-info.outputs.container_runtime }}" || true; free=$(df -BGB --output=avail / | tail -1); echo "Free disk space before: ${{ steps.disk-check-cleaned.outputs.free }} [${{ steps.disk-check.outputs.free }} before cleanup], and after: ${free}."; if [[ ${free%GB} -lt 5 ]]; then\ diff --git a/.github/actions/junit2jira/action.yaml b/.github/actions/junit2jira/action.yaml index 96b39cbcd6dcb..2de93401cf986 100644 --- a/.github/actions/junit2jira/action.yaml +++ b/.github/actions/junit2jira/action.yaml @@ -11,6 +11,9 @@ inputs: gcp-account: description: 'Account to be used to upload tests data' required: true + jira-user: + description: 'User used to authenticate with Jira.' + required: true jira-token: description: 'Token used to authenticate with Jira.' required: true @@ -31,7 +34,7 @@ runs: - name: Download junit2jira shell: bash env: - VERSION: "v0.0.25" + VERSION: "v0.0.27" run: | set -u LOCATION="https://github.com/stackrox/junit2jira/releases/download/$VERSION/junit2jira" @@ -40,11 +43,28 @@ runs: curl --retry 5 --retry-connrefused --silent --show-error --fail --location --output junit2jira "$LOCATION" chmod +x junit2jira fi + - name: Capture job failure as JUnit if no test failures exist + shell: bash + if: always() + env: + STEPS_JSON: ${{ toJSON(steps) }} + run: | + set -u + source scripts/ci/lib.sh + export ARTIFACT_DIR="${{ inputs.directory }}" + + capture_job_failure_as_junit \ + "${{ inputs.directory }}" \ + "${{ github.job }}" \ + "${{ job.status }}" \ + "$STEPS_JSON" \ + "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - name: Check files id: run shell: bash env: GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ inputs.gcp-account }} + JIRA_USER: ${{ inputs.jira-user }} JIRA_TOKEN: ${{ inputs.jira-token }} if: "${{ env.GCP_SERVICE_ACCOUNT_STACKROX_CI != '' || env.JIRA_TOKEN != '' }}" run: | @@ -62,6 +82,7 @@ runs: -build-link "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ -build-tag "${{ github.ref_name }}@${{ github.sha }}" \ -csv-output "${csv_output}" \ + -jira-url "https://redhat.atlassian.net/" \ -job-name "${{ github.job }}" \ -junit-reports-dir "${{ inputs.directory }}" \ -orchestrator "${{ runner.name }} ${{ runner.os }}-${{ runner.arch }}" \ diff --git a/.github/actions/notify-slack-on-failure/action.yaml b/.github/actions/notify-slack-on-failure/action.yaml new file mode 100644 index 0000000000000..e5c014585b920 --- /dev/null +++ b/.github/actions/notify-slack-on-failure/action.yaml @@ -0,0 +1,61 @@ +name: Notify Slack on failure +description: > + Posts a Slack failure notification for the current workflow run. By + default, collects ::error:: annotations and per-job status via the GitHub + API and builds a message from that data; pass `message` to post a fixed, pre-built + message instead (no GitHub API calls). Requires `permissions: {checks: + read, actions: read}` on the calling job unless `message` is set. +inputs: + slack-webhook: + description: "Slack incoming webhook URL to post to." + required: true + github-token: + description: "Token used to query the Actions/Checks API. Not needed when `message` is set." + required: false + default: ${{ github.token }} + mention: + description: "Optional Slack mention/subteam tag prepended to the message (e.g. '')." + required: false + default: "" + context: + description: "Optional extra line of context inserted near the top of the message (e.g. 'Source: alpine'). Ignored when `message` is set." + required: false + default: "" + job-name-filter: + description: > + Optional substring used to narrow annotation/job-status collection to + jobs whose name contains it. Use this when the calling job is one of + several matrix legs sharing the same workflow run, so the message only + covers this leg. Ignored when `message` is set. + required: false + default: "" + include-successful-jobs: + description: > + If "true", also inspect jobs that did not fail: any job with real + ::error:: annotations is included even if it concluded successfully. + Jobs with no annotations are omitted either way. Use this when the + caller wants to alert on a job that intentionally does not fail + despite reporting real errors. Ignored when `message` is set. + required: false + default: "" + message: + description: "If set, this literal text is posted instead of collecting annotations/job status." + required: false + default: "" +runs: + using: composite + steps: + - shell: bash + run: "${{ github.action_path }}/notify-slack-on-failure.sh" + env: + SLACK_WEBHOOK: ${{ inputs.slack-webhook }} + GH_TOKEN: ${{ inputs.github-token }} + MENTION: ${{ inputs.mention }} + CONTEXT: ${{ inputs.context }} + JOB_NAME_FILTER: ${{ inputs.job-name-filter }} + INCLUDE_SUCCESSFUL_JOBS: ${{ inputs.include-successful-jobs }} + MESSAGE: ${{ inputs.message }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + WORKFLOW: ${{ github.workflow }} diff --git a/.github/actions/notify-slack-on-failure/notify-slack-on-failure.sh b/.github/actions/notify-slack-on-failure/notify-slack-on-failure.sh new file mode 100755 index 0000000000000..27ed01e711099 --- /dev/null +++ b/.github/actions/notify-slack-on-failure/notify-slack-on-failure.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Posts a Slack failure notification to $SLACK_WEBHOOK. +# +# Two modes: +# - $MESSAGE is set: post that text verbatim (prefixed by $MENTION), no +# GitHub API calls are made. Use this when the caller already has a +# specific, curated message (e.g. a list of failed sources). +# - $MESSAGE is unset: collect ::error:: annotations and per-job status for +# the current workflow run via the GitHub API (optionally narrowed to +# jobs whose name contains $JOB_NAME_FILTER, useful when this script runs +# once per matrix leg of a reusable workflow), and build a message from that data. +# If collection itself fails, a degraded but still-informative message is +# sent instead of nothing at all. +# +# Required env vars: SLACK_WEBHOOK, RUN_URL, WORKFLOW, REPO +# Used only when $MESSAGE is unset: GH_TOKEN, RUN_ID +# Optional: MENTION, CONTEXT, MESSAGE, JOB_NAME_FILTER, INCLUDE_SUCCESSFUL_JOBS + +set -euo pipefail + +: "${SLACK_WEBHOOK:?SLACK_WEBHOOK is required}" +: "${RUN_URL:?RUN_URL is required}" +: "${WORKFLOW:?WORKFLOW is required}" +: "${REPO:?REPO is required}" + +MENTION="${MENTION:-}" +CONTEXT="${CONTEXT:-}" +MESSAGE="${MESSAGE:-}" +JOB_NAME_FILTER="${JOB_NAME_FILTER:-}" +INCLUDE_SUCCESSFUL_JOBS="${INCLUDE_SUCCESSFUL_JOBS:-}" + +max_messages_per_job=10 + +# collect_annotations sets the global SUMMARY and JOB_STATUS variables from +# the current workflow run's jobs/annotations, optionally narrowed to job +# names containing $JOB_NAME_FILTER. Returns non-zero on any GitHub API +# failure. +collect_annotations() { + : "${GH_TOKEN:?GH_TOKEN is required to collect annotations}" + : "${RUN_ID:?RUN_ID is required to collect annotations}" + + # --paginate output is multiple JSON docs back-to-back; jq below handles + # that as a stream, no --slurp needed. + local jobs_json + if ! jobs_json=$(gh api "repos/${REPO}/actions/runs/${RUN_ID}/jobs" --paginate); then + return 1 + fi + + if [ -n "$JOB_NAME_FILTER" ]; then + jobs_json=$(printf '%s' "$jobs_json" \ + | jq --arg filter "$JOB_NAME_FILTER" '{ jobs: [.jobs[] | select(.name | contains($filter))] }') + fi + + # Jobs to inspect for annotations: only failure-like ones by default; every + # job (success included) when INCLUDE_SUCCESSFUL_JOBS is set. Either way, + # a successful job only ends up in SUMMARY below if it actually has real + # annotations (see the skip check in the loop). + local jobs_filter='.jobs[]' + if [ "$INCLUDE_SUCCESSFUL_JOBS" != "true" ]; then + jobs_filter='.jobs[] | select(.conclusion | IN("failure","timed_out","action_required","neutral","stale","startup_failure"))' + fi + + # Plain text: rendered inside a Slack code block, not mrkdwn. + SUMMARY="" + while IFS=$'\t' read -r job_id job_name job_conclusion failed_step; do + # Exclude GitHub's generic "Process completed..." annotation (noise). + # --paginate: a job can have >30 annotations (API's page size). + local messages + if ! messages=$(gh api "repos/${REPO}/check-runs/${job_id}/annotations" --paginate \ + --jq '.[] | select(.annotation_level == "failure") | select(.message | test("^Process completed with exit code") | not) | .message'); then + # Distinct fallback so an API failure isn't confused with "no annotations". + messages="(failed to fetch annotations for this job via the GitHub API; see the workflow run directly for details)" + fi + + local is_failure_like=false + case "$job_conclusion" in + failure | timed_out | action_required | neutral | stale | startup_failure) is_failure_like=true ;; + esac + + # A successful job with nothing to report contributes nothing (only + # relevant when INCLUDE_SUCCESSFUL_JOBS pulled it in above). A job + # that actually failed always shows up, even with no annotations. + if [ -z "$messages" ] && [ "$is_failure_like" = false ]; then + continue + fi + + # Blank line between job blocks, but not before the first one. + if [ -n "$SUMMARY" ]; then + SUMMARY+=$'\n\n' + fi + SUMMARY+="${job_name}: ${failed_step}"$'\n' + if [ -n "$messages" ]; then + local total + total=$(printf '%s\n' "$messages" | grep -c . || true) + SUMMARY+="$(printf '%s\n' "$messages" | sed -n "1,${max_messages_per_job}s/^/ - /p")" + if [ "$total" -gt "$max_messages_per_job" ]; then + SUMMARY+=$'\n'" - ... and $((total - max_messages_per_job)) more (see the workflow run for full details)" + fi + else + SUMMARY+=" - (no detailed error annotations found; see the workflow run for details)" + fi + done < <(printf '%s' "$jobs_json" | jq -r "$jobs_filter"' | [.id, .name, (.conclusion // "in_progress"), ([.steps[]? | select(.conclusion | IN("failure","timed_out","action_required","neutral","stale","startup_failure")) | .name][0] // "unknown step")] | @tsv') + + if [ -z "$SUMMARY" ]; then + SUMMARY="(no failed jobs found; see the workflow run for details)" + fi + + # Safety-net truncation, on top of the per-job cap above. + SUMMARY="${SUMMARY:0:3000}" + + # Status of every (filtered) job, icons matching GitHub's own. + JOB_STATUS="" + while IFS=$'\t' read -r job_name job_conclusion; do + local icon + case "$job_conclusion" in + success) icon="✅" ;; + skipped) icon="⏭️" ;; + cancelled) icon="🚫" ;; + in_progress) icon="🟡" ;; + failure | timed_out | action_required | neutral | stale | startup_failure) icon="❌" ;; + *) icon="❔" ;; # genuinely unrecognized future conclusion value + esac + JOB_STATUS+="${icon} ${job_name}"$'\n' + done < <(printf '%s' "$jobs_json" | jq -r '.jobs[] | [.name, (.conclusion // "in_progress")] | @tsv') + + JOB_STATUS="${JOB_STATUS:0:3000}" +} + +context_line="" +if [ -n "$CONTEXT" ]; then + context_line="*Context:* ${CONTEXT}"$'\n' +fi + +if [ -n "$MESSAGE" ]; then + text="$MESSAGE" +elif collect_annotations; then + # shellcheck disable=SC2016 # backticks below are a Slack code-block + # delimiter, not shell command substitution. + text=$(printf '*Failed Workflow:* <%s|%s>\n*Repository:* %s\n%s*Job Status:*\n```\n%s```\n*Failed steps:*\n```\n%s\n```' \ + "$RUN_URL" "$WORKFLOW" "$REPO" "$context_line" "$JOB_STATUS" "$SUMMARY") +else + # Runs even if annotation collection failed, so a degraded message still + # reaches on-call instead of nothing at all. + text=$(printf ':warning: *Failed Workflow:* <%s|%s>\n*Repository:* %s\n%sCould not collect failure annotations for this run (the collection step itself failed) -- please check the workflow run directly for details.' \ + "$RUN_URL" "$WORKFLOW" "$REPO" "$context_line") +fi + +if [ -n "$MENTION" ]; then + text="${MENTION} ${text}" +fi + +# --retry-all-errors means a slow-but-eventually-successful post could be +# retried and sent twice; unlikely in practice, but possible. +jq -n --arg text "$text" '{text: $text}' \ + | curl --fail --silent --show-error \ + --retry 3 --retry-all-errors --retry-delay 5 \ + --connect-timeout 10 --max-time 300 \ + -X POST -H 'Content-type: application/json' --data @- "$SLACK_WEBHOOK" diff --git a/.github/actions/upload-artifact-with-retry/action.yaml b/.github/actions/upload-artifact-with-retry/action.yaml index 31b9e0c295ee2..9d68c29d65e11 100644 --- a/.github/actions/upload-artifact-with-retry/action.yaml +++ b/.github/actions/upload-artifact-with-retry/action.yaml @@ -16,27 +16,34 @@ inputs: error: Fail the action with an error message ignore: Do not output any warnings or errors, the action does not fail default: 'warn' + compression-level: + description: 'Compression level: 0 (none) to 9 (max). Default is 6 (zstd default)' + required: false + default: '6' runs: using: composite steps: - id: upload-artifact-try1 - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # ratchet:actions/upload-artifact@v7 with: name: ${{ inputs.name }} path: ${{ inputs.path }} if-no-files-found: ${{ inputs.if-no-files-found }} + compression-level: ${{ inputs.compression-level }} continue-on-error: true - id: upload-artifact-try2 if: steps.upload-artifact-try1.outcome == 'failure' - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # ratchet:actions/upload-artifact@v7 with: name: ${{ inputs.name }} path: ${{ inputs.path }} if-no-files-found: ${{ inputs.if-no-files-found }} + compression-level: ${{ inputs.compression-level }} continue-on-error: true - if: steps.upload-artifact-try2.outcome == 'failure' - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # ratchet:actions/upload-artifact@v7 with: name: ${{ inputs.name }} path: ${{ inputs.path }} if-no-files-found: ${{ inputs.if-no-files-found }} + compression-level: ${{ inputs.compression-level }} diff --git a/.github/contributing.md b/.github/contributing.md index 9c6a4dd770731..40d1fd617fd9f 100644 --- a/.github/contributing.md +++ b/.github/contributing.md @@ -97,7 +97,8 @@ draft PR to a regular PR, you can revert it to a draft PR by clicking "Convert to draft" under the list of reviewers in the right sidebar of the "Conversation" view. -- Use the `auto-retest` label if you want CI to re-run E2E test on failures. +- Comment `/retest` on a PR to rerun all failed CI jobs (both Prow and GitHub Actions). +- Use the `auto-retest` label if you want CI to automatically issue `/retest` commands on failures. If you want to retest given job no matter of it result (e.g. validate for resolving flake) use `/retest-times ` in addition to the `auto-retest` label. Note that these options only work if GitHub checks are passing. To debug please look at diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 7c8f2504f6a93..12679b4870145 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -114,7 +114,21 @@ updates: prefix: chore - package-ecosystem: 'gomod' - directory: '/tools/retest/' + directory: '/tools/check-workflow-run' + schedule: + interval: 'weekly' + day: 'wednesday' + open-pull-requests-limit: 3 + labels: + - "dependencies" + - "auto-merge" + - "auto-retest" + commit-message: + include: scope + prefix: chore + + - package-ecosystem: 'gomod' + directory: '/tools/build/' schedule: interval: 'weekly' day: 'wednesday' @@ -157,6 +171,21 @@ updates: include: scope prefix: chore + - package-ecosystem: 'gomod' + directory: '/operator/tools/kubebuilder/' + schedule: + interval: 'weekly' + day: 'wednesday' + open-pull-requests-limit: 3 + labels: + - "dependencies" + - "area/operator" + - "auto-merge" + - "auto-retest" + commit-message: + include: scope + prefix: chore + - package-ecosystem: 'gomod' directory: '/operator/tools/kustomize/' schedule: @@ -187,6 +216,21 @@ updates: include: scope prefix: chore + - package-ecosystem: 'gomod' + directory: '/operator/tools/crdify/' + schedule: + interval: 'weekly' + day: 'wednesday' + open-pull-requests-limit: 3 + labels: + - "dependencies" + - "area/operator" + - "auto-merge" + - "auto-retest" + commit-message: + include: scope + prefix: chore + - package-ecosystem: 'gomod' directory: '/operator/tools/operator-sdk/' schedule: @@ -222,7 +266,7 @@ updates: prefix: chore - package-ecosystem: 'gomod' - directory: '/scanner/tools/' + directory: '/scanner/hack/quay' schedule: interval: 'weekly' day: 'wednesday' @@ -236,8 +280,23 @@ updates: include: scope prefix: chore + - package-ecosystem: 'gomod' + directory: '/tests/performance/scale' + schedule: + interval: 'weekly' + day: 'wednesday' + open-pull-requests-limit: 3 + labels: + - "dependencies" + - "auto-merge" + - "auto-retest" + commit-message: + include: scope + prefix: chore + - package-ecosystem: 'docker' directory: 'operator/' + exclude-paths: [ '**/konflux*.Dockerfile' ] schedule: interval: 'weekly' day: 'wednesday' @@ -250,16 +309,22 @@ updates: commit-message: include: scope prefix: chore + ignore: + # ubi9/go-toolset has both Go-based (1.x) and RHEL-based (9.x) tags. + # Only track Go version tags. + - dependency-name: "ubi9/go-toolset" + versions: [">= 2"] - package-ecosystem: 'docker' directory: 'operator/tests/controller/metrics' + exclude-paths: [ '**/konflux*.Dockerfile' ] schedule: interval: 'monthly' open-pull-requests-limit: 3 labels: - "dependencies" - "area/operator" - - "auto-merge-any" # these images do not follow semver + - "auto-merge" - "auto-retest" commit-message: include: scope @@ -267,6 +332,7 @@ updates: - package-ecosystem: 'docker' directory: 'image/rhel' + exclude-paths: [ '**/konflux*.Dockerfile' ] schedule: interval: 'weekly' day: 'wednesday' @@ -280,6 +346,7 @@ updates: - package-ecosystem: 'docker' directory: 'image/postgres' + exclude-paths: [ '**/konflux*.Dockerfile' ] schedule: interval: 'weekly' day: 'wednesday' @@ -293,6 +360,7 @@ updates: - package-ecosystem: 'docker' directory: 'scanner/image/scanner' + exclude-paths: [ '**/konflux*.Dockerfile' ] schedule: interval: 'weekly' day: 'wednesday' @@ -306,6 +374,7 @@ updates: - package-ecosystem: 'docker' directory: 'scanner/image/db' + exclude-paths: [ '**/konflux*.Dockerfile' ] schedule: interval: 'weekly' day: 'wednesday' @@ -317,6 +386,19 @@ updates: include: scope prefix: chore + - package-ecosystem: 'docker' + directory: '.github/test-images' + schedule: + interval: 'weekly' + day: 'wednesday' + open-pull-requests-limit: 1 + labels: + - "dependencies" + - "auto-retest" + commit-message: + include: scope + prefix: chore + - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/docker/images.hcl b/.github/docker/images.hcl new file mode 100644 index 0000000000000..b4b6d47ac7760 --- /dev/null +++ b/.github/docker/images.hcl @@ -0,0 +1,62 @@ +# Docker Buildx bake file for container images +# Builds: central-db, scanner-v4-db +# +# Usage: +# docker buildx bake -f .github/docker/images.hcl --print databases +# docker buildx bake -f .github/docker/images.hcl --push databases +# +# Final cache verification + +variable "TAG" { + default = "latest" +} + +variable "PLATFORMS" { + default = "linux/amd64,linux/arm64" +} + +variable "REGISTRY" { + default = "quay.io/stackrox-io" +} + +variable "QUAY_TAG_EXPIRATION" { + default = "13w" +} + +# Group to build database images +group "databases" { + targets = ["central-db", "scanner-v4-db"] +} + +# Central database (PostgreSQL 16) +target "central-db" { + context = "image/postgres" + dockerfile = "Dockerfile" + platforms = split(",", PLATFORMS) + tags = [ + "${REGISTRY}/central-db:${TAG}" + ] + args = { + MAIN_IMAGE_TAG = TAG + QUAY_TAG_EXPIRATION = QUAY_TAG_EXPIRATION + } + cache-from = ["type=gha,scope=central-db"] + cache-to = ["type=gha,mode=max,scope=central-db"] +} + +# Scanner V4 database (PostgreSQL 15) +target "scanner-v4-db" { + context = "scanner/image/db" + dockerfile = "Dockerfile" + platforms = split(",", PLATFORMS) + tags = [ + "${REGISTRY}/scanner-v4-db:${TAG}" + ] + args = { + LABEL_VERSION = TAG + LABEL_RELEASE = TAG + QUAY_TAG_EXPIRATION = QUAY_TAG_EXPIRATION + } + cache-from = ["type=gha,scope=scanner-v4-db"] + cache-to = ["type=gha,mode=max,scope=scanner-v4-db"] +} diff --git a/.github/labeler.yml b/.github/labeler.yml index 4451b601031db..7a345d52c9ba6 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -60,17 +60,20 @@ ci-all-qa-tests: - changed-files: - any-glob-to-any-file: - COLLECTOR_VERSION - - SCANNER_VERSION + - FACT_VERSION backport: - base-branch: 'release-*' konflux-build: +# This label triggers Konflux pipelines. +# The list here should be kept in sync with the CODEOWNERS section for Konflux related files. - changed-files: - any-glob-to-any-file: - '**/konflux*.Dockerfile' - .tekton/**/* - .konflux/**/* + - rpms.* ai-review: - changed-files: @@ -82,3 +85,14 @@ ai-review: - roxctl/**/* - scripts/ci/**/ - sensor/**/* + +coderabbit-review: +- changed-files: + - any-glob-to-any-file: + - .github/**/* + - .openshift-ci/**/* + - pkg/auth/**/* + - pkg/sac/**/* + - roxctl/**/* + - scripts/ci/**/ + - sensor/**/* diff --git a/.github/properties b/.github/properties index 455075aa33e7b..c5ad5ec6aed7f 100644 --- a/.github/properties +++ b/.github/properties @@ -2,4 +2,3 @@ docs-repository=openshift/openshift-docs slack-channel=C05AZF8T7GW dry-slack-channel=C03KSV3N6N8 jira-project=ROX -jira-projects=ROX, RS, RTOOLS diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 9985276aa8626..f942d95eab682 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -17,6 +17,8 @@ // This tells Renovate to combine all updates in one PR so that we have fewer PRs to deal with. "group:all", ], + // Don't apply dependency cooldown because via MintMaker we take only Red Hat dependencies. + "minimumReleaseAge": null, // The number of PRs that can be open against the repo. "prConcurrentLimit": 20, // `null` means the branch limit should be the same as PR limit. @@ -28,15 +30,18 @@ // Allowed syntax: https://docs.renovatebot.com/configuration-options/#schedule // The time was selected (with the help of https://time.fyi/timezones) so that Renovate isn't active during business // hours from Germany to US West Coast. This way, after we merge a PR, a new one does not pop up immediately after - // that. - "after 3am and before 7am", + // that. Another reason for this schedule is to include at least one 4 hour interval from the moment of scheduled + // MintMaker execution + // Between 0a.m. and 7:59a.m. every day. + "* 0-7 * * *", ], // Tell Renovate not to update PRs when outside of schedule. "updateNotScheduled": false, "tekton": { "schedule": [ // Override Konflux custom schedule for this manager to our intended one. - "after 3am and before 7am", + // Between 0a.m. and 7:59a.m. every day. + "* 0-7 * * *", ], "packageRules": [ // Note: the packageRules from the Konflux config (find URL in comments above) get merged with these. @@ -45,6 +50,8 @@ "matchPackageNames": [ "/^quay.io/rhacs-eng/konflux-tasks/", ], + "recreateWhen": "always", + "rebaseWhen": "behind-base-branch", }, ], }, @@ -56,20 +63,16 @@ ], "schedule": [ // Override Konflux custom schedule for this manager to our intended one. - "after 3am and before 7am", + // Between 0a.m. and 7:59a.m. every day. + "* 0-7 * * *", ], - "postUpgradeTasks": { - "commands": [ - // Refresh the rpm lockfile after updating image references in the dockerfile. - "rpm-lockfile-prototype rpms.in.yaml", - ], - }, }, "rpm-lockfile": { "schedule": [ // Override Konflux custom schedule for this manager to our intended one. // Note that MintMaker will create security updates outside of schedule. - "after 3am and before 7am", + // Between 0a.m. and 7:59a.m. every day. + "* 0-7 * * *", ], }, "enabledManagers": [ @@ -78,4 +81,8 @@ "dockerfile", "rpm-lockfile", ], + "labels": [ + // mintmaker-auto-merge label triggers the corresponding workflow. + "mintmaker-auto-merge", + ], } diff --git a/.github/test-images/postgres-ci.conf b/.github/test-images/postgres-ci.conf new file mode 100644 index 0000000000000..737d8f561ff81 --- /dev/null +++ b/.github/test-images/postgres-ci.conf @@ -0,0 +1,11 @@ +# Shared PostgreSQL settings for CI unit tests and benchmarks. +# Mounted into sclorg containers via /opt/app-root/src/postgresql-cfg/ and copied +# into /etc/postgresql/*/main/conf.d/ for the runner's pre-installed Postgres. +# tmpfs has limited space; minimize WAL footprint since durability is irrelevant. +listen_addresses = 'localhost' +fsync = off +full_page_writes = off +wal_level = minimal +max_wal_senders = 0 +max_wal_size = 64MB +synchronous_commit = off diff --git a/.github/test-images/postgres.dockerfile b/.github/test-images/postgres.dockerfile new file mode 100644 index 0000000000000..bb498f0494622 --- /dev/null +++ b/.github/test-images/postgres.dockerfile @@ -0,0 +1,3 @@ +# Postgres 16 image for unit test CI. Referenced by unit-tests.yaml. +# Dependabot auto-bumps this digest via the docker ecosystem config. +FROM quay.io/sclorg/postgresql-16-c9s@sha256:d33717fd33c1a4b11afef94568d0c20812a19d62fb8c633fadee1e491e2edd4a diff --git a/.github/workflows/add-new-pr-to-oss-triaging.yml b/.github/workflows/add-new-pr-to-oss-triaging.yml index dfb5b52a95100..9d7a10996440f 100644 --- a/.github/workflows/add-new-pr-to-oss-triaging.yml +++ b/.github/workflows/add-new-pr-to-oss-triaging.yml @@ -9,7 +9,7 @@ env: jobs: check-pr-if-external: name: Add external label to pull request if outside StackRox - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: GH_TOKEN: ${{ github.token }} BASE_REPO: ${{ github.repository }} diff --git a/.github/workflows/add-vms-to-cluster.yml b/.github/workflows/add-vms-to-cluster.yml new file mode 100644 index 0000000000000..0ace3f5f384e7 --- /dev/null +++ b/.github/workflows/add-vms-to-cluster.yml @@ -0,0 +1,52 @@ +name: "Ensure VMs on Cluster" +run-name: "Ensure ${{ inputs.num-vms }} ${{ inputs.vm-os }} VMs on ${{ inputs.cluster-name }}" + +on: + workflow_dispatch: + inputs: + cluster-name: + description: "Infra cluster name (from `infractl list`)" + required: true + type: string + num-vms: + description: "Desired count of {os}-1..N, not how many to add. Existing VMs are reused; 2 is a no-op if rhel9-1 and rhel9-2 already exist." + required: false + default: 1 + type: number + vm-os: + description: "VM OS" + required: false + default: "rhel9" + type: choice + options: + - rhel8 + - rhel9 + - rhel10 + ssh-public-key: + description: "Optional SSH public key for human access to VMs" + required: false + default: "" + type: string + +jobs: + add-vms: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + persist-credentials: false + + - name: Add VMs to cluster + uses: ./scripts/ci/add-vms + with: + cluster-name: ${{ inputs.cluster-name }} + num-vms: ${{ inputs.num-vms }} + vm-os: ${{ inputs.vm-os }} + ssh-public-key: ${{ inputs.ssh-public-key }} + infra-token: ${{ secrets.INFRA_TOKEN }} + quay-username: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + quay-password: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} diff --git a/.github/workflows/audit-backport-prs.yml b/.github/workflows/audit-backport-prs.yml new file mode 100644 index 0000000000000..b220cb7c4e725 --- /dev/null +++ b/.github/workflows/audit-backport-prs.yml @@ -0,0 +1,118 @@ +name: Audit Backport PRs + +on: + pull_request: + paths: + - '.github/workflows/audit-backport-prs.yml' + - 'scripts/ci/backport-audit.py' + - 'scripts/ci/backport_audit/**' + schedule: + # Run twice a month: 1st and 15th at 9:00 AM UTC + - cron: '0 9 1,15 * *' + workflow_dispatch: + inputs: + release_branches: + description: 'Release branches (comma-separated, or "all" for auto-detect)' + required: false + default: 'all' + dry_run: + description: 'Dry run (post to test Slack channel)' + required: false + default: true + type: boolean + +env: + ACCEPT_RAW: "Accept: application/vnd.github.v3.raw" + GH_TOKEN: ${{ github.token }} + GH_NO_UPDATE_NOTIFIER: 1 + +jobs: + properties: + name: Read repository properties + runs-on: ubuntu-24.04 + outputs: + slack-channel: ${{ fromJSON(format('["{0}","{1}"]', steps.properties.outputs.slack-channel, steps.properties.outputs.dry-slack-channel))[(github.event_name == 'workflow_dispatch' && inputs.dry_run) || github.event_name == 'pull_request'] }} + steps: + - name: Read workflow properties file + id: properties + env: + PROPERTIES_URL: /repos/${{ github.repository }}/contents/.github/properties?ref=${{ github.event.repository.default_branch }} + run: gh api -H "$ACCEPT_RAW" "$PROPERTIES_URL" >> "$GITHUB_OUTPUT" + + audit: + runs-on: ubuntu-24.04 + needs: [properties] + permissions: + contents: read + pull-requests: read + issues: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 # Need full history for git tag operations + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # ratchet:actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Run backport audit + env: + JIRA_USER: ${{ vars.RHACS_BOT_GITHUB_EMAIL }} + JIRA_TOKEN: ${{ secrets.JIRA_TOKEN }} + GITHUB_TOKEN: ${{ github.token }} + RELEASE_BRANCHES: ${{ inputs.release_branches || 'all' }} + run: | + python3 scripts/ci/backport-audit.py \ + --branches "$RELEASE_BRANCHES" \ + --output-dir . \ + --github-run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + - name: Show report in summary + if: always() + run: | + if [ -f backport-audit-report.md ]; then + cat backport-audit-report.md >> "$GITHUB_STEP_SUMMARY" + else + echo "⚠️ Report file not generated" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # ratchet:actions/upload-artifact@v7.0.1 + if: always() + with: + name: backport-audit-report + path: | + backport-audit-report.md + slack-payload.json + + - name: Post report to Slack + if: success() + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + SLACK_CHANNEL: ${{ needs.properties.outputs.slack-channel }} + run: | + if [ -f slack-payload.json ]; then + # Update channel in payload if needed + if [ -n "$SLACK_CHANNEL" ]; then + jq --arg channel "$SLACK_CHANNEL" '.channel = $channel' slack-payload.json > slack-payload-updated.json + mv slack-payload-updated.json slack-payload.json + fi + + # Post to Slack and capture response + response=$(curl -X POST https://slack.com/api/chat.postMessage \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H "Content-Type: application/json" \ + -d @slack-payload.json) + + # Check if Slack API returned success + if ! echo "$response" | jq -e '.ok == true' > /dev/null; then + echo "❌ Slack API error:" + echo "$response" + exit 1 + fi + + echo "✅ Successfully posted to Slack" + else + echo "⚠️ Slack payload file not found, skipping Slack post" + fi diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 9e2033cd97412..1a18592aebca3 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -1,30 +1,29 @@ name: auto-merge on: - pull_request_target: - types: - - labeled + schedule: + - cron: '*/15 * * * *' # Every 15 minutes + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: auto-merge + cancel-in-progress: false jobs: - auto-merge-minor: - name: Auto-merge semver minor and patch bumps - runs-on: ubuntu-latest - if: github.actor == 'dependabot[bot]' && github.event.label.name == 'auto-merge' - steps: - - uses: ahmadnassri/action-dependabot-auto-merge@v2.6 - with: - github-token: '${{ secrets.RHACS_BOT_GITHUB_TOKEN }}' - command: "squash and merge" - approve: true - target: minor + auto-merge: + name: Enable auto-merge for eligible PRs + runs-on: ubuntu-24.04 + if: github.repository_owner == 'stackrox' - auto-merge-any: - name: Auto-merge any version bumps - runs-on: ubuntu-latest - if: github.actor == 'dependabot[bot]' && github.event.label.name == 'auto-merge-any' steps: - - uses: ahmadnassri/action-dependabot-auto-merge@v2.6 - with: - github-token: '${{ secrets.RHACS_BOT_GITHUB_TOKEN }}' - command: "squash and merge" - approve: true + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + + - name: Run auto-merge action + uses: stackrox/actions/automerge@8fcfacbfc4629361cca5573aba147be2155ba7e9 + with: + github-token: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/backport-pr.yaml b/.github/workflows/backport-pr.yaml index fd4f91621decb..dc54e31818fce 100644 --- a/.github/workflows/backport-pr.yaml +++ b/.github/workflows/backport-pr.yaml @@ -1,14 +1,22 @@ name: Backport PR to release branch on: - pull_request_target: + # From security considerations, it will not be possible for PRs from forks to be backported with this workflow. + pull_request: types: - closed - labeled +# Disables permissions of the default GITHUB_TOKEN, it's not used anyway. +permissions: {} + +defaults: + run: + shell: bash + jobs: backport: name: Backport - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # Only react to merged PRs for security reasons. # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target. if: > @@ -17,11 +25,40 @@ jobs: github.event.action == 'closed' || ( github.event.action == 'labeled' - && contains(github.event.label.name, 'backport') + && startsWith(github.event.label.name, 'backport release-') ) ) steps: - - uses: tibdex/backport@v2.0.4 + - uses: tibdex/backport@9565281eda0731b1d20c4025c43339fb0a23812e # ratchet:tibdex/backport@v2.0.4 + id: backport with: github_token: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} - title_template: "<%= title %>" + title_template: "<%= title %> (<%= base %>)" + label_pattern: '^backport (?(release-[0-9]+\.[0-9]+))$' + labels_template: '["backport"]' + + - name: Assign backport PRs to original author + if: steps.backport.outcome == 'success' && steps.backport.outputs.created_pull_requests != '{}' + env: + GH_TOKEN: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} + CREATED_PRS: ${{ steps.backport.outputs.created_pull_requests }} + ORIGINAL_AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + set -u + echo "Created PRs: ${CREATED_PRS}" + echo "Original author: ${ORIGINAL_AUTHOR}" + + # Check if the original author can be assigned + if gh api --silent "repos/${{ github.repository }}/assignees/${ORIGINAL_AUTHOR}" 2>/dev/null; then + echo "Author ${ORIGINAL_AUTHOR} is eligible for assignment" + + # Parse the JSON object and extract PR numbers + for pr_number in $(echo "${CREATED_PRS}" | jq -r '.[]'); do + echo "Assigning PR #${pr_number} to ${ORIGINAL_AUTHOR}" + if ! gh pr edit "${pr_number}" --add-assignee "${ORIGINAL_AUTHOR}" --repo "${{ github.repository }}"; then + >&2 echo "Warning: Failed to assign PR #${pr_number} to ${ORIGINAL_AUTHOR}" + fi + done + else + >&2 echo "Warning: Author ${ORIGINAL_AUTHOR} cannot be assigned to PRs in this repository" + fi diff --git a/.github/workflows/batch-load-test-metrics.yml b/.github/workflows/batch-load-test-metrics.yml index de06ff50f3905..f798db49af93d 100644 --- a/.github/workflows/batch-load-test-metrics.yml +++ b/.github/workflows/batch-load-test-metrics.yml @@ -10,12 +10,12 @@ concurrency: Batch load test metrics jobs: batch-load-test-metrics: if: ${{ github.repository_owner == 'stackrox' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11@sha256:df3459cf038e73775314b3a5d36d14dda41e6df438a0273c9efa8d23e0f2358a # ratchet:quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - name: Load env: GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 750e389193dac..607c34fe7803f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,13 +28,52 @@ jobs: define-job-matrix: outputs: matrix: ${{ steps.define-job-matrix.outputs.matrix }} - runs-on: ubuntu-latest + build-tag: ${{ steps.build-tag.outputs.tag }} + short-commit: ${{ steps.short-commit.outputs.shortcommit }} + gotags: ${{ steps.gotags.outputs.gotags }} + quay-tag-expiration: ${{ steps.quay-tag-expiration.outputs.expiration }} + runs-on: ubuntu-slim steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: + fetch-depth: 0 ref: ${{ inputs.commit || github.event.pull_request.head.sha }} + - uses: ./.github/actions/handle-tagged-build + + - name: Compute build tag + id: build-tag + run: | + TAG=$(make --quiet --no-print-directory tag) + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + + - name: Compute short commit + id: short-commit + run: | + SHORTCOMMIT=$(make --quiet --no-print-directory shortcommit) + echo "shortcommit=${SHORTCOMMIT}" >> "$GITHUB_OUTPUT" + + - name: Compute GOTAGS + id: gotags + run: | + GOTAGS="" + if [[ "${GITHUB_REF}" =~ ^refs/tags/ ]]; then + # Tag builds (release or nightly) get release GOTAGS + GOTAGS="release" + fi + echo "gotags=${GOTAGS}" >> "$GITHUB_OUTPUT" + + - name: Compute Quay tag expiration + id: quay-tag-expiration + run: | + EXPIRATION="13w" + if [[ "${GITHUB_REF}" =~ ^refs/tags/ ]]; then + # Tag builds (release or nightly) never expire + EXPIRATION="never" + fi + echo "expiration=${EXPIRATION}" >> "$GITHUB_OUTPUT" + - name: Define the matrix for build jobs id: define-job-matrix run: | @@ -46,7 +85,8 @@ jobs: "push_main_multiarch_manifests": { "name":[] }, "build_and_push_operator": { "name":[], "arch":[] }, "push_operator_multiarch_manifests": { "name":[] }, - "scan_images_with_roxctl": { "name":[], "image":[], "exclude":[] } + "build_and_push_scanner": { "name":[], "goos":[], "arch":[], "registry":[] }, + "push_scanner_manifests": { "name":[], "registry":[] } }' # The base matrix @@ -64,14 +104,19 @@ jobs: matrix="$(jq '.push_operator_multiarch_manifests.name += ["RHACS_BRANDING", "STACKROX_BRANDING"]' <<< "$matrix")" matrix="$(jq '.push_operator_multiarch_manifests.archs = [.build_and_push_operator.arch | join(",")]' <<< "$matrix")" - matrix="$(jq '.scan_images_with_roxctl.name += ["RHACS_BRANDING", "STACKROX_BRANDING"]' <<< "$matrix")" - matrix="$(jq '.scan_images_with_roxctl.image += ["central-db", "collector", "main", "roxctl", "scanner", "scanner-db", "scanner-db-slim", "scanner-slim", "stackrox-operator"]' <<< "$matrix")" - # TODO(ROX-27191): remove the exclusion once there's a community operator. - matrix="$(jq '.scan_images_with_roxctl.exclude += [{ "name": "STACKROX_BRANDING", "image": "stackrox-operator" }]' <<< "$matrix")" + matrix="$(jq '.build_and_push_scanner.name += ["default"]' <<< "$matrix")" + matrix="$(jq '.build_and_push_scanner.goos += ["linux"]' <<< "$matrix")" + matrix="$(jq '.build_and_push_scanner.arch += ["amd64", "arm64"]' <<< "$matrix")" + matrix="$(jq '.build_and_push_scanner.registry += ["quay.io/stackrox-io", "quay.io/rhacs-eng"]' <<< "$matrix")" + + matrix="$(jq '.push_scanner_manifests.name += ["default"]' <<< "$matrix")" + matrix="$(jq '.push_scanner_manifests.registry += ["quay.io/stackrox-io", "quay.io/rhacs-eng"]' <<< "$matrix")" if ! is_in_PR_context || pr_has_label ci-build-all-arch; then matrix="$(jq '.pre_build_go_binaries.arch += ["ppc64le", "s390x"]' <<< "$matrix")" matrix="$(jq '.build_and_push_main.arch += ["ppc64le", "s390x"]' <<< "$matrix")" + matrix="$(jq '.build_and_push_operator.arch += ["ppc64le", "s390x"]' <<< "$matrix")" + matrix="$(jq '.build_and_push_scanner.arch += ["ppc64le", "s390x"]' <<< "$matrix")" fi; # Conditionally add a prerelease build (binaries built with GOTAGS=release) @@ -80,6 +125,8 @@ jobs: matrix="$(jq '.pre_build_go_binaries.name += ["prerelease"]' <<< "$matrix")" matrix="$(jq '.build_and_push_main.name += ["prerelease"]' <<< "$matrix")" matrix="$(jq '.push_main_multiarch_manifests.name += ["prerelease"]' <<< "$matrix")" + matrix="$(jq '.build_and_push_scanner.name += ["prerelease"]' <<< "$matrix")" + matrix="$(jq '.push_scanner_manifests.name += ["prerelease"]' <<< "$matrix")" fi fi @@ -88,6 +135,8 @@ jobs: matrix="$(jq '.pre_build_go_binaries.name += ["race-condition-debug"]' <<< "$matrix")" matrix="$(jq '.build_and_push_main.name += ["race-condition-debug"]' <<< "$matrix")" matrix="$(jq '.push_main_multiarch_manifests.name += ["race-condition-debug"]' <<< "$matrix")" + matrix="$(jq '.build_and_push_scanner.name += ["race-condition-debug"]' <<< "$matrix")" + matrix="$(jq '.push_scanner_manifests.name += ["race-condition-debug"]' <<< "$matrix")" # Exclude "arm64", "ppc64le", "s390x" matrix="$(jq '.pre_build_go_binaries.exclude = [{ "name": "race-condition-debug", "arch": "arm64" }]' <<< "$matrix")" matrix="$(jq '.pre_build_go_binaries.exclude += [{ "name": "race-condition-debug", "arch": "ppc64le" }]' <<< "$matrix")" @@ -95,6 +144,9 @@ jobs: matrix="$(jq '.build_and_push_main.exclude = [{ "name": "race-condition-debug", "arch": "arm64" }]' <<< "$matrix")" matrix="$(jq '.build_and_push_main.exclude += [{ "name": "race-condition-debug", "arch": "ppc64le" }]' <<< "$matrix")" matrix="$(jq '.build_and_push_main.exclude += [{ "name": "race-condition-debug", "arch": "s390x" }]' <<< "$matrix")" + matrix="$(jq '.build_and_push_scanner.exclude = [{ "name": "race-condition-debug", "arch": "arm64" }]' <<< "$matrix")" + matrix="$(jq '.build_and_push_scanner.exclude += [{ "name": "race-condition-debug", "arch": "ppc64le" }]' <<< "$matrix")" + matrix="$(jq '.build_and_push_scanner.exclude += [{ "name": "race-condition-debug", "arch": "s390x" }]' <<< "$matrix")" fi echo "Job matrix after conditionals:" @@ -112,15 +164,10 @@ jobs: ROX_PRODUCT_BRANDING: ${{ matrix.branding }} # For consistency with `image/rhel/konflux.Dockerfile`, see comment there. UI_PKG_INSTALL_EXTRA_ARGS: --ignore-scripts - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ inputs.commit || github.event.pull_request.head.sha }} @@ -129,13 +176,23 @@ jobs: with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - uses: ./.github/actions/cache-ui-dependencies + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # ratchet:actions/setup-node@v6 + with: + node-version-file: ui/package.json - - uses: ./.github/actions/handle-tagged-build + - name: Cache UI dependencies + id: cache-ui + uses: ./.github/actions/cache-ui-dependencies - name: Fetch UI deps + if: steps.cache-ui.outputs.cache-hit != 'true' run: make -C ui deps + - name: Skip UI deps on cache hit + if: steps.cache-ui.outputs.cache-hit == 'true' + run: touch ui/deps + - name: Build UI run: make -C ui build @@ -147,58 +204,94 @@ jobs: ui/apps/platform/package-lock.json pre-build-cli: - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt + needs: define-job-matrix + runs-on: ubuntu-24.04 + env: + BUILD_TAG: ${{ needs.define-job-matrix.outputs.build-tag }} + SHORTCOMMIT: ${{ needs.define-job-matrix.outputs.short-commit }} + GOTAGS: ${{ needs.define-job-matrix.outputs.gotags }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: - fetch-depth: 0 ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Create CLI stub files + if: | + !(needs.define-job-matrix.outputs.gotags == 'release' + || github.ref_name == 'master' + || github.event_name == 'workflow_call' + || contains(github.event.pull_request.labels.*.name, 'ci-build-cli')) + run: | + cat > stub.txt << 'EOF' + This is a placeholder file. CLI binaries were not built for this PR build. + + To build CLI binaries in PR builds: + - Add the "ci-build-cli" label to your pull request + - Re-push to trigger a new build + + To download the latest roxctl binary: + - Visit https://mirror.openshift.com/pub/rhacs/assets/latest/bin/ + + EOF + + mkdir -p bin/{linux_{amd64,arm64,ppc64le,s390x},darwin_{amd64,arm64},windows_amd64} + + for arch in linux_amd64 linux_arm64 linux_ppc64le linux_s390x darwin_amd64 darwin_arm64; do + cp stub.txt bin/${arch}/roxctl + cp stub.txt bin/${arch}/roxagent + done + cp stub.txt bin/windows_amd64/roxctl.exe + + - name: PR labels + uses: joerick/pr-labels-action@0543b277721e852d821c6738d449f2f4dea03d5f # ratchet:joerick/pr-labels-action@v1.0.9 + - name: Cache Go dependencies + if: | + needs.define-job-matrix.outputs.gotags == 'release' + || github.ref_name == 'master' + || github.event_name == 'workflow_call' + || contains(github.event.pull_request.labels.*.name, 'ci-build-cli') uses: ./.github/actions/cache-go-dependencies - - uses: ./.github/actions/handle-tagged-build - - name: Build CLI + if: | + needs.define-job-matrix.outputs.gotags == 'release' + || github.ref_name == 'master' + || github.event_name == 'workflow_call' + || contains(github.event.pull_request.labels.*.name, 'ci-build-cli') run: make cli - name: Bundle build to preserve permissions - run: tar -cvzf cli-build.tgz bin + run: tar -cvf cli-build.tar bin - uses: ./.github/actions/upload-artifact-with-retry with: name: cli-build - path: cli-build.tgz + path: cli-build.tar + compression-level: 1 pre-build-go-binaries: strategy: # Supports three go binary builds: - # default - built with environment defaults (see handle-tagged-build & env.mk) + # default - built with environment defaults (see env.mk) # prerelease - built with GOTAGS=release # race-condition-debug - built with -race matrix: ${{ fromJson(needs.define-job-matrix.outputs.matrix).pre_build_go_binaries }} needs: define-job-matrix - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt + runs-on: ubuntu-24.04 + env: + BUILD_TAG: ${{ needs.define-job-matrix.outputs.build-tag }} + SHORTCOMMIT: ${{ needs.define-job-matrix.outputs.short-commit }} + GOTAGS: ${{ needs.define-job-matrix.outputs.gotags }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: - fetch-depth: 0 ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - uses: ./.github/actions/job-preamble @@ -209,11 +302,13 @@ jobs: env: GOARCH: ${{ matrix.arch }} uses: ./.github/actions/cache-go-dependencies - - - uses: ./.github/actions/handle-tagged-build + with: + # Only race needs a separate cache (-race/CGO_ENABLED=1 produces 39% unique entries). + # Default and prerelease share 93% of entries and should use the same cache. + key-suffix: ${{ (matrix.name == 'race-condition-debug' && 'race') || '' }} - name: PR labels - uses: joerick/pr-labels-action@v1.0.9 + uses: joerick/pr-labels-action@0543b277721e852d821c6738d449f2f4dea03d5f # ratchet:joerick/pr-labels-action@v1.0.9 - name: Setup build env for debugging support run: | @@ -239,34 +334,39 @@ jobs: ) run: echo "RACE=true" >> "$GITHUB_ENV" - - name: Build Go Binaries + - name: Install musl-tools for race builds + if: | + matrix.arch == 'amd64' + && + ( + contains(github.event.pull_request.labels.*.name, 'ci-race-tests') + || + matrix.name == 'race-condition-debug' + ) run: | - if [[ "${{ matrix.arch }}" != "amd64" ]]; then - echo "Building non-amd binary" - GOOS=linux GOARCH=${{ matrix.arch }} CGO_ENABLED=0 make build-prep main-build-nodeps - else - echo "Building amd binary" - GOOS=linux GOARCH=${{ matrix.arch }} CGO_ENABLED=1 make build-prep main-build-nodeps - fi + sudo apt-get update -qq && sudo apt-get install -y -qq musl-tools musl-dev linux-libc-dev + # musl-gcc searches /usr/include/x86_64-linux-musl by default + sudo ln -sf /usr/include/linux /usr/include/x86_64-linux-musl/linux + sudo ln -sf /usr/include/asm-generic /usr/include/x86_64-linux-musl/asm-generic + sudo ln -sf /usr/include/asm /usr/include/x86_64-linux-musl/asm + + - name: Build Go Binaries + run: GOOS=linux GOARCH=${{ matrix.arch }} CGO_ENABLED=0 make build-prep main-build-nodeps roxctl_linux-${{ matrix.arch }} - name: Bundle the build to preserve permissions - run: tar -cvzf go-binaries-build.tgz bin/linux_${{ matrix.arch }} + run: tar -cvf go-binaries-build.tar bin/linux_${{ matrix.arch }} - uses: ./.github/actions/upload-artifact-with-retry with: name: go-binaries-build-${{ matrix.arch }}-${{ matrix.name }} - path: go-binaries-build.tgz + path: go-binaries-build.tar + compression-level: 1 pre-build-docs: - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ inputs.commit || github.event.pull_request.head.sha }} @@ -278,10 +378,8 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies - - uses: ./.github/actions/handle-tagged-build - - - name: Resolve mods for protos - run: go mod tidy + - name: Download scanner module for proto generation + run: go mod download github.com/stackrox/scanner - name: Generate the swagger docs run: | @@ -295,14 +393,200 @@ jobs: path: | image/rhel/docs + pre-build-oss-notice: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ inputs.commit || github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # ratchet:actions/setup-node@v6 + with: + node-version-file: ui/package.json + + - name: Cache Go dependencies + uses: ./.github/actions/cache-go-dependencies + + - name: Cache UI dependencies + id: cache-ui + uses: ./.github/actions/cache-ui-dependencies + + - name: Fetch UI deps + if: steps.cache-ui.outputs.cache-hit != 'true' + run: make -C ui deps + + - name: Skip UI deps on cache hit + if: steps.cache-ui.outputs.cache-hit == 'true' + run: touch ui/deps + + - name: Generate OSS notice + run: make ossls-notice + + - uses: ./.github/actions/upload-artifact-with-retry + with: + name: oss-notice + path: image/rhel/THIRD_PARTY_NOTICES + + build-and-push-db: + runs-on: ubuntu-24.04 + needs: + - define-job-matrix + if: ${{ !cancelled() }} + env: + BUILD_TAG: ${{ needs.define-job-matrix.outputs.build-tag }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + ref: ${{ inputs.commit || github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Login to docker.io to mitigate rate limiting on downloading images + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # ratchet:docker/login-action@v4 + continue-on-error: true + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + with: + username: ${{ secrets.DOCKERHUB_CI_ACCOUNT_USERNAME }} + password: ${{ secrets.DOCKERHUB_CI_ACCOUNT_PASSWORD }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # ratchet:docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # ratchet:docker/setup-buildx-action@v4 + + - name: Determine platforms + id: platforms + run: | + PLATFORMS="linux/amd64,linux/arm64" + if [[ "${{ github.event_name }}" != "pull_request" ]] || \ + [[ "${{ contains(github.event.pull_request.labels.*.name, 'ci-build-all-arch') }}" == "true" ]]; then + PLATFORMS="linux/amd64,linux/arm64,linux/ppc64le,linux/s390x" + fi + echo "platforms=${PLATFORMS}" >> "$GITHUB_OUTPUT" + + - name: Login to quay.io/stackrox-io + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # ratchet:docker/login-action@v4 + with: + registry: quay.io + username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + + # scanner/image/db/Dockerfile does COPY init-bundles/db-init.dump.zst. + # This file is not in git - the Makefile creates it as empty by default + # (or populated if SCANNER_DB_INIT_BUNDLE_ENABLED=force, which is not used). + # Since we bypass make and build directly with buildx, we create the empty + # file here to match the default make behavior. The init script ignores it. + - name: Create empty init bundle for scanner-v4-db + run: | + mkdir -p scanner/image/db/init-bundles + touch scanner/image/db/init-bundles/db-init.dump.zst + + - name: Build and push DB images in parallel with bake + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + uses: docker/bake-action@018cb6412ab401ebaa809aa5f85966b74628600f # ratchet:docker/bake-action@v7.4.0 + with: + source: . + files: .github/docker/images.hcl + targets: databases + push: true + env: + TAG: ${{ env.BUILD_TAG }} + PLATFORMS: ${{ steps.platforms.outputs.platforms }} + REGISTRY: quay.io/stackrox-io + QUAY_TAG_EXPIRATION: ${{ needs.define-job-matrix.outputs.quay-tag-expiration }} + + # We can't push to both quay.io/stackrox-io and quay.io/rhacs-eng in a single + # build because they require different credentials (different organizations). + # Instead, we build once and push to stackrox-io, then use skopeo to copy the + # multi-arch manifest list to rhacs-eng. This is more efficient than building + # twice and matches the pattern used for collector retag. + - name: Login to quay.io/rhacs-eng + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # ratchet:docker/login-action@v4 + with: + registry: quay.io + username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + + - name: Copy central-db to rhacs-eng registry + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + run: | + skopeo copy --retry-times 5 --all \ + "docker://quay.io/stackrox-io/central-db:${{ env.BUILD_TAG }}" \ + "docker://quay.io/rhacs-eng/central-db:${{ env.BUILD_TAG }}" + + - name: Copy scanner-v4-db to rhacs-eng registry + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + run: | + skopeo copy --retry-times 5 --all \ + "docker://quay.io/stackrox-io/scanner-v4-db:${{ env.BUILD_TAG }}" \ + "docker://quay.io/rhacs-eng/scanner-v4-db:${{ env.BUILD_TAG }}" + + build-operator-bundle: + runs-on: ubuntu-24.04 + env: + QUAY_ORG: rhacs-eng + QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + ROX_OPERATOR_SKIP_PROTO_GENERATED_SRCS: true + ROX_PRODUCT_BRANDING: RHACS_BRANDING + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ inputs.commit || github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Cache Go dependencies + uses: ./.github/actions/cache-go-dependencies + + - uses: ./.github/actions/handle-tagged-build + + - name: Build Operator Bundle + run: make -C operator/ bundle bundle-build + + - name: Push Operator Bundle and Index + # Skip for external contributions. + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + run: | + ./scripts/ci/lib.sh registry_rw_login "quay.io/${QUAY_ORG}" + make -C operator/ docker-push-bundle | cat + make -C operator/ index-build + make -C operator/ docker-push-index | cat + build-and-push-main: - runs-on: ubuntu-latest + # Use native arm64 runner to avoid QEMU emulation overhead (~2x slower). + runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} needs: - define-job-matrix - - pre-build-ui - pre-build-cli - - pre-build-go-binaries - pre-build-docs + - pre-build-go-binaries + - pre-build-oss-notice + - pre-build-ui # This step will run even if required pre-* steps failed. By following this # approach as much of the build matrix as possible is completed despite # transient issues e.g. docker pull timeouts. In this way some e2e jobs that @@ -319,18 +603,9 @@ jobs: env: GO_BINARIES_BUILD_ARTIFACT: "" ROX_PRODUCT_BRANDING: "" - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt - env: - QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} - QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} - QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + BUILD_TAG: ${{ needs.define-job-matrix.outputs.build-tag }} + SHORTCOMMIT: ${{ needs.define-job-matrix.outputs.short-commit }} + GOTAGS: ${{ needs.define-job-matrix.outputs.gotags }} steps: - name: Setup build env run: | @@ -365,9 +640,8 @@ jobs: fi - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: - fetch-depth: 0 ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - uses: ./.github/actions/job-preamble @@ -375,27 +649,27 @@ jobs: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - name: Login to docker.io to mitigate rate limiting on downloading images - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # ratchet:docker/login-action@v4 + continue-on-error: true # Skip for external contributions. if: | - github.event_name == 'push' || !github.event.pull_request.head.repo.fork + matrix.arch != 'amd64' && matrix.arch != 'arm64' && + (github.event_name == 'push' || !github.event.pull_request.head.repo.fork) with: username: ${{ secrets.DOCKERHUB_CI_ACCOUNT_USERNAME }} password: ${{ secrets.DOCKERHUB_CI_ACCOUNT_PASSWORD }} - name: Set up QEMU - if: matrix.arch != 'amd64' - uses: docker/setup-qemu-action@v3 + if: matrix.arch != 'amd64' && matrix.arch != 'arm64' + uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # ratchet:docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # ratchet:docker/setup-buildx-action@v4 - name: Checkout submodules run: | git submodule update --init - - uses: ./.github/actions/handle-tagged-build - - uses: ./.github/actions/download-artifact-with-retry with: name: ui-${{ env.ROX_PRODUCT_BRANDING }}-build @@ -407,7 +681,7 @@ jobs: - name: Unpack cli build run: | - tar xvzf cli-build.tgz + tar xvf cli-build.tar - uses: ./.github/actions/download-artifact-with-retry with: @@ -415,22 +689,17 @@ jobs: - name: Unpack Go binaries build run: | - tar xvzf go-binaries-build.tgz + tar xvf go-binaries-build.tar - uses: ./.github/actions/download-artifact-with-retry with: name: docs-build path: image/rhel/docs - # needed to restore node_modules for ossls-nostice - - uses: ./.github/actions/cache-ui-dependencies - - # explicitly fetch deps just in case cache was not ready - - name: Fetch UI deps - run: make -C ui deps - - - name: Generate OSS notice - run: make ossls-notice + - uses: ./.github/actions/download-artifact-with-retry + with: + name: oss-notice + path: image/rhel/THIRD_PARTY_NOTICES - name: Set build tag for prerelease images if: matrix.name == 'prerelease' @@ -445,8 +714,13 @@ jobs: GOOS=linux GOARCH=${{ matrix.arch }} scripts/lib.sh retry 6 true make docker-build-main-image - name: Check debugger presence in the main image + if: github.event_name != 'pull_request' run: make check-debugger + - name: Prepare roxctl build context + run: | + cp -r image/rhel/THIRD_PARTY_NOTICES image/roxctl/THIRD_PARTY_NOTICES + - name: Build roxctl image run: | GOOS=linux GOARCH=${{ matrix.arch }} scripts/lib.sh retry 6 true make docker-build-roxctl-image @@ -456,6 +730,9 @@ jobs: # Skip for external contributions. if: | github.event_name == 'push' || !github.event.pull_request.head.repo.fork + env: + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} run: | ./scripts/ci/lib.sh registry_ro_login "quay.io/rhacs-eng" @@ -463,6 +740,11 @@ jobs: # Skip for external contributions. if: | github.event_name == 'push' || !github.event.pull_request.head.repo.fork + env: + QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} run: | source ./scripts/ci/lib.sh echo "Will determine context from: ${{ github.event_name }} & ${{ github.ref_name }}" @@ -472,18 +754,41 @@ jobs: fi push_main_image_set "$push_context" "${{ env.ROX_PRODUCT_BRANDING }}" "${{ matrix.arch }}" + push-matching-collector-scanner: + runs-on: ubuntu-24.04 + needs: + - define-job-matrix + if: ${{ !cancelled() }} + strategy: + fail-fast: false + matrix: + name: [RHACS_BRANDING, STACKROX_BRANDING] + env: + ROX_PRODUCT_BRANDING: ${{ matrix.name }} + BUILD_TAG: ${{ needs.define-job-matrix.outputs.build-tag }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + ref: ${{ inputs.commit || github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Push matching collector and scanner images - # Skip for external contributions. - if: | - github.event_name == 'push' || !github.event.pull_request.head.repo.fork + if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork + env: + QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} run: | - # Need to free up some space before push_matching_collector_scanner_images() does its pull. - docker system prune --all --force - source ./scripts/ci/lib.sh - push_matching_collector_scanner_images "${{ env.ROX_PRODUCT_BRANDING }}" "${{ matrix.arch }}" + source ./scripts/ci/lib.sh + push_matching_collector_scanner_images "${{ env.ROX_PRODUCT_BRANDING }}" push-main-manifests: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - define-job-matrix - build-and-push-main @@ -493,6 +798,8 @@ jobs: # some e2e jobs that require some part of the build matrix can still # proceed. if: ${{ !cancelled() }} + permissions: + pull-requests: write strategy: # Supports four image builds (see Setup build env): # STACKROX_BRANDING @@ -503,18 +810,9 @@ jobs: matrix: ${{ fromJson(needs.define-job-matrix.outputs.matrix).push_main_multiarch_manifests }} env: ROX_PRODUCT_BRANDING: "" - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt - env: - QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} - QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} - QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + BUILD_TAG: ${{ needs.define-job-matrix.outputs.build-tag }} + SHORTCOMMIT: ${{ needs.define-job-matrix.outputs.short-commit }} + GOTAGS: ${{ needs.define-job-matrix.outputs.gotags }} steps: - name: Setup build env run: | @@ -541,21 +839,21 @@ jobs: } >> "$GITHUB_ENV" - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: - fetch-depth: 0 ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # ratchet:docker/setup-buildx-action@v4 + - name: Checkout submodules run: | git submodule update --init - - uses: ./.github/actions/handle-tagged-build - - name: Set build tag for prerelease images if: matrix.name == 'prerelease' run: echo "BUILD_TAG=$(make --quiet --no-print-directory tag)-prerelease" >> "$GITHUB_ENV" @@ -568,6 +866,11 @@ jobs: # Skip for external contributions. if: | github.event_name == 'push' || !github.event.pull_request.head.repo.fork + env: + QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} run: | source ./scripts/ci/lib.sh echo "Will determine context from: ${{ github.event_name }} & ${{ github.ref_name }}" @@ -584,63 +887,74 @@ jobs: fi push_image_manifest_lists "$push_context" "${{ env.ROX_PRODUCT_BRANDING }}" "$architectures" - - name: Comment on the PR + - name: Prepare build comment # Skip for external contributions. if: | github.event_name == 'push' || !github.event.pull_request.head.repo.fork - env: - GITHUB_TOKEN: "${{ secrets.RHACS_BOT_GITHUB_TOKEN }}" run: | - source ./scripts/ci/lib.sh - add_build_comment_to_pr + cat > build-comment.md <> "$GITHUB_ENV" + + - name: Set build tag for race condition images + if: matrix.name == 'race-condition-debug' + run: echo "BUILD_TAG=$(make -C scanner --quiet --no-print-directory tag)-rcd" >> "$GITHUB_ENV" + + - name: Build Scanner and ScannerDB images + run: scripts/lib.sh retry 6 true make -C scanner GOOS=${{ matrix.goos }} GOARCH=${{ matrix.arch }} images + + - name: Push Scanner and ScannerDB images if: | - matrix.name != 'STACKROX_BRANDING' && (github.event_name == 'push' || !github.event.pull_request.head.repo.fork) + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + env: + QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} run: | - make -C operator/ docker-push-index | cat + source ./scripts/ci/lib.sh + push_scanner_image_set "${{ matrix.registry }}" "${{ matrix.arch }}" + + push-scanner-manifests: + runs-on: ubuntu-24.04 + needs: + - define-job-matrix + - build-and-push-scanner + # This step will run even if some of the build-and-push-scanner steps failed. + # By following this approach as much of the build matrix as possible is + # completed despite transient issues e.g. docker pull timeouts. In this way + # some e2e jobs that require some part of the build matrix can still + # proceed. + if: ${{ !cancelled() }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.define-job-matrix.outputs.matrix).push_scanner_manifests }} + env: + BUILD_TAG: ${{ needs.define-job-matrix.outputs.build-tag }} + SHORTCOMMIT: ${{ needs.define-job-matrix.outputs.short-commit }} + GOTAGS: ${{ needs.define-job-matrix.outputs.gotags }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + ref: ${{ inputs.commit || github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # ratchet:docker/setup-buildx-action@v4 + + - name: Set build tag for prerelease images + if: matrix.name == 'prerelease' + run: echo "BUILD_TAG=$(make -C scanner --quiet --no-print-directory tag)-prerelease" >> "$GITHUB_ENV" + + - name: Set build tag for race condition images + if: matrix.name == 'race-condition-debug' + run: echo "BUILD_TAG=$(make -C scanner --quiet --no-print-directory tag)-rcd" >> "$GITHUB_ENV" + + - name: Push Scanner and ScannerDB image manifests + if: | + github.event_name == 'push' || !github.event.pull_request.head.repo.fork + env: + QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + run: | + source ./scripts/ci/lib.sh + + architectures="amd64,arm64" + if ! is_in_PR_context || pr_has_label ci-build-all-arch; then + architectures="amd64,arm64,ppc64le,s390x" + fi + if [[ "${{ matrix.name }}" == "race-condition-debug" ]]; then + architectures="amd64" + fi + + push_scanner_image_manifest_lists "${{ matrix.registry }}" "$architectures" push-operator-manifests: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - define-job-matrix - build-and-push-operator @@ -760,35 +1189,31 @@ jobs: matrix: ${{ fromJson(needs.define-job-matrix.outputs.matrix).push_operator_multiarch_manifests }} env: ROX_PRODUCT_BRANDING: ${{ matrix.name }} - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt - env: - QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} - QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} - QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + BUILD_TAG: ${{ needs.define-job-matrix.outputs.build-tag }} + SHORTCOMMIT: ${{ needs.define-job-matrix.outputs.short-commit }} + GOTAGS: ${{ needs.define-job-matrix.outputs.gotags }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: - fetch-depth: 0 ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - uses: ./.github/actions/handle-tagged-build + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # ratchet:docker/setup-buildx-action@v4 - name: Push operator manifest lists # Skip for external contributions. if: | github.event_name == 'push' || !github.event.pull_request.head.repo.fork + env: + QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} run: | source ./scripts/ci/lib.sh echo "Will determine context from: ${{ github.event_name }} & ${{ github.ref_name }}" @@ -803,15 +1228,13 @@ jobs: contains(github.event.pull_request.labels.*.name, 'scan-go-binaries') env: ARTIFACT_DIR: junit-reports/ - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + runs-on: ubuntu-24.04 needs: - pre-build-cli - pre-build-go-binaries steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 1 ref: ${{ inputs.commit || github.event.pull_request.head.sha }} @@ -826,7 +1249,7 @@ jobs: - name: Unpack cli build run: | - tar xvzf cli-build.tgz + tar xvf cli-build.tar - uses: ./.github/actions/download-artifact-with-retry with: @@ -834,14 +1257,14 @@ jobs: - name: Unpack Go binaries build run: | - tar xvzf go-binaries-build.tgz + tar xvf go-binaries-build.tar - name: Scan run: | ./scripts/ci/govulncheck.sh - name: Publish Test Report - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 if: always() with: paths: 'junit-reports/**/*.xml' @@ -850,71 +1273,11 @@ jobs: uses: ./.github/actions/junit2jira if: always() with: + jira-user: ${{ secrets.JIRA_USER }} jira-token: ${{ secrets.JIRA_TOKEN }} gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} directory: 'junit-reports' - scan-images-with-roxctl: - if: github.event_name == 'push' || - contains(github.event.pull_request.labels.*.name, 'scan-images-with-roxctl') - needs: - - define-job-matrix - - build-and-push-main - - build-and-push-operator - - push-main-manifests - name: Check images for vulnerabilities - runs-on: ubuntu-latest - permissions: - # Needed for stackrox/central-login to create the JWT token. - id-token: write - security-events: write - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.define-job-matrix.outputs.matrix).scan_images_with_roxctl }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - - - uses: ./.github/actions/job-preamble - with: - gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - - uses: ./.github/actions/handle-tagged-build - - - name: Central login - uses: stackrox/central-login@v1 - with: - endpoint: ${{ vars.ACS_DOGFOODING_CENTRAL_URL }} - - - name: Install roxctl - uses: stackrox/roxctl-installer-action@v1 - with: - central-endpoint: ${{ vars.ACS_DOGFOODING_CENTRAL_URL }} - central-token: ${{ env.ROX_API_TOKEN }} - - - name: Scan images for vulnerabilities - run: | - release_tag="$(make --quiet --no-print-directory tag)" - if [[ "${{ matrix.image }}" =~ "operator" ]]; then - release_tag="$(make -C operator --quiet --no-print-directory tag)" - fi - - registry="$(./scripts/ci/lib.sh registry_from_branding "${{ matrix.name }}")" - - roxctl image scan --retries=10 --retry-delay=15 --force --severity=CRITICAL,IMPORTANT --output=sarif \ - --image="${registry}/${{ matrix.image }}:${release_tag}" \ - | tee results.sarif - - # TODO: re-enable roxctl scan results upload once quota issue has been resolved - # - name: Upload roxctl scan results to GitHub Security tab - # uses: github/codeql-action/upload-sarif@v3 - # with: - # category: ${{ matrix.image }} - # sarif_file: results.sarif - slack-on-build-failure: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -926,21 +1289,24 @@ jobs: contains(github.event.pull_request.labels.*.name, 'ci-test-github-action-slack-messages') ) name: Post failure message to Slack - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - - pre-build-ui - - pre-build-cli - - pre-build-go-binaries - - pre-build-docs + - build-and-push-db - build-and-push-main - build-and-push-operator - - scan-images-with-roxctl + - build-and-push-scanner + - build-operator-bundle + - pre-build-cli + - pre-build-docs + - pre-build-go-binaries + - pre-build-ui + - push-matching-collector-scanner - scan-go-binaries permissions: actions: read steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - name: Slack message env: diff --git a/.github/workflows/check-crd-compatibility.yaml b/.github/workflows/check-crd-compatibility.yaml index 2e32fdb61d995..d5d2ba0ffd62b 100644 --- a/.github/workflows/check-crd-compatibility.yaml +++ b/.github/workflows/check-crd-compatibility.yaml @@ -17,15 +17,20 @@ concurrency: cancel-in-progress: true jobs: + detect-changes: + uses: ./.github/workflows/detect-changes.yml + check-crd-compatibility: - runs-on: ubuntu-latest + needs: detect-changes + if: ${{ !needs.detect-changes.outputs.ui_only }} + runs-on: ubuntu-24.04 container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11@sha256:df3459cf038e73775314b3a5d36d14dda41e6df438a0273c9efa8d23e0f2358a # ratchet:quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11 outputs: released_version: ${{ steps.get_previous_released_version.outputs.released_version }} steps: - name: Checkout stackrox/stackrox - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} @@ -35,7 +40,7 @@ jobs: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - name: Checkout crd-schema-checker - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: repository: openshift/crd-schema-checker path: .crd-schema-checker @@ -53,7 +58,7 @@ jobs: echo "released_version=$released_version" >> "$GITHUB_OUTPUT" - name: Checkout previous released version - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ steps.get_previous_released_version.outputs.released_version }} @@ -80,3 +85,20 @@ jobs: args="--disabled-validators=ConditionsMustHaveProperSSATags,ListsMustHaveSSATags,NoBools,NoMaps" set -x .crd-schema-checker/crd-schema-checker check-manifests $args --existing-crd-filename=$old_path/$policy_file --new-crd-filename=$new_path/$policy_file + + - name: Run crdify for platform operator + run: | + set -euo pipefail + old_path="$PWD/.old-stackrox/operator/config/crd/bases" + new_path="$PWD/operator/config/crd/bases" + crdify=".github/workflows/scripts/crdify.sh" + $crdify "${old_path}/platform.stackrox.io_securedclusters.yaml" "${new_path}/platform.stackrox.io_securedclusters.yaml" + $crdify "${old_path}/platform.stackrox.io_centrals.yaml" "${new_path}/platform.stackrox.io_centrals.yaml" + + - name: Run crdify for config operator + run: | + set -euo pipefail + old_path="$PWD/.old-stackrox/config-controller/config/crd/bases" + new_path="$PWD/config-controller/config/crd/bases" + crdify=".github/workflows/scripts/crdify.sh" + $crdify "${old_path}/config.stackrox.io_securitypolicies.yaml" "${new_path}/config.stackrox.io_securitypolicies.yaml" diff --git a/.github/workflows/check-crd-frozen-on-release.yaml b/.github/workflows/check-crd-frozen-on-release.yaml new file mode 100644 index 0000000000000..981253e8cfb15 --- /dev/null +++ b/.github/workflows/check-crd-frozen-on-release.yaml @@ -0,0 +1,140 @@ +# On release branches that already shipped their first (.0) release, CRD schemas +# must stay frozen: no changes to the platform (Central/SecuredCluster) and +# config (SecurityPolicy) CRDs are allowed in a patch release. +# +# The freeze only kicks in once the first release of the branch has been tagged. +# The tag is derived from the branch name: release-X.Y -> X.Y.0. Before X.Y.0 is +# tagged the branch is still stabilizing toward its first release, so CRD changes +# are still permitted and this check is skipped. +# +# When the freeze is active: +# - For PRs targeting a release branch, we diff the release branch (target) +# against the PR tip. +# - For commits pushed to a release branch, we diff the first release tag (X.Y.0) +# against the current commit, so only changes made after the first release are +# considered. +# +# See https://redhat.atlassian.net/browse/ROX-36823 +name: Check CRDs Frozen on Release Branches + +on: + push: + branches: + - release-* + pull_request: + # Only PRs whose base (target) branch is a release branch. + branches: + - release-* + types: + - opened + - reopened + - synchronize + +permissions: + contents: read + +jobs: + check-crds-frozen: + runs-on: ubuntu-24.04 + steps: + - name: Checkout stackrox/stackrox + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + # Full history so we can compute the point from which the release branch + # (or the PR) diverged. + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Ensure CRD schemas are unchanged + env: + EVENT_NAME: ${{ github.event_name }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + shell: bash + run: | + set -euo pipefail + + # Determine the release branch this event concerns. + if [[ "$EVENT_NAME" == "pull_request" ]]; then + branch="$BASE_REF" + else + branch="$GITHUB_REF_NAME" + fi + + # Derive the first release tag from the branch name: release-X.Y -> X.Y.0. + # Anything that is not a plain release-X.Y branch (e.g. a scanner branch) + # is out of scope for this check. + if [[ ! "$branch" =~ ^release-([0-9]+)\.([0-9]+)$ ]]; then + echo "Branch '$branch' is not a standard release-X.Y branch; skipping CRD freeze check." + exit 0 + fi + first_release_tag="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}.0" + + # Only freeze CRDs for patch releases, i.e. once the first (.0) release of + # this branch has been tagged. Before that the branch is still + # stabilizing and CRD changes are allowed. + # + # `git ls-remote --exit-code` distinguishes its outcomes: 0 = the tag + # exists, 2 = the tag is definitively absent, any other code = a + # transport/other error. We must not treat a transient network failure as + # "not tagged" (that would silently disable the freeze), so we retry and + # fail closed if we cannot get an authoritative answer. + tag_ref="refs/tags/${first_release_tag}" + rc=0 + for attempt in 1 2 3; do + rc=0 + git ls-remote --exit-code --tags origin "$tag_ref" >/dev/null 2>&1 || rc=$? + if [[ "$rc" -eq 0 || "$rc" -eq 2 ]]; then + break + fi + echo "git ls-remote for '${tag_ref}' failed (exit $rc); retry ${attempt}/3..." >&2 + sleep 5 + done + + case "$rc" in + 0) + echo "First release '${first_release_tag}' is tagged; enforcing CRD freeze for patch releases." + ;; + 2) + echo "First release '${first_release_tag}' is not tagged yet; branch is still stabilizing. CRD changes are allowed." + exit 0 + ;; + *) + echo "::error::Could not determine whether '${first_release_tag}' is tagged (git ls-remote exit $rc). Failing closed." + exit 1 + ;; + esac + + # Platform and config CRD manifests that must remain frozen on release + # branches. + crd_files=( + operator/config/crd/bases/platform.stackrox.io_centrals.yaml + operator/config/crd/bases/platform.stackrox.io_securedclusters.yaml + config-controller/config/crd/bases/config.stackrox.io_securitypolicies.yaml + ) + + if [[ "$EVENT_NAME" == "pull_request" ]]; then + # Diff the release branch (target) against the PR tip. + base="$BASE_SHA" + head="$HEAD_SHA" + echo "Comparing release branch target ($base) against PR tip ($head)" + else + # Diff the first release tag (X.Y.0) against the current commit, so + # only changes made after the first release are considered. + head="$GITHUB_SHA" + git fetch --no-tags --quiet origin "refs/tags/${first_release_tag}:refs/tags/${first_release_tag}" + base="$first_release_tag" + echo "Comparing first release tag $base against current commit ($head)" + fi + + diff="$(git diff "$base" "$head" -- "${crd_files[@]}")" + + if [[ -n "$diff" ]]; then + echo "::error::CRD schema changes are not allowed on release branches (ROX-36823)." + echo "The following platform/config CRD changes were detected:" + echo "$diff" + exit 1 + fi + + echo "No platform/config CRD schema changes detected. CRDs are frozen. ✅" diff --git a/.github/workflows/check-pr-title.yaml b/.github/workflows/check-pr-title.yaml index 5212dfd3ac09b..49afd75bf89a4 100644 --- a/.github/workflows/check-pr-title.yaml +++ b/.github/workflows/check-pr-title.yaml @@ -6,7 +6,7 @@ on: jobs: check-title: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: PR_TITLE: ${{ github.event.pull_request.title }} steps: diff --git a/.github/workflows/ci-failures-report.yml b/.github/workflows/ci-failures-report.yml index 369a09976720b..a2e0e03d9a7bf 100644 --- a/.github/workflows/ci-failures-report.yml +++ b/.github/workflows/ci-failures-report.yml @@ -1,5 +1,10 @@ name: CI Failures Report on: + pull_request: + paths: + - '.github/workflows/ci-failures-report.yml' + - 'scripts/ci/metrics.sh' + - 'scripts/ci/sql/test_failure_streaks.sql' workflow_dispatch: inputs: test: @@ -13,12 +18,10 @@ on: jobs: report-e2e-failures-to-slack: if: ${{ github.repository_owner == 'stackrox' }} - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - name: Report env: GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} @@ -32,9 +35,18 @@ jobs: # Avoid BQ init message in first report bq --quiet --format=pretty query --use_legacy_sql=false 'select 1' + + # Use test mode for PRs and manual dispatch with test=true, production mode for scheduled runs + if [[ "${{ github.event_name }}" == "pull_request" ]] || [[ "${{ github.event.inputs.test }}" == "true" ]]; then + is_test="true" + else + is_test="false" + fi + n=10 - slack_top_n_failures ${n} "qa-e2e" ":testgrid: Top ${n} QA E2E Test failures in this and previous week" "${{github.event.inputs.test}}" - slack_top_n_failures ${n} "operator-e2e" ":operator: Top ${n} Operator E2E Test failures in this and previous week" "${{github.event.inputs.test}}" - slack_top_n_failures ${n} "ui-e2e" ":computer: Top ${n} UI E2E Test failures in this and previous week" "${{github.event.inputs.test}}" - slack_top_n_failures ${n} "nongroovy-e2e" ":go: Top ${n} NonGroovy E2E Test failures in this and previous week" "${{github.event.inputs.test}}" - slack_top_n_failures ${n} "go" ":github: Top ${n} unit test failures in this and previous week" "${{github.event.inputs.test}}" + slack_top_n_failures ${n} "qa-e2e" ":testgrid: Top ${n} QA E2E Test failures in this and previous week" "${is_test}" + slack_top_n_failures ${n} "operator-e2e" ":operator: Top ${n} Operator E2E Test failures in this and previous week" "${is_test}" + slack_top_n_failures ${n} "ui-e2e" ":computer: Top ${n} UI E2E Test failures in this and previous week" "${is_test}" + slack_top_n_failures ${n} "nongroovy-e2e" ":go: Top ${n} NonGroovy E2E Test failures in this and previous week" "${is_test}" + slack_top_n_failures ${n} "go" ":github: Top ${n} unit test failures in this and previous week" "${is_test}" + slack_test_failure_streaks 3 ${n} ":fire: Top ${n} tests failing 3+ times in a row (still failing today)" "${is_test}" diff --git a/.github/workflows/create-clusters.yml b/.github/workflows/create-clusters.yml index ace80e23e87e4..2693c27c9311c 100644 --- a/.github/workflows/create-clusters.yml +++ b/.github/workflows/create-clusters.yml @@ -42,7 +42,7 @@ concurrency: Release automation ${{ inputs.version }} jobs: trim-cluster-names: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: cluster-with-fake-load-name-cleaned: ${{ steps.cluster-names.outputs.cluster-with-fake-load-name-cleaned }} cluster-with-real-load-name-cleaned: ${{ steps.cluster-names.outputs.cluster-with-real-load-name-cleaned }} @@ -64,7 +64,7 @@ jobs: clusters: name: Setup demo clusters needs: trim-cluster-names - uses: stackrox/actions/.github/workflows/create-demo-clusters.yml@v1 + uses: stackrox/actions/.github/workflows/create-demo-clusters.yml@f3893761af7aff7477e3dde7a4d9656b173fc687 # ratchet:stackrox/actions/.github/workflows/create-demo-clusters.yml@v1 secrets: inherit with: version: ${{github.event.inputs.version}} diff --git a/.github/workflows/cut-rc.yml b/.github/workflows/cut-rc.yml index 24f1d3784ef91..d28a3ce950815 100644 --- a/.github/workflows/cut-rc.yml +++ b/.github/workflows/cut-rc.yml @@ -37,6 +37,8 @@ on: default: true type: boolean +cache-mode: read + env: main_branch: ${{github.event.repository.default_branch}} script_url: /repos/${{github.repository}}/contents/.github/workflows/scripts/common.sh?ref=${{ github.ref_name }} @@ -60,10 +62,10 @@ concurrency: Release automation ${{ inputs.version }} jobs: properties: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: slack-channel: ${{ fromJSON(format('["{0}","{1}"]', steps.fetch.outputs.dry-slack-channel, steps.fetch.outputs.slack-channel))[github.event.inputs.dry-run != 'true'] }} - jira-projects: ${{ steps.fetch.outputs.jira-projects }} + jira-project: ${{ steps.fetch.outputs.jira-project }} steps: - name: Read workflow properties file id: fetch @@ -73,7 +75,7 @@ jobs: run-parameters: name: Run parameters - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - run: | [ "$DRY_RUN" = "true" ] && echo "::warning::This is a dry run" @@ -96,18 +98,20 @@ jobs: check-jira: name: Check Jira tickets for release needs: [variables, properties] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Query JIRA # Checking unequal with "false" because closed milestones have unset input values. if: github.event.inputs.check-jira-issues != 'false' env: + JIRA_USER: ${{ vars.RHACS_BOT_GITHUB_EMAIL }} JIRA_TOKEN: ${{ secrets.JIRA_TOKEN }} + JIRA_BASE_URL: ${{ vars.JIRA_BASE_URL }} run: | set -uo pipefail gh api -H "$ACCEPT_RAW" "${{env.script_url}}" | bash -s -- \ check-jira-issues \ - "${{needs.properties.outputs.jira-projects}}" \ + "${{needs.properties.outputs.jira-project}}" \ "${{needs.variables.outputs.release}}" \ "${{needs.variables.outputs.patch}}" \ "${{needs.variables.outputs.named-release-patch}}" @@ -115,7 +119,7 @@ jobs: postpone-prs: name: Postpone open PRs needs: variables - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Check open PRs id: check @@ -171,11 +175,11 @@ jobs: cut-rc: name: Tag RC for milestone ${{needs.variables.outputs.milestone}} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [variables, properties, postpone-prs, check-jira] steps: - name: Check out code - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: ref: ${{needs.variables.outputs.branch}} token: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} @@ -183,6 +187,13 @@ jobs: run: | git config user.name "${{github.event.sender.login}}" git config user.email noreply@github.com + - name: Check CHANGELOG entry exists for release "${{ needs.variables.outputs.named-release-patch }}" + run: | + set -uo pipefail + gh api -H "$ACCEPT_RAW" "${{ env.script_url }}" | bash -s -- \ + check-changelog \ + "${{ needs.variables.outputs.branch }}" \ + "${{ needs.variables.outputs.named-release-patch }}" - run: | # Could be optimized: done only if there are closed PRs to cherry-pick git fetch origin ${{env.main_branch}}:${{env.main_branch}} --unshallow @@ -198,7 +209,7 @@ jobs: - name: Post to Slack about picked cherries if: failure() && steps.cherry-pick.outputs.bad-cherries != '' - uses: slackapi/slack-github-action@v2.1.1 + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # ratchet:slackapi/slack-github-action@v3.0.3 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} @@ -278,7 +289,7 @@ jobs: "true" - name: Post to Slack - uses: slackapi/slack-github-action@v2.1.1 + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # ratchet:slackapi/slack-github-action@v3.0.3 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} @@ -299,7 +310,7 @@ jobs: :arrow_right: Once all checks pass and you're ready for release, run the <${{ github.server_url }}/${{ github.repository }}/actions/workflows/finish-release.yml|Finish Release> workflow and delete the `${{ needs.variables.outputs.next-milestone }}` milestone to avoid confusion. trim-cluster-names: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: cluster-with-fake-load-name-cleaned: ${{ steps.cluster-names.outputs.cluster-with-fake-load-name-cleaned }} cluster-with-real-load-name-cleaned: ${{ steps.cluster-names.outputs.cluster-with-real-load-name-cleaned }} @@ -322,7 +333,7 @@ jobs: name: Setup demo clusters secrets: inherit needs: [variables, cut-rc, trim-cluster-names] - uses: stackrox/actions/.github/workflows/create-demo-clusters.yml@v1 + uses: stackrox/actions/.github/workflows/create-demo-clusters.yml@f3893761af7aff7477e3dde7a4d9656b173fc687 # ratchet:stackrox/actions/.github/workflows/create-demo-clusters.yml@v1 if: >- # Skip if no clusters are going to be created. Checking unequal with "false" because closed milestones have unset input values. github.event.inputs.create-k8s-cluster != 'false' || github.event.inputs.create-os4-cluster != 'false' || diff --git a/.github/workflows/detect-changes.yml b/.github/workflows/detect-changes.yml new file mode 100644 index 0000000000000..8cced67cfd949 --- /dev/null +++ b/.github/workflows/detect-changes.yml @@ -0,0 +1,106 @@ +name: Detect changed files + +# Outputs the list of files changed in a PR, one per line, wrapped +# with ^ and $ anchors (e.g. ^ui/src/App.tsx$). Jobs match with +# contains() using anchors for precise substring matching: +# '^scripts/' — full tree under a root directory +# '.go$' — file extension (won't match .gotmpl) +# '^go.mod$' — exact root file +# +# Empty output (push events, API failure, 3000+ files) is falsy, +# so jobs prefix with: !needs.detect-changes.outputs.files || +# to default to "run" when the file list is unavailable. +# +# Optional run_all input: patterns that affect all jobs. If any +# changed file matches, output stays empty so all jobs run. + +on: + workflow_call: + inputs: + run_all: + description: > + Newline-separated list of patterns (same ^/$ anchored + format as the output). If any changed file matches any + of these, output is empty and all jobs run. + type: string + required: false + default: '' + outputs: + files: + description: > + Newline-separated list of changed filenames, or empty + string on push/failure (falsy, so jobs default to run). + value: ${{ jobs.detect.outputs.files }} + ui_only: + description: > + 'true' when every changed file is under ui/, empty + otherwise. Skip non-UI jobs with: if: !needs.X.outputs.ui_only + value: ${{ jobs.detect.outputs.ui_only }} + +jobs: + detect: + runs-on: ubuntu-24.04 + outputs: + files: ${{ steps.list.outputs.files }} + ui_only: ${{ steps.list.outputs.ui_only }} + steps: + - name: List changed files + id: list + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_ALL: ${{ inputs.run_all }} + WORKFLOW_FILE_PATH: ".github/workflows/detect-changes.yml" + shell: bash + run: | + set -eo pipefail + + if [[ -z "${PR_NUMBER}" ]]; then + echo "Not a PR" >&2 + exit 0 + fi + + if ! output="$(gh api "repos/${GH_REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename')"; then + echo "::warning::GitHub API call failed" >&2 + exit 0 + fi + + mapfile -t files < <(echo -n "${output}") + echo "Changed files (${#files[@]}):" >&2 + printf ' %s\n' "${files[@]}" >&2 + + if [[ ${#files[@]} -ge 3000 ]]; then + echo "::warning::PR has 3000+ changed files" >&2 + exit 0 + fi + + anchored=$(printf '^%s$\n' "${files[@]}") + + if echo "${anchored}" | grep -qF "^${WORKFLOW_FILE_PATH}$"; then + echo "detect-changes.yml modified, running all jobs" >&2 + exit 0 + fi + + if [[ -n "${RUN_ALL}" ]]; then + while IFS= read -r pattern; do + [[ -z "$pattern" ]] && continue + if echo "${anchored}" | grep -qF "$pattern"; then + echo "run_all: matched '${pattern}', running all jobs" >&2 + exit 0 + fi + done <<< "${RUN_ALL}" + fi + + if [[ ${#files[@]} -gt 0 ]]; then + { + echo "files<> "${GITHUB_OUTPUT}" + + if ! echo "${anchored}" | grep -qv '^\^ui/'; then + echo "All changed files are under ui/" >&2 + echo "ui_only=true" >> "${GITHUB_OUTPUT}" + fi + fi diff --git a/.github/workflows/e2e-byodb-tests.yaml b/.github/workflows/e2e-byodb-tests.yaml new file mode 100644 index 0000000000000..f9a52ad5547a6 --- /dev/null +++ b/.github/workflows/e2e-byodb-tests.yaml @@ -0,0 +1,227 @@ +name: e2e-byodb-tests + +on: + workflow_call: + inputs: + tag: + description: Image tag to test. + type: string + required: true + should-run: + description: true if should really run the job. false - otherwise, i.e. the job should be skipped. + type: boolean + required: true + workflow_dispatch: + inputs: + tag: + description: Image tag to test. + type: string + required: true + +permissions: + contents: read + pull-requests: read + +defaults: + run: + shell: bash + +jobs: + gke-byodb-qa-e2e-tests: + if: github.event_name == 'workflow_dispatch' || inputs.should-run + runs-on: ubuntu-24.04 + env: + MAIN_IMAGE_TAG: ${{ inputs.tag }} + BUILD_TAG: ${{ inputs.tag }} + GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + CI_JOB_NAME: gke-byodb-qa-e2e-tests + ARTIFACT_DIR: ./junit-reports + LOAD_BALANCER: lb + ORCHESTRATOR_FLAVOR: k8s + KUBERNETES_PROVIDER: gke + SCANNER_V4_DB_STORAGE_CLASS: stackrox-gke-ssd + POSTGRES_VERSION: "17" + LONGTERM_LICENSE: "unused" + BYODB_TEST: "true" + # roxctl helm values pin scannerV4.disable=true unless this is set. + ROX_SCANNER_V4: "true" + ROX_RISK_REPROCESSING_INTERVAL: "15s" + ROX_SENSOR_CONNECTION_RETRY_MAX_INTERVAL: "30s" + PYTHONUNBUFFERED: "1" + timeout-minutes: 361 + steps: + + - name: Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Docker login to Quay.io + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: docker login -u "${REGISTRY_USERNAME}" --password-stdin quay.io <<<"${REGISTRY_PASSWORD}" + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + with: + install_components: "gke-gcloud-auth-plugin,beta" + + - name: Create GKE cluster + env: + MACHINE_TYPE: e2-standard-8 + NUM_NODES: "1" + GKE_SPOT: "false" + timeout-minutes: 20 + run: | + scripts/ci/gke.sh provision_gke_cluster byodb + echo "KUBECONFIG=${HOME}/.kube/config" >> "$GITHUB_ENV" + + - name: Build roxctl + run: | + make roxctl_linux-amd64 + sudo cp bin/linux_amd64/roxctl /usr/local/bin/roxctl + echo "${HOME}/go/bin" >> "$GITHUB_PATH" + + - name: Verify kubectl access and wait for cluster to stabilize + timeout-minutes: 5 + run: | + kubectl cluster-info + kubectl get nodes + scripts/ci/gke.sh wait_for_cluster + + - name: Create artifact directory + run: mkdir -p "$ARTIFACT_DIR" + + - name: Retrieve roxie version from repository + id: roxie-version + run: echo "version=$(cat ROXIE_VERSION)" >> "$GITHUB_OUTPUT" + + - name: Install roxie + uses: stackrox/actions/roxie/install-cli@bcda9b2d6d2b644c267e699479656973e84ef7fa # ratchet:stackrox/actions@v1 + with: + version: "v${{ steps.roxie-version.outputs.version }}" + gh-token: ${{ github.token }} + + - name: Run pre-test + shell: python + env: + QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: | + import sys + sys.path.append('.openshift-ci') + from pre_tests import PreSystemTests + PreSystemTests().run() + + - name: Run BYODB test + env: + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ECR_REGISTRY_REGION: ${{ secrets.AWS_ECR_REGISTRY_REGION }} + AWS_ECR_REGISTRY_NAME: ${{ secrets.AWS_ECR_REGISTRY_NAME }} + AWS_ASSUME_ROLE_ACCESS_KEY_ID: ${{ secrets.AWS_ASSUME_ROLE_ACCESS_KEY_ID }} + AWS_ASSUME_ROLE_SECRET_ACCESS_KEY: ${{ secrets.AWS_ASSUME_ROLE_SECRET_ACCESS_KEY }} + AWS_ASSUME_ROLE_ROLE_ID: ${{ secrets.AWS_ASSUME_ROLE_ROLE_ID }} + AWS_ASSUME_ROLE_EXTERNAL_ID: ${{ secrets.AWS_ASSUME_ROLE_EXTERNAL_ID }} + AWS_ASSUME_ROLE_TEST_CONDITION_ID: ${{ secrets.AWS_ASSUME_ROLE_TEST_CONDITION_ID }} + AZURE_REGISTRY_PASSWORD: ${{ secrets.AZURE_REGISTRY_PASSWORD }} + GOOGLE_CREDENTIALS_GCR_SCANNER_V2: ${{ secrets.GOOGLE_CREDENTIALS_GCR_SCANNER_V2 }} + GOOGLE_CREDENTIALS_GCR_NO_ACCESS_KEY_V2: ${{ secrets.GOOGLE_CREDENTIALS_GCR_NO_ACCESS_KEY_V2 }} + GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2: ${{ secrets.GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2 }} + GHCR_REGISTRY_USER: ${{ secrets.GHCR_REGISTRY_USER }} + GHCR_REGISTRY_PASSWORD: ${{ secrets.GHCR_REGISTRY_PASSWORD }} + QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + REDHAT_USERNAME: ${{ secrets.RH_REGISTRY_USERNAME_RO }} + REDHAT_PASSWORD: ${{ secrets.RH_REGISTRY_PASSWORD_RO }} + SLACK_FIXABLE_VULNS_CHANNEL: ${{ secrets.SLACK_FIXABLE_VULNS_CHANNEL }} + SLACK_MAIN_WEBHOOK: ${{ secrets.SLACK_MAIN_WEBHOOK }} + SLACK_ALT_WEBHOOK: ${{ secrets.SLACK_ALT_WEBHOOK }} + OCM_OFFLINE_TOKEN: ${{ secrets.OCM_OFFLINE_TOKEN }} + CLOUD_SOURCES_TEST_OCM_CLIENT_ID: ${{ secrets.CLOUD_SOURCES_TEST_OCM_CLIENT_ID }} + CLOUD_SOURCES_TEST_OCM_CLIENT_SECRET: ${{ secrets.CLOUD_SOURCES_TEST_OCM_CLIENT_SECRET }} + BUILD_ID: ${{ github.run_id }} + USE_ROXIE_DEPLOY: true + timeout-minutes: 245 + run: tests/byodb/run.sh /tmp/byodb-test-logs + + - name: Persist env vars for post-test + if: always() + run: | + source tests/e2e/lib.sh + wait_for_api + echo "API_ENDPOINT=${API_ENDPOINT}" >> "$GITHUB_ENV" + echo "ROX_ADMIN_PASSWORD=$(cat deploy/k8s/central-deploy/password)" >> "$GITHUB_ENV" + + - name: Run post-test + if: always() + shell: python + env: + ROX_USERNAME: admin + run: | + import os, sys + sys.path.append('.openshift-ci') + from post_tests import PostClusterTest + post = PostClusterTest( + collect_central_artifacts=False, + ) + if os.environ.get("ORCHESTRATOR_FLAVOR") == "k8s": + post.openshift_namespaces = [] + post.run(['/tmp/byodb-test-logs', '${{ env.ARTIFACT_DIR }}']) + + - name: Run final-post + if: always() + shell: python + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import FinalPost + FinalPost(store_qa_tests_data=False).run() + + - name: Teardown GKE cluster + if: always() + continue-on-error: true + run: | + if [[ -z "${CLUSTER_NAME:-}" ]]; then + echo "No cluster to teardown" + exit 0 + fi + if ! scripts/ci/gke.sh teardown_gke_cluster false; then + echo "::warning::WARNING: Could not tear down cluster ${CLUSTER_NAME}. See Teardown step logs for details." + exit 6 + fi + + - name: Publish summary for failed tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "fail" + - name: Publish summary for passed and skipped tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "pass, skip" + + - name: Report junit failures in jira + if: (!cancelled()) + id: junit2jira + uses: ./.github/actions/junit2jira + with: + create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} + jira-token: ${{ secrets.JIRA_TOKEN }} + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + directory: ${{ env.ARTIFACT_DIR }} diff --git a/.github/workflows/e2e-db-backup-restore-tests.yaml b/.github/workflows/e2e-db-backup-restore-tests.yaml new file mode 100644 index 0000000000000..7ea0636f763b1 --- /dev/null +++ b/.github/workflows/e2e-db-backup-restore-tests.yaml @@ -0,0 +1,182 @@ +name: e2e-db-backup-restore-tests + +on: + workflow_call: + inputs: + tag: + description: Image tag to test. + type: string + required: true + should-run: + description: true if should really run the job. false - otherwise, i.e. the job should be skipped. + type: boolean + required: true + workflow_dispatch: + inputs: + tag: + description: Image tag to test. + type: string + required: true + +permissions: + contents: read + pull-requests: read + +defaults: + run: + shell: bash + +jobs: + gke-db-backup-restore-tests: + if: github.event_name == 'workflow_dispatch' || inputs.should-run + runs-on: ubuntu-24.04 + env: + MAIN_IMAGE_TAG: ${{ inputs.tag }} + BUILD_TAG: ${{ inputs.tag }} + CI_JOB_NAME: gke-db-backup-restore-tests + ARTIFACT_DIR: ./junit-reports + ROX_SCANNER_V4: "false" + STORAGE_CLASS: faster + PYTHONUNBUFFERED: "1" + timeout-minutes: 60 + steps: + + - name: Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + with: + install_components: "gke-gcloud-auth-plugin,beta" + + - name: Create GKE cluster + env: + MACHINE_TYPE: e2-standard-8 + NUM_NODES: "1" + GKE_SPOT: "false" + timeout-minutes: 20 + run: | + scripts/ci/gke.sh provision_gke_cluster dbbr + echo "KUBECONFIG=${HOME}/.kube/config" >> "$GITHUB_ENV" + + - name: Build roxctl + # The test uses roxctl for central backup/restore operations + run: | + make roxctl_linux-amd64 + echo "$(pwd)/bin/linux_amd64" >> "$GITHUB_PATH" + + - name: Retrieve roxie version from repository + id: roxie-version + run: echo "version=$(cat ROXIE_VERSION)" >> "$GITHUB_OUTPUT" + + - name: Install roxie + uses: stackrox/actions/roxie/install-cli@bcda9b2d6d2b644c267e699479656973e84ef7fa # ratchet:stackrox/actions@v1 + with: + version: "v${{ steps.roxie-version.outputs.version }}" + gh-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify kubectl access and wait for cluster to stabilize + timeout-minutes: 5 + run: | + kubectl cluster-info + kubectl get nodes + scripts/ci/gke.sh wait_for_cluster + + - name: Create artifact directory + run: mkdir -p "$ARTIFACT_DIR" + + - name: Run pre-test + shell: python + env: + QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + run: | + import sys + sys.path.append('.openshift-ci') + from pre_tests import PreSystemTests + PreSystemTests().run() + + - name: Deploy StackRox and run DB backup/restore test + env: + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: | + source tests/e2e/lib.sh + source tests/scripts/setup-certs.sh + source scripts/ci/sensor-wait.sh + + export_test_environment + setup_deployment_env false false + + roxie_config="$(mktemp)" + merge_yaml "$roxie_config" <<'EOF' + central: + resourceProfile: ci + securedCluster: + resourceProfile: ci + EOF + deploy_stackrox_with_roxie_compat "$roxie_config" + rm -f "$roxie_config" + + db_backup_and_restore_test /tmp/db-backup-restore-test + + - name: Run post-test + if: always() + shell: python + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import CheckStackroxLogs + CheckStackroxLogs( + check_for_errors_in_stackrox_logs=True, + artifact_destination_prefix="db-test", + ).run(['/tmp/db-backup-restore-test', '${{ env.ARTIFACT_DIR }}']) + + - name: Run final-post + if: always() + shell: python + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import FinalPost + FinalPost(store_qa_tests_data=False).run() + + - name: Teardown GKE cluster + if: always() + continue-on-error: true + run: | + if [[ -z "${CLUSTER_NAME:-}" ]]; then + echo "No cluster to teardown" + exit 0 + fi + if ! scripts/ci/gke.sh teardown_gke_cluster false; then + echo "::warning::WARNING: Could not tear down cluster ${CLUSTER_NAME}. See Teardown step logs for details." + exit 6 + fi + + - name: Publish summary for all tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: all + + - name: Report junit failures in jira + if: (!cancelled()) + id: junit2jira + uses: ./.github/actions/junit2jira + with: + create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} + jira-token: ${{ secrets.JIRA_TOKEN }} + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + directory: ${{ env.ARTIFACT_DIR }} diff --git a/.github/workflows/e2e-dispatch.yaml b/.github/workflows/e2e-dispatch.yaml new file mode 100644 index 0000000000000..4335fefeb05cb --- /dev/null +++ b/.github/workflows/e2e-dispatch.yaml @@ -0,0 +1,319 @@ +name: e2e-dispatch + +on: + push: + tags: + - '*' + branches: + - master + - release-* + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + +permissions: + contents: read + pull-requests: read + +defaults: + run: + shell: bash + +jobs: + detect-changes: + uses: ./.github/workflows/detect-changes.yml + + # Determines whether we should dispatch jobs that call real E2E workflows. + # This allows to skip running E2E tests on draft PRs without corresponding labels or on untrusted forks. + should-dispatch: + needs: [ detect-changes ] + runs-on: ubuntu-slim + outputs: + wait-for-gha-images: ${{ steps.determine.outputs.wait-for-gha-images }} + wait-for-konflux-images: ${{ steps.determine.outputs.wait-for-konflux-images }} + jobs: ${{ steps.determine.outputs.jobs }} + steps: + - name: Determine whether should dispatch workflows + id: determine + shell: python + env: + GH_TOKEN: ${{ github.token }} + PYTHONUNBUFFERED: "1" + input__event_name: ${{ github.event_name }} + input__is_fork: ${{ github.event.pull_request.head.repo.fork }} + input__is_draft: ${{ github.event.pull_request.draft }} + input__ui_only_change: ${{ needs.detect-changes.outputs.ui_only }} + input__repo: ${{ github.repository }} + input__pr_number: ${{ github.event.pull_request.number }} + run: | + import json + import os + import subprocess + from types import SimpleNamespace + from dataclasses import dataclass + + @dataclass + class Job: + name: str + require_pr_label: bool = False + require_konflux_build: bool = False + + def should_run(self, input): + if input.is_fork == 'true': + return False + + if input.event_name != 'pull_request': + return True + elif self.name in input.pr_labels: + return True + + # We are in a PR context, job is not explicitly listed among the PR labels. + + # TODO(ROX-21073): re-enable Konflux for all PR branches + if self.require_konflux_build and 'konflux-build' not in input.pr_labels: + return False + + if input.ui_only_change == 'true': + # For now, all the workflows that we dispatch should be skipped for PRs that only change UI. + # As we add more E2E test workflows, we may want to make this more flexible. + return False + if self.require_pr_label: + return False + if input.is_draft != 'true': + return True + + return False + + + input = SimpleNamespace(**{ + key.removeprefix('input__'): val + for (key, val) in os.environ.items() + if key.startswith('input__')}) + + if input.event_name != 'pull_request': + input.pr_labels = [] + else: + p = subprocess.run( + ['gh', 'pr', 'view', input.pr_number, '--repo', input.repo, '--json', 'labels', '--jq', '.labels[].name'], + check=True, encoding='utf-8', stdout=subprocess.PIPE, stdin=subprocess.DEVNULL) + input.pr_labels = p.stdout.strip().splitlines() + + for k, v in vars(input).items(): + print(f'{k} = {v}') + + jobs_registry = [ + Job(name='e2e-byodb-tests', require_pr_label=True), + Job(name='e2e-db-backup-restore-tests'), + Job(name='e2e-gke-upgrade-tests'), + Job(name='e2e-nongroovy-tests-gke'), + Job(name='e2e-nongroovy-tests-gke-konflux', require_konflux_build=True), + Job(name='e2e-qa-tests-gke', require_pr_label=True), + Job(name='e2e-qa-tests-gke-konflux', require_konflux_build=True), + Job(name='e2e-qa-tests-ocp-fips-konflux', require_konflux_build=True), + ] + + with open(os.environ['GITHUB_OUTPUT'], mode='at', encoding='utf-8') as file: + def write(job, val): + line = f'{job}={val}' + print(line) + file.write(line + '\n') + + jobs = {job.name: job.should_run(input) for job in jobs_registry} + + write('jobs', json.dumps(jobs)) + + # Find out when we can complete the workflow early by not even waiting for GHA-built images. + wait_for_gha_images = input.event_name != 'pull_request' or input.is_draft != 'true' or any(jobs.values()) + write('wait-for-gha-images', str(wait_for_gha_images).lower()) + + # Waiting for konflux-built images in PR request context requires the 'konflux-build' PR label. + wait_for_konflux_images = input.event_name != 'pull_request' or ('konflux-build' in input.pr_labels and any(jobs.values())) + write('wait-for-konflux-images', str(wait_for_konflux_images).lower()) + + wait-for-images: + needs: [ should-dispatch ] + if: needs.should-dispatch.outputs.wait-for-gha-images == 'true' + runs-on: ubuntu-24.04 + env: + MAIN_IMAGE_TAG_DIR: "." + IMAGE_PREFIX: "" + OPERATOR_IMAGE_PREFIX: "stackrox-" + DEV_TAG_SUFFIX: "" + # MAIN_IMAGE_TAG can be used for overriding when you want to run workflows for particular images. + # MAIN_IMAGE_TAG: 4.12.x-350-gc76d76b0fb + outputs: + tag: ${{ steps.image-references.outputs.tag }} + tag_suffix: ${{ steps.image-references.outputs.tag_suffix }} + concurrency: + group: wait-for-images-${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + steps: + - name: Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: ./.github/actions/handle-tagged-build + - name: Determine image references + id: image-references + run: &image-references | + IMAGE_TAG="${MAIN_IMAGE_TAG:-"$(make -C "$MAIN_IMAGE_TAG_DIR" --quiet --no-print-directory tag)"}" + echo "IMAGE_TAG=$IMAGE_TAG" | tee -a "$GITHUB_ENV" + echo "tag=$IMAGE_TAG" | tee -a "$GITHUB_OUTPUT" + + OPERATOR_IMAGE_TAG="$(BUILD_TAG="${IMAGE_TAG}" make -C operator --quiet --no-print-directory tag)" + echo "OPERATOR_IMAGE_TAG=${OPERATOR_IMAGE_TAG}" | tee -a "$GITHUB_ENV" + + # Mirrors the logic in https://github.com/stackrox/konflux-tasks/blob/main/tasks/determine-image-tag-task.yaml. + source_branch="${GITHUB_HEAD_REF:-}" + target_branch="${GITHUB_BASE_REF:-${GITHUB_REF}}" + TAG_SUFFIX="" + if [[ "${source_branch}" == *konflux-release-like* ]]; then + echo "Source branch contains 'konflux-release-like', suppressing suffix." + elif ! grep -qE '^((refs/heads/)?release-[0-9a-z]+\.[0-9a-z]+|refs/tags/[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?)$' <<< "${target_branch}"; then + TAG_SUFFIX="${DEV_TAG_SUFFIX}" + fi + echo "TAG_SUFFIX=${TAG_SUFFIX}" | tee -a "$GITHUB_ENV" + echo "tag_suffix=${TAG_SUFFIX}" | tee -a "$GITHUB_OUTPUT" + + - name: Wait for images + uses: stackrox/actions/release/wait-for-image@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/release/wait-for-image@v1 + with: + token: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + limit: 1800 # Seconds, corresponding to 30m. + image: &stackrox-required-images | + rhacs-eng/${{ env.IMAGE_PREFIX }}central-db:${{ env.IMAGE_TAG }}${{ env.TAG_SUFFIX }} + rhacs-eng/${{ env.IMAGE_PREFIX }}collector:${{ env.IMAGE_TAG }}${{ env.TAG_SUFFIX }} + rhacs-eng/${{ env.IMAGE_PREFIX }}fact:${{ env.IMAGE_TAG }}${{ env.TAG_SUFFIX }} + rhacs-eng/${{ env.IMAGE_PREFIX }}main:${{ env.IMAGE_TAG }}${{ env.TAG_SUFFIX }} + rhacs-eng/${{ env.IMAGE_PREFIX }}roxctl:${{ env.IMAGE_TAG }}${{ env.TAG_SUFFIX }} + rhacs-eng/${{ env.IMAGE_PREFIX }}scanner-v4-db:${{ env.IMAGE_TAG }}${{ env.TAG_SUFFIX }} + rhacs-eng/${{ env.IMAGE_PREFIX }}scanner-v4:${{ env.IMAGE_TAG }}${{ env.TAG_SUFFIX }} + rhacs-eng/${{ env.OPERATOR_IMAGE_PREFIX }}operator-bundle:v${{ env.OPERATOR_IMAGE_TAG }}${{ env.TAG_SUFFIX }} + rhacs-eng/${{ env.OPERATOR_IMAGE_PREFIX }}operator:${{ env.OPERATOR_IMAGE_TAG }}${{ env.TAG_SUFFIX }} + + wait-for-konflux-images: + needs: [ should-dispatch ] + if: needs.should-dispatch.outputs.wait-for-konflux-images == 'true' + runs-on: ubuntu-24.04 + env: + MAIN_IMAGE_TAG_DIR: "operator" + IMAGE_PREFIX: "release-" + OPERATOR_IMAGE_PREFIX: "release-" + DEV_TAG_SUFFIX: "-fast" + outputs: + tag: ${{ steps.image-references.outputs.tag }} + tag_suffix: ${{ steps.image-references.outputs.tag_suffix }} + concurrency: + group: wait-for-konflux-images-${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + steps: + - name: Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: ./.github/actions/handle-tagged-build + - name: Determine image references + id: image-references + run: *image-references + + - name: Wait for Konflux images + uses: stackrox/actions/release/wait-for-image@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/release/wait-for-image@v1 + with: + token: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + image: *stackrox-required-images + limit: 14100 # Seconds, corresponding to 3h55m, which is the timeout for building the operator-index (.tekton/operator-index-build.yaml) + 5m buffer. + + e2e-byodb-tests: + needs: [ wait-for-images, should-dispatch ] + uses: ./.github/workflows/e2e-byodb-tests.yaml + with: + tag: ${{ needs.wait-for-images.outputs.tag }} + should-run: ${{ fromJSON(needs.should-dispatch.outputs.jobs).e2e-byodb-tests }} + secrets: inherit + concurrency: + group: e2e-byodb-tests-${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + + e2e-db-backup-restore-tests: + needs: [ wait-for-images, should-dispatch ] + uses: ./.github/workflows/e2e-db-backup-restore-tests.yaml + with: + tag: ${{ needs.wait-for-images.outputs.tag }} + should-run: ${{ fromJSON(needs.should-dispatch.outputs.jobs).e2e-db-backup-restore-tests }} + secrets: inherit + concurrency: + group: e2e-db-backup-restore-tests-${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + + e2e-gke-upgrade-tests: + needs: [ wait-for-images, should-dispatch ] + uses: ./.github/workflows/e2e-gke-upgrade-tests.yaml + with: + tag: ${{ needs.wait-for-images.outputs.tag }} + should-run: ${{ fromJSON(needs.should-dispatch.outputs.jobs).e2e-gke-upgrade-tests }} + secrets: inherit + concurrency: + group: e2e-gke-upgrade-tests-${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + + e2e-nongroovy-tests-gke: + needs: [ wait-for-images, should-dispatch ] + uses: ./.github/workflows/e2e-nongroovy-tests-gke.yaml + with: + tag: ${{ needs.wait-for-images.outputs.tag }} + should-run: ${{ fromJSON(needs.should-dispatch.outputs.jobs).e2e-nongroovy-tests-gke }} + secrets: inherit + concurrency: + group: e2e-nongroovy-tests-gke-${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + + e2e-nongroovy-tests-gke-konflux: + needs: [ wait-for-konflux-images, should-dispatch ] + uses: ./.github/workflows/e2e-nongroovy-tests-gke-konflux.yaml + with: + tag: ${{ needs.wait-for-konflux-images.outputs.tag }}${{ needs.wait-for-konflux-images.outputs.tag_suffix }} + should-run: ${{ fromJSON(needs.should-dispatch.outputs.jobs).e2e-nongroovy-tests-gke-konflux }} + secrets: inherit + concurrency: + group: e2e-nongroovy-tests-gke-konflux-${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + + e2e-qa-tests-gke: + needs: [ wait-for-images, should-dispatch ] + uses: ./.github/workflows/e2e-qa-tests-gke.yaml + with: + tag: ${{ needs.wait-for-images.outputs.tag }} + should-run: ${{ fromJSON(needs.should-dispatch.outputs.jobs).e2e-qa-tests-gke }} + secrets: inherit + concurrency: + group: e2e-qa-tests-gke-${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + + e2e-qa-tests-ocp-fips-konflux: + needs: [ wait-for-konflux-images, should-dispatch ] + uses: ./.github/workflows/e2e-qa-tests-ocp-fips-konflux.yaml + with: + tag: ${{ needs.wait-for-konflux-images.outputs.tag }}${{ needs.wait-for-konflux-images.outputs.tag_suffix }} + should-run: ${{ fromJSON(needs.should-dispatch.outputs.jobs).e2e-qa-tests-ocp-fips-konflux }} + secrets: inherit + concurrency: + group: e2e-qa-tests-ocp-fips-konflux-${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + + e2e-qa-tests-gke-konflux: + needs: [ wait-for-konflux-images, should-dispatch ] + uses: ./.github/workflows/e2e-qa-tests-gke-konflux.yaml + with: + tag: ${{ needs.wait-for-konflux-images.outputs.tag }}${{ needs.wait-for-konflux-images.outputs.tag_suffix }} + should-run: ${{ fromJSON(needs.should-dispatch.outputs.jobs).e2e-qa-tests-gke-konflux }} + secrets: inherit + concurrency: + group: e2e-qa-tests-gke-konflux-${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true diff --git a/.github/workflows/e2e-gke-upgrade-tests.yaml b/.github/workflows/e2e-gke-upgrade-tests.yaml new file mode 100644 index 0000000000000..bf779c3a02626 --- /dev/null +++ b/.github/workflows/e2e-gke-upgrade-tests.yaml @@ -0,0 +1,502 @@ +name: e2e-gke-upgrade-tests + +on: + workflow_call: + inputs: + tag: + description: Image tag to test. + type: string + required: true + should-run: + description: true if should really run the job. false - otherwise, i.e. the job should be skipped. + type: boolean + required: true + workflow_dispatch: + inputs: + tag: + description: Image tag to test. + type: string + required: true + +permissions: + contents: read + pull-requests: read + +defaults: + run: + shell: bash + +jobs: + + # Verifies Central DB upgrades/rollbacks. + gke-upgrade-tests-central: + if: github.event_name == 'workflow_dispatch' || inputs.should-run + runs-on: ubuntu-24.04 + env: + MAIN_IMAGE_TAG: ${{ inputs.tag }} + BUILD_TAG: ${{ inputs.tag }} + # Not making CI_JOB_NAME match *gke-upgrade-tests because JUnit reporting will be broken since the former + # gke-upgrade-tests expect all sensor, central and postgres tests to run together in a single sequence. + CI_JOB_NAME: gke-upgrade-tests-central + ARTIFACT_DIR: ./junit-reports + # Make sure Python steps don't buffer stdout/stderr. + PYTHONUNBUFFERED: "1" + timeout-minutes: 180 # Keep in sync with cluster's lifespan + steps: + + - name: Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Create GKE cluster + env: + MACHINE_TYPE: e2-standard-8 + # Leave room for the scale workload alongside both Scanner versions. + NUM_NODES: "4" + GKE_SPOT: "false" + timeout-minutes: 20 + run: | + scripts/ci/gke.sh provision_gke_cluster cent-up + echo "KUBECONFIG=${HOME}/.kube/config" >> "$GITHUB_ENV" + + - name: Docker login to Quay.io + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: docker login -u "${REGISTRY_USERNAME}" --password-stdin quay.io <<<"${REGISTRY_PASSWORD}" + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + with: + install_components: "gke-gcloud-auth-plugin" + + - name: Build roxctl + run: make roxctl_linux-amd64 + + - name: Retrieve roxie version from repository + id: roxie-version + run: echo "version=$(cat ROXIE_VERSION)" >> "$GITHUB_OUTPUT" + + - name: Install roxie + uses: stackrox/actions/roxie/install-cli@bcda9b2d6d2b644c267e699479656973e84ef7fa # ratchet:stackrox/actions@v1 + with: + version: "v${{ steps.roxie-version.outputs.version }}" + gh-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify kubectl access and wait for cluster to stabilize + timeout-minutes: 5 + run: | + kubectl cluster-info + kubectl get nodes + scripts/ci/gke.sh wait_for_cluster + + - name: Create artifact directory + run: mkdir -p "$ARTIFACT_DIR" + + - name: Run pre-test + shell: python + env: + QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + run: | + import sys + sys.path.append('.openshift-ci') + from pre_tests import PreSystemTests + PreSystemTests().run() + + - name: Fix GOPATH and PATH for building and running roxctl during tests + run: | + GOPATH="$(go env GOPATH)" + echo "GOPATH=${GOPATH}" | tee -a "${GITHUB_ENV}" + echo "${GOPATH}/bin" | tee -a "${GITHUB_PATH}" + + - name: Run central upgrade test + env: + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + GOOGLE_CREDENTIALS_GCR_SCANNER_V2: ${{ secrets.GOOGLE_CREDENTIALS_GCR_SCANNER_V2 }} + GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2: ${{ secrets.GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2 }} + REDHAT_USERNAME: ${{ secrets.RH_REGISTRY_USERNAME_RO }} + REDHAT_PASSWORD: ${{ secrets.RH_REGISTRY_PASSWORD_RO }} + run: | + tests/upgrade/postgres_run.sh /tmp/postgres-upgrade-test-logs + # This makes sure Stackrox_Deployment / Was Stackrox deployed to the cluster? test does not fail. + touch /tmp/stackrox_ci_state_deployed + + - name: Run post-test + if: always() + shell: python + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import PostClusterTest + PostClusterTest(collect_central_artifacts=True).run(['/tmp/postgres-upgrade-test-logs', '${{ env.ARTIFACT_DIR }}']) + + - name: Run final-post + if: always() + shell: python + env: + QA_TEST_DEBUG_LOGS: /tmp/qa-tests-backend-logs + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import FinalPost + FinalPost(store_qa_tests_data=True).run() + + - name: Teardown GKE cluster + if: always() + continue-on-error: true + run: | + if [[ -z "${CLUSTER_NAME:-}" ]]; then + echo "No cluster to teardown" + exit 0 + fi + if ! scripts/ci/gke.sh teardown_gke_cluster false; then + echo "::warning::WARNING: Could not tear down cluster ${CLUSTER_NAME}. See Teardown step logs for details." + exit 6 + fi + + - name: Publish summary for failed tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "fail" + - name: Publish summary for passed and skipped tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "pass, skip" + + - name: Report junit failures in jira + if: (!cancelled()) + id: junit2jira + uses: ./.github/actions/junit2jira + with: + create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} + jira-token: ${{ secrets.JIRA_TOKEN }} + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + directory: ${{ env.ARTIFACT_DIR }} + + check-postgres-upgrade: + if: github.event_name == 'workflow_dispatch' || inputs.should-run + runs-on: ubuntu-latest + outputs: + required: ${{ steps.check.outputs.required }} + steps: + - name: Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Check whether PostgreSQL needs a major-version upgrade + id: check + run: tests/upgrade/postgres_upgrade_run.sh --check + + # Verifies we can do Postgres major version upgrade. + gke-upgrade-tests-postgres: + needs: check-postgres-upgrade + if: needs.check-postgres-upgrade.outputs.required == 'true' + runs-on: ubuntu-24.04 + env: + MAIN_IMAGE_TAG: ${{ inputs.tag }} + BUILD_TAG: ${{ inputs.tag }} + # Not making CI_JOB_NAME match *gke-upgrade-tests because JUnit reporting will be broken since the former + # gke-upgrade-tests expect all sensor, central and postgres tests to run together in a single sequence. + CI_JOB_NAME: gke-upgrade-tests-postgres + ARTIFACT_DIR: ./junit-reports + # Make sure Python steps don't buffer stdout/stderr. + PYTHONUNBUFFERED: "1" + timeout-minutes: 180 # Keep in sync with cluster's lifespan + steps: + + - name: Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Create GKE cluster + env: + MACHINE_TYPE: e2-standard-8 + # Leave room for the scale workload alongside both Scanner versions. + NUM_NODES: "4" + GKE_SPOT: "false" + timeout-minutes: 20 + run: | + scripts/ci/gke.sh provision_gke_cluster pg-up + echo "KUBECONFIG=${HOME}/.kube/config" >> "$GITHUB_ENV" + + - name: Docker login to Quay.io + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: docker login -u "${REGISTRY_USERNAME}" --password-stdin quay.io <<<"${REGISTRY_PASSWORD}" + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + with: + install_components: "gke-gcloud-auth-plugin" + + - name: Build roxctl + run: make roxctl_linux-amd64 + + - name: Retrieve roxie version from repository + id: roxie-version + run: echo "version=$(cat ROXIE_VERSION)" >> "$GITHUB_OUTPUT" + + - name: Install roxie + uses: stackrox/actions/roxie/install-cli@bcda9b2d6d2b644c267e699479656973e84ef7fa # ratchet:stackrox/actions@v1 + with: + version: "v${{ steps.roxie-version.outputs.version }}" + gh-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify kubectl access and wait for cluster to stabilize + timeout-minutes: 5 + run: | + kubectl cluster-info + kubectl get nodes + scripts/ci/gke.sh wait_for_cluster + + - name: Create artifact directory + run: mkdir -p "$ARTIFACT_DIR" + + - name: Run pre-test + shell: python + env: + QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + run: | + import sys + sys.path.append('.openshift-ci') + from pre_tests import PreSystemTests + PreSystemTests().run() + + - name: Fix GOPATH and PATH for building and running roxctl during tests + run: | + GOPATH="$(go env GOPATH)" + echo "GOPATH=${GOPATH}" | tee -a "${GITHUB_ENV}" + echo "${GOPATH}/bin" | tee -a "${GITHUB_PATH}" + + - name: Run postgres version upgrade test + env: + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + GOOGLE_CREDENTIALS_GCR_SCANNER_V2: ${{ secrets.GOOGLE_CREDENTIALS_GCR_SCANNER_V2 }} + REDHAT_USERNAME: ${{ secrets.RH_REGISTRY_USERNAME_RO }} + REDHAT_PASSWORD: ${{ secrets.RH_REGISTRY_PASSWORD_RO }} + run: | + tests/upgrade/postgres_upgrade_run.sh /tmp/postgres-version-upgrade-test-logs + # This makes sure Stackrox_Deployment / Was Stackrox deployed to the cluster? test does not fail. + touch /tmp/stackrox_ci_state_deployed + + - name: Run post-test + if: always() + shell: python + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import PostClusterTest + PostClusterTest(collect_central_artifacts=True).run(['/tmp/postgres-version-upgrade-test-logs', '${{ env.ARTIFACT_DIR }}']) + + - name: Run final-post + if: always() + shell: python + env: + QA_TEST_DEBUG_LOGS: /tmp/qa-tests-backend-logs + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import FinalPost + FinalPost(store_qa_tests_data=True).run() + + - name: Teardown GKE cluster + if: always() + continue-on-error: true + run: | + if [[ -z "${CLUSTER_NAME:-}" ]]; then + echo "No cluster to teardown" + exit 0 + fi + if ! scripts/ci/gke.sh teardown_gke_cluster false; then + echo "::warning::WARNING: Could not tear down cluster ${CLUSTER_NAME}. See Teardown step logs for details." + exit 6 + fi + + - name: Publish summary for failed tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "fail" + - name: Publish summary for passed and skipped tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "pass, skip" + + - name: Report junit failures in jira + if: (!cancelled()) + id: junit2jira + uses: ./.github/actions/junit2jira + with: + create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} + jira-token: ${{ secrets.JIRA_TOKEN }} + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + directory: ${{ env.ARTIFACT_DIR }} + + # Verifies Sensor/Secured Cluster `upgrader` (applicable to manifest installations only). + gke-upgrade-tests-sensor: + if: github.event_name == 'workflow_dispatch' || inputs.should-run + runs-on: ubuntu-24.04 + env: + MAIN_IMAGE_TAG: ${{ inputs.tag }} + BUILD_TAG: ${{ inputs.tag }} + # Not making CI_JOB_NAME match *gke-upgrade-tests because JUnit reporting will be broken since the former + # gke-upgrade-tests expect all sensor, central and postgres tests to run together in a single sequence. + CI_JOB_NAME: gke-upgrade-tests-sensor + ARTIFACT_DIR: ./junit-reports + # Make sure Python steps don't buffer stdout/stderr. + PYTHONUNBUFFERED: "1" + timeout-minutes: 60 # Keep in sync with cluster's lifespan + steps: + + - name: Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Create GKE cluster + env: + MACHINE_TYPE: e2-standard-8 + NUM_NODES: "1" + GKE_SPOT: "false" + timeout-minutes: 20 + run: | + scripts/ci/gke.sh provision_gke_cluster sens-up + echo "KUBECONFIG=${HOME}/.kube/config" >> "$GITHUB_ENV" + + - name: Docker login to Quay.io + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: docker login -u "${REGISTRY_USERNAME}" --password-stdin quay.io <<<"${REGISTRY_PASSWORD}" + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + with: + install_components: "gke-gcloud-auth-plugin" + + - name: Build roxctl and upgrader + run: make roxctl_linux-amd64 upgrader + + - name: Verify kubectl access and wait for cluster to stabilize + timeout-minutes: 5 + run: | + kubectl cluster-info + kubectl get nodes + scripts/ci/gke.sh wait_for_cluster + + - name: Create artifact directory + run: mkdir -p "$ARTIFACT_DIR" + + - name: Run pre-test + shell: python + env: + QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + run: | + import sys + sys.path.append('.openshift-ci') + from pre_tests import PreSystemTests + PreSystemTests().run() + + - name: Run sensor upgrade test + env: + LONGTERM_LICENSE: todo - remove me + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + LOAD_BALANCER: lb + run: tests/upgrade/postgres_sensor_run.sh /tmp/postgres-sensor-upgrade-test-logs + + - name: Run post-test + if: always() + shell: python + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import PostClusterTest + # The particular test removes StackRox resources, therefore not collecting their artifacts. + PostClusterTest(collect_central_artifacts=False).run(['/tmp/postgres-sensor-upgrade-test-logs', '${{ env.ARTIFACT_DIR }}']) + + - name: Run final-post + if: always() + shell: python + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import FinalPost + FinalPost(store_qa_tests_data=False).run() + + - name: Teardown GKE cluster + if: always() + continue-on-error: true + run: | + if [[ -z "${CLUSTER_NAME:-}" ]]; then + echo "No cluster to teardown" + exit 0 + fi + if ! scripts/ci/gke.sh teardown_gke_cluster false; then + echo "::warning::WARNING: Could not tear down cluster ${CLUSTER_NAME}. See Teardown step logs for details." + exit 6 + fi + + - name: Publish summary for failed tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "fail" + - name: Publish summary for passed and skipped tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "pass, skip" + + - name: Report junit failures in jira + if: (!cancelled()) + id: junit2jira + uses: ./.github/actions/junit2jira + with: + create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} + jira-token: ${{ secrets.JIRA_TOKEN }} + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + directory: ${{ env.ARTIFACT_DIR }} diff --git a/.github/workflows/e2e-nongroovy-tests-gke-konflux.yaml b/.github/workflows/e2e-nongroovy-tests-gke-konflux.yaml new file mode 100644 index 0000000000000..1a7c511cb7970 --- /dev/null +++ b/.github/workflows/e2e-nongroovy-tests-gke-konflux.yaml @@ -0,0 +1,38 @@ +name: e2e-nongroovy-tests-gke-konflux + +on: + workflow_call: + inputs: + tag: + description: | + Image tag to test. The caller needs to make sure that the tag carries the + "-fast" Konflux suffix if required (e.g. for development builds). + type: string + required: true + should-run: + description: true if should really run the job. false - otherwise, i.e. the job should be skipped. + type: boolean + required: true + workflow_dispatch: + inputs: + tag: + description: | + Image tag to test. You need to make sure that the tag carries the + "-fast" Konflux suffix if required (e.g. for development builds). + type: string + required: true + +permissions: + contents: read + pull-requests: read + +jobs: + run: + uses: ./.github/workflows/e2e-nongroovy-tests.yaml + with: + should-run: ${{ github.event_name == 'workflow_dispatch' || inputs.should-run }} + tag: ${{ inputs.tag }} + ci-job-name: gke-nongroovy-e2e-tests-konflux + extra-env-vars: | + USE_KONFLUX_IMAGES=true + secrets: inherit diff --git a/.github/workflows/e2e-nongroovy-tests-gke.yaml b/.github/workflows/e2e-nongroovy-tests-gke.yaml new file mode 100644 index 0000000000000..d0370a2f6ace0 --- /dev/null +++ b/.github/workflows/e2e-nongroovy-tests-gke.yaml @@ -0,0 +1,32 @@ +name: e2e-nongroovy-tests-gke + +on: + workflow_call: + inputs: + tag: + description: Image tag to test. + type: string + required: true + should-run: + description: true if should really run the job. false - otherwise, i.e. the job should be skipped. + type: boolean + required: true + workflow_dispatch: + inputs: + tag: + description: Image tag to test. + type: string + required: true + +permissions: + contents: read + pull-requests: read + +jobs: + run: + uses: ./.github/workflows/e2e-nongroovy-tests.yaml + with: + should-run: ${{ github.event_name == 'workflow_dispatch' || inputs.should-run }} + tag: ${{ inputs.tag }} + ci-job-name: gke-nongroovy-e2e-tests + secrets: inherit diff --git a/.github/workflows/e2e-nongroovy-tests.yaml b/.github/workflows/e2e-nongroovy-tests.yaml new file mode 100644 index 0000000000000..768d5dfea1439 --- /dev/null +++ b/.github/workflows/e2e-nongroovy-tests.yaml @@ -0,0 +1,244 @@ +name: e2e-nongroovy-tests + +on: + workflow_call: + inputs: + tag: + description: Image tag to test. + type: string + required: true + should-run: + description: true if should really run the job. false - otherwise, i.e. the job should be skipped. + type: boolean + required: true + ci-job-name: + description: Unique job name, e.g. gke-nongroovy-e2e-tests-konflux. Propagated to tests as CI_JOB_NAME for test dispatching. + type: string + required: true + extra-env-vars: + description: Additional environment variables as multi-line KEY=VALUE pairs (e.g., USE_ROXIE_DEPLOY=true) + type: string + required: false + +permissions: + contents: read + pull-requests: read + +defaults: + run: + shell: bash + +jobs: + gke-nongroovy-e2e-tests: + if: inputs.should-run + runs-on: ubuntu-24.04 + env: + MAIN_IMAGE_TAG: ${{ inputs.tag }} + BUILD_TAG: ${{ inputs.tag }} + GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + CI_JOB_NAME: ${{ inputs.ci-job-name }} + ARTIFACT_DIR: ./junit-reports + LOAD_BALANCER: lb + ORCHESTRATOR_FLAVOR: k8s + KUBERNETES_PROVIDER: gke + SENSOR_SCANNER_V4_SUPPORT: "true" + ROX_DEPLOY_SENSOR_WITH_CRS: "true" + SENSOR_HELM_MANAGED: "true" + STORAGE: pvc + STORAGE_CLASS: faster + SCANNER_V4_DB_STORAGE_CLASS: faster + ROX_SCANNER_V4: "true" + PYTHONUNBUFFERED: "1" + timeout-minutes: 361 + steps: + + - name: Checkout repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Set additional env vars + if: inputs.extra-env-vars != '' + env: + EXTRA_ENV_VARS: ${{ inputs.extra-env-vars }} + run: | + while IFS= read -r line; do + if [[ -n "$line" ]]; then + echo "$line" >> "$GITHUB_ENV" + fi + done <<< "$EXTRA_ENV_VARS" + + - name: Docker login to Quay.io + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: docker login -u "${REGISTRY_USERNAME}" --password-stdin quay.io <<<"${REGISTRY_PASSWORD}" + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + with: + install_components: "gke-gcloud-auth-plugin,beta" + + - name: Create GKE cluster + env: + MACHINE_TYPE: e2-standard-8 + NUM_NODES: "1" + GKE_SPOT: "false" + timeout-minutes: 20 + run: | + scripts/ci/gke.sh provision_gke_cluster nongroovy + echo "KUBECONFIG=${HOME}/.kube/config" >> "$GITHUB_ENV" + + - name: Retrieving roxie version from repository + id: roxie-version + run: echo "version=$(cat ROXIE_VERSION)" >> "$GITHUB_OUTPUT" + + - name: Install roxie + uses: stackrox/actions/roxie/install-cli@43bb707aa4f32ead5bfb8d5ab11fb409f971ffc8 # ratchet:stackrox/actions@v1 + with: + version: "v${{ steps.roxie-version.outputs.version }}" + + - name: Build roxctl + # roxctl is used for generate (deploy) and CLI e2e tests (token-file, etc). + # Copy to /usr/local/bin so it survives bats tests which mv the binary out of bin/. + # A release Central must be generated with a release roxctl; otherwise generate + # copies ROX_* into the bundle and TestFeatureFlagSettings cannot fail the way + # nightlies do. + run: | + if [[ "${GITHUB_REF}" =~ ^refs/tags/ ]] || [[ "${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-release-build') }}" == "true" ]]; then + export GOTAGS=release + echo "Building release roxctl (tag or ci-release-build)" + fi + make roxctl_linux-amd64 + sudo cp bin/linux_amd64/roxctl /usr/local/bin/roxctl + echo "${HOME}/go/bin" >> "$GITHUB_PATH" + + - name: Verify kubectl access and wait for cluster to stabilize + timeout-minutes: 5 + run: | + kubectl cluster-info + kubectl get nodes + scripts/ci/gke.sh wait_for_cluster + + - name: Create artifact directory + run: mkdir -p "$ARTIFACT_DIR" + + - name: Run pre-test + shell: python + env: + QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: | + import sys + sys.path.append('.openshift-ci') + from pre_tests import PreSystemTests + PreSystemTests().run() + + - name: Install bats + run: | + sudo npm install -g bats bats-support bats-assert + sudo mkdir -p /usr/lib/node_modules + sudo ln -s /usr/local/lib/node_modules/bats-support /usr/lib/node_modules/bats-support + sudo ln -s /usr/local/lib/node_modules/bats-assert /usr/lib/node_modules/bats-assert + + - name: Run e2e tests + # SETUP_WORKLOAD_IDENTITIES is intentionally omitted; WIF tests run in the Prow pipeline. + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_S3_BACKUP_TEST_BUCKET_NAME: ${{ secrets.AWS_S3_BACKUP_TEST_BUCKET_NAME }} + AWS_S3_BACKUP_TEST_BUCKET_REGION: ${{ secrets.AWS_S3_BACKUP_TEST_BUCKET_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + BUILD_ID: ${{ github.run_id }} + CLOUDFLARE_R2_BACKUP_TEST_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_BACKUP_TEST_ACCESS_KEY_ID }} + CLOUDFLARE_R2_BACKUP_TEST_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_BACKUP_TEST_ACCOUNT_ID }} + CLOUDFLARE_R2_BACKUP_TEST_BUCKET_NAME: ${{ secrets.CLOUDFLARE_R2_BACKUP_TEST_BUCKET_NAME }} + CLOUDFLARE_R2_BACKUP_TEST_REGION: ${{ secrets.CLOUDFLARE_R2_BACKUP_TEST_REGION }} + CLOUDFLARE_R2_BACKUP_TEST_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_BACKUP_TEST_SECRET_ACCESS_KEY }} + GCP_GCS_BACKUP_TEST_BUCKET_NAME_V2: ${{ secrets.GCP_GCS_BACKUP_TEST_BUCKET_NAME_V2 }} + GOOGLE_GCS_BACKUP_SERVICE_ACCOUNT_V2: ${{ secrets.GOOGLE_GCS_BACKUP_SERVICE_ACCOUNT_V2 }} + ODF_S3_BACKUP_TEST_ACCESS_KEY_ID: ${{ secrets.ODF_S3_BACKUP_TEST_ACCESS_KEY_ID }} + ODF_S3_BACKUP_TEST_BUCKET_NAME: ${{ secrets.ODF_S3_BACKUP_TEST_BUCKET_NAME }} + ODF_S3_BACKUP_TEST_ENDPOINT: ${{ secrets.ODF_S3_BACKUP_TEST_ENDPOINT }} + ODF_S3_BACKUP_TEST_REGION: ${{ secrets.ODF_S3_BACKUP_TEST_REGION }} + ODF_S3_BACKUP_TEST_SECRET_ACCESS_KEY: ${{ secrets.ODF_S3_BACKUP_TEST_SECRET_ACCESS_KEY }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + timeout-minutes: 245 + run: tests/e2e/run.sh /tmp/e2e-test-logs + + - name: Ensure central stability before post-test + if: always() + run: | + source tests/e2e/lib.sh + wait_for_api + + - name: Run post-test + if: always() + shell: python + env: + ROX_USERNAME: admin + run: | + import os, sys + sys.path.append('.openshift-ci') + from post_tests import PostClusterTest + post = PostClusterTest( + check_stackrox_logs=False, + ) + if os.environ.get("ORCHESTRATOR_FLAVOR") == "k8s": + post.openshift_namespaces = [] + post.run(['/tmp/e2e-test-logs', '${{ env.ARTIFACT_DIR }}']) + + - name: Run final-post + if: always() + shell: python + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import FinalPost + FinalPost(store_qa_tests_data=False).run() + + - name: Teardown GKE cluster + if: always() + continue-on-error: true + run: | + if [[ -z "${CLUSTER_NAME:-}" ]]; then + echo "No cluster to teardown" + exit 0 + fi + if ! scripts/ci/gke.sh teardown_gke_cluster false; then + echo "::warning::WARNING: Could not tear down cluster ${CLUSTER_NAME}. See Teardown step logs for details." + exit 6 + fi + + - name: Publish summary for failed tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "fail" + - name: Publish summary for passed and skipped tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "pass, skip" + + - name: Report junit failures in jira + if: (!cancelled()) + id: junit2jira + uses: ./.github/actions/junit2jira + with: + create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} + jira-token: ${{ secrets.JIRA_TOKEN }} + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + directory: ${{ env.ARTIFACT_DIR }} diff --git a/.github/workflows/e2e-qa-tests-gke-konflux.yaml b/.github/workflows/e2e-qa-tests-gke-konflux.yaml new file mode 100644 index 0000000000000..8b050adcc6ef9 --- /dev/null +++ b/.github/workflows/e2e-qa-tests-gke-konflux.yaml @@ -0,0 +1,40 @@ +name: e2e-qa-tests-gke-konflux + +on: + workflow_call: + inputs: + tag: + description: | + Image tag to test. The caller needs to make sure that the tag carries the + "-fast" Konflux suffix if required (e.g. for development builds). + type: string + required: true + should-run: + description: true if should really run the job. false - otherwise, i.e. the job should be skipped. + type: boolean + required: true + workflow_dispatch: + inputs: + tag: + description: | + Image tag to test. You need to make sure that the tag carries the + "-fast" Konflux suffix if required (e.g. for development builds). + type: string + required: true + +jobs: + run: + uses: ./.github/workflows/e2e-qa-tests.yaml + with: + should-run: ${{ github.event_name == 'workflow_dispatch' || inputs.should-run }} + tag: ${{ inputs.tag }} + ci-job-name: gke-qa-e2e-tests-konflux + cluster-name-prefix: e2e-qa-gke-konflux + cluster-provisioning-method: direct-gke + extra-env-vars: | + ORCHESTRATOR_FLAVOR=k8s + KUBERNETES_PROVIDER=gke + USE_KONFLUX_IMAGES=true + STORAGE_CLASS=stackrox-gke-ssd + SCANNER_V4_DB_STORAGE_CLASS=stackrox-gke-ssd + secrets: inherit diff --git a/.github/workflows/e2e-qa-tests-gke.yaml b/.github/workflows/e2e-qa-tests-gke.yaml new file mode 100644 index 0000000000000..6fb0aca91414d --- /dev/null +++ b/.github/workflows/e2e-qa-tests-gke.yaml @@ -0,0 +1,35 @@ +name: e2e-qa-tests-gke + +on: + workflow_call: + inputs: + tag: + description: Image tag to test. + type: string + required: true + should-run: + description: true if should really run the job. false - otherwise, i.e. the job should be skipped. + type: boolean + required: true + workflow_dispatch: + inputs: + tag: + description: Image tag to test. + type: string + required: true + +jobs: + run: + uses: ./.github/workflows/e2e-qa-tests.yaml + with: + should-run: ${{ github.event_name == 'workflow_dispatch' || inputs.should-run }} + tag: ${{ inputs.tag }} + ci-job-name: gke-qa-e2e-tests + cluster-name-prefix: e2e-qa-tests-gke + cluster-provisioning-method: direct-gke + extra-env-vars: | + ORCHESTRATOR_FLAVOR=k8s + KUBERNETES_PROVIDER=gke + STORAGE_CLASS=stackrox-gke-ssd + SCANNER_V4_DB_STORAGE_CLASS=stackrox-gke-ssd + secrets: inherit diff --git a/.github/workflows/e2e-qa-tests-ocp-fips-konflux.yaml b/.github/workflows/e2e-qa-tests-ocp-fips-konflux.yaml new file mode 100644 index 0000000000000..7670ec997452d --- /dev/null +++ b/.github/workflows/e2e-qa-tests-ocp-fips-konflux.yaml @@ -0,0 +1,52 @@ +name: e2e-qa-tests-ocp-fips-konflux + +on: + workflow_call: + inputs: + tag: + description: | + Image tag to test. The caller needs to make sure that the tag carries the + "-fast" Konflux suffix if required (e.g. for development builds). + type: string + required: true + should-run: + description: true if should really run the job. false - otherwise, i.e. the job should be skipped. + type: boolean + required: true + workflow_dispatch: + inputs: + tag: + description: | + Image tag to test. You need to make sure that the tag carries the + "-fast" Konflux suffix if required (e.g. for development builds). + type: string + required: true + +jobs: + run: + uses: ./.github/workflows/e2e-qa-tests.yaml + with: + should-run: ${{ github.event_name == 'workflow_dispatch' || inputs.should-run }} + tag: ${{ inputs.tag }} + ci-job-name: ocp-qa-e2e-tests-fips-konflux + cluster-name-prefix: e2e-qa-ocp-fps-knflx + cluster-provisioning-method: infra + cluster-wait-timeout-minutes: 60 + infra-flavor: openshift-4 + infra-args: | + openshift-version=ocp/stable + master-node-type=e2-standard-4 + master-node-count=3 + worker-node-type=e2-standard-8 + worker-node-count=3 + ssd-storage-class=true + fips-enabled=true + infra-lifespan: 5h + # TODO(ROX-36332): setup workload identities + # SETUP_WORKLOAD_IDENTITIES=true + extra-env-vars: | + ORCHESTRATOR_FLAVOR=openshift + KUBERNETES_PROVIDER=ocp + SFA_AGENT=true + USE_KONFLUX_IMAGES=true + secrets: inherit diff --git a/.github/workflows/e2e-qa-tests.yaml b/.github/workflows/e2e-qa-tests.yaml new file mode 100644 index 0000000000000..543d5288be977 --- /dev/null +++ b/.github/workflows/e2e-qa-tests.yaml @@ -0,0 +1,376 @@ +name: e2e-qa-tests + +on: + workflow_call: + inputs: + should-run: + description: "true if should really run the job. false - otherwise, i.e. the job should be skipped." + required: true + type: boolean + tag: + description: "Image tag to test." + required: true + type: string + ci-job-name: + description: "Unique job name, e.g. e2e-qa-tests-. Used to configure the image prefetcher and name uploaded artifacts. Propagated to tests as CI_JOB_NAME for test dispatching." + required: true + type: string + cluster-name-prefix: + description: "Used as prefix in provisioned cluster names. Maximum 20 characters." + required: true + type: string + cluster-provisioning-method: + description: "Selects cluster provisioning method to use (supported: direct-gke, infra)." + type: string + default: "direct-gke" + extra-env-vars: + description: "Environment variables for this flavor of QA E2E tests, as multi-line KEY=VALUE pairs" + required: false + type: string + infra-flavor: + description: "Infra cluster flavor, e.g. gke-default, ocp-4." + type: string + default: "gke-default" + infra-args: + description: "Infra cluster args as multi-line KEY=VALUE pairs, one per line." + type: string + default: | + nodes=3 + machine-type=e2-standard-8 + infra-lifespan: + description: "Infra cluster lifespan (auto-deleted after expiry)." + type: string + default: "5h" + cluster-wait-timeout-minutes: + description: "Timeout for waiting on cluster provisioning (OCP needs longer)." + type: number + default: 25 + job-timeout-minutes: + description: "Overall job timeout." + type: number + default: 361 + part1-timeout-minutes: + description: "Part 1 test timeout." + type: number + default: 245 + part2-timeout-minutes: + description: "Part 2 test timeout." + type: number + default: 35 + db-backup-restore-timeout-minutes: + description: "DB backup and restore test timeout." + type: number + default: 30 + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + qa-e2e-tests: + if: inputs.should-run + runs-on: ubuntu-24.04 + env: + MAIN_IMAGE_TAG: ${{ inputs.tag }} + BUILD_TAG: ${{ inputs.tag }} + CI_JOB_NAME: ${{ inputs.ci-job-name }} + ARTIFACT_DIR: ./junit-reports/ + USE_LOCAL_ROXCTL: "true" + ROX_RISK_REPROCESSING_INTERVAL: 15s + ROX_SENSOR_CONNECTION_RETRY_MAX_INTERVAL: 30s + GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + QA_TEST_DEBUG_LOGS: /tmp/qa-tests-backend-logs + PYTHONUNBUFFERED: "1" + CLUSTER_NAME_PREFIX: "${{ inputs.cluster-name-prefix }}" + CLUSTER_NAME_PREFIX_MAX: "20" + timeout-minutes: ${{ inputs.job-timeout-minutes }} + steps: + - name: Validate workflow input + run: | + case "${{ inputs.cluster-provisioning-method }}" in + direct-gke|infra) ;; + *) echo "::error::Unknown cluster-provisioning-method: '${{ inputs.cluster-provisioning-method }}'. Supported: direct-gke, infra"; exit 1 ;; + esac + + if [[ "${#CLUSTER_NAME_PREFIX}" -gt "$CLUSTER_NAME_PREFIX_MAX" ]]; then + echo "::error::Workflow input cluster-name-prefix must not be larger than $CLUSTER_NAME_PREFIX_MAX characters (is ${#CLUSTER_NAME_PREFIX} characters long)." + exit 1 + fi + + - name: Checkout repo + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # ratchet:actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Compute cluster name for infra-provisioned cluster + if: inputs.cluster-provisioning-method == 'infra' + # infractl enforces a 28-char limit for cluster names. + run: | + cluster_name_max=28 + + hash_length=$(( cluster_name_max - 1 - CLUSTER_NAME_PREFIX_MAX )) + hash_input="${{ github.run_id }}-${{ github.run_attempt }}" + hash="$(echo "$hash_input" | sha256sum)" + hash_truncated="${hash:0:$hash_length}" + + cluster_name="${CLUSTER_NAME_PREFIX}-${hash_truncated}" + echo "CLUSTER_NAME=${cluster_name}" >> "$GITHUB_ENV" + + - name: Prepare infra args + if: inputs.cluster-provisioning-method == 'infra' + id: prepare-infra-args + env: + INFRA_ARGS: ${{ inputs.infra-args }} + run: | + args=$(echo "$INFRA_ARGS" | grep -v '^$' | paste -sd ',' -) + echo "args=$args" >> "$GITHUB_OUTPUT" + + - name: Trigger cluster creation via infra + if: inputs.cluster-provisioning-method == 'infra' + uses: stackrox/actions/infra/create-cluster@d624eeae7073b3acb838aab7705730b2de58a4a0 # ratchet:stackrox/actions/infra/create-cluster@v1 + with: + token: ${{ secrets.INFRA_TOKEN }} + flavor: ${{ inputs.infra-flavor }} + name: ${{ env.CLUSTER_NAME }} + lifespan: ${{ inputs.infra-lifespan }} + args: ${{ steps.prepare-infra-args.outputs.args }} + wait: false + + - name: Set up GCloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + with: + install_components: "gke-gcloud-auth-plugin,beta" + + # In the direct-gke provisioning case, this step also exports CLUSTER_NAME + # into the environments. + - name: Create cluster via direct-gke + if: inputs.cluster-provisioning-method == 'direct-gke' + timeout-minutes: ${{ inputs.cluster-wait-timeout-minutes }} + env: + MACHINE_TYPE: "e2-standard-8" + NUM_NODES: "3" + GKE_SPOT: "true" + run: | + scripts/ci/gke.sh provision_gke_cluster "$CLUSTER_NAME_PREFIX" + echo "KUBECONFIG=${HOME}/.kube/config" >> "$GITHUB_ENV" + + - name: Docker login to Quay.io + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: docker login -u "${REGISTRY_USERNAME}" --password-stdin quay.io <<<"${REGISTRY_PASSWORD}" + + - name: Set flavor-specific env vars + env: + EXTRA_ENV_VARS: ${{ inputs.extra-env-vars }} + run: | + while IFS= read -r line; do + if [[ -n "$line" ]]; then + echo "$line" >> "$GITHUB_ENV" + fi + done <<< "$EXTRA_ENV_VARS" + + - name: Install roxctl + uses: stackrox/roxctl-installer-action@47fb4f5b275066b8322369e6e33fa010915b0d13 # ratchet:stackrox/roxctl-installer-action@v1 + with: + roxctl-release: latest + + - name: Retrieving roxie version from repository + id: roxie-version + run: echo "version=$(cat ROXIE_VERSION)" >> "$GITHUB_OUTPUT" + + - name: Install roxie + uses: stackrox/actions/roxie/install-cli@bcda9b2d6d2b644c267e699479656973e84ef7fa # ratchet:stackrox/actions@v1 + with: + version: "v${{ steps.roxie-version.outputs.version }}" + gh-token: ${{ github.token }} + + - name: Install oc + if: startsWith(inputs.infra-flavor, 'openshift') + run: | + curl -fsSL --retry 5 --retry-all-errors https://mirror.openshift.com/pub/openshift-v5/clients/ocp/stable/openshift-client-linux.tar.gz \ + | tar xz -C /usr/local/bin oc + oc version --client + + - uses: stackrox/actions/infra/install-infractl@3b27674f0fa69f2d8c31ab26aa63c9f76401c64e # ratchet:stackrox/actions/infra/install-infractl@v1 + if: inputs.cluster-provisioning-method == 'infra' + + - name: Wait for infra cluster and get kubeconfig + if: inputs.cluster-provisioning-method == 'infra' + env: + INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }} + timeout-minutes: ${{ inputs.cluster-wait-timeout-minutes }} + run: | + infractl wait "${CLUSTER_NAME}" + infractl artifacts "${CLUSTER_NAME}" --download-dir artifacts > /dev/null + echo "KUBECONFIG=${PWD}/artifacts/kubeconfig" >> "$GITHUB_ENV" + + # gcloud container clusters create is synchronous, so the cluster is + # already RUNNING at this point. wait_for_cluster provides additional + # guarantees by polling until all kube-system pods are ready. + - name: Wait for cluster to stabilize + timeout-minutes: 5 + env: + CLUSTER_PROVISIONING_METHOD: ${{ inputs.cluster-provisioning-method }} + run: | + kubectl cluster-info + kubectl get nodes + if [[ "$CLUSTER_PROVISIONING_METHOD" == "direct-gke" ]]; then + scripts/ci/gke.sh wait_for_cluster + fi + + - name: Create artifact directory + run: mkdir -p "$ARTIFACT_DIR" + + - name: Download Go module dependencies + # The qa-tests-backend Makefile copies scanner protos from the Go module + # cache (via go list -m). Unlike Prow's apollo-ci image, GHA runners + # don't have the cache pre-populated. + run: go mod download github.com/stackrox/scanner + + - name: Run pre-test + shell: python + env: + QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: | + import sys + sys.path.append('.openshift-ci') + from pre_tests import PreSystemTests + PreSystemTests().run() + + - name: Run Part 1 tests + env: &test_env + USE_ROXIE_DEPLOY: true + QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + REDHAT_USERNAME: ${{ secrets.REDHAT_USERNAME }} + REDHAT_PASSWORD: ${{ secrets.REDHAT_PASSWORD }} + GOOGLE_CREDENTIALS_GCR_SCANNER_V2: ${{ secrets.GOOGLE_CREDENTIALS_GCR_SCANNER_V2 }} + GOOGLE_CREDENTIALS_GCR_NO_ACCESS_KEY_V2: ${{ secrets.GOOGLE_CREDENTIALS_GCR_NO_ACCESS_KEY_V2 }} + GHCR_REGISTRY_USER: ${{ secrets.GHCR_REGISTRY_USER }} + GHCR_REGISTRY_PASSWORD: ${{ secrets.GHCR_REGISTRY_PASSWORD }} + AZURE_REGISTRY_PASSWORD: ${{ secrets.AZURE_REGISTRY_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ECR_REGISTRY_NAME: ${{ secrets.AWS_ECR_REGISTRY_NAME }} + AWS_ECR_REGISTRY_REGION: ${{ secrets.AWS_ECR_REGISTRY_REGION }} + AWS_ASSUME_ROLE_ACCESS_KEY_ID: ${{ secrets.AWS_ASSUME_ROLE_ACCESS_KEY_ID }} + AWS_ASSUME_ROLE_SECRET_ACCESS_KEY: ${{ secrets.AWS_ASSUME_ROLE_SECRET_ACCESS_KEY }} + AWS_ASSUME_ROLE_ROLE_ID: ${{ secrets.AWS_ASSUME_ROLE_ROLE_ID }} + AWS_ASSUME_ROLE_EXTERNAL_ID: ${{ secrets.AWS_ASSUME_ROLE_EXTERNAL_ID }} + AWS_ASSUME_ROLE_TEST_CONDITION_ID: ${{ secrets.AWS_ASSUME_ROLE_TEST_CONDITION_ID }} + SLACK_FIXABLE_VULNS_CHANNEL: ${{ secrets.SLACK_FIXABLE_VULNS_CHANNEL }} + SLACK_ALT_WEBHOOK: ${{ secrets.SLACK_ALT_WEBHOOK }} + GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2: ${{ secrets.GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2 }} + QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} + OCM_OFFLINE_TOKEN: ${{ secrets.OCM_OFFLINE_TOKEN }} + CLOUD_SOURCES_TEST_OCM_CLIENT_ID: ${{ secrets.CLOUD_SOURCES_TEST_OCM_CLIENT_ID }} + CLOUD_SOURCES_TEST_OCM_CLIENT_SECRET: ${{ secrets.CLOUD_SOURCES_TEST_OCM_CLIENT_SECRET }} + SLACK_MAIN_WEBHOOK: ${{ secrets.SLACK_MAIN_WEBHOOK }} + GCP_SERVICE_ACCOUNT_EMAIL_STACKROX_CI_WORKLOAD_IDENTITY: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL_STACKROX_CI_WORKLOAD_IDENTITY }} + GCP_PROJECT_NUMBER_WORKLOAD_IDENTITY: ${{ secrets.GCP_PROJECT_NUMBER_WORKLOAD_IDENTITY }} + timeout-minutes: ${{ inputs.part1-timeout-minutes }} + run: qa-tests-backend/scripts/run-part-1.sh + + - name: Run Part 1 post-test + if: always() && env.KUBECONFIG != '' + shell: python + run: | + import os, sys + sys.path.append('.openshift-ci') + from post_tests import PostClusterTest + post = PostClusterTest( + check_stackrox_logs=True, + artifact_destination_prefix="part-1", + ) + post.run([os.environ['ARTIFACT_DIR']]) + + - name: Run Part 2 tests + id: run-part-2 + env: *test_env + timeout-minutes: ${{ inputs.part2-timeout-minutes }} + run: qa-tests-backend/scripts/run-part-2.sh + + - name: Run Part 2 post-test + if: always() && steps.run-part-2.outcome != 'skipped' && env.KUBECONFIG != '' + shell: python + run: | + import os, sys + sys.path.append('.openshift-ci') + from post_tests import PostClusterTest + post = PostClusterTest( + check_stackrox_logs=True, + artifact_destination_prefix="part-2", + ) + post.run([os.environ['ARTIFACT_DIR']]) + + - name: Run final-post + if: always() + shell: python + run: | + import sys + sys.path.append('.openshift-ci') + from post_tests import FinalPost + FinalPost(store_qa_tests_data=True).run() + + - name: Publish summary for failed tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "fail" + - name: Publish summary for passed and skipped tests + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 + if: always() + with: + paths: ${{ env.ARTIFACT_DIR }}/**/*.xml + show: "pass, skip" + + - name: Report junit failures in jira + if: (!cancelled()) + id: junit2jira + uses: ./.github/actions/junit2jira + with: + create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} + jira-token: ${{ secrets.JIRA_TOKEN }} + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + directory: ${{ env.ARTIFACT_DIR }} + + # CLUSTER_NAME is set before cluster creation, so it may be non-empty + # even when creation ultimately failed. We intentionally attempt teardown + # here anyway, just in case a cluster has been provisioned even though + # a failure during cluster creation has been surfaced. + - name: Delete cluster via direct-gke + if: always() && inputs.cluster-provisioning-method == 'direct-gke' + continue-on-error: true + run: | + if [[ -z "${CLUSTER_NAME:-}" ]]; then + echo "No cluster to teardown" + exit 0 + fi + if ! scripts/ci/gke.sh teardown_gke_cluster false; then + echo "::warning::WARNING: Could not delete the cluster ${CLUSTER_NAME}. See the step logs for details." + exit 6 + fi + + - name: Delete cluster via infra + if: always() && inputs.cluster-provisioning-method == 'infra' + continue-on-error: true + env: + INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }} + run: | + infractl delete "${CLUSTER_NAME}" diff --git a/.github/workflows/emailsender-central-compatibility.yaml b/.github/workflows/emailsender-central-compatibility.yaml index 9de49411e5612..f5d3f3952dae0 100644 --- a/.github/workflows/emailsender-central-compatibility.yaml +++ b/.github/workflows/emailsender-central-compatibility.yaml @@ -19,64 +19,71 @@ on: - 'central/auth/m2m/**' - '.github/workflows/emailsender-central-compatibility.yaml' +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + jobs: e2e-test-on-kind: timeout-minutes: 60 name: "Test on kind cluster" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ !github.event.pull_request.head.repo.fork }} # do not run for PRs from forks permissions: id-token: write contents: read steps: - - name: Cancel Previous Runs - uses: n1hility/cancel-previous-runs@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: Login to Quay.io - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # ratchet:docker/login-action@v4 with: registry: quay.io username: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} password: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} - name: Checkout stackrox/acs-fleet-manager repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: repository: stackrox/acs-fleet-manager path: acs-fleet-manager ref: main fetch-depth: 0 - name: Checkout stackrox/stackrox repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: repository: stackrox/stackrox ref: ${{ github.event.pull_request.head.sha }} path: stackrox fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # ratchet:actions/setup-go@v6 with: - go-version-file: stackrox/go.mod - - name: Cache go module - uses: actions/cache@v5 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('stackrox/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + # Version needs to be compatible with both repos. acs-fleet-manager's Go version usually gets bumped first + go-version-file: acs-fleet-manager/go.mod + cache: false # rarely runs, and repo checkout is in stackrox/ subdir - uses: ./stackrox/.github/actions/job-preamble with: free-disk-space: 40 - - name: Create Kind cluster" - uses: helm/kind-action@v1 + + - name: Retrieve roxie version from repository + id: roxie-version + run: echo "version=$(cat stackrox/ROXIE_VERSION)" >> "$GITHUB_OUTPUT" + + - name: Install roxie + uses: stackrox/actions/roxie/install-cli@bcda9b2d6d2b644c267e699479656973e84ef7fa # ratchet:stackrox/actions@v1 + with: + version: "v${{ steps.roxie-version.outputs.version }}" + gh-token: ${{ github.token }} + + - name: Create Kind cluster + uses: helm/kind-action@06c1ae10762d3b9c1644e7fe69596ae519e015a2 # ratchet:helm/kind-action@v1.15.0 with: cluster_name: kind + version: v0.32.0 - name: Run Test env: IMG_WAIT_TIMEOUT_SECONDS: 2100 run: acs-fleet-manager/scripts/ci/central_compatibility/entrypoint.sh - name: Upload logs - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # ratchet:actions/upload-artifact@v7 if: failure() with: name: logs-directory-${{ github.sha }} diff --git a/.github/workflows/finish-release.yml b/.github/workflows/finish-release.yml index 54dc553dcf474..3e7a22ecaeed7 100644 --- a/.github/workflows/finish-release.yml +++ b/.github/workflows/finish-release.yml @@ -34,7 +34,7 @@ concurrency: Release automation ${{ inputs.version }} jobs: properties: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: slack-channel: ${{ fromJSON(format('["{0}","{1}"]', steps.fetch.outputs.dry-slack-channel, steps.fetch.outputs.slack-channel))[github.event.inputs.dry-run != 'true'] }} steps: @@ -47,7 +47,7 @@ jobs: run-parameters: if: github.event_name == 'workflow_dispatch' name: Run parameters - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - run: | [ "$DRY_RUN" = "true" ] && echo "::warning::This is a dry run" @@ -68,13 +68,24 @@ jobs: version: ${{ inputs.version }} assert-format: A.B.C[-N] + preflight-checks: + needs: [variables] + runs-on: ubuntu-24.04 + steps: + - name: Verify that this release is listed in scanner’s RELEASE_VERSION file on master + run: | + set -uo pipefail + gh api -H "$ACCEPT_RAW" "${{ env.script_url }}" | bash -s -- \ + check-scanner-updater-version \ + "${{ needs.variables.outputs.named-release-patch }}" + publish-release: name: Tag Release ${{ needs.variables.outputs.named-release-patch }} - runs-on: ubuntu-latest - needs: [variables, properties] + runs-on: ubuntu-24.04 + needs: [variables, properties, preflight-checks] steps: - name: Check out code - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: ref: ${{ needs.variables.outputs.branch }} token: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} @@ -145,7 +156,7 @@ jobs: fi - name: Post to Slack - uses: slackapi/slack-github-action@v2.1.1 + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # ratchet:slackapi/slack-github-action@v3.0.3 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} @@ -172,13 +183,13 @@ jobs: update-infra-demo-version: name: Update infra demo default versions - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [publish-release] env: BRANCH: "bump-demo-versions-${{ inputs.version }}" steps: - name: Checkout stackrox/infra repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: repository: stackrox/infra token: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/fixxxer.yaml b/.github/workflows/fixxxer.yaml index 94ec15650b54d..f2706f6355ef5 100644 --- a/.github/workflows/fixxxer.yaml +++ b/.github/workflows/fixxxer.yaml @@ -7,14 +7,17 @@ on: permissions: contents: write +env: + ROX_PRODUCT_BRANDING: RHACS_BRANDING + jobs: pr_commented: name: Run PR Fixxxer # TODO: support leading and trailing whitespace too if: ${{ github.event.issue.pull_request && github.event.comment.body == '/fixxx' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11@sha256:df3459cf038e73775314b3a5d36d14dda41e6df438a0273c9efa8d23e0f2358a # ratchet:quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11 steps: - name: Fetch PR metadata @@ -55,7 +58,7 @@ jobs: STEPS_CONTEXT: ${{ toJson(steps) }} run: echo "$STEPS_CONTEXT" | jq . - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: ref: ${{ steps.pr-metadata.outputs.branch }} fetch-depth: 0 @@ -70,7 +73,7 @@ jobs: - run: ./scripts/fixxxer.sh - - uses: ad-m/github-push-action@v1.0.0 + - uses: ad-m/github-push-action@881a6320fdb16eb5318c5054f31c218aec2b324c # ratchet:ad-m/github-push-action@v1.3.0 with: branch: ${{ steps.pr-metadata.outputs.branch }} github_token: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/konflux-auto-retest.yml b/.github/workflows/konflux-auto-retest.yml index f0ecb4cfc89ea..2a76babeb7de6 100644 --- a/.github/workflows/konflux-auto-retest.yml +++ b/.github/workflows/konflux-auto-retest.yml @@ -1,17 +1,18 @@ -name: Retest Konflux Builds +name: Periodic Retest Failed Konflux Builds on: - check_run: - types: [completed] - pull_request: - types: [synchronize] + schedule: + - cron: '5,15,25,35,45,55 * * * *' + workflow_dispatch: jobs: - retest-failed-builds: - uses: stackrox/actions/.github/workflows/retest-konflux-builds.yml@v1 + retest: + uses: stackrox/actions/.github/workflows/periodic-retest-konflux-builds.yml@6691d5c300d4af17546145bdbbb9862c53975af4 # ratchet:stackrox/actions/.github/workflows/periodic-retest-konflux-builds.yml@v1 permissions: pull-requests: write issues: write + checks: read + contents: read with: max_retries: 3 - check_name_suffix: '-on-push' + retest_command: '/konflux-retest' diff --git a/.github/workflows/labeler.yaml b/.github/workflows/labeler.yaml index a2b40b7d73bc6..53e135c2eebf5 100644 --- a/.github/workflows/labeler.yaml +++ b/.github/workflows/labeler.yaml @@ -8,10 +8,10 @@ on: jobs: label-pr: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork }} steps: - - uses: actions/labeler@v6 + - uses: actions/labeler@b8dd2d9be0f68b860e7dae5dae7d772984eacd6d # ratchet:actions/labeler@v6 with: repo-token: "${{ github.token }}" sync-labels: False diff --git a/.github/workflows/mintmaker-auto-merge.yml b/.github/workflows/mintmaker-auto-merge.yml new file mode 100644 index 0000000000000..c288ba0ad0e3b --- /dev/null +++ b/.github/workflows/mintmaker-auto-merge.yml @@ -0,0 +1,30 @@ +name: mintmaker-auto-merge + +on: + pull_request: + types: + - labeled + branches: + # Only enabled for PRs targeting master because release branches lack protection rules. + - master + +jobs: + mintmaker-auto-approve-and-merge: + name: Auto-approve and auto-merge MintMaker updates + runs-on: ubuntu-slim + if: github.event.pull_request.user.login == 'red-hat-konflux[bot]' && github.event.label.name == 'mintmaker-auto-merge' + steps: + - env: + GH_TOKEN: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} + run: | + set -euo pipefail + + gh pr review ${{ github.event.pull_request.number }} \ + --approve --body "Auto-approved by automation." \ + --repo "${{ github.repository }}" + + # This enables auto-merge due to the `--auto` argument. Once branch protection rules are satisfied, the merge + # will happen. If no rules are defined, PR will be merged instantly. + gh pr merge ${{ github.event.pull_request.number }} \ + --auto --delete-branch --squash \ + --repo "${{ github.repository }}" diff --git a/.github/workflows/nightly-tag.yml b/.github/workflows/nightly-tag.yml index fef2d6783d405..6ef44bae5fd8f 100644 --- a/.github/workflows/nightly-tag.yml +++ b/.github/workflows/nightly-tag.yml @@ -2,14 +2,14 @@ name: Create nightly tag on: workflow_dispatch: schedule: - - cron: 0 0 * * 1-5 + - cron: 0 2 * * 1-5 jobs: create-nightly-tag: if: ${{ github.repository_owner == 'stackrox' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: ref: nightlies - name: Rebase branch and create new tag @@ -20,10 +20,10 @@ jobs: slack-on-failure: needs: create-nightly-tag if: failure() - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Slack on failure - uses: slackapi/slack-github-action@v2.1.1 + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # ratchet:slackapi/slack-github-action@v3.0.3 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/notify-milestone-change.yml b/.github/workflows/notify-milestone-change.yml index e08e5f29b09f8..dc327ead0b241 100644 --- a/.github/workflows/notify-milestone-change.yml +++ b/.github/workflows/notify-milestone-change.yml @@ -18,7 +18,7 @@ env: jobs: properties: name: Read repository properties - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: docs-repository: ${{ steps.properties.outputs.docs-repository }} slack-channel: ${{ fromJSON(format('["{0}","{1}"]', steps.properties.outputs.dry-slack-channel, steps.properties.outputs.slack-channel))[github.repository == 'stackrox/stackrox'] }} @@ -31,16 +31,19 @@ jobs: run: gh api -H "$ACCEPT_RAW" "$PROPERTIES_URL" >> "$GITHUB_OUTPUT" notify: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [properties] steps: - name: Determine Jira context for PR id: context env: PR_TITLE: ${{ github.event_name == 'workflow_dispatch' && inputs.test-pr-title || github.event.pull_request.title }} + JIRA_USER: ${{ vars.RHACS_BOT_GITHUB_EMAIL }} + JIRA_TOKEN: ${{ secrets.JIRA_TOKEN }} + JIRA_BASE_URL: ${{ vars.JIRA_BASE_URL }} run: | # For manual testing, simulate a milestoned event - if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event.action}}" = "milestoned" ]; then + if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event.action }}" = "milestoned" ]; then EVENT_ACTION="added to" EVENT_ICON=":white_check_mark:" echo "call-to-action=:arrow_right: Release Engineers : Review whether this PR meets the criteria for being included in the release." >> "${GITHUB_OUTPUT}" @@ -75,32 +78,34 @@ jobs: ISSUE_ID="$(echo "${PR_TITLE}" | grep --only-matching 'ROX-[0-9]\+' || true)" if [ -n "${ISSUE_ID}" ]; then - JIRA_INFO="$(curl \ - -sSL \ - --retry 5 --retry-all-errors \ - --location "https://issues.redhat.com/rest/api/2/issue/${ISSUE_ID}" \ - --header "Authorization: Bearer ${{ secrets.JIRA_TOKEN }}" \ - | jq -r '{ key: .key, summary: .fields.summary, priority: .fields.priority.name, assignee: .fields.assignee.displayName, status: .fields.status.name, type: .fields.issuetype.name }')" + JIRA_INFO=$(curl -sSL --request GET \ + --user "${JIRA_USER}:${JIRA_TOKEN}" \ + --header "Accept: application/json" \ + "https://${JIRA_BASE_URL}/rest/api/3/issue/${ISSUE_ID}") KEY="$(echo "${JIRA_INFO}" | jq -r .key)" - SUMMARY="$(echo "${JIRA_INFO}" | jq -r .summary)" - PRIORITY="$(echo "${JIRA_INFO}" | jq -r .priority)" - ASSIGNEE="$(echo "${JIRA_INFO}" | jq -r .assignee)" - STATUS="$(echo "${JIRA_INFO}" | jq -r .status)" - TYPE="$(echo "${JIRA_INFO}" | jq -r .type)" + if [ "${KEY}" != "null" ]; then + SUMMARY="$(echo "${JIRA_INFO}" | jq -r .fields.summary)" + PRIORITY="$(echo "${JIRA_INFO}" | jq -r .fields.priority.name)" + ASSIGNEE="$(echo "${JIRA_INFO}" | jq -r '.fields.assignee.displayName // "Unassigned"')" + STATUS="$(echo "${JIRA_INFO}" | jq -r .fields.status.name)" + TYPE="$(echo "${JIRA_INFO}" | jq -r .fields.issuetype.name)" - # Escape quotes in SUMMARY for YAML compatibility - ESCAPED_SUMMARY="${SUMMARY//\"/\\\"}" + # Escape quotes in SUMMARY for YAML compatibility + ESCAPED_SUMMARY="${SUMMARY//\"/\\\"}" - # Need to use echo to preserve newlines in the output (GH action limitation) - # shellcheck disable=SC2028 - echo "jira-info=\n> Type: \`${TYPE}\`\n> Assignee: \`${ASSIGNEE}\`\n> Status: \`${STATUS}\`\n> Priority: \`${PRIORITY}\`" >> "${GITHUB_OUTPUT}" + # Need to use echo to preserve newlines in the output (GH action limitation) + # shellcheck disable=SC2028 + echo "jira-info=\n> Type: \`${TYPE}\`\n> Assignee: \`${ASSIGNEE}\`\n> Status: \`${STATUS}\`\n> Priority: \`${PRIORITY}\`" >> "${GITHUB_OUTPUT}" + else + echo "jira-info=:exclamation: We could not find the Jira issue for this PR." >> "${GITHUB_OUTPUT}" + fi else - echo "jira-info=:exclamation: We could not find a Jira issue for this PR." >> "${GITHUB_OUTPUT}" + echo "jira-info=:exclamation: We could not find the Jira issue from the title of this PR." >> "${GITHUB_OUTPUT}" fi - name: Send Slack notification - uses: slackapi/slack-github-action@v2.1.1 + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # ratchet:slackapi/slack-github-action@v3.0.3 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index b71360dff760a..3e33fc821ee70 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -10,31 +10,30 @@ concurrency: performance-tests-${{ github.ref }} jobs: wait-for-images: name: Wait for images - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ contains(github.event.pull_request.labels.*.name, 'ci-performance-tests') }} - strategy: - matrix: - image: [main, roxctl] steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Get PR image tag run: | - MAIN_IMAGE_TAG="$(make tag)" + MAIN_IMAGE_TAG="$(make --quiet --no-print-directory tag)" echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" >> "$GITHUB_ENV" - - name: Wait for the ${{ matrix.image }} - uses: stackrox/actions/release/wait-for-image@v1 + - name: Wait for images + uses: stackrox/actions/release/wait-for-image@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/release/wait-for-image@v1 with: token: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} - image: rhacs-eng/${{ matrix.image }}:${{ env.MAIN_IMAGE_TAG }} + image: | + rhacs-eng/main:${{ env.MAIN_IMAGE_TAG }} + rhacs-eng/roxctl:${{ env.MAIN_IMAGE_TAG }} run-k6-performance-tests: needs: [wait-for-images] name: Run k6 performance tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ contains(github.event.pull_request.labels.*.name, 'ci-performance-tests') }} env: USE_GKE_GCLOUD_AUTH_PLUGIN: "True" @@ -47,20 +46,30 @@ jobs: SCALE_WORKLOAD_CLUSTERS: 10 steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Checkout workflow scripts - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: repository: "stackrox/workflow" path: workflow + - name: Retrieve roxie version from repository + id: roxie-version + run: echo "version=$(cat ROXIE_VERSION)" >> "$GITHUB_OUTPUT" + + - name: Install roxie + uses: stackrox/actions/roxie/install-cli@bcda9b2d6d2b644c267e699479656973e84ef7fa # ratchet:stackrox/actions@v1 + with: + version: "v${{ steps.roxie-version.outputs.version }}" + gh-token: ${{ secrets.GITHUB_TOKEN }} + # Create scale cluster with larger nodes. - name: Create testing cluster - uses: stackrox/actions/infra/create-cluster@v1 + uses: stackrox/actions/infra/create-cluster@d624eeae7073b3acb838aab7705730b2de58a4a0 # ratchet:stackrox/actions/infra/create-cluster@v1 with: token: ${{ secrets.INFRA_TOKEN }} flavor: gke-default @@ -70,12 +79,12 @@ jobs: wait: true # Fetch infractl artifacts - - uses: stackrox/actions/infra/install-infractl@v1 - - uses: "google-github-actions/auth@v3" + - uses: stackrox/actions/infra/install-infractl@3b27674f0fa69f2d8c31ab26aa63c9f76401c64e # ratchet:stackrox/actions/infra/install-infractl@v1 + - uses: 'google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093' # ratchet:google-github-actions/auth@v3 with: credentials_json: "${{ secrets.GCP_PERFORMANCE_TESTER_CI_SA }}" - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v3" + uses: 'google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db' # ratchet:google-github-actions/setup-gcloud@v3 with: install_components: "gke-gcloud-auth-plugin" - name: Test cluster readiness @@ -98,12 +107,8 @@ jobs: - name: Clean previous installation run: | source ./workflow/env.sh - ./scripts/k8s/cleanup.sh stackrox - - # Delete all namespaces created by scale script. - for i in $(seq 1 "${SCALE_WORKLOAD_CLUSTERS}"); do - ./scripts/k8s/cleanup.sh "stackrox${i}" - done + roxie teardown central --single-namespace + ./scale/dev/kill-sensors.sh - name: Get PR image tag run: | @@ -113,37 +118,29 @@ jobs: # Install Central + CentralDB - name: Launch central services env: - LOAD_BALANCER: lb - ROX_POSTGRES_DATASTORE: "true" - MONITORING_SUPPORT: "false" REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} run: | - source ./workflow/env.sh + set -euo pipefail - # ROX-29641: We are facing issues with workload generation on GKE when Scanner v4 is enabled. - export ROX_SCANNER_V4=false ./scale/dev/launch_central.sh - + source .roxie.env # Required for k6 tests to run. - echo "ROX_ADMIN_PASSWORD=$(cat ./deploy/k8s/central-deploy/password)" >> "$GITHUB_ENV" - echo "ROX_PASSWORD=$(cat ./deploy/k8s/central-deploy/password)" >> "$GITHUB_ENV" - echo "::add-mask::$(cat ./deploy/k8s/central-deploy/password)" + echo "ROX_ADMIN_PASSWORD=${ROX_ADMIN_PASSWORD}" >> "$GITHUB_ENV" + echo "ROX_PASSWORD=${ROX_ADMIN_PASSWORD}" >> "$GITHUB_ENV" + echo "::add-mask::${ROX_ADMIN_PASSWORD}" - LB_IP="$(kubectl -n stackrox get svc/central-loadbalancer -o json | jq -r '.status.loadBalancer.ingress[0] | .ip // .hostname')" - echo "HOST=https://${LB_IP}:443" >> "$GITHUB_ENV" - echo "API_ENDPOINT=${LB_IP}:443" >> "$GITHUB_ENV" - echo "::add-mask::${LB_IP}" + echo "HOST=${ROX_BASE_URL}" >> "$GITHUB_ENV" + echo "API_ENDPOINT=${API_ENDPOINT}" >> "$GITHUB_ENV" + echo "::add-mask::${API_ENDPOINT}" - name: Generate workload env: - LOAD_BALANCER: lb - ROX_POSTGRES_DATASTORE: "true" - MONITORING_SUPPORT: "false" REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} run: | source ./workflow/env.sh + source .roxie.env ./scale/dev/run-many.sh "${SCALE_WORKLOAD_TYPE}" "${SCALE_WORKLOAD_CLUSTERS}" num_deployments=$(yq '.deploymentWorkload[] | select(.deploymentType == "Deployment") | .numDeployments' "./scale/workloads/${SCALE_WORKLOAD_TYPE}.yaml") @@ -161,18 +158,18 @@ jobs: mkdir -p performance-results - name: Run local k6 test - uses: grafana/k6-action@v0.3.1 + uses: grafana/k6-action@e4714b734f2b0afaabeb7b4a69142745548ab9ec # ratchet:grafana/k6-action@v0.3.1 with: filename: ./tests/performance/tests/testK6Integration.js flags: --out json=performance-results/raw.json --out csv=performance-results/raw.csv --iterations 50 --duration 20m --tag github_run_id=${{ github.run_id }} --tag github_ref=${{ github.ref }} --tag github_sha=${{ github.sha }} - name: Publish JUnit test report in job summary - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 with: paths: performance-results/report.xml - name: Upload results to GCS - uses: google-github-actions/upload-cloud-storage@v3 + uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # ratchet:google-github-actions/upload-cloud-storage@v3 with: path: performance-results destination: acs-performance-test-results/${{ github.ref }}/${{ github.sha }}/${{ github.run_id }}/${{ github.run_attempt }} @@ -198,7 +195,7 @@ jobs: cat pr-comment.md >> "$GITHUB_STEP_SUMMARY" - name: Add text report and links as PR comment - uses: mshick/add-pr-comment@v2 + uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # ratchet:mshick/add-pr-comment@v3 with: message-id: performance-evaluation message-path: | diff --git a/.github/workflows/release-ci.yaml b/.github/workflows/release-ci.yaml index 1ec4ae0e1fbe8..7a631fd36b3ad 100644 --- a/.github/workflows/release-ci.yaml +++ b/.github/workflows/release-ci.yaml @@ -9,29 +9,28 @@ on: jobs: run-parameters: name: Run parameters - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - run: | echo "Event: ${{ github.event_name }}" >> "$GITHUB_STEP_SUMMARY" echo "Ref: ${{ github.ref_name }}" >> "$GITHUB_STEP_SUMMARY" build: + permissions: + actions: read + contents: read + pull-requests: write name: Build uses: ./.github/workflows/build.yaml secrets: inherit - build-scanner-v4: - name: Build Scanner v4 - uses: ./.github/workflows/scanner-build.yaml - secrets: inherit - check-is-release: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: is_release: ${{ steps.is_release_check.outputs.is_release }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.ref_name }} @@ -44,39 +43,39 @@ jobs: echo "is_release=false" | tee -a "$GITHUB_OUTPUT" fi - check-scanner-version: - runs-on: ubuntu-latest + check-collector-version: + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.ref_name }} - - name: Check scanner version + - name: Check collector version run: | scripts/ci/lib.sh \ - check_scanner_version + check_collector_version - check-collector-version: - runs-on: ubuntu-latest + check-fact-version: + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.ref_name }} - - name: Check collector version + - name: Check fact version run: | scripts/ci/lib.sh \ - check_collector_version + check_fact_version publish-helm-charts: - needs: [build, check-scanner-version, check-collector-version, check-is-release] + needs: [build, check-collector-version, check-fact-version, check-is-release] if: needs.check-is-release.outputs.is_release == 'true' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.ref_name }} @@ -92,7 +91,7 @@ jobs: SLACK_MAIN_WEBHOOK: ${{ secrets.SLACK_MAIN_WEBHOOK }} RELEASE_WORKFLOW_NOTIFY_WEBHOOK: ${{ secrets.RELEASE_WORKFLOW_NOTIFY_WEBHOOK }} run: | - tar xzf cli-build.tgz + tar xf cli-build.tar mkdir -p "${HOME}/.local/bin" mv bin/linux_amd64/roxctl "${HOME}/.local/bin/roxctl" scripts/ci/lib.sh \ @@ -100,20 +99,20 @@ jobs: # Publish `roxagent` and `roxctl`. publish-cli: - needs: [build, check-scanner-version, check-collector-version, check-is-release] + needs: [build, check-collector-version, check-fact-version, check-is-release] if: needs.check-is-release.outputs.is_release == 'true' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.ref_name }} - - uses: 'google-github-actions/auth@v3' + - uses: 'google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093' # ratchet:google-github-actions/auth@v3 with: credentials_json: '${{ secrets.GCLOUD_SERVICE_ACCOUNT_CI_ROX }}' - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v3' + uses: 'google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db' # ratchet:google-github-actions/setup-gcloud@v3 - uses: ./.github/actions/download-artifact-with-retry with: name: cli-build @@ -121,25 +120,25 @@ jobs: env: STACKROX_TAG: ${{ github.ref_name }} run: | - tar xzf cli-build.tgz + tar xf cli-build.tar scripts/ci/lib.sh \ publish_cli "${STACKROX_TAG}" publish-openapispec: - needs: [build, check-scanner-version, check-collector-version, check-is-release] + needs: [build, check-collector-version, check-fact-version, check-is-release] if: needs.check-is-release.outputs.is_release == 'true' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.ref_name }} - - uses: 'google-github-actions/auth@v3' + - uses: 'google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093' # ratchet:google-github-actions/auth@v3 with: credentials_json: '${{ secrets.GCLOUD_SERVICE_ACCOUNT_CI_ROX }}' - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v3' + uses: 'google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db' # ratchet:google-github-actions/setup-gcloud@v3 - name: Publish OpenAPI spec to Google Cloud storage env: STACKROX_TAG: ${{ github.ref_name }} diff --git a/.github/workflows/retest_comment.yml b/.github/workflows/retest_comment.yml new file mode 100644 index 0000000000000..3d8dffdb4a2a0 --- /dev/null +++ b/.github/workflows/retest_comment.yml @@ -0,0 +1,34 @@ +name: /retest for GitHub Actions + +on: + issue_comment: + types: [created] + +jobs: + rerun-failed: + if: >- + github.event.issue.pull_request && + github.event.comment.body == '/retest' + runs-on: ubuntu-slim + steps: + - name: Fetch PR head SHA + id: pr + env: + GH_TOKEN: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} + run: | + sha=$(gh api "${{ github.event.issue.pull_request.url }}" --jq '.head.sha') + echo "sha=$sha" >> "$GITHUB_OUTPUT" + + - name: Rerun failed workflows + env: + GH_TOKEN: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} + run: | + failed_runs=$(gh api "repos/${{ github.repository }}/actions/runs?head_sha=${{ steps.pr.outputs.sha }}&event=pull_request&status=failure" --jq '.workflow_runs[].id') + if [ -z "$failed_runs" ]; then + echo "No failed workflow runs found" + exit 0 + fi + for run_id in $failed_runs; do + echo "Rerunning failed jobs for run $run_id" + gh run rerun "$run_id" --failed --repo "${{ github.repository }}" + done diff --git a/.github/workflows/retest_periodic.yml b/.github/workflows/retest_periodic.yml index ae15f50e59843..3c371a6102804 100644 --- a/.github/workflows/retest_periodic.yml +++ b/.github/workflows/retest_periodic.yml @@ -10,20 +10,23 @@ jobs: retest: if: ${{ github.repository_owner == 'stackrox' }} timeout-minutes: 10 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: sparse-checkout: | .github tools/retest - name: Setup Go environment - uses: actions/setup-go@v6 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # ratchet:actions/setup-go@v6 with: go-version-file: go.mod + - name: Test + run: cd tools/retest && go test ./... + - name: Retest env: GITHUB_TOKEN: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/scan-image-vulnerabilities.yaml b/.github/workflows/scan-image-vulnerabilities.yaml index 1f022e587d7fb..bb4befe0852d1 100644 --- a/.github/workflows/scan-image-vulnerabilities.yaml +++ b/.github/workflows/scan-image-vulnerabilities.yaml @@ -21,7 +21,7 @@ run-name: ${{ format('Scan images for vulnerabilities ({0})', inputs.version || jobs: run-parameters: name: Run parameters - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: wait-for-images-limit: ${{ steps.run-parameters.outputs.wait-for-images-limit }} version: ${{ steps.run-parameters.outputs.version }} @@ -47,7 +47,7 @@ jobs: scan-image-vulnerabilities-upstream: name: "Upstream: ${{ matrix.image }}:${{ needs.run-parameters.outputs.version }}" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [run-parameters] permissions: # Needed for stackrox/central-login to create the JWT token. @@ -59,12 +59,9 @@ jobs: [ "central-db", "collector", + "fact", "main", "roxctl", - "scanner", - "scanner-db", - "scanner-db-slim", - "scanner-slim", "scanner-v4", "scanner-v4-db", "stackrox-operator", @@ -82,7 +79,7 @@ jobs: echo "tag=${tag}" >> "${GITHUB_OUTPUT}" - name: Scan image vulnerabilities - uses: stackrox/actions/release/scan-image-vulnerabilities@v1 + uses: stackrox/actions/release/scan-image-vulnerabilities@df8c8f5830fc09d37908be1a9744365a93c5b3cc # ratchet:stackrox/actions/release/scan-image-vulnerabilities@v1 id: scan-image with: image: rhacs-eng/${{ matrix.image }} @@ -94,7 +91,7 @@ jobs: - name: Upload scan result artifact if: always() - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # ratchet:actions/upload-artifact@v7 with: name: upstream_quay_io_rhacs-eng_${{ matrix.image }}_${{ needs.run-parameters.outputs.version }}.json path: ${{ steps.scan-image.outputs.result-path }} @@ -102,7 +99,7 @@ jobs: scan-image-vulnerabilities-konflux: name: "Konflux: ${{ matrix.image }}:${{ needs.run-parameters.outputs.version }}" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [run-parameters] permissions: # Needed for stackrox/central-login to create the JWT token. @@ -114,12 +111,9 @@ jobs: [ "release-central-db", "release-collector", + "release-fact", "release-main", "release-roxctl", - "release-scanner", - "release-scanner-db", - "release-scanner-db-slim", - "release-scanner-slim", "release-scanner-v4", "release-scanner-v4-db", "release-operator", @@ -139,7 +133,7 @@ jobs: echo "tag=${tag}" >> "${GITHUB_OUTPUT}" - name: Scan image vulnerabilities - uses: stackrox/actions/release/scan-image-vulnerabilities@v1 + uses: stackrox/actions/release/scan-image-vulnerabilities@df8c8f5830fc09d37908be1a9744365a93c5b3cc # ratchet:stackrox/actions/release/scan-image-vulnerabilities@v1 id: scan-image with: image: rhacs-eng/${{ matrix.image }} @@ -151,7 +145,7 @@ jobs: - name: Upload scan result artifact if: always() - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # ratchet:actions/upload-artifact@v7 with: name: konflux_quay_io_rhacs-eng_${{ matrix.image }}_${{ needs.run-parameters.outputs.version }}.json path: ${{ steps.scan-image.outputs.result-path }} diff --git a/.github/workflows/scanner-build.yaml b/.github/workflows/scanner-build.yaml deleted file mode 100644 index 1493f701f16ae..0000000000000 --- a/.github/workflows/scanner-build.yaml +++ /dev/null @@ -1,379 +0,0 @@ -name: Scanner build and push images - -on: - workflow_dispatch: - inputs: - commit: - description: 'Commit SHA to build' - required: true - type: string - workflow_call: - push: - tags: - - '*-nightly-*' - branches: - - master - pull_request: - types: - - opened - - reopened - - synchronize - -defaults: - run: - # This enables `-o pipefail` for all jobs as compared to when shell isn't set. - # See https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#defaultsrunshell - shell: bash - -jobs: - define-scanner-job-matrix: - outputs: - matrix: ${{ steps.define-scanner-job-matrix.outputs.matrix }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - - - name: Define the matrix for build jobs - id: define-scanner-job-matrix - run: | - source './scripts/ci/lib.sh' - - # If goarch is updated, be sure to update architectures in "push-scanner-manifests" below. - matrix='{ - "pre_build_scanner_go_binary": { "name":["default"], "goos":["linux"], "goarch":["amd64", "arm64"] }, - "build_and_push_scanner": { }, - "push_scanner_manifests": { "name":["default"], "registry":["quay.io/stackrox-io", "quay.io/rhacs-eng"] }, - "scan_images_with_roxctl": { "image":["scanner-v4", "scanner-v4-db"], "registry":["quay.io/stackrox-io", "quay.io/rhacs-eng"] } - }' - - if ! is_in_PR_context || pr_has_label ci-build-all-arch; then - matrix="$(jq '.pre_build_scanner_go_binary.goarch += ["ppc64le", "s390x"]' <<< "$matrix")" - fi; - - # Conditionally add a prerelease build (binaries built with GOTAGS=release) - if ! is_tagged; then - if ! is_in_PR_context || pr_has_label ci-build-prerelease; then - matrix="$(jq '.pre_build_scanner_go_binary.name += ["prerelease"]' <<< "$matrix")" - matrix="$(jq '.push_scanner_manifests.name += ["prerelease"]' <<< "$matrix")" - fi - fi - - # Conditionally add a -race debug build (binaries built with -race) - if ! is_in_PR_context || pr_has_label ci-build-race-condition-debug; then - matrix="$(jq '.pre_build_scanner_go_binary.name += ["race-condition-debug"]' <<< "$matrix")" - matrix="$(jq '.push_scanner_manifests.name += ["race-condition-debug"]' <<< "$matrix")" - fi - - matrix="$(jq '.build_and_push_scanner = .pre_build_scanner_go_binary' <<< "$matrix")" - matrix="$(jq '.build_and_push_scanner.registry = .push_scanner_manifests.registry' <<< "$matrix")" - - echo "Job matrix after conditionals:" - jq <<< "$matrix" - - condensed="$(jq -c <<< "$matrix")" - echo "matrix=$condensed" >> "$GITHUB_OUTPUT" - - pre-build-scanner-go-binary: - needs: define-scanner-job-matrix - runs-on: ubuntu-latest - strategy: - # Supports three go binary builds: - # default - built with environment defaults (see handle-tagged-build & env.mk) - # prerelease - built with GOTAGS=release - # race-condition-debug - built with -race - matrix: ${{ fromJson(needs.define-scanner-job-matrix.outputs.matrix).pre_build_scanner_go_binary }} - container: - image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - - - uses: ./.github/actions/job-preamble - with: - gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - - name: Cache Go dependencies - uses: ./.github/actions/cache-go-dependencies - - - uses: ./.github/actions/handle-tagged-build - - - name: Setup Go build environment for release - if: | - contains(github.event.pull_request.labels.*.name, 'ci-release-build') - || - matrix.name == 'prerelease' - run: echo "GOTAGS=release" >> "$GITHUB_ENV" - - - name: Setup Go build environment for -race - if: | - matrix.arch == 'amd64' - && - ( - contains(github.event.pull_request.labels.*.name, 'ci-race-tests') - || - matrix.name == 'race-condition-debug' - ) - run: echo "RACE=true" >> "$GITHUB_ENV" - - - name: Build Go binary - run: | - if [[ "${{ matrix.goarch }}" != "amd64" ]]; then - echo "Building non-amd binary (CGO_ENABLED=0)" - make -C scanner CGO_ENABLED=0 GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} image/scanner/bin/scanner - else - echo "Building amd binary (CGO_ENABLED=1)" - make -C scanner CGO_ENABLED=1 GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} image/scanner/bin/scanner - fi - - - name: Archive Go binary to preserve permissions - run: tar -cvzf go-binary.tgz scanner/image/scanner/bin/scanner - - - uses: ./.github/actions/upload-artifact-with-retry - with: - name: go-binary-${{ matrix.name }}-${{ matrix.goos }}-${{ matrix.goarch }} - path: go-binary.tgz - - scan-scanner-go-binary: - needs: pre-build-scanner-go-binary - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.1 - if: contains(github.event.pull_request.labels.*.name, 'scan-go-binaries') - env: - ARTIFACT_DIR: junit-reports/ - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 1 - ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - - - uses: ./.github/actions/job-preamble - with: - gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - - uses: ./.github/actions/download-artifact-with-retry - with: - name: go-binary-default-linux-amd64 - - - name: Unpack Go binary - run: | - tar xvzf go-binary.tgz - - - name: Scan - run: | - ./scripts/ci/govulncheck.sh scanner/image/scanner/bin/scanner - - - name: Publish Test Report - uses: test-summary/action@v2 - if: always() - with: - paths: 'junit-reports/**/*.xml' - - - name: Report junit failures in jira - uses: ./.github/actions/junit2jira - if: always() - with: - jira-token: ${{ secrets.JIRA_TOKEN }} - gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - directory: 'junit-reports' - - build-and-push-scanner: - needs: - - define-scanner-job-matrix - - pre-build-scanner-go-binary - runs-on: ubuntu-latest - strategy: - fail-fast: false - # Supports three go binary and image builds: - # default - built with environment defaults (see handle-tagged-build & env.mk) - # prerelease - built with GOTAGS=release - # race-condition-debug - built with -race - matrix: ${{ fromJson(needs.define-scanner-job-matrix.outputs.matrix).build_and_push_scanner }} - container: - image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt - env: - QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - - - uses: ./.github/actions/job-preamble - with: - gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - - - name: Login to docker.io to mitigate rate limiting on downloading images - uses: docker/login-action@v3 - # Skip for external contributions. - if: | - github.event_name == 'push' || !github.event.pull_request.head.repo.fork - with: - username: ${{ secrets.DOCKERHUB_CI_ACCOUNT_USERNAME }} - password: ${{ secrets.DOCKERHUB_CI_ACCOUNT_PASSWORD }} - - - name: Set up QEMU - if: matrix.goarch != 'amd64' - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - uses: ./.github/actions/handle-tagged-build - - - uses: ./.github/actions/download-artifact-with-retry - with: - name: go-binary-${{ matrix.name }}-${{ matrix.goos }}-${{ matrix.goarch }} - - - name: Unpack Go binary - run: tar xvzf go-binary.tgz - - - name: Set build tag for prerelease images - if: matrix.name == 'prerelease' - run: echo "BUILD_TAG=$(make -C scanner --quiet --no-print-directory tag)-prerelease" >> "$GITHUB_ENV" - - - name: Set build tag for race condition images - if: matrix.name == 'race-condition-debug' - run: echo "BUILD_TAG=$(make -C scanner --quiet --no-print-directory tag)-rcd" >> "$GITHUB_ENV" - - - name: Build Scanner and ScannerDB images - run: scripts/lib.sh retry 6 true make -C scanner GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} images - - - name: Push Scanner and ScannerDB images - # Skip for external contributions. - if: | - github.event_name == 'push' || !github.event.pull_request.head.repo.fork - run: | - source ./scripts/ci/lib.sh - push_scanner_image_set "${{ matrix.registry }}" "${{ matrix.goarch }}" - - push-scanner-manifests: - needs: - - define-scanner-job-matrix - - build-and-push-scanner - runs-on: ubuntu-latest - strategy: - fail-fast: false - # Supports three image builds: - # default - # prerelease - # race-condition-debug - matrix: ${{ fromJson(needs.define-scanner-job-matrix.outputs.matrix).push_scanner_manifests }} - container: - image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.1 - env: - QUAY_RHACS_ENG_RW_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - QUAY_RHACS_ENG_RW_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - QUAY_STACKROX_IO_RW_USERNAME: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - QUAY_STACKROX_IO_RW_PASSWORD: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - - - name: Ignore dubious repository ownership - run: | - # Prevent fatal error "detected dubious ownership in repository" from recent git. - git config --global --add safe.directory "$(pwd)" - - - uses: ./.github/actions/handle-tagged-build - - - name: Set build tag for prerelease images - if: matrix.name == 'prerelease' - run: echo "BUILD_TAG=$(make -C scanner --quiet --no-print-directory tag)-prerelease" >> "$GITHUB_ENV" - - - name: Set build tag for race condition images - if: matrix.name == 'race-condition-debug' - run: echo "BUILD_TAG=$(make -C scanner --quiet --no-print-directory tag)-rcd" >> "$GITHUB_ENV" - - - name: Push Scanner and ScannerDB image manifests - # Skip for external contributions. - if: | - github.event_name == 'push' || !github.event.pull_request.head.repo.fork - run: | - source ./scripts/ci/lib.sh - - # If this is updated, be sure to update goarch in define-scanner-job-matrix above. - architectures="amd64,arm64" - if ! is_in_PR_context || pr_has_label ci-build-all-arch; then - architectures="amd64,arm64,ppc64le,s390x" - fi - if [[ "${{ matrix.name }}" == "race-condition-debug" ]]; then - architectures="amd64" - fi - - push_scanner_image_manifest_lists "${{ matrix.registry }}" "$architectures" - - scan-images-with-roxctl: - if: github.event_name == 'push' || - contains(github.event.pull_request.labels.*.name, 'scan-images-with-roxctl') - needs: - - define-scanner-job-matrix - - build-and-push-scanner - - push-scanner-manifests - name: Check images for vulnerabilities - runs-on: ubuntu-latest - permissions: - # Needed for stackrox/central-login to create the JWT token. - id-token: write - security-events: write - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.define-scanner-job-matrix.outputs.matrix).scan_images_with_roxctl }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: ${{ inputs.commit || github.event.pull_request.head.sha }} - - - uses: ./.github/actions/job-preamble - with: - gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - - uses: ./.github/actions/handle-tagged-build - - - name: Central login - uses: stackrox/central-login@v1 - with: - endpoint: ${{ vars.ACS_DOGFOODING_CENTRAL_URL }} - - - name: Install roxctl - uses: stackrox/roxctl-installer-action@v1 - with: - central-endpoint: ${{ vars.ACS_DOGFOODING_CENTRAL_URL }} - central-token: ${{ env.ROX_API_TOKEN }} - - - name: Scan images for vulnerabilities - run: | - release_tag="$(make --quiet --no-print-directory tag)" - roxctl image scan --retries=10 --retry-delay=15 --force --severity=IMPORTANT,CRITICAL --output=sarif \ - --image="${{ matrix.registry }}/${{ matrix.image }}:${release_tag}" \ - | tee results.sarif - - # TODO: re-enable roxctl scan results upload once quota issue has been resolved - # - name: Upload roxctl scan results to GitHub Security tab - # uses: github/codeql-action/upload-sarif@v3 - # with: - # category: ${{ matrix.image }} - # sarif_file: results.sarif diff --git a/.github/workflows/scanner-db-integration-tests.yaml b/.github/workflows/scanner-db-integration-tests.yaml index a17a3c2766522..effbe1190271e 100644 --- a/.github/workflows/scanner-db-integration-tests.yaml +++ b/.github/workflows/scanner-db-integration-tests.yaml @@ -14,13 +14,18 @@ on: - synchronize jobs: + detect-changes: + uses: ./.github/workflows/detect-changes.yml + db-integration-tests: - runs-on: ubuntu-latest + needs: detect-changes + if: ${{ !needs.detect-changes.outputs.ui_only }} + runs-on: ubuntu-24.04 container: - image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.1 + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.11@sha256:cea78f2b3a5a38c9ccda678712daaed90b7c372d6b5df963feb758b32dccebe1 # ratchet:quay.io/stackrox-io/apollo-ci:scanner-test-0.5.11 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 diff --git a/.github/workflows/scanner-e2e-test.yaml b/.github/workflows/scanner-e2e-test.yaml index 1157dfe233ef6..b2d0e9670e165 100644 --- a/.github/workflows/scanner-e2e-test.yaml +++ b/.github/workflows/scanner-e2e-test.yaml @@ -14,32 +14,29 @@ jobs: if: > github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'scanner-functional-tests') - runs-on: ubuntu-latest - strategy: - matrix: - image: - - scanner-v4 - - scanner-v4-db + runs-on: ubuntu-24.04 steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: "${{ github.event.pull_request.head.sha }}" - name: Get tag run: | - echo "tag=$(make tag)" >> "$GITHUB_ENV" - - name: Wait for image ${{ matrix.image }} - uses: stackrox/actions/release/wait-for-image@v1 + echo "tag=$(make --quiet --no-print-directory tag)" >> "$GITHUB_ENV" + - name: Wait for images + uses: stackrox/actions/release/wait-for-image@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/release/wait-for-image@v1 with: token: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} - image: rhacs-eng/${{ matrix.image }}:${{ env.tag }} + image: | + rhacs-eng/scanner-v4:${{ env.tag }} + rhacs-eng/scanner-v4-db:${{ env.tag }} scanner-ft-run: name: Deploy and run needs: - scanner-ft-images - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: SCANNER_E2E_QUAY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} SCANNER_E2E_QUAY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} @@ -48,7 +45,7 @@ jobs: SCANNER_E2E_REDHAT_PASSWORD: ${{ secrets.REDHAT_PASSWORD }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: "${{ github.event.pull_request.head.sha }}" @@ -58,7 +55,7 @@ jobs: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - name: Create cluster (minikube) - uses: medyagh/setup-minikube@latest + uses: medyagh/setup-minikube@e9e035a86bbc3caea26a450bd4dbf9d0c453682e # ratchet:medyagh/setup-minikube@latest with: driver: docker cpus: max diff --git a/.github/workflows/scanner-mapping-update.yaml b/.github/workflows/scanner-mapping-update.yaml index 82b440dbcf09f..236c427937429 100644 --- a/.github/workflows/scanner-mapping-update.yaml +++ b/.github/workflows/scanner-mapping-update.yaml @@ -7,20 +7,20 @@ on: jobs: update-scanner-mappings: if: ${{ github.repository_owner == 'stackrox' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: REPOMAPPING_DIR: repomapping steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - name: Authenticate with Google Cloud - uses: google-github-actions/auth@v3 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 with: credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 - name: Download repository mappings run: scanner/image/scanner/download-mappings.sh "${{env.REPOMAPPING_DIR}}" @@ -34,7 +34,18 @@ jobs: # Upload mapping.zip to Google Cloud Storage gsutil cp -r "redhat-repository-mappings" "gs://definitions.stackrox.io/v4" - - name: Send Slack notification on workflow failure - if: failure() - run: | - curl -X POST -H 'Content-type: application/json' --data '{"text":" Failure in Scanner V4 Updater Workflow: Update Repository Mappings"}' ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + send-notification: + needs: update-scanner-mappings + if: failure() + runs-on: ubuntu-24.04 + permissions: + contents: read + checks: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + - uses: ./.github/actions/notify-slack-on-failure + with: + slack-webhook: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + mention: "" diff --git a/.github/workflows/scanner-nvd-update.yaml b/.github/workflows/scanner-nvd-update.yaml index 902dd4f6c03bb..8294ffb27ab71 100644 --- a/.github/workflows/scanner-nvd-update.yaml +++ b/.github/workflows/scanner-nvd-update.yaml @@ -12,7 +12,7 @@ on: jobs: fetch-nvd-feeds: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: > github.repository_owner == 'stackrox' && github.event_name == 'schedule' || @@ -22,10 +22,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # ratchet:actions/setup-python@v6 with: python-version: '3.10' @@ -51,7 +51,7 @@ jobs: if-no-files-found: error fetch-nvd-api: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: > github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || @@ -60,10 +60,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # ratchet:actions/setup-python@v6 with: python-version: '3.10' @@ -94,12 +94,12 @@ jobs: upload-pr-nvd-feeds: needs: - fetch-nvd-feeds - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-nvd') steps: # Checkout master to get the latest local actions - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 @@ -109,18 +109,18 @@ jobs: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - name: Checkout specific reference - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Authenticate with test GCS bucket - uses: google-github-actions/auth@v3 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 with: credentials_json: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 - name: Download NVD uses: ./.github/actions/download-artifact-with-retry @@ -145,7 +145,7 @@ jobs: # Checkout again to get the latest local actions for cleanup phase - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 if: always() with: fetch-depth: 0 @@ -153,11 +153,11 @@ jobs: upload-nvd-feeds: needs: - fetch-nvd-feeds - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: github.event_name != 'pull_request' steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.ref }} @@ -168,12 +168,12 @@ jobs: path: . - name: Authenticate with Google Cloud - uses: google-github-actions/auth@v3 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 with: credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 - name: Upload NVD Feeds to Google Cloud Storage run: | @@ -182,11 +182,11 @@ jobs: upload-nvd-api: needs: - fetch-nvd-api - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: github.event_name != 'pull_request' steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.ref }} @@ -197,12 +197,12 @@ jobs: path: . - name: Authenticate with Google Cloud - uses: google-github-actions/auth@v3 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 with: credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 - name: Upload NVD API to Google Cloud Storage run: @@ -212,9 +212,15 @@ jobs: needs: - upload-nvd-feeds - upload-nvd-api - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ failure() && github.ref_name == 'master' }} + permissions: + contents: read + checks: read + actions: read steps: - - name: Send Slack notification on workflow failure - run: | - curl -X POST -H 'Content-type: application/json' --data '{"text":"<${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Workflow ${{ github.workflow }}> failed in repository ${{ github.repository }}: Failed to update NVD CVEs"}' ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + - uses: ./.github/actions/notify-slack-on-failure + with: + slack-webhook: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} diff --git a/.github/workflows/scanner-offline-bundle-update.yaml b/.github/workflows/scanner-offline-bundle-update.yaml index cdba51a879ede..10213e553e435 100644 --- a/.github/workflows/scanner-offline-bundle-update.yaml +++ b/.github/workflows/scanner-offline-bundle-update.yaml @@ -11,7 +11,7 @@ on: jobs: offline-bundle-matrix: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: > github.repository_owner == 'stackrox' && github.event_name == 'schedule' || @@ -21,7 +21,7 @@ jobs: matrix: ${{ steps.output-matrix.outputs.matrix }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 @@ -36,7 +36,7 @@ jobs: offline-bundle-generate: needs: offline-bundle-matrix - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -46,10 +46,10 @@ jobs: SUPPORTED_RELEASES: ${{ matrix.versions.supported_releases }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 - uses: ./.github/actions/job-preamble with: @@ -69,11 +69,11 @@ jobs: if-no-files-found: error offline-bundle-upload: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - offline-bundle-generate steps: - - uses: actions/download-artifact@v7 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # ratchet:actions/download-artifact@v8 with: path: bundles pattern: offline-bundle-* @@ -81,18 +81,18 @@ jobs: - name: Authenticate with Google Cloud (release) if: github.event_name == 'schedule' - uses: google-github-actions/auth@v3 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 with: credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} - name: Authenticate with Google Cloud (pull request) if: github.event_name == 'pull_request' - uses: google-github-actions/auth@v3 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 with: credentials_json: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 - name: Set GCS bucket (release) if: github.event_name == 'schedule' @@ -125,9 +125,15 @@ jobs: - offline-bundle-matrix - offline-bundle-generate - offline-bundle-upload - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: github.event_name == 'schedule' && failure() + permissions: + contents: read + checks: read + actions: read steps: - - name: Send Slack notification on workflow failure - run: | - curl -X POST -H 'Content-type: application/json' --data '{"text":"<${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Workflow ${{ github.workflow }}> failed in repository ${{ github.repository }}: Failed to update offline vulnerabilities bundle"}' ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + - uses: ./.github/actions/notify-slack-on-failure + with: + slack-webhook: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} diff --git a/.github/workflows/scanner-updater-bundle.yaml b/.github/workflows/scanner-updater-bundle.yaml new file mode 100644 index 0000000000000..de4e98dc7c40a --- /dev/null +++ b/.github/workflows/scanner-updater-bundle.yaml @@ -0,0 +1,229 @@ +name: Scanner updater bundle +# Downloads per-source vulnerability data from GCS, validates each source, +# and assembles vulnerabilities.zip for each bundle stream defined in +# VULNERABILITY_BUNDLE_VERSION. Missing or empty sources are skipped with a +# Slack notification; if no sources are valid the job fails. + +on: + schedule: + - cron: "47 * * * *" + workflow_dispatch: + inputs: + force: + description: "Force rebuild even if no sources are newer than the bundle" + required: false + type: boolean + default: false + +permissions: + contents: read + +jobs: + build-matrix: + if: github.repository_owner == 'stackrox' + runs-on: ubuntu-24.04 + outputs: + streams: ${{ steps.emit.outputs.streams }} + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Build stream matrix + id: emit + run: | + set -euo pipefail + streams=$(./.github/workflows/scripts/scanner-get-released-tags.sh | jq -c '[.[].version]') + echo "streams=${streams}" >> "$GITHUB_OUTPUT" + + bundle: + needs: build-matrix + runs-on: ubuntu-24.04 + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.11@sha256:cea78f2b3a5a38c9ccda678712daaed90b7c372d6b5df963feb758b32dccebe1 + volumes: + - /tmp:/tmp + - /usr:/mnt/usr + - /opt:/mnt/opt + strategy: + fail-fast: false + matrix: + stream: ${{ fromJson(needs.build-matrix.outputs.streams) }} + concurrency: + group: scanner-updater-bundle-${{ matrix.stream }} + cancel-in-progress: false + defaults: + run: + shell: bash + env: + STREAM: ${{ matrix.stream }} + SOURCE_CONFIG: scanner/updater/ci/source-config.yaml + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Authenticate with GCS + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + + - name: Check for changes + id: check-changes + env: + FORCE: ${{ inputs.force == true }} + run: | + set -euo pipefail + + sc=./.github/workflows/scripts/scanner-updater-config.sh + bucket=$("$sc" bucket) + prefix=$("$sc" prefix) + + echo "GCS_BUCKET=$bucket" >> "$GITHUB_ENV" + echo "GCS_PREFIX=$prefix" >> "$GITHUB_ENV" + + if [[ "$FORCE" == "true" ]]; then + echo "Force rebuild requested" + echo "changed=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + bundle_obj=$("$sc" bundle-object "$STREAM") + + # Get the bundle's creation timestamp. If the bundle does not + # exist or the query fails, assume changes and proceed. + if ! bundle_ts=$(gcloud storage objects describe "$bundle_obj" \ + --format="value(creation_time)" 2>/dev/null) || [ -z "$bundle_ts" ]; then + echo "Bundle not found or query failed; assuming changes" + echo "changed=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + bundle_epoch=$(date -d "$bundle_ts" +%s) + echo "Bundle created: $bundle_ts (epoch: $bundle_epoch)" + + # Check if any source has been updated after the bundle. + changed=false + while IFS= read -r source; do + object="gs://${bucket}/${prefix}/${STREAM}/sources/${source}.json.zst" + newer=$(GCS_OBJECT="$object" \ + ./.github/workflows/scripts/scanner-updater-check-freshness.sh --newer-than "$bundle_epoch") || { + echo "Source '${source}': check failed; assuming changes" + changed=true; break + } + if [[ "$newer" == "true" ]]; then + echo "Source '${source}' is newer than bundle" + changed=true; break + fi + done < <("$sc" sources "$STREAM") + + echo "changed=${changed}" >> "$GITHUB_OUTPUT" + if [ "$changed" = "false" ]; then + echo "No sources newer than bundle; skipping" + fi + + - name: Download sources + if: steps.check-changes.outputs.changed == 'true' + run: | + set -euxo pipefail + + sc=./.github/workflows/scripts/scanner-updater-config.sh + mkdir -p bundles + + while IFS= read -r source; do + object="gs://${GCS_BUCKET}/${GCS_PREFIX}/${STREAM}/sources/${source}.json.zst" + dest="bundles/${source}.json.zst" + + # Retry if the source is overwritten between describe and cp. + downloaded=false + for attempt in 1 2 3; do + meta=$(gcloud storage objects describe "$object" --format=json) + gen=$(jq -r '.generation' <<< "$meta") + timestamp=$(jq -re '.custom_fields["refresh-completed-at"]' <<< "$meta") + + if gcloud storage cp "$object" "$dest" --if-generation-match="$gen"; then + touch -d "$timestamp" "$dest" + downloaded=true + break + fi + echo "Source '${source}': download attempt ${attempt} failed, retrying" + done + + if [[ "$downloaded" != "true" ]]; then + echo "::error::${source}: download failed after 3 attempts" + exit 1 + fi + done < <("$sc" sources "$STREAM") + + - name: Validate sources + id: download-validate + if: steps.check-changes.outputs.changed == 'true' + run: | + set -euo pipefail + + valid_files=() + failed_sources=() + while IFS= read -r source; do + dest="bundles/${source}.json.zst" + if [ ! -f "$dest" ] || [ ! -s "$dest" ]; then + echo "::error::${source}: missing or empty" + failed_sources+=("$source") + else + echo "source '${source}': OK" + valid_files+=("$dest") + fi + done < <(./.github/workflows/scripts/scanner-updater-config.sh sources "$STREAM") + + ( IFS=','; echo "failed=${failed_sources[*]}" >> "$GITHUB_OUTPUT" ) + + if [ "${#valid_files[@]}" -eq 0 ]; then + echo "::error::no valid source files found for stream '${STREAM}'; cannot assemble bundle" + exit 1 + fi + + printf '%s\n' "${valid_files[@]}" > valid-files.txt + echo "Validated: ${#valid_files[@]} of $(( ${#valid_files[@]} + ${#failed_sources[@]} )) source(s) OK" + + - name: Assemble and upload bundle + if: steps.check-changes.outputs.changed == 'true' + run: | + set -euo pipefail + + zip vulnerabilities.zip -@ < valid-files.txt + + bundle_obj="gs://${GCS_BUCKET}/${GCS_PREFIX}/${STREAM}/vulnerabilities.zip" + gcloud storage cp vulnerabilities.zip "$bundle_obj" + echo "Uploaded bundle for stream '${STREAM}' -> ${bundle_obj}" + + - name: Notify on partial failure + if: >- + always() && steps.download-validate.outcome == 'success' && + steps.download-validate.outputs.failed != '' + uses: ./.github/actions/notify-slack-on-failure + with: + slack-webhook: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + message: >- + Vulnerability bundler: validation failures for stream '${{ env.STREAM }}' + (sources: ${{ steps.download-validate.outputs.failed }}): + <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs> + + notify: + needs: [bundle] + if: always() && needs.bundle.result == 'failure' + runs-on: ubuntu-24.04 + permissions: + contents: read + checks: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + - uses: ./.github/actions/notify-slack-on-failure + with: + slack-webhook: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} diff --git a/.github/workflows/scanner-updater-export.yaml b/.github/workflows/scanner-updater-export.yaml new file mode 100644 index 0000000000000..cbe0b3a3fa41c --- /dev/null +++ b/.github/workflows/scanner-updater-export.yaml @@ -0,0 +1,211 @@ +name: Scanner updater export + +on: + workflow_call: + inputs: + source: + description: "Updater name (e.g. alpine, nvd, osv)" + required: true + type: string + ref: + description: "Git reference to build the updater from" + required: true + type: string + object: + description: "Full GCS object path (gs://bucket/prefix/stream/sources/updater.json.zst)" + required: true + type: string + manual_url: + description: "URL to manual vulnerability data YAML" + required: true + type: string + skip_freshness_check: + description: "Skip the freshness recheck and run the update unconditionally" + required: false + type: boolean + default: false + secrets: + GCP_SERVICE_ACCOUNT_STACKROX_CI: + required: true + GOOGLE_SA_CIRCLECI_SCANNER: + required: true + GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER: + required: true + SLACK_ONCALL_SCANNER_WEBHOOK: + required: true + +permissions: + contents: read + +jobs: + recheck: + if: ${{ !inputs.skip_freshness_check }} + runs-on: ubuntu-24.04 + outputs: + result: ${{ steps.check.outputs.result }} + concurrency: + group: scanner-updater-export-${{ inputs.object }} + cancel-in-progress: false + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Authenticate with GCS + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + + - name: Recheck freshness + id: check + env: + GCS_OBJECT: ${{ inputs.object }} + UPDATER_SOURCE: ${{ inputs.source }} + SOURCE_CONFIG: scanner/updater/ci/source-config.yaml + run: | + interval_secs=$(./.github/workflows/scripts/scanner-updater-config.sh interval-secs "$UPDATER_SOURCE") + + fresh=$(./.github/workflows/scripts/scanner-updater-check-freshness.sh --max-age "$interval_secs") || { + echo "result=error" >> "$GITHUB_OUTPUT" + exit 0 + } + if [[ "$fresh" == "true" ]]; then + echo "result=skip" >> "$GITHUB_OUTPUT" + else + echo "result=run" >> "$GITHUB_OUTPUT" + fi + + - name: Fail on recheck error + if: steps.check.outputs.result == 'error' + run: | + echo "::error::freshness recheck failed for ${{ inputs.source }} (object: ${{ inputs.object }})" + exit 1 + + update: + needs: recheck + if: always() && (needs.recheck.result == 'skipped' || needs.recheck.outputs.result == 'run') + runs-on: ubuntu-24.04 + concurrency: + group: scanner-updater-export-${{ inputs.object }} + cancel-in-progress: false + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.11@sha256:cea78f2b3a5a38c9ccda678712daaed90b7c372d6b5df963feb758b32dccebe1 + volumes: + - /tmp:/tmp + - /usr:/mnt/usr + - /opt:/mnt/opt + env: + UPDATER_SOURCE: ${{ inputs.source }} + ROX_GIT_REF: ${{ inputs.ref }} + GCS_OBJECT: ${{ inputs.object }} + STACKROX_RHEL_VEX_COMPRESSED_FILE_TIMEOUT: "10m" + STACKROX_RHEL_VEX_IGNORE_KERNEL_PACKAGES: "true" + steps: + - name: Checkout repository (master for local actions) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: ./.github/actions/job-preamble + with: + free-disk-space: 50 + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + + - name: Checkout updater ref + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: Download NVD + if: inputs.source == 'nvd' + run: | + set -euo pipefail + url="https://definitions.stackrox.io/v4/nvd/nvd-feeds.zip" + curl --fail --silent --show-error \ + --retry 3 --retry-all-errors --retry-delay 5 \ + --connect-timeout 10 --max-time 300 \ + -o nvd.zip "$url" + echo "STACKROX_NVD_ZIP_PATH=$PWD/nvd.zip" >> "$GITHUB_ENV" + + - name: Build updater + run: | + echo "Building updater for $UPDATER_SOURCE from ref $ROX_GIT_REF..." + make tag + make -C scanner bin/updater + + - name: Run updater + env: + MANUAL_URL: ${{ inputs.manual_url }} + run: | + set -euo pipefail + export STACKROX_SCANNER_V4_UPDATER_SOURCES="$UPDATER_SOURCE" + mkdir -p bundles + scanner/bin/updater export --manual-url "$MANUAL_URL" bundles + + - name: Authenticate with GCS for upload + if: ${{ !inputs.skip_freshness_check }} + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} + + - name: Authenticate with GCS for upload (PR) + if: ${{ inputs.skip_freshness_check }} + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + + - name: Upload to GCS + run: | + set -euo pipefail + src="bundles/${UPDATER_SOURCE}.json.zst" + if [[ ! -f "$src" ]]; then + echo "::error::expected output file not found: $src" + exit 1 + fi + + digest="sha256:$(sha256sum "$src" | cut -d' ' -f1)" + completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + gcloud storage cp "$src" "$GCS_OBJECT" \ + --custom-metadata="git-ref=$ROX_GIT_REF,content-digest=$digest,refresh-completed-at=$completed_at" + + echo "Uploaded $src -> $GCS_OBJECT (digest=$digest)" + + - name: Checkout repository (cleanup) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + if: always() + with: + fetch-depth: 0 + persist-credentials: false + + notify: + needs: [recheck, update] + if: >- + !inputs.skip_freshness_check && + always() && + (needs.recheck.result == 'failure' || needs.update.result == 'failure') + runs-on: ubuntu-24.04 + permissions: + contents: read + checks: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + - uses: ./.github/actions/notify-slack-on-failure + with: + slack-webhook: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + context: "Source: ${{ inputs.source }} (object: ${{ inputs.object }})" + # Scopes annotation/status collection to this matrix leg's own jobs. + job-name-filter: "update (${{ inputs.source }}, ${{ inputs.ref }}," diff --git a/.github/workflows/scanner-updater-schedule.yaml b/.github/workflows/scanner-updater-schedule.yaml new file mode 100644 index 0000000000000..7cdf390562a86 --- /dev/null +++ b/.github/workflows/scanner-updater-schedule.yaml @@ -0,0 +1,172 @@ +name: Scanner updater schedule + +on: + schedule: + - cron: "17 * * * *" + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-matrix: + permissions: + contents: read + actions: read + if: >- + github.repository_owner == 'stackrox' && ( + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns') + ) + runs-on: ubuntu-24.04 + outputs: + needs-notification: ${{ steps.matrix-scheduled.outputs.has_errors }} + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Authenticate with GCS + if: github.event_name != 'pull_request' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} + + - name: Set up Cloud SDK + if: github.event_name != 'pull_request' + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + + - name: Build updater matrix (scheduled/dispatch) + if: github.event_name != 'pull_request' + id: matrix-scheduled + env: + SOURCE_CONFIG: scanner/updater/ci/source-config.yaml + run: | + set -euo pipefail + # Keep cadence eligibility stable if this job is queued or rerun. + schedule_time=$(GH_TOKEN="${{ github.token }}" gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}" --jq '.created_at') + export SCHEDULE_TIME="$schedule_time" + result=$(./.github/workflows/scripts/scanner-updater-matrix.sh) + { + echo "has_errors=$(echo "$result" | jq -r '.has_errors')" + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "errors<<$EOF" + echo "$result" | jq -r '.errors | join(", ")' + echo "$EOF" + } >> "$GITHUB_OUTPUT" + echo "$result" > matrix-result.json + + - name: Log matrix errors + # This annotation is what send-notification (at the end of this + # workflow) picks up via include-successful-jobs, since this job + # doesn't fail just because some sources had scheduling errors. + if: github.event_name != 'pull_request' && steps.matrix-scheduled.outputs.has_errors == 'true' + env: + ERRORS: ${{ steps.matrix-scheduled.outputs.errors }} + run: | + echo "::error::vulnerability updater scheduling errors: ${ERRORS}" + + - name: Build updater matrix (PR) + if: github.event_name == 'pull_request' + env: + SOURCE_CONFIG: scanner/updater/ci/source-config.yaml + PR_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + sc=./.github/workflows/scripts/scanner-updater-config.sh + bucket="scanner-v4-test" + prefix=$("$sc" prefix) + matrix="{\"include\":[{\"source\":\"manual\",\"ref\":\"${PR_SHA}\",\"object\":\"gs://${bucket}/${prefix}/pr-${PR_NUMBER}/sources/manual.json.zst\"}]}" + printf '{"has_due":true,"has_errors":false,"errors":[],"matrix":%s}\n' "$matrix" > matrix-result.json + + - name: Validate matrix builder (PR) + if: github.event_name == 'pull_request' + env: + SOURCE_CONFIG: scanner/updater/ci/source-config.yaml + run: | + echo "Running matrix builder for validation (output not used for scheduling):" + ./.github/workflows/scripts/scanner-updater-matrix.sh | jq . + + - name: Upload matrix result + uses: ./.github/actions/upload-artifact-with-retry + with: + name: matrix-result + path: matrix-result.json + + plan: + needs: build-matrix + runs-on: ubuntu-24.04 + outputs: + has_due: ${{ steps.emit.outputs.has_due }} + matrix: ${{ steps.emit.outputs.matrix }} + steps: + - name: Download matrix result + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # ratchet:actions/download-artifact@v8 + with: + name: matrix-result + + - name: Emit matrix outputs + id: emit + run: | + set -euo pipefail + result=$(cat matrix-result.json) + { + echo "has_due=$(echo "$result" | jq -r '.has_due')" + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "matrix<<$EOF" + echo "$result" | jq -c '.matrix' + echo "$EOF" + } >> "$GITHUB_OUTPUT" + + update: + needs: plan + if: needs.plan.outputs.has_due == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.plan.outputs.matrix) }} + # Propagates to scanner-updater-export.yaml's notify job. + permissions: + contents: read + checks: read + actions: read + uses: ./.github/workflows/scanner-updater-export.yaml + with: + source: ${{ matrix.source }} + ref: ${{ matrix.ref }} + object: ${{ matrix.object }} + manual_url: ${{ github.event_name == 'pull_request' && format('https://raw.githubusercontent.com/stackrox/stackrox/{0}/scanner/updater/manual/vulns.yaml', github.event.pull_request.head.sha) || 'https://raw.githubusercontent.com/stackrox/stackrox/master/scanner/updater/manual/vulns.yaml' }} + skip_freshness_check: ${{ github.event_name == 'pull_request' }} + secrets: + GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + GOOGLE_SA_CIRCLECI_SCANNER: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} + GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} + SLACK_ONCALL_SCANNER_WEBHOOK: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + + send-notification: + # Covers both hard failures in build-matrix/plan and data-level scheduling + # errors reported by the matrix script (build-matrix succeeds either way). + needs: [build-matrix, plan] + if: >- + github.event_name != 'pull_request' && + (failure() || needs.build-matrix.outputs.needs-notification == 'true') + runs-on: ubuntu-24.04 + permissions: + contents: read + checks: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + - uses: ./.github/actions/notify-slack-on-failure + with: + slack-webhook: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + include-successful-jobs: "true" diff --git a/.github/workflows/scanner-versioned-definitions-update.yaml b/.github/workflows/scanner-versioned-definitions-update.yaml index 54622a9d776cf..c9ba26d32df8f 100644 --- a/.github/workflows/scanner-versioned-definitions-update.yaml +++ b/.github/workflows/scanner-versioned-definitions-update.yaml @@ -1,8 +1,6 @@ name: Scanner versioned vulnerabilities update on: - schedule: - - cron: "30 */4 * * *" pull_request: types: - opened @@ -15,7 +13,7 @@ on: required: false ga_ref: description: > - WARNING - THIS WLL UPDATE A GA BUNDLE STREAM: Sets the git reference + WARNING - THIS WLL UPDATE A GA BUNDLE STREAM if not a dry-run: Sets the git reference to build a bundle that will be uploaded to the GA version stream. required: false rc_ref: @@ -24,6 +22,11 @@ on: release candidate version stream, appending `-rc` to the version stream vN (eg. v2-rc). required: false + vuln_bundles: + description: > + Sets a specific updater to run. If unset, all updaters are built. + Not supported by older git references, please double-check + required: false job: type: choice description: "Choose the NVD source" @@ -32,16 +35,20 @@ on: - nvd-feeds required: true default: nvd-feeds + dry_run: + description: "Dry-run" + default: false + type: boolean jobs: parse-versions: if: github.repository_owner == 'stackrox' && github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns') - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: versions: ${{ steps.set-versions.outputs.versions }} steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 - name: Parse VULNERABILITY_BUNDLE_VERSION @@ -50,6 +57,7 @@ jobs: INPUT_GA_REF: ${{ github.event.inputs.ga_ref || '' }} INPUT_RC_REF: ${{ github.event.inputs.rc_ref || '' }} INPUT_STREAM: ${{ github.event.inputs.stream || '' }} + INPUT_DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }} run: | set -o pipefail @@ -73,8 +81,11 @@ jobs: allow_rc=true fi if [ -z "$ref" ]; then - echo >&2 "abort: invalid input: could not determine the reference" - exit 1 + # Allow empty ref when dry_run is true (follow normal flow) + if [ "$INPUT_DRY_RUN" == "false" ]; then + echo >&2 "abort: invalid input: could not determine the reference" + exit 1 + fi fi # Configure environment to generate the matrix with versions and @@ -91,12 +102,12 @@ jobs: prepare-environment: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns') - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: manual_url: ${{ steps.set-manual-url.outputs.manual_url }} steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 @@ -104,7 +115,6 @@ jobs: if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-nvd') run: | set -eu - since_time=$(date -u -d '24 hours ago' '+%a, %d %b %Y %H:%M:%S GMT') NVD_BUNDLE_TYPE=${{ github.event.inputs.job || 'nvd-feeds' }} case "$NVD_BUNDLE_TYPE" in nvd-api) @@ -121,21 +131,15 @@ jobs: # Replace / with -, so the branch name isn't truncated when pushed to GCS. dir=${branch////-} url="https://storage.googleapis.com/scanner-v4-test/nvd/$dir/$nvd_file" - - code=$(curl \ - -o nvd.zip \ - -w "%{http_code}" \ - -H "If-Modified-Since: $since_time" \ - "$url") - - echo "code: $code" - echo "$code" | grep -q 200 + curl --fail --silent --show-error \ + --retry 3 --retry-all-errors --retry-delay 5 \ + --connect-timeout 10 --max-time 300 \ + -o nvd.zip "$url" - name: Download NVD if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-nvd') run: | set -eu - since_time=$(date -u -d '24 hours ago' '+%a, %d %b %Y %H:%M:%S GMT') NVD_BUNDLE_TYPE=${{ github.event.inputs.job || 'nvd-feeds' }} case "$NVD_BUNDLE_TYPE" in nvd-api) @@ -149,15 +153,10 @@ jobs: exit 1 esac url="https://definitions.stackrox.io/v4/nvd/$nvd_file" - - code=$(curl \ - -o nvd.zip \ - -w "%{http_code}" \ - -H "If-Modified-Since: $since_time" \ - "$url") - - echo "code: $code" - echo "$code" | grep -q 200 + curl --fail --silent --show-error \ + --retry 3 --retry-all-errors --retry-delay 5 \ + --connect-timeout 10 --max-time 300 \ + -o nvd.zip "$url" - name: Set MANUAL_URL id: set-manual-url @@ -188,9 +187,9 @@ jobs: needs: - parse-versions - prepare-environment - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 container: - image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.1 + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.11@sha256:cea78f2b3a5a38c9ccda678712daaed90b7c372d6b5df963feb758b32dccebe1 # ratchet:quay.io/stackrox-io/apollo-ci:scanner-test-0.5.11 volumes: # The updater makes heavy use of /tmp files. - /tmp:/tmp @@ -204,10 +203,11 @@ jobs: env: SCANNER_BUNDLE_VERSION: ${{ matrix.version }} ROX_GIT_REF: ${{ matrix.ref }} + STACKROX_RHEL_VEX_COMPRESSED_FILE_TIMEOUT: "10m" steps: # Checkout master to get the latest local actions - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 @@ -217,7 +217,7 @@ jobs: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - name: Checkout specific reference - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ env.ROX_GIT_REF }} @@ -244,13 +244,13 @@ jobs: unzip -l "$path" echo "STACKROX_NVD_ZIP_PATH=$path" >> "$GITHUB_ENV" - - name: Run Updater (single bundle) - if: ${{ env.SCANNER_BUNDLE_VERSION == 'v1' }} - run: | - scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" "definitions/${SCANNER_BUNDLE_VERSION}" - - name: Run updater (multi bundle) + env: + VULN_BUNDLES: ${{ github.event.inputs.vuln_bundles }} run: | + if [ -n "$VULN_BUNDLES" ]; then + export STACKROX_SCANNER_V4_UPDATER_SOURCES="$VULN_BUNDLES" + fi tag=$(make -s --no-print-directory tag) # In pre-4.9, the exporter requires --split to generate multi-bundles. case "$tag" in @@ -263,15 +263,75 @@ jobs: esac zip definitions/"$SCANNER_BUNDLE_VERSION"/vulnerabilities.zip bundles/*.json.zst + - name: Validate bundle + # ROX-33993: fails on any per-source issue; "Upload definitions artifacts" + # still runs via if: always() so healthy sources still reach users. + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + + files=(bundles/*.json.zst) + if [ "${#files[@]}" -eq 0 ]; then + echo "::error::no vulnerability bundle files found in bundles/ directory" + exit 1 + fi + + # Accumulate failures so all sources are checked before exiting. + failures=() + fail() { + echo "::error::$*" + failures+=("$*") + } + + # Counts JSON objects in an NDJSON stream, failing on any non-object value. + # shellcheck disable=SC2016 # $i is jq syntax, not a shell variable. + jq_filter='reduce inputs as $i (0; if ($i | type) == "object" then . + 1 else error("malformed JSON value: \($i | type)") end)' + for f in "${files[@]}"; do + name=$(basename "$f" .json.zst) + + if [ ! -s "$f" ]; then + fail "source '$name': file $f is empty (0 bytes)" + continue + fi + + if ! zstd -t "$f" >/dev/null 2>&1; then + fail "source '$name': file $f is not a valid zstd archive (corrupted/truncated?)" + continue + fi + + if ! record_count=$(zstd -dc "$f" | jq -n "$jq_filter" 2>/dev/null); then + fail "source '$name': file $f decompressed successfully, but a malformed JSON value was found" + continue + fi + + if [ "$record_count" -eq 0 ]; then + fail "source '$name': file $f decompresses fine but contains 0 JSON objects (expected >0)" + continue + fi + + echo "source '$name': OK ($record_count records)" + done + + if [ "${#failures[@]}" -gt 0 ]; then + echo "::error::vulnerability bundle validation failed: ${#failures[@]} of ${#files[@]} source(s) broken (affected source(s) will be missing from the published bundle; this does not block publishing)" + printf ' - %s\n' "${failures[@]}" + exit 1 + fi + + echo "vulnerability bundle validation passed: all ${#files[@]} sources OK" + + # Uploads the definitions as a temporary GitHub Actions artifact. The actual publishing to GCS happens on: upload-definitions - name: Upload definitions artifacts - uses: actions/upload-artifact@v6 + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # ratchet:actions/upload-artifact@v7 with: name: artifact_${{ env.SCANNER_BUNDLE_VERSION }} path: definitions # Checkout again to get the latest local actions for cleanup phase - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 if: always() with: fetch-depth: 0 @@ -280,18 +340,20 @@ jobs: if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns') needs: - prepare-environment - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 container: - image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.1 + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.11@sha256:cea78f2b3a5a38c9ccda678712daaed90b7c372d6b5df963feb758b32dccebe1 # ratchet:quay.io/stackrox-io/apollo-ci:scanner-test-0.5.11 volumes: # The updater makes heavy use of /tmp files. - /tmp:/tmp - /usr:/mnt/usr - /opt:/mnt/opt + env: + STACKROX_RHEL_VEX_COMPRESSED_FILE_TIMEOUT: "10m" steps: # Checkout master to get the latest local actions - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 @@ -301,19 +363,19 @@ jobs: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - name: Checkout specific reference - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Authenticate with test GCS bucket if: github.event_name == 'pull_request' - uses: google-github-actions/auth@v3 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 with: credentials_json: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 - name: Download NVD uses: ./.github/actions/download-artifact-with-retry @@ -359,7 +421,7 @@ jobs: # Checkout again to get the latest local actions for cleanup phase - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 if: always() with: fetch-depth: 0 @@ -367,15 +429,15 @@ jobs: upload-definitions: needs: - build-and-run - if: ${{ (failure() || success()) && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} - runs-on: ubuntu-latest + if: ${{ (failure() || success()) && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'false')) }} + runs-on: ubuntu-24.04 steps: # Checkout to run ./.github/actions/download-artifact-with-retry - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 - - uses: actions/download-artifact@v7 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # ratchet:actions/download-artifact@v8 with: # Vulnerability bundles are named `artifact_${{ env.SCANNER_BUNDLE_VERSION }}`. pattern: "artifact_*" @@ -422,12 +484,12 @@ jobs: done - name: Authenticate with Google Cloud - uses: google-github-actions/auth@v3 + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 with: credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v3 + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 - name: Upload vulnerability bundles to GCS working-directory: ./definitions_files @@ -444,45 +506,18 @@ jobs: echo "Copy upstream dev (dev) to downstream dev (1.0.0)" gsutil cp -r gs://definitions.stackrox.io/v4/vulnerability-bundles/dev/* gs://definitions.stackrox.io/v4/vulnerability-bundles/1.0.0/ - - name: Copy v1 to pre-versioned bundles - # Run on schedule or when dispatched to build "GA" (only possible if RC was not requested). - if: github.event.inputs.rc_ref == '' - run: | - # Only execute if v1 files were created by the build job. - if ! [ -d "definitions_files/v1" ]; then - echo "error: no definition files for v1 were created: skipping..." - exit 0 - fi - # Using v1 bundle for released versions listed in RELEASE_VERSION. - single=gs://definitions.stackrox.io/v4/vulnerability-bundles/v1/vulns.json.zst - multi=gs://definitions.stackrox.io/v4/vulnerability-bundles/v1/vulnerabilities.zip - # Parse all supported pre-4.6 releases and copy the versioned bundle v1 to ensure - # these releases get updates. - grep -E "^4\.(4|5)\.[0-9]+$" scanner/updater/version/RELEASE_VERSION | while read -r release; do - case "$release" in - 4.4.*) - gsutil cp "$single" "gs://definitions.stackrox.io/v4/vulnerability-bundles/$release/" - ;; - 4.5.*) - gsutil cp "$single" "gs://definitions.stackrox.io/v4/vulnerability-bundles/$release/" - gsutil cp "$multi" "gs://definitions.stackrox.io/v4/vulnerability-bundles/$release/" - ;; - *) - echo "Should not happen!" - echo "Error: unexpected release version: $release" - echo "Ignoring..." - ;; - esac - echo "Copied v1 into $release" - done - send-notification: needs: - build-and-run - upload-definitions - runs-on: ubuntu-latest - if: ${{ failure() && github.ref_name == 'master' }} + runs-on: ubuntu-24.04 + if: ${{ failure() && github.ref_name == 'master' && (github.event_name == 'schedule' || github.event.inputs.dry_run == 'false') }} + permissions: + checks: read + actions: read steps: - - name: Send Slack notification on workflow failure - run: | - curl -X POST -H 'Content-type: application/json' --data '{"text":"<${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Workflow ${{ github.workflow }}> failed in repository ${{ github.repository }}: Failed to update vulnerabilities"}' ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + - uses: ./.github/actions/notify-slack-on-failure + with: + slack-webhook: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} diff --git a/.github/workflows/scripts/check-jira-issues.sh b/.github/workflows/scripts/check-jira-issues.sh index 4956013f0e0cb..2176c0f7b364d 100644 --- a/.github/workflows/scripts/check-jira-issues.sh +++ b/.github/workflows/scripts/check-jira-issues.sh @@ -2,90 +2,81 @@ # # Checks if there are open issues for the given release # and adds comments to such issues. -# + set -euo pipefail -PROJECTS="$1" +PROJECT="$1" RELEASE="$2" PATCH="$3" NAMED_RELEASE_PATCH="$4" check_not_empty \ - PROJECTS \ + PROJECT \ RELEASE \ PATCH \ NAMED_RELEASE_PATCH \ \ JIRA_TOKEN \ + JIRA_BASE_URL \ + JIRA_USER \ DRY_RUN -# TODO: Jira returns 400 if requested fixVersion does not exist. That means -# the named release must exist on Jira, which is not given. -JQL_OPEN_ISSUES="project IN ($PROJECTS) \ -AND fixVersion = \"$RELEASE.$PATCH\" \ -AND statusCategory != done \ -AND (Component IS EMPTY or Component NOT IN (Documentation, \"ACS Cloud Service\")) \ -AND type NOT IN (Epic, \"Feature Request\") \ +# Finds closed issues that have open PRs for the given release. +JQL_CLOSED_WITH_PR="project = $PROJECT \ +AND fixVersion = '$RELEASE.$PATCH' \ +AND statusCategory = Done \ +AND development[pullrequests].open > 0 \ +AND (Component IS EMPTY or Component NOT IN (Documentation, 'ACS Cloud Service')) \ ORDER BY assignee" -JQL_CLOSED_WITH_PR="project IN ($PROJECTS) \ -AND fixVersion = \"$RELEASE.$PATCH\" \ -AND statusCategory = done \ -AND (Component IS EMPTY or Component NOT IN (Documentation, \"ACS Cloud Service\")) \ -AND issue.property[development].openprs > 0 \ +# Finds open issues for the given release. +JQL_OPEN_ISSUES="project = $PROJECT \ +AND fixVersion = '$RELEASE.$PATCH' \ +AND statusCategory != done \ +AND (Component IS EMPTY or Component NOT IN (Documentation, 'ACS Cloud Service')) \ +AND type NOT IN (Epic, 'Feature Request') \ ORDER BY assignee" -get_issues() { - curl --fail -sSL --get --data-urlencode "jql=$1" \ - -H "Authorization: Bearer $JIRA_TOKEN" \ - -H "Accept: application/json" \ - "https://issues.redhat.com/rest/api/2/search" -} - comment_on_issues_list() { - ISSUES=$1 + local FILE_NAME="$1" + local COMMENT="$2" while read -r KEY; do - comment_on_single_issue "$KEY" - done < <(cut -d " " -f 1 <<< "$ISSUES") -} - -comment_on_single_issue() { - curl --fail --output /dev/null -sSL -X POST \ - -H "Authorization: Bearer $JIRA_TOKEN" \ - -H "Content-Type: application/json" \ - --data "{\"body\": \ -\"Release $NAMED_RELEASE_PATCH is ongoing. \ -Please update the status of this issue and notify the release engineer.\"}" \ - "https://issues.redhat.com/rest/api/2/issue/$1/comment" + comment_on_issue "$KEY" "$COMMENT" + done < <(jq -r ".[] | .key" "$FILE_NAME") } get_issues_summary() { + local FILE_NAME="$1" read -r GH_MD_FORMAT_LINE < + +set -euo pipefail + +TAG="$1" +check_not_empty TAG +VERSION="${TAG/-rc.[0-9]*/}" +SCANNER_VERSION_PATH="scanner/updater/version/RELEASE_VERSION" + +# Fetch the file content from the master branch (raw). +SCANNER_VERSION=$(gh api -H "Accept: application/vnd.github.v3.raw+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "/repos/${GITHUB_REPOSITORY}/contents/$SCANNER_VERSION_PATH?ref=master") + +if ! grep -Fqx "$VERSION" <<<"$SCANNER_VERSION"; then + gh_log error "Release version $VERSION (inferred from the tag '$TAG') is not present in $SCANNER_VERSION_PATH in master branch" + gh_summary "Release version not found in $SCANNER_VERSION_PATH in master branch" + gh_summary "Most likely, this is because the PR to update scanner version file created by \`start-release\` workflow is not merged" + gh_summary "➡️ Please check the PR created by \`start-release\` workflow that started this release." + gh_summary "➡️ There should be $VERSION in the \`RELEASE_VERSION\` file in master." + exit 1 +fi + +gh_summary "✅ Version ${VERSION} present in $SCANNER_VERSION_PATH" diff --git a/.github/workflows/scripts/common.sh b/.github/workflows/scripts/common.sh index 7d1d424a05274..b2da96eb48504 100644 --- a/.github/workflows/scripts/common.sh +++ b/.github/workflows/scripts/common.sh @@ -78,6 +78,146 @@ gh_summary() { } export -f gh_summary +########## +## JIRA ## +########## +jira_check_not_empty() { + check_not_empty \ + JIRA_USER \ + JIRA_TOKEN \ + JIRA_BASE_URL +} +export -f jira_check_not_empty + +# Retrieves issues from JIRA using JQL and stores them in JSON format in the OUTPUT_FILE. +# If additional fields are needed, they can be added to the "fields" array. +# +# Example: +# get_issues "project = ROX AND fixVersion = 4.9.0" "issues.json" +get_issues() { + jira_check_not_empty + local JQL="$1" + local OUTPUT_FILE="$2" + + # Initializations + PAGE_SIZE=20 + next_page_token="" + echo "[]" > "${OUTPUT_FILE}" + + # Paginate through the results + while true; do + cat < /tmp/request.json +{ + "jql": "$JQL", + "maxResults": "${PAGE_SIZE}", + "nextPageToken": "${next_page_token}", + "fields": [ + "key", + "assignee", + "issuetype", + "status", + "summary" + ] +} +EOF + + response=$(curl --fail -sSL --request POST \ + --url "https://${JIRA_BASE_URL}/rest/api/3/search/jql" \ + --user "${JIRA_USER}:${JIRA_TOKEN}" \ + --header "Accept: application/json" \ + --header "Content-Type: application/json" \ + --data-binary @/tmp/request.json \ + --retry 5 --retry-all-errors + ) + if [ -z "${response}" ]; then + gh_log error "Failed to retrieve issues from JIRA." + exit 1 + fi + + # Append the issues from the current page to $OUTPUT_FILE + # Need to use a temporary file as jq does not support in-place editing. + jq --argjson new_issues "$(echo "$response" | jq '.issues')" '. + $new_issues' "${OUTPUT_FILE}" > "${OUTPUT_FILE}.tmp" + mv "${OUTPUT_FILE}.tmp" "${OUTPUT_FILE}" + + if [[ "$(echo "${response}" | jq '.isLast')" = "true" ]]; then + # No more pages, exit the loop. + break + else + next_page_token="$(echo "${response}" | jq -r '.nextPageToken')" + fi + done +} +export -f get_issues + +# Retrieves a JIRA issue with ISSUE_KEY and returns it in JSON format. +# +# Example: +# get_issue "ROX-29997" > rox-29997.json +get_issue() { + jira_check_not_empty + local ISSUE_KEY="$1" + + curl --fail -sSL --request GET \ + --user "${JIRA_USER}:${JIRA_TOKEN}" \ + --header "Accept: application/json" \ + --retry 5 --retry-all-errors \ + "https://${JIRA_BASE_URL}/rest/api/3/issue/${ISSUE_KEY}" +} +export -f get_issue + +# Adds COMMENT to a JIRA issue with ISSUE_KEY. +# +# Example: +# comment_on_issue "ROX-29997" "Please update the status of this issue and notify the release engineer." +comment_on_issue() { + jira_check_not_empty + local ISSUE_KEY="$1" + local COMMENT="$2" + + ## Commenting on a single issue + cat < /tmp/request.json +{ + "body": { + "version": 1, + "type": "doc", + "content": [ + { + "type": "paragraph", + "content": [ + { + "type": "text", + "text": "${COMMENT}" + } + ] + } + ] + } +} +EOF + + curl --fail --output /dev/null -sSL --request POST \ + --user "${JIRA_USER}:${JIRA_TOKEN}" \ + --header "Accept: application/json" \ + --header "Content-Type: application/json" \ + --data-binary @/tmp/request.json \ + "https://${JIRA_BASE_URL}/rest/api/3/issue/${ISSUE_KEY}/comment" + + gh_log debug "Commented on issue ${ISSUE_KEY}" +} +export -f comment_on_issue + +get_jira_release() { + jira_check_not_empty + local PROJECT="$1" + local VERSION="$2" + + curl --fail -sSL --request GET \ + --user "${JIRA_USER}:${JIRA_TOKEN}" \ + --header "Accept: application/json" \ + --retry 5 --retry-all-errors \ + "https://${JIRA_BASE_URL}/rest/api/3/project/${PROJECT}/versions" | jq -r ".[] | select(.name == \"${VERSION}\")" +} +export -f get_jira_release # Fetches the CHANGELOG from a release branch # and returns the content of the section for a given version. diff --git a/.github/workflows/scripts/crdify.sh b/.github/workflows/scripts/crdify.sh new file mode 100755 index 0000000000000..37e9ce64b545f --- /dev/null +++ b/.github/workflows/scripts/crdify.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Run crdify on copies of CRDs after stripping out known false positives. +# Usage: +# crdify.sh [crdify flags] + +set -euo pipefail + +ROOT_DIR="$(dirname "${BASH_SOURCE[0]}")/../../.." +readonly ROOT_DIR + +if [[ $# -lt 2 ]]; then + echo "usage: $0 [crdify flags]" >&2 + exit 2 +fi + +old_crd="${1:?missing required argument 'old-crd'}" +new_crd="${2:?missing required argument 'new-crd'}" +shift +shift + +if [[ ! -f "$old_crd" ]]; then + echo "::error::CRD file not found: $old_crd" >&2 + exit 1 +fi + +if [[ ! -f "$new_crd" ]]; then + echo "::error::CRD file not found: $new_crd" >&2 + exit 1 +fi + + +# The addition of this particular validation rule, even though considered a breaking change by definition +# (https://github.com/kubernetes/community/blob/main/contributors/devel/sig-architecture/api_changes.md#backward-compatibility-gotchas) +# is fine in practice, since the `excludeByKind` branch is being added in the same release cycle as the rule itself, +# meaning no production CRS could have existed that violated the validation. +filter_spec_exclusions_one_of() { + local input="$1" + local output="$2" + + local message='at most one of the fields in [deployment excludeByKind] may be set' + local rule='(has(self.deployment)?1:0)+(has(self.excludeByKind)?1:0) <= 1' + + # Remove matching rule, and the validations list if empty afterwards. + message="${message}" rule="${rule}" \ + go run -C "${ROOT_DIR}/operator/tools/yq" github.com/mikefarah/yq/v4 eval ' + del( + .spec.versions[].schema.openAPIV3Schema.properties.spec.properties.exclusions.items."x-kubernetes-validations"[] + | select(.message == strenv(message) and .rule == strenv(rule)) + ) + | del( + .spec.versions[].schema.openAPIV3Schema.properties.spec.properties.exclusions.items."x-kubernetes-validations" + | select(length == 0) + ) + ' "$input" > "$output" +} + + +crd_basename="$(basename "${old_crd}")" +case "$crd_basename" in +config.stackrox.io_securitypolicies.yaml) + filtered_dir="$(mktemp -d)" + echo >&2 "Filtering known false positives from ${old_crd} and ${new_crd}" + filter_spec_exclusions_one_of "${old_crd}" "${filtered_dir}/old-${crd_basename}" + filter_spec_exclusions_one_of "${new_crd}" "${filtered_dir}/new-${crd_basename}" + old_crd="${filtered_dir}/old-${crd_basename}" + new_crd="${filtered_dir}/new-${crd_basename}" + ;; +*) + echo >&2 "No filtering rules for ${old_crd}, running crdify on original CRDs." + ;; +esac + +set -x +go run -C "${ROOT_DIR}/operator/tools/crdify" sigs.k8s.io/crdify --config=config.yaml "$@" \ + "file://${old_crd}" "file://${new_crd}" diff --git a/.github/workflows/scripts/local-env.sh b/.github/workflows/scripts/local-env.sh index b3465e5e33041..27714cc83a357 100755 --- a/.github/workflows/scripts/local-env.sh +++ b/.github/workflows/scripts/local-env.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # # Sets some necessary variables up for local testing. -# JIRA_TOKEN still has to be set manually. +# JIRA_TOKEN, JIRA_USER and JIRA_BASE_URL still have to be set manually. # # Usage example: # bash local-env.sh check-jira-release 3.73.0 ROX diff --git a/.github/workflows/scripts/scanner-get-released-tags.sh b/.github/workflows/scripts/scanner-get-released-tags.sh index d6073401abd4d..871393b666b8d 100755 --- a/.github/workflows/scripts/scanner-get-released-tags.sh +++ b/.github/workflows/scripts/scanner-get-released-tags.sh @@ -87,10 +87,6 @@ while read -r line; do read -r version ref <<< "$line" - # We only append `-rc` if reference is a "release version" reference, and - # use RC is true, setting to false by default. - append_rc="false" - case $ref in heads/*) resolved_tag="${ref#heads/}" @@ -101,7 +97,6 @@ while read -r line; do *) # Assume anything else is a release version reference. resolved_tag=${version_map[$ref]:-} - [[ "$use_rc" == "true" ]] && append_rc="true" ;; esac @@ -112,9 +107,9 @@ while read -r line; do fi case "$version" in - dev|v1) - # These vulnerability bundle versions are special, and we cannot - # validate them by fetching the vulnerability version. + dev) + # This vulnerability bundle version is special, and we cannot + # validate it by fetching the vulnerability version. ;; *) actual_version=$(fetch_version "$resolved_tag") @@ -127,7 +122,8 @@ while read -r line; do ;; esac - if [[ "$append_rc" == "true" ]]; then + # RC builds upload to the RC stream (e.g. v2-rc). + if [[ "$use_rc" == "true" ]]; then version="$version-rc" fi diff --git a/.github/workflows/scripts/scanner-offline-bundle-matrix.sh b/.github/workflows/scripts/scanner-offline-bundle-matrix.sh index 23a3823fc35b9..965b468dd817b 100755 --- a/.github/workflows/scripts/scanner-offline-bundle-matrix.sh +++ b/.github/workflows/scripts/scanner-offline-bundle-matrix.sh @@ -39,8 +39,9 @@ main() { declare -A releases while read -r tag version _; do - # Sanity check. - if echo "$version" | grep -q '^[0-4]\.[0-3]'; then + # Sanity check: skip pre-V4 versions (before 4.4.0). + smallest=$(printf '%s\n' "$version" 4.4.0 | sort -V | head -1) + if [[ "$smallest" = "$version" && "$version" != "4.4.0" ]]; then echo >&2 "info: skipping pre-V4 tag: $version" continue fi diff --git a/.github/workflows/scripts/scanner-update-nvd-api.py b/.github/workflows/scripts/scanner-update-nvd-api.py index c57d88d179704..14ad025324d98 100644 --- a/.github/workflows/scripts/scanner-update-nvd-api.py +++ b/.github/workflows/scripts/scanner-update-nvd-api.py @@ -14,6 +14,8 @@ BASE_URL = os.getenv('SCANNER_NVD_URL', "https://services.nvd.nist.gov/rest/json/cves/2.0") API_KEY = os.getenv('SCANNER_NVD_API_KEY', None) +BACKOFF_TABLE = [15, 30, 60, 120, 240, 300] + class ApiLoader: def __init__(self): @@ -46,8 +48,8 @@ def get_dates(year, interval): def fetch_data(start, end): index, total = 0, 0 logging.info(f"Fetching page from {start} to {end}") - backoff_time = 10 - max_retries = 3 + max_retries = 10 + attempt = 0 while total == 0 or index < total: try: @@ -62,7 +64,7 @@ def fetch_data(start, end): req = request.Request(url) req.headers = {'apiKey': API_KEY} - with request.urlopen(req, timeout=45) as response: + with request.urlopen(req, timeout=120) as response: data = json.loads(response.read().decode()) total = data['totalResults'] @@ -72,17 +74,16 @@ def fetch_data(start, end): yield {"cve": item['cve']} index += data['resultsPerPage'] - max_retries = 3 # Reset retries - backoff_time = 1 + attempt = 0 except Exception as e: - logging.error(f"Failed to download page at index {index}: {e}") - if max_retries > 0: - logging.info(f"Retrying in {backoff_time} seconds...") - time.sleep(backoff_time) - backoff_time *= 2 # Increase the backoff time exponentially - max_retries -= 1 + if attempt < max_retries: + backoff = BACKOFF_TABLE[min(attempt, len(BACKOFF_TABLE) - 1)] + logging.warning(f"Failed to download page at index {index} (attempt {attempt + 1}/{max_retries + 1}): {e}") + logging.info(f"Retrying in {backoff} seconds...") + time.sleep(backoff) + attempt += 1 else: - raise # Re-raise the exception if max retries have been exceeded + raise time.sleep(2) @staticmethod diff --git a/.github/workflows/scripts/scanner-update-nvd-feeds.py b/.github/workflows/scripts/scanner-update-nvd-feeds.py index 50302593054a3..613e27b449b85 100644 --- a/.github/workflows/scripts/scanner-update-nvd-feeds.py +++ b/.github/workflows/scripts/scanner-update-nvd-feeds.py @@ -5,9 +5,11 @@ """ import gzip +import io import json import logging import os +import time from urllib import request from urllib import parse @@ -18,6 +20,9 @@ DEFAULT_BASE_URL = "https://nvd.nist.gov/feeds/json/cve/2.0/" +BACKOFF_TABLE = [15, 30, 60, 120, 240, 300] + + class FeedLoader: log = logging.getLogger("FeedLoader") @@ -27,10 +32,22 @@ def __init__(self, base_url): def fetch(self, year): url = parse.urljoin(self._base_url, f"nvdcve-2.0-{year}.json.gz") - self.log.info("fetching and decompressing: %s", url) - with request.urlopen(url, timeout=60) as resp: - with gzip.GzipFile(fileobj=resp) as gz: - data = json.load(gz) + max_retries = 10 + for attempt in range(max_retries + 1): + try: + self.log.info("fetching and decompressing: %s", url) + with request.urlopen(url, timeout=120) as resp: + with gzip.GzipFile(fileobj=io.BytesIO(resp.read())) as gz: + data = json.load(gz) + break + except Exception as e: + if attempt < max_retries: + backoff = BACKOFF_TABLE[min(attempt, len(BACKOFF_TABLE) - 1)] + self.log.warning("failed to fetch %s (attempt %d/%d): %s", url, attempt + 1, max_retries + 1, e) + self.log.info("retrying in %d seconds...", backoff) + time.sleep(backoff) + else: + raise yield from (v for v in data["vulnerabilities"] if v["cve"]["vulnStatus"].lower() != "rejected") diff --git a/.github/workflows/scripts/scanner-updater-check-freshness.sh b/.github/workflows/scripts/scanner-updater-check-freshness.sh new file mode 100755 index 0000000000000..562c838df0577 --- /dev/null +++ b/.github/workflows/scripts/scanner-updater-check-freshness.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# +# Check whether a GCS object is newer than a reference point. +# +# Uses an unauthenticated HTTP HEAD request against the public GCS +# download URL to classify the object state (200/404/other), then +# gcloud for the object timestamp when it exists. +# +# Exactly one of --newer-than or --max-age must be provided. +# +# --newer-than : +# The object is "fresh" if its timestamp is after . +# +# --max-age : +# The object is "fresh" if its age is below +# (equivalent to --newer-than "$(date +%s) - N"). +# +# Output (stdout): +# "true" — object exists and is newer than the threshold +# "false" — object is not newer, or does not exist (404) +# +# Exit codes: +# 0 — check completed (result is on stdout) +# non-zero — unexpected failure (stdout is irrelevant) +# +# Inputs (environment): +# GCS_OBJECT — full GCS object path (gs://bucket/path/to/object) +# +# Requires: curl, gcloud, date. + +set -euo pipefail + +trap 'echo "::error::unexpected failure in check-freshness.sh at line $LINENO" >&2; exit 2' ERR + +threshold="" +while [[ $# -gt 0 ]]; do + case "$1" in + --newer-than) + if [[ -n "$threshold" ]]; then + echo "::error::--newer-than and --max-age are mutually exclusive" >&2 + exit 1 + fi + threshold="${2:?--newer-than requires an epoch value}" + shift 2 + ;; + --max-age) + if [[ -n "$threshold" ]]; then + echo "::error::--newer-than and --max-age are mutually exclusive" >&2 + exit 1 + fi + max_age="${2:?--max-age requires a value in seconds}" + threshold=$(( $(date +%s) - max_age )) + shift 2 + ;; + *) + echo "::error::unknown argument: $1" >&2 + exit 1 + ;; + esac +done + +if [[ -z "$threshold" ]]; then + echo "::error::one of --newer-than or --max-age is required" >&2 + exit 1 +fi + +object="${GCS_OBJECT:?GCS_OBJECT is required}" + +# Derive the public download URL from the gs:// path. +download_url="https://storage.googleapis.com/${object#gs://}" + +http_code=$(curl --head --silent --show-error -o /dev/null -w "%{http_code}" \ + --connect-timeout 10 --max-time 15 \ + "$download_url") + +case "$http_code" in + 200) + creation_time=$(timeout 30 gcloud storage objects describe "$object" \ + --format="value(creation_time)") + object_epoch=$(date -d "$creation_time" +%s) + if (( object_epoch > threshold )); then + echo >&2 "INFO: ${object}: newer than threshold (${object_epoch} > ${threshold})" + echo "true" + else + echo >&2 "INFO: ${object}: not newer than threshold (${object_epoch} <= ${threshold})" + echo "false" + fi + ;; + 404) + echo >&2 "INFO: ${object}: not found" + echo "false" + ;; + *) + echo "::error::${object}: GCS returned HTTP $http_code" >&2 + exit 1 + ;; +esac diff --git a/.github/workflows/scripts/scanner-updater-config.sh b/.github/workflows/scripts/scanner-updater-config.sh new file mode 100755 index 0000000000000..95a63e7200a3a --- /dev/null +++ b/.github/workflows/scripts/scanner-updater-config.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Query source-config.yaml for scanner updater workflows. +# +# Usage: +# scanner-updater-config.sh [args...] +# +# Commands: +# bucket Storage bucket name. +# prefix Storage prefix path. +# sources List updater sources for a stream, one per line. +# source-object GCS object path for a source. +# bundle-object GCS object path for a bundle. +# interval-secs Updater refresh interval in seconds. +# attempt-cadence-secs Attempt cadence in seconds; 0 if unset. +# +# Inputs (environment): +# SOURCE_CONFIG — path to source-config.yaml +# +# Requires: yq. + +set -euo pipefail + +config="${SOURCE_CONFIG:?SOURCE_CONFIG is required}" + +if [[ ! -f "$config" ]]; then + echo "::error::config file not found: $config" >&2 + exit 1 +fi + +if ! yq eval '.' "$config" > /dev/null 2>&1; then + echo "::error::config file is not valid YAML: $config" >&2 + exit 1 +fi + +duration_to_seconds() { + local dur="${1:?missing required argument 'duration'}"; shift + local total=0 remaining="$dur" + while [[ -n "$remaining" ]]; do + if [[ "$remaining" =~ ^([0-9]+)h(.*)$ ]]; then + total=$(( total + BASH_REMATCH[1] * 3600 )); remaining="${BASH_REMATCH[2]}" + elif [[ "$remaining" =~ ^([0-9]+)m(.*)$ ]]; then + total=$(( total + BASH_REMATCH[1] * 60 )); remaining="${BASH_REMATCH[2]}" + elif [[ "$remaining" =~ ^([0-9]+)s(.*)$ ]]; then + total=$(( total + BASH_REMATCH[1] )); remaining="${BASH_REMATCH[2]}" + else + echo "::error::cannot parse duration: $remaining" >&2; exit 1 + fi + done + echo "$total" +} + +command="${1:?missing required argument 'command'}"; shift + +case "$command" in + bucket) + yq eval '.storage.bucket' "$config" + ;; + prefix) + yq eval '.storage.prefix' "$config" + ;; + sources) + stream="${1:?missing required argument 'stream'}"; shift + yq eval ".bundles.\"${stream}\"[]" "$config" + ;; + source-object) + stream="${1:?missing required argument 'stream'}"; shift + source="${1:?missing required argument 'source'}"; shift + bucket=$(yq eval '.storage.bucket' "$config") + prefix=$(yq eval '.storage.prefix' "$config") + echo "gs://${bucket}/${prefix}/${stream}/sources/${source}.json.zst" + ;; + bundle-object) + stream="${1:?missing required argument 'stream'}"; shift + bucket=$(yq eval '.storage.bucket' "$config") + prefix=$(yq eval '.storage.prefix' "$config") + echo "gs://${bucket}/${prefix}/${stream}/vulnerabilities.zip" + ;; + interval-secs) + source="${1:?missing required argument 'source'}"; shift + default_interval=$(yq eval '.updaters.default.interval // "4h"' "$config") + interval=$(yq eval ".updaters.\"${source}\".interval // \"$default_interval\"" "$config") + duration_to_seconds "$interval" + ;; + attempt-cadence-secs) + source="${1:?missing required argument 'source'}"; shift + cadence=$(yq eval ".updaters.\"${source}\".attempt_cadence // .updaters.default.attempt_cadence // \"\"" "$config") + if [[ -z "$cadence" ]]; then + echo 0 + exit 0 + fi + cadence_secs=$(duration_to_seconds "$cadence") + if (( cadence_secs <= 0 || cadence_secs % 3600 != 0 )); then + echo "::error::attempt_cadence must be a positive whole-hour duration: $cadence" >&2 + exit 1 + fi + echo "$cadence_secs" + ;; + *) + echo "::error::unknown command: $command" >&2 + exit 1 + ;; +esac diff --git a/.github/workflows/scripts/scanner-updater-matrix.sh b/.github/workflows/scripts/scanner-updater-matrix.sh new file mode 100755 index 0000000000000..4e99bfd923e37 --- /dev/null +++ b/.github/workflows/scripts/scanner-updater-matrix.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# +# Build a GitHub Actions matrix of updater/stream pairs that are due +# for a refresh. +# +# Expands stream/ref pairs (from scanner-get-released-tags.sh) against +# per-stream updater lists (from source-config.yaml), then calls +# scanner-updater-check-freshness.sh for each pair. Only due pairs +# are emitted. +# +# Inputs (environment): +# SOURCE_CONFIG — path to source-config.yaml +# SCHEDULE_TIME — workflow run creation time; defaults to the current time. +# +# Outputs (stdout): +# JSON object with "has_due", "has_errors", "errors", and "matrix" +# for GitHub Actions. +# +# Requires: yq, jq, and scanner-get-released-tags.sh and +# scanner-updater-check-freshness.sh in the same repo checkout. + +set -euo pipefail + +for cmd in yq jq; do + if ! command -v "$cmd" &>/dev/null; then + echo "::error::$cmd is required but not found" + exit 1 + fi +done + +sc="$(dirname "$0")/scanner-updater-config.sh" + +bucket=$("$sc" bucket) +prefix=$("$sc" prefix) + +schedule_epoch=$(date -u -d "${SCHEDULE_TIME:-now}" +%s) +schedule_hour=$(( schedule_epoch / 3600 )) + +# Expand stream/ref/updater triples, check freshness, emit due pairs. +./.github/workflows/scripts/scanner-get-released-tags.sh \ + | jq -r '.[] | "\(.version) \(.ref)"' \ + | while read -r stream ref; do + "$sc" sources "$stream" \ + | while read -r updater; do + echo "$stream $ref $updater" + done + done \ + | while read -r stream ref updater; do + object="gs://${bucket}/${prefix}/${stream}/sources/${updater}.json.zst" + interval_secs=$("$sc" interval-secs "$updater") + cadence_secs=$("$sc" attempt-cadence-secs "$updater") + if (( cadence_secs > 0 )); then + cadence_hours=$(( cadence_secs / 3600 )) + if (( schedule_hour % cadence_hours != 0 )); then + continue + fi + fi + + fresh=$(GCS_OBJECT="$object" \ + ./.github/workflows/scripts/scanner-updater-check-freshness.sh --max-age "$interval_secs") || { + echo "::error::${object}: freshness check failed" >&2 + jq -n --arg msg "${object}: freshness check failed" '{error: $msg}' + continue + } + if [[ "$fresh" == "true" ]]; then + continue + fi + jq -n \ + --arg source "$updater" \ + --arg ref "$ref" \ + --arg object "$object" \ + '{source: $source, ref: $ref, object: $object}' + done \ + | jq -s '{ + has_due: (map(select(.source)) | length > 0 | tostring), + has_errors: (map(select(.error)) | length > 0 | tostring), + errors: [.[] | .error // empty], + matrix: {include: [.[] | select(.source)]} + }' diff --git a/.github/workflows/scripts/validate-signing-key-bundle.sh b/.github/workflows/scripts/validate-signing-key-bundle.sh new file mode 100755 index 0000000000000..3be88e7ab3b70 --- /dev/null +++ b/.github/workflows/scripts/validate-signing-key-bundle.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Validates a Red Hat signing key bundle JSON file. +# +# Checks mirror the Go ParseKeyBundle logic in pkg/signatures/key_bundle.go: +# - valid JSON with non-empty "cosignKeys" array +# - each key has a non-empty "name" (after trimming) without path separators +# - no duplicate key names +# - each "publicKey" value is a valid PEM-encoded public key +# +# USAGE: validate-signing-key-bundle.sh + +set -euo pipefail + +bundle="${1:?Usage: validate-signing-key-bundle.sh }" + +if [[ ! -f "$bundle" ]]; then + echo "ERROR: file not found: $bundle" >&2 + exit 1 +fi + +# 1. Valid JSON structure with non-empty cosignKeys array. +key_count=$(jq -e '.cosignKeys | length' "$bundle") || { + echo "ERROR: invalid JSON or missing 'cosignKeys' array" >&2 + exit 1 +} + +if [[ "$key_count" -eq 0 ]]; then + echo "ERROR: key bundle must contain at least one key" >&2 + exit 1 +fi + +echo "Found $key_count key(s), validating..." + +# 2-4. Validate each key entry. +seen_names=() +for i in $(seq 0 $((key_count - 1))); do + raw_name=$(jq -r ".cosignKeys[$i].name // empty" "$bundle") + pem_data=$(jq -r ".cosignKeys[$i].publicKey // empty" "$bundle") + + # Trim whitespace to match Go's strings.TrimSpace before all checks. + name="${raw_name#"${raw_name%%[![:space:]]*}"}" + name="${name%"${name##*[![:space:]]}"}" + + if [[ -z "$name" ]]; then + echo "ERROR: key at index $i has empty name" >&2 + exit 1 + fi + + if [[ "$name" == */* || "$name" == *\\* ]]; then + echo "ERROR: key name '$name' contains path separators" >&2 + exit 1 + fi + + for seen in "${seen_names[@]+"${seen_names[@]}"}"; do + if [[ "$seen" == "$name" ]]; then + echo "ERROR: duplicate key name '$name'" >&2 + exit 1 + fi + done + seen_names+=("$name") + + if [[ -z "$pem_data" ]]; then + echo "ERROR: key '$name' has empty publicKey" >&2 + exit 1 + fi + + if ! echo "$pem_data" | openssl pkey -pubin -noout 2>/dev/null; then + echo "ERROR: key '$name' has invalid PEM public key" >&2 + exit 1 + fi + + echo " [$i] '$name' — valid" +done + +echo "Bundle validation passed: $key_count key(s) OK" diff --git a/.github/workflows/scripts/verify-release.sh b/.github/workflows/scripts/verify-release.sh index fb207b421809e..528cf5c4660ad 100644 --- a/.github/workflows/scripts/verify-release.sh +++ b/.github/workflows/scripts/verify-release.sh @@ -17,7 +17,11 @@ check_args() { LATEST_VERSION \ RELEASE \ PROJECT \ - ERRATA_NAME + ERRATA_NAME \ + \ + JIRA_TOKEN \ + JIRA_BASE_URL \ + JIRA_USER } mark_failed() { @@ -45,7 +49,7 @@ check_dir_not_empty() { check_url_page_exists() { URL="$1" - curl -sSLf "$URL" --output /dev/null || { + curl -sSLf --retry 5 --retry-all-errors "$URL" --output /dev/null || { mark_failed gh_log error "Retrieving $URL failed." } @@ -54,7 +58,7 @@ check_url_page_exists() { check_url_yaml_contains() { URL="$1" QUERY="$2" - curl -sSLf "$URL" 2>/dev/null | yq -e "$QUERY" >/dev/null || { + curl -sSLf --retry 5 --retry-all-errors "$URL" 2>/dev/null | yq -e "$QUERY" >/dev/null || { mark_failed gh_log error "The Helm index does not contain the new version." } @@ -83,31 +87,38 @@ validate_helm_charts() { check_url_yaml_contains "https://charts.stackrox.io/index.yaml?v=$(date +%s)" ".entries.central-services[] | select( .appVersion == \"${RELEASE_PATCH}\")" check_url_yaml_contains "https://charts.stackrox.io/index.yaml?v=$(date +%s)" ".entries.secured-cluster-services[] | select( .appVersion == \"${RELEASE_PATCH}\")" + # The operator chart is published to charts.stackrox.io starting from 4.11.0. + if [[ $(printf '%s\n' "4.11.0" "${RELEASE_PATCH}" | sort -V | head -n1) = 4.11.0 ]]; then + check_url_yaml_contains "https://charts.stackrox.io/index.yaml?v=$(date +%s)" ".entries.stackrox-operator[] | select( .appVersion == \"${RELEASE_PATCH}\")" + fi } validate_images() { RELEASE_PATCH="$1" - check_docker_image "registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8:${RELEASE_PATCH}" + check_docker_image "registry.redhat.io/advanced-cluster-security/rhacs-operator-bundle:${RELEASE_PATCH}" check_docker_image "quay.io/stackrox-io/main:${RELEASE_PATCH}" check_docker_image "quay.io/rhacs-eng/main:${RELEASE_PATCH}" } validate_docs() { RELEASE="$1" - check_url_page_exists "https://docs.openshift.com/acs/${RELEASE}/welcome/index.html" + check_url_page_exists "https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_security_for_kubernetes/${RELEASE}" } validate_jira_release() { PROJECT="$1" RELEASE_PATCH="$2" - IS_RELEASED=$(curl -sSLf \ - -H "Authorization: Bearer $JIRA_TOKEN" \ - "https://issues.redhat.com/rest/api/2/project/${PROJECT}/versions" \ - | jq -r ".[] | select(.name == \"${RELEASE_PATCH}\") | .released") - if [ "${IS_RELEASED}" != "true" ]; then + release=$(get_jira_release "$PROJECT" "$RELEASE_PATCH") + if [ -n "${release}" ]; then + IS_RELEASED=$(echo "$release" | jq -r ".released") + if [ "${IS_RELEASED}" != "true" ]; then + mark_failed + gh_log error "JIRA Release $RELEASE_PATCH has not been marked as done." + fi + else mark_failed - gh_log error "JIRA Release $RELEASE_PATCH has not been marked as done." + gh_log error "Couldn't find JIRA release \`$RELEASE_PATCH\`." fi } diff --git a/.github/workflows/signing-key-bundle-update.yaml b/.github/workflows/signing-key-bundle-update.yaml new file mode 100644 index 0000000000000..fda57fde6bd44 --- /dev/null +++ b/.github/workflows/signing-key-bundle-update.yaml @@ -0,0 +1,63 @@ +name: Red Hat signing key bundle update + +on: + push: + branches: + - master + paths: + - pkg/signatures/bundle.json + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate-and-upload: + if: ${{ github.repository_owner == 'stackrox' }} + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + persist-credentials: false + + - name: Validate key bundle + run: .github/workflows/scripts/validate-signing-key-bundle.sh pkg/signatures/bundle.json + + - name: Authenticate with Google Cloud + if: github.ref == 'refs/heads/master' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # ratchet:google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} + + - name: Set up Cloud SDK + if: github.ref == 'refs/heads/master' + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3 + + - name: Upload key bundle to GCS + if: github.ref == 'refs/heads/master' + run: | + gsutil -h "Content-Type: application/json" \ + cp pkg/signatures/bundle.json \ + gs://definitions.stackrox.io/signing-keys/bundle.json + + send-notification: + needs: + - validate-and-upload + runs-on: ubuntu-24.04 + if: failure() + permissions: + contents: read + checks: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + - uses: ./.github/actions/notify-slack-on-failure + with: + slack-webhook: ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} + mention: "" diff --git a/.github/workflows/start-release.yml b/.github/workflows/start-release.yml index 4521e3f6bc438..d23caf47e4a8b 100644 --- a/.github/workflows/start-release.yml +++ b/.github/workflows/start-release.yml @@ -42,12 +42,12 @@ concurrency: Release automation ${{ inputs.version }} jobs: start-sha: name: Find commit for release branch - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: start-sha: ${{ fromJSON(format('["{0}","{1}"]', env.USER_REF, steps.find.outputs.merge_base))[env.USER_REF == ''] }} steps: - name: Check out code - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: ref: ${{ github.event.repository.default_branch }} fetch-depth: 0 @@ -67,7 +67,7 @@ jobs: properties: name: Read repository properties - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: docs-repository: ${{ steps.properties.outputs.docs-repository }} slack-channel: ${{ fromJSON(format('["{0}","{1}"]', steps.properties.outputs.dry-slack-channel, steps.properties.outputs.slack-channel))[github.event.inputs.dry-run != 'true'] }} @@ -81,7 +81,7 @@ jobs: run-parameters: name: Run parameters - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - run: | [ "$DRY_RUN" = "true" ] && echo "::warning::This is a dry run" @@ -102,14 +102,16 @@ jobs: check-jira: name: Check Jira release needs: [variables, properties] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: release-date: ${{steps.check-jira-release.outputs.date}} steps: - name: Check that Jira release ${{needs.variables.outputs.release}}.${{needs.variables.outputs.patch}} is not released id: check-jira-release env: + JIRA_USER: ${{ vars.RHACS_BOT_GITHUB_EMAIL }} JIRA_TOKEN: ${{ secrets.JIRA_TOKEN }} + JIRA_BASE_URL: ${{ vars.JIRA_BASE_URL }} run: | set -uo pipefail gh api -H "$ACCEPT_RAW" "${{env.script_url}}" | bash -s -- \ @@ -119,19 +121,19 @@ jobs: branch: name: Prepare release branch - needs: [variables, start-sha] - runs-on: ubuntu-latest + needs: [variables, start-sha, check-jira] + runs-on: ubuntu-24.04 steps: - name: Check out code (normal release) if: needs.variables.outputs.patch == 0 - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: ref: ${{ needs.start-sha.outputs.start-sha }} fetch-depth: 0 token: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} - name: Check out code (patch release) if: needs.variables.outputs.patch != 0 - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: ref: ${{ needs.variables.outputs.branch }} submodules: true @@ -191,7 +193,7 @@ jobs: name: Configure OpenShift CI jobs needs: [variables, branch] if: needs.variables.outputs.patch == 0 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: RELEASE: "${{needs.variables.outputs.release}}" BRANCH: "stackrox-release-${{needs.variables.outputs.release}}" @@ -212,7 +214,7 @@ jobs: run: | gh repo sync stackrox/openshift-release --source openshift/release - name: Check out stackrox/openshift-release - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: repository: stackrox/openshift-release token: "${{ secrets.RHACS_BOT_GITHUB_TOKEN }}" @@ -260,10 +262,10 @@ jobs: name: Patch CHANGELOG.md on the default branch needs: [variables, branch, start-sha] if: needs.variables.outputs.patch == 0 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Check out code - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: # Working on `start-sha` instead of `env.main_branch` may help # to spot changes happened to CHANGELOG.md after `start-sha`. @@ -286,10 +288,10 @@ jobs: patch-scanner-versions: name: Patch Scanner updater configuration needs: [variables, branch] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Check out code - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: ref: ${{ env.main_branch }} - name: Initialize mandatory git config @@ -308,7 +310,7 @@ jobs: milestone: name: Create milestone needs: [variables] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Create ${{needs.variables.outputs.next-milestone}} milestone if: env.DRY_RUN == 'false' @@ -334,11 +336,11 @@ jobs: notify: name: Notify everybody needs: [variables, properties, branch, milestone] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Post to Slack (normal release) if: needs.variables.outputs.patch == 0 - uses: slackapi/slack-github-action@v2.1.1 + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # ratchet:slackapi/slack-github-action@v3.0.3 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} @@ -370,7 +372,7 @@ jobs: - name: Post to Slack (patch release) if: needs.variables.outputs.patch != 0 - uses: slackapi/slack-github-action@v2.1.1 + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # ratchet:slackapi/slack-github-action@v3.0.3 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml index c6f183a8739af..3989115a09bab 100644 --- a/.github/workflows/style.yaml +++ b/.github/workflows/style.yaml @@ -19,15 +19,18 @@ concurrency: cancel-in-progress: true jobs: + detect-changes: + uses: ./.github/workflows/detect-changes.yml + check-generated-files: + needs: detect-changes + if: ${{ !needs.detect-changes.outputs.ui_only }} env: ARTIFACT_DIR: junit-reports/ - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} @@ -39,8 +42,11 @@ jobs: - name: Create artifacts dir run: mkdir -p "$ARTIFACT_DIR" - - name: Cache Go dependencies - uses: ./.github/actions/cache-go-dependencies + # No GOCACHE here — this job must validate go.mod/go.sum against + # real source files, not a cached module index that may be stale. + + - name: Download scanner module for proto generation + run: go mod download github.com/stackrox/scanner - name: Check Generated run: scripts/ci/jobs/check-generated.sh @@ -48,12 +54,10 @@ jobs: misc-checks: env: ARTIFACT_DIR: junit-reports/ - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} @@ -68,7 +72,8 @@ jobs: - name: Ensure no trailing whitespaces if: github.event_name == 'pull_request' # Markdown files aren't checked for trailing whitespaces because it's a valid linebreak there. - run: git diff --check "${{ github.event.pull_request.base.sha }}"...HEAD ':(exclude)*.md' + # Batch files are excluded because they are generated (e.g. gradle wrapper) and use Windows line endings. + run: git diff --check "${{ github.event.pull_request.base.sha }}"...HEAD ':(exclude)*.md' ':(exclude)*.bat' - name: Check PR fixes run: scripts/ci/jobs/check-pr-fixes.sh @@ -88,16 +93,14 @@ jobs: - name: Check dependabot knows about pinned prefetched images run: scripts/ci/jobs/check-dependabot-pinned-image.sh + - name: Check dependabot gomod configurations match go.mod files + run: if [ -s scripts/ci/jobs/check-dependabot-gomod.sh ]; then scripts/ci/jobs/check-dependabot-gomod.sh; fi + style-check: - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} @@ -107,46 +110,83 @@ jobs: free-disk-space: '30' gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # ratchet:actions/setup-node@v6 + with: + node-version-file: ui/package.json + + - name: Install xmlstarlet for shellcheck output to junit + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq xmlstarlet + + - name: Install pycodestyle + run: | + python3 -m pip install -r .openshift-ci/dev-requirements.txt + + - name: Verify tools + run: | + yq --version + xmlstarlet --version + pycodestyle --version + - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies + - name: Download scanner module for proto generation + run: go mod download github.com/stackrox/scanner + - name: Cache UI dependencies + id: cache-ui uses: ./.github/actions/cache-ui-dependencies - name: Cache QA Test dependencies uses: ./.github/actions/cache-gradle-dependencies - name: Fetch UI deps + if: steps.cache-ui.outputs.cache-hit != 'true' run: make -C ui deps + - name: Skip UI deps on cache hit + if: steps.cache-ui.outputs.cache-hit == 'true' + run: touch ui/deps + + - name: Cache ESLint results + uses: ./.github/actions/cache-eslint + - name: make style-slim run: make style-slim golangci-lint: + needs: detect-changes + if: ${{ !needs.detect-changes.outputs.ui_only }} timeout-minutes: 240 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - uses: actions/setup-go@v6 - with: - go-version-file: 'tools/linters/go.mod' - - name: Check Cache golangci-lint run: make golangci-lint-cache-status - name: Cache golangci-lint - uses: actions/cache@v5 + if: github.event_name == 'push' && github.ref_name == github.event.repository.default_branch + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # ratchet:actions/cache@v6 with: - path: /github/home/.cache/golangci-lint + path: ${{ env.HOME }}/.cache/golangci-lint + key: go-lint-v2-${{ hashFiles('**/go.sum') }} + restore-keys: | + go-lint-v2- + + - name: Restore golangci-lint analysis cache + if: ${{ !(github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # ratchet:actions/cache/restore@v6 + with: + path: ${{ env.HOME }}/.cache/golangci-lint key: go-lint-v2-${{ hashFiles('**/go.sum') }} restore-keys: | - go-lint-v2-${{ hashFiles('**/go.sum') }} go-lint-v2- - name: Check cache golangci-lint @@ -162,71 +202,74 @@ jobs: - name: Check Cache golangci-lint run: make golangci-lint-cache-status - check-collector-and-scanner-images-exist: - # This job ensures that COLLECTOR_VERSION or SCANNER_VERSION files cannot be updated to a version for which the - # image was not successfully built on Konflux (suffix "-fast"). It also verifies that GHA-built image is there (no - # suffix) so that the failure also happens in this job. - runs-on: ubuntu-latest - strategy: - matrix: - image: ["collector", "scanner", "scanner-slim", "scanner-db", "scanner-db-slim"] + check-dependent-images-exist: + needs: detect-changes + if: ${{ !needs.detect-changes.outputs.ui_only }} + # This job ensures that COLLECTOR_VERSION or FACT_VERSION files cannot be updated to a version + # for which the image was not successfully built on Konflux (suffix "-fast"). It also verifies that GHA-built image + # is there (no suffix) so that the failure also happens in this job. + runs-on: ubuntu-24.04 steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - name: Get image tag from COLLECTOR|SCANNER_VERSION file - id: image-tag + - name: Get image tags from COLLECTOR_|FACT_VERSION file run: | - if [[ "${{ matrix.image }}" == "collector" ]]; then - makefile_target="collector-tag" - else - makefile_target="scanner-tag" - fi - echo "image-tag=$(make --quiet --no-print-directory "${makefile_target}")" >> "${GITHUB_OUTPUT}" + { + echo "collector_tag=$(make --quiet --no-print-directory collector-tag)" + echo "fact_tag=$(make --quiet --no-print-directory fact-tag)" + } >> "$GITHUB_ENV" - name: Check GHA-built image exists - uses: stackrox/actions/release/wait-for-image@v1 + uses: stackrox/actions/release/wait-for-image@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/release/wait-for-image@v1 # Skip for external contributions. if: | github.event_name == 'push' || !github.event.pull_request.head.repo.fork with: token: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} - image: rhacs-eng/${{ matrix.image }}:${{ steps.image-tag.outputs.image-tag }} limit: 300 + image: | + rhacs-eng/collector:${{ env.collector_tag }} + rhacs-eng/fact:${{ env.fact_tag }} - name: Check Konflux-built image exists - uses: stackrox/actions/release/wait-for-image@v1 + uses: stackrox/actions/release/wait-for-image@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/release/wait-for-image@v1 # Skip for external contributions. if: | github.event_name == 'push' || !github.event.pull_request.head.repo.fork with: token: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }} - image: rhacs-eng/release-${{ matrix.image }}:${{ steps.image-tag.outputs.image-tag }}-fast limit: 300 + image: | + rhacs-eng/release-collector:${{ env.collector_tag }}-fast + rhacs-eng/release-fact:${{ env.fact_tag }}-fast github-actions-lint: - runs-on: ubuntu-latest + needs: detect-changes + if: >- + !needs.detect-changes.outputs.files || + contains(needs.detect-changes.outputs.files, '^.github/') + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - name: Download actionlint - id: get_actionlint - run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/2ab3a12c7848f6c15faca9a92612ef4261d0e370/scripts/download-actionlint.bash) 1.6.27 - shell: bash - name: Check workflow files - run: ${{ steps.get_actionlint.outputs.executable }} -color - shell: bash + run: make github-actions-lint github-actions-shellcheck: - runs-on: ubuntu-latest + needs: detect-changes + if: >- + !needs.detect-changes.outputs.files || + contains(needs.detect-changes.outputs.files, '^.github/') + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} @@ -234,12 +277,14 @@ jobs: run: shellcheck -P SCRIPTDIR -x ./.github/workflows/scripts/*.sh openshift-ci-lint: - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + needs: detect-changes + if: >- + !needs.detect-changes.outputs.files || + contains(needs.detect-changes.outputs.files, '^.openshift-ci/') + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} @@ -248,6 +293,9 @@ jobs: with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Install Python linters + run: python3 -m pip install -r .openshift-ci/dev-requirements.txt + - name: pycodestyle run: make -C .openshift-ci style @@ -265,13 +313,13 @@ jobs: contains(github.event.pull_request.labels.*.name, 'ci-test-github-action-slack-messages') ) name: Post failure message to Slack - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - check-generated-files - misc-checks - style-check - golangci-lint - - check-collector-and-scanner-images-exist + - check-dependent-images-exist - github-actions-lint - github-actions-shellcheck - openshift-ci-lint @@ -279,7 +327,7 @@ jobs: actions: read steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - name: Slack message env: diff --git a/.github/workflows/tag-more-reviewers.yaml b/.github/workflows/tag-more-reviewers.yaml index f6ce74aac7c50..4b94157c38a6d 100644 --- a/.github/workflows/tag-more-reviewers.yaml +++ b/.github/workflows/tag-more-reviewers.yaml @@ -18,7 +18,7 @@ jobs: github.event.pull_request.user.login != 'red-hat-konflux[bot]' env: GH_TOKEN: ${{ secrets.RHACS_BOT_GITHUB_TOKEN }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Tag Konflux Maintainers for review run: | diff --git a/.github/workflows/tagged-ci.yaml b/.github/workflows/tagged-ci.yaml index ca55fe959f008..e936f4f4ea0ca 100644 --- a/.github/workflows/tagged-ci.yaml +++ b/.github/workflows/tagged-ci.yaml @@ -8,10 +8,10 @@ on: jobs: slack-tagged-notice: name: Slack Tagged Notice - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.ref_name }} diff --git a/.github/workflows/triage-report_periodic.yml b/.github/workflows/triage-report_periodic.yml index 7910423e2a329..2df7169befde0 100644 --- a/.github/workflows/triage-report_periodic.yml +++ b/.github/workflows/triage-report_periodic.yml @@ -2,16 +2,17 @@ name: Triage Report on: workflow_dispatch: schedule: - - cron: 0 15 * * 3,5 + - cron: 0 15 * * 1-5 jobs: triage-report: if: ${{ github.repository_owner == 'stackrox' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - name: Report to Slack env: + JIRA_USER: ${{ secrets.JIRA_USER }} JIRA_TOKEN: ${{ secrets.JIRA_TOKEN }} SLACK_WEBHOOK: ${{ secrets.SLACK_ACS_SUPPORT_TRIAGE_WEBHOOK }} run: ./scripts/run-support-triage-report.sh diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index b529981e09341..48ea27c9d0873 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -16,37 +16,52 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + # Stable version ldflags for test caching: Go's test cache keys include + # link ActionID which includes -X ldflags. Fixed BUILD_TAG/SHORTCOMMIT + # prevent per-commit cache misses. + BUILD_TAG: 0.0.0 + SHORTCOMMIT: "0000000" + jobs: + detect-changes: + uses: ./.github/workflows/detect-changes.yml + with: + run_all: | + ^Makefile$ + ^make/ + ^.github/workflows/unit-tests.yaml$ + ^.github/actions/job-preamble/ + go: + needs: detect-changes strategy: fail-fast: false matrix: gotags: [ 'GOTAGS=""', 'GOTAGS=release' ] - runs-on: ubuntu-latest + exclude: + - gotags: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-release-build') && 'GOTAGS=release' }} + runs-on: ubuntu-24.04 outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt steps: - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + free-disk-space: 30 - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies + with: + key-suffix: ${{ matrix.gotags }} - name: Go Unit Tests run: ${{ matrix.gotags }} make go-unit-tests - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # ratchet:codecov/codecov-action@v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: go-unit-tests @@ -56,7 +71,7 @@ jobs: run: make generate-junit-reports - name: Publish Test Report - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 if: always() with: paths: 'junit-reports/report.xml' @@ -67,12 +82,15 @@ jobs: - name: Go Operator Integration Tests run: ${{ matrix.gotags }} make -C operator/ test-integration + - name: Go Operator Helm Tests + run: ${{ matrix.gotags }} make -C operator/ test-helm + - name: Generate junit report if: always() run: make generate-junit-reports - name: Publish Test Report - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 if: always() with: paths: 'junit-reports/report.xml' @@ -83,53 +101,78 @@ jobs: uses: ./.github/actions/junit2jira with: create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} jira-token: ${{ secrets.JIRA_TOKEN }} gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} directory: 'junit-reports' go-postgres: + needs: detect-changes strategy: fail-fast: false matrix: gotags: [ 'GOTAGS=""', 'GOTAGS=release' ] - pg: [ '15' ] - runs-on: ubuntu-latest + pg: [ '16' ] + exclude: + - gotags: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-release-build') && 'GOTAGS=release' }} + runs-on: ubuntu-24.04 outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt + env: + POSTGRES_PASSWORD: testpassword steps: - - name: Set Postgres version - run: | - echo "/usr/pgsql-${{ matrix.pg }}/bin" >> "${GITHUB_PATH}" - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + + # Runner's postgres is initialized with C.UTF-8 locale (GHA default). + # Tests (e.g. central/splunk) require byte-order collation. + - name: Start Postgres (pre-installed) + if: matrix.pg == 'default' + run: | + # Run on tmpfs to avoid disk I/O overhead in tests. + PG_VER=$(pg_lsclusters -h | awk '{print $1}') + sudo pg_dropcluster --stop "$PG_VER" main + sudo mount -t tmpfs -o size=1G tmpfs /var/lib/postgresql + # the unit tests expect a local postgres with passwordless access. + sudo pg_createcluster "$PG_VER" main -- --auth=trust + sudo cp .github/test-images/postgres-ci.conf "/etc/postgresql/$PG_VER/main/conf.d/ci.conf" + sudo systemctl start postgresql.service + + # Image pinned in .github/test-images/postgres.dockerfile (Dependabot auto-bumps). + - name: Start Postgres (${{ matrix.pg }}) + if: matrix.pg != 'default' + run: | + docker build -t postgres-test -f .github/test-images/postgres.dockerfile . + docker run --rm -d --name postgres --network=host \ + --tmpfs /var/lib/pgsql/data:rw,size=1G \ + -v "$GITHUB_WORKSPACE/.github/test-images/postgres-ci.conf:/opt/app-root/src/postgresql-cfg/ci.conf:ro" \ + -e POSTGRESQL_ADMIN_PASSWORD="$POSTGRES_PASSWORD" \ + -e LANG=C.UTF-8 \ + postgres-test - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - name: Run Postgres - run: | - su postgres -c 'initdb -D /tmp/data' - su postgres -c 'pg_ctl -D /tmp/data start' - - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies + with: + key-suffix: ${{ matrix.gotags }} - name: Is Postgres ready - run: pg_isready -h 127.0.0.1 + run: | + if [ "${{ matrix.pg }}" = "default" ]; then + timeout 60 bash -c 'until pg_isready; do sleep 1; done' \ + || { sudo journalctl -u postgresql --no-pager -n 20; exit 1; } + else + timeout 60 bash -c 'until pg_isready -h 127.0.0.1; do sleep 1; done' \ + || { docker logs postgres; exit 1; } + fi - name: Go Unit Tests run: ${{ matrix.gotags }} make go-postgres-unit-tests - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # ratchet:codecov/codecov-action@v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: go-unit-tests @@ -139,7 +182,7 @@ jobs: run: make generate-junit-reports - name: Publish Test Report - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 if: always() with: paths: 'junit-reports/report.xml' @@ -150,51 +193,63 @@ jobs: uses: ./.github/actions/junit2jira with: create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} jira-token: ${{ secrets.JIRA_TOKEN }} gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} directory: 'junit-reports' + # Benchmarks run each bench function once (-benchtime=1x -count=1) to verify + # they compile and execute without errors. A single postgres version is + # sufficient since go-postgres already tests against multiple versions. go-bench: - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 - volumes: - - /usr:/mnt/usr - - /opt:/mnt/opt + needs: detect-changes + if: ${{ !needs.detect-changes.outputs.ui_only }} + runs-on: ubuntu-24.04 steps: - - name: Set Postgres version - run: | - echo "/usr/pgsql-15/bin" >> "${GITHUB_PATH}" - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + + - name: Start Postgres + run: | + # Run on tmpfs to avoid disk I/O overhead in tests. + PG_VER=$(pg_lsclusters -h | awk '{print $1}') + sudo pg_dropcluster --stop "$PG_VER" main + sudo mount -t tmpfs -o size=1G tmpfs /var/lib/postgresql + # the unit tests expect a local postgres with passwordless access. + sudo pg_createcluster "$PG_VER" main -- --auth=trust + sudo cp .github/test-images/postgres-ci.conf "/etc/postgresql/$PG_VER/main/conf.d/ci.conf" + sudo systemctl start postgresql.service - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} - - name: Run Postgres - run: | - su postgres -c 'initdb -D /tmp/data' - su postgres -c 'pg_ctl -D /tmp/data start' - - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies - name: Is Postgres ready - run: pg_isready -h 127.0.0.1 + run: | + timeout 60 bash -c 'until pg_isready -h 127.0.0.1; do sleep 1; done' \ + || { sudo journalctl -u postgresql --no-pager -n 20; exit 1; } - name: Go Bench Tests run: make go-postgres-bench-tests + - name: Postgres diagnostics + if: failure() + run: | + pg_isready -h 127.0.0.1 || true + pg_lsclusters || true + df -h /var/lib/postgresql || true + sudo journalctl -u postgresql --no-pager -n 40 || true + sudo cat /var/log/postgresql/postgresql-*-main.log | tail -60 || true + - name: Generate junit report if: always() run: make generate-junit-reports - name: Publish Test Report - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 if: always() with: paths: 'junit-reports/report.xml' @@ -203,27 +258,44 @@ jobs: # https://github.com/jstemmer/go-junit-report/issues/174 ui: - runs-on: ubuntu-latest + needs: detect-changes + if: >- + !needs.detect-changes.outputs.files || + contains(needs.detect-changes.outputs.files, '^ui/') || + contains(needs.detect-changes.outputs.files, '^.github/actions/cache-ui-dependencies/') + runs-on: ubuntu-24.04 outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # ratchet:actions/setup-node@v6 + with: + node-version-file: ui/package.json + - name: Cache UI dependencies + id: cache-ui uses: ./.github/actions/cache-ui-dependencies + - name: Install UI dependencies + if: steps.cache-ui.outputs.cache-hit != 'true' + run: make -C ui deps + + - name: Skip UI deps on cache hit + if: steps.cache-ui.outputs.cache-hit == 'true' + run: touch ui/deps + - name: UI Unit Tests run: make ui-test - name: Publish Test Report - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 if: always() with: paths: 'ui/apps/platform/test-results/reports/*.xml' @@ -234,29 +306,51 @@ jobs: uses: ./.github/actions/junit2jira with: create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} jira-token: ${{ secrets.JIRA_TOKEN }} gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} directory: 'ui/apps/platform/test-results/reports' ui-component: - runs-on: ubuntu-latest + needs: detect-changes + if: >- + !needs.detect-changes.outputs.files || + contains(needs.detect-changes.outputs.files, '^ui/') || + contains(needs.detect-changes.outputs.files, '^.github/actions/cache-ui-dependencies/') + runs-on: ubuntu-24.04 outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} - container: - image: quay.io/stackrox-io/apollo-ci:stackrox-ui-test-0.5.1 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # ratchet:actions/setup-node@v6 + with: + node-version-file: ui/package.json + - name: Cache UI dependencies + id: cache-ui uses: ./.github/actions/cache-ui-dependencies + - name: Install UI dependencies + if: steps.cache-ui.outputs.cache-hit != 'true' + run: make -C ui deps + + - name: Skip UI deps on cache hit + if: steps.cache-ui.outputs.cache-hit == 'true' + run: touch ui/deps + - name: UI Component Tests - run: make ui-component-tests + uses: cypress-io/github-action@dace029018fcdf86e0df89a31bc3cfa5b32570d8 # ratchet:cypress-io/github-action@v7 + with: + working-directory: ui/apps/platform + install: false + command: npm run test-component -- --browser chrome - name: Upload Cypress test result artifacts uses: ./.github/actions/upload-artifact-with-retry @@ -266,7 +360,7 @@ jobs: path: ui/apps/platform/cypress/test-results/ - name: Publish Test Report - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 if: always() with: paths: 'ui/apps/platform/cypress/test-results/reports/**/*.xml' @@ -277,24 +371,26 @@ jobs: uses: ./.github/actions/junit2jira with: create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} jira-token: ${{ secrets.JIRA_TOKEN }} gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} directory: 'ui/apps/platform/cypress/test-results/reports' local-roxctl-tests: - runs-on: ubuntu-latest + needs: detect-changes + if: ${{ !needs.detect-changes.outputs.ui_only }} + runs-on: ubuntu-24.04 outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11@sha256:df3459cf038e73775314b3a5d36d14dda41e6df438a0273c9efa8d23e0f2358a # ratchet:quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11 volumes: - /usr:/mnt/usr - /opt:/mnt/opt steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: - fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - uses: ./.github/actions/job-preamble @@ -310,7 +406,7 @@ jobs: run: ./scripts/ci/jobs/local-roxctl-tests.sh - name: Publish Test Report - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 if: always() with: paths: 'roxctl-test-output/*.xml' @@ -321,21 +417,23 @@ jobs: uses: ./.github/actions/junit2jira with: create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} jira-token: ${{ secrets.JIRA_TOKEN }} gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} directory: 'roxctl-test-output' shell-unit-tests: - runs-on: ubuntu-latest + needs: detect-changes + if: ${{ !needs.detect-changes.outputs.ui_only }} + runs-on: ubuntu-24.04 outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11@sha256:df3459cf038e73775314b3a5d36d14dda41e6df438a0273c9efa8d23e0f2358a # ratchet:quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: - fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - uses: ./.github/actions/job-preamble @@ -346,7 +444,7 @@ jobs: run: make shell-unit-tests - name: Publish Test Report - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 if: always() with: paths: 'shell-test-output/*.xml' @@ -357,17 +455,20 @@ jobs: uses: ./.github/actions/junit2jira with: create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} jira-token: ${{ secrets.JIRA_TOKEN }} gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} directory: 'shell-test-output' openshift-ci-unit-tests: - runs-on: ubuntu-latest + needs: detect-changes + if: ${{ !needs.detect-changes.outputs.ui_only }} + runs-on: ubuntu-24.04 container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11@sha256:df3459cf038e73775314b3a5d36d14dda41e6df438a0273c9efa8d23e0f2358a # ratchet:quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} @@ -380,21 +481,16 @@ jobs: run: make -C .openshift-ci test sensor-integration-tests: + needs: detect-changes + if: ${{ !needs.detect-changes.outputs.ui_only }} env: KUBECONFIG: "/tmp/kubeconfig" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} steps: - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - uses: ./.github/actions/job-preamble with: @@ -403,11 +499,36 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies + - name: Login to Quay.io + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # ratchet:docker/login-action@v4 + with: + registry: quay.io + username: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + password: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + - name: Create k8s Kind Cluster - uses: helm/kind-action@v1 + uses: helm/kind-action@06c1ae10762d3b9c1644e7fe69596ae519e015a2 # ratchet:helm/kind-action@v1.15.0 with: kubeconfig: "${{ env.KUBECONFIG }}" + - name: Load test images into Kind + run: | + source scripts/lib.sh + # Read images from prefetch list (single source of truth) + while IFS= read -r image; do + # Skip empty lines and comments + [[ -z "$image" || "$image" =~ ^# ]] && continue + echo "Loading $image into Kind..." + # Pull directly inside Kind node to avoid docker save/import issues with multi-arch + # This bypasses the problematic "kind load docker-image" which uses --all-platforms + # GitHub Actions runners are linux/amd64, so we only need that platform + # Use --user flag to pass quay.io credentials to ctr + retry 5 true docker exec chart-testing-control-plane ctr -n k8s.io images pull \ + --user "${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}:${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}" \ + --platform linux/amd64 \ + "$image" + done < sensor/tests/images-to-prefetch.txt + - name: Run sensor integration tests run: make sensor-integration-test @@ -416,7 +537,7 @@ jobs: run: make generate-junit-reports - name: Publish Test Report - uses: test-summary/action@v2 + uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2 if: always() with: paths: 'junit-reports/report.xml' @@ -427,6 +548,7 @@ jobs: uses: ./.github/actions/junit2jira with: create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} jira-token: ${{ secrets.JIRA_TOKEN }} gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} directory: 'junit-reports' @@ -442,7 +564,7 @@ jobs: contains(github.event.pull_request.labels.*.name, 'ci-test-github-action-slack-messages') ) name: Post failure message to Slack - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: - go - go-bench @@ -457,12 +579,12 @@ jobs: actions: read steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - name: Slack message env: GITHUB_CONTEXT: ${{ toJSON(github) }} - mention_author: ${{ needs.go.outputs.new-jiras || needs.go-postgres.outputs.new-jiras || needs.local-roxctl-tests.outputs.new-jiras || needs.ui.outputs.new-jiras || needs.go.outputs.new-jiras || needs.shell-unit-tests.outputs.new-jiras || needs.sensor-integration-tests.outputs.new-jiras }} + mention_author: true run: | source scripts/ci/lib.sh slack_workflow_failure diff --git a/.github/workflows/update_collector_periodic.yaml b/.github/workflows/update_collector_periodic.yaml index db4a50239f689..6eec999095b3d 100644 --- a/.github/workflows/update_collector_periodic.yaml +++ b/.github/workflows/update_collector_periodic.yaml @@ -2,18 +2,18 @@ name: Update collector version on: workflow_dispatch: schedule: - - cron: 0 5 * * 1-5 + - cron: 0 5 * * 1 jobs: update-collector: if: ${{ github.repository_owner == 'stackrox' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: ref: master # this is our target branch fetch-depth: 0 # we need to fetch all branches - name: Checkout collector repo - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: repository: stackrox/collector path: deps/collector @@ -22,7 +22,7 @@ jobs: run: make -sC deps/collector tag | tee COLLECTOR_VERSION - name: Create Pull Request id: cpr - uses: peter-evans/create-pull-request@v8 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # ratchet:peter-evans/create-pull-request@v8 with: token: '${{ secrets.RHACS_BOT_GITHUB_TOKEN }}' commit-message: Update COLLECTOR_VERSION @@ -37,12 +37,13 @@ jobs: labels: | ci-all-qa-tests dependencies + konflux-build team-reviewers: | collector-team draft: false - name: Enable Pull Request Automerge if: steps.cpr.outputs.pull-request-operation == 'created' - uses: peter-evans/enable-pull-request-automerge@v3 + uses: peter-evans/enable-pull-request-automerge@a660677d5469627102a1c1e11409dd063606628d # ratchet:peter-evans/enable-pull-request-automerge@v3 with: token: '${{ secrets.RHACS_BOT_GITHUB_TOKEN }}' pull-request-number: '${{ steps.cpr.outputs.pull-request-number }}' diff --git a/.github/workflows/update_fact_periodic.yaml b/.github/workflows/update_fact_periodic.yaml new file mode 100644 index 0000000000000..1ba1644308290 --- /dev/null +++ b/.github/workflows/update_fact_periodic.yaml @@ -0,0 +1,50 @@ +name: Update fact version +on: + workflow_dispatch: + schedule: + - cron: 0 5 * * 1 +jobs: + update-fact: + if: ${{ github.repository_owner == 'stackrox' }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + ref: master # this is our target branch + fetch-depth: 0 # we need to fetch all branches + - name: Checkout fact repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + with: + repository: stackrox/fact + path: deps/fact + fetch-depth: 0 # we need to fetch tags + - name: Update version + run: make -sC deps/fact tag | tee FACT_VERSION + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # ratchet:peter-evans/create-pull-request@v8 + with: + token: '${{ secrets.RHACS_BOT_GITHUB_TOKEN }}' + commit-message: Update FACT_VERSION + committer: '${{ secrets.RHACS_BOT_GITHUB_USERNAME }} <${{ secrets.RHACS_BOT_GITHUB_EMAIL }}>' + author: '${{ secrets.RHACS_BOT_GITHUB_USERNAME }} <${{ secrets.RHACS_BOT_GITHUB_EMAIL }}>' + branch: update_fact_version + signoff: false + delete-branch: true + title: 'chore(fact): Update FACT_VERSION' + body: | + Daily update of FACT_VERSION to latest master version + labels: | + ci-all-qa-tests + dependencies + konflux-build + team-reviewers: | + collector-team + draft: false + - name: Enable Pull Request Automerge + if: steps.cpr.outputs.pull-request-operation == 'created' + uses: peter-evans/enable-pull-request-automerge@a660677d5469627102a1c1e11409dd063606628d # ratchet:peter-evans/enable-pull-request-automerge@v3 + with: + token: '${{ secrets.RHACS_BOT_GITHUB_TOKEN }}' + pull-request-number: '${{ steps.cpr.outputs.pull-request-number }}' + merge-method: squash diff --git a/.github/workflows/update_scanner_periodic.yaml b/.github/workflows/update_scanner_periodic.yaml deleted file mode 100644 index 55acd4e23e780..0000000000000 --- a/.github/workflows/update_scanner_periodic.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: Update scanner version -on: - workflow_dispatch: - schedule: - - cron: 0 5 * * 1 -jobs: - update-scanner: - if: ${{ github.repository_owner == 'stackrox' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: master # this is our target branch - fetch-depth: 0 # we need to fetch all branches - - name: Checkout scanner repo - uses: actions/checkout@v6 - with: - repository: stackrox/scanner - path: deps/scanner - fetch-depth: 0 # we need to fetch tags - - name: Update version - run: make -sC deps/scanner tag | tee SCANNER_VERSION - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v8 - with: - token: '${{ secrets.RHACS_BOT_GITHUB_TOKEN }}' - commit-message: Update SCANNER_VERSION - committer: '${{ secrets.RHACS_BOT_GITHUB_USERNAME }} <${{ secrets.RHACS_BOT_GITHUB_EMAIL }}>' - author: '${{ secrets.RHACS_BOT_GITHUB_USERNAME }} <${{ secrets.RHACS_BOT_GITHUB_EMAIL }}>' - branch: update_scanner_version - signoff: false - delete-branch: true - title: 'chore(scanner): Update SCANNER_VERSION' - body: | - Weekly update of SCANNER_VERSION to latest master version - labels: | - ci-all-qa-tests - dependencies - team-reviewers: scanner - draft: false - - name: Enable Pull Request Automerge - if: steps.cpr.outputs.pull-request-operation == 'created' - uses: peter-evans/enable-pull-request-automerge@v3 - with: - token: '${{ secrets.RHACS_BOT_GITHUB_TOKEN }}' - pull-request-number: '${{ steps.cpr.outputs.pull-request-number }}' - merge-method: squash diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml index 82eb2190d4369..71e46fe283973 100644 --- a/.github/workflows/upgrade-test.yml +++ b/.github/workflows/upgrade-test.yml @@ -40,7 +40,7 @@ concurrency: Release automation ${{ inputs.milestone }} jobs: properties: name: Read repository properties - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: slack-channel: ${{ fromJSON(format('["{0}","{1}"]', steps.properties.outputs.dry-slack-channel, steps.properties.outputs.slack-channel))[inputs.dry-run != 'true'] }} steps: @@ -52,7 +52,7 @@ jobs: run-parameters: name: Run parameters - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - run: | [ "$DRY_RUN" = "true" ] && echo "::warning::This is a dry run" @@ -66,13 +66,13 @@ jobs: create-clusters: needs: [properties] if: github.event.inputs.dry-run != 'true' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: matrix: name: [test1, test2] steps: - name: Create the ${{ matrix.name }} cluster - uses: stackrox/actions/infra/create-cluster@v1 + uses: stackrox/actions/infra/create-cluster@d624eeae7073b3acb838aab7705730b2de58a4a0 # ratchet:stackrox/actions/infra/create-cluster@v1 with: token: ${{ secrets.INFRA_TOKEN }} flavor: gke-default @@ -83,18 +83,18 @@ jobs: prepare-clusters: needs: [properties, create-clusters] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Check out code - uses: actions/checkout@v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: ref: ${{ github.ref_name }} - - uses: stackrox/actions/infra/install-infractl@v1 - - uses: "google-github-actions/auth@v3" + - uses: stackrox/actions/infra/install-infractl@3b27674f0fa69f2d8c31ab26aa63c9f76401c64e # ratchet:stackrox/actions/infra/install-infractl@v1 + - uses: 'google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093' # ratchet:google-github-actions/auth@v3 with: credentials_json: "${{ secrets.GCP_RELEASE_AUTOMATION_SA }}" - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v3" + uses: 'google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db' # ratchet:google-github-actions/setup-gcloud@v3 with: install_components: "gke-gcloud-auth-plugin" - name: Prepare clusters for upgrade test @@ -111,7 +111,7 @@ jobs: "${{inputs.previous-version}}" \ "${{inputs.milestone}}" - name: Post to Slack about cluster creation - uses: slackapi/slack-github-action@v2.1.1 + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # ratchet:slackapi/slack-github-action@v3.0.3 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} @@ -136,10 +136,10 @@ jobs: name: Notify about failed cluster creation needs: [properties, prepare-clusters] if: always() && needs.prepare-clusters.result == 'failure' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Post to Slack - uses: slackapi/slack-github-action@v2.1.1 + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # ratchet:slackapi/slack-github-action@v3.0.3 with: method: chat.postMessage token: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/variables.yml b/.github/workflows/variables.yml index f574eec48faa4..6fedd4d7f5546 100644 --- a/.github/workflows/variables.yml +++ b/.github/workflows/variables.yml @@ -56,7 +56,7 @@ on: jobs: parse: name: Parse ${{inputs.version}} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: release: ${{steps.parse.outputs.release}} major: ${{steps.parse.outputs.major}} diff --git a/.github/workflows/verify-release.yml b/.github/workflows/verify-release.yml index f04d9379a4145..3792e9c80958b 100644 --- a/.github/workflows/verify-release.yml +++ b/.github/workflows/verify-release.yml @@ -32,7 +32,7 @@ concurrency: Release automation ${{ inputs.version }} jobs: run-parameters: name: Run parameters - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - run: | { @@ -52,7 +52,7 @@ jobs: properties: name: Read repository properties - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: jira-project: ${{ steps.properties.outputs.jira-project }} steps: @@ -64,20 +64,22 @@ jobs: verify-release: name: Verify all artifacts for release ${{ needs.variables.outputs.named-release-patch }} are published - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [variables, properties] env: + JIRA_USER: ${{ vars.RHACS_BOT_GITHUB_EMAIL }} JIRA_TOKEN: ${{ secrets.JIRA_TOKEN }} + JIRA_BASE_URL: ${{ vars.JIRA_BASE_URL }} steps: - name: Login to Quay.io - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # ratchet:docker/login-action@v4 with: registry: quay.io username: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} password: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} - name: Login to registry.redhat.io - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # ratchet:docker/login-action@v4 with: registry: registry.redhat.io username: ${{ secrets.RH_REGISTRY_USERNAME_RO }} diff --git a/.gitignore b/.gitignore index 73713a2db78c8..e51e9ea58251c 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,8 @@ combined_coverage.dat /stackrox-central-services-chart/ /stackrox-secured-cluster-services-chart/ .vscode +.kilo/ +.opencode/ # Used for mounting into the qa e2e-test docker container. .gitconfig diff --git a/.golangci.yml b/.golangci.yml index 00bef1461e32f..1bc4853049192 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,12 +1,13 @@ version: "2" run: timeout: 240m - go: "1.25" + go: "1.26" build-tags: - integration - scanner_db_integration - sql_integration - test_e2e + - test_e2e_vm modules-download-mode: readonly output: formats: @@ -28,7 +29,9 @@ linters: - gosec - govet - ineffassign + - modernize - nolintlint + - prealloc - protogetter - revive # replaces golint - rowserrcheck @@ -164,8 +167,12 @@ linters: - policy\.NewErr.* - retry\.MakeRetryable - retry\.WithRetry + - sensor\.DiagnoseConnectionFailure - status\.Error - utils\.Should + modernize: + disable: + - any exclusions: generated: lax rules: @@ -182,6 +189,12 @@ linters: - linters: - forbidigo path: roxctl/central/generate/interactive.go + - linters: + - prealloc + path-except: ^(pkg)/ + - linters: + - prealloc + path: _test\.go - linters: - forbidigo - wrapcheck diff --git a/.konflux/konflux.check-go-builder-version.Dockerfile b/.konflux/konflux.check-go-builder-version.Dockerfile new file mode 100644 index 0000000000000..f9b99226d5379 --- /dev/null +++ b/.konflux/konflux.check-go-builder-version.Dockerfile @@ -0,0 +1,29 @@ +FROM brew.registry.redhat.io/rh-osbs/openshift-golang-builder:rhel_9_golang_1.26@sha256:e74b03198cccc24c24a27071ad4d6c2edcfe0bb10c6de99c609ce6e3c06d6d04 + +WORKDIR /workspace + +# Copy only go.mod to check version compatibility +COPY go.mod . + +# Validate Go version compatibility +# go mod tidy will fail if go.mod requires a Go version higher than available in the builder +RUN echo "Go version of the builder:" && \ + go version 2>/dev/null +RUN echo "go.mod version requirement:" && \ + grep -E '^(go|toolchain) ' go.mod +RUN echo "If the following command fails, either roll back your changes to go.mod or update the openshift-golang-builder base images." +RUN echo "Checking go.mod compatibility..." && \ + go mod tidy +RUN echo "SUCCESS: Go version is compatible with go.mod" + +# Test that go mod tidy actually fails on incompatible versions +# This validates we're not relying on behavior that silently changed +RUN echo "Testing go mod tidy failure detection..." +RUN go mod edit -go=1.200.0 2>/dev/null +RUN if go mod tidy; then \ + echo "ERROR: go mod tidy succeeded with incompatible version"; \ + echo "Our assumption about go mod tidy behavior is broken!"; \ + exit 1; \ + else \ + echo "SUCCESS: go mod tidy correctly detects an incompatible Go version"; \ + fi diff --git a/.openshift-ci/Dockerfile.build_root b/.openshift-ci/Dockerfile.build_root index 158314ac8b810..fc24145a32636 100644 --- a/.openshift-ci/Dockerfile.build_root +++ b/.openshift-ci/Dockerfile.build_root @@ -27,4 +27,4 @@ # For an example, see https://github.com/stackrox/stackrox/pull/2762 and its counterpart # https://github.com/openshift/release/pull/31561 -FROM quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.1 +FROM quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11 diff --git a/.openshift-ci/begin.sh b/.openshift-ci/begin.sh index dcb3e34f61b65..b5411a9922b01 100755 --- a/.openshift-ci/begin.sh +++ b/.openshift-ci/begin.sh @@ -12,10 +12,46 @@ set -euo pipefail info "Start of CI handling" +# Skip expensive setup for non-UI jobs when the PR only touches ui/ files. +if [[ "${JOB_NAME:-}" =~ nongroovy|upgrade ]] && changes_limited_to "ui/"; then + info "Skipping begin: UI-only PR, non-UI job ${JOB_NAME}" + exit 0 +fi + openshift_ci_mods openshift_ci_import_creds setup_gcp + +# Pre-flight DNS check (mitigation for DPTP-5138) +info "Pre-flight DNS health check (DPTP-5138 mitigation)" + +_check_dns() { + local domain="$1" + getent hosts "$domain" >/dev/null 2>&1 +} + +dns_check_failed=false +for domain in oauth2.googleapis.com compute.googleapis.com storage.googleapis.com; do + if retry 5 true _check_dns "$domain"; then + info "✓ DNS OK for $domain" + else + warn "⚠️ DNS resolution FAILED for $domain after retries" + dns_check_failed=true + fi +done + +if [[ "$dns_check_failed" == "true" ]]; then + local error_msg="DNS is broken after retries (likely persistent node-resolver issue - see https://redhat.atlassian.net/browse/DPTP-5138). Failing fast to avoid wasted cluster creation time." + + # Generate structured JUnit report for better CI dashboard visibility + if [[ -n "${ARTIFACT_DIR:-}" ]] && command -v save_junit_failure >/dev/null 2>&1; then + save_junit_failure "DNS_Preflight_Check" "DNS resolution failed" "$error_msg" + fi + + die "$error_msg" +fi + set_ci_shared_export started_at "$(date -u +%s)" if [[ -z "${SHARED_DIR:-}" ]]; then @@ -26,7 +62,12 @@ fi if [[ "${JOB_NAME:-}" =~ -ocp- ]]; then info "Setting worker node type and count for OCP 4 jobs" set_ci_shared_export WORKER_NODE_COUNT 2 - set_ci_shared_export WORKER_NODE_TYPE e2-standard-8 + if [[ "${JOB_NAME:-}" =~ vm-scanning ]]; then + # Selecting nodes with KVM support + set_ci_shared_export WORKER_NODE_TYPE n2-standard-8 + else + set_ci_shared_export WORKER_NODE_TYPE e2-standard-8 + fi fi if [[ "${JOB_NAME:-}" =~ -eks- ]]; then diff --git a/.openshift-ci/ci_tests.py b/.openshift-ci/ci_tests.py index 3676771516f01..b3d602f46e2ec 100755 --- a/.openshift-ci/ci_tests.py +++ b/.openshift-ci/ci_tests.py @@ -6,6 +6,7 @@ import os import subprocess +import tempfile from common import popen_graceful_kill @@ -42,39 +43,8 @@ def run(self): pass -class UpgradeTest(BaseTest): - TEST_TIMEOUT = 60 * 60 * 2 - TEST_OUTPUT_DIR = "/tmp/postgres-upgrade-test-logs" - TEST_PG_UPGRADE_OUTPUT_DIR = "/tmp/postgres-version-upgrade-test-logs" - TEST_SENSOR_OUTPUT_DIR = "/tmp/postgres-sensor-upgrade-test-logs" - - def run(self): - print("Executing the Upgrade Test") - - self.run_with_graceful_kill( - [ - "tests/upgrade/postgres_sensor_run.sh", - self.TEST_SENSOR_OUTPUT_DIR, - ], - self.TEST_TIMEOUT, - output_dir=self.TEST_SENSOR_OUTPUT_DIR, - ) - - self.run_with_graceful_kill( - ["tests/upgrade/postgres_run.sh", self.TEST_OUTPUT_DIR], - self.TEST_TIMEOUT, - output_dir=self.TEST_OUTPUT_DIR, - ) - - self.run_with_graceful_kill( - ["tests/upgrade/postgres_upgrade_run.sh", self.TEST_PG_UPGRADE_OUTPUT_DIR], - self.TEST_TIMEOUT, - output_dir=self.TEST_PG_UPGRADE_OUTPUT_DIR, - ) - - class OperatorE2eTest(BaseTest): - OLM_SETUP_TIMEOUT_SEC = 60 * 10 + OLM_SETUP_TIMEOUT_SEC = 60 * 2 TEST_TIMEOUT_SEC = 60 * 60 * 2 OPERATOR_CLUSTER_TYPE_OPENSHIFT4 = "openshift4" @@ -94,38 +64,17 @@ def run(self): "-p", '[{"op": "add", "path": "/spec/disableAllDefaultSources", "value": true}]'], self.OLM_SETUP_TIMEOUT_SEC, ) + print("Bouncing catalog operator pod to clear its cache") olm_ns = "openshift-operator-lifecycle-manager" - else: - print("Installing OLM") - attempts = 3 - for attempt in range(1, attempts + 1): - try: - self.run_with_graceful_kill( - ["make", "-C", "operator", "olm-install"], - self.OLM_SETUP_TIMEOUT_SEC, - ) - break - except Exception as ex: - if attempt == attempts: - raise - print(f"OLM install failed with {ex} (attempt {attempt}/{attempts}), retrying...") - print("Removing unused catalog source(s)") self.run_with_graceful_kill( - ["kubectl", "delete", "catalogsource.operators.coreos.com", - "--namespace=olm", "--all"], + ["kubectl", "delete", "pods", + f"--namespace={olm_ns}", "--selector", "app=catalog-operator", "--now=true"], self.OLM_SETUP_TIMEOUT_SEC, ) - olm_ns = "olm" - print("Bouncing catalog operator pod to clear its cache") - self.run_with_graceful_kill( - ["kubectl", "delete", "pods", - f"--namespace={olm_ns}", "--selector", "app=catalog-operator", "--now=true"], - self.OLM_SETUP_TIMEOUT_SEC, - ) print("Executing operator e2e tests") self.run_with_graceful_kill( - ["operator/tests/run.sh"], + ["operator/tests/run.sh", self._operator_cluster_type], self.TEST_TIMEOUT_SEC, output_dir="/tmp/operator-e2e-misc-logs", ) @@ -213,7 +162,7 @@ def run(self): class UIE2eTest(BaseTest): - TEST_TIMEOUT = 2 * 60 * 60 + TEST_TIMEOUT = 3 * 60 * 60 def run(self): print("Executing UI e2e test") @@ -243,7 +192,7 @@ def run(self): class NonGroovyE2e(BaseTest): - TEST_TIMEOUT = 90 * 60 + TEST_TIMEOUT = 3 * 60 * 60 TEST_OUTPUT_DIR = "/tmp/e2e-test-logs" def run(self): @@ -256,26 +205,21 @@ def run(self): ) -class SensorIntegration(BaseTest): - TEST_TIMEOUT = 90 * 60 - TEST_OUTPUT_DIR = "/tmp/sensor-integration-test-logs" +class VMScanningE2e(BaseTest): + TEST_TIMEOUT = 2 * 60 * 60 + TEST_OUTPUT_DIR_PREFIX = "vm-scanning-test-logs-" def run(self): - print("Executing the Sensor Integration Tests") + print("Executing VM scanning e2e tests") + output_dir = tempfile.mkdtemp(prefix=self.TEST_OUTPUT_DIR_PREFIX, dir=os.getenv("TMPDIR", "/tmp")) self.run_with_graceful_kill( - ["tests/e2e/sensor.sh", self.TEST_OUTPUT_DIR], + ["tests/e2e/run-vm-scanning.sh", output_dir], self.TEST_TIMEOUT, - output_dir=self.TEST_OUTPUT_DIR, + output_dir=output_dir, ) -class SensorIntegrationOCP(SensorIntegration): - def run(self): - # TODO(ROX-17875): make them work on OCP. - print("Skipping the Sensor Integration Tests for OCP") - - class ScaleTest(BaseTest): TEST_TIMEOUT = 90 * 60 PPROF_ZIP_OUTPUT = "/tmp/scale-test/pprof.zip" @@ -317,7 +261,7 @@ def run(self): class BYODBTest(BaseTest): - TEST_TIMEOUT = 60 * 60 * 2 + TEST_TIMEOUT = 60 * 60 * 3 TEST_OUTPUT_DIR = "/tmp/byodb-test-logs" def run(self): diff --git a/.openshift-ci/common.py b/.openshift-ci/common.py index 44b109710e24f..fdf0d0bd212e5 100644 --- a/.openshift-ci/common.py +++ b/.openshift-ci/common.py @@ -1,5 +1,6 @@ from datetime import datetime import os +import re import subprocess @@ -27,3 +28,32 @@ def log_print(*args): now = datetime.now() time = now.strftime("%H:%M:%S") print(f"{time}:", *args) + + +def enable_sfa_for_ocp(): + """Enable the Fact (SFA) agent on OCP >= 4.16. + + SFA is not supported on older OCP versions. This function checks + CLUSTER_FLAVOR_VARIANT to determine the OCP version and sets + SFA_AGENT=true if the version is 4.16 or later. + """ + try: + ocp_variant = os.environ.get("CLUSTER_FLAVOR_VARIANT", "") + + # A special case, OCP release candidate version. Most certainly newer + # than we require. + if ocp_variant == "openshift-4-ocp/candidate": + os.environ["SFA_AGENT"] = "true" + log_print("Enabled SFA agent for OCP", ocp_variant) + return + + expr = r"openshift-4-ocp/\w+-(?P\d+)\.(?P\d+)" + m = re.match(expr, ocp_variant) + if m: + major = int(m.group("major")) + minor = int(m.group("minor")) + if (major, minor) >= (4, 16): + os.environ["SFA_AGENT"] = "true" + log_print("Enabled SFA agent for OCP", ocp_variant) + except (ValueError, TypeError, AttributeError) as ex: + log_print(f"Could not identify OCP version, SFA is disabled: {ex}") diff --git a/.openshift-ci/compatibility_test.py b/.openshift-ci/compatibility_test.py index ca6ffea982d95..00a0967fe7375 100644 --- a/.openshift-ci/compatibility_test.py +++ b/.openshift-ci/compatibility_test.py @@ -9,7 +9,6 @@ from pre_tests import ( PreSystemTests, - CollectionMethodOverridePreTest ) from post_tests import PostClusterTest, FinalPost from runners import ClusterTestSetsRunner, TestSet @@ -23,7 +22,7 @@ def run_compatibility_tests(testfunc, cluster_name): # start logging logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - # Get the test tuples (central_version, sensor_version) for supported versions with available helm charts + # Get the test tuples (central_version, sensor_version) for supported versions test_tuples = get_compatibility_test_tuples() if len(test_tuples) > 0: @@ -32,9 +31,6 @@ def run_compatibility_tests(testfunc, cluster_name): os.environ["ROX_TELEMETRY_STORAGE_KEY_V1"] = 'DISABLED' test_versions = f'{test_tuple.central_version}--{test_tuple.sensor_version}' - # expected version string is like 74.x.x for ACS 3.74 versions - is_3_74_sensor = test_tuple.sensor_version.startswith('74') - logging.info("Running compatibility tests for central-v%s, sensor-v%s with function %s", test_tuple.central_version, test_tuple.sensor_version, testfunc.__name__) @@ -43,12 +39,9 @@ def run_compatibility_tests(testfunc, cluster_name): f'version compatibility tests: {test_versions}', test=testfunc(test_tuple.central_version, test_tuple.sensor_version), post=PostClusterTest( - collect_collector_metrics=not is_3_74_sensor, check_stackrox_logs=True, artifact_destination_prefix=test_versions, - ), - # Collection not supported on 3.74 - pre=CollectionMethodOverridePreTest("NO_COLLECTION" if is_3_74_sensor else "core_bpf") + ) ) ) ClusterTestSetsRunner( diff --git a/.openshift-ci/dev-requirements.txt b/.openshift-ci/dev-requirements.txt index 4b112aa426cd4..e5bdc0337bc31 100644 --- a/.openshift-ci/dev-requirements.txt +++ b/.openshift-ci/dev-requirements.txt @@ -1,4 +1,4 @@ -# These versions should match those used in the current CI test image (stackrox-test-0.5.1). +# These versions should match those used in the current CI test image (stackrox-test-0.5.11). # See .github/workflows/{lint,style}.yaml for that. # And the stackrox/rox-ci-image repo for the original source and pinned versions. pycodestyle==2.14.0 diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh index 968ff14e0c72d..bd8e4ce1a1a27 100755 --- a/.openshift-ci/dispatch.sh +++ b/.openshift-ci/dispatch.sh @@ -12,6 +12,23 @@ source "$ROOT/tests/e2e/lib.sh" set -euo pipefail +if [[ "$#" -lt 1 ]]; then + die "usage: dispatch [<...other parameters...>]" +fi + +ci_job="$1" +shift + +# Skip non-UI jobs early when the PR only touches files under ui/. +case "$ci_job" in + *nongroovy*|*upgrade*) + if changes_limited_to "ui/"; then + info "Skipping $ci_job: all changes are under ui/" + exit 0 + fi + ;; +esac + if [[ -f "${SHARED_DIR:-}/shared_env" ]]; then # shellcheck disable=SC1091 source "${SHARED_DIR:-}/shared_env" @@ -21,17 +38,19 @@ openshift_ci_mods openshift_ci_import_creds create_exit_trap -if [[ "$#" -lt 1 ]]; then - die "usage: dispatch [<...other parameters...>]" -fi +# Enable Scanner V4 by default for all e2e jobs. openshift/release currently +# hardcodes ROX_SCANNER_V4=false; this override can be removed once that is +# dropped. Individual job scripts can opt out via os.environ["ROX_SCANNER_V4"]. +# +# TODO(ROX-35345): remove this +export ROX_SCANNER_V4=true -ci_job="$1" -shift ci_export CI_JOB_NAME "$ci_job" case "$ci_job" in gke*qa-e2e-tests|gke*nongroovy-e2e-tests|gke*upgrade-tests|gke-ui-e2e-tests|\ - eks-qa-e2e-tests|osd*qa-e2e-tests|gke*sensor-integration-tests) + eks-qa-e2e-tests|osd*qa-e2e-tests|\ + *vm-scanning-e2e-tests) openshift_ci_e2e_mods ;; *-operator-e2e-tests) @@ -40,7 +59,8 @@ case "$ci_job" in esac case "$ci_job" in - eks-qa-e2e-tests|osd*qa-e2e-tests) + eks-qa-e2e-tests|osd*qa-e2e-tests|ocp*ui-e2e-tests|\ + *vm-scanning-e2e-tests) setup_automation_flavor_e2e_cluster "$ci_job" ;; esac diff --git a/.openshift-ci/end.sh b/.openshift-ci/end.sh index d1081ebbba79b..71a402d174457 100755 --- a/.openshift-ci/end.sh +++ b/.openshift-ci/end.sh @@ -119,7 +119,7 @@ generate_cluster_junit() { if [[ "${CREATE_CLUSTER_OUTCOME:-}" == "${OUTCOME_PASSED}" ]]; then save_junit_success "${_JUNIT_CLUSTER_CLASS}" "${_JUNIT_CLUSTER_CREATE_DESCRIPTION} ${cluster_flavor_variant}" - elif [[ "${CREATE_CLUSTER_OUTCOME:-}" == "${OUTCOME_FAILED}" ]]; then + else save_junit_failure "${_JUNIT_CLUSTER_CLASS}" "${_JUNIT_CLUSTER_CREATE_DESCRIPTION} ${cluster_flavor_variant}" \ "${cluster_create_debug}" fi @@ -133,7 +133,7 @@ generate_cluster_junit() { if [[ "${DESTROY_CLUSTER_OUTCOME:-}" == "${OUTCOME_PASSED}" ]]; then save_junit_success "${_JUNIT_CLUSTER_CLASS}" "${_JUNIT_CLUSTER_DESTROY_DESCRIPTION} ${cluster_flavor_variant}" - elif [[ "${DESTROY_CLUSTER_OUTCOME:-}" == "${OUTCOME_FAILED}" && "${CREATE_CLUSTER_OUTCOME:-}" == "${OUTCOME_PASSED}" ]]; then + elif [[ "${CREATE_CLUSTER_OUTCOME:-}" == "${OUTCOME_PASSED}" ]]; then # Only record cluster destroy failures when the create succeeded. Otherwise it is mostly noise. save_junit_failure "${_JUNIT_CLUSTER_CLASS}" "${_JUNIT_CLUSTER_DESTROY_DESCRIPTION} ${cluster_flavor_variant}" \ "${cluster_destroy_debug}" diff --git a/.openshift-ci/get_compatibility_test_tuples.py b/.openshift-ci/get_compatibility_test_tuples.py index 98f3ff4702071..02c5ff99df27e 100755 --- a/.openshift-ci/get_compatibility_test_tuples.py +++ b/.openshift-ci/get_compatibility_test_tuples.py @@ -4,6 +4,7 @@ Returns the central-sensor version tuples to be used for compatibility testing. """ import logging +import re import subprocess import sys from collections import namedtuple @@ -11,7 +12,6 @@ from get_latest_helm_chart_versions import ( get_supported_helm_chart_versions, - get_latest_helm_chart_version_for_specific_release, ) @@ -19,17 +19,17 @@ # get_compatibility_test_tuples.py provides the function get_compatibility_test_tuples() which is called in our # compatibility tests that returns all tuples of central and sensor versions for which: # 1. Central is latest and Sensor is a supported older version OR -# 2. Sensor is latest and Central is a supported older version OR -# 3. Is a support exception -# These are returned only if Helm charts for both versions could be found. +# 2. Sensor is latest and Central is a supported older version +# These are returned only if charts for both versions could be found. # After running get_compatibility_test_tuples.py I received the following output: # INFO:root:Listing supported versions tuples: -# INFO:root:Tuple 1: {Central v4.6.x-736-g48077a980e-dirty - Sensor v400.5.3} -# INFO:root:Tuple 2: {Central v4.6.x-736-g48077a980e-dirty - Sensor v400.4.5} -# INFO:root:Tuple 3: {Central v400.5.3 - Sensor v4.6.x-736-g48077a980e-dirty} -# INFO:root:Tuple 4: {Central v400.4.5 - Sensor v4.6.x-736-g48077a980e-dirty} -# INFO:root:Tuple 5: {Central v4.6.x-736-g48077a980e-dirty - Sensor v74.9.0} -# If no supported versions with available Helm charts are found, an empty list is returned. +# INFO:root:Tuple 1: {Central v4.12.x-793-gf6bf3cc40e - Sensor v4.11.2} +# INFO:root:Tuple 2: {Central v4.12.x-793-gf6bf3cc40e - Sensor v4.10.6} +# INFO:root:Tuple 3: {Central v4.12.x-793-gf6bf3cc40e - Sensor v4.9.10} +# INFO:root:Tuple 4: {Central v4.11.2 - Sensor v4.12.x-793-gf6bf3cc40e} +# INFO:root:Tuple 5: {Central v4.10.6 - Sensor v4.12.x-793-gf6bf3cc40e} +# INFO:root:Tuple 6: {Central v4.9.10 - Sensor v4.12.x-793-gf6bf3cc40e} +# If no supported versions with available charts are found, an empty list is returned. def main(): logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) test_tuples = get_compatibility_test_tuples() @@ -44,36 +44,40 @@ def main(): ) -# Returns True if the helm_version is newer than the current_version -def is_newer_version(current_version: str, helm_version: str): - helm_version_split = helm_version.split(sep='.') - current_version_split = current_version.split(sep='.') - # Remove the trailing 0s - # Helm chart versions have two trailing 0s in the major version number - helm_version_split[0] = helm_version_split[0][:-2] - # Remove commit hash from the current version - current_version_split = current_version_split[:-1] - # If we are in a release branch, we will have patch version with '-rc' - if len(current_version_split) > 2: - # Remove '-rc' if present - current_version_split[2] = str(current_version_split[2]).rstrip("-rc") - - for (current, helm) in zip(current_version_split, helm_version_split): - if current > helm: +# Returns True if the product_version is newer than the current_version. +# +# current_version examples: '4.11.x-736-g48077a980e', '4.11.1-rc.2', '4.11.1' +# product_version examples: '4.9.3', '4.11.0' +def is_newer_version(current_version: str, product_version: str): + product_parts = [int(x) for x in product_version.split('.')] + + # Parse the numeric major.minor[.patch] prefix from current_version, stopping + # at any component that does not begin with a digit (e.g. 'x-736-ghash'). + # This correctly handles: + # dev: '4.11.x-736-ghash' -> [4, 11] + # rc: '4.11.1-rc.2' -> [4, 11, 1] + # release: '4.11.1' -> [4, 11, 1] + current_parts = [] + for part in current_version.split('.')[:3]: + m = re.match(r'^(\d+)', part) + if not m: break - if current < helm: + current_parts.append(int(m.group(1))) + + for (current, product) in zip(current_parts, product_parts): + if current > product: + return False + if current < product: return True return False def get_compatibility_test_tuples(): - Release = namedtuple("Release", ["major", "minor"]) - # start logging logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - central_chart_versions, sensor_chart_versions = get_supported_helm_chart_versions() + central_versions, sensor_versions = get_supported_helm_chart_versions() makefile_path = Path(__file__).parent.parent latest_tag = subprocess.check_output( @@ -89,57 +93,35 @@ def get_compatibility_test_tuples(): # versions. # There is no risk in excluding newer versions as the compatibility tests in # their respective branches will test against older versions. - central_chart_versions = [i for i in central_chart_versions - if not - is_newer_version(current_version=latest_tag, - helm_version=i)] - sensor_chart_versions = [i for i in sensor_chart_versions - if not - is_newer_version(current_version=latest_tag, - helm_version=i)] - - if len(central_chart_versions) == 0: - logging.info("Found no older central chart versions to test against according to the product lifecycles API.") - if len(sensor_chart_versions) == 0: - logging.info("Found no older sensor chart versions to test against according to the product lifecycles API.") - if len(central_chart_versions) == 0 or len(sensor_chart_versions) == 0: - logging.info("However versions with support exceptions will still be tested against.") - - ChartVersions = namedtuple( - "Chart_versions", ["central_version", "sensor_version"]) - - # Latest central vs sensor versions in sensor_chart_versions + central_versions = [i for i in central_versions + if not is_newer_version(current_version=latest_tag, + product_version=i)] + sensor_versions = [i for i in sensor_versions + if not is_newer_version(current_version=latest_tag, + product_version=i)] + + if len(central_versions) == 0: + logging.info("Found no older central versions to test against according to the product lifecycles API.") + if len(sensor_versions) == 0: + logging.info("Found no older sensor versions to test against according to the product lifecycles API.") + + VersionTuple = namedtuple("VersionTuple", ["central_version", "sensor_version"]) + + # Latest central vs older sensor versions test_tuples = [ - ChartVersions(central_version=latest_tag, - sensor_version=sensor_chart_version) - for sensor_chart_version in sensor_chart_versions + VersionTuple(central_version=latest_tag, + sensor_version=sensor_version) + for sensor_version in sensor_versions ] - # Latest sensor vs central versions in central_chart_versions + # Older central versions vs latest sensor test_tuples.extend( [ - ChartVersions(central_version=central_chart_version, - sensor_version=latest_tag) - for central_chart_version in central_chart_versions + VersionTuple(central_version=central_version, + sensor_version=latest_tag) + for central_version in central_versions ] ) - # Currently there are no support exceptions, the last one expired on 2024-06-30, see: - # https://issues.redhat.com/browse/ROX-18223 - # however a new support exception is being negotiated, add it here when it's ready - support_exceptions = [ - ChartVersions( - central_version=latest_tag, - sensor_version=get_latest_helm_chart_version_for_specific_release( - "stackrox-secured-cluster-services", Release(major=3, minor=74) - ), - ) - ] - - test_tuples.extend( - support_exception - for support_exception in support_exceptions - if support_exception not in test_tuples - ) return test_tuples diff --git a/.openshift-ci/get_latest_helm_chart_versions.py b/.openshift-ci/get_latest_helm_chart_versions.py index ae5c4b0f5d33f..c61a18049c978 100755 --- a/.openshift-ci/get_latest_helm_chart_versions.py +++ b/.openshift-ci/get_latest_helm_chart_versions.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ -Looks up N previous releases and outputs a Helm chart version for the most +Looks up supported releases and outputs a product version for the most recent patch for each found release. """ @@ -46,11 +46,13 @@ # with the current cadence and support period. NUM_RELEASES_DEFAULT = 3 -# For support exceptions we may need to get the latest patch for a specific -# release that is not within the last N versions. In that case -# get_latest_helm_chart_version_for_specific_release will provide the latest -# patch of the input release. -sample_support_exception = Release(major=3, minor=74) +# From this release onward the stackrox-operator chart is used instead of the +# per-component legacy charts (stackrox-central-services, +# stackrox-secured-cluster-services), which are not published for new releases. +# Drop legacy chart inspection can be dropped, once 4.10 goes EOL. +OPERATOR_CHART_FIRST_RELEASE = Release(major=4, minor=11) + +OPERATOR_CHART_NAME = "stackrox-operator" def main(argv): @@ -60,25 +62,29 @@ def main(argv): "stackrox-secured-cluster-services", num_releases ) logging.info( - f"Helm chart versions for the latest {num_releases} releases:") + f"Product versions for the latest {num_releases} releases:") logging.info("\n".join(helm_versions)) - helm_version_specific = get_latest_helm_chart_version_for_specific_release( - "stackrox-secured-cluster-services", sample_support_exception - ) - logging.info( - f"Latest chart version for the {sample_support_exception} " - f"releases is {helm_version_specific}" - ) supported_versions_string = [[f"{version.major}.{version.minor}"] for version in get_supported_releases()] supported_central_versions_from_api, supported_sensor_versions_from_api = get_supported_helm_chart_versions() logging.info( f"\nThe product lifecycles API denotes support for the following versions: {supported_versions_string}\n" - f"Found helm charts for the following supported versions: " + f"Found product versions for the following supported versions: " f"central{supported_central_versions_from_api} - sensor{supported_sensor_versions_from_api}" ) +def _is_legacy_release(release: Release) -> bool: + """Returns True for releases that use per-component charts (pre-4.11).""" + return release < OPERATOR_CHART_FIRST_RELEASE + + +def _chart_to_product_version(chart: dict) -> str: + """Return a product version string (e.g. '4.9.3') from a chart dict.""" + av = chart["parsed_app_version"] + return f"{av.major}.{av.minor}.{av.patch}" + + def get_latest_helm_chart_versions(chart_name, num_releases=NUM_RELEASES_DEFAULT): add_helm_repo() try: @@ -110,22 +116,42 @@ def __get_supported_helm_chart_versions(): supported_central_versions = [] supported_sensor_versions = [] + charts = read_charts() # one helm search, reused for all releases + supported_releases = get_supported_releases() for release in supported_releases: - if __does_chart_exist("stackrox-central-services", release): - supported_central_versions.append(__get_latest_helm_chart_version_for_specific_release( - "stackrox-central-services", release) - ) - else: - logging.debug(f"Supported version \"{release.major}.{release.minor}\" has no corresponding helm chart for " - f"stackrox-central-services.") - if __does_chart_exist("stackrox-secured-cluster-services", release): - supported_sensor_versions.append(__get_latest_helm_chart_version_for_specific_release( - "stackrox-secured-cluster-services", release) - ) + if _is_legacy_release(release): + # Pre-4.11: separate charts for central and sensor. + for chart_name, version_list in [ + ("stackrox-central-services", supported_central_versions), + ("stackrox-secured-cluster-services", supported_sensor_versions), + ]: + filtered = filter_charts_by_name(charts, chart_name) + if any(version_to_release(c["parsed_app_version"]) == release + for c in filtered): + chart = get_latest_chart_for_specific_release(filtered, release) + version_list.append(_chart_to_product_version(chart)) + else: + logging.debug( + f"Supported version \"{release.major}.{release.minor}\" has no " + f"corresponding chart for {chart_name}." + ) else: - logging.debug(f"Supported version \"{release.major}.{release.minor}\" has no corresponding helm chart for " - f"stackrox-secured-cluster-services.") + # 4.11+: one operator chart covers both central and sensor. + filtered = filter_charts_by_name(charts, OPERATOR_CHART_NAME) + if any(version_to_release(c["parsed_app_version"]) == release + for c in filtered): + chart = get_latest_chart_for_specific_release(filtered, release) + version = _chart_to_product_version(chart) + supported_central_versions.append(version) + supported_sensor_versions.append(version) + else: + logging.warning( + f"Supported version \"{release.major}.{release.minor}\" has no " + f"corresponding chart for {OPERATOR_CHART_NAME}. " + f"This release will be skipped in compatibility testing." + ) + return supported_central_versions, supported_sensor_versions @@ -164,15 +190,6 @@ def __get_data_from_product_lifecycles_api(): return [] -def __does_chart_exist(chart_name, release): - charts = read_charts() - filtered_charts = filter_charts_by_name(charts, chart_name) - for fchart in filtered_charts: - if version_to_release(fchart["parsed_app_version"]) == release: - return True - return False - - def __get_latest_helm_chart_versions(chart_name, num_releases): charts = read_charts() logging.info(f"Discovered total {len(charts)} charts") @@ -187,8 +204,7 @@ def __get_latest_helm_chart_versions(chart_name, num_releases): logging.debug( f"Identified these charts as {num_releases} latest: {latest_charts}") - # Specifically remove 400.1.6 which is affected by a max message size bug, but is no longer supported. - return [c["version"] for c in latest_charts if c["version"] != "400.1.6"] + return [_chart_to_product_version(c) for c in latest_charts] def __get_latest_helm_chart_version_for_specific_release(chart_name, release): @@ -204,7 +220,7 @@ def __get_latest_helm_chart_version_for_specific_release(chart_name, release): logging.debug( f"Identified {latest_chart} as latest version of release {release}") - return latest_chart["version"] + return _chart_to_product_version(latest_chart) def read_charts(): diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py index 0abe84832d6d1..64f27674b5204 100644 --- a/.openshift-ci/post_tests.py +++ b/.openshift-ci/post_tests.py @@ -15,10 +15,13 @@ class PostTestsConstants: API_TIMEOUT = 5 * 60 - COLLECT_TIMEOUT = 15 * 60 + COLLECT_TIMEOUT = 30 * 60 COLLECT_INFRA_TIMEOUT = 12 * 60 CHECK_TIMEOUT = 5 * 60 STORE_TIMEOUT = 5 * 60 + # k8s service logs include many namespaces, pod logs, and OpenShift + # must-gather audit archives; gsutil uploads routinely exceed STORE_TIMEOUT. + STORE_LARGE_TIMEOUT = 20 * 60 FIXUP_TIMEOUT = 5 * 60 ARTIFACTS_TIMEOUT = 3 * 60 # QA_TEST_DEBUG_LOGS - where the QA tests store failure logs. @@ -31,6 +34,7 @@ class PostTestsConstants: DIAGNOSTIC_OUTPUT = "diagnostic-bundle" CENTRAL_DATA_OUTPUT = "central-data" STACKROX_LOG_DIR = "/tmp/stackrox-logs" + VM_GUEST_LOG_DIR = "/tmp/vm-diagnostics" class NullPostTest: @@ -84,6 +88,12 @@ def run(self, test_outputs=None): self.store_artifacts(test_outputs) self.handle_run_failure() + def _store_timeout_for(self, source): + artifact_name = os.path.basename(source.rstrip(os.sep)) + if artifact_name == "k8s-logs": + return self.STORE_LARGE_TIMEOUT + return self.STORE_TIMEOUT + def store_artifacts(self, test_outputs=None): if test_outputs is not None: self.data_to_store = test_outputs + self.data_to_store @@ -99,7 +109,7 @@ def store_artifacts(self, test_outputs=None): ) self.run_with_best_effort( args, - timeout=self.STORE_TIMEOUT, + timeout=self._store_timeout_for(source), ) self._store_osci_artifacts() @@ -142,12 +152,13 @@ def _store_osci_artifact(self, source): class PostClusterTest(StoreArtifacts): """The standard cluster test suite of debug gathering and analysis""" - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-positional-arguments def __init__( self, collect_collector_metrics=True, collect_central_artifacts=True, collect_service_logs=True, + collect_vm_guest_logs=False, check_stackrox_logs=False, artifact_destination_prefix=None, ): @@ -160,10 +171,11 @@ def __init__( self.service_logs_destination = self.K8S_LOG_DIR self._collect_collector_metrics = collect_collector_metrics self._collect_service_logs = collect_service_logs + self._collect_vm_guest_logs = collect_vm_guest_logs self._check_stackrox_logs = check_stackrox_logs self.k8s_namespaces = [ "stackrox", - "stackrox-operator", + "rhacs-operator-system", "proxies", "squid", "kube-system", @@ -177,6 +189,7 @@ def __init__( "openshift-etcd", "openshift-controller-manager", "openshift-compliance", + "openshift-kube-apiserver", "openshift-ingress", "openshift-marketplace", ] @@ -192,6 +205,8 @@ def run(self, test_outputs=None): self.grab_central_data() if self._collect_service_logs: self.collect_service_logs() + if self._collect_vm_guest_logs: + self.collect_vm_guest_logs() if self._check_stackrox_logs: self.check_stackrox_logs() self.store_artifacts(test_outputs) @@ -222,6 +237,23 @@ def collect_service_logs(self): ) self.data_to_store.append(self.service_logs_destination) + def collect_vm_guest_logs(self): + # Per-VM virtctl ssh is capped in the script. Cleanup is a second + # process because a collect timeout SIGKILLs the first before it can + # delete namespaces. + self.run_with_best_effort( + [ + "scripts/ci/collect-vm-guest-logs.sh", + self.VM_GUEST_LOG_DIR, + ], + timeout=10 * 60, + ) + self.data_to_store.append(self.VM_GUEST_LOG_DIR) + self.run_with_best_effort( + ["scripts/ci/delete-vm-ns.sh"], + timeout=60, + ) + def collect_collector_metrics(self): self.run_with_best_effort( [ diff --git a/.openshift-ci/pre_tests.py b/.openshift-ci/pre_tests.py index 8b4d16e4e29d6..1902074fcc020 100644 --- a/.openshift-ci/pre_tests.py +++ b/.openshift-ci/pre_tests.py @@ -2,7 +2,6 @@ PreTests - something to run before test but after resource provisioning. """ -import os import subprocess @@ -57,15 +56,3 @@ def run(self): check=True, timeout=self.START_PREFETCH_TIMEOUT, ) - - -class CollectionMethodOverridePreTest: - """ - CollectionPreTest - allows finer control over collection method - for individual test jobs - """ - def __init__(self, method): - self._collection_method = method - - def run(self): - os.environ['COLLECTION_METHOD'] = self._collection_method diff --git a/.openshift-ci/tests/test_common.py b/.openshift-ci/tests/test_common.py new file mode 100644 index 0000000000000..0fe018e0e7c3b --- /dev/null +++ b/.openshift-ci/tests/test_common.py @@ -0,0 +1,52 @@ +import os +import unittest + +from common import enable_sfa_for_ocp + + +class TestEnableSfaForOcp(unittest.TestCase): + def setUp(self): + os.environ.pop("SFA_AGENT", None) + os.environ.pop("CLUSTER_FLAVOR_VARIANT", None) + + def tearDown(self): + os.environ.pop("SFA_AGENT", None) + os.environ.pop("CLUSTER_FLAVOR_VARIANT", None) + + def test_enables_sfa_for_ocp_4_16(self): + os.environ["CLUSTER_FLAVOR_VARIANT"] = "openshift-4-ocp/stable-4.16" + enable_sfa_for_ocp() + self.assertEqual(os.environ.get("SFA_AGENT"), "true") + + def test_enables_sfa_for_ocp_4_17(self): + os.environ["CLUSTER_FLAVOR_VARIANT"] = "openshift-4-ocp/stable-4.17" + enable_sfa_for_ocp() + self.assertEqual(os.environ.get("SFA_AGENT"), "true") + + def test_enables_sfa_for_ocp_5_1(self): + os.environ["CLUSTER_FLAVOR_VARIANT"] = "openshift-4-ocp/stable-5.1" + enable_sfa_for_ocp() + self.assertEqual(os.environ.get("SFA_AGENT"), "true") + + def test_does_not_enable_sfa_for_ocp_4_15(self): + os.environ["CLUSTER_FLAVOR_VARIANT"] = "openshift-4-ocp/stable-4.15" + enable_sfa_for_ocp() + self.assertIsNone(os.environ.get("SFA_AGENT")) + + def test_does_not_enable_sfa_when_variant_missing(self): + enable_sfa_for_ocp() + self.assertIsNone(os.environ.get("SFA_AGENT")) + + def test_does_not_enable_sfa_for_non_ocp_variant(self): + os.environ["CLUSTER_FLAVOR_VARIANT"] = "gke-default" + enable_sfa_for_ocp() + self.assertIsNone(os.environ.get("SFA_AGENT")) + + def test_does_not_enable_sfa_for_malformed_variant(self): + os.environ["CLUSTER_FLAVOR_VARIANT"] = "openshift-4-ocp/" + enable_sfa_for_ocp() + self.assertIsNone(os.environ.get("SFA_AGENT")) + + +if __name__ == "__main__": + unittest.main() diff --git a/.openshift-ci/tests/test_get_compatibility_test_tuples.py b/.openshift-ci/tests/test_get_compatibility_test_tuples.py new file mode 100644 index 0000000000000..fabf092c277a9 --- /dev/null +++ b/.openshift-ci/tests/test_get_compatibility_test_tuples.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +""" +Unit tests for get_compatibility_test_tuples.py + +These tests verify that version comparison works correctly, +particularly for double-digit minor versions (4.10+) where +string comparison fails. +""" + +import unittest + +from get_compatibility_test_tuples import is_newer_version + + +class TestIsNewerVersion(unittest.TestCase): + """Test the is_newer_version function to prevent regression of string vs int comparison bug""" + + def test_double_digit_minor_version_not_newer(self): + """4.11 should NOT consider 4.9 as newer (this was the bug)""" + self.assertFalse( + is_newer_version('4.11.x-94-g75a2cb6b34', '4.9.5'), + "4.9 should not be considered newer than 4.11" + ) + + def test_double_digit_minor_version_4_10_vs_4_9(self): + """4.10 should NOT consider 4.9 as newer""" + self.assertFalse( + is_newer_version('4.10.x-50-ghash', '4.9.5'), + "4.9 should not be considered newer than 4.10" + ) + + def test_double_digit_minor_version_4_11_vs_4_10(self): + """4.11 should NOT consider 4.10 as newer""" + self.assertFalse( + is_newer_version('4.11.x-94-ghash', '4.10.2'), + "4.10 should not be considered newer than 4.11" + ) + + def test_double_digit_minor_version_is_newer(self): + """4.10 should consider 4.11 as newer""" + self.assertTrue( + is_newer_version('4.10.x-50-ghash', '4.11.1'), + "4.11 should be considered newer than 4.10" + ) + + def test_single_digit_minor_version_not_newer(self): + """4.8 should NOT consider 4.7 as newer (existing behavior)""" + self.assertFalse( + is_newer_version('4.8.x-325-ghash', '4.7.0'), + "4.7 should not be considered newer than 4.8" + ) + + def test_single_digit_minor_version_is_newer(self): + """4.6 should consider 4.9 as newer (existing behavior)""" + self.assertTrue( + is_newer_version('4.6.x-736-ghash', '4.9.5'), + "4.9 should be considered newer than 4.6" + ) + + def test_equal_major_minor_versions(self): + """Equal major.minor versions should not be considered newer""" + self.assertFalse( + is_newer_version('4.9.x-100-ghash', '4.9.0'), + "Same version should not be considered newer" + ) + + def test_major_version_difference(self): + """Higher major version should be considered newer""" + self.assertTrue( + is_newer_version('4.11.x-94-ghash', '5.0.0'), + "5.0 should be considered newer than 4.11" + ) + + def test_major_version_not_newer(self): + """Lower major version should not be considered newer""" + self.assertFalse( + is_newer_version('4.11.x-94-ghash', '3.74.9'), + "3.74 should not be considered newer than 4.11" + ) + + def test_release_candidate_version(self): + """Test with release candidate versions""" + self.assertTrue( + is_newer_version('4.11.1-rc.2', '4.12.0'), + "4.12 should be considered newer than 4.11.1-rc.2" + ) + + def test_release_candidate_not_newer(self): + """Test with release candidate versions - not newer""" + self.assertFalse( + is_newer_version('4.11.1-rc.2', '4.10.5'), + "4.10 should not be considered newer than 4.11.1-rc.2" + ) + + def test_patch_version_differences(self): + """Test that patch versions are handled correctly""" + # 4.11.2 should not consider 4.11.1 as newer + self.assertFalse( + is_newer_version('4.11.2-10-ghash', '4.11.1'), + "4.11.1 should not be considered newer than 4.11.2" + ) + + def test_three_digit_minor_version(self): + """Test edge case with hypothetical three-digit minor versions""" + # If we ever reach 4.100, this should work + self.assertFalse( + is_newer_version('4.100.x-1-ghash', '4.99.0'), + "4.99 should not be considered newer than 4.100" + ) + + def test_product_version_with_different_patch_levels(self): + """Test comparison with different patch versions""" + self.assertFalse( + is_newer_version('4.9.x-100-ghash', '4.9.10'), + "4.9.10 should not be considered newer than 4.9" + ) + + def test_plain_release_tag_patch_newer(self): + """Plain release tag: newer patch should be detected""" + self.assertTrue( + is_newer_version('4.11.1', '4.11.5'), + "4.11.5 should be considered newer than plain release 4.11.1" + ) + + def test_plain_release_tag_patch_not_newer(self): + """Plain release tag: older patch should not be considered newer""" + self.assertFalse( + is_newer_version('4.11.5', '4.11.1'), + "4.11.1 should not be considered newer than plain release 4.11.5" + ) + + def test_plain_release_tag_same_version(self): + """Plain release tag: same version is not newer""" + self.assertFalse( + is_newer_version('4.11.1', '4.11.1'), + "Same plain release version should not be considered newer" + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/.tekton/basic-component-pipeline.yaml b/.tekton/basic-component-pipeline.yaml index d44396956e29e..642db21c6996b 100644 --- a/.tekton/basic-component-pipeline.yaml +++ b/.tekton/basic-component-pipeline.yaml @@ -21,21 +21,7 @@ spec: - name: name value: slack-webhook-notification - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:69945a30c11387a766e3d0ae33991b68e865a290c09da1fea44f193d358926ba - - name: kind - value: task - resolver: bundles - - - name: show-sbom - params: - - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - taskRef: - params: - - name: name - value: show-sbom - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:beb0616db051952b4b861dd8c3e00fa1c0eccbd926feddf71194d3bb3ace9ce7 + value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:02842e9c752056c47a409f69ed462cfd38cbd6b15facb796564a610b549222ea - name: kind value: task resolver: bundles @@ -49,7 +35,7 @@ spec: - name: name value: post-bigquery-metrics - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -77,10 +63,6 @@ spec: description: Path to the Dockerfile inside the context specified by parameter path-context name: dockerfile type: string - - default: "false" - description: Force rebuild image - name: rebuild - type: string - default: "false" description: Skip checks against built image name: skip-checks @@ -135,6 +117,26 @@ spec: default: 'false' description: Enable cache proxy configuration type: string + - name: sast-target-dirs + type: string + default: . + description: Target directories to scan with SAST tools. Multiple values should be separated with commas. + - name: enable-package-registry-proxy + default: 'true' + description: Use the package registry proxy when prefetching dependencies + type: string + - name: source-date-epoch + type: string + default: '' + description: Sets the image created time and the SOURCE_DATE_EPOCH build argument. On its own, it does not change file timestamps inside the layers (set rewrite-timestamp to "true" for that). Leave empty to keep the actual build time. + - name: rewrite-timestamp + type: string + default: 'false' + description: When "true", clamp file modification times in the image layers to at most source-date-epoch. Does nothing unless source-date-epoch is set. + - name: omit-history + type: string + default: 'false' + description: When "true", omit the build history (history timestamps, layer metadata, etc.) from the resulting image. results: - description: "" name: IMAGE_URL @@ -159,15 +161,6 @@ spec: - name: init params: - - name: image-url - # We can't provide a StackRox-style tag because it is not known at this time (requires cloning source, etc.) - # As a workaround, we still provide a unique tag that's based on a revision in order for this task to comply with - # its expected input. We later actually add this tag on a built image with the apply-index-image-tag task. - value: $(params.output-image-repo):konflux-$(params.revision) - - name: rebuild - value: $(params.rebuild) - - name: skip-checks - value: $(params.skip-checks) - name: enable-cache-proxy value: $(params.enable-cache-proxy) taskRef: @@ -175,7 +168,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:b349d24cb896573695802d6913d311640b44675ec082b3ad167721946a6a0a71 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4.3@sha256:7854d7babe4136e53e337f6433d952532314433b48bb62b422ddc02700934d85 - name: kind value: task resolver: bundles @@ -199,14 +192,10 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:56f65a16d3d0485c64ad85af2c1f3e9b0bb4d02d63f2fd0ebb9498d219ca723d + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] workspaces: - name: basic-auth workspace: git-auth @@ -222,7 +211,7 @@ spec: - name: name value: determine-image-expiration - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -238,7 +227,7 @@ spec: - name: name value: determine-image-tag - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -253,12 +242,16 @@ spec: value: $(params.output-image-repo):konflux-$(params.revision).prefetch - name: ociArtifactExpiresAfter value: $(params.oci-artifact-expires-after) + - name: ACTIVATION_KEY + value: subscription-manager-activation-key-prod + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) taskRef: params: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:8eac535f436874ca27c70434ff356ba941d7b649c0e861bca6f4e9fc5e215478 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.10.3@sha256:0c386f2c26b46b978280c5c54caac26bcd524975f8f7cc06bb080ba00eae3ad4 - name: kind value: task resolver: bundles @@ -306,31 +299,29 @@ spec: value: $(tasks.init.results.http-proxy) - name: NO_PROXY value: $(tasks.init.results.no-proxy) + - name: SOURCE_DATE_EPOCH + value: $(params.source-date-epoch) + - name: REWRITE_TIMESTAMP + value: $(params.rewrite-timestamp) + - name: OMIT_HISTORY + value: $(params.omit-history) taskRef: params: - name: name value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.7@sha256:fe66734cde6713f530b19e222680acc5e0d6a533866429659ea8561b746d9ce7 + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.12.2@sha256:a68dce8ed79be8d9e10d98bd2b56895c27fc34f1ed7d67cbc9ec0a91d64cf641 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] - name: build-image-index params: - name: IMAGE value: $(params.output-image-repo):$(tasks.determine-image-tag.results.IMAGE_TAG) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - name: IMAGES value: - $(tasks.build-images.results.IMAGE_REF[*]) - - name: IMAGE_EXPIRES_AFTER - value: $(tasks.determine-image-expiration.results.IMAGE_EXPIRES_AFTER) - name: BUILDAH_FORMAT value: $(params.buildah-format) taskRef: @@ -338,39 +329,12 @@ spec: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.2@sha256:985d1efe861b02524a7679ecd855624b3d4e3a2e835b6f8a97ec7d135898ec0b + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.4@sha256:654446d8ec028b20ed6ada8a96486ff16692a25192755aa04a99cea8e0afb7e2 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] runAfter: [build-images] - - name: apply-index-image-tag - params: - - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE_DIGEST - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: ADDITIONAL_TAGS - value: - - konflux-$(params.revision) - taskRef: - params: - - name: name - value: apply-tags - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.2@sha256:c89cd10b2a3f4c43789c5f06ef2b86f528b28f156c20af5e751fa8c0facd457d - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] - - name: build-source-image params: - name: BINARY_IMAGE @@ -386,14 +350,11 @@ spec: - name: name value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:c35ba219390d77a48ee19347e5ee8d13e5c23e3984299e02291d6da1ed8a986c + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3.2@sha256:45091b5022754d452b4de60c7d7f4c6e03626efff55f96e9af937400fe3506eb - name: kind value: task resolver: bundles when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] - input: $(params.build-source-image) operator: in values: ["true"] @@ -409,7 +370,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:808fe09bb5b8503de569de097ae5dd619a7488110f79e8e215e69862ee3fce6d + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:0ccc688a77e9b7b0b8973c132a1e840844137e77f887be4a0bec8893b0776872 - name: kind value: task resolver: bundles @@ -418,12 +379,27 @@ spec: operator: in values: ["false"] - - name: clair-scan - matrix: + - name: roxctl-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: params: - - name: image-platform - value: - - $(params.build-platforms) + - name: name + value: roxctl-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:97e2b2cdca9110fdc8a93ba585a1a1a743f989f2fd85f4073f6e5ea9ad2ce828 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: ["false"] + + - name: clair-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -434,7 +410,7 @@ spec: - name: name value: clair-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:7c2a32de9021f16f6e8df08a55f539f12e00ea4d96f6fb37f9ea04167032c61f + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.4.1@sha256:219964897ac4d5a9473ec297a1b39ae078a34b76b45d04c4672bd569cc295c29 - name: kind value: task resolver: bundles @@ -457,7 +433,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:04f75593558f79a27da2336400bc63d460bf0c5669e3c13f40ee2fb650b1ad1e + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:fa53ef450ba538addafee1211ff6ac15b55865ce652fec15ac99b554680cb642 - name: kind value: task resolver: bundles @@ -476,12 +452,14 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d44336d7bcbd1f7cedee639357a493bd1f661e2859e49e11a34644bdf6819c4e + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d00c884c78489a52dc638e6ced0a01429670d0754f5b1305ad576709cf660312 - name: kind value: task resolver: bundles @@ -500,12 +478,14 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.3@sha256:e5a8d3e8e7be7246a1460385b95c084ea6e8fe7520d40fe4389deb90f1bf5176 + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:15d654c9576b9c4ee1f78b09fce541acf9afe33f1c2363fd6219a79dbf62c95d - name: kind value: task resolver: bundles @@ -524,12 +504,14 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:0eca130f289a1a1069a1b92943479f79aa7324e4e68d6396fd777ccd97058f50 + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5@sha256:17b9587eba53fabaab35061275306bfe1f99661d55e14e22f62321fa7cb5ff3e - name: kind value: task resolver: bundles @@ -554,7 +536,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:f3d2d179cddcc07d0228d9f52959a233037a3afa2619d0a8b2effbb467db80c3 + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.3@sha256:9df272d2180bd67e6a3e04d5cd30ffa7ba27af4e599a1922f7069f5c89888e5b - name: kind value: task resolver: bundles @@ -574,7 +556,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:0b10508c82ccb0f5a06a66ce7af56e9bfd40651ddefdf0f499988e897771ee28 + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:d9df5d325f81a9fa132cb597c465aa9210b9b0fa3716b85b32be98d92c9c881a - name: kind value: task resolver: bundles @@ -600,7 +582,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:08bba4a659ecd48f871bef00b80af58954e5a09fcbb28a1783ddd640c4f6535e + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3.1@sha256:4c567d11296850230fe32e76d129834d3bf337a0cae9989b64279b7cfaa4e735 - name: kind value: task resolver: bundles diff --git a/.tekton/central-db-build.yaml b/.tekton/central-db-build.yaml index 9076f0e1063c6..aebf7613820b1 100644 --- a/.tekton/central-db-build.yaml +++ b/.tekton/central-db-build.yaml @@ -8,6 +8,7 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest central-db-on-push" # TODO(ROX-21073): re-enable for all PR branches pipelinesascode.tekton.dev/on-cel-expression: | ( @@ -60,21 +61,31 @@ spec: - name: extra-labels value: # X.Y in the cpe label must be adjusted for every version stream. - - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el8" + - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el9" pipelineRef: name: basic-component-pipeline taskRunSpecs: + # The following are overrides to default step resources to prevent occasional or deterministic OOM kills. - pipelineTaskName: build-source-image stepSpecs: - # Bump memory for source image build because the one with defaults tends to get sometimes OOM-killed. - name: build computeResources: limits: + cpu: 1 memory: 3Gi requests: + cpu: 1 memory: 3Gi + - pipelineTaskName: rpms-signature-scan + stepSpecs: + - name: rpms-signature-scan + computeResources: + limits: + memory: 512Mi + requests: + memory: 512Mi taskRunTemplate: serviceAccountName: build-pipeline-central-db diff --git a/.tekton/checks.yaml b/.tekton/checks.yaml new file mode 100644 index 0000000000000..5e7ce9e0f1912 --- /dev/null +++ b/.tekton/checks.yaml @@ -0,0 +1,152 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun + +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/stackrox/stackrox?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest checks" + pipelinesascode.tekton.dev/on-cel-expression: | + (event == "push" && target_branch.matches("^(master|release-.*|refs/tags/.*)$")) || + (event == "pull_request" && body.action != "ready_for_review") + labels: + appstudio.openshift.io/application: acs + name: checks + namespace: rh-acs-tenant + +spec: + + params: + + taskRunTemplate: + serviceAccountName: build-pipeline-operator-bundle + + timeouts: + tasks: 10m + finally: 2m + pipeline: 12m + + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' + + pipelineSpec: + + finally: + - name: slack-notification + params: + - name: message + value: ':x: `{{event_type}}` pipeline for (revision <$(params.git-url)/commit/$(params.revision)|$(params.revision)>) has failed.' + - name: key-name + value: 'acs-konflux-notifications' + when: + - input: $(tasks.status) + operator: in + values: ["Failed"] + taskRef: + params: + - name: name + value: slack-webhook-notification + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:02842e9c752056c47a409f69ed462cfd38cbd6b15facb796564a610b549222ea + - name: kind + value: task + resolver: bundles + + - name: post-metric-end + params: + - name: AGGREGATE_TASKS_STATUS + value: $(tasks.status) + taskRef: &post-bigquery-metrics-ref + params: + - name: name + value: post-bigquery-metrics + - name: bundle + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 + - name: kind + value: task + resolver: bundles + + params: + - default: "{{source_url}}" + description: Source Repository URL + name: git-url + type: string + - default: "{{revision}}" + description: Revision of the Source Repository + name: revision + type: string + - default: quay.io/rhacs-eng/konflux-checks + description: Output Image Repository (for OCI artifacts) + name: output-image-repo + type: string + - default: "1d" + description: This sets the expiration time for intermediate OCI artifacts produced and used during builds after which they can be garbage collected. + name: oci-artifact-expires-after + type: string + + + workspaces: + - name: git-auth + + tasks: + + - name: post-metric-start + taskRef: *post-bigquery-metrics-ref + + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: depth + value: "1" + - name: fetchTags + value: "false" + - name: ociStorage + value: $(params.output-image-repo):konflux-$(params.revision).git + - name: ociArtifactExpiresAfter + value: $(params.oci-artifact-expires-after) + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 + - name: kind + value: task + resolver: bundles + workspaces: + - name: basic-auth + workspace: git-auth + + - name: check-go-builder-version + params: + - name: IMAGE + value: $(params.output-image-repo):$(params.revision)-check-go-builder-version + - name: DOCKERFILE + value: .konflux/konflux.check-go-builder-version.Dockerfile + - name: CONTEXT + value: . + - name: IMAGE_EXPIRES_AFTER + value: $(params.oci-artifact-expires-after) + - name: COMMIT_SHA + value: $(params.revision) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: HERMETIC + value: "true" + taskRef: + params: + - name: name + value: buildah-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.12.2@sha256:e1bbe61a3a4bc543be9b5f6fc4174acaa2dd08fb5f7fe4d9db9f3193a7e1b03c + - name: kind + value: task + resolver: bundles diff --git a/.tekton/create-custom-snapshot.yaml b/.tekton/create-custom-snapshot.yaml index 4e865b0fd322b..b96b14d7b28b3 100644 --- a/.tekton/create-custom-snapshot.yaml +++ b/.tekton/create-custom-snapshot.yaml @@ -8,17 +8,14 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest create-custom-snapshot" + # Unlike other pipelines, this one is enabled for all PRs. See the description of determine-actual-build task. + # The condition for a label is to make sure the konflux-build label addition activates this pipeline, same as others. # TODO(ROX-21073): re-enable for all PR branches pipelinesascode.tekton.dev/on-cel-expression: | - ( - event == "push" && target_branch.matches("^(master|release-.*|refs/tags/.*)$") - ) || ( - event == "pull_request" && ( - target_branch.startsWith("release-") || - source_branch.matches("(konflux|renovate|appstudio|rhtap)") || - (has(body.pull_request) && has(body.pull_request.labels) && body.pull_request.labels.exists(l, l.name == "konflux-build")) - ) && body.action != "ready_for_review" - ) + (event == "push" && target_branch.matches("^(master|release-.*|refs/tags/.*)$")) || + (event == "pull_request" && body.action != "ready_for_review") || + (has(body.pull_request) && has(body.pull_request.labels) && body.pull_request.labels.exists(l, l.name == "konflux-build")) # The empty `on-label` annotation is a workaround to make sure the pipeline gets triggered when the label gets first # added to the PR. See the Slack tread linked from ROX-30580. pipelinesascode.tekton.dev/on-label: "[]" @@ -64,7 +61,7 @@ spec: - name: name value: slack-webhook-notification - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:69945a30c11387a766e3d0ae33991b68e865a290c09da1fea44f193d358926ba + value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:02842e9c752056c47a409f69ed462cfd38cbd6b15facb796564a610b549222ea - name: kind value: task resolver: bundles @@ -78,7 +75,7 @@ spec: - name: name value: post-bigquery-metrics - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -123,6 +120,68 @@ spec: - name: post-metric-start taskRef: *post-bigquery-metrics-ref + - name: determine-actual-build + taskSpec: + description: | + Determines whether the actual tasks in the pipeline should be skipped or executed. + Tasks will be skipped on PRs where Konflux builds aren't enabled (TODO: ROX-21073), see + `pipelinesascode.tekton.dev/on-cel-expression` annotation in `*-build.yaml` files. + This all is needed because we want to always run this `create-custom-snapshot` pipeline and make it part of + required GitHub checks in branch protection rules. + results: + - name: SHOULD_REALLY_WORK + steps: + - name: determine + image: registry.redhat.io/ubi9-minimal:latest@sha256:c4c285d42941ebb2bd4da878479008e1e4536bb599aad1dbe568f89de9053661 + imagePullPolicy: IfNotPresent + # The script expresses the same condition as pipelinesascode.tekton.dev/on-cel-expression in other pipelines + # but in a more verbose way because, as it turns out, both syntax and fields of CEL expressions are slightly + # different. + script: | + set -euo pipefail + + function assert_boolean() { + if [[ "${2:-}" != "" ]]; then + echo "${2}" + fi + echo "${1}: ${!1}" + if [[ "${!1}" != "true" && "${!1}" != "false" ]]; then + >&2 echo "Error: not a boolean value: ${!1}" + exit 2 + fi + } + + is_pull_request="true" + if [[ '{{ cel: target_branch }}' == '{{ cel: source_branch }}' ]]; then + is_pull_request="false" + fi + assert_boolean is_pull_request + + is_matching_target_branch='{{ cel: target_branch.startsWith("release-") }}' + assert_boolean is_matching_target_branch "target_branch: {{ target_branch }}" + + is_matching_source_branch='{{ cel: source_branch.matches("(konflux|renovate|appstudio|rhtap)") }}' + assert_boolean is_matching_source_branch "source_branch: {{ source_branch }}" + + # The label will be findable this way for pull request events. + has_konflux_build_label_as_pr='{{ cel: has(body.pull_request) && has(body.pull_request.labels) && body.pull_request.labels.exists(l, l.name == "konflux-build") }}' + assert_boolean has_konflux_build_label_as_pr + + # The label will be findable this way on PRs for /test comments and alike. + has_konflux_build_label_as_issue='{{ cel: has(body.issue) && has(body.issue.labels) && body.issue.labels.exists(l, l.name == "konflux-build") }}' + assert_boolean has_konflux_build_label_as_issue + + result="false" + if [[ "${is_pull_request}" != "true" || \ + "${is_matching_target_branch}" == "true" || \ + "${is_matching_source_branch}" == "true" || \ + "${has_konflux_build_label_as_pr}" == "true" || \ + "${has_konflux_build_label_as_issue}" == "true" ]]; then + result="true" + fi + echo -n "The result is: " + echo -n "${result}" | tee "$(results.SHOULD_REALLY_WORK.path)" + - name: clone-repository params: - name: url @@ -142,13 +201,17 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:56f65a16d3d0485c64ad85af2c1f3e9b0bb4d02d63f2fd0ebb9498d219ca723d + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 - name: kind value: task resolver: bundles workspaces: - name: basic-auth workspace: git-auth + when: + - input: $(tasks.determine-actual-build.results.SHOULD_REALLY_WORK) + operator: in + values: [ "true" ] - name: determine-image-tag params: @@ -161,10 +224,14 @@ spec: - name: name value: determine-image-tag - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles + when: + - input: $(tasks.determine-actual-build.results.SHOULD_REALLY_WORK) + operator: in + values: [ "true" ] - name: wait-for-bundle-image params: @@ -175,12 +242,16 @@ spec: - name: name value: wait-for-image - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles # The timemout should be kept in sync with the pipeline timeout in the operator-bundle-build.yaml timeout: 3h25m + when: + - input: $(tasks.determine-actual-build.results.SHOULD_REALLY_WORK) + operator: in + values: [ "true" ] - name: create-acs-style-snapshot params: @@ -194,17 +265,22 @@ spec: value: | [ { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-central-db-rhel8", + "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-central-db-rhel9", "internalRepo": "quay.io/rhacs-eng/release-central-db", "component": "central-db" }, { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-collector-rhel8", + "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-collector-rhel9", "internalRepo": "quay.io/rhacs-eng/release-collector", "component": "collector" }, { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8", + "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-fact-rhel9", + "internalRepo": "quay.io/rhacs-eng/release-fact", + "component": "fact" + }, + { + "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-main-rhel9", "internalRepo": "quay.io/rhacs-eng/release-main", "component": "main" }, @@ -214,42 +290,22 @@ spec: "component": "operator-bundle" }, { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-rhel8-operator", + "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-rhel9-operator", "internalRepo": "quay.io/rhacs-eng/release-operator", "component": "operator" }, { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-roxctl-rhel8", + "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-roxctl-rhel9", "internalRepo": "quay.io/rhacs-eng/release-roxctl", "component": "roxctl" }, { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-scanner-db-rhel8", - "internalRepo": "quay.io/rhacs-eng/release-scanner-db", - "component": "scanner-db" - }, - { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-scanner-db-slim-rhel8", - "internalRepo": "quay.io/rhacs-eng/release-scanner-db-slim", - "component": "scanner-db-slim" - }, - { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-scanner-rhel8", - "internalRepo": "quay.io/rhacs-eng/release-scanner", - "component": "scanner" - }, - { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-scanner-slim-rhel8", - "internalRepo": "quay.io/rhacs-eng/release-scanner-slim", - "component": "scanner-slim" - }, - { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-db-rhel8", + "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-db-rhel9", "internalRepo": "quay.io/rhacs-eng/release-scanner-v4-db", "component": "scanner-v4-db" }, { - "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-rhel8", + "externalRepo": "registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-rhel9", "internalRepo": "quay.io/rhacs-eng/release-scanner-v4", "component": "scanner-v4" } @@ -259,7 +315,11 @@ spec: - name: name value: create-snapshot-from-bundle - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles + when: + - input: $(tasks.determine-actual-build.results.SHOULD_REALLY_WORK) + operator: in + values: [ "true" ] diff --git a/.tekton/images-mirror-set.yaml b/.tekton/images-mirror-set.yaml index 3cc93ea45b2df..f8f22c411a424 100644 --- a/.tekton/images-mirror-set.yaml +++ b/.tekton/images-mirror-set.yaml @@ -7,40 +7,31 @@ spec: - source: registry.redhat.io/advanced-cluster-security/rhacs-operator-bundle mirrors: - quay.io/rhacs-eng/release-operator-bundle - - source: registry.redhat.io/advanced-cluster-security/rhacs-central-db-rhel8 + - source: registry.redhat.io/advanced-cluster-security/rhacs-central-db-rhel9 mirrors: - quay.io/rhacs-eng/release-central-db - - source: registry.redhat.io/advanced-cluster-security/rhacs-collector-rhel8 + - source: registry.redhat.io/advanced-cluster-security/rhacs-collector-rhel9 mirrors: - quay.io/rhacs-eng/release-collector - - source: registry.redhat.io/advanced-cluster-security/rhacs-collector-slim-rhel8 + - source: registry.redhat.io/advanced-cluster-security/rhacs-collector-slim-rhel9 mirrors: - quay.io/rhacs-eng/release-collector - quay.io/rhacs-eng/release-collector-slim - - source: registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8 + - source: registry.redhat.io/advanced-cluster-security/rhacs-fact-rhel9 + mirrors: + - quay.io/rhacs-eng/release-fact + - source: registry.redhat.io/advanced-cluster-security/rhacs-main-rhel9 mirrors: - quay.io/rhacs-eng/release-main - - source: registry.redhat.io/advanced-cluster-security/rhacs-rhel8-operator + - source: registry.redhat.io/advanced-cluster-security/rhacs-rhel9-operator mirrors: - quay.io/rhacs-eng/release-operator - - source: registry.redhat.io/advanced-cluster-security/rhacs-roxctl-rhel8 + - source: registry.redhat.io/advanced-cluster-security/rhacs-roxctl-rhel9 mirrors: - quay.io/rhacs-eng/release-roxctl - - source: registry.redhat.io/advanced-cluster-security/rhacs-scanner-db-rhel8 - mirrors: - - quay.io/rhacs-eng/release-scanner-db - - source: registry.redhat.io/advanced-cluster-security/rhacs-scanner-db-slim-rhel8 - mirrors: - - quay.io/rhacs-eng/release-scanner-db-slim - - source: registry.redhat.io/advanced-cluster-security/rhacs-scanner-rhel8 - mirrors: - - quay.io/rhacs-eng/release-scanner - - source: registry.redhat.io/advanced-cluster-security/rhacs-scanner-slim-rhel8 - mirrors: - - quay.io/rhacs-eng/release-scanner-slim - - source: registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-db-rhel8 + - source: registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-db-rhel9 mirrors: - quay.io/rhacs-eng/release-scanner-v4-db - - source: registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-rhel8 + - source: registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-rhel9 mirrors: - quay.io/rhacs-eng/release-scanner-v4 diff --git a/.tekton/main-build.yaml b/.tekton/main-build.yaml index a59403e0cbc35..5967d48f41745 100644 --- a/.tekton/main-build.yaml +++ b/.tekton/main-build.yaml @@ -8,6 +8,7 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest main-on-push" # TODO(ROX-21073): re-enable for all PR branches pipelinesascode.tekton.dev/on-cel-expression: | ( @@ -64,42 +65,66 @@ spec: - name: extra-labels value: # X.Y in the cpe label must be adjusted for every version stream. - - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el8" + - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el9" pipelineRef: name: main-pipeline taskRunSpecs: + # The following are overrides to default step resources to prevent occasional or deterministic OOM kills. - pipelineTaskName: prefetch-dependencies + podTemplate: &go-memlimit + env: + - name: GOMEMLIMIT + value: 2500MiB stepSpecs: - # Limit CPU for Go dependencies prefetch, as it tends to be OOM-killed when it gets lots of it without OOTB limit. - name: prefetch-dependencies computeResources: + limits: + memory: 6Gi + requests: + memory: 6Gi + - name: create-trusted-artifact + computeResources: &ta-resources limits: cpu: 2 + memory: 7Gi requests: cpu: 2 - # For all sbom-syft-generate steps: - # Memory is increased for the syft command to succeed. Otherwise, there's an error like this in log and container - # exits with 137 (OOMKilled): - # /tekton/scripts/script-2-h7nll: line 7: 33 Killed syft dir:$(cat /shared/container_path) --output cyclonedx-json=/var/workdir/sbom-image.json + memory: 7Gi - pipelineTaskName: build-images + podTemplate: *go-memlimit stepSpecs: - - name: sbom-syft-generate - computeResources: - limits: - memory: 3Gi - requests: - memory: 3Gi + - name: use-trusted-artifact + computeResources: *ta-resources - pipelineTaskName: build-source-image + podTemplate: *go-memlimit stepSpecs: - # Bump memory for source image build because the one with defaults tends to get sometimes OOM-killed. + - name: use-trusted-artifact + computeResources: *ta-resources - name: build computeResources: limits: + cpu: 1 memory: 3Gi requests: + cpu: 1 memory: 3Gi + - pipelineTaskName: sast-shell-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources + - pipelineTaskName: sast-unicode-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources + - pipelineTaskName: sast-snyk-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources taskRunTemplate: serviceAccountName: build-pipeline-main diff --git a/.tekton/main-pipeline.yaml b/.tekton/main-pipeline.yaml index 223d6e9194a92..fb022d858b32c 100644 --- a/.tekton/main-pipeline.yaml +++ b/.tekton/main-pipeline.yaml @@ -21,21 +21,7 @@ spec: - name: name value: slack-webhook-notification - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:69945a30c11387a766e3d0ae33991b68e865a290c09da1fea44f193d358926ba - - name: kind - value: task - resolver: bundles - - - name: show-sbom - params: - - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - taskRef: - params: - - name: name - value: show-sbom - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:beb0616db051952b4b861dd8c3e00fa1c0eccbd926feddf71194d3bb3ace9ce7 + value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:02842e9c752056c47a409f69ed462cfd38cbd6b15facb796564a610b549222ea - name: kind value: task resolver: bundles @@ -49,7 +35,7 @@ spec: - name: name value: post-bigquery-metrics - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -77,10 +63,6 @@ spec: description: Path to the Dockerfile inside the context specified by parameter path-context name: dockerfile type: string - - default: "false" - description: Force rebuild image - name: rebuild - type: string - default: "false" description: Skip checks against built image name: skip-checks @@ -136,6 +118,26 @@ spec: default: 'false' description: Enable cache proxy configuration type: string + - name: sast-target-dirs + type: string + default: . + description: Target directories to scan with SAST tools. Multiple values should be separated with commas. + - name: enable-package-registry-proxy + default: 'true' + description: Use the package registry proxy when prefetching dependencies + type: string + - name: source-date-epoch + type: string + default: '' + description: Sets the image created time and the SOURCE_DATE_EPOCH build argument. On its own, it does not change file timestamps inside the layers (set rewrite-timestamp to "true" for that). Leave empty to keep the actual build time. + - name: rewrite-timestamp + type: string + default: 'false' + description: When "true", clamp file modification times in the image layers to at most source-date-epoch. Does nothing unless source-date-epoch is set. + - name: omit-history + type: string + default: 'false' + description: When "true", omit the build history (history timestamps, layer metadata, etc.) from the resulting image. results: - description: "" name: IMAGE_URL @@ -160,15 +162,6 @@ spec: - name: init params: - - name: image-url - # We can't provide a StackRox-style tag because it is not known at this time (requires cloning source, etc.) - # As a workaround, we still provide a unique tag that's based on a revision in order for this task to comply with - # its expected input. We later actually add this tag on a built image with the apply-index-image-tag task. - value: $(params.output-image-repo):konflux-$(params.revision) - - name: rebuild - value: $(params.rebuild) - - name: skip-checks - value: $(params.skip-checks) - name: enable-cache-proxy value: $(params.enable-cache-proxy) taskRef: @@ -176,7 +169,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:b349d24cb896573695802d6913d311640b44675ec082b3ad167721946a6a0a71 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4.3@sha256:7854d7babe4136e53e337f6433d952532314433b48bb62b422ddc02700934d85 - name: kind value: task resolver: bundles @@ -200,14 +193,10 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:56f65a16d3d0485c64ad85af2c1f3e9b0bb4d02d63f2fd0ebb9498d219ca723d + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] workspaces: - name: basic-auth workspace: git-auth @@ -223,7 +212,7 @@ spec: - name: name value: determine-image-expiration - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -239,7 +228,7 @@ spec: - name: name value: determine-image-tag - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -259,7 +248,7 @@ spec: - name: name value: fetch-external-networks - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -276,15 +265,14 @@ spec: value: $(params.oci-artifact-expires-after) - name: ACTIVATION_KEY value: subscription-manager-activation-key-prod - # Required for the RPM prefetching support. - - name: dev-package-managers - value: "true" + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) taskRef: params: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:8eac535f436874ca27c70434ff356ba941d7b649c0e861bca6f4e9fc5e215478 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.10.3@sha256:0c386f2c26b46b978280c5c54caac26bcd524975f8f7cc06bb080ba00eae3ad4 - name: kind value: task resolver: bundles @@ -332,32 +320,30 @@ spec: value: $(tasks.init.results.http-proxy) - name: NO_PROXY value: $(tasks.init.results.no-proxy) + - name: SOURCE_DATE_EPOCH + value: $(params.source-date-epoch) + - name: REWRITE_TIMESTAMP + value: $(params.rewrite-timestamp) + - name: OMIT_HISTORY + value: $(params.omit-history) taskRef: params: - name: name value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.7@sha256:fe66734cde6713f530b19e222680acc5e0d6a533866429659ea8561b746d9ce7 + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.12.2@sha256:a68dce8ed79be8d9e10d98bd2b56895c27fc34f1ed7d67cbc9ec0a91d64cf641 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] timeout: 2h0m0s - name: build-image-index params: - name: IMAGE value: $(params.output-image-repo):$(tasks.determine-image-tag.results.IMAGE_TAG) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - name: IMAGES value: - $(tasks.build-images.results.IMAGE_REF[*]) - - name: IMAGE_EXPIRES_AFTER - value: $(tasks.determine-image-expiration.results.IMAGE_EXPIRES_AFTER) - name: BUILDAH_FORMAT value: $(params.buildah-format) taskRef: @@ -365,39 +351,12 @@ spec: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.2@sha256:985d1efe861b02524a7679ecd855624b3d4e3a2e835b6f8a97ec7d135898ec0b + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.4@sha256:654446d8ec028b20ed6ada8a96486ff16692a25192755aa04a99cea8e0afb7e2 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] runAfter: [build-images] - - name: apply-index-image-tag - params: - - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE_DIGEST - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: ADDITIONAL_TAGS - value: - - konflux-$(params.revision) - taskRef: - params: - - name: name - value: apply-tags - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.2@sha256:c89cd10b2a3f4c43789c5f06ef2b86f528b28f156c20af5e751fa8c0facd457d - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] - - name: build-source-image params: - name: BINARY_IMAGE @@ -413,14 +372,11 @@ spec: - name: name value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:c35ba219390d77a48ee19347e5ee8d13e5c23e3984299e02291d6da1ed8a986c + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3.2@sha256:45091b5022754d452b4de60c7d7f4c6e03626efff55f96e9af937400fe3506eb - name: kind value: task resolver: bundles when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] - input: $(params.build-source-image) operator: in values: ["true"] @@ -436,7 +392,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:808fe09bb5b8503de569de097ae5dd619a7488110f79e8e215e69862ee3fce6d + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:0ccc688a77e9b7b0b8973c132a1e840844137e77f887be4a0bec8893b0776872 - name: kind value: task resolver: bundles @@ -445,12 +401,27 @@ spec: operator: in values: ["false"] - - name: clair-scan - matrix: + - name: roxctl-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: params: - - name: image-platform - value: - - $(params.build-platforms) + - name: name + value: roxctl-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:97e2b2cdca9110fdc8a93ba585a1a1a743f989f2fd85f4073f6e5ea9ad2ce828 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: ["false"] + + - name: clair-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -461,7 +432,7 @@ spec: - name: name value: clair-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:7c2a32de9021f16f6e8df08a55f539f12e00ea4d96f6fb37f9ea04167032c61f + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.4.1@sha256:219964897ac4d5a9473ec297a1b39ae078a34b76b45d04c4672bd569cc295c29 - name: kind value: task resolver: bundles @@ -484,7 +455,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:04f75593558f79a27da2336400bc63d460bf0c5669e3c13f40ee2fb650b1ad1e + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:fa53ef450ba538addafee1211ff6ac15b55865ce652fec15ac99b554680cb642 - name: kind value: task resolver: bundles @@ -503,12 +474,14 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d44336d7bcbd1f7cedee639357a493bd1f661e2859e49e11a34644bdf6819c4e + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d00c884c78489a52dc638e6ced0a01429670d0754f5b1305ad576709cf660312 - name: kind value: task resolver: bundles @@ -527,12 +500,14 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.3@sha256:e5a8d3e8e7be7246a1460385b95c084ea6e8fe7520d40fe4389deb90f1bf5176 + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:15d654c9576b9c4ee1f78b09fce541acf9afe33f1c2363fd6219a79dbf62c95d - name: kind value: task resolver: bundles @@ -551,12 +526,14 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:0eca130f289a1a1069a1b92943479f79aa7324e4e68d6396fd777ccd97058f50 + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5@sha256:17b9587eba53fabaab35061275306bfe1f99661d55e14e22f62321fa7cb5ff3e - name: kind value: task resolver: bundles @@ -581,7 +558,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:f3d2d179cddcc07d0228d9f52959a233037a3afa2619d0a8b2effbb467db80c3 + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.3@sha256:9df272d2180bd67e6a3e04d5cd30ffa7ba27af4e599a1922f7069f5c89888e5b - name: kind value: task resolver: bundles @@ -601,7 +578,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:0b10508c82ccb0f5a06a66ce7af56e9bfd40651ddefdf0f499988e897771ee28 + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:d9df5d325f81a9fa132cb597c465aa9210b9b0fa3716b85b32be98d92c9c881a - name: kind value: task resolver: bundles @@ -627,7 +604,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:08bba4a659ecd48f871bef00b80af58954e5a09fcbb28a1783ddd640c4f6535e + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3.1@sha256:4c567d11296850230fe32e76d129834d3bf337a0cae9989b64279b7cfaa4e735 - name: kind value: task resolver: bundles diff --git a/.tekton/operator-build.yaml b/.tekton/operator-build.yaml index c7f3b2df2c97e..c0e836893797e 100644 --- a/.tekton/operator-build.yaml +++ b/.tekton/operator-build.yaml @@ -8,6 +8,7 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest operator-on-push" # TODO(ROX-21073): re-enable for all PR branches pipelinesascode.tekton.dev/on-cel-expression: | ( @@ -49,7 +50,11 @@ spec: - name: hermetic value: "true" - name: prefetch-input - value: '{"type": "gomod", "path": "."}' + value: | + [ + { "type": "gomod", "path": "." }, + { "type": "rpm", "path": "." } + ] - name: build-source-image value: 'true' - name: clone-depth @@ -59,30 +64,66 @@ spec: - name: extra-labels value: # X.Y in the cpe label must be adjusted for every version stream. - - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el8" + - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el9" pipelineRef: name: basic-component-pipeline taskRunSpecs: + # The following are overrides to default step resources to prevent occasional or deterministic OOM kills. - pipelineTaskName: prefetch-dependencies + podTemplate: &go-memlimit + env: + - name: GOMEMLIMIT + value: 2500MiB stepSpecs: - # Limit CPU for Go dependencies prefetch, as it tends to be OOM-killed when it gets lots of it without OOTB limit. - name: prefetch-dependencies computeResources: + limits: + memory: 6Gi + requests: + memory: 6Gi + - name: create-trusted-artifact + computeResources: &ta-resources limits: cpu: 2 + memory: 7Gi requests: cpu: 2 + memory: 7Gi + - pipelineTaskName: build-images + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources - pipelineTaskName: build-source-image + podTemplate: *go-memlimit stepSpecs: - # Bump memory for source image build because the one with defaults tends to get sometimes OOM-killed. + - name: use-trusted-artifact + computeResources: *ta-resources - name: build computeResources: limits: + cpu: 1 memory: 3Gi requests: + cpu: 1 memory: 3Gi + - pipelineTaskName: sast-shell-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources + - pipelineTaskName: sast-unicode-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources + - pipelineTaskName: sast-snyk-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources taskRunTemplate: serviceAccountName: build-pipeline-operator diff --git a/.tekton/operator-bundle-build.yaml b/.tekton/operator-bundle-build.yaml index 08f39fc7e2c11..b5fbb29ba97a4 100644 --- a/.tekton/operator-bundle-build.yaml +++ b/.tekton/operator-bundle-build.yaml @@ -8,6 +8,7 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest operator-bundle-on-push" # TODO(ROX-21073): re-enable for all PR branches pipelinesascode.tekton.dev/on-cel-expression: | ( @@ -49,7 +50,7 @@ spec: - name: prefetch-input value: | [ - { "type": "pip", "path": "operator/bundle_helpers" } + { "type": "gomod", "path": "." } ] - name: build-source-image value: 'true' @@ -60,21 +61,66 @@ spec: - name: extra-labels value: # X.Y in the cpe label must be adjusted for every version stream. - - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el8" + - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el9" pipelineRef: name: operator-bundle-pipeline taskRunSpecs: + # The following are overrides to default step resources to prevent occasional or deterministic OOM kills. + - pipelineTaskName: prefetch-dependencies + podTemplate: &go-memlimit + env: + - name: GOMEMLIMIT + value: 2500MiB + stepSpecs: + - name: prefetch-dependencies + computeResources: + limits: + memory: 6Gi + requests: + memory: 6Gi + - name: create-trusted-artifact + computeResources: &ta-resources + limits: + cpu: 2 + memory: 7Gi + requests: + cpu: 2 + memory: 7Gi + - pipelineTaskName: build-container + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources - pipelineTaskName: build-source-image + podTemplate: *go-memlimit stepSpecs: - # Bump memory for source image build because the one with defaults tends to get sometimes OOM-killed. + - name: use-trusted-artifact + computeResources: *ta-resources - name: build computeResources: limits: + cpu: 1 memory: 3Gi requests: + cpu: 1 memory: 3Gi + - pipelineTaskName: sast-shell-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources + - pipelineTaskName: sast-unicode-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources + - pipelineTaskName: sast-snyk-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources taskRunTemplate: serviceAccountName: build-pipeline-operator-bundle diff --git a/.tekton/operator-bundle-pipeline.yaml b/.tekton/operator-bundle-pipeline.yaml index 228a65276b860..f7da2ba4be86b 100644 --- a/.tekton/operator-bundle-pipeline.yaml +++ b/.tekton/operator-bundle-pipeline.yaml @@ -21,21 +21,7 @@ spec: - name: name value: slack-webhook-notification - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:69945a30c11387a766e3d0ae33991b68e865a290c09da1fea44f193d358926ba - - name: kind - value: task - resolver: bundles - - - name: show-sbom - params: - - name: IMAGE_URL - value: $(tasks.build-container.results.IMAGE_URL) - taskRef: - params: - - name: name - value: show-sbom - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:beb0616db051952b4b861dd8c3e00fa1c0eccbd926feddf71194d3bb3ace9ce7 + value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:02842e9c752056c47a409f69ed462cfd38cbd6b15facb796564a610b549222ea - name: kind value: task resolver: bundles @@ -49,7 +35,7 @@ spec: - name: name value: post-bigquery-metrics - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -77,10 +63,6 @@ spec: description: Path to the Dockerfile inside the context specified by parameter path-context name: dockerfile type: string - - default: "false" - description: Force rebuild image - name: rebuild - type: string - default: "false" description: Skip checks against built image name: skip-checks @@ -127,7 +109,7 @@ spec: - name: operator-image-catalog-repo description: Repository within the Red Hat Container Catalog where the operator image is pushed to during the release. type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-rhel8-operator" + default: "registry.redhat.io/advanced-cluster-security/rhacs-rhel9-operator" - name: main-image-build-repo description: Repository where the (unreleased) main image is pushed by its build pipeline. @@ -136,43 +118,7 @@ spec: - name: main-image-catalog-repo description: Repository within the Red Hat Container Catalog where the main image is pushed to during the release. type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-main-rhel8" - - - name: scanner-image-build-repo - description: Repository where the (unreleased) Scanner V2 image is pushed by its build pipeline. - type: string - default: "quay.io/rhacs-eng/release-scanner" - - name: scanner-image-catalog-repo - description: Repository within the Red Hat Container Catalog where the Scanner V2 image is pushed to during the release. - type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-scanner-rhel8" - - - name: scanner-db-image-build-repo - description: Repository where the (unreleased) Scanner V2 DB image is pushed by its build pipeline. - type: string - default: "quay.io/rhacs-eng/release-scanner-db" - - name: scanner-db-image-catalog-repo - description: Repository within the Red Hat Container Catalog where the Scanner V2 DB image is pushed to during the release. - type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-scanner-db-rhel8" - - - name: scanner-slim-image-build-repo - description: Repository where the (unreleased) Scanner V2 slim image is pushed by its build pipeline. - type: string - default: "quay.io/rhacs-eng/release-scanner-slim" - - name: scanner-slim-image-catalog-repo - description: Repository within the Red Hat Container Catalog where the Scanner V2 slim image is pushed to during the release. - type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-scanner-slim-rhel8" - - - name: scanner-db-slim-image-build-repo - description: Repository where the (unreleased) Scanner V2 DB slim image is pushed by its build pipeline. - type: string - default: "quay.io/rhacs-eng/release-scanner-db-slim" - - name: scanner-db-slim-image-catalog-repo - description: Repository within the Red Hat Container Catalog where the Scanner V2 DB slim image is pushed to during the release. - type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-scanner-db-slim-rhel8" + default: "registry.redhat.io/advanced-cluster-security/rhacs-main-rhel9" - name: scanner-v4-image-build-repo description: Repository where the (unreleased) Scanner V4 image is pushed by its build pipeline. @@ -181,7 +127,7 @@ spec: - name: scanner-v4-image-catalog-repo description: Repository within the Red Hat Container Catalog where the Scanner V4 image is pushed to during the release. type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-rhel8" + default: "registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-rhel9" - name: scanner-v4-db-image-build-repo description: Repository where the (unreleased) Scanner V4 DB image is pushed by its build pipeline. @@ -190,7 +136,7 @@ spec: - name: scanner-v4-db-image-catalog-repo description: Repository within the Red Hat Container Catalog where the Scanner V4 DB image is pushed to during the release. type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-db-rhel8" + default: "registry.redhat.io/advanced-cluster-security/rhacs-scanner-v4-db-rhel9" - name: collector-image-build-repo description: Repository where the (unreleased) collector full image is pushed by its build pipeline. @@ -199,7 +145,16 @@ spec: - name: collector-image-catalog-repo description: Repository within the Red Hat Container Catalog where the collector full image is pushed to during the release. type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-collector-rhel8" + default: "registry.redhat.io/advanced-cluster-security/rhacs-collector-rhel9" + + - name: fact-image-build-repo + description: Repository where the (unreleased) fact image is pushed by its build pipeline. + type: string + default: "quay.io/rhacs-eng/release-fact" + - name: fact-image-catalog-repo + description: Repository within the Red Hat Container Catalog where the fact image is pushed to during the release. + type: string + default: "registry.redhat.io/advanced-cluster-security/rhacs-fact-rhel9" - name: roxctl-image-build-repo description: Repository where the (unreleased) roxctl image is pushed by its build pipeline. @@ -208,7 +163,7 @@ spec: - name: roxctl-image-catalog-repo description: Repository within the Red Hat Container Catalog where the roxctl image is pushed to during the release. type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-roxctl-rhel8" + default: "registry.redhat.io/advanced-cluster-security/rhacs-roxctl-rhel9" - name: central-db-image-build-repo description: Repository where the (unreleased) Central DB image is pushed by its build pipeline. @@ -217,12 +172,32 @@ spec: - name: central-db-image-catalog-repo description: Repository within the Red Hat Container Catalog where the Central DB image is pushed to during the release. type: string - default: "registry.redhat.io/advanced-cluster-security/rhacs-central-db-rhel8" + default: "registry.redhat.io/advanced-cluster-security/rhacs-central-db-rhel9" - name: enable-cache-proxy default: 'false' description: Enable cache proxy configuration type: string + - name: sast-target-dirs + type: string + default: . + description: Target directories to scan with SAST tools. Multiple values should be separated with commas. + - name: enable-package-registry-proxy + default: 'true' + description: Use the package registry proxy when prefetching dependencies + type: string + - name: source-date-epoch + type: string + default: '' + description: Sets the image created time and the SOURCE_DATE_EPOCH build argument. On its own, it does not change file timestamps inside the layers (set rewrite-timestamp to "true" for that). Leave empty to keep the actual build time. + - name: rewrite-timestamp + type: string + default: 'false' + description: When "true", clamp file modification times in the image layers to at most source-date-epoch. Does nothing unless source-date-epoch is set. + - name: omit-history + type: string + default: 'false' + description: When "true", omit the build history (history timestamps, layer metadata, etc.) from the resulting image. results: - description: "" name: IMAGE_URL @@ -247,15 +222,6 @@ spec: - name: init params: - - name: image-url - # We can't provide a StackRox-style tag because it is not known at this time (requires cloning source, etc.) - # As a workaround, we still provide a unique tag that's based on a revision to this task to comply with its - # expected input. We later actually add this tag on a built image with apply-tags task. - value: $(params.output-image-repo):konflux-$(params.revision) - - name: rebuild - value: $(params.rebuild) - - name: skip-checks - value: $(params.skip-checks) - name: enable-cache-proxy value: $(params.enable-cache-proxy) taskRef: @@ -263,7 +229,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:b349d24cb896573695802d6913d311640b44675ec082b3ad167721946a6a0a71 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4.3@sha256:7854d7babe4136e53e337f6433d952532314433b48bb62b422ddc02700934d85 - name: kind value: task resolver: bundles @@ -287,14 +253,10 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:56f65a16d3d0485c64ad85af2c1f3e9b0bb4d02d63f2fd0ebb9498d219ca723d + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] workspaces: - name: basic-auth workspace: git-auth @@ -310,7 +272,7 @@ spec: - name: name value: determine-image-expiration - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -326,7 +288,7 @@ spec: - name: name value: determine-image-tag - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -341,12 +303,16 @@ spec: value: $(params.output-image-repo):konflux-$(params.revision).prefetch - name: ociArtifactExpiresAfter value: $(params.oci-artifact-expires-after) + - name: ACTIVATION_KEY + value: subscription-manager-activation-key-prod + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) taskRef: params: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:8eac535f436874ca27c70434ff356ba941d7b649c0e861bca6f4e9fc5e215478 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.10.3@sha256:0c386f2c26b46b978280c5c54caac26bcd524975f8f7cc06bb080ba00eae3ad4 - name: kind value: task resolver: bundles @@ -363,7 +329,7 @@ spec: - name: name value: wait-for-image - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -378,38 +344,6 @@ spec: # This timeout must be the same as the pipeline timeout in `main-build.yaml`. timeout: 2h40m - - name: wait-for-scanner-image - params: - - name: IMAGE - value: "$(params.scanner-image-build-repo):$(tasks.determine-image-tag.results.IMAGE_TAG)" - taskRef: *wait-for-image-ref - # This timeout must be the same as the pipeline timeout in `scanner-retag.yaml` - timeout: 40m - - - name: wait-for-scanner-db-image - params: - - name: IMAGE - value: "$(params.scanner-db-image-build-repo):$(tasks.determine-image-tag.results.IMAGE_TAG)" - taskRef: *wait-for-image-ref - # This timeout must be the same as the pipeline timeout in `scanner-db-retag.yaml` - timeout: 40m - - - name: wait-for-scanner-slim-image - params: - - name: IMAGE - value: "$(params.scanner-slim-image-build-repo):$(tasks.determine-image-tag.results.IMAGE_TAG)" - taskRef: *wait-for-image-ref - # This timeout must be the same as the pipeline timeout in `scanner-slim-retag.yaml` - timeout: 40m - - - name: wait-for-scanner-db-slim-image - params: - - name: IMAGE - value: "$(params.scanner-db-slim-image-build-repo):$(tasks.determine-image-tag.results.IMAGE_TAG)" - taskRef: *wait-for-image-ref - # This timeout must be the same as the pipeline timeout in `scanner-db-slim-retag.yaml` - timeout: 40m - - name: wait-for-scanner-v4-image params: - name: IMAGE @@ -434,6 +368,14 @@ spec: # This timeout must be the same as the pipeline timeout in `collector-retag.yaml` timeout: 40m + - name: wait-for-fact-image + params: + - name: IMAGE + value: "$(params.fact-image-build-repo):$(tasks.determine-image-tag.results.IMAGE_TAG)" + taskRef: *wait-for-image-ref + # This timeout must be the same as the pipeline timeout in `retag-fact.yaml` + timeout: 40m + - name: wait-for-roxctl-image params: - name: IMAGE @@ -472,13 +414,10 @@ spec: - OPERATOR_IMAGE_TAG=$(tasks.determine-image-tag.results.IMAGE_TAG) - OPERATOR_IMAGE_REF=$(params.operator-image-catalog-repo)@$(tasks.wait-for-operator-image.results.IMAGE_DIGEST) - RELATED_IMAGE_MAIN=$(params.main-image-catalog-repo)@$(tasks.wait-for-main-image.results.IMAGE_DIGEST) - - RELATED_IMAGE_SCANNER=$(params.scanner-image-catalog-repo)@$(tasks.wait-for-scanner-image.results.IMAGE_DIGEST) - - RELATED_IMAGE_SCANNER_DB=$(params.scanner-db-image-catalog-repo)@$(tasks.wait-for-scanner-db-image.results.IMAGE_DIGEST) - - RELATED_IMAGE_SCANNER_SLIM=$(params.scanner-slim-image-catalog-repo)@$(tasks.wait-for-scanner-slim-image.results.IMAGE_DIGEST) - - RELATED_IMAGE_SCANNER_DB_SLIM=$(params.scanner-db-slim-image-catalog-repo)@$(tasks.wait-for-scanner-db-slim-image.results.IMAGE_DIGEST) - RELATED_IMAGE_SCANNER_V4=$(params.scanner-v4-image-catalog-repo)@$(tasks.wait-for-scanner-v4-image.results.IMAGE_DIGEST) - RELATED_IMAGE_SCANNER_V4_DB=$(params.scanner-v4-db-image-catalog-repo)@$(tasks.wait-for-scanner-v4-db-image.results.IMAGE_DIGEST) - RELATED_IMAGE_COLLECTOR=$(params.collector-image-catalog-repo)@$(tasks.wait-for-collector-image.results.IMAGE_DIGEST) + - RELATED_IMAGE_FACT=$(params.fact-image-catalog-repo)@$(tasks.wait-for-fact-image.results.IMAGE_DIGEST) - RELATED_IMAGE_ROXCTL=$(params.roxctl-image-catalog-repo)@$(tasks.wait-for-roxctl-image.results.IMAGE_DIGEST) - RELATED_IMAGE_CENTRAL_DB=$(params.central-db-image-catalog-repo)@$(tasks.wait-for-central-db-image.results.IMAGE_DIGEST) - name: SOURCE_ARTIFACT @@ -495,19 +434,21 @@ spec: value: $(tasks.init.results.http-proxy) - name: NO_PROXY value: $(tasks.init.results.no-proxy) + - name: SOURCE_DATE_EPOCH + value: $(params.source-date-epoch) + - name: REWRITE_TIMESTAMP + value: $(params.rewrite-timestamp) + - name: OMIT_HISTORY + value: $(params.omit-history) taskRef: params: - name: name value: buildah-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.7@sha256:eeb86e8739801237bbefa84e1ab7873c887ca598ef859dd6d06548948b11b95f + value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.12.2@sha256:e1bbe61a3a4bc543be9b5f6fc4174acaa2dd08fb5f7fe4d9db9f3193a7e1b03c - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] - name: apply-tags params: @@ -523,7 +464,7 @@ spec: - name: name value: apply-tags - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.2@sha256:c89cd10b2a3f4c43789c5f06ef2b86f528b28f156c20af5e751fa8c0facd457d + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3.1@sha256:c177fc2b2482b1299e713c51bf258eba4972c016ef260d9f45335c2550dc7b2b - name: kind value: task resolver: bundles @@ -543,14 +484,11 @@ spec: - name: name value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:c35ba219390d77a48ee19347e5ee8d13e5c23e3984299e02291d6da1ed8a986c + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3.2@sha256:45091b5022754d452b4de60c7d7f4c6e03626efff55f96e9af937400fe3506eb - name: kind value: task resolver: bundles when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] - input: $(params.build-source-image) operator: in values: ["true"] @@ -566,7 +504,27 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:808fe09bb5b8503de569de097ae5dd619a7488110f79e8e215e69862ee3fce6d + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:0ccc688a77e9b7b0b8973c132a1e840844137e77f887be4a0bec8893b0776872 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: ["false"] + + - name: roxctl-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + taskRef: + params: + - name: name + value: roxctl-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:97e2b2cdca9110fdc8a93ba585a1a1a743f989f2fd85f4073f6e5ea9ad2ce828 - name: kind value: task resolver: bundles @@ -586,7 +544,7 @@ spec: - name: name value: clair-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:7c2a32de9021f16f6e8df08a55f539f12e00ea4d96f6fb37f9ea04167032c61f + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.4.1@sha256:219964897ac4d5a9473ec297a1b39ae078a34b76b45d04c4672bd569cc295c29 - name: kind value: task resolver: bundles @@ -608,7 +566,7 @@ spec: - name: name value: fips-operator-bundle-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-fips-operator-bundle-check-oci-ta:0.1@sha256:d3b0730dac6a72db1de4a90f3f2703fb261365b2202cb79a9cf7cc56cec0671f + value: quay.io/konflux-ci/tekton-catalog/task-fips-operator-bundle-check-oci-ta:0.1@sha256:1d8d73418bfdc9b0461cf91ba2ac836cc9dde3e0b871c36eed1135287fc7f6a8 - name: kind value: task resolver: bundles @@ -627,12 +585,14 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d44336d7bcbd1f7cedee639357a493bd1f661e2859e49e11a34644bdf6819c4e + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d00c884c78489a52dc638e6ced0a01429670d0754f5b1305ad576709cf660312 - name: kind value: task resolver: bundles @@ -651,12 +611,14 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.3@sha256:e5a8d3e8e7be7246a1460385b95c084ea6e8fe7520d40fe4389deb90f1bf5176 + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:15d654c9576b9c4ee1f78b09fce541acf9afe33f1c2363fd6219a79dbf62c95d - name: kind value: task resolver: bundles @@ -675,12 +637,14 @@ spec: value: $(tasks.build-container.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-container.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:0eca130f289a1a1069a1b92943479f79aa7324e4e68d6396fd777ccd97058f50 + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5@sha256:17b9587eba53fabaab35061275306bfe1f99661d55e14e22f62321fa7cb5ff3e - name: kind value: task resolver: bundles @@ -700,7 +664,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:f3d2d179cddcc07d0228d9f52959a233037a3afa2619d0a8b2effbb467db80c3 + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.3@sha256:9df272d2180bd67e6a3e04d5cd30ffa7ba27af4e599a1922f7069f5c89888e5b - name: kind value: task resolver: bundles @@ -720,7 +684,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:0b10508c82ccb0f5a06a66ce7af56e9bfd40651ddefdf0f499988e897771ee28 + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:d9df5d325f81a9fa132cb597c465aa9210b9b0fa3716b85b32be98d92c9c881a - name: kind value: task resolver: bundles @@ -746,7 +710,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:08bba4a659ecd48f871bef00b80af58954e5a09fcbb28a1783ddd640c4f6535e + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3.1@sha256:4c567d11296850230fe32e76d129834d3bf337a0cae9989b64279b7cfaa4e735 - name: kind value: task resolver: bundles diff --git a/.tekton/operator-index-build.yaml b/.tekton/operator-index-build.yaml new file mode 100644 index 0000000000000..3d4a96d8d9a0b --- /dev/null +++ b/.tekton/operator-index-build.yaml @@ -0,0 +1,439 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun + +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/stackrox/stackrox?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest operator-index-on-push" + # TODO(ROX-21073): re-enable for all PR branches + pipelinesascode.tekton.dev/on-cel-expression: | + ( + event == "push" && target_branch.matches("^(master|release-.*|refs/tags/.*)$") + ) || ( + event == "pull_request" && ( + target_branch.startsWith("release-") || + source_branch.matches("(konflux|renovate|appstudio|rhtap)") || + (has(body.pull_request) && has(body.pull_request.labels) && body.pull_request.labels.exists(l, l.name == "konflux-build")) + ) && body.action != "ready_for_review" + ) + # The empty `on-label` annotation is a workaround to make sure the pipeline gets triggered when the label gets first + # added to the PR. See the Slack tread linked from ROX-30580. + pipelinesascode.tekton.dev/on-label: "[]" + labels: + appstudio.openshift.io/application: acs + pipelines.appstudio.openshift.io/type: build + name: operator-index-on-push + namespace: rh-acs-tenant + +spec: + + params: + - name: git-url + value: '{{source_url}}' + - name: image-expires-after + value: '13w' + - name: output-image-repo + value: quay.io/rhacs-eng/stackrox-operator-index + - name: revision + value: '{{revision}}' + - name: clone-depth + value: '0' + - name: clone-fetch-tags + value: 'true' + - name: bundle-image-repo + value: quay.io/rhacs-eng/release-operator-bundle + + taskRunTemplate: + serviceAccountName: build-pipeline-operator-bundle + + # The operator-index build waits for the operator-bundle to be built and pushed. + # The timeout must account for: + # 1. Bundle build timeout (3h20m from operator-bundle-build.yaml) + # 2. Index build itself (~10-30 minutes) + # Total: 3h20m + 30m = 3h50m + timeouts: + tasks: 3h50m + finally: 10m + pipeline: 4h + + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' + + pipelineSpec: + + finally: + - name: slack-notification + params: + - name: message + value: ':x: `{{event_type}}` pipeline for (`$(params.output-image-repo)`, revision <$(params.git-url)/commit/$(params.revision)|$(params.revision)>) has failed.' + - name: key-name + value: 'acs-konflux-notifications' + when: + # Run when any task has Failed + - input: $(tasks.status) + operator: in + values: [ "Failed" ] + taskRef: + params: + - name: name + value: slack-webhook-notification + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:02842e9c752056c47a409f69ed462cfd38cbd6b15facb796564a610b549222ea + - name: kind + value: task + resolver: bundles + + - name: post-metric-end + params: + - name: AGGREGATE_TASKS_STATUS + value: $(tasks.status) + taskRef: &post-bigquery-metrics-ref + params: + - name: name + value: post-bigquery-metrics + - name: bundle + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 + - name: kind + value: task + resolver: bundles + + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Output Image Repository + name: output-image-repo + type: string + - description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + type: string + - default: "0" + description: Depth of the git clone in number of commits. Use "1" for shallow clone. Use "0" for deep clone, i.e. to fetch all commits. + name: clone-depth + type: string + - default: "true" + description: Fetch tags with git clone + name: clone-fetch-tags + type: string + - default: "1d" + description: This sets the expiration time for intermediate OCI artifacts produced and used during builds after which they can be garbage collected. + name: oci-artifact-expires-after + type: string + - name: extra-labels + type: array + description: Additional labels to put on the built containers. + default: [] + - default: + - linux/amd64 + - linux/arm64 + - linux/ppc64le + - linux/s390x + description: > + List of platforms to build the container images for. The available + set of values is determined by the configuration of the multi-platform-controller + on the cluster: https://konflux.pages.redhat.com/docs/users/getting-started/multi-platform-builds.html + name: build-platforms + type: array + - name: bundle-image-repo + description: Repository where the operator bundle image is pushed by operator-bundle-build.yaml. + type: string + default: "quay.io/rhacs-eng/release-operator-bundle" + - name: buildah-format + default: docker + type: string + description: The format for the resulting image's mediaType. Valid values are oci or docker. + + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + + workspaces: + - name: git-auth + + tasks: + + - name: post-metric-start + taskRef: *post-bigquery-metrics-ref + + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: depth + value: $(params.clone-depth) + - name: fetchTags + value: $(params.clone-fetch-tags) + - name: ociStorage + value: $(params.output-image-repo):konflux-$(params.revision).git + - name: ociArtifactExpiresAfter + value: $(params.oci-artifact-expires-after) + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 + - name: kind + value: task + resolver: bundles + workspaces: + - name: basic-auth + workspace: git-auth + + - name: determine-image-expiration + params: + - name: DEFAULT_IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + taskRef: + params: + - name: name + value: determine-image-expiration + - name: bundle + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 + - name: kind + value: task + resolver: bundles + + - name: determine-image-tag + params: + - name: TAG_SUFFIX + value: "-fast" + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + taskRef: + params: + - name: name + value: determine-image-tag + - name: bundle + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 + - name: kind + value: task + resolver: bundles + + - name: determine-index-tag + params: + - name: BASE_TAG + value: $(tasks.determine-image-tag.results.IMAGE_TAG) + taskSpec: + params: + - name: BASE_TAG + description: Base image tag from determine-image-tag task + type: string + results: + - name: INDEX_TAG + description: Full operator-index image tag with v prefix and -fast suffix to avoid collision with GHA builds + steps: + - name: compute-index-tag + image: registry.access.redhat.com/ubi9-minimal:latest@sha256:c4c285d42941ebb2bd4da878479008e1e4536bb599aad1dbe568f89de9053661 + imagePullPolicy: IfNotPresent + script: | + #!/bin/bash + set -euo pipefail + base_tag="$(params.BASE_TAG)" + + # Add "v" prefix and ensure "-fast" suffix + # This ensures Konflux operator-index images never collide with GHA operator-index builds + if [[ "${base_tag}" == *-fast ]]; then + index_tag="v${base_tag}" + else + # No -fast suffix (e.g., from tagged builds where suffix was suppressed) + index_tag="v${base_tag}-fast" + fi + + echo -n "${index_tag}" | tee $(results.INDEX_TAG.path) + + - name: wait-for-bundle-image + params: + - name: IMAGE + value: "$(params.bundle-image-repo):v$(tasks.determine-image-tag.results.IMAGE_TAG)" + taskRef: + params: + - name: name + value: wait-for-image + - name: bundle + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 + - name: kind + value: task + resolver: bundles + # This timeout must be the same as the pipeline timeout in `operator-bundle-build.yaml`. + timeout: 3h20m + + - name: generate-index-content + runAfter: + - wait-for-bundle-image + params: + - name: BUNDLE_IMAGE + value: "$(params.bundle-image-repo):v$(tasks.determine-image-tag.results.IMAGE_TAG)" + - name: INDEX_IMAGE + value: $(params.output-image-repo):$(tasks.determine-index-tag.results.INDEX_TAG) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image-repo):konflux-$(params.revision).index-build + - name: ociArtifactExpiresAfter + value: $(params.oci-artifact-expires-after) + taskSpec: + description: >- + Builds operator index (catalog) using opm tool. This task generates the index YAML + and Dockerfile which are then used by buildah to create multi-platform images. + params: + - name: BUNDLE_IMAGE + description: The operator bundle image to include in the index + type: string + - name: INDEX_IMAGE + description: The target index image name + type: string + - name: SOURCE_ARTIFACT + description: The Trusted Artifact URI pointing to the application source code. + type: string + - name: ociStorage + description: The OCI repository where the index build artifacts will be stored temporarily. + type: string + - name: ociArtifactExpiresAfter + description: Expiration time for the OCI artifacts. + type: string + default: "1d" + results: + - name: SOURCE_ARTIFACT + description: The Trusted Artifact URI pointing to the index build directory with Dockerfile and YAML. + type: string + stepTemplate: + volumeMounts: + - mountPath: /var/workdir + name: workdir + steps: + - name: use-trusted-artifact + image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:9b180776a41d9a22a1c51539f1647c60defbbd55b44bbebdd4130e33512d8b0d + imagePullPolicy: IfNotPresent + args: + - use + - $(params.SOURCE_ARTIFACT)=/var/workdir/source + - name: build-index-content + image: registry.access.redhat.com/ubi9/go-toolset:9.8-1792010711@sha256:60fd6fb2998c40ad0d6561c34e83588b986a0cf7771f64e5e5c295ba87101f9b + imagePullPolicy: IfNotPresent + workingDir: /var/workdir/source/operator + securityContext: + runAsUser: 0 + env: + - name: BUNDLE_IMAGE + value: $(params.BUNDLE_IMAGE) + - name: INDEX_IMAGE + value: $(params.INDEX_IMAGE) + script: | + #!/bin/bash + set -euo pipefail + + echo "Building operator index for bundle: ${BUNDLE_IMAGE}" + echo "Target index image: ${INDEX_IMAGE}" + + echo "Building yq..." + GOFLAGS="-mod=mod" make yq + export YQ="${PWD}/.gotools/bin/yq" + + # Use the existing build script to generate index content + ./hack/build-index-image.sh \ + --index-tag "${INDEX_IMAGE}" \ + --bundle-tag "${BUNDLE_IMAGE}" \ + --single-bundle \ + --skip-build \ + --base-dir "." + + echo "Index content generated successfully" + - name: create-trusted-artifact + image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:9b180776a41d9a22a1c51539f1647c60defbbd55b44bbebdd4130e33512d8b0d + imagePullPolicy: IfNotPresent + args: + - create + - --store + - $(params.ociStorage) + - $(results.SOURCE_ARTIFACT.path)=/var/workdir/source/operator/build/index + env: + - name: IMAGE_EXPIRES_AFTER + value: $(params.ociArtifactExpiresAfter) + volumes: + - name: workdir + emptyDir: {} + + - name: build-images + runAfter: + - generate-index-content + matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + params: + - name: IMAGE + value: $(params.output-image-repo):$(tasks.determine-index-tag.results.INDEX_TAG) + - name: DOCKERFILE + # The Dockerfile is generated in the build-index-image.sh script (via OPM generate dockerfile) + value: rhacs-operator-index.Dockerfile + - name: CONTEXT + value: . + - name: IMAGE_EXPIRES_AFTER + value: $(tasks.determine-image-expiration.results.IMAGE_EXPIRES_AFTER) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: SOURCE_ARTIFACT + value: $(tasks.generate-index-content.results.SOURCE_ARTIFACT) + - name: BUILD_TIMESTAMP + value: "$(tasks.clone-repository.results.commit-timestamp)" + - name: LABELS + value: ["$(params.extra-labels[*])"] + - name: BUILDAH_FORMAT + value: $(params.buildah-format) + - name: IMAGE_APPEND_PLATFORM + value: "true" + taskRef: + params: + - name: name + value: buildah-remote-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.12.2@sha256:a68dce8ed79be8d9e10d98bd2b56895c27fc34f1ed7d67cbc9ec0a91d64cf641 + - name: kind + value: task + resolver: bundles + + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image-repo):$(tasks.determine-index-tag.results.INDEX_TAG) + - name: IMAGES + value: + - $(tasks.build-images.results.IMAGE_REF[*]) + - name: BUILDAH_FORMAT + value: $(params.buildah-format) + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.4@sha256:654446d8ec028b20ed6ada8a96486ff16692a25192755aa04a99cea8e0afb7e2 + - name: kind + value: task + resolver: bundles diff --git a/.tekton/retag-collector.yaml b/.tekton/retag-collector.yaml index 402dfb62469e1..d9c526eba9dc5 100644 --- a/.tekton/retag-collector.yaml +++ b/.tekton/retag-collector.yaml @@ -8,6 +8,7 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest retag-collector" # TODO(ROX-21073): re-enable for all PR branches pipelinesascode.tekton.dev/on-cel-expression: | ( diff --git a/.tekton/retag-fact.yaml b/.tekton/retag-fact.yaml new file mode 100644 index 0000000000000..e41677be00dd6 --- /dev/null +++ b/.tekton/retag-fact.yaml @@ -0,0 +1,60 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun + +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/stackrox/stackrox?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest retag-fact" + # TODO(ROX-21073): re-enable for all PR branches + pipelinesascode.tekton.dev/on-cel-expression: | + ( + event == "push" && target_branch.matches("^(master|release-.*|refs/tags/.*)$") + ) || ( + event == "pull_request" && ( + target_branch.startsWith("release-") || + source_branch.matches("(konflux|renovate|appstudio|rhtap)") || + (has(body.pull_request) && has(body.pull_request.labels) && body.pull_request.labels.exists(l, l.name == "konflux-build")) + ) && body.action != "ready_for_review" + ) + # The empty `on-label` annotation is a workaround to make sure the pipeline gets triggered when the label gets first + # added to the PR. See the Slack tread linked from ROX-30580. + pipelinesascode.tekton.dev/on-label: "[]" + labels: + appstudio.openshift.io/application: acs + name: retag-fact + namespace: rh-acs-tenant + +spec: + + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: input-image-repo + value: quay.io/rhacs-eng/release-fact + - name: input-image-tag-makefile-target + value: fact-tag + - name: output-image-repo + value: quay.io/rhacs-eng/release-fact + + pipelineRef: + name: retag-pipeline + + taskRunTemplate: + serviceAccountName: build-pipeline-fact + + timeouts: + tasks: 30m + # Reserve time for final tasks to run. + finally: 10m + pipeline: 40m + + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' diff --git a/.tekton/retag-pipeline.yaml b/.tekton/retag-pipeline.yaml index 71cd2995cde74..023b634a93706 100644 --- a/.tekton/retag-pipeline.yaml +++ b/.tekton/retag-pipeline.yaml @@ -21,7 +21,7 @@ spec: - name: name value: slack-webhook-notification - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:69945a30c11387a766e3d0ae33991b68e865a290c09da1fea44f193d358926ba + value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:02842e9c752056c47a409f69ed462cfd38cbd6b15facb796564a610b549222ea - name: kind value: task resolver: bundles @@ -35,7 +35,7 @@ spec: - name: name value: post-bigquery-metrics - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -117,7 +117,7 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:56f65a16d3d0485c64ad85af2c1f3e9b0bb4d02d63f2fd0ebb9498d219ca723d + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 - name: kind value: task resolver: bundles @@ -136,7 +136,7 @@ spec: - name: name value: determine-image-tag - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -154,7 +154,7 @@ spec: - name: name value: determine-dependency-image-tag - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -170,7 +170,7 @@ spec: - name: name value: wait-for-image - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -195,7 +195,7 @@ spec: - name: name value: retag-image - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles diff --git a/.tekton/retag-scanner-db-slim.yaml b/.tekton/retag-scanner-db-slim.yaml deleted file mode 100644 index 581a18e658190..0000000000000 --- a/.tekton/retag-scanner-db-slim.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun - -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/stackrox/stackrox?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "500" - # TODO(ROX-21073): re-enable for all PR branches - pipelinesascode.tekton.dev/on-cel-expression: | - ( - event == "push" && target_branch.matches("^(master|release-.*|refs/tags/.*)$") - ) || ( - event == "pull_request" && ( - target_branch.startsWith("release-") || - source_branch.matches("(konflux|renovate|appstudio|rhtap)") || - (has(body.pull_request) && has(body.pull_request.labels) && body.pull_request.labels.exists(l, l.name == "konflux-build")) - ) && body.action != "ready_for_review" - ) - # The empty `on-label` annotation is a workaround to make sure the pipeline gets triggered when the label gets first - # added to the PR. See the Slack tread linked from ROX-30580. - pipelinesascode.tekton.dev/on-label: "[]" - labels: - appstudio.openshift.io/application: acs - name: retag-scanner-db-slim - namespace: rh-acs-tenant - -spec: - - params: - - name: git-url - value: '{{source_url}}' - - name: revision - value: '{{revision}}' - - name: input-image-repo - value: quay.io/rhacs-eng/release-scanner-db-slim - - name: input-image-tag-makefile-target - value: scanner-tag - - name: output-image-repo - value: quay.io/rhacs-eng/release-scanner-db-slim - - pipelineRef: - name: retag-pipeline - - taskRunTemplate: - serviceAccountName: build-pipeline-scanner-db-slim - - timeouts: - tasks: 30m - # Reserve time for final tasks to run. - finally: 10m - pipeline: 40m - - workspaces: - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' diff --git a/.tekton/retag-scanner-db.yaml b/.tekton/retag-scanner-db.yaml deleted file mode 100644 index 500aa96403280..0000000000000 --- a/.tekton/retag-scanner-db.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun - -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/stackrox/stackrox?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "500" - # TODO(ROX-21073): re-enable for all PR branches - pipelinesascode.tekton.dev/on-cel-expression: | - ( - event == "push" && target_branch.matches("^(master|release-.*|refs/tags/.*)$") - ) || ( - event == "pull_request" && ( - target_branch.startsWith("release-") || - source_branch.matches("(konflux|renovate|appstudio|rhtap)") || - (has(body.pull_request) && has(body.pull_request.labels) && body.pull_request.labels.exists(l, l.name == "konflux-build")) - ) && body.action != "ready_for_review" - ) - # The empty `on-label` annotation is a workaround to make sure the pipeline gets triggered when the label gets first - # added to the PR. See the Slack tread linked from ROX-30580. - pipelinesascode.tekton.dev/on-label: "[]" - labels: - appstudio.openshift.io/application: acs - name: retag-scanner-db - namespace: rh-acs-tenant - -spec: - - params: - - name: git-url - value: '{{source_url}}' - - name: revision - value: '{{revision}}' - - name: input-image-repo - value: quay.io/rhacs-eng/release-scanner-db - - name: input-image-tag-makefile-target - value: scanner-tag - - name: output-image-repo - value: quay.io/rhacs-eng/release-scanner-db - - pipelineRef: - name: retag-pipeline - - taskRunTemplate: - serviceAccountName: build-pipeline-scanner-db - - timeouts: - tasks: 30m - # Reserve time for final tasks to run. - finally: 10m - pipeline: 40m - - workspaces: - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' diff --git a/.tekton/retag-scanner-slim.yaml b/.tekton/retag-scanner-slim.yaml deleted file mode 100644 index 9a0347886450c..0000000000000 --- a/.tekton/retag-scanner-slim.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun - -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/stackrox/stackrox?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "500" - # TODO(ROX-21073): re-enable for all PR branches - pipelinesascode.tekton.dev/on-cel-expression: | - ( - event == "push" && target_branch.matches("^(master|release-.*|refs/tags/.*)$") - ) || ( - event == "pull_request" && ( - target_branch.startsWith("release-") || - source_branch.matches("(konflux|renovate|appstudio|rhtap)") || - (has(body.pull_request) && has(body.pull_request.labels) && body.pull_request.labels.exists(l, l.name == "konflux-build")) - ) && body.action != "ready_for_review" - ) - # The empty `on-label` annotation is a workaround to make sure the pipeline gets triggered when the label gets first - # added to the PR. See the Slack tread linked from ROX-30580. - pipelinesascode.tekton.dev/on-label: "[]" - labels: - appstudio.openshift.io/application: acs - name: retag-scanner-slim - namespace: rh-acs-tenant - -spec: - - params: - - name: git-url - value: '{{source_url}}' - - name: revision - value: '{{revision}}' - - name: input-image-repo - value: quay.io/rhacs-eng/release-scanner-slim - - name: input-image-tag-makefile-target - value: scanner-tag - - name: output-image-repo - value: quay.io/rhacs-eng/release-scanner-slim - - pipelineRef: - name: retag-pipeline - - taskRunTemplate: - serviceAccountName: build-pipeline-scanner-slim - - timeouts: - tasks: 30m - # Reserve time for final tasks to run. - finally: 10m - pipeline: 40m - - workspaces: - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' diff --git a/.tekton/retag-scanner.yaml b/.tekton/retag-scanner.yaml deleted file mode 100644 index 1ef222b253e72..0000000000000 --- a/.tekton/retag-scanner.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun - -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/stackrox/stackrox?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "500" - # TODO(ROX-21073): re-enable for all PR branches - pipelinesascode.tekton.dev/on-cel-expression: | - ( - event == "push" && target_branch.matches("^(master|release-.*|refs/tags/.*)$") - ) || ( - event == "pull_request" && ( - target_branch.startsWith("release-") || - source_branch.matches("(konflux|renovate|appstudio|rhtap)") || - (has(body.pull_request) && has(body.pull_request.labels) && body.pull_request.labels.exists(l, l.name == "konflux-build")) - ) && body.action != "ready_for_review" - ) - # The empty `on-label` annotation is a workaround to make sure the pipeline gets triggered when the label gets first - # added to the PR. See the Slack tread linked from ROX-30580. - pipelinesascode.tekton.dev/on-label: "[]" - labels: - appstudio.openshift.io/application: acs - name: retag-scanner - namespace: rh-acs-tenant - -spec: - - params: - - name: git-url - value: '{{source_url}}' - - name: revision - value: '{{revision}}' - - name: input-image-repo - value: quay.io/rhacs-eng/release-scanner - - name: input-image-tag-makefile-target - value: scanner-tag - - name: output-image-repo - value: quay.io/rhacs-eng/release-scanner - - pipelineRef: - name: retag-pipeline - - taskRunTemplate: - serviceAccountName: build-pipeline-scanner - - timeouts: - tasks: 30m - # Reserve time for final tasks to run. - finally: 10m - pipeline: 40m - - workspaces: - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' diff --git a/.tekton/roxctl-build.yaml b/.tekton/roxctl-build.yaml index a5386f4c3a007..6868f0e903112 100644 --- a/.tekton/roxctl-build.yaml +++ b/.tekton/roxctl-build.yaml @@ -8,6 +8,7 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest roxctl-on-push" # TODO(ROX-21073): re-enable for all PR branches pipelinesascode.tekton.dev/on-cel-expression: | ( @@ -49,7 +50,11 @@ spec: - name: hermetic value: 'true' - name: prefetch-input - value: '{"type": "gomod", "path": "."}' + value: | + [ + { "type": "gomod", "path": "." }, + { "type": "rpm", "path": "." } + ] - name: build-source-image value: 'true' - name: clone-depth @@ -59,30 +64,66 @@ spec: - name: extra-labels value: # X.Y in the cpe label must be adjusted for every version stream. - - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el8" + - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el9" pipelineRef: name: basic-component-pipeline taskRunSpecs: + # The following are overrides to default step resources to prevent occasional or deterministic OOM kills. - pipelineTaskName: prefetch-dependencies + podTemplate: &go-memlimit + env: + - name: GOMEMLIMIT + value: 2500MiB stepSpecs: - # Limit CPU for Go dependencies prefetch, as it tends to be OOM-killed when it gets lots of it without OOTB limit. - name: prefetch-dependencies computeResources: + limits: + memory: 6Gi + requests: + memory: 6Gi + - name: create-trusted-artifact + computeResources: &ta-resources limits: cpu: 2 + memory: 7Gi requests: cpu: 2 + memory: 7Gi + - pipelineTaskName: build-images + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources - pipelineTaskName: build-source-image + podTemplate: *go-memlimit stepSpecs: - # Bump memory for source image build because the one with defaults tends to get sometimes OOM-killed. + - name: use-trusted-artifact + computeResources: *ta-resources - name: build computeResources: limits: + cpu: 1 memory: 3Gi requests: + cpu: 1 memory: 3Gi + - pipelineTaskName: sast-shell-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources + - pipelineTaskName: sast-unicode-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources + - pipelineTaskName: sast-snyk-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources taskRunTemplate: serviceAccountName: build-pipeline-roxctl diff --git a/.tekton/scanner-v4-build.yaml b/.tekton/scanner-v4-build.yaml index c6c890a261eb1..66cb33cb30448 100644 --- a/.tekton/scanner-v4-build.yaml +++ b/.tekton/scanner-v4-build.yaml @@ -8,6 +8,7 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest scanner-v4-on-push" # TODO(ROX-21073): re-enable for all PR branches pipelinesascode.tekton.dev/on-cel-expression: | ( @@ -49,7 +50,11 @@ spec: - name: hermetic value: "true" - name: prefetch-input - value: '{"type": "gomod", "path": "."}' + value: | + [ + { "type": "gomod", "path": "." }, + { "type": "rpm" } + ] - name: build-source-image value: 'true' - name: clone-depth @@ -59,30 +64,66 @@ spec: - name: extra-labels value: # X.Y in the cpe label must be adjusted for every version stream. - - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el8" + - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el9" pipelineRef: name: scanner-v4-pipeline taskRunSpecs: + # The following are overrides to default step resources to prevent occasional or deterministic OOM kills. - pipelineTaskName: prefetch-dependencies + podTemplate: &go-memlimit + env: + - name: GOMEMLIMIT + value: 2500MiB stepSpecs: - # Limit CPU for Go dependencies prefetch, as it tends to be OOM-killed when it gets lots of it without OOTB limit. - name: prefetch-dependencies computeResources: + limits: + memory: 6Gi + requests: + memory: 6Gi + - name: create-trusted-artifact + computeResources: &ta-resources limits: cpu: 2 + memory: 7Gi requests: cpu: 2 + memory: 7Gi + - pipelineTaskName: build-images + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources - pipelineTaskName: build-source-image + podTemplate: *go-memlimit stepSpecs: - # Bump memory for source image build because the one with defaults tends to get sometimes OOM-killed. + - name: use-trusted-artifact + computeResources: *ta-resources - name: build computeResources: limits: + cpu: 1 memory: 3Gi requests: + cpu: 1 memory: 3Gi + - pipelineTaskName: sast-shell-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources + - pipelineTaskName: sast-unicode-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources + - pipelineTaskName: sast-snyk-check + podTemplate: *go-memlimit + stepSpecs: + - name: use-trusted-artifact + computeResources: *ta-resources taskRunTemplate: serviceAccountName: build-pipeline-scanner-v4 diff --git a/.tekton/scanner-v4-db-build.yaml b/.tekton/scanner-v4-db-build.yaml index 5b74dc9b763eb..430695e3c9e8f 100644 --- a/.tekton/scanner-v4-db-build.yaml +++ b/.tekton/scanner-v4-db-build.yaml @@ -8,6 +8,7 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" + pipelinesascode.tekton.dev/on-comment: "/konflux-retest scanner-v4-db-on-push" # TODO(ROX-21073): re-enable for all PR branches pipelinesascode.tekton.dev/on-cel-expression: | ( @@ -60,21 +61,31 @@ spec: - name: extra-labels value: # X.Y in the cpe label must be adjusted for every version stream. - - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el8" + - "cpe=cpe:/a:redhat:advanced_cluster_security:X.Y::el9" pipelineRef: name: basic-component-pipeline taskRunSpecs: + # The following are overrides to default step resources to prevent occasional or deterministic OOM kills. - pipelineTaskName: build-source-image stepSpecs: - # Bump memory for source image build because the one with defaults tends to get sometimes OOM-killed. - name: build computeResources: limits: + cpu: 1 memory: 3Gi requests: + cpu: 1 memory: 3Gi + - pipelineTaskName: rpms-signature-scan + stepSpecs: + - name: rpms-signature-scan + computeResources: + limits: + memory: 512Mi + requests: + memory: 512Mi taskRunTemplate: serviceAccountName: build-pipeline-scanner-v4-db diff --git a/.tekton/scanner-v4-pipeline.yaml b/.tekton/scanner-v4-pipeline.yaml index de426d4da0bc1..05368bff8ca56 100644 --- a/.tekton/scanner-v4-pipeline.yaml +++ b/.tekton/scanner-v4-pipeline.yaml @@ -21,21 +21,7 @@ spec: - name: name value: slack-webhook-notification - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:69945a30c11387a766e3d0ae33991b68e865a290c09da1fea44f193d358926ba - - name: kind - value: task - resolver: bundles - - - name: show-sbom - params: - - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - taskRef: - params: - - name: name - value: show-sbom - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:beb0616db051952b4b861dd8c3e00fa1c0eccbd926feddf71194d3bb3ace9ce7 + value: quay.io/konflux-ci/tekton-catalog/task-slack-webhook-notification:0.1@sha256:02842e9c752056c47a409f69ed462cfd38cbd6b15facb796564a610b549222ea - name: kind value: task resolver: bundles @@ -49,7 +35,7 @@ spec: - name: name value: post-bigquery-metrics - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -77,10 +63,6 @@ spec: description: Path to the Dockerfile inside the context specified by parameter path-context name: dockerfile type: string - - default: "false" - description: Force rebuild image - name: rebuild - type: string - default: "false" description: Skip checks against built image name: skip-checks @@ -135,6 +117,26 @@ spec: default: 'false' description: Enable cache proxy configuration type: string + - name: sast-target-dirs + type: string + default: . + description: Target directories to scan with SAST tools. Multiple values should be separated with commas. + - name: enable-package-registry-proxy + default: 'true' + description: Use the package registry proxy when prefetching dependencies + type: string + - name: source-date-epoch + type: string + default: '' + description: Sets the image created time and the SOURCE_DATE_EPOCH build argument. On its own, it does not change file timestamps inside the layers (set rewrite-timestamp to "true" for that). Leave empty to keep the actual build time. + - name: rewrite-timestamp + type: string + default: 'false' + description: When "true", clamp file modification times in the image layers to at most source-date-epoch. Does nothing unless source-date-epoch is set. + - name: omit-history + type: string + default: 'false' + description: When "true", omit the build history (history timestamps, layer metadata, etc.) from the resulting image. results: - description: "" name: IMAGE_URL @@ -159,15 +161,6 @@ spec: - name: init params: - - name: image-url - # We can't provide a StackRox-style tag because it is not known at this time (requires cloning source, etc.) - # As a workaround, we still provide a unique tag that's based on a revision in order for this task to comply with - # its expected input. We later actually add this tag on a built image with the apply-index-image-tag task. - value: $(params.output-image-repo):konflux-$(params.revision) - - name: rebuild - value: $(params.rebuild) - - name: skip-checks - value: $(params.skip-checks) - name: enable-cache-proxy value: $(params.enable-cache-proxy) taskRef: @@ -175,7 +168,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:b349d24cb896573695802d6913d311640b44675ec082b3ad167721946a6a0a71 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4.3@sha256:7854d7babe4136e53e337f6433d952532314433b48bb62b422ddc02700934d85 - name: kind value: task resolver: bundles @@ -199,14 +192,10 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:56f65a16d3d0485c64ad85af2c1f3e9b0bb4d02d63f2fd0ebb9498d219ca723d + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.2.6@sha256:a3678910c2bf9187a66a96066193cb41046fbb7097160bf2be6a374ed5e49d30 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] workspaces: - name: basic-auth workspace: git-auth @@ -222,7 +211,7 @@ spec: - name: name value: determine-image-expiration - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -238,7 +227,7 @@ spec: - name: name value: determine-image-tag - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -258,7 +247,7 @@ spec: - name: name value: fetch-scanner-v4-vuln-mappings - name: bundle - value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:c3b069313d4de5d67dc662e31cbcdde02e134be5fe7dcdb5bceeb670f89fcf52 + value: quay.io/rhacs-eng/konflux-tasks:latest@sha256:18defc6b58a3792ef4d1c7268ce22167c833e7846523b97e0f3219094df0d510 - name: kind value: task resolver: bundles @@ -273,12 +262,16 @@ spec: value: $(params.output-image-repo):konflux-$(params.revision).prefetch - name: ociArtifactExpiresAfter value: $(params.oci-artifact-expires-after) + - name: ACTIVATION_KEY + value: subscription-manager-activation-key-prod + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) taskRef: params: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:8eac535f436874ca27c70434ff356ba941d7b649c0e861bca6f4e9fc5e215478 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.10.3@sha256:0c386f2c26b46b978280c5c54caac26bcd524975f8f7cc06bb080ba00eae3ad4 - name: kind value: task resolver: bundles @@ -326,31 +319,29 @@ spec: value: $(tasks.init.results.http-proxy) - name: NO_PROXY value: $(tasks.init.results.no-proxy) + - name: SOURCE_DATE_EPOCH + value: $(params.source-date-epoch) + - name: REWRITE_TIMESTAMP + value: $(params.rewrite-timestamp) + - name: OMIT_HISTORY + value: $(params.omit-history) taskRef: params: - name: name value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.7@sha256:fe66734cde6713f530b19e222680acc5e0d6a533866429659ea8561b746d9ce7 + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.12.2@sha256:a68dce8ed79be8d9e10d98bd2b56895c27fc34f1ed7d67cbc9ec0a91d64cf641 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] - name: build-image-index params: - name: IMAGE value: $(params.output-image-repo):$(tasks.determine-image-tag.results.IMAGE_TAG) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - name: IMAGES value: - $(tasks.build-images.results.IMAGE_REF[*]) - - name: IMAGE_EXPIRES_AFTER - value: $(tasks.determine-image-expiration.results.IMAGE_EXPIRES_AFTER) - name: BUILDAH_FORMAT value: $(params.buildah-format) taskRef: @@ -358,39 +349,12 @@ spec: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.2@sha256:985d1efe861b02524a7679ecd855624b3d4e3a2e835b6f8a97ec7d135898ec0b + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.4@sha256:654446d8ec028b20ed6ada8a96486ff16692a25192755aa04a99cea8e0afb7e2 - name: kind value: task resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] runAfter: [build-images] - - name: apply-index-image-tag - params: - - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE_DIGEST - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: ADDITIONAL_TAGS - value: - - konflux-$(params.revision) - taskRef: - params: - - name: name - value: apply-tags - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.2@sha256:c89cd10b2a3f4c43789c5f06ef2b86f528b28f156c20af5e751fa8c0facd457d - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] - - name: build-source-image params: - name: BINARY_IMAGE @@ -406,14 +370,11 @@ spec: - name: name value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:c35ba219390d77a48ee19347e5ee8d13e5c23e3984299e02291d6da1ed8a986c + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3.2@sha256:45091b5022754d452b4de60c7d7f4c6e03626efff55f96e9af937400fe3506eb - name: kind value: task resolver: bundles when: - - input: $(tasks.init.results.build) - operator: in - values: ["true"] - input: $(params.build-source-image) operator: in values: ["true"] @@ -429,7 +390,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:808fe09bb5b8503de569de097ae5dd619a7488110f79e8e215e69862ee3fce6d + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:0ccc688a77e9b7b0b8973c132a1e840844137e77f887be4a0bec8893b0776872 - name: kind value: task resolver: bundles @@ -438,12 +399,27 @@ spec: operator: in values: ["false"] - - name: clair-scan - matrix: + - name: roxctl-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: params: - - name: image-platform - value: - - $(params.build-platforms) + - name: name + value: roxctl-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-roxctl-scan:0.1@sha256:97e2b2cdca9110fdc8a93ba585a1a1a743f989f2fd85f4073f6e5ea9ad2ce828 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: ["false"] + + - name: clair-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -454,7 +430,7 @@ spec: - name: name value: clair-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:7c2a32de9021f16f6e8df08a55f539f12e00ea4d96f6fb37f9ea04167032c61f + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.4.1@sha256:219964897ac4d5a9473ec297a1b39ae078a34b76b45d04c4672bd569cc295c29 - name: kind value: task resolver: bundles @@ -477,7 +453,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:04f75593558f79a27da2336400bc63d460bf0c5669e3c13f40ee2fb650b1ad1e + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:fa53ef450ba538addafee1211ff6ac15b55865ce652fec15ac99b554680cb642 - name: kind value: task resolver: bundles @@ -496,12 +472,14 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d44336d7bcbd1f7cedee639357a493bd1f661e2859e49e11a34644bdf6819c4e + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d00c884c78489a52dc638e6ced0a01429670d0754f5b1305ad576709cf660312 - name: kind value: task resolver: bundles @@ -520,12 +498,14 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.3@sha256:e5a8d3e8e7be7246a1460385b95c084ea6e8fe7520d40fe4389deb90f1bf5176 + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:15d654c9576b9c4ee1f78b09fce541acf9afe33f1c2363fd6219a79dbf62c95d - name: kind value: task resolver: bundles @@ -544,12 +524,14 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) taskRef: params: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:0eca130f289a1a1069a1b92943479f79aa7324e4e68d6396fd777ccd97058f50 + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.5@sha256:17b9587eba53fabaab35061275306bfe1f99661d55e14e22f62321fa7cb5ff3e - name: kind value: task resolver: bundles @@ -574,7 +556,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:f3d2d179cddcc07d0228d9f52959a233037a3afa2619d0a8b2effbb467db80c3 + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3.3@sha256:9df272d2180bd67e6a3e04d5cd30ffa7ba27af4e599a1922f7069f5c89888e5b - name: kind value: task resolver: bundles @@ -594,7 +576,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:0b10508c82ccb0f5a06a66ce7af56e9bfd40651ddefdf0f499988e897771ee28 + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2.2@sha256:d9df5d325f81a9fa132cb597c465aa9210b9b0fa3716b85b32be98d92c9c881a - name: kind value: task resolver: bundles @@ -620,7 +602,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:08bba4a659ecd48f871bef00b80af58954e5a09fcbb28a1783ddd640c4f6535e + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3.1@sha256:4c567d11296850230fe32e76d129834d3bf337a0cae9989b64279b7cfaa4e735 - name: kind value: task resolver: bundles diff --git a/AGENTS.md b/AGENTS.md index 18e37bb07003f..52856e8ddb89a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,8 +85,33 @@ When creating pull requests, you must follow these requirements: - `make go-generated-srcs` - Generate Go code (mockgen, stringer, easyjson) - `make generated-srcs` - Generate all source code +### Adding Database Columns +When the user needs to add a new column to an existing database table: + +**IMPORTANT: Schema-only changes do NOT require a migration.** Adding columns or tables +is handled automatically by GORM AutoMigrate on every Central startup. Indexes are managed +by the code generator and applied outside of GORM (see `migrator/README.md`). +Simply update the proto definition (add field, add search/sql tags) and regenerate the +schema code. A migration is ONLY needed when existing data must be backfilled or transformed. +If the new column can tolerate its zero value until normal operation populates it, no +migration is needed. + +When a migration IS needed: +- Read `migrator/README.md` for the authoritative migration workflow and examples +- Prompt the user about their specific needs (column type, backfill requirements, performance considerations) +- Present options based on similar migrations in `migrator/migrations/` directory +- Reference the "Code Generation Commands" section above for regenerating schema/store code +- Suggest running code generation commands in the background if appropriate (they can be slow) +- After code generation, run code quality checks from the "Code Quality Commands" section above + +The migrator README contains detailed examples of frozen schemas, GORM usage patterns, and migration best practices. + ### Local Development Commands -- `./deploy/deploy-local.sh` - Deploy StackRox locally (requires existing k8s cluster) + +- `roxie deploy` - Deploy StackRox with roxie, also works for local clusters, see deploy/AGENTS.md for more + information on roxie. +- `./deploy/deploy-local.sh` - Deploy StackRox locally (requires existing k8s cluster), still supported, + but deprecated way. - `make install-dev-tools` - Install development tools (linters, generators) ### Single Test Examples @@ -122,6 +147,11 @@ StackRox is a Kubernetes-native security platform with a distributed microservic - **Secured Cluster Services**: Deployed per monitored cluster (Sensor, Admission Controller) - **Multi-cluster support**: One Central instance monitors multiple Kubernetes clusters +### CI Image (Prow / OpenShift CI) +- The CI image is built from the **[stackrox/rox-ci-image](https://github.com/stackrox/rox-ci-image)** repo, not this repo. +- Tool versions bundled in the CI image (e.g. roxie, helm, docker) are pinned in Dockerfiles in that repo. +- `.openshift-ci/Dockerfile.build_root` in this repo only references the resulting image for validation purposes. + ### Key Directories - `/central/` - Central management service code - `/sensor/` - Sensor agent code for cluster monitoring @@ -138,9 +168,15 @@ StackRox is a Kubernetes-native security platform with a distributed microservic ### Detailed Documentation +**In-code documentation:** Packages with non-obvious architecture or operational +gotchas often contain a markdown file (e.g. `README.md`) co-located with the +source code. Before researching a subsystem from scratch, check for markdown +files in its package directory. + When working on specific areas, refer to these detailed guides: **Operator Development:** +- `operator/ARCHITECTURE.md` - Internal architecture, reconciliation pipeline, and package layout - `operator/EXTENDING_CRDS.md` - How to add new fields to CRDs (Central/SecuredCluster) - `operator/DEFAULTING.md` - Defaulting mechanisms and best practices for CRD fields @@ -149,6 +185,20 @@ When working on specific areas, refer to these detailed guides: - `image/templates/CHART_TEMPLATING.md` - Meta-templating system, feature flags, and chart instantiation - `image/templates/CHANGING_CHARTS.md` - How to add/modify Helm values fields and cluster config +When making code changes, keep this in-code documentation up-to-date. + +## Development + +### Style and Conventions +- Go code follows golangci-lint standards +- Additional project specific style guide in `.github/go-coding-style.md` +- Protocol buffers have enforced style guidelines +- Shell scripts are checked with shellcheck +- UI code uses TypeScript with React conventions +- All generated code should not be manually edited +- Table-driven tests are defined as maps with the test name as the key. +- Tests use `assert` or `require` provided by `github.com/stretchr/testify`. + ### Development Workflow 1. Use `make install-dev-tools` to set up development environment 2. Run `make proto-generated-srcs` when protobuf files change @@ -168,11 +218,3 @@ When working on specific areas, refer to these detailed guides: - Use `docker run --rm --env POSTGRES_USER="$USER" --env POSTGRES_HOST_AUTH_METHOD=trust --publish 5432:5432 docker.io/library/postgres:15` for test setup - Integration tests in `/qa-tests-backend/` use Groovy/Spock framework - Tests marked with `//go:build sql_integration` require database connectivity - -### Style and Conventions -- Go code follows golangci-lint standards -- Additional project specific style guide in `.github/go-coding-style.md` -- Protocol buffers have enforced style guidelines -- Shell scripts are checked with shellcheck -- UI code uses TypeScript with React conventions -- All generated code should not be manually edited diff --git a/BUILD_IMAGE_VERSION b/BUILD_IMAGE_VERSION index 44776a28929be..490f8fe04f8a2 100644 --- a/BUILD_IMAGE_VERSION +++ b/BUILD_IMAGE_VERSION @@ -1 +1 @@ -stackrox-build-0.5.1 +stackrox-build-0.5.11 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c51d8dd67efd..ec4b4f4196c90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ + # Changelog This file helps upstream users learn about what is new in a release. @@ -11,6 +12,131 @@ Changes should still be described appropriately in JIRA/doc input pages, for inc ## [NEXT RELEASE] +### Added Features + +### Removed Features + +- ROX-35079: The `app.k8s.io/v1beta1/Application` CR is no longer shipped. + +### Deprecated Features + +### Technical Changes +- ROX-36534: roxctl binaries in the Central image are now stored as `.tar.gz` archives. Central extracts and streams the binary on download, so the user-facing download behavior is unchanged. +- ROX-35137: Central now prunes deployments whose cluster no longer exists in the database. +- ROX-37048: Central writes its version in `Rh-Central-Version` gRPC response header to authenticated clients. `roxctl` compares this version with its own version and reports a warning to stderr if the version skew is outside of a supported range. +- ROX-37315: Image exclusions no longer turn off deploy-time checks. Previously, an image exclusion on a policy with Build and Deploy stages stopped that policy from raising deploy-time violations or blocking any deployment. Image exclusions now apply only at Build, so after upgrading, affected policies raise deploy-time violations again and block deployments if enforcement is on. To skip apps at deploy time, use deployment exclusions. +- ROX-37265: Feature flag `ROX_SCANNER_V4_RED_HAT_LAYERS_RED_HAT_VULNS_ONLY` will now correctly filter non Red Hat vulnerabilities from RH image layers in newer Konflux built images where it previously had no affect. The flag must now be set on Central INSTEAD of Scanner V4 Matcher. +- ROX-37385: Scanner V4 node scans now record the operating system (for example `rhcos:9.6`) for OpenShift 4.19 and later nodes. RHCOS switched to RHEL-based dotted versions such as `9.6.20250701-0` in OpenShift 4.19, and Central previously stored an empty operating system for these nodes. + +- ROX-37149: `roxctl` now honors the `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` environment variables when connecting to Central over HTTP, so it works where direct egress to Central is blocked but a proxy is available. + +## [5.0.0] + + + +### Added Features + +- ROX-32148: Virtual machine scanning is now enabled by default. RHEL VMs created with OpenShift Virtualization can be scanned for package vulnerabilities after installing roxagent in the guest. +- ROX-34997: The Central CR now supports `spec.central.rolloutStrategy` (`Recreate` or `RollingUpdate`) to configure the central deployment rollout strategy. Default remains `Recreate`. +- ROX-35181: Administrative events are now exposed as configurable custom Prometheus metrics (`rox_central_admin_event_*`), aggregated by Type, Level, Domain, ResourceType, and ResourceName. Requires permission to read Administration resource, globally scoped. +- ROX-35545: Added ACL change as a file access operation for runtime policies. +- ROX-35546: File access policies now detect extended attribute (xattr) changes. +- ROX-32461: Red Hat OpenShift Data Foundation is now officially supported as an S3-compatible backup target. +- ROX-35962: On OCP, central API is exposed via a new `central-ocp` service, signed and rotated by OCP. +- ROX-35508: Scanner V4 now suppresses duplicate OSV.dev vulnerability records when Red Hat VEX data covers the same CVE for a Red Hat product image, showing Red Hat's own severity/CVSS/remediation data instead of a conflicting OSV.dev one. Enabled by default; disable via `ROX_SCANNER_V4_SUPPRESS_OSV_WITH_RED_HAT_VEX=false` if needed. +- ROX-34488: Added support for cosign signature discovery via OCI 1.1 referrers, including + DSSE envelope verification for sigstore bundle-format signatures. +- ROX-36858: Added more supported labels to the image and node vulnerability central custom metrics. +- Pruning (garbage collection) and vulnerability report scheduling now run in a separate `central-worker` deployment by default instead of inside Central. Disable with the Helm value `centralWorker.enabled=false` or `spec.centralWorker.enabled: false` in the Central CR. + +### Removed Features + +- Compliance container no longer collects Scanner V2 node inventories. Node scanning continues via Scanner V4 index reports, as long as Scanner V4 is enabled. +- ROX-36654: Removed legacy scanner (StackRox Scanner) across all installation methods. + +### Deprecated Features + +- ROX-26281: block creation of new GCR integrations. Users are directed to use Google Artifact Registry instead. +- ROX-35079: installation of the `app.k8s.io/v1beta1/Application` resource when central is installed is deprecated. It will be removed in a future release. +- The `priority` field on API responses for deployments, images, nodes, and components is deprecated and will be removed in a future release. Use the `risk_score` field on the same objects instead. Sorting by "Risk Priority" in search queries is also deprecated; sort by "Risk Score" instead. For clusters and namespaces, the `priority` field will be removed and replaced by a `risk_score` field where applicable. +- ROX-37014: The Images by severity, Nodes by severity, CVEs by severity, and Virtual machines by severity counts—including summary widgets and table columns—are deprecated and will be removed in a future release to improve page performance + +### Technical Changes +- ROX-36784: Scanner V4 node indexing on OpenShift now reads the host RPM database Claircore reports: SQLite on RHEL 9+ (`/usr/share/rpm`, `/usr/lib/sysimage/rpm`) and Berkeley DB on RHEL 8 (`/usr/share/rpm`, `/usr/lib/sysimage/rpm-ostree-base-db`). +- ROX-36824: Diagnostic bundles now redact the value of the `openshift.io/token-secret.value` annotation on secrets. Previously this OpenShift-managed annotation, which contains a plaintext service account token on generated dockercfg secrets, was included unredacted in the bundle. +- ROX-36660: The **Fixable → CVE is not yet fixable** policy criterion now matches Scanner V4 CVEs that have no fix version. Scanner V4 leaves `Fixed By` unset instead of empty (Scanner V2 always set an empty string), so the matcher previously skipped those CVEs. +- ROX-36490: The virtual machine enhanced data model (`ROX_VIRTUAL_MACHINES_ENHANCED_DATA_MODEL`) is now enabled by default. +- ROX-32969: The `roxctl-linux` symlink has been removed from the `/assets/downloads/cli/` directory inside the main container image. Only the architecture-specific binaries (`roxctl-linux-amd64`, `roxctl-linux-arm64`, etc.) remain. This change does not affect CLI downloads from the Central UI or any other supported download path. +- ROX-33078: Fixed telemetry gatherer failing to report database size metrics when using an external database. The database name is now read from the connection config instead of using the hardcoded default. +- ROX-35006: Go runtime upgraded to 1.26. Unbracketed IPv6 addresses (e.g. `2001:db8::1`) are no longer accepted; use bracketed format instead (e.g. `[2001:db8::1]:443`). +- ROX-34804: The machine access configuration for `config-controller` now validates the audience (`aud` claim) of the service account token. The expected audience is `central.stackrox.io`. When users have added their own role bindings to this machine access configuration, the audience check is not enforced by default to keep backwards compatibility. It is recommended to set the expected audience to `central.stackrox.io` after ensuring that all exchange tokens are being created with this audience claim. + +- ROX-34535: Fixes an issue where if ScannerV2 is disabled or unavailable on initial startup the central deployment leaks GRPC connections until the scanner becomes available. +- ROX-36509: Improved Central memory efficiency by optimizing process filter data structures in high-cardinality scenarios. The `ROX_PROCESS_FILTER_FAN_OUT_LEVELS` environment variable now accepts values up to 255; higher values are automatically clamped with a warning. +- ROX-36432: Fixed node scan results (`scan.scanTime`) silently freezing on installs whose default Scanner V4 integration lacked the `NODE_SCANNER` category. Central now restores the default categories on startup. Affected nodes move from Scanner V2 results to Scanner V4 results, so their vulnerability counts may change. + +## [4.11.0] + +### Added Features +- The config-controller now periodically reconciles SecurityPolicy CRs (default: every 30 minutes), detecting drift if policies are modified or deleted directly in Central. The interval is configurable via the `ROX_CONFIG_CONTROLLER_RECONCILE_INTERVAL` environment variable. +- ROX-26769: Central API for generating CRSs now supports specifying an upper bound for cluster + registrations using the new field "max_registrations". + roxctl's "central crs generate" supports specifying a maximum number of cluster registrations + using the new parameter "--max-clusters". +- ROX-24311: Detection and enforcement for pods/attach Kubernetes event. +- ROX-33099: New Operator Helm Chart is now the only recommended way to install on non-OpenShift clusters. +- ROX-33098 (Tech Preview): Effective path and Actual Path have been combined into a single File Path policy criterion. +- ROX-33156 (Tech Preview): A new default policy category called "File Activity Monitoring" is now available. +- ROX-33673: A new default policy has been added to detect missing egress NetworkPolicy associated with deployments. The policy is disabled by default. +- ROX-33336: The Operator now reads the cluster-wide TLS profile from `apiserver.config.openshift.io/cluster` on OpenShift and propagates it to all managed ACS components via environment variables. The Operator's own metrics server always honors the cluster TLS profile when running on OpenShift. +- ROX-26033: Compliance now tracks tailored profiles and custom rules from the Compliance Operator. Tailored profiles can be included in scan configurations, and their check results are shown in the Coverage page and CSV reports. +- ROX-34407: Deprecated fields to select optional columns NVD CVSS, EPSS Probability and Advisory from Vulnerability Reporting. These columns will be included by default next to similar columns. This change also affects column order in reports. +- ROX-33108: Added Component Version Column in Vulnerability Reporting. +- ROX-32865: Images are now uniquely identified by the combination of name and digest, rather + than by digest alone. This new data model resolves several long-standing issues when multiple + images share the same digest but have different names (e.g., different registries or tags): + - Deployments now correctly distinguish images with the same digest but different names, + so each deployment shows its own image reference in VM dashboards and vice-versa. + - Vulnerability exceptions (deferrals, false positives) can now be correctly scoped to a + specific image name. Previously, a deferral/false-positive created for one image name + would leak to all images sharing the same digest. + - Policies now evaluate correctly per the deployed/checked image name and respect its + vulnerability exceptions, rather than being affected by shared-digest exception leakage. + +### Removed Features + +### Deprecated Features + +- The following `roxctl` commands related to manifest-based and Helm-based installation are now deprecated. + They will be removed in a future release. Please use the operator for deployment management instead. + - `roxctl sensor generate {k8s,openshift}` + - `roxctl sensor get-bundle` + - `roxctl sensor generate-certs` + - `roxctl central generate {interactive,k8s,openshift}` + - `roxctl helm output {central-services,secured-cluster-services}` + - `roxctl helm derive-local-values` +- Deprecated gRPC endpoints for manifest-based and Helm-based installation: `GetCAConfig`, + `SensorUpgradeService` (all RPCs), `DeploymentFormat` enum, `PostCluster`, and + `PutCluster`. The REST endpoints `/api/extensions/clusters/zip`, + `/api/extensions/clusters/helm-config.yaml`, and `/api/extensions/helm-charts/` + are also deprecated. + They will be removed in a future release. Please use the operator for deployment management instead. +- Plaintext (non-TLS) Central endpoints, configured via the `ROX_PLAINTEXT_ENDPOINTS` environment + variable, are deprecated and will be removed in a future release. Modern load balancers and + ingress controllers support TLS passthrough, making plaintext endpoints unnecessary. + +### Technical Changes + +- ROX-34351: ACS components now always prevent Istio sidecar injection, removing the need for the + `env.istio` Helm value and `--istio-support` CLI flag. Existing Istio service mesh deployments + continue to work without configuration changes. +- OpenShift 3 support removed from all installation methods. +- ROX-34524: Fixed proxy bypass for non-HTTP schemes (e.g., TLS checks) when only `HTTPS_PROXY`/`HTTP_PROXY` are configured without `ALL_PROXY`. Previously, sensor's lazy TLS registry initialization would fail in proxy-only environments. + +## [4.10.0] + + + ### Added Features - ROX-31443: Automatic HTTP to HTTPS redirection is now enabled for Central OpenShift routes (passthrough and reencrypt). @@ -19,6 +145,11 @@ Changes should still be described appropriately in JIRA/doc input pages, for inc - ROX-31738: Added the `spec.customize.deploymentDefaults` field to Central and SecuredCluster CRDs, for configuring global default scheduling constraints for Deployments. This was previously possible on a per-component basis. - ROX-30094, ROX-30610, ROX-30740: Add new namespaces to Layered Products default config regex. - ROX-31960, ROX-32449: include and exclude filters for custom metrics. +- ROX-33603: Switch to UBI9/rhel9 base images. +- ROX-30641: Added a new policy criteria "Days Since CVE Fix Was Available". +- ROX-32630: The OpenShift console plugin integrates the ACS vulnerability management view into OpenShift console. It is enabled by default for operator-deployed secured clusters. +- Tech preview: operator-based installation available for community StackRox build. More information in [a separate README file](operator/install/README.md). +- ROX-30585, ROX-30196 (Tech Preview): Added file activity monitoring, including new policy criteria for deployment or node file activity. ### Removed Features - ROX-31727: `/v1/cve/requests` APIs (deprecated in 4.3.0) for managing vulnerability exceptions have been removed. @@ -28,7 +159,15 @@ Changes should still be described appropriately in JIRA/doc input pages, for inc ### Deprecated Features +- ROX-32851: The `roxctl netpol generate`, `roxctl netpol connectivity map`, and `roxctl netpol connectivity diff` commands are deprecated because they rely on the unmaintained NP-Guard library and will be removed in a future release. +- ROX-32867: The Compliance V1 feature has been deprecated, and it is planned to be removed in a future release. This includes: + - The Compliance Dashboard + - The Compliance V1 API endpoints + - The Compliance Configuration Management Board + ### Technical Changes +- ROX-32239: Process indicator filtering is now configurable via environment variables. `ROX_PROCESS_FILTER_MAX_EXACT_PATH_MATCHES` is the maximum number of times an exact path (same deployment+container+process+args) can appear before being filtered. `ROX_PROCESS_FILTER_FAN_OUT_LEVELS` is the maximum number of unique process executable paths per container. `ROX_PROCESS_FILTER_MAX_PROCESS_PATHS` is an integer array where each value represents the number of distinct child arguments for that arg position. +- ROX-32679: Added preset process filtering modes via `ROX_PROCESS_FILTER_MODE` environment variable. Available modes: `default`, `aggressive`, and `minimal`. Setting this environment variables controls the same parameters as the `ROX_PROCESS_FILTER_MAX_EXACT_PATH_MATCHES`, `ROX_PROCESS_FILTER_FAN_OUT_LEVELS`, and `ROX_PROCESS_FILTER_MAX_PROCESS_PATHS` environment variables. Individual filter settings can override preset values. - ROX-30769: Update Node.js requirement for ui folder to 22.13.0 - ROX-31295: The lower limit for `ROX_MAX_PARALLEL_IMAGE_SCAN_INTERNAL` on Sensor has been reduced to one (from 10). - ROX-32125: The operator now adopts secrets that have the `app.stackrox.io/managed-by: operator` label but no `ownerReferences`. This fixes reconciliation failures after backup/restore operations that strip `ownerReferences` from secrets. diff --git a/CI_TRIAGE.md b/CI_TRIAGE.md index 43136d780564c..79f80deab0c06 100644 --- a/CI_TRIAGE.md +++ b/CI_TRIAGE.md @@ -8,7 +8,6 @@ To enable automated CI triage capabilities, you need to configure the following - [mcp-atlassian](https://github.com/sooperset/mcp-atlassian) - [Github MCP](https://github.com/github/github-mcp-server/blob/main/docs/installation-guides/install-claude.md) -- [Prow MCP](https://github.com/redhat-community-ai-tools/prowject) Example MCPs configuration template: - Install gopls https://go.dev/gopls/#installation diff --git a/COLLECTOR_VERSION b/COLLECTOR_VERSION index cbd15a0dc8b42..12a33ce182018 100644 --- a/COLLECTOR_VERSION +++ b/COLLECTOR_VERSION @@ -1 +1 @@ -3.23.x-110-gae27601472 +3.25.0-188-g237a0275dc diff --git a/FACT_VERSION b/FACT_VERSION index 6454293b3009a..3d828935ed6bc 100644 --- a/FACT_VERSION +++ b/FACT_VERSION @@ -1 +1 @@ -0.1.x-103-gacf65d8889 +0.3.x-340-gcbfa5e1fc2 diff --git a/Makefile b/Makefile index fd8462d39ad70..54fca4db3901d 100644 --- a/Makefile +++ b/Makefile @@ -8,15 +8,28 @@ BENCHTIME ?= 1x BENCHTIMEOUT ?= 20m BENCHCOUNT ?= 1 +CONTAINER_ENGINE ?= docker +export CONTAINER_ENGINE + podman = +# If CONTAINER_ENGINE is explicitly set to podman, detect it +ifeq ($(CONTAINER_ENGINE),podman) + podman = yes # docker --version might not contain any traces of podman in the latest # version, search for more output -ifneq (,$(findstring podman,$(shell docker --version 2>/dev/null))) +else ifneq (,$(findstring podman,$(shell $(CONTAINER_ENGINE) --version 2>/dev/null))) + podman = yes +else ifneq (,$(findstring Podman,$(shell $(CONTAINER_ENGINE) version 2>/dev/null))) podman = yes endif -ifneq (,$(findstring Podman,$(shell docker version 2>/dev/null))) +# if docker is not available and podman is installed, use podman instead +ifeq (,$(shell command -v docker 2>/dev/null)) +ifneq (,$(shell command -v podman 2>/dev/null)) podman = yes + CONTAINER_ENGINE=podman endif +endif + ifdef podman # Disable selinux for local podman builds. @@ -118,11 +131,14 @@ include make/gotools.mk $(call go-tool, BUF_BIN, github.com/bufbuild/buf/cmd/buf, tools/proto) $(call go-tool, GOLANGCILINT_BIN, github.com/golangci/golangci-lint/v2/cmd/golangci-lint, tools/linters) $(call go-tool, EASYJSON_BIN, github.com/mailru/easyjson/easyjson) +$(call go-tool, OSSLS_BIN, github.com/stackrox/ossls, tools/build) $(call go-tool, ROXVET_BIN, ./tools/roxvet) $(call go-tool, STRINGER_BIN, golang.org/x/tools/cmd/stringer) $(call go-tool, MOCKGEN_BIN, go.uber.org/mock/mockgen) $(call go-tool, GO_JUNIT_REPORT_BIN, github.com/jstemmer/go-junit-report/v2, tools/test) $(call go-tool, PROTOLOCK_BIN, github.com/nilslice/protolock/cmd/protolock, tools/linters) +$(call go-tool, RATCHET_BIN, github.com/sethvargo/ratchet, tools/linters) +$(call go-tool, ACTIONLINT_BIN, actionlint.kjanat.dev/cmd/actionlint, tools/linters) $(call go-tool, GOVULNCHECK_BIN, golang.org/x/vuln/cmd/govulncheck, tools/linters) $(call go-tool, IMAGE_PREFETCHER_DEPLOY_BIN, github.com/stackrox/image-prefetcher/deploy, tools/test) $(call go-tool, PROMETHEUS_METRIC_PARSER_BIN, github.com/stackrox/prometheus-metric-parser, tools/test) @@ -136,7 +152,9 @@ style: golangci-lint style-slim .PHONY: style-slim style-slim: \ blanks \ + check-cert-watcher-sync \ check-service-protos \ + github-actions-pin-check \ newlines \ no-large-files \ openshift-ci-style \ @@ -162,10 +180,6 @@ ifdef CI @echo 'The environment indicates we are in CI; running linters in check mode.' @echo 'If this fails, run `make style`.' $(GOLANGCILINT_BIN) --version - @echo "Running with no tags and no tests..." - @# The first run is meant to have limited scope to warmup the cache. - @# Adding it as first allowed to shorten the runtime of the following runs to about 5 min each - $(GOLANGCILINT_BIN) run $(GOLANGCILINT_FLAGS) --tests=false @echo "Running with no tags..." @# We need to enable unused linter here as it will not work without tests or in release tag. $(GOLANGCILINT_BIN) run $(GOLANGCILINT_FLAGS) --enable=unused @@ -209,6 +223,16 @@ update-shellcheck-skip: $(SILENT)rm -f scripts/style/shellcheck_skip.txt $(SILENT)$(BASE_DIR)/scripts/style/shellcheck.sh update_failing_list +.PHONY: github-actions-lint +github-actions-lint: $(ACTIONLINT_BIN) + @echo "+ $@" + $(ACTIONLINT_BIN) -color -config-file .github/actionlint.yaml .github/workflows/*.yaml .github/workflows/*.yml + +.PHONY: github-actions-pin-check +github-actions-pin-check: $(RATCHET_BIN) + @echo "+ $@" + $(RATCHET_BIN) lint -format actions .github/workflows/*.yaml .github/workflows/*.yml .github/actions/*/action.yaml + .PHONY: fast-central-build fast-central-build: central-build-nodeps @@ -225,7 +249,7 @@ config-controller-build-nodeps: .PHONY: fast-central fast-central: deps @echo "+ $@" - docker run $(DOCKER_OPTS) -e CGO_ENABLED --rm $(GOPATH_WD_OVERRIDES) $(LOCAL_VOLUME_ARGS) $(BUILD_IMAGE) make fast-central-build + $(CONTAINER_ENGINE) run $(DOCKER_OPTS) -e CGO_ENABLED --rm $(GOPATH_WD_OVERRIDES) $(LOCAL_VOLUME_ARGS) $(BUILD_IMAGE) make fast-central-build $(SILENT)$(BASE_DIR)/scripts/k8s/kill-pod.sh central # fast is a dev mode options when using local dev @@ -243,7 +267,7 @@ fast-sensor-kubernetes: sensor-kubernetes-build-dockerized .PHONY: fast-migrator fast-migrator: @echo "+ $@" - docker run $(DOCKER_OPTS) -e CGO_ENABLED --rm $(GOPATH_WD_OVERRIDES) $(LOCAL_VOLUME_ARGS) $(BUILD_IMAGE) make fast-migrator-build + $(CONTAINER_ENGINE) run $(DOCKER_OPTS) -e CGO_ENABLED --rm $(GOPATH_WD_OVERRIDES) $(LOCAL_VOLUME_ARGS) $(BUILD_IMAGE) make fast-migrator-build .PHONY: fast-migrator-build fast-migrator-build: migrator-build-nodeps @@ -253,6 +277,12 @@ migrator-build-nodeps: @echo "+ $@" $(GOBUILD) migrator +.PHONY: check-cert-watcher-sync +check-cert-watcher-sync: + @echo "+ $@" + @diff image/postgres/scripts/cert-watcher.sh scanner/image/db/scripts/cert-watcher.sh \ + || (echo "ERROR: cert-watcher.sh files are out of sync" && exit 1) + .PHONY: check-service-protos check-service-protos: @echo "+ $@" @@ -342,22 +372,23 @@ config-controller-gen: .PHONY: generated-srcs generated-srcs: go-generated-srcs config-controller-gen -deps: $(shell find $(BASE_DIR) -name "go.sum") - @echo "+ $@" - $(SILENT)touch deps - -%/go.sum: %/go.mod - $(SILENT)cd $* - @echo "+ $@" - $(SILENT)$(eval GOMOCK_REFLECT_DIRS=`find . -type d -name 'gomock_reflect_*'`) - $(SILENT)test -z $(GOMOCK_REFLECT_DIRS) || { echo "Found leftover gomock directories. Please remove them and rerun make deps!"; echo $(GOMOCK_REFLECT_DIRS); exit 1; } ifdef CI - $(SILENT)GOTOOLCHAIN=local go mod tidy || { >&2 echo "Go toolchain does not match with installed Go version. This is a compatibility check that prevents breaking downstream builds. If you really need to update the toolchain version, ask in #forum-acs-golang" ; exit 1 ; } - $(SILENT)git diff --exit-code -- go.mod go.sum || { echo "go.mod/go.sum files were updated after running 'go mod tidy', run this command on your local machine and commit the results." ; exit 1 ; } +# In CI, go.mod/go.sum must be committed — not auto-updated. +# go mod tidy is validated by check-generated-files (scripts/ci/jobs/check-generated.sh). +deps: + @echo "deps: CI is set — skipping go mod tidy (validated by check-generated-files)" >&2 else +deps: go.mod + @echo "+ $@" + $(SILENT)GOMOCK_REFLECT_DIRS=$$(find . -type d -name 'gomock_reflect_*'); \ + if [ -n "$$GOMOCK_REFLECT_DIRS" ]; then \ + echo "Found leftover gomock directories. Please remove them and rerun make deps!"; \ + echo "$$GOMOCK_REFLECT_DIRS"; \ + exit 1; \ + fi $(SILENT)go mod tidy -endif $(SILENT)touch $@ +endif .PHONY: clean-deps clean-deps: @@ -451,8 +482,8 @@ bin/$(HOST_OS)_$(GOARCH)/admission-control: build-prep .PHONY: build-volumes build-volumes: $(SILENT)mkdir -p $(CURDIR)/linux-gocache - $(SILENT)docker volume inspect $(GOPATH_VOLUME_NAME) >/dev/null 2>&1 || docker volume create $(GOPATH_VOLUME_NAME) - $(SILENT)docker volume inspect $(GOCACHE_VOLUME_NAME) >/dev/null 2>&1 || docker volume create $(GOCACHE_VOLUME_NAME) + $(SILENT)$(CONTAINER_ENGINE) volume inspect $(GOPATH_VOLUME_NAME) >/dev/null 2>&1 || $(CONTAINER_ENGINE) volume create $(GOPATH_VOLUME_NAME) + $(SILENT)$(CONTAINER_ENGINE) volume inspect $(GOCACHE_VOLUME_NAME) >/dev/null 2>&1 || $(CONTAINER_ENGINE) volume create $(GOCACHE_VOLUME_NAME) .PHONY: main-build main-build: build-prep main-build-dockerized @@ -461,12 +492,12 @@ main-build: build-prep main-build-dockerized .PHONY: sensor-build-dockerized sensor-build-dockerized: build-volumes @echo "+ $@" - docker run $(DOCKER_OPTS) --rm -e CI -e BUILD_TAG -e GOTAGS -e DEBUG_BUILD -e CGO_ENABLED $(GOPATH_WD_OVERRIDES) $(LOCAL_VOLUME_ARGS) $(BUILD_IMAGE) make sensor-build + $(CONTAINER_ENGINE) run $(DOCKER_OPTS) --rm -e CI -e BUILD_TAG -e GOTAGS -e DEBUG_BUILD -e CGO_ENABLED $(GOPATH_WD_OVERRIDES) $(LOCAL_VOLUME_ARGS) $(BUILD_IMAGE) make sensor-build .PHONY: sensor-kubernetes-build-dockerized sensor-kubernetes-build-dockerized: build-volumes @echo "+ $@" - docker run $(DOCKER_OPTS) -e CI -e BUILD_TAG -e GOTAGS -e DEBUG_BUILD -e CGO_ENABLED $(GOPATH_WD_OVERRIDES) $(LOCAL_VOLUME_ARGS) $(BUILD_IMAGE) make sensor-kubernetes-build + $(CONTAINER_ENGINE) run $(DOCKER_OPTS) -e CI -e BUILD_TAG -e GOTAGS -e DEBUG_BUILD -e CGO_ENABLED $(GOPATH_WD_OVERRIDES) $(LOCAL_VOLUME_ARGS) $(BUILD_IMAGE) make sensor-kubernetes-build .PHONY: sensor-build sensor-build: @@ -480,18 +511,23 @@ sensor-kubernetes-build: .PHONY: main-build-dockerized main-build-dockerized: build-volumes @echo "+ $@" - docker run $(DOCKER_OPTS) -i -e RACE -e CI -e BUILD_TAG -e SHORTCOMMIT -e GOTAGS -e DEBUG_BUILD -e CGO_ENABLED --rm $(GOPATH_WD_OVERRIDES) $(LOCAL_VOLUME_ARGS) $(BUILD_IMAGE) make main-build-nodeps + $(CONTAINER_ENGINE) run $(DOCKER_OPTS) -i -e RACE -e CI -e BUILD_TAG -e SHORTCOMMIT -e GOTAGS -e DEBUG_BUILD -e CGO_ENABLED --rm $(GOPATH_WD_OVERRIDES) $(LOCAL_VOLUME_ARGS) $(BUILD_IMAGE) make main-build-nodeps .PHONY: main-build-nodeps main-build-nodeps: $(GOBUILD) \ central \ + central/worker \ compliance/cmd/compliance \ config-controller \ migrator \ + operator/cmd \ + scanner/cmd/scanner \ sensor/admission-control \ sensor/kubernetes \ - sensor/upgrader + sensor/upgrader \ + compliance/virtualmachines/roxagent + mv bin/linux_$(GOARCH)/cmd bin/linux_$(GOARCH)/stackrox-operator ifndef CI CGO_ENABLED=0 $(GOBUILD) roxctl endif @@ -523,6 +559,13 @@ swagger-docs: $(MERGED_API_SWAGGER_SPEC) $(MERGED_API_SWAGGER_SPEC_V2) UNIT_TEST_PACKAGES ?= ./... +# Root-module test discovery must not pass tests from nested Go modules to the +# root module's go list. Those modules are tested separately in their own +# working directories. +NESTED_GO_MODULE_DIRS := $(shell find . -mindepth 2 -name go.mod -exec dirname {} \; | sed 's@^\./@@') +NESTED_GO_MODULE_REGEX := $(shell printf '%s\n' $(NESTED_GO_MODULE_DIRS) | paste -sd'|' -) +NESTED_GO_MODULES_WITH_TESTS := $(shell for dir in $(NESTED_GO_MODULE_DIRS); do git ls-files "$$dir/*_test.go" | grep -q . && echo "$$dir"; done) + .PHONY: test-prep test-prep: @echo "+ $@" @@ -532,8 +575,15 @@ test-prep: go-unit-tests: build-prep test-prep set -o pipefail ; \ CGO_ENABLED=1 GOEXPERIMENT=cgocheck2 MUTEX_WATCHDOG_TIMEOUT_SECS=30 GOTAGS=$(GOTAGS),test scripts/go-test.sh -timeout 25m -race -cover -coverprofile test-output/coverage.out -v \ - $(shell git ls-files -- '*_test.go' | sed -e 's@^@./@g' | xargs -n 1 dirname | sort | uniq | xargs go list| grep -v '^github.com/stackrox/rox/tests$$' | grep -Ev $(UNIT_TEST_IGNORE)) \ + $(shell git ls-files -- '*_test.go' | grep -v -E '^($(NESTED_GO_MODULE_REGEX))(/|$$)' | scripts/go-files-to-packages.sh | grep -Ev $(UNIT_TEST_IGNORE)) \ | tee $(GO_TEST_OUTPUT_PATH) + # Nested modules have independent coverage profiles, so keep their coverage + # out of the root profile while including their verbose output in JUnit. + @set -o pipefail; \ + for module_dir in $(NESTED_GO_MODULES_WITH_TESTS); do \ + echo "Run tests in $$module_dir"; \ + (cd "$$module_dir" && go test -buildvcs=false -race -v ./...) 2>&1 | tee -a $(GO_TEST_OUTPUT_PATH); \ + done # Exercise the logging package for all supported logging levels to make sure that initialization works properly @echo "Run log tests" for encoding in console json; do \ @@ -554,19 +604,19 @@ sensor-pipeline-benchmark: build-prep test-prep go-postgres-unit-tests: build-prep test-prep set -o pipefail ; \ CGO_ENABLED=1 GOEXPERIMENT=cgocheck2 MUTEX_WATCHDOG_TIMEOUT_SECS=30 GOTAGS=$(GOTAGS),test,sql_integration scripts/go-test.sh -timeout 15m -race -cover -coverprofile test-output/coverage.out -v \ - $(shell git grep -rl "//go:build sql_integration" central pkg tools | sed -e 's@^@./@g' | xargs -n 1 dirname | sort | uniq | xargs go list -tags sql_integration | grep -v '^github.com/stackrox/rox/tests$$' | grep -Ev $(UNIT_TEST_IGNORE)) \ + $(shell git grep -rl "//go:build sql_integration" central pkg tools | scripts/go-files-to-packages.sh -tags sql_integration | grep -Ev $(UNIT_TEST_IGNORE)) \ | tee $(GO_TEST_OUTPUT_PATH) @# The -p 1 passed to go test is required to ensure that tests of different packages are not run in parallel, so as to avoid conflicts when interacting with the DB. set -o pipefail ; \ CGO_ENABLED=1 GOEXPERIMENT=cgocheck2 MUTEX_WATCHDOG_TIMEOUT_SECS=30 GOTAGS=$(GOTAGS),test,sql_integration scripts/go-test.sh -p 1 -race -cover -coverprofile test-output/migrator-coverage.out -v \ - $(shell git grep -rl "//go:build sql_integration" migrator | sed -e 's@^@./@g' | xargs -n 1 dirname | sort | uniq | xargs go list -tags sql_integration | grep -v '^github.com/stackrox/rox/tests$$' | grep -Ev $(UNIT_TEST_IGNORE)) \ + $(shell git grep -rl "//go:build sql_integration" migrator | scripts/go-files-to-packages.sh -tags sql_integration | grep -Ev $(UNIT_TEST_IGNORE)) \ | tee -a $(GO_TEST_OUTPUT_PATH) .PHONY: go-postgres-bench-tests go-postgres-bench-tests: build-prep test-prep set -o pipefail ; \ CGO_ENABLED=1 GOEXPERIMENT=cgocheck2 MUTEX_WATCHDOG_TIMEOUT_SECS=30 GOTAGS=$(GOTAGS),test,sql_integration scripts/go-test.sh -run=nonthing -bench=. -benchtime=$(BENCHTIME) -benchmem -timeout $(BENCHTIMEOUT) -count $(BENCHCOUNT) -v \ - $(shell git grep -rl "testing.B" central pkg migrator tools | sed -e 's@^@./@g' | xargs -n 1 dirname | sort | uniq | xargs go list -tags sql_integration | grep -v '^github.com/stackrox/rox/tests$$' | grep -Ev $(UNIT_TEST_IGNORE)) \ + $(shell git grep -rl "testing.B" central pkg migrator tools | scripts/go-files-to-packages.sh -tags sql_integration | grep -Ev $(UNIT_TEST_IGNORE)) \ | tee $(GO_TEST_OUTPUT_PATH) .PHONY: shell-unit-tests @@ -574,6 +624,7 @@ shell-unit-tests: @echo "+ $@" $(SILENT)mkdir -p shell-test-output bats --print-output-on-failure --verbose-run --recursive --report-formatter junit --output shell-test-output \ + scanner/image/db/bats \ scripts \ tests/e2e/bats @@ -631,7 +682,7 @@ main-image: all-builds make docker-build-main-image .PHONY: docker-build-main-image -docker-build-main-image: copy-binaries-to-image-dir central-db-image +docker-build-main-image: copy-binaries-to-image-dir $(DOCKERBUILD) \ -t stackrox/main:$(TAG) \ -t $(DEFAULT_IMAGE_REGISTRY)/main:$(TAG) \ @@ -662,6 +713,7 @@ docker-build-roxctl-image: copy-go-binaries-to-image-dir: cp bin/linux_$(GOARCH)/central image/rhel/bin/central cp bin/linux_$(GOARCH)/config-controller image/rhel/bin/config-controller + cp bin/linux_$(GOARCH)/worker image/rhel/bin/central-worker ifdef CI cp bin/linux_amd64/roxctl image/rhel/bin/roxctl-linux-amd64 cp bin/linux_arm64/roxctl image/rhel/bin/roxctl-linux-arm64 @@ -681,6 +733,7 @@ endif cp bin/linux_$(GOARCH)/upgrader image/rhel/bin/sensor-upgrader cp bin/linux_$(GOARCH)/admission-control image/rhel/bin/admission-control cp bin/linux_$(GOARCH)/compliance image/rhel/bin/compliance + cp bin/linux_$(GOARCH)/roxagent image/rhel/bin/roxagent # Workaround to bug in lima: https://github.com/lima-vm/lima/issues/602 find image/rhel/bin -not -path "*/.*" -type f -exec chmod +x {} \; @@ -700,7 +753,7 @@ scale-image: scale-build clean-image cp bin/linux_$(GOARCH)/profiler scale/image/rhel/bin/profiler cp bin/linux_$(GOARCH)/chaos scale/image/rhel/bin/chaos chmod +w scale/image/rhel/bin/* - docker build \ + $(CONTAINER_ENGINE) build \ -t stackrox/scale:$(TAG) \ -t quay.io/rhacs-eng/scale:$(TAG) \ -f scale/image/Dockerfile scale @@ -709,7 +762,7 @@ webhookserver-image: webhookserver-build -mkdir webhookserver/bin cp bin/linux_$(GOARCH)/webhookserver webhookserver/bin/webhookserver chmod +w webhookserver/bin/webhookserver - docker build \ + $(CONTAINER_ENGINE) build \ -t stackrox/webhookserver:1.2 \ -t quay.io/rhacs-eng/webhookserver:1.2 \ -f webhookserver/Dockerfile webhookserver @@ -718,7 +771,7 @@ syslog-image: syslog-build -mkdir qa-tests-backend/test-images/syslog/bin cp bin/linux_$(GOARCH)/syslog qa-tests-backend/test-images/syslog/bin/syslog chmod +w qa-tests-backend/test-images/syslog/bin/syslog - docker build \ + $(DOCKERBUILD) \ -t stackrox/qa:syslog_server_1_0 \ -t quay.io/rhacs-eng/qa:syslog_server_1_0 \ -f qa-tests-backend/test-images/syslog/Dockerfile qa-tests-backend/test-images/syslog @@ -773,14 +826,14 @@ product-branding: @echo $(ROX_PRODUCT_BRANDING) .PHONY: ossls-audit -ossls-audit: deps - ossls version - ossls audit +ossls-audit: deps $(OSSLS_BIN) + $(OSSLS_BIN) version + $(OSSLS_BIN) audit .PHONY: ossls-notice -ossls-notice: deps - ossls version - ossls audit --export image/rhel/THIRD_PARTY_NOTICES +ossls-notice: deps $(OSSLS_BIN) + $(OSSLS_BIN) version + $(OSSLS_BIN) audit --export image/rhel/THIRD_PARTY_NOTICES .PHONY: collector-tag collector-tag: @@ -808,28 +861,17 @@ install-dev-tools: gotools-all @echo "+ $@" .PHONY: roxvet -roxvet: skip-dirs := operator/pkg/clientset +roxvet: skip-dirs := operator/pkg/clientset /generated/ /mocks roxvet: $(ROXVET_BIN) @echo "+ $@" @# TODO(ROX-7574): Add options to ignore specific files or paths in roxvet $(SILENT)go list -e ./... \ | $(foreach d,$(skip-dirs),grep -v '$(d)' |) \ - xargs -n 1000 go vet -vettool "$(ROXVET_BIN)" -donotcompareproto -gogoprotofunctions -tags "sql_integration test_e2e test race destructive integration scanner_db_integration compliance externalbackups" - $(SILENT)go list -e ./... \ - | $(foreach d,$(skip-dirs),grep -v '$(d)' |) \ - xargs -n 1000 go vet -vettool "$(ROXVET_BIN)" + xargs -n 1000 go vet -vettool "$(ROXVET_BIN)" -tags "sql_integration test_e2e test_e2e_vm test race destructive integration scanner_db_integration compliance externalbackups" ########## ## Misc ## ########## -.PHONY: clean-offline-bundle -clean-offline-bundle: - $(SILENT)find scripts/offline-bundle -name '*.img' -delete -o -name '*.tgz' -delete -o -name 'bin' -type d -exec rm -r "{}" \; - -.PHONY: offline-bundle -offline-bundle: clean-offline-bundle - $(SILENT)./scripts/offline-bundle/create.sh - .PHONY: check-debugger check-debugger: /usr/bin/env DEBUG_BUILD="$(DEBUG_BUILD)" BUILD_TAG="$(BUILD_TAG)" TAG="$(TAG)" ./scripts/check-debugger.sh @@ -853,6 +895,10 @@ mitre: bootstrap_migration: $(SILENT)if [[ "x${DESCRIPTION}" == "x" ]]; then echo "Please set a description for your migration in the DESCRIPTION environment variable"; else go run tools/generate-helpers/bootstrap-migration/main.go --root . --description "${DESCRIPTION}" ;fi +.PHONY: bootstrap_background_migration +bootstrap_background_migration: + $(SILENT)if [[ "x${DESCRIPTION}" == "x" ]]; then echo "Please set a description for your migration in the DESCRIPTION environment variable"; else go run tools/generate-helpers/bootstrap-background-migration/main.go --root . --description "${DESCRIPTION}" ;fi + .PHONY: image-prefetcher-deploy-bin image-prefetcher-deploy-bin: $(IMAGE_PREFETCHER_DEPLOY_BIN) ## download and install diff --git a/README.md b/README.md index 6d647e0be8dde..4d4087e11c61c 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ - [Deploying StackRox](#deploying-stackrox) - [Quick Installation using Helm](#quick-installation-using-helm) - [Manual Installation using Helm](#manual-installation-using-helm) + - [Installation using the Operator](#installation-using-the-operator) - [Installation via Scripts](#installation-via-scripts) - [Kubernetes Distributions (EKS, AKS, GKE)](#kubernetes-distributions-eks-aks-gke) - [OpenShift](#openshift) @@ -57,6 +58,13 @@ To [report a vulnerability or bug](https://github.com/stackrox/stackrox/security ## Deploying StackRox +### Installation using the Operator + +As of release 4.10 it's possible to [install StackRox using the operator](operator/install). + +> [!WARNING] +> The following installation methods are deprecated. + ### Quick Installation using Helm StackRox offers quick installation via Helm Charts. Follow the [Helm Installation Guide](https://helm.sh/docs/intro/install/) to get `helm` CLI on your system. @@ -294,11 +302,11 @@ After a few minutes, all resources should be deployed. -### Accessing the StackRox User Interface (UI) +## Accessing the StackRox User Interface (UI)
Click to expand -After the deployment has completed (Helm or script install) a port-forward should exist, so you can connect to https://localhost:8000/. Run the following +After the deployment has completed (operator, Helm or script install) a port-forward should exist, so you can connect to https://localhost:8000/. Run the following ```sh kubectl port-forward -n 'stackrox' svc/central "8000:443" @@ -317,14 +325,13 @@ Then go to https://localhost:8000/ in your web browser. --- ## Development +- Start with [**pull request guidelines**](./.github/contributing.md) if you plan to make a contribution +- Look at [**shepherding**](./.github/shepherding.md) for non-trivial code submissions +- Check out our [**AI policy**](./.github/using-llms.md) +- **Go coding style guide**: [go-coding-style.md](./.github/go-coding-style.md) - **UI Dev Docs**: Refer to [ui/README.md](./ui/README.md) - - **E2E Dev Docs**: Refer to [qa-tests-backend/README.md](./qa-tests-backend/README.md) -- **Pull request guidelines**: [contributing.md](./.github/contributing.md) - -- **Go coding style guide**: [go-coding-style.md](./.github/go-coding-style.md) - ### Quickstart #### Build Tooling diff --git a/ROXIE_VERSION b/ROXIE_VERSION new file mode 100644 index 0000000000000..1f77169993b92 --- /dev/null +++ b/ROXIE_VERSION @@ -0,0 +1 @@ +0.4.13 diff --git a/SCANNER_VERSION b/SCANNER_VERSION index 0aede1756389d..a8877242d134b 100644 --- a/SCANNER_VERSION +++ b/SCANNER_VERSION @@ -1 +1 @@ -2.38.x-24-gcd5fb7a6d1 +2.40.x-150-g4a18014bc5 diff --git a/SECURITY.md b/SECURITY.md index b597cb17647f0..37a22a5f32203 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,4 +1,79 @@ -# Reporting a Vulnerability +# Reporting a Security Vulnerability or Incident -If you've found a security issue that you'd like to disclose confidentially please contact Red Hat's Product Security team. -Details at https://access.redhat.com/security/team/contact +Please do not report security vulnerabilities or security incidents via public +channels such as GitHub Issues or Pull Requests. To ensure coordinated +disclosure, direct all security questions and vulnerability reports to: + +- **Email**: secalert@redhat.com + +Further details at https://access.redhat.com/security/team/contact + +## Submission Guidelines + +To help us triage and resolve the issue efficiently, please include the +following in your report: + +- **Title**: A concise, descriptive summary of the issue. +- **Reporter Details**: Your name or handle and affiliation. +- **Technical Description**: Detailed information regarding the vulnerability. +- **Affected Versions**: The specific version(s) or range(s) of software tested. +- **Reproduction Steps**: A minimal, functional example to reproduce the issue. +- **Impact Assessment**: Potential exploit scenarios and perceived severity (optional). +- **Suggested Fix**: Any proposed patches or mitigations (optional). +- **Disclosure Status**: Whether this has been shared with other parties or published, and your plan for future sharing (e.g., at a conference). + +## Response Timeline + +We aim to provide an initial acknowledgement of your report within 10 business +days. All confirmed security vulnerabilities and incidents will be addressed +according to severity level and impact on the project. + +## Remediation Process + +### Fix Development + +1. Embargoed fixes are developed in **private branches** or forks to prevent premature disclosure. +2. All security patches undergo **peer review** by at least one other maintainer with relevant domain knowledge. +3. Patches are tested against the **reproduction steps** from the original report and validated against **regression test suites**. + +### Release and Distribution + +1. Security fixes are distributed through the project's **normal release channels** (container images, operator index, helm registry). +2. Release notes must reference the associated CVE ID(s) and advisory. + +## Coordinated Vulnerability Disclosure + +### Disclosure Model + +The StackRox project follows a **Coordinated Vulnerability Disclosure (CVD)** model aligned with OpenSSF guidance: + +1. **Embargo Period:** Vulnerability details remain confidential until a fix is available and released. +2. **Disclosure Timing:** Public disclosure occurs simultaneously with or immediately after the availability of a patched release. +3. **Pre-notification:** For Critical severity issues affecting widely-deployed components, the Security Team may provide advance notice to major downstream users under embargo, up to 7 days before public disclosure. + +## Supported Versions + +We regularly perform patch releases for at least two most recent minor versions, +which contain fixes for relevant security vulnerabilities and important bugs. +Prior releases might receive critical security fixes on a best-effort basis. +However, we cannot guarantee that security fixes will be back-ported to older +unsupported versions. + +# Secure Development Practices + +We follow established industry best practices for secure development incl. but not limited to + +- Secured version control of source code +- Mandatory peer reviews +- Build time vulnerability scanning +- Automatic update of 3rd party dependencies +- Automatic end 2 end testing of security relevant features (e.g. authentication) + +# EU Cyber Resilience Act — Open Source Steward Statement + +This project is stewarded by **Red Hat, Inc.**, an open source software steward as defined in Article 3(14) of the [EU Cyber Resilience Act (Regulation 2024/2847)](https://eur-lex.europa.eu/eli/reg/2024/2847/oj/eng). +Contact: [cra-steward@redhat.com](mailto:cra-steward@redhat.com) + +For guidance on reporting actively exploited vulnerabilities and severe incidents under the CRA, see [Reporting Actively Exploited Vulnerabilities and Severe Incidents](https://github.com/RedHatProductSecurity/CRA/blob/main/Guidelines/eu-cra-incident-and-vulnerability-reporting-guidelines.md). + +Refer to [Red Hat's security practices and vulnerability management policy](https://access.redhat.com/security/) for detailed information. diff --git a/central/administration/events/datastore/internal/writer/writer_impl.go b/central/administration/events/datastore/internal/writer/writer_impl.go index c290b6bb20d5b..f1cced29f9578 100644 --- a/central/administration/events/datastore/internal/writer/writer_impl.go +++ b/central/administration/events/datastore/internal/writer/writer_impl.go @@ -7,6 +7,8 @@ import ( "github.com/pkg/errors" "github.com/stackrox/rox/central/administration/events/datastore/internal/store" + "github.com/stackrox/rox/central/metrics" + "github.com/stackrox/rox/central/metrics/custom/refresh" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/administration/events" "github.com/stackrox/rox/pkg/protoutils" @@ -75,7 +77,7 @@ func (c *writerImpl) flushNoLock(ctx context.Context) error { // After successful DB operations, reset the buffer. c.resetNoLock() - + refresh.RefreshTracker(metrics.AdminEvents) return nil } diff --git a/central/administration/usage/datastore/securedunits/datastore_impl_test.go b/central/administration/usage/datastore/securedunits/datastore_impl_test.go index 411eab9679a0c..670f7d66d0fc7 100644 --- a/central/administration/usage/datastore/securedunits/datastore_impl_test.go +++ b/central/administration/usage/datastore/securedunits/datastore_impl_test.go @@ -100,7 +100,7 @@ func (suite *UsageDataStoreTestSuite) TestWalk() { const N = page + 2 first := time.Now() last := first - for i := 0; i < N; i++ { + for i := range N { ts, _ := protocompat.ConvertTimeToTimestampOrError(last) err = suite.datastore.Add(suite.hasWriteCtx, &storage.SecuredUnits{ Timestamp: ts, @@ -134,7 +134,7 @@ func (suite *UsageDataStoreTestSuite) TestGetMax() { const N = page + 2 first := time.Now() last := first - for i := 0; i < N; i++ { + for i := range N { ts, _ := protocompat.ConvertTimeToTimestampOrError(last) err = suite.datastore.Add(suite.hasWriteCtx, &storage.SecuredUnits{ Timestamp: ts, diff --git a/central/administration/usage/injector/injector_impl.go b/central/administration/usage/injector/injector_impl.go index 140e724b29099..624f6f4ac7aeb 100644 --- a/central/administration/usage/injector/injector_impl.go +++ b/central/administration/usage/injector/injector_impl.go @@ -50,11 +50,9 @@ func (i *injectorImpl) gatherLoop() { for { select { case <-i.tickChan: - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { i.gather(ctx) - }() + }) case <-i.stop.Done(): cancel() wg.Wait() @@ -68,11 +66,9 @@ func (i *injectorImpl) gatherLoop() { // Start initiates periodic data injections to the database with the // collected usage. func (i *injectorImpl) Start() { - i.gatherersGroup.Add(1) - go func() { - defer i.gatherersGroup.Done() + i.gatherersGroup.Go(func() { i.gatherLoop() - }() + }) } // Stop stops the scheduled timer and wait for the gatherer to stop. diff --git a/central/alert/datastore/bench_postgres_test.go b/central/alert/datastore/bench_postgres_test.go index b663464a2788a..3e2c88cc1e9d5 100644 --- a/central/alert/datastore/bench_postgres_test.go +++ b/central/alert/datastore/bench_postgres_test.go @@ -54,7 +54,7 @@ func BenchmarkAlertDatabaseOps(b *testing.B) { // Keep the count low in CI. You can run w/ higher numbers locally. totalAlerts := 1000 - for i := 0; i < totalAlerts; i++ { + for range totalAlerts { id := uuid.NewV4().String() ids = append(ids, id) a := fixtures.GetAlertWithID(id) @@ -99,7 +99,7 @@ func BenchmarkAlertDatabaseOps(b *testing.B) { require.NoError(b, datastore.UpsertAlert(ctx, a)) } - var expected []*violationsBySeverity + expected := make([]*violationsBySeverity, 0, len(sevToCount)) for sev, count := range sevToCount { expected = append(expected, &violationsBySeverity{count, int(sev)}) } @@ -299,7 +299,11 @@ func runSearchRawAlerts(ctx context.Context, t testing.TB, datastore DataStore, } func runSelectQuery(ctx context.Context, t testing.TB, testDB *pgtest.TestPostgres, q *v1.Query, expected []*violationsBySeverity) { - results, err := postgres.RunSelectRequestForSchema[violationsBySeverity](ctx, testDB.DB, schema.AlertsSchema, q) + var results []*violationsBySeverity + err := postgres.RunSelectRequestForSchemaFn[violationsBySeverity](ctx, testDB.DB, schema.AlertsSchema, q, func(r *violationsBySeverity) error { + results = append(results, r) + return nil + }) require.NoError(t, err) assert.ElementsMatch(t, expected, results) } diff --git a/central/alert/datastore/datastore.go b/central/alert/datastore/datastore.go index a69d1033d0f26..d8cdabd28d890 100644 --- a/central/alert/datastore/datastore.go +++ b/central/alert/datastore/datastore.go @@ -6,6 +6,7 @@ import ( "github.com/stackrox/rox/central/alert/datastore/internal/store" pgStore "github.com/stackrox/rox/central/alert/datastore/internal/store/postgres" + alertviews "github.com/stackrox/rox/central/alert/views" platformmatcher "github.com/stackrox/rox/central/platform/matcher" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" @@ -29,6 +30,12 @@ type DataStore interface { SearchAlerts(ctx context.Context, q *v1.Query) ([]*v1.SearchResult, error) SearchRawAlerts(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*storage.Alert, error) SearchListAlerts(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*storage.ListAlert, error) + SearchAlertPolicyNamesAndSeverities(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*alertviews.PolicyNameAndSeverity, error) + SearchAlertPolicySeverityCounts(ctx context.Context, q *v1.Query, excludeResolved bool) (*alertviews.PolicySeverityCounts, error) + SearchAlertPolicyGroups(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*alertviews.AlertPolicyGroup, error) + SearchAlertTimeseriesEvents(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*alertviews.AlertTimeseriesEvent, error) + SearchAlertDeploymentIDs(ctx context.Context, q *v1.Query, excludeResolved bool) ([]string, error) + SearchAlertMatchKeys(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*alertviews.AlertMatchKey, error) WalkByQuery(ctx context.Context, q *v1.Query, db func(d *storage.Alert) error) error WalkAll(ctx context.Context, fn func(alert *storage.ListAlert) error) error @@ -44,9 +51,10 @@ type DataStore interface { } // New returns a new soleInstance of DataStore using the input store, and searcher. -func New(alertStore store.Store, platformMatcher platformmatcher.PlatformMatcher) DataStore { +func New(db postgres.DB, alertStore store.Store, platformMatcher platformmatcher.PlatformMatcher) DataStore { ds := &datastoreImpl{ storage: alertStore, + db: db, keyedMutex: concurrency.NewKeyedMutex(mutexPoolSize), keyFence: concurrency.NewKeyFence(), platformMatcher: platformMatcher, @@ -59,5 +67,5 @@ func GetTestPostgresDataStore(t testing.TB, pool postgres.DB) DataStore { alertStore := pgStore.New(pool) mockCtrl := gomock.NewController(t) - return New(alertStore, platformmatcher.GetTestPlatformMatcherWithDefaultPlatformComponentConfig(mockCtrl)) + return New(pool, alertStore, platformmatcher.GetTestPlatformMatcherWithDefaultPlatformComponentConfig(mockCtrl)) } diff --git a/central/alert/datastore/datastore_impl.go b/central/alert/datastore/datastore_impl.go index 02295fcb8c26d..3ced80f0d633c 100644 --- a/central/alert/datastore/datastore_impl.go +++ b/central/alert/datastore/datastore_impl.go @@ -9,22 +9,30 @@ import ( "github.com/pkg/errors" "github.com/stackrox/rox/central/alert/datastore/internal/store" alertutils "github.com/stackrox/rox/central/alert/utils" + alertviews "github.com/stackrox/rox/central/alert/views" "github.com/stackrox/rox/central/metrics" platformmatcher "github.com/stackrox/rox/central/platform/matcher" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/alert/convert" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/errorhelpers" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgutils" + "github.com/stackrox/rox/pkg/postgres/schema" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" searchCommon "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/paginated" + pgSearch "github.com/stackrox/rox/pkg/search/postgres" "github.com/stackrox/rox/pkg/sync" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) const whenUnlimited = 100 @@ -38,7 +46,10 @@ var ( // datastoreImpl is a transaction script with methods that provide the domain logic for CRUD uses cases for Alert // objects. type datastoreImpl struct { - storage store.Store + storage store.Store + // TODO(ROX-31142): No way to call RunSelectRequestForSchemaFn via + // the store so we have to allow for access to the DB here. + db postgres.DB keyedMutex *concurrency.KeyedMutex keyFence concurrency.KeyFence platformMatcher platformmatcher.PlatformMatcher @@ -66,9 +77,40 @@ func (ds *datastoreImpl) Count(ctx context.Context, q *v1.Query, excludeResolved func (ds *datastoreImpl) SearchListAlerts(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*storage.ListAlert, error) { defer metrics.SetDatastoreFunctionDuration(time.Now(), "Alert", "SearchListAlerts") + if q == nil { + q = search.EmptyQuery() + } + if excludeResolved { q = applyDefaultState(q) } + + if env.ListAlertUseLegacyQuery.BooleanSetting() { + return ds.searchListAlertsLegacy(ctx, q) + } + + cloned := q.CloneVT() + cloned.Selects = alertviews.ListAlertSelectProtos + + var sc alertviews.ListAlertScanner + listAlerts := make([]*storage.ListAlert, 0, paginated.GetLimit(cloned.GetPagination().GetLimit(), whenUnlimited)) + err := pgSearch.RunSelectDirectFn(ctx, ds.db, schema.AlertsSchema, cloned, alertviews.ListAlertArrayFields, + &pgSearch.DirectScanConfig{ + ScanDests: sc.Dests, + OnRow: func() error { + listAlerts = append(listAlerts, sc.Build()) + return nil + }, + }) + if err != nil { + return nil, err + } + return listAlerts, nil +} + +// searchListAlertsLegacy is the original SearchListAlerts implementation that +// deserializes the full alert blob. Used as a fallback via ROX_LIST_ALERT_LEGACY_QUERY. +func (ds *datastoreImpl) searchListAlertsLegacy(ctx context.Context, q *v1.Query) ([]*storage.ListAlert, error) { listAlerts := make([]*storage.ListAlert, 0, paginated.GetLimit(q.GetPagination().GetLimit(), whenUnlimited)) err := ds.storage.GetByQueryFn(ctx, q, func(alert *storage.Alert) error { listAlerts = append(listAlerts, convert.AlertToListAlert(alert)) @@ -80,29 +122,179 @@ func (ds *datastoreImpl) SearchListAlerts(ctx context.Context, q *v1.Query, excl return listAlerts, nil } -// SearchAlerts returns search results for the given request. This will exclude resolved alerts by default unless Violation State = Resolved is explicitly specified in the query -func (ds *datastoreImpl) SearchAlerts(ctx context.Context, q *v1.Query) ([]*v1.SearchResult, error) { - defer metrics.SetDatastoreFunctionDuration(time.Now(), "Alert", "SearchAlerts") +// SearchAlertPolicyNamesAndSeverities returns lightweight policy name and severity pairs +// for matching alerts. +func (ds *datastoreImpl) SearchAlertPolicyNamesAndSeverities(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*alertviews.PolicyNameAndSeverity, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "Alert", "SearchAlertPolicyNamesAndSeverities") + + if excludeResolved { + q = applyDefaultState(q) + } + clonedQuery := q.CloneVT() + clonedQuery.Selects = []*v1.QuerySelect{ + search.NewQuerySelect(search.PolicyName).Proto(), + search.NewQuerySelect(search.Severity).Proto(), + } + + var results []*alertviews.PolicyNameAndSeverity + err := pgSearch.RunSelectRequestForSchemaFn(ctx, ds.db, schema.AlertsSchema, clonedQuery, func(r *alertviews.PolicyNameAndSeverity) error { + results = append(results, r) + return nil + }) + if err != nil { + return nil, err + } + return results, nil +} + +// SearchAlertPolicySeverityCounts returns the count of distinct policies per severity level +// for matching alerts. Uses a SQL aggregate query to avoid deserializing alert blobs. +func (ds *datastoreImpl) SearchAlertPolicySeverityCounts(ctx context.Context, q *v1.Query, excludeResolved bool) (*alertviews.PolicySeverityCounts, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "Alert", "SearchAlertPolicySeverityCounts") - results, err := ds.Search(ctx, q, true) + if excludeResolved { + q = applyDefaultState(q) + } + countQuery := alertviews.WithPolicySeverityCountQuery(q) + + result, err := pgSearch.RunSelectOneForSchema[alertviews.PolicySeverityCounts](ctx, ds.db, schema.AlertsSchema, countQuery) + if err != nil { + return &alertviews.PolicySeverityCounts{}, err + } + if result == nil { + return &alertviews.PolicySeverityCounts{}, nil + } + return result, nil +} + +// SearchAlertPolicyGroups returns alerts grouped by policy with a count per group. +// Uses a SQL aggregate query to avoid deserializing alert blobs. +func (ds *datastoreImpl) SearchAlertPolicyGroups(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*alertviews.AlertPolicyGroup, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "Alert", "SearchAlertPolicyGroups") + + if excludeResolved { + q = applyDefaultState(q) + } + groupQuery := alertviews.WithAlertPolicyGroupQuery(q) + + var results []*alertviews.AlertPolicyGroup + err := pgSearch.RunSelectRequestForSchemaFn(ctx, ds.db, schema.AlertsSchema, groupQuery, func(r *alertviews.AlertPolicyGroup) error { + results = append(results, r) + return nil + }) if err != nil { return nil, err } - alerts, missingIndices, err := ds.storage.GetMany(ctx, searchCommon.ResultsToIDs(results)) + return results, nil +} + +// SearchAlertTimeseriesEvents returns lightweight alert event data for timeseries display. +// Uses a SQL projection query to avoid deserializing full alert protobuf blobs. +func (ds *datastoreImpl) SearchAlertTimeseriesEvents(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*alertviews.AlertTimeseriesEvent, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "Alert", "SearchAlertTimeseriesEvents") + + if excludeResolved { + q = applyDefaultState(q) + } + timeseriesQuery := alertviews.WithAlertTimeseriesQuery(q) + + var results []*alertviews.AlertTimeseriesEvent + err := pgSearch.RunSelectRequestForSchemaFn(ctx, ds.db, schema.AlertsSchema, timeseriesQuery, func(r *alertviews.AlertTimeseriesEvent) error { + results = append(results, r) + return nil + }) if err != nil { return nil, err } - listAlerts := make([]*storage.ListAlert, 0, len(alerts)) - for _, alert := range alerts { - listAlerts = append(listAlerts, convert.AlertToListAlert(alert)) + return results, nil +} + +// SearchAlertDeploymentIDs returns distinct deployment IDs from alerts matching the query. +// Uses a SQL projection to avoid deserializing full alert protobuf blobs. +func (ds *datastoreImpl) SearchAlertDeploymentIDs(ctx context.Context, q *v1.Query, excludeResolved bool) ([]string, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "Alert", "SearchAlertDeploymentIDs") + + if excludeResolved { + q = applyDefaultState(q) + } + clonedQuery := q.CloneVT() + clonedQuery.Selects = []*v1.QuerySelect{ + search.NewQuerySelect(search.DeploymentID).Distinct().Proto(), + } + // No sorting needed — we only collect unique IDs. + clonedQuery.Pagination = nil + + var ids []string + err := pgSearch.RunSelectRequestForSchemaFn(ctx, ds.db, schema.AlertsSchema, clonedQuery, func(r *alertviews.DeploymentIDResult) error { + if id := r.GetDeploymentID(); id != "" { + ids = append(ids, id) + } + return nil + }) + if err != nil { + return nil, err + } + return ids, nil +} + +// SearchAlertMatchKeys returns lightweight alert match keys for the given query. +// Projects only inline columns needed for alert matching, avoiding TOAST I/O. +func (ds *datastoreImpl) SearchAlertMatchKeys(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*alertviews.AlertMatchKey, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "Alert", "SearchAlertMatchKeys") + + if excludeResolved { + q = applyDefaultState(q) + } + matchKeyQuery := alertviews.WithAlertMatchKeyQuery(q) + + var results []*alertviews.AlertMatchKey + err := pgSearch.RunSelectRequestForSchemaFn(ctx, ds.db, schema.AlertsSchema, matchKeyQuery, func(r *alertviews.AlertMatchKey) error { + results = append(results, r) + return nil + }) + if err != nil { + return nil, err + } + return results, nil +} + +// SearchAlerts returns search results for the given request. This will exclude resolved alerts by default unless Violation State = Resolved is explicitly specified in the query +func (ds *datastoreImpl) SearchAlerts(ctx context.Context, q *v1.Query) ([]*v1.SearchResult, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "Alert", "SearchAlerts") + + if q == nil { + q = search.EmptyQuery() + } + + // Clone the query and add select fields for SearchResult construction + clonedQuery := q.CloneVT() + selectSelects := []*v1.QuerySelect{ + search.NewQuerySelect(search.PolicyName).Proto(), + search.NewQuerySelect(search.Cluster).Proto(), + search.NewQuerySelect(search.Namespace).Proto(), + search.NewQuerySelect(search.DeploymentName).Proto(), + search.NewQuerySelect(search.EntityType).Proto(), + search.NewQuerySelect(search.ResourceName).Proto(), + search.NewQuerySelect(search.ResourceType).Proto(), } - results = searchCommon.RemoveMissingResults(results, missingIndices) + clonedQuery.Selects = append(clonedQuery.GetSelects(), selectSelects...) - protoResults := make([]*v1.SearchResult, 0, len(alerts)) - for i, alert := range listAlerts { - protoResults = append(protoResults, convertAlert(alert, results[i])) + results, err := ds.Search(ctx, clonedQuery, false) + if err != nil { + return nil, err } - return protoResults, nil + + // Populate Name and Location fields from FieldValues for each result + for i := range results { + + if results[i].FieldValues != nil { + if nameVal, ok := results[i].FieldValues[strings.ToLower(search.PolicyName.String())]; ok { + results[i].Name = nameVal + } + } + } + + return search.ResultsToSearchResultProtos(results, &AlertSearchResultConverter{}), nil } // SearchRawAlerts returns search results for the given request in the form of a slice of alerts. @@ -308,8 +500,12 @@ func (ds *datastoreImpl) updateAlertNoLock(ctx context.Context, alerts ...*stora return nil } - if features.PlatformComponents.Enabled() { - for _, alert := range alerts { + for _, alert := range alerts { + // Compute and cache enforcement count so it can be queried directly + // from the column without deserializing the full alert blob. + alert.EnforcementCount = convert.EnforcementCount(alert) + + if features.PlatformComponents.Enabled() { alert.EntityType = alertutils.GetEntityType(alert) match, err := ds.platformMatcher.MatchAlert(alert) if err != nil { @@ -347,10 +543,29 @@ func (ds *datastoreImpl) WalkAll(ctx context.Context, fn func(*storage.ListAlert return sac.ErrResourceAccessDenied } + if env.ListAlertUseLegacyQuery.BooleanSetting() { + return ds.walkAllLegacy(ctx, fn) + } + + q := searchCommon.EmptyQuery() + q.Selects = alertviews.ListAlertSelectProtos + + var sc alertviews.ListAlertScanner + return pgSearch.RunSelectDirectFn(ctx, ds.db, schema.AlertsSchema, q, alertviews.ListAlertArrayFields, + &pgSearch.DirectScanConfig{ + ScanDests: sc.Dests, + OnRow: func() error { + return fn(sc.Build()) + }, + }) +} + +// walkAllLegacy is the original WalkAll implementation that deserializes the +// full alert blob. Used as a fallback via ROX_LIST_ALERT_LEGACY_QUERY. +func (ds *datastoreImpl) walkAllLegacy(ctx context.Context, fn func(*storage.ListAlert) error) error { walkFn := func() error { return ds.storage.Walk(ctx, func(alert *storage.Alert) error { - listAlert := convert.AlertToListAlert(alert) - return fn(listAlert) + return fn(convert.AlertToListAlert(alert)) }) } return pgutils.RetryIfPostgres(ctx, walkFn) @@ -388,36 +603,56 @@ func applyDefaultState(q *v1.Query) *v1.Query { storage.ViolationState_ACTIVE.String(), storage.ViolationState_ATTEMPTED.String()).ProtoQuery()) cq.Pagination = q.GetPagination() + cq.Selects = q.GetSelects() return cq } return q } -// convertAlert returns proto search result from an alert object and the internal search result -func convertAlert(alert *storage.ListAlert, result searchCommon.Result) *v1.SearchResult { - entityInfo := alert.GetCommonEntityInfo() +// AlertSearchResultConverter implements search.SearchResultConverter for alert search results. +// This enables single-pass query construction for SearchResult protos. +type AlertSearchResultConverter struct{} + +func (c *AlertSearchResultConverter) BuildName(result *search.Result) string { + return result.Name +} + +func (c *AlertSearchResultConverter) BuildLocation(result *search.Result) string { + fv := result.FieldValues + clusterName := fv[strings.ToLower(search.Cluster.String())] + namespace := fv[strings.ToLower(search.Namespace.String())] + deploymentName := fv[strings.ToLower(search.DeploymentName.String())] + entityType := fv[strings.ToLower(search.EntityType.String())] + resourceName := fv[strings.ToLower(search.ResourceName.String())] + resourceType := fv[strings.ToLower(search.ResourceType.String())] + + nodeName := fv[strings.ToLower(search.Node.String())] + var entityName string - switch entity := alert.GetEntity().(type) { - case *storage.ListAlert_Resource: - entityName = entity.Resource.GetName() - case *storage.ListAlert_Deployment: - entityName = entity.Deployment.GetName() + switch entityType { + case "DEPLOYMENT": + entityName = deploymentName + resourceType = entityType + case "RESOURCE": + entityName = resourceName + case "NODE": + entityName = nodeName + resourceType = entityType } - resourceTypeTitleCase := strings.Title(strings.ToLower(entityInfo.GetResourceType().String())) + var location string - if entityInfo.GetNamespace() != "" { + + casePkg := cases.Title(language.English) + if namespace != "" { location = fmt.Sprintf("/%s/%s/%s/%s", - entityInfo.GetClusterName(), entityInfo.GetNamespace(), resourceTypeTitleCase, entityName) + clusterName, namespace, casePkg.String(resourceType), entityName) } else { location = fmt.Sprintf("/%s/%s/%s", - entityInfo.GetClusterName(), resourceTypeTitleCase, entityName) - } - return &v1.SearchResult{ - Category: v1.SearchCategory_ALERTS, - Id: alert.GetId(), - Name: alert.GetPolicy().GetName(), - FieldToMatches: searchCommon.GetProtoMatchesMap(result.Matches), - Score: result.Score, - Location: location, + clusterName, casePkg.String(resourceType), entityName) } + return location +} + +func (c *AlertSearchResultConverter) GetCategory() v1.SearchCategory { + return v1.SearchCategory_ALERTS } diff --git a/central/alert/datastore/datastore_impl_test.go b/central/alert/datastore/datastore_impl_test.go index 5e7dd0397d4a2..bf6eda87a1e85 100644 --- a/central/alert/datastore/datastore_impl_test.go +++ b/central/alert/datastore/datastore_impl_test.go @@ -6,8 +6,10 @@ import ( "context" "fmt" "testing" + "time" "github.com/stackrox/rox/central/alert/datastore/internal/store/postgres" + alertviews "github.com/stackrox/rox/central/alert/views" matcherMocks "github.com/stackrox/rox/central/platform/matcher/mocks" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" @@ -17,9 +19,9 @@ import ( "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/protoutils" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/search" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" ) @@ -50,7 +52,7 @@ func (s *AlertDatastoreImplSuite) SetupTest() { s.matcher = matcherMocks.NewMockPlatformMatcher(s.mockCtrl) store := postgres.New(s.testPostgres.DB) - s.datastore = New(store, s.matcher) + s.datastore = New(s.testPostgres.DB, store, s.matcher) // Initialize alert tracking s.createdAlertIDs = []string{} @@ -207,7 +209,7 @@ func (s *AlertDatastoreImplSuite) TestCountResolved() { s.Equal(2, results) } -// TestSearchAlerts tests the SearchAlerts functionality with real data +// TestSearchAlerts tests the SearchAlerts functionality with deployment alert func (s *AlertDatastoreImplSuite) TestSearchAlerts() { alert := fixtures.GetAlert() alert.EntityType = storage.Alert_DEPLOYMENT @@ -225,6 +227,53 @@ func (s *AlertDatastoreImplSuite) TestSearchAlerts() { s.Len(searchResults, 1) s.Equal(alert.GetId(), searchResults[0].GetId()) s.Equal(alert.GetPolicy().GetName(), searchResults[0].GetName()) + s.Equal(v1.SearchCategory_ALERTS, searchResults[0].GetCategory()) + expectedLocation := "/prod cluster/stackrox/Deployment/nginx_server" + s.Equal(searchResults[0].GetLocation(), expectedLocation) +} + +// TestSearchAlerts tests the SearchAlerts functionality with resource alert +func (s *AlertDatastoreImplSuite) TestSearchResourceAlerts() { + alert := fixtures.GetResourceAlert() + alert.PlatformComponent = false + + // Mock platform matcher to return false for platform component + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + + // Upsert the alert + s.NoError(s.datastore.UpsertAlert(ctx, alert)) + + // Test SearchAlerts + searchResults, err := s.datastore.SearchAlerts(ctx, search.EmptyQuery()) + s.NoError(err) + s.Len(searchResults, 1) + s.Equal(alert.GetId(), searchResults[0].GetId()) + s.Equal(alert.GetPolicy().GetName(), searchResults[0].GetName()) + s.Equal(v1.SearchCategory_ALERTS, searchResults[0].GetCategory()) + expectedLocation := "/prod cluster/stackrox/Secrets/my-secret" + s.Equal(searchResults[0].GetLocation(), expectedLocation) +} + +// Test non namespaced resource alert +func (s *AlertDatastoreImplSuite) TestSearchNonNameSpacedAlerts() { + alert := fixtures.GetScopedResourceAlert(fixtureconsts.Alert1, fixtureconsts.Cluster1, "") + alert.PlatformComponent = false + + // Mock platform matcher to return false for platform component + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + + // Upsert the alert + s.NoError(s.datastore.UpsertAlert(ctx, alert)) + + // Test SearchAlerts + searchResults, err := s.datastore.SearchAlerts(ctx, search.EmptyQuery()) + s.NoError(err) + s.Len(searchResults, 1) + s.Equal(alert.GetId(), searchResults[0].GetId()) + s.Equal(alert.GetPolicy().GetName(), searchResults[0].GetName()) + s.Equal(v1.SearchCategory_ALERTS, searchResults[0].GetCategory()) + expectedLocation := "/prod cluster/Secrets/my-secret" + s.Equal(searchResults[0].GetLocation(), expectedLocation) } // TestSearchRawAlerts tests the SearchRawAlerts functionality with real data @@ -264,6 +313,93 @@ func (s *AlertDatastoreImplSuite) TestSearchRawAlerts() { s.Equal(storage.ViolationState_RESOLVED, rawAlerts[0].GetState()) } +// TestSearchAlertPolicyNamesAndSeverities tests the column-projection search +func (s *AlertDatastoreImplSuite) TestSearchAlertPolicyNamesAndSeverities() { + alert := fixtures.GetAlert() + alert.EntityType = storage.Alert_DEPLOYMENT + alert.PlatformComponent = false + + // Mock platform matcher to return false for platform component + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + + // Upsert the alert + s.NoError(s.datastore.UpsertAlert(ctx, alert)) + + // Test 1: Search excluding resolved returns the alert's policy name and severity + results, err := s.datastore.SearchAlertPolicyNamesAndSeverities(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(results, 1) + s.Equal(alert.GetPolicy().GetName(), results[0].GetPolicyName()) + s.Equal(alert.GetPolicy().GetSeverity(), results[0].GetSeverity()) + + // Test 2: Search with deployment ID filter + q := search.NewQueryBuilder(). + AddExactMatches(search.DeploymentID, alert.GetDeployment().GetId()). + ProtoQuery() + results, err = s.datastore.SearchAlertPolicyNamesAndSeverities(ctx, q, true) + s.NoError(err) + s.Len(results, 1) + s.Equal(alert.GetPolicy().GetName(), results[0].GetPolicyName()) + s.Equal(alert.GetPolicy().GetSeverity(), results[0].GetSeverity()) + + // Test 3: Mark alert as resolved + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + resolvedAlerts, err := s.datastore.MarkAlertsResolvedBatch(ctx, alert.GetId()) + s.NoError(err) + s.Len(resolvedAlerts, 1) + + // Test 4: Search excluding resolved should return 0 + results, err = s.datastore.SearchAlertPolicyNamesAndSeverities(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(results, 0) + + // Test 5: Search including resolved should return 1 + results, err = s.datastore.SearchAlertPolicyNamesAndSeverities(ctx, search.EmptyQuery(), false) + s.NoError(err) + s.Len(results, 1) + s.Equal(alert.GetPolicy().GetName(), results[0].GetPolicyName()) + s.Equal(alert.GetPolicy().GetSeverity(), results[0].GetSeverity()) +} + +// TestSearchAlertPolicyNamesAndSeveritiesMultiple tests with multiple alerts of different severities +func (s *AlertDatastoreImplSuite) TestSearchAlertPolicyNamesAndSeveritiesMultiple() { + severities := []storage.Severity{ + storage.Severity_LOW_SEVERITY, + storage.Severity_MEDIUM_SEVERITY, + storage.Severity_HIGH_SEVERITY, + storage.Severity_CRITICAL_SEVERITY, + } + alertIDs := []string{fixtureconsts.Alert1, fixtureconsts.Alert2, fixtureconsts.Alert3, fixtureconsts.Alert4} + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil).Times(len(alertIDs)) + + for i, id := range alertIDs { + alert := fixtures.GetAlert() + alert.Id = id + alert.EntityType = storage.Alert_DEPLOYMENT + alert.PlatformComponent = false + alert.Policy.Name = fmt.Sprintf("Policy %d", i+1) + alert.Policy.Severity = severities[i] + s.createAndTrackAlert(alert) + } + + results, err := s.datastore.SearchAlertPolicyNamesAndSeverities(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(results, len(alertIDs)) + + // Collect results into a map for order-independent assertions + resultMap := make(map[string]*alertviews.PolicyNameAndSeverity) + for _, r := range results { + resultMap[r.GetPolicyName()] = r + } + + for i, sev := range severities { + name := fmt.Sprintf("Policy %d", i+1) + s.Contains(resultMap, name) + s.Equal(sev, resultMap[name].GetSeverity()) + } +} + // TestSearchListAlerts tests the SearchListAlerts functionality with pagination func (s *AlertDatastoreImplSuite) TestSearchListAlerts() { // Create multiple alerts to test pagination @@ -275,7 +411,7 @@ func (s *AlertDatastoreImplSuite) TestSearchListAlerts() { fixtureconsts.Deployment5, } - var createdAlerts []*storage.Alert + createdAlerts := make([]*storage.Alert, 0, len(alertIDs)) for i, id := range alertIDs { alert := fixtures.GetAlert() alert.Id = id @@ -378,39 +514,75 @@ func (s *AlertDatastoreImplSuite) TestSearchListAlerts() { s.NotNil(matchingCreatedAlert, "Should find matching created alert") expectedListAlert := convert.AlertToListAlert(matchingCreatedAlert) + // PostgreSQL timestamps have microsecond precision, so round both + // sides to microseconds before comparing. + expectedListAlert.Time = protoutils.RoundTimestamp(expectedListAlert.GetTime(), time.Microsecond) + returnedAlert.Time = protoutils.RoundTimestamp(returnedAlert.GetTime(), time.Microsecond) protoassert.Equal(s.T(), expectedListAlert, returnedAlert) } -// TestConvertAlert covers the same functionality as searcher_impl_test.go TestConvertAlert -func (s *AlertDatastoreImplSuite) TestConvertAlert() { - nonNamespacedResourceAlert := fixtures.GetResourceAlert() - nonNamespacedResourceAlert.GetResource().Namespace = "" - - for _, testCase := range []struct { - desc string - alert *storage.ListAlert - expectedLocation string - }{ - { - desc: "Deployment alert", - alert: convert.AlertToListAlert(fixtures.GetAlert()), - expectedLocation: "/prod cluster/stackrox/Deployment/nginx_server", - }, - { - desc: "Namespaced resource alert", - alert: convert.AlertToListAlert(fixtures.GetResourceAlert()), - expectedLocation: "/prod cluster/stackrox/Secrets/my-secret", - }, - { - desc: "Non-namespaced resource alert", - alert: convert.AlertToListAlert(nonNamespacedResourceAlert), - expectedLocation: "/prod cluster/Secrets/my-secret", - }, - } { - s.T().Run(testCase.desc, func(t *testing.T) { - res := convertAlert(testCase.alert, search.Result{}) - assert.Equal(t, testCase.expectedLocation, res.GetLocation()) - }) +// TestSearchListAlertsEntityTypes verifies that the projection-based SearchListAlerts +// correctly handles deployment, resource, and node entity types, including +// deployment_type, enforcement_count, and categories. +func (s *AlertDatastoreImplSuite) TestSearchListAlertsEntityTypes() { + // 1. Deployment alert with deployment_type and enforcement. + deployAlert := fixtures.GetAlert() + deployAlert.Id = fixtureconsts.Deployment1 + deployAlert.GetDeployment().Type = "DaemonSet" + deployAlert.EntityType = storage.Alert_DEPLOYMENT + deployAlert.PlatformComponent = false + deployAlert.Enforcement = &storage.Alert_Enforcement{ + Action: storage.EnforcementAction_SCALE_TO_ZERO_ENFORCEMENT, + } + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + s.createAndTrackAlert(deployAlert) + + // 2. Resource alert. + resourceAlert := fixtures.GetResourceAlert() + resourceAlert.Id = fixtureconsts.Deployment2 + resourceAlert.EntityType = storage.Alert_RESOURCE + resourceAlert.PlatformComponent = false + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + s.createAndTrackAlert(resourceAlert) + + // 3. Node alert. + nodeAlert := fixtures.GetScopedNodeAlert(fixtureconsts.Deployment3, fixtureconsts.Cluster1, fixtureconsts.Node1, "test-node") + nodeAlert.EntityType = storage.Alert_NODE + nodeAlert.PlatformComponent = false + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + s.createAndTrackAlert(nodeAlert) + + // Search all. + listAlerts, err := s.datastore.SearchListAlerts(ctx, search.EmptyQuery(), false) + s.NoError(err) + s.Len(listAlerts, 3) + + // Verify each entity type by comparing with convert.AlertToListAlert. + for _, la := range listAlerts { + switch la.GetId() { + case deployAlert.GetId(): + expected := convert.AlertToListAlert(deployAlert) + s.Equal("DaemonSet", la.GetDeployment().GetDeploymentType()) + s.Equal(expected.GetEnforcementCount(), la.GetEnforcementCount()) + s.Equal(expected.GetPolicy().GetCategories(), la.GetPolicy().GetCategories()) + s.NotNil(la.GetCommonEntityInfo()) + s.Equal(storage.ListAlert_DEPLOYMENT, la.GetCommonEntityInfo().GetResourceType()) + + case resourceAlert.GetId(): + s.NotNil(la.GetResource()) + s.Equal(resourceAlert.GetResource().GetName(), la.GetResource().GetName()) + s.NotNil(la.GetCommonEntityInfo()) + s.Equal(storage.ListAlert_ResourceType(storage.Alert_Resource_SECRETS), la.GetCommonEntityInfo().GetResourceType()) + + case nodeAlert.GetId(): + s.NotNil(la.GetNode()) + s.Equal("test-node", la.GetNode().GetName()) + s.NotNil(la.GetCommonEntityInfo()) + s.Equal(nodeAlert.GetClusterName(), la.GetCommonEntityInfo().GetClusterName()) + + default: + s.Failf("unexpected alert ID", "got %s", la.GetId()) + } } } @@ -586,6 +758,305 @@ func (s *AlertDatastoreImplSuite) TestWalkAll() { } } +// TestSearchAlertPolicyGroups tests the SearchAlertPolicyGroups aggregate query +func (s *AlertDatastoreImplSuite) TestSearchAlertPolicyGroups() { + alert := fixtures.GetAlert() + alert.EntityType = storage.Alert_DEPLOYMENT + alert.PlatformComponent = false + + // Mock platform matcher to return false for platform component + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + + // Upsert the alert + s.NoError(s.datastore.UpsertAlert(ctx, alert)) + s.createdAlertIDs = append(s.createdAlertIDs, alert.GetId()) + + // Test 1: Search excluding resolved returns the alert's policy group + results, err := s.datastore.SearchAlertPolicyGroups(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(results, 1) + s.Equal(alert.GetPolicy().GetId(), results[0].PolicyID) + s.Equal(alert.GetPolicy().GetName(), results[0].PolicyName) + s.Equal(int(alert.GetPolicy().GetSeverity()), results[0].Severity) + s.Equal(alert.GetPolicy().GetDescription(), results[0].Description) + s.ElementsMatch(alert.GetPolicy().GetCategories(), results[0].Categories) + s.Equal(1, results[0].NumAlerts) + + // Test 2: Mark alert as resolved + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + resolvedAlerts, err := s.datastore.MarkAlertsResolvedBatch(ctx, alert.GetId()) + s.NoError(err) + s.Len(resolvedAlerts, 1) + + // Test 3: Search excluding resolved should return 0 + results, err = s.datastore.SearchAlertPolicyGroups(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(results, 0) + + // Test 4: Search including resolved should return 1 + results, err = s.datastore.SearchAlertPolicyGroups(ctx, search.EmptyQuery(), false) + s.NoError(err) + s.Len(results, 1) + s.Equal(alert.GetPolicy().GetId(), results[0].PolicyID) + s.ElementsMatch(alert.GetPolicy().GetCategories(), results[0].Categories) +} + +// TestSearchAlertPolicyGroupsMultiple tests with multiple alerts grouped by policy +func (s *AlertDatastoreImplSuite) TestSearchAlertPolicyGroupsMultiple() { + alertIDs := []string{fixtureconsts.Alert1, fixtureconsts.Alert2, fixtureconsts.Alert3, fixtureconsts.Alert4} + policyCategories := [][]string{ + {"Image Assurance"}, + {"Image Assurance"}, + {"Network", "Security"}, + {"Network", "Security"}, + } + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil).Times(len(alertIDs)) + + for i, id := range alertIDs { + alert := fixtures.GetAlert() + alert.Id = id + alert.EntityType = storage.Alert_DEPLOYMENT + alert.PlatformComponent = false + // Use two distinct policies: first two alerts share policy A, last two share policy B + if i < 2 { + alert.Policy.Id = "policy-a" + alert.Policy.Name = "Policy A" + alert.Policy.Severity = storage.Severity_LOW_SEVERITY + alert.Policy.Description = "low severity policy" + } else { + alert.Policy.Id = "policy-b" + alert.Policy.Name = "Policy B" + alert.Policy.Severity = storage.Severity_HIGH_SEVERITY + alert.Policy.Description = "high severity policy" + } + alert.Policy.Categories = policyCategories[i] + s.createAndTrackAlert(alert) + } + + results, err := s.datastore.SearchAlertPolicyGroups(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(results, 2) + + // Collect results into a map for order-independent assertions + resultMap := make(map[string]*alertviews.AlertPolicyGroup) + for _, r := range results { + resultMap[r.PolicyID] = r + } + + s.Contains(resultMap, "policy-a") + s.Equal("Policy A", resultMap["policy-a"].PolicyName) + s.Equal(int(storage.Severity_LOW_SEVERITY), resultMap["policy-a"].Severity) + s.ElementsMatch([]string{"Image Assurance"}, resultMap["policy-a"].Categories) + s.Equal(2, resultMap["policy-a"].NumAlerts) + + s.Contains(resultMap, "policy-b") + s.Equal("Policy B", resultMap["policy-b"].PolicyName) + s.Equal(int(storage.Severity_HIGH_SEVERITY), resultMap["policy-b"].Severity) + s.ElementsMatch([]string{"Network", "Security"}, resultMap["policy-b"].Categories) + s.Equal(2, resultMap["policy-b"].NumAlerts) +} + +// TestSearchAlertTimeseriesEvents tests the SearchAlertTimeseriesEvents column-projection query +func (s *AlertDatastoreImplSuite) TestSearchAlertTimeseriesEvents() { + alert := fixtures.GetAlert() + alert.EntityType = storage.Alert_DEPLOYMENT + alert.PlatformComponent = false + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + s.createAndTrackAlert(alert) + + // Test 1: Search excluding resolved returns the alert's timeseries event fields + results, err := s.datastore.SearchAlertTimeseriesEvents(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(results, 1) + s.Equal(alert.GetId(), results[0].GetAlertID()) + s.Equal(alert.GetDeployment().GetClusterName(), results[0].GetClusterName()) + s.Equal(alert.GetPolicy().GetSeverity(), results[0].GetSeverity()) + s.Equal(alert.GetState(), results[0].GetState()) + s.NotNil(results[0].Time) + + // Test 2: Mark alert as resolved + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + resolvedAlerts, err := s.datastore.MarkAlertsResolvedBatch(ctx, alert.GetId()) + s.NoError(err) + s.Len(resolvedAlerts, 1) + + // Test 3: Search excluding resolved should return 0 + results, err = s.datastore.SearchAlertTimeseriesEvents(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(results, 0) + + // Test 4: Search including resolved should return 1 + results, err = s.datastore.SearchAlertTimeseriesEvents(ctx, search.EmptyQuery(), false) + s.NoError(err) + s.Len(results, 1) + s.Equal(alert.GetId(), results[0].GetAlertID()) + s.Equal(storage.ViolationState_RESOLVED, results[0].GetState()) +} + +// TestSearchAlertTimeseriesEventsMultiple tests with multiple alerts across clusters/severities +func (s *AlertDatastoreImplSuite) TestSearchAlertTimeseriesEventsMultiple() { + alertIDs := []string{fixtureconsts.Alert1, fixtureconsts.Alert2, fixtureconsts.Alert3, fixtureconsts.Alert4} + clusterNames := []string{"dev", "dev", "prod", "prod"} + severities := []storage.Severity{ + storage.Severity_CRITICAL_SEVERITY, + storage.Severity_HIGH_SEVERITY, + storage.Severity_LOW_SEVERITY, + storage.Severity_MEDIUM_SEVERITY, + } + states := []storage.ViolationState{ + storage.ViolationState_RESOLVED, + storage.ViolationState_ACTIVE, + storage.ViolationState_RESOLVED, + storage.ViolationState_ACTIVE, + } + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil).Times(len(alertIDs)) + + for i, id := range alertIDs { + alert := fixtures.GetAlert() + alert.Id = id + alert.EntityType = storage.Alert_DEPLOYMENT + alert.PlatformComponent = false + alert.Policy.Severity = severities[i] + alert.State = states[i] + alert.GetDeployment().ClusterName = clusterNames[i] + alert.GetDeployment().ClusterId = fmt.Sprintf("cluster-%s", clusterNames[i]) + alert.ClusterName = clusterNames[i] + alert.ClusterId = fmt.Sprintf("cluster-%s", clusterNames[i]) + s.createAndTrackAlert(alert) + } + + // Search including all states to get all 4 alerts + results, err := s.datastore.SearchAlertTimeseriesEvents(ctx, search.EmptyQuery(), false) + s.NoError(err) + s.Len(results, len(alertIDs)) + + // Collect results into a map for order-independent assertions + resultMap := make(map[string]*alertviews.AlertTimeseriesEvent) + for _, r := range results { + resultMap[r.GetAlertID()] = r + } + + for i, id := range alertIDs { + s.Contains(resultMap, id) + s.Equal(clusterNames[i], resultMap[id].GetClusterName()) + s.Equal(severities[i], resultMap[id].GetSeverity()) + s.Equal(states[i], resultMap[id].GetState()) + } + + // Search excluding resolved should return only active alerts + results, err = s.datastore.SearchAlertTimeseriesEvents(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(results, 2) + for _, r := range results { + s.NotEqual(storage.ViolationState_RESOLVED, r.GetState()) + } +} + +// TestSearchAlertDeploymentIDs tests the deployment ID projection query +func (s *AlertDatastoreImplSuite) TestSearchAlertDeploymentIDs() { + alert := fixtures.GetAlert() + alert.EntityType = storage.Alert_DEPLOYMENT + alert.PlatformComponent = false + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + s.createAndTrackAlert(alert) + + // Test 1: Search excluding resolved returns the deployment ID + ids, err := s.datastore.SearchAlertDeploymentIDs(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(ids, 1) + s.Equal(alert.GetDeployment().GetId(), ids[0]) + + // Test 2: Search with policy filter + q := search.NewQueryBuilder(). + AddExactMatches(search.PolicyID, alert.GetPolicy().GetId()). + ProtoQuery() + ids, err = s.datastore.SearchAlertDeploymentIDs(ctx, q, true) + s.NoError(err) + s.Len(ids, 1) + s.Equal(alert.GetDeployment().GetId(), ids[0]) + + // Test 3: Mark alert as resolved + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + _, err = s.datastore.MarkAlertsResolvedBatch(ctx, alert.GetId()) + s.NoError(err) + + // Test 4: Search excluding resolved should return 0 + ids, err = s.datastore.SearchAlertDeploymentIDs(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(ids, 0) + + // Test 5: Search including resolved should return 1 + ids, err = s.datastore.SearchAlertDeploymentIDs(ctx, search.EmptyQuery(), false) + s.NoError(err) + s.Len(ids, 1) +} + +// TestSearchAlertDeploymentIDsDeduplication tests that duplicate deployment IDs are collapsed +func (s *AlertDatastoreImplSuite) TestSearchAlertDeploymentIDsDeduplication() { + // Create multiple alerts for the same deployment (different policies) + alertIDs := []string{fixtureconsts.Alert1, fixtureconsts.Alert2, fixtureconsts.Alert3} + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil).Times(len(alertIDs)) + + sharedDeploymentID := fixtureconsts.Deployment1 + for i, id := range alertIDs { + alert := fixtures.GetAlert() + alert.Id = id + alert.EntityType = storage.Alert_DEPLOYMENT + alert.PlatformComponent = false + alert.Policy.Id = fmt.Sprintf("policy-%d", i) + alert.Policy.Name = fmt.Sprintf("Policy %d", i) + alert.GetDeployment().Id = sharedDeploymentID + s.createAndTrackAlert(alert) + } + + // Should return only 1 unique deployment ID despite 3 alerts + ids, err := s.datastore.SearchAlertDeploymentIDs(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(ids, 1) + s.Equal(sharedDeploymentID, ids[0]) +} + +// TestSearchAlertDeploymentIDsMultipleDeployments tests with alerts across different deployments +func (s *AlertDatastoreImplSuite) TestSearchAlertDeploymentIDsMultipleDeployments() { + alertIDs := []string{fixtureconsts.Alert1, fixtureconsts.Alert2, fixtureconsts.Alert3} + deploymentIDs := []string{fixtureconsts.Deployment1, fixtureconsts.Deployment2, fixtureconsts.Deployment3} + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil).Times(len(alertIDs)) + + for i, id := range alertIDs { + alert := fixtures.GetAlert() + alert.Id = id + alert.EntityType = storage.Alert_DEPLOYMENT + alert.PlatformComponent = false + alert.GetDeployment().Id = deploymentIDs[i] + s.createAndTrackAlert(alert) + } + + ids, err := s.datastore.SearchAlertDeploymentIDs(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(ids, 3) + s.ElementsMatch(deploymentIDs, ids) +} + +// TestSearchAlertDeploymentIDsResourceAlerts tests that resource alerts (no deployment) return empty deployment IDs +func (s *AlertDatastoreImplSuite) TestSearchAlertDeploymentIDsResourceAlerts() { + alert := fixtures.GetResourceAlert() + alert.PlatformComponent = false + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + s.createAndTrackAlert(alert) + + // Resource alerts have no deployment ID — should be filtered out + ids, err := s.datastore.SearchAlertDeploymentIDs(ctx, search.EmptyQuery(), true) + s.NoError(err) + s.Len(ids, 0) +} + // TestUpsert_PlatformComponentAndEntityTypeAssignment tests platform component assignment logic // Moved from datastore_test.go and converted to use real data instead of mocks func (s *AlertDatastoreImplSuite) TestUpsert_PlatformComponentAndEntityTypeAssignment() { @@ -703,3 +1174,120 @@ func (s *AlertDatastoreImplSuite) TestUpsert_PlatformComponentAndEntityTypeAssig s.Equal(storage.Alert_DEPLOYMENT, storedAlert.GetEntityType()) s.True(storedAlert.GetPlatformComponent()) } + +func (s *AlertDatastoreImplSuite) TestSearchAlertMatchKeysDeployment() { + alert := fixtures.GetAlert() + alert.Id = fixtureconsts.Alert1 + alert.State = storage.ViolationState_ACTIVE + alert.LifecycleStage = storage.LifecycleStage_DEPLOY + alert.GetDeployment().Id = fixtureconsts.Deployment2 + alert.GetDeployment().Inactive = false + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + s.createAndTrackAlert(alert) + + q := search.NewQueryBuilder(). + AddExactMatches(search.ViolationState, storage.ViolationState_ACTIVE.String()). + AddExactMatches(search.DeploymentID, fixtureconsts.Deployment2). + ProtoQuery() + + keys, err := s.datastore.SearchAlertMatchKeys(ctx, q, false) + s.NoError(err) + s.Require().Len(keys, 1) + + key := keys[0] + s.Equal(alert.GetId(), key.GetId()) + s.Equal(alert.GetPolicy().GetId(), key.GetPolicyId()) + s.Equal(alert.GetState(), key.GetState()) + s.Equal(alert.GetLifecycleStage(), key.GetLifecycleStage()) + s.Equal(fixtureconsts.Deployment2, key.GetDeploymentId()) + s.True(key.HasDeployment()) + s.False(key.IsDeploymentInactive()) + s.False(key.HasResource()) + s.False(key.HasNode()) +} + +func (s *AlertDatastoreImplSuite) TestSearchAlertMatchKeysResource() { + alert := fixtures.GetResourceAlert() + alert.State = storage.ViolationState_ACTIVE + alert.LifecycleStage = storage.LifecycleStage_RUNTIME + res := alert.GetResource() + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + s.createAndTrackAlert(alert) + + q := search.NewQueryBuilder(). + AddExactMatches(search.ViolationState, storage.ViolationState_ACTIVE.String()). + AddExactMatches(search.AlertID, alert.GetId()). + ProtoQuery() + + keys, err := s.datastore.SearchAlertMatchKeys(ctx, q, false) + s.NoError(err) + s.Require().Len(keys, 1) + + key := keys[0] + s.Equal(alert.GetId(), key.GetId()) + s.Equal(alert.GetPolicy().GetId(), key.GetPolicyId()) + s.Equal(alert.GetState(), key.GetState()) + s.Equal(alert.GetLifecycleStage(), key.GetLifecycleStage()) + s.False(key.HasDeployment()) + s.True(key.HasResource()) + s.Equal(res.GetResourceType(), key.GetResourceType()) + s.Equal(res.GetName(), key.GetResourceName()) + s.Equal(res.GetClusterId(), key.GetClusterId()) + s.Equal(res.GetNamespace(), key.GetNamespace()) +} + +func (s *AlertDatastoreImplSuite) TestSearchAlertMatchKeysNode() { + alert := fixtures.GetNodeAlert() + alert.State = storage.ViolationState_ACTIVE + alert.LifecycleStage = storage.LifecycleStage_RUNTIME + node := alert.GetNode() + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil) + s.createAndTrackAlert(alert) + + q := search.NewQueryBuilder(). + AddExactMatches(search.ViolationState, storage.ViolationState_ACTIVE.String()). + AddExactMatches(search.AlertID, alert.GetId()). + ProtoQuery() + + keys, err := s.datastore.SearchAlertMatchKeys(ctx, q, false) + s.NoError(err) + s.Require().Len(keys, 1) + + key := keys[0] + s.Equal(alert.GetId(), key.GetId()) + s.Equal(alert.GetPolicy().GetId(), key.GetPolicyId()) + s.Equal(alert.GetState(), key.GetState()) + s.Equal(alert.GetLifecycleStage(), key.GetLifecycleStage()) + s.False(key.HasDeployment()) + s.False(key.HasResource()) + s.True(key.HasNode()) + s.Equal(node.GetId(), key.GetNodeId()) + s.Equal(node.GetName(), key.GetNodeName()) + s.Equal(node.GetClusterId(), key.GetClusterId()) +} + +func (s *AlertDatastoreImplSuite) TestSearchAlertMatchKeysExcludesResolved() { + active := fixtures.GetAlert() + active.Id = fixtureconsts.Alert1 + active.State = storage.ViolationState_ACTIVE + + resolved := fixtures.GetAlert() + resolved.Id = fixtureconsts.Alert2 + resolved.State = storage.ViolationState_RESOLVED + + s.matcher.EXPECT().MatchAlert(gomock.Any()).Return(false, nil).Times(2) + s.createAndTrackAlert(active) + s.createAndTrackAlert(resolved) + + q := search.NewQueryBuilder(). + AddExactMatches(search.AlertID, active.GetId(), resolved.GetId()). + ProtoQuery() + + keys, err := s.datastore.SearchAlertMatchKeys(ctx, q, true) + s.NoError(err) + s.Require().Len(keys, 1) + s.Equal(active.GetId(), keys[0].GetId()) +} diff --git a/central/alert/datastore/datastore_sac_test.go b/central/alert/datastore/datastore_sac_test.go index fc1299d40656b..79b44ff42ea28 100644 --- a/central/alert/datastore/datastore_sac_test.go +++ b/central/alert/datastore/datastore_sac_test.go @@ -779,6 +779,63 @@ func (s *alertDatastoreSACTestSuite) TestAlertUnrestrictedListAlerts() { } } +func (s *alertDatastoreSACTestSuite) runSearchAlertPolicyNamesAndSeveritiesTest(testparams alertSACSearchResult) { + ctx := s.testContexts[testparams.scopeKey] + results, err := s.datastore.SearchAlertPolicyNamesAndSeverities(ctx, nil, true) + s.NoError(err) + expectedCount := testutils.AggregateCounts(s.T(), testparams.resultCounts) + s.Equal(expectedCount, len(results)) +} + +func (s *alertDatastoreSACTestSuite) TestAlertScopedSearchAlertPolicyNamesAndSeverities() { + for name, c := range alertScopedSACSearchTestCases { + s.Run(name, func() { + s.runSearchAlertPolicyNamesAndSeveritiesTest(c) + }) + } +} + +func (s *alertDatastoreSACTestSuite) TestAlertUnrestrictedSearchAlertPolicyNamesAndSeverities() { + for name, c := range alertUnrestrictedSACObjectSearchTestCases { + s.Run(name, func() { + s.runSearchAlertPolicyNamesAndSeveritiesTest(c) + }) + } +} + +func (s *alertDatastoreSACTestSuite) runSearchAlertPolicyGroupsTest(testparams alertSACSearchResult) { + ctx := s.testContexts[testparams.scopeKey] + results, err := s.datastore.SearchAlertPolicyGroups(ctx, nil, true) + s.NoError(err) + totalCount := 0 + for _, r := range results { + totalCount += r.NumAlerts + } + expectedCount := testutils.AggregateCounts(s.T(), testparams.resultCounts) + s.Equal(expectedCount, totalCount) + + // Verify categories are populated for all groups + for _, r := range results { + s.NotEmpty(r.Categories, "categories should be populated for policy %s", r.PolicyName) + } +} + +func (s *alertDatastoreSACTestSuite) TestAlertScopedSearchAlertPolicyGroups() { + for name, c := range alertScopedSACSearchTestCases { + s.Run(name, func() { + s.runSearchAlertPolicyGroupsTest(c) + }) + } +} + +func (s *alertDatastoreSACTestSuite) TestAlertUnrestrictedSearchAlertPolicyGroups() { + for name, c := range alertUnrestrictedSACObjectSearchTestCases { + s.Run(name, func() { + s.runSearchAlertPolicyGroupsTest(c) + }) + } +} + func (s *alertDatastoreSACTestSuite) countSearchRawAlertsResultsPerClusterAndNamespace(results []*storage.Alert) map[string]map[string]int { resultDistribution := make(map[string]map[string]int, 0) for _, result := range results { diff --git a/central/alert/datastore/internal/store/postgres/bench_test.go b/central/alert/datastore/internal/store/postgres/bench_test.go index 3a654cb7f925f..1e5f60fb9a15f 100644 --- a/central/alert/datastore/internal/store/postgres/bench_test.go +++ b/central/alert/datastore/internal/store/postgres/bench_test.go @@ -16,7 +16,7 @@ import ( func BenchmarkMany(b *testing.B) { var alerts []*storage.Alert const alertsNum = 10000 - for i := 0; i < alertsNum; i++ { + for range alertsNum { alert := &storage.Alert{} err := testutils.FullInit(alert, testutils.UniqueInitializer(), testutils.JSONFieldsFilter) if err != nil { @@ -25,7 +25,7 @@ func BenchmarkMany(b *testing.B) { alerts = append(alerts, alert) } - var idx []string + idx := make([]string, 0, len(alerts)) for _, a := range alerts { idx = append(idx, a.GetId()) } diff --git a/central/alert/datastore/internal/store/postgres/store.go b/central/alert/datastore/internal/store/postgres/store.go index b064e6717c056..80a9e61c911c5 100644 --- a/central/alert/datastore/internal/store/postgres/store.go +++ b/central/alert/datastore/internal/store/postgres/store.go @@ -141,6 +141,7 @@ func insertIntoAlerts(batch *pgx.Batch, obj *storage.Alert) error { pgutils.NilOrUUID(obj.GetNamespaceId()), pgutils.NilOrUUID(obj.GetDeployment().GetId()), obj.GetDeployment().GetName(), + obj.GetDeployment().GetType(), obj.GetDeployment().GetInactive(), obj.GetImage().GetId(), obj.GetImage().GetName().GetRegistry(), @@ -157,10 +158,11 @@ func insertIntoAlerts(batch *pgx.Batch, obj *storage.Alert) error { obj.GetState(), obj.GetPlatformComponent(), obj.GetEntityType(), + obj.GetEnforcementCount(), serialized, } - finalStr := "INSERT INTO alerts (Id, Policy_Id, Policy_Name, Policy_Description, Policy_Disabled, Policy_Categories, Policy_Severity, Policy_EnforcementActions, Policy_LastUpdated, Policy_SORTName, Policy_SORTLifecycleStage, Policy_SORTEnforcement, LifecycleStage, ClusterId, ClusterName, Namespace, NamespaceId, Deployment_Id, Deployment_Name, Deployment_Inactive, Image_Id, Image_Name_Registry, Image_Name_Remote, Image_Name_Tag, Image_Name_FullName, Image_IdV2, Node_Id, Node_Name, Resource_ResourceType, Resource_Name, Enforcement_Action, Time, State, PlatformComponent, EntityType, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, Policy_Id = EXCLUDED.Policy_Id, Policy_Name = EXCLUDED.Policy_Name, Policy_Description = EXCLUDED.Policy_Description, Policy_Disabled = EXCLUDED.Policy_Disabled, Policy_Categories = EXCLUDED.Policy_Categories, Policy_Severity = EXCLUDED.Policy_Severity, Policy_EnforcementActions = EXCLUDED.Policy_EnforcementActions, Policy_LastUpdated = EXCLUDED.Policy_LastUpdated, Policy_SORTName = EXCLUDED.Policy_SORTName, Policy_SORTLifecycleStage = EXCLUDED.Policy_SORTLifecycleStage, Policy_SORTEnforcement = EXCLUDED.Policy_SORTEnforcement, LifecycleStage = EXCLUDED.LifecycleStage, ClusterId = EXCLUDED.ClusterId, ClusterName = EXCLUDED.ClusterName, Namespace = EXCLUDED.Namespace, NamespaceId = EXCLUDED.NamespaceId, Deployment_Id = EXCLUDED.Deployment_Id, Deployment_Name = EXCLUDED.Deployment_Name, Deployment_Inactive = EXCLUDED.Deployment_Inactive, Image_Id = EXCLUDED.Image_Id, Image_Name_Registry = EXCLUDED.Image_Name_Registry, Image_Name_Remote = EXCLUDED.Image_Name_Remote, Image_Name_Tag = EXCLUDED.Image_Name_Tag, Image_Name_FullName = EXCLUDED.Image_Name_FullName, Image_IdV2 = EXCLUDED.Image_IdV2, Node_Id = EXCLUDED.Node_Id, Node_Name = EXCLUDED.Node_Name, Resource_ResourceType = EXCLUDED.Resource_ResourceType, Resource_Name = EXCLUDED.Resource_Name, Enforcement_Action = EXCLUDED.Enforcement_Action, Time = EXCLUDED.Time, State = EXCLUDED.State, PlatformComponent = EXCLUDED.PlatformComponent, EntityType = EXCLUDED.EntityType, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO alerts (Id, Policy_Id, Policy_Name, Policy_Description, Policy_Disabled, Policy_Categories, Policy_Severity, Policy_EnforcementActions, Policy_LastUpdated, Policy_SORTName, Policy_SORTLifecycleStage, Policy_SORTEnforcement, LifecycleStage, ClusterId, ClusterName, Namespace, NamespaceId, Deployment_Id, Deployment_Name, Deployment_Type, Deployment_Inactive, Image_Id, Image_Name_Registry, Image_Name_Remote, Image_Name_Tag, Image_Name_FullName, Image_IdV2, Node_Id, Node_Name, Resource_ResourceType, Resource_Name, Enforcement_Action, Time, State, PlatformComponent, EntityType, EnforcementCount, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, Policy_Id = EXCLUDED.Policy_Id, Policy_Name = EXCLUDED.Policy_Name, Policy_Description = EXCLUDED.Policy_Description, Policy_Disabled = EXCLUDED.Policy_Disabled, Policy_Categories = EXCLUDED.Policy_Categories, Policy_Severity = EXCLUDED.Policy_Severity, Policy_EnforcementActions = EXCLUDED.Policy_EnforcementActions, Policy_LastUpdated = EXCLUDED.Policy_LastUpdated, Policy_SORTName = EXCLUDED.Policy_SORTName, Policy_SORTLifecycleStage = EXCLUDED.Policy_SORTLifecycleStage, Policy_SORTEnforcement = EXCLUDED.Policy_SORTEnforcement, LifecycleStage = EXCLUDED.LifecycleStage, ClusterId = EXCLUDED.ClusterId, ClusterName = EXCLUDED.ClusterName, Namespace = EXCLUDED.Namespace, NamespaceId = EXCLUDED.NamespaceId, Deployment_Id = EXCLUDED.Deployment_Id, Deployment_Name = EXCLUDED.Deployment_Name, Deployment_Type = EXCLUDED.Deployment_Type, Deployment_Inactive = EXCLUDED.Deployment_Inactive, Image_Id = EXCLUDED.Image_Id, Image_Name_Registry = EXCLUDED.Image_Name_Registry, Image_Name_Remote = EXCLUDED.Image_Name_Remote, Image_Name_Tag = EXCLUDED.Image_Name_Tag, Image_Name_FullName = EXCLUDED.Image_Name_FullName, Image_IdV2 = EXCLUDED.Image_IdV2, Node_Id = EXCLUDED.Node_Id, Node_Name = EXCLUDED.Node_Name, Resource_ResourceType = EXCLUDED.Resource_ResourceType, Resource_Name = EXCLUDED.Resource_Name, Enforcement_Action = EXCLUDED.Enforcement_Action, Time = EXCLUDED.Time, State = EXCLUDED.State, PlatformComponent = EXCLUDED.PlatformComponent, EntityType = EXCLUDED.EntityType, EnforcementCount = EXCLUDED.EnforcementCount, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) return nil @@ -186,6 +188,7 @@ var copyColsAlerts = []string{ "namespaceid", "deployment_id", "deployment_name", + "deployment_type", "deployment_inactive", "image_id", "image_name_registry", @@ -202,6 +205,7 @@ var copyColsAlerts = []string{ "state", "platformcomponent", "entitytype", + "enforcementcount", "serialized", } @@ -255,6 +259,7 @@ func copyFromAlerts(ctx context.Context, s pgSearch.Deleter, tx *postgres.Tx, ob pgutils.NilOrUUID(obj.GetNamespaceId()), pgutils.NilOrUUID(obj.GetDeployment().GetId()), obj.GetDeployment().GetName(), + obj.GetDeployment().GetType(), obj.GetDeployment().GetInactive(), obj.GetImage().GetId(), obj.GetImage().GetName().GetRegistry(), @@ -271,6 +276,7 @@ func copyFromAlerts(ctx context.Context, s pgSearch.Deleter, tx *postgres.Tx, ob obj.GetState(), obj.GetPlatformComponent(), obj.GetEntityType(), + obj.GetEnforcementCount(), serialized, }, nil }) diff --git a/central/alert/datastore/mocks/datastore.go b/central/alert/datastore/mocks/datastore.go index 9bb026ba452bc..d43ea3fac1963 100644 --- a/central/alert/datastore/mocks/datastore.go +++ b/central/alert/datastore/mocks/datastore.go @@ -13,6 +13,7 @@ import ( context "context" reflect "reflect" + views "github.com/stackrox/rox/central/alert/views" v1 "github.com/stackrox/rox/generated/api/v1" storage "github.com/stackrox/rox/generated/storage" search "github.com/stackrox/rox/pkg/search" @@ -162,6 +163,96 @@ func (mr *MockDataStoreMockRecorder) Search(ctx, q, excludeResolved any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockDataStore)(nil).Search), ctx, q, excludeResolved) } +// SearchAlertDeploymentIDs mocks base method. +func (m *MockDataStore) SearchAlertDeploymentIDs(ctx context.Context, q *v1.Query, excludeResolved bool) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchAlertDeploymentIDs", ctx, q, excludeResolved) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchAlertDeploymentIDs indicates an expected call of SearchAlertDeploymentIDs. +func (mr *MockDataStoreMockRecorder) SearchAlertDeploymentIDs(ctx, q, excludeResolved any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchAlertDeploymentIDs", reflect.TypeOf((*MockDataStore)(nil).SearchAlertDeploymentIDs), ctx, q, excludeResolved) +} + +// SearchAlertMatchKeys mocks base method. +func (m *MockDataStore) SearchAlertMatchKeys(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*views.AlertMatchKey, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchAlertMatchKeys", ctx, q, excludeResolved) + ret0, _ := ret[0].([]*views.AlertMatchKey) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchAlertMatchKeys indicates an expected call of SearchAlertMatchKeys. +func (mr *MockDataStoreMockRecorder) SearchAlertMatchKeys(ctx, q, excludeResolved any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchAlertMatchKeys", reflect.TypeOf((*MockDataStore)(nil).SearchAlertMatchKeys), ctx, q, excludeResolved) +} + +// SearchAlertPolicyGroups mocks base method. +func (m *MockDataStore) SearchAlertPolicyGroups(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*views.AlertPolicyGroup, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchAlertPolicyGroups", ctx, q, excludeResolved) + ret0, _ := ret[0].([]*views.AlertPolicyGroup) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchAlertPolicyGroups indicates an expected call of SearchAlertPolicyGroups. +func (mr *MockDataStoreMockRecorder) SearchAlertPolicyGroups(ctx, q, excludeResolved any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchAlertPolicyGroups", reflect.TypeOf((*MockDataStore)(nil).SearchAlertPolicyGroups), ctx, q, excludeResolved) +} + +// SearchAlertPolicyNamesAndSeverities mocks base method. +func (m *MockDataStore) SearchAlertPolicyNamesAndSeverities(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*views.PolicyNameAndSeverity, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchAlertPolicyNamesAndSeverities", ctx, q, excludeResolved) + ret0, _ := ret[0].([]*views.PolicyNameAndSeverity) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchAlertPolicyNamesAndSeverities indicates an expected call of SearchAlertPolicyNamesAndSeverities. +func (mr *MockDataStoreMockRecorder) SearchAlertPolicyNamesAndSeverities(ctx, q, excludeResolved any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchAlertPolicyNamesAndSeverities", reflect.TypeOf((*MockDataStore)(nil).SearchAlertPolicyNamesAndSeverities), ctx, q, excludeResolved) +} + +// SearchAlertPolicySeverityCounts mocks base method. +func (m *MockDataStore) SearchAlertPolicySeverityCounts(ctx context.Context, q *v1.Query, excludeResolved bool) (*views.PolicySeverityCounts, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchAlertPolicySeverityCounts", ctx, q, excludeResolved) + ret0, _ := ret[0].(*views.PolicySeverityCounts) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchAlertPolicySeverityCounts indicates an expected call of SearchAlertPolicySeverityCounts. +func (mr *MockDataStoreMockRecorder) SearchAlertPolicySeverityCounts(ctx, q, excludeResolved any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchAlertPolicySeverityCounts", reflect.TypeOf((*MockDataStore)(nil).SearchAlertPolicySeverityCounts), ctx, q, excludeResolved) +} + +// SearchAlertTimeseriesEvents mocks base method. +func (m *MockDataStore) SearchAlertTimeseriesEvents(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*views.AlertTimeseriesEvent, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchAlertTimeseriesEvents", ctx, q, excludeResolved) + ret0, _ := ret[0].([]*views.AlertTimeseriesEvent) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchAlertTimeseriesEvents indicates an expected call of SearchAlertTimeseriesEvents. +func (mr *MockDataStoreMockRecorder) SearchAlertTimeseriesEvents(ctx, q, excludeResolved any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchAlertTimeseriesEvents", reflect.TypeOf((*MockDataStore)(nil).SearchAlertTimeseriesEvents), ctx, q, excludeResolved) +} + // SearchAlerts mocks base method. func (m *MockDataStore) SearchAlerts(ctx context.Context, q *v1.Query) ([]*v1.SearchResult, error) { m.ctrl.T.Helper() diff --git a/central/alert/datastore/singleton.go b/central/alert/datastore/singleton.go index a2280121871ac..9d38ad8ddc27c 100644 --- a/central/alert/datastore/singleton.go +++ b/central/alert/datastore/singleton.go @@ -13,8 +13,9 @@ var ( ) func initialize() { - storage := pgStore.New(globaldb.GetPostgres()) - soleInstance = New(storage, platformmatcher.Singleton()) + db := globaldb.GetPostgres() + storage := pgStore.New(db) + soleInstance = New(db, storage, platformmatcher.Singleton()) } // Singleton returns the sole instance of the DataStore service. diff --git a/central/alert/service/service_impl.go b/central/alert/service/service_impl.go index 3b23e23fca32c..2bb1dfebe5bc3 100644 --- a/central/alert/service/service_impl.go +++ b/central/alert/service/service_impl.go @@ -11,6 +11,7 @@ import ( "github.com/pkg/errors" "github.com/stackrox/rox/central/alert/datastore" "github.com/stackrox/rox/central/alert/mappings" + alertviews "github.com/stackrox/rox/central/alert/views" baselineDatastore "github.com/stackrox/rox/central/processbaseline/datastore" "github.com/stackrox/rox/central/sensor/service/connection" v1 "github.com/stackrox/rox/generated/api/v1" @@ -114,10 +115,8 @@ func (s *serviceImpl) GetAlert(ctx context.Context, request *v1.ResourceByID) (* // listAlertsRequestToQuery converts a v1.ListAlertsRequest to a search query func listAlertsRequestToQuery(request *v1.ListAlertsRequest, sort bool) (*v1.Query, error) { - var q *v1.Query - if request.GetQuery() == "" { - q = search.EmptyQuery() - } else { + q := search.EmptyQuery() + if request.GetQuery() != "" { var err error q, err = search.ParseQuery(request.GetQuery()) if err != nil { @@ -177,21 +176,24 @@ func ensureAllAlertsAreFetched(req *v1.ListAlertsRequest) *v1.ListAlertsRequest return req } -// GetAlertsGroup returns alerts according to the request, grouped by category and policy. +// GetAlertsGroup returns alerts according to the request, grouped by policy. func (s *serviceImpl) GetAlertsGroup(ctx context.Context, request *v1.ListAlertsRequest) (*v1.GetAlertsGroupResponse, error) { - request = ensureAllAlertsAreFetched(request) - q, err := listAlertsRequestToQuery(request, false) - if err != nil { - return nil, err + q := search.EmptyQuery() + if request.GetQuery() != "" { + var err error + q, err = search.ParseQuery(request.GetQuery()) + if err != nil { + return nil, err + } } - alerts, err := s.dataStore.SearchListAlerts(ctx, q, true) + + groups, err := s.dataStore.SearchAlertPolicyGroups(ctx, q, true) if err != nil { log.Error(err) return nil, err } - response := alertsGroupResponseFrom(alerts) - return response, nil + return alertsGroupResponseFromPolicyGroups(groups), nil } // GetAlertsCounts returns alert counts by severity according to the request. @@ -263,11 +265,11 @@ func (s *serviceImpl) GetAlertTimeseries(ctx context.Context, req *v1.ListAlerts return nil, err } - alerts, err := s.dataStore.SearchListAlerts(ctx, q, true) + events, err := s.dataStore.SearchAlertTimeseriesEvents(ctx, q, true) if err != nil { return nil, err } - response := alertTimeseriesResponseFrom(alerts) + response := alertTimeseriesResponseFrom(events) return response, nil } @@ -460,33 +462,26 @@ func (s *serviceImpl) DeleteAlerts(ctx context.Context, request *v1.DeleteAlerts return response, nil } -// alertsGroupResponseFrom returns a slice of storage.ListAlert objects translated into a v1.GetAlertsGroupResponse object. -func alertsGroupResponseFrom(alerts []*storage.ListAlert) (output *v1.GetAlertsGroupResponse) { - policiesMap := make(map[string]*storage.ListAlertPolicy) - alertCountsByPolicy := make(map[string]int) - - for _, a := range alerts { - if _, ok := policiesMap[a.GetPolicy().GetId()]; !ok { - policiesMap[a.GetPolicy().GetId()] = a.GetPolicy() - } - alertCountsByPolicy[a.GetPolicy().GetId()]++ +// alertsGroupResponseFromPolicyGroups converts aggregate AlertPolicyGroup results +// into a GetAlertsGroupResponse. Results are already sorted by policy name from +// the SQL query. +func alertsGroupResponseFromPolicyGroups(groups []*alertviews.AlertPolicyGroup) *v1.GetAlertsGroupResponse { + output := &v1.GetAlertsGroupResponse{ + AlertsByPolicies: make([]*v1.GetAlertsGroupResponse_PolicyGroup, 0, len(groups)), } - - output = new(v1.GetAlertsGroupResponse) - output.AlertsByPolicies = make([]*v1.GetAlertsGroupResponse_PolicyGroup, 0, len(policiesMap)) - - for id, p := range policiesMap { + for _, g := range groups { output.AlertsByPolicies = append(output.AlertsByPolicies, &v1.GetAlertsGroupResponse_PolicyGroup{ - Policy: p, - NumAlerts: int64(alertCountsByPolicy[id]), + Policy: &storage.ListAlertPolicy{ + Id: g.PolicyID, + Name: g.PolicyName, + Severity: g.GetPolicySeverity(), + Description: g.Description, + Categories: g.Categories, + }, + NumAlerts: int64(g.NumAlerts), }) } - - sort.Slice(output.GetAlertsByPolicies(), func(i, j int) bool { - return output.GetAlertsByPolicies()[i].GetPolicy().GetName() < output.GetAlertsByPolicies()[j].GetPolicy().GetName() - }) - - return + return output } // alertsCountsResponseFrom returns a slice of search.Result objects translated into a v1.GetAlertsCountsResponse @@ -500,11 +495,11 @@ func alertsCountsResponseFrom(alerts []search.Result, groupBy v1.GetAlertsCounts return nil, false } -// alertTimeseriesResponseFrom returns a slice of storage.ListAlert objects translated into a v1.GetAlertTimeseriesResponse +// alertTimeseriesResponseFrom returns a slice of AlertTimeseriesEvent objects translated into a v1.GetAlertTimeseriesResponse // object. -func alertTimeseriesResponseFrom(alerts []*storage.ListAlert) *v1.GetAlertTimeseriesResponse { +func alertTimeseriesResponseFrom(events []*alertviews.AlertTimeseriesEvent) *v1.GetAlertTimeseriesResponse { response := new(v1.GetAlertTimeseriesResponse) - for cluster, severityMap := range getGroupToAlertEvents(alerts) { + for cluster, severityMap := range getGroupToAlertEvents(events) { alertCluster := &v1.GetAlertTimeseriesResponse_ClusterAlerts{Cluster: cluster} for severity, alertEvents := range severityMap { // Sort the alert events so they are chronological @@ -579,25 +574,27 @@ func getMapOfAlertCounts(alerts []search.Result, groupByFunc func(alert search.R return } -func getGroupToAlertEvents(alerts []*storage.ListAlert) (clusters map[string]map[storage.Severity][]*v1.AlertEvent) { +func getGroupToAlertEvents(events []*alertviews.AlertTimeseriesEvent) (clusters map[string]map[storage.Severity][]*v1.AlertEvent) { clusters = make(map[string]map[storage.Severity][]*v1.AlertEvent) - for _, a := range alerts { - alertCluster := a.GetDeployment().GetClusterName() - if clusters[alertCluster] == nil { - clusters[alertCluster] = make(map[storage.Severity][]*v1.AlertEvent) + for _, e := range events { + clusterName := e.GetClusterName() + if clusters[clusterName] == nil { + clusters[clusterName] = make(map[storage.Severity][]*v1.AlertEvent) } - eventList := clusters[alertCluster][a.GetPolicy().GetSeverity()] - eventList = append(eventList, &v1.AlertEvent{Time: a.GetTime().GetSeconds() * 1000, Id: a.GetId(), Type: v1.Type_CREATED}) - if a.GetState() == storage.ViolationState_RESOLVED { - eventList = append(eventList, &v1.AlertEvent{Time: a.GetTime().GetSeconds() * 1000, Id: a.GetId(), Type: v1.Type_REMOVED}) + severity := e.GetSeverity() + timeMillis := e.GetTimeMillis() + eventList := clusters[clusterName][severity] + eventList = append(eventList, &v1.AlertEvent{Time: timeMillis, Id: e.GetAlertID(), Type: v1.Type_CREATED}) + if e.GetState() == storage.ViolationState_RESOLVED { + eventList = append(eventList, &v1.AlertEvent{Time: timeMillis, Id: e.GetAlertID(), Type: v1.Type_REMOVED}) } - clusters[alertCluster][a.GetPolicy().GetSeverity()] = eventList + clusters[clusterName][severity] = eventList } - for _, v1 := range clusters { - for k2, v2 := range v1 { - sort.SliceStable(v2, func(i, j int) bool { return v2[i].GetTime() < v2[j].GetTime() }) - v1[k2] = v2 + for _, severityMap := range clusters { + for k, v := range severityMap { + sort.SliceStable(v, func(i, j int) bool { return v[i].GetTime() < v[j].GetTime() }) + severityMap[k] = v } } return diff --git a/central/alert/service/service_test.go b/central/alert/service/service_test.go index 0489c312eef9e..6a0ea469eaa7c 100644 --- a/central/alert/service/service_test.go +++ b/central/alert/service/service_test.go @@ -4,10 +4,12 @@ import ( "context" "math" "testing" + "time" "github.com/pkg/errors" dataStoreMocks "github.com/stackrox/rox/central/alert/datastore/mocks" "github.com/stackrox/rox/central/alert/mappings" + alertviews "github.com/stackrox/rox/central/alert/views" "github.com/stackrox/rox/central/alerttest" baselineMocks "github.com/stackrox/rox/central/processbaseline/datastore/mocks" v1 "github.com/stackrox/rox/generated/api/v1" @@ -203,36 +205,20 @@ type getAlertsGroupsTests struct { } func (s *getAlertsGroupsTests) TestGetAlertsGroupForOneCategory() { - fakeListAlertSlice := []*storage.ListAlert{ + fakeGroups := []*alertviews.AlertPolicyGroup{ { - Id: "id1", - Policy: &storage.ListAlertPolicy{ - Categories: []string{"Image Assurance"}, - Id: "id1", - Name: "policy1", - Severity: storage.Severity_LOW_SEVERITY, - }, - Time: protocompat.GetProtoTimestampFromSeconds(300), - }, - { - Id: "id2", - Policy: &storage.ListAlertPolicy{ - Categories: []string{"Image Assurance"}, - Id: "id2", - Name: "policy2", - Severity: storage.Severity_HIGH_SEVERITY, - }, - Time: protocompat.GetProtoTimestampFromSeconds(200), + PolicyID: "id1", + PolicyName: "policy1", + Severity: int(storage.Severity_LOW_SEVERITY), + Categories: []string{"Image Assurance"}, + NumAlerts: 2, }, { - Id: "id3", - Policy: &storage.ListAlertPolicy{ - Categories: []string{"Image Assurance"}, - Id: "id1", - Name: "policy1", - Severity: storage.Severity_LOW_SEVERITY, - }, - Time: protocompat.GetProtoTimestampFromSeconds(100), + PolicyID: "id2", + PolicyName: "policy2", + Severity: int(storage.Severity_HIGH_SEVERITY), + Categories: []string{"Network", "Security"}, + NumAlerts: 1, }, } @@ -240,69 +226,53 @@ func (s *getAlertsGroupsTests) TestGetAlertsGroupForOneCategory() { AlertsByPolicies: []*v1.GetAlertsGroupResponse_PolicyGroup{ { Policy: &storage.ListAlertPolicy{ - Categories: []string{"Image Assurance"}, Id: "id1", Name: "policy1", Severity: storage.Severity_LOW_SEVERITY, + Categories: []string{"Image Assurance"}, }, NumAlerts: 2, }, { Policy: &storage.ListAlertPolicy{ - Categories: []string{"Image Assurance"}, Id: "id2", Name: "policy2", Severity: storage.Severity_HIGH_SEVERITY, + Categories: []string{"Network", "Security"}, }, NumAlerts: 1, }, }, } - s.testGetAlertsGroupFor(fakeListAlertSlice, expected) + s.testGetAlertsGroupFor(fakeGroups, expected) } -func (s *getAlertsGroupsTests) TestGetAlertsGroupForMultipleCategories() { - fakeListAlertSlice := []*storage.ListAlert{ +func (s *getAlertsGroupsTests) TestGetAlertsGroupForMultiplePolicies() { + fakeGroups := []*alertviews.AlertPolicyGroup{ { - Id: "id1", - Policy: &storage.ListAlertPolicy{ - Categories: []string{"Image Assurance"}, - Id: "id1", - Name: "policy1", - Severity: storage.Severity_LOW_SEVERITY, - }, - Time: protocompat.GetProtoTimestampFromSeconds(300), + PolicyID: "id1", + PolicyName: "policy1", + Severity: int(storage.Severity_LOW_SEVERITY), + Description: "low severity policy", + Categories: []string{"Image Assurance"}, + NumAlerts: 2, }, { - Id: "id2", - Policy: &storage.ListAlertPolicy{ - Categories: []string{"Image Assurance", "Privileges Capabilities"}, - Id: "id2", - Name: "policy2", - Severity: storage.Severity_HIGH_SEVERITY, - }, - Time: protocompat.GetProtoTimestampFromSeconds(200), + PolicyID: "id2", + PolicyName: "policy2", + Severity: int(storage.Severity_HIGH_SEVERITY), + Description: "high severity policy", + Categories: []string{"Network"}, + NumAlerts: 1, }, { - Id: "id3", - Policy: &storage.ListAlertPolicy{ - Categories: []string{"Container Configuration"}, - Id: "id30", - Name: "policy30", - Severity: storage.Severity_CRITICAL_SEVERITY, - }, - Time: protocompat.GetProtoTimestampFromSeconds(150), - }, - { - Id: "id4", - Policy: &storage.ListAlertPolicy{ - Categories: []string{"Image Assurance"}, - Id: "id1", - Name: "policy1", - Severity: storage.Severity_LOW_SEVERITY, - }, - Time: protocompat.GetProtoTimestampFromSeconds(100), + PolicyID: "id30", + PolicyName: "policy30", + Severity: int(storage.Severity_CRITICAL_SEVERITY), + Description: "critical policy", + Categories: []string{"Security", "DevOps"}, + NumAlerts: 1, }, } @@ -310,44 +280,44 @@ func (s *getAlertsGroupsTests) TestGetAlertsGroupForMultipleCategories() { AlertsByPolicies: []*v1.GetAlertsGroupResponse_PolicyGroup{ { Policy: &storage.ListAlertPolicy{ - Categories: []string{"Image Assurance"}, - Id: "id1", - Name: "policy1", - Severity: storage.Severity_LOW_SEVERITY, + Id: "id1", + Name: "policy1", + Severity: storage.Severity_LOW_SEVERITY, + Description: "low severity policy", + Categories: []string{"Image Assurance"}, }, NumAlerts: 2, }, { Policy: &storage.ListAlertPolicy{ - Categories: []string{"Image Assurance", "Privileges Capabilities"}, - Id: "id2", - Name: "policy2", - Severity: storage.Severity_HIGH_SEVERITY, + Id: "id2", + Name: "policy2", + Severity: storage.Severity_HIGH_SEVERITY, + Description: "high severity policy", + Categories: []string{"Network"}, }, NumAlerts: 1, }, { Policy: &storage.ListAlertPolicy{ - Categories: []string{"Container Configuration"}, - Id: "id30", - Name: "policy30", - Severity: storage.Severity_CRITICAL_SEVERITY, + Id: "id30", + Name: "policy30", + Severity: storage.Severity_CRITICAL_SEVERITY, + Description: "critical policy", + Categories: []string{"Security", "DevOps"}, }, NumAlerts: 1, }, }, } - s.testGetAlertsGroupFor(fakeListAlertSlice, expected) + s.testGetAlertsGroupFor(fakeGroups, expected) } -func (s *getAlertsGroupsTests) testGetAlertsGroupFor(fakeListAlertSlice []*storage.ListAlert, expected *v1.GetAlertsGroupResponse) { +func (s *getAlertsGroupsTests) testGetAlertsGroupFor(fakeGroups []*alertviews.AlertPolicyGroup, expected *v1.GetAlertsGroupResponse) { fakeContext := context.Background() protoQuery := search.NewQueryBuilder().ProtoQuery() - protoQuery.Pagination = &v1.QueryPagination{ - Limit: math.MaxInt32, - } - s.datastoreMock.EXPECT().SearchListAlerts(fakeContext, protoQuery, true).Return(fakeListAlertSlice, nil) + s.datastoreMock.EXPECT().SearchAlertPolicyGroups(fakeContext, protoQuery, true).Return(fakeGroups, nil) result, err := s.service.GetAlertsGroup(fakeContext, &v1.ListAlertsRequest{ Query: "", @@ -360,10 +330,7 @@ func (s *getAlertsGroupsTests) testGetAlertsGroupFor(fakeListAlertSlice []*stora func (s *getAlertsGroupsTests) TestGetAlertsGroupWhenTheDataAccessLayerFails() { fakeContext := context.Background() protoQuery := search.NewQueryBuilder().ProtoQuery() - protoQuery.Pagination = &v1.QueryPagination{ - Limit: math.MaxInt32, - } - s.datastoreMock.EXPECT().SearchListAlerts(fakeContext, protoQuery, true).Return(nil, errFake) + s.datastoreMock.EXPECT().SearchAlertPolicyGroups(fakeContext, protoQuery, true).Return(nil, errFake) result, err := s.service.GetAlertsGroup(fakeContext, &v1.ListAlertsRequest{ Query: "", @@ -760,40 +727,36 @@ type getAlertTimeseriesTests struct { } func (s *getAlertTimeseriesTests) TestGetAlertTimeseries() { - alerts := []*storage.ListAlert{ + time1 := time.Unix(1, 0) + time6 := time.Unix(6, 0) + events := []*alertviews.AlertTimeseriesEvent{ { - Id: "id1", - Time: protocompat.GetProtoTimestampFromSeconds(1), - - State: storage.ViolationState_RESOLVED, - Entity: &storage.ListAlert_Deployment{Deployment: &storage.ListAlertDeployment{ClusterName: "dev"}}, - CommonEntityInfo: &storage.ListAlert_CommonEntityInfo{ClusterName: "dev"}, - Policy: &storage.ListAlertPolicy{Severity: storage.Severity_CRITICAL_SEVERITY}, + AlertID: "id1", + ClusterName: "dev", + Severity: int(storage.Severity_CRITICAL_SEVERITY), + Time: &time1, + State: int(storage.ViolationState_RESOLVED), }, { - Id: "id2", - Time: protocompat.GetProtoTimestampFromSeconds(6), - - Entity: &storage.ListAlert_Deployment{Deployment: &storage.ListAlertDeployment{ClusterName: "dev"}}, - CommonEntityInfo: &storage.ListAlert_CommonEntityInfo{ClusterName: "dev"}, - Policy: &storage.ListAlertPolicy{Severity: storage.Severity_HIGH_SEVERITY}, + AlertID: "id2", + ClusterName: "dev", + Severity: int(storage.Severity_HIGH_SEVERITY), + Time: &time6, + State: int(storage.ViolationState_ACTIVE), }, { - Id: "id3", - Time: protocompat.GetProtoTimestampFromSeconds(1), - - State: storage.ViolationState_RESOLVED, - Entity: &storage.ListAlert_Deployment{Deployment: &storage.ListAlertDeployment{ClusterName: "prod"}}, - CommonEntityInfo: &storage.ListAlert_CommonEntityInfo{ClusterName: "prod"}, - Policy: &storage.ListAlertPolicy{Severity: storage.Severity_LOW_SEVERITY}, + AlertID: "id3", + ClusterName: "prod", + Severity: int(storage.Severity_LOW_SEVERITY), + Time: &time1, + State: int(storage.ViolationState_RESOLVED), }, { - Id: "id4", - Time: protocompat.GetProtoTimestampFromSeconds(6), - - Entity: &storage.ListAlert_Deployment{Deployment: &storage.ListAlertDeployment{ClusterName: "prod"}}, - CommonEntityInfo: &storage.ListAlert_CommonEntityInfo{ClusterName: "prod"}, - Policy: &storage.ListAlertPolicy{Severity: storage.Severity_MEDIUM_SEVERITY}, + AlertID: "id4", + ClusterName: "prod", + Severity: int(storage.Severity_MEDIUM_SEVERITY), + Time: &time6, + State: int(storage.ViolationState_ACTIVE), }, } @@ -863,7 +826,7 @@ func (s *getAlertTimeseriesTests) TestGetAlertTimeseries() { } fakeContext := context.Background() protoQuery := search.NewQueryBuilder().WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery() - s.datastoreMock.EXPECT().SearchListAlerts(fakeContext, protoQuery, true).Return(alerts, nil) + s.datastoreMock.EXPECT().SearchAlertTimeseriesEvents(fakeContext, protoQuery, true).Return(events, nil) result, err := s.service.GetAlertTimeseries(fakeContext, &v1.ListAlertsRequest{ Query: "", @@ -876,7 +839,7 @@ func (s *getAlertTimeseriesTests) TestGetAlertTimeseries() { func (s *getAlertTimeseriesTests) TestGetAlertTimeseriesWhenTheDataAccessLayerFails() { fakeContext := context.Background() protoQuery := search.NewQueryBuilder().WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery() - s.datastoreMock.EXPECT().SearchListAlerts(fakeContext, protoQuery, true).Return(nil, errFake) + s.datastoreMock.EXPECT().SearchAlertTimeseriesEvents(fakeContext, protoQuery, true).Return(nil, errFake) result, err := s.service.GetAlertTimeseries(fakeContext, &v1.ListAlertsRequest{ Query: "", diff --git a/central/alert/views/list_alert_scanner.go b/central/alert/views/list_alert_scanner.go new file mode 100644 index 0000000000000..d603e88a99a4f --- /dev/null +++ b/central/alert/views/list_alert_scanner.go @@ -0,0 +1,179 @@ +package views + +import ( + "github.com/jackc/pgx/v5/pgtype" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/search" +) + +var ( + log = logging.LoggerForModule() + + // ListAlertSelectProtos defines the column projections for ListAlert queries. + // Order must match the scan destinations in ListAlertScanner.Dests. + ListAlertSelectProtos = []*v1.QuerySelect{ + search.NewQuerySelect(search.AlertID).Proto(), + search.NewQuerySelect(search.LifecycleStage).Proto(), + search.NewQuerySelect(search.ViolationTime).Proto(), + search.NewQuerySelect(search.ViolationState).Proto(), + search.NewQuerySelect(search.PolicyID).Proto(), + search.NewQuerySelect(search.PolicyName).Proto(), + search.NewQuerySelect(search.Severity).Proto(), + search.NewQuerySelect(search.Description).Proto(), + search.NewQuerySelect(search.Category).Proto(), + search.NewQuerySelect(search.EnforcementAction).Proto(), + search.NewQuerySelect(search.EnforcementCount).Proto(), + search.NewQuerySelect(search.EntityType).Proto(), + search.NewQuerySelect(search.ClusterID).Proto(), + search.NewQuerySelect(search.Cluster).Proto(), + search.NewQuerySelect(search.Namespace).Proto(), + search.NewQuerySelect(search.NamespaceID).Proto(), + search.NewQuerySelect(search.DeploymentID).Proto(), + search.NewQuerySelect(search.DeploymentName).Proto(), + search.NewQuerySelect(search.DeploymentType).Proto(), + search.NewQuerySelect(search.Inactive).Proto(), + search.NewQuerySelect(search.NodeID).Proto(), + search.NewQuerySelect(search.Node).Proto(), + search.NewQuerySelect(search.ResourceName).Proto(), + search.NewQuerySelect(search.ResourceType).Proto(), + } + + // ListAlertArrayFields tells the query builder that "category" is a + // parent-table array column, not a child table requiring a JOIN. + ListAlertArrayFields = map[string]bool{ + "category": true, + } +) + +// ListAlertScanner holds pgtype scan destinations for the column projection +// query and converts scanned values into a *storage.ListAlert. +type ListAlertScanner struct { + ID pgtype.Text + LifecycleStage pgtype.Int4 + ViolationTime pgtype.Timestamp + State pgtype.Int4 + PolicyID pgtype.Text + PolicyName pgtype.Text + Severity pgtype.Int4 + Description pgtype.Text + Categories pgtype.FlatArray[string] + EnforcementAction pgtype.Int4 + EnforcementCount pgtype.Int4 + EntityType pgtype.Int4 + ClusterID pgtype.Text + ClusterName pgtype.Text + Namespace pgtype.Text + NamespaceID pgtype.Text + DeploymentID pgtype.Text + DeploymentName pgtype.Text + DeploymentType pgtype.Text + DeploymentInactive pgtype.Bool + NodeID pgtype.Text + NodeName pgtype.Text + ResourceName pgtype.Text + ResourceType pgtype.Int4 +} + +// Dests returns scan destination pointers in the order matching listAlertSelectProtos. +func (s *ListAlertScanner) Dests() []any { + return []any{ + &s.ID, &s.LifecycleStage, &s.ViolationTime, &s.State, + &s.PolicyID, &s.PolicyName, &s.Severity, &s.Description, &s.Categories, + &s.EnforcementAction, &s.EnforcementCount, &s.EntityType, + &s.ClusterID, &s.ClusterName, &s.Namespace, &s.NamespaceID, + &s.DeploymentID, &s.DeploymentName, &s.DeploymentType, &s.DeploymentInactive, + &s.NodeID, &s.NodeName, &s.ResourceName, &s.ResourceType, + } +} + +// Build converts the scanned column values into a *storage.ListAlert. +func (s *ListAlertScanner) Build() *storage.ListAlert { + la := &storage.ListAlert{ + Id: s.ID.String, + LifecycleStage: storage.LifecycleStage(s.LifecycleStage.Int32), + State: storage.ViolationState(s.State.Int32), + Policy: &storage.ListAlertPolicy{ + Id: s.PolicyID.String, + Name: s.PolicyName.String, + Severity: storage.Severity(s.Severity.Int32), + Description: s.Description.String, + Categories: []string(s.Categories), + }, + EnforcementAction: storage.EnforcementAction(s.EnforcementAction.Int32), + } + + if storage.ViolationState(s.State.Int32) == storage.ViolationState_ACTIVE { + la.EnforcementCount = s.EnforcementCount.Int32 + } + + if s.ViolationTime.Valid { + la.Time = protocompat.ConvertTimeToTimestampOrNil(&s.ViolationTime.Time) + } + + if !s.EntityType.Valid { + log.Warnf("alert %s has NULL entity type, skipping entity info", s.ID.String) + return la + } + + et := storage.Alert_EntityType(s.EntityType.Int32) + switch et { + case storage.Alert_DEPLOYMENT: + la.CommonEntityInfo = &storage.ListAlert_CommonEntityInfo{ + ClusterName: s.ClusterName.String, + ClusterId: s.ClusterID.String, + Namespace: s.Namespace.String, + NamespaceId: s.NamespaceID.String, + ResourceType: storage.ListAlert_DEPLOYMENT, + } + la.Entity = &storage.ListAlert_Deployment{ + Deployment: &storage.ListAlertDeployment{ + Id: s.DeploymentID.String, + Name: s.DeploymentName.String, + ClusterName: s.ClusterName.String, + ClusterId: s.ClusterID.String, + Namespace: s.Namespace.String, + NamespaceId: s.NamespaceID.String, + DeploymentType: deploymentTypeOrDefault(s.DeploymentType), + Inactive: s.DeploymentInactive.Bool, + }, + } + case storage.Alert_RESOURCE: + la.CommonEntityInfo = &storage.ListAlert_CommonEntityInfo{ + ClusterName: s.ClusterName.String, + ClusterId: s.ClusterID.String, + Namespace: s.Namespace.String, + NamespaceId: s.NamespaceID.String, + ResourceType: storage.ListAlert_ResourceType(s.ResourceType.Int32), + } + la.Entity = &storage.ListAlert_Resource{ + Resource: &storage.ListAlert_ResourceEntity{ + Name: s.ResourceName.String, + }, + } + case storage.Alert_NODE: + la.CommonEntityInfo = &storage.ListAlert_CommonEntityInfo{ + ClusterName: s.ClusterName.String, + ClusterId: s.ClusterID.String, + ResourceType: storage.ListAlert_NODE, + } + la.Entity = &storage.ListAlert_Node{ + Node: &storage.ListAlert_NodeEntity{ + Name: s.NodeName.String, + }, + } + default: + log.Warnf("alert %s has unhandled entity type %d, skipping entity info", s.ID.String, s.EntityType.Int32) + } + + return la +} + +func deploymentTypeOrDefault(dt pgtype.Text) string { + if dt.Valid && dt.String != "" { + return dt.String + } + return "Deployment" +} diff --git a/central/alert/views/list_alert_scanner_test.go b/central/alert/views/list_alert_scanner_test.go new file mode 100644 index 0000000000000..f1885ebd0a5dd --- /dev/null +++ b/central/alert/views/list_alert_scanner_test.go @@ -0,0 +1,79 @@ +package views + +import ( + "testing" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/stackrox/rox/pkg/search" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestListAlertSelectProtosMatchDests(t *testing.T) { + var sc ListAlertScanner + dests := sc.Dests() + + require.Equal(t, len(ListAlertSelectProtos), len(dests), + "ListAlertSelectProtos and Dests() must have the same length") + + expectedOrder := []search.FieldLabel{ + search.AlertID, + search.LifecycleStage, + search.ViolationTime, + search.ViolationState, + search.PolicyID, + search.PolicyName, + search.Severity, + search.Description, + search.Category, + search.EnforcementAction, + search.EnforcementCount, + search.EntityType, + search.ClusterID, + search.Cluster, + search.Namespace, + search.NamespaceID, + search.DeploymentID, + search.DeploymentName, + search.DeploymentType, + search.Inactive, + search.NodeID, + search.Node, + search.ResourceName, + search.ResourceType, + } + + require.Equal(t, len(expectedOrder), len(ListAlertSelectProtos), + "expectedOrder must match ListAlertSelectProtos length") + + for i, sel := range ListAlertSelectProtos { + assert.Equal(t, expectedOrder[i].String(), sel.GetField().GetName(), + "ListAlertSelectProtos[%d] field name mismatch", i) + } +} + +func TestDeploymentTypeOrDefault(t *testing.T) { + cases := map[string]struct { + input pgtype.Text + expected string + }{ + "valid value": { + input: pgtype.Text{String: "DaemonSet", Valid: true}, + expected: "DaemonSet", + }, + "null": { + input: pgtype.Text{Valid: false}, + expected: "Deployment", + }, + "empty string": { + input: pgtype.Text{String: "", Valid: true}, + expected: "Deployment", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, deploymentTypeOrDefault(tc.input)) + }) + } +} diff --git a/central/alert/views/views.go b/central/alert/views/views.go new file mode 100644 index 0000000000000..8bc73a2559201 --- /dev/null +++ b/central/alert/views/views.go @@ -0,0 +1,312 @@ +package views + +import ( + "time" + + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" +) + +// DeploymentIDResult is a lightweight projection of alert data containing only +// the deployment ID. Used by failingDeployments to avoid deserializing full +// alert protobuf blobs when only the deployment ID is needed. +// DeploymentID is a pointer to handle NULL values from resource/image alerts. +type DeploymentIDResult struct { + DeploymentID *string `db:"deployment_id"` +} + +// GetDeploymentID returns the deployment ID, or an empty string if NULL. +func (d *DeploymentIDResult) GetDeploymentID() string { + if d.DeploymentID == nil { + return "" + } + return *d.DeploymentID +} + +// PolicyNameAndSeverity is a lightweight projection of alert data containing only +// the policy name and severity. Used by risk scoring to avoid deserializing full +// alert protobuf blobs when only these two fields are needed. +type PolicyNameAndSeverity struct { + PolicyName string `db:"policy"` + Severity int `db:"severity"` +} + +// GetPolicyName returns the policy name. +func (p *PolicyNameAndSeverity) GetPolicyName() string { + return p.PolicyName +} + +// GetSeverity returns the severity as a storage.Severity enum value. +func (p *PolicyNameAndSeverity) GetSeverity() storage.Severity { + return storage.Severity(p.Severity) +} + +// PolicySeverityCounts holds the count of distinct policies per severity level. +// Used by GraphQL resolvers that only need aggregate policy counts, avoiding +// deserialization of individual alert protobuf blobs. +type PolicySeverityCounts struct { + LowCount int `db:"low_policy_count"` + MediumCount int `db:"medium_policy_count"` + HighCount int `db:"high_policy_count"` + CriticalCount int `db:"critical_policy_count"` +} + +// AlertPolicyGroup holds the result of a GROUP BY query that counts alerts per policy. +// Used by GetAlertsGroup to avoid deserializing full alert protobuf blobs. +type AlertPolicyGroup struct { + PolicyID string `db:"policy_id"` + PolicyName string `db:"policy"` + Severity int `db:"severity"` + Description string `db:"description"` + Categories []string `db:"category"` + NumAlerts int `db:"alert_id_count"` +} + +// GetPolicySeverity returns the severity as a storage.Severity enum value. +func (g *AlertPolicyGroup) GetPolicySeverity() storage.Severity { + return storage.Severity(g.Severity) +} + +// AlertTimeseriesEvent is a lightweight projection of alert data containing only +// the fields needed by GetAlertTimeseries: id, cluster name, severity, time, and state. +// Used to avoid deserializing full alert protobuf blobs. +type AlertTimeseriesEvent struct { + AlertID string `db:"alert_id"` + ClusterName string `db:"cluster"` + Severity int `db:"severity"` + Time *time.Time `db:"violation_time"` + State int `db:"violation_state"` +} + +// GetAlertID returns the alert ID. +func (e *AlertTimeseriesEvent) GetAlertID() string { + return e.AlertID +} + +// GetClusterName returns the cluster name. +func (e *AlertTimeseriesEvent) GetClusterName() string { + return e.ClusterName +} + +// GetSeverity returns the severity as a storage.Severity enum value. +func (e *AlertTimeseriesEvent) GetSeverity() storage.Severity { + return storage.Severity(e.Severity) +} + +// GetTimeMillis returns the violation time in milliseconds since epoch. +func (e *AlertTimeseriesEvent) GetTimeMillis() int64 { + if e.Time == nil { + return 0 + } + return e.Time.Unix() * 1000 +} + +// GetState returns the violation state as a storage.ViolationState enum value. +func (e *AlertTimeseriesEvent) GetState() storage.ViolationState { + return storage.ViolationState(e.State) +} + +// WithAlertTimeseriesQuery augments a query with SELECT on the 5 fields needed +// by GetAlertTimeseries: alert_id, cluster, severity, violation_time, and violation_state. +// Results are sorted by violation time ascending. +func WithAlertTimeseriesQuery(q *v1.Query) *v1.Query { + cloned := q.CloneVT() + cloned.Selects = []*v1.QuerySelect{ + search.NewQuerySelect(search.AlertID).Proto(), + search.NewQuerySelect(search.Cluster).Proto(), + search.NewQuerySelect(search.Severity).Proto(), + search.NewQuerySelect(search.ViolationTime).Proto(), + search.NewQuerySelect(search.ViolationState).Proto(), + } + cloned.Pagination = &v1.QueryPagination{ + SortOptions: []*v1.QuerySortOption{ + { + Field: search.ViolationTime.String(), + }, + }, + } + return cloned +} + +// WithAlertPolicyGroupQuery augments a query with COUNT(alert_id) and GROUP BY +// on policy_id, policy name, severity, and description. Results are sorted by +// policy name ascending. +func WithAlertPolicyGroupQuery(q *v1.Query) *v1.Query { + cloned := q.CloneVT() + cloned.Selects = []*v1.QuerySelect{ + search.NewQuerySelect(search.PolicyID).Proto(), + search.NewQuerySelect(search.PolicyName).Proto(), + search.NewQuerySelect(search.Severity).Proto(), + search.NewQuerySelect(search.Description).Proto(), + search.NewQuerySelect(search.Category).Proto(), + search.NewQuerySelect(search.AlertID).AggrFunc(aggregatefunc.Count).Proto(), + } + cloned.GroupBy = &v1.QueryGroupBy{ + Fields: []string{ + search.PolicyID.String(), + search.PolicyName.String(), + search.Severity.String(), + search.Description.String(), + search.Category.String(), + }, + } + cloned.Pagination = &v1.QueryPagination{ + SortOptions: []*v1.QuerySortOption{ + { + Field: search.PolicyName.String(), + }, + }, + } + return cloned +} + +// WithPolicySeverityCountQuery augments a query with filtered COUNT(DISTINCT policy_id) +// selects, one per severity level. Returns a single row with 4 integer columns. +func WithPolicySeverityCountQuery(q *v1.Query) *v1.Query { + cloned := q.CloneVT() + cloned.Selects = []*v1.QuerySelect{ + buildSelectCountBySeverity("low_policy_count", storage.Severity_LOW_SEVERITY), + buildSelectCountBySeverity("medium_policy_count", storage.Severity_MEDIUM_SEVERITY), + buildSelectCountBySeverity("high_policy_count", storage.Severity_HIGH_SEVERITY), + buildSelectCountBySeverity("critical_policy_count", storage.Severity_CRITICAL_SEVERITY), + } + return cloned +} + +func buildSelectCountBySeverity(filterName string, severity storage.Severity) *v1.QuerySelect { + return search.NewQuerySelect(search.PolicyID). + Distinct(). + AggrFunc(aggregatefunc.Count). + Filter(filterName, + search.NewQueryBuilder(). + AddExactMatches(search.Severity, severity.String()). + ProtoQuery(), + ).Proto() +} + +// AlertMatcher provides the fields needed to match alerts by policy and entity +// without deserializing full protobuf blobs. Both AlertMatchKey (lightweight +// DB projection) and *storage.Alert (via an adapter) can implement this. +type AlertMatcher interface { + GetId() string + GetPolicyId() string + GetState() storage.ViolationState + GetLifecycleStage() storage.LifecycleStage + HasDeployment() bool + GetDeploymentId() string + IsDeploymentInactive() bool + HasResource() bool + GetResourceType() storage.Alert_Resource_ResourceType + GetResourceName() string + HasNode() bool + GetNodeId() string + GetNodeName() string + GetClusterId() string + GetNamespace() string +} + +// AlertMatchKey is a lightweight projection of alert data containing only the +// fields needed by mergeManyAlerts to match incoming alerts against previous +// alerts and determine resolution/inactive status. Avoids TOAST I/O by +// reading only inline columns. +type AlertMatchKey struct { + ID string `db:"alert_id"` + PolicyID string `db:"policy_id"` + State int `db:"violation_state"` + LifecycleStage int `db:"lifecycle_stage"` + DeploymentID *string `db:"deployment_id"` + DeploymentInactive *bool `db:"inactive_deployment"` + ResourceType *int `db:"resource_type"` + ResourceName *string `db:"resource"` + ClusterID *string `db:"cluster_id"` + Namespace *string `db:"namespace"` + NodeID *string `db:"node_id"` + NodeName *string `db:"node"` +} + +func (k *AlertMatchKey) GetId() string { return k.ID } +func (k *AlertMatchKey) GetPolicyId() string { return k.PolicyID } +func (k *AlertMatchKey) GetState() storage.ViolationState { return storage.ViolationState(k.State) } +func (k *AlertMatchKey) GetLifecycleStage() storage.LifecycleStage { + return storage.LifecycleStage(k.LifecycleStage) +} + +func (k *AlertMatchKey) HasDeployment() bool { + return k.DeploymentID != nil && *k.DeploymentID != "" +} +func (k *AlertMatchKey) GetDeploymentId() string { + if k.DeploymentID == nil { + return "" + } + return *k.DeploymentID +} +func (k *AlertMatchKey) IsDeploymentInactive() bool { + return k.DeploymentInactive != nil && *k.DeploymentInactive +} +func (k *AlertMatchKey) HasResource() bool { + return k.ResourceName != nil && *k.ResourceName != "" +} +func (k *AlertMatchKey) GetResourceType() storage.Alert_Resource_ResourceType { + if k.ResourceType == nil { + return 0 + } + return storage.Alert_Resource_ResourceType(*k.ResourceType) +} +func (k *AlertMatchKey) GetResourceName() string { + if k.ResourceName == nil { + return "" + } + return *k.ResourceName +} +func (k *AlertMatchKey) HasNode() bool { + return k.NodeID != nil && *k.NodeID != "" +} +func (k *AlertMatchKey) GetNodeId() string { + if k.NodeID == nil { + return "" + } + return *k.NodeID +} +func (k *AlertMatchKey) GetNodeName() string { + if k.NodeName == nil { + return "" + } + return *k.NodeName +} +func (k *AlertMatchKey) GetClusterId() string { + if k.ClusterID == nil { + return "" + } + return *k.ClusterID +} +func (k *AlertMatchKey) GetNamespace() string { + if k.Namespace == nil { + return "" + } + return *k.Namespace +} + +// WithAlertMatchKeyQuery augments a query with SELECT on the 12 inline columns +// needed for alert matching: alert_id, policy_id, violation_state, lifecycle_stage, +// deployment_id, inactive_deployment, resource_type, resource, cluster_id, +// namespace, node_id, and node. +func WithAlertMatchKeyQuery(q *v1.Query) *v1.Query { + cloned := q.CloneVT() + cloned.Selects = []*v1.QuerySelect{ + search.NewQuerySelect(search.AlertID).Proto(), + search.NewQuerySelect(search.PolicyID).Proto(), + search.NewQuerySelect(search.ViolationState).Proto(), + search.NewQuerySelect(search.LifecycleStage).Proto(), + search.NewQuerySelect(search.DeploymentID).Proto(), + search.NewQuerySelect(search.Inactive).Proto(), + search.NewQuerySelect(search.ResourceType).Proto(), + search.NewQuerySelect(search.ResourceName).Proto(), + search.NewQuerySelect(search.ClusterID).Proto(), + search.NewQuerySelect(search.Namespace).Proto(), + search.NewQuerySelect(search.NodeID).Proto(), + search.NewQuerySelect(search.Node).Proto(), + } + return cloned +} diff --git a/central/apitoken/backend/backend_impl.go b/central/apitoken/backend/backend_impl.go index 1edb2c4425059..4f0ad5951e638 100644 --- a/central/apitoken/backend/backend_impl.go +++ b/central/apitoken/backend/backend_impl.go @@ -31,8 +31,12 @@ func (c *backendImpl) IssueRoleToken(ctx context.Context, name string, roleNames if expireAt != nil && expireAt.Before(time.Now()) { return "", nil, errox.InvalidArgs.New("Expiration date cannot be in the past") } - claims := tokens.RoxClaims{RoleNames: roleNames, Name: name, ExpireAt: expireAt} - tokenInfo, err := c.issuer.Issue(ctx, claims) + claims := tokens.RoxClaims{RoleNames: roleNames, Name: name} + var opts []tokens.Option + if expireAt != nil { + opts = append(opts, tokens.WithExpiry(*expireAt)) + } + tokenInfo, err := c.issuer.Issue(ctx, claims, opts...) if err != nil { return "", nil, err } diff --git a/central/audit/audit.go b/central/audit/audit.go index c2f0f698014a8..f5038230a755d 100644 --- a/central/audit/audit.go +++ b/central/audit/audit.go @@ -2,10 +2,12 @@ package audit import ( "context" + "fmt" "net/http" "github.com/pkg/errors" v1 "github.com/stackrox/rox/generated/api/v1" + tokenServiceV1 "github.com/stackrox/rox/generated/internalapi/central/v1" "github.com/stackrox/rox/generated/storage" auditPkg "github.com/stackrox/rox/pkg/audit" "github.com/stackrox/rox/pkg/env" @@ -17,6 +19,7 @@ import ( "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/protoutils" "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/utils" "google.golang.org/grpc" ) @@ -56,13 +59,24 @@ func (a *audit) SendAuditMessage(ctx context.Context, req interface{}, grpcMetho a.notifications.ProcessAuditMessage(ctx, am) } -var requestInteractionMap = map[string]v1.Audit_Interaction{ - http.MethodPost: v1.Audit_CREATE, - http.MethodPut: v1.Audit_UPDATE, - http.MethodPatch: v1.Audit_UPDATE, - http.MethodDelete: v1.Audit_DELETE, - defaultGRPCMethod: v1.Audit_UPDATE, -} +var ( + requestInteractionMap = map[string]v1.Audit_Interaction{ + http.MethodPost: v1.Audit_CREATE, + http.MethodPut: v1.Audit_UPDATE, + http.MethodPatch: v1.Audit_UPDATE, + http.MethodDelete: v1.Audit_DELETE, + defaultGRPCMethod: v1.Audit_UPDATE, + } + + // auditableServiceEndpoints contains service-to-service endpoints that should be audited. + // When adding new security-sensitive internal RPCs, add their full method name constant + // here to ensure they appear in audit logs. These use generated gRPC full method name + // constants, so any rename or signature change to the RPC will cause a compile-time + // error rather than silently breaking auditing. + auditableServiceEndpoints = set.NewFrozenSet( + tokenServiceV1.TokenService_GenerateTokenForPermissionsAndScope_FullMethodName, + ) +) func (a *audit) newAuditMessage(ctx context.Context, req interface{}, grpcFullMethod string, authError interceptor.AuthStatus, requestError error) *v1.Audit_Message { @@ -73,15 +87,22 @@ func (a *audit) newAuditMessage(ctx context.Context, req interface{}, grpcFullMe } identity := authn.IdentityFromContextOrNil(ctx) - // Ignore requests from services + isServiceIdentity := false + // Selectively audit service-to-service requests for security-sensitive operations. if identity != nil { - if identity.Service() != nil { - return nil + if svc := identity.Service(); svc != nil { + isServiceIdentity = true + if !auditableServiceEndpoints.Contains(grpcFullMethod) { + return nil + } + // For service requests, populate user info from service identity. + msg.User = serviceIdentityUserInfo(svc) + } else { + msg.User = utils.IfThenElse(a.withoutPermissions, + stripPermissionsFromUserInfo(identity.User()), + identity.User(), + ) } - msg.User = utils.IfThenElse(a.withoutPermissions, - stripPermissionsFromUserInfo(identity.User()), - identity.User(), - ) } var method, endpoint string @@ -96,7 +117,12 @@ func (a *audit) newAuditMessage(ctx context.Context, req interface{}, grpcFullMe } else { method = defaultGRPCMethod endpoint = grpcFullMethod - msg.Method = v1.Audit_CLI + // Service-to-service gRPC calls should be marked as API, not CLI. + if isServiceIdentity { + msg.Method = v1.Audit_API + } else { + msg.Method = v1.Audit_CLI + } } interaction, ok := requestInteractionMap[method] @@ -156,6 +182,16 @@ func calculateAuditStatus(authError interceptor.AuthStatus, requestError error) } } +// serviceIdentityUserInfo builds a UserInfo from a service identity for audit logging. +// Username uses the "service::" format to clearly distinguish +// service identities from regular users in audit log queries. +func serviceIdentityUserInfo(svc *storage.ServiceIdentity) *storage.UserInfo { + return &storage.UserInfo{ + Username: fmt.Sprintf("service:%s:%s", svc.GetType().String(), svc.GetId()), + FriendlyName: fmt.Sprintf("Service: %s (ID: %s)", svc.GetType().String(), svc.GetId()), + } +} + func stripPermissionsFromUserInfo(userInfo *storage.UserInfo) *storage.UserInfo { userInfoWithoutPermissions := userInfo.CloneVT() userInfoWithoutPermissions.Permissions = nil diff --git a/central/audit/audit_test.go b/central/audit/audit_test.go index bad1915048b7f..d6dce3110fa44 100644 --- a/central/audit/audit_test.go +++ b/central/audit/audit_test.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" v1 "github.com/stackrox/rox/generated/api/v1" + tokenServiceV1 "github.com/stackrox/rox/generated/internalapi/central/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/grpc/authn" identityMocks "github.com/stackrox/rox/pkg/grpc/authn/mocks" @@ -130,6 +131,110 @@ func (suite *AuditLogTestSuite) TestPermissionsRemoval() { } } +func (suite *AuditLogTestSuite) TestServiceRequestsForInternalTokenEndpointAreAudited() { + // Create mock service identity. + serviceIdentity := &storage.ServiceIdentity{ + Id: "test-sensor-12345", + Type: storage.ServiceType_SENSOR_SERVICE, + } + suite.identityMock.EXPECT().Service().Return(serviceIdentity).AnyTimes() + suite.identityMock.EXPECT().User().Return(nil).AnyTimes() + + ctxWithServiceIdentity := authn.ContextWithIdentity(context.Background(), suite.identityMock, suite.T()) + ctxWithAuth := interceptor.ContextWithAuthStatus(ctxWithServiceIdentity, nil) + + a := &audit{notifications: suite.notifierMock} + + // Test internal token generation endpoint - should be audited. + msg := a.newAuditMessage(ctxWithAuth, "test-request", + tokenServiceV1.TokenService_GenerateTokenForPermissionsAndScope_FullMethodName, + interceptor.AuthStatus{Error: nil}, nil) + + suite.NotNil(msg, "Service request for internal token endpoint should be audited") + suite.NotNil(msg.GetUser(), "Audit message should contain user info from service identity") + suite.Equal("service:SENSOR_SERVICE:test-sensor-12345", msg.GetUser().GetUsername()) + suite.Equal("Service: SENSOR_SERVICE (ID: test-sensor-12345)", msg.GetUser().GetFriendlyName()) + suite.Equal(v1.Audit_API, msg.GetMethod(), "Service requests should have method=API") + suite.Equal(tokenServiceV1.TokenService_GenerateTokenForPermissionsAndScope_FullMethodName, + msg.GetRequest().GetEndpoint(), "Audit message should contain the correct endpoint") + suite.Equal(v1.Audit_UPDATE, msg.GetInteraction(), "gRPC requests should have interaction=UPDATE") + suite.Equal(v1.Audit_REQUEST_SUCCEEDED, msg.GetStatus(), "Successful request should have REQUEST_SUCCEEDED status") +} + +func (suite *AuditLogTestSuite) TestServiceRequestsForOtherEndpointsAreNotAudited() { + // Create mock service identity. + serviceIdentity := &storage.ServiceIdentity{ + Id: "test-sensor-12345", + Type: storage.ServiceType_SENSOR_SERVICE, + } + suite.identityMock.EXPECT().Service().Return(serviceIdentity).AnyTimes() + + ctxWithServiceIdentity := authn.ContextWithIdentity(context.Background(), suite.identityMock, suite.T()) + ctxWithAuth := interceptor.ContextWithAuthStatus(ctxWithServiceIdentity, nil) + + a := &audit{notifications: suite.notifierMock} + + // Test non-token endpoint - should NOT be audited. + msg := a.newAuditMessage(ctxWithAuth, "test-request", + "/v1.SomeOtherService/SomeMethod", + interceptor.AuthStatus{Error: nil}, nil) + + suite.Nil(msg, "Service request for non-token endpoint should not be audited") +} + +func (suite *AuditLogTestSuite) TestServiceRequestsForInternalTokenEndpointWithAuthErrorAreAudited() { + // Create mock service identity. + serviceIdentity := &storage.ServiceIdentity{ + Id: "test-sensor-12345", + Type: storage.ServiceType_SENSOR_SERVICE, + } + suite.identityMock.EXPECT().Service().Return(serviceIdentity).AnyTimes() + suite.identityMock.EXPECT().User().Return(nil).AnyTimes() + + ctxWithServiceIdentity := authn.ContextWithIdentity(context.Background(), suite.identityMock, suite.T()) + ctxWithAuth := interceptor.ContextWithAuthStatus(ctxWithServiceIdentity, nil) + + a := &audit{notifications: suite.notifierMock} + + // Simulate an auth failure for the internal token endpoint. + authErr := errors.New("test auth error") + msg := a.newAuditMessage(ctxWithAuth, "test-request", + tokenServiceV1.TokenService_GenerateTokenForPermissionsAndScope_FullMethodName, + interceptor.AuthStatus{Error: authErr}, nil) + + suite.NotNil(msg, "Service request for internal token endpoint with auth error should still be audited") + suite.NotNil(msg.GetUser(), "Audit message should contain user info from service identity") + suite.Equal("service:SENSOR_SERVICE:test-sensor-12345", msg.GetUser().GetUsername()) + suite.Equal("Service: SENSOR_SERVICE (ID: test-sensor-12345)", msg.GetUser().GetFriendlyName()) + suite.Equal(v1.Audit_API, msg.GetMethod(), "Service requests should have method=API") + suite.Equal(v1.Audit_AUTH_FAILED, msg.GetStatus(), "Auth error should result in AUTH_FAILED status") + suite.Contains(msg.GetStatusReason(), authErr.Error(), "Audit message should contain auth error details") +} + +func (suite *AuditLogTestSuite) TestUserRequestsContinueToBeAudited() { + userInfo := &storage.UserInfo{ + Username: "test-user", + FriendlyName: "Test User", + } + suite.identityMock.EXPECT().Service().Return(nil).AnyTimes() + suite.identityMock.EXPECT().User().Return(userInfo).AnyTimes() + + ctxWithUserIdentity := authn.ContextWithIdentity(context.Background(), suite.identityMock, suite.T()) + ctxWithAuth := interceptor.ContextWithAuthStatus(ctxWithUserIdentity, nil) + + a := &audit{notifications: suite.notifierMock} + + // Test any endpoint with user identity - should be audited. + msg := a.newAuditMessage(ctxWithAuth, "test-request", + "/v1.SomeService/SomeMethod", + interceptor.AuthStatus{Error: nil}, nil) + + suite.NotNil(msg, "User request should be audited") + suite.NotNil(msg.GetUser(), "Audit message should contain user info") + suite.Equal("test-user", msg.GetUser().GetUsername()) + suite.Equal(v1.Audit_CLI, msg.GetMethod(), "User gRPC requests should have method=CLI") +} + func handler(err error) func(ctx context.Context, req interface{}) (interface{}, error) { return func(ctx context.Context, req interface{}) (interface{}, error) { return nil, err diff --git a/central/auth/datastore/datastore_impl.go b/central/auth/datastore/datastore_impl.go index be8cbef83eda2..d9a063149981a 100644 --- a/central/auth/datastore/datastore_impl.go +++ b/central/auth/datastore/datastore_impl.go @@ -23,6 +23,8 @@ import ( "github.com/stackrox/rox/pkg/uuid" ) +const configControllerM2MAudience = "central.stackrox.io" + var ( _ DataStore = (*datastoreImpl)(nil) @@ -101,7 +103,7 @@ func (d *datastoreImpl) upsertAuthM2MConfigNoLock(ctx context.Context, // Upsert the token exchanger first, ensuring the config is valid and a token exchanger can be successfully // created from it. if err := d.set.UpsertTokenExchanger(ctx, config); err != nil { - return nil, d.wrapRollBackSet(ctx, err, storedConfig, config, existingExchanger) + return nil, d.wrapRollback(ctx, tx, err, storedConfig, config, existingExchanger) } // Upsert the config to the DB after the token exchanger has been successfully added. @@ -257,6 +259,7 @@ func newKubeM2MConfig(kubeSAIssuer string) *storage.AuthMachineToMachineConfig { TokenExpirationDuration: "1h", Mappings: []*storage.AuthMachineToMachineConfig_Mapping{}, Issuer: kubeSAIssuer, + Audience: configControllerM2MAudience, } } @@ -287,6 +290,14 @@ func (d *datastoreImpl) configureConfigControllerAccess(kubeSAConfig *storage.Au }) } + // Above we ensure that the config-controller mapping is always present. + // But we only set the audience on upgraded configs when the config-controller + // mapping is the only mapping. If users added their own mappings, their tokens + // may not carry the custom audience, so enforcing it could break their setup. + if kubeSAConfig.GetAudience() == "" && len(kubeSAConfig.GetMappings()) == 1 { + kubeSAConfig.Audience = configControllerM2MAudience + } + ctx := sac.WithGlobalAccessScopeChecker(context.Background(), sac.AllowFixedScopes( sac.AccessModeScopeKeys(storage.Access_READ_WRITE_ACCESS), sac.ResourceScopeKeys(resources.Access))) diff --git a/central/auth/datastore/datastore_impl_test.go b/central/auth/datastore/datastore_impl_test.go index c851bee44cdba..c588ccf1bf4ac 100644 --- a/central/auth/datastore/datastore_impl_test.go +++ b/central/auth/datastore/datastore_impl_test.go @@ -4,6 +4,7 @@ package datastore import ( "context" + "errors" "fmt" "testing" @@ -162,6 +163,7 @@ func (s *datastorePostgresTestSuite) TestKubeSAM2MConfigPersistsAfterDelete() { s.Equal("sub", kubeSAConfig.GetMappings()[0].GetKey()) s.Equal("Configuration Controller", kubeSAConfig.GetMappings()[0].GetRole()) s.Contains(kubeSAConfig.GetMappings()[0].GetValueExpression(), "config-controller") + s.Equal(configControllerM2MAudience, kubeSAConfig.GetAudience()) } s.kubeSAM2MConfig(authDataStoreMutator, authDataStoreValidator) @@ -175,6 +177,7 @@ func (s *datastorePostgresTestSuite) TestKubeSAM2MConfigPersistsAfterRestart() { s.Equal("sub", kubeSAConfig.GetMappings()[0].GetKey()) s.Equal("Configuration Controller", kubeSAConfig.GetMappings()[0].GetRole()) s.Contains(kubeSAConfig.GetMappings()[0].GetValueExpression(), "config-controller") + s.Equal(configControllerM2MAudience, kubeSAConfig.GetAudience()) } s.kubeSAM2MConfig(authDataStoreMutator, authDataStoreValidator) @@ -222,11 +225,109 @@ func (s *datastorePostgresTestSuite) TestKubeSAM2MConfigPersistsAfterModificatio s.FailNowf("Failed to find role mapping", "key=%s; role=%s; valueExpression=%s", mapping.GetKey(), mapping.GetRole(), mapping.GetValueExpression()) } } + s.Equal(configControllerM2MAudience, kubeSAConfig.GetAudience()) } s.kubeSAM2MConfig(authDataStoreMutator, authDataStoreValidator) } +// TestKubeSAM2MConfigAudienceSetOnUpgradeWithSingleMapping simulates upgrading +// from a Central version that did not set an audience on the kube SA M2M config. +// When the config only has the config-controller mapping, the audience should be +// backfilled on restart. +func (s *datastorePostgresTestSuite) TestKubeSAM2MConfigAudienceSetOnUpgradeWithSingleMapping() { + controller := gomock.NewController(s.T()) + defer controller.Finish() + authStore := store.New(s.pool.DB) + + // Seed a pre-existing config without audience (simulates pre-upgrade state). + legacyConfig := &storage.AuthMachineToMachineConfig{ + Id: uuid.NewV4().String(), + Type: storage.AuthMachineToMachineConfig_KUBE_SERVICE_ACCOUNT, + TokenExpirationDuration: "1h", + Issuer: testIssuer, + Mappings: []*storage.AuthMachineToMachineConfig_Mapping{ + { + Key: "sub", + ValueExpression: configControllerServiceAccountName, + Role: configController, + }, + }, + } + s.Require().NoError(authStore.Upsert(s.ctx, legacyConfig)) + + // Simulate Central restart with InitializeTokenExchangers. + mockSet := mocks.NewMockTokenExchangerSet(controller) + mockSet.EXPECT().UpsertTokenExchanger(gomock.Any(), kubeSAMatcher{}).Return(nil).MinTimes(1) + mockSet.EXPECT().GetTokenExchanger(gomock.Any()).Return(nil, false).AnyTimes() + + authDataStore := New(authStore, s.roleDataStore, mockSet) + s.NoError(authDataStore.InitializeTokenExchangers()) + + var kubeSAConfig *storage.AuthMachineToMachineConfig + err := authDataStore.ForEachAuthM2MConfig(s.ctx, func(obj *storage.AuthMachineToMachineConfig) error { + if obj.GetIssuer() == testIssuer { + kubeSAConfig = obj + } + return nil + }) + s.NoError(err) + s.Require().NotNil(kubeSAConfig) + s.Equal(1, len(kubeSAConfig.GetMappings())) + s.Equal(configControllerM2MAudience, kubeSAConfig.GetAudience(), "audience should be backfilled when only the config-controller mapping exists") +} + +// TestKubeSAM2MConfigAudienceNotSetOnUpgradeWithMultipleMappings simulates +// upgrading from a Central version that did not set an audience, where the user +// has added their own role mappings to the kube SA M2M config. The audience must +// NOT be backfilled because the user's tokens won't carry the custom audience. +func (s *datastorePostgresTestSuite) TestKubeSAM2MConfigAudienceNotSetOnUpgradeWithMultipleMappings() { + controller := gomock.NewController(s.T()) + defer controller.Finish() + authStore := store.New(s.pool.DB) + + // Seed a pre-existing config without audience that has an additional user mapping. + legacyConfig := &storage.AuthMachineToMachineConfig{ + Id: uuid.NewV4().String(), + Type: storage.AuthMachineToMachineConfig_KUBE_SERVICE_ACCOUNT, + TokenExpirationDuration: "1h", + Issuer: testIssuer, + Mappings: []*storage.AuthMachineToMachineConfig_Mapping{ + { + Key: "sub", + ValueExpression: configControllerServiceAccountName, + Role: configController, + }, + { + Key: "sub", + ValueExpression: "system:serviceaccount:my-namespace:my-service-account", + Role: testRole1, + }, + }, + } + s.Require().NoError(authStore.Upsert(s.ctx, legacyConfig)) + + // Simulate Central restart with InitializeTokenExchangers. + mockSet := mocks.NewMockTokenExchangerSet(controller) + mockSet.EXPECT().UpsertTokenExchanger(gomock.Any(), kubeSAMatcher{}).Return(nil).MinTimes(1) + mockSet.EXPECT().GetTokenExchanger(gomock.Any()).Return(nil, false).AnyTimes() + + authDataStore := New(authStore, s.roleDataStore, mockSet) + s.NoError(authDataStore.InitializeTokenExchangers()) + + var kubeSAConfig *storage.AuthMachineToMachineConfig + err := authDataStore.ForEachAuthM2MConfig(s.ctx, func(obj *storage.AuthMachineToMachineConfig) error { + if obj.GetIssuer() == testIssuer { + kubeSAConfig = obj + } + return nil + }) + s.NoError(err) + s.Require().NotNil(kubeSAConfig) + s.Equal(2, len(kubeSAConfig.GetMappings())) + s.Empty(kubeSAConfig.GetAudience(), "audience must not be set when user-defined mappings exist") +} + func (s *datastorePostgresTestSuite) TestAddFKConstraint() { config, err := s.authDataStore.UpsertAuthM2MConfig(s.ctx, &storage.AuthMachineToMachineConfig{ Id: "80c053c2-24a7-4b97-bd69-85b3a511241e", @@ -329,7 +430,7 @@ func (s *datastorePostgresTestSuite) addRoles( Traits: objectTraits.CloneVT(), })) - for _, role := range testRoles.AsSlice() { + for role := range testRoles.All() { s.Require().NoError(roleStore.Upsert(s.ctx, &storage.Role{ Name: namePrefix + role, Description: "test role", @@ -448,3 +549,65 @@ func (s *datastorePostgresTestSuite) TestDeclarativeUpserts() { }) } } + +// TestUpsertTokenExchangerFailureRollsBackTransaction verifies that when UpsertTokenExchanger +// fails, the database transaction is properly rolled back and no data is persisted. +// This test was added to prevent connection leaks where the transaction was not being +// rolled back when the token exchanger creation failed (e.g., OIDC provider errors). +func (s *datastorePostgresTestSuite) TestUpsertTokenExchangerFailureRollsBackTransaction() { + controller := gomock.NewController(s.T()) + defer controller.Finish() + + authStore := store.New(s.pool.DB) + + // Mock UpsertTokenExchanger to return an error (simulating OIDC provider failure) + tokenExchangerError := errors.New("creating OIDC provider: 404 Not Found: NoSuchBucket") + mockSet := mocks.NewMockTokenExchangerSet(controller) + mockSet.EXPECT().GetTokenExchanger(gomock.Any()).Return(nil, false).AnyTimes() + mockSet.EXPECT().UpsertTokenExchanger(gomock.Any(), gomock.Any()).Return(tokenExchangerError).Times(1) + // Expect rollback operations to be called + mockSet.EXPECT().RemoveTokenExchanger(gomock.Any()).Return(nil).Times(1) + + authDataStore := New(authStore, s.roleDataStore, mockSet) + + testConfigID := uuid.NewV4().String() + testIssuerURL := "https://storage.googleapis.com/test-bucket" + + config := &storage.AuthMachineToMachineConfig{ + Id: testConfigID, + Type: storage.AuthMachineToMachineConfig_GENERIC, + TokenExpirationDuration: "5m", + Mappings: []*storage.AuthMachineToMachineConfig_Mapping{ + { + Key: "sub", + ValueExpression: "test-value", + Role: testRole1, + }, + }, + Issuer: testIssuerURL, + } + + // Attempt to upsert - should fail + result, err := authDataStore.UpsertAuthM2MConfig(s.ctx, config) + s.Error(err, "UpsertAuthM2MConfig should return an error when UpsertTokenExchanger fails") + s.Nil(result, "Result should be nil when upsert fails") + s.Contains(err.Error(), "NoSuchBucket", "Error should contain the original error message") + + // Verify NO config was persisted (transaction was rolled back) + // Use a fresh datastore with a permissive mock to read from the database + readMockSet := mocks.NewMockTokenExchangerSet(controller) + readMockSet.EXPECT().GetTokenExchanger(gomock.Any()).Return(nil, false).AnyTimes() + readMockSet.EXPECT().UpsertTokenExchanger(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + readDataStore := New(authStore, s.roleDataStore, readMockSet) + + var foundConfig *storage.AuthMachineToMachineConfig + err = readDataStore.ForEachAuthM2MConfig(s.ctx, func(obj *storage.AuthMachineToMachineConfig) error { + if obj.GetId() == testConfigID || obj.GetIssuer() == testIssuerURL { + foundConfig = obj + } + return nil + }) + s.NoError(err) + s.Nil(foundConfig, "Config should NOT be persisted when UpsertTokenExchanger fails - transaction should have been rolled back") +} diff --git a/central/auth/internaltokens/service/metrics.go b/central/auth/internaltokens/service/metrics.go new file mode 100644 index 0000000000000..7729a5160bcc5 --- /dev/null +++ b/central/auth/internaltokens/service/metrics.go @@ -0,0 +1,49 @@ +package service + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stackrox/rox/pkg/metrics" +) + +// Result label values for tokenGenerationTotal. +const ( + tokenGenResultSuccess = "success" + tokenGenResultInvalidArgs = "invalid_args" + tokenGenResultRoleCreationError = "role_creation_error" + tokenGenResultIssuanceError = "token_issuance_error" // #nosec G101 not a hardcoded credential +) + +var ( + // tokenGenerationTotal counts token generation attempts by result. + tokenGenerationTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "internal_token_generation_total", + Help: "Total number of internal token generation attempts by result.", + }, []string{"result"}) + + // tokenGenerationDuration tracks the latency of token generation attempts by result. + tokenGenerationDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "internal_token_generation_duration_seconds", + Help: "Duration of internal token generation attempts in seconds by result.", + Buckets: prometheus.DefBuckets, + }, []string{"result"}) +) + +func init() { + metrics.EmplaceCollector( + tokenGenerationTotal, + tokenGenerationDuration, + ) +} + +// observeTokenGeneration increments the token generation counter and observes +// the generation duration for all results. +func observeTokenGeneration(result string, duration time.Duration) { + tokenGenerationTotal.WithLabelValues(result).Inc() + tokenGenerationDuration.WithLabelValues(result).Observe(duration.Seconds()) +} diff --git a/central/auth/internaltokens/service/policy.go b/central/auth/internaltokens/service/policy.go new file mode 100644 index 0000000000000..e4e669dba4559 --- /dev/null +++ b/central/auth/internaltokens/service/policy.go @@ -0,0 +1,139 @@ +package service + +import ( + "context" + "time" + + v1 "github.com/stackrox/rox/generated/internalapi/central/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/grpc/authn" + "google.golang.org/protobuf/types/known/durationpb" +) + +const ( + // defaultMaxLifetime is the maximum token lifetime enforced by the + // default policy. + defaultMaxLifetime = 1 * time.Hour +) + +type tokenPolicy struct { + maxLifetime *durationpb.Duration + allowedPermissions map[string]v1.Access +} + +// newTokenPolicy creates a tokenPolicy with the given maximum lifetime and +// allowed permissions. +func newTokenPolicy(maxLifetime time.Duration, allowedPermissions map[string]v1.Access) *tokenPolicy { + var maxLT *durationpb.Duration + if maxLifetime > 0 { + maxLT = durationpb.New(maxLifetime) + } + return &tokenPolicy{ + maxLifetime: maxLT, + allowedPermissions: allowedPermissions, + } +} + +// defaultTokenPolicy returns the hardcoded policy for internal tokens issued +// to sensors. It caps lifetime to 1 hour and allows read access to Deployment +// and Image resources. +// This policy should be kept in sync with the needs of the ocp console plugin. +func defaultTokenPolicy() *tokenPolicy { + // defaultAllowedPermissions lists the resource permissions that the default + // policy allows sensors to request. These must stay in sync with the needs + // of the known token consumers (including the OCP console plugin). + defaultAllowedPermissions := map[string]v1.Access{ + "Deployment": v1.Access_READ_ACCESS, + "Image": v1.Access_READ_ACCESS, + } + return newTokenPolicy(defaultMaxLifetime, defaultAllowedPermissions) +} + +// validatePermissions checks that every requested permission is present in the +// allowlist with an access level no greater than the allowed level. +func (p *tokenPolicy) validatePermissions(requested map[string]v1.Access) error { + for resource, requestedAccess := range requested { + allowedAccess, ok := p.allowedPermissions[resource] + if !ok { + return errox.InvalidArgs.Newf( + "permission %q for resource %q is not allowed", requestedAccess.String(), resource) + } + if requestedAccess > allowedAccess { + return errox.InvalidArgs.Newf( + "requested permission %q for resource %q exceeds the allowed maximum permission %q", + requestedAccess.String(), resource, allowedAccess.String()) + } + } + return nil +} + +// validateClusterScope checks that every ClusterScope in the request references +// only the requesting sensor's own cluster. +func (p *tokenPolicy) validateClusterScope(scopes []*v1.ClusterScope, sensorClusterID string) error { + for _, scope := range scopes { + if scope.GetClusterId() != sensorClusterID { + return errox.InvalidArgs.Newf( + "cluster scope references cluster %q, but requesting sensor belongs to cluster %q", + scope.GetClusterId(), sensorClusterID) + } + } + return nil +} + +// enforceLifetimeLimit validates that the requested lifetime is a positive duration +// and caps it to the configured maximum. If the lifetime is capped, a cloned +// request with the adjusted lifetime is returned. +func (p *tokenPolicy) enforceLifetimeLimit( + req *v1.GenerateTokenForPermissionsAndScopeRequest, +) (*v1.GenerateTokenForPermissionsAndScopeRequest, error) { + lifetime := req.GetLifetime() + if lifetime == nil { + return nil, errox.InvalidArgs.New("token lifetime is required") + } + if err := lifetime.CheckValid(); err != nil { + return nil, errox.InvalidArgs.CausedByf("converting requested token lifetime: %v", err) + } + if lifetime.AsDuration() <= 0 { + return nil, errox.InvalidArgs.CausedBy("token lifetime must be positive") + } + if p.maxLifetime != nil && lifetime.AsDuration() > p.maxLifetime.AsDuration() { + req = req.CloneVT() + req.Lifetime = p.maxLifetime + } + return req, nil +} + +// validateSensorIdentity validates and extracts the sensor identity from the +// context. Returns the sensor's cluster ID. +func (p *tokenPolicy) validateSensorIdentity(ctx context.Context) (string, error) { + identity := authn.IdentityFromContextOrNil(ctx) + if identity == nil || identity.Service() == nil { + return "", errox.NotAuthorized.New("missing service identity") + } + if identity.Service().GetType() != storage.ServiceType_SENSOR_SERVICE { + return "", errox.NotAuthorized.CausedByf( + "only sensor may access this API, unexpected service type %q", + identity.Service().GetType()) + } + return identity.Service().GetId(), nil +} + +// enforce validates the request against all policy checks and returns the +// request with a capped lifetime if necessary. +func (p *tokenPolicy) enforce( + ctx context.Context, + req *v1.GenerateTokenForPermissionsAndScopeRequest, +) (*v1.GenerateTokenForPermissionsAndScopeRequest, error) { + sensorClusterID, err := p.validateSensorIdentity(ctx) + if err != nil { + return nil, err + } + if err := p.validatePermissions(req.GetPermissions()); err != nil { + return nil, err + } + if err := p.validateClusterScope(req.GetClusterScopes(), sensorClusterID); err != nil { + return nil, err + } + return p.enforceLifetimeLimit(req) +} diff --git a/central/auth/internaltokens/service/policy_test.go b/central/auth/internaltokens/service/policy_test.go new file mode 100644 index 0000000000000..784432bb487ae --- /dev/null +++ b/central/auth/internaltokens/service/policy_test.go @@ -0,0 +1,324 @@ +package service + +import ( + "testing" + "time" + + v1 "github.com/stackrox/rox/generated/internalapi/central/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" + "github.com/stackrox/rox/pkg/grpc/authn" + authnMocks "github.com/stackrox/rox/pkg/grpc/authn/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/durationpb" +) + +// This test is a sanity check in case defaultAllowedPermissions is modified. +func TestDefaultTokenPolicyIsCompatibleWithOCPPlugin(t *testing.T) { + policy := defaultTokenPolicy() + require.NotNil(t, policy) + + // Needed for vulnerability management in the ocp console plugin. + minimumPermissionSet := map[string]v1.Access{ + "Deployment": v1.Access_READ_ACCESS, + "Image": v1.Access_READ_ACCESS, + } + assert.Equal(t, minimumPermissionSet, policy.allowedPermissions, + "default allowed permissions must be compatible with ocp plugin") +} + +func TestValidatePermissions(t *testing.T) { + policy := newTokenPolicy(0, map[string]v1.Access{ + "Deployment": v1.Access_READ_ACCESS, + "Image": v1.Access_READ_ACCESS, + }) + + for name, tc := range map[string]struct { + requested map[string]v1.Access + expectError bool + }{ + "nil permissions": { + requested: nil, + }, + "empty permissions": { + requested: map[string]v1.Access{}, + }, + "valid subset - single": { + requested: map[string]v1.Access{ + "Deployment": v1.Access_READ_ACCESS, + }, + }, + "valid subset - both": { + requested: map[string]v1.Access{ + "Deployment": v1.Access_READ_ACCESS, + "Image": v1.Access_READ_ACCESS, + }, + }, + "lower access than allowed": { + requested: map[string]v1.Access{ + "Deployment": v1.Access_NO_ACCESS, + }, + }, + "resource not in allowlist": { + requested: map[string]v1.Access{ + "NetworkGraph": v1.Access_READ_ACCESS, + }, + expectError: true, + }, + "access exceeds allowlist": { + requested: map[string]v1.Access{ + "Deployment": v1.Access_READ_WRITE_ACCESS, + }, + expectError: true, + }, + "mixed - one valid, one not allowed resource": { + requested: map[string]v1.Access{ + "Deployment": v1.Access_READ_ACCESS, + "NetworkGraph": v1.Access_READ_ACCESS, + }, + expectError: true, + }, + } { + t.Run(name, func(t *testing.T) { + err := policy.validatePermissions(tc.requested) + if tc.expectError { + assert.ErrorIs(t, err, errox.InvalidArgs) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateClusterScope(t *testing.T) { + policy := newTokenPolicy(0, nil) + + for name, tc := range map[string]struct { + scopes []*v1.ClusterScope + sensorClusterID string + expectError bool + }{ + "nil scopes": { + scopes: nil, + sensorClusterID: fixtureconsts.Cluster1, + }, + "empty scopes": { + scopes: []*v1.ClusterScope{}, + sensorClusterID: fixtureconsts.Cluster1, + }, + "matching cluster": { + scopes: []*v1.ClusterScope{ + {ClusterId: fixtureconsts.Cluster1}, + }, + sensorClusterID: fixtureconsts.Cluster1, + }, + "multiple matching clusters": { + scopes: []*v1.ClusterScope{ + {ClusterId: fixtureconsts.Cluster1}, + {ClusterId: fixtureconsts.Cluster1, Namespaces: []string{"ns1"}}, + }, + sensorClusterID: fixtureconsts.Cluster1, + }, + "mismatched cluster": { + scopes: []*v1.ClusterScope{ + {ClusterId: fixtureconsts.Cluster2}, + }, + sensorClusterID: fixtureconsts.Cluster1, + expectError: true, + }, + "one matching, one mismatched": { + scopes: []*v1.ClusterScope{ + {ClusterId: fixtureconsts.Cluster1}, + {ClusterId: fixtureconsts.Cluster2}, + }, + sensorClusterID: fixtureconsts.Cluster1, + expectError: true, + }, + } { + t.Run(name, func(t *testing.T) { + err := policy.validateClusterScope(tc.scopes, tc.sensorClusterID) + if tc.expectError { + assert.ErrorIs(t, err, errox.InvalidArgs) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestEnforce(t *testing.T) { + policyDuration := 1 * time.Hour + policy := newTokenPolicy(policyDuration, map[string]v1.Access{ + "Deployment": v1.Access_READ_ACCESS, + }) + + for name, tc := range map[string]struct { + req *v1.GenerateTokenForPermissionsAndScopeRequest + clusterID string + expectError bool + expectCapped bool + }{ + "permission validation failure": { + req: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"NetworkGraph": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: durationpb.New(5 * time.Minute), + }, + clusterID: fixtureconsts.Cluster1, + expectError: true, + }, + "cluster scope violation": { + req: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster2}}, + Lifetime: durationpb.New(5 * time.Minute), + }, + clusterID: fixtureconsts.Cluster1, + expectError: true, + }, + "nil lifetime": { + req: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + }, + clusterID: fixtureconsts.Cluster1, + expectError: true, + }, + "invalid proto duration": { + req: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: &durationpb.Duration{Seconds: 60, Nanos: -654321987}, + }, + clusterID: fixtureconsts.Cluster1, + expectError: true, + }, + "zero lifetime": { + req: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: durationpb.New(0), + }, + clusterID: fixtureconsts.Cluster1, + expectError: true, + }, + "negative lifetime": { + req: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: durationpb.New(-5 * time.Minute), + }, + clusterID: fixtureconsts.Cluster1, + expectError: true, + }, + "lifetime within limit": { + req: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: durationpb.New(30 * time.Minute), + }, + clusterID: fixtureconsts.Cluster1, + }, + "lifetime exceeds limit": { + req: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: durationpb.New(2 * time.Hour), + }, + clusterID: fixtureconsts.Cluster1, + expectCapped: true, + }, + } { + t.Run(name, func(t *testing.T) { + ctx := sensorContext(t, gomock.NewController(t), tc.clusterID) + result, err := policy.enforce(ctx, tc.req) + if tc.expectError { + assert.ErrorIs(t, err, errox.InvalidArgs) + assert.Nil(t, result) + } else { + require.NoError(t, err) + require.NotNil(t, result) + if tc.expectCapped { + assert.Equal(t, durationpb.New(policyDuration), result.GetLifetime()) + } else { + assert.Equal(t, tc.req.GetLifetime(), result.GetLifetime()) + } + } + }) + } + + t.Run("missing identity rejects request", func(t *testing.T) { + req := &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: durationpb.New(5 * time.Minute), + } + result, err := policy.enforce(t.Context(), req) + assert.ErrorIs(t, err, errox.NotAuthorized) + assert.Nil(t, result) + }) + + t.Run("identity with nil service rejects request", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockIdentity := authnMocks.NewMockIdentity(ctrl) + mockIdentity.EXPECT().Service().Return(nil).AnyTimes() + ctx := authn.ContextWithIdentity(t.Context(), mockIdentity, t) + + req := &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: durationpb.New(5 * time.Minute), + } + result, err := policy.enforce(ctx, req) + assert.ErrorIs(t, err, errox.NotAuthorized) + assert.Nil(t, result) + }) + + t.Run("non-sensor service identity rejects request", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockIdentity := authnMocks.NewMockIdentity(ctrl) + mockIdentity.EXPECT().Service().Return(&storage.ServiceIdentity{ + Id: "some-service-id", + Type: storage.ServiceType_CENTRAL_SERVICE, + }).AnyTimes() + ctx := authn.ContextWithIdentity(t.Context(), mockIdentity, t) + + req := &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: durationpb.New(5 * time.Minute), + } + result, err := policy.enforce(ctx, req) + assert.ErrorIs(t, err, errox.NotAuthorized) + assert.Nil(t, result) + }) + + t.Run("zero maxLifetime applies no cap", func(t *testing.T) { + p := newTokenPolicy(0, map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}) + req := &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: durationpb.New(2 * time.Hour), + } + ctx := sensorContext(t, gomock.NewController(t), fixtureconsts.Cluster1) + result, err := p.enforce(ctx, req) + require.NoError(t, err) + assert.Equal(t, 2*time.Hour, result.GetLifetime().AsDuration()) + }) + + t.Run("negative maxLifetime applies no cap", func(t *testing.T) { + p := newTokenPolicy(-policyDuration, map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}) + req := &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{"Deployment": v1.Access_READ_ACCESS}, + ClusterScopes: []*v1.ClusterScope{{ClusterId: fixtureconsts.Cluster1}}, + Lifetime: durationpb.New(2 * time.Hour), + } + ctx := sensorContext(t, gomock.NewController(t), fixtureconsts.Cluster1) + result, err := p.enforce(ctx, req) + require.NoError(t, err) + assert.Equal(t, 2*time.Hour, result.GetLifetime().AsDuration()) + }) +} diff --git a/central/auth/internaltokens/service/role_manager.go b/central/auth/internaltokens/service/role_manager.go index 069a6a240b35c..91b99b1ac9541 100644 --- a/central/auth/internaltokens/service/role_manager.go +++ b/central/auth/internaltokens/service/role_manager.go @@ -2,85 +2,52 @@ package service import ( "context" - "fmt" - "slices" - "strings" - "github.com/pkg/errors" - clusterDatastore "github.com/stackrox/rox/central/cluster/datastore" - roleDatastore "github.com/stackrox/rox/central/role/datastore" v1 "github.com/stackrox/rox/generated/internalapi/central/v1" "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/declarativeconfig" + "github.com/stackrox/rox/pkg/auth/tokens" ) const ( - permissionSetNameFormat = "Generated permission set for %s" - accessScopeNameFormat = "Generated access scope for %s" - roleNameFormat = "Generated role for permission set %s and access scope %s" - - accessScopeDescription = "Generated access scope for internal authentication tokens" - permissionSetDescription = "Generated permission set for internal authentication tokens" - roleDescription = "Generated role for internal authentication tokens" - - primaryListSeparator = ";" - keyValueSeparator = ":" - secondaryListSeparator = "," - clusterWildCard = "*" + internalRoleName = "internal role" ) -type roleManager struct { - clusterStore clusterDatastore.DataStore - roleStore roleDatastore.DataStore -} - -var ( - generatedObjectTraits = &storage.Traits{Origin: storage.Traits_IMPERATIVE} -) +type roleManager struct{} -// createPermissionSet creates a dynamic permission set, granting the requested permissions. -// The returned information is the ID of the created permission set, or an error if any occurred -// in the creation process. -func (rm *roleManager) createPermissionSet( - ctx context.Context, +func (rm *roleManager) createRoleForRoxClaims( + _ context.Context, req *v1.GenerateTokenForPermissionsAndScopeRequest, -) (string, error) { - // TODO: Consider pruning the generated permission sets after some idle time. - permissionSet := &storage.PermissionSet{ - Description: permissionSetDescription, - ResourceToAccess: make(map[string]storage.Access), - Traits: generatedObjectTraits.CloneVT(), +) (*tokens.InternalRole, error) { + role := &tokens.InternalRole{ + RoleName: internalRoleName, + } + requestedPermissions := req.GetPermissions() + if len(requestedPermissions) > 0 { + role.Permissions = make(map[storage.Access][]string) + } + for resource, v1Access := range req.GetPermissions() { + access := convertAccess(v1Access) + if _, found := role.Permissions[access]; !found { + role.Permissions[access] = make([]string, 0) + } + role.Permissions[access] = append(role.Permissions[access], resource) } - var b strings.Builder - permissions := req.GetPermissions() - resources := make([]string, 0, len(permissions)) - for res := range permissions { - resources = append(resources, res) + if len(req.GetClusterScopes()) > 0 { + role.Clusters = make(tokens.ClusterScopes) } - slices.Sort(resources) - for ix, resource := range resources { - if ix > 0 { - b.WriteString(primaryListSeparator) + for _, requestedScope := range req.GetClusterScopes() { + clusterID := requestedScope.GetClusterId() + if requestedScope.GetFullClusterAccess() { + role.Clusters[clusterID] = []string{"*"} + continue } - access := permissions[resource] - accessString := access.String() - b.WriteString(resource) - b.WriteString(keyValueSeparator) - b.WriteString(accessString) - permissionSet.ResourceToAccess[resource] = convertAccess(access) - } - permissionSetID := declarativeconfig.NewDeclarativePermissionSetUUID(b.String()).String() - permissionSet.Id = permissionSetID - permissionSet.Name = fmt.Sprintf(permissionSetNameFormat, permissionSetID) - err := rm.roleStore.UpsertPermissionSet(ctx, permissionSet) - if err != nil { - return "", errors.Wrap(err, "storing permission set") + role.Clusters[clusterID] = append(role.Clusters[clusterID], requestedScope.GetNamespaces()...) } - return permissionSet.GetId(), nil + return role, nil } -func convertAccess(in v1.Access) storage.Access { - switch in { +func convertAccess(access v1.Access) storage.Access { + switch access { case v1.Access_READ_ACCESS: return storage.Access_READ_ACCESS case v1.Access_READ_WRITE_ACCESS: @@ -89,94 +56,3 @@ func convertAccess(in v1.Access) storage.Access { return storage.Access_NO_ACCESS } } - -// createAccessScope creates a dynamic access scope, granting the requested scope. -// The returned information is the identifier of the created access scope, -// or an error if any occurred in the creation process. -func (rm *roleManager) createAccessScope( - ctx context.Context, - req *v1.GenerateTokenForPermissionsAndScopeRequest, -) (string, error) { - // TODO: Consider pruning the generated access scopes after some idle time. - accessScope := &storage.SimpleAccessScope{ - Description: accessScopeDescription, - Rules: &storage.SimpleAccessScope_Rules{}, - Traits: generatedObjectTraits.CloneVT(), - } - var b strings.Builder - fullAccessClusters := make([]string, 0) - clusterNamespaces := make([]*storage.SimpleAccessScope_Rules_Namespace, 0) - for ix, clusterScope := range req.GetClusterScopes() { - clusterID := clusterScope.GetClusterId() - clusterName, found, err := rm.clusterStore.GetClusterName(ctx, clusterID) - if err != nil { - return "", errors.Wrap(err, "retrieving cluster name") - } - if !found { - continue - } - if ix > 0 { - b.WriteString(primaryListSeparator) - } - b.WriteString(clusterName) - b.WriteString(keyValueSeparator) - if clusterScope.GetFullClusterAccess() { - fullAccessClusters = append(fullAccessClusters, clusterName) - b.WriteString(clusterWildCard) - } else { - for namespaceIndex, namespace := range clusterScope.GetNamespaces() { - clusterNamespaces = append(clusterNamespaces, &storage.SimpleAccessScope_Rules_Namespace{ - ClusterName: clusterName, - NamespaceName: namespace, - }) - if namespaceIndex > 0 { - b.WriteString(secondaryListSeparator) - } - b.WriteString(namespace) - } - } - } - accessScope.Rules.IncludedClusters = fullAccessClusters - accessScope.Rules.IncludedNamespaces = clusterNamespaces - accessScopeID := declarativeconfig.NewDeclarativeAccessScopeUUID(b.String()).String() - accessScope.Id = accessScopeID - accessScope.Name = fmt.Sprintf(accessScopeNameFormat, accessScopeID) - - err := rm.roleStore.UpsertAccessScope(ctx, accessScope) - if err != nil { - return "", errors.Wrap(err, "storing access scope") - } - - return accessScope.GetId(), nil -} - -// createRole creates a dynamic role, granting the requested permissions and scope. -// The returned information is the name of the created role, or an error if any occurred -// in the creation process. -func (rm *roleManager) createRole( - ctx context.Context, - req *v1.GenerateTokenForPermissionsAndScopeRequest, -) (string, error) { - // TODO: Consider pruning the generated roles after some idle time. - permissionSetID, err := rm.createPermissionSet(ctx, req) - if err != nil { - return "", errors.Wrap(err, "creating permission set for role") - } - accessScopeID, err := rm.createAccessScope(ctx, req) - if err != nil { - return "", errors.Wrap(err, "creating access scope for role") - } - resultRole := &storage.Role{ - Name: fmt.Sprintf(roleNameFormat, permissionSetID, accessScopeID), - Description: roleDescription, - PermissionSetId: permissionSetID, - AccessScopeId: accessScopeID, - Traits: generatedObjectTraits.CloneVT(), - } - err = rm.roleStore.UpsertRole(ctx, resultRole) - if err != nil { - return "", errors.Wrap(err, "storing role") - } - - return resultRole.GetName(), nil -} diff --git a/central/auth/internaltokens/service/role_manager_test.go b/central/auth/internaltokens/service/role_manager_test.go index 2c1bc8df5aa16..6d43c3366ca3e 100644 --- a/central/auth/internaltokens/service/role_manager_test.go +++ b/central/auth/internaltokens/service/role_manager_test.go @@ -1,712 +1 @@ package service - -import ( - "errors" - "fmt" - "slices" - "strings" - "testing" - - clusterDataStoreMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" - roleDataStoreMocks "github.com/stackrox/rox/central/role/datastore/mocks" - v1 "github.com/stackrox/rox/generated/internalapi/central/v1" - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/declarativeconfig" - "github.com/stackrox/rox/pkg/protomock" - "github.com/stackrox/rox/pkg/sac" - "github.com/stretchr/testify/assert" - "go.uber.org/mock/gomock" -) - -func TestCreatePermissionSet(t *testing.T) { - nilPermissions := (map[string]v1.Access)(nil) - requestWithoutPermissions := &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: nilPermissions, - } - noPermissions := make(map[string]v1.Access) - requestForNoPermissions := &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: noPermissions, - } - onePermission := map[string]v1.Access{ - "Deployment": v1.Access_READ_ACCESS, - } - requestForOnePermission := &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: onePermission, - } - manyPermissions := map[string]v1.Access{ - "Deployment": v1.Access_READ_ACCESS, - "Namespace": v1.Access_READ_ACCESS, - "NetworkGraph": v1.Access_READ_ACCESS, - } - requestForManyPermissions := &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: manyPermissions, - } - for name, tc := range map[string]struct { - input *v1.GenerateTokenForPermissionsAndScopeRequest - expectedPermissionSet *storage.PermissionSet - expectedStoreError error - }{ - "nil request, successful storage (no access permissions)": { - input: nil, - expectedPermissionSet: testPermissionSet(nilPermissions), - expectedStoreError: nil, - }, - "nil request, failed storage (no access permissions)": { - input: nil, - expectedPermissionSet: testPermissionSet(nilPermissions), - expectedStoreError: errDummy, - }, - "request with nil permissions, successful storage (no access permissions)": { - input: requestWithoutPermissions, - expectedPermissionSet: testPermissionSet(nilPermissions), - expectedStoreError: nil, - }, - "request with nil permissions, failed storage (no access permissions)": { - input: requestWithoutPermissions, - expectedPermissionSet: testPermissionSet(nilPermissions), - expectedStoreError: errDummy, - }, - "request for no permissions, successful storage (no access permissions)": { - input: requestForNoPermissions, - expectedPermissionSet: testPermissionSet(noPermissions), - expectedStoreError: nil, - }, - "request for no permissions, failed storage (no access permissions)": { - input: requestForNoPermissions, - expectedPermissionSet: testPermissionSet(noPermissions), - expectedStoreError: errDummy, - }, - "request for one permissions, successful storage": { - input: requestForOnePermission, - expectedPermissionSet: testPermissionSet(onePermission), - expectedStoreError: nil, - }, - "request for one permissions, failed storage": { - input: requestForOnePermission, - expectedPermissionSet: testPermissionSet(onePermission), - expectedStoreError: errDummy, - }, - "request for many permissions, successful storage": { - input: requestForManyPermissions, - expectedPermissionSet: testPermissionSet(manyPermissions), - expectedStoreError: nil, - }, - "request for many permissions, failed storage": { - input: requestForManyPermissions, - expectedPermissionSet: testPermissionSet(manyPermissions), - expectedStoreError: errDummy, - }, - } { - t.Run(name, func(it *testing.T) { - ctx := sac.WithAllAccess(it.Context()) - mockCtrl := gomock.NewController(it) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - roleMgr := &roleManager{ - roleStore: mockRoleStore, - } - mockRoleStore.EXPECT(). - UpsertPermissionSet(gomock.Any(), protomock.GoMockMatcherEqualMessage(tc.expectedPermissionSet)). - Times(1). - Return(tc.expectedStoreError) - - psID, err := roleMgr.createPermissionSet(ctx, tc.input) - - if tc.expectedStoreError != nil { - assert.Empty(it, psID) - assert.ErrorIs(it, err, tc.expectedStoreError) - } else { - assert.Equal(it, tc.expectedPermissionSet.GetId(), psID) - assert.NoError(it, err) - } - }) - } -} - -func TestCreateAccessScope(t *testing.T) { - targetCluster1 := "cluster 1" - targetCluster2 := "cluster 2" - targetCluster3 := "cluster 3" - targetNamespaceA := "namespace A" - targetNamespaceB := "namespace B" - targetNamespaceC := "namespace C" - requestFullCluster := &v1.ClusterScope{ - ClusterId: targetCluster1, - FullClusterAccess: true, - } - requestSingleNamespace := &v1.ClusterScope{ - ClusterId: targetCluster2, - FullClusterAccess: false, - Namespaces: []string{targetNamespaceA}, - } - requestMultipleNamespaces := &v1.ClusterScope{ - ClusterId: targetCluster3, - FullClusterAccess: false, - Namespaces: []string{targetNamespaceB, targetNamespaceC}, - } - for name, tc := range map[string]struct { - input *v1.GenerateTokenForPermissionsAndScopeRequest - expectedAccessScope *storage.SimpleAccessScope - expectedStoreError error - }{ - "nil input, successful storage (empty scope)": { - input: nil, - expectedAccessScope: testAccessScope(nil), - expectedStoreError: nil, - }, - "nil input, failed storage (empty scope)": { - input: nil, - expectedAccessScope: testAccessScope(nil), - expectedStoreError: errDummy, - }, - "input with nil scope, successful storage (empty scope)": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{}, - expectedAccessScope: testAccessScope(nil), - expectedStoreError: nil, - }, - "input with nil scope, failed storage (empty scope)": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{}, - expectedAccessScope: testAccessScope(nil), - expectedStoreError: errDummy, - }, - "input with empty scope, successful storage (empty scope)": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: make([]*v1.ClusterScope, 0), - }, - expectedAccessScope: testAccessScope(make([]*v1.ClusterScope, 0)), - expectedStoreError: nil, - }, - "input with empty scope, failed storage (empty scope)": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: make([]*v1.ClusterScope, 0), - }, - expectedAccessScope: testAccessScope(make([]*v1.ClusterScope, 0)), - expectedStoreError: errDummy, - }, - "input with single full cluster scope, successful storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{requestFullCluster}, - }, - expectedAccessScope: testAccessScope([]*v1.ClusterScope{requestFullCluster}), - expectedStoreError: nil, - }, - "input with single full cluster scope, failed storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{requestFullCluster}, - }, - expectedAccessScope: testAccessScope([]*v1.ClusterScope{requestFullCluster}), - expectedStoreError: errDummy, - }, - "input with single namespace scope, successful storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, - }, - expectedAccessScope: testAccessScope([]*v1.ClusterScope{requestSingleNamespace}), - expectedStoreError: nil, - }, - "input with single namespace scope, failed storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, - }, - expectedAccessScope: testAccessScope([]*v1.ClusterScope{requestSingleNamespace}), - expectedStoreError: errDummy, - }, - "input with multi namespace scope, successful storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{requestMultipleNamespaces}, - }, - expectedAccessScope: testAccessScope([]*v1.ClusterScope{requestMultipleNamespaces}), - expectedStoreError: nil, - }, - "input with multi namespace scope, failed storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{requestMultipleNamespaces}, - }, - expectedAccessScope: testAccessScope([]*v1.ClusterScope{requestMultipleNamespaces}), - expectedStoreError: errDummy, - }, - "input with multi cluster-namespace scope, successful storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{ - requestSingleNamespace, - requestMultipleNamespaces, - }, - }, - expectedAccessScope: testAccessScope([]*v1.ClusterScope{ - requestSingleNamespace, - requestMultipleNamespaces, - }), - expectedStoreError: nil, - }, - "input with multi cluster-namespace scope, failed storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{ - requestSingleNamespace, - requestMultipleNamespaces, - }, - }, - expectedAccessScope: testAccessScope([]*v1.ClusterScope{ - requestSingleNamespace, - requestMultipleNamespaces, - }), - expectedStoreError: errDummy, - }, - "input with complex scope mix, successful storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{ - requestFullCluster, - requestSingleNamespace, - requestMultipleNamespaces, - }, - }, - expectedAccessScope: testAccessScope([]*v1.ClusterScope{ - requestFullCluster, - requestSingleNamespace, - requestMultipleNamespaces, - }), - expectedStoreError: nil, - }, - "input with complex scope mix, failed storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{ - requestFullCluster, - requestSingleNamespace, - requestMultipleNamespaces, - }, - }, - expectedAccessScope: testAccessScope([]*v1.ClusterScope{ - requestFullCluster, - requestSingleNamespace, - requestMultipleNamespaces, - }), - expectedStoreError: errDummy, - }, - } { - t.Run(name, func(it *testing.T) { - ctx := sac.WithAllAccess(it.Context()) - mockCtrl := gomock.NewController(it) - mockClusterStore := clusterDataStoreMocks.NewMockDataStore(mockCtrl) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - roleMgr := &roleManager{ - clusterStore: mockClusterStore, - roleStore: mockRoleStore, - } - setClusterStoreExpectations(tc.input, mockClusterStore) - mockRoleStore.EXPECT(). - UpsertAccessScope(gomock.Any(), protomock.GoMockMatcherEqualMessage(tc.expectedAccessScope)). - Times(1). - Return(tc.expectedStoreError) - - asID, err := roleMgr.createAccessScope(ctx, tc.input) - - if tc.expectedStoreError != nil { - assert.Empty(it, asID) - assert.ErrorIs(it, err, tc.expectedStoreError) - } else { - assert.Equal(it, tc.expectedAccessScope.GetId(), asID) - assert.NoError(it, err) - } - }) - } - t.Run("Cluster name lookup errors are propagated", func(it *testing.T) { - ctx := sac.WithAllAccess(it.Context()) - mockCtrl := gomock.NewController(it) - mockClusterStore := clusterDataStoreMocks.NewMockDataStore(mockCtrl) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - roleMgr := &roleManager{ - clusterStore: mockClusterStore, - roleStore: mockRoleStore, - } - input := &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{requestFullCluster}, - } - mockClusterStore.EXPECT(). - GetClusterName(gomock.Any(), requestFullCluster.GetClusterId()). - Times(1). - Return("", false, errDummy) - - accessScopeId, err := roleMgr.createAccessScope(ctx, input) - assert.Empty(it, accessScopeId) - assert.ErrorIs(it, err, errDummy) - }) - t.Run("Cluster name lookup misses are excluded from resulting scope", func(it *testing.T) { - ctx := sac.WithAllAccess(it.Context()) - mockCtrl := gomock.NewController(it) - mockClusterStore := clusterDataStoreMocks.NewMockDataStore(mockCtrl) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - roleMgr := &roleManager{ - clusterStore: mockClusterStore, - roleStore: mockRoleStore, - } - input := &v1.GenerateTokenForPermissionsAndScopeRequest{ - ClusterScopes: []*v1.ClusterScope{requestFullCluster, requestSingleNamespace}, - } - - expectedAccessScope := testAccessScope([]*v1.ClusterScope{nil, requestSingleNamespace}) - mockClusterStore.EXPECT(). - GetClusterName(gomock.Any(), requestFullCluster.GetClusterId()). - Times(1). - Return("", false, nil) - mockClusterStore.EXPECT(). - GetClusterName(gomock.Any(), requestSingleNamespace.GetClusterId()). - Times(1). - Return(requestSingleNamespace.GetClusterId(), true, nil) - mockRoleStore.EXPECT(). - UpsertAccessScope(gomock.Any(), protomock.GoMockMatcherEqualMessage(expectedAccessScope)). - Times(1). - Return(nil) - - accessScopeId, err := roleMgr.createAccessScope(ctx, input) - assert.Equal(it, expectedAccessScope.GetId(), accessScopeId) - assert.NoError(it, err) - }) -} - -func TestCreateRole(t *testing.T) { - noPermission := make(map[string]v1.Access) - deploymentPermission := map[string]v1.Access{ - "Deployment": v1.Access_READ_ACCESS, - } - targetCluster1 := "cluster 1" - targetCluster2 := "cluster 2" - targetNamespaceA := "namespace A" - requestFullCluster := &v1.ClusterScope{ - ClusterId: targetCluster1, - FullClusterAccess: true, - } - requestSingleNamespace := &v1.ClusterScope{ - ClusterId: targetCluster2, - FullClusterAccess: false, - Namespaces: []string{targetNamespaceA}, - } - for name, tc := range map[string]struct { - input *v1.GenerateTokenForPermissionsAndScopeRequest - expectedPermissionSet *storage.PermissionSet - expectedAccessScope *storage.SimpleAccessScope - expectedRole *storage.Role - expectedRoleStoreError error - }{ - "nil input, successful storage (role with no permission and empty scope)": { - input: nil, - expectedPermissionSet: testPermissionSet(noPermission), - expectedAccessScope: testAccessScope(make([]*v1.ClusterScope, 0)), - expectedRole: testRole(noPermission, make([]*v1.ClusterScope, 0)), - expectedRoleStoreError: nil, - }, - "nil input, failed storage (role with no permission and empty scope)": { - input: nil, - expectedPermissionSet: testPermissionSet(noPermission), - expectedAccessScope: testAccessScope(make([]*v1.ClusterScope, 0)), - expectedRole: testRole(noPermission, make([]*v1.ClusterScope, 0)), - expectedRoleStoreError: errDummy, - }, - "request for single full cluster access to deployments, successful storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: deploymentPermission, - ClusterScopes: []*v1.ClusterScope{requestFullCluster}, - }, - expectedPermissionSet: testPermissionSet(deploymentPermission), - expectedAccessScope: testAccessScope([]*v1.ClusterScope{requestFullCluster}), - expectedRole: testRole(deploymentPermission, []*v1.ClusterScope{requestFullCluster}), - expectedRoleStoreError: nil, - }, - "request for single full cluster access to deployments, failed storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: deploymentPermission, - ClusterScopes: []*v1.ClusterScope{requestFullCluster}, - }, - expectedPermissionSet: testPermissionSet(deploymentPermission), - expectedAccessScope: testAccessScope([]*v1.ClusterScope{requestFullCluster}), - expectedRole: testRole(deploymentPermission, []*v1.ClusterScope{requestFullCluster}), - expectedRoleStoreError: errDummy, - }, - "request for single namespace access to deployments, successful storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: deploymentPermission, - ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, - }, - expectedPermissionSet: testPermissionSet(deploymentPermission), - expectedAccessScope: testAccessScope([]*v1.ClusterScope{requestSingleNamespace}), - expectedRole: testRole(deploymentPermission, []*v1.ClusterScope{requestSingleNamespace}), - expectedRoleStoreError: nil, - }, - "request for single namespace access to deployments, failed storage": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: deploymentPermission, - ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, - }, - expectedPermissionSet: testPermissionSet(deploymentPermission), - expectedAccessScope: testAccessScope([]*v1.ClusterScope{requestSingleNamespace}), - expectedRole: testRole(deploymentPermission, []*v1.ClusterScope{requestSingleNamespace}), - expectedRoleStoreError: errDummy, - }, - } { - t.Run(name, func(it *testing.T) { - ctx := sac.WithAllAccess(it.Context()) - mockCtrl := gomock.NewController(it) - mockClusterStore := clusterDataStoreMocks.NewMockDataStore(mockCtrl) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - roleMgr := &roleManager{ - clusterStore: mockClusterStore, - roleStore: mockRoleStore, - } - setClusterStoreExpectations(tc.input, mockClusterStore) - setNormalRoleStoreExpectations( - tc.expectedPermissionSet, - tc.expectedAccessScope, - tc.expectedRole, - tc.expectedRoleStoreError, - mockRoleStore, - ) - - roleName, err := roleMgr.createRole(ctx, tc.input) - - if tc.expectedRoleStoreError != nil { - assert.Empty(it, roleName) - assert.ErrorIs(it, err, tc.expectedRoleStoreError) - } else { - assert.Equal(it, tc.expectedRole.GetName(), roleName) - assert.NoError(it, err) - } - }) - } - - t.Run("access scope creation failure", func(it *testing.T) { - ctx := sac.WithAllAccess(it.Context()) - mockCtrl := gomock.NewController(it) - mockClusterStore := clusterDataStoreMocks.NewMockDataStore(mockCtrl) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - roleMgr := &roleManager{ - clusterStore: mockClusterStore, - roleStore: mockRoleStore, - } - expectedPermissionSet := testPermissionSet(deploymentPermission) - expectedAccessScope := testAccessScope([]*v1.ClusterScope{requestSingleNamespace}) - accessScopeCreationErr := errors.New("access scope creation error") - mockRoleStore.EXPECT(). - UpsertPermissionSet(gomock.Any(), protomock.GoMockMatcherEqualMessage(expectedPermissionSet)). - Times(1). - Return(nil) - mockRoleStore.EXPECT(). - UpsertAccessScope(gomock.Any(), protomock.GoMockMatcherEqualMessage(expectedAccessScope)). - Times(1). - Return(accessScopeCreationErr) - - input := &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: deploymentPermission, - ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, - } - - setClusterStoreExpectations(input, mockClusterStore) - - roleName, err := roleMgr.createRole(ctx, input) - - assert.Empty(it, roleName) - assert.ErrorIs(it, err, accessScopeCreationErr) - }) - - t.Run("permission set creation failure", func(it *testing.T) { - ctx := sac.WithAllAccess(it.Context()) - mockCtrl := gomock.NewController(it) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - roleMgr := &roleManager{ - roleStore: mockRoleStore, - } - expectedPermissionSet := testPermissionSet(deploymentPermission) - permissionSetCreationErr := errors.New("permission set creation error") - mockRoleStore.EXPECT(). - UpsertPermissionSet(gomock.Any(), protomock.GoMockMatcherEqualMessage(expectedPermissionSet)). - Times(1). - Return(permissionSetCreationErr) - - input := &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: deploymentPermission, - ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, - } - - roleName, err := roleMgr.createRole(ctx, input) - - assert.Empty(it, roleName) - assert.ErrorIs(it, err, permissionSetCreationErr) - }) -} - -func TestConvertAccess(t *testing.T) { - for name, tc := range map[string]struct { - in v1.Access - out storage.Access - }{ - v1.Access_NO_ACCESS.String(): { - in: v1.Access_NO_ACCESS, - out: storage.Access_NO_ACCESS, - }, - v1.Access_READ_ACCESS.String(): { - in: v1.Access_READ_ACCESS, - out: storage.Access_READ_ACCESS, - }, - v1.Access_READ_WRITE_ACCESS.String(): { - in: v1.Access_READ_WRITE_ACCESS, - out: storage.Access_READ_WRITE_ACCESS, - }, - "Out of range -> default no access": { - in: v1.Access(-1), - out: storage.Access_NO_ACCESS, - }, - } { - t.Run(name, func(it *testing.T) { - converted := convertAccess(tc.in) - assert.Equal(it, tc.out, converted) - }) - } -} - -func computePermissionSetID(permissions map[string]v1.Access) string { - resources := make([]string, 0, len(permissions)) - for res := range permissions { - resources = append(resources, res) - } - slices.Sort(resources) - individualPermissions := make([]string, 0, len(resources)) - for _, resource := range resources { - access := permissions[resource] - individualPermissions = append( - individualPermissions, - fmt.Sprintf("%s%s%s", resource, keyValueSeparator, access.String()), - ) - } - permissionString := strings.Join(individualPermissions, primaryListSeparator) - return declarativeconfig.NewDeclarativePermissionSetUUID(permissionString).String() -} - -func computeAccessScopeID(targetScopes []*v1.ClusterScope) string { - clusterScopes := make([]string, 0, len(targetScopes)) - for _, targetScope := range targetScopes { - if targetScope == nil { - clusterScopes = append(clusterScopes, "") - continue - } - var namespaceScope string - if targetScope.GetFullClusterAccess() { - namespaceScope = clusterWildCard - } else { - namespaceScope = strings.Join(targetScope.GetNamespaces(), secondaryListSeparator) - } - clusterScopes = append( - clusterScopes, - fmt.Sprintf("%s%s%s", targetScope.GetClusterId(), keyValueSeparator, namespaceScope), - ) - } - accessScopeString := strings.Join(clusterScopes, primaryListSeparator) - return declarativeconfig.NewDeclarativeAccessScopeUUID(accessScopeString).String() -} - -func computeRoleName(permissions map[string]v1.Access, targetScopes []*v1.ClusterScope) string { - permissionSetID := computePermissionSetID(permissions) - accessScopeID := computeAccessScopeID(targetScopes) - return fmt.Sprintf(roleNameFormat, permissionSetID, accessScopeID) -} - -func testPermissionSet(permissions map[string]v1.Access) *storage.PermissionSet { - resources := make([]string, 0, len(permissions)) - for res := range permissions { - resources = append(resources, res) - } - permissionSetID := computePermissionSetID(permissions) - permissionSet := &storage.PermissionSet{ - Id: permissionSetID, - Name: fmt.Sprintf(permissionSetNameFormat, permissionSetID), - Description: permissionSetDescription, - ResourceToAccess: make(map[string]storage.Access), - Traits: generatedObjectTraits.CloneVT(), - } - for _, resource := range resources { - permissionSet.ResourceToAccess[resource] = convertAccess(permissions[resource]) - } - return permissionSet -} - -func testAccessScope(targetScopes []*v1.ClusterScope) *storage.SimpleAccessScope { - accessScopeID := computeAccessScopeID(targetScopes) - accessScope := &storage.SimpleAccessScope{ - Id: accessScopeID, - Name: fmt.Sprintf(accessScopeNameFormat, accessScopeID), - Description: accessScopeDescription, - Rules: &storage.SimpleAccessScope_Rules{ - IncludedClusters: make([]string, 0), - IncludedNamespaces: make([]*storage.SimpleAccessScope_Rules_Namespace, 0), - }, - Traits: generatedObjectTraits.CloneVT(), - } - for _, targetScope := range targetScopes { - if targetScope == nil { - continue - } - if targetScope.GetFullClusterAccess() { - accessScope.Rules.IncludedClusters = append( - accessScope.Rules.IncludedClusters, - targetScope.GetClusterId(), - ) - } else { - for _, namespace := range targetScope.GetNamespaces() { - accessScope.Rules.IncludedNamespaces = append( - accessScope.Rules.IncludedNamespaces, - &storage.SimpleAccessScope_Rules_Namespace{ - ClusterName: targetScope.GetClusterId(), - NamespaceName: namespace, - }, - ) - } - } - } - return accessScope -} - -func testRole(permissions map[string]v1.Access, targetScopes []*v1.ClusterScope) *storage.Role { - permissionSetID := computePermissionSetID(permissions) - accessScopeID := computeAccessScopeID(targetScopes) - role := &storage.Role{ - Name: computeRoleName(permissions, targetScopes), - Description: roleDescription, - PermissionSetId: permissionSetID, - AccessScopeId: accessScopeID, - Traits: generatedObjectTraits.CloneVT(), - } - return role -} - -func setClusterStoreExpectations( - input *v1.GenerateTokenForPermissionsAndScopeRequest, - mockClusterStore *clusterDataStoreMocks.MockDataStore, -) { - for _, clusterScope := range input.GetClusterScopes() { - clusterIdName := clusterScope.GetClusterId() - mockClusterStore.EXPECT(). - GetClusterName(gomock.Any(), clusterIdName). - Times(1). - Return(clusterIdName, true, nil) - } -} - -func setNormalRoleStoreExpectations( - permissionSet *storage.PermissionSet, - accessScope *storage.SimpleAccessScope, - role *storage.Role, - roleStoreError error, - mockRoleStore *roleDataStoreMocks.MockDataStore, -) { - mockRoleStore.EXPECT(). - UpsertPermissionSet( - gomock.Any(), - protomock.GoMockMatcherEqualMessage(permissionSet), - ).Times(1).Return(nil) - mockRoleStore.EXPECT(). - UpsertAccessScope( - gomock.Any(), - protomock.GoMockMatcherEqualMessage(accessScope), - ).Times(1).Return(nil) - mockRoleStore.EXPECT(). - UpsertRole( - gomock.Any(), - protomock.GoMockMatcherEqualMessage(role), - ). - Times(1).Return(roleStoreError) -} diff --git a/central/auth/internaltokens/service/service_impl.go b/central/auth/internaltokens/service/service_impl.go index 4d8da5c693755..7dc08030ab131 100644 --- a/central/auth/internaltokens/service/service_impl.go +++ b/central/auth/internaltokens/service/service_impl.go @@ -7,6 +7,7 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/pkg/errors" + "github.com/stackrox/rox/central/pruning" v1 "github.com/stackrox/rox/generated/internalapi/central/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/auth/tokens" @@ -49,6 +50,8 @@ type serviceImpl struct { now func() time.Time + policy *tokenPolicy + v1.UnimplementedTokenServiceServer } @@ -71,23 +74,41 @@ func (s *serviceImpl) GenerateTokenForPermissionsAndScope( ctx context.Context, req *v1.GenerateTokenForPermissionsAndScopeRequest, ) (*v1.GenerateTokenForPermissionsAndScopeResponse, error) { - roleName, err := s.roleManager.createRole(ctx, req) + start := time.Now() + + // Enable dynamic RBAC pruning. + pruning.EnableDynamicRBACPruning() + + // Validate request against policy and modify request if necessary. + req, err := s.policy.enforce(ctx, req) if err != nil { - return nil, errors.Wrap(err, "creating and storing target role") + return nil, err } + + // Calculate expiry first so we can set it on the RBAC objects. expiresAt, err := s.getExpiresAt(ctx, req) if err != nil { + observeTokenGeneration(tokenGenResultInvalidArgs, time.Since(start)) return nil, errors.Wrap(err, "getting expiration time") } + tokenRole, err := s.roleManager.createRoleForRoxClaims(ctx, req) + if err != nil { + observeTokenGeneration(tokenGenResultRoleCreationError, time.Since(start)) + return nil, errors.Wrap(err, "building ephemeral role") + } + roleName := tokenRole.GetRoleName() claimName := fmt.Sprintf(claimNameFormat, roleName, expiresAt.Format(time.RFC3339Nano)) roxClaims := tokens.RoxClaims{ - RoleNames: []string{roleName}, - Name: claimName, + InternalRoles: []*tokens.InternalRole{tokenRole}, + Name: claimName, + Requester: req.GetRequester(), } tokenInfo, err := s.issuer.Issue(ctx, roxClaims, tokens.WithExpiry(expiresAt)) if err != nil { + observeTokenGeneration(tokenGenResultIssuanceError, time.Since(start)) return nil, err } + observeTokenGeneration(tokenGenResultSuccess, time.Since(start)) response := &v1.GenerateTokenForPermissionsAndScopeResponse{ Token: tokenInfo.Token, } @@ -101,10 +122,7 @@ func (s *serviceImpl) getExpiresAt( ) (time.Time, error) { duration, err := protocompat.DurationFromProto(req.GetLifetime()) if err != nil { - return time.Time{}, errox.InvalidArgs.CausedByf("converting requested token validity duration: %v", err) - } - if duration <= 0 { - return time.Time{}, errox.InvalidArgs.CausedBy("token validity duration should be positive") + return time.Time{}, errox.InvalidArgs.CausedByf("converting requested token lifetime: %v", err) } return s.now().Add(duration), nil } diff --git a/central/auth/internaltokens/service/service_impl_test.go b/central/auth/internaltokens/service/service_impl_test.go index 882060218b0ec..32008b67406db 100644 --- a/central/auth/internaltokens/service/service_impl_test.go +++ b/central/auth/internaltokens/service/service_impl_test.go @@ -1,33 +1,52 @@ package service import ( + "context" "errors" "fmt" "testing" "time" - clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" clusterDataStoreMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" - roleDataStore "github.com/stackrox/rox/central/role/datastore" roleDataStoreMocks "github.com/stackrox/rox/central/role/datastore/mocks" v1 "github.com/stackrox/rox/generated/internalapi/central/v1" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/auth/tokens" tokensMocks "github.com/stackrox/rox/pkg/auth/tokens/mocks" + "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" + "github.com/stackrox/rox/pkg/grpc/authn" + authnMocks "github.com/stackrox/rox/pkg/grpc/authn/mocks" "github.com/stackrox/rox/pkg/protoassert" - "github.com/stackrox/rox/pkg/protomock" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" "google.golang.org/protobuf/types/known/durationpb" ) +const ( + deploymentResource = "Deployment" + imageResource = "Image" + networkGraphResource = "NetworkGraph" +) + var ( errDummy = errors.New("test error") - expectedExpiration = time.Date(1989, time.November, 9, 18, 10, 35, 987654321, time.UTC) + // permissivePolicy allows the permissions used in existing tests. + permissivePolicy = newTokenPolicy(1*time.Hour, map[string]v1.Access{ + "Deployment": v1.Access_READ_ACCESS, + "Image": v1.Access_READ_ACCESS, + }) ) -func testClock() time.Time { - return time.Date(1989, time.November, 9, 18, 05, 35, 987654321, time.UTC) +// sensorContext returns a context with a mock sensor identity injected. +func sensorContext(t testing.TB, ctrl *gomock.Controller, clusterID string) context.Context { + mockIdentity := authnMocks.NewMockIdentity(ctrl) + mockIdentity.EXPECT().Service().Return(&storage.ServiceIdentity{ + Id: clusterID, + Type: storage.ServiceType_SENSOR_SERVICE, + }).AnyTimes() + return authn.ContextWithIdentity(t.Context(), mockIdentity, t) } func TestGetExpiresAt(t *testing.T) { @@ -55,15 +74,6 @@ func TestGetExpiresAt(t *testing.T) { }, expectsErr: true, }, - "input with negative requested validity": { - input: &v1.GenerateTokenForPermissionsAndScopeRequest{ - Lifetime: &durationpb.Duration{ - Seconds: -60, - Nanos: -654321987, - }, - }, - expectsErr: true, - }, "valid input": { input: &v1.GenerateTokenForPermissionsAndScopeRequest{ Lifetime: &durationpb.Duration{ @@ -71,11 +81,11 @@ func TestGetExpiresAt(t *testing.T) { }, }, expectsErr: false, - expectedExpiration: expectedExpiration, + expectedExpiration: testTokenExpiry, }, } { t.Run(name, func(it *testing.T) { - svc := &serviceImpl{now: testClock} + svc := &serviceImpl{now: testClock, policy: permissivePolicy} expiresAt, err := svc.getExpiresAt(it.Context(), tc.input) if tc.expectsErr { fmt.Println(err.Error()) @@ -89,143 +99,401 @@ func TestGetExpiresAt(t *testing.T) { } } +type mockContainer struct { + clusterStore *clusterDataStoreMocks.MockDataStore + roleStore *roleDataStoreMocks.MockDataStore + issuer *tokensMocks.MockIssuer +} + func TestGenerateTokenForPermissionsAndScope(t *testing.T) { deploymentPermission := map[string]v1.Access{ - "Deployment": v1.Access_READ_ACCESS, + deploymentResource: v1.Access_READ_ACCESS, } requestSingleNamespace := &v1.ClusterScope{ - ClusterId: "cluster 1", + ClusterId: fixtureconsts.Cluster1, FullClusterAccess: false, Namespaces: []string{"namespace A"}, } - deploymentPS := testPermissionSet(deploymentPermission) - singleNSScope := testAccessScope( - []*v1.ClusterScope{requestSingleNamespace}, - ) - expectedRole := testRole( - deploymentPermission, - []*v1.ClusterScope{requestSingleNamespace}, - ) + requestFullCluster := &v1.ClusterScope{ + ClusterId: fixtureconsts.Cluster1, + FullClusterAccess: true, + } createService := func( + t testing.TB, issuer tokens.Issuer, - clusterStore clusterDataStore.DataStore, - roleStore roleDataStore.DataStore, + policy *tokenPolicy, ) *serviceImpl { + t.Helper() return &serviceImpl{ - issuer: issuer, - roleManager: &roleManager{ - clusterStore: clusterStore, - roleStore: roleStore, - }, - now: testClock, + issuer: issuer, + roleManager: &roleManager{}, + now: testClock, + policy: policy, } } - t.Run("no requested validity", func(it *testing.T) { - input := &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: deploymentPermission, - ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, - Lifetime: nil, - } - - mockCtrl := gomock.NewController(it) - mockClusterStore := clusterDataStoreMocks.NewMockDataStore(mockCtrl) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - svc := createService(nil, mockClusterStore, mockRoleStore) - setClusterStoreExpectations(input, mockClusterStore) - setNormalRoleStoreExpectations(deploymentPS, singleNSScope, expectedRole, nil, mockRoleStore) - - rsp, err := svc.GenerateTokenForPermissionsAndScope(t.Context(), input) - assert.Nil(it, rsp) - assert.Error(it, err) - }) - t.Run("failed role creation", func(it *testing.T) { - input := &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: deploymentPermission, - ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, - Lifetime: nil, - } - - mockCtrl := gomock.NewController(it) - mockClusterStore := clusterDataStoreMocks.NewMockDataStore(mockCtrl) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - svc := createService(nil, mockClusterStore, mockRoleStore) - mockRoleStore.EXPECT(). - UpsertPermissionSet( - gomock.Any(), - protomock.GoMockMatcherEqualMessage(deploymentPS), - ).Times(1).Return(errDummy) - - rsp, err := svc.GenerateTokenForPermissionsAndScope(t.Context(), input) - assert.Nil(it, rsp) - assert.Error(it, err) - }) - t.Run("token issuer failure", func(it *testing.T) { - input := &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: deploymentPermission, - ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, - Lifetime: &durationpb.Duration{Seconds: 300}, - } - - mockCtrl := gomock.NewController(it) - mockClusterStore := clusterDataStoreMocks.NewMockDataStore(mockCtrl) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - mockIssuer := tokensMocks.NewMockIssuer(mockCtrl) - svc := createService(mockIssuer, mockClusterStore, mockRoleStore) - setClusterStoreExpectations(input, mockClusterStore) - setNormalRoleStoreExpectations(deploymentPS, singleNSScope, expectedRole, nil, mockRoleStore) - expectedClaims := tokens.RoxClaims{ - RoleNames: []string{expectedRole.GetName()}, - Name: fmt.Sprintf( - claimNameFormat, - expectedRole.GetName(), - expectedExpiration.Format(time.RFC3339Nano), - ), - } - mockIssuer.EXPECT(). - Issue(gomock.Any(), expectedClaims, gomock.Any()). - Times(1).Return(nil, errDummy) - - rsp, err := svc.GenerateTokenForPermissionsAndScope(t.Context(), input) - assert.Nil(it, rsp) - assert.Error(it, err) - }) - t.Run("success", func(it *testing.T) { - input := &v1.GenerateTokenForPermissionsAndScopeRequest{ - Permissions: deploymentPermission, - ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, - Lifetime: &durationpb.Duration{Seconds: 300}, - } + // error cases + for name, tc := range map[string]struct { + input *v1.GenerateTokenForPermissionsAndScopeRequest + setup func(*mockContainer) + expectedErr error + }{ + "nil request": { + // The request processing fails at the step where + // the token expiration is computed (missing input). + input: nil, + expectedErr: errox.InvalidArgs, + }, + "no requested validity": { + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: deploymentPermission, + ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, + // The missing lifetime field causes computation of the + // token expiration to fail. + Lifetime: nil, + }, + expectedErr: errox.InvalidArgs, + }, + "token issuer failure": { + // The input is valid. The mock setup lets the flow succeed + // up to the point where the (mocked) token issuer is called. + // The expected claims are the outcome of the process so far, + // the issuer error is propagated. + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: deploymentPermission, + ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, + Lifetime: testExpirationDuration, + }, + setup: func(mocks *mockContainer) { + expectedClaims := tokens.RoxClaims{ + InternalRoles: []*tokens.InternalRole{ + { + RoleName: internalRoleName, + Permissions: map[storage.Access][]string{ + storage.Access_READ_ACCESS: {deploymentResource}, + }, + Clusters: tokens.ClusterScopes{ + fixtureconsts.Cluster1: {"namespace A"}, + }, + }, + }, + Name: fmt.Sprintf( + claimNameFormat, + internalRoleName, + testTokenExpiry.Format(time.RFC3339Nano), + ), + } + mocks.issuer.EXPECT(). + Issue(gomock.Any(), expectedClaims, gomock.Any()). + Times(1).Return(nil, errDummy) + }, + expectedErr: errDummy, + }, + "permission not in allowlist rejects request": { + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{ + // The tokenPolicy used for these error tests (permissivePolicy) + // does not allow actions on the NetworkGraph resource. + networkGraphResource: v1.Access_READ_ACCESS, + }, + ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, + Lifetime: testExpirationDuration, + }, + expectedErr: errox.InvalidArgs, + }, + "access level exceeds allowlist rejects request": { + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{ + // The tokenPolicy used for these error tests (permissivePolicy) + // only allows read actions on the Deployment resource. + deploymentResource: v1.Access_READ_WRITE_ACCESS, + }, + ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, + Lifetime: testExpirationDuration, + }, + expectedErr: errox.InvalidArgs, + }, + "cluster scope mismatch rejects request": { + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: deploymentPermission, + ClusterScopes: []*v1.ClusterScope{ + // The context in use for these error tests identifies + // the source cluster as "cluster 1" (testSensorClusterID). + // The requested scope for "other-cluster" does not match. + {ClusterId: "other-cluster", Namespaces: []string{"ns"}}, + }, + Lifetime: testExpirationDuration, + }, + expectedErr: errox.InvalidArgs, + }, + } { + t.Run(name, func(it *testing.T) { + mockCtrl := gomock.NewController(it) + mocks := &mockContainer{ + clusterStore: clusterDataStoreMocks.NewMockDataStore(mockCtrl), + roleStore: roleDataStoreMocks.NewMockDataStore(mockCtrl), + issuer: tokensMocks.NewMockIssuer(mockCtrl), + } + if tc.setup != nil { + tc.setup(mocks) + } + svc := createService(it, mocks.issuer, permissivePolicy) + ctx := sensorContext(it, mockCtrl, fixtureconsts.Cluster1) - mockCtrl := gomock.NewController(it) - mockClusterStore := clusterDataStoreMocks.NewMockDataStore(mockCtrl) - mockRoleStore := roleDataStoreMocks.NewMockDataStore(mockCtrl) - mockIssuer := tokensMocks.NewMockIssuer(mockCtrl) - svc := createService(mockIssuer, mockClusterStore, mockRoleStore) - setClusterStoreExpectations(input, mockClusterStore) - setNormalRoleStoreExpectations(deploymentPS, singleNSScope, expectedRole, nil, mockRoleStore) - expectedClaims := tokens.RoxClaims{ - RoleNames: []string{expectedRole.GetName()}, - Name: fmt.Sprintf( - "Generated claims for role %s expiring at %s", - expectedRole.GetName(), - expectedExpiration.Format(time.RFC3339Nano), - ), - } - mockIssuer.EXPECT(). - Issue(gomock.Any(), expectedClaims, gomock.Any()). - Times(1).Return(&tokens.TokenInfo{Token: "the quick brown fox jumps over the lazy dog"}, nil) + rsp, err := svc.GenerateTokenForPermissionsAndScope(ctx, tc.input) + assert.Nil(it, rsp) + assert.ErrorIs(it, err, tc.expectedErr) + }) + } - rsp, err := svc.GenerateTokenForPermissionsAndScope(t.Context(), input) - assert.NotNil(it, rsp) - protoassert.Equal( - it, - &v1.GenerateTokenForPermissionsAndScopeResponse{ + // success cases + for name, tc := range map[string]struct { + input *v1.GenerateTokenForPermissionsAndScopeRequest + policy *tokenPolicy + setup func(*testing.T, *mockContainer) + expectedRsp *v1.GenerateTokenForPermissionsAndScopeResponse + }{ + "success - standard request": { + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: deploymentPermission, + ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, + Lifetime: testExpirationDuration, + }, + policy: permissivePolicy, + setup: func(_ *testing.T, mocks *mockContainer) { + expectedClaims := tokens.RoxClaims{ + InternalRoles: []*tokens.InternalRole{ + { + RoleName: internalRoleName, + Permissions: map[storage.Access][]string{ + storage.Access_READ_ACCESS: {deploymentResource}, + }, + Clusters: tokens.ClusterScopes{ + fixtureconsts.Cluster1: {"namespace A"}, + }, + }, + }, + Name: fmt.Sprintf( + "Generated claims for role %s expiring at %s", + internalRoleName, + testTokenExpiry.Format(time.RFC3339Nano), + ), + } + mocks.issuer.EXPECT(). + Issue(gomock.Any(), expectedClaims, gomock.Any()). + Times(1).Return(&tokens.TokenInfo{Token: "the quick brown fox jumps over the lazy dog"}, nil) + }, + expectedRsp: &v1.GenerateTokenForPermissionsAndScopeResponse{ Token: "the quick brown fox jumps over the lazy dog", }, - rsp, - ) - assert.NoError(it, err) - }) + }, + "lifetime capping": { + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: deploymentPermission, + ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, + // 300 seconds requested, but policy caps at 10 seconds. + Lifetime: testExpirationDuration, + }, + policy: newTokenPolicy(10*time.Second, map[string]v1.Access{ + deploymentResource: v1.Access_READ_ACCESS, + }), + setup: func(st *testing.T, mocks *mockContainer) { + cappedExpiry := testClock().Add(10 * time.Second) + expectedClaims := tokens.RoxClaims{ + InternalRoles: []*tokens.InternalRole{ + { + RoleName: internalRoleName, + Permissions: map[storage.Access][]string{ + storage.Access_READ_ACCESS: {deploymentResource}, + }, + Clusters: tokens.ClusterScopes{ + fixtureconsts.Cluster1: {"namespace A"}, + }, + }, + }, + Name: fmt.Sprintf( + claimNameFormat, + internalRoleName, + // This part of the user claims passed to the issuer validate + // that the token lifetime was reduced to match the policy. + cappedExpiry.Format(time.RFC3339Nano), + ), + } + mocks.issuer.EXPECT(). + Issue(gomock.Any(), expectedClaims, gomock.Any()). + Times(1).Return(&tokens.TokenInfo{Token: "capped-token"}, nil) + }, + expectedRsp: &v1.GenerateTokenForPermissionsAndScopeResponse{ + Token: "capped-token", + }, + }, + "success - with custom requester": { + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: deploymentPermission, + ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, + Lifetime: testExpirationDuration, + // The requester information should be propagated to the claims + // passed to the issuers. + Requester: "custom requester", + }, + policy: permissivePolicy, + setup: func(t *testing.T, mocks *mockContainer) { + expectedClaims := tokens.RoxClaims{ + InternalRoles: []*tokens.InternalRole{ + { + RoleName: internalRoleName, + Permissions: map[storage.Access][]string{ + storage.Access_READ_ACCESS: {deploymentResource}, + }, + Clusters: tokens.ClusterScopes{ + fixtureconsts.Cluster1: {"namespace A"}, + }, + }, + }, + Name: fmt.Sprintf( + "Generated claims for role %s expiring at %s", + internalRoleName, + testTokenExpiry.Format(time.RFC3339Nano), + ), + Requester: "custom requester", + } + mocks.issuer.EXPECT(). + Issue(gomock.Any(), expectedClaims, gomock.Any()). + Times(1).Return(&tokens.TokenInfo{Token: "the quick brown fox jumps over the lazy dog"}, nil) + }, + expectedRsp: &v1.GenerateTokenForPermissionsAndScopeResponse{ + Token: "the quick brown fox jumps over the lazy dog", + }, + }, + "success - no requested permissions": { + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + // No requested permission results in empty permissions for the user role in the token claims. + Permissions: nil, + ClusterScopes: []*v1.ClusterScope{requestSingleNamespace}, + Lifetime: testExpirationDuration, + }, + policy: permissivePolicy, + setup: func(t *testing.T, mocks *mockContainer) { + expectedClaims := tokens.RoxClaims{ + InternalRoles: []*tokens.InternalRole{ + { + RoleName: internalRoleName, + Clusters: tokens.ClusterScopes{ + fixtureconsts.Cluster1: {"namespace A"}, + }, + }, + }, + Name: fmt.Sprintf( + "Generated claims for role %s expiring at %s", + internalRoleName, + testTokenExpiry.Format(time.RFC3339Nano), + ), + } + mocks.issuer.EXPECT(). + Issue(gomock.Any(), expectedClaims, gomock.Any()). + Times(1).Return(&tokens.TokenInfo{Token: "In the days when everybody started fair, the Leopard lived in a place called the High Veldt."}, nil) + }, + expectedRsp: &v1.GenerateTokenForPermissionsAndScopeResponse{ + Token: "In the days when everybody started fair, the Leopard lived in a place called the High Veldt.", + }, + }, + "success - no requested scope": { + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: deploymentPermission, + // No requested scope results in empty scope for the user role in the token claims. + ClusterScopes: nil, + Lifetime: testExpirationDuration, + }, + policy: permissivePolicy, + setup: func(t *testing.T, mocks *mockContainer) { + expectedClaims := tokens.RoxClaims{ + InternalRoles: []*tokens.InternalRole{ + { + RoleName: internalRoleName, + Permissions: map[storage.Access][]string{ + storage.Access_READ_ACCESS: {deploymentResource}, + }, + Clusters: nil, + }, + }, + Name: fmt.Sprintf( + "Generated claims for role %s expiring at %s", + internalRoleName, + testTokenExpiry.Format(time.RFC3339Nano), + ), + } + mocks.issuer.EXPECT(). + Issue(gomock.Any(), expectedClaims, gomock.Any()). + Times(1).Return(&tokens.TokenInfo{Token: "In the high and far-off times the elephant had no trunk."}, nil) + }, + expectedRsp: &v1.GenerateTokenForPermissionsAndScopeResponse{ + Token: "In the high and far-off times the elephant had no trunk.", + }, + }, + "success - multiple permissions and cluster scopes": { + input: &v1.GenerateTokenForPermissionsAndScopeRequest{ + Permissions: map[string]v1.Access{ + deploymentResource: v1.Access_READ_ACCESS, + imageResource: v1.Access_READ_WRITE_ACCESS, + }, + ClusterScopes: []*v1.ClusterScope{ + requestSingleNamespace, + requestFullCluster, + }, + Lifetime: testExpirationDuration, + }, + policy: newTokenPolicy(300*time.Second, map[string]v1.Access{ + deploymentResource: v1.Access_READ_ACCESS, + imageResource: v1.Access_READ_WRITE_ACCESS, + }), + setup: func(t *testing.T, mocks *mockContainer) { + expectedClaims := tokens.RoxClaims{ + InternalRoles: []*tokens.InternalRole{ + { + RoleName: internalRoleName, + Permissions: map[storage.Access][]string{ + storage.Access_READ_ACCESS: {deploymentResource}, + storage.Access_READ_WRITE_ACCESS: {imageResource}, + }, + Clusters: tokens.ClusterScopes{ + fixtureconsts.Cluster1: {"*"}, + }, + }, + }, + Name: fmt.Sprintf( + "Generated claims for role %s expiring at %s", + internalRoleName, + testTokenExpiry.Format(time.RFC3339Nano), + ), + } + mocks.issuer.EXPECT(). + Issue(gomock.Any(), expectedClaims, gomock.Any()). + Times(1). + Return(&tokens.TokenInfo{Token: "Hear and attend and listen; for this befell and happened and became and was, when the Tame animals were wild."}, nil) + }, + expectedRsp: &v1.GenerateTokenForPermissionsAndScopeResponse{ + Token: "Hear and attend and listen; for this befell and happened and became and was, when the Tame animals were wild.", + }, + }, + } { + t.Run(name, func(it *testing.T) { + mockCtrl := gomock.NewController(it) + mocks := &mockContainer{ + clusterStore: clusterDataStoreMocks.NewMockDataStore(mockCtrl), + roleStore: roleDataStoreMocks.NewMockDataStore(mockCtrl), + issuer: tokensMocks.NewMockIssuer(mockCtrl), + } + svc := createService(it, mocks.issuer, tc.policy) + if tc.setup != nil { + tc.setup(it, mocks) + } + ctx := sensorContext(it, mockCtrl, fixtureconsts.Cluster1) + + rsp, err := svc.GenerateTokenForPermissionsAndScope(ctx, tc.input) + assert.NotNil(it, rsp) + protoassert.Equal(it, tc.expectedRsp, rsp) + assert.NoError(it, err) + }) + } } diff --git a/central/auth/internaltokens/service/singleton.go b/central/auth/internaltokens/service/singleton.go index b8d14eb447c20..8ca92ff5aea84 100644 --- a/central/auth/internaltokens/service/singleton.go +++ b/central/auth/internaltokens/service/singleton.go @@ -3,9 +3,7 @@ package service import ( "time" - clusterStore "github.com/stackrox/rox/central/cluster/datastore" "github.com/stackrox/rox/central/jwt" - roleStore "github.com/stackrox/rox/central/role/datastore" "github.com/stackrox/rox/pkg/auth/authproviders/tokenbased" "github.com/stackrox/rox/pkg/auth/tokens" "github.com/stackrox/rox/pkg/sync" @@ -28,12 +26,10 @@ var ( func Singleton() Service { once.Do(func() { s = &serviceImpl{ - issuer: getTokenIssuer(jwt.IssuerFactorySingleton()), - roleManager: &roleManager{ - clusterStore: clusterStore.Singleton(), - roleStore: roleStore.Singleton(), - }, - now: time.Now, + issuer: getTokenIssuer(jwt.IssuerFactorySingleton()), + roleManager: &roleManager{}, + now: time.Now, + policy: defaultTokenPolicy(), } }) return s diff --git a/central/auth/internaltokens/service/timeutils_test.go b/central/auth/internaltokens/service/timeutils_test.go new file mode 100644 index 0000000000000..a105c82a41f84 --- /dev/null +++ b/central/auth/internaltokens/service/timeutils_test.go @@ -0,0 +1,27 @@ +package service + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/durationpb" +) + +var ( + testClockTime = time.Date(1989, time.November, 9, 18, 05, 35, 987654321, time.UTC) + // testExpirationDuration is the constant token expiration duration. + testExpirationDuration = &durationpb.Duration{Seconds: 300} + // testTokenExpiry is the timestamp of the token expiration. + testTokenExpiry = testClock().Add(testExpirationDuration.AsDuration()) +) + +// testClock is the clock function injection for testing purposes. +func testClock() time.Time { + return testClockTime +} + +func Test_clock(t *testing.T) { + now := testClock() + assert.Equal(t, testExpirationDuration.GetSeconds(), int64(testTokenExpiry.Sub(now).Seconds())) +} diff --git a/central/auth/m2m/generic_token_verifier.go b/central/auth/m2m/generic_token_verifier.go index 0bc6a7fb47e84..770329ee7b4fe 100644 --- a/central/auth/m2m/generic_token_verifier.go +++ b/central/auth/m2m/generic_token_verifier.go @@ -8,21 +8,22 @@ import ( type genericTokenVerifier struct { provider *oidc.Provider + audience string } var _ tokenVerifier = (*genericTokenVerifier)(nil) func (g *genericTokenVerifier) VerifyIDToken(ctx context.Context, rawIDToken string) (*IDToken, error) { - verifier := g.provider.Verifier(&oidc.Config{ - // We currently provide no config to expose the client ID that's associated with the ID token. - // The reason for this is the following: - // - A magnitude of client IDs would have to be configured (i.e. in the case of GitHub actions, this would be - // all repository URLs including their potential customizations). - // - Client IDs (i.e. the "sub" claim) _may_ be part for the mappings within the - // config. So, essentially the client ID check is deferred to a latter point, as the mappings _may_ be used - // for mapping, but it currently isn't a requirement. - SkipClientIDCheck: true, - }) + // When an expected audience is configured, validate the token's aud claim via the go-oidc + // ClientID check. Otherwise skip it for backward compatibility: a single client ID cannot + // cover all cases (e.g. GitHub Actions tokens use per-repository audience values), and the + // audience may still be verified indirectly through claim mappings. + oidcConfig := &oidc.Config{SkipClientIDCheck: true} + if g.audience != "" { + oidcConfig.ClientID = g.audience + oidcConfig.SkipClientIDCheck = false + } + verifier := g.provider.Verifier(oidcConfig) token, err := verifier.Verify(ctx, rawIDToken) if err != nil { diff --git a/central/auth/m2m/generic_token_verifier_test.go b/central/auth/m2m/generic_token_verifier_test.go index cac4427e9b6f2..dd820bdda4cfa 100644 --- a/central/auth/m2m/generic_token_verifier_test.go +++ b/central/auth/m2m/generic_token_verifier_test.go @@ -13,25 +13,23 @@ import ( "time" "github.com/coreos/go-oidc/v3/oidc" - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func Test_genericTokenVerifier_VerifyIDToken(t *testing.T) { - // Generate RSA key pair for signing. +func newTestOIDCServer(t *testing.T) (*httptest.Server, *rsa.PrivateKey, string) { + t.Helper() privateKey, err := rsa.GenerateKey(rand.Reader, 2048) require.NoError(t, err) publicKey := &privateKey.PublicKey - keyID := "test-key-id" - // Create a mock OIDC server. mux := http.NewServeMux() server := httptest.NewServer(mux) - defer server.Close() + t.Cleanup(server.Close) - mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) { discovery := map[string]any{ "issuer": server.URL, "authorization_endpoint": server.URL + "/auth", @@ -40,16 +38,15 @@ func Test_genericTokenVerifier_VerifyIDToken(t *testing.T) { "id_token_signing_alg_values_supported": []string{"RS256"}, } w.Header().Set("Content-Type", "application/json") - err := json.NewEncoder(w).Encode(discovery) - require.NoError(t, err) + if err := json.NewEncoder(w).Encode(discovery); err != nil { + t.Errorf("encoding discovery document: %v", err) + return + } }) - // JWKS endpoint with real RSA public key. - mux.HandleFunc("/jwks", func(w http.ResponseWriter, r *http.Request) { - // Convert RSA public key to JWK format + mux.HandleFunc("/jwks", func(w http.ResponseWriter, _ *http.Request) { nBytes := publicKey.N.Bytes() eBytes := big.NewInt(int64(publicKey.E)).Bytes() - jwks := map[string]any{ "keys": []map[string]any{ { @@ -63,21 +60,33 @@ func Test_genericTokenVerifier_VerifyIDToken(t *testing.T) { }, } w.Header().Set("Content-Type", "application/json") - err := json.NewEncoder(w).Encode(jwks) - require.NoError(t, err) + if err := json.NewEncoder(w).Encode(jwks); err != nil { + t.Errorf("encoding JWKS: %v", err) + return + } }) - // Create real OIDC provider pointing to our mock server. + return server, privateKey, keyID +} + +func signToken(t *testing.T, privateKey *rsa.PrivateKey, keyID string, claims jwt.MapClaims) string { + t.Helper() + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + token.Header["kid"] = keyID + tokenString, err := token.SignedString(privateKey) + require.NoError(t, err) + return tokenString +} + +func Test_genericTokenVerifier_VerifyIDToken(t *testing.T) { + server, privateKey, keyID := newTestOIDCServer(t) + ctx := context.Background() provider, err := oidc.NewProvider(ctx, server.URL) require.NoError(t, err) - // Create verifier with real provider. - verifier := &genericTokenVerifier{provider: provider} - - // Create a valid JWT token. now := time.Now() - claims := jwt.MapClaims{ + tokenString := signToken(t, privateKey, keyID, jwt.MapClaims{ "iss": server.URL, "sub": "test-subject", "aud": []string{"test-audience"}, @@ -85,34 +94,97 @@ func Test_genericTokenVerifier_VerifyIDToken(t *testing.T) { "iat": now.Unix(), "email": "test@example.com", "name": "Test User", - } - - token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) - token.Header["kid"] = keyID + }) - tokenString, err := token.SignedString(privateKey) - require.NoError(t, err) + verifier := &genericTokenVerifier{provider: provider} - // Test with valid token. idToken, err := verifier.VerifyIDToken(ctx, tokenString) - - // Should succeed. require.NoError(t, err) require.NotNil(t, idToken) assert.Equal(t, "test-subject", idToken.Subject) assert.Equal(t, []string{"test-audience"}, idToken.Audience) - // Test claims extraction. var extractedClaims map[string]any err = idToken.Claims(&extractedClaims) require.NoError(t, err) assert.Equal(t, "test@example.com", extractedClaims["email"]) assert.Equal(t, "Test User", extractedClaims["name"]) - // Test with invalid token - should fail JWT parsing/verification. idToken, err = verifier.VerifyIDToken(ctx, "invalid.jwt.token") assert.Nil(t, idToken) assert.Error(t, err) assert.NotContains(t, err.Error(), "404") assert.NotContains(t, err.Error(), "connection") } + +func Test_genericTokenVerifier_AudienceValidation(t *testing.T) { + server, privateKey, keyID := newTestOIDCServer(t) + + ctx := context.Background() + provider, err := oidc.NewProvider(ctx, server.URL) + require.NoError(t, err) + + now := time.Now() + baseClaims := func(aud any) jwt.MapClaims { + claims := jwt.MapClaims{ + "iss": server.URL, + "sub": "test-subject", + "exp": now.Add(time.Hour).Unix(), + "iat": now.Unix(), + } + if aud != nil { + claims["aud"] = aud + } + return claims + } + + testCases := map[string]struct { + audience string + tokenAudience any + expectError bool + }{ + "matching audience passes": { + audience: "my-client-id", + tokenAudience: "my-client-id", + }, + "matching audience in array passes": { + audience: "my-client-id", + tokenAudience: []string{"other-client", "my-client-id"}, + }, + "non-matching audience is rejected": { + audience: "my-client-id", + tokenAudience: "wrong-audience", + expectError: true, + }, + "non-matching audience array is rejected": { + audience: "my-client-id", + tokenAudience: []string{"wrong-audience", "another-wrong"}, + expectError: true, + }, + "empty audience config skips check": { + audience: "", + tokenAudience: "any-audience-value", + }, + "token without aud claim is rejected when audience is configured": { + audience: "my-client-id", + tokenAudience: nil, + expectError: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + verifier := &genericTokenVerifier{provider: provider, audience: tc.audience} + tokenString := signToken(t, privateKey, keyID, baseClaims(tc.tokenAudience)) + + idToken, err := verifier.VerifyIDToken(ctx, tokenString) + if tc.expectError { + assert.Error(t, err) + assert.Nil(t, idToken) + } else { + assert.NoError(t, err) + assert.NotNil(t, idToken) + } + }) + } +} diff --git a/central/auth/m2m/id_token.go b/central/auth/m2m/id_token.go index b5851c50b8935..c957e25abad51 100644 --- a/central/auth/m2m/id_token.go +++ b/central/auth/m2m/id_token.go @@ -1,7 +1,7 @@ package m2m import ( - "github.com/golang-jwt/jwt/v4" + "github.com/golang-jwt/jwt/v5" "github.com/pkg/errors" "github.com/stackrox/rox/pkg/errox" ) diff --git a/central/auth/m2m/role_mapper.go b/central/auth/m2m/role_mapper.go index 4b5107851a495..e75e85443f139 100644 --- a/central/auth/m2m/role_mapper.go +++ b/central/auth/m2m/role_mapper.go @@ -3,6 +3,7 @@ package m2m import ( "context" "regexp" + "slices" "github.com/pkg/errors" roleDataStore "github.com/stackrox/rox/central/role/datastore" @@ -44,7 +45,7 @@ func resolveRolesForClaims(ctx context.Context, claims map[string][]string, role rolesForUser := set.NewStringSet() for i, mapping := range mappings { - if valuesMatch(expressions[i], claims[mapping.GetKey()]) { + if anyValueMatches(expressions[i], claims[mapping.GetKey()]) { rolesForUser.Add(mapping.GetRole()) } } @@ -72,11 +73,6 @@ func resolveRolesForClaims(ctx context.Context, claims map[string][]string, role return resolvedRoles, nil } -func valuesMatch(expr *regexp.Regexp, claimValues []string) bool { - for _, claimValue := range claimValues { - if expr.MatchString(claimValue) { - return true - } - } - return false +func anyValueMatches(expr *regexp.Regexp, claimValues []string) bool { + return slices.ContainsFunc(claimValues, expr.MatchString) } diff --git a/central/auth/m2m/token_verifier.go b/central/auth/m2m/token_verifier.go index b6c74932f2f92..79c6b945d627d 100644 --- a/central/auth/m2m/token_verifier.go +++ b/central/auth/m2m/token_verifier.go @@ -40,5 +40,5 @@ func tokenVerifierFromConfig(ctx context.Context, config *storage.AuthMachineToM return nil, errors.Wrapf(err, "creating OIDC provider for issuer %q", config.GetIssuer()) } - return &genericTokenVerifier{provider: provider}, nil + return &genericTokenVerifier{provider: provider, audience: config.GetAudience()}, nil } diff --git a/central/auth/openshift/metrics_auth.go b/central/auth/openshift/metrics_auth.go new file mode 100644 index 0000000000000..98dc0928c3830 --- /dev/null +++ b/central/auth/openshift/metrics_auth.go @@ -0,0 +1,237 @@ +package openshift + +import ( + "context" + + groupDataStore "github.com/stackrox/rox/central/group/datastore" + rolePkg "github.com/stackrox/rox/central/role" + roleDataStore "github.com/stackrox/rox/central/role/datastore" + "github.com/stackrox/rox/central/tlsconfig" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/auth/authproviders" + "github.com/stackrox/rox/pkg/auth/authproviders/userpki" + permissionsUtils "github.com/stackrox/rox/pkg/auth/permissions/utils" + "github.com/stackrox/rox/pkg/declarativeconfig" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/k8scfgwatch" + "github.com/stackrox/rox/pkg/k8sutil" + "github.com/stackrox/rox/pkg/logging" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + authProviderID = "b3070020-ecc3-4f34-a3f6-ad28ffc9b80f" + permissionSetID = "b3070020-ecc3-4f34-a3f6-ad28ffc9b80e" + + // Auth provider is hidden from the login screen but visible in Access Control. + // It trusts the Kubernetes client CA from extension-apiserver-authentication, + // which signs Prometheus client certs. This adds the CA to userTrustRoots on the + // public :8443 endpoint (optional client cert verification). + authProviderName = "OpenShift Platform Client Certificates" + roleName = "OpenShift Prometheus Metrics Reader" + permissionSetName = "OpenShift Prometheus Metrics Reader" +) + +var log = logging.LoggerForModule() + +// SeedMetricsAuthProvider ensures that the userpki auth provider, permission set, role, +// and group mapping required for OpenShift Prometheus to scrape /metrics exist. +// +// The auth provider uses the Kubernetes client CA from the +// extension-apiserver-authentication ConfigMap (same CA that signs Prometheus client +// certs). A ConfigMap watcher keeps the CA current across rotations. +// +// The permission set and role use IMPERATIVE origin. The auth provider and group +// use DEFAULT origin so they cannot be modified through the user-facing APIs. +func SeedMetricsAuthProvider(ctx context.Context, registry authproviders.Registry, roleDS roleDataStore.DataStore, groupDS groupDataStore.DataStore) { + if !tlsconfig.OpenShiftTLSConfigured() { + return + } + + clientset, err := newK8sClient() + if err != nil { + log.Warnf("Cannot create Kubernetes client for OpenShift metrics auth: %v", err) + return + } + + onCA := func(caPEM string) { + ensurePermissionSet(ctx, roleDS) + ensureRole(ctx, roleDS) + ensureAuthProvider(ctx, registry, caPEM) + ensureGroup(ctx, groupDS) + } + + if caPEM := readClientCA(ctx, clientset); caPEM != "" { + onCA(caPEM) + } + + watchClientCA(ctx, clientset, onCA) +} + +func newK8sClient() (kubernetes.Interface, error) { + config, err := k8sutil.GetK8sInClusterConfig() + if err != nil { + return nil, err + } + return kubernetes.NewForConfig(config) +} + +func readClientCA(ctx context.Context, clientset kubernetes.Interface) string { + cm, err := clientset.CoreV1().ConfigMaps( + env.SecureMetricsClientCANamespace.Setting(), + ).Get(ctx, env.SecureMetricsClientCAConfigMap.Setting(), metav1.GetOptions{}) + if err != nil { + log.Warnf("Cannot read client CA ConfigMap %s/%s: %v", + env.SecureMetricsClientCANamespace.Setting(), + env.SecureMetricsClientCAConfigMap.Setting(), err) + return "" + } + pem, ok := cm.Data[env.SecureMetricsClientCAKey.Setting()] + if !ok || pem == "" { + log.Warnf("Client CA key %q not found in ConfigMap %s/%s", + env.SecureMetricsClientCAKey.Setting(), + env.SecureMetricsClientCANamespace.Setting(), + env.SecureMetricsClientCAConfigMap.Setting()) + return "" + } + return pem +} + +func watchClientCA(ctx context.Context, clientset kubernetes.Interface, onCA func(string)) { + watcher := k8scfgwatch.NewConfigMapWatcher(clientset, func(cm *v1.ConfigMap) { + if cm == nil { + return + } + pem, ok := cm.Data[env.SecureMetricsClientCAKey.Setting()] + if !ok || pem == "" { + return + } + onCA(pem) + }) + watcher.Watch( + ctx, + env.SecureMetricsClientCANamespace.Setting(), + env.SecureMetricsClientCAConfigMap.Setting(), + ) +} + +func refreshCA(ctx context.Context, registry authproviders.Registry, provider authproviders.Provider, caPEM string) { + config := provider.StorageView().GetConfig() + if config[userpki.ConfigKeys] == caPEM { + return + } + log.Info("Client CA changed, updating OpenShift metrics auth provider") + config[userpki.ConfigKeys] = caPEM + + factory := provider.BackendFactory() + if factory == nil { + log.Warn("Auth provider has no backend factory, cannot apply the new client CA") + return + } + // WithConfig on its own only rewrites the stored config. The provider keeps its + // existing backend, which parsed the certificates once at construction time, and the + // registry then re-registers those same stale certificates with the client CA + // manager. Central would keep advertising the superseded CA until restart, so the + // backend has to be rebuilt from the new config as part of the same update. + if _, err := registry.UpdateProvider(declarativeconfig.WithModifyDefaultResource(ctx), authProviderID, + authproviders.WithConfig(config), + authproviders.WithBackendFromFactory(ctx, factory), + ); err != nil { + log.Warnf("Failed to update client CA in auth provider: %v", err) + } +} + +func ensurePermissionSet(ctx context.Context, roleDS roleDataStore.DataStore) { + if _, exists, err := roleDS.GetPermissionSet(ctx, permissionSetID); err != nil { + log.Warnf("Failed to check OpenShift metrics permission set: %v", err) + return + } else if exists { + return + } + + ps := &storage.PermissionSet{ + Id: permissionSetID, + Name: permissionSetName, + Description: "For OpenShift Prometheus: read access to Administration for /metrics scraping", + // Let's give no permissions by default to avoid unexpected sensitive data exposure. + ResourceToAccess: permissionsUtils.FromResourcesWithAccess(), + } + if err := roleDS.AddPermissionSet(ctx, ps); err != nil { + log.Warnf("Failed to create OpenShift metrics permission set: %v", err) + } +} + +func ensureRole(ctx context.Context, roleDS roleDataStore.DataStore) { + if _, exists, err := roleDS.GetRole(ctx, roleName); err != nil { + log.Warnf("Failed to check OpenShift metrics role: %v", err) + return + } else if exists { + return + } + + role := &storage.Role{ + Name: roleName, + Description: "Maps OpenShift Prometheus service account to /metrics read access", + PermissionSetId: permissionSetID, + AccessScopeId: rolePkg.AccessScopeIncludeAll.GetId(), + } + if err := roleDS.AddRole(ctx, role); err != nil { + log.Warnf("Failed to create OpenShift metrics role: %v", err) + } +} + +func ensureAuthProvider(ctx context.Context, registry authproviders.Registry, caPEM string) { + if existing := registry.GetProvider(authProviderID); existing != nil { + refreshCA(ctx, registry, existing, caPEM) + return + } + + _, err := registry.CreateProvider(declarativeconfig.WithModifyDefaultResource(ctx), + authproviders.WithType(userpki.TypeName), + authproviders.WithID(authProviderID), + authproviders.WithName(authProviderName), + authproviders.WithEnabled(true), + authproviders.WithActive(true), + authproviders.WithVisibility(storage.Traits_HIDDEN), + authproviders.WithOrigin(storage.Traits_DEFAULT), + authproviders.WithConfig(map[string]string{ + userpki.ConfigKeys: caPEM, + }), + ) + if err != nil { + log.Warnf("Failed to create OpenShift metrics auth provider: %v", err) + } +} + +func ensureGroup(ctx context.Context, groupDS groupDataStore.DataStore) { + cn := env.SecureMetricsClientCertCN.Setting() + existing, err := groupDS.GetFiltered(ctx, func(g *storage.Group) bool { + p := g.GetProps() + return p.GetAuthProviderId() == authProviderID && p.GetKey() == "name" && p.GetValue() == cn + }) + if err != nil { + log.Warnf("Failed to check OpenShift metrics group: %v", err) + return + } + if len(existing) > 0 { + return + } + group := &storage.Group{ + Props: &storage.GroupProperties{ + AuthProviderId: authProviderID, + Key: "name", + Value: cn, + Traits: &storage.Traits{ + Origin: storage.Traits_DEFAULT, + }, + }, + RoleName: roleName, + } + // Group uses DEFAULT origin so a user can't repoint or delete the mapping and thereby + // orphan (and delete) the role out from under it. + if err := groupDS.Add(declarativeconfig.WithModifyDefaultResource(ctx), group); err != nil { + log.Warnf("Failed to create OpenShift metrics group: %v", err) + } +} diff --git a/central/auth/openshift/metrics_auth_test.go b/central/auth/openshift/metrics_auth_test.go new file mode 100644 index 0000000000000..9355236e8d6d4 --- /dev/null +++ b/central/auth/openshift/metrics_auth_test.go @@ -0,0 +1,264 @@ +package openshift + +import ( + "context" + "crypto/x509" + "errors" + "testing" + + groupMocks "github.com/stackrox/rox/central/group/datastore/mocks" + roleMocks "github.com/stackrox/rox/central/role/datastore/mocks" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/auth/authproviders" + providerMocks "github.com/stackrox/rox/pkg/auth/authproviders/mocks" + "github.com/stackrox/rox/pkg/auth/authproviders/userpki" + "github.com/stackrox/rox/pkg/auth/permissions" + "github.com/stackrox/rox/pkg/auth/tokens" + "github.com/stackrox/rox/pkg/certgen" + "github.com/stackrox/rox/pkg/declarativeconfig" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +const testCAPEM = "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n" + +func setupMocks(t *testing.T) (*providerMocks.MockRegistry, *roleMocks.MockDataStore, *groupMocks.MockDataStore) { + ctrl := gomock.NewController(t) + return providerMocks.NewMockRegistry(ctrl), + roleMocks.NewMockDataStore(ctrl), + groupMocks.NewMockDataStore(ctrl) +} + +func TestSeed_FirstBoot_CreatesAllObjects(t *testing.T) { + registry, roleDS, groupDS := setupMocks(t) + ctx := context.Background() + + roleDS.EXPECT().GetPermissionSet(gomock.Any(), permissionSetID).Return(nil, false, nil) + roleDS.EXPECT().AddPermissionSet(gomock.Any(), gomock.Any()).Return(nil) + roleDS.EXPECT().GetRole(gomock.Any(), roleName).Return(nil, false, nil) + roleDS.EXPECT().AddRole(gomock.Any(), gomock.Any()).Return(nil) + + providerCreated := false + registry.EXPECT().GetProvider(authProviderID).Return(nil) + registry.EXPECT().CreateProvider(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ ...interface{}) (interface{}, error) { + providerCreated = true + return nil, nil + }) + groupDS.EXPECT().GetFiltered(gomock.Any(), gomock.Any()).Return(nil, nil) + groupDS.EXPECT().Add(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ *storage.Group) error { + if !providerCreated { + t.Error("group Add called before provider creation") + } + return nil + }) + + ensurePermissionSet(ctx, roleDS) + ensureRole(ctx, roleDS) + ensureAuthProvider(ctx, registry, testCAPEM) + ensureGroup(ctx, groupDS) +} + +func TestSeed_SubsequentBoot_CAUnchanged_NoUpdate(t *testing.T) { + registry, _, _ := setupMocks(t) + ctx := context.Background() + + existing := providerMocks.NewMockProvider(gomock.NewController(t)) + existing.EXPECT().StorageView().Return(&storage.AuthProvider{ + Config: map[string]string{userpki.ConfigKeys: testCAPEM}, + }) + registry.EXPECT().GetProvider(authProviderID).Return(existing) + + ensureAuthProvider(ctx, registry, testCAPEM) +} + +func TestSeed_PartialRecovery_PermissionSetExists_RoleMissing(t *testing.T) { + registry, roleDS, groupDS := setupMocks(t) + ctx := context.Background() + + roleDS.EXPECT().GetPermissionSet(gomock.Any(), permissionSetID).Return(&storage.PermissionSet{}, true, nil) + roleDS.EXPECT().GetRole(gomock.Any(), roleName).Return(nil, false, nil) + roleDS.EXPECT().AddRole(gomock.Any(), gomock.Any()).Return(nil) + registry.EXPECT().GetProvider(authProviderID).Return(nil) + registry.EXPECT().CreateProvider(gomock.Any(), gomock.Any()).Return(nil, nil) + groupDS.EXPECT().GetFiltered(gomock.Any(), gomock.Any()).Return(nil, nil) + groupDS.EXPECT().Add(gomock.Any(), gomock.Any()).Return(nil) + + ensurePermissionSet(ctx, roleDS) + ensureRole(ctx, roleDS) + ensureAuthProvider(ctx, registry, testCAPEM) + ensureGroup(ctx, groupDS) +} + +func TestSeed_Group_IsDefaultOriginAndImmutable(t *testing.T) { + _, _, groupDS := setupMocks(t) + ctx := context.Background() + + groupDS.EXPECT().GetFiltered(gomock.Any(), gomock.Any()).Return(nil, nil) + groupDS.EXPECT().Add(gomock.Any(), gomock.Any()).DoAndReturn( + func(addCtx context.Context, group *storage.Group) error { + if group.GetProps().GetTraits().GetOrigin() != storage.Traits_DEFAULT { + t.Errorf("expected group to use DEFAULT origin, got %v", group.GetProps().GetTraits().GetOrigin()) + } + assert.False(t, declarativeconfig.CanModifyResource(ctx, group.GetProps())) + if !declarativeconfig.CanModifyResource(addCtx, group.GetProps()) { + t.Error("expected ensureGroup to pass a context that can modify DEFAULT-origin resources") + } + return nil + }) + + ensureGroup(ctx, groupDS) +} + +func TestSeed_SubsequentBoot_GroupExists_NotOverwritten(t *testing.T) { + _, _, groupDS := setupMocks(t) + ctx := context.Background() + + groupDS.EXPECT().GetFiltered(gomock.Any(), gomock.Any()).Return([]*storage.Group{{ + Props: &storage.GroupProperties{AuthProviderId: authProviderID, Key: "name", Value: "system:serviceaccount:openshift-monitoring:prometheus-k8s"}, + RoleName: "User Modified Role", + }}, nil) + + ensureGroup(ctx, groupDS) +} + +// TestSeed_CARotation_RegistersNewCA runs against a real auth provider registry because +// the interesting part of a rotation happens past the registry: a mocked registry only +// shows that UpdateProvider was called, not whether the new CA reached the client CA +// manager. Persisting the bundle without also rebuilding the provider backend leaves +// Central advertising the superseded CA until restart, breaking Prometheus scrapes +// roughly monthly when OpenShift rotates its CSR signer. +func TestSeed_CARotation_RegistersNewCA(t *testing.T) { + ctx := context.Background() + caManager := &recordingCAManager{} + store := &inMemoryAuthProviderStore{providers: map[string]*storage.AuthProvider{}} + + registry := authproviders.NewStoreBackedRegistry("/sso/", "/auth/response/generic", + store, noopIssuerFactory{}, noopRoleMapperFactory{}, nil) + require.NoError(t, registry.RegisterBackendFactory(ctx, userpki.TypeName, + userpki.NewFactoryFactory(caManager))) + require.NoError(t, registry.Init()) + + oldCAPEM, oldCA := generateCA(t) + newCAPEM, newCA := generateCA(t) + + ensureAuthProvider(ctx, registry, oldCAPEM) + require.Equal(t, []string{oldCA.Subject.String()}, registeredSubjects(caManager.certs)) + + ensureAuthProvider(ctx, registry, newCAPEM) + + provider := registry.GetProvider(authProviderID) + require.NotNil(t, provider) + assert.Equal(t, storage.Traits_DEFAULT, provider.StorageView().GetTraits().GetOrigin()) + assert.Equal(t, newCAPEM, provider.StorageView().GetConfig()[userpki.ConfigKeys], + "rotated CA should be persisted in the auth provider config") + assert.Equal(t, []string{newCA.Subject.String()}, registeredSubjects(caManager.certs), + "rotated CA should also be registered with the client CA manager") +} + +func generateCA(t *testing.T) (string, *x509.Certificate) { + ca, err := certgen.GenerateCA() + require.NoError(t, err) + return string(ca.CertPEM()), ca.Certificate() +} + +func registeredSubjects(certs []*x509.Certificate) []string { + subjects := make([]string, 0, len(certs)) + for _, cert := range certs { + subjects = append(subjects, cert.Subject.String()) + } + return subjects +} + +// recordingCAManager stands in for central/tlsconfig.Manager, which is what Central wires +// into the userpki backend factory. It records the certificates the backend hands over, +// i.e. what ends up in the client CA pool advertised during the TLS handshake. +type recordingCAManager struct { + certs []*x509.Certificate +} + +func (c *recordingCAManager) RegisterAuthProvider(_ authproviders.Provider, certs []*x509.Certificate) { + c.certs = certs +} + +func (c *recordingCAManager) UnregisterAuthProvider(authproviders.Provider) { + c.certs = nil +} + +func (c *recordingCAManager) GetProviderForFingerprint(string) authproviders.Provider { + return nil +} + +type inMemoryAuthProviderStore struct { + providers map[string]*storage.AuthProvider +} + +func (s *inMemoryAuthProviderStore) GetAuthProvider(_ context.Context, id string) (*storage.AuthProvider, bool, error) { + provider, found := s.providers[id] + return provider, found, nil +} + +func (s *inMemoryAuthProviderStore) ForEachAuthProvider(_ context.Context, fn func(*storage.AuthProvider) error) error { + for _, provider := range s.providers { + if err := fn(provider); err != nil { + return err + } + } + return nil +} + +func (s *inMemoryAuthProviderStore) GetAuthProvidersFiltered(_ context.Context, + filter func(*storage.AuthProvider) bool) ([]*storage.AuthProvider, error) { + var filtered []*storage.AuthProvider + for _, provider := range s.providers { + if filter(provider) { + filtered = append(filtered, provider) + } + } + return filtered, nil +} + +func (s *inMemoryAuthProviderStore) AuthProviderExistsWithName(_ context.Context, name string) (bool, error) { + for _, provider := range s.providers { + if provider.GetName() == name { + return true, nil + } + } + return false, nil +} + +func (s *inMemoryAuthProviderStore) AddAuthProvider(ctx context.Context, provider *storage.AuthProvider) error { + if !declarativeconfig.CanModifyResource(ctx, provider) { + return errors.New("cannot create provider with this origin context") + } + s.providers[provider.GetId()] = provider.CloneVT() + return nil +} + +func (s *inMemoryAuthProviderStore) UpdateAuthProvider(ctx context.Context, provider *storage.AuthProvider) error { + if !declarativeconfig.CanModifyResource(ctx, s.providers[provider.GetId()]) || + !declarativeconfig.CanModifyResource(ctx, provider) { + return errors.New("cannot update provider with this origin context") + } + s.providers[provider.GetId()] = provider.CloneVT() + return nil +} + +func (s *inMemoryAuthProviderStore) RemoveAuthProvider(_ context.Context, id string, _ bool) error { + delete(s.providers, id) + return nil +} + +type noopIssuerFactory struct{} + +func (noopIssuerFactory) CreateIssuer(tokens.Source, ...tokens.Option) (tokens.Issuer, error) { + return nil, nil +} + +func (noopIssuerFactory) UnregisterSource(tokens.Source) error { return nil } + +type noopRoleMapperFactory struct{} + +func (noopRoleMapperFactory) GetRoleMapper(string) permissions.RoleMapper { return nil } diff --git a/central/auth/service/service_impl_test.go b/central/auth/service/service_impl_test.go index 21962705a4ff7..c5b8a79e4e99d 100644 --- a/central/auth/service/service_impl_test.go +++ b/central/auth/service/service_impl_test.go @@ -791,7 +791,7 @@ func (s *authServiceAccessControlTestSuite) addRoles() { }, })) - for _, role := range testRoles.AsSlice() { + for role := range testRoles.All() { s.Require().NoError(s.roleDS.AddRole(s.accessCtx, &storage.Role{ Name: role, Description: "test role", diff --git a/central/authprovider/registry/singleton.go b/central/authprovider/registry/singleton.go index def0c6fad846d..a57a9f90435ac 100644 --- a/central/authprovider/registry/singleton.go +++ b/central/authprovider/registry/singleton.go @@ -2,6 +2,7 @@ package registry import ( authProviderDS "github.com/stackrox/rox/central/authprovider/datastore" + clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" groupDataStore "github.com/stackrox/rox/central/group/datastore" "github.com/stackrox/rox/central/jwt" "github.com/stackrox/rox/central/role/mapper" @@ -24,7 +25,7 @@ func initialize() { registry = authproviders.NewStoreBackedRegistry( ssoURLPathPrefix, tokenRedirectURLPath, authProviderDS.Singleton(), jwt.IssuerFactorySingleton(), - mapper.FactorySingleton()) + mapper.FactorySingleton(), clusterDataStore.Singleton()) // Set the auth provider registry function in the group datastore to break circular dependency groupDataStore.SetAuthProviderRegistryFunc(func() authproviders.Registry { return registry diff --git a/central/authprovider/service/service_impl.go b/central/authprovider/service/service_impl.go index df7d7ad5fd1d3..3e3f6f1384eb0 100644 --- a/central/authprovider/service/service_impl.go +++ b/central/authprovider/service/service_impl.go @@ -192,6 +192,9 @@ func (s *serviceImpl) GetAuthProviders(_ context.Context, request *v1.GetAuthPro // PostAuthProvider inserts a new auth provider into the system. func (s *serviceImpl) PostAuthProvider(ctx context.Context, request *v1.PostAuthProviderRequest) (*storage.AuthProvider, error) { providerReq := request.GetProvider() + if err := normalizeAuthProviderEndpoints(providerReq); err != nil { + return nil, err + } if providerReq.GetName() == "" { return nil, errox.InvalidArgs.CausedBy("no auth provider name specified") } @@ -216,6 +219,9 @@ func (s *serviceImpl) PostAuthProvider(ctx context.Context, request *v1.PostAuth // PutAuthProvider upserts an auth provider into the system. func (s *serviceImpl) PutAuthProvider(ctx context.Context, request *storage.AuthProvider) (*storage.AuthProvider, error) { + if err := normalizeAuthProviderEndpoints(request); err != nil { + return nil, err + } if request.GetId() == "" { return nil, errox.InvalidArgs.CausedBy("auth provider id is empty") } @@ -355,3 +361,24 @@ func (s *serviceImpl) ExchangeToken(ctx context.Context, request *v1.ExchangeTok } return response, nil } + +func normalizeAuthProviderEndpoints(provider *storage.AuthProvider) error { + if provider == nil { + return nil + } + if ep := provider.GetUiEndpoint(); ep != "" { + normalized, err := authproviders.NormalizeUIEndpoint(ep) + if err != nil { + return err + } + provider.UiEndpoint = normalized + } + for i, ep := range provider.GetExtraUiEndpoints() { + normalized, err := authproviders.NormalizeUIEndpoint(ep) + if err != nil { + return err + } + provider.ExtraUiEndpoints[i] = normalized + } + return nil +} diff --git a/central/authprovider/service/service_impl_postgres_test.go b/central/authprovider/service/service_impl_postgres_test.go index 7cc7ea126330b..c12bbb8a5e8ff 100644 --- a/central/authprovider/service/service_impl_postgres_test.go +++ b/central/authprovider/service/service_impl_postgres_test.go @@ -55,6 +55,7 @@ func (s *authProviderServiceTestSuite) SetupSuite() { authProviderDS, tokenIssuerFactory, mapperFactory, + nil, ) ctx := sac.WithAllAccess(context.Background()) diff --git a/central/authprovider/service/service_impl_test.go b/central/authprovider/service/service_impl_test.go index bb9f9b7194faf..8144ac20c5286 100644 --- a/central/authprovider/service/service_impl_test.go +++ b/central/authprovider/service/service_impl_test.go @@ -12,6 +12,7 @@ import ( permissionsMocks "github.com/stackrox/rox/pkg/auth/permissions/mocks" authTokenMocks "github.com/stackrox/rox/pkg/auth/tokens/mocks" "github.com/stackrox/rox/pkg/errox" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" ) @@ -57,6 +58,7 @@ func (s *mockedAuthProviderServiceTestSuite) SetupTest() { s.providerMockStore, tokenIssuerFactory, mapperFactory, + nil, ) s.providerMockBEFactory = authProviderMocks.NewMockBackendFactory(s.mockCtrl) @@ -152,3 +154,54 @@ func (s *mockedAuthProviderServiceTestSuite) expectProviderAdditionToStore() { Times(1). Return(nil) } + +func TestNormalizeAuthProviderEndpoints(t *testing.T) { + cases := map[string]struct { + input *storage.AuthProvider + expectedEndpoint string + expectedExtraEPs []string + wantErr bool + }{ + "nil provider": { + input: nil, + }, + "strips scheme and infers port": { + input: &storage.AuthProvider{ + UiEndpoint: "https://central.example.com", + ExtraUiEndpoints: []string{"http://localhost:8080", "central.example.com"}, + }, + expectedEndpoint: "central.example.com:443", + expectedExtraEPs: []string{"localhost:8080", "central.example.com:443"}, + }, + "already canonical": { + input: &storage.AuthProvider{ + UiEndpoint: "central.example.com:443", + ExtraUiEndpoints: []string{"localhost:8080"}, + }, + expectedEndpoint: "central.example.com:443", + expectedExtraEPs: []string{"localhost:8080"}, + }, + "rejects empty extra endpoint": { + input: &storage.AuthProvider{ + UiEndpoint: "central.example.com:443", + ExtraUiEndpoints: []string{""}, + }, + wantErr: true, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + err := normalizeAuthProviderEndpoints(tc.input) + if tc.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + if tc.input != nil { + assert.Equal(t, tc.expectedEndpoint, tc.input.GetUiEndpoint()) + assert.Equal(t, tc.expectedExtraEPs, tc.input.GetExtraUiEndpoints()) + } + }) + } +} diff --git a/central/backgroundmigrations/BACKGROUND_MIGRATIONS.md b/central/backgroundmigrations/BACKGROUND_MIGRATIONS.md new file mode 100644 index 0000000000000..09b684e60d2b1 --- /dev/null +++ b/central/backgroundmigrations/BACKGROUND_MIGRATIONS.md @@ -0,0 +1,598 @@ +# Background Migrations Guide + +Background migrations run **after** Central is fully started and serving traffic. +They are the **only** path for all data backfills and transformations. They wait for the +rollout to fully complete (all old-version pods terminated) before starting, which is the +primary mechanism that prevents conflicts with old Central code. Use a startup migration +only for DDL/schema changes that GORM AutoMigrate cannot handle. + +For an overview of both migration types and the decision flowchart, see +[README.md](../../migrator/README.md). + +## When to use background migrations + +- All data backfills and transformations, regardless of table size. Central must tolerate partially migrated state (some rows backfilled, some not) while the migration runs +- Creating indexes not covered by the code generator (e.g., partial indexes, expression + indexes). Standard non-unique indexes are handled by proto tags (`sql:"index"`) and + the background migration runner's index reconciliation. Unique indexes are created at + startup via GORM — see [Index Management](../../migrator/README.md#index-management). + +**Do NOT use background migrations** for: +- Schema-only changes (GORM AutoMigrate handles these) +- Standard index creation (use `sql:"index"` proto tags instead). + Unique indexes are always created at startup via GORM. +- Setting static column defaults, use a startup migration with `ALTER COLUMN SET DEFAULT` because PSQL optimizes for that use case + +## How it works + +``` +Central starts and begins serving traffic + │ + ▼ +Rollout checker polls K8s API + "Are all old-version pods terminated?" + │ + NO ──┤──► Wait 60s, retry + │ + YES ─┘ + │ + ▼ +Acquire advisory lock (only one Central runs migrations) + │ + ▼ +Read current seq num from background_migration_versions table + │ + ▼ +For each migration from current to target: + ├── Run migration.Run(ctx, db) + └── Update seq num in DB (checkpoint) + │ + ▼ +Reconcile indexes (CREATE INDEX CONCURRENTLY) +``` + +The rollout checker ensures old-version Central pods are fully terminated before +migrations begin. This prevents the old code from writing data in the pre-migration +format while the migration is transforming it. + +If the migration fails, the runner retries after 60 seconds. If Central shuts down +mid-migration, the next startup resumes from the last checkpoint. + +## Getting started + +### 1. Create the migration package + +**The part of creating the migration manually is going to be replaced with [ROX-35101](https://redhat.atlassian.net/browse/ROX-35101)** + +Unlike startup migrations, there is no `make bootstrap_migration` tool for background +migrations. Create the package manually. + +Create a new directory under `central/backgroundmigrations/migrations/`. Follow the +naming convention `m_{N}_to_m_{N+1}_{description}/`. + +``` +central/backgroundmigrations/migrations/m_000_to_m_001_backfill_foo/ +└── migration.go +``` + +### 2. Register the migration + +In `migration.go`, register via `init()`: + +```go +package m000tom001 + +import ( + "context" + + "github.com/stackrox/rox/central/backgroundmigrations/migrations" + "github.com/stackrox/rox/central/backgroundmigrations/types" + "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/postgres" +) + +var log = logging.LoggerForModule() + +func init() { + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, + VersionAfterSeqNum: 1, + Description: "Backfill foo column on process_indicators", + Run: run, + }) +} + +func run(ctx context.Context, db postgres.DB) error { + // Your migration logic here + return nil +} +``` + +### 3. Import in runner/all.go + +Add a blank import in `central/backgroundmigrations/runner/all.go`: + +```go +import ( + _ "github.com/stackrox/rox/central/backgroundmigrations/migrations/m_000_to_m_001_backfill_foo" +) +``` + +### 4. Bump the sequence number + +Check the current value in `central/backgroundmigrations/seq_num.go` and increment it. +Your migration's `StartingSeqNum` must equal the current value, and +`VersionAfterSeqNum` must be `StartingSeqNum + 1`. + +```go +const CurrentBgMigrationSeqNum = 1 // was 0 +``` + +## The contract + +Your `Run` function must satisfy these requirements: + +1. **Idempotent**: Safe to re-run after a crash, rollback, or manual override. Previously + migrated rows must be processed again on re-run and must produce a consistent result with the serialized field. +2. **Context-aware**: Check `ctx.Done()` between units of work (e.g., between batches) for + graceful shutdown. When cancelled, return `ctx.Err()`. +3. **Conflict-free with Central**: Central is live and processing events. Your migration + must not conflict with concurrent reads and writes. See + [Concurrency with Central](#concurrency-with-central) below. +4. **Efficient on re-run**: On crash, rollback, or retry the migration starts from the + beginning. Iterate by primary key and only write rows that actually need a change -- + already-migrated rows are read but not written, making re-runs fast. Avoid filtering + by `WHERE new_column IS NULL` as the sole pagination mechanism unless there is an + index on that column; without an index every batch scan is expensive. + +## Concurrency with Central + +Background migrations wait for the rollout to fully complete before starting — at that +point, only new Central code is running. However, the new Central is actively receiving +sensor data, API calls, and processing events while the migration runs. You cannot +control what events arrive, so the migration must account for concurrent writes from +the new Central. + +### Use `SELECT ... FOR UPDATE SKIP LOCKED` + +The standard pattern is to iterate by primary key in batches, locking rows for update +while skipping any rows currently locked by Central. See +[Example 1](#example-1-backfill-a-column-from-a-serialized-proto) for a complete +implementation. + +```go +rows, err := conn.Query(ctx, ` + SELECT id, serialized FROM my_table + WHERE id > $1 + ORDER BY id + LIMIT $2 + FOR UPDATE SKIP LOCKED`, lastID, batchSize) +``` + +- **`FOR UPDATE`** acquires row-level locks, preventing Central from modifying those + rows while the migration processes them. +- **`SKIP LOCKED`** skips rows currently locked by the new Central rather than blocking. + Since the rollout is complete at this point, any locked row is being written by the new + Central version which already populates the new column consistently. +- **Primary key iteration** (`id > $1 ORDER BY id`) uses the primary key index for + efficient pagination. The full table scan happens once; writes are far more expensive + than the read pass, so this is less of a bottleneck. + +Within each batch, only update rows that actually need a change (e.g., where the target +column is still NULL or differs from the expected value). This avoids unnecessary writes +and accelerates migration progress on re-runs. + +### What the application code must handle + +While the migration is in progress, some rows have the new column populated and some don't. +The application code must tolerate both states: + +```go +// In your datastore/query code, handle the zero value gracefully: +if row.NewColumn != "" { + // Use the backfilled value +} else { + // Fall back to parsing from serialized blob or using a default +} +``` + +The application code must also populate the new column on INSERT and UPDATE so that +new rows written by Central are already in the post-migration format. + +When using stores this is handled by the code generation automatically. + +## Example 1: Backfill a column from a serialized proto + +This example adds a `signal_hostname` column to `process_indicators` by extracting it from +the serialized protobuf blob. The table has millions of rows in production. + +### Prerequisites + +1. The proto field already exists in `storage.ProcessIndicator` +2. The column is already added to the GORM schema (GORM AutoMigrate creates it on startup) +3. The application datastore code already populates `signal_hostname` on new inserts and updates + +### Migration implementation + +```go +package m000tom001 + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/stackrox/rox/central/backgroundmigrations/migrations" + "github.com/stackrox/rox/central/backgroundmigrations/types" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/postgres" +) + +var ( + log = logging.LoggerForModule() + batchSize = 5000 +) + +func init() { + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, + VersionAfterSeqNum: 1, + Description: "Backfill signal_hostname on process_indicators from serialized proto", + Run: run, + }) +} + +type indicatorRow struct { + id string + serialized []byte + signalHostname *string +} + +func run(ctx context.Context, db postgres.DB) error { + totalUpdated := 0 + lastID := "" + + for { + if ctx.Err() != nil { + log.Infof("shutdown requested after %d rows, will restart on next start", totalUpdated) + return ctx.Err() + } + + rowsUpdated, newLastID, err := migrateBatch(ctx, db, lastID) + if err != nil { + return fmt.Errorf("batch after id %s: %w", lastID, err) + } + if newLastID == "" { + break + } + + lastID = newLastID + totalUpdated += rowsUpdated + } + + log.Infof("backfill complete: %d rows updated", totalUpdated) + return nil +} + +// migrateBatch runs one batch inside an explicit transaction so that the +// FOR UPDATE SKIP LOCKED row locks are held through the UPDATE statements. +func migrateBatch(ctx context.Context, db postgres.DB, lastID string) (int, string, error) { + conn, err := db.Acquire(ctx) + if err != nil { + return 0, "", fmt.Errorf("acquiring connection: %w", err) + } + defer conn.Release() + + tx, err := conn.Begin(ctx) + if err != nil { + return 0, "", fmt.Errorf("begin tx: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + rows, err := fetchBatch(ctx, tx, lastID) + if err != nil { + return 0, "", err + } + if len(rows) == 0 { + return 0, "", nil + } + + batch, err := buildBatchUpdates(rows) + if err != nil { + return 0, "", err + } + + if batch.Len() > 0 { + if err := sendBatch(ctx, tx, batch); err != nil { + return 0, "", err + } + } + + if err := tx.Commit(ctx); err != nil { + return 0, "", fmt.Errorf("commit: %w", err) + } + + return batch.Len(), rows[len(rows)-1].id, nil +} + +func fetchBatch(ctx context.Context, tx pgx.Tx, lastID string) ([]indicatorRow, error) { + rows, err := tx.Query(ctx, ` + SELECT id, serialized, signal_hostname + FROM process_indicators + WHERE id > $1 + ORDER BY id + LIMIT $2 + FOR UPDATE SKIP LOCKED`, lastID, batchSize) + if err != nil { + return nil, fmt.Errorf("querying batch: %w", err) + } + defer rows.Close() + + result := make([]indicatorRow, 0, batchSize) + for rows.Next() { + var r indicatorRow + if err := rows.Scan(&r.id, &r.serialized, &r.signalHostname); err != nil { + return nil, fmt.Errorf("scanning row: %w", err) + } + result = append(result, r) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating rows: %w", err) + } + return result, nil +} + +func buildBatchUpdates(rows []indicatorRow) (*pgx.Batch, error) { + batch := &pgx.Batch{} + for _, r := range rows { + indicator := &storage.ProcessIndicator{} + if err := indicator.UnmarshalVT(r.serialized); err != nil { + return nil, fmt.Errorf("unmarshal row %s: %w", r.id, err) + } + + hostname := indicator.GetSignal().GetHostname() + + // Only write if the column value differs from what the serialized + // blob says it should be. The serialized blob is the source of truth + // and may have changed (e.g. after a rollback), so we cannot skip + // rows just because the column is non-NULL. + if r.signalHostname != nil && *r.signalHostname == hostname { + continue + } + + batch.Queue( + "UPDATE process_indicators SET signal_hostname = $1 WHERE id = $2", + hostname, r.id, + ) + } + return batch, nil +} + +func sendBatch(ctx context.Context, tx pgx.Tx, batch *pgx.Batch) error { + results := tx.SendBatch(ctx, batch) + for i := 0; i < batch.Len(); i++ { + if _, err := results.Exec(); err != nil { + _ = results.Close() + return fmt.Errorf("batch exec statement %d: %w", i, err) + } + } + return results.Close() +} +``` + +### Key design decisions + +| Decision | Rationale | +|---|---| +| Explicit transaction per batch | Ensures `FOR UPDATE` locks are held through the UPDATE statements. | +| `FOR UPDATE SKIP LOCKED` | Locks rows to prevent conflicts with Central. Skips rows currently locked by Central. | +| Primary key iteration (`id > $1 ORDER BY id`) | Uses the primary key index for efficient pagination. | +| Skip rows where column already matches serialized blob | Avoids unnecessary writes. | +| `ctx.Err()` check per batch | Allows graceful shutdown. | +| `pgx.Batch` for writes | Sends all updates in a single round-trip, reducing network overhead. | + +## Example 2: Batched SQL JOIN backfill + +When the value you need exists in another table (no deserialization required), but the +target table is too large for a single `UPDATE ... FROM` (which locks all affected rows +for the full statement duration). + +```go +func run(ctx context.Context, db postgres.DB) error { + conn, err := db.Acquire(ctx) + if err != nil { + return fmt.Errorf("acquiring connection: %w", err) + } + defer conn.Release() + + totalUpdated := 0 + lastID := "" + + for { + if ctx.Err() != nil { + log.Infof("shutdown requested after %d rows, will resume on next start", totalUpdated) + return ctx.Err() + } + + updated, newLastID, err := backfillBatch(ctx, conn, lastID) + if err != nil { + return fmt.Errorf("batch after id %s: %w", lastID, err) + } + if newLastID == "" { + break + } + + lastID = newLastID + totalUpdated += updated + } + + log.Infof("backfill complete: %d rows updated", totalUpdated) + return nil +} + +func backfillBatch(ctx context.Context, conn *postgres.Conn, lastID string) (int, string, error) { + // Iterate by primary key, join to get the value, only update rows that need it. + // FOR UPDATE SKIP LOCKED on the CTE prevents conflicts with Central. + tag, err := conn.Exec(ctx, ` + WITH batch AS ( + SELECT a.id + FROM alerts a + WHERE a.id > $1 + ORDER BY a.id + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + UPDATE alerts a + SET deployment_type = d.type + FROM batch + JOIN deployments d ON a.deployment_id = d.id + WHERE a.id = batch.id + AND (a.deployment_type IS NULL OR a.deployment_type != d.type)`, + lastID, batchSize) + if err != nil { + return 0, "", fmt.Errorf("updating batch: %w", err) + } + + // Get the last ID in the batch for keyset pagination. + var newLastID string + err = conn.QueryRow(ctx, ` + SELECT id FROM alerts + WHERE id > $1 + ORDER BY id + LIMIT 1 OFFSET $2 - 1`, lastID, batchSize).Scan(&newLastID) + if err != nil { + // No more rows to process. + return int(tag.RowsAffected()), "", nil + } + + return int(tag.RowsAffected()), newLastID, nil +} +``` + +This avoids holding locks on the entire table — each batch locks only its subset of rows +and commits before the next batch begins. + +## Operations + +### Skipping a migration + +To skip specific background migrations (e.g., during an emergency), set the environment +variable on the Central deployment: + +```bash +ROX_SKIP_BACKGROUND_MIGRATIONS=0,1 # comma-separated seq nums to skip +``` + +The runner will log that the migration was skipped and advance the seq num past it. +Remove the env var after the situation is resolved. + +### Forcing a re-run (override) + +To force background migrations to re-run from a specific sequence number: + +```bash +ROX_BACKGROUND_MIGRATION_OVERRIDE_SEQ_NUM=0 # start from this seq num +ROX_BACKGROUND_MIGRATION_OVERRIDE_TAG=rerun-v1 # unique tag for this override +``` + +Both env vars must be set. The tag is persisted to the DB to prevent re-applying the same +override on subsequent pod restarts. To trigger another re-run, change the tag value. + +This is useful when: +- A migration completed but produced incorrect results that were fixed in a code update +- A rollback to a version unaware of background migration occured, since that version does not automatically reset the background migrations sequence number +- Debugging a migration in a test environment + +### Monitoring + +Two Prometheus metrics track background migration progress: + +| Metric | Description | +|---|---| +| `rox_central_background_migration_seq_num` | Current completed seq num | +| `rox_central_background_migration_complete` | `1` when all migrations are done, `0` otherwise | + +### Rollback behavior + +When Central is rolled back to a previous version: + +1. The old binary has a lower `CurrentBgMigrationSeqNum` +2. The runner detects `dbSeqNum > targetSeqNum` (rollback) +3. The runner resets the background DB seq num to the target +4. On the next roll-forward, background migrations re-run from the reset point + +This is why idempotency is critical — migrations will be re-executed after rollback + upgrade. + +## Feature flag + +Background migrations are gated by the feature flag `ROX_BACKGROUND_MIGRATION` (enabled by +default), defined in `pkg/features/list.go`. When disabled, no background migrations run. +The flag is checked in `central/main.go` before starting the runner. + +## Testing + +Use the same test patterns as startup migrations (see +[STARTUP_MIGRATIONS.md](../../migrator/STARTUP_MIGRATIONS.md#testing)). Tests require +the `//go:build sql_integration` tag and a running PostgreSQL instance. + +### What to test + +- **Backfill correctness**: new column gets the right value from the serialized blob +- **Idempotency**: running twice produces the same result +- **Conflict-free with existing data**: rows already populated by Central are not overwritten +- **Graceful shutdown**: cancellation mid-migration doesn't corrupt data; re-run completes +- **Batch boundaries**: test with small batch sizes to exercise pagination logic + +### Performance testing on a long-running cluster + +Background migration runtime is not a primary concern since they run after +Central is serving. However, they must not starve Central of CPU or memory, and +must not cause SQL conflicts or deadlocks with concurrent Central operations. + +**For migrations touching [high-cardinality tables](../../migrator/README.md#high-cardinality-tables-always-use-background-migrations), +testing with a long running cluster or a similar real load environment is mandatory.** + + + +For the testing process: + +1. **Set up a long-running cluster under load**: Deploy Central with active + sensor connections and realistic workload (events, API calls, policy + evaluations). The cluster should be exercising the tables the migration + touches. +2. **Populate target tables**: If the cluster's organic data is insufficient, + generate synthetic rows to reach representative volumes (500k–1M rows) in + the target table before triggering the migration. +3. **Monitor during migration**: + - **Logs**: Watch for SQL conflicts, deadlocks, lock wait timeouts, or + retry storms. Any `deadlock detected` or `could not serialize access` + errors indicate the batch strategy needs adjustment. + - **Metrics**: Track Central's memory RSS, heap usage, and CPU utilization. + The migration must not cause memory spikes that could trigger OOM kills + or CPU saturation that degrades API latency. + - **Central responsiveness**: Verify that API response times and sensor + connections remain healthy throughout the migration. +4. **Tune batch size**: Adjust `batchSize` to balance throughput against + resource impact. Smaller batches reduce peak memory, shorten lock hold + times, and lower the chance of conflicts with Central. Larger batches + improve throughput. Find the sweet spot where Central remains responsive + and no conflicts or deadlocks appear in the logs. + +**The simplest way to set up such environment to run the [GHA](https://github.com/stackrox/stackrox/actions/workflows/create-clusters.yml) at a stable release (e.g 4.11.0), then update central to a PR image.** +Read this [section](https://redhat.atlassian.net/wiki/spaces/StackRox/pages/309338714/Long+Running+Clusters) of the release docs for more information. + +## Checklist + +- [ ] Migration is idempotent (safe to re-run after rollback or crash) +- [ ] `ctx.Done()` is checked between batches for graceful shutdown +- [ ] Iterates by primary key (not by `WHERE new_column IS NULL`) for efficient pagination +- [ ] Uses `FOR UPDATE SKIP LOCKED` to handle concurrent Central writes +- [ ] Only updates rows where the column value differs from the serialized source of truth +- [ ] Application code already populates the new column on INSERT/UPDATE +- [ ] Application code tolerates partial migration state (some rows backfilled, some not) +- [ ] Registered in `central/backgroundmigrations/runner/all.go` +- [ ] `CurrentBgMigrationSeqNum` incremented in `central/backgroundmigrations/seq_num.go` +- [ ] Tests cover correctness, idempotency, existing data, and graceful shutdown +- [ ] Long-running cluster test for high-cardinality tables (no deadlocks, stable memory/CPU) +- [ ] No feature flag dependencies in migration code diff --git a/central/backgroundmigrations/migrations/example/migration.go b/central/backgroundmigrations/migrations/example/migration.go new file mode 100644 index 0000000000000..b9596891ccfcf --- /dev/null +++ b/central/backgroundmigrations/migrations/example/migration.go @@ -0,0 +1,27 @@ +package example + +import ( + "context" + + "github.com/stackrox/rox/central/backgroundmigrations/migrations" + "github.com/stackrox/rox/central/backgroundmigrations/types" + "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/postgres" +) + +var log = logging.LoggerForModule() + +// You must import a migration in ../../runner/all.go for this to be executed +func init() { + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, + VersionAfterSeqNum: 1, + Description: "Test migration that logs a message", + Run: func(ctx context.Context, db postgres.DB) error { + log.Info("Background migration m_000_to_m_001: starting") + log.Info("Background migration m_000_to_m_001: this is an example migration that simply logs") + log.Info("Background migration m_000_to_m_001: completed successfully") + return nil + }, + }) +} diff --git a/central/backgroundmigrations/migrations/registry.go b/central/backgroundmigrations/migrations/registry.go new file mode 100644 index 0000000000000..1bd75245b7e27 --- /dev/null +++ b/central/backgroundmigrations/migrations/registry.go @@ -0,0 +1,32 @@ +package migrations + +import ( + "fmt" + "testing" + + "github.com/stackrox/rox/central/backgroundmigrations/types" +) + +var migrations map[int]types.BackgroundMigration = make(map[int]types.BackgroundMigration) + +// MustRegister adds a background migration to the registry. It panics on error. +func MustRegister(m types.BackgroundMigration) { + if m.VersionAfterSeqNum != m.StartingSeqNum+1 { + panic(fmt.Sprintf("Background Migration at seq num %d has VersionAfterSeqNum %d, expected %d", m.StartingSeqNum, m.VersionAfterSeqNum, m.StartingSeqNum+1)) + } + if _, ok := migrations[m.StartingSeqNum]; ok { + panic(fmt.Sprintf("Found multiple background migrations starting at seq num %d", m.StartingSeqNum)) + } + migrations[m.StartingSeqNum] = m +} + +// Get the BackgroundMigration at the given sequence number. +func Get(startingSeqNum int) (types.BackgroundMigration, bool) { + m, ok := migrations[startingSeqNum] + return m, ok +} + +// ResetRegistryForTesting clears all registered migrations. Test use only. +func ResetRegistryForTesting(t *testing.T) { + migrations = make(map[int]types.BackgroundMigration) +} diff --git a/central/backgroundmigrations/migrations/registry_test.go b/central/backgroundmigrations/migrations/registry_test.go new file mode 100644 index 0000000000000..6d30f2a78f9a8 --- /dev/null +++ b/central/backgroundmigrations/migrations/registry_test.go @@ -0,0 +1,33 @@ +package migrations + +import ( + "testing" + + "github.com/stackrox/rox/central/backgroundmigrations/types" + "github.com/stretchr/testify/assert" +) + +func TestRegistryPanicsOnDuplicate(t *testing.T) { + ResetRegistryForTesting(t) + MustRegister(types.BackgroundMigration{StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "first"}) + assert.Panics(t, func() { + MustRegister(types.BackgroundMigration{StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "duplicate"}) + }) +} + +func TestRegistryGet(t *testing.T) { + ResetRegistryForTesting(t) + MustRegister(types.BackgroundMigration{StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "m0"}) + MustRegister(types.BackgroundMigration{StartingSeqNum: 1, VersionAfterSeqNum: 2, Description: "m1"}) + + m, ok := Get(0) + assert.True(t, ok) + assert.Equal(t, "m0", m.Description) + + m, ok = Get(1) + assert.True(t, ok) + assert.Equal(t, "m1", m.Description) + + _, ok = Get(99) + assert.False(t, ok) +} diff --git a/central/backgroundmigrations/runner/all.go b/central/backgroundmigrations/runner/all.go new file mode 100644 index 0000000000000..bfd204fa2cede --- /dev/null +++ b/central/backgroundmigrations/runner/all.go @@ -0,0 +1,6 @@ +package runner + +// Import background migration packages here to register them via init(). +// import ( +// _ "github.com/stackrox/rox/central/backgroundmigrations/example" +// ) diff --git a/central/backgroundmigrations/runner/metrics.go b/central/backgroundmigrations/runner/metrics.go new file mode 100644 index 0000000000000..c8937ce3788f5 --- /dev/null +++ b/central/backgroundmigrations/runner/metrics.go @@ -0,0 +1,29 @@ +package runner + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/stackrox/rox/pkg/metrics" +) + +var ( + bgMigrationSeqNumGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "background_migration_seq_num", + Help: "Current sequence number of completed background migrations", + }) + + bgMigrationCompleteGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "background_migration_complete", + Help: "1 if all background migrations have finished, 0 otherwise", + }) +) + +func init() { + prometheus.MustRegister( + bgMigrationSeqNumGauge, + bgMigrationCompleteGauge, + ) +} diff --git a/central/backgroundmigrations/runner/rollout.go b/central/backgroundmigrations/runner/rollout.go new file mode 100644 index 0000000000000..47f9cf820bcbd --- /dev/null +++ b/central/backgroundmigrations/runner/rollout.go @@ -0,0 +1,123 @@ +package runner + +import ( + "context" + "fmt" + + "github.com/stackrox/rox/pkg/env" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const deploymentName = "central" + +// RolloutChecker checks whether the Central deployment rollout is complete. +type RolloutChecker interface { + IsRolloutDone(ctx context.Context) (bool, error) +} + +type k8sRolloutChecker struct { + client kubernetes.Interface + inCluster bool +} + +// NewCentralRolloutChecker creates a RolloutChecker that queries the K8s API. +func NewCentralRolloutChecker() RolloutChecker { + cfg, err := rest.InClusterConfig() + rc := &k8sRolloutChecker{inCluster: true} + + if err != nil { + log.Warnf("failed to get in cluster kubernetes config, assuming not running in a kubernetes cluster: %v", err) + rc.inCluster = false + return rc + } + + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + log.Warnf("failed to create in cluster K8s client, assuming not running in a kubernetes cluster: %v", err) + rc.inCluster = false + return rc + } + + rc.client = client + + return rc +} + +// IsRolloutDone checks whether the Central deployment rollout is complete. +func (c *k8sRolloutChecker) IsRolloutDone(ctx context.Context) (bool, error) { + if !c.inCluster { + return true, nil + } + + namespace := env.Namespace.Setting() + deployment, err := c.client.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("error getting deployment %s/%s: %w", namespace, deploymentName, err) + } + + if done := c.checkRolloutStatus(deployment); !done { + return false, nil + } + + terminatingPods, err := c.checkTerminatingPods(ctx, deployment) + if err != nil { + return false, err + } + + if len(terminatingPods) > 0 { + log.Infof("deployment %s/%s rollout complete but pods still terminating: %v", + namespace, deploymentName, terminatingPods) + return false, nil + } + + log.Infof("deployment %s/%s rollout complete, no terminating pods", namespace, deploymentName) + return true, nil +} + +func (c *k8sRolloutChecker) checkRolloutStatus(deployment *appsv1.Deployment) bool { + replicas := int32(1) + if deployment.Spec.Replicas != nil { + replicas = *deployment.Spec.Replicas + } + if deployment.Status.UpdatedReplicas != replicas || + deployment.Status.AvailableReplicas != replicas || + deployment.Status.ObservedGeneration < deployment.Generation { + namespace := env.Namespace.Setting() + log.Infof("deployment %s/%s rollout in progress (updated=%d, available=%d, desired=%d)", + namespace, deploymentName, + deployment.Status.UpdatedReplicas, + deployment.Status.AvailableReplicas, + replicas) + return false + } + return true +} + +// checkTerminatingPods queries for terminating pods matching the deployment selector +// TODO: once the min. k8s version we support is 1.35 we can use deployment.Status.TerminatingReplicas instead +func (c *k8sRolloutChecker) checkTerminatingPods(ctx context.Context, deployment *appsv1.Deployment) ([]string, error) { + namespace := env.Namespace.Setting() + + selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector) + if err != nil { + return nil, fmt.Errorf("error parsing deployment selector: %w", err) + } + + pods, err := c.client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector.String(), + }) + if err != nil { + return nil, fmt.Errorf("error listing pods for deployment %s/%s: %w", namespace, deploymentName, err) + } + + var terminating []string + for _, pod := range pods.Items { + if pod.DeletionTimestamp != nil { + terminating = append(terminating, pod.Name) + } + } + return terminating, nil +} diff --git a/central/backgroundmigrations/runner/rollout_test.go b/central/backgroundmigrations/runner/rollout_test.go new file mode 100644 index 0000000000000..3ffc276c231a4 --- /dev/null +++ b/central/backgroundmigrations/runner/rollout_test.go @@ -0,0 +1,123 @@ +package runner + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +const ( + testNamespace = "stackrox" +) + +func newTestRolloutChecker(client *fake.Clientset) *k8sRolloutChecker { + return &k8sRolloutChecker{ + client: client, + inCluster: true, + } +} + +func readyDeployment() *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentName, + Namespace: testNamespace, + Generation: 1, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: new(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "central"}, + }, + }, + Status: appsv1.DeploymentStatus{ + UpdatedReplicas: 1, + AvailableReplicas: 1, + ObservedGeneration: 1, + }, + } +} + +func TestRolloutDone(t *testing.T) { + client := fake.NewSimpleClientset(readyDeployment()) + checker := newTestRolloutChecker(client) + + done, err := checker.IsRolloutDone(context.Background()) + assert.NoError(t, err) + assert.True(t, done) +} + +func TestRolloutNotInCluster(t *testing.T) { + checker := &k8sRolloutChecker{ + inCluster: false, + } + + done, err := checker.IsRolloutDone(context.Background()) + assert.NoError(t, err) + assert.True(t, done) +} + +func TestRolloutNotDoneWhenReplicasNotReady(t *testing.T) { + dep := readyDeployment() + dep.Status.UpdatedReplicas = 0 + dep.Status.AvailableReplicas = 0 + + client := fake.NewSimpleClientset(dep) + checker := newTestRolloutChecker(client) + + done, err := checker.IsRolloutDone(context.Background()) + assert.NoError(t, err) + assert.False(t, done) +} + +func TestRolloutNotDoneWhenObservedGenerationBehind(t *testing.T) { + dep := readyDeployment() + dep.Generation = 2 + dep.Status.ObservedGeneration = 1 + + client := fake.NewSimpleClientset(dep) + checker := newTestRolloutChecker(client) + + done, err := checker.IsRolloutDone(context.Background()) + assert.NoError(t, err) + assert.False(t, done) +} + +func TestRolloutNotDoneWithTerminatingPods(t *testing.T) { + dep := readyDeployment() + now := metav1.Now() + terminatingPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "central-old", + Namespace: testNamespace, + Labels: map[string]string{"app": "central"}, + DeletionTimestamp: &now, + }, + } + + client := fake.NewSimpleClientset(dep, terminatingPod) + checker := newTestRolloutChecker(client) + + done, err := checker.IsRolloutDone(context.Background()) + assert.NoError(t, err) + assert.False(t, done) +} + +func TestRolloutCheckDefaultsToOneReplica(t *testing.T) { + dep := readyDeployment() + dep.Spec.Replicas = nil + dep.Status.UpdatedReplicas = 1 + dep.Status.AvailableReplicas = 1 + + client := fake.NewSimpleClientset(dep) + checker := newTestRolloutChecker(client) + + done, err := checker.IsRolloutDone(context.Background()) + assert.NoError(t, err) + assert.True(t, done) +} diff --git a/central/backgroundmigrations/runner/runner.go b/central/backgroundmigrations/runner/runner.go new file mode 100644 index 0000000000000..69a1ee701414e --- /dev/null +++ b/central/backgroundmigrations/runner/runner.go @@ -0,0 +1,314 @@ +package runner + +import ( + "context" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/jackc/pgx/v5" + "github.com/pkg/errors" + "github.com/stackrox/rox/central/backgroundmigrations" + "github.com/stackrox/rox/central/backgroundmigrations/migrations" + "github.com/stackrox/rox/central/globaldb" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/dblock" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/schema" + "github.com/stackrox/rox/pkg/set" + "github.com/stackrox/rox/pkg/sync" +) + +var ( + once sync.Once + instance *Runner +) + +var log = logging.LoggerForModule() + +// Singleton returns the singleton Runner instance. +func Singleton() *Runner { + once.Do(func() { + instance = NewRunner(globaldb.GetPostgres(), NewCentralRolloutChecker()) + }) + return instance +} + +const ( + // bgMigrationAdvisoryLockID is a unique identifier for the background migration advisory lock. + // This value is arbitrary but must be consistent across all Central instances and must + // differ from the migrator advisory lock ID. + bgMigrationAdvisoryLockID int64 = 2_846_193_750_482_637_519 + + retryInterval = 60 * time.Second +) + +// Runner executes background migrations after Central is ready. +type Runner struct { + db postgres.DB + rolloutChecker RolloutChecker + stopper concurrency.Stopper + started atomic.Bool + targetSeqNum int + retryInterval time.Duration + skipMigrations set.IntSet +} + +// NewRunner creates a new Runner. +func NewRunner(db postgres.DB, rolloutChecker RolloutChecker) *Runner { + return &Runner{ + db: db, + rolloutChecker: rolloutChecker, + stopper: concurrency.NewStopper(), + targetSeqNum: backgroundmigrations.CurrentBgMigrationSeqNum, + retryInterval: retryInterval, + skipMigrations: parseSkipMigrations(), + } +} + +// Start launches the background migration goroutine. +func (r *Runner) Start() { + if !r.started.CompareAndSwap(false, true) { + return + } + + go r.run() +} + +// Stop requests graceful shutdown and waits for the runner to finish. +func (r *Runner) Stop() { + r.stopper.Client().Stop() + if r.started.Load() { + _ = r.stopper.Client().Stopped().Wait() + } +} + +func (r *Runner) run() { + defer r.stopper.Flow().ReportStopped() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + select { + case <-r.stopper.Flow().StopRequested(): + cancel() + case <-ctx.Done(): + } + }() + + for { + err := r.runOnce(ctx) + if err == nil { + return + } + + log.Errorf("background migrations failed, retrying in %v: %v", r.retryInterval, err) + select { + case <-ctx.Done(): + log.Infof("background migrations stopped") + return + case <-time.After(r.retryInterval): + } + } +} + +func (r *Runner) runOnce(ctx context.Context) error { + done, err := r.rolloutChecker.IsRolloutDone(ctx) + if err != nil { + return errors.Wrap(err, "rollout check") + } + if !done { + return errors.New("rollout not yet complete") + } + + release, err := r.acquireLock(ctx) + if err != nil { + return errors.Wrap(err, "acquiring lock") + } + defer release() + + if err := r.runMigrations(ctx); err != nil { + return errors.Wrap(err, "running background migrations") + } + + return nil +} + +// acquireLock attempts to acquire the advisory lock once. +// Returns (release, nil) on success, or an error if the lock could not be acquired. +func (r *Runner) acquireLock(ctx context.Context) (func(), error) { + acquired, release, err := dblock.TryAcquireAdvisoryLock(ctx, r.db, bgMigrationAdvisoryLockID) + if err != nil { + return nil, errors.Wrap(err, "acquiring advisory lock") + } + if !acquired { + return nil, errors.New("advisory lock held by another instance") + } + return release, nil +} + +func (r *Runner) runMigrations(ctx context.Context) error { + dbSeqNum, dbOverrideTag, err := r.readState(ctx) + if err != nil { + return errors.Wrap(err, "reading current state") + } + bgMigrationSeqNumGauge.Set(float64(dbSeqNum)) + + overrideSeqNum, overrideTag, shouldOverride := r.checkSeqNumOverrideConfig(r.targetSeqNum, dbOverrideTag) + if shouldOverride { + log.Infof("applying override tag %q, resetting seq num from %d to %d", overrideTag, dbSeqNum, overrideSeqNum) + if err := r.writeState(ctx, overrideSeqNum, overrideTag); err != nil { + return errors.Wrap(err, "writing override state") + } + dbSeqNum = overrideSeqNum + } + + if !shouldOverride && dbOverrideTag != "" && overrideTag == "" { + // reset old override tags if it exists + log.Infof("override env var removed, clearing stale override tag %q from DB", dbOverrideTag) + if err := r.writeOverrideTag(ctx, ""); err != nil { + return errors.Wrap(err, "clearing override tag") + } + } + + if dbSeqNum > r.targetSeqNum { + log.Warnf("rollback detected (db=%d, current=%d). Resetting to current seq num.", dbSeqNum, r.targetSeqNum) + if err := r.writeSeqNum(ctx, r.targetSeqNum); err != nil { + return errors.Wrap(err, "resetting seq num after rollback") + } + dbSeqNum = r.targetSeqNum + } + + // apply potential changes from overrides or rollbacks to the metric + bgMigrationSeqNumGauge.Set(float64(dbSeqNum)) + + if dbSeqNum == r.targetSeqNum { + log.Infof("up to date at seq num %d, reconciling background indexes...", dbSeqNum) + if err := schema.ApplyAllIndexes(ctx, r.db, env.BackgroundIndexTimeout.DurationSetting()); err != nil { + return errors.Wrap(err, "creating background indexes") + } + bgMigrationCompleteGauge.Set(1) + return nil + } + + log.Infof("running migrations from %d to %d", dbSeqNum, r.targetSeqNum) + + for seqNum := dbSeqNum; seqNum < r.targetSeqNum; seqNum++ { + if ctx.Err() != nil { + return ctx.Err() + } + + migration, ok := migrations.Get(seqNum) + if !ok { + return errors.Errorf("no migration found starting at %d", seqNum) + } + + if r.skipMigrations.Contains(seqNum) { + log.Infof("skipping migration %d based on %s", seqNum, env.SkipBackgroundMigrations.EnvVar()) + if err := r.writeSeqNum(ctx, migration.VersionAfterSeqNum); err != nil { + return errors.Wrapf(err, "updating seq num to %d after skipping migration %d", migration.VersionAfterSeqNum, seqNum) + } + bgMigrationSeqNumGauge.Set(float64(migration.VersionAfterSeqNum)) + continue + } + + log.Infof("running migration %d: %s", seqNum, migration.Description) + + if err := migration.Run(ctx, r.db); err != nil { + return errors.Wrapf(err, "migration %d failed", seqNum) + } + + if err := r.writeSeqNum(ctx, migration.VersionAfterSeqNum); err != nil { + return errors.Wrapf(err, "updating seq num to %d", migration.VersionAfterSeqNum) + } + + bgMigrationSeqNumGauge.Set(float64(migration.VersionAfterSeqNum)) + + log.Infof("completed migration %d, now at seq num %d", seqNum, migration.VersionAfterSeqNum) + } + + log.Infof("all numbered migrations complete, at seq num %d. Reconciling background indexes...", r.targetSeqNum) + + if err := schema.ApplyAllIndexes(ctx, r.db, env.BackgroundIndexTimeout.DurationSetting()); err != nil { + return errors.Wrap(err, "creating background indexes") + } + + bgMigrationCompleteGauge.Set(1) + + log.Infof("background migrations and index reconciliation complete") + return nil +} + +func (r *Runner) readState(ctx context.Context) (int, string, error) { + row := r.db.QueryRow(ctx, "SELECT seqnum, override_tag FROM "+schema.BackgroundMigrationVersionsTableName+" LIMIT 1") + var seqNum int32 + var overrideTag string + if err := row.Scan(&seqNum, &overrideTag); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + if err := r.seedInitialRow(ctx); err != nil { + return 0, "", errors.Wrap(err, "seeding initial row") + } + return 0, "", nil + } + return 0, "", err + } + return int(seqNum), overrideTag, nil +} + +func (r *Runner) seedInitialRow(ctx context.Context) error { + _, err := r.db.Exec(ctx, "INSERT INTO "+schema.BackgroundMigrationVersionsTableName+" (seqnum, override_tag) VALUES (0, '')") + return err +} + +func (r *Runner) writeSeqNum(ctx context.Context, seqNum int) error { + _, err := r.db.Exec(ctx, "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = $1", int32(seqNum)) + return err +} + +func (r *Runner) writeOverrideTag(ctx context.Context, overrideTag string) error { + _, err := r.db.Exec(ctx, "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET override_tag = $1", overrideTag) + return err +} + +func (r *Runner) writeState(ctx context.Context, seqNum int, overrideTag string) error { + _, err := r.db.Exec(ctx, "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = $1, override_tag = $2", int32(seqNum), overrideTag) + return err +} + +// checkSeqNumOverrideConfig checks env configuration for sequence number overrides and applies them. +// returns the configuration and whether it needs to be applied +func (r *Runner) checkSeqNumOverrideConfig(currSeqNum int, dbOverrideTag string) (seqNum int, tag string, shouldOverride bool) { + seqNum = env.BackgroundMigrationOverrideSeqNum.IntegerSetting() + tag = env.BackgroundMigrationOverrideTag.Setting() + + if seqNum < 0 || tag == "" || tag == dbOverrideTag { + return seqNum, tag, shouldOverride + } + + if seqNum >= currSeqNum { + log.Infof("override background seq num %d is greater or equal current seq num %d, ignoring override", seqNum, currSeqNum) + return seqNum, tag, shouldOverride + } + + return seqNum, tag, true +} + +func parseSkipMigrations() set.IntSet { + val := env.SkipBackgroundMigrations.Setting() + if val == "" { + return set.NewIntSet() + } + s := set.NewIntSet() + for entry := range strings.SplitSeq(val, ",") { + n, err := strconv.Atoi(strings.TrimSpace(entry)) + if err != nil { + log.Errorf("could not parse %q from %s, not skipping", entry, env.SkipBackgroundMigrations.EnvVar()) + continue + } + s.Add(n) + } + return s +} diff --git a/central/backgroundmigrations/runner/runner_test.go b/central/backgroundmigrations/runner/runner_test.go new file mode 100644 index 0000000000000..3da2d3e65d5e1 --- /dev/null +++ b/central/backgroundmigrations/runner/runner_test.go @@ -0,0 +1,611 @@ +//go:build sql_integration + +package runner + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stackrox/rox/central/backgroundmigrations/migrations" + "github.com/stackrox/rox/central/backgroundmigrations/types" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/postgres/pgtest/conn" + "github.com/stackrox/rox/pkg/postgres/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const testTimeout = 1 * time.Second + +type doneRolloutChecker struct{} + +func (n *doneRolloutChecker) IsRolloutDone(_ context.Context) (bool, error) { return true, nil } + +type notDoneRolloutChecker struct{} + +func (c *notDoneRolloutChecker) IsRolloutDone(_ context.Context) (bool, error) { return false, nil } + +// countingRolloutChecker returns not-done for the first N calls, then done. +type countingRolloutChecker struct { + notDoneCount int + callCount int +} + +func (c *countingRolloutChecker) IsRolloutDone(_ context.Context) (bool, error) { + c.callCount++ + if c.callCount <= c.notDoneCount { + return false, errors.New("transient rollout check error") + } + return true, nil +} + +var _ RolloutChecker = &doneRolloutChecker{} +var _ RolloutChecker = &k8sRolloutChecker{} +var _ RolloutChecker = ¬DoneRolloutChecker{} +var _ RolloutChecker = &countingRolloutChecker{} + +type RunnerTestSuite struct { + suite.Suite + db postgres.DB + ctx context.Context +} + +func TestRunnerSuite(t *testing.T) { + suite.Run(t, new(RunnerTestSuite)) +} + +func (s *RunnerTestSuite) SetupSuite() { + s.ctx = context.Background() + + // Create a test database with a pool that supports multiple connections. + // The advisory lock holds one connection, so we need at least 2. + database := pgtest.CreateADatabaseForT(s.T()) + source := conn.GetConnectionStringWithDatabaseName(s.T(), database) + config, err := postgres.ParseConfig(source) + s.Require().NoError(err) + + pool, err := postgres.New(s.ctx, config) + s.Require().NoError(err) + s.T().Cleanup(func() { + pool.Close() + pgtest.DropDatabase(s.T(), database) + }) + s.db = pool + + gormDB := conn.OpenGormDB(s.T(), source, false) + schema.ApplyAllSchemas(s.ctx, gormDB) +} + +// waitForAdvisoryUnlock ensures no pool connection holds the background migration +// advisory lock. Tests that use Start()/Stop() may leave the lock briefly held +// due to connection pool timing, causing the next test to fail with +// "advisory lock held by another instance". +func (s *RunnerTestSuite) waitForAdvisoryUnlock() { + require.EventuallyWithT(s.T(), func(t *assert.CollectT) { + dbConn, err := s.db.Acquire(s.ctx) + require.NoError(t, err) + require.NotNil(t, dbConn) + + defer dbConn.Release() + var acquired bool + err = dbConn.QueryRow(s.ctx, "SELECT pg_try_advisory_lock($1)", bgMigrationAdvisoryLockID).Scan(&acquired) + require.NoError(t, err) + require.True(t, acquired) + + _, err = dbConn.Exec(s.ctx, "SELECT pg_advisory_unlock($1)", bgMigrationAdvisoryLockID) + assert.NoError(t, err) + }, time.Second, 50*time.Millisecond, "advisory lock still held") +} + +func (s *RunnerTestSuite) SetupTest() { + migrations.ResetRegistryForTesting(s.T()) + + s.waitForAdvisoryUnlock() + + _, err := s.db.Exec(s.ctx, + "DELETE FROM "+schema.BackgroundMigrationVersionsTableName) + s.Require().NoError(err) + _, err = s.db.Exec(s.ctx, + "INSERT INTO "+schema.BackgroundMigrationVersionsTableName+" (seqnum, override_tag) VALUES (0, '')") + s.Require().NoError(err) +} + +func (s *RunnerTestSuite) newRunner(rolloutChecker RolloutChecker, targetSeqNum int) *Runner { + r := NewRunner(s.db, rolloutChecker) + r.targetSeqNum = targetSeqNum + r.retryInterval = 10 * time.Millisecond + return r +} + +// requireStoppedWithin starts the runner and fails the test if it doesn't stop within the timeout. +func requireStoppedWithin(t *testing.T, runner *Runner, timeout time.Duration) { + t.Helper() + done := make(chan struct{}) + go func() { + _ = runner.stopper.Client().Stopped().Wait() + close(done) + }() + runner.Start() + select { + case <-done: + case <-time.After(timeout): + t.Fatal("runner did not stop within timeout") + } +} + +func (s *RunnerTestSuite) TestUpToDate() { + runner := s.newRunner(&doneRolloutChecker{}, 0) + err := runner.runOnce(s.ctx) + s.Require().NoError(err) + + seqNum, _, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(0, seqNum) + + s.Equal(0.0, testutil.ToFloat64(bgMigrationSeqNumGauge)) + s.Equal(1.0, testutil.ToFloat64(bgMigrationCompleteGauge)) +} + +func (s *RunnerTestSuite) TestDetectsRollback() { + // Set DB to seq 5, but current is 2. + _, err := s.db.Exec(s.ctx, + "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = 5") + s.Require().NoError(err) + + runner := s.newRunner(&doneRolloutChecker{}, 2) + err = runner.runOnce(s.ctx) + s.Require().NoError(err) + + seqNum, _, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(2, seqNum) + + s.Equal(2.0, testutil.ToFloat64(bgMigrationSeqNumGauge)) + s.Equal(1.0, testutil.ToFloat64(bgMigrationCompleteGauge)) +} + +func (s *RunnerTestSuite) TestRunsOnlyNewMigrations() { + // DB at seq 1, current target is 3 — should only run migrations 1 and 2. + _, err := s.db.Exec(s.ctx, + "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = 1") + s.Require().NoError(err) + + ran := []int{} + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "m0", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 0); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 1, VersionAfterSeqNum: 2, Description: "m1", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 1); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 2, VersionAfterSeqNum: 3, Description: "m2", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 2); return nil }, + }) + + runner := s.newRunner(&doneRolloutChecker{}, 3) + err = runner.runOnce(s.ctx) + s.Require().NoError(err) + + s.Equal([]int{1, 2}, ran) + + seqNum, _, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(3, seqNum) + + s.Equal(3.0, testutil.ToFloat64(bgMigrationSeqNumGauge)) + s.Equal(1.0, testutil.ToFloat64(bgMigrationCompleteGauge)) +} + +func (s *RunnerTestSuite) TestRetriesOnMigrationError() { + callCount := 0 + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "fails-then-succeeds", + Run: func(_ context.Context, _ postgres.DB) error { + callCount++ + if callCount == 1 { + return errors.New("transient failure") + } + return nil + }, + }) + + runner := s.newRunner(&doneRolloutChecker{}, 1) + requireStoppedWithin(s.T(), runner, testTimeout) + + s.Equal(2, callCount) +} + +func (s *RunnerTestSuite) TestRetryStopsOnShutdown() { + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "always-fails", + Run: func(_ context.Context, _ postgres.DB) error { return errors.New("permanent failure") }, + }) + + runner := s.newRunner(&doneRolloutChecker{}, 1) + runner.Start() + + time.Sleep(50 * time.Millisecond) + runner.Stop() + + done := make(chan struct{}) + go func() { + _ = runner.stopper.Client().Stopped().Wait() + close(done) + }() + select { + case <-done: + case <-time.After(testTimeout): + s.T().Fatal("runner did not stop after Stop() within timeout") + } +} + +func (s *RunnerTestSuite) TestMigrationRespectsContext() { + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "ctx-aware", + Run: func(ctx context.Context, _ postgres.DB) error { return ctx.Err() }, + }) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + m, ok := migrations.Get(0) + require.True(s.T(), ok) + err := m.Run(ctx, nil) + assert.ErrorIs(s.T(), err, context.Canceled) +} + +func (s *RunnerTestSuite) TestStopDuringRolloutRetry() { + runner := s.newRunner(¬DoneRolloutChecker{}, 0) + runner.Start() + + time.Sleep(50 * time.Millisecond) + runner.Stop() + + done := make(chan struct{}) + go func() { + _ = runner.stopper.Client().Stopped().Wait() + close(done) + }() + select { + case <-done: + case <-time.After(testTimeout): + s.T().Fatal("runner did not stop after Stop() within timeout") + } +} + +func (s *RunnerTestSuite) TestRetriesOnRolloutCheckError() { + checker := &countingRolloutChecker{notDoneCount: 2} + runner := s.newRunner(checker, 0) + requireStoppedWithin(s.T(), runner, testTimeout) + + s.Equal(3, checker.callCount) +} + +func (s *RunnerTestSuite) TestLockReleasedAfterMigrations() { + runner := s.newRunner(&doneRolloutChecker{}, 0) + err := runner.runOnce(s.ctx) + s.Require().NoError(err) + + // If the lock was released, we should be able to acquire it again. + runner2 := s.newRunner(&doneRolloutChecker{}, 0) + err = runner2.runOnce(s.ctx) + s.Require().NoError(err) +} + +func (s *RunnerTestSuite) TestStopCancelsRunningMigration() { + migrationStarted := make(chan struct{}) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "long-running", + Run: func(ctx context.Context, _ postgres.DB) error { + close(migrationStarted) + <-ctx.Done() + return ctx.Err() + }, + }) + + runner := s.newRunner(&doneRolloutChecker{}, 1) + runner.Start() + + select { + case <-migrationStarted: + case <-time.After(testTimeout): + s.T().Fatal("migration did not start within timeout") + } + + runner.Stop() + + done := make(chan struct{}) + go func() { + _ = runner.stopper.Client().Stopped().Wait() + close(done) + }() + select { + case <-done: + case <-time.After(testTimeout): + s.T().Fatal("runner did not stop after cancelling migration within timeout") + } +} + +func (s *RunnerTestSuite) TestOverrideAppliesWithNewTag() { + // DB at seq 3, override resets to 0. + _, err := s.db.Exec(s.ctx, + "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = 3") + s.Require().NoError(err) + + ran := []int{} + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "m0", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 0); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 1, VersionAfterSeqNum: 2, Description: "m1", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 1); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 2, VersionAfterSeqNum: 3, Description: "m2", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 2); return nil }, + }) + + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_SEQ_NUM", "0") + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_TAG", "ROX-123") + + runner := s.newRunner(&doneRolloutChecker{}, 3) + err = runner.runOnce(s.ctx) + s.Require().NoError(err) + + s.Equal([]int{0, 1, 2}, ran) + + _, overrideTag, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal("ROX-123", overrideTag) +} + +func (s *RunnerTestSuite) TestOverrideSkipsWhenTagMatches() { + // DB already has the same tag — override was already applied. + _, err := s.db.Exec(s.ctx, + "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = 3, override_tag = 'ROX-123'") + s.Require().NoError(err) + + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_SEQ_NUM", "0") + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_TAG", "ROX-123") + + runner := s.newRunner(&doneRolloutChecker{}, 3) + err = runner.runOnce(s.ctx) + s.Require().NoError(err) + + seqNum, _, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(3, seqNum) +} + +func (s *RunnerTestSuite) TestOverrideRerunsWithNewTag() { + _, err := s.db.Exec(s.ctx, + "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = 3, override_tag = 'ROX-123'") + s.Require().NoError(err) + + ran := []int{} + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "m0", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 0); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 1, VersionAfterSeqNum: 2, Description: "m1", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 1); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 2, VersionAfterSeqNum: 3, Description: "m2", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 2); return nil }, + }) + + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_SEQ_NUM", "0") + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_TAG", "ROX-456") + + runner := s.newRunner(&doneRolloutChecker{}, 3) + err = runner.runOnce(s.ctx) + s.Require().NoError(err) + + s.Equal([]int{0, 1, 2}, ran) +} + +func (s *RunnerTestSuite) TestOverrideIgnoredWithoutTag() { + _, err := s.db.Exec(s.ctx, + "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = 3") + s.Require().NoError(err) + + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_SEQ_NUM", "0") + + runner := s.newRunner(&doneRolloutChecker{}, 3) + err = runner.runOnce(s.ctx) + s.Require().NoError(err) + + seqNum, _, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(3, seqNum) +} + +func (s *RunnerTestSuite) TestOverrideIgnoredWhenSeqNumExceedsCurrent() { + _, err := s.db.Exec(s.ctx, + "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = 3") + s.Require().NoError(err) + + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_SEQ_NUM", "10") + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_TAG", "ROX-999") + + runner := s.newRunner(&doneRolloutChecker{}, 3) + err = runner.runOnce(s.ctx) + s.Require().NoError(err) + + seqNum, _, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(3, seqNum) +} + +func (s *RunnerTestSuite) TestSeedsInitialRowWhenTableEmpty() { + _, err := s.db.Exec(s.ctx, "DELETE FROM "+schema.BackgroundMigrationVersionsTableName) + s.Require().NoError(err) + + runner := s.newRunner(&doneRolloutChecker{}, 0) + err = runner.runOnce(s.ctx) + s.Require().NoError(err) + + seqNum, overrideTag, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(0, seqNum) + s.Equal("", overrideTag) +} + +func (s *RunnerTestSuite) TestOverrideTagClearedWhenEnvRemoved() { + // DB has a stale override tag from a previous run, but env vars are not set. + _, err := s.db.Exec(s.ctx, + "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = 3, override_tag = 'ROX-123'") + s.Require().NoError(err) + + runner := s.newRunner(&doneRolloutChecker{}, 3) + err = runner.runOnce(s.ctx) + s.Require().NoError(err) + + seqNum, overrideTag, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(3, seqNum) + s.Equal("", overrideTag) +} + +func (s *RunnerTestSuite) TestOverrideNotReappliedOnRestart() { + ran := []int{} + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "m0", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 0); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 1, VersionAfterSeqNum: 2, Description: "m1", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 1); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 2, VersionAfterSeqNum: 3, Description: "m2", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 2); return nil }, + }) + + _, err := s.db.Exec(s.ctx, + "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = 3") + s.Require().NoError(err) + + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_SEQ_NUM", "0") + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_TAG", "ROX-123") + + // First run: override applies, migrations re-run. + runner1 := s.newRunner(&doneRolloutChecker{}, 3) + err = runner1.runOnce(s.ctx) + s.Require().NoError(err) + s.Equal([]int{0, 1, 2}, ran) + + seqNum, overrideTag, err := runner1.readState(s.ctx) + s.Require().NoError(err) + s.Equal(3, seqNum) + s.Equal("ROX-123", overrideTag, "override tag must be preserved after first run") + + // Second run (simulating restart): env vars still set, tag matches DB. + // Migrations must NOT re-run. + ran = []int{} + runner2 := s.newRunner(&doneRolloutChecker{}, 3) + err = runner2.runOnce(s.ctx) + s.Require().NoError(err) + s.Empty(ran, "migrations must not re-run when override tag already matches") + + seqNum, overrideTag, err = runner2.readState(s.ctx) + s.Require().NoError(err) + s.Equal(3, seqNum) + s.Equal("ROX-123", overrideTag, "override tag must remain after second run") +} + +func (s *RunnerTestSuite) TestSkipMigrations() { + ran := []int{} + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "m0", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 0); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 1, VersionAfterSeqNum: 2, Description: "m1", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 1); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 2, VersionAfterSeqNum: 3, Description: "m2", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 2); return nil }, + }) + + s.T().Setenv("ROX_SKIP_BACKGROUND_MIGRATIONS", "1") + + runner := s.newRunner(&doneRolloutChecker{}, 3) + err := runner.runOnce(s.ctx) + s.Require().NoError(err) + + s.Equal([]int{0, 2}, ran) + + seqNum, _, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(3, seqNum) +} + +func (s *RunnerTestSuite) TestSkipMultipleMigrations() { + ran := []int{} + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "m0", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 0); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 1, VersionAfterSeqNum: 2, Description: "m1", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 1); return nil }, + }) + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 2, VersionAfterSeqNum: 3, Description: "m2", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 2); return nil }, + }) + + s.T().Setenv("ROX_SKIP_BACKGROUND_MIGRATIONS", "0, 2") + + runner := s.newRunner(&doneRolloutChecker{}, 3) + err := runner.runOnce(s.ctx) + s.Require().NoError(err) + + s.Equal([]int{1}, ran) + + seqNum, _, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(3, seqNum) +} + +func (s *RunnerTestSuite) TestSkipMigrationsEmptyEnv() { + ran := []int{} + migrations.MustRegister(types.BackgroundMigration{ + StartingSeqNum: 0, VersionAfterSeqNum: 1, Description: "m0", + Run: func(_ context.Context, _ postgres.DB) error { ran = append(ran, 0); return nil }, + }) + + runner := s.newRunner(&doneRolloutChecker{}, 1) + err := runner.runOnce(s.ctx) + s.Require().NoError(err) + + s.Equal([]int{0}, ran) +} + +func (s *RunnerTestSuite) TestOverrideIgnoredWithoutSeqNum() { + _, err := s.db.Exec(s.ctx, + "UPDATE "+schema.BackgroundMigrationVersionsTableName+" SET seqnum = 3") + s.Require().NoError(err) + + s.T().Setenv("ROX_BACKGROUND_MIGRATION_OVERRIDE_TAG", "ROX-123") + + runner := s.newRunner(&doneRolloutChecker{}, 3) + err = runner.runOnce(s.ctx) + s.Require().NoError(err) + + seqNum, _, err := runner.readState(s.ctx) + s.Require().NoError(err) + s.Equal(3, seqNum) +} diff --git a/central/backgroundmigrations/seq_num.go b/central/backgroundmigrations/seq_num.go new file mode 100644 index 0000000000000..60c3a226b5524 --- /dev/null +++ b/central/backgroundmigrations/seq_num.go @@ -0,0 +1,6 @@ +package backgroundmigrations + +// CurrentBgMigrationSeqNum is the current background migration sequence number. +// This must be incremented every time a new background migration is added. +// It is independent from the schema migration sequence number in pkg/migrations. +const CurrentBgMigrationSeqNum = 0 diff --git a/central/backgroundmigrations/types/migration.go b/central/backgroundmigrations/types/migration.go new file mode 100644 index 0000000000000..c794e790d0674 --- /dev/null +++ b/central/backgroundmigrations/types/migration.go @@ -0,0 +1,23 @@ +package types + +import ( + "context" + + "github.com/stackrox/rox/pkg/postgres" +) + +// BackgroundMigration defines a long-running migration that runs in Central after startup. +// Examples for long running migration use cases: Backfilling a column from serialized values, Index Creation +type BackgroundMigration struct { + // StartingSeqNum is the seqnum before this migration runs. + StartingSeqNum int + // VersionAfterSeqNum is StartingSeqNum + 1. + VersionAfterSeqNum int + // Description is a human-readable description of the migration. + Description string + // Run executes the migration. Contract: + // - Must be idempotent (safe to re-run after rollback) + // - Must check ctx.Done() between units of work for graceful shutdown + // - Must be consistent after re-run, previously migrated rows need to be rechecked on a re-run + Run func(ctx context.Context, db postgres.DB) error +} diff --git a/central/baseimage/datastore/datastore_impl.go b/central/baseimage/datastore/datastore_impl.go index 97359d5d8e764..3a825e34b9c8e 100644 --- a/central/baseimage/datastore/datastore_impl.go +++ b/central/baseimage/datastore/datastore_impl.go @@ -60,10 +60,7 @@ func (ds *datastoreImpl) UpsertImages( // Process in chunks. This reduces pressure on storage and avoids parameter/time limits. for start := 0; start < len(batch); start += upsertChunkSize { - end := start + upsertChunkSize - if end > len(batch) { - end = len(batch) - } + end := min(start+upsertChunkSize, len(batch)) if err := ds.storage.UpsertMany(ctx, batch[start:end]); err != nil { return fmt.Errorf("upsert images chunk [%d:%d]: %w", start, end, err) diff --git a/central/baseimage/datastore/repository/datastore.go b/central/baseimage/datastore/repository/datastore.go index d66b2484993ab..a6495a5bb9e12 100644 --- a/central/baseimage/datastore/repository/datastore.go +++ b/central/baseimage/datastore/repository/datastore.go @@ -2,11 +2,43 @@ package repository import ( "context" + "time" repoStore "github.com/stackrox/rox/central/baseimage/store/repository/postgres" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" ) +// FailureCountOp specifies the operation to perform on the failure count. +type FailureCountOp int + +const ( + // FailureCountNoOp leaves the failure count unchanged. + FailureCountNoOp FailureCountOp = iota + // FailureCountReset sets the failure count to zero. + FailureCountReset + // FailureCountIncrement increments the failure count by one. + FailureCountIncrement +) + +// ConfigUpdate contains user-configurable fields. +type ConfigUpdate struct { + RepositoryPath *string + TagPattern *string +} + +// StatusUpdate contains fields for internal lifecycle updates. +type StatusUpdate struct { + Status storage.BaseImageRepository_Status + LastPolledAt *time.Time + LastFailureMessage *string + FailureCountOp FailureCountOp + + // OnlyIfStatus makes the update conditional: it only applies if the current + // status matches one of the specified values. If empty, the update is unconditional. + OnlyIfStatus []storage.BaseImageRepository_Status +} + // DataStore provides access to base image repositories. // //go:generate mockgen-wrapper @@ -21,18 +53,27 @@ type DataStore interface { ListRepositories(ctx context.Context) ([]*storage.BaseImageRepository, error) // UpsertRepository inserts or updates the given base image repository. - // Returns the updated repository and an error, if any. + // Returns the updated repository or an error if the operation failed. UpsertRepository(ctx context.Context, repo *storage.BaseImageRepository) (*storage.BaseImageRepository, error) // DeleteRepository removes the base image repository with the specified ID. // Returns an error if deletion fails. DeleteRepository(ctx context.Context, id string) error + + // UpdateConfiguration updates user-configurable fields of an existing repository. + // Returns NotFound if the repository does not exist. + // Returns ErrScanInProgress if the repository is being scanned as per its status. + UpdateConfiguration(ctx context.Context, id string, update ConfigUpdate) (*storage.BaseImageRepository, error) + + // UpdateStatus updates internal lifecycle fields of a repository. + // Returns (nil, nil) if the repository does not exist or if conditions are not met. + UpdateStatus(ctx context.Context, id string, update StatusUpdate) (*storage.BaseImageRepository, error) } // New returns a base image repository DataStore. -func New(s repoStore.Store) DataStore { - ds := &datastoreImpl{ - store: s, +func New(s repoStore.Store, wf concurrency.KeyFence) DataStore { + return &datastoreImpl{ + store: s, + writeFence: wf, } - return ds } diff --git a/central/baseimage/datastore/repository/datastore_impl.go b/central/baseimage/datastore/repository/datastore_impl.go index cef2857d15f3f..77ecc99a156bb 100644 --- a/central/baseimage/datastore/repository/datastore_impl.go +++ b/central/baseimage/datastore/repository/datastore_impl.go @@ -4,12 +4,16 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" + "fmt" + "slices" "time" - "github.com/pkg/errors" repoStore "github.com/stackrox/rox/central/baseimage/store/repository/postgres" "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/grpc/authn" "github.com/stackrox/rox/pkg/postgres/pgutils" "github.com/stackrox/rox/pkg/sac" @@ -26,8 +30,43 @@ var ( baseImageRepositorySAC = sac.ForResource(resources.ImageAdministration) ) +// ErrScanInProgress is returned when a config update is attempted while repository scanning is in progress. +var ErrScanInProgress = errors.New("repository scanning in progress") + type datastoreImpl struct { - store repoStore.Store + store repoStore.Store + writeFence concurrency.KeyFence +} + +func (d *datastoreImpl) withWriteLock(id string, fn func() error) error { + return d.writeFence.DoStatusWithLock(concurrency.DiscreteKeySet([]byte(id)), fn) +} + +func (d *datastoreImpl) withWriteLockedRepository(ctx context.Context, id string, fn func(repo *storage.BaseImageRepository) error) error { + return d.withWriteLock(id, func() error { + repo, found, err := d.store.Get(ctx, id) + if err != nil { + return fmt.Errorf("fetching repository %q: %w", id, err) + } + if !found { + return fn(nil) + } + return fn(repo) + }) +} + +func resetTimestampAndHash(repo *storage.BaseImageRepository) { + repo.UpdatedAt = timestamppb.Now() + hash := sha256.Sum256([]byte(repo.GetTagPattern())) + repo.PatternHash = hex.EncodeToString(hash[:]) +} + +func resetRepositoryState(repo *storage.BaseImageRepository) { + resetTimestampAndHash(repo) + repo.Status = storage.BaseImageRepository_CREATED + repo.FailureCount = 0 + repo.HealthStatus = storage.BaseImageRepository_HEALTHY + repo.LastFailureMessage = "" } func (d *datastoreImpl) GetRepository(ctx context.Context, id string) (*storage.BaseImageRepository, bool, error) { @@ -68,31 +107,114 @@ func (d *datastoreImpl) UpsertRepository(ctx context.Context, repo *storage.Base } if repo.GetId() == "" { - // Generate ID if not provided repo.Id = uuid.NewV4().String() - // Fill in the CreatedBy field. slimUser := authn.UserFromContext(ctx) if slimUser == nil { - return nil, errors.New("Could not determine user identity from provided context") + return nil, errors.New("could not determine user identity from provided context") } repo.CreatedBy = slimUser } - repo.UpdatedAt = timestamppb.Now() - hash := sha256.Sum256([]byte(repo.GetTagPattern())) - repo.PatternHash = hex.EncodeToString(hash[:]) + resetTimestampAndHash(repo) + if err := d.store.Upsert(ctx, repo); err != nil { return nil, err } return repo, nil } +func (d *datastoreImpl) UpdateConfiguration(ctx context.Context, id string, update ConfigUpdate) (*storage.BaseImageRepository, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), resourceType, "UpdateConfiguration") + + if err := sac.VerifyAuthzOK(baseImageRepositorySAC.WriteAllowed(ctx)); err != nil { + return nil, err + } + + var updated *storage.BaseImageRepository + err := d.withWriteLockedRepository(ctx, id, func(existing *storage.BaseImageRepository) error { + if existing == nil { + return errox.NotFound.Newf("base image repository %q not found", id) + } + switch existing.GetStatus() { + case storage.BaseImageRepository_QUEUED, storage.BaseImageRepository_IN_PROGRESS: + return ErrScanInProgress + } + + if update.RepositoryPath != nil { + existing.RepositoryPath = *update.RepositoryPath + } + if update.TagPattern != nil { + existing.TagPattern = *update.TagPattern + } + resetRepositoryState(existing) + + if err := d.store.Upsert(ctx, existing); err != nil { + return fmt.Errorf("updating repository: %w", err) + } + updated = existing + return nil + }) + if err != nil { + return nil, err + } + return updated, nil +} + +func (d *datastoreImpl) UpdateStatus(ctx context.Context, id string, update StatusUpdate) (*storage.BaseImageRepository, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), resourceType, "UpdateStatus") + + if err := sac.VerifyAuthzOK(baseImageRepositorySAC.WriteAllowed(ctx)); err != nil { + return nil, err + } + + var updated *storage.BaseImageRepository + err := d.withWriteLockedRepository(ctx, id, func(existing *storage.BaseImageRepository) error { + if existing == nil { + return nil + } + + if len(update.OnlyIfStatus) > 0 && !statusMatches(existing.GetStatus(), update.OnlyIfStatus) { + return nil + } + + existing.Status = update.Status + if update.LastPolledAt != nil { + existing.LastPolledAt = timestamppb.New(*update.LastPolledAt) + } + if update.LastFailureMessage != nil { + existing.LastFailureMessage = *update.LastFailureMessage + } + switch update.FailureCountOp { + case FailureCountReset: + existing.FailureCount = 0 + case FailureCountIncrement: + existing.FailureCount++ + } + + if err := d.store.Upsert(ctx, existing); err != nil { + return fmt.Errorf("updating repository: %w", err) + } + updated = existing + return nil + }) + if err != nil { + return nil, err + } + return updated, nil +} + func (d *datastoreImpl) DeleteRepository(ctx context.Context, id string) error { defer metrics.SetDatastoreFunctionDuration(time.Now(), resourceType, "DeleteRepository") if err := sac.VerifyAuthzOK(baseImageRepositorySAC.WriteAllowed(ctx)); err != nil { return err } - return d.store.Delete(ctx, id) + return d.withWriteLock(id, func() error { + return d.store.Delete(ctx, id) + }) +} + +func statusMatches(current storage.BaseImageRepository_Status, allowed []storage.BaseImageRepository_Status) bool { + return slices.Contains(allowed, current) } diff --git a/central/baseimage/datastore/repository/datastore_impl_test.go b/central/baseimage/datastore/repository/datastore_impl_test.go index 49cbfda2b3a0a..0015eb4224b19 100644 --- a/central/baseimage/datastore/repository/datastore_impl_test.go +++ b/central/baseimage/datastore/repository/datastore_impl_test.go @@ -5,10 +5,14 @@ package repository import ( "context" "testing" + "time" repoStore "github.com/stackrox/rox/central/baseimage/store/repository/postgres" + v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/auth/authproviders" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/grpc/authn" "github.com/stackrox/rox/pkg/grpc/authn/basic" mockIdentity "github.com/stackrox/rox/pkg/grpc/authn/mocks" @@ -17,9 +21,11 @@ import ( "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/uuid" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" ) func TestBaseImageRepositoryDatastore(t *testing.T) { @@ -30,7 +36,6 @@ type BaseImageRepositoryDatastoreTestSuite struct { suite.Suite mockCtrl *gomock.Controller - mockID *mockIdentity.MockIdentity datastore DataStore storage repoStore.Store @@ -39,6 +44,8 @@ type BaseImageRepositoryDatastoreTestSuite struct { normalUserCtx context.Context } +var _ repoStore.Store = (*blockingRepositoryStore)(nil) + var _ interface { suite.SetupAllSuite suite.TearDownAllSuite @@ -48,13 +55,12 @@ func (s *BaseImageRepositoryDatastoreTestSuite) SetupSuite() { s.mockCtrl = gomock.NewController(s.T()) s.pool = pgtest.ForT(s.T()) s.storage = repoStore.New(s.pool) - s.datastore = New(s.storage) + s.datastore = New(s.storage, concurrency.NewKeyFence()) mockID := mockIdentity.NewMockIdentity(s.mockCtrl) mockID.EXPECT().UID().Return("uid").AnyTimes() mockID.EXPECT().FullName().Return("name").AnyTimes() mockID.EXPECT().FriendlyName().Return("name").AnyTimes() - s.mockID = mockID ctx := sac.WithGlobalAccessScopeChecker(s.T().Context(), sac.AllowFixedScopes( @@ -68,12 +74,12 @@ func (s *BaseImageRepositoryDatastoreTestSuite) SetupSuite() { } func (s *BaseImageRepositoryDatastoreTestSuite) SetupTest() { - ctx := s.imgAdminCtx - _, err := s.pool.Exec(ctx, "TRUNCATE base_image_repositories CASCADE") + _, err := s.pool.Exec(s.imgAdminCtx, "TRUNCATE base_image_repositories CASCADE") s.NoError(err) } func (s *BaseImageRepositoryDatastoreTestSuite) TearDownSuite() { + s.mockCtrl.Finish() s.pool.Close() } @@ -83,33 +89,40 @@ func (s *BaseImageRepositoryDatastoreTestSuite) mustGetRepository(ctx context.Co return repo, found } +func newRepository(path, tagPattern string) *storage.BaseImageRepository { + return &storage.BaseImageRepository{ + RepositoryPath: path, + TagPattern: tagPattern, + } +} + func (s *BaseImageRepositoryDatastoreTestSuite) TestBaseImageRepositoryDatastore() { ctx := s.imgAdminCtx - // Test repository lifecycle: Create, Read, Update, Delete - repo := &storage.BaseImageRepository{ - RepositoryPath: "registry.example.com/test-repo", - TagPattern: "v*", - } + // Test repository lifecycle: Create, Read, Update, Delete. + repo := newRepository("registry.example.com/test-repo", "v*") - // Test Create + // Test Create. created, err := s.datastore.UpsertRepository(ctx, repo) s.NoError(err) s.NotEmpty(created.GetId()) s.Equal(repo.GetRepositoryPath(), created.GetRepositoryPath()) s.Equal(repo.GetTagPattern(), created.GetTagPattern()) + s.Equal(storage.BaseImageRepository_CREATED, created.GetStatus()) + s.Zero(created.GetFailureCount()) + s.Empty(created.GetLastFailureMessage()) - // Verify it exists + // Verify it exists. retrieved, found := s.mustGetRepository(ctx, created.GetId()) s.True(found) protoassert.Equal(s.T(), created, retrieved) - // Get All - verify it's in the list + // Get all and verify the repository is returned. allRepos, err := s.datastore.ListRepositories(ctx) s.NoError(err) protoassert.SliceContains(s.T(), allRepos, created) - // Update - change tag pattern + // Update the tag pattern. updated := &storage.BaseImageRepository{ Id: created.GetId(), RepositoryPath: created.GetRepositoryPath(), @@ -119,8 +132,11 @@ func (s *BaseImageRepositoryDatastoreTestSuite) TestBaseImageRepositoryDatastore s.NoError(err) s.Equal(created.GetId(), updatedResult.GetId()) s.Equal("latest", updatedResult.GetTagPattern()) + s.Equal(storage.BaseImageRepository_CREATED, updatedResult.GetStatus()) + s.Zero(updatedResult.GetFailureCount()) + s.Empty(updatedResult.GetLastFailureMessage()) - // Delete - remove the repository + // Delete the repository. s.NoError(s.datastore.DeleteRepository(ctx, created.GetId())) _, found = s.mustGetRepository(ctx, created.GetId()) s.False(found) @@ -134,36 +150,28 @@ func (s *BaseImageRepositoryDatastoreTestSuite) TestGetRepositoryNotFound() { } func (s *BaseImageRepositoryDatastoreTestSuite) TestDeleteRepositoryNotFound() { - // No error when deleting non-existent repository s.NoError(s.datastore.DeleteRepository(s.imgAdminCtx, uuid.NewV4().String())) } func (s *BaseImageRepositoryDatastoreTestSuite) TestUniqueRepositoryPathConstraint() { ctx := s.imgAdminCtx - // Create first repository - repo1 := &storage.BaseImageRepository{ - RepositoryPath: "registry.example.com/unique-repo", - TagPattern: "v1.*", - } + // Create the first repository. + repo1 := newRepository("registry.example.com/unique-repo", "v1.*") createdRepo1, err := s.datastore.UpsertRepository(ctx, repo1) s.NoError(err) s.NotEmpty(createdRepo1.GetId()) - // Try to create a second repository with the same repository_path but different tag_pattern - repo2 := &storage.BaseImageRepository{ - RepositoryPath: "registry.example.com/unique-repo", // Same path - TagPattern: "v2.*", // Different pattern - } + // Creating a second repository with the same path should fail. + repo2 := newRepository("registry.example.com/unique-repo", "v2.*") - // This should fail due to unique constraint on repository_path _, err = s.datastore.UpsertRepository(ctx, repo2) s.Error(err, "Creating second repository with same path should fail due to unique constraint") } func (s *BaseImageRepositoryDatastoreTestSuite) TestListRepositoriesAccessDenied() { - // Create mock auth provider + // Create a context with a user that has no image administration access. authProvider, err := authproviders.NewProvider( authproviders.WithEnabled(true), authproviders.WithID(uuid.NewDummy().String()), @@ -171,38 +179,565 @@ func (s *BaseImageRepositoryDatastoreTestSuite) TestListRepositoriesAccessDenied ) s.Require().NoError(err) - // Create context with a normal user that has no access noAccessCtx := sac.WithNoAccess(basic.ContextWithNoAccessIdentity(s.T(), authProvider)) - // Attempt to list repositories with no access user should be denied repos, err := s.datastore.ListRepositories(noAccessCtx) s.Error(err, "ListRepositories should fail for user with no access") s.Nil(repos, "No repositories should be returned when access is denied") } func (s *BaseImageRepositoryDatastoreTestSuite) TestUserCtxOperationsDenied() { - // Test all datastore operations with userCtx (has GlobalAccessScopeChecker but mock identity has no permissions) - // Operations should fail due to lack of permissions, not missing scope checker - - repo := &storage.BaseImageRepository{ - RepositoryPath: "registry.example.com/test-repo", - TagPattern: "v*", - } + // Exercise all datastore methods with a user context that lacks permissions. + repo := newRepository("registry.example.com/test-repo", "v*") - // Test ListRepositories repos, err := s.datastore.ListRepositories(s.normalUserCtx) s.Error(err, "ListRepositories should fail for user context without permissions") s.Nil(repos, "No repositories should be returned when access is denied due to permissions") - // Test UpsertRepository _, err = s.datastore.UpsertRepository(s.normalUserCtx, repo) s.Error(err, "UpsertRepository should fail for user context without permissions") - // Test GetRepository _, _, err = s.datastore.GetRepository(s.normalUserCtx, "non-existent-id") s.Error(err, "GetRepository should fail for user context without permissions") - // Test DeleteRepository err = s.datastore.DeleteRepository(s.normalUserCtx, "non-existent-id") s.Error(err, "DeleteRepository should fail for user context without permissions") + + _, err = s.datastore.UpdateStatus(s.normalUserCtx, "non-existent-id", StatusUpdate{Status: storage.BaseImageRepository_READY}) + s.Error(err, "UpdateStatus should fail for user context without permissions") + + _, err = s.datastore.UpdateConfiguration(s.normalUserCtx, "non-existent-id", ConfigUpdate{TagPattern: new("v*")}) + s.Error(err, "UpdateConfiguration should fail for user context without permissions") +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_UpdatesLastPolledAt() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo", "v*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + s.Nil(created.GetLastPolledAt(), "LastPolledAt should be nil initially") + + pollTime := time.Now().Truncate(time.Microsecond) + _, err = s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_IN_PROGRESS, + LastPolledAt: &pollTime, + }) + s.NoError(err) + + retrieved, found := s.mustGetRepository(ctx, created.GetId()) + s.True(found) + s.NotNil(retrieved.GetLastPolledAt()) + s.Equal(pollTime.UTC(), retrieved.GetLastPolledAt().AsTime().UTC()) + s.Equal(storage.BaseImageRepository_IN_PROGRESS, retrieved.GetStatus()) +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_DoesNotChangeUpdatedAt() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo", "v*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + originalUpdatedAt := created.GetUpdatedAt() + s.NotNil(originalUpdatedAt) + + time.Sleep(10 * time.Millisecond) + + pollTime := time.Now() + _, err = s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_READY, + LastPolledAt: &pollTime, + }) + s.NoError(err) + + retrieved, found := s.mustGetRepository(ctx, created.GetId()) + s.True(found) + s.Equal(originalUpdatedAt.AsTime(), retrieved.GetUpdatedAt().AsTime(), + "UpdatedAt should not change when updating repository status") +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_FailureFields() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo", "v1.*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + + failureMessage := "polling failure" + pollTime := time.Now().Truncate(time.Microsecond) + _, err = s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_FAILED, + LastPolledAt: &pollTime, + LastFailureMessage: &failureMessage, + FailureCountOp: FailureCountIncrement, + }) + s.Require().NoError(err) + + retrieved, found := s.mustGetRepository(ctx, created.GetId()) + s.True(found) + s.Equal(storage.BaseImageRepository_FAILED, retrieved.GetStatus()) + s.Equal(int32(1), retrieved.GetFailureCount()) + s.Equal(failureMessage, retrieved.GetLastFailureMessage()) + s.Equal(pollTime.UTC(), retrieved.GetLastPolledAt().AsTime().UTC()) + + clearedFailureMessage := "" + _, err = s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_READY, + LastFailureMessage: &clearedFailureMessage, + FailureCountOp: FailureCountReset, + }) + s.Require().NoError(err) + + retrieved, found = s.mustGetRepository(ctx, created.GetId()) + s.True(found) + s.Equal(storage.BaseImageRepository_READY, retrieved.GetStatus()) + s.Zero(retrieved.GetFailureCount()) + s.Empty(retrieved.GetLastFailureMessage()) +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_NonExistentRepo() { + ctx := s.imgAdminCtx + + nonExistentID := uuid.NewV4().String() + updated, err := s.datastore.UpdateStatus(ctx, nonExistentID, StatusUpdate{Status: storage.BaseImageRepository_READY}) + + s.Nil(updated) + s.NoError(err, "UpdateStatus on non-existent repo should return nil") +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpsertRepository_InsertsProvidedIDWhenRepositoryDoesNotExist() { + repo := newRepository("registry.example.com/test-repo", "v*") + repo.Id = uuid.NewV4().String() + + updated, err := s.datastore.UpsertRepository(s.imgAdminCtx, repo) + + s.NoError(err) + s.NotNil(updated) + s.Equal(repo.GetId(), updated.GetId()) +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateConfiguration_NotFound() { + updated, err := s.datastore.UpdateConfiguration(s.imgAdminCtx, uuid.NewV4().String(), ConfigUpdate{TagPattern: new("v*")}) + + s.Nil(updated) + s.ErrorIs(err, errox.NotFound) +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_FailureCountOperations() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo", "v1.*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + + // Increment failure count. + failureMessage := "polling failure" + updated, err := s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_FAILED, + LastFailureMessage: &failureMessage, + FailureCountOp: FailureCountIncrement, + }) + s.Require().NoError(err) + s.Equal(int32(1), updated.GetFailureCount()) + + // Increment again. + updated, err = s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_FAILED, + FailureCountOp: FailureCountIncrement, + }) + s.Require().NoError(err) + s.Equal(int32(2), updated.GetFailureCount()) + + // Reset failure count. + updated, err = s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_READY, + FailureCountOp: FailureCountReset, + }) + s.Require().NoError(err) + s.Zero(updated.GetFailureCount()) + s.Equal(storage.BaseImageRepository_READY, updated.GetStatus()) +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestConcurrency_OperationsSerialize() { + ctx := s.imgAdminCtx + repo := &storage.BaseImageRepository{Id: "repo-id", Status: storage.BaseImageRepository_CREATED} + store := &blockingRepositoryStore{ + repo: repo, + getEntered: make(chan struct{}, 1), + deleteEntered: make(chan struct{}, 1), + unblockGet: make(chan struct{}), + } + ds := New(store, concurrency.NewKeyFence()) + + statusUpdateDone := make(chan struct{}) + go func() { + _, _ = ds.UpdateStatus(ctx, repo.GetId(), StatusUpdate{Status: storage.BaseImageRepository_QUEUED}) + close(statusUpdateDone) + }() + + <-store.getEntered + + deleteDone := make(chan struct{}) + go func() { + _ = ds.DeleteRepository(ctx, repo.GetId()) + close(deleteDone) + }() + + select { + case <-store.deleteEntered: + s.Fail("DeleteRepository reached store.Delete while UpdateRepositoryStatus held the fence") + default: + } + + close(store.unblockGet) + <-statusUpdateDone + <-store.deleteEntered + <-deleteDone +} + +// UpdateStatus conditional update tests (OnlyIfStatus) + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_OnlyIfStatus_MatchesCreated() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo", "v*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + s.Equal(storage.BaseImageRepository_CREATED, created.GetStatus()) + + // Update should succeed because status matches. + updated, err := s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_QUEUED, + OnlyIfStatus: []storage.BaseImageRepository_Status{ + storage.BaseImageRepository_CREATED, + storage.BaseImageRepository_READY, + storage.BaseImageRepository_FAILED, + }, + }) + s.NoError(err) + s.NotNil(updated) + s.Equal(storage.BaseImageRepository_QUEUED, updated.GetStatus()) + + retrieved, found := s.mustGetRepository(ctx, created.GetId()) + s.True(found) + s.Equal(storage.BaseImageRepository_QUEUED, retrieved.GetStatus()) +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_OnlyIfStatus_MatchesReady() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo", "v*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + + // Set status to READY. + _, err = s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_READY, + }) + s.Require().NoError(err) + + // Conditional update should succeed. + updated, err := s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_QUEUED, + OnlyIfStatus: []storage.BaseImageRepository_Status{ + storage.BaseImageRepository_CREATED, + storage.BaseImageRepository_READY, + storage.BaseImageRepository_FAILED, + }, + }) + s.NoError(err) + s.NotNil(updated) + s.Equal(storage.BaseImageRepository_QUEUED, updated.GetStatus()) +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_OnlyIfStatus_MatchesFailed() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo", "v*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + + // Set status to FAILED. + _, err = s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_FAILED, + }) + s.Require().NoError(err) + + // Conditional update should succeed. + updated, err := s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_QUEUED, + OnlyIfStatus: []storage.BaseImageRepository_Status{ + storage.BaseImageRepository_CREATED, + storage.BaseImageRepository_READY, + storage.BaseImageRepository_FAILED, + }, + }) + s.NoError(err) + s.NotNil(updated) + s.Equal(storage.BaseImageRepository_QUEUED, updated.GetStatus()) +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_OnlyIfStatus_DoesNotMatchQueued() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo", "v*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + + // First update to QUEUED succeeds. + updated, err := s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_QUEUED, + OnlyIfStatus: []storage.BaseImageRepository_Status{ + storage.BaseImageRepository_CREATED, + storage.BaseImageRepository_READY, + storage.BaseImageRepository_FAILED, + }, + }) + s.Require().NoError(err) + s.Require().NotNil(updated) + s.Equal(storage.BaseImageRepository_QUEUED, updated.GetStatus()) + + // Second update should return nil because status is QUEUED. + updated2, err := s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_QUEUED, + OnlyIfStatus: []storage.BaseImageRepository_Status{ + storage.BaseImageRepository_CREATED, + storage.BaseImageRepository_READY, + storage.BaseImageRepository_FAILED, + }, + }) + s.NoError(err) + s.Nil(updated2, "should not update already queued repository") +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_OnlyIfStatus_DoesNotMatchInProgress() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo", "v*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + + // Set status to IN_PROGRESS. + _, err = s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_IN_PROGRESS, + }) + s.Require().NoError(err) + + // Conditional update should return nil because status doesn't match. + updated, err := s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_QUEUED, + OnlyIfStatus: []storage.BaseImageRepository_Status{ + storage.BaseImageRepository_CREATED, + storage.BaseImageRepository_READY, + storage.BaseImageRepository_FAILED, + }, + }) + s.NoError(err) + s.Nil(updated, "should not update in-progress repository") + + retrieved, found := s.mustGetRepository(ctx, created.GetId()) + s.True(found) + s.Equal(storage.BaseImageRepository_IN_PROGRESS, retrieved.GetStatus()) +} + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateStatus_OnlyIfStatus_ConcurrentUpdates() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo", "v*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + + // Launch multiple concurrent conditional update attempts. + const numGoroutines = 10 + results := make(chan *storage.BaseImageRepository, numGoroutines) + errs := make(chan error, numGoroutines) + + for range numGoroutines { + go func() { + updated, err := s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: storage.BaseImageRepository_QUEUED, + OnlyIfStatus: []storage.BaseImageRepository_Status{ + storage.BaseImageRepository_CREATED, + storage.BaseImageRepository_READY, + storage.BaseImageRepository_FAILED, + }, + }) + results <- updated + errs <- err + }() + } + + // Collect results. + var updatedCount int + for range numGoroutines { + updated := <-results + err := <-errs + s.NoError(err) + if updated != nil { + updatedCount++ + } + } + + s.Equal(1, updatedCount, "exactly one goroutine should succeed in updating") + + retrieved, found := s.mustGetRepository(ctx, created.GetId()) + s.True(found) + s.Equal(storage.BaseImageRepository_QUEUED, retrieved.GetStatus()) +} + +// UpdateConfiguration blocked during scan tests + +func (s *BaseImageRepositoryDatastoreTestSuite) TestUpdateConfiguration_ScanInProgressBlocking() { + cases := []struct { + name string + statusTransitions []storage.BaseImageRepository_Status + expectBlocked bool + }{ + { + name: "blocked while QUEUED", + statusTransitions: []storage.BaseImageRepository_Status{storage.BaseImageRepository_QUEUED}, + expectBlocked: true, + }, + { + name: "blocked while IN_PROGRESS", + statusTransitions: []storage.BaseImageRepository_Status{storage.BaseImageRepository_IN_PROGRESS}, + expectBlocked: true, + }, + { + name: "allowed after scan completes READY", + statusTransitions: []storage.BaseImageRepository_Status{ + storage.BaseImageRepository_QUEUED, + storage.BaseImageRepository_IN_PROGRESS, + storage.BaseImageRepository_READY, + }, + expectBlocked: false, + }, + { + name: "allowed after scan completes FAILED", + statusTransitions: []storage.BaseImageRepository_Status{ + storage.BaseImageRepository_QUEUED, + storage.BaseImageRepository_IN_PROGRESS, + storage.BaseImageRepository_FAILED, + }, + expectBlocked: false, + }, + { + name: "allowed when CREATED", + statusTransitions: []storage.BaseImageRepository_Status{}, + expectBlocked: false, + }, + } + + for _, tc := range cases { + s.Run(tc.name, func() { + ctx := s.imgAdminCtx + + repo := newRepository("registry.example.com/test-repo-"+tc.name, "v*") + created, err := s.datastore.UpsertRepository(ctx, repo) + s.Require().NoError(err) + + for _, status := range tc.statusTransitions { + _, err = s.datastore.UpdateStatus(ctx, created.GetId(), StatusUpdate{ + Status: status, + }) + s.Require().NoError(err) + } + + newPattern := "2.*" + updated, err := s.datastore.UpdateConfiguration(ctx, created.GetId(), ConfigUpdate{ + TagPattern: &newPattern, + }) + + if tc.expectBlocked { + s.ErrorIs(err, ErrScanInProgress) + s.Nil(updated) + } else { + s.NoError(err) + s.NotNil(updated) + s.Equal("2.*", updated.GetTagPattern()) + } + }) + } +} + +type blockingRepositoryStore struct { + repo *storage.BaseImageRepository + + getEntered chan struct{} + deleteEntered chan struct{} + unblockGet chan struct{} +} + +func (s *blockingRepositoryStore) Upsert(ctx context.Context, obj *storage.BaseImageRepository) error { + s.repo = proto.Clone(obj).(*storage.BaseImageRepository) + return nil +} + +func (s *blockingRepositoryStore) UpsertMany(context.Context, []*storage.BaseImageRepository) error { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) Delete(context.Context, string) error { + select { + case s.deleteEntered <- struct{}{}: + default: + } + s.repo = nil + return nil +} + +func (s *blockingRepositoryStore) DeleteByQuery(context.Context, *v1.Query) error { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) DeleteByQueryWithIDs(context.Context, *v1.Query) ([]string, error) { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) DeleteMany(context.Context, []string) error { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) PruneMany(context.Context, []string) error { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) Count(context.Context, *v1.Query) (int, error) { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) Exists(context.Context, string) (bool, error) { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) Search(context.Context, *v1.Query) ([]search.Result, error) { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) Get(context.Context, string) (*storage.BaseImageRepository, bool, error) { + select { + case s.getEntered <- struct{}{}: + default: + } + <-s.unblockGet + if s.repo == nil { + return nil, false, nil + } + return proto.Clone(s.repo).(*storage.BaseImageRepository), true, nil +} + +func (s *blockingRepositoryStore) GetMany(context.Context, []string) ([]*storage.BaseImageRepository, []int, error) { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) GetIDs(context.Context) ([]string, error) { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) Walk(context.Context, func(*storage.BaseImageRepository) error) error { + panic("unexpected call") +} + +func (s *blockingRepositoryStore) WalkByQuery(context.Context, *v1.Query, func(*storage.BaseImageRepository) error) error { + panic("unexpected call") } diff --git a/central/baseimage/datastore/repository/mocks/datastore.go b/central/baseimage/datastore/repository/mocks/datastore.go index b56a122206f7c..66c8cfb8bc86b 100644 --- a/central/baseimage/datastore/repository/mocks/datastore.go +++ b/central/baseimage/datastore/repository/mocks/datastore.go @@ -13,6 +13,7 @@ import ( context "context" reflect "reflect" + repository "github.com/stackrox/rox/central/baseimage/datastore/repository" storage "github.com/stackrox/rox/generated/storage" gomock "go.uber.org/mock/gomock" ) @@ -86,6 +87,36 @@ func (mr *MockDataStoreMockRecorder) ListRepositories(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRepositories", reflect.TypeOf((*MockDataStore)(nil).ListRepositories), ctx) } +// UpdateConfiguration mocks base method. +func (m *MockDataStore) UpdateConfiguration(ctx context.Context, id string, update repository.ConfigUpdate) (*storage.BaseImageRepository, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateConfiguration", ctx, id, update) + ret0, _ := ret[0].(*storage.BaseImageRepository) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateConfiguration indicates an expected call of UpdateConfiguration. +func (mr *MockDataStoreMockRecorder) UpdateConfiguration(ctx, id, update any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfiguration", reflect.TypeOf((*MockDataStore)(nil).UpdateConfiguration), ctx, id, update) +} + +// UpdateStatus mocks base method. +func (m *MockDataStore) UpdateStatus(ctx context.Context, id string, update repository.StatusUpdate) (*storage.BaseImageRepository, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateStatus", ctx, id, update) + ret0, _ := ret[0].(*storage.BaseImageRepository) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateStatus indicates an expected call of UpdateStatus. +func (mr *MockDataStoreMockRecorder) UpdateStatus(ctx, id, update any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateStatus", reflect.TypeOf((*MockDataStore)(nil).UpdateStatus), ctx, id, update) +} + // UpsertRepository mocks base method. func (m *MockDataStore) UpsertRepository(ctx context.Context, repo *storage.BaseImageRepository) (*storage.BaseImageRepository, error) { m.ctrl.T.Helper() diff --git a/central/baseimage/datastore/repository/singleton.go b/central/baseimage/datastore/repository/singleton.go index 65b52f0bc6a5d..20139008223db 100644 --- a/central/baseimage/datastore/repository/singleton.go +++ b/central/baseimage/datastore/repository/singleton.go @@ -3,6 +3,7 @@ package repository import ( repoStore "github.com/stackrox/rox/central/baseimage/store/repository/postgres" "github.com/stackrox/rox/central/globaldb" + "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/sync" ) @@ -16,7 +17,7 @@ var ( // Singleton returns the global datastore instance for base image repositories. func Singleton() DataStore { once.Do(func() { - ds = New(repoStore.New(globaldb.GetPostgres())) + ds = New(repoStore.New(globaldb.GetPostgres()), concurrency.NewKeyFence()) log.Info("Initialized base image repository datastore with PostgreSQL backend") }) return ds diff --git a/central/baseimage/datastore/tag/datastore_impl_test.go b/central/baseimage/datastore/tag/datastore_impl_test.go index df17083431205..939b323df3c81 100644 --- a/central/baseimage/datastore/tag/datastore_impl_test.go +++ b/central/baseimage/datastore/tag/datastore_impl_test.go @@ -11,6 +11,7 @@ import ( repoStore "github.com/stackrox/rox/central/baseimage/store/repository/postgres" tagStore "github.com/stackrox/rox/central/baseimage/store/tag/postgres" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/uuid" @@ -33,7 +34,7 @@ func TestTagDataStoreIntegration(t *testing.T) { func (s *TagDataStoreIntegrationSuite) SetupSuite() { s.testDB = pgtest.ForT(s.T()) s.dataStore = New(tagStore.New(s.testDB.DB)) - s.repoDS = repoDS.New(repoStore.New(s.testDB.DB)) + s.repoDS = repoDS.New(repoStore.New(s.testDB.DB), concurrency.NewKeyFence()) s.ctx = sac.WithAllAccess(context.Background()) } @@ -175,7 +176,7 @@ func (s *TagDataStoreIntegrationSuite) TestUpsertMany_BatchOperation() { // Create 100 tags tags := make([]*storage.BaseImageTag, 100) - for i := 0; i < 100; i++ { + for i := range 100 { tags[i] = &storage.BaseImageTag{ Id: uuid.NewV4().String(), BaseImageRepositoryId: repo.GetId(), @@ -254,7 +255,7 @@ func (s *TagDataStoreIntegrationSuite) TestDeleteMany_BatchOperation() { // Create 10 tags tags := make([]*storage.BaseImageTag, 10) tagIDs := make([]string, 10) - for i := 0; i < 10; i++ { + for i := range 10 { tagID := uuid.NewV4().String() tags[i] = &storage.BaseImageTag{ Id: tagID, diff --git a/central/baseimage/matcher/matcher.go b/central/baseimage/matcher/matcher.go index a46315dd2e308..8f5bc09e3a104 100644 --- a/central/baseimage/matcher/matcher.go +++ b/central/baseimage/matcher/matcher.go @@ -8,5 +8,5 @@ import ( //go:generate mockgen-wrapper type Matcher interface { - MatchWithBaseImages(ctx context.Context, layers []string) ([]*storage.BaseImageInfo, error) + MatchWithBaseImages(ctx context.Context, layers []string) ([]*storage.BaseImage, error) } diff --git a/central/baseimage/matcher/matcher_impl.go b/central/baseimage/matcher/matcher_impl.go index 96a3f7a2032d9..9b43bc5482dea 100644 --- a/central/baseimage/matcher/matcher_impl.go +++ b/central/baseimage/matcher/matcher_impl.go @@ -28,7 +28,7 @@ func New( } } -func (m matcherImpl) MatchWithBaseImages(ctx context.Context, layers []string) ([]*storage.BaseImageInfo, error) { +func (m matcherImpl) MatchWithBaseImages(ctx context.Context, layers []string) ([]*storage.BaseImage, error) { start := time.Now() defer func() { @@ -45,15 +45,20 @@ func (m matcherImpl) MatchWithBaseImages(ctx context.Context, layers []string) ( if err != nil { return nil, fmt.Errorf("listing candidates for layer %s: %w", firstLayer, err) } - var baseImages []*storage.BaseImageInfo + var baseImages []*storage.BaseImage + maxLayers := 0 + for _, c := range candidates { candidateLayers := c.GetLayers() slices.SortFunc(candidateLayers, func(a, b *storage.BaseImageLayer) int { return int(a.GetIndex() - b.GetIndex()) }) + + // base images should always have less layers than a target image if len(layers) <= len(candidateLayers) { continue } + match := true for i, l := range candidateLayers { if layers[i] != l.GetLayerDigest() { @@ -63,13 +68,18 @@ func (m matcherImpl) MatchWithBaseImages(ctx context.Context, layers []string) ( } if match { - baseImages = append(baseImages, &storage.BaseImageInfo{ - BaseImageId: c.GetId(), - BaseImageFullName: fmt.Sprintf("%s:%s", c.GetRepository(), c.GetTag()), - BaseImageDigest: c.GetManifestDigest(), - Created: c.GetCreated(), - }) + n := len(candidateLayers) + + if n > maxLayers { + // Found a better (longer) match: reset the slice and update max + maxLayers = n + baseImages = []*storage.BaseImage{c} + } else if n == maxLayers { + // Found another match of the same (maximum) length + baseImages = append(baseImages, c) + } } } + return baseImages, nil } diff --git a/central/baseimage/matcher/matcher_impl_test.go b/central/baseimage/matcher/matcher_impl_test.go index f208c2bcfeef2..3cce996fdb865 100644 --- a/central/baseimage/matcher/matcher_impl_test.go +++ b/central/baseimage/matcher/matcher_impl_test.go @@ -27,7 +27,7 @@ func TestMatchWithBaseImages(t *testing.T) { desc string imgLayers []string mockSetup func() - expected []*storage.BaseImageInfo + expected []*storage.BaseImage }{ { desc: "Match found: Base image layers are returned out of order", @@ -49,12 +49,13 @@ func TestMatchWithBaseImages(t *testing.T) { }, }, nil) }, - expected: []*storage.BaseImageInfo{ + expected: []*storage.BaseImage{ { - BaseImageId: "base-1", - BaseImageFullName: "rhel:8", - BaseImageDigest: "sha-base", - Created: testCreatedTime, + Id: "base-1", + Repository: "rhel", + Tag: "8", + ManifestDigest: "sha-base", + Created: testCreatedTime, }, }, }, @@ -99,8 +100,8 @@ func TestMatchWithBaseImages(t *testing.T) { }, }, nil) }, - expected: []*storage.BaseImageInfo{ - {BaseImageId: "match"}, + expected: []*storage.BaseImage{ + {Id: "match"}, }, }, { @@ -120,6 +121,67 @@ func TestMatchWithBaseImages(t *testing.T) { }, nil) }, expected: nil, + }, { + desc: "Max layers only: Returns 3-layer match and ignores 2-layer match", + imgLayers: []string{"L1", "L2", "L3", "L4"}, + mockSetup: func() { + mockDS.EXPECT(). + ListCandidateBaseImages(gomock.Any(), "L1"). + Return([]*storage.BaseImage{ + { + Id: "short-match", // 2 layers + Layers: []*storage.BaseImageLayer{ + {LayerDigest: "L1", Index: 0}, + {LayerDigest: "L2", Index: 1}, + }, + }, + { + Id: "long-match", // 3 layers - the winner + Layers: []*storage.BaseImageLayer{ + {LayerDigest: "L1", Index: 0}, + {LayerDigest: "L2", Index: 1}, + {LayerDigest: "L3", Index: 2}, + }, + }, + }, nil) + }, + expected: []*storage.BaseImage{ + {Id: "long-match"}, + }, + }, + { + desc: "Max layers only: Returns multiple matches if they have same max length", + imgLayers: []string{"L1", "L2", "L3"}, + mockSetup: func() { + mockDS.EXPECT(). + ListCandidateBaseImages(gomock.Any(), "L1"). + Return([]*storage.BaseImage{ + { + Id: "match-A", + Layers: []*storage.BaseImageLayer{ + {LayerDigest: "L1", Index: 0}, + {LayerDigest: "L2", Index: 1}, + }, + }, + { + Id: "match-B", + Layers: []*storage.BaseImageLayer{ + {LayerDigest: "L1", Index: 0}, + {LayerDigest: "L2", Index: 1}, + }, + }, + { + Id: "too-short", + Layers: []*storage.BaseImageLayer{ + {LayerDigest: "L1", Index: 0}, + }, + }, + }, nil) + }, + expected: []*storage.BaseImage{ + {Id: "match-A"}, + {Id: "match-B"}, + }, }, } @@ -134,9 +196,10 @@ func TestMatchWithBaseImages(t *testing.T) { } else { require.Equal(t, len(tc.expected), len(actual)) for i := range tc.expected { - assert.Equal(t, tc.expected[i].GetBaseImageId(), actual[i].GetBaseImageId()) - if tc.expected[i].GetBaseImageFullName() != "" { - assert.Equal(t, tc.expected[i].GetBaseImageFullName(), actual[i].GetBaseImageFullName()) + assert.Equal(t, tc.expected[i].GetId(), actual[i].GetId()) + if tc.expected[i].GetRepository() != "" { + assert.Equal(t, tc.expected[i].GetRepository(), actual[i].GetRepository()) + assert.Equal(t, tc.expected[i].GetTag(), actual[i].GetTag()) } if tc.expected[i].GetCreated() != nil { assert.Equal(t, tc.expected[i].GetCreated(), actual[i].GetCreated()) diff --git a/central/baseimage/matcher/mocks/matcher.go b/central/baseimage/matcher/mocks/matcher.go index fb015f20f03e7..567d6287b7d72 100644 --- a/central/baseimage/matcher/mocks/matcher.go +++ b/central/baseimage/matcher/mocks/matcher.go @@ -42,10 +42,10 @@ func (m *MockMatcher) EXPECT() *MockMatcherMockRecorder { } // MatchWithBaseImages mocks base method. -func (m *MockMatcher) MatchWithBaseImages(ctx context.Context, layers []string) ([]*storage.BaseImageInfo, error) { +func (m *MockMatcher) MatchWithBaseImages(ctx context.Context, layers []string) ([]*storage.BaseImage, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "MatchWithBaseImages", ctx, layers) - ret0, _ := ret[0].([]*storage.BaseImageInfo) + ret0, _ := ret[0].([]*storage.BaseImage) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/central/baseimage/service/service_impl.go b/central/baseimage/service/service_impl.go index fff566432b080..2c163a6817495 100644 --- a/central/baseimage/service/service_impl.go +++ b/central/baseimage/service/service_impl.go @@ -2,8 +2,11 @@ package service import ( "context" + "errors" "fmt" "path" + "strings" + "unicode" "github.com/distribution/reference" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" @@ -99,20 +102,16 @@ func (s *serviceImpl) UpdateBaseImageTagPattern(ctx context.Context, req *v2.Upd return nil, status.Errorf(codes.InvalidArgument, "invalid base image tag pattern: %v", err) } - // First get the existing repository - existing, found, err := s.datastore.GetRepository(ctx, req.GetId()) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get base image repository: %v", err) - } - if !found { - return nil, status.Errorf(codes.NotFound, "base image repository with ID %q not found", req.GetId()) - } - - // Update the repository - existing.TagPattern = req.GetBaseImageTagPattern() - - _, err = s.datastore.UpsertRepository(ctx, existing) + _, err := s.datastore.UpdateConfiguration(ctx, req.GetId(), repository.ConfigUpdate{ + TagPattern: new(req.GetBaseImageTagPattern()), + }) if err != nil { + if errors.Is(err, repository.ErrScanInProgress) { + return nil, errox.ResourceExhausted.Newf("base image repository with ID %q is currently being scanned and cannot be changed, try again later", req.GetId()) + } + if errors.Is(err, errox.NotFound) { + return nil, errox.NotFound.Newf("base image repository with ID %q not found", req.GetId()) + } return nil, status.Errorf(codes.Internal, "failed to update base image repository: %v", err) } @@ -162,23 +161,22 @@ func convertStorageToAPI(repo *storage.BaseImageRepository) *v2.BaseImageReferen } } -// validateBaseImageRepository validate the provided base image repository path, -// Returns an error if the repository path is malformed or no matching image integration is found. +// validateBaseImageRepository returns an error when the repository path is malformed or no matching image integration exists. func (s *serviceImpl) validateBaseImageRepository(ctx context.Context, repoPath string) error { imageName, ref, err := imgUtils.GenerateImageNameFromString(repoPath) if err != nil { return errox.InvalidArgs.Newf("invalid base image repository path '%s'", repoPath).CausedBy(err) } - // Reject references that include tags - these should be in separate fields + // Reject references that include tags. These should be stored in a separate field. if imageName.GetTag() != "" { return errox.InvalidArgs.Newf("repository path '%s' must not include tag - please put tag in the tag pattern field", repoPath) } - // Reject references that include digests + // Reject references that include digests. if _, ok := ref.(reference.Digested); ok { return errox.InvalidArgs.Newf("repository path '%s' must not include digest", repoPath) } - // Check delegated registry config (only if feature is enabled). + // Check delegated registry config only if the feature is enabled. var shouldDelegate bool if features.DelegatedBaseImageScanning.Enabled() { _, shouldDelegate, _ = s.delegator.GetDelegateClusterID(ctx, imageName) @@ -193,14 +191,29 @@ func (s *serviceImpl) validateBaseImageRepository(ctx context.Context, repoPath return nil } -// isValidTagPattern checks if the given string is a valid [path.Match] glob pattern. +// maxTagPatternLength bounds a tag pattern to the maximum length of a container image +// tag: one leading character plus up to 127 more (see [reference.TagRegexp]). +const maxTagPatternLength = 128 + +// isValidTagPattern checks that the given string is a glob pattern that could match a +// container image tag. Besides validating the [path.Match] glob syntax, it rejects +// characters that are illegal in a tag ('/', ':', '@', whitespace). Their presence +// signals that the repository path or a registry port leaked into the tag field - +// common when the registry includes a port and the user mistypes the input. Such a +// pattern can never match a real tag, so it is rejected here instead of silently +// matching nothing. func isValidTagPattern(tagPattern string) (bool, error) { - // Reject empty tag patterns, should use * instead + // Reject empty tag patterns. Use * to match all tags. if tagPattern == "" { return false, errox.InvalidArgs.New("tag pattern cannot be empty") } - // Validate the pattern by attempting to match it against an empty string. - // If the pattern is malformed, path.Match will return an error. + if len(tagPattern) > maxTagPatternLength { + return false, errox.InvalidArgs.Newf("tag pattern is too long: %d bytes (max %d)", len(tagPattern), maxTagPatternLength) + } + if strings.ContainsAny(tagPattern, "/:@") || strings.ContainsFunc(tagPattern, unicode.IsSpace) { + return false, errox.InvalidArgs.Newf("tag pattern '%s' must not contain '/', ':', '@' or whitespace - for a registry with a port, set baseImageRepoPath to 'registry:port/repo' and baseImageTagPattern to the tag pattern (e.g. '1.*')", tagPattern) + } + // path.Match validates the pattern and returns an error for malformed input. _, err := path.Match(tagPattern, "") if err != nil { return false, errox.InvalidArgs.CausedBy(fmt.Errorf("invalid tag pattern: %w", err)) diff --git a/central/baseimage/service/service_impl_test.go b/central/baseimage/service/service_impl_test.go index 8cfc91399706e..5c94a73ec53f6 100644 --- a/central/baseimage/service/service_impl_test.go +++ b/central/baseimage/service/service_impl_test.go @@ -6,13 +6,14 @@ import ( "strings" "testing" + "github.com/stackrox/rox/central/baseimage/datastore/repository" "github.com/stackrox/rox/central/baseimage/datastore/repository/mocks" v2 "github.com/stackrox/rox/generated/api/v2" "github.com/stackrox/rox/generated/storage" delegatorMocks "github.com/stackrox/rox/pkg/delegatedregistry/mocks" + "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/features" integrationMocks "github.com/stackrox/rox/pkg/images/integration/mocks" - "github.com/stackrox/rox/pkg/pointers" registryMocks "github.com/stackrox/rox/pkg/registries/mocks" "github.com/stackrox/rox/pkg/testutils" "github.com/stretchr/testify/suite" @@ -85,32 +86,32 @@ func (suite *ServiceTestSuite) TestValidateBaseImageRepository() { input: "test_com", expectedValid: true, expectedErrMsg: "", - expectRegistryMatch: pointers.Bool(true), - expectDelegation: pointers.Bool(false), + expectRegistryMatch: new(true), + expectDelegation: new(false), }, { description: "accepts standard Docker registry format with organization", input: "docker.io/library/nginx", expectedValid: true, expectedErrMsg: "", - expectRegistryMatch: pointers.Bool(true), - expectDelegation: pointers.Bool(false), + expectRegistryMatch: new(true), + expectDelegation: new(false), }, { description: "accepts IPv4 registry with custom port", input: "192.168.1.1:5000/myapp", expectedValid: true, expectedErrMsg: "", - expectRegistryMatch: pointers.Bool(true), - expectDelegation: pointers.Bool(false), + expectRegistryMatch: new(true), + expectDelegation: new(false), }, { description: "accepts IPv6 registry with repository path", input: "[2001:db8::1]/repo", expectedValid: true, expectedErrMsg: "", - expectRegistryMatch: pointers.Bool(true), - expectDelegation: pointers.Bool(false), + expectRegistryMatch: new(true), + expectDelegation: new(false), }, { description: "rejects empty repository path", @@ -147,24 +148,24 @@ func (suite *ServiceTestSuite) TestValidateBaseImageRepository() { input: "docker.io/library/nginx", expectedValid: false, expectedErrMsg: "no matching image integration found: please add an image integration for 'docker.io/library/nginx'", - expectRegistryMatch: pointers.Bool(false), - expectDelegation: pointers.Bool(false), + expectRegistryMatch: new(false), + expectDelegation: new(false), }, { description: "accepts repository path with matching central integration", input: "delegated.registry.com/myapp", expectedValid: true, expectedErrMsg: "", - expectRegistryMatch: pointers.Bool(true), - expectDelegation: pointers.Bool(false), + expectRegistryMatch: new(true), + expectDelegation: new(false), }, { description: "rejects repository path when neither central integration nor delegation exists", input: "unknown.registry.com/myapp", expectedValid: false, expectedErrMsg: "no matching image integration found: please add an image integration for 'unknown.registry.com/myapp'", - expectRegistryMatch: pointers.Bool(false), - expectDelegation: pointers.Bool(false), + expectRegistryMatch: new(false), + expectDelegation: new(false), }, } @@ -232,6 +233,42 @@ func (suite *ServiceTestSuite) TestIsValidTagPattern() { expectedValid: true, expectedErrMsg: "", }, + { + description: "rejects pattern with slash (repo path leaked into tag field)", + input: "5000/library/ubuntu*", + expectedValid: false, + expectedErrMsg: "set baseImageRepoPath to 'registry:port/repo' and baseImageTagPattern to the tag pattern", + }, + { + description: "rejects pattern with colon (registry port leaked into tag field)", + input: "5000:latest", + expectedValid: false, + expectedErrMsg: "must not contain", + }, + { + description: "rejects pattern with digest separator", + input: "sha256@abc", + expectedValid: false, + expectedErrMsg: "must not contain", + }, + { + description: "rejects pattern with whitespace", + input: "1. *", + expectedValid: false, + expectedErrMsg: "must not contain", + }, + { + description: "rejects pattern with non-ASCII whitespace", + input: "1.\u00a0*", + expectedValid: false, + expectedErrMsg: "must not contain", + }, + { + description: "rejects pattern longer than max tag length", + input: strings.Repeat("a", maxTagPatternLength+1), + expectedValid: false, + expectedErrMsg: "129 bytes (max 128)", + }, } for _, tt := range tests { @@ -333,6 +370,7 @@ func (suite *ServiceTestSuite) TestUpdateBaseImageTagPattern() { mockSetup func() expectedError bool errorContains string + errorIs error }{ { description: "updates tag pattern with valid glob", @@ -341,15 +379,7 @@ func (suite *ServiceTestSuite) TestUpdateBaseImageTagPattern() { BaseImageTagPattern: "8.*", }, mockSetup: func() { - existing := &storage.BaseImageRepository{ - Id: "test-id", - RepositoryPath: "nginx", - TagPattern: "latest", - } - suite.mockDatastore.EXPECT().GetRepository(gomock.Any(), "test-id"). - Return(existing, true, nil) - suite.mockDatastore.EXPECT().UpsertRepository(gomock.Any(), gomock.Any()). - Return(nil, nil) + suite.mockDatastore.EXPECT().UpdateConfiguration(gomock.Any(), "test-id", gomock.Any()).Return(&storage.BaseImageRepository{}, nil) }, expectedError: false, }, @@ -373,6 +403,32 @@ func (suite *ServiceTestSuite) TestUpdateBaseImageTagPattern() { expectedError: true, errorContains: "base image reference ID is required", }, + { + description: "returns retriable error when repository scanning is already in progress", + request: &v2.UpdateBaseImageTagPatternRequest{ + Id: "test-id", + BaseImageTagPattern: "8.*", + }, + mockSetup: func() { + suite.mockDatastore.EXPECT().UpdateConfiguration(gomock.Any(), "test-id", gomock.Any()).Return(nil, repository.ErrScanInProgress) + }, + expectedError: true, + errorContains: "is currently being scanned and cannot be changed", + errorIs: errox.ResourceExhausted, + }, + { + description: "returns not found when repository disappears before update", + request: &v2.UpdateBaseImageTagPatternRequest{ + Id: "test-id", + BaseImageTagPattern: "8.*", + }, + mockSetup: func() { + suite.mockDatastore.EXPECT().UpdateConfiguration(gomock.Any(), "test-id", gomock.Any()).Return(nil, errox.NotFound.Newf("base image repository %q not found", "test-id")) + }, + expectedError: true, + errorContains: "base image repository with ID \"test-id\" not found", + errorIs: errox.NotFound, + }, } for _, tt := range tests { @@ -384,6 +440,9 @@ func (suite *ServiceTestSuite) TestUpdateBaseImageTagPattern() { if tt.expectedError { suite.Error(err, "expected error") suite.Contains(err.Error(), tt.errorContains, "error message should contain expected text") + if tt.errorIs != nil { + suite.ErrorIs(err, tt.errorIs) + } } else { suite.NoError(err, "expected no error") } diff --git a/central/baseimage/service/singleton.go b/central/baseimage/service/singleton.go index a3ad7d955ed19..6b7f5ef540112 100644 --- a/central/baseimage/service/singleton.go +++ b/central/baseimage/service/singleton.go @@ -27,6 +27,7 @@ func initialize() { scanwaiter.Singleton(), scanwaiterv2.Singleton(), sachelper.NewClusterNamespaceSacHelper(clusterDataStore.Singleton(), namespaceDataStore.Singleton()), + clusterDataStore.Singleton(), ) as = New(repository.Singleton(), imageintegration.Set(), scanDelegator) diff --git a/central/baseimage/store/tag/postgres/store_integration_test.go b/central/baseimage/store/tag/postgres/store_integration_test.go index e752cc59a3360..636e1d909b6b4 100644 --- a/central/baseimage/store/tag/postgres/store_integration_test.go +++ b/central/baseimage/store/tag/postgres/store_integration_test.go @@ -92,7 +92,7 @@ func (s *BaseImageTagsIntegrationSuite) TestCascadeDelete() { s.NoError(s.repoStore.Upsert(ctx, repo)) // Create multiple tags - for i := 0; i < 3; i++ { + for i := range 3 { tag := &storage.BaseImageTag{ Id: uuid.NewV4().String(), BaseImageRepositoryId: repo.GetId(), diff --git a/central/baseimage/watcher/metrics.go b/central/baseimage/watcher/metrics.go index 8dd40cf3373be..22572ef0691e3 100644 --- a/central/baseimage/watcher/metrics.go +++ b/central/baseimage/watcher/metrics.go @@ -24,7 +24,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: "base_image_watcher", Name: "poll_duration_seconds", - Help: "Time taken to complete a poll cycle", + Help: "Time spent completing a poll cycle", Buckets: prometheus.ExponentialBuckets(0.1, 2, 10), // 0.1s to ~102s }, []string{"error"}, @@ -63,7 +63,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: "base_image_watcher", Name: "scan_duration_seconds", - Help: "Time taken to scan a repository (list tags + fetch metadata)", + Help: "Time spent scanning a repository (list tags + fetch metadata)", Buckets: prometheus.ExponentialBuckets(0.1, 2, 12), // 0.1s to ~409s }, []string{ "registry_domain", diff --git a/central/baseimage/watcher/singleton.go b/central/baseimage/watcher/singleton.go index d796d5d278114..5d48ef671fa25 100644 --- a/central/baseimage/watcher/singleton.go +++ b/central/baseimage/watcher/singleton.go @@ -32,6 +32,7 @@ func Singleton() Watcher { scanwaiter.Singleton(), scanwaiterv2.Singleton(), sachelper.NewClusterNamespaceSacHelper(clusterDataStore.Singleton(), namespaceDataStore.Singleton()), + clusterDataStore.Singleton(), ) watcher = New( @@ -41,8 +42,10 @@ func Singleton() Watcher { imageintegration.Set().RegistrySet(), scanDelegator, env.BaseImageWatcherPollInterval.DurationSetting(), + env.BaseImageWatcherSchedulerCadence.DurationSetting(), env.BaseImageWatcherTagBatchSize.IntegerSetting(), env.BaseImageWatcherPerRepoTagLimit.IntegerSetting(), + env.BaseImageWatcherMaxConcurrentRepositories.IntegerSetting(), features.DelegatedBaseImageScanning.Enabled(), ) }) diff --git a/central/baseimage/watcher/watcher_impl.go b/central/baseimage/watcher/watcher_impl.go index db40f19079e10..e71775e697abc 100644 --- a/central/baseimage/watcher/watcher_impl.go +++ b/central/baseimage/watcher/watcher_impl.go @@ -1,8 +1,10 @@ package watcher import ( + "cmp" "context" "fmt" + "slices" "time" "github.com/pkg/errors" @@ -13,7 +15,6 @@ import ( "github.com/stackrox/rox/pkg/baseimage/reposcan" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/delegatedregistry" - "github.com/stackrox/rox/pkg/env" imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/protocompat" @@ -38,37 +39,54 @@ type watcherImpl struct { startedOnce sync.Once stoppedOnce sync.Once - pollInterval time.Duration - batchSize int - tagLimit int + pollInterval time.Duration + schedulerCadence time.Duration + batchSize int + tagLimit int delegationEnabled bool + + // sem limits concurrent repository scans to maxConcurrent. + sem *semaphore.Weighted + + // wg tracks active scan goroutines for graceful shutdown. + wg sync.WaitGroup + + // pendingStatus holds repos where scan completed but status update failed. + pendingStatus map[string]repoDS.StatusUpdate + pendingStatusMu sync.RWMutex } // New creates a new base image watcher. func New( - repoDS repoDS.DataStore, + repositoryDS repoDS.DataStore, tagDS tagDS.DataStore, baseImageDS baseImageDS.DataStore, registries registries.Set, delegator delegatedregistry.Delegator, pollInterval time.Duration, + schedulerCadence time.Duration, batchSize int, tagLimit int, + maxConcurrent int, delegationEnabled bool, ) Watcher { return &watcherImpl{ - repoDS: repoDS, - tagDS: tagDS, - baseImageDS: baseImageDS, - delegator: delegator, - localScanner: reposcan.NewLocalScanner(registries), - stopper: concurrency.NewStopper(), - pollInterval: pollInterval, - batchSize: batchSize, - tagLimit: tagLimit, + repoDS: repositoryDS, + tagDS: tagDS, + baseImageDS: baseImageDS, + delegator: delegator, + localScanner: reposcan.NewLocalScanner(registries), + stopper: concurrency.NewStopper(), + pollInterval: pollInterval, + schedulerCadence: schedulerCadence, + batchSize: batchSize, + tagLimit: tagLimit, delegationEnabled: delegationEnabled, + + sem: semaphore.NewWeighted(int64(maxConcurrent)), + pendingStatus: make(map[string]repoDS.StatusUpdate), } } @@ -80,31 +98,36 @@ func (w *watcherImpl) Start() { }) } -// Stop signals shutdown and blocks until polling goroutine exits. +// Stop signals shutdown and blocks until all goroutines exit. // Subsequent calls are no-ops. func (w *watcherImpl) Stop() { w.stoppedOnce.Do(func() { w.stopper.Client().Stop() _ = w.stopper.Client().Stopped().Wait() + w.wg.Wait() }) } -// run is the main polling loop, runs until Stop() is called. +// run is the main scheduling loop, runs until Stop() is called. func (w *watcherImpl) run() { + log.Info("Base image watcher started") + defer w.stopper.Flow().ReportStopped() - log.Info("Base image watcher started") + ctx := sac.WithAllAccess(concurrency.AsContext(w.stopper.LowLevel().GetStopRequestSignal())) - // Poll immediately on start - w.pollOnce() + // Fail in-flight scans before starting the scheduler. At this point no scan + // goroutines exist, so any repo in QUEUED/IN_PROGRESS is leftover from a + // previous crash. + w.failInFlightScans(ctx) - ticker := time.NewTicker(w.pollInterval) + ticker := time.NewTicker(w.schedulerCadence) defer ticker.Stop() for { select { case <-ticker.C: - w.pollOnce() + w.schedulerPass(ctx) case <-w.stopper.Flow().StopRequested(): log.Info("Base image watcher stopped") return @@ -112,91 +135,205 @@ func (w *watcherImpl) run() { } } -// pollOnce executes a single poll cycle with metric tracking. -func (w *watcherImpl) pollOnce() { +// schedulerPass executes a single scheduler pass with metric tracking. +func (w *watcherImpl) schedulerPass(ctx context.Context) { + w.updateStatus(ctx) + start := time.Now() - err := w.doPoll() + claimed, err := w.doSchedulerPass(ctx) recordPollDuration(time.Since(start).Seconds(), err) if err != nil { - log.Errorf("Base image watcher poll cycle failed: duration=%v: %v", time.Since(start), err) + log.Errorf("Base image watcher scheduler pass failed: duration=%v: %v", time.Since(start), err) + } else if claimed > 0 { + log.Infof("Base image watcher scheduler pass completed: duration=%v claimed=%d", time.Since(start), claimed) } else { - log.Infof("Base image watcher poll cycle completed: duration=%v", time.Since(start)) + log.Debugf("Base image watcher scheduler pass completed: duration=%v claimed=%d", time.Since(start), claimed) } } -// doPoll contains the core poll logic. -func (w *watcherImpl) doPoll() error { - log.Debug("Starting base image watcher poll cycle") - - ctx := concurrency.AsContext(w.stopper.LowLevel().GetStopRequestSignal()) - - // Use all access since the watcher is an internal Central component. - ctx = sac.WithAllAccess(ctx) +// doSchedulerPass lists repositories, claims due ones, and spawns scans. +func (w *watcherImpl) doSchedulerPass(ctx context.Context) (int, error) { + log.Debug("Starting base image watcher scheduler pass") repos, err := w.repoDS.ListRepositories(ctx) if err != nil { - return fmt.Errorf("listing repositories: %w", err) + return 0, fmt.Errorf("listing repositories: %w", err) } + recordRepositoryCount(len(repos)) + if len(repos) == 0 { - log.Info("No base image repositories configured, skipping poll cycle") - recordRepositoryCount(0) - return nil + log.Debug("No base image repositories configured") + return 0, nil } - log.Debugf("Processing repositories: count=%d", len(repos)) - recordRepositoryCount(len(repos)) + // Sort by last_polled_at ascending so repos waiting longest get priority. + // Repos with nil last_polled_at (never scanned) come first. + slices.SortFunc(repos, func(a, b *storage.BaseImageRepository) int { + aTime := a.GetLastPolledAt().AsTime() + bTime := b.GetLastPolledAt().AsTime() + return cmp.Compare(aTime.UnixNano(), bTime.UnixNano()) + }) + + // Claim due repositories and spawn scans. + var claimedCount int + for _, repo := range repos { + if !isRepositoryDue(repo, w.pollInterval) { + continue + } + if !w.sem.TryAcquire(1) { + break + } + claimedRepo, err := w.repoDS.UpdateStatus(ctx, repo.GetId(), repoDS.StatusUpdate{ + Status: storage.BaseImageRepository_QUEUED, + OnlyIfStatus: []storage.BaseImageRepository_Status{ + storage.BaseImageRepository_CREATED, + storage.BaseImageRepository_READY, + storage.BaseImageRepository_FAILED, + }, + }) + if err != nil || claimedRepo == nil { + w.sem.Release(1) + if err != nil { + log.Errorf("Failed to claim repository %q: %v", repo.GetRepositoryPath(), err) + } + continue + } + claimedCount++ + w.wg.Add(1) + go w.scanRepository(ctx, claimedRepo) + } + + if claimedCount > 0 { + log.Debugf("Claimed repositories for scanning: count=%d", claimedCount) + } + + return claimedCount, nil +} - // Process repositories concurrently with bounded parallelism. - maxConcurrent := env.BaseImageWatcherMaxConcurrentRepositories.IntegerSetting() - sem := semaphore.NewWeighted(int64(maxConcurrent)) - wg := &sync.WaitGroup{} - defer wg.Wait() // Wait for any goroutines launched before returning +// failInFlightScans marks repositories in QUEUED or IN_PROGRESS state as FAILED. +func (w *watcherImpl) failInFlightScans(ctx context.Context) { + repos, err := w.repoDS.ListRepositories(ctx) + if err != nil { + log.Errorf("Failed to list repositories: %v", err) + return + } for _, repo := range repos { - if err := sem.Acquire(ctx, 1); err != nil { - return fmt.Errorf("interrupted during semaphore acquire: %w", err) + status := repo.GetStatus() + if status != storage.BaseImageRepository_QUEUED && status != storage.BaseImageRepository_IN_PROGRESS { + continue + } + msg := "scan interrupted by restart" + _, err := w.repoDS.UpdateStatus(ctx, repo.GetId(), repoDS.StatusUpdate{ + Status: storage.BaseImageRepository_FAILED, + LastFailureMessage: &msg, + FailureCountOp: repoDS.FailureCountIncrement, + }) + if err != nil { + log.Errorf("Failed to update in-flight scan %q: %v", repo.GetRepositoryPath(), err) + } else { + log.Warnf("Marked in-flight scan as failed: %q was %s", repo.GetRepositoryPath(), status) } - wg.Add(1) - go func(r *storage.BaseImageRepository) { - defer sem.Release(1) - defer wg.Done() - w.processRepository(ctx, r) - }(repo) } +} - return nil +// scanRepository scans a claimed repository (QUEUED → IN_PROGRESS → READY/FAILED). +func (w *watcherImpl) scanRepository(ctx context.Context, repo *storage.BaseImageRepository) { + defer w.wg.Done() + defer w.sem.Release(1) + + id := repo.GetId() + log.Debugf("Scanning repository: repository=%q pattern=%q", + repo.GetRepositoryPath(), repo.GetTagPattern()) + + scanErr := w.doScan(ctx, repo) + + // Build final status update. + now := time.Now() + update := repoDS.StatusUpdate{ + Status: storage.BaseImageRepository_READY, + LastPolledAt: &now, + FailureCountOp: repoDS.FailureCountReset, + } + if scanErr != nil { + update.Status = storage.BaseImageRepository_FAILED + s := scanErr.Error() + update.LastFailureMessage = &s + update.FailureCountOp = repoDS.FailureCountIncrement + } + + // Persist. + _, err := w.repoDS.UpdateStatus(ctx, id, update) + if err != nil { + log.Errorf("Failed to save status for %q: %v", repo.GetRepositoryPath(), err) + concurrency.WithLock(&w.pendingStatusMu, func() { + w.pendingStatus[id] = update + }) + } +} + +// updateStatus retries pending status updates from previous failures. +func (w *watcherImpl) updateStatus(ctx context.Context) { + ids := concurrency.WithRLock1(&w.pendingStatusMu, func() []string { + result := make([]string, 0, len(w.pendingStatus)) + for id := range w.pendingStatus { + result = append(result, id) + } + return result + }) + + for _, id := range ids { + update, ok := concurrency.WithRLock2(&w.pendingStatusMu, func() (repoDS.StatusUpdate, bool) { + u, exists := w.pendingStatus[id] + return u, exists + }) + if !ok { + continue + } + + now := time.Now() + update.LastPolledAt = &now + _, err := w.repoDS.UpdateStatus(ctx, id, update) + if err != nil { + log.Errorf("Failed to update status for %q: %v", id, err) + } else { + concurrency.WithLock(&w.pendingStatusMu, func() { + delete(w.pendingStatus, id) + }) + } + } } -// processRepository processes a single repository. -func (w *watcherImpl) processRepository(ctx context.Context, repo *storage.BaseImageRepository) { - log.Debugf("Processing repository: repository=%q pattern=%q", - repo.GetRepositoryPath(), - repo.GetTagPattern()) +// doScan performs the actual scan of a repository. Returns an error if the scan fails. +func (w *watcherImpl) doScan(ctx context.Context, repo *storage.BaseImageRepository) error { + // Transition to IN_PROGRESS. + _, err := w.repoDS.UpdateStatus(ctx, repo.GetId(), repoDS.StatusUpdate{ + Status: storage.BaseImageRepository_IN_PROGRESS, + }) + if err != nil { + return fmt.Errorf("failed to set IN_PROGRESS: %w", err) + } // Validate repository ID is a valid UUID. if _, err := uuid.FromString(repo.GetId()); err != nil { - utils.Should(fmt.Errorf("repository ID is not a valid UUID: id=%q repository=%q: %w", - repo.GetId(), repo.GetRepositoryPath(), err)) - return + return fmt.Errorf("repository ID is not a valid UUID: id=%q repository=%q: %w", + repo.GetId(), repo.GetRepositoryPath(), err) } name, _, err := imageUtils.GenerateImageNameFromString(repo.GetRepositoryPath()) if err != nil { - utils.Should(fmt.Errorf("failed to parse repository path %q: %w", repo.GetRepositoryPath(), err)) - return + return fmt.Errorf("failed to parse repository path %q: %w", repo.GetRepositoryPath(), err) } if repo.GetTagPattern() == "" { - utils.Should(fmt.Errorf("tag pattern is empty: repository: %q", repo.GetRepositoryPath())) - return + return fmt.Errorf("tag pattern is empty: repository: %q", repo.GetRepositoryPath()) } // Check for context cancellation (shutdown during processing) select { case <-ctx.Done(): - log.Warnf("Repository processing cancelled: repository=%q", repo.GetRepositoryPath()) - return + return fmt.Errorf("repository processing cancelled: repository=%q", repo.GetRepositoryPath()) default: } @@ -219,8 +356,7 @@ func (w *watcherImpl) processRepository(ctx context.Context, repo *storage.BaseI // Fetch existing tags from cache (sorted by created timestamp, newest first). tags, err := w.tagDS.ListTagsByRepository(ctx, repo.GetId()) if err != nil { - log.Errorf("Failed to list tags: repository=%q: %v", repo.GetRepositoryPath(), err) - return + return fmt.Errorf("failed to list tags: repository=%q: %w", repo.GetRepositoryPath(), err) } // Build scan request. @@ -230,7 +366,7 @@ func (w *watcherImpl) processRepository(ctx context.Context, repo *storage.BaseI SkipTags: make(map[string]struct{}), } for i, t := range tags { - if i < env.BaseImageWatcherPerRepoTagLimit.IntegerSetting() { + if i < w.tagLimit { req.CheckTags[t.GetTag()] = t } else { req.SkipTags[t.GetTag()] = struct{}{} @@ -246,12 +382,14 @@ func (w *watcherImpl) processRepository(ctx context.Context, repo *storage.BaseI var metadataCount, errorCount, deleteCount int var adds []*storage.BaseImageTag var dels []string + var scanErr error for event, err := range scanner.ScanRepository(ctx, repo, req) { log.Debugf("Processing repository: scan event: err=%v event=%v repo=%v", err, event, repo) if err != nil { log.Errorf("Error during repository scan: repository=%q: %v", repo.GetRepositoryPath(), err) + scanErr = err break } @@ -335,6 +473,8 @@ func (w *watcherImpl) processRepository(ctx context.Context, repo *storage.BaseI log.Infof("Repository scan completed: repository=%q pattern=%q processed=%d metadata=%d errors=%d deletes=%d", repo.GetRepositoryPath(), repo.GetTagPattern(), metadataCount+errorCount+deleteCount, metadataCount, errorCount, deleteCount) + + return scanErr } // promoteTags promotes the top-N tags by created timestamp from cache to base_images. @@ -430,3 +570,19 @@ func tagUUID(repoID, tag string) (string, error) { } return uuid.NewV5(repoUUID, tag).String(), nil } + +// isRepositoryDue returns true if a repository is eligible for polling. +func isRepositoryDue(repo *storage.BaseImageRepository, pollInterval time.Duration) bool { + switch repo.GetStatus() { + case storage.BaseImageRepository_CREATED: + return true + case storage.BaseImageRepository_READY, storage.BaseImageRepository_FAILED: + lastPolled := repo.GetLastPolledAt() + if lastPolled == nil { + return true + } + return lastPolled.AsTime().Add(pollInterval).Before(time.Now()) + default: + return false + } +} diff --git a/central/baseimage/watcher/watcher_impl_test.go b/central/baseimage/watcher/watcher_impl_test.go index ba331a9b0d90a..747e3910fc64e 100644 --- a/central/baseimage/watcher/watcher_impl_test.go +++ b/central/baseimage/watcher/watcher_impl_test.go @@ -7,21 +7,25 @@ import ( "time" baseImageDSMocks "github.com/stackrox/rox/central/baseimage/datastore/mocks" + repoDS "github.com/stackrox/rox/central/baseimage/datastore/repository" repoDSMocks "github.com/stackrox/rox/central/baseimage/datastore/repository/mocks" tagDSMocks "github.com/stackrox/rox/central/baseimage/datastore/tag/mocks" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/baseimage/reposcan" "github.com/stackrox/rox/pkg/baseimage/tagfetcher" delegatedRegistryMocks "github.com/stackrox/rox/pkg/delegatedregistry/mocks" - "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/protocompat" registryMocks "github.com/stackrox/rox/pkg/registries/mocks" "github.com/stackrox/rox/pkg/registries/types" registryTypesMocks "github.com/stackrox/rox/pkg/registries/types/mocks" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sync" + "github.com/stackrox/rox/pkg/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -54,26 +58,36 @@ func createTestWatcher( mockTagDS.EXPECT().ListTagsByRepository(gomock.Any(), gomock.Any()).Return([]*storage.BaseImageTag{}, nil).AnyTimes() mockTagDS.EXPECT().UpsertMany(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - return New(mockRepoDS, mockTagDS, mockBaseImageDS, mockRegistrySet, mockDelegator, poll, 10, 0, delegationEnabled) + // Allow repository status updates (for new scheduling architecture). + mockRepoDS.EXPECT().UpdateStatus(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + + return New(mockRepoDS, mockTagDS, mockBaseImageDS, mockRegistrySet, mockDelegator, poll, 10*time.Millisecond, 10, 100, 5, delegationEnabled) +} + +func runSchedulerPassAndWait(w Watcher) { + impl := w.(*watcherImpl) + ctx := sac.WithAllAccess(context.Background()) + impl.schedulerPass(ctx) + impl.wg.Wait() } func TestWatcher_StartsAndStops(t *testing.T) { ctrl := gomock.NewController(t) mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) - // Polls immediately on start + 1 timed poll after 100ms (with 150ms sleep) + // Scheduler runs on cadence tick, number of calls depends on timing. mockRepoDS.EXPECT(). ListRepositories(gomock.Any()). Return([]*storage.BaseImageRepository{}, nil). - Times(2) + AnyTimes() w := createTestWatcher(ctrl, mockRepoDS, nil, nil, nil, 100*time.Millisecond, false) // Start watcher w.Start() - // Let it run briefly (should trigger 2 polls: immediate + 1 timed) - time.Sleep(150 * time.Millisecond) + // Let it run briefly + time.Sleep(50 * time.Millisecond) // Stop should complete quickly done := make(chan struct{}) @@ -90,31 +104,34 @@ func TestWatcher_StartsAndStops(t *testing.T) { } } -func TestWatcher_PollsImmediately(t *testing.T) { +func TestWatcher_PollsOnFirstSchedulerTick(t *testing.T) { ctrl := gomock.NewController(t) mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) - pollCalled := make(chan struct{}) + pollCalled := make(chan struct{}, 1) mockRepoDS.EXPECT(). ListRepositories(gomock.Any()). DoAndReturn(func(ctx context.Context) ([]*storage.BaseImageRepository, error) { - close(pollCalled) + select { + case pollCalled <- struct{}{}: + default: + } return []*storage.BaseImageRepository{}, nil }). - Times(1) + AnyTimes() w := createTestWatcher(ctrl, mockRepoDS, nil, nil, nil, 1*time.Hour, false) w.Start() defer w.Stop() - // Verify immediate poll happened + // Verify poll happened on first scheduler tick (10ms cadence in test) select { case <-pollCalled: // Success case <-time.After(1 * time.Second): - t.Fatal("Watcher did not poll immediately on start") + t.Fatal("Watcher did not poll on first scheduler tick") } } @@ -136,6 +153,14 @@ func TestWatcher_ProcessesMultipleRepositories(t *testing.T) { Return(repos, nil). Times(1) + // Each repository is claimed for polling (UpdateStatus with OnlyIfStatus). + for _, repo := range repos { + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + MinTimes(1) + } + // Each repository will be processed: 3 delegation checks mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). @@ -159,7 +184,7 @@ func TestWatcher_ProcessesMultipleRepositories(t *testing.T) { w := createTestWatcher(ctrl, mockRepoDS, mockTagDS, mockRegistrySet, mockDelegator, 1*time.Hour, true) assert.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) } @@ -175,7 +200,7 @@ func TestWatcher_HandlesDatastoreError(t *testing.T) { w := createTestWatcher(ctrl, mockRepoDS, nil, nil, nil, 1*time.Hour, false) assert.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) } @@ -183,11 +208,11 @@ func TestWatcher_StartIsIdempotent(t *testing.T) { ctrl := gomock.NewController(t) mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) - // Polls immediately + 1 timed poll after 100ms (with 150ms sleep) + // Scheduler runs on cadence tick, number of calls depends on timing. mockRepoDS.EXPECT(). ListRepositories(gomock.Any()). Return([]*storage.BaseImageRepository{}, nil). - Times(2) + AnyTimes() w := createTestWatcher(ctrl, mockRepoDS, nil, nil, nil, 100*time.Millisecond, false) @@ -196,7 +221,7 @@ func TestWatcher_StartIsIdempotent(t *testing.T) { w.Start() w.Start() - time.Sleep(150 * time.Millisecond) + time.Sleep(50 * time.Millisecond) // Should stop cleanly w.Stop() @@ -206,16 +231,16 @@ func TestWatcher_StopIsIdempotent(t *testing.T) { ctrl := gomock.NewController(t) mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) - // Polls immediately + 1 timed poll after 100ms (with 150ms sleep) + // Scheduler runs on cadence tick, number of calls depends on timing. mockRepoDS.EXPECT(). ListRepositories(gomock.Any()). Return([]*storage.BaseImageRepository{}, nil). - Times(2) + AnyTimes() w := createTestWatcher(ctrl, mockRepoDS, nil, nil, nil, 100*time.Millisecond, false) w.Start() - time.Sleep(150 * time.Millisecond) + time.Sleep(50 * time.Millisecond) // Call Stop multiple times (only first should take effect) w.Stop() @@ -232,13 +257,17 @@ func TestWatcher_StopsGracefullyDuringPoll(t *testing.T) { // Block during ListRepositories blockCh := make(chan struct{}) + callCount := 0 mockRepoDS.EXPECT(). ListRepositories(gomock.Any()). DoAndReturn(func(ctx context.Context) ([]*storage.BaseImageRepository, error) { - <-blockCh + callCount++ + if callCount == 1 { + <-blockCh + } return []*storage.BaseImageRepository{}, nil }). - Times(1) + AnyTimes() w := createTestWatcher(ctrl, mockRepoDS, nil, nil, nil, 1*time.Hour, false) @@ -286,6 +315,11 @@ func TestWatcher_AccessesAllProtoFields(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + // One delegation check for the repository mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). @@ -309,7 +343,7 @@ func TestWatcher_AccessesAllProtoFields(t *testing.T) { w := createTestWatcher(ctrl, mockRepoDS, mockTagDS, mockRegistrySet, mockDelegator, 1*time.Hour, true) // Should not panic when accessing proto fields - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) // Verify fields are accessible require.NotNil(t, repo) @@ -338,6 +372,11 @@ func TestWatcher_DelegationError(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + // Delegation check returns error - should continue with Central-based processing mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). @@ -362,7 +401,7 @@ func TestWatcher_DelegationError(t *testing.T) { // Should not panic on delegation error assert.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) } @@ -383,6 +422,11 @@ func TestWatcher_ShouldDelegate(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + // Fetch existing tags from cache mockTagDS.EXPECT(). ListTagsByRepository(gomock.Any(), gomock.Any()). @@ -402,7 +446,7 @@ func TestWatcher_ShouldDelegate(t *testing.T) { // Should not panic when delegation is required assert.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) } @@ -425,6 +469,11 @@ func TestWatcher_NoMatchingRegistry(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). Return("", false, nil). @@ -454,7 +503,7 @@ func TestWatcher_NoMatchingRegistry(t *testing.T) { // Should not panic when no matching registry found assert.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) } @@ -477,6 +526,11 @@ func TestWatcher_MatchingRegistryWithTagListError(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). Return("", false, nil). @@ -510,7 +564,7 @@ func TestWatcher_MatchingRegistryWithTagListError(t *testing.T) { // Should not panic on tag listing error assert.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) } @@ -533,6 +587,11 @@ func TestWatcher_MatchingRegistrySuccess(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). Return("", false, nil). @@ -585,7 +644,7 @@ func TestWatcher_MatchingRegistrySuccess(t *testing.T) { // Should complete successfully assert.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) } @@ -602,16 +661,22 @@ func TestWatcher_ContextCancellation(t *testing.T) { TagPattern: "*", } + // Scheduler may call ListRepositories multiple times based on timing. mockRepoDS.EXPECT(). ListRepositories(gomock.Any()). Return([]*storage.BaseImageRepository{repo}, nil). - Times(1) + AnyTimes() + + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + AnyTimes() // Fetch existing tags from cache (happens before delegation check) mockTagDS.EXPECT(). ListTagsByRepository(gomock.Any(), gomock.Any()). Return([]*storage.BaseImageTag{}, nil). - Times(1) + AnyTimes() // Block on GetDelegateClusterID until context is cancelled mockDelegator.EXPECT(). @@ -620,13 +685,13 @@ func TestWatcher_ContextCancellation(t *testing.T) { <-ctx.Done() return "", false, ctx.Err() }). - Times(1) + AnyTimes() // After delegation error, processing continues and scanner calls GetAllUnique mockRegistrySet.EXPECT(). GetAllUnique(). Return(nil). - Times(1) + AnyTimes() // No tags stored (no matching registry), so no UpsertMany/DeleteMany calls expected @@ -701,6 +766,11 @@ func TestWatcher_IncrementalUpdate_CheckTagsConstruction(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). Return("", false, nil). @@ -777,16 +847,13 @@ func TestWatcher_IncrementalUpdate_CheckTagsConstruction(t *testing.T) { // Execute poll require.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) } // TestWatcher_IncrementalUpdate_SkipTagsWithLargeCache tests that when there are // more cached tags than the tag limit, the excess tags go into SkipTags. func TestWatcher_IncrementalUpdate_SkipTagsWithLargeCache(t *testing.T) { - // Set a low limit for testing (automatically restored after test) - t.Setenv(env.BaseImageWatcherPerRepoTagLimit.EnvVar(), "2") - ctrl := gomock.NewController(t) mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) mockTagDS := tagDSMocks.NewMockDataStore(ctrl) @@ -800,7 +867,7 @@ func TestWatcher_IncrementalUpdate_SkipTagsWithLargeCache(t *testing.T) { TagPattern: "1.*", } - // Create 5 cached tags, but limit is 2, so only first 2 should be in CheckTags + // Create 5 cached tags, but limit is 2, so only first 2 should be in CheckTags. now := time.Now() cachedTags := []*storage.BaseImageTag{ {Tag: "1.25", Created: timestamppb.New(now.Add(-1 * time.Hour)), ManifestDigest: "sha256:25"}, @@ -820,22 +887,28 @@ func TestWatcher_IncrementalUpdate_SkipTagsWithLargeCache(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + AnyTimes() + mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). Return("", false, nil). Times(1) + // ListTagsByRepository is called twice: once for building the scan request, once in promoteTags. mockTagDS.EXPECT(). ListTagsByRepository(gomock.Any(), repo.GetId()). Return(cachedTags, nil). - Times(1) + AnyTimes() mockRegistry.EXPECT(). Match(gomock.Any()). Return(true). Times(1) - // Registry returns all tags + // Registry returns all tags. mockRegistry.EXPECT(). ListTags(gomock.Any(), gomock.Any()). Return([]string{"1.25", "1.24", "1.23", "1.22", "1.21"}, nil). @@ -846,12 +919,12 @@ func TestWatcher_IncrementalUpdate_SkipTagsWithLargeCache(t *testing.T) { Return(&storage.ImageIntegration{Id: "integration-1"}). Times(1) - // Only first 2 tags should have metadata fetched (CheckTags), rest are skipped + // Only first 2 tags should have metadata fetched (CheckTags), rest are skipped. mockRegistry.EXPECT(). Metadata(gomock.Any()). DoAndReturn(func(img *storage.Image) (*storage.ImageMetadata, error) { tag := img.GetName().GetTag() - // Should only see 1.25 and 1.24 + // Should only see 1.25 and 1.24. require.Contains(t, []string{"1.25", "1.24"}, tag, "Unexpected metadata fetch for tag outside CheckTags") return &storage.ImageMetadata{ V1: &storage.V1Metadata{}, @@ -860,7 +933,7 @@ func TestWatcher_IncrementalUpdate_SkipTagsWithLargeCache(t *testing.T) { }, }, nil }). - Times(2) // Only CheckTags (first 2) + Times(2) mockRegistrySet.EXPECT(). GetAllUnique(). @@ -875,10 +948,15 @@ func TestWatcher_IncrementalUpdate_SkipTagsWithLargeCache(t *testing.T) { DeleteMany(gomock.Any(), gomock.Any()). AnyTimes() - w := createTestWatcher(ctrl, mockRepoDS, mockTagDS, mockRegistrySet, mockDelegator, 1*time.Hour, true) + mockBaseImageDS := baseImageDSMocks.NewMockDataStore(ctrl) + mockBaseImageDS.EXPECT().ReplaceByRepository(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + // Create watcher with tagLimit=2 to test skip tag behavior. + w := New(mockRepoDS, mockTagDS, mockBaseImageDS, mockRegistrySet, mockDelegator, + 1*time.Hour, 10*time.Millisecond, 10, 2, 5, true) require.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) } @@ -903,6 +981,11 @@ func TestWatcher_TagBatch_FlushAfterScan(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). Return("", false, nil). @@ -970,7 +1053,7 @@ func TestWatcher_TagBatch_FlushAfterScan(t *testing.T) { w := createTestWatcher(ctrl, mockRepoDS, mockTagDS, mockRegistrySet, mockDelegator, 1*time.Hour, true) require.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) require.True(t, upsertCalled, "UpsertMany should have been called during Flush") @@ -1023,6 +1106,11 @@ func TestWatcher_TagBatch_DeleteEvent(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). Return("", false, nil). @@ -1088,7 +1176,7 @@ func TestWatcher_TagBatch_DeleteEvent(t *testing.T) { w := createTestWatcher(ctrl, mockRepoDS, mockTagDS, mockRegistrySet, mockDelegator, 1*time.Hour, true) require.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) require.True(t, deleteCalled, "DeleteMany should have been called for deleted tag") @@ -1386,6 +1474,11 @@ func TestWatcher_DelegatedFeatureFlag_Disabled(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + // GetDelegateClusterID should NOT be called when delegation is disabled. // If it were called, this test would fail due to missing expectation. @@ -1404,7 +1497,7 @@ func TestWatcher_DelegatedFeatureFlag_Disabled(t *testing.T) { w := createTestWatcher(ctrl, mockRepoDS, mockTagDS, mockRegistrySet, mockDelegator, 1*time.Hour, false) assert.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) }) } @@ -1427,6 +1520,11 @@ func TestWatcher_DelegatedFeatureFlag_Enabled(t *testing.T) { Return([]*storage.BaseImageRepository{repo}, nil). Times(1) + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + // GetDelegateClusterID should be called when feature is enabled. mockDelegator.EXPECT(). GetDelegateClusterID(gomock.Any(), gomock.Any()). @@ -1448,6 +1546,588 @@ func TestWatcher_DelegatedFeatureFlag_Enabled(t *testing.T) { w := createTestWatcher(ctrl, mockRepoDS, mockTagDS, mockRegistrySet, mockDelegator, 1*time.Hour, true) assert.NotPanics(t, func() { - w.(*watcherImpl).pollOnce() + runSchedulerPassAndWait(w) + }) +} + +func TestIsRepositoryDue(t *testing.T) { + now := time.Now() + fiveHoursAgo := now.Add(-5 * time.Hour) + threeHoursAgo := now.Add(-3 * time.Hour) + pollInterval := 4 * time.Hour + + cases := []struct { + name string + status storage.BaseImageRepository_Status + lastPolledAt *time.Time + expected bool + }{ + {"CREATED always due", storage.BaseImageRepository_CREATED, nil, true}, + {"QUEUED never due", storage.BaseImageRepository_QUEUED, nil, false}, + {"IN_PROGRESS never due", storage.BaseImageRepository_IN_PROGRESS, nil, false}, + {"READY nil lastPolled due", storage.BaseImageRepository_READY, nil, true}, + {"READY recently polled not due", storage.BaseImageRepository_READY, &now, false}, + {"READY within interval not due", storage.BaseImageRepository_READY, &threeHoursAgo, false}, + {"READY interval elapsed due", storage.BaseImageRepository_READY, &fiveHoursAgo, true}, + {"FAILED nil lastPolled due", storage.BaseImageRepository_FAILED, nil, true}, + {"FAILED recently polled not due", storage.BaseImageRepository_FAILED, &now, false}, + {"FAILED within interval not due", storage.BaseImageRepository_FAILED, &threeHoursAgo, false}, + {"FAILED interval elapsed due", storage.BaseImageRepository_FAILED, &fiveHoursAgo, true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + repo := &storage.BaseImageRepository{ + Id: uuid.NewV4().String(), + Status: tc.status, + LastPolledAt: protocompat.ConvertTimeToTimestampOrNil(tc.lastPolledAt), + } + assert.Equal(t, tc.expected, isRepositoryDue(repo, pollInterval)) + }) + } +} + +func TestWatcher_ScanFailure_SetsFailedStatus(t *testing.T) { + ctrl := gomock.NewController(t) + mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) + mockTagDS := tagDSMocks.NewMockDataStore(ctrl) + mockBaseImageDS := baseImageDSMocks.NewMockDataStore(ctrl) + mockRegistrySet := registryMocks.NewMockSet(ctrl) + mockDelegator := delegatedRegistryMocks.NewMockDelegator(ctrl) + + repo := &storage.BaseImageRepository{ + Id: "00000000-0000-0000-0000-0000000000ff", + RepositoryPath: "docker.io/library/nginx", + TagPattern: "*", + } + + mockRepoDS.EXPECT(). + ListRepositories(gomock.Any()). + Return([]*storage.BaseImageRepository{repo}, nil). + Times(1) + + // First UpdateStatus: claiming (QUEUED). + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + + // Second UpdateStatus: IN_PROGRESS. + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + + // Third UpdateStatus: FAILED with failure message. + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + DoAndReturn(func(ctx context.Context, id string, update repoDS.StatusUpdate) (*storage.BaseImageRepository, error) { + assert.Equal(t, storage.BaseImageRepository_FAILED, update.Status) + assert.Equal(t, repoDS.FailureCountIncrement, update.FailureCountOp) + assert.NotNil(t, update.LastFailureMessage) + assert.Contains(t, *update.LastFailureMessage, "failed to list tags") + return repo, nil + }). + Times(1) + + // Delegation check (happens before tag list). + mockDelegator.EXPECT(). + GetDelegateClusterID(gomock.Any(), gomock.Any()). + Return("", false, nil). + Times(1) + + // ListTagsByRepository fails, causing doScan to return error. + mockTagDS.EXPECT(). + ListTagsByRepository(gomock.Any(), gomock.Any()). + Return(nil, errors.New("database connection failed")). + Times(1) + + w := New(mockRepoDS, mockTagDS, mockBaseImageDS, mockRegistrySet, mockDelegator, 1*time.Hour, 10*time.Millisecond, 10, 10, 5, true) + + assert.NotPanics(t, func() { + runSchedulerPassAndWait(w) }) } + +func TestWatcher_SchedulerCadence_SkipsNotDueRepos(t *testing.T) { + ctrl := gomock.NewController(t) + mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) + + // Repository was polled recently, not due for rescan. + repo := &storage.BaseImageRepository{ + Id: "00000000-0000-0000-0000-0000000000ff", + RepositoryPath: "docker.io/library/nginx", + TagPattern: "*", + Status: storage.BaseImageRepository_READY, + LastPolledAt: protocompat.TimestampNow(), + } + + mockRepoDS.EXPECT(). + ListRepositories(gomock.Any()). + Return([]*storage.BaseImageRepository{repo}, nil). + Times(1) + + // UpdateStatus should NOT be called because repo is not due. + // No expectation set = test fails if called. + + w := createTestWatcher(ctrl, mockRepoDS, nil, nil, nil, 4*time.Hour, false) + + ctx := sac.WithAllAccess(context.Background()) + claimed, err := w.(*watcherImpl).doSchedulerPass(ctx) + assert.NoError(t, err) + assert.Equal(t, 0, claimed, "should not claim repos that are not due") +} + +func TestWatcher_SchedulerFairness_SortsReposByLastPolledAt(t *testing.T) { + ctrl := gomock.NewController(t) + mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) + mockTagDS := tagDSMocks.NewMockDataStore(ctrl) + mockBaseImageDS := baseImageDSMocks.NewMockDataStore(ctrl) + mockRegistrySet := registryMocks.NewMockSet(ctrl) + mockDelegator := delegatedRegistryMocks.NewMockDelegator(ctrl) + + now := time.Now() + oldestPolledAt := now.Add(-6 * time.Hour) + recentPolledAt := now.Add(-5 * time.Hour) + neverScanned := &storage.BaseImageRepository{ + Id: "00000000-0000-0000-0000-000000000001", + RepositoryPath: "docker.io/library/never", + TagPattern: "*", + Status: storage.BaseImageRepository_CREATED, + } + oldestScanned := &storage.BaseImageRepository{ + Id: "00000000-0000-0000-0000-000000000002", + RepositoryPath: "docker.io/library/oldest", + TagPattern: "*", + Status: storage.BaseImageRepository_READY, + LastPolledAt: protocompat.ConvertTimeToTimestampOrNil(&oldestPolledAt), + } + recentlyScanned := &storage.BaseImageRepository{ + Id: "00000000-0000-0000-0000-000000000003", + RepositoryPath: "docker.io/library/recent", + TagPattern: "*", + Status: storage.BaseImageRepository_READY, + LastPolledAt: protocompat.ConvertTimeToTimestampOrNil(&recentPolledAt), + } + + mockRepoDS.EXPECT(). + ListRepositories(gomock.Any()). + Return([]*storage.BaseImageRepository{recentlyScanned, oldestScanned, neverScanned}, nil). + Times(1) + + claimOrder := make([]string, 0, 2) + blockScan := make(chan struct{}) + startedScan := make(chan struct{}, 2) + + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, id string, update repoDS.StatusUpdate) (*storage.BaseImageRepository, error) { + switch update.Status { + case storage.BaseImageRepository_QUEUED: + claimOrder = append(claimOrder, id) + switch id { + case neverScanned.GetId(): + return neverScanned, nil + case oldestScanned.GetId(): + return oldestScanned, nil + case recentlyScanned.GetId(): + return recentlyScanned, nil + default: + return nil, errors.New("unexpected repository claimed") + } + case storage.BaseImageRepository_IN_PROGRESS: + startedScan <- struct{}{} + <-blockScan + return nil, context.Canceled + default: + return nil, errors.New("unexpected status transition") + } + }). + AnyTimes() + + w := New(mockRepoDS, mockTagDS, mockBaseImageDS, mockRegistrySet, mockDelegator, 4*time.Hour, 10*time.Millisecond, 10, 100, 2, false) + + ctx := sac.WithAllAccess(context.Background()) + claimed, err := w.(*watcherImpl).doSchedulerPass(ctx) + require.NoError(t, err) + require.Equal(t, 2, claimed) + + for range 2 { + select { + case <-startedScan: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for claimed scans to start") + } + } + close(blockScan) + w.(*watcherImpl).wg.Wait() + + assert.Equal(t, []string{neverScanned.GetId(), oldestScanned.GetId()}, claimOrder) +} + +func TestWatcher_RegistryError_SetsFailedStatus(t *testing.T) { + ctrl := gomock.NewController(t) + mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) + mockTagDS := tagDSMocks.NewMockDataStore(ctrl) + mockRegistrySet := registryMocks.NewMockSet(ctrl) + mockDelegator := delegatedRegistryMocks.NewMockDelegator(ctrl) + mockRegistry := registryTypesMocks.NewMockImageRegistry(ctrl) + + repo := &storage.BaseImageRepository{ + Id: "00000000-0000-0000-0000-0000000000ff", + RepositoryPath: "docker.io/library/nginx", + TagPattern: "*", + } + + mockRepoDS.EXPECT(). + ListRepositories(gomock.Any()). + Return([]*storage.BaseImageRepository{repo}, nil). + Times(1) + + // First UpdateStatus: claiming (QUEUED). + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, update repoDS.StatusUpdate) (*storage.BaseImageRepository, error) { + assert.Equal(t, storage.BaseImageRepository_QUEUED, update.Status) + return repo, nil + }). + Times(1) + + // Second UpdateStatus: IN_PROGRESS. + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, update repoDS.StatusUpdate) (*storage.BaseImageRepository, error) { + assert.Equal(t, storage.BaseImageRepository_IN_PROGRESS, update.Status) + return repo, nil + }). + Times(1) + + // Third UpdateStatus: FAILED with registry error message. + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, update repoDS.StatusUpdate) (*storage.BaseImageRepository, error) { + assert.Equal(t, storage.BaseImageRepository_FAILED, update.Status) + assert.Equal(t, repoDS.FailureCountIncrement, update.FailureCountOp) + require.NotNil(t, update.LastFailureMessage) + assert.Contains(t, *update.LastFailureMessage, "registry connection failed") + return repo, nil + }). + Times(1) + + mockDelegator.EXPECT(). + GetDelegateClusterID(gomock.Any(), gomock.Any()). + Return("", false, nil). + Times(1) + + // Fetch existing tags from cache (for incremental scan). + mockTagDS.EXPECT(). + ListTagsByRepository(gomock.Any(), gomock.Any()). + Return([]*storage.BaseImageTag{}, nil). + Times(1) + + // Registry matches but ListTags fails with registry error. + mockRegistry.EXPECT(). + Match(gomock.Any()). + Return(true). + Times(1) + + mockRegistry.EXPECT(). + ListTags(gomock.Any(), gomock.Any()). + Return(nil, errox.InvariantViolation.New("registry connection failed")). + Times(1) + + mockRegistrySet.EXPECT(). + GetAllUnique(). + Return([]types.ImageRegistry{mockRegistry}). + Times(1) + + w := createTestWatcher(ctrl, mockRepoDS, mockTagDS, mockRegistrySet, mockDelegator, 1*time.Hour, true) + + assert.NotPanics(t, func() { + runSchedulerPassAndWait(w) + }) +} + +func TestWatcher_LastPolledAt_SetAtScanCompletion(t *testing.T) { + ctrl := gomock.NewController(t) + mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) + mockTagDS := tagDSMocks.NewMockDataStore(ctrl) + mockRegistrySet := registryMocks.NewMockSet(ctrl) + mockDelegator := delegatedRegistryMocks.NewMockDelegator(ctrl) + mockRegistry := registryTypesMocks.NewMockImageRegistry(ctrl) + + repo := &storage.BaseImageRepository{ + Id: "00000000-0000-0000-0000-0000000000ff", + RepositoryPath: "docker.io/library/nginx", + TagPattern: "*", + } + + mockRepoDS.EXPECT(). + ListRepositories(gomock.Any()). + Return([]*storage.BaseImageRepository{repo}, nil). + Times(1) + + // First UpdateStatus: claiming (QUEUED). + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + Return(repo, nil). + Times(1) + + // Capture time before IN_PROGRESS transition. + var inProgressTime time.Time + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, update repoDS.StatusUpdate) (*storage.BaseImageRepository, error) { + assert.Equal(t, storage.BaseImageRepository_IN_PROGRESS, update.Status) + inProgressTime = time.Now() + return repo, nil + }). + Times(1) + + // Final UpdateStatus: READY with LastPolledAt. + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repo.GetId(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, update repoDS.StatusUpdate) (*storage.BaseImageRepository, error) { + assert.Equal(t, storage.BaseImageRepository_READY, update.Status) + require.NotNil(t, update.LastPolledAt) + // LastPolledAt should be AFTER inProgressTime (set at scan completion, not start). + assert.True(t, update.LastPolledAt.After(inProgressTime) || update.LastPolledAt.Equal(inProgressTime), + "LastPolledAt should be >= inProgressTime (set at completion, not start)") + return repo, nil + }). + Times(1) + + mockDelegator.EXPECT(). + GetDelegateClusterID(gomock.Any(), gomock.Any()). + Return("", false, nil). + Times(1) + + // Empty tags from cache. + mockTagDS.EXPECT(). + ListTagsByRepository(gomock.Any(), gomock.Any()). + Return([]*storage.BaseImageTag{}, nil). + AnyTimes() + + // Registry matches, returns empty tags (fast scan). + mockRegistry.EXPECT(). + Match(gomock.Any()). + Return(true). + Times(1) + + mockRegistry.EXPECT(). + ListTags(gomock.Any(), gomock.Any()). + Return([]string{}, nil). + Times(1) + + mockRegistrySet.EXPECT(). + GetAllUnique(). + Return([]types.ImageRegistry{mockRegistry}). + Times(1) + + w := createTestWatcher(ctrl, mockRepoDS, mockTagDS, mockRegistrySet, mockDelegator, 1*time.Hour, true) + + runSchedulerPassAndWait(w) +} + +func TestWatcher_PendingStatus(t *testing.T) { + repoID := "00000000-0000-0000-0000-0000000000ff" + + tests := []struct { + name string + initial *repoDS.StatusUpdate // nil = empty pending + retryErr error // error returned by UpdateStatus retry + expectPending bool + expectStatus storage.BaseImageRepository_Status + }{ + { + name: "retry succeeds, removed from pending", + initial: &repoDS.StatusUpdate{ + Status: storage.BaseImageRepository_READY, + FailureCountOp: repoDS.FailureCountReset, + }, + retryErr: nil, + expectPending: false, + }, + { + name: "retry fails, stays in pending", + initial: &repoDS.StatusUpdate{ + Status: storage.BaseImageRepository_FAILED, + FailureCountOp: repoDS.FailureCountIncrement, + }, + retryErr: errors.New("db still down"), + expectPending: true, + expectStatus: storage.BaseImageRepository_FAILED, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) + + // No repos due. + mockRepoDS.EXPECT(). + ListRepositories(gomock.Any()). + Return([]*storage.BaseImageRepository{}, nil). + Times(1) + + // Retry attempt. + if tt.initial != nil { + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repoID, gomock.Any()). + Return(nil, tt.retryErr). + Times(1) + } + + w := New(mockRepoDS, nil, nil, nil, nil, 1*time.Hour, 10*time.Millisecond, 10, 0, 5, false) + impl := w.(*watcherImpl) + + if tt.initial != nil { + impl.pendingStatus[repoID] = *tt.initial + } + + ctx := sac.WithAllAccess(context.Background()) + impl.schedulerPass(ctx) + + impl.pendingStatusMu.Lock() + defer impl.pendingStatusMu.Unlock() + + if tt.expectPending { + assert.Len(t, impl.pendingStatus, 1) + u, ok := impl.pendingStatus[repoID] + assert.True(t, ok) + assert.Equal(t, tt.expectStatus, u.Status) + } else { + assert.Len(t, impl.pendingStatus, 0) + } + }) + } +} + +// TestWatcher_RecoveryDoesNotCorruptActiveScans verifies that recovery does not +// mark a repo as FAILED while it is actively being scanned. +// +// Bug scenario: +// 1. Tick 1: repo claimed (QUEUED → IN_PROGRESS), scan goroutine spawned +// 2. Scan is slow, still running when tick 2 fires +// 3. Tick 2: recovery sees IN_PROGRESS, marks as FAILED (BUG!) +// 4. Scan finishes, updates to READY, but damage is done +// +// The test expects no FAILED status from recovery while a scan is active. +func TestWatcher_RecoveryDoesNotCorruptActiveScans(t *testing.T) { + ctrl := gomock.NewController(t) + mockRepoDS := repoDSMocks.NewMockDataStore(ctrl) + mockTagDS := tagDSMocks.NewMockDataStore(ctrl) + mockRegistrySet := registryMocks.NewMockSet(ctrl) + mockDelegator := delegatedRegistryMocks.NewMockDelegator(ctrl) + mockRegistry := registryTypesMocks.NewMockImageRegistry(ctrl) + mockBaseImageDS := baseImageDSMocks.NewMockDataStore(ctrl) + + repoID := uuid.NewV4().String() + repo := &storage.BaseImageRepository{ + Id: repoID, + RepositoryPath: "docker.io/library/nginx", + TagPattern: "*", + Status: storage.BaseImageRepository_CREATED, + } + + // Track status transitions. + var statusMu sync.Mutex + var statusHistory []storage.BaseImageRepository_Status + var recoveryCorruptedScan bool + + // Scan will block until we release it. + scanStarted := make(chan struct{}) + scanRelease := make(chan struct{}) + var scanStartedOnce sync.Once + + // ListRepositories returns a copy of the repo to avoid data races. + mockRepoDS.EXPECT(). + ListRepositories(gomock.Any()). + DoAndReturn(func(_ context.Context) ([]*storage.BaseImageRepository, error) { + statusMu.Lock() + defer statusMu.Unlock() + clone := proto.Clone(repo).(*storage.BaseImageRepository) + return []*storage.BaseImageRepository{clone}, nil + }). + AnyTimes() + + // UpdateStatus tracks transitions. + mockRepoDS.EXPECT(). + UpdateStatus(gomock.Any(), repoID, gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, update repoDS.StatusUpdate) (*storage.BaseImageRepository, error) { + statusMu.Lock() + defer statusMu.Unlock() + + // Detect recovery corruption: FAILED while scan is in progress. + if update.Status == storage.BaseImageRepository_FAILED && + repo.GetStatus() == storage.BaseImageRepository_IN_PROGRESS { + // Check if this is from recovery (has the restart message). + if update.LastFailureMessage != nil && *update.LastFailureMessage == "scan interrupted by restart" { + recoveryCorruptedScan = true + } + } + + repo.Status = update.Status + statusHistory = append(statusHistory, update.Status) + clone := proto.Clone(repo).(*storage.BaseImageRepository) + return clone, nil + }). + AnyTimes() + + mockDelegator.EXPECT(). + GetDelegateClusterID(gomock.Any(), gomock.Any()). + Return("", false, nil). + AnyTimes() + + mockTagDS.EXPECT(). + ListTagsByRepository(gomock.Any(), repoID). + DoAndReturn(func(_ context.Context, _ string) ([]*storage.BaseImageTag, error) { + // Signal that scan has started (only once), then block. + scanStartedOnce.Do(func() { close(scanStarted) }) + <-scanRelease + return []*storage.BaseImageTag{}, nil + }). + AnyTimes() + + mockRegistry.EXPECT().Match(gomock.Any()).Return(true).AnyTimes() + mockRegistry.EXPECT().ListTags(gomock.Any(), gomock.Any()).Return([]string{}, nil).AnyTimes() + mockRegistry.EXPECT().Source().Return(&storage.ImageIntegration{Id: "integration-1"}).AnyTimes() + mockRegistrySet.EXPECT().GetAllUnique().Return([]types.ImageRegistry{mockRegistry}).AnyTimes() + mockBaseImageDS.EXPECT().ReplaceByRepository(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + // Short cadence so second tick fires while scan is blocked. + w := New(mockRepoDS, mockTagDS, mockBaseImageDS, mockRegistrySet, mockDelegator, + 1*time.Hour, // pollInterval + 20*time.Millisecond, // schedulerCadence - short to trigger multiple ticks + 10, 100, 5, true) + + w.Start() + + // Wait for scan to start (first tick claimed and spawned goroutine). + select { + case <-scanStarted: + case <-time.After(5 * time.Second): + t.Fatal("Timed out waiting for scan to start") + } + + // Let a few more ticks fire while scan is blocked. + time.Sleep(100 * time.Millisecond) + + // Release the scan. + close(scanRelease) + + // Wait for scan goroutines to complete. + time.Sleep(50 * time.Millisecond) + + w.Stop() + + // Verify no recovery corruption. + statusMu.Lock() + defer statusMu.Unlock() + + t.Logf("Status history: %v", statusHistory) + + assert.False(t, recoveryCorruptedScan, + "Recovery should not mark actively scanning repo as FAILED") +} diff --git a/central/blob/datastore/datastore.go b/central/blob/datastore/datastore.go index 07ada065363d1..032932e4afbd2 100644 --- a/central/blob/datastore/datastore.go +++ b/central/blob/datastore/datastore.go @@ -10,6 +10,7 @@ import ( v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" pkgSearch "github.com/stackrox/rox/pkg/search" @@ -27,6 +28,7 @@ var ( type Datastore interface { GetIDs(ctx context.Context) ([]string, error) Upsert(ctx context.Context, obj *storage.Blob, reader io.Reader) error + UpsertWithWriter(ctx context.Context, tx *postgres.Tx, obj *storage.Blob, writeFn func(io.Writer) error) error Get(ctx context.Context, name string, writer io.Writer) (*storage.Blob, bool, error) Delete(ctx context.Context, name string) error GetMetadata(ctx context.Context, name string) (*storage.Blob, bool, error) @@ -53,6 +55,12 @@ func (d *datastoreImpl) Upsert(ctx context.Context, obj *storage.Blob, reader io return d.store.Upsert(ctx, obj, reader) } +// UpsertWithWriter writes a blob using the caller's transaction and a callback +// that writes directly to the large object. +func (d *datastoreImpl) UpsertWithWriter(ctx context.Context, tx *postgres.Tx, obj *storage.Blob, writeFn func(io.Writer) error) error { + return d.store.UpsertWithWriter(ctx, tx, obj, writeFn) +} + // Get retrieves a blob from the database func (d *datastoreImpl) Get(ctx context.Context, name string, writer io.Writer) (*storage.Blob, bool, error) { return d.store.Get(ctx, name, writer) diff --git a/central/blob/datastore/datastore_test.go b/central/blob/datastore/datastore_test.go index 98d6ee9650af8..c1cdb0c220b8c 100644 --- a/central/blob/datastore/datastore_test.go +++ b/central/blob/datastore/datastore_test.go @@ -52,7 +52,7 @@ func (s *blobTestSuite) SetupTest() { func (s *blobTestSuite) createBlobs(prefix string, size int, n int, modTime time.Time) []*storage.Blob { var blobs []*storage.Blob - for i := 0; i < n; i++ { + for i := range n { blob := &storage.Blob{ Name: fmt.Sprintf("%s/test/%d", prefix, i), ModifiedTime: protoconv.MustConvertTimeToTimestamp(modTime), diff --git a/central/blob/datastore/mocks/datastore.go b/central/blob/datastore/mocks/datastore.go index b6ed7a5d2e1a6..84fb9b90926ab 100644 --- a/central/blob/datastore/mocks/datastore.go +++ b/central/blob/datastore/mocks/datastore.go @@ -17,6 +17,7 @@ import ( v1 "github.com/stackrox/rox/generated/api/v1" storage "github.com/stackrox/rox/generated/storage" + postgres "github.com/stackrox/rox/pkg/postgres" search "github.com/stackrox/rox/pkg/search" gomock "go.uber.org/mock/gomock" ) @@ -181,3 +182,17 @@ func (mr *MockDatastoreMockRecorder) Upsert(ctx, obj, reader any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upsert", reflect.TypeOf((*MockDatastore)(nil).Upsert), ctx, obj, reader) } + +// UpsertWithWriter mocks base method. +func (m *MockDatastore) UpsertWithWriter(ctx context.Context, tx *postgres.Tx, obj *storage.Blob, writeFn func(io.Writer) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertWithWriter", ctx, tx, obj, writeFn) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertWithWriter indicates an expected call of UpsertWithWriter. +func (mr *MockDatastoreMockRecorder) UpsertWithWriter(ctx, tx, obj, writeFn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertWithWriter", reflect.TypeOf((*MockDatastore)(nil).UpsertWithWriter), ctx, tx, obj, writeFn) +} diff --git a/central/blob/datastore/store/mocks/store.go b/central/blob/datastore/store/mocks/store.go index 8982a9a900b89..53e4bab0a7ee4 100644 --- a/central/blob/datastore/store/mocks/store.go +++ b/central/blob/datastore/store/mocks/store.go @@ -16,6 +16,7 @@ import ( v1 "github.com/stackrox/rox/generated/api/v1" storage "github.com/stackrox/rox/generated/storage" + postgres "github.com/stackrox/rox/pkg/postgres" search "github.com/stackrox/rox/pkg/search" gomock "go.uber.org/mock/gomock" ) @@ -163,3 +164,17 @@ func (mr *MockStoreMockRecorder) Upsert(ctx, obj, reader any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upsert", reflect.TypeOf((*MockStore)(nil).Upsert), ctx, obj, reader) } + +// UpsertWithWriter mocks base method. +func (m *MockStore) UpsertWithWriter(ctx context.Context, tx *postgres.Tx, obj *storage.Blob, writeFn func(io.Writer) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertWithWriter", ctx, tx, obj, writeFn) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertWithWriter indicates an expected call of UpsertWithWriter. +func (mr *MockStoreMockRecorder) UpsertWithWriter(ctx, tx, obj, writeFn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertWithWriter", reflect.TypeOf((*MockStore)(nil).UpsertWithWriter), ctx, tx, obj, writeFn) +} diff --git a/central/blob/datastore/store/store.go b/central/blob/datastore/store/store.go index 3fd5064c5cbc8..c5d3fd693053b 100644 --- a/central/blob/datastore/store/store.go +++ b/central/blob/datastore/store/store.go @@ -30,6 +30,7 @@ type Store interface { GetIDs(ctx context.Context) ([]string, error) GetMetadata(ctx context.Context, name string) (*storage.Blob, bool, error) Upsert(ctx context.Context, obj *storage.Blob, reader io.Reader) error + UpsertWithWriter(ctx context.Context, tx *pgPkg.Tx, obj *storage.Blob, writeFn func(io.Writer) error) error Delete(ctx context.Context, name string) error } @@ -139,6 +140,81 @@ func (s *storeImpl) Upsert(ctx context.Context, obj *storage.Blob, reader io.Rea return tx.Commit(ctx) } +// UpsertWithWriter writes a blob using the caller's transaction and a callback +// that writes directly to the large object. The caller is responsible for +// committing or rolling back the transaction. ctx must already carry the +// transaction (via postgres.ContextWithTx) so that internal store operations +// reuse the same connection. writeFn receives a writer backed by the PostgreSQL +// large object; all writes happen on the calling goroutine, so the caller can +// safely interleave other operations on the same transaction (e.g. cursor reads). +func (s *storeImpl) UpsertWithWriter(ctx context.Context, tx *pgPkg.Tx, obj *storage.Blob, writeFn func(io.Writer) error) error { + if err := sac.VerifyAuthzOK(scopeChecker.WriteAllowed(ctx)); err != nil { + return err + } + ctx = sac.WithGlobalAccessScopeChecker(ctx, + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS, storage.Access_READ_WRITE_ACCESS), + sac.ResourceScopeKeys(resources.Administration))) + + existingBlob, exists, err := s.store.Get(ctx, obj.GetName()) + if err != nil { + return err + } + + los := tx.LargeObjects() + var lo *pgx.LargeObject + if exists { + lo, err = los.Open(ctx, existingBlob.GetOid(), pgx.LargeObjectModeWrite) + if err != nil { + return errors.Wrapf(err, "opening blob with oid %d", existingBlob.GetOid()) + } + if err := lo.Truncate(0); err != nil { + return errors.Wrapf(err, "truncating blob with oid %d", existingBlob.GetOid()) + } + obj.Oid = existingBlob.GetOid() + } else { + oid, err := los.Create(ctx, 0) + if err != nil { + return errors.Wrap(err, "error creating new blob") + } + lo, err = los.Open(ctx, oid, pgx.LargeObjectModeWrite) + if err != nil { + return errors.Wrapf(err, "opening blob with oid %d", oid) + } + obj.Oid = oid + } + + cw := &countingWriter{w: lo} + if err := writeFn(cw); err != nil { + return err + } + + if err := lo.Close(); err != nil { + return errors.Wrap(err, "closing large object for blob") + } + if obj.GetLength() < 0 { + obj.Length = cw.n + } else if cw.n != obj.GetLength() { + return errors.Errorf("Blob metadata mismatch. Blob metadata shows %d in length, but data has length of %d", obj.GetLength(), cw.n) + } + + if err := s.store.Upsert(ctx, obj); err != nil { + return errors.Wrapf(err, "error upserting blob %q", obj.GetName()) + } + return nil +} + +type countingWriter struct { + w io.Writer + n int64 +} + +func (cw *countingWriter) Write(p []byte) (int, error) { + n, err := cw.w.Write(p) + cw.n += int64(n) + return n, err +} + // Get returns a blob from the database func (s *storeImpl) Get(ctx context.Context, name string, writer io.Writer) (*storage.Blob, bool, error) { tx, err := s.db.Begin(ctx) diff --git a/central/blob/datastore/test_utils.go b/central/blob/datastore/test_utils.go index 5f6e54c03d647..845e95189aff1 100644 --- a/central/blob/datastore/test_utils.go +++ b/central/blob/datastore/test_utils.go @@ -7,8 +7,8 @@ import ( pgPkg "github.com/stackrox/rox/pkg/postgres" ) -// NewTestDatastore creates the data store for testing. -func NewTestDatastore(_ *testing.T, db pgPkg.DB) Datastore { +// NewTestDatastore creates the data store for testing and benchmarks. +func NewTestDatastore(_ testing.TB, db pgPkg.DB) Datastore { return &datastoreImpl{ store: store.New(db), } diff --git a/central/certgen/central_scanner.go b/central/certgen/central_scanner.go index ac0d482b346df..485a91162e519 100644 --- a/central/certgen/central_scanner.go +++ b/central/certgen/central_scanner.go @@ -77,33 +77,19 @@ func (s *serviceImpl) scannerHandler(w http.ResponseWriter, r *http.Request) { return } - secrets, ca, err := initializeSecretsWithCACertAndKey() - if err != nil { - httputil.WriteGRPCStyleError(w, codes.Internal, err) - return - } - - namespace := env.Namespace.Setting() - if r.URL.Query().Get("v") == "4" { - s.scannerV4Handler(w, secrets, ca, namespace) + if r.URL.Query().Get("v") != "4" { + httputil.WriteErrorf(w, http.StatusBadRequest, + "legacy Scanner certificate generation is not supported; pass v=4 for Scanner V4") return } - if err := certgen.IssueScannerCerts(secrets, ca, mtls.WithNamespace(namespace)); err != nil { - httputil.WriteGRPCStyleError(w, codes.Internal, err) - return - } - - rendered, err := renderer.RenderScannerTLSSecretOnly(renderer.Config{ - K8sConfig: &renderer.K8sConfig{}, - SecretsByteMap: secrets, - }, defaults.GetImageFlavorFromEnv()) + secrets, ca, err := initializeSecretsWithCACertAndKey() if err != nil { - httputil.WriteGRPCStyleErrorf(w, codes.Internal, "failed to render scanner TLS file: %v", err) + httputil.WriteGRPCStyleError(w, codes.Internal, err) return } - writeFile(w, rendered, "scanner-tls.yaml") + s.scannerV4Handler(w, secrets, ca, env.Namespace.Setting()) } func (s *serviceImpl) scannerV4Handler(w http.ResponseWriter, secrets map[string][]byte, ca mtls.CA, namespace string) { diff --git a/central/certgen/central_scanner_test.go b/central/certgen/central_scanner_test.go new file mode 100644 index 0000000000000..3ee7daee90088 --- /dev/null +++ b/central/certgen/central_scanner_test.go @@ -0,0 +1,42 @@ +package certgen + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestScannerHandlerRejectsUnsupportedVersion(t *testing.T) { + s := &serviceImpl{} + cases := map[string]struct { + method string + rawURL string + want int + }{ + "should reject GET": { + method: http.MethodGet, + rawURL: "/api/extensions/certgen/scanner?v=4", + want: http.StatusMethodNotAllowed, + }, + "should reject scanner v2": { + method: http.MethodPost, + rawURL: "/api/extensions/certgen/scanner", + want: http.StatusBadRequest, + }, + "should reject unknown version": { + method: http.MethodPost, + rawURL: "/api/extensions/certgen/scanner?v=2", + want: http.StatusBadRequest, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + req := httptest.NewRequest(tc.method, tc.rawURL, nil) + rec := httptest.NewRecorder() + s.scannerHandler(rec, req) + assert.Equal(t, tc.want, rec.Code) + }) + } +} diff --git a/central/cli/download.go b/central/cli/download.go index e0ad06abf0ecc..cdc705a28dcb3 100644 --- a/central/cli/download.go +++ b/central/cli/download.go @@ -1,17 +1,85 @@ package cli import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" "net/http" + "os" "path/filepath" + "strconv" + "strings" + + "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/utils" ) const downloadPath = "/assets/downloads/cli" -// Handler returns a handler for serving files from Central's downloads folder +var log = logging.LoggerForModule() + +// Handler serves roxctl binaries from Central UI. +// Binaries are stored as .tar.gz and extracted on the fly on each request. func Handler() http.HandlerFunc { + return handlerWithDir(downloadPath) +} + +func handlerWithDir(dir string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // Need to remove the rest of the URL path so that it just contains the file wanted - r.URL.Path = filepath.Base(r.URL.Path) - http.FileServer(http.Dir(downloadPath)).ServeHTTP(w, r) + if r.Method != http.MethodGet && r.Method != http.MethodHead { + w.Header().Set("Allow", http.MethodGet+", "+http.MethodHead) + http.Error(w, fmt.Sprintf("method %s not allowed", r.Method), http.StatusMethodNotAllowed) + return + } + + filename := filepath.Base(r.URL.Path) + tarPath := filepath.Join(dir, strings.TrimSuffix(filename, ".exe")+".tar.gz") + + f, err := os.Open(tarPath) + if err != nil { + log.Warnf("error opening tarball for requested binary %q: %v", filename, err) + http.Error(w, "not found", http.StatusNotFound) + return + } + defer utils.IgnoreError(f.Close) + + gz, err := gzip.NewReader(f) + if err != nil { + log.Warnf("error opening gzip reader for %s: %v", tarPath, err) + http.Error(w, "invalid archive", http.StatusInternalServerError) + return + } + defer utils.IgnoreError(gz.Close) + + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + log.Warnf("binary %q not found in %s; tarball entry name may not match the requested filename", filename, tarPath) + http.Error(w, "not found", http.StatusNotFound) + return + } + if err != nil { + log.Warnf("error reading archive %s: %v", tarPath, err) + http.Error(w, "error reading archive", http.StatusInternalServerError) + return + } + if hdr.Name != filename { + continue + } + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename)) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.FormatInt(hdr.Size, 10)) + w.Header().Set("Accept-Ranges", "none") + w.Header().Set("Cache-Control", "no-cache") + if r.Method == http.MethodGet { + if _, err = io.Copy(w, tr); err != nil { + log.Warnf("failed to stream %s: %v", filename, err) + panic(http.ErrAbortHandler) + } + } + return + } } } diff --git a/central/cli/download_test.go b/central/cli/download_test.go new file mode 100644 index 0000000000000..7a32eee16e2d3 --- /dev/null +++ b/central/cli/download_test.go @@ -0,0 +1,139 @@ +package cli + +import ( + "archive/tar" + "compress/gzip" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func writeTarball(t *testing.T, dir, tarName, entryName string, content []byte) { + t.Helper() + f, err := os.Create(filepath.Join(dir, tarName+".tar.gz")) + require.NoError(t, err) + + gz := gzip.NewWriter(f) + tw := tar.NewWriter(gz) + err = tw.WriteHeader(&tar.Header{ + Name: entryName, + Size: int64(len(content)), + Mode: 0755, + }) + require.NoError(t, err) + _, err = tw.Write(content) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gz.Close()) + require.NoError(t, f.Close()) +} + +type HandlerSuite struct { + suite.Suite + dir string + handler http.HandlerFunc + payload []byte +} + +func TestHandlerSuite(t *testing.T) { + suite.Run(t, new(HandlerSuite)) +} + +func (s *HandlerSuite) SetupTest() { + s.dir = s.T().TempDir() + s.handler = handlerWithDir(s.dir) + s.payload = []byte("fake roxctl binary content") + writeTarball(s.T(), s.dir, "roxctl-linux-amd64", "roxctl-linux-amd64", s.payload) + // Windows: tarball strips .exe from the name (required by release pipeline), + // but the entry inside the tarball preserves .exe. + writeTarball(s.T(), s.dir, "roxctl-windows-amd64", "roxctl-windows-amd64.exe", s.payload) +} + +func (s *HandlerSuite) TestServesLinuxBinaryViaGET() { + req := httptest.NewRequest(http.MethodGet, "/api/cli/download/roxctl-linux-amd64", nil) + rr := httptest.NewRecorder() + s.handler(rr, req) + s.Equal(http.StatusOK, rr.Code) + s.Equal(`attachment; filename="roxctl-linux-amd64"`, rr.Header().Get("Content-Disposition")) + s.Equal("application/octet-stream", rr.Header().Get("Content-Type")) + s.Equal("26", rr.Header().Get("Content-Length")) + s.Equal("none", rr.Header().Get("Accept-Ranges")) + s.Equal("no-cache", rr.Header().Get("Cache-Control")) + s.Equal(s.payload, rr.Body.Bytes()) +} + +func (s *HandlerSuite) TestServesWindowsBinaryViaGET() { + req := httptest.NewRequest(http.MethodGet, "/api/cli/download/roxctl-windows-amd64.exe", nil) + rr := httptest.NewRecorder() + s.handler(rr, req) + s.Equal(http.StatusOK, rr.Code) + s.Equal(`attachment; filename="roxctl-windows-amd64.exe"`, rr.Header().Get("Content-Disposition")) + s.Equal(s.payload, rr.Body.Bytes()) +} + +func (s *HandlerSuite) TestReturnsHeadersButNoBodyForHEAD() { + req := httptest.NewRequest(http.MethodHead, "/api/cli/download/roxctl-linux-amd64", nil) + rr := httptest.NewRecorder() + s.handler(rr, req) + s.Equal(http.StatusOK, rr.Code) + s.Equal("26", rr.Header().Get("Content-Length")) + s.Empty(rr.Body.Bytes()) +} + +func (s *HandlerSuite) TestReturns404ForMissingTarball() { + req := httptest.NewRequest(http.MethodGet, "/api/cli/download/roxctl-darwin-amd64", nil) + rr := httptest.NewRecorder() + s.handler(rr, req) + s.Equal(http.StatusNotFound, rr.Code) +} + +func (s *HandlerSuite) TestReturns404WhenBinaryNotInTarball() { + dir := s.T().TempDir() + // Tarball exists but entry name does not match the requested filename. + writeTarball(s.T(), dir, "roxctl-linux-amd64", "wrong-entry-name", s.payload) + handler := handlerWithDir(dir) + req := httptest.NewRequest(http.MethodGet, "/api/cli/download/roxctl-linux-amd64", nil) + rr := httptest.NewRecorder() + handler(rr, req) + s.Equal(http.StatusNotFound, rr.Code) +} + +func (s *HandlerSuite) TestReturns405ForUnsupportedMethod() { + req := httptest.NewRequest(http.MethodPost, "/api/cli/download/roxctl-linux-amd64", nil) + rr := httptest.NewRecorder() + s.handler(rr, req) + s.Equal(http.StatusMethodNotAllowed, rr.Code) + s.Equal("GET, HEAD", rr.Header().Get("Allow")) +} + +func (s *HandlerSuite) TestReturns500ForCorruptedGzip() { + dir := s.T().TempDir() + err := os.WriteFile(filepath.Join(dir, "roxctl-linux-amd64.tar.gz"), []byte("not a gzip"), 0600) + s.Require().NoError(err) + handler := handlerWithDir(dir) + req := httptest.NewRequest(http.MethodGet, "/api/cli/download/roxctl-linux-amd64", nil) + rr := httptest.NewRecorder() + handler(rr, req) + s.Equal(http.StatusInternalServerError, rr.Code) +} + +func (s *HandlerSuite) TestReturns500ForCorruptedTar() { + dir := s.T().TempDir() + f, err := os.Create(filepath.Join(dir, "roxctl-linux-amd64.tar.gz")) + s.Require().NoError(err) + gz := gzip.NewWriter(f) + _, err = gz.Write([]byte("valid gzip but not a tar")) + s.Require().NoError(err) + s.Require().NoError(gz.Close()) + s.Require().NoError(f.Close()) + handler := handlerWithDir(dir) + req := httptest.NewRequest(http.MethodGet, "/api/cli/download/roxctl-linux-amd64", nil) + rr := httptest.NewRecorder() + handler(rr, req) + s.Equal(http.StatusInternalServerError, rr.Code) +} diff --git a/central/cloudsources/service/service_impl_postgres_test.go b/central/cloudsources/service/service_impl_postgres_test.go index 30a34c5bace82..9d6b7bb0f4619 100644 --- a/central/cloudsources/service/service_impl_postgres_test.go +++ b/central/cloudsources/service/service_impl_postgres_test.go @@ -477,7 +477,7 @@ func (s *servicePostgresTestSuite) TestCloudSourceTest() { func (s *servicePostgresTestSuite) addCloudSources(num int) []*v1.CloudSource { cloudSources := fixtures.GetManyStorageCloudSources(num) - result := []*v1.CloudSource{} + result := make([]*v1.CloudSource, 0, len(cloudSources)) for _, cs := range cloudSources { s.Require().NoError(s.datastore.UpsertCloudSource(s.writeCtx, cs)) result = append(result, storagetov1.CloudSource(cs)) diff --git a/central/cluster/datastore/datastore.go b/central/cluster/datastore/datastore.go index 47fd9d2e061a5..fc119157ef87e 100644 --- a/central/cluster/datastore/datastore.go +++ b/central/cluster/datastore/datastore.go @@ -6,6 +6,7 @@ import ( alertDataStore "github.com/stackrox/rox/central/alert/datastore" clusterStore "github.com/stackrox/rox/central/cluster/store/cluster" clusterHealthStore "github.com/stackrox/rox/central/cluster/store/clusterhealth" + clusterInitStore "github.com/stackrox/rox/central/clusterinit/store" compliancePruning "github.com/stackrox/rox/central/complianceoperator/v2/pruner" clusterCVEDS "github.com/stackrox/rox/central/cve/cluster/datastore" deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" @@ -22,6 +23,8 @@ import ( secretDataStore "github.com/stackrox/rox/central/secret/datastore" "github.com/stackrox/rox/central/sensor/service/connection" serviceAccountDataStore "github.com/stackrox/rox/central/serviceaccount/datastore" + virtualMachineDataStore "github.com/stackrox/rox/central/virtualmachine/datastore" + virtualMachineV2DataStore "github.com/stackrox/rox/central/virtualmachine/v2/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" @@ -43,9 +46,11 @@ var ( //go:generate mockgen-wrapper type DataStore interface { GetCluster(ctx context.Context, id string) (*storage.Cluster, bool, error) + GetClusterID(ctx context.Context, name string) (string, bool, error) GetClusterName(ctx context.Context, id string) (string, bool, error) GetClusters(ctx context.Context) ([]*storage.Cluster, error) GetClustersForSAC() ([]effectiveaccessscope.Cluster, error) + GetClusterLabels(ctx context.Context, clusterID string) (map[string]string, error) CountClusters(ctx context.Context) (int, error) Exists(ctx context.Context, id string) (bool, error) WalkClusters(ctx context.Context, fn func(obj *storage.Cluster) error) error @@ -71,6 +76,8 @@ type DataStore interface { SearchResults(ctx context.Context, q *v1.Query) ([]*v1.SearchResult, error) LookupOrCreateClusterFromConfig(ctx context.Context, clusterID, bundleID string, hello *central.SensorHello) (*storage.Cluster, error) + + MatchProcessIndicator(ctx context.Context, indicator *storage.ProcessIndicator) (bool, error) } // New returns an instance of DataStore. @@ -95,6 +102,9 @@ func New( clusterRanker *ranking.Ranker, networkBaselineMgr networkBaselineManager.Manager, compliancePruner compliancePruning.Pruner, + clusterInitStore clusterInitStore.Store, + virtualMachines virtualMachineDataStore.DataStore, + virtualMachinesV2 virtualMachineV2DataStore.DataStore, ) (DataStore, error) { ds := &datastoreImpl{ clusterStorage: clusterStorage, @@ -117,8 +127,12 @@ func New( clusterRanker: clusterRanker, networkBaselineMgr: networkBaselineMgr, idToNameCache: simplecache.New(), + idToNamespaceFilterCache: simplecache.New(), nameToIDCache: simplecache.New(), compliancePruner: compliancePruner, + clusterInitStore: clusterInitStore, + virtualMachineDataStore: virtualMachines, + virtualMachineV2DataStore: virtualMachinesV2, } if err := ds.buildCache(sac.WithAllAccess(context.Background())); err != nil { diff --git a/central/cluster/datastore/datastore_impl.go b/central/cluster/datastore/datastore_impl.go index 817045978a532..fea57dfb7bf47 100644 --- a/central/cluster/datastore/datastore_impl.go +++ b/central/cluster/datastore/datastore_impl.go @@ -3,19 +3,26 @@ package datastore import ( "context" "fmt" + "maps" + "math" + "regexp" "slices" "strings" "time" "github.com/pkg/errors" alertDataStore "github.com/stackrox/rox/central/alert/datastore" + "github.com/stackrox/rox/central/cluster/lifecycle" clusterStore "github.com/stackrox/rox/central/cluster/store/cluster" clusterHealthStore "github.com/stackrox/rox/central/cluster/store/clusterhealth" + clusterInitStore "github.com/stackrox/rox/central/clusterinit/store" compliancePruning "github.com/stackrox/rox/central/complianceoperator/v2/pruner" "github.com/stackrox/rox/central/convert/storagetoeffectiveaccessscope" clusterCVEDS "github.com/stackrox/rox/central/cve/cluster/datastore" deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" imageIntegrationDataStore "github.com/stackrox/rox/central/imageintegration/datastore" + "github.com/stackrox/rox/central/metrics" + "github.com/stackrox/rox/central/metrics/custom/refresh" namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" networkBaselineManager "github.com/stackrox/rox/central/networkbaseline/manager" netEntityDataStore "github.com/stackrox/rox/central/networkgraph/entity/datastore" @@ -29,11 +36,13 @@ import ( "github.com/stackrox/rox/central/sensor/service/common" "github.com/stackrox/rox/central/sensor/service/connection" serviceAccountDataStore "github.com/stackrox/rox/central/serviceaccount/datastore" + virtualMachineDataStore "github.com/stackrox/rox/central/virtualmachine/datastore" + virtualMachineV2DataStore "github.com/stackrox/rox/central/virtualmachine/v2/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/centralsensor" - clusterValidation "github.com/stackrox/rox/pkg/cluster" + clusterPkg "github.com/stackrox/rox/pkg/cluster" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/errox" @@ -53,6 +62,8 @@ import ( "github.com/stackrox/rox/pkg/sliceutils" "github.com/stackrox/rox/pkg/sync" "github.com/stackrox/rox/pkg/uuid" + "github.com/stackrox/rox/pkg/version/productstreams" + "github.com/stackrox/rox/pkg/version/versioncompatibility" ) const ( @@ -71,6 +82,7 @@ var ( type datastoreImpl struct { clusterStorage clusterStore.Store + clusterInitStore clusterInitStore.Store clusterHealthStorage clusterHealthStore.Store clusterCVEDataStore clusterCVEDS.DataStore alertDataStore alertDataStore.DataStore @@ -88,12 +100,16 @@ type datastoreImpl struct { compliancePruner compliancePruning.Pruner cm connection.Manager networkBaselineMgr networkBaselineManager.Manager + virtualMachineDataStore virtualMachineDataStore.DataStore + // virtualMachineV2DataStore is nil when VirtualMachinesEnhancedDataModel is off. + virtualMachineV2DataStore virtualMachineV2DataStore.DataStore notifier notifierProcessor.Processor clusterRanker *ranking.Ranker - idToNameCache simplecache.Cache - nameToIDCache simplecache.Cache + idToNameCache simplecache.Cache + idToNamespaceFilterCache simplecache.Cache + nameToIDCache simplecache.Cache lock sync.Mutex } @@ -116,7 +132,12 @@ func (ds *datastoreImpl) UpdateClusterUpgradeStatus(ctx context.Context, id stri } cluster.Status.UpgradeStatus = upgradeStatus - return ds.clusterStorage.Upsert(ctx, cluster) + + err = ds.clusterStorage.Upsert(ctx, cluster) + if err == nil { + refresh.RefreshTracker(metrics.Health) + } + return err } func (ds *datastoreImpl) UpdateClusterCertExpiryStatus(ctx context.Context, id string, clusterCertExpiryStatus *storage.ClusterCertExpiryStatus) error { @@ -137,7 +158,11 @@ func (ds *datastoreImpl) UpdateClusterCertExpiryStatus(ctx context.Context, id s } cluster.Status.CertExpiryStatus = clusterCertExpiryStatus - return ds.clusterStorage.Upsert(ctx, cluster) + err = ds.clusterStorage.Upsert(ctx, cluster) + if err == nil { + refresh.RefreshTracker(metrics.Expiry) + } + return err } func (ds *datastoreImpl) UpdateClusterStatus(ctx context.Context, id string, status *storage.ClusterStatus) error { @@ -154,7 +179,11 @@ func (ds *datastoreImpl) UpdateClusterStatus(ctx context.Context, id string, sta status.CertExpiryStatus = cluster.GetStatus().GetCertExpiryStatus() cluster.Status = status - return ds.clusterStorage.Upsert(ctx, cluster) + err = ds.clusterStorage.Upsert(ctx, cluster) + if err == nil { + refresh.RefreshTracker(metrics.Health) + } + return err } func (ds *datastoreImpl) buildCache(ctx context.Context) error { @@ -178,6 +207,22 @@ func (ds *datastoreImpl) buildCache(ctx context.Context) error { for _, c := range clusters { ds.idToNameCache.Add(c.GetId(), c.GetName()) ds.nameToIDCache.Add(c.GetName(), c.GetId()) + + if filter := clusterPkg.GetNamespaceFilter(c); filter != nil { + compiledFilter, err := regexp.Compile(*filter) + + if err == nil { + ds.idToNamespaceFilterCache.Add(c.GetId(), compiledFilter) + } else { + log.Errorf("Could not compile filter regexp: %v", err) + } + } else { + // We got empty filter building the cluster. There should be no + // prior cache at this point, but just in case make sure it's + // invalidated. + ds.idToNamespaceFilterCache.Remove(c.GetId()) + } + c.HealthStatus = clusterHealthStatuses[c.GetId()] } return nil @@ -280,6 +325,7 @@ func (ds *datastoreImpl) searchRawClusters(ctx context.Context, q *v1.Query) ([] } ds.populateHealthInfos(ctx, clusters...) + ds.populateSensorVersionCompatibility(clusters...) ds.updateClusterPriority(clusters...) return clusters, nil } @@ -294,6 +340,7 @@ func (ds *datastoreImpl) GetCluster(ctx context.Context, id string) (*storage.Cl } ds.populateHealthInfos(ctx, cluster) + ds.populateSensorVersionCompatibility(cluster) ds.updateClusterPriority(cluster) return cluster, true, nil } @@ -309,6 +356,7 @@ func (ds *datastoreImpl) GetClusters(ctx context.Context) ([]*storage.Cluster, e } ds.populateHealthInfos(ctx, clusters...) + ds.populateSensorVersionCompatibility(clusters...) ds.updateClusterPriority(clusters...) return clusters, nil } @@ -331,6 +379,38 @@ func (ds *datastoreImpl) GetClusterName(ctx context.Context, id string) (string, return val.(string), true, nil } +func (ds *datastoreImpl) GetClusterID(ctx context.Context, name string) (string, bool, error) { + idVal, ok := ds.nameToIDCache.Get(name) + if !ok { + return "", false, nil + } + id := idVal.(string) + if allowed, err := clusterSAC.ReadAllowed(ctx, sac.ClusterScopeKey(id)); err != nil || !allowed { + return "", false, err + } + return id, true, nil +} + +// Figure out if an indicator matches provided namespace filter. We consider +// SAC errors or failure to find the pre-compiled filter regex as not matching +// cases. +func (ds *datastoreImpl) MatchProcessIndicator(ctx context.Context, + indicator *storage.ProcessIndicator) (bool, error) { + + id := indicator.GetClusterId() + + if ok, err := clusterSAC.ReadAllowed(ctx, sac.ClusterScopeKey(id)); err != nil || !ok { + return false, err + } + + filter, ok := ds.idToNamespaceFilterCache.Get(id) + if !ok { + return false, nil + } + + return filter.(*regexp.Regexp).MatchString(indicator.GetNamespace()), nil +} + func (ds *datastoreImpl) Exists(ctx context.Context, id string) (bool, error) { if ok, err := clusterSAC.ReadAllowed(ctx, sac.ClusterScopeKey(id)); err != nil || !ok { return false, err @@ -344,6 +424,7 @@ func (ds *datastoreImpl) WalkClusters(ctx context.Context, fn func(obj *storage. return ds.clusterStorage.Walk(ctx, func(cluster *storage.Cluster) error { clonedCluster := cluster.CloneVT() ds.populateHealthInfos(ctx, clonedCluster) + ds.populateSensorVersionCompatibility(clonedCluster) ds.updateClusterPriority(clonedCluster) return fn(clonedCluster) }) @@ -495,6 +576,7 @@ func (ds *datastoreImpl) UpdateClusterHealth(ctx context.Context, id string, clu if clusterHealthStatus.GetSensorHealthStatus() == oldHealth.GetSensorHealthStatus() && clusterHealthStatus.GetCollectorHealthStatus() == oldHealth.GetCollectorHealthStatus() { return nil } + defer refresh.RefreshTracker(metrics.Health) cluster, exists, err := ds.clusterStorage.Get(ctx, id) if err != nil { @@ -555,9 +637,7 @@ func (ds *datastoreImpl) UpdateAuditLogFileStates(ctx context.Context, id string if cluster.GetAuditLogState() == nil { cluster.AuditLogState = make(map[string]*storage.AuditLogFileState) } - for node, state := range states { - cluster.AuditLogState[node] = state - } + maps.Copy(cluster.GetAuditLogState(), states) return ds.clusterStorage.Upsert(ctx, cluster) } @@ -579,18 +659,22 @@ func (ds *datastoreImpl) RemoveCluster(ctx context.Context, id string, done *con return err } + release := lifecycle.Singleton().BeginDeletion(id) if err := ds.clusterStorage.Delete(ctx, id); err != nil { + release() return errors.Wrapf(err, "failed to remove cluster %q", id) } ds.idToNameCache.Remove(id) + ds.idToNamespaceFilterCache.Remove(id) ds.nameToIDCache.Remove(cluster.GetName()) deleteRelatedCtx := sac.WithAllAccess(context.Background()) - go ds.postRemoveCluster(deleteRelatedCtx, cluster, done) + go ds.postRemoveCluster(deleteRelatedCtx, cluster, done, release) return nil } -func (ds *datastoreImpl) postRemoveCluster(ctx context.Context, cluster *storage.Cluster, done *concurrency.Signal) { +func (ds *datastoreImpl) postRemoveCluster(ctx context.Context, cluster *storage.Cluster, done *concurrency.Signal, release func()) { + defer release() // Terminate the cluster connection to prevent new data from being stored. if ds.cm != nil { ds.cm.CloseConnection(cluster.GetId()) @@ -617,6 +701,8 @@ func (ds *datastoreImpl) postRemoveCluster(ctx context.Context, cluster *storage log.Errorf("failed to remove nodes for cluster %s: %v", cluster.GetId(), err) } + ds.removeClusterVirtualMachines(ctx, cluster) + if err := ds.netEntityDataStore.DeleteExternalNetworkEntitiesForCluster(ctx, cluster.GetId()); err != nil { log.Errorf("failed to delete external network graph entities for removed cluster %s: %v", cluster.GetId(), err) } @@ -694,6 +780,43 @@ func (ds *datastoreImpl) removeClusterPods(ctx context.Context, cluster *storage } } +// removeClusterVirtualMachines deletes V1 and V2 VM inventory for the cluster. +// Child scan/component/CVE rows cascade from the VM; ClusterID has no FK to clusters. +func (ds *datastoreImpl) removeClusterVirtualMachines(ctx context.Context, cluster *storage.Cluster) { + q := pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.ClusterID, cluster.GetId()).ProtoQuery() + q.Pagination = &v1.QueryPagination{Limit: math.MaxInt32} + + if ds.virtualMachineDataStore != nil { + vms, err := ds.virtualMachineDataStore.SearchRawVirtualMachines(ctx, q) + if err != nil { + log.Errorf("Failed to get virtual machines for removed cluster %s: %v", cluster.GetId(), err) + } else if ids := virtualMachineIDs(vms); len(ids) > 0 { + if err := ds.virtualMachineDataStore.DeleteVirtualMachines(ctx, ids...); err != nil { + log.Errorf("Failed to remove virtual machines as part of removal of cluster %s: %v", cluster.GetId(), err) + } + } + } + + if ds.virtualMachineV2DataStore != nil { + results, err := ds.virtualMachineV2DataStore.Search(ctx, q) + if err != nil { + log.Errorf("Failed to get v2 virtual machines for removed cluster %s: %v", cluster.GetId(), err) + } else if ids := pkgSearch.ResultsToIDs(results); len(ids) > 0 { + if err := ds.virtualMachineV2DataStore.DeleteVirtualMachines(ctx, ids...); err != nil { + log.Errorf("Failed to remove v2 virtual machines as part of removal of cluster %s: %v", cluster.GetId(), err) + } + } + } +} + +func virtualMachineIDs(vms []*storage.VirtualMachine) []string { + ids := make([]string, 0, len(vms)) + for _, vm := range vms { + ids = append(ids, vm.GetId()) + } + return ids +} + func (ds *datastoreImpl) removeClusterDeployments(ctx context.Context, cluster *storage.Cluster) []string { q := pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.ClusterID, cluster.GetId()).ProtoQuery() deployments, err := ds.deploymentDataStore.Search(ctx, q) @@ -878,6 +1001,42 @@ func (ds *datastoreImpl) populateHealthInfos(ctx context.Context, clusters ...*s } } +func (ds *datastoreImpl) populateSensorVersionCompatibility(clusters ...*storage.Cluster) { + for _, cluster := range clusters { + if cluster.GetStatus() == nil { + continue + } + sensorXY, err := productstreams.ParseXYFromVersionString(cluster.GetStatus().GetSensorVersion()) + if err != nil { + cluster.Status.SensorVersionCompatibility = storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_UNKNOWN + continue + } + compat, err := versioncompatibility.ClassifyVersion(sensorXY) + if err != nil { + cluster.Status.SensorVersionCompatibility = storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_UNKNOWN + continue + } + cluster.Status.SensorVersionCompatibility = compatibilityToProto(compat) + } +} + +func compatibilityToProto(c versioncompatibility.Compatibility) storage.SensorVersionCompatibility { + switch c { + case versioncompatibility.Matched: + return storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_MATCHED + case versioncompatibility.CompatibleBehind: + return storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_COMPATIBLE_BEHIND + case versioncompatibility.CompatibleAhead: + return storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_COMPATIBLE_AHEAD + case versioncompatibility.IncompatibleBehind: + return storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_INCOMPATIBLE_BEHIND + case versioncompatibility.IncompatibleAhead: + return storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_INCOMPATIBLE_AHEAD + default: + return storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_UNKNOWN + } +} + func (ds *datastoreImpl) updateClusterNoLock(ctx context.Context, cluster *storage.Cluster) error { if err := normalizeCluster(cluster); err != nil { return err @@ -891,134 +1050,210 @@ func (ds *datastoreImpl) updateClusterNoLock(ctx context.Context, cluster *stora } ds.idToNameCache.Add(cluster.GetId(), cluster.GetName()) ds.nameToIDCache.Add(cluster.GetName(), cluster.GetId()) + + if filter := clusterPkg.GetNamespaceFilter(cluster); filter != nil { + compiledFilter, err := regexp.Compile(*filter) + + if err == nil { + ds.idToNamespaceFilterCache.Add(cluster.GetId(), compiledFilter) + } else { + log.Errorf("Could not compile filter regexp: %v", err) + } + } else { + // We got empty filter updating the cluster. Make sure the cache is + // invalidated. + ds.idToNamespaceFilterCache.Remove(cluster.GetId()) + } return nil } -// registrantID can be the ID of an init bundle or of a CRS. -func (ds *datastoreImpl) LookupOrCreateClusterFromConfig(ctx context.Context, clusterID, registrantID string, hello *central.SensorHello) (*storage.Cluster, error) { - if err := checkWriteSac(ctx, clusterID); err != nil { - return nil, err +// clusterConfigData holds extracted configuration from SensorHello. +// This allows pure functions to work with structured data instead of raw protobuf messages. +type clusterConfigData struct { + clusterName string + manager storage.ManagerType + helmConfig *storage.CompleteClusterConfig + isNotManagedManually bool + deploymentIdentification *storage.SensorDeploymentIdentification + capabilities []string +} + +// extractClusterConfig extracts relevant configuration data from SensorHello. +// This is a pure function that performs the extraction once. +func extractClusterConfig(hello *central.SensorHello) clusterConfigData { + helmInit := hello.GetHelmManagedConfigInit() + return clusterConfigData{ + clusterName: helmInit.GetClusterName(), + manager: helmInit.GetManagedBy(), + helmConfig: helmInit.GetClusterConfig(), + isNotManagedManually: centralsensor.SecuredClusterIsNotManagedManually(helmInit), + deploymentIdentification: hello.GetDeploymentIdentification(), + capabilities: hello.GetCapabilities(), } +} - helmConfig := hello.GetHelmManagedConfigInit() - manager := helmConfig.GetManagedBy() +// shouldUpdateCluster determines if an existing cluster needs updating. +// Returns true if any of: sensor capabilities, config fingerprint, init bundle ID, or manager type has changed. +func shouldUpdateCluster(existing *storage.Cluster, config clusterConfigData, registrantID string) bool { + return !(set.NewSet(existing.GetSensorCapabilities()...).Equal(set.NewSet(config.capabilities...)) && + existing.GetInitBundleId() == registrantID && + existing.GetHelmConfig().GetConfigFingerprint() == config.helmConfig.GetConfigFingerprint() && + existing.GetManagedBy() == config.manager) +} - ds.lock.Lock() - defer ds.lock.Unlock() +func buildNewClusterFromConfig(clusterName, registrantID string, config clusterConfigData) *storage.Cluster { + cluster := &storage.Cluster{ + Name: clusterName, + InitBundleId: registrantID, + ManagedBy: config.manager, + MostRecentSensorId: config.deploymentIdentification.CloneVT(), + SensorCapabilities: sliceutils.CopySliceSorted(config.capabilities), + } + if config.isNotManagedManually { + configureFromHelmConfig(cluster, config.helmConfig) + } + + return cluster +} - clusterName := helmConfig.GetClusterName() +// applyConfigToCluster applies configuration updates to a cluster. +// Returns a new cluster object with updates applied (immutable pattern). +func applyConfigToCluster(cluster *storage.Cluster, config clusterConfigData) *storage.Cluster { + updated := cluster.CloneVT() + updated.ManagedBy = config.manager + updated.SensorCapabilities = sliceutils.CopySliceSorted(config.capabilities) - if clusterID == "" && clusterName != "" { - // Try to look up cluster ID by name, if this is for an existing cluster - clusterIDVal, _ := ds.nameToIDCache.Get(clusterName) - clusterID, _ = clusterIDVal.(string) + if config.isNotManagedManually { + configureFromHelmConfig(updated, config.helmConfig) + } else { + updated.HelmConfig = nil } - isExisting := false - var cluster *storage.Cluster - if clusterID != "" { - clusterByID, exist, err := ds.GetCluster(ctx, clusterID) - if err != nil { - return nil, err - } else if !exist { - return nil, errors.Errorf("cluster with ID %q does not exist", clusterID) - } + return updated +} - isExisting = true +// checkGracePeriodForReconnect checks if reconnection is allowed based on grace period. +// For Helm/Operator managed clusters, prevents cluster moves within the grace period. +// (Note: see ROX-32981 for further discussion) +func checkGracePeriodForReconnect(cluster *storage.Cluster, deploymentID *storage.SensorDeploymentIdentification, manager storage.ManagerType) error { + // In a scale test environment, allow Sensors to reconnect in under the time limit. + if env.ScaleTestEnabled.BooleanSetting() { + return nil + } - cluster = clusterByID + lastContact := protoconv.ConvertTimestampToTimeOrDefault(cluster.GetHealthStatus().GetLastContact(), time.Time{}) + timeLeftInGracePeriod := clusterMoveGracePeriod - time.Since(lastContact) - // If a name is specified, validate it (otherwise, accept any name) - if clusterName != "" && clusterName != clusterByID.GetName() { - return nil, errors.Errorf("Name mismatch for cluster %q: expected %q, but %q was specified. Set the cluster.name/clusterName attribute in your Helm config to %q, or remove it", clusterID, cluster.GetName(), clusterName, cluster.GetName()) + if timeLeftInGracePeriod > 0 { + if err := common.CheckConnReplace(deploymentID, cluster.GetMostRecentSensorId()); err != nil { + // Fallback value - should be overridden by switch unless ManagerType is extended. + // This should never surface to the user. + managerPretty := "non-manually" + switch manager { + case storage.ManagerType_MANAGER_TYPE_HELM_CHART: + managerPretty = "Helm" + case storage.ManagerType_MANAGER_TYPE_KUBERNETES_OPERATOR: + managerPretty = "Operator" + } + return errors.Errorf("registering %s-managed cluster is not allowed: %s. If you recently re-deployed, please wait for another %v", + managerPretty, err, timeLeftInGracePeriod) } + } + return nil +} - } else if clusterName != "" { - // At this point, we can be sure that the cluster does not exist. - cluster = &storage.Cluster{ - Name: clusterName, - InitBundleId: registrantID, - MostRecentSensorId: hello.GetDeploymentIdentification().CloneVT(), - SensorCapabilities: sliceutils.CopySliceSorted(hello.GetCapabilities()), - } - clusterConfig := helmConfig.GetClusterConfig() - configureFromHelmConfig(cluster, clusterConfig) +// Returns the cluster, a bool indicating whether it was an existing cluster (true) or newly created (false), and an error. +// The bool is important because existing clusters need grace period checks and update checks, while new clusters skip those. +func (ds *datastoreImpl) lookupOrCreateCluster(ctx context.Context, clusterID, clusterName, registrantID string, config clusterConfigData) (*storage.Cluster, bool, error) { + if clusterID == "" && clusterName == "" { + return nil, false, errors.New("neither a cluster ID nor a cluster name was specified") + } - if centralsensor.SecuredClusterIsNotManagedManually(helmConfig) { - cluster.HelmConfig = clusterConfig.CloneVT() + // Try to resolve cluster ID from name if not provided + if clusterID == "" { + if cachedID, ok := ds.nameToIDCache.Get(clusterName); ok { + clusterID, _ = cachedID.(string) } + } - if _, err := ds.addClusterNoLock(ctx, cluster); err != nil { - return nil, errors.Wrapf(err, "failed to dynamically add cluster with name %q", clusterName) + // Path 1: Lookup existing cluster by ID + if clusterID != "" { + cluster, exists, err := ds.GetCluster(ctx, clusterID) + if err != nil { + return nil, false, err } - } else { - return nil, errors.New("neither a cluster ID nor a cluster name was specified") - } - - if manager != storage.ManagerType_MANAGER_TYPE_MANUAL && isExisting { - // This is short-cut for clusters whose Helm config fingerprint and init bundle ID is unchanged. - // Applies to Helm- and Operator-managed clusters, not to manually managed clusters. - - // Check if the newly incoming request may replace the old connection - lastContact := protoconv.ConvertTimestampToTimeOrDefault(cluster.GetHealthStatus().GetLastContact(), time.Time{}) - timeLeftInGracePeriod := clusterMoveGracePeriod - time.Since(lastContact) - - // In a scale test environment, allow Sensors to reconnect in under the time limit - if timeLeftInGracePeriod > 0 && !env.ScaleTestEnabled.BooleanSetting() { - if err := common.CheckConnReplace(hello.GetDeploymentIdentification(), cluster.GetMostRecentSensorId()); err != nil { - managerPretty := "non-manually" // Unless we extend the `ManagerType` and forget to extend the switch here, this should never surface to the user. - switch manager { - case storage.ManagerType_MANAGER_TYPE_HELM_CHART: - managerPretty = "Helm" - case storage.ManagerType_MANAGER_TYPE_KUBERNETES_OPERATOR: - managerPretty = "Operator" - } - return nil, errors.Errorf("registering %s-managed cluster is not allowed: %s. If you recently re-deployed, please wait for another %v", - managerPretty, err, timeLeftInGracePeriod) - } + if !exists { + return nil, false, errors.Errorf("cluster with ID %q does not exist", clusterID) } - if sensorCapabilitiesEqual(cluster, hello) && - cluster.GetInitBundleId() == registrantID && - cluster.GetHelmConfig().GetConfigFingerprint() == helmConfig.GetClusterConfig().GetConfigFingerprint() && - cluster.GetManagedBy() == manager { - // No change in either of - // * sensor capabilities - // * fingerprint of the Helm configuration - // * in init bundle ID - // * manager type - // - // => there is no need to update the cluster, return immediately. - return cluster, nil + // Validate name matches if specified + if clusterName != "" && clusterName != cluster.GetName() { + return nil, false, errors.Errorf("name mismatch for cluster %q: expected %q, but %q was specified. Set the cluster.name/clusterName attribute in your Helm config to %q, or remove it", + clusterID, cluster.GetName(), clusterName, cluster.GetName()) } + + return cluster, true, nil } - clusterConfig := helmConfig.GetClusterConfig() - currentCluster := cluster + // Path 2: Create new cluster by name + cluster := buildNewClusterFromConfig(clusterName, registrantID, config) - cluster = cluster.CloneVT() - cluster.ManagedBy = manager - cluster.InitBundleId = registrantID - cluster.SensorCapabilities = sliceutils.CopySliceSorted(hello.GetCapabilities()) - if centralsensor.SecuredClusterIsNotManagedManually(helmConfig) { - configureFromHelmConfig(cluster, clusterConfig) - cluster.HelmConfig = clusterConfig.CloneVT() - } else { - cluster.HelmConfig = nil + if err := ds.clusterInitStore.InitiateClusterRegistration(ctx, registrantID, clusterName); err != nil { + return nil, false, errors.Wrapf(err, "initiating registrations of cluster %s using init artifact %s", clusterName, registrantID) + } + + if _, err := ds.addClusterNoLock(ctx, cluster); err != nil { + return nil, false, errors.Wrapf(err, "failed to dynamically add cluster with name %q", clusterName) + } + + return cluster, false, nil +} + +// registrantID can be one of +// * ID of an init bundle, when connecting with an init bundle certificate. +// * ID of a CRS, when connecting with a CRS certificate. +// * Empty, when connecting with non-init service certificates. +func (ds *datastoreImpl) LookupOrCreateClusterFromConfig(ctx context.Context, clusterID, registrantID string, hello *central.SensorHello) (*storage.Cluster, error) { + if err := checkWriteSac(ctx, clusterID); err != nil { + return nil, err + } + + config := extractClusterConfig(hello) + + ds.lock.Lock() + defer ds.lock.Unlock() + + cluster, isExisting, err := ds.lookupOrCreateCluster(ctx, clusterID, config.clusterName, registrantID, config) + if err != nil { + return nil, err + } + + // New clusters are fully built and persisted by lookupOrCreateCluster; no further update needed. + if !isExisting { + return cluster, nil } - if !currentCluster.EqualVT(cluster) { - // Cluster is dirty and needs to be updated in the DB. - if err := ds.updateClusterNoLock(ctx, cluster); err != nil { + // For existing clusters, check if update is needed + if config.manager != storage.ManagerType_MANAGER_TYPE_MANUAL { + if err := checkGracePeriodForReconnect(cluster, config.deploymentIdentification, config.manager); err != nil { return nil, err } + + if !shouldUpdateCluster(cluster, config, registrantID) { + return cluster, nil + } } - return cluster, nil -} + updatedCluster := applyConfigToCluster(cluster, config) + + // Persist if changed + if !cluster.EqualVT(updatedCluster) { + if err := ds.updateClusterNoLock(ctx, updatedCluster); err != nil { + return nil, err + } + } -func sensorCapabilitiesEqual(cluster *storage.Cluster, hello *central.SensorHello) bool { - return set.NewSet(cluster.GetSensorCapabilities()...).Equal(set.NewSet(hello.GetCapabilities()...)) + return updatedCluster, nil } func normalizeCluster(cluster *storage.Cluster) error { @@ -1033,7 +1268,7 @@ func normalizeCluster(cluster *storage.Cluster) error { } func validateInput(cluster *storage.Cluster) error { - return clusterValidation.Validate(cluster).ToError() + return clusterPkg.Validate(cluster).ToError() } // addDefaults enriches the provided non-nil cluster object with defaults for @@ -1110,6 +1345,8 @@ func configureFromHelmConfig(cluster *storage.Cluster, helmConfig *storage.Compl if features.AdmissionControllerConfig.Enabled() { cluster.AdmissionControllerFailOnError = staticConfig.GetAdmissionControllerFailOnError() } + cluster.HelmConfig = helmConfig.CloneVT() + } func (ds *datastoreImpl) collectClusters(ctx context.Context) ([]*storage.Cluster, error) { @@ -1127,6 +1364,18 @@ func (ds *datastoreImpl) collectClusters(ctx context.Context) ([]*storage.Cluste return clusters, nil } +// GetClusterLabels returns the labels for the specified cluster. +func (ds *datastoreImpl) GetClusterLabels(ctx context.Context, clusterID string) (map[string]string, error) { + cluster, exists, err := ds.GetCluster(ctx, clusterID) + if err != nil { + return nil, err + } + if !exists { + return nil, nil + } + return cluster.GetLabels(), nil +} + // ClusterSearchResultConverter implements search.SearchResultConverter for cluster search results. // This enables single-pass query construction for SearchResult protos. type ClusterSearchResultConverter struct{} diff --git a/central/cluster/datastore/datastore_impl_postgres_test.go b/central/cluster/datastore/datastore_impl_postgres_test.go index c70b6646d11c7..ed85aee19f140 100644 --- a/central/cluster/datastore/datastore_impl_postgres_test.go +++ b/central/cluster/datastore/datastore_impl_postgres_test.go @@ -11,6 +11,7 @@ import ( alertDatastore "github.com/stackrox/rox/central/alert/datastore" clusterPostgresStore "github.com/stackrox/rox/central/cluster/store/cluster/postgres" clusterHealthPostgresStore "github.com/stackrox/rox/central/cluster/store/clusterhealth/postgres" + clusterInitStore "github.com/stackrox/rox/central/clusterinit/store" compliancePruning "github.com/stackrox/rox/central/complianceoperator/v2/pruner" clusterCVEDataStore "github.com/stackrox/rox/central/cve/cluster/datastore" deploymentDatastore "github.com/stackrox/rox/central/deployment/datastore" @@ -29,6 +30,10 @@ import ( secretDataStore "github.com/stackrox/rox/central/secret/datastore" "github.com/stackrox/rox/central/sensor/service/connection" serviceAccountDataStore "github.com/stackrox/rox/central/serviceaccount/datastore" + vmCVEDatastore "github.com/stackrox/rox/central/virtualmachine/cve/v2/datastore" + vmDatastore "github.com/stackrox/rox/central/virtualmachine/datastore" + vmV2Datastore "github.com/stackrox/rox/central/virtualmachine/v2/datastore" + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" @@ -45,8 +50,10 @@ import ( pkgSearch "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/scoped" "github.com/stackrox/rox/pkg/uuid" + versionTestutils "github.com/stackrox/rox/pkg/version/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "google.golang.org/protobuf/types/known/timestamppb" ) const ( @@ -74,12 +81,16 @@ type ClusterPostgresDataStoreTestSuite struct { roleBindingDatastore k8sRoleBindingDataStore.DataStore imageIntegrationDatastore imageIntegrationDataStore.DataStore clusterDatastore DataStore + clusterInitStore clusterInitStore.Store + vmDatastore vmDatastore.DataStore + vmV2Datastore vmV2Datastore.DataStore + vmCVEDatastore vmCVEDatastore.DataStore clusterHealthDBStore clusterHealthPostgresStore.Store } func (s *ClusterPostgresDataStoreTestSuite) SetupTest() { - + versionTestutils.SetMainVersion(s.T(), "4.5.0-testing") s.ctx = sac.WithAllAccess(context.Background()) s.db = pgtest.ForT(s.T()) clusterDBStore := clusterPostgresStore.New(s.db) @@ -107,11 +118,16 @@ func (s *ClusterPostgresDataStoreTestSuite) SetupTest() { s.roleDatastore = k8sRoleDataStore.GetTestPostgresDataStore(s.T(), s.db.DB) s.roleBindingDatastore = k8sRoleBindingDataStore.GetTestPostgresDataStore(s.T(), s.db.DB) s.imageIntegrationDatastore = imageIntegrationDataStore.GetTestPostgresDataStore(s.T(), s.db.DB) + s.clusterInitStore = clusterInitStore.GetTestPostgresDataStore(s.T(), s.db.DB) + s.vmDatastore = vmDatastore.GetTestPostgresDataStore(s.T(), s.db) + s.vmV2Datastore = vmV2Datastore.GetTestPostgresDataStore(s.T(), s.db) + s.vmCVEDatastore = vmCVEDatastore.GetTestPostgresDataStore(s.T(), s.db) s.clusterDatastore, err = New(clusterDBStore, s.clusterHealthDBStore, clusterCVEStore, s.alertDatastore, s.imageIntegrationDatastore, s.nsDatastore, s.deploymentDatastore, nodeStore, s.podDatastore, s.secretDatastore, netFlowStore, netEntityStore, s.serviceAccountDatastore, s.roleDatastore, s.roleBindingDatastore, sensorCnxMgr, nil, - clusterRanker, networkBaselineM, compliancePruner) + clusterRanker, networkBaselineM, compliancePruner, s.clusterInitStore, + s.vmDatastore, s.vmV2Datastore) s.NoError(err) } @@ -165,6 +181,43 @@ func (s *ClusterPostgresDataStoreTestSuite) TestRemoveCluster() { s.NotEmpty(imageIntegrationId) s.NoError(imageIntegrationAddErr) + testV1VM := &storage.VirtualMachine{ + Id: fixtureconsts.VirtualMachine1, + Name: "retired-vm-v1", + Namespace: "default", + ClusterId: clusterId, + Scan: &storage.VirtualMachineScan{ + Components: []*storage.EmbeddedVirtualMachineScanComponent{ + {Name: "openssl", Version: "1.1.1"}, + }, + }, + } + s.NoError(s.vmDatastore.UpsertVirtualMachine(ctx, testV1VM)) + + testV2VM := &storage.VirtualMachineV2{ + Id: fixtureconsts.VirtualMachine2, + Name: "retired-vm-v2", + Namespace: "default", + ClusterId: clusterId, + ClusterName: testCluster.GetName(), + GuestOs: "rhel9", + State: storage.VirtualMachineV2_RUNNING, + } + s.NoError(s.vmV2Datastore.UpsertVirtualMachine(ctx, testV2VM)) + s.NoError(s.vmV2Datastore.UpsertScan(ctx, testV2VM.GetId(), clusterRemovalTestScanParts(testV2VM.GetId()))) + + keptCluster := &storage.Cluster{Name: fixtureconsts.ClusterName2, MainImage: mainImage, CentralApiEndpoint: centralEndpoint} + keptClusterID, keptClusterAddErr := s.clusterDatastore.AddCluster(ctx, keptCluster) + s.NotEmpty(keptClusterID) + s.NoError(keptClusterAddErr) + keptV1VM := &storage.VirtualMachine{ + Id: fixtureconsts.VirtualMachineFake, + Name: "kept-vm-v1", + Namespace: "default", + ClusterId: keptClusterID, + } + s.NoError(s.vmDatastore.UpsertVirtualMachine(ctx, keptV1VM)) + // Remove cluster and verify that the removal has been cascaded to all related components doneSignal := concurrency.NewSignal() clusterRemoveErr := s.clusterDatastore.RemoveCluster(ctx, clusterId, &doneSignal) @@ -208,6 +261,66 @@ func (s *ClusterPostgresDataStoreTestSuite) TestRemoveCluster() { _, imageIntegrationFound, imageIntegrationGetErr := s.imageIntegrationDatastore.GetImageIntegration(ctx, testImageIntegration.GetId()) s.NoError(imageIntegrationGetErr) s.False(imageIntegrationFound) + + v1Count, v1CountErr := s.vmDatastore.CountVirtualMachines(ctx, pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.ClusterID, clusterId).ProtoQuery()) + s.NoError(v1CountErr) + s.Zero(v1Count) + _, v1Found, v1GetErr := s.vmDatastore.GetVirtualMachine(ctx, testV1VM.GetId()) + s.NoError(v1GetErr) + s.False(v1Found) + + v2Count, v2CountErr := s.vmV2Datastore.CountVirtualMachines(ctx, pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.ClusterID, clusterId).ProtoQuery()) + s.NoError(v2CountErr) + s.Zero(v2Count) + _, v2Found, v2GetErr := s.vmV2Datastore.GetVirtualMachine(ctx, testV2VM.GetId()) + s.NoError(v2GetErr) + s.False(v2Found) + + cves, cveSearchErr := s.vmCVEDatastore.SearchRawVMCVEs(ctx, pkgSearch.EmptyQuery()) + s.NoError(cveSearchErr) + s.Empty(cves) + + _, keptFound, keptGetErr := s.vmDatastore.GetVirtualMachine(ctx, keptV1VM.GetId()) + s.NoError(keptGetErr) + s.True(keptFound) +} + +func clusterRemovalTestScanParts(vmID string) common.VMScanParts { + scanID := uuid.NewV4().String() + compID := uuid.NewV4().String() + cveID := uuid.NewV4().String() + return common.VMScanParts{ + Scan: &storage.VirtualMachineScanV2{ + Id: scanID, + VmV2Id: vmID, + ScanOs: "rhel9", + }, + Components: []*storage.VirtualMachineComponentV2{ + { + Id: compID, + VmScanId: scanID, + Name: "openssl", + Version: "1.1.1", + Source: storage.SourceType_OS, + OperatingSystem: "rhel:9", + }, + }, + CVEs: []*storage.VirtualMachineCVEV2{ + { + Id: cveID, + VmV2Id: vmID, + VmComponentId: compID, + CveBaseInfo: &storage.CVEInfo{ + Cve: "CVE-2024-0001", + Summary: "test vulnerability", + }, + PreferredCvss: 7.5, + Severity: storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, + IsFixable: true, + HasFixedBy: &storage.VirtualMachineCVEV2_FixedBy{FixedBy: "1.1.2"}, + }, + }, + } } func (s *ClusterPostgresDataStoreTestSuite) TestPopulateClusterHealthInfo() { @@ -457,6 +570,13 @@ func (s *ClusterPostgresDataStoreTestSuite) TestLookupOrCreateClusterFromConfig( } ctx := sac.WithAllAccess(context.Background()) + _ = s.clusterInitStore.Delete(ctx, bundleID) + err = s.clusterInitStore.Add(ctx, &storage.InitBundleMeta{ + Id: bundleID, + Name: "human-readable-bundle-name", + CreatedAt: timestamppb.New(time.Now()), + }) + s.NoError(err, "adding InitBundleMeta (init bundle or CRS) to clusterInitStore") if c.shouldClusterBeUpserted { clusterID, err = s.clusterDatastore.AddCluster(ctx, c.cluster) @@ -1023,7 +1143,7 @@ func (s *ClusterPostgresDataStoreTestSuite) TestSearchRawClustersConsistentOrder const numIterations = 10 var firstResults []*storage.Cluster - for i := 0; i < numIterations; i++ { + for i := range numIterations { results, err := s.clusterDatastore.SearchRawClusters(ctx, pkgSearch.EmptyQuery()) s.Require().NoError(err) s.Require().NotEmpty(results) @@ -1047,7 +1167,7 @@ func (s *ClusterPostgresDataStoreTestSuite) TestSearchRawClustersConsistentOrder query := pkgSearch.NewQueryBuilder().AddMapQuery(pkgSearch.ClusterLabel, "test", "ordering").ProtoQuery() var firstFilteredResults []*storage.Cluster - for i := 0; i < numIterations; i++ { + for i := range numIterations { results, err := s.clusterDatastore.SearchRawClusters(ctx, query) s.Require().NoError(err) s.Require().Len(results, len(clusterNames), "Should return all test clusters") diff --git a/central/cluster/datastore/datastore_impl_test.go b/central/cluster/datastore/datastore_impl_test.go index fe052d2afaf8b..0ba81c3c61ccf 100644 --- a/central/cluster/datastore/datastore_impl_test.go +++ b/central/cluster/datastore/datastore_impl_test.go @@ -2,7 +2,10 @@ package datastore import ( "errors" + "math" + "regexp" "testing" + "time" alertDataStoreMocks "github.com/stackrox/rox/central/alert/datastore/mocks" clusterStoreMocks "github.com/stackrox/rox/central/cluster/store/cluster/mocks" @@ -11,6 +14,8 @@ import ( clusterCVEDataStoreMocks "github.com/stackrox/rox/central/cve/cluster/datastore/mocks" deploymentDataStoreMocks "github.com/stackrox/rox/central/deployment/datastore/mocks" imageIntegrationDataStoreMocks "github.com/stackrox/rox/central/imageintegration/datastore/mocks" + "github.com/stackrox/rox/central/metrics" + "github.com/stackrox/rox/central/metrics/custom/refresh" namespaceDataStoreMocks "github.com/stackrox/rox/central/namespace/datastore/mocks" networkBaselineManagerMocks "github.com/stackrox/rox/central/networkbaseline/manager/mocks" netEntityDataStoreMocks "github.com/stackrox/rox/central/networkgraph/entity/datastore/mocks" @@ -23,20 +28,35 @@ import ( secretDataStoreMocks "github.com/stackrox/rox/central/secret/datastore/mocks" connectionMocks "github.com/stackrox/rox/central/sensor/service/connection/mocks" serviceAccountDataStoreMocks "github.com/stackrox/rox/central/serviceaccount/datastore/mocks" + virtualMachineDSMocks "github.com/stackrox/rox/central/virtualmachine/datastore/mocks" + virtualMachineV2DSMocks "github.com/stackrox/rox/central/virtualmachine/v2/datastore/mocks" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + clusterPkg "github.com/stackrox/rox/pkg/cluster" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" notifierMocks "github.com/stackrox/rox/pkg/notifier/mocks" + "github.com/stackrox/rox/pkg/protoconv" "github.com/stackrox/rox/pkg/protomock" "github.com/stackrox/rox/pkg/sac" pkgSearch "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/simplecache" "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" ) +type recordingRefresher struct { + calls []string +} + +func (r *recordingRefresher) RefreshTracker(prefix string) { + r.calls = append(r.calls, prefix) +} + func TestClusterDataStore(t *testing.T) { suite.Run(t, new(clusterDataStoreTestSuite)) } @@ -64,12 +84,15 @@ type clusterDataStoreTestSuite struct { k8sRoleBindingDS *roleBindingDataStoreMocks.MockDataStore secretDS *secretDataStoreMocks.MockDataStore serviceAccountDS *serviceAccountDataStoreMocks.MockDataStore + vmDS *virtualMachineDSMocks.MockDataStore + vmV2DS *virtualMachineV2DSMocks.MockDataStore networkBaselineMgr *networkBaselineManagerMocks.MockManager notifierProcessor *notifierMocks.MockProcessor compliancePruner *compliancePrunerMocks.MockPruner + refresher *recordingRefresher datastore *datastoreImpl } @@ -95,12 +118,16 @@ func (s *clusterDataStoreTestSuite) SetupTest() { s.k8sRoleBindingDS = roleBindingDataStoreMocks.NewMockDataStore(s.mockCtrl) s.secretDS = secretDataStoreMocks.NewMockDataStore(s.mockCtrl) s.serviceAccountDS = serviceAccountDataStoreMocks.NewMockDataStore(s.mockCtrl) + s.vmDS = virtualMachineDSMocks.NewMockDataStore(s.mockCtrl) + s.vmV2DS = virtualMachineV2DSMocks.NewMockDataStore(s.mockCtrl) s.networkBaselineMgr = networkBaselineManagerMocks.NewMockManager(s.mockCtrl) s.notifierProcessor = notifierMocks.NewMockProcessor(s.mockCtrl) s.compliancePruner = compliancePrunerMocks.NewMockPruner(s.mockCtrl) + s.refresher = &recordingRefresher{} + refresh.SetSingleton(s.refresher) // Create datastore instance with all mocks s.datastore = &datastoreImpl{ @@ -119,18 +146,27 @@ func (s *clusterDataStoreTestSuite) SetupTest() { serviceAccountDataStore: s.serviceAccountDS, roleDataStore: s.k8sRoleDS, roleBindingDataStore: s.k8sRoleBindingDS, + virtualMachineDataStore: s.vmDS, + virtualMachineV2DataStore: s.vmV2DS, cm: s.sensorConnectionMgr, notifier: s.notifierProcessor, clusterRanker: ranking.NewRanker(), networkBaselineMgr: s.networkBaselineMgr, compliancePruner: s.compliancePruner, idToNameCache: simplecache.New(), + idToNamespaceFilterCache: simplecache.New(), nameToIDCache: simplecache.New(), } } func (s *clusterDataStoreTestSuite) TearDownTest() { s.mockCtrl.Finish() + refresh.SetSingleton(nil) + + // reset caches + s.datastore.idToNameCache = simplecache.New() + s.datastore.idToNamespaceFilterCache = simplecache.New() + s.datastore.nameToIDCache = simplecache.New() } type lookupResult struct { @@ -151,6 +187,9 @@ func (s *clusterDataStoreTestSuite) TestPostRemoveCluster() { clusterIDSearchQuery := pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.ClusterID, clusterID).ProtoQuery() matchClusterIDSearchQuery := protomock.GoMockMatcherEqualMessage(clusterIDSearchQuery) + vmClusterIDSearchQuery := clusterIDSearchQuery.CloneVT() + vmClusterIDSearchQuery.Pagination = &v1.QueryPagination{Limit: math.MaxInt32} + matchVMClusterIDSearchQuery := protomock.GoMockMatcherEqualMessage(vmClusterIDSearchQuery) testError := errors.New("test error") @@ -174,8 +213,8 @@ func (s *clusterDataStoreTestSuite) TestPostRemoveCluster() { }, } { s.Run(name, func() { - var searchResults []pkgSearch.Result - var resultIDs []string + searchResults := make([]pkgSearch.Result, 0, len(tc.results)) + resultIDs := make([]string, 0, len(tc.results)) for _, result := range tc.results { searchResults = append(searchResults, pkgSearch.Result{ID: result.ID}) resultIDs = append(resultIDs, result.ID) @@ -226,8 +265,8 @@ func (s *clusterDataStoreTestSuite) TestPostRemoveCluster() { Times(1). Return(result.status) // For each deployment, get alerts and mark them stale - var alertIDs []string - var resolvedAlerts []*storage.Alert + alertIDs := make([]string, 0, len(tc.results)) + resolvedAlerts := make([]*storage.Alert, 0, len(tc.results)) for _, alertResult := range tc.results { alertID := uuid.NewV5FromNonUUIDs(result.ID, alertResult.ID).String() alertIDs = append(alertIDs, alertID) @@ -274,6 +313,31 @@ func (s *clusterDataStoreTestSuite) TestPostRemoveCluster() { Times(1). Return(tc.status) + // Remove virtual machines (v1 and v2) + v1VMs := make([]*storage.VirtualMachine, 0, len(tc.results)) + for _, result := range tc.results { + v1VMs = append(v1VMs, &storage.VirtualMachine{Id: result.ID}) + } + s.vmDS.EXPECT(). + SearchRawVirtualMachines(gomock.Any(), matchVMClusterIDSearchQuery). + Times(1). + Return(v1VMs, tc.status) + s.vmV2DS.EXPECT(). + Search(gomock.Any(), matchVMClusterIDSearchQuery). + Times(1). + Return(searchResults, tc.status) + if tc.status == nil && len(resultIDs) > 0 { + var deleteErr error + for _, result := range tc.results { + if result.status != nil { + deleteErr = result.status + break + } + } + s.vmDS.EXPECT().DeleteVirtualMachines(gomock.Any(), resultIDs[0], resultIDs[1]).Times(1).Return(deleteErr) + s.vmV2DS.EXPECT().DeleteVirtualMachines(gomock.Any(), resultIDs[0], resultIDs[1]).Times(1).Return(deleteErr) + } + // 9. Delete external network entities for cluster s.networkEntityDS.EXPECT(). DeleteExternalNetworkEntitiesForCluster(gomock.Any(), clusterID). @@ -307,7 +371,7 @@ func (s *clusterDataStoreTestSuite) TestPostRemoveCluster() { } // 13. Remove secrets - var listSecrets []*storage.ListSecret + listSecrets := make([]*storage.ListSecret, 0, len(tc.results)) for _, result := range tc.results { listSecrets = append(listSecrets, &storage.ListSecret{Id: result.ID}) s.secretDS.EXPECT(). @@ -370,9 +434,686 @@ func (s *clusterDataStoreTestSuite) TestPostRemoveCluster() { ctx := sac.WithAllAccess(s.T().Context()) doneSignal := concurrency.NewSignal() - s.datastore.postRemoveCluster(ctx, removedCluster, &doneSignal) + s.datastore.postRemoveCluster(ctx, removedCluster, &doneSignal, func() {}) doneSignal.Wait() }) } } + +func TestExtractClusterConfig(t *testing.T) { + t.Run("extracts all fields correctly", func(t *testing.T) { + hello := ¢ral.SensorHello{ + HelmManagedConfigInit: ¢ral.HelmManagedConfigInit{ + ClusterName: "test-cluster", + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + ClusterConfig: &storage.CompleteClusterConfig{ + ConfigFingerprint: "fingerprint123", + }, + }, + DeploymentIdentification: &storage.SensorDeploymentIdentification{ + AppNamespace: "stackrox", + }, + Capabilities: []string{"cap1", "cap2"}, + } + + config := extractClusterConfig(hello) + + assert.Equal(t, "test-cluster", config.clusterName) + assert.Equal(t, storage.ManagerType_MANAGER_TYPE_HELM_CHART, config.manager) + assert.Equal(t, "fingerprint123", config.helmConfig.GetConfigFingerprint()) + assert.Equal(t, "stackrox", config.deploymentIdentification.GetAppNamespace()) + assert.Equal(t, []string{"cap1", "cap2"}, config.capabilities) + assert.True(t, config.isNotManagedManually) + }) + + t.Run("handles nil values gracefully", func(t *testing.T) { + hello := ¢ral.SensorHello{} + + config := extractClusterConfig(hello) + + assert.Empty(t, config.clusterName) + assert.Equal(t, storage.ManagerType_MANAGER_TYPE_UNKNOWN, config.manager) + assert.Nil(t, config.helmConfig) + assert.False(t, config.isNotManagedManually) + }) +} + +func TestShouldUpdateCluster(t *testing.T) { + baseConfig := clusterConfigData{ + manager: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + helmConfig: &storage.CompleteClusterConfig{ + ConfigFingerprint: "fp123", + }, + capabilities: []string{"cap1", "cap2"}, + } + + t.Run("returns false when nothing changed", func(t *testing.T) { + cluster := &storage.Cluster{ + SensorCapabilities: []string{"cap1", "cap2"}, + InitBundleId: "bundle123", + HelmConfig: &storage.CompleteClusterConfig{ + ConfigFingerprint: "fp123", + }, + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + } + + needsUpdate := shouldUpdateCluster(cluster, baseConfig, "bundle123") + assert.False(t, needsUpdate) + }) + + t.Run("returns true when capabilities changed", func(t *testing.T) { + cluster := &storage.Cluster{ + SensorCapabilities: []string{"cap1", "cap3"}, + InitBundleId: "bundle123", + HelmConfig: &storage.CompleteClusterConfig{ + ConfigFingerprint: "fp123", + }, + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + } + + needsUpdate := shouldUpdateCluster(cluster, baseConfig, "bundle123") + assert.True(t, needsUpdate) + }) + + t.Run("returns true when init bundle ID changed", func(t *testing.T) { + cluster := &storage.Cluster{ + SensorCapabilities: []string{"cap1", "cap2"}, + InitBundleId: "old-bundle", + HelmConfig: &storage.CompleteClusterConfig{ + ConfigFingerprint: "fp123", + }, + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + } + + needsUpdate := shouldUpdateCluster(cluster, baseConfig, "new-bundle") + assert.True(t, needsUpdate) + }) + + t.Run("returns true when fingerprint changed", func(t *testing.T) { + cluster := &storage.Cluster{ + SensorCapabilities: []string{"cap1", "cap2"}, + InitBundleId: "bundle123", + HelmConfig: &storage.CompleteClusterConfig{ + ConfigFingerprint: "old-fp", + }, + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + } + + needsUpdate := shouldUpdateCluster(cluster, baseConfig, "bundle123") + assert.True(t, needsUpdate) + }) + + t.Run("returns true when manager type changed", func(t *testing.T) { + cluster := &storage.Cluster{ + SensorCapabilities: []string{"cap1", "cap2"}, + InitBundleId: "bundle123", + HelmConfig: &storage.CompleteClusterConfig{ + ConfigFingerprint: "fp123", + }, + ManagedBy: storage.ManagerType_MANAGER_TYPE_KUBERNETES_OPERATOR, + } + + needsUpdate := shouldUpdateCluster(cluster, baseConfig, "bundle123") + assert.True(t, needsUpdate) + }) + + t.Run("handles capability order independence", func(t *testing.T) { + cluster := &storage.Cluster{ + SensorCapabilities: []string{"cap2", "cap1"}, // Different order + InitBundleId: "bundle123", + HelmConfig: &storage.CompleteClusterConfig{ + ConfigFingerprint: "fp123", + }, + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + } + + needsUpdate := shouldUpdateCluster(cluster, baseConfig, "bundle123") + assert.False(t, needsUpdate, "capability order should not matter") + }) +} + +func TestBuildNewClusterFromConfig(t *testing.T) { + t.Run("builds cluster with all fields", func(t *testing.T) { + config := clusterConfigData{ + manager: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + helmConfig: &storage.CompleteClusterConfig{ + StaticConfig: &storage.StaticClusterConfig{ + Type: storage.ClusterType_KUBERNETES_CLUSTER, + MainImage: "stackrox/main:latest", + }, + }, + isNotManagedManually: true, + deploymentIdentification: &storage.SensorDeploymentIdentification{ + AppNamespace: "stackrox", + }, + capabilities: []string{"cap1", "cap2"}, + } + + cluster := buildNewClusterFromConfig("test-cluster", "bundle-123", config) + + assert.Equal(t, "test-cluster", cluster.GetName()) + assert.Equal(t, "bundle-123", cluster.GetInitBundleId()) + assert.Equal(t, storage.ManagerType_MANAGER_TYPE_HELM_CHART, cluster.GetManagedBy()) + assert.Equal(t, "stackrox", cluster.GetMostRecentSensorId().GetAppNamespace()) + assert.ElementsMatch(t, []string{"cap1", "cap2"}, cluster.GetSensorCapabilities()) + assert.NotNil(t, cluster.GetHelmConfig()) + }) + + t.Run("does not set HelmConfig for manually managed clusters", func(t *testing.T) { + config := clusterConfigData{ + manager: storage.ManagerType_MANAGER_TYPE_MANUAL, + helmConfig: nil, + isNotManagedManually: false, + deploymentIdentification: &storage.SensorDeploymentIdentification{}, + capabilities: []string{}, + } + + cluster := buildNewClusterFromConfig("test-cluster", "bundle-123", config) + + assert.Nil(t, cluster.GetHelmConfig()) + }) + + t.Run("capabilities are sorted", func(t *testing.T) { + config := clusterConfigData{ + manager: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + helmConfig: &storage.CompleteClusterConfig{ + StaticConfig: &storage.StaticClusterConfig{}, + }, + isNotManagedManually: true, + deploymentIdentification: &storage.SensorDeploymentIdentification{}, + capabilities: []string{"zzz", "aaa", "mmm"}, + } + + cluster := buildNewClusterFromConfig("test-cluster", "bundle-123", config) + + assert.Equal(t, []string{"aaa", "mmm", "zzz"}, cluster.GetSensorCapabilities()) + }) +} + +func TestApplyConfigToCluster(t *testing.T) { + t.Run("applies all updates for Helm-managed cluster", func(t *testing.T) { + original := &storage.Cluster{ + Id: "cluster-id", + Name: "test-cluster", + ManagedBy: storage.ManagerType_MANAGER_TYPE_MANUAL, + InitBundleId: "old-bundle", + SensorCapabilities: []string{"old-cap"}, + } + + config := clusterConfigData{ + manager: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + helmConfig: &storage.CompleteClusterConfig{ + ConfigFingerprint: "new-fp", + StaticConfig: &storage.StaticClusterConfig{ + Type: storage.ClusterType_KUBERNETES_CLUSTER, + }, + }, + isNotManagedManually: true, + capabilities: []string{"new-cap1", "new-cap2"}, + } + + updated := applyConfigToCluster(original, config) + + assert.Equal(t, storage.ManagerType_MANAGER_TYPE_HELM_CHART, updated.GetManagedBy()) + assert.ElementsMatch(t, []string{"new-cap1", "new-cap2"}, updated.GetSensorCapabilities()) + assert.NotNil(t, updated.GetHelmConfig()) + assert.Equal(t, "new-fp", updated.GetHelmConfig().GetConfigFingerprint()) + }) + + t.Run("clears HelmConfig for manually managed cluster", func(t *testing.T) { + original := &storage.Cluster{ + Id: "cluster-id", + Name: "test-cluster", + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + InitBundleId: "bundle", + HelmConfig: &storage.CompleteClusterConfig{ + ConfigFingerprint: "old-fp", + }, + } + + config := clusterConfigData{ + manager: storage.ManagerType_MANAGER_TYPE_MANUAL, + helmConfig: &storage.CompleteClusterConfig{}, + isNotManagedManually: false, + capabilities: []string{}, + } + + updated := applyConfigToCluster(original, config) + + assert.Equal(t, storage.ManagerType_MANAGER_TYPE_MANUAL, updated.GetManagedBy()) + assert.Nil(t, updated.GetHelmConfig()) + }) + + t.Run("does not mutate original cluster", func(t *testing.T) { + original := &storage.Cluster{ + Id: "cluster-id", + Name: "test-cluster", + ManagedBy: storage.ManagerType_MANAGER_TYPE_MANUAL, + InitBundleId: "old-bundle", + } + + config := clusterConfigData{ + manager: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + helmConfig: &storage.CompleteClusterConfig{}, + isNotManagedManually: true, + capabilities: []string{"cap1"}, + } + + updated := applyConfigToCluster(original, config) + + // Original should be unchanged + assert.Equal(t, storage.ManagerType_MANAGER_TYPE_MANUAL, original.GetManagedBy()) + assert.Equal(t, "old-bundle", original.GetInitBundleId()) + + // Updated should have new values + assert.Equal(t, storage.ManagerType_MANAGER_TYPE_HELM_CHART, updated.GetManagedBy()) + }) +} + +func TestCheckGracePeriodForReconnect(t *testing.T) { + deploymentID := &storage.SensorDeploymentIdentification{ + AppNamespace: "stackrox", + SystemNamespaceId: "123", + AppNamespaceId: "456", + DefaultNamespaceId: "789", + } + + t.Run("allows reconnect when last contact is old", func(t *testing.T) { + // Cluster with old last contact (outside grace period) + cluster := &storage.Cluster{ + HealthStatus: &storage.ClusterHealthStatus{ + LastContact: nil, // Defaults to zero time, definitely outside grace period + }, + MostRecentSensorId: deploymentID.CloneVT(), + } + + err := checkGracePeriodForReconnect(cluster, deploymentID, storage.ManagerType_MANAGER_TYPE_HELM_CHART) + assert.NoError(t, err) + }) + + t.Run("allows reconnect with matching deployment ID even within grace period", func(t *testing.T) { + // Same deployment ID should always be allowed, even with recent contact + cluster := &storage.Cluster{ + HealthStatus: &storage.ClusterHealthStatus{ + LastContact: protoconv.ConvertTimeToTimestampOrNil( + time.Now().Add(-1 * time.Minute)), // 1 minute ago + }, + MostRecentSensorId: deploymentID.CloneVT(), + } + + err := checkGracePeriodForReconnect(cluster, deploymentID, storage.ManagerType_MANAGER_TYPE_HELM_CHART) + assert.NoError(t, err) + }) + + t.Run("returns error during grace period with different deployment IDs", func(t *testing.T) { + // Control the environment variable to ensure grace period is enforced + t.Setenv("ROX_SCALE_TEST", "false") + + // Create a cluster with RECENT last contact (within 3-minute grace period) + cluster := &storage.Cluster{ + HealthStatus: &storage.ClusterHealthStatus{ + LastContact: protoconv.ConvertTimeToTimestampOrNil( + time.Now().Add(-1 * time.Minute)), // 1 minute ago + }, + MostRecentSensorId: &storage.SensorDeploymentIdentification{ + AppNamespace: "stackrox", + SystemNamespaceId: "old-cluster-kube-system-uid", + }, + } + + // New deployment from a DIFFERENT cluster (different SystemNamespaceId) + newDeploymentID := &storage.SensorDeploymentIdentification{ + AppNamespace: "stackrox", // Same namespace name + SystemNamespaceId: "new-cluster-kube-system-uid", // Different cluster! + } + + err := checkGracePeriodForReconnect(cluster, newDeploymentID, storage.ManagerType_MANAGER_TYPE_HELM_CHART) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "registering Helm-managed cluster is not allowed") + assert.Contains(t, err.Error(), "please wait") + }) + + t.Run("returns error during grace period for operator-managed cluster", func(t *testing.T) { + // Control the environment variable to ensure grace period is enforced + t.Setenv("ROX_SCALE_TEST", "false") + + // Create a cluster with RECENT last contact + cluster := &storage.Cluster{ + HealthStatus: &storage.ClusterHealthStatus{ + LastContact: protoconv.ConvertTimeToTimestampOrNil( + time.Now().Add(-90 * time.Second)), // 90 seconds ago + }, + MostRecentSensorId: &storage.SensorDeploymentIdentification{ + AppNamespace: "stackrox", + DefaultNamespaceId: "default-namespace-uid-1", + }, + } + + // New deployment from a different cluster (different DefaultNamespaceId) + newDeploymentID := &storage.SensorDeploymentIdentification{ + AppNamespace: "stackrox", + DefaultNamespaceId: "default-namespace-uid-2", // Different cluster! + } + + err := checkGracePeriodForReconnect(cluster, newDeploymentID, storage.ManagerType_MANAGER_TYPE_KUBERNETES_OPERATOR) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "registering Operator-managed cluster is not allowed") + assert.Contains(t, err.Error(), "please wait") + }) +} + +func (s *clusterDataStoreTestSuite) TestGetClusterLabels() { + ctx := sac.WithAllAccess(s.T().Context()) + clusterID := fixtureconsts.Cluster1 + + testCases := []struct { + name string + clusterLabels map[string]string + clusterExists bool + getClusterErr error + expectedLabels map[string]string + expectedErr bool + }{ + { + name: "cluster exists with labels", + clusterLabels: map[string]string{"env": "prod", "team": "platform"}, + clusterExists: true, + getClusterErr: nil, + expectedLabels: map[string]string{"env": "prod", "team": "platform"}, + expectedErr: false, + }, + { + name: "cluster exists with no labels", + clusterLabels: map[string]string{}, + clusterExists: true, + getClusterErr: nil, + expectedLabels: map[string]string{}, + expectedErr: false, + }, + { + name: "cluster does not exist", + clusterExists: false, + getClusterErr: nil, + expectedLabels: nil, + expectedErr: false, + }, + { + name: "error fetching cluster", + clusterExists: false, + getClusterErr: errors.New("database error"), + expectedErr: true, + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + // Setup mock expectations + if tc.getClusterErr != nil { + s.clusterStore.EXPECT(). + Get(gomock.Any(), clusterID). + Return(nil, false, tc.getClusterErr) + } else if tc.clusterExists { + cluster := &storage.Cluster{ + Id: clusterID, + Labels: tc.clusterLabels, + } + s.clusterStore.EXPECT(). + Get(gomock.Any(), clusterID). + Return(cluster, true, nil) + s.clusterHealthStore.EXPECT(). + GetMany(gomock.Any(), gomock.Any()). + Return(nil, nil, nil). + AnyTimes() + } else { + s.clusterStore.EXPECT(). + Get(gomock.Any(), clusterID). + Return(nil, false, nil) + } + + // Call GetClusterLabels + labels, err := s.datastore.GetClusterLabels(ctx, clusterID) + + // Verify results + if tc.expectedErr { + s.Error(err) + } else { + s.NoError(err) + s.Equal(tc.expectedLabels, labels) + } + }) + } +} + +func (s *clusterDataStoreTestSuite) TestProcessMatching() { + clusterID := fixtureconsts.Cluster1 + testCluster := &storage.Cluster{ + Id: clusterID, + Name: "test", + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + HelmConfig: &storage.CompleteClusterConfig{ + DynamicConfig: &storage.DynamicClusterConfig{ + ProcessIndicators: &storage.DynamicClusterConfig_ProcessIndicatorsConfig{ + ExcludeNamespaceFilter: "test-.*", + NoPersistence: false, + }, + }, + }, + } + + ctx := sac.WithAllAccess(s.T().Context()) + + // populate regexp cache + filter := clusterPkg.GetNamespaceFilter(testCluster) + s.datastore.idToNamespaceFilterCache.Add(clusterID, regexp.MustCompile(*filter)) + + indicator := &storage.ProcessIndicator{ + Id: uuid.NewV4().String(), + DeploymentId: uuid.NewV4().String(), + ContainerName: uuid.NewV4().String(), + PodId: uuid.NewV4().String(), + ClusterId: clusterID, + Namespace: "test-namespace", + } + + // The process with matching namespace should be found + match, err := s.datastore.MatchProcessIndicator(ctx, indicator) + s.NoError(err) + assert.True(s.T(), match) + + // We change the namespace to not match anymore, and the process passes + indicator.Namespace = "other-namespace" + match, err = s.datastore.MatchProcessIndicator(ctx, indicator) + s.NoError(err) + assert.False(s.T(), match) + + // We change the namespace to match the openshift pattern, + // and ask to exclude it + indicator.Namespace = "openshift-test" + testCluster.HelmConfig.DynamicConfig.ProcessIndicators.ExcludeOpenshiftNs = true + + // We need to update the cache to take persistence into account + filter = clusterPkg.GetNamespaceFilter(testCluster) + s.datastore.idToNamespaceFilterCache.Add(clusterID, regexp.MustCompile(*filter)) + + match, err = s.datastore.MatchProcessIndicator(ctx, indicator) + s.NoError(err) + assert.True(s.T(), match) + + // No matter how the namespace looks like, if no persistence is requested, + // the process will match + indicator.Namespace = "something-completely-different" + testCluster.HelmConfig.DynamicConfig.ProcessIndicators.NoPersistence = true + + // We need to update the cache to take persistence into account + filter = clusterPkg.GetNamespaceFilter(testCluster) + s.datastore.idToNamespaceFilterCache.Add(clusterID, regexp.MustCompile(*filter)) + + match, err = s.datastore.MatchProcessIndicator(ctx, indicator) + s.NoError(err) + assert.True(s.T(), match) +} + +func (s *clusterDataStoreTestSuite) TestUpdateCluster() { + clusterID := fixtureconsts.Cluster1 + testCluster := &storage.Cluster{ + Id: clusterID, + Name: "test", + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + MainImage: "docker.io/stackrox/rox:latest", + HelmConfig: &storage.CompleteClusterConfig{ + DynamicConfig: &storage.DynamicClusterConfig{ + ProcessIndicators: &storage.DynamicClusterConfig_ProcessIndicatorsConfig{ + ExcludeNamespaceFilter: "test-.*", + NoPersistence: false, + }, + }, + }, + } + + ctx := sac.WithAllAccess(s.T().Context()) + + s.clusterStore.EXPECT(). + Upsert(gomock.Any(), gomock.Any()). + Return(nil) + + err := s.datastore.updateClusterNoLock(ctx, testCluster) + s.NoError(err) + + filter, ok := s.datastore.idToNamespaceFilterCache.Get(clusterID) + assert.True(s.T(), ok) + assert.Equal(s.T(), filter.(*regexp.Regexp).String(), "test-.*") +} + +func (s *clusterDataStoreTestSuite) TestUpdateClusterIncorrectFilter() { + clusterID := fixtureconsts.Cluster1 + testCluster := &storage.Cluster{ + Id: clusterID, + Name: "test", + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + MainImage: "docker.io/stackrox/rox:latest", + HelmConfig: &storage.CompleteClusterConfig{ + DynamicConfig: &storage.DynamicClusterConfig{ + ProcessIndicators: &storage.DynamicClusterConfig_ProcessIndicatorsConfig{ + ExcludeNamespaceFilter: "[", + NoPersistence: false, + }, + }, + }, + } + + ctx := sac.WithAllAccess(s.T().Context()) + + s.clusterStore.EXPECT(). + Upsert(gomock.Any(), gomock.Any()). + Return(nil) + + err := s.datastore.updateClusterNoLock(ctx, testCluster) + s.NoError(err) + + filter, ok := s.datastore.idToNamespaceFilterCache.Get(clusterID) + assert.False(s.T(), ok) + assert.Nil(s.T(), filter) +} + +func (s *clusterDataStoreTestSuite) TestUpdateClusterCleanupCache() { + clusterID := fixtureconsts.Cluster1 + testCluster := &storage.Cluster{ + Id: clusterID, + Name: "test", + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + MainImage: "docker.io/stackrox/rox:latest", + HelmConfig: &storage.CompleteClusterConfig{ + DynamicConfig: &storage.DynamicClusterConfig{ + ProcessIndicators: &storage.DynamicClusterConfig_ProcessIndicatorsConfig{ + ExcludeNamespaceFilter: "", + NoPersistence: false, + }, + }, + }, + } + + ctx := sac.WithAllAccess(s.T().Context()) + + s.clusterStore.EXPECT(). + Upsert(gomock.Any(), gomock.Any()). + Return(nil) + + // populate regexp cache + s.datastore.idToNamespaceFilterCache.Add(clusterID, regexp.MustCompile("test-.*")) + + err := s.datastore.updateClusterNoLock(ctx, testCluster) + s.NoError(err) + + filter, ok := s.datastore.idToNamespaceFilterCache.Get(clusterID) + assert.False(s.T(), ok) + assert.Nil(s.T(), filter) +} + +func (s *clusterDataStoreTestSuite) TestMetricsRefreshOnClusterStatusUpdate() { + ctx := sac.WithAllAccess(s.T().Context()) + clusterID := fixtureconsts.Cluster1 + + for name, tc := range map[string]struct { + update func() error + upsertErr error + expectedCalls []string + }{ + "UpdateClusterUpgradeStatus refreshes on success": { + update: func() error { + return s.datastore.UpdateClusterUpgradeStatus(ctx, clusterID, &storage.ClusterUpgradeStatus{}) + }, + expectedCalls: []string{metrics.Health}, + }, + "UpdateClusterUpgradeStatus skips refresh on error": { + update: func() error { + return s.datastore.UpdateClusterUpgradeStatus(ctx, clusterID, &storage.ClusterUpgradeStatus{}) + }, + upsertErr: errors.New("upsert failed"), + }, + "UpdateClusterCertExpiryStatus refreshes on success": { + update: func() error { + return s.datastore.UpdateClusterCertExpiryStatus(ctx, clusterID, &storage.ClusterCertExpiryStatus{}) + }, + expectedCalls: []string{metrics.Expiry}, + }, + "UpdateClusterCertExpiryStatus skips refresh on error": { + update: func() error { + return s.datastore.UpdateClusterCertExpiryStatus(ctx, clusterID, &storage.ClusterCertExpiryStatus{}) + }, + upsertErr: errors.New("upsert failed"), + }, + "UpdateClusterStatus refreshes on success": { + update: func() error { + return s.datastore.UpdateClusterStatus(ctx, clusterID, &storage.ClusterStatus{}) + }, + expectedCalls: []string{metrics.Health}, + }, + "UpdateClusterStatus skips refresh on error": { + update: func() error { + return s.datastore.UpdateClusterStatus(ctx, clusterID, &storage.ClusterStatus{}) + }, + upsertErr: errors.New("upsert failed"), + }, + } { + s.Run(name, func() { + s.refresher.calls = nil + s.clusterStore.EXPECT(). + Get(gomock.Any(), clusterID). + Return(&storage.Cluster{Id: clusterID, Status: &storage.ClusterStatus{}}, true, nil) + s.clusterStore.EXPECT(). + Upsert(gomock.Any(), gomock.Any()). + Return(tc.upsertErr) + + err := tc.update() + if tc.upsertErr != nil { + s.Error(err) + } else { + s.NoError(err) + } + s.Equal(tc.expectedCalls, s.refresher.calls) + }) + } +} diff --git a/central/cluster/datastore/datastore_sac_test.go b/central/cluster/datastore/datastore_sac_test.go index e0de49cbe6a1f..28d0efe70edff 100644 --- a/central/cluster/datastore/datastore_sac_test.go +++ b/central/cluster/datastore/datastore_sac_test.go @@ -18,6 +18,7 @@ import ( "github.com/stackrox/rox/pkg/sac/testconsts" "github.com/stackrox/rox/pkg/sac/testutils" searchPkg "github.com/stackrox/rox/pkg/search" + versionTestutils "github.com/stackrox/rox/pkg/version/testutils" "github.com/stretchr/testify/suite" ) @@ -44,6 +45,7 @@ type clusterDatastoreSACSuite struct { } func (s *clusterDatastoreSACSuite) SetupSuite() { + versionTestutils.SetMainVersion(s.T(), "4.5.0-testing") var err error s.pgTestBase = pgtest.ForT(s.T()) s.NotNil(s.pgTestBase) diff --git a/central/cluster/datastore/datastore_test_constructors.go b/central/cluster/datastore/datastore_test_constructors.go index c85bd284a5de7..b231c9eacfc82 100644 --- a/central/cluster/datastore/datastore_test_constructors.go +++ b/central/cluster/datastore/datastore_test_constructors.go @@ -6,6 +6,7 @@ import ( alertDataStore "github.com/stackrox/rox/central/alert/datastore" clusterPostgresStore "github.com/stackrox/rox/central/cluster/store/cluster/postgres" clusterHealthPostgresStore "github.com/stackrox/rox/central/cluster/store/clusterhealth/postgres" + clusterInitStore "github.com/stackrox/rox/central/clusterinit/store" compliancePruning "github.com/stackrox/rox/central/complianceoperator/v2/pruner" clusterCVEDataStore "github.com/stackrox/rox/central/cve/cluster/datastore" deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" @@ -24,6 +25,8 @@ import ( secretDataStore "github.com/stackrox/rox/central/secret/datastore" "github.com/stackrox/rox/central/sensor/service/connection" serviceAccountDataStore "github.com/stackrox/rox/central/serviceaccount/datastore" + virtualMachineDataStore "github.com/stackrox/rox/central/virtualmachine/datastore" + virtualMachineV2DataStore "github.com/stackrox/rox/central/virtualmachine/v2/datastore" "github.com/stackrox/rox/pkg/postgres" ) @@ -63,6 +66,8 @@ func GetTestPostgresDataStore(t testing.TB, pool postgres.DB) (DataStore, error) hashStore := datastore.GetTestPostgresDataStore(t, pool) + clusterInitStore := clusterInitStore.GetTestPostgresDataStore(t, pool) + sensorCnxMgr := connection.NewManager(hashManager.NewManager(hashStore)) clusterRanker := ranking.ClusterRanker() @@ -72,5 +77,7 @@ func GetTestPostgresDataStore(t testing.TB, pool postgres.DB) (DataStore, error) alertStore, iiStore, namespaceStore, deploymentStore, nodeStore, podStore, secretStore, netFlowStore, netEntityStore, serviceAccountStore, k8sRoleStore, k8sRoleBindingStore, sensorCnxMgr, nil, - clusterRanker, networkBaselineManager, compliancePruner) + clusterRanker, networkBaselineManager, compliancePruner, clusterInitStore, + virtualMachineDataStore.GetTestPostgresDataStore(t, pool), + virtualMachineV2DataStore.GetTestPostgresDataStore(t, pool)) } diff --git a/central/cluster/datastore/mocks/datastore.go b/central/cluster/datastore/mocks/datastore.go index 4573b88184dab..ba643ce95df76 100644 --- a/central/cluster/datastore/mocks/datastore.go +++ b/central/cluster/datastore/mocks/datastore.go @@ -122,6 +122,37 @@ func (mr *MockDataStoreMockRecorder) GetCluster(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCluster", reflect.TypeOf((*MockDataStore)(nil).GetCluster), ctx, id) } +// GetClusterID mocks base method. +func (m *MockDataStore) GetClusterID(ctx context.Context, name string) (string, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetClusterID", ctx, name) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetClusterID indicates an expected call of GetClusterID. +func (mr *MockDataStoreMockRecorder) GetClusterID(ctx, name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterID", reflect.TypeOf((*MockDataStore)(nil).GetClusterID), ctx, name) +} + +// GetClusterLabels mocks base method. +func (m *MockDataStore) GetClusterLabels(ctx context.Context, clusterID string) (map[string]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetClusterLabels", ctx, clusterID) + ret0, _ := ret[0].(map[string]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetClusterLabels indicates an expected call of GetClusterLabels. +func (mr *MockDataStoreMockRecorder) GetClusterLabels(ctx, clusterID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterLabels", reflect.TypeOf((*MockDataStore)(nil).GetClusterLabels), ctx, clusterID) +} + // GetClusterName mocks base method. func (m *MockDataStore) GetClusterName(ctx context.Context, id string) (string, bool, error) { m.ctrl.T.Helper() @@ -183,6 +214,21 @@ func (mr *MockDataStoreMockRecorder) LookupOrCreateClusterFromConfig(ctx, cluste return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LookupOrCreateClusterFromConfig", reflect.TypeOf((*MockDataStore)(nil).LookupOrCreateClusterFromConfig), ctx, clusterID, bundleID, hello) } +// MatchProcessIndicator mocks base method. +func (m *MockDataStore) MatchProcessIndicator(ctx context.Context, indicator *storage.ProcessIndicator) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MatchProcessIndicator", ctx, indicator) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MatchProcessIndicator indicates an expected call of MatchProcessIndicator. +func (mr *MockDataStoreMockRecorder) MatchProcessIndicator(ctx, indicator any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MatchProcessIndicator", reflect.TypeOf((*MockDataStore)(nil).MatchProcessIndicator), ctx, indicator) +} + // RemoveCluster mocks base method. func (m *MockDataStore) RemoveCluster(ctx context.Context, id string, done *concurrency.Signal) error { m.ctrl.T.Helper() diff --git a/central/cluster/datastore/singleton.go b/central/cluster/datastore/singleton.go index a4956600f7ffc..17f7b0408f68e 100644 --- a/central/cluster/datastore/singleton.go +++ b/central/cluster/datastore/singleton.go @@ -4,6 +4,7 @@ import ( alertDataStore "github.com/stackrox/rox/central/alert/datastore" clusterPostgres "github.com/stackrox/rox/central/cluster/store/cluster/postgres" clusterHealthPostgres "github.com/stackrox/rox/central/cluster/store/clusterhealth/postgres" + clusterInitStoreSingleton "github.com/stackrox/rox/central/clusterinit/store/singleton" compliancePruning "github.com/stackrox/rox/central/complianceoperator/v2/pruner" clusterCVEDS "github.com/stackrox/rox/central/cve/cluster/datastore" deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" @@ -22,6 +23,8 @@ import ( secretDataStore "github.com/stackrox/rox/central/secret/datastore" "github.com/stackrox/rox/central/sensor/service/connection" serviceAccountDataStore "github.com/stackrox/rox/central/serviceaccount/datastore" + virtualMachineDataStore "github.com/stackrox/rox/central/virtualmachine/datastore" + virtualMachineV2DataStore "github.com/stackrox/rox/central/virtualmachine/v2/datastore" "github.com/stackrox/rox/pkg/sync" "github.com/stackrox/rox/pkg/utils" ) @@ -58,7 +61,10 @@ func initialize() { notifierProcessor.Singleton(), ranking.ClusterRanker(), networkBaselineManager.Singleton(), - compliancePruning.Singleton()) + compliancePruning.Singleton(), + clusterInitStoreSingleton.Singleton(), + virtualMachineDataStore.Singleton(), + virtualMachineV2DataStore.Singleton()) utils.CrashOnError(err) } diff --git a/central/cluster/datastore/telemetry.go b/central/cluster/datastore/telemetry.go index 3a20123ba5a87..bd636efd5f689 100644 --- a/central/cluster/datastore/telemetry.go +++ b/central/cluster/datastore/telemetry.go @@ -2,11 +2,16 @@ package datastore import ( "context" + "encoding/json" + "maps" + "slices" + "strings" "time" "github.com/stackrox/rox/central/telemetry/centralclient" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + clusterPkg "github.com/stackrox/rox/pkg/cluster" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/telemetry/phonehome" @@ -40,14 +45,15 @@ func trackClusterRegistered(cluster *storage.Cluster) { func makeClusterProperties(cluster *storage.Cluster) map[string]any { return map[string]any{ - "Main Image": cluster.GetMainImage(), - "Admission Controller": cluster.GetAdmissionController(), - "Collection Method": cluster.GetCollectionMethod().String(), - "Collector Image": cluster.GetCollectorImage(), - "Managed By": cluster.GetManagedBy().String(), - "Priority": cluster.GetPriority(), - "Cluster Type": cluster.GetType().String(), - "Slim Collector": cluster.GetSlimCollector(), + "Main Image": cluster.GetMainImage(), + "Admission Controller": cluster.GetAdmissionController(), + "Collection Method": cluster.GetCollectionMethod().String(), + "Collector Image": cluster.GetCollectorImage(), + "Managed By": cluster.GetManagedBy().String(), + "Priority": cluster.GetPriority(), + "Cluster Type": cluster.GetType().String(), + "Slim Collector": cluster.GetSlimCollector(), + "Auto Lock Process Baselines": clusterPkg.GetAutoLockProcessBaselinesEnabled(cluster), } } @@ -77,9 +83,58 @@ var Gather phonehome.GatherFunc = func(ctx context.Context) (map[string]any, err return props, nil } +type versionEntry struct { + Version string `json:"version"` + Count int32 `json:"count"` +} + +// buildVMTraits returns Segment trait key/value pairs for VM telemetry. +// Returns nil when the Sensor lacks VirtualMachineTelemetryCap (old Sensor, +// leave existing traits untouched). +func buildVMTraits(hasCapability bool, vmm *central.VirtualMachineMetrics) map[string]any { + if !hasCapability { + return nil + } + if vmm == nil { + return map[string]any{ + "VM Scanning Enabled": false, + "VM Tracked Count": int32(0), + "VM Scanned Count": int32(0), + "VM Unscanned Count": int32(0), + "Roxagent Version Counts": "[]", + } + } + + traits := map[string]any{ + "VM Scanning Enabled": true, + "VM Tracked Count": vmm.GetTrackedVms(), + "VM Scanned Count": vmm.GetVmsScanned(), + "VM Unscanned Count": max(int32(0), vmm.GetTrackedVms()-vmm.GetVmsScanned()), + "Roxagent Version Counts": "[]", + } + + if vc := vmm.GetRoxagentVersionCounts(); len(vc) > 0 { + entries := make([]versionEntry, 0, len(vc)) + for v, c := range vc { + entries = append(entries, versionEntry{Version: v, Count: c}) + } + slices.SortFunc(entries, func(a, b versionEntry) int { + return strings.Compare(a.Version, b.Version) + }) + raw, err := json.Marshal(entries) + if err != nil { + log.Warnf("Failed to marshal roxagent version counts: %v", err) + return traits + } + traits["Roxagent Version Counts"] = string(raw) + } + + return traits +} + // UpdateSecuredClusterIdentity is called by the clustermetrics pipeline on // the reception of the cluster metrics from a sensor. -func UpdateSecuredClusterIdentity(ctx context.Context, clusterID string, metrics *central.ClusterMetrics) { +func UpdateSecuredClusterIdentity(ctx context.Context, clusterID string, metrics *central.ClusterMetrics, hasVMTelemetryCap bool) { c := centralclient.Singleton() // This is a shortcut to avoid calling the cluster datastore in case // telemetry is for sure not enabled. @@ -125,6 +180,12 @@ func UpdateSecuredClusterIdentity(ctx context.Context, clusterID string, metrics } props["Orchestrator Version"] = omd.GetVersion() + props["Sensor Version Compatibility"] = cluster.GetStatus().GetSensorVersionCompatibility().String() + + if vmTraits := buildVMTraits(hasVMTelemetryCap, metrics.GetVirtualMachineMetrics()); vmTraits != nil { + maps.Copy(props, vmTraits) + } + c.Track("Updated Secured Cluster Identity", nil, append( c.WithGroups(), telemeter.WithClient(clusterID, securedClusterClient, cluster.GetMainImage()), diff --git a/central/cluster/datastore/telemetry_test.go b/central/cluster/datastore/telemetry_test.go new file mode 100644 index 0000000000000..9ff1275ad590b --- /dev/null +++ b/central/cluster/datastore/telemetry_test.go @@ -0,0 +1,86 @@ +package datastore + +import ( + "testing" + + "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stretchr/testify/assert" +) + +func TestBuildVMTraits(t *testing.T) { + tests := map[string]struct { + hasCapability bool + metrics *central.VirtualMachineMetrics + wantTraits map[string]any + }{ + "should set VM Scanning Enabled true with sorted version JSON when metrics are non-nil": { + hasCapability: true, + metrics: ¢ral.VirtualMachineMetrics{ + TrackedVms: 10, + VmsScanned: 7, + RoxagentVersionCounts: map[string]int32{ + "4.7.0-12-gabcdef0123": 4, + "4.8.0-3-gfedcba9876": 2, + "unknown": 1, + }, + }, + wantTraits: map[string]any{ + "VM Scanning Enabled": true, + "VM Tracked Count": int32(10), + "VM Scanned Count": int32(7), + "VM Unscanned Count": int32(3), + "Roxagent Version Counts": `[{"version":"4.7.0-12-gabcdef0123","count":4},{"version":"4.8.0-3-gfedcba9876","count":2},{"version":"unknown","count":1}]`, + }, + }, + "should zero traits including version counts when capable Sensor sends nil metrics (feature off)": { + hasCapability: true, + metrics: nil, + wantTraits: map[string]any{ + "VM Scanning Enabled": false, + "VM Tracked Count": int32(0), + "VM Scanned Count": int32(0), + "VM Unscanned Count": int32(0), + "Roxagent Version Counts": "[]", + }, + }, + "should set empty version counts for non-nil-but-all-zero metrics (feature on, zero VMs)": { + hasCapability: true, + metrics: ¢ral.VirtualMachineMetrics{}, + wantTraits: map[string]any{ + "VM Scanning Enabled": true, + "VM Tracked Count": int32(0), + "VM Scanned Count": int32(0), + "VM Unscanned Count": int32(0), + "Roxagent Version Counts": "[]", + }, + }, + "should omit unknown from version JSON when the map has no unknown bucket": { + hasCapability: true, + metrics: ¢ral.VirtualMachineMetrics{ + TrackedVms: 4, + VmsScanned: 4, + RoxagentVersionCounts: map[string]int32{ + "4.7.0-12-gabcdef0123": 4, + }, + }, + wantTraits: map[string]any{ + "VM Scanning Enabled": true, + "VM Tracked Count": int32(4), + "VM Scanned Count": int32(4), + "VM Unscanned Count": int32(0), + "Roxagent Version Counts": `[{"version":"4.7.0-12-gabcdef0123","count":4}]`, + }, + }, + "should return nil traits when Sensor lacks capability (old Sensor)": { + hasCapability: false, + metrics: nil, + wantTraits: nil, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.wantTraits, buildVMTraits(tc.hasCapability, tc.metrics)) + }) + } +} diff --git a/central/cluster/datastore/test_support.go b/central/cluster/datastore/test_support.go new file mode 100644 index 0000000000000..8ba049e9529fc --- /dev/null +++ b/central/cluster/datastore/test_support.go @@ -0,0 +1,21 @@ +//go:build sql_integration + +package datastore + +import ( + "testing" + + clusterInitStore "github.com/stackrox/rox/central/clusterinit/store" +) + +// For certain tests (central/sensor/service/service_impl_test.go) we need to interact +// with a cluster data store and the underlying cluster init store at the same time. +// For this purpose we need a way for these tests to extract the cluster init store +// from a cluster data store. +func IntrospectClusterInitStore(t *testing.T, storeInterface DataStore) clusterInitStore.Store { + store, ok := storeInterface.(*datastoreImpl) + if !ok { + t.Fatal("unexpected datastore type") + } + return store.clusterInitStore +} diff --git a/central/cluster/lifecycle/gate.go b/central/cluster/lifecycle/gate.go new file mode 100644 index 0000000000000..cb1513f543220 --- /dev/null +++ b/central/cluster/lifecycle/gate.go @@ -0,0 +1,93 @@ +package lifecycle + +import ( + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/sync" +) + +// Gate serializes cluster deletion with writes that must not outlive the cluster. +type Gate struct { + mu sync.Mutex + entries map[string]*entry +} + +type entry struct { + mu sync.RWMutex + stateMu sync.Mutex + deleting bool + refs int +} + +var singleton = New() + +// Singleton returns the process-wide cluster lifecycle gate. +func Singleton() *Gate { + return singleton +} + +// New creates a cluster lifecycle gate. +func New() *Gate { + return &Gate{entries: make(map[string]*entry)} +} + +// Enter acquires a shared lifecycle lease unless deletion has started. The +// returned release function must be called when active is true. +func (g *Gate) Enter(clusterID string) (active bool, release func()) { + e := g.retain(clusterID) + + active = concurrency.WithLock1(&e.stateMu, func() bool { + if e.deleting { + return false + } + e.mu.RLock() + return true + }) + if !active { + g.release(clusterID, e) + return false, nil + } + + return true, func() { + concurrency.UnsafeRUnlock(&e.mu) + g.release(clusterID, e) + } +} + +// BeginDeletion prevents new active writes and waits for in-flight writes to finish. +// The returned function must be called once cluster cleanup completes. +func (g *Gate) BeginDeletion(clusterID string) func() { + e := g.retain(clusterID) + concurrency.WithLock(&e.stateMu, func() { + e.deleting = true + }) + // Drain in-flight read locks, then release. + concurrency.WithLock(&e.mu, func() {}) + + return func() { + concurrency.WithLock(&e.stateMu, func() { + e.deleting = false + }) + g.release(clusterID, e) + } +} + +func (g *Gate) retain(clusterID string) *entry { + g.mu.Lock() + defer g.mu.Unlock() + e := g.entries[clusterID] + if e == nil { + e = &entry{} + g.entries[clusterID] = e + } + e.refs++ + return e +} + +func (g *Gate) release(clusterID string, e *entry) { + g.mu.Lock() + defer g.mu.Unlock() + e.refs-- + if e.refs == 0 && !e.deleting { + delete(g.entries, clusterID) + } +} diff --git a/central/cluster/lifecycle/gate_test.go b/central/cluster/lifecycle/gate_test.go new file mode 100644 index 0000000000000..5ad0da55a097f --- /dev/null +++ b/central/cluster/lifecycle/gate_test.go @@ -0,0 +1,72 @@ +package lifecycle + +import ( + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestGateBlocksDeletionUntilActiveWriteCompletes(t *testing.T) { + clusterID := "cluster-id" + + gate := New() + writeStarted := make(chan struct{}) + releaseWrite := make(chan struct{}) + writeFinished := make(chan struct{}) + go func() { + active, release := gate.Enter(clusterID) + if !active { + t.Error("active write was rejected") + return + } + defer release() + close(writeStarted) + <-releaseWrite + close(writeFinished) + }() + <-writeStarted + + deletionStarted := make(chan func(), 1) + go func() { deletionStarted <- gate.BeginDeletion(clusterID) }() + require.Eventually(t, func() bool { + active, release := gate.Enter(clusterID) + if release != nil { + release() + } + + return !active + }, time.Second, 5*time.Millisecond) + + // yield -> to give additional cycles to BeginDeletion. + runtime.Gosched() + require.Empty(t, deletionStarted) + + close(releaseWrite) + <-writeFinished + releaseDeletion := <-deletionStarted + releaseDeletion() + + active, release := gate.Enter(clusterID) + require.True(t, active) + release() +} + +// TestBeginDeletionDropsWriteLockBeforeCleanup checks that BeginDeletion only +// holds the write lock long enough to drain in-flight leases. The dev mutex +// watchdog aborts the process if that lock stays held through cleanup. +func TestBeginDeletionDropsWriteLockBeforeCleanup(t *testing.T) { + gate := New() + releaseDeletion := gate.BeginDeletion("cluster-id") + t.Cleanup(releaseDeletion) + + e := gate.entries["cluster-id"] + require.NotNil(t, e) + require.True(t, e.mu.TryLock(), "write lock still held after BeginDeletion returned") + e.mu.Unlock() + + active, release := gate.Enter("cluster-id") + require.False(t, active) + require.Nil(t, release) +} diff --git a/central/cluster/service/service_impl.go b/central/cluster/service/service_impl.go index 7cc70c935b006..e43bec0a2b472 100644 --- a/central/cluster/service/service_impl.go +++ b/central/cluster/service/service_impl.go @@ -20,9 +20,11 @@ import ( "github.com/stackrox/rox/pkg/grpc/authz/user" "github.com/stackrox/rox/pkg/images/defaults" "github.com/stackrox/rox/pkg/maputil" + "github.com/stackrox/rox/pkg/postgres/schema" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/predicate" "github.com/stackrox/rox/pkg/timeutil" "google.golang.org/grpc" ) @@ -41,6 +43,18 @@ var ( v1.ClustersService_DeleteCluster_FullMethodName, }, })) + + // skewOptionsMap contains only the SensorVersionCompatibility field. It is + // computed at runtime in the datastore layer and has no DB column + // (storage/cluster.proto sql:"-"), so it can't be pushed down to the DB. + // It is used to split an incoming query into a DB-safe part and a part + // that must be evaluated in memory against the already-populated field. + skewOptionsMap = search.NewOptionsMap(v1.SearchCategory_CLUSTERS).Add( + search.SensorVersionCompatibility, + schema.ClustersSchema.OptionsMap.MustGet(search.SensorVersionCompatibility.String()), + ) + + clusterPredicateFactory = predicate.NewFactory("cluster", (*storage.Cluster)(nil)) ) // ClusterService is the struct that manages the cluster API @@ -174,16 +188,48 @@ func (s *serviceImpl) getClusterRetentionInfo(ctx context.Context, cluster *stor // GetClusters returns the currently defined clusters. func (s *serviceImpl) GetClusters(ctx context.Context, req *v1.GetClustersRequest) (*v1.ClustersList, error) { - q, err := search.ParseQuery(req.GetQuery(), search.MatchAllIfEmpty()) + fullQuery, err := search.ParseQuery(req.GetQuery(), search.MatchAllIfEmpty()) if err != nil { return nil, errors.Wrapf(errox.InvalidArgs, "invalid query %q: %v", req.GetQuery(), err) } - clusters, err := s.datastore.SearchRawClusters(ctx, q) + // Split the query: dbQuery contains everything the DB can filter on; + // skewQuery contains only the SensorVersionCompatibility field, which + // is computed at runtime and has no DB column. + dbQuery, _ := search.InverseFilterQueryWithMap(fullQuery, skewOptionsMap) + skewQuery, _ := search.FilterQueryWithMap(fullQuery, skewOptionsMap) + + clusters, err := s.datastore.SearchRawClusters(ctx, dbQuery) if err != nil { return nil, err } + if skewQuery != nil { + pred, err := clusterPredicateFactory.GeneratePredicate(skewQuery) + if err != nil { + return nil, errors.Wrapf(errox.InvalidArgs, "building predicate for version compatibility filter %q: %v", req.GetQuery(), err) + } + var filtered []*storage.Cluster + for _, cluster := range clusters { + // TODO(ROX-36353): Try to move this to the db layer instead here + // Clusters where sensor has never connected have nil Status. + // Initialize it so the predicate can match UNKNOWN. + if cluster.GetStatus() == nil { + cluster.Status = &storage.ClusterStatus{ + SensorVersionCompatibility: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_UNKNOWN, + } + } + if pred.Matches(cluster) { + filtered = append(filtered, cluster) + } + } + clusters = filtered + } + + // If we want to add pagination in the future it will have to be done + // in memory at this point, after both DB-side and in-memory filters + // have been applied. + clusterIDToRetentionInfoMap, err := s.getClusterIDToRetentionInfoMap(ctx, clusters) if err != nil { return nil, err diff --git a/central/cluster/service/service_impl_postgres_test.go b/central/cluster/service/service_impl_postgres_test.go new file mode 100644 index 0000000000000..793f05230b2f3 --- /dev/null +++ b/central/cluster/service/service_impl_postgres_test.go @@ -0,0 +1,167 @@ +//go:build sql_integration + +// Exercises the real service.serviceImpl.GetClusters implementation (query +// splitting, in-memory skew filtering) against a real Postgres-backed cluster +// datastore. +// +// Requires a local Postgres reachable on port 5432, e.g.: +// +// docker run --rm --env POSTGRES_USER="$USER" \ +// --env POSTGRES_HOST_AUTH_METHOD=trust --publish 5432:5432 \ +// docker.io/library/postgres:15 +// +// Run with: +// +// go test -tags sql_integration -v -run TestGetClustersSkewFilteringPostgres \ +// ./central/cluster/service/... +package service + +import ( + "context" + "fmt" + "testing" + + "github.com/stackrox/rox/central/cluster/datastore" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/version/testutils" + "github.com/stretchr/testify/require" +) + +func assertClustersAre(t *testing.T, clusters []*storage.Cluster, expectedNames ...string) { + t.Helper() + names := make([]string, len(clusters)) + for i, c := range clusters { + names[i] = c.GetName() + } + require.ElementsMatch(t, expectedNames, names) +} + +func TestGetClustersSkewFilteringPostgres(t *testing.T) { + testutils.SetMainVersion(t, "4.5.0-testing") + + db := pgtest.ForT(t) + ds, err := datastore.GetTestPostgresDataStore(t, db) + require.NoError(t, err) + + ctx := sac.WithAllAccess(context.Background()) + + type fixture struct { + name string + sensorVersion string + labels map[string]string + } + fixtures := []fixture{ + {name: "a-matched", sensorVersion: "4.5.9", labels: map[string]string{"name": "a-matched", "env": "prod"}}, + {name: "b-matched", sensorVersion: "4.5.0", labels: map[string]string{"name": "b-matched", "env": "staging"}}, + {name: "c-behind", sensorVersion: "4.4.0", labels: map[string]string{"name": "c-behind", "env": "prod"}}, + {name: "d-ahead", sensorVersion: "4.6.0", labels: map[string]string{"name": "d-ahead", "env": "prod"}}, + {name: "e-incompatible-behind", sensorVersion: "1.0.0", labels: map[string]string{"name": "e-incompatible-behind", "env": "prod"}}, + {name: "f-incompatible-ahead", sensorVersion: "5.5.0", labels: map[string]string{"name": "f-incompatible-ahead", "env": "prod"}}, + {name: "g-unknown", sensorVersion: "", labels: map[string]string{"name": "g-unknown", "env": "prod"}}, + } + + for _, f := range fixtures { + // MainImage uses a fixed tag because the sensor version is set + // separately via UpdateClusterStatus, which is what the datastore + // uses for compatibility classification. + cluster := &storage.Cluster{ + Name: f.name, + MainImage: "docker.io/stackrox/main:latest", + CentralApiEndpoint: "central.stackrox:443", + Labels: f.labels, + } + id, err := ds.AddCluster(ctx, cluster) + require.NoError(t, err) + if f.sensorVersion != "" { + require.NoError(t, ds.UpdateClusterStatus(ctx, id, &storage.ClusterStatus{ + SensorVersion: f.sensorVersion, + })) + } + } + + svc := New(ds, nil, nil, nil) + + t.Run("filter by compatible behind", func(t *testing.T) { + query := fmt.Sprintf("%s:%s", search.SensorVersionCompatibility, + storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_COMPATIBLE_BEHIND.String()) + + resp, err := svc.GetClusters(ctx, &v1.GetClustersRequest{Query: query}) + require.NoError(t, err) + assertClustersAre(t, resp.GetClusters(), "c-behind") + }) + + t.Run("filter by compatible ahead", func(t *testing.T) { + query := fmt.Sprintf("%s:%s", search.SensorVersionCompatibility, + storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_COMPATIBLE_AHEAD.String()) + + resp, err := svc.GetClusters(ctx, &v1.GetClustersRequest{Query: query}) + require.NoError(t, err) + assertClustersAre(t, resp.GetClusters(), "d-ahead") + }) + + t.Run("filter by matched returns multiple", func(t *testing.T) { + query := fmt.Sprintf("%s:%s", search.SensorVersionCompatibility, + storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_MATCHED.String()) + + resp, err := svc.GetClusters(ctx, &v1.GetClustersRequest{Query: query}) + require.NoError(t, err) + assertClustersAre(t, resp.GetClusters(), "a-matched", "b-matched") + }) + + t.Run("filter by incompatible behind", func(t *testing.T) { + query := fmt.Sprintf("%s:%s", search.SensorVersionCompatibility, + storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_INCOMPATIBLE_BEHIND.String()) + + resp, err := svc.GetClusters(ctx, &v1.GetClustersRequest{Query: query}) + require.NoError(t, err) + assertClustersAre(t, resp.GetClusters(), "e-incompatible-behind") + }) + + t.Run("filter by incompatible ahead", func(t *testing.T) { + query := fmt.Sprintf("%s:%s", search.SensorVersionCompatibility, + storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_INCOMPATIBLE_AHEAD.String()) + + resp, err := svc.GetClusters(ctx, &v1.GetClustersRequest{Query: query}) + require.NoError(t, err) + assertClustersAre(t, resp.GetClusters(), "f-incompatible-ahead") + }) + + t.Run("filter by unknown", func(t *testing.T) { + query := fmt.Sprintf("%s:%s", search.SensorVersionCompatibility, + storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_UNKNOWN.String()) + + resp, err := svc.GetClusters(ctx, &v1.GetClustersRequest{Query: query}) + require.NoError(t, err) + assertClustersAre(t, resp.GetClusters(), "g-unknown") + }) + + t.Run("db filter narrows results before skew filter", func(t *testing.T) { + query := fmt.Sprintf("%s:%s+%s:%s", search.SensorVersionCompatibility, + storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_MATCHED.String(), + search.ClusterLabel, "env=prod") + + resp, err := svc.GetClusters(ctx, &v1.GetClustersRequest{Query: query}) + require.NoError(t, err) + assertClustersAre(t, resp.GetClusters(), "a-matched") + }) + + t.Run("no filter returns all clusters", func(t *testing.T) { + resp, err := svc.GetClusters(ctx, &v1.GetClustersRequest{ + Query: search.EmptyQuery().String(), + }) + require.NoError(t, err) + require.Len(t, resp.GetClusters(), 7) + }) + + t.Run("db filter only without skew filter", func(t *testing.T) { + query := fmt.Sprintf("%s:%s", search.ClusterLabel, "env=staging") + + resp, err := svc.GetClusters(ctx, &v1.GetClustersRequest{Query: query}) + require.NoError(t, err) + assertClustersAre(t, resp.GetClusters(), "b-matched") + }) +} diff --git a/central/cluster/service/service_impl_test.go b/central/cluster/service/service_impl_test.go index 3a58093e89645..69ca32343887b 100644 --- a/central/cluster/service/service_impl_test.go +++ b/central/cluster/service/service_impl_test.go @@ -223,6 +223,134 @@ func (suite *ClusterServiceTestSuite) TestGetClustersWithRetentionInfoMap() { } } +func (suite *ClusterServiceTestSuite) TestGetClustersSkewFiltering() { + allClusters := []*storage.Cluster{ + { + Id: "matched-1", + Status: &storage.ClusterStatus{ + SensorVersionCompatibility: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_MATCHED, + }, + }, + { + Id: "matched-2", + Status: &storage.ClusterStatus{ + SensorVersionCompatibility: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_MATCHED, + }, + }, + { + Id: "compatible-behind", + Status: &storage.ClusterStatus{ + SensorVersionCompatibility: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_COMPATIBLE_BEHIND, + }, + }, + { + Id: "compatible-ahead", + Status: &storage.ClusterStatus{ + SensorVersionCompatibility: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_COMPATIBLE_AHEAD, + }, + }, + { + Id: "incompatible-behind", + Status: &storage.ClusterStatus{ + SensorVersionCompatibility: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_INCOMPATIBLE_BEHIND, + }, + }, + { + Id: "incompatible-ahead", + Status: &storage.ClusterStatus{ + SensorVersionCompatibility: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_INCOMPATIBLE_AHEAD, + }, + }, + } + + tests := map[string]struct { + filterValue string + expectedIDs []string + }{ + "filter by matched returns multiple": { + filterValue: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_MATCHED.String(), + expectedIDs: []string{"matched-1", "matched-2"}, + }, + "filter by compatible behind": { + filterValue: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_COMPATIBLE_BEHIND.String(), + expectedIDs: []string{"compatible-behind"}, + }, + "filter by compatible ahead": { + filterValue: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_COMPATIBLE_AHEAD.String(), + expectedIDs: []string{"compatible-ahead"}, + }, + "filter by incompatible behind": { + filterValue: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_INCOMPATIBLE_BEHIND.String(), + expectedIDs: []string{"incompatible-behind"}, + }, + "filter by incompatible ahead": { + filterValue: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_INCOMPATIBLE_AHEAD.String(), + expectedIDs: []string{"incompatible-ahead"}, + }, + "filter by unknown returns empty": { + filterValue: storage.SensorVersionCompatibility_SENSOR_VERSION_COMPATIBILITY_UNKNOWN.String(), + expectedIDs: nil, + }, + } + + for name, tt := range tests { + suite.Run(name, func() { + suite.dataStore.EXPECT().SearchRawClusters(gomock.Any(), gomock.Any()).Times(1).Return(allClusters, nil) + + clusterService := New(suite.dataStore, nil, nil, nil) + + query := search.NewQueryBuilder().AddStrings( + search.SensorVersionCompatibility, tt.filterValue, + ).Query() + + results, err := clusterService.GetClusters(context.Background(), &v1.GetClustersRequest{Query: query}) + suite.NoError(err) + suite.Require().Len(results.GetClusters(), len(tt.expectedIDs)) + + gotIDs := make([]string, len(results.GetClusters())) + for i, c := range results.GetClusters() { + gotIDs[i] = c.GetId() + } + suite.ElementsMatch(tt.expectedIDs, gotIDs) + }) + } +} + +func (suite *ClusterServiceTestSuite) TestGetClustersInvalidSkewFilterValue() { + clusters := []*storage.Cluster{ + {Id: "cluster-1"}, + } + + suite.dataStore.EXPECT().SearchRawClusters(gomock.Any(), gomock.Any()).Times(1).Return(clusters, nil) + + clusterService := New(suite.dataStore, nil, nil, nil) + + query := search.NewQueryBuilder().AddStrings( + search.SensorVersionCompatibility, "not-a-valid-compatibility", + ).Query() + + results, err := clusterService.GetClusters(context.Background(), &v1.GetClustersRequest{Query: query}) + suite.NoError(err) + suite.Empty(results.GetClusters()) +} + +func (suite *ClusterServiceTestSuite) TestGetClustersNoSkewFilterReturnsAll() { + clusters := []*storage.Cluster{ + {Id: "cluster-1"}, + {Id: "cluster-2"}, + } + + suite.dataStore.EXPECT().SearchRawClusters(gomock.Any(), gomock.Any()).Times(1).Return(clusters, nil) + + clusterService := New(suite.dataStore, nil, nil, nil) + + results, err := clusterService.GetClusters(context.Background(), &v1.GetClustersRequest{ + Query: search.EmptyQuery().String(), + }) + suite.NoError(err) + suite.Len(results.GetClusters(), 2) +} + func daysAgo(days int) time.Time { return time.Now().Add(-time.Duration(days) * 24 * time.Hour) } diff --git a/central/cluster/store/cluster/store.go b/central/cluster/store/cluster/store.go index 4d5ce8c59647e..a65704bbe16d6 100644 --- a/central/cluster/store/cluster/store.go +++ b/central/cluster/store/cluster/store.go @@ -22,5 +22,6 @@ type Store interface { Walk(ctx context.Context, fn func(obj *storage.Cluster) error) error WalkByQuery(ctx context.Context, query *v1.Query, fn func(obj *storage.Cluster) error) error + // Deprecated: It will not clone the object so it should be used only for SAC. GetAllFromCacheForSAC() []*storage.Cluster } diff --git a/central/clustercveedge/datastore/datastore_impl.go b/central/clustercveedge/datastore/datastore_impl.go index faf6472654852..f446db0dc19bc 100644 --- a/central/clustercveedge/datastore/datastore_impl.go +++ b/central/clustercveedge/datastore/datastore_impl.go @@ -90,7 +90,3 @@ func (c *ClusterCVEEdgeSearchResultConverter) BuildLocation(result *searchPkg.Re func (c *ClusterCVEEdgeSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_CLUSTER_VULN_EDGE } - -func (c *ClusterCVEEdgeSearchResultConverter) GetScore(result *searchPkg.Result) float64 { - return result.Score -} diff --git a/central/clusterinit/backend/backend.go b/central/clusterinit/backend/backend.go index 9cc43d2ff095d..54d4771b16673 100644 --- a/central/clusterinit/backend/backend.go +++ b/central/clusterinit/backend/backend.go @@ -46,7 +46,7 @@ type Backend interface { GetAllCRS(ctx context.Context) ([]*storage.InitBundleMeta, error) GetCAConfig(ctx context.Context) (*CAConfig, error) Issue(ctx context.Context, name string) (*InitBundleWithMeta, error) - IssueCRS(ctx context.Context, name string, validUntil time.Time) (*CRSWithMeta, error) + IssueCRS(ctx context.Context, name string, validUntil time.Time, maxRegistrations uint64) (*CRSWithMeta, error) Revoke(ctx context.Context, id string) error CheckRevoked(ctx context.Context, id string) error authn.ValidateCertChain diff --git a/central/clusterinit/backend/backend_impl.go b/central/clusterinit/backend/backend_impl.go index 133785401dca7..fffd79fad9e6b 100644 --- a/central/clusterinit/backend/backend_impl.go +++ b/central/clusterinit/backend/backend_impl.go @@ -12,6 +12,7 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/crs" + "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/grpc/authn" "github.com/stackrox/rox/pkg/mtls" "github.com/stackrox/rox/pkg/protocompat" @@ -21,6 +22,10 @@ import ( const ( currentCrsVersion = 1 + + // We enforce an upper bound for this setting because for this feature it is required to maintain a list of cluster names + // per CRS in the storage and therefore we need to keep the storage requirements under control. + maxRegistrationsUpperLimit = 100 ) var _ authn.ValidateCertChain = (*backendImpl)(nil) @@ -156,7 +161,7 @@ func (b *backendImpl) Issue(ctx context.Context, name string) (*InitBundleWithMe }, nil } -func (b *backendImpl) IssueCRS(ctx context.Context, name string, validUntil time.Time) (*CRSWithMeta, error) { +func (b *backendImpl) IssueCRS(ctx context.Context, name string, validUntil time.Time, maxRegistrations uint64) (*CRSWithMeta, error) { if err := access.CheckAccess(ctx, storage.Access_READ_WRITE_ACCESS); err != nil { return nil, err } @@ -166,6 +171,10 @@ func (b *backendImpl) IssueCRS(ctx context.Context, name string, validUntil time return nil, err } + if maxRegistrations > maxRegistrationsUpperLimit { + return nil, errox.InvalidArgs.Newf("cluster registration limit must be in the range 0...%d", maxRegistrationsUpperLimit) + } + caCert, err := b.certProvider.GetCA() if err != nil { return nil, errors.Wrap(err, "retrieving CA certificate") @@ -189,12 +198,13 @@ func (b *backendImpl) IssueCRS(ctx context.Context, name string, validUntil time // On the storage side we are reusing the InitBundleMeta. meta := &storage.InitBundleMeta{ - Id: id.String(), - Name: name, - CreatedAt: protocompat.TimestampNow(), - CreatedBy: user, - ExpiresAt: expiryTimestamp, - Version: storage.InitBundleMeta_CRS, + Id: id.String(), + Name: name, + CreatedAt: protocompat.TimestampNow(), + CreatedBy: user, + ExpiresAt: expiryTimestamp, + Version: storage.InitBundleMeta_CRS, + MaxRegistrations: maxRegistrations, } if err := b.store.Add(storeCtx, meta); err != nil { diff --git a/central/clusterinit/backend/backend_test.go b/central/clusterinit/backend/backend_test.go index 2e9aa94a5abba..fe3884fc931bd 100644 --- a/central/clusterinit/backend/backend_test.go +++ b/central/clusterinit/backend/backend_test.go @@ -227,11 +227,11 @@ func (s *clusterInitBackendTestSuite) TestCRSNameMustBeUnique() { crsName := "test1" // Issue new CRS. - _, err := s.backend.IssueCRS(ctx, crsName, time.Time{}) + _, err := s.backend.IssueCRS(ctx, crsName, time.Time{}, 0) s.Require().NoError(err) // Attempt to issue again with same name. - _, err = s.backend.IssueCRS(ctx, crsName, time.Time{}) + _, err = s.backend.IssueCRS(ctx, crsName, time.Time{}, 0) s.Require().Error(err) s.Require().ErrorIs(err, store.ErrInitBundleDuplicateName) } @@ -239,7 +239,7 @@ func (s *clusterInitBackendTestSuite) TestCRSNameMustBeUnique() { func (s *clusterInitBackendTestSuite) TestCRSDefaultExpiration() { expectedNotAfter := time.Now().UTC().Add(24 * time.Hour) - crsWithMeta, err := s.backend.IssueCRS(s.ctx, "crs-default-expiration", time.Time{}.UTC()) + crsWithMeta, err := s.backend.IssueCRS(s.ctx, "crs-default-expiration", time.Time{}.UTC(), 0) s.Require().NoError(err) certPEM := []byte(crsWithMeta.CRS.Cert) @@ -259,7 +259,7 @@ func (s *clusterInitBackendTestSuite) TestCRSExpirationValidUntil() { validUntil, err := time.Parse(time.RFC3339, "2106-01-02T15:04:05Z") s.Require().NoError(err) - crsWithMeta, err := s.backend.IssueCRS(ctx, crsName, validUntil) + crsWithMeta, err := s.backend.IssueCRS(ctx, crsName, validUntil, 0) s.Require().NoError(err) certPEM := []byte(crsWithMeta.CRS.Cert) @@ -275,7 +275,7 @@ func (s *clusterInitBackendTestSuite) TestCRSLifecycle() { crsName := "test1" // Issue new CRS. - crsWithMeta, err := s.backend.IssueCRS(ctx, crsName, time.Time{}) + crsWithMeta, err := s.backend.IssueCRS(ctx, crsName, time.Time{}, 0) s.Require().NoError(err) id := crsWithMeta.Meta.GetId() @@ -354,14 +354,18 @@ func (s *clusterInitBackendTestSuite) TestCRSLifecycle() { } } +func (s *clusterInitBackendTestSuite) TestCrsIssuingWithTooLargeRegistrationLimit() { + crsName := "test-crs-exceeding-limit" + _, err := s.backend.IssueCRS(s.ctx, crsName, time.Time{}, 101) + s.Require().Error(err, "issuing CRS with maxRegistrations=101 succeeded.") +} + // Tests if attempt to issue two init bundles with the same name fails as expected. func (s *clusterInitBackendTestSuite) TestIssuingWithDuplicateName() { - ctx := s.ctx - - _, err := s.backend.Issue(ctx, "test2") + _, err := s.backend.Issue(s.ctx, "test2") s.Require().NoError(err) - _, err = s.backend.Issue(ctx, "test2") + _, err = s.backend.Issue(s.ctx, "test2") s.Require().Error(err, "issuing two init bundles with the same name") } @@ -382,7 +386,7 @@ func (s *clusterInitBackendTestSuite) TestValidateClientCertificateNotFound() { err := s.backend.ValidateClientCertificate(ctx, certs) s.Require().Error(err) - s.Equal(fmt.Sprintf("failed checking init bundle status %[1]q: retrieving init bundle %[1]q: init bundle not found", id), err.Error()) + s.Equal(fmt.Sprintf("failed checking init bundle status %[1]q: retrieving init bundle %[1]q: init bundle or CRS not found", id), err.Error()) } func (s *clusterInitBackendTestSuite) TestValidateClientCertificateEphemeralInitBundle() { diff --git a/central/clusterinit/backend/mocks/backend.go b/central/clusterinit/backend/mocks/backend.go index dbcb55ba72597..7ce7d3eabded1 100644 --- a/central/clusterinit/backend/mocks/backend.go +++ b/central/clusterinit/backend/mocks/backend.go @@ -119,18 +119,18 @@ func (mr *MockBackendMockRecorder) Issue(ctx, name any) *gomock.Call { } // IssueCRS mocks base method. -func (m *MockBackend) IssueCRS(ctx context.Context, name string, validUntil time.Time) (*backend.CRSWithMeta, error) { +func (m *MockBackend) IssueCRS(ctx context.Context, name string, validUntil time.Time, maxRegistrations uint64) (*backend.CRSWithMeta, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IssueCRS", ctx, name, validUntil) + ret := m.ctrl.Call(m, "IssueCRS", ctx, name, validUntil, maxRegistrations) ret0, _ := ret[0].(*backend.CRSWithMeta) ret1, _ := ret[1].(error) return ret0, ret1 } // IssueCRS indicates an expected call of IssueCRS. -func (mr *MockBackendMockRecorder) IssueCRS(ctx, name, validUntil any) *gomock.Call { +func (mr *MockBackendMockRecorder) IssueCRS(ctx, name, validUntil, maxRegistrations any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IssueCRS", reflect.TypeOf((*MockBackend)(nil).IssueCRS), ctx, name, validUntil) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IssueCRS", reflect.TypeOf((*MockBackend)(nil).IssueCRS), ctx, name, validUntil, maxRegistrations) } // Revoke mocks base method. diff --git a/central/clusterinit/backend/render.go b/central/clusterinit/backend/render.go index ea31101f22c07..962e99593d950 100644 --- a/central/clusterinit/backend/render.go +++ b/central/clusterinit/backend/render.go @@ -40,6 +40,7 @@ const ( # It is used for setting up StackRox secured clusters. # NOTE: This file contains secret data that allows connecting new secured clusters to central, # and needs to be handled and stored accordingly. +# ` ) @@ -174,6 +175,16 @@ func (b *CRSWithMeta) RenderAsK8sSecret() ([]byte, error) { var buf bytes.Buffer _, _ = fmt.Fprint(&buf, crsHeader) + var what string + switch b.Meta.GetMaxRegistrations() { + case 0: + what = "any number of clusters" + case 1: + what = "only one cluster" + default: + what = fmt.Sprintf("at most %v clusters", b.Meta.GetMaxRegistrations()) + } + _, _ = fmt.Fprintf(&buf, "# This Cluster Registration Secret can be used for registering %s.\n", what) crs, err := crs.SerializeSecret(b.CRS) if err != nil { diff --git a/central/clusterinit/service/service_impl.go b/central/clusterinit/service/service_impl.go index df69adf42c4a1..a38491fc3e67e 100644 --- a/central/clusterinit/service/service_impl.go +++ b/central/clusterinit/service/service_impl.go @@ -137,7 +137,7 @@ func (s *serviceImpl) GenerateCRS(ctx context.Context, request *v1.CRSGenRequest return nil, status.Error(codes.Unimplemented, "support for generating Cluster Registration Secrets (CRS) is not enabled") } - generated, err := s.backend.IssueCRS(ctx, request.GetName(), time.Time{}) + generated, err := s.backend.IssueCRS(ctx, request.GetName(), time.Time{}, 0) if err != nil { if errors.Is(err, store.ErrInitBundleDuplicateName) { return nil, status.Errorf(codes.AlreadyExists, "generating new CRS: %s", err) @@ -162,10 +162,6 @@ func (s *serviceImpl) GenerateCRSExtended(ctx context.Context, request *v1.CRSGe return nil, status.Error(codes.Unimplemented, "support for generating Cluster Registration Secrets (CRS) is not enabled") } - if request.GetMaxRegistrations() != 0 { - return nil, errox.NotImplemented.CausedBy("max-registration limits not supported") - } - reqValidUntil := request.GetValidUntil() reqValidFor := request.GetValidFor() if reqValidUntil != nil && reqValidFor != nil { @@ -177,7 +173,9 @@ func (s *serviceImpl) GenerateCRSExtended(ctx context.Context, request *v1.CRSGe validUntil = protocompat.NilOrNow(reqValidUntil).Add(reqValidFor.AsDuration()) } - generated, err := s.backend.IssueCRS(ctx, request.GetName(), validUntil) + reqMaxRegistrations := request.GetMaxRegistrations() + + generated, err := s.backend.IssueCRS(ctx, request.GetName(), validUntil, reqMaxRegistrations) if err != nil { if errors.Is(err, store.ErrInitBundleDuplicateName) { return nil, status.Errorf(codes.AlreadyExists, "generating new CRS: %s", err) diff --git a/central/clusterinit/service/service_impl_test.go b/central/clusterinit/service/service_impl_test.go index ba88f77356366..aa3fff09458b0 100644 --- a/central/clusterinit/service/service_impl_test.go +++ b/central/clusterinit/service/service_impl_test.go @@ -59,8 +59,8 @@ func TestGenerateCRS(t *testing.T) { mockBackend := mocks.NewMockBackend(mockCtrl) service := New(mockBackend, mockStore) - mockBackend.EXPECT().IssueCRS(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).DoAndReturn( - func(_ context.Context, _ string, validUntil time.Time) (*backend.CRSWithMeta, error) { + mockBackend.EXPECT().IssueCRS(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(1).DoAndReturn( + func(_ context.Context, _ string, validUntil time.Time, maxRegistrations uint64) (*backend.CRSWithMeta, error) { assert.True(t, validUntil.IsZero()) crsWithMeta := &backend.CRSWithMeta{ CRS: &crs.CRS{}, @@ -84,8 +84,8 @@ func TestGenerateCRSWithoutValidity(t *testing.T) { mockBackend := mocks.NewMockBackend(mockCtrl) service := New(mockBackend, mockStore) - mockBackend.EXPECT().IssueCRS(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).DoAndReturn( - func(_ context.Context, _ string, validUntil time.Time) (*backend.CRSWithMeta, error) { + mockBackend.EXPECT().IssueCRS(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(1).DoAndReturn( + func(_ context.Context, _ string, validUntil time.Time, maxRegistrations uint64) (*backend.CRSWithMeta, error) { assert.True(t, validUntil.IsZero()) crsWithMeta := &backend.CRSWithMeta{ CRS: &crs.CRS{}, @@ -111,10 +111,10 @@ func TestGenerateCRSWithValidUntil(t *testing.T) { reqValidUntil, err := time.Parse(time.RFC3339, "2100-01-02T13:04:05Z") assert.NoError(t, err, "parsing RFC3339 timestamp failed") - mockBackend.EXPECT().IssueCRS(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).DoAndReturn( + mockBackend.EXPECT().IssueCRS(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(1).DoAndReturn( // Verify that the validUntil timestamp passed to the backend matches what is specified // in the service request. - func(_ context.Context, _ string, validUntil time.Time) (*backend.CRSWithMeta, error) { + func(_ context.Context, _ string, validUntil time.Time, maxRegistrations uint64) (*backend.CRSWithMeta, error) { assert.True(t, validUntil.Equal(reqValidUntil)) crsWithMeta := &backend.CRSWithMeta{ CRS: &crs.CRS{}, @@ -142,8 +142,8 @@ func TestGenerateCRSWithValidFor(t *testing.T) { expectedValidUntil := time.Now().Add(reqValidFor) epsilon := 10 * time.Second - mockBackend.EXPECT().IssueCRS(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).DoAndReturn( - func(_ context.Context, _ string, validUntil time.Time) (*backend.CRSWithMeta, error) { + mockBackend.EXPECT().IssueCRS(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(1).DoAndReturn( + func(_ context.Context, _ string, validUntil time.Time, maxRegistrations uint64) (*backend.CRSWithMeta, error) { // Verify that the validUntil passed to the backend matches now() + validFor. timeDelta := validUntil.Sub(expectedValidUntil) assert.Less(t, timeDelta, epsilon, "CRS valid for longer than expected") @@ -173,8 +173,6 @@ func TestGenerateCRSWithValidForAndValidUntil(t *testing.T) { assert.NoError(t, err, "parsing RFC3339 timestamp failed") reqValidFor := 10 * time.Minute - mockBackend.EXPECT().IssueCRS(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) - request := &v1.CRSGenRequestExtended{ Name: "secured-cluster", ValidUntil: timestamppb.New(reqValidUntil), diff --git a/central/clusterinit/service/transformation.go b/central/clusterinit/service/transformation.go index aad16592995a4..7beaf0c683b40 100644 --- a/central/clusterinit/service/transformation.go +++ b/central/clusterinit/service/transformation.go @@ -24,10 +24,13 @@ func initBundleMetaStorageToV1WithImpactedClusters(meta *storage.InitBundleMeta, func crsMetaStorageToV1(meta *storage.InitBundleMeta) *v1.CRSMeta { return &v1.CRSMeta{ - Id: meta.GetId(), - Name: meta.GetName(), - CreatedAt: meta.GetCreatedAt(), - CreatedBy: meta.GetCreatedBy(), - ExpiresAt: meta.GetExpiresAt(), + Id: meta.GetId(), + Name: meta.GetName(), + CreatedAt: meta.GetCreatedAt(), + CreatedBy: meta.GetCreatedBy(), + ExpiresAt: meta.GetExpiresAt(), + MaxRegistrations: meta.GetMaxRegistrations(), + RegistrationsInitiated: meta.GetRegistrationsInitiated(), + RegistrationsCompleted: meta.GetRegistrationsCompleted(), } } diff --git a/central/clusterinit/store/mocks/store.go b/central/clusterinit/store/mocks/store.go new file mode 100644 index 0000000000000..96cfbd674ebd2 --- /dev/null +++ b/central/clusterinit/store/mocks/store.go @@ -0,0 +1,268 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: store.go +// +// Generated by this command: +// +// mockgen -package mocks -destination mocks/store.go -source store.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + storage "github.com/stackrox/rox/generated/storage" + gomock "go.uber.org/mock/gomock" +) + +// MockStore is a mock of Store interface. +type MockStore struct { + ctrl *gomock.Controller + recorder *MockStoreMockRecorder + isgomock struct{} +} + +// MockStoreMockRecorder is the mock recorder for MockStore. +type MockStoreMockRecorder struct { + mock *MockStore +} + +// NewMockStore creates a new mock instance. +func NewMockStore(ctrl *gomock.Controller) *MockStore { + mock := &MockStore{ctrl: ctrl} + mock.recorder = &MockStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStore) EXPECT() *MockStoreMockRecorder { + return m.recorder +} + +// Add mocks base method. +func (m *MockStore) Add(ctx context.Context, bundleMeta *storage.InitBundleMeta) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Add", ctx, bundleMeta) + ret0, _ := ret[0].(error) + return ret0 +} + +// Add indicates an expected call of Add. +func (mr *MockStoreMockRecorder) Add(ctx, bundleMeta any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockStore)(nil).Add), ctx, bundleMeta) +} + +// Delete mocks base method. +func (m *MockStore) Delete(ctx context.Context, id string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, id) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockStoreMockRecorder) Delete(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockStore)(nil).Delete), ctx, id) +} + +// Get mocks base method. +func (m *MockStore) Get(ctx context.Context, id string) (*storage.InitBundleMeta, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, id) + ret0, _ := ret[0].(*storage.InitBundleMeta) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockStoreMockRecorder) Get(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockStore)(nil).Get), ctx, id) +} + +// GetAll mocks base method. +func (m *MockStore) GetAll(ctx context.Context) ([]*storage.InitBundleMeta, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAll", ctx) + ret0, _ := ret[0].([]*storage.InitBundleMeta) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAll indicates an expected call of GetAll. +func (mr *MockStoreMockRecorder) GetAll(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockStore)(nil).GetAll), ctx) +} + +// GetAllCRS mocks base method. +func (m *MockStore) GetAllCRS(ctx context.Context) ([]*storage.InitBundleMeta, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllCRS", ctx) + ret0, _ := ret[0].([]*storage.InitBundleMeta) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllCRS indicates an expected call of GetAllCRS. +func (mr *MockStoreMockRecorder) GetAllCRS(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllCRS", reflect.TypeOf((*MockStore)(nil).GetAllCRS), ctx) +} + +// InitiateClusterRegistration mocks base method. +func (m *MockStore) InitiateClusterRegistration(ctx context.Context, initArtifactId, clusterName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitiateClusterRegistration", ctx, initArtifactId, clusterName) + ret0, _ := ret[0].(error) + return ret0 +} + +// InitiateClusterRegistration indicates an expected call of InitiateClusterRegistration. +func (mr *MockStoreMockRecorder) InitiateClusterRegistration(ctx, initArtifactId, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitiateClusterRegistration", reflect.TypeOf((*MockStore)(nil).InitiateClusterRegistration), ctx, initArtifactId, clusterName) +} + +// MarkClusterRegistrationComplete mocks base method. +func (m *MockStore) MarkClusterRegistrationComplete(ctx context.Context, initArtifactId, clusterName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkClusterRegistrationComplete", ctx, initArtifactId, clusterName) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkClusterRegistrationComplete indicates an expected call of MarkClusterRegistrationComplete. +func (mr *MockStoreMockRecorder) MarkClusterRegistrationComplete(ctx, initArtifactId, clusterName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkClusterRegistrationComplete", reflect.TypeOf((*MockStore)(nil).MarkClusterRegistrationComplete), ctx, initArtifactId, clusterName) +} + +// Revoke mocks base method. +func (m *MockStore) Revoke(ctx context.Context, id string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Revoke", ctx, id) + ret0, _ := ret[0].(error) + return ret0 +} + +// Revoke indicates an expected call of Revoke. +func (mr *MockStoreMockRecorder) Revoke(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Revoke", reflect.TypeOf((*MockStore)(nil).Revoke), ctx, id) +} + +// Upsert mocks base method. +func (m *MockStore) Upsert(ctx context.Context, crs *storage.InitBundleMeta) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Upsert", ctx, crs) + ret0, _ := ret[0].(error) + return ret0 +} + +// Upsert indicates an expected call of Upsert. +func (mr *MockStoreMockRecorder) Upsert(ctx, crs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upsert", reflect.TypeOf((*MockStore)(nil).Upsert), ctx, crs) +} + +// MockUnderlyingStore is a mock of UnderlyingStore interface. +type MockUnderlyingStore struct { + ctrl *gomock.Controller + recorder *MockUnderlyingStoreMockRecorder + isgomock struct{} +} + +// MockUnderlyingStoreMockRecorder is the mock recorder for MockUnderlyingStore. +type MockUnderlyingStoreMockRecorder struct { + mock *MockUnderlyingStore +} + +// NewMockUnderlyingStore creates a new mock instance. +func NewMockUnderlyingStore(ctrl *gomock.Controller) *MockUnderlyingStore { + mock := &MockUnderlyingStore{ctrl: ctrl} + mock.recorder = &MockUnderlyingStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockUnderlyingStore) EXPECT() *MockUnderlyingStoreMockRecorder { + return m.recorder +} + +// Delete mocks base method. +func (m *MockUnderlyingStore) Delete(ctx context.Context, id string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, id) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockUnderlyingStoreMockRecorder) Delete(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockUnderlyingStore)(nil).Delete), ctx, id) +} + +// Exists mocks base method. +func (m *MockUnderlyingStore) Exists(ctx context.Context, id string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, id) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockUnderlyingStoreMockRecorder) Exists(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockUnderlyingStore)(nil).Exists), ctx, id) +} + +// Get mocks base method. +func (m *MockUnderlyingStore) Get(ctx context.Context, id string) (*storage.InitBundleMeta, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, id) + ret0, _ := ret[0].(*storage.InitBundleMeta) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Get indicates an expected call of Get. +func (mr *MockUnderlyingStoreMockRecorder) Get(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockUnderlyingStore)(nil).Get), ctx, id) +} + +// Upsert mocks base method. +func (m *MockUnderlyingStore) Upsert(ctx context.Context, obj *storage.InitBundleMeta) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Upsert", ctx, obj) + ret0, _ := ret[0].(error) + return ret0 +} + +// Upsert indicates an expected call of Upsert. +func (mr *MockUnderlyingStoreMockRecorder) Upsert(ctx, obj any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upsert", reflect.TypeOf((*MockUnderlyingStore)(nil).Upsert), ctx, obj) +} + +// Walk mocks base method. +func (m *MockUnderlyingStore) Walk(ctx context.Context, fn func(*storage.InitBundleMeta) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Walk", ctx, fn) + ret0, _ := ret[0].(error) + return ret0 +} + +// Walk indicates an expected call of Walk. +func (mr *MockUnderlyingStoreMockRecorder) Walk(ctx, fn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Walk", reflect.TypeOf((*MockUnderlyingStore)(nil).Walk), ctx, fn) +} diff --git a/central/clusterinit/store/store.go b/central/clusterinit/store/store.go index 3fd82e25828a7..cc4b74a6cee5a 100644 --- a/central/clusterinit/store/store.go +++ b/central/clusterinit/store/store.go @@ -5,27 +5,38 @@ import ( "github.com/pkg/errors" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/sync" ) var ( - // ErrInitBundleNotFound signals that a requested init bundle could not be located in the store. - ErrInitBundleNotFound = errors.New("init bundle not found") + // ErrInitBundleNotFound signals that a requested init bundle or CRS could not be located in the store. + ErrInitBundleNotFound = errors.New("init bundle or CRS not found") - // ErrInitBundleIDCollision signals that an init bundle could not be added to the store due to an ID collision. - ErrInitBundleIDCollision = errors.New("init bundle ID collision") + // ErrInitBundleIDCollision signals that an init bundle or a CRS could not be added to the store due to an ID collision. + ErrInitBundleIDCollision = errors.New("init bundle or CRS ID collision") - // ErrInitBundleDuplicateName signals that an init bundle or CRS could not be added because the name already exists on a non-revoked init bundle or CRS. + // ErrInitBundleDuplicateName signals that an init bundle or a CRS could not be added because the name already exists for + // a non-revoked init bundle or CRS. ErrInitBundleDuplicateName = errors.New("init bundle or CRS already exists") + + log = logging.LoggerForModule() ) // Store interface for managing persisted cluster init bundles. +// +//go:generate mockgen-wrapper type Store interface { GetAll(ctx context.Context) ([]*storage.InitBundleMeta, error) GetAllCRS(ctx context.Context) ([]*storage.InitBundleMeta, error) Get(ctx context.Context, id string) (*storage.InitBundleMeta, error) Add(ctx context.Context, bundleMeta *storage.InitBundleMeta) error + Delete(ctx context.Context, id string) error + Upsert(ctx context.Context, crs *storage.InitBundleMeta) error Revoke(ctx context.Context, id string) error + InitiateClusterRegistration(ctx context.Context, initArtifactId, clusterName string) error + MarkClusterRegistrationComplete(ctx context.Context, initArtifactId, clusterName string) error } // UnderlyingStore is the base store that actually accesses the data @@ -109,6 +120,13 @@ func (w *storeImpl) Add(ctx context.Context, meta *storage.InitBundleMeta) error return w.store.Upsert(ctx, meta) } +func (w *storeImpl) Delete(ctx context.Context, id string) error { + w.uniqueUpdateMutex.Lock() + defer w.uniqueUpdateMutex.Unlock() + + return w.store.Delete(ctx, id) +} + func (w *storeImpl) checkDuplicateName(ctx context.Context, meta *storage.InitBundleMeta) error { metas, err := w.GetAll(ctx) if err != nil { @@ -127,6 +145,13 @@ func (w *storeImpl) checkDuplicateName(ctx context.Context, meta *storage.InitBu return nil } +func (w *storeImpl) Upsert(ctx context.Context, crs *storage.InitBundleMeta) error { + w.uniqueUpdateMutex.Lock() + defer w.uniqueUpdateMutex.Unlock() + + return w.store.Upsert(ctx, crs) +} + func (w *storeImpl) Revoke(ctx context.Context, id string) error { w.uniqueUpdateMutex.Lock() defer w.uniqueUpdateMutex.Unlock() @@ -145,3 +170,122 @@ func (w *storeImpl) Revoke(ctx context.Context, id string) error { } return nil } + +// InitiateClusterRegistration checks if another registration using the CRS with the provided CRS ID is possible. +// If the provided id belongs to an init bundle, then registration is always allowed, without any bookkeeping. +func (w *storeImpl) InitiateClusterRegistration(ctx context.Context, initArtifactId, clusterName string) error { + w.uniqueUpdateMutex.Lock() + defer w.uniqueUpdateMutex.Unlock() + + initArtifactMeta, err := w.Get(ctx, initArtifactId) + if err != nil { + return errors.Wrapf(err, "retrieving init artifact meta data for ID %q", initArtifactId) + } + + log.Infof("Attempting registration for cluster %s using %s %s.", clusterName, initArtifactMeta.GetVersion().String(), initArtifactMeta.GetId()) + if initArtifactMeta.GetIsRevoked() { + log.Warnf("Init artifact %s is revoked, registration of cluster %s not allowed.", initArtifactId, clusterName) + return errors.Errorf("Init artifact %s is revoked", initArtifactMeta.GetId()) + } + + if initArtifactMeta.GetVersion() == storage.InitBundleMeta_INIT_BUNDLE { + return nil + } + + crsMeta := initArtifactMeta + maxRegistrations := crsMeta.GetMaxRegistrations() + + if maxRegistrations == 0 { + // We don't do any bookkeeping in this case to prevent the clusterinit storage holding the CRS IDs from growing unbounded. + log.Infof("Allowing registration of cluster %s using CRS %s without registration limit.", clusterName, crsMeta.GetId()) + return nil + } + + // Bookkeeping for registration-limited CRS. + registrationsInitiatedSet := set.NewStringSet(crsMeta.GetRegistrationsInitiated()...) + registrationsCompletedSet := set.NewStringSet(crsMeta.GetRegistrationsCompleted()...) + numRegistrationsTotal := uint64(len(registrationsInitiatedSet) + len(registrationsCompletedSet)) + if numRegistrationsTotal >= maxRegistrations { + log.Warnf("maximum number of cluster registrations (%d/%d) with the provided cluster registration secret %s/%q reached.", + numRegistrationsTotal, maxRegistrations, + crsMeta.GetId(), crsMeta.GetName()) + return errors.New("maximum number of allowed cluster registrations reached") + } + if registrationsCompletedSet.Contains(clusterName) { + return errors.Errorf("cluster %s already registered with cluster registration secret %s/%q", clusterName, crsMeta.GetId(), crsMeta.GetName()) + } + + if !registrationsInitiatedSet.Add(clusterName) { + log.Warnf("Attempting to initiate registration of cluster %s, even though it is already associated with CRS %s.", clusterName, crsMeta.GetId()) + return nil + } + + crsMeta.RegistrationsInitiated = registrationsInitiatedSet.AsSlice() + if err := w.store.Upsert(ctx, crsMeta); err != nil { + return errors.Wrapf(err, "updating meta data for cluster registration secret %s/%q", crsMeta.GetId(), crsMeta.GetName()) + } + log.Infof("Added cluster %s to list of initiated registrations for CRS %s (%s).", clusterName, crsMeta.GetName(), crsMeta.GetId()) + + return nil +} + +func (w *storeImpl) MarkClusterRegistrationComplete(ctx context.Context, initArtifactId, clusterName string) error { + w.uniqueUpdateMutex.Lock() + defer w.uniqueUpdateMutex.Unlock() + + initArtifactMeta, err := w.Get(ctx, initArtifactId) + if err != nil { + return errors.Wrapf(err, "retrieving init artifact meta data for ID %q", initArtifactId) + } + + log.Infof("Completing registration of cluster %s using %s %s.", clusterName, initArtifactMeta.GetVersion().String(), initArtifactMeta.GetId()) + + if initArtifactMeta.GetVersion() == storage.InitBundleMeta_INIT_BUNDLE { + return nil + } + + crsMeta := initArtifactMeta + maxRegistrations := crsMeta.GetMaxRegistrations() + + if maxRegistrations == 0 { + log.Infof("Completing the registration of cluster %s using CRS %s without allowed registration limit.", clusterName, crsMeta.GetId()) + return nil + } + + // Bookkeeping for registration-limited CRS. + + log.Infof("Marking registration of cluster %s using CRS %s as complete.", clusterName, crsMeta.GetId()) + registrationsInitiatedSet := set.NewStringSet(crsMeta.GetRegistrationsInitiated()...) + registrationsCompletedSet := set.NewStringSet(crsMeta.GetRegistrationsCompleted()...) + + if registrationsCompletedSet.Contains(clusterName) { + // Already done? + log.Infof("Registration of cluster %s using CRS %s already completed.", clusterName, initArtifactId) + return nil + } + + if !registrationsInitiatedSet.Contains(clusterName) { + return errors.Errorf("registration for cluster %s using cluster registration secret %s/%q not initiated", clusterName, crsMeta.GetId(), crsMeta.GetName()) + } + + _ = registrationsInitiatedSet.Remove(clusterName) + _ = registrationsCompletedSet.Add(clusterName) + updatedRegistrationsInitiated := uint64(len(registrationsInitiatedSet)) + updatedRegistrationsCompleted := uint64(len(registrationsCompletedSet)) + // Revoke CRS, if the limit for completed registrations is reached and if no registrations are currently in flight. + if updatedRegistrationsCompleted >= maxRegistrations && updatedRegistrationsInitiated == 0 { + crsMeta.IsRevoked = true + } + + crsMeta.RegistrationsInitiated = registrationsInitiatedSet.AsSlice() + crsMeta.RegistrationsCompleted = registrationsCompletedSet.AsSlice() + if err := w.store.Upsert(ctx, crsMeta); err != nil { + return errors.Wrapf(err, "updating meta data for cluster registration secret %q", crsMeta.GetId()) + } + + if crsMeta.GetIsRevoked() { + log.Infof("Marked CRS %s as revoked.", crsMeta.GetId()) + } + + return nil +} diff --git a/central/clusterinit/store/store_test_constructors.go b/central/clusterinit/store/store_test_constructors.go new file mode 100644 index 0000000000000..c4d2c3981761b --- /dev/null +++ b/central/clusterinit/store/store_test_constructors.go @@ -0,0 +1,14 @@ +package store + +import ( + "testing" + + pgStore "github.com/stackrox/rox/central/clusterinit/store/postgres" + "github.com/stackrox/rox/pkg/postgres" +) + +// GetTestPostgresDataStore provides a datastore connected to postgres for testing purposes. +func GetTestPostgresDataStore(_ testing.TB, pool postgres.DB) Store { + dbStore := pgStore.New(pool) + return NewStore(dbStore) +} diff --git a/central/clusterinit/store/tests/store_test.go b/central/clusterinit/store/tests/store_test.go index 5e1115de3dcb8..dd1498a856ff8 100644 --- a/central/clusterinit/store/tests/store_test.go +++ b/central/clusterinit/store/tests/store_test.go @@ -5,7 +5,10 @@ package tests import ( "context" "errors" + "fmt" + randpkg "math/rand/v2" "testing" + "time" "github.com/stackrox/rox/central/clusterinit/store" pgStore "github.com/stackrox/rox/central/clusterinit/store/postgres" @@ -13,7 +16,13 @@ import ( "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/uuid" "github.com/stretchr/testify/suite" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var ( + rand = randpkg.New(randpkg.NewPCG(3, 14159)) ) func TestClusterInitStore(t *testing.T) { @@ -118,3 +127,104 @@ func (s *clusterInitStoreTestSuite) TestRevokeToken() { s.Equal(toReuseMetaName.GetName(), reused.GetName()) s.Equal(toRevokeMeta.GetName(), reused.GetName()) } + +// Tests auto revocation for a CRS with maxRegistrations == 0 (unlimited registrations). + +func (s *clusterInitStoreTestSuite) TestCrsWithoutMaxRegistrations() { + clusterName := fmt.Sprintf("some-cluster-%s", uuid.NewV4().String()) + crsId := uuid.NewV4().String() + crsMeta := &storage.InitBundleMeta{ + Id: crsId, + Name: fmt.Sprintf("test-crs-unlimited-1-%d", rand.IntN(10000)), + CreatedAt: timestamppb.New(time.Now()), + Version: storage.InitBundleMeta_CRS, + } + err := s.store.Add(s.ctx, crsMeta) + s.Require().NoError(err, "adding CRS %s failed", crsId) + + err = s.store.InitiateClusterRegistration(s.ctx, crsId, clusterName) + s.NoErrorf(err, "recording initiated registration for CRS %s failed", crsId) + + crsMeta, err = s.store.Get(s.ctx, crsMeta.GetId()) + s.NoErrorf(err, "retrieving CRS %s failed", crsId) + s.Empty(crsMeta.GetRegistrationsInitiated(), "CRS %s has registrationsInitiated non-empty, even though registrations are unlimited", crsId) + + err = s.store.MarkClusterRegistrationComplete(s.ctx, crsId, clusterName) + s.NoErrorf(err, "recording completed registration for CRS %s failed", crsId) + s.Empty(crsMeta.GetRegistrationsInitiated(), "CRS %s has registrationsInitiated non-empty, even though registrations are unlimited", crsId) + s.Empty(crsMeta.GetRegistrationsCompleted(), "CRS %s has registrationsCompleted non-empty, even though registrations are unlimited", crsId) + s.Falsef(crsMeta.GetIsRevoked(), "CRS %s is revoked", crsId) +} + +// Tests auto revocation for a CRS with maxRegistrations == 1. + +func (s *clusterInitStoreTestSuite) TestCrsAutoRevocationOneShot() { + clusterName := fmt.Sprintf("some-cluster-%s", uuid.NewV4().String()) + crsId := uuid.NewV4().String() + crsMeta := &storage.InitBundleMeta{ + Id: crsId, + Name: fmt.Sprintf("test-crs-auto-revocation-1-%d", rand.IntN(10000)), + CreatedAt: timestamppb.New(time.Now()), + Version: storage.InitBundleMeta_CRS, + MaxRegistrations: 1, + } + err := s.store.Add(s.ctx, crsMeta) + s.Require().NoError(err, "adding CRS %s failed", crsId) + + err = s.store.InitiateClusterRegistration(s.ctx, crsId, clusterName) + s.NoErrorf(err, "recording initiated registration for CRS %s failed", crsId) + + err = s.store.InitiateClusterRegistration(s.ctx, crsId, clusterName) + s.Error(err, "cluster registration still possible") + + err = s.store.MarkClusterRegistrationComplete(s.ctx, crsId, clusterName) + s.NoErrorf(err, "recording completed registration for CRS %s failed", crsId) + + crsMeta, err = s.store.Get(s.ctx, crsId) + s.NoErrorf(err, "receiving CRS %s", crsId) + s.Truef(crsMeta.GetIsRevoked(), "CRS %s is not revoked", crsId) + +} + +// Tests auto revocation for a CRS with maxRegistrations > 1. +func (s *clusterInitStoreTestSuite) TestCrsAutoRevocationAfterTwoRegistrations() { + clusterName := fmt.Sprintf("some-cluster-%s", uuid.NewV4().String()) + crsId := uuid.NewV4().String() + crsMeta := &storage.InitBundleMeta{ + Id: crsId, + Name: fmt.Sprintf("test-crs-auto-revocation-2-%d", rand.IntN(10000)), + CreatedAt: timestamppb.New(time.Now()), + Version: storage.InitBundleMeta_CRS, + MaxRegistrations: 2, + } + err := s.store.Add(s.ctx, crsMeta) + s.Require().NoError(err, "adding CRS %s failed", crsId) + + err = s.store.InitiateClusterRegistration(s.ctx, crsId, clusterName) + s.NoErrorf(err, "recording initiated registration for CRS %s failed", crsId) + + err = s.store.InitiateClusterRegistration(s.ctx, crsId, clusterName) + s.NoError(err, "cluster registration not possible") + + err = s.store.MarkClusterRegistrationComplete(s.ctx, crsId, clusterName) + s.NoErrorf(err, "recording completed registration for CRS %s failed", crsId) + + crsMeta, err = s.store.Get(s.ctx, crsId) + s.NoErrorf(err, "receiving CRS %s", crsId) + s.Falsef(crsMeta.GetIsRevoked(), "CRS %s is not revoked", crsId) + + clusterName = fmt.Sprintf("some-cluster-%s", uuid.NewV4().String()) + + err = s.store.InitiateClusterRegistration(s.ctx, crsId, clusterName) + s.NoErrorf(err, "recording initiated registration for CRS %s failed", crsId) + + err = s.store.InitiateClusterRegistration(s.ctx, crsId, clusterName) + s.Error(err, "cluster registration still possible") + + err = s.store.MarkClusterRegistrationComplete(s.ctx, crsId, clusterName) + s.NoErrorf(err, "recording completed registration for CRS %s failed", crsId) + + crsMeta, err = s.store.Get(s.ctx, crsId) + s.NoErrorf(err, "receiving CRS %s", crsId) + s.Truef(crsMeta.GetIsRevoked(), "CRS %s is not revoked", crsId) +} diff --git a/central/clusters/deployer.go b/central/clusters/deployer.go index 0c9669792a8ae..c7a8f1f38b449 100644 --- a/central/clusters/deployer.go +++ b/central/clusters/deployer.go @@ -19,7 +19,6 @@ import ( // RenderOptions are options that control the rendering. type RenderOptions struct { CreateUpgraderSA bool - IstioVersion string DisablePodSecurityPolicies bool } @@ -33,7 +32,6 @@ func FieldsFromClusterAndRenderOpts(c *storage.Cluster, imageFlavor *defaults.Im baseValues := getBaseMetaValues(c, imageFlavor, imageFlavor.ChartRepo, &opts) setMainOverride(mainImage, baseValues) - deriveScannerRemoteFromMain(mainImage, baseValues) baseValues.EnablePodSecurityPolicies = !opts.DisablePodSecurityPolicies collector := determineCollectorImage(mainImage, collectorImage, imageFlavor) @@ -62,15 +60,6 @@ func MakeClusterImageNames(flavor *defaults.ImageFlavor, c *storage.Cluster) (*s return mainImageName, collectorImageName, nil } -// deriveScannerRemoteFromMain sets scanner-slim image remote, so that it comes from the same location as the main image -func deriveScannerRemoteFromMain(mainImage *storage.ImageName, metaValues *charts.MetaValues) { - scannerRemoteSlice := strings.Split(mainImage.GetRemote(), "/") - if len(scannerRemoteSlice) > 0 { - scannerRemoteSlice[len(scannerRemoteSlice)-1] = metaValues.ScannerSlimImageRemote - metaValues.ScannerSlimImageRemote = strings.Join(scannerRemoteSlice, "/") - } -} - // setMainOverride adds main image values to meta values as defined in secured cluster object. func setMainOverride(mainImage *storage.ImageName, metaValues *charts.MetaValues) { metaValues.MainRegistry = mainImage.GetRegistry() @@ -152,9 +141,6 @@ func getBaseMetaValues(c *storage.Cluster, imageFlavor *defaults.ImageFlavor, ch FactImageTag: versions.FactVersion, FactImageRemote: imageFlavor.FactImageName, - ScannerImageTag: versions.ScannerVersion, - ScannerSlimImageRemote: imageFlavor.ScannerSlimImageName, - KubectlOutput: true, Versions: versions, @@ -178,11 +164,10 @@ func getBaseMetaValues(c *storage.Cluster, imageFlavor *defaults.ImageFlavor, ch } func getFeatureFlagsAsManifestBundleEnv() map[string]string { - // For the environment variables we need to filter out ROX_SCANNER_V4, because it would - // wrongly enable Scanner V4 delegated scanning on secured clusters which are set up - // using manifest bundles. But delegated scanning is not supported for manifest bundle - // installed secured clusters. - skipFeatureFlags := set.NewFrozenStringSet("ROX_SCANNER_V4") + // ROX_SCANNER_V4 would enable delegated scanning, which bundles do not support. + // ROX_VIRTUAL_MACHINES is already set from virtualMachines.enabled; a duplicate + // env name would make kubectl apply fail to patch an existing Sensor Deployment. + skipFeatureFlags := set.NewFrozenStringSet("ROX_SCANNER_V4", "ROX_VIRTUAL_MACHINES") featureFlagVals := make(map[string]string) for _, feature := range features.Flags { envVar := feature.EnvVar() diff --git a/central/clusters/deployer_test.go b/central/clusters/deployer_test.go index 8b15d537962cd..08e800b6dc56d 100644 --- a/central/clusters/deployer_test.go +++ b/central/clusters/deployer_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/helm/charts" "github.com/stackrox/rox/pkg/images/defaults" flavorUtils "github.com/stackrox/rox/pkg/images/defaults/testutils" @@ -17,10 +18,6 @@ func getCollectorImage(fields *charts.MetaValues) string { return fmt.Sprintf("%s/%s:%s", fields.CollectorRegistry, fields.CollectorImageRemote, fields.CollectorImageTag) } -func getScannerSlim(fields *charts.MetaValues) string { - return fmt.Sprintf("%s/%s:%s", fields.MainRegistry, fields.ScannerSlimImageRemote, fields.ScannerImageTag) -} - func getMain(fields *charts.MetaValues) string { return fmt.Sprintf("%s/%s:%s", fields.MainRegistry, fields.ImageRemote, fields.ImageTag) } @@ -113,121 +110,102 @@ func testMetaValueGenerationWithImageFlavor(s *deployerTestSuite, flavor default defaultMainImage := flavor.MainImage() defaultCollectorImageNoTag := flavor.CollectorImageNoTag() defaultCollectorImage := flavor.CollectorImage() - defaultScannerSlimImage := flavor.ScannerSlimImage() var cases = map[string]struct { - cluster *storage.Cluster - expectedErrorMessage string - expectedMain string - expectedCollectorRef string - expectedScannerSlimRef string + cluster *storage.Cluster + expectedErrorMessage string + expectedMain string + expectedCollectorRef string }{ // we're testing possible main & collector combinations, grouped by main image setting // default main image "default main / no collector": { - cluster: makeTestCluster(defaultMainImageNoTag, ""), - expectedMain: defaultMainImage, - expectedCollectorRef: defaultCollectorImage, - expectedScannerSlimRef: defaultScannerSlimImage, + cluster: makeTestCluster(defaultMainImageNoTag, ""), + expectedMain: defaultMainImage, + expectedCollectorRef: defaultCollectorImage, }, "default main / default collector": { - cluster: makeTestCluster(defaultMainImageNoTag, defaultCollectorImageNoTag), - expectedMain: defaultMainImage, - expectedCollectorRef: defaultCollectorImage, - expectedScannerSlimRef: defaultScannerSlimImage, + cluster: makeTestCluster(defaultMainImageNoTag, defaultCollectorImageNoTag), + expectedMain: defaultMainImage, + expectedCollectorRef: defaultCollectorImage, }, "default main / default collector: custom tag": { - cluster: makeTestCluster(defaultMainImageNoTag, fmt.Sprintf("%s:custom", defaultCollectorImageNoTag)), - expectedMain: defaultMainImage, - expectedCollectorRef: flavor.CollectorImage(), - expectedScannerSlimRef: defaultScannerSlimImage, + cluster: makeTestCluster(defaultMainImageNoTag, fmt.Sprintf("%s:custom", defaultCollectorImageNoTag)), + expectedMain: defaultMainImage, + expectedCollectorRef: flavor.CollectorImage(), }, "default main / custom collector: with namespace": { - cluster: makeTestCluster(defaultMainImage, "quay.io/rhacs/collector"), - expectedMain: defaultMainImage, - expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/collector:%s", flavor.CollectorImageTag), - expectedScannerSlimRef: defaultScannerSlimImage, + cluster: makeTestCluster(defaultMainImage, "quay.io/rhacs/collector"), + expectedMain: defaultMainImage, + expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/collector:%s", flavor.CollectorImageTag), }, "default main / custom collector: with namespace & custom tag": { - cluster: makeTestCluster(defaultMainImage, "quay.io/rhacs/collector:custom"), - expectedMain: defaultMainImage, - expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/collector:%s", flavor.CollectorImageTag), - expectedScannerSlimRef: defaultScannerSlimImage, + cluster: makeTestCluster(defaultMainImage, "quay.io/rhacs/collector:custom"), + expectedMain: defaultMainImage, + expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/collector:%s", flavor.CollectorImageTag), }, "default main / custom collector: without namespace": { - cluster: makeTestCluster(defaultMainImage, "example.io/collector"), - expectedMain: defaultMainImage, - expectedCollectorRef: fmt.Sprintf("example.io/collector:%s", flavor.CollectorImageTag), - expectedScannerSlimRef: defaultScannerSlimImage, + cluster: makeTestCluster(defaultMainImage, "example.io/collector"), + expectedMain: defaultMainImage, + expectedCollectorRef: fmt.Sprintf("example.io/collector:%s", flavor.CollectorImageTag), }, "default main / custom collector: without namespace & custom tag": { - cluster: makeTestCluster(defaultMainImage, "example.io/collector:custom"), - expectedMain: defaultMainImage, - expectedCollectorRef: fmt.Sprintf("example.io/collector:%s", flavor.CollectorImageTag), - expectedScannerSlimRef: defaultScannerSlimImage, + cluster: makeTestCluster(defaultMainImage, "example.io/collector:custom"), + expectedMain: defaultMainImage, + expectedCollectorRef: fmt.Sprintf("example.io/collector:%s", flavor.CollectorImageTag), }, "default main: custom tag / no collector": { - cluster: makeTestCluster(fmt.Sprintf("%s:custom", defaultMainImageNoTag), ""), - expectedMain: fmt.Sprintf("%s:custom", defaultMainImageNoTag), - expectedCollectorRef: defaultCollectorImage, - expectedScannerSlimRef: defaultScannerSlimImage, + cluster: makeTestCluster(fmt.Sprintf("%s:custom", defaultMainImageNoTag), ""), + expectedMain: fmt.Sprintf("%s:custom", defaultMainImageNoTag), + expectedCollectorRef: defaultCollectorImage, }, "default main: custom tag / default collector": { - cluster: makeTestCluster(fmt.Sprintf("%s:custom", defaultMainImageNoTag), ""), - expectedMain: fmt.Sprintf("%s:custom", defaultMainImageNoTag), - expectedCollectorRef: defaultCollectorImage, - expectedScannerSlimRef: defaultScannerSlimImage, + cluster: makeTestCluster(fmt.Sprintf("%s:custom", defaultMainImageNoTag), ""), + expectedMain: fmt.Sprintf("%s:custom", defaultMainImageNoTag), + expectedCollectorRef: defaultCollectorImage, }, "default main: custom registry / no collector": { - cluster: makeTestCluster("quay.io/rhacs/"+flavor.MainImageName, ""), - expectedMain: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.MainImageName, flavor.MainImageTag), - expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.CollectorImageName, flavor.CollectorImageTag), - expectedScannerSlimRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.ScannerSlimImageName, flavor.ScannerImageTag), + cluster: makeTestCluster("quay.io/rhacs/"+flavor.MainImageName, ""), + expectedMain: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.MainImageName, flavor.MainImageTag), + expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.CollectorImageName, flavor.CollectorImageTag), }, // custom main image "custom main: with namespace / no collector": { - cluster: makeTestCluster("quay.io/rhacs/main", ""), - expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), - expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.CollectorImageName, flavor.CollectorImageTag), - expectedScannerSlimRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.ScannerSlimImageName, flavor.ScannerImageTag), + cluster: makeTestCluster("quay.io/rhacs/main", ""), + expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), + expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.CollectorImageName, flavor.CollectorImageTag), }, "custom main: without namespace / no collector": { - cluster: makeTestCluster("example.io/main", ""), - expectedMain: fmt.Sprintf("example.io/main:%s", flavor.MainImageTag), - expectedCollectorRef: fmt.Sprintf("example.io/%s:%s", flavor.CollectorImageName, flavor.CollectorImageTag), - expectedScannerSlimRef: fmt.Sprintf("example.io/%s:%s", flavor.ScannerSlimImageName, flavor.ScannerImageTag), + cluster: makeTestCluster("example.io/main", ""), + expectedMain: fmt.Sprintf("example.io/main:%s", flavor.MainImageTag), + expectedCollectorRef: fmt.Sprintf("example.io/%s:%s", flavor.CollectorImageName, flavor.CollectorImageTag), }, "custom main / default collector": { - cluster: makeTestCluster("quay.io/rhacs/main", defaultCollectorImageNoTag), - expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), - expectedCollectorRef: defaultCollectorImage, - expectedScannerSlimRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.ScannerSlimImageName, flavor.ScannerImageTag), + cluster: makeTestCluster("quay.io/rhacs/main", defaultCollectorImageNoTag), + expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), + expectedCollectorRef: defaultCollectorImage, }, "custom main / custom collector: with namespace": { - cluster: makeTestCluster("quay.io/rhacs/main", "quay.io/rhacs/collector"), - expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), - expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/collector:%s", flavor.CollectorImageTag), - expectedScannerSlimRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.ScannerSlimImageName, flavor.ScannerImageTag), + cluster: makeTestCluster("quay.io/rhacs/main", "quay.io/rhacs/collector"), + expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), + expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/collector:%s", flavor.CollectorImageTag), }, "custom main / custom collector: with namespace & custom tag": { - cluster: makeTestCluster("quay.io/rhacs/main", "quay.io/rhacs/collector:custom"), - expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), - expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/collector:%s", flavor.CollectorImageTag), - expectedScannerSlimRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.ScannerSlimImageName, flavor.ScannerImageTag), + cluster: makeTestCluster("quay.io/rhacs/main", "quay.io/rhacs/collector:custom"), + expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), + expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/collector:%s", flavor.CollectorImageTag), }, "custom main / custom collector: without namespace": { - cluster: makeTestCluster("quay.io/rhacs/main", "example.io/collector"), - expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), - expectedCollectorRef: fmt.Sprintf("example.io/collector:%s", flavor.CollectorImageTag), - expectedScannerSlimRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.ScannerSlimImageName, flavor.ScannerImageTag), + cluster: makeTestCluster("quay.io/rhacs/main", "example.io/collector"), + expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), + expectedCollectorRef: fmt.Sprintf("example.io/collector:%s", flavor.CollectorImageTag), }, "custom main / custom collector: without namespace & custom tag": { - cluster: makeTestCluster("quay.io/rhacs/main", "example.io/collector:custom"), - expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), - expectedCollectorRef: fmt.Sprintf("example.io/collector:%s", flavor.CollectorImageTag), - expectedScannerSlimRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.ScannerSlimImageName, flavor.ScannerImageTag), + cluster: makeTestCluster("quay.io/rhacs/main", "example.io/collector:custom"), + expectedMain: fmt.Sprintf("quay.io/rhacs/main:%s", flavor.MainImageTag), + expectedCollectorRef: fmt.Sprintf("example.io/collector:%s", flavor.CollectorImageTag), }, /* // These tests are failing right now but should succeed after ROX-15691 has been implemented @@ -243,16 +221,14 @@ func testMetaValueGenerationWithImageFlavor(s *deployerTestSuite, flavor default }, */ "custom main / custom collector: same registry, different namespaces": { - cluster: makeTestCluster("quay.io/namespace-a/main", "quay.io/namespace-b/collector"), - expectedMain: fmt.Sprintf("quay.io/namespace-a/main:%s", flavor.MainImageTag), - expectedCollectorRef: fmt.Sprintf("quay.io/namespace-b/collector:%s", flavor.CollectorImageTag), - expectedScannerSlimRef: fmt.Sprintf("quay.io/namespace-a/%s:%s", flavor.ScannerSlimImageName, flavor.ScannerImageTag), + cluster: makeTestCluster("quay.io/namespace-a/main", "quay.io/namespace-b/collector"), + expectedMain: fmt.Sprintf("quay.io/namespace-a/main:%s", flavor.MainImageTag), + expectedCollectorRef: fmt.Sprintf("quay.io/namespace-b/collector:%s", flavor.CollectorImageTag), }, "custom main: non-default name / no collector": { - cluster: makeTestCluster("quay.io/rhacs/customname", ""), - expectedMain: fmt.Sprintf("quay.io/rhacs/customname:%s", flavor.MainImageTag), - expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.CollectorImageName, flavor.CollectorImageTag), - expectedScannerSlimRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.ScannerSlimImageName, flavor.ScannerImageTag), + cluster: makeTestCluster("quay.io/rhacs/customname", ""), + expectedMain: fmt.Sprintf("quay.io/rhacs/customname:%s", flavor.MainImageTag), + expectedCollectorRef: fmt.Sprintf("quay.io/rhacs/%s:%s", flavor.CollectorImageName, flavor.CollectorImageTag), }, // Expected fail cases "expectedError: empty main image": { @@ -279,7 +255,6 @@ func testMetaValueGenerationWithImageFlavor(s *deployerTestSuite, flavor default s.NoError(err) s.Equal(c.expectedMain, getMain(fields), "Main image does not match") s.Equal(c.expectedCollectorRef, getCollectorImage(fields), "Collector image does not match") - s.Equal(c.expectedScannerSlimRef, getScannerSlim(fields), "Scanner slim image does not match") } }) } @@ -312,6 +287,15 @@ func (s *deployerTestSuite) TestFieldsFromClusterAndRenderOpts() { } +// TestGetFeatureFlagsAsManifestBundleEnv_OmitsChartOwnedFlags checks that +// certain flags (normally already set from chart config) are not also injected as EnvVars. +func TestGetFeatureFlagsAsManifestBundleEnv_OmitsChartOwnedFlags(t *testing.T) { + t.Parallel() + env := getFeatureFlagsAsManifestBundleEnv() + assert.NotContains(t, env, features.ScannerV4.EnvVar()) + assert.NotContains(t, env, features.VirtualMachines.EnvVar()) +} + func TestRequiredFieldsArePresent(t *testing.T) { testutils.SetExampleVersion(t) @@ -328,7 +312,6 @@ func TestRequiredFieldsArePresent(t *testing.T) { assert.NotEmpty(t, fields.Versions.ChartVersion) assert.NotEmpty(t, fields.Versions.MainVersion) assert.NotEmpty(t, fields.Versions.CollectorVersion) - assert.NotEmpty(t, fields.Versions.ScannerVersion) assert.NotEmpty(t, fields.ChartRepo.URL) } diff --git a/central/clusters/zip/render_test.go b/central/clusters/zip/render_test.go index 2caf9dfa05cc0..d07839e397119 100644 --- a/central/clusters/zip/render_test.go +++ b/central/clusters/zip/render_test.go @@ -3,7 +3,6 @@ package zip import ( "bufio" "bytes" - "strings" "testing" "github.com/stackrox/rox/central/clusters" @@ -86,25 +85,10 @@ func doTestRenderOpenshif(t *testing.T, clusterType storage.ClusterType) { assert.True(t, foundMain) assert.Equal(t, "compliance", complianceCont.Name) - if clusterType == storage.ClusterType_OPENSHIFT4_CLUSTER { - nInvCont, found := findContainer(ds.Spec.Template.Spec.Containers, "node-inventory") - assert.True(t, found, "node-inventory container should exist under collector DS") - assert.Equal(t, "node-inventory", nInvCont.Name) - - expectedScannerParts := strings.Split(strings.ReplaceAll(complianceCont.Image, "/main:", "/scanner-slim:"), ":") - assert.Truef(t, strings.HasPrefix(nInvCont.Image, expectedScannerParts[0]), "scanner-slim image (%q) should be from the same registry as main (%q)", nInvCont.Image, complianceCont.Image) - - value, exists := getEnvVarValue(complianceCont.Env, env.NodeInventoryContainerEnabled.EnvVar()) - assert.True(t, exists) - assert.Equal(t, "true", value, "compliance should have %s=true", env.NodeInventoryContainerEnabled.EnvVar()) - } else { - _, foundNInv := findContainer(ds.Spec.Template.Spec.Containers, "node-inventory") - assert.False(t, foundNInv, "node-inventory container must not exist under collector DS") - - value, exists := getEnvVarValue(complianceCont.Env, env.NodeInventoryContainerEnabled.EnvVar()) - assert.True(t, exists) - assert.Equalf(t, "false", value, "compliance should have %s=false", env.NodeInventoryContainerEnabled.EnvVar()) - } + _, exists := getEnvVarValue(complianceCont.Env, env.NodeInventoryContainerEnabled.EnvVar()) + assert.False(t, exists, "compliance should not set %s; Go default is false", env.NodeInventoryContainerEnabled.EnvVar()) + _, foundNInv := findContainer(ds.Spec.Template.Spec.Containers, "node-inventory") + assert.False(t, foundNInv, "node-inventory container must not exist under collector DS") } cases := map[string]func(object runtime.Object){ @@ -135,7 +119,7 @@ func TestRenderWithNoCollection(t *testing.T) { cluster := &storage.Cluster{ Name: "cluster", MainImage: "stackrox/main:abc", - Type: storage.ClusterType_OPENSHIFT_CLUSTER, + Type: storage.ClusterType_OPENSHIFT4_CLUSTER, CollectionMethod: storage.CollectionMethod_NO_COLLECTION, } diff --git a/central/clusters/zip/zip.go b/central/clusters/zip/zip.go index fa1ee404f3cf4..c4774ac2078ea 100644 --- a/central/clusters/zip/zip.go +++ b/central/clusters/zip/zip.go @@ -17,7 +17,6 @@ import ( helmUtil "github.com/stackrox/rox/pkg/helm/util" "github.com/stackrox/rox/pkg/httputil" "github.com/stackrox/rox/pkg/images/defaults" - "github.com/stackrox/rox/pkg/istioutils" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/renderer" "github.com/stackrox/rox/pkg/zip" @@ -60,14 +59,6 @@ func renderBaseFiles(cluster *storage.Cluster, renderOpts clusters.RenderOptions IsInstall: true, }, } - if renderOpts.IstioVersion != "" { - istioAPIResources, err := istioutils.GetAPIResourcesByVersion(renderOpts.IstioVersion) - if err != nil { - return nil, errors.Wrap(err, "unable to retrieve Istio API resources") - } - opts.APIVersions = helmUtil.VersionSetFromResources(istioAPIResources...) - } - baseFiles, err := renderer.RenderSensor(fields, &certs, opts) if err != nil { return nil, errors.Wrap(err, "unable to get required cluster information") @@ -135,7 +126,6 @@ func (z zipHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { renderOpts := clusters.RenderOptions{ CreateUpgraderSA: createUpgraderSA, - IstioVersion: params.IstioVersion, DisablePodSecurityPolicies: params.DisablePodSecurityPolicies, } diff --git a/central/compliance/aggregation/aggregation_test.go b/central/compliance/aggregation/aggregation_test.go index c4db4dc168cd1..bd8a81574d0f3 100644 --- a/central/compliance/aggregation/aggregation_test.go +++ b/central/compliance/aggregation/aggregation_test.go @@ -29,7 +29,7 @@ const registeredStandardID = "standard1" func mockStandardsRepo(t require.TestingT) standards.Repository { controls := make([]metadata.Control, 0, 8) - for i := 0; i < 9; i++ { + for i := range 9 { controls = append(controls, metadata.Control{ ID: fmt.Sprintf("control%d", i), }) @@ -544,11 +544,11 @@ func TestIsValidCheck(t *testing.T) { func mockBenchmarkRunResult() *storage.ComplianceRunResults { deploymentResults := make(map[string]*storage.ComplianceRunResults_EntityResults) deployments := make(map[string]*storage.ComplianceDomain_Deployment) - for i := 0; i < 10000; i++ { + for range 10000 { results := &storage.ComplianceRunResults_EntityResults{ ControlResults: make(map[string]*storage.ComplianceResultValue), } - for i := 0; i < 50; i++ { + for i := range 50 { results.ControlResults[fmt.Sprintf("%s:control%d", registeredStandardID, i)] = &storage.ComplianceResultValue{ OverallState: storage.ComplianceState_COMPLIANCE_STATE_FAILURE, } @@ -584,11 +584,10 @@ func mockBenchmarkRunResult() *storage.ComplianceRunResults { func BenchmarkAggregatedResults(b *testing.B) { result := mockBenchmarkRunResult() - b.ResetTimer() a := &aggregatorImpl{ standards: mockStandardsRepo(b), } - for i := 0; i < b.N; i++ { + for b.Loop() { a.getAggregatedResults(nil, storage.ComplianceAggregation_CHECK, []*storage.ComplianceRunResults{result}, &mask{}) } } diff --git a/central/compliance/aggregation/sort.go b/central/compliance/aggregation/sort.go index 1240b1bd4e3d0..209ea632f69d1 100644 --- a/central/compliance/aggregation/sort.go +++ b/central/compliance/aggregation/sort.go @@ -15,16 +15,10 @@ func sortAggregations(results []*storage.ComplianceAggregation_Result) { } func aBeforeB(a, b []*storage.ComplianceAggregation_AggregationKey) bool { - // Get the length of the smallest aggregtion key set. - var minLen int - if len(a) < len(b) { - minLen = len(a) - } else { - minLen = len(b) - } + minLen := min(len(a), len(b)) // Try to choose a or b based on scope. Lowest scope type works first. - for i := 0; i < minLen; i++ { + for i := range minLen { if a[i].GetScope() < b[i].GetScope() { return true } else if a[i].GetScope() > b[i].GetScope() { @@ -40,7 +34,7 @@ func aBeforeB(a, b []*storage.ComplianceAggregation_AggregationKey) bool { } // Same exact scopes, so we have to choose order based on ids. - for i := 0; i < minLen; i++ { + for i := range minLen { var cmp int if a[i].GetScope() == storage.ComplianceAggregation_CONTROL { // We want to use version string comparison for control ids (1_20_a > 1_2_a) diff --git a/central/compliance/checks/common/checks.go b/central/compliance/checks/common/checks.go index d07bb2ef847a6..7dd7d395a513f 100644 --- a/central/compliance/checks/common/checks.go +++ b/central/compliance/checks/common/checks.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "regexp" + "slices" "github.com/stackrox/rox/central/compliance/framework" "github.com/stackrox/rox/generated/storage" @@ -28,11 +29,9 @@ func CheckNotifierInUseByCluster(ctx framework.ComplianceContext) { continue } - for _, notifierID := range policy.GetNotifiers() { - if notifiers.Contains(notifierID) { - framework.Pass(ctx, "At least one enabled policy has a notifier configured.") - return - } + if slices.ContainsFunc(policy.GetNotifiers(), notifiers.Contains) { + framework.Pass(ctx, "At least one enabled policy has a notifier configured.") + return } } @@ -106,13 +105,7 @@ func CheckAnyPolicyInLifecycleStageEnforced(ctx framework.ComplianceContext, lif // PolicyIsInLifecycleStage returns whether the given policy is in the given lifecycle stage. func PolicyIsInLifecycleStage(policy *storage.Policy, targetStage storage.LifecycleStage) bool { - for _, policyStage := range policy.GetLifecycleStages() { - if policyStage == targetStage { - return true - } - } - - return false + return slices.Contains(policy.GetLifecycleStages(), targetStage) } // AnyPoliciesEnforced checks if any policy in the given set is being enforced. @@ -333,12 +326,7 @@ func deploymentHasSpecifiedNetworkPolicy(policyType storage.NetworkPolicyType, d } func policyIsOfType(spec *storage.NetworkPolicySpec, policyType storage.NetworkPolicyType) bool { - for _, ty := range spec.GetPolicyTypes() { - if ty == policyType { - return true - } - } - return false + return slices.Contains(spec.GetPolicyTypes(), policyType) } func isKubeSystem(deployment *storage.Deployment) bool { diff --git a/central/compliance/checks/common/vulns.go b/central/compliance/checks/common/vulns.go index 3c01a37faee0b..7039174db8eba 100644 --- a/central/compliance/checks/common/vulns.go +++ b/central/compliance/checks/common/vulns.go @@ -1,6 +1,7 @@ package common import ( + "slices" "strings" "unicode" @@ -39,12 +40,9 @@ func CheckAtLeastOnePolicyEnabledReferringToVulns(ctx framework.ComplianceContex framework.Passf(ctx, "Policy %q is enabled, and targets vulnerabilities", policy.GetName()) continue } - for _, cveField := range policyfields.GetCVEs(policy) { - if checkCVEIsNotSingleRegexMatch(cveField) { - vulnPolicyIDs.Add(policy.GetId()) - framework.Passf(ctx, "Policy %q is enabled, and targets vulnerabilities", policy.GetName()) - break - } + if slices.ContainsFunc(policyfields.GetCVEs(policy), checkCVEIsNotSingleRegexMatch) { + vulnPolicyIDs.Add(policy.GetId()) + framework.Passf(ctx, "Policy %q is enabled, and targets vulnerabilities", policy.GetName()) } } if vulnPolicyIDs.Cardinality() == 0 { diff --git a/central/compliance/data/factory.go b/central/compliance/data/factory.go index c20314b9e5088..5331539120961 100644 --- a/central/compliance/data/factory.go +++ b/central/compliance/data/factory.go @@ -11,6 +11,7 @@ import ( imageStore "github.com/stackrox/rox/central/image/datastore" "github.com/stackrox/rox/central/imageintegration" imageIntegrationStore "github.com/stackrox/rox/central/imageintegration/datastore" + imageMapperDatastore "github.com/stackrox/rox/central/imagev2/datastore/mapper/datastore" "github.com/stackrox/rox/central/networkgraph/entity/networktree" nfDS "github.com/stackrox/rox/central/networkgraph/flow/datastore" npDS "github.com/stackrox/rox/central/networkpolicies/datastore" @@ -54,7 +55,7 @@ func NewDefaultFactory() RepositoryFactory { networkPoliciesStore: npDS.Singleton(), networkGraphEvaluator: graph.Singleton(), policyStore: policiesStore.Singleton(), - imageStore: imageStore.Singleton(), + imageStore: imageMapperDatastore.Singleton(), imageIntegrationStore: imageIntegrationStore.Singleton(), imageIntegrationsSet: imageintegration.Set(), processIndicatorStore: processIndicatorStore.Singleton(), diff --git a/central/compliance/datastore/datastore_impl.go b/central/compliance/datastore/datastore_impl.go index 6e0e4c0bbb2f0..22a05eaca8fb8 100644 --- a/central/compliance/datastore/datastore_impl.go +++ b/central/compliance/datastore/datastore_impl.go @@ -28,7 +28,7 @@ type datastoreImpl struct { filter SacFilter storedAggregationMutex sync.RWMutex - aggregationSequenceNumber uint64 + aggregationSequenceNumber atomic.Uint64 } func (ds *datastoreImpl) GetSpecificRunResults(ctx context.Context, clusterID, standardID, runID string, flags types.GetFlags) (types.ResultsWithStatus, error) { @@ -145,7 +145,7 @@ func (ds *datastoreImpl) StoreRunResults(ctx context.Context, results *storage.C defer func() { // Atomic because it will be atomically read outside the mutex - atomic.AddUint64(&ds.aggregationSequenceNumber, 1) + ds.aggregationSequenceNumber.Add(1) }() if err := ds.storage.ClearAggregationResults(ctx); err != nil { @@ -185,7 +185,7 @@ func (ds *datastoreImpl) PerformStoredAggregation(ctx context.Context, args *Sto } // Get the aggregation sequence number before performing the aggregation. We don't need to be in a lock here. - aggregationSequenceNumber := atomic.LoadUint64(&ds.aggregationSequenceNumber) + aggregationSequenceNumber := ds.aggregationSequenceNumber.Load() // This performs the actual aggregation. It must occur after getting the sequence number. results, sources, domainMap, err = args.AggregationFunc() if err != nil { @@ -198,7 +198,7 @@ func (ds *datastoreImpl) PerformStoredAggregation(ctx context.Context, args *Sto // storing the aggregation result ds.storedAggregationMutex.RLock() defer ds.storedAggregationMutex.RUnlock() - curAggSeqNum := atomic.LoadUint64(&ds.aggregationSequenceNumber) + curAggSeqNum := ds.aggregationSequenceNumber.Load() // Storing aggregation results is only permitted if the compliance data hasn't changed if aggregationSequenceNumber != curAggSeqNum { return diff --git a/central/compliance/datastore/util.go b/central/compliance/datastore/util.go index 4d1ef013895b8..5d648116bffde 100644 --- a/central/compliance/datastore/util.go +++ b/central/compliance/datastore/util.go @@ -10,7 +10,7 @@ import ( // metadata of all referenced cluster/standard pair and their successful/failed runs. func ValidResultsAndSources(allResults map[compliance.ClusterStandardPair]types.ResultsWithStatus) ([]*storage.ComplianceRunResults, []*storage.ComplianceAggregation_Source) { var validResults []*storage.ComplianceRunResults - var sources []*storage.ComplianceAggregation_Source + sources := make([]*storage.ComplianceAggregation_Source, 0, len(allResults)) for key, res := range allResults { if res.LastSuccessfulResults != nil { diff --git a/central/compliance/framework/check.go b/central/compliance/framework/check.go index 1b64543312124..7485849109d9a 100644 --- a/central/compliance/framework/check.go +++ b/central/compliance/framework/check.go @@ -1,6 +1,10 @@ package framework -import pkgFramework "github.com/stackrox/rox/pkg/compliance/framework" +import ( + "slices" + + pkgFramework "github.com/stackrox/rox/pkg/compliance/framework" +) // A Check is a single piece of logic executed as part of a compliance run. It usually corresponds to one or multiple // controls in a compliance standard. @@ -73,12 +77,7 @@ func (c *checkFromFunc) AppliesToScope(scope pkgFramework.TargetKind) bool { if c.metadata.Scope == scope { return true } - for _, addlScope := range c.metadata.AdditionalScopes { - if addlScope == scope { - return true - } - } - return false + return slices.Contains(c.metadata.AdditionalScopes, scope) } func (c *checkFromFunc) DataDependencies() []string { diff --git a/central/compliance/handlers/csv.go b/central/compliance/handlers/csv.go index c2d572339832a..4518d4775a21d 100644 --- a/central/compliance/handlers/csv.go +++ b/central/compliance/handlers/csv.go @@ -99,7 +99,8 @@ func stateToString(s storage.ComplianceState) string { func (c *csvResults) addRow(row complianceRow) { // standard, cluster, type, namespace, object, control, state, evidence - value := []string{ + value := make([]string, 0, 10) + value = append(value, row.standardID, row.clusterName, row.objectNamespace, @@ -108,7 +109,7 @@ func (c *csvResults) addRow(row complianceRow) { row.controlName, row.controlDescription, stateToString(row.result.GetOverallState()), - } + ) lines := make([]string, 0, len(row.result.GetEvidence())) for i, ev := range row.result.GetEvidence() { diff --git a/central/compliance/manager/manager_impl.go b/central/compliance/manager/manager_impl.go index c6d21b23f2616..a7611bc3b52f9 100644 --- a/central/compliance/manager/manager_impl.go +++ b/central/compliance/manager/manager_impl.go @@ -352,7 +352,7 @@ func (m *manager) createAndLaunchRuns(ctx context.Context, clusterStandardPairs var dataPromise dataPromise if standard.HasAnyDataDependency(scrapeDataDeps...) { if scrapeBasedPromise == nil { - var standardIDs []string + standardIDs := make([]string, 0, len(standardImpls)) for _, standard := range standardImpls { standardIDs = append(standardIDs, standard.ID) } diff --git a/central/compliance/standards/types.go b/central/compliance/standards/types.go index 297648673f163..eb1db04e67974 100644 --- a/central/compliance/standards/types.go +++ b/central/compliance/standards/types.go @@ -1,6 +1,7 @@ package standards import ( + "slices" "sort" "github.com/stackrox/rox/central/compliance/framework" @@ -71,9 +72,7 @@ func (s *Standard) protoScopes() []v1.ComplianceStandardMetadata_Scope { scopes = append(scopes, v1.ComplianceStandardMetadata_NODE) } } - sort.Slice(scopes, func(i, j int) bool { - return scopes[i] < scopes[j] - }) + slices.Sort(scopes) return scopes } @@ -110,12 +109,7 @@ func (s *Standard) ToProto() *v1.ComplianceStandard { // HasAnyDataDependency checks if the given standard requires at least one of the given data dependencies. func (s *Standard) HasAnyDataDependency(deps ...string) bool { - for _, dep := range deps { - if s.allDataDeps.Contains(dep) { - return true - } - } - return false + return slices.ContainsFunc(deps, s.allDataDeps.Contains) } // AllDataDependencies returns all data dependencies of all checks in this standard in sorted order. @@ -260,9 +254,7 @@ func newStandard(standardMD metadata.Standard, checkRegistry framework.CheckRegi for s := range scopeMap { scopes = append(scopes, s) } - sort.Slice(scopes, func(i, j int) bool { - return scopes[i] < scopes[j] - }) + slices.Sort(scopes) s.scopes = scopes return s diff --git a/central/compliance/standards/utils.go b/central/compliance/standards/utils.go index d91b57642a252..472170385deba 100644 --- a/central/compliance/standards/utils.go +++ b/central/compliance/standards/utils.go @@ -6,8 +6,9 @@ import ( // GetSupportedStandards returns the ids of all supported compliance standards func GetSupportedStandards() []string { - var registeredStandards []string - for _, standard := range RegistrySingleton().AllStandards() { + allStandards := RegistrySingleton().AllStandards() + registeredStandards := make([]string, 0, len(allStandards)) + for _, standard := range allStandards { registeredStandards = append(registeredStandards, standard.ID) } return registeredStandards diff --git a/central/complianceoperator/manager/convoy_test.go b/central/complianceoperator/manager/convoy_test.go new file mode 100644 index 0000000000000..ead4ff90db917 --- /dev/null +++ b/central/complianceoperator/manager/convoy_test.go @@ -0,0 +1,195 @@ +//go:build sql_integration + +package manager + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/ComplianceAsCode/compliance-operator/pkg/apis/compliance/v1alpha1" + "github.com/stackrox/rox/central/compliance/datastore/mocks" + "github.com/stackrox/rox/central/compliance/framework" + "github.com/stackrox/rox/central/compliance/standards" + "github.com/stackrox/rox/central/compliance/standards/metadata" + checkResultsDatastore "github.com/stackrox/rox/central/complianceoperator/checkresults/datastore" + checkResultsStore "github.com/stackrox/rox/central/complianceoperator/checkresults/store/postgres" + profileDatastore "github.com/stackrox/rox/central/complianceoperator/profiles/datastore" + profileStore "github.com/stackrox/rox/central/complianceoperator/profiles/store/postgres" + rulesDatastore "github.com/stackrox/rox/central/complianceoperator/rules/datastore" + rulesStore "github.com/stackrox/rox/central/complianceoperator/rules/store/postgres" + scansDatastore "github.com/stackrox/rox/central/complianceoperator/scans/datastore" + scansStore "github.com/stackrox/rox/central/complianceoperator/scans/store/postgres" + scanSettingBindingDatastore "github.com/stackrox/rox/central/complianceoperator/scansettingbinding/datastore" + scanSettingBindingStore "github.com/stackrox/rox/central/complianceoperator/scansettingbinding/store/postgres" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sync" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func newManagerTB(tb testing.TB) *managerImpl { + tb.Helper() + registry, err := standards.NewRegistry(framework.RegistrySingleton(), metadata.AllStandards...) + require.NoError(tb, err) + + db := pgtest.ForT(tb) + prof := profileStore.New(db) + ssb := scanSettingBindingStore.New(db) + rules := rulesStore.New(db) + rulesDS, err := rulesDatastore.NewDatastore(rules) + require.NoError(tb, err) + scans := scansStore.New(db) + scansDS := scansDatastore.NewDatastore(scans) + checks := checkResultsStore.New(db) + + ctrl := gomock.NewController(tb) + compliance := mocks.NewMockDataStore(ctrl) + + mgr, err := NewManager(registry, profileDatastore.NewDatastore(prof), scansDS, scanSettingBindingDatastore.NewDatastore(ssb), rulesDS, checkResultsDatastore.NewDatastore(checks), compliance) + require.NoError(tb, err) + return mgr.(*managerImpl) +} + +func makeConvoyRule(clusterID, ruleName string) *storage.ComplianceOperatorRule { + return &storage.ComplianceOperatorRule{ + Id: uuid.NewV4().String(), + Name: fmt.Sprintf("%s-ext", ruleName), + Annotations: map[string]string{ + v1alpha1.RuleIDAnnotationKey: ruleName, + }, + ClusterId: clusterID, + Title: fmt.Sprintf("Rule %s", ruleName), + } +} + +func makeConvoyProfile(clusterID, profileName string, ruleNames []string) *storage.ComplianceOperatorProfile { + rules := make([]*storage.ComplianceOperatorProfile_Rule, len(ruleNames)) + for i, name := range ruleNames { + rules[i] = &storage.ComplianceOperatorProfile_Rule{Name: name} + } + return &storage.ComplianceOperatorProfile{ + Id: uuid.NewV4().String(), + Name: profileName, + ClusterId: clusterID, + Description: fmt.Sprintf("Profile %s on cluster %s", profileName, clusterID), + Annotations: map[string]string{ + v1alpha1.ProductTypeAnnotation: string(v1alpha1.ScanTypePlatform), + }, + Rules: rules, + } +} + +func seedProfiles(tb testing.TB, mgr *managerImpl, numClusters, profilesPerCluster int, ruleNames []string) { + tb.Helper() + for c := range numClusters { + for p := range profilesPerCluster { + profile := makeConvoyProfile(fmt.Sprintf("cluster-%d", c), fmt.Sprintf("profile-%d", p), ruleNames) + require.NoError(tb, mgr.AddProfile(profile)) + } + } +} + +// TestConcurrentAddProfileCompletesWithoutConvoy verifies that without the old +// startup Walk competing for resources, concurrent AddProfile calls from sensor +// reconnects complete within a reasonable deadline. +// +// Background: The old NewManager did a Walk of all profiles at startup, calling +// addProfileNoLock for each — which itself did another Walk. This O(N²) pattern +// combined with concurrent sensor AddProfile calls caused a lock convoy that +// exhausted context deadlines and panicked Central. See the customer stack trace: +// +// panic: context deadline exceeded +// iterating over rows +// RunCursorQueryForSchemaFn → walkByQuery → Walk → +// complianceoperator/profiles/datastore.Walk → +// complianceoperator/manager.NewManager +// +// The fix removes the startup Walk (sensors repopulate on reconnect) and +// throttles concurrent profile/rule pipeline operations via semaphore. +func TestConcurrentAddProfileCompletesWithoutConvoy(t *testing.T) { + mgr := newManagerTB(t) + + ruleNames := []string{"rule-0", "rule-1", "rule-2"} + for _, name := range ruleNames { + require.NoError(t, mgr.AddRule(makeConvoyRule("seed", name))) + } + + seedProfiles(t, mgr, 50, 10, ruleNames) + t.Logf("Seeded 500 profiles") + + deadline := 30 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), deadline) + defer cancel() + + sensorClusters := 20 + var wg sync.WaitGroup + errs := make([]error, sensorClusters) + wg.Add(sensorClusters) + + start := time.Now() + for c := range sensorClusters { + go func(clusterIdx int) { + defer wg.Done() + clusterID := fmt.Sprintf("reconnecting-%d", clusterIdx) + for p := range 10 { + if ctx.Err() != nil { + errs[clusterIdx] = ctx.Err() + return + } + profile := makeConvoyProfile(clusterID, fmt.Sprintf("profile-%d", p), ruleNames) + if err := mgr.AddProfile(profile); err != nil { + errs[clusterIdx] = err + return + } + } + }(c) + } + wg.Wait() + elapsed := time.Since(start) + t.Logf("20 clusters × 10 profiles completed in %v", elapsed) + + for i, err := range errs { + assert.NoError(t, err, "sensor cluster %d should not error", i) + } + assert.Less(t, elapsed, deadline, "should complete well within deadline") +} + +// BenchmarkAddProfileConvoy measures concurrent AddProfile throughput. +func BenchmarkAddProfileConvoy(b *testing.B) { + for _, numClusters := range []int{5, 10, 20} { + b.Run(fmt.Sprintf("clusters_%d", numClusters), func(b *testing.B) { + mgr := newManagerTB(b) + + ruleNames := []string{"rule-0", "rule-1", "rule-2"} + for _, name := range ruleNames { + require.NoError(b, mgr.AddRule(makeConvoyRule("seed", name))) + } + + seedProfiles(b, mgr, 5, 10, ruleNames) + + profilesPerCluster := 10 + + b.ResetTimer() + for b.Loop() { + var wg sync.WaitGroup + wg.Add(numClusters) + for c := range numClusters { + go func(clusterIdx int) { + defer wg.Done() + clusterID := fmt.Sprintf("reconnecting-%d", clusterIdx) + for p := range profilesPerCluster { + profile := makeConvoyProfile(clusterID, fmt.Sprintf("profile-%d", p), ruleNames) + _ = mgr.AddProfile(profile) + } + }(c) + } + wg.Wait() + } + }) + } +} diff --git a/central/complianceoperator/manager/manager.go b/central/complianceoperator/manager/manager.go index a5ab54f0dd9cd..5d347b853e887 100644 --- a/central/complianceoperator/manager/manager.go +++ b/central/complianceoperator/manager/manager.go @@ -80,13 +80,10 @@ func NewManager(registry *standards.Registry, profiles profileDatastore.DataStor rules: rules, results: results, } - // Postgres retries in addProfileNoLock(...) - err := profiles.Walk(allAccessCtx, func(profile *storage.ComplianceOperatorProfile) error { - return mgr.addProfileNoLock(profile) - }) - if err != nil { - return nil, err - } + // The registry starts empty and is populated as sensors reconnect and push + // profiles through AddProfile. Sensors always re-send all compliance data + // on reconnect (compliance types skip deduping), so the registry will be + // fully populated once all sensors have reconnected. return mgr, nil } diff --git a/central/complianceoperator/pipelines/complianceoperatorprofiles/pipeline.go b/central/complianceoperator/pipelines/complianceoperatorprofiles/pipeline.go index 4c6e03a25ad86..0940d35b2a8b8 100644 --- a/central/complianceoperator/pipelines/complianceoperatorprofiles/pipeline.go +++ b/central/complianceoperator/pipelines/complianceoperatorprofiles/pipeline.go @@ -3,6 +3,7 @@ package complianceoperatorprofiles import ( "context" + "github.com/pkg/errors" "github.com/stackrox/rox/central/complianceoperator/manager" "github.com/stackrox/rox/central/complianceoperator/profiles/datastore" countMetrics "github.com/stackrox/rox/central/metrics" @@ -12,12 +13,17 @@ import ( "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/centralsensor" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/metrics" "github.com/stackrox/rox/pkg/postgres/pgutils" "github.com/stackrox/rox/pkg/set" + "golang.org/x/sync/semaphore" ) var ( + log = logging.LoggerForModule() + _ pipeline.Fragment = (*pipelineImpl)(nil) ) @@ -28,15 +34,18 @@ func GetPipeline() pipeline.Fragment { // NewPipeline returns a new instance of Pipeline. func NewPipeline(datastore datastore.DataStore, manager manager.Manager) pipeline.Fragment { + maxConcurrency := int64(env.ComplianceV1MaxConcurrency.IntegerSetting()) return &pipelineImpl{ datastore: datastore, manager: manager, + semaphore: semaphore.NewWeighted(maxConcurrency), } } type pipelineImpl struct { datastore datastore.DataStore manager manager.Manager + semaphore *semaphore.Weighted } func (s *pipelineImpl) Capabilities() []centralsensor.CentralCapability { @@ -69,9 +78,14 @@ func (s *pipelineImpl) Match(msg *central.MsgFromSensor) bool { } // Run runs the pipeline template on the input and returns the output. -func (s *pipelineImpl) Run(_ context.Context, clusterID string, msg *central.MsgFromSensor, _ common.MessageInjector) error { +func (s *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.MsgFromSensor, _ common.MessageInjector) error { defer countMetrics.IncrementResourceProcessedCounter(pipeline.ActionToOperation(msg.GetEvent().GetAction()), metrics.ComplianceOperatorProfile) + if err := s.acquireSemaphore(ctx); err != nil { + return err + } + defer s.semaphore.Release(1) + event := msg.GetEvent() profile := event.GetComplianceOperatorProfile() profile.ClusterId = clusterID @@ -84,4 +98,27 @@ func (s *pipelineImpl) Run(_ context.Context, clusterID string, msg *central.Msg } } +func (s *pipelineImpl) acquireSemaphore(ctx context.Context) error { + waitTime := env.ComplianceV1SemaphoreWaitTime.DurationSetting() + + acquireCtx := ctx + if waitTime > 0 { + var cancel context.CancelFunc + acquireCtx, cancel = context.WithTimeout(ctx, waitTime) + defer cancel() + } + + if err := s.semaphore.Acquire(acquireCtx, 1); err != nil { + if ctx.Err() != nil { + log.Debugf("Unable to acquire semaphore for compliance profile update: %v", err) + } else if errors.Is(err, context.DeadlineExceeded) { + log.Warnf("Timed out waiting to process compliance profile (waited %v): %v", waitTime, err) + } else { + log.Errorf("Unexpected error acquiring compliance profile semaphore: %v", err) + } + return err + } + return nil +} + func (s *pipelineImpl) OnFinish(_ string) {} diff --git a/central/complianceoperator/pipelines/complianceoperatorprofiles/pipeline_test.go b/central/complianceoperator/pipelines/complianceoperatorprofiles/pipeline_test.go index f5cbd4f83f0f9..fe60ef6a4d5ed 100644 --- a/central/complianceoperator/pipelines/complianceoperatorprofiles/pipeline_test.go +++ b/central/complianceoperator/pipelines/complianceoperatorprofiles/pipeline_test.go @@ -2,17 +2,24 @@ package complianceoperatorprofiles import ( "context" + "sync/atomic" "testing" + "time" managerMocks "github.com/stackrox/rox/central/complianceoperator/manager/mocks" v1ProfileMocks "github.com/stackrox/rox/central/complianceoperator/profiles/datastore/mocks" "github.com/stackrox/rox/central/convert/testutils" "github.com/stackrox/rox/central/sensor/service/pipeline/reconciliation" "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" + "golang.org/x/sync/semaphore" ) func TestPipeline(t *testing.T) { @@ -101,3 +108,108 @@ func (s *PipelineTestSuite) TestRunReconcileNoOp() { err := pipeline.Reconcile(ctx, fixtureconsts.Cluster1, reconciliation.NewStoreMap()) s.NoError(err) } + +func TestThrottlesConcurrency(t *testing.T) { + ctrl := gomock.NewController(t) + mgr := managerMocks.NewMockManager(ctrl) + ds := v1ProfileMocks.NewMockDataStore(ctrl) + + const maxConcurrency = 3 + p := &pipelineImpl{ + datastore: ds, + manager: mgr, + semaphore: semaphore.NewWeighted(maxConcurrency), + } + + var currentConcurrency atomic.Int32 + var peakConcurrency atomic.Int32 + var completed atomic.Int32 + + const numProfiles = 20 + + for range numProfiles { + profile := &storage.ComplianceOperatorProfile{ + Id: uuid.NewV4().String(), + Name: "profile", + } + mgr.EXPECT().AddProfile(gomock.Any()).Do(func(_ *storage.ComplianceOperatorProfile) { + cur := currentConcurrency.Add(1) + for { + peak := peakConcurrency.Load() + if cur <= peak { + break + } + if peakConcurrency.CompareAndSwap(peak, cur) { + break + } + } + time.Sleep(50 * time.Millisecond) + currentConcurrency.Add(-1) + completed.Add(1) + }).Return(nil) + + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Id: profile.GetId(), + Action: central.ResourceAction_CREATE_RESOURCE, + Resource: ¢ral.SensorEvent_ComplianceOperatorProfile{ + ComplianceOperatorProfile: profile, + }, + }, + }, + } + + go func() { + err := p.Run(context.Background(), "cluster-1", msg, nil) + assert.NoError(t, err) + }() + } + + require.Eventually(t, func() bool { + return completed.Load() == numProfiles + }, 10*time.Second, 10*time.Millisecond) + + assert.LessOrEqual(t, peakConcurrency.Load(), int32(maxConcurrency), + "peak concurrency (%d) should not exceed the semaphore limit (%d)", + peakConcurrency.Load(), maxConcurrency) + t.Logf("Peak concurrency: %d (limit: %d)", peakConcurrency.Load(), maxConcurrency) +} + +func TestRespectsContextCancellation(t *testing.T) { + ctrl := gomock.NewController(t) + mgr := managerMocks.NewMockManager(ctrl) + ds := v1ProfileMocks.NewMockDataStore(ctrl) + + sem := semaphore.NewWeighted(1) + require.NoError(t, sem.Acquire(context.Background(), 1)) + + p := &pipelineImpl{ + datastore: ds, + manager: mgr, + semaphore: sem, + } + + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Id: uuid.NewV4().String(), + Action: central.ResourceAction_CREATE_RESOURCE, + Resource: ¢ral.SensorEvent_ComplianceOperatorProfile{ + ComplianceOperatorProfile: &storage.ComplianceOperatorProfile{ + Id: uuid.NewV4().String(), + Name: "profile", + }, + }, + }, + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := p.Run(ctx, "cluster-1", msg, nil) + assert.Error(t, err, "Run should return an error when context is cancelled") + + sem.Release(1) +} diff --git a/central/complianceoperator/pipelines/complianceoperatorrules/pipeline.go b/central/complianceoperator/pipelines/complianceoperatorrules/pipeline.go index 848e5200a9179..c40f45fcadae7 100644 --- a/central/complianceoperator/pipelines/complianceoperatorrules/pipeline.go +++ b/central/complianceoperator/pipelines/complianceoperatorrules/pipeline.go @@ -14,12 +14,17 @@ import ( "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/centralsensor" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/metrics" "github.com/stackrox/rox/pkg/postgres/pgutils" "github.com/stackrox/rox/pkg/set" + "golang.org/x/sync/semaphore" ) var ( + log = logging.LoggerForModule() + _ pipeline.Fragment = (*pipelineImpl)(nil) ) @@ -30,15 +35,18 @@ func GetPipeline() pipeline.Fragment { // NewPipeline returns a new instance of Pipeline. func NewPipeline(datastore datastore.DataStore, manager manager.Manager) pipeline.Fragment { + maxConcurrency := int64(env.ComplianceV1MaxConcurrency.IntegerSetting()) return &pipelineImpl{ datastore: datastore, manager: manager, + semaphore: semaphore.NewWeighted(maxConcurrency), } } type pipelineImpl struct { datastore datastore.DataStore manager manager.Manager + semaphore *semaphore.Weighted } func (s *pipelineImpl) Capabilities() []centralsensor.CentralCapability { @@ -70,9 +78,14 @@ func (s *pipelineImpl) Match(msg *central.MsgFromSensor) bool { } // Run runs the pipeline template on the input and returns the output. -func (s *pipelineImpl) Run(_ context.Context, clusterID string, msg *central.MsgFromSensor, _ common.MessageInjector) error { +func (s *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.MsgFromSensor, _ common.MessageInjector) error { defer countMetrics.IncrementResourceProcessedCounter(pipeline.ActionToOperation(msg.GetEvent().GetAction()), metrics.ComplianceOperatorRule) + if err := s.acquireSemaphore(ctx); err != nil { + return err + } + defer s.semaphore.Release(1) + event := msg.GetEvent() rule := event.GetComplianceOperatorRule() rule.ClusterId = clusterID @@ -89,4 +102,27 @@ func (s *pipelineImpl) Run(_ context.Context, clusterID string, msg *central.Msg } } +func (s *pipelineImpl) acquireSemaphore(ctx context.Context) error { + waitTime := env.ComplianceV1SemaphoreWaitTime.DurationSetting() + + acquireCtx := ctx + if waitTime > 0 { + var cancel context.CancelFunc + acquireCtx, cancel = context.WithTimeout(ctx, waitTime) + defer cancel() + } + + if err := s.semaphore.Acquire(acquireCtx, 1); err != nil { + if ctx.Err() != nil { + log.Debugf("Unable to acquire semaphore for compliance rule update: %v", err) + } else if errors.Is(err, context.DeadlineExceeded) { + log.Warnf("Timed out waiting to process compliance rule (waited %v): %v", waitTime, err) + } else { + log.Errorf("Unexpected error acquiring compliance rule semaphore: %v", err) + } + return err + } + return nil +} + func (s *pipelineImpl) OnFinish(_ string) {} diff --git a/central/complianceoperator/pipelines/complianceoperatorrules/pipeline_test.go b/central/complianceoperator/pipelines/complianceoperatorrules/pipeline_test.go new file mode 100644 index 0000000000000..b882ea119ba2d --- /dev/null +++ b/central/complianceoperator/pipelines/complianceoperatorrules/pipeline_test.go @@ -0,0 +1,124 @@ +package complianceoperatorrules + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/ComplianceAsCode/compliance-operator/pkg/apis/compliance/v1alpha1" + managerMocks "github.com/stackrox/rox/central/complianceoperator/manager/mocks" + "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "golang.org/x/sync/semaphore" +) + +func makeRuleMsg(ruleID, ruleName string) *central.MsgFromSensor { + return ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Id: ruleID, + Action: central.ResourceAction_CREATE_RESOURCE, + Resource: ¢ral.SensorEvent_ComplianceOperatorRule{ + ComplianceOperatorRule: &storage.ComplianceOperatorRule{ + Id: ruleID, + Name: ruleName, + Annotations: map[string]string{ + v1alpha1.RuleIDAnnotationKey: ruleName, + }, + }, + }, + }, + }, + } +} + +func TestRunCallsAddRule(t *testing.T) { + ctrl := gomock.NewController(t) + mgr := managerMocks.NewMockManager(ctrl) + p := &pipelineImpl{ + manager: mgr, + semaphore: semaphore.NewWeighted(5), + } + + mgr.EXPECT().AddRule(gomock.Any()).Return(nil) + + err := p.Run(context.Background(), "cluster-1", makeRuleMsg(uuid.NewV4().String(), "test-rule"), nil) + assert.NoError(t, err) +} + +func TestThrottlesConcurrency(t *testing.T) { + ctrl := gomock.NewController(t) + mgr := managerMocks.NewMockManager(ctrl) + + const maxConcurrency = 3 + p := &pipelineImpl{ + manager: mgr, + semaphore: semaphore.NewWeighted(maxConcurrency), + } + + var currentConcurrency atomic.Int32 + var peakConcurrency atomic.Int32 + var completed atomic.Int32 + + const numRules = 20 + + for range numRules { + mgr.EXPECT().AddRule(gomock.Any()).Do(func(_ *storage.ComplianceOperatorRule) { + cur := currentConcurrency.Add(1) + for { + peak := peakConcurrency.Load() + if cur <= peak { + break + } + if peakConcurrency.CompareAndSwap(peak, cur) { + break + } + } + time.Sleep(50 * time.Millisecond) + currentConcurrency.Add(-1) + completed.Add(1) + }).Return(nil) + + msg := makeRuleMsg(uuid.NewV4().String(), "test-rule") + + go func() { + err := p.Run(context.Background(), "cluster-1", msg, nil) + assert.NoError(t, err) + }() + } + + require.Eventually(t, func() bool { + return completed.Load() == numRules + }, 10*time.Second, 10*time.Millisecond) + + assert.LessOrEqual(t, peakConcurrency.Load(), int32(maxConcurrency), + "peak concurrency (%d) should not exceed the semaphore limit (%d)", + peakConcurrency.Load(), maxConcurrency) + t.Logf("Peak concurrency: %d (limit: %d)", peakConcurrency.Load(), maxConcurrency) +} + +func TestRespectsContextCancellation(t *testing.T) { + ctrl := gomock.NewController(t) + mgr := managerMocks.NewMockManager(ctrl) + + sem := semaphore.NewWeighted(1) + require.NoError(t, sem.Acquire(context.Background(), 1)) + + p := &pipelineImpl{ + manager: mgr, + semaphore: sem, + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := p.Run(ctx, "cluster-1", makeRuleMsg(uuid.NewV4().String(), "test-rule"), nil) + assert.Error(t, err, "Run should return an error when context is cancelled") + + sem.Release(1) +} diff --git a/central/complianceoperator/v2/benchmark/benchmark.go b/central/complianceoperator/v2/benchmark/benchmark.go index 28c677d0f6d84..53c3980bea2f1 100644 --- a/central/complianceoperator/v2/benchmark/benchmark.go +++ b/central/complianceoperator/v2/benchmark/benchmark.go @@ -22,11 +22,11 @@ var ( regexp.MustCompile(".+-nerc-cip$|.+-nerc-cip-.+"): {shortName: "NERC-CIP", provider: "NERC"}, regexp.MustCompile(".+-pci-dss$|.+-pci-dss-.+"): {shortName: "PCI-DSS", provider: "PCI"}, - regexp.MustCompile("^ocp4-cis$|^ocp4-cis-.+"): {shortName: "CIS-OCP", provider: "CIS"}, - regexp.MustCompile("^ocp4-high$|^ocp4-high-.+"): {shortName: "NIST-800-53", provider: "NIST"}, - regexp.MustCompile("^ocp4-moderate$|^ocp4-moderate-.+"): {shortName: "NIST-800-53", provider: "NIST"}, - regexp.MustCompile("^rhcos4-high$|^rhcos4-high-.+"): {shortName: "NIST-800-53", provider: "NIST"}, - regexp.MustCompile("^rhcos4-moderate$|^rhcos4-moderate-.+"): {shortName: "NIST-800-53", provider: "NIST"}, + regexp.MustCompile("^ocp[45]-cis(-.+)?$"): {shortName: "CIS-OCP", provider: "CIS"}, + regexp.MustCompile("^ocp[45]-high(-.+)?$"): {shortName: "NIST-800-53", provider: "NIST"}, + regexp.MustCompile("^ocp[45]-moderate(-.+)?$"): {shortName: "NIST-800-53", provider: "NIST"}, + regexp.MustCompile("^rhcos[45]-high(-.+)?$"): {shortName: "NIST-800-53", provider: "NIST"}, + regexp.MustCompile("^rhcos[45]-moderate(-.+)?$"): {shortName: "NIST-800-53", provider: "NIST"}, } log = logging.LoggerForModule() diff --git a/central/complianceoperator/v2/benchmark/fixtures/bsi-benchmark-2022.json b/central/complianceoperator/v2/benchmark/fixtures/bsi-benchmark-2022.json index 5351530aa4bfe..02cb0df653f4b 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/bsi-benchmark-2022.json +++ b/central/complianceoperator/v2/benchmark/fixtures/bsi-benchmark-2022.json @@ -1,34 +1,19 @@ { - "id": "29a99c1e-7d12-416b-8055-78a04e2eac7c", "name": "Bundesamt für Sicherheit in der Informationstechnik", - "version": "2022", - "description": "", "provider": "Germany’s Federal Office for Information Security", "shortName": "BSI", "profiles": [ - { - "profileName": "ocp4-bsi", - "profileVersion": "2022" - }, - { - "profileName": "ocp4-bsi-node", - "profileVersion": "2022" - }, - { - "profileName": "ocp4-bsi-2022", - "profileVersion": "2022" - }, - { - "profileName": "ocp4-bsi-node-2022", - "profileVersion": "2022" - }, - { - "profileName": "rhcos4-bsi", - "profileVersion": "2022" - }, - { - "profileName": "rhcos4-bsi-2022", - "profileVersion": "2022" - } + { "profileName": "ocp4-bsi" }, + { "profileName": "ocp4-bsi-node" }, + { "profileName": "ocp4-bsi-2022" }, + { "profileName": "ocp4-bsi-node-2022" }, + { "profileName": "rhcos4-bsi" }, + { "profileName": "rhcos4-bsi-2022" }, + { "profileName": "ocp5-bsi" }, + { "profileName": "ocp5-bsi-node" }, + { "profileName": "ocp5-bsi-2022" }, + { "profileName": "ocp5-bsi-node-2022" }, + { "profileName": "rhcos5-bsi" }, + { "profileName": "rhcos5-bsi-2022" } ] } diff --git a/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-4.json b/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-4.json index 1223661881805..c2eb1d26c2b20 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-4.json +++ b/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-4.json @@ -1,18 +1,11 @@ { - "id": "93946d52-61e6-46dc-9071-39ebf07d0cd8", "name": "CIS Red Hat OpenShift Container Platform Benchmark", - "version": "v1.4.0", - "description": "", "provider": "CIS", "shortName": "CIS-OCP", "profiles": [ - { - "profileName": "ocp4-cis-1-4", - "profileVersion": "1.4.0" - }, - { - "profileName": "ocp4-cis-node-1-4", - "profileVersion": "1.4.0" - } + { "profileName": "ocp4-cis-1-4" }, + { "profileName": "ocp4-cis-node-1-4" }, + { "profileName": "ocp5-cis-1-4" }, + { "profileName": "ocp5-cis-node-1-4" } ] } diff --git a/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-5.json b/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-5.json index ca4252836530b..1b26b8c2cfe5d 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-5.json +++ b/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-5.json @@ -1,18 +1,11 @@ { - "id": "4d0d96f7-3782-41ba-b3b7-c040686a421c", "name": "CIS Red Hat OpenShift Container Platform Benchmark", - "version": "v1.5.0", - "description": "", "provider": "CIS", "shortName": "CIS-OCP", "profiles": [ - { - "profileName": "ocp4-cis-1-5", - "profileVersion": "1.5.0" - }, - { - "profileName": "ocp4-cis-node-1-5", - "profileVersion": "1.5.0" - } + { "profileName": "ocp4-cis-1-5" }, + { "profileName": "ocp4-cis-node-1-5" }, + { "profileName": "ocp5-cis-1-5" }, + { "profileName": "ocp5-cis-node-1-5" } ] -} \ No newline at end of file +} diff --git a/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-7.json b/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-7.json index aeb39eb89ca0d..2a7f7909aa843 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-7.json +++ b/central/complianceoperator/v2/benchmark/fixtures/cis-benchmark-1-7.json @@ -1,26 +1,15 @@ { - "id": "58f60633-4f14-42cf-afe0-1cfbc6329466", "name": "CIS Red Hat OpenShift Container Platform Benchmark", - "version": "v1.7.0", - "description": "", "provider": "CIS", "shortName": "CIS-OCP", "profiles": [ - { - "profileName": "ocp4-cis", - "profileVersion": "1.7.0" - }, - { - "profileName": "ocp4-cis-node", - "profileVersion": "1.7.0" - }, - { - "profileName": "ocp4-cis-1-7", - "profileVersion": "1.7.0" - }, - { - "profileName": "ocp4-cis-node-1-7", - "profileVersion": "1.7.0" - } + { "profileName": "ocp4-cis" }, + { "profileName": "ocp4-cis-node" }, + { "profileName": "ocp4-cis-1-7" }, + { "profileName": "ocp4-cis-node-1-7" }, + { "profileName": "ocp5-cis" }, + { "profileName": "ocp5-cis-node" }, + { "profileName": "ocp5-cis-1-7" }, + { "profileName": "ocp5-cis-node-1-7" } ] } diff --git a/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v1r1.json b/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v1r1.json index 79046413ef6a0..dbdea4bdf9cd3 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v1r1.json +++ b/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v1r1.json @@ -1,22 +1,13 @@ { - "id": "4fdab844-1464-4bc3-8d8f-026e0c2821cf", "name": "Defense Information Systems Agency Security Technical Implementation Guide", - "version": "V1R1", - "description": "", "provider": "DISA", "shortName": "STIG", "profiles": [ - { - "profileName": "ocp4-stig-v1r1", - "profileVersion": "V1R1" - }, - { - "profileName": "ocp4-stig-node-v1r1", - "profileVersion": "V1R1" - }, - { - "profileName": "rhcos4-stig-v1r1", - "profileVersion": "V1R1" - } + { "profileName": "ocp4-stig-v1r1" }, + { "profileName": "ocp4-stig-node-v1r1" }, + { "profileName": "rhcos4-stig-v1r1" }, + { "profileName": "ocp5-stig-v1r1" }, + { "profileName": "ocp5-stig-node-v1r1" }, + { "profileName": "rhcos5-stig-v1r1" } ] -} \ No newline at end of file +} diff --git a/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r1.json b/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r1.json index ced904c661493..1e766128d4e01 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r1.json +++ b/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r1.json @@ -1,22 +1,13 @@ { - "id": "65490c0b-2db8-4613-a8a3-a59df8c6a561", "name": "Defense Information Systems Agency Security Technical Implementation Guide", - "version": "V2R1", - "description": "", "provider": "DISA", "shortName": "STIG", "profiles": [ - { - "profileName": "ocp4-stig-v2r1", - "profileVersion": "V2R1" - }, - { - "profileName": "ocp4-stig-node-v2r1", - "profileVersion": "V2R1" - }, - { - "profileName": "rhcos4-stig-v2r1", - "profileVersion": "V2R1" - } + { "profileName": "ocp4-stig-v2r1" }, + { "profileName": "ocp4-stig-node-v2r1" }, + { "profileName": "rhcos4-stig-v2r1" }, + { "profileName": "ocp5-stig-v2r1" }, + { "profileName": "ocp5-stig-node-v2r1" }, + { "profileName": "rhcos5-stig-v2r1" } ] } diff --git a/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r2.json b/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r2.json index d5bd0e568fdf1..3905957e625bb 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r2.json +++ b/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r2.json @@ -1,22 +1,13 @@ { - "id": "411f67c1-db12-43f7-b408-3480b810ce1e", "name": "Defense Information Systems Agency Security Technical Implementation Guide", - "version": "V2R2", - "description": "", "provider": "DISA", "shortName": "STIG", "profiles": [ - { - "profileName": "ocp4-stig-v2r2", - "profileVersion": "V2R2" - }, - { - "profileName": "ocp4-stig-node-v2r2", - "profileVersion": "V2R2" - }, - { - "profileName": "rhcos4-stig-v2r2", - "profileVersion": "V2R2" - } + { "profileName": "ocp4-stig-v2r2" }, + { "profileName": "ocp4-stig-node-v2r2" }, + { "profileName": "rhcos4-stig-v2r2" }, + { "profileName": "ocp5-stig-v2r2" }, + { "profileName": "ocp5-stig-node-v2r2" }, + { "profileName": "rhcos5-stig-v2r2" } ] } diff --git a/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r3.json b/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r3.json index 85a89d4edfe8d..9d84177943bfc 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r3.json +++ b/central/complianceoperator/v2/benchmark/fixtures/disa-stig-benchmark-v2r3.json @@ -1,34 +1,19 @@ { - "id": "aaedc84e-c526-4b06-bbba-d0612d509b03", "name": "Defense Information Systems Agency Security Technical Implementation Guide", - "version": "V2R3", - "description": "", "provider": "DISA", "shortName": "STIG", "profiles": [ - { - "profileName": "ocp4-stig", - "profileVersion": "V2R3" - }, - { - "profileName": "ocp4-stig-node", - "profileVersion": "V2R3" - }, - { - "profileName": "rhcos4-stig", - "profileVersion": "V2R3" - }, - { - "profileName": "ocp4-stig-v2r3", - "profileVersion": "V2R3" - }, - { - "profileName": "ocp4-stig-node-v2r3", - "profileVersion": "V2R3" - }, - { - "profileName": "rhcos4-stig-v2r3", - "profileVersion": "V2R3" - } + { "profileName": "ocp4-stig" }, + { "profileName": "ocp4-stig-node" }, + { "profileName": "rhcos4-stig" }, + { "profileName": "ocp4-stig-v2r3" }, + { "profileName": "ocp4-stig-node-v2r3" }, + { "profileName": "rhcos4-stig-v2r3" }, + { "profileName": "ocp5-stig" }, + { "profileName": "ocp5-stig-node" }, + { "profileName": "rhcos5-stig" }, + { "profileName": "ocp5-stig-v2r3" }, + { "profileName": "ocp5-stig-node-v2r3" }, + { "profileName": "rhcos5-stig-v2r3" } ] } diff --git a/central/complianceoperator/v2/benchmark/fixtures/e8-benchmark.json b/central/complianceoperator/v2/benchmark/fixtures/e8-benchmark.json index e45518bdbd084..08dea2d91bc10 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/e8-benchmark.json +++ b/central/complianceoperator/v2/benchmark/fixtures/e8-benchmark.json @@ -1,18 +1,11 @@ { - "id": "dcd2955f-8337-4f02-b713-60f1237229f0", "name": "Australian Cyber Security Centre (ACSC) Essential Eight", - "version": "", - "description": "", "provider": "ACSC", "shortName": "E8", "profiles": [ - { - "profileName": "ocp4-e8", - "profileVersion": "latest" - }, - { - "profileName": "rhcos4-e8", - "profileVersion": "latest" - } + { "profileName": "ocp4-e8" }, + { "profileName": "rhcos4-e8" }, + { "profileName": "ocp5-e8" }, + { "profileName": "rhcos5-e8" } ] -} \ No newline at end of file +} diff --git a/central/complianceoperator/v2/benchmark/fixtures/nerc-benchmark.json b/central/complianceoperator/v2/benchmark/fixtures/nerc-benchmark.json index ea50908bcbbea..902f626a95900 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/nerc-benchmark.json +++ b/central/complianceoperator/v2/benchmark/fixtures/nerc-benchmark.json @@ -1,22 +1,13 @@ { - "id": "199c1777-bc5c-4f5e-bf1f-02afa8bd4f59", "name": "North American Electric Reliability Corporation (NERC) Critical Infrastructure Protection (CIP) cybersecurity standards profile", - "version": "", - "description": "", "provider": "NERC", "shortName": "NERC-CIP", "profiles": [ - { - "profileName": "ocp4-nerc-cip", - "profileVersion": "latest" - }, - { - "profileName": "ocp4-nerc-cip-node", - "profileVersion": "latest" - }, - { - "profileName": "rhcos4-nerc-cip", - "profileVersion": "latest" - } + { "profileName": "ocp4-nerc-cip" }, + { "profileName": "ocp4-nerc-cip-node" }, + { "profileName": "rhcos4-nerc-cip" }, + { "profileName": "ocp5-nerc-cip" }, + { "profileName": "ocp5-nerc-cip-node" }, + { "profileName": "rhcos5-nerc-cip" } ] -} \ No newline at end of file +} diff --git a/central/complianceoperator/v2/benchmark/fixtures/nist-high-benchmark-revision-4.json b/central/complianceoperator/v2/benchmark/fixtures/nist-high-benchmark-revision-4.json index 8fb691bde8f5e..e2fe05c6f5106 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/nist-high-benchmark-revision-4.json +++ b/central/complianceoperator/v2/benchmark/fixtures/nist-high-benchmark-revision-4.json @@ -1,34 +1,19 @@ { - "id": "f5859d89-1a5a-4876-9ddc-fbcc52b79fe5", "name": "NIST 800-53 High-Impact Baseline", - "version": "Revision 4", - "description": "", "provider": "NIST", "shortName": "NIST-800-53", "profiles": [ - { - "profileName": "ocp4-high", - "profileVersion": "Revision 4" - }, - { - "profileName": "ocp4-high-node", - "profileVersion": "Revision 4" - }, - { - "profileName": "rhcos4-high", - "profileVersion": "Revision 4" - }, - { - "profileName": "ocp4-high-rev-4", - "profileVersion": "Revision 4" - }, - { - "profileName": "ocp4-high-node-rev-4", - "profileVersion": "Revision 4" - }, - { - "profileName": "rhcos4-high-rev-4", - "profileVersion": "Revision 4" - } + { "profileName": "ocp4-high" }, + { "profileName": "ocp4-high-node" }, + { "profileName": "rhcos4-high" }, + { "profileName": "ocp4-high-rev-4" }, + { "profileName": "ocp4-high-node-rev-4" }, + { "profileName": "rhcos4-high-rev-4" }, + { "profileName": "ocp5-high" }, + { "profileName": "ocp5-high-node" }, + { "profileName": "rhcos5-high" }, + { "profileName": "ocp5-high-rev-4" }, + { "profileName": "ocp5-high-node-rev-4" }, + { "profileName": "rhcos5-high-rev-4" } ] -} \ No newline at end of file +} diff --git a/central/complianceoperator/v2/benchmark/fixtures/nist-moderate-benchmark-revision-4.json b/central/complianceoperator/v2/benchmark/fixtures/nist-moderate-benchmark-revision-4.json index e911d147b05b1..f0e58f494995c 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/nist-moderate-benchmark-revision-4.json +++ b/central/complianceoperator/v2/benchmark/fixtures/nist-moderate-benchmark-revision-4.json @@ -1,34 +1,19 @@ { - "id": "13d6b465-1467-4bfd-af0f-a3e2205048f1", "name": "NIST 800-53 Moderate-Impact Baseline", - "version": "Revision 4", - "description": "", "provider": "NIST", "shortName": "NIST-800-53", "profiles": [ - { - "profileName": "ocp4-moderate", - "profileVersion": "Revision 4" - }, - { - "profileName": "ocp4-moderate-node", - "profileVersion": "Revision 4" - }, - { - "profileName": "rhcos4-moderate", - "profileVersion": "Revision 4" - }, - { - "profileName": "ocp4-moderate-rev-4", - "profileVersion": "Revision 4" - }, - { - "profileName": "ocp4-moderate-node-rev-4", - "profileVersion": "Revision 4" - }, - { - "profileName": "rhcos4-moderate-rev-4", - "profileVersion": "Revision 4" - } + { "profileName": "ocp4-moderate" }, + { "profileName": "ocp4-moderate-node" }, + { "profileName": "rhcos4-moderate" }, + { "profileName": "ocp4-moderate-rev-4" }, + { "profileName": "ocp4-moderate-node-rev-4" }, + { "profileName": "rhcos4-moderate-rev-4" }, + { "profileName": "ocp5-moderate" }, + { "profileName": "ocp5-moderate-node" }, + { "profileName": "rhcos5-moderate" }, + { "profileName": "ocp5-moderate-rev-4" }, + { "profileName": "ocp5-moderate-node-rev-4" }, + { "profileName": "rhcos5-moderate-rev-4" } ] -} \ No newline at end of file +} diff --git a/central/complianceoperator/v2/benchmark/fixtures/pci-benchmark-3-2-1.json b/central/complianceoperator/v2/benchmark/fixtures/pci-benchmark-3-2-1.json index a55d38b890fa5..4627c8f76eece 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/pci-benchmark-3-2-1.json +++ b/central/complianceoperator/v2/benchmark/fixtures/pci-benchmark-3-2-1.json @@ -1,26 +1,15 @@ { - "id": "9f204799-c73d-46a1-8890-cd2ceec12d7e", "name": "PCI-DSS v3.2.1 Control Baseline", - "version": "3.2.1", - "description": "", "provider": "PCI", "shortName": "PCI-DSS", "profiles": [ - { - "profileName": "ocp4-pci-dss", - "profileVersion": "3.2.1" - }, - { - "profileName": "ocp4-pci-dss-node", - "profileVersion": "3.2.1" - }, - { - "profileName": "ocp4-pci-dss-3-2", - "profileVersion": "3.2.1" - }, - { - "profileName": "ocp4-pci-dss-node-3-2", - "profileVersion": "3.2.1" - } + { "profileName": "ocp4-pci-dss" }, + { "profileName": "ocp4-pci-dss-node" }, + { "profileName": "ocp4-pci-dss-3-2" }, + { "profileName": "ocp4-pci-dss-node-3-2" }, + { "profileName": "ocp5-pci-dss" }, + { "profileName": "ocp5-pci-dss-node" }, + { "profileName": "ocp5-pci-dss-3-2" }, + { "profileName": "ocp5-pci-dss-node-3-2" } ] -} \ No newline at end of file +} diff --git a/central/complianceoperator/v2/benchmark/fixtures/pci-benchmark-4-0.json b/central/complianceoperator/v2/benchmark/fixtures/pci-benchmark-4-0.json index 6270b80346ed0..2e8d506a5bc57 100644 --- a/central/complianceoperator/v2/benchmark/fixtures/pci-benchmark-4-0.json +++ b/central/complianceoperator/v2/benchmark/fixtures/pci-benchmark-4-0.json @@ -1,18 +1,11 @@ { - "id": "62fc8ca1-0cd2-4969-aae7-1e366ca5408e", "name": "PCI-DSS v4.0.0 Control Baseline", - "version": "4.0.0", - "description": "", "provider": "PCI", "shortName": "PCI-DSS", "profiles": [ - { - "profileName": "ocp4-pci-dss-4-0", - "profileVersion": "4.0.0" - }, - { - "profileName": "ocp4-pci-dss-node-4-0", - "profileVersion": "4.0.0" - } + { "profileName": "ocp4-pci-dss-4-0" }, + { "profileName": "ocp4-pci-dss-node-4-0" }, + { "profileName": "ocp5-pci-dss-4-0" }, + { "profileName": "ocp5-pci-dss-node-4-0" } ] } diff --git a/central/complianceoperator/v2/checkresults/datastore/datastore_impl.go b/central/complianceoperator/v2/checkresults/datastore/datastore_impl.go index ed2471e3ffd2d..12890e2a447b1 100644 --- a/central/complianceoperator/v2/checkresults/datastore/datastore_impl.go +++ b/central/complianceoperator/v2/checkresults/datastore/datastore_impl.go @@ -7,7 +7,6 @@ import ( "github.com/pkg/errors" store "github.com/stackrox/rox/central/complianceoperator/v2/checkresults/store/postgres" - complianceUtils "github.com/stackrox/rox/central/complianceoperator/v2/utils" "github.com/stackrox/rox/central/metrics" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" @@ -19,7 +18,6 @@ import ( "github.com/stackrox/rox/pkg/search" pgSearch "github.com/stackrox/rox/pkg/search/postgres" "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" - "github.com/stackrox/rox/pkg/utils" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -77,12 +75,6 @@ func (d *datastoreImpl) GetComplianceCheckResult(ctx context.Context, compliance func (d *datastoreImpl) ComplianceCheckResultStats(ctx context.Context, query *v1.Query) ([]*ResourceResultCountByClusterScan, error) { defer metrics.SetDatastoreFunctionDuration(time.Now(), "ComplianceOperatorCheckResultV2", "ComplianceCheckResultStats") - var err error - query, err = complianceUtils.WithSACFilter(ctx, resources.Compliance, query) - if err != nil { - return nil, err - } - cloned := query.CloneVT() cloned.Selects = []*v1.QuerySelect{ search.NewQuerySelect(search.ClusterID).Proto(), @@ -115,7 +107,11 @@ func (d *datastoreImpl) ComplianceCheckResultStats(ctx context.Context, query *v } countQuery := d.withCountByResultSelectQuery(cloned, search.ClusterID) - countResults, err := pgSearch.RunSelectRequestForSchema[ResourceResultCountByClusterScan](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, countQuery) + var countResults []*ResourceResultCountByClusterScan + err := pgSearch.RunSelectRequestForSchemaFn[ResourceResultCountByClusterScan](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, countQuery, func(r *ResourceResultCountByClusterScan) error { + countResults = append(countResults, r) + return nil + }) if err != nil { return nil, err } @@ -127,12 +123,6 @@ func (d *datastoreImpl) ComplianceCheckResultStats(ctx context.Context, query *v func (d *datastoreImpl) ComplianceProfileResultStats(ctx context.Context, query *v1.Query) ([]*ResourceResultCountByProfile, error) { defer metrics.SetDatastoreFunctionDuration(time.Now(), "ComplianceOperatorCheckResultV2", "ComplianceProfileResultStats") - var err error - query, err = complianceUtils.WithSACFilter(ctx, resources.Compliance, query) - if err != nil { - return nil, err - } - cloned := query.CloneVT() cloned.Selects = []*v1.QuerySelect{ search.NewQuerySelect(search.ComplianceOperatorProfileName).Proto(), @@ -155,7 +145,11 @@ func (d *datastoreImpl) ComplianceProfileResultStats(ctx context.Context, query } countQuery := d.withCountByResultSelectQuery(cloned, search.ComplianceOperatorProfileName) - countResults, err := pgSearch.RunSelectRequestForSchema[ResourceResultCountByProfile](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, countQuery) + var countResults []*ResourceResultCountByProfile + err := pgSearch.RunSelectRequestForSchemaFn[ResourceResultCountByProfile](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, countQuery, func(r *ResourceResultCountByProfile) error { + countResults = append(countResults, r) + return nil + }) if err != nil { return nil, err } @@ -167,12 +161,6 @@ func (d *datastoreImpl) ComplianceProfileResultStats(ctx context.Context, query func (d *datastoreImpl) ComplianceProfileResults(ctx context.Context, query *v1.Query) ([]*ResourceResultsByProfile, error) { defer metrics.SetDatastoreFunctionDuration(time.Now(), "ComplianceOperatorCheckResultV2", "ComplianceProfileResultStats") - var err error - query, err = complianceUtils.WithSACFilter(ctx, resources.Compliance, query) - if err != nil { - return nil, err - } - cloned := query.CloneVT() cloned.Selects = []*v1.QuerySelect{ search.NewQuerySelect(search.ComplianceOperatorProfileName).Proto(), @@ -210,7 +198,11 @@ func (d *datastoreImpl) ComplianceProfileResults(ctx context.Context, query *v1. } countQuery := d.withCountByResultSelectQuery(cloned, search.ComplianceOperatorProfileName) - results, err := pgSearch.RunSelectRequestForSchema[ResourceResultsByProfile](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, countQuery) + var results []*ResourceResultsByProfile + err := pgSearch.RunSelectRequestForSchemaFn[ResourceResultsByProfile](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, countQuery, func(r *ResourceResultsByProfile) error { + results = append(results, r) + return nil + }) if err != nil { return nil, err } @@ -222,12 +214,6 @@ func (d *datastoreImpl) ComplianceProfileResults(ctx context.Context, query *v1. func (d *datastoreImpl) ComplianceClusterStats(ctx context.Context, query *v1.Query) ([]*ResultStatusCountByCluster, error) { defer metrics.SetDatastoreFunctionDuration(time.Now(), "ComplianceOperatorCheckResultV2", "ComplianceClusterStats") - var err error - query, err = complianceUtils.WithSACFilter(ctx, resources.Compliance, query) - if err != nil { - return nil, err - } - cloned := query.CloneVT() cloned.Selects = []*v1.QuerySelect{ search.NewQuerySelect(search.ClusterID).Proto(), @@ -257,7 +243,11 @@ func (d *datastoreImpl) ComplianceClusterStats(ctx context.Context, query *v1.Qu } countQuery := d.withCountByResultSelectQuery(cloned, search.ClusterID) - countResults, err := pgSearch.RunSelectRequestForSchema[ResultStatusCountByCluster](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, countQuery) + var countResults []*ResultStatusCountByCluster + err := pgSearch.RunSelectRequestForSchemaFn[ResultStatusCountByCluster](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, countQuery, func(r *ResultStatusCountByCluster) error { + countResults = append(countResults, r) + return nil + }) if err != nil { return nil, errors.Wrap(err, "unable to retrieve data") } @@ -267,11 +257,6 @@ func (d *datastoreImpl) ComplianceClusterStats(ctx context.Context, query *v1.Qu // CountByField retrieves the distinct scan result counts specified by query based on specified search field func (d *datastoreImpl) CountByField(ctx context.Context, query *v1.Query, field search.FieldLabel) (int, error) { - var err error - query, err = complianceUtils.WithSACFilter(ctx, resources.Compliance, query) - if err != nil { - return 0, err - } switch field { case search.ClusterID: @@ -295,71 +280,19 @@ func (d *datastoreImpl) WalkByQuery(ctx context.Context, query *v1.Query, fn fun } func (d *datastoreImpl) countByCluster(ctx context.Context, query *v1.Query, field search.FieldLabel) (int, error) { - var results []*clusterCount - results, err := pgSearch.RunSelectRequestForSchema[clusterCount](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, withCountQuery(query, search.ClusterID)) - if err != nil { - return 0, err - } - if len(results) == 0 { - return 0, nil - } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", query.String()) - utils.Should(err) - return 0, err - } - return results[0].TotalCount, nil + return pgSearch.RunDistinctCountForSchema(ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, query, search.ClusterID) } func (d *datastoreImpl) countByProfile(ctx context.Context, query *v1.Query) (int, error) { - var results []*profileCount - results, err := pgSearch.RunSelectRequestForSchema[profileCount](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, withCountQuery(query, search.ComplianceOperatorProfileName)) - if err != nil { - return 0, err - } - if len(results) == 0 { - return 0, nil - } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", query.String()) - utils.Should(err) - return 0, err - } - return results[0].TotalCount, nil + return pgSearch.RunDistinctCountForSchema(ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, query, search.ComplianceOperatorProfileName) } func (d *datastoreImpl) countByCheck(ctx context.Context, query *v1.Query) (int, error) { - var results []*complianceCheckCount - results, err := pgSearch.RunSelectRequestForSchema[complianceCheckCount](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, withCountQuery(query, search.ComplianceOperatorCheckName)) - if err != nil { - return 0, err - } - if len(results) == 0 { - return 0, nil - } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", query.String()) - utils.Should(err) - return 0, err - } - return results[0].TotalCount, nil + return pgSearch.RunDistinctCountForSchema(ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, query, search.ComplianceOperatorCheckName) } func (d *datastoreImpl) countByConfiguration(ctx context.Context, query *v1.Query) (int, error) { - var results []*configurationCount - results, err := pgSearch.RunSelectRequestForSchema[configurationCount](ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, withCountQuery(query, search.ComplianceOperatorScanConfigName)) - if err != nil { - return 0, err - } - if len(results) == 0 { - return 0, nil - } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", query.String()) - utils.Should(err) - return 0, err - } - return results[0].TotalCount, nil + return pgSearch.RunDistinctCountForSchema(ctx, d.db, schema.ComplianceOperatorCheckResultV2Schema, query, search.ComplianceOperatorScanConfigName) } func (d *datastoreImpl) CountCheckResults(ctx context.Context, q *v1.Query) (int, error) { @@ -389,14 +322,6 @@ func (d *datastoreImpl) DeleteResultsByScans(ctx context.Context, scanRefIds []s return d.store.DeleteByQuery(ctx, query) } -func withCountQuery(q *v1.Query, field search.FieldLabel) *v1.Query { - cloned := q.CloneVT() - cloned.Selects = []*v1.QuerySelect{ - search.NewQuerySelect(field).AggrFunc(aggregatefunc.Count).Distinct().Proto(), - } - return cloned -} - func (d *datastoreImpl) withCountByResultSelectQuery(q *v1.Query, countOn search.FieldLabel) *v1.Query { cloned := q.CloneVT() cloned.Selects = append(cloned.Selects, diff --git a/central/complianceoperator/v2/checkresults/datastore/datastore_impl_benchmark_test.go b/central/complianceoperator/v2/checkresults/datastore/datastore_impl_benchmark_test.go index be2d6fea6d038..98fdbd0471b5d 100644 --- a/central/complianceoperator/v2/checkresults/datastore/datastore_impl_benchmark_test.go +++ b/central/complianceoperator/v2/checkresults/datastore/datastore_impl_benchmark_test.go @@ -49,7 +49,7 @@ func cleanupDatabase(b *testing.B, pool *pgtest.TestPostgres) { func setupTest(b *testing.B, pool *pgtest.TestPostgres, datastore DataStore, numConfigs int, numClusters int, numResults int) { scanConfigs := make([]string, 0, numConfigs) - for i := 0; i < numConfigs; i++ { + for i := range numConfigs { scanConfigName := fmt.Sprintf("scan-config-%d", i) _, err := pool.DB.Exec(context.Background(), "insert into compliance_operator_scan_configuration_v2 (id, scanconfigname) values ($1, $2)", uuid.NewV4().String(), scanConfigName) require.NoError(b, err) @@ -59,7 +59,7 @@ func setupTest(b *testing.B, pool *pgtest.TestPostgres, datastore DataStore, num clusters := make([]clusterInfo, 0, numClusters) profiles := make(map[string]string, numClusters) - for i := 0; i < numClusters; i++ { + for i := range numClusters { clusterID := uuid.NewV4().String() clusterName := fmt.Sprintf("cluster-%d", i) scanRefID := uuid.NewV4().String() @@ -81,7 +81,7 @@ func setupTest(b *testing.B, pool *pgtest.TestPostgres, datastore DataStore, num require.NoError(b, err) } - for i := 0; i < numResults; i++ { + for i := range numResults { resultName := fmt.Sprintf("check-result-%d", i) for _, cluster := range clusters { for _, scanConfig := range scanConfigs { diff --git a/central/complianceoperator/v2/checkresults/datastore/stats_views.go b/central/complianceoperator/v2/checkresults/datastore/stats_views.go index 6667f872648ef..a70ce2c96668d 100644 --- a/central/complianceoperator/v2/checkresults/datastore/stats_views.go +++ b/central/complianceoperator/v2/checkresults/datastore/stats_views.go @@ -33,22 +33,6 @@ type ResultStatusCountByCluster struct { LastScanTime *time.Time `db:"compliance_scan_last_executed_time_max"` } -type clusterCount struct { - TotalCount int `db:"cluster_id_count"` -} - -type profileCount struct { - TotalCount int `db:"compliance_profile_name_count"` -} - -type complianceCheckCount struct { - TotalCount int `db:"compliance_check_name_count"` -} - -type configurationCount struct { - TotalCount int `db:"compliance_scan_config_name_count"` -} - // ResourceResultCountByProfile represents shape of the stats query for compliance operator results type ResourceResultCountByProfile struct { PassCount int `db:"compliance_pass_count"` diff --git a/central/complianceoperator/v2/compliancemanager/manager_impl.go b/central/complianceoperator/v2/compliancemanager/manager_impl.go index c8f808687230d..221f15ff3353a 100644 --- a/central/complianceoperator/v2/compliancemanager/manager_impl.go +++ b/central/complianceoperator/v2/compliancemanager/manager_impl.go @@ -13,10 +13,12 @@ import ( profileDatastore "github.com/stackrox/rox/central/complianceoperator/v2/profiles/datastore" compScanSetting "github.com/stackrox/rox/central/complianceoperator/v2/scanconfigurations/datastore" scansDatastore "github.com/stackrox/rox/central/complianceoperator/v2/scans/datastore" + ssbDatastore "github.com/stackrox/rox/central/complianceoperator/v2/scansettingbindings/datastore" "github.com/stackrox/rox/central/convert/internaltov2storage" "github.com/stackrox/rox/central/sensor/service/connection" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/errorhelpers" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/protocompat" @@ -45,6 +47,7 @@ type managerImpl struct { sensorConnMgr connection.Manager integrationDS compIntegration.DataStore scanSettingDS compScanSetting.DataStore + ssbDS ssbDatastore.DataStore clusterDS clusterDatastore.DataStore profileDS profileDatastore.DataStore scansDS scansDatastore.DataStore @@ -57,11 +60,12 @@ type managerImpl struct { } // New returns on instance of Manager interface that provides functionality to process compliance requests and forward them to Sensor. -func New(sensorConnMgr connection.Manager, integrationDS compIntegration.DataStore, scanSettingDS compScanSetting.DataStore, clusterDS clusterDatastore.DataStore, profileDS profileDatastore.DataStore, scansDS scansDatastore.DataStore, resultsDS resultsDatastore.DataStore) Manager { +func New(sensorConnMgr connection.Manager, integrationDS compIntegration.DataStore, scanSettingDS compScanSetting.DataStore, ssbDS ssbDatastore.DataStore, clusterDS clusterDatastore.DataStore, profileDS profileDatastore.DataStore, scansDS scansDatastore.DataStore, resultsDS resultsDatastore.DataStore) Manager { return &managerImpl{ sensorConnMgr: sensorConnMgr, integrationDS: integrationDS, scanSettingDS: scanSettingDS, + ssbDS: ssbDS, runningRequests: make(map[string]clusterScan), clusterDS: clusterDS, profileDS: profileDS, @@ -128,7 +132,7 @@ func (m *managerImpl) ProcessScanRequest(ctx context.Context, scanRequest *stora // Check if scan configuration already exists by name. scanConfig, err := m.scanSettingDS.GetScanConfigurationByName(ctx, scanRequest.GetScanConfigName()) if err != nil { - err = errors.Wrapf(err, "Unable to create scan configuration named %q.", scanRequest.GetScanConfigName()) + err = errors.Wrapf(err, "Unable to create scan configuration named %q", scanRequest.GetScanConfigName()) log.Error(err) return nil, err } @@ -139,7 +143,12 @@ func (m *managerImpl) ProcessScanRequest(ctx context.Context, scanRequest *stora scanRequest.Id = uuid.NewV4().String() scanRequest.CreatedTime = protocompat.TimestampNow() - return m.processRequestToSensor(ctx, scanRequest, cron, clusters, true) + validatedProfiles, err := m.validateScan(ctx, scanRequest, clusters) + if err != nil { + return nil, err + } + + return m.processRequestToSensor(ctx, scanRequest, cron, clusters, true, validatedProfiles) } // UpdateScanRequest processes a request to apply a compliance scan configuration to one or more Sensors. @@ -165,7 +174,7 @@ func (m *managerImpl) UpdateScanRequest(ctx context.Context, scanRequest *storag // Verify the scan configuration ID is valid oldScanConfig, found, err := m.scanSettingDS.GetScanConfiguration(ctx, scanRequest.GetId()) if err != nil { - err = errors.Wrapf(err, "Unable to find scan configuration with ID %q.", scanRequest.GetId()) + err = errors.Wrapf(err, "Unable to find scan configuration with ID %q", scanRequest.GetId()) log.Error(err) return nil, err } @@ -178,13 +187,18 @@ func (m *managerImpl) UpdateScanRequest(ctx context.Context, scanRequest *storag return nil, errors.New("Changing the scan schedule name is not allowed.") } + validatedProfiles, err := m.validateScan(ctx, scanRequest, clusters) + if err != nil { + return nil, err + } + // TODO(ROX-22398): if we restrict cluster deletion, this is where we would do it before any updates are done. m.removeObsoleteResultsByClusters(ctx, oldScanConfig, scanRequest) m.removeObsoleteResultsByProfiles(ctx, oldScanConfig, scanRequest) // Use the created time from the DB scanRequest.CreatedTime = oldScanConfig.GetCreatedTime() - scanRequest, err = m.processRequestToSensor(ctx, scanRequest, cron, clusters, false) + scanRequest, err = m.processRequestToSensor(ctx, scanRequest, cron, clusters, false, validatedProfiles) if err != nil { return nil, err } @@ -258,44 +272,75 @@ func (m *managerImpl) removeObsoleteResultsByProfiles(ctx context.Context, oldSc } } -func (m *managerImpl) processRequestToSensor(ctx context.Context, scanRequest *storage.ComplianceOperatorScanConfigurationV2, cron string, clusters []string, createScanRequest bool) (*storage.ComplianceOperatorScanConfigurationV2, error) { - var profiles []string +func (m *managerImpl) validateScan(ctx context.Context, scanRequest *storage.ComplianceOperatorScanConfigurationV2, clusters []string) ([]*storage.ComplianceOperatorProfileV2, error) { + profiles := make([]string, 0, len(scanRequest.GetProfiles())) for _, profile := range scanRequest.GetProfiles() { profiles = append(profiles, profile.GetProfileName()) } - // Check if there are any existing clusters that have a scan configuration with any of profiles being referenced by the scan request. - // If so, then we cannot create the scan configuration. + // Check if there are any existing clusters that have a scan configuration with any of profiles + // being referenced by the scan request. If so, then we cannot create the scan configuration. err := m.scanSettingDS.ScanConfigurationProfileExists(ctx, scanRequest.GetId(), profiles, clusters) if err != nil { log.Error(err) - return nil, errors.Wrapf(err, "Unable to create scan configuration named %q.", scanRequest.GetScanConfigName()) + return nil, err + } + + // Check if any non-ACS-managed ScanSettingBindings already reference the requested profiles. + if err := m.checkForExternalSSBConflicts(ctx, profiles, clusters); err != nil { + return nil, err } - // Get all profiles from the database to validate that they exist and are compatible + // Validate that all profiles exist in the database and have compatible kinds. returnedProfiles, err := m.profileDS.SearchProfiles(ctx, search.NewQueryBuilder(). AddExactMatches(search.ClusterID, clusters[0]). AddExactMatches(search.ComplianceOperatorProfileName, profiles...).ProtoQuery()) - if err != nil { - return nil, errors.Wrapf(err, "Unable to retrieve profiles for scan configuration named %q.", scanRequest.GetScanConfigName()) + return nil, errors.Wrapf(err, "Unable to retrieve profiles for scan configuration named %q", scanRequest.GetScanConfigName()) } if len(returnedProfiles) != len(profiles) { return nil, errors.Errorf("Unable to find all profiles for scan configuration named %q.", scanRequest.GetScanConfigName()) } - err = m.scanSettingDS.UpsertScanConfiguration(ctx, scanRequest) + // UNSPECIFIED should not appear here because centralToStorageProfileKind normalizes it to + // PROFILE at ingestion time, but we allow it through defensively — StorageToCentralProfileKind + // will map it to PROFILE. Truly unknown kinds are rejected. + for _, p := range returnedProfiles { + switch p.GetOperatorKind() { + case storage.ComplianceOperatorProfileV2_PROFILE, storage.ComplianceOperatorProfileV2_TAILORED_PROFILE: + // valid + case storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED: + log.Warnf("Profile %q in scan configuration %q has UNSPECIFIED operator kind; treating as PROFILE", p.GetName(), scanRequest.GetScanConfigName()) + default: + return nil, errors.Errorf("profile %q has unsupported operator kind %v (scan configuration %q)", p.GetName(), p.GetOperatorKind(), scanRequest.GetScanConfigName()) + } + } + + return returnedProfiles, nil +} + +func (m *managerImpl) processRequestToSensor(ctx context.Context, scanRequest *storage.ComplianceOperatorScanConfigurationV2, cron string, clusters []string, createScanRequest bool, validatedProfiles []*storage.ComplianceOperatorProfileV2) (*storage.ComplianceOperatorScanConfigurationV2, error) { + profiles := make([]string, 0, len(validatedProfiles)) + for _, p := range validatedProfiles { + profiles = append(profiles, p.GetName()) + } + + scanRequest.ProfileRefs = internaltov2storage.ProfileV2ToScanConfigRefs(validatedProfiles) + + err := m.scanSettingDS.UpsertScanConfiguration(ctx, scanRequest) if err != nil { log.Error(err) return nil, errors.Errorf("Unable to save scan configuration named %q.", scanRequest.GetScanConfigName()) } + profileRefs := internaltov2storage.ScanConfigRefsToCentral(scanRequest.GetProfileRefs()) + for _, clusterID := range clusters { // id for the request message to sensor sensorRequestID := uuid.NewV4().String() - sensorMessage := buildScanConfigSensorMsg(sensorRequestID, cron, profiles, scanRequest.GetScanConfigName(), createScanRequest) + sensorMessage := buildScanConfigSensorMsg(sensorRequestID, cron, profiles, profileRefs, scanRequest.GetScanConfigName(), createScanRequest) err := m.sensorConnMgr.SendMessage(clusterID, sensorMessage) var status string if err != nil { @@ -415,6 +460,58 @@ func (m *managerImpl) HandleScanRequestResponse(ctx context.Context, requestID s return nil } +// checkForExternalSSBConflicts checks if any non-ACS-managed ScanSettingBindings in the target +// clusters already reference any of the requested profiles. ACS-managed SSBs are skipped because +// conflicts with those are already enforced by ScanConfigurationProfileExists. +func (m *managerImpl) checkForExternalSSBConflicts(ctx context.Context, profiles []string, clusters []string) error { + requestedProfiles := set.NewStringSet(profiles...) + if requestedProfiles.Cardinality() == 0 { + return nil + } + + var errList errorhelpers.ErrorList + for _, clusterID := range clusters { + ssbs, err := m.ssbDS.GetScanSettingBindingsByCluster(ctx, clusterID) + if err != nil { + return errors.Wrapf(err, "getting scan setting bindings for cluster %q", clusterID) + } + + clusterRef := "" + for _, ssb := range ssbs { + if ssb.GetLabels()["app.kubernetes.io/name"] == "stackrox" { + continue + } + + var conflicting []string + for _, profileName := range ssb.GetProfileNames() { + if requestedProfiles.Contains(profileName) { + conflicting = append(conflicting, profileName) + } + } + if len(conflicting) > 0 { + if clusterRef == "" { + clusterRef = clusterID + if name, exists, err := m.clusterDS.GetClusterName(ctx, clusterID); err == nil && exists { + clusterRef = name + } + } + quoted := make([]string, 0, len(conflicting)) + for _, p := range conflicting { + quoted = append(quoted, fmt.Sprintf("%q", p)) + } + errList.AddStringf( + "profiles [%s] conflict with external ScanSettingBinding %q in cluster %q", + strings.Join(quoted, ", "), ssb.GetName(), clusterRef) + } + } + } + + if err := errList.ToError(); err != nil { + return fmt.Errorf("%w, remove the external ScanSettingBindings or choose different profiles", err) + } + return nil +} + func (m *managerImpl) ProcessRescanRequest(ctx context.Context, scanID string) error { if !features.ComplianceEnhancements.Enabled() { return errors.Errorf("Compliance is disabled. Cannot run compliance scan for configuration with ID %s", scanID) @@ -476,7 +573,7 @@ func (m *managerImpl) DeleteScan(ctx context.Context, scanID string) error { // Remove the scan configuration from the database scanConfigName, err := m.scanSettingDS.DeleteScanConfiguration(ctx, scanID) if err != nil { - return errors.Wrapf(err, "Unable to delete scan configuration ID %q.", scanID) + return errors.Wrapf(err, "Unable to delete scan configuration ID %q", scanID) } if scanConfigName == "" { @@ -536,7 +633,7 @@ func convertSchedule(scanRequest *storage.ComplianceOperatorScanConfigurationV2) if scanRequest.GetSchedule() != nil { cron, err = schedule.ConvertToCronTab(scanRequest.GetSchedule()) if err != nil { - err = errors.Wrapf(err, "Unable to convert schedule for scan configuration named %q to cron.", scanRequest.GetScanConfigName()) + err = errors.Wrapf(err, "Unable to convert schedule for scan configuration named %q to cron", scanRequest.GetScanConfigName()) log.Error(err) return "", err } diff --git a/central/complianceoperator/v2/compliancemanager/manager_impl_ssb_test.go b/central/complianceoperator/v2/compliancemanager/manager_impl_ssb_test.go new file mode 100644 index 0000000000000..eb4bb5df48826 --- /dev/null +++ b/central/complianceoperator/v2/compliancemanager/manager_impl_ssb_test.go @@ -0,0 +1,109 @@ +package compliancemanager + +import ( + "github.com/pkg/errors" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/sac/testconsts" + "github.com/stackrox/rox/pkg/sac/testutils" + "go.uber.org/mock/gomock" +) + +func (suite *complianceManagerTestSuite) expectPassesPreSSBChecks() { + ctx := suite.testContexts[testutils.UnrestrictedReadWriteCtx] + suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.scanConfigDS.EXPECT().GetScanConfigurationByName(ctx, mockScanName).Return(nil, nil).Times(1) +} + +func (suite *complianceManagerTestSuite) expectSingleClusterSuccess() { + ctx := suite.testContexts[testutils.UnrestrictedReadWriteCtx] + suite.profileDS.EXPECT().SearchProfiles(ctx, gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ + getTestProfile("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1), + }, nil).Times(1) + suite.scanConfigDS.EXPECT().UpsertScanConfiguration(ctx, gomock.Any()).Return(nil).Times(1) + suite.connectionMgr.EXPECT().SendMessage(testconsts.Cluster1, gomock.Any()).Return(nil).Times(1) + suite.clusterDatastore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).Return("test_cluster", true, nil).Times(1) + suite.scanConfigDS.EXPECT().UpdateClusterStatus(ctx, gomock.Any(), testconsts.Cluster1, "", "test_cluster") +} + +func (suite *complianceManagerTestSuite) TestProcessScanRequestSSBConflicts() { + ctx := suite.testContexts[testutils.UnrestrictedReadWriteCtx] + + suite.Run("Profile conflict with external ScanSettingBinding", func() { + suite.expectPassesPreSSBChecks() + suite.clusterDatastore.EXPECT().GetClusterName(gomock.Any(), testconsts.Cluster1).Return("test_cluster", true, nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return([]*storage.ComplianceOperatorScanSettingBindingV2{ + {Name: "external-argocd-ssb", ClusterId: testconsts.Cluster1, ProfileNames: []string{"ocp4-cis"}, Labels: map[string]string{"managed-by": "argocd"}}, + }, nil).Times(1) + + config, err := suite.manager.ProcessScanRequest(ctx, getTestRecNoID(), []string{testconsts.Cluster1}) + suite.Require().ErrorContains(err, "conflict with external ScanSettingBinding") + suite.Require().ErrorContains(err, "test_cluster") + suite.Require().Nil(config) + }) + + suite.Run("No conflict when external SSB uses different profiles", func() { + suite.expectPassesPreSSBChecks() + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return([]*storage.ComplianceOperatorScanSettingBindingV2{ + {Name: "external-argocd-ssb", ClusterId: testconsts.Cluster1, ProfileNames: []string{"ocp4-moderate"}, Labels: map[string]string{"managed-by": "argocd"}}, + }, nil).Times(1) + suite.expectSingleClusterSuccess() + + config, err := suite.manager.ProcessScanRequest(ctx, getTestRecNoID(), []string{testconsts.Cluster1}) + suite.Require().NoError(err) + suite.Require().NotNil(config) + }) + + suite.Run("Stackrox-managed SSB is ignored during conflict check", func() { + suite.expectPassesPreSSBChecks() + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return([]*storage.ComplianceOperatorScanSettingBindingV2{ + {Name: "stackrox-managed-ssb", ClusterId: testconsts.Cluster1, ProfileNames: []string{"ocp4-cis"}, Labels: map[string]string{"app.kubernetes.io/name": "stackrox"}}, + }, nil).Times(1) + suite.expectSingleClusterSuccess() + + config, err := suite.manager.ProcessScanRequest(ctx, getTestRecNoID(), []string{testconsts.Cluster1}) + suite.Require().NoError(err) + suite.Require().NotNil(config) + }) + + suite.Run("SSB datastore error propagates", func() { + suite.expectPassesPreSSBChecks() + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, errors.New("db connection failed")).Times(1) + + config, err := suite.manager.ProcessScanRequest(ctx, getTestRecNoID(), []string{testconsts.Cluster1}) + suite.Require().ErrorContains(err, "getting scan setting bindings for cluster") + suite.Require().Nil(config) + }) + + suite.Run("Conflicts across multiple clusters reported together", func() { + suite.expectPassesPreSSBChecks() + clusters := []string{testconsts.Cluster1, testconsts.Cluster2} + suite.clusterDatastore.EXPECT().GetClusterName(gomock.Any(), testconsts.Cluster1).Return("cluster_one", true, nil).Times(1) + suite.clusterDatastore.EXPECT().GetClusterName(gomock.Any(), testconsts.Cluster2).Return("cluster_two", true, nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return([]*storage.ComplianceOperatorScanSettingBindingV2{ + {Name: "external-ssb-cluster1", ClusterId: testconsts.Cluster1, ProfileNames: []string{"ocp4-cis"}, Labels: map[string]string{"managed-by": "argocd"}}, + }, nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster2).Return([]*storage.ComplianceOperatorScanSettingBindingV2{ + {Name: "external-ssb-cluster2", ClusterId: testconsts.Cluster2, ProfileNames: []string{"ocp4-cis"}, Labels: map[string]string{"managed-by": "argocd"}}, + }, nil).Times(1) + + config, err := suite.manager.ProcessScanRequest(ctx, getTestRecNoID(), clusters) + suite.Require().ErrorContains(err, "external-ssb-cluster1") + suite.Require().ErrorContains(err, "external-ssb-cluster2") + suite.Require().ErrorContains(err, "cluster_one") + suite.Require().ErrorContains(err, "cluster_two") + suite.Require().ErrorContains(err, "remove the external ScanSettingBindings") + suite.Require().Nil(config) + }) + + suite.Run("SSB with nil labels is treated as external", func() { + suite.expectPassesPreSSBChecks() + suite.clusterDatastore.EXPECT().GetClusterName(gomock.Any(), testconsts.Cluster1).Return("test_cluster", true, nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return([]*storage.ComplianceOperatorScanSettingBindingV2{ + {Name: "unlabeled-ssb", ClusterId: testconsts.Cluster1, ProfileNames: []string{"ocp4-cis"}}, + }, nil).Times(1) + + config, err := suite.manager.ProcessScanRequest(ctx, getTestRecNoID(), []string{testconsts.Cluster1}) + suite.Require().ErrorContains(err, "conflict with external ScanSettingBinding") + suite.Require().Nil(config) + }) +} diff --git a/central/complianceoperator/v2/compliancemanager/manager_impl_test.go b/central/complianceoperator/v2/compliancemanager/manager_impl_test.go index 75f6cb1ec2c6d..d184241a1dee7 100644 --- a/central/complianceoperator/v2/compliancemanager/manager_impl_test.go +++ b/central/complianceoperator/v2/compliancemanager/manager_impl_test.go @@ -12,8 +12,10 @@ import ( profileMocks "github.com/stackrox/rox/central/complianceoperator/v2/profiles/datastore/mocks" scanConfigMocks "github.com/stackrox/rox/central/complianceoperator/v2/scanconfigurations/datastore/mocks" scansMocks "github.com/stackrox/rox/central/complianceoperator/v2/scans/datastore/mocks" + ssbMocks "github.com/stackrox/rox/central/complianceoperator/v2/scansettingbindings/datastore/mocks" "github.com/stackrox/rox/central/convert/internaltov2storage" sensorMocks "github.com/stackrox/rox/central/sensor/service/connection/mocks" + "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" @@ -69,6 +71,7 @@ type complianceManagerTestSuite struct { mockCtrl *gomock.Controller integrationDS *mocks.MockDataStore scanConfigDS *scanConfigMocks.MockDataStore + ssbDS *ssbMocks.MockDataStore profileDS *profileMocks.MockDataStore scansDS *scansMocks.MockDataStore resultsDS *resultsMocks.MockDataStore @@ -97,12 +100,13 @@ func (suite *complianceManagerTestSuite) SetupTest() { suite.integrationDS = mocks.NewMockDataStore(suite.mockCtrl) suite.scanConfigDS = scanConfigMocks.NewMockDataStore(suite.mockCtrl) + suite.ssbDS = ssbMocks.NewMockDataStore(suite.mockCtrl) suite.connectionMgr = sensorMocks.NewMockManager(suite.mockCtrl) suite.clusterDatastore = clusterDatastoreMocks.NewMockDataStore(suite.mockCtrl) suite.profileDS = profileMocks.NewMockDataStore(suite.mockCtrl) suite.scansDS = scansMocks.NewMockDataStore(suite.mockCtrl) suite.resultsDS = resultsMocks.NewMockDataStore(suite.mockCtrl) - suite.manager = New(suite.connectionMgr, suite.integrationDS, suite.scanConfigDS, suite.clusterDatastore, suite.profileDS, suite.scansDS, suite.resultsDS) + suite.manager = New(suite.connectionMgr, suite.integrationDS, suite.scanConfigDS, suite.ssbDS, suite.clusterDatastore, suite.profileDS, suite.scansDS, suite.resultsDS) } func (suite *complianceManagerTestSuite) TearDownTest() { @@ -204,6 +208,7 @@ func (suite *complianceManagerTestSuite) TestProcessScanRequest() { clusters: []string{testconsts.Cluster1}, setMocks: func() { suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) suite.profileDS.EXPECT().SearchProfiles(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ getTestProfile("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1), }, nil).Times(1) @@ -223,6 +228,7 @@ func (suite *complianceManagerTestSuite) TestProcessScanRequest() { setMocks: func() { suite.scanConfigDS.EXPECT().GetScanConfigurationByName(gomock.Any(), mockScanName).Return(nil, nil).Times(1) suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) suite.profileDS.EXPECT().SearchProfiles(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ getTestProfile("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1), getTestProfile("rhcos4-cis", "1.0.0", "node", "rhcos4", testconsts.Cluster1, 1), @@ -242,6 +248,7 @@ func (suite *complianceManagerTestSuite) TestProcessScanRequest() { setMocks: func() { suite.scanConfigDS.EXPECT().GetScanConfigurationByName(gomock.Any(), mockScanName).Return(nil, nil).Times(1) suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) suite.profileDS.EXPECT().SearchProfiles(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ getTestProfile("ocp4-cis-node", "1.0.0", "node", "ocp4", testconsts.Cluster1, 1), getTestProfile("rhcos4-cis", "1.0.0", "node", "rhcos4", testconsts.Cluster1, 1), @@ -283,6 +290,7 @@ func (suite *complianceManagerTestSuite) TestProcessScanRequest() { clusters: []string{testconsts.Cluster1}, setMocks: func() { suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) suite.profileDS.EXPECT().SearchProfiles(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ getTestProfile("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1), }, nil).Times(1) @@ -293,12 +301,13 @@ func (suite *complianceManagerTestSuite) TestProcessScanRequest() { expectedErr: fmt.Sprintf("Unable to save scan configuration named %q", mockScanName), }, { - desc: "Error from sensor", + desc: "SendMessage failure from sensor", testRequest: getTestRecNoID(), testContext: suite.testContexts[testutils.UnrestrictedReadWriteCtx], clusters: []string{testconsts.Cluster1}, setMocks: func() { suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) suite.scanConfigDS.EXPECT().GetScanConfigurationByName(suite.testContexts[testutils.UnrestrictedReadWriteCtx], mockScanName).Return(nil, nil).Times(1) suite.profileDS.EXPECT().SearchProfiles(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ getTestProfile("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1), @@ -347,6 +356,7 @@ func (suite *complianceManagerTestSuite) TestProcessScanRequest() { setMocks: func() { suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) suite.scanConfigDS.EXPECT().GetScanConfigurationByName(suite.testContexts[testutils.UnrestrictedReadWriteCtx], mockScanName).Return(nil, nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) suite.profileDS.EXPECT().SearchProfiles(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ getTestProfile("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1), }, nil).Times(1) @@ -404,6 +414,7 @@ func (suite *complianceManagerTestSuite) TestUpdateScanRequest() { clusters: []string{testconsts.Cluster1}, setMocks: func() { suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) suite.scanConfigDS.EXPECT().GetScanConfiguration(gomock.Any(), mockScanID).Return(getTestRec(), true, nil).Times(1) suite.profileDS.EXPECT().SearchProfiles(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ getTestProfile("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1), @@ -414,12 +425,13 @@ func (suite *complianceManagerTestSuite) TestUpdateScanRequest() { expectedErr: fmt.Sprintf("Unable to save scan configuration named %q", mockScanName), }, { - desc: "Error from sensor", + desc: "SendMessage failure from sensor", testRequest: getTestRec(), testContext: suite.testContexts[testutils.UnrestrictedReadWriteCtx], clusters: []string{testconsts.Cluster1}, setMocks: func() { suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) suite.scanConfigDS.EXPECT().GetScanConfiguration(gomock.Any(), mockScanID).Return(getTestRec(), true, nil).Times(1) suite.profileDS.EXPECT().SearchProfiles(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ getTestProfile("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1), @@ -459,6 +471,7 @@ func (suite *complianceManagerTestSuite) TestUpdateScanRequest() { setMocks: func() { suite.scanConfigDS.EXPECT().GetScanConfiguration(gomock.Any(), mockScanID).Return(getTestRec(), true, nil).Times(1) suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) suite.profileDS.EXPECT().SearchProfiles(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ getTestProfile("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1), }, nil).Times(1) @@ -477,14 +490,15 @@ func (suite *complianceManagerTestSuite) TestUpdateScanRequest() { setMocks: func() { suite.scanConfigDS.EXPECT().GetScanConfiguration(gomock.Any(), mockScanID).Return(getTestRecMultiCluster(), true, nil).Times(1) suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) suite.profileDS.EXPECT().SearchProfiles(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ getTestProfile("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1), }, nil).Times(1) - suite.scanConfigDS.EXPECT().UpsertScanConfiguration(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return(nil).Times(1) suite.scanConfigDS.EXPECT().RemoveClusterStatus(gomock.Any(), mockScanID, testconsts.Cluster2).Return(nil).Times(1) - suite.connectionMgr.EXPECT().SendMessage(testconsts.Cluster1, gomock.Any()).Return(nil).Times(1) suite.connectionMgr.EXPECT().SendMessage(testconsts.Cluster2, gomock.Any()).Return(nil).Times(1) suite.resultsDS.EXPECT().DeleteResultsByScanConfigAndCluster(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.scanConfigDS.EXPECT().UpsertScanConfiguration(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any()).Return(nil).Times(1) + suite.connectionMgr.EXPECT().SendMessage(testconsts.Cluster1, gomock.Any()).Return(nil).Times(1) suite.clusterDatastore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).Return("test_cluster", true, nil).Times(1) suite.scanConfigDS.EXPECT().UpdateClusterStatus(suite.testContexts[testutils.UnrestrictedReadWriteCtx], gomock.Any(), testconsts.Cluster1, "", "test_cluster") }, @@ -507,6 +521,101 @@ func (suite *complianceManagerTestSuite) TestUpdateScanRequest() { } } +func (suite *complianceManagerTestSuite) TestProcessScanRequestSendsProfileRefsWithKinds() { + ctx := suite.testContexts[testutils.UnrestrictedReadWriteCtx] + req := getTestRecNoIDValidProfile() + + suite.scanConfigDS.EXPECT().GetScanConfigurationByName(gomock.Any(), mockScanName).Return(nil, nil).Times(1) + suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) + suite.profileDS.EXPECT().SearchProfiles(ctx, gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ + getTestProfileWithKind("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1, storage.ComplianceOperatorProfileV2_PROFILE), + getTestProfileWithKind("rhcos4-cis", "1.0.0", "node", "rhcos4", testconsts.Cluster1, 1, storage.ComplianceOperatorProfileV2_TAILORED_PROFILE), + }, nil).Times(1) + suite.scanConfigDS.EXPECT().UpsertScanConfiguration(ctx, gomock.Any()).Return(nil).Times(1) + suite.connectionMgr.EXPECT().SendMessage(testconsts.Cluster1, gomock.Any()).DoAndReturn( + func(_ string, msg *central.MsgToSensor) error { + scanSettings := msg.GetComplianceRequest().GetApplyScanConfig().GetScheduledScan().GetScanSettings() + suite.Require().NotNil(scanSettings) + refs := scanSettings.GetProfileRefs() + suite.Require().Len(refs, 2) + actualKinds := map[string]central.ComplianceOperatorProfileV2_OperatorKind{} + for _, ref := range refs { + actualKinds[ref.GetName()] = ref.GetKind() + } + suite.Equal(map[string]central.ComplianceOperatorProfileV2_OperatorKind{ + "ocp4-cis": central.ComplianceOperatorProfileV2_PROFILE, + "rhcos4-cis": central.ComplianceOperatorProfileV2_TAILORED_PROFILE, + }, actualKinds) + // Legacy field is still populated for compatibility. + suite.ElementsMatch([]string{"ocp4-cis", "rhcos4-cis"}, scanSettings.GetProfiles()) + return nil + }, + ).Times(1) + suite.clusterDatastore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).Return("test_cluster", true, nil).Times(1) + suite.scanConfigDS.EXPECT().UpdateClusterStatus(ctx, gomock.Any(), testconsts.Cluster1, "", "test_cluster") + + config, err := suite.manager.ProcessScanRequest(ctx, req, []string{testconsts.Cluster1}) + suite.Require().NoError(err) + suite.Require().NotNil(config) +} + +// TestProcessScanRequestUnspecifiedKindFallsBackToProfile ensures that a profile with +// OPERATOR_KIND_UNSPECIFIED in the DB (stored by older Central before kind tracking) is +// treated as PROFILE and the scan config is sent successfully. +func (suite *complianceManagerTestSuite) TestProcessScanRequestUnspecifiedKindFallsBackToProfile() { + ctx := suite.testContexts[testutils.UnrestrictedReadWriteCtx] + req := getTestRecNoIDValidProfile() + + suite.scanConfigDS.EXPECT().GetScanConfigurationByName(gomock.Any(), mockScanName).Return(nil, nil).Times(1) + suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) + suite.profileDS.EXPECT().SearchProfiles(ctx, gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ + getTestProfileWithKind("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1, storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED), + getTestProfileWithKind("rhcos4-cis", "1.0.0", "node", "rhcos4", testconsts.Cluster1, 1, storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED), + }, nil).Times(1) + suite.scanConfigDS.EXPECT().UpsertScanConfiguration(ctx, gomock.Any()).Return(nil).Times(1) + suite.connectionMgr.EXPECT().SendMessage(testconsts.Cluster1, gomock.Any()).DoAndReturn( + func(_ string, msg *central.MsgToSensor) error { + refs := msg.GetComplianceRequest().GetApplyScanConfig().GetScheduledScan().GetScanSettings().GetProfileRefs() + suite.Require().Len(refs, 2) + for _, ref := range refs { + suite.Equal(central.ComplianceOperatorProfileV2_PROFILE, ref.GetKind(), + "profile %q with UNSPECIFIED storage kind should be sent as PROFILE", ref.GetName()) + } + return nil + }, + ).Times(1) + suite.clusterDatastore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).Return("test_cluster", true, nil).Times(1) + suite.scanConfigDS.EXPECT().UpdateClusterStatus(ctx, gomock.Any(), testconsts.Cluster1, "", "test_cluster") + + config, err := suite.manager.ProcessScanRequest(ctx, req, []string{testconsts.Cluster1}) + suite.Require().NoError(err) + suite.Require().NotNil(config) +} + +// TestProcessScanRequestRejectsUnknownProfileKind ensures Central rejects the request (no persist, +// no send) when a profile has a truly unknown operator kind (not UNSPECIFIED, PROFILE, or TAILORED_PROFILE). +func (suite *complianceManagerTestSuite) TestProcessScanRequestRejectsUnknownProfileKind() { + ctx := suite.testContexts[testutils.UnrestrictedReadWriteCtx] + req := getTestRecNoIDValidProfile() + + suite.scanConfigDS.EXPECT().GetScanConfigurationByName(gomock.Any(), mockScanName).Return(nil, nil).Times(1) + suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), testconsts.Cluster1).Return(nil, nil).Times(1) + suite.profileDS.EXPECT().SearchProfiles(ctx, gomock.Any()).Return([]*storage.ComplianceOperatorProfileV2{ + getTestProfileWithKind("ocp4-cis", "1.0.0", "platform", "ocp4", testconsts.Cluster1, 1, storage.ComplianceOperatorProfileV2_PROFILE), + getTestProfileWithKind("bad-profile", "1.0.0", "node", "ocp4", testconsts.Cluster1, 1, storage.ComplianceOperatorProfileV2_OperatorKind(999)), + }, nil).Times(1) + // No UpsertScanConfiguration, no SendMessage — we fail before that. + + config, err := suite.manager.ProcessScanRequest(ctx, req, []string{testconsts.Cluster1}) + suite.Require().Error(err) + suite.Require().Nil(config) + suite.Require().ErrorContains(err, "unsupported operator kind") + suite.Require().ErrorContains(err, "bad-profile") +} + func (suite *complianceManagerTestSuite) TestDeleteScanConfiguration() { cases := []processScanConfigTestCase{ { @@ -557,7 +666,7 @@ func getTestProfile(profileName string, version string, platform string, product if ruleCount > 0 { rules = make([]*storage.ComplianceOperatorProfileV2_Rule, 0, ruleCount) - for i := 0; i < ruleCount; i++ { + for i := range ruleCount { rules = append(rules, &storage.ComplianceOperatorProfileV2_Rule{ RuleName: fmt.Sprintf("name-%d", i), }) @@ -578,12 +687,19 @@ func getTestProfile(profileName string, version string, platform string, product ClusterId: clusterID, Title: "A Title", Rules: rules, + OperatorKind: storage.ComplianceOperatorProfileV2_PROFILE, } } +func getTestProfileWithKind(profileName string, version string, platform string, product string, clusterID string, ruleCount int, kind storage.ComplianceOperatorProfileV2_OperatorKind) *storage.ComplianceOperatorProfileV2 { + profile := getTestProfile(profileName, version, platform, product, clusterID, ruleCount) + profile.OperatorKind = kind + return profile +} + func getTestScans(scanConfigName string, clusterID string, profileID string, count int) []*storage.ComplianceOperatorScanV2 { scans := make([]*storage.ComplianceOperatorScanV2, 0, count) - for i := 0; i < count; i++ { + for i := range count { scanName := fmt.Sprintf("scan-%s-%s-%s-%d", scanConfigName, clusterID, profileID, i) scans = append(scans, &storage.ComplianceOperatorScanV2{ @@ -729,19 +845,21 @@ func (suite *complianceManagerTestSuite) TestRemoveObsoleteResultsByClusters() { // Default scan update calls. suite.scanConfigDS.EXPECT().GetScanConfiguration(gomock.Any(), mockScanID).Return(oldScanConfig, true, nil).Times(1) suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) - testProfiles := make([]*storage.ComplianceOperatorProfileV2, 0) + for _, clusterID := range tc.newClusterIDs { + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), clusterID).Return(nil, nil).Times(1) + } + testProfiles := make([]*storage.ComplianceOperatorProfileV2, 0, len(tc.profileNames)) for _, profileName := range tc.profileNames { testProfiles = append(testProfiles, getTestProfile(profileName, "1.0.0", "platform", "ocp4", tc.newClusterIDs[0], 1)) } suite.profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Return(testProfiles, nil).Times(1) - suite.scanConfigDS.EXPECT().UpsertScanConfiguration(gomock.Any(), gomock.Any()).Return(nil).Times(1) - // Remove cluster calls. - suite.connectionMgr.EXPECT().SendMessage(gomock.Any(), gomock.Any()).Return(nil).Times(len(expectedRemovedClusters)) + // Remove cluster calls (fire-and-forget via processClusterDelete). suite.scanConfigDS.EXPECT().RemoveClusterStatus(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(len(expectedRemovedClusters)) - // Update cluster calls. - suite.connectionMgr.EXPECT().SendMessage(gomock.Any(), gomock.Any()).Return(nil).Times(len(tc.newClusterIDs)) + totalSendMessages := len(expectedRemovedClusters) + len(tc.newClusterIDs) + suite.connectionMgr.EXPECT().SendMessage(gomock.Any(), gomock.Any()).Return(nil).Times(totalSendMessages) + suite.scanConfigDS.EXPECT().UpsertScanConfiguration(gomock.Any(), gomock.Any()).Return(nil).Times(1) suite.clusterDatastore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).Return("test_cluster", true, nil).Times(len(tc.newClusterIDs)) suite.scanConfigDS.EXPECT().UpdateClusterStatus(gomock.Any(), gomock.Any(), gomock.Any(), "", "test_cluster").Return(nil).Times(len(tc.newClusterIDs)) @@ -798,7 +916,7 @@ func (suite *complianceManagerTestSuite) TestRemoveObsoleteResultsByProfiles() { } } - newTestProfiles := make([]*storage.ComplianceOperatorProfileV2, 0) + newTestProfiles := make([]*storage.ComplianceOperatorProfileV2, 0, len(tc.newProfileNames)) for _, profileName := range tc.newProfileNames { testProfile := getTestProfile(profileName, "1.0.0", "platform", "ocp4", tc.clusterIDs[0], 1) newTestProfiles = append(newTestProfiles, testProfile) @@ -806,10 +924,13 @@ func (suite *complianceManagerTestSuite) TestRemoveObsoleteResultsByProfiles() { suite.scanConfigDS.EXPECT().GetScanConfiguration(gomock.Any(), mockScanID).Return(oldScanConfig, true, nil).Times(1) suite.scanConfigDS.EXPECT().ScanConfigurationProfileExists(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + for _, clusterID := range tc.clusterIDs { + suite.ssbDS.EXPECT().GetScanSettingBindingsByCluster(gomock.Any(), clusterID).Return(nil, nil).Times(1) + } suite.profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Return(newTestProfiles, nil).Times(1) - suite.scanConfigDS.EXPECT().UpsertScanConfiguration(gomock.Any(), gomock.Any()).Return(nil).Times(1) suite.connectionMgr.EXPECT().SendMessage(gomock.Any(), gomock.Any()).Return(nil).Times(len(tc.clusterIDs)) + suite.scanConfigDS.EXPECT().UpsertScanConfiguration(gomock.Any(), gomock.Any()).Return(nil).Times(1) suite.clusterDatastore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).Return("test_cluster", true, nil).Times(len(tc.clusterIDs)) suite.scanConfigDS.EXPECT().UpdateClusterStatus(gomock.Any(), gomock.Any(), gomock.Any(), "", "test_cluster").Return(nil).Times(len(tc.clusterIDs)) diff --git a/central/complianceoperator/v2/compliancemanager/singleton.go b/central/complianceoperator/v2/compliancemanager/singleton.go index b97a2936b9e6d..8dd71451f8a26 100644 --- a/central/complianceoperator/v2/compliancemanager/singleton.go +++ b/central/complianceoperator/v2/compliancemanager/singleton.go @@ -7,6 +7,7 @@ import ( profileDatastore "github.com/stackrox/rox/central/complianceoperator/v2/profiles/datastore" compScanSetting "github.com/stackrox/rox/central/complianceoperator/v2/scanconfigurations/datastore" scansDatastore "github.com/stackrox/rox/central/complianceoperator/v2/scans/datastore" + ssbDatastore "github.com/stackrox/rox/central/complianceoperator/v2/scansettingbindings/datastore" "github.com/stackrox/rox/central/sensor/service/connection" "github.com/stackrox/rox/pkg/sync" ) @@ -19,7 +20,7 @@ var ( // Singleton returns the compliance operator manager func Singleton() Manager { once.Do(func() { - manager = New(connection.ManagerSingleton(), compIntegration.Singleton(), compScanSetting.Singleton(), clusterDatastore.Singleton(), profileDatastore.Singleton(), scansDatastore.Singleton(), resultsDatastore.Singleton()) + manager = New(connection.ManagerSingleton(), compIntegration.Singleton(), compScanSetting.Singleton(), ssbDatastore.Singleton(), clusterDatastore.Singleton(), profileDatastore.Singleton(), scansDatastore.Singleton(), resultsDatastore.Singleton()) }) return manager } diff --git a/central/complianceoperator/v2/compliancemanager/utils.go b/central/complianceoperator/v2/compliancemanager/utils.go index 7fbdf19f0ea81..816f72df1be61 100644 --- a/central/complianceoperator/v2/compliancemanager/utils.go +++ b/central/complianceoperator/v2/compliancemanager/utils.go @@ -2,7 +2,7 @@ package compliancemanager import "github.com/stackrox/rox/generated/internalapi/central" -func buildScanConfigSensorMsg(msgID string, cron string, profiles []string, configName string, createConfig bool) *central.MsgToSensor { +func buildScanConfigSensorMsg(msgID string, cron string, profiles []string, profileRefs []*central.ApplyComplianceScanConfigRequest_BaseScanSettings_ProfileReference, configName string, createConfig bool) *central.MsgToSensor { if createConfig { return ¢ral.MsgToSensor{ Msg: ¢ral.MsgToSensor_ComplianceRequest{ @@ -16,6 +16,7 @@ func buildScanConfigSensorMsg(msgID string, cron string, profiles []string, conf ScanName: configName, StrictNodeScan: true, Profiles: profiles, + ProfileRefs: profileRefs, }, Cron: cron, }, @@ -39,6 +40,7 @@ func buildScanConfigSensorMsg(msgID string, cron string, profiles []string, conf ScanName: configName, StrictNodeScan: true, Profiles: profiles, + ProfileRefs: profileRefs, }, Cron: cron, }, diff --git a/central/complianceoperator/v2/integration/datastore/datastore_impl.go b/central/complianceoperator/v2/integration/datastore/datastore_impl.go index ef67c1cdf92f0..73a27d50d4ffe 100644 --- a/central/complianceoperator/v2/integration/datastore/datastore_impl.go +++ b/central/complianceoperator/v2/integration/datastore/datastore_impl.go @@ -5,7 +5,6 @@ import ( "github.com/pkg/errors" store "github.com/stackrox/rox/central/complianceoperator/v2/integration/store/postgres" - complianceUtils "github.com/stackrox/rox/central/complianceoperator/v2/utils" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/errox" @@ -45,12 +44,6 @@ func (ds *datastoreImpl) GetComplianceIntegrations(ctx context.Context, query *v // GetComplianceIntegrationsView provides an in memory layer on top of the underlying DB based storage. func (ds *datastoreImpl) GetComplianceIntegrationsView(ctx context.Context, query *v1.Query) ([]*IntegrationDetails, error) { - var err error - query, err = complianceUtils.WithSACFilter(ctx, resources.Compliance, query) - if err != nil { - return nil, err - } - cloned := query.CloneVT() cloned.Selects = []*v1.QuerySelect{ search.NewQuerySelect(search.Cluster).Proto(), @@ -63,7 +56,11 @@ func (ds *datastoreImpl) GetComplianceIntegrationsView(ctx context.Context, quer search.NewQuerySelect(search.ComplianceOperatorIntegrationID).Proto(), } - results, err := pgSearch.RunSelectRequestForSchema[IntegrationDetails](ctx, ds.db, schema.ComplianceIntegrationsSchema, cloned) + var results []*IntegrationDetails + err := pgSearch.RunSelectRequestForSchemaFn[IntegrationDetails](ctx, ds.db, schema.ComplianceIntegrationsSchema, cloned, func(r *IntegrationDetails) error { + results = append(results, r) + return nil + }) if err != nil { return nil, err } diff --git a/central/complianceoperator/v2/integration/datastore/datastore_impl_test.go b/central/complianceoperator/v2/integration/datastore/datastore_impl_test.go index 224f7543f46f9..6a6e86f08e7c2 100644 --- a/central/complianceoperator/v2/integration/datastore/datastore_impl_test.go +++ b/central/complianceoperator/v2/integration/datastore/datastore_impl_test.go @@ -11,7 +11,6 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/sac" @@ -404,7 +403,7 @@ func (s *complianceIntegrationDataStoreTestSuite) TestCountIntegrations() { } func (s *complianceIntegrationDataStoreTestSuite) addBaseIntegrations(testIntegrations []*storage.ComplianceIntegration) []string { - var ids []string + ids := make([]string, 0, len(testIntegrations)) for _, integration := range testIntegrations { id, err := s.dataStore.AddComplianceIntegration(s.hasWriteCtx, integration) s.NoError(err) @@ -450,32 +449,32 @@ func getDefaultTestIntegrationViews() []*IntegrationDetails { { ID: "", Version: "2", - OperatorInstalled: pointers.Bool(true), - OperatorStatus: pointers.Pointer(storage.COStatus_HEALTHY), + OperatorInstalled: new(true), + OperatorStatus: new(storage.COStatus_HEALTHY), ClusterID: testconsts.Cluster1, ClusterName: "cluster1", - Type: pointers.Pointer(storage.ClusterType_KUBERNETES_CLUSTER), - StatusProviderMetadataClusterType: pointers.Pointer(storage.ClusterMetadata_AKS), + Type: new(storage.ClusterType_KUBERNETES_CLUSTER), + StatusProviderMetadataClusterType: new(storage.ClusterMetadata_AKS), }, { ID: "", ClusterID: testconsts.Cluster2, Version: "2", - OperatorStatus: pointers.Pointer(storage.COStatus_HEALTHY), + OperatorStatus: new(storage.COStatus_HEALTHY), ClusterName: "cluster2", - Type: pointers.Pointer(storage.ClusterType_OPENSHIFT_CLUSTER), - StatusProviderMetadataClusterType: pointers.Pointer(storage.ClusterMetadata_ARO), - OperatorInstalled: pointers.Bool(true), + Type: new(storage.ClusterType_OPENSHIFT_CLUSTER), + StatusProviderMetadataClusterType: new(storage.ClusterMetadata_ARO), + OperatorInstalled: new(true), }, { ID: "", ClusterID: testconsts.Cluster3, Version: "2", - OperatorStatus: pointers.Pointer(storage.COStatus_HEALTHY), + OperatorStatus: new(storage.COStatus_HEALTHY), ClusterName: "cluster3", - Type: pointers.Pointer(storage.ClusterType_OPENSHIFT4_CLUSTER), - StatusProviderMetadataClusterType: pointers.Pointer(storage.ClusterMetadata_OCP), - OperatorInstalled: pointers.Bool(true), + Type: new(storage.ClusterType_OPENSHIFT4_CLUSTER), + StatusProviderMetadataClusterType: new(storage.ClusterMetadata_OCP), + OperatorInstalled: new(true), }, } diff --git a/central/complianceoperator/v2/integration/service/convert_test.go b/central/complianceoperator/v2/integration/service/convert_test.go index 2978b4af1775b..d4e15caf744e7 100644 --- a/central/complianceoperator/v2/integration/service/convert_test.go +++ b/central/complianceoperator/v2/integration/service/convert_test.go @@ -10,7 +10,6 @@ import ( apiV2 "github.com/stackrox/rox/generated/api/v2" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/sac/testconsts" "github.com/stackrox/rox/pkg/uuid" @@ -44,12 +43,12 @@ func TestConvertStorageIntegrationToV2(t *testing.T) { view: &datastore.IntegrationDetails{ ID: testID, Version: "22", - OperatorInstalled: pointers.Bool(true), - OperatorStatus: pointers.Pointer(storage.COStatus_HEALTHY), + OperatorInstalled: new(true), + OperatorStatus: new(storage.COStatus_HEALTHY), ClusterID: fixtureconsts.Cluster1, ClusterName: mockClusterName, - Type: pointers.Pointer(storage.ClusterType_OPENSHIFT_CLUSTER), - StatusProviderMetadataClusterType: pointers.Pointer(storage.ClusterMetadata_OCP), + Type: new(storage.ClusterType_OPENSHIFT_CLUSTER), + StatusProviderMetadataClusterType: new(storage.ClusterMetadata_OCP), }, expected: &apiV2.ComplianceIntegration{ Id: testID, @@ -77,12 +76,12 @@ func TestConvertStorageIntegrationToV2(t *testing.T) { view: &datastore.IntegrationDetails{ ID: testID, Version: "22", - OperatorInstalled: pointers.Bool(true), - OperatorStatus: pointers.Pointer(storage.COStatus_HEALTHY), + OperatorInstalled: new(true), + OperatorStatus: new(storage.COStatus_HEALTHY), ClusterID: testconsts.Cluster1, ClusterName: mockClusterName, - Type: pointers.Pointer(storage.ClusterType_OPENSHIFT_CLUSTER), - StatusProviderMetadataClusterType: pointers.Pointer(storage.ClusterMetadata_OCP), + Type: new(storage.ClusterType_OPENSHIFT_CLUSTER), + StatusProviderMetadataClusterType: new(storage.ClusterMetadata_OCP), }, expected: nil, clusterError: true, diff --git a/central/complianceoperator/v2/integration/service/service_impl_test.go b/central/complianceoperator/v2/integration/service/service_impl_test.go index 687a6b8695e34..af6d72a119aa3 100644 --- a/central/complianceoperator/v2/integration/service/service_impl_test.go +++ b/central/complianceoperator/v2/integration/service/service_impl_test.go @@ -15,7 +15,6 @@ import ( "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" "github.com/stackrox/rox/pkg/grpc/testutils" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/search" @@ -205,12 +204,12 @@ func (s *ComplianceIntegrationServiceTestSuite) TestListComplianceIntegrations() Return([]*datastore.IntegrationDetails{{ ID: uuid.NewDummy().String(), Version: "22", - OperatorInstalled: pointers.Bool(true), - OperatorStatus: pointers.Pointer(storage.COStatus_HEALTHY), + OperatorInstalled: new(true), + OperatorStatus: new(storage.COStatus_HEALTHY), ClusterID: fixtureconsts.Cluster1, ClusterName: mockClusterName, - Type: pointers.Pointer(storage.ClusterType_OPENSHIFT_CLUSTER), - StatusProviderMetadataClusterType: pointers.Pointer(storage.ClusterMetadata_OCP), + Type: new(storage.ClusterType_OPENSHIFT_CLUSTER), + StatusProviderMetadataClusterType: new(storage.ClusterMetadata_OCP), }, }, nil).Times(1) diff --git a/central/complianceoperator/v2/pipelines/complianceoperatorprofilesv2/pipeline_test.go b/central/complianceoperator/v2/pipelines/complianceoperatorprofilesv2/pipeline_test.go index 91725e0beba49..38f00fc7d2263 100644 --- a/central/complianceoperator/v2/pipelines/complianceoperatorprofilesv2/pipeline_test.go +++ b/central/complianceoperator/v2/pipelines/complianceoperatorprofilesv2/pipeline_test.go @@ -5,6 +5,7 @@ import ( "testing" v2ProfileMocks "github.com/stackrox/rox/central/complianceoperator/v2/profiles/datastore/mocks" + "github.com/stackrox/rox/central/convert/internaltov2storage" "github.com/stackrox/rox/central/convert/testutils" "github.com/stackrox/rox/central/sensor/service/pipeline/reconciliation" "github.com/stackrox/rox/generated/internalapi/central" @@ -49,7 +50,8 @@ func (s *PipelineTestSuite) TestRunCreate() { ctx := context.Background() pipeline := NewPipeline(s.v2ProfileDS) - s.v2ProfileDS.EXPECT().UpsertProfile(ctx, testutils.GetProfileV2Storage(s.T())).Return(nil).Times(1) + expectedProfile := internaltov2storage.ComplianceOperatorProfileV2(testutils.GetProfileV2SensorMsg(s.T()), fixtureconsts.Cluster1) + s.v2ProfileDS.EXPECT().UpsertProfile(ctx, expectedProfile).Return(nil).Times(1) msg := ¢ral.MsgFromSensor{ Msg: ¢ral.MsgFromSensor_Event{ diff --git a/central/complianceoperator/v2/pipelines/complianceoperatorrulesv2/pipeline.go b/central/complianceoperator/v2/pipelines/complianceoperatorrulesv2/pipeline.go index 891b46de0c6d3..2338bfc2cd918 100644 --- a/central/complianceoperator/v2/pipelines/complianceoperatorrulesv2/pipeline.go +++ b/central/complianceoperator/v2/pipelines/complianceoperatorrulesv2/pipeline.go @@ -80,7 +80,7 @@ func (s *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.M event := msg.GetEvent() rule := event.GetComplianceOperatorRuleV2() - if val := rule.GetAnnotations()[v1alpha1.RuleIDAnnotationKey]; val == "" { + if rule.GetOperatorKind() != central.ComplianceOperatorRuleV2_CUSTOM_RULE && rule.GetAnnotations()[v1alpha1.RuleIDAnnotationKey] == "" { return errors.Errorf("Rule %s is missing the annotation %s", rule.GetName(), v1alpha1.RuleIDAnnotationKey) } diff --git a/central/complianceoperator/v2/pipelines/complianceoperatorrulesv2/pipeline_test.go b/central/complianceoperator/v2/pipelines/complianceoperatorrulesv2/pipeline_test.go index 7731c7950d474..8fa2ac626061f 100644 --- a/central/complianceoperator/v2/pipelines/complianceoperatorrulesv2/pipeline_test.go +++ b/central/complianceoperator/v2/pipelines/complianceoperatorrulesv2/pipeline_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/ComplianceAsCode/compliance-operator/pkg/apis/compliance/v1alpha1" v2Mocks "github.com/stackrox/rox/central/complianceoperator/v2/rules/datastore/mocks" "github.com/stackrox/rox/central/convert/testutils" "github.com/stackrox/rox/central/sensor/service/pipeline/reconciliation" @@ -151,3 +152,62 @@ func (s *PipelineTestSuite) TestMatch() { s.Require().True(s.pipeline.Match(v2Msg)) s.Require().False(s.pipeline.Match(otherMsg)) } + +// TestRunCreateRuleMissingAnnotationFails tests that rules missing the RuleID annotation are rejected. +// OPERATOR_KIND_UNSPECIFIED covers the legacy sensor compatibility path where older sensors don't set OperatorKind. +func (s *PipelineTestSuite) TestRunCreateRuleMissingAnnotationFails() { + cases := []central.ComplianceOperatorRuleV2_OperatorKind{ + central.ComplianceOperatorRuleV2_OPERATOR_KIND_UNSPECIFIED, + central.ComplianceOperatorRuleV2_RULE, + } + for _, kind := range cases { + s.Run(kind.String(), func() { + ctx := context.Background() + + rule := testutils.GetRuleV2SensorMsg(s.T()).CloneVT() + rule.OperatorKind = kind + delete(rule.GetAnnotations(), v1alpha1.RuleIDAnnotationKey) + + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Id: testutils.RuleUID, + Action: central.ResourceAction_CREATE_RESOURCE, + Resource: ¢ral.SensorEvent_ComplianceOperatorRuleV2{ + ComplianceOperatorRuleV2: rule, + }, + }, + }, + } + + err := s.pipeline.Run(ctx, fixtureconsts.Cluster1, msg, nil) + s.Error(err) + s.Contains(err.Error(), v1alpha1.RuleIDAnnotationKey) + }) + } +} + +func (s *PipelineTestSuite) TestRunCreateCustomRuleMissingRuleIDAnnotationSucceeds() { + ctx := context.Background() + + rule := testutils.GetRuleV2SensorMsg(s.T()).CloneVT() + rule.OperatorKind = central.ComplianceOperatorRuleV2_CUSTOM_RULE + delete(rule.GetAnnotations(), v1alpha1.RuleIDAnnotationKey) + + s.v2DS.EXPECT().UpsertRule(ctx, gomock.Any()).Return(nil).Times(1) + + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Id: testutils.RuleUID, + Action: central.ResourceAction_CREATE_RESOURCE, + Resource: ¢ral.SensorEvent_ComplianceOperatorRuleV2{ + ComplianceOperatorRuleV2: rule, + }, + }, + }, + } + + err := s.pipeline.Run(ctx, fixtureconsts.Cluster1, msg, nil) + s.NoError(err) +} diff --git a/central/complianceoperator/v2/profiles/datastore/datastore_impl.go b/central/complianceoperator/v2/profiles/datastore/datastore_impl.go index c7f0d4108d99b..eb15e5c8ddd58 100644 --- a/central/complianceoperator/v2/profiles/datastore/datastore_impl.go +++ b/central/complianceoperator/v2/profiles/datastore/datastore_impl.go @@ -119,18 +119,17 @@ func (d *datastoreImpl) GetProfilesNames(ctx context.Context, q *v1.Query, clust } parsedQuery.Pagination = q.GetPagination() - var results []*distinctProfileName - results, err = pgSearch.RunSelectRequestForSchema[distinctProfileName](ctx, d.db, schema.ComplianceOperatorProfileV2Schema, parsedQuery) + var profileNames []string + err = pgSearch.RunSelectRequestForSchemaFn[distinctProfileName](ctx, d.db, schema.ComplianceOperatorProfileV2Schema, parsedQuery, func(r *distinctProfileName) error { + profileNames = append(profileNames, r.ProfileName) + return nil + }) if err != nil { return nil, err } - if len(results) == 0 { + if len(profileNames) == 0 { return nil, nil } - profileNames := make([]string, 0, len(results)) - for _, result := range results { - profileNames = append(profileNames, result.ProfileName) - } return profileNames, err } @@ -158,12 +157,15 @@ func (d *datastoreImpl) CountDistinctProfiles(ctx context.Context, q *v1.Query, }, } - var results []*distinctProfileCount - results, err := pgSearch.RunSelectRequestForSchema[distinctProfileCount](ctx, d.db, schema.ComplianceOperatorProfileV2Schema, withCountQuery(query, search.ComplianceOperatorProfileName)) + var count int + err := pgSearch.RunSelectRequestForSchemaFn[distinctProfileCount](ctx, d.db, schema.ComplianceOperatorProfileV2Schema, withCountQuery(query, search.ComplianceOperatorProfileName), func(r *distinctProfileCount) error { + count++ + return nil + }) if err != nil { return 0, err } - return len(results), nil + return count, nil } func withCountQuery(query *v1.Query, field search.FieldLabel) *v1.Query { diff --git a/central/complianceoperator/v2/profiles/datastore/datastore_impl_test.go b/central/complianceoperator/v2/profiles/datastore/datastore_impl_test.go index 77ff8b84c8bd0..62e96cbc003bd 100644 --- a/central/complianceoperator/v2/profiles/datastore/datastore_impl_test.go +++ b/central/complianceoperator/v2/profiles/datastore/datastore_impl_test.go @@ -400,7 +400,7 @@ func getTestProfile(profileUID string, profileName string, version string, clust if ruleCount > 0 { rules = make([]*storage.ComplianceOperatorProfileV2_Rule, 0, ruleCount) - for i := 0; i < ruleCount; i++ { + for i := range ruleCount { rules = append(rules, &storage.ComplianceOperatorProfileV2_Rule{ RuleName: fmt.Sprintf("name-%d", i), }) diff --git a/central/complianceoperator/v2/profiles/service/service_impl.go b/central/complianceoperator/v2/profiles/service/service_impl.go index 6be54016dc67c..bc622b3e5cbf4 100644 --- a/central/complianceoperator/v2/profiles/service/service_impl.go +++ b/central/complianceoperator/v2/profiles/service/service_impl.go @@ -163,6 +163,9 @@ func (s *serviceImpl) ListProfileSummaries(ctx context.Context, request *v2.Clus if err != nil { return nil, errors.Wrapf(errox.InvalidArgs, "Unable to retrieve compliance profiles for %v", request) } + if len(profileNames) == 0 { + return &v2.ListComplianceProfileSummaryResponse{}, nil + } // Build query to get the filtered list by profile names profileQuery := search.NewQueryBuilder().AddSelectFields().AddExactMatches(search.ComplianceOperatorProfileName, profileNames...).ProtoQuery() diff --git a/central/complianceoperator/v2/profiles/service/service_impl_test.go b/central/complianceoperator/v2/profiles/service/service_impl_test.go index bdd456453967f..c301d0710042a 100644 --- a/central/complianceoperator/v2/profiles/service/service_impl_test.go +++ b/central/complianceoperator/v2/profiles/service/service_impl_test.go @@ -156,6 +156,26 @@ func (s *ComplianceProfilesServiceTestSuite) TestListProfileSummaries() { setMocks: func() { }, }, + { + desc: "No profiles match filter", + query: &apiV2.ClustersProfileSummaryRequest{ + ClusterIds: []string{fixtureconsts.Cluster1}, + Query: &apiV2.RawQuery{Query: "Compliance Profile Name:nonexistent"}, + }, + expectedErr: nil, + expectedResp: []*apiV2.ComplianceProfileSummary{}, + found: true, + setMocks: func() { + profileQuery, _ := search.ParseQuery("Compliance Profile Name:nonexistent", search.MatchAllIfEmpty()) + paginated.FillPaginationV2(profileQuery, nil, maxPaginationLimit) + profileQuery.Pagination.SortOptions = []*apiV1.QuerySortOption{ + { + Field: search.ComplianceOperatorProfileName.String(), + }, + } + s.profileDatastore.EXPECT().GetProfilesNames(gomock.Any(), profileQuery, []string{fixtureconsts.Cluster1}).Return(nil, nil).Times(1) + }, + }, { desc: "Query with cluster 1", query: &apiV2.ClustersProfileSummaryRequest{ClusterIds: []string{fixtureconsts.Cluster1}}, @@ -174,6 +194,7 @@ func (s *ComplianceProfilesServiceTestSuite) TestListProfileSummaries() { ShortName: "CIS-OCP", Version: "1-5", }}, + OperatorKind: apiV2.ComplianceProfileSummary_PROFILE, }, }, found: true, @@ -200,6 +221,7 @@ func (s *ComplianceProfilesServiceTestSuite) TestListProfileSummaries() { Description: "this is a test", Title: "A Title", ProfileVersion: "1-5", + OperatorKind: storage.ComplianceOperatorProfileV2_PROFILE, Rules: []*storage.ComplianceOperatorProfileV2_Rule{ { RuleName: "test 1", diff --git a/central/complianceoperator/v2/profiles/store/postgres/store.go b/central/complianceoperator/v2/profiles/store/postgres/store.go index 371d2c43f517f..94553a512ed79 100644 --- a/central/complianceoperator/v2/profiles/store/postgres/store.go +++ b/central/complianceoperator/v2/profiles/store/postgres/store.go @@ -129,10 +129,11 @@ func insertIntoComplianceOperatorProfileV2(batch *pgx.Batch, obj *storage.Compli obj.GetStandard(), pgutils.NilOrUUID(obj.GetClusterId()), pgutils.NilOrUUID(obj.GetProfileRefId()), + obj.GetOperatorKind(), serialized, } - finalStr := "INSERT INTO compliance_operator_profile_v2 (Id, ProfileId, Name, ProfileVersion, ProductType, Standard, ClusterId, ProfileRefId, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, ProfileId = EXCLUDED.ProfileId, Name = EXCLUDED.Name, ProfileVersion = EXCLUDED.ProfileVersion, ProductType = EXCLUDED.ProductType, Standard = EXCLUDED.Standard, ClusterId = EXCLUDED.ClusterId, ProfileRefId = EXCLUDED.ProfileRefId, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO compliance_operator_profile_v2 (Id, ProfileId, Name, ProfileVersion, ProductType, Standard, ClusterId, ProfileRefId, OperatorKind, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, ProfileId = EXCLUDED.ProfileId, Name = EXCLUDED.Name, ProfileVersion = EXCLUDED.ProfileVersion, ProductType = EXCLUDED.ProductType, Standard = EXCLUDED.Standard, ClusterId = EXCLUDED.ClusterId, ProfileRefId = EXCLUDED.ProfileRefId, OperatorKind = EXCLUDED.OperatorKind, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) var query string @@ -172,6 +173,7 @@ var copyColsComplianceOperatorProfileV2 = []string{ "standard", "clusterid", "profilerefid", + "operatorkind", "serialized", } @@ -214,6 +216,7 @@ func copyFromComplianceOperatorProfileV2(ctx context.Context, s pgSearch.Deleter obj.GetStandard(), pgutils.NilOrUUID(obj.GetClusterId()), pgutils.NilOrUUID(obj.GetProfileRefId()), + obj.GetOperatorKind(), serialized, }, nil }) diff --git a/central/complianceoperator/v2/report/datastore/datastore_impl_test.go b/central/complianceoperator/v2/report/datastore/datastore_impl_test.go index 0fd841377480c..04f68983e2326 100644 --- a/central/complianceoperator/v2/report/datastore/datastore_impl_test.go +++ b/central/complianceoperator/v2/report/datastore/datastore_impl_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/google/uuid" reportStorage "github.com/stackrox/rox/central/complianceoperator/v2/report/store/postgres" scanConfigDS "github.com/stackrox/rox/central/complianceoperator/v2/scanconfigurations/datastore" "github.com/stackrox/rox/generated/storage" @@ -20,6 +19,7 @@ import ( "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/timestamp" + "github.com/stackrox/rox/pkg/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -194,25 +194,25 @@ func (s *complianceReportSnapshotDataStoreSuite) TestDeleteOrphaned() { storage.ComplianceOperatorReportStatus_FAILURE, storage.ComplianceOperatorReportStatus_PARTIAL_ERROR, } - var reports []*storage.ComplianceOperatorReportSnapshotV2 + reports := make([]*storage.ComplianceOperatorReportSnapshotV2, 0, len(runStates)*2+2) for _, state := range runStates { - id, _ := uuid.NewUUID() + id := uuid.NewV4() reports = append(reports, getTestReport(id.String(), uuidScanConfigStub1, getStatus(state, timestamp.Now().Protobuf(), nil, "", storage.ComplianceOperatorReportStatus_SCHEDULED, storage.ComplianceOperatorReportStatus_EMAIL), user)) - id, _ = uuid.NewUUID() + id = uuid.NewV4() reports = append(reports, getTestReport(id.String(), uuidScanConfigStub1, getStatus(state, timestamp.Now().Protobuf(), nil, "", storage.ComplianceOperatorReportStatus_SCHEDULED, storage.ComplianceOperatorReportStatus_DOWNLOAD), user)) } - id, _ := uuid.NewUUID() + id := uuid.NewV4() reports = append(reports, getTestReport(id.String(), uuidScanConfigStub1, getStatus(storage.ComplianceOperatorReportStatus_PARTIAL_SCAN_ERROR_EMAIL, timestamp.Now().Protobuf(), nil, "", storage.ComplianceOperatorReportStatus_SCHEDULED, storage.ComplianceOperatorReportStatus_EMAIL), user)) - id, _ = uuid.NewUUID() + id = uuid.NewV4() reports = append(reports, getTestReport(id.String(), uuidScanConfigStub1, getStatus(storage.ComplianceOperatorReportStatus_PARTIAL_SCAN_ERROR_DOWNLOAD, timestamp.Now().Protobuf(), nil, "", storage.ComplianceOperatorReportStatus_SCHEDULED, storage.ComplianceOperatorReportStatus_DOWNLOAD), diff --git a/central/complianceoperator/v2/report/errors.go b/central/complianceoperator/v2/report/errors.go index b73b6523761ec..af919a72dfe01 100644 --- a/central/complianceoperator/v2/report/errors.go +++ b/central/complianceoperator/v2/report/errors.go @@ -6,6 +6,7 @@ const ( COMPLIANCE_VERSION_ERROR = "Compliance Operator Version is older than 1.6.0" COMPLIANCE_NOT_INSTALLED = "Compliance Operator is not installed" INTERNAL_ERROR = "Internal Error" + SCAN_NOT_APPLICABLE = "All scans were not applicable for this cluster" SCAN_REMOVED_FMT = "Scan %s was removed" SCAN_TIMEOUT_FMT = "Timeout waiting for scan %s to finish" SCAN_TIMEOUT_SENSOR_DISCONNECTED_FMT = "Timeout waiting for scan %s to finish (Sensor disconnect during the scan)" diff --git a/central/complianceoperator/v2/report/manager/format/formatter.go b/central/complianceoperator/v2/report/manager/format/formatter.go index 72477ebc5e39a..7692db82356e1 100644 --- a/central/complianceoperator/v2/report/manager/format/formatter.go +++ b/central/complianceoperator/v2/report/manager/format/formatter.go @@ -5,6 +5,7 @@ import ( "bytes" "fmt" "io" + "time" "github.com/pkg/errors" "github.com/stackrox/rox/central/complianceoperator/v2/report" @@ -23,12 +24,14 @@ var ( "Control Reference", "Check(CCR)", "Profile(version)", + "Profile type", "Check Description", "Cluster", "Status", "Remediation", "Rationale", "Instructions", + "Assessment Time", } failedClusterCSVHeader = []string{ "Cluster ID", @@ -119,12 +122,14 @@ func generateRecord(row *report.ResultRow) []string { row.ControlRef, row.CheckName, row.Profile, + row.ProfileType, row.Description, row.ClusterName, row.Status, row.Remediation, row.Rationale, row.Instructions, + row.AssessmentTime, } } @@ -146,8 +151,25 @@ func getFileName(format string, clusterName string, timestamp *timestamppb.Times return fmt.Sprintf(format, fmt.Sprintf("%s_%d-%d-%d", clusterName, year, month, day)) } +type timestampedZipWriter struct { + w *zip.Writer +} + +func (t *timestampedZipWriter) Create(name string) (io.Writer, error) { + header := &zip.FileHeader{ + Name: name, + Method: zip.Deflate, + Modified: time.Now(), + } + return t.w.CreateHeader(header) +} + +func (t *timestampedZipWriter) Close() error { + return t.w.Close() +} + func createNewZipWriter(buf *bytes.Buffer) ZipWriter { - return zip.NewWriter(buf) + return ×tampedZipWriter{w: zip.NewWriter(buf)} } func createNewCSVWriter(header csv.Header, sort bool) CSVWriter { diff --git a/central/complianceoperator/v2/report/manager/format/formatter_test.go b/central/complianceoperator/v2/report/manager/format/formatter_test.go index aead83f4beab9..66f9836969343 100644 --- a/central/complianceoperator/v2/report/manager/format/formatter_test.go +++ b/central/complianceoperator/v2/report/manager/format/formatter_test.go @@ -1,15 +1,19 @@ package format import ( + "archive/zip" "bytes" "fmt" "strings" "testing" + "time" "github.com/pkg/errors" "github.com/stackrox/rox/central/complianceoperator/v2/report" "github.com/stackrox/rox/central/complianceoperator/v2/report/manager/format/mocks" "github.com/stackrox/rox/pkg/csv" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" "google.golang.org/protobuf/types/known/timestamppb" @@ -20,6 +24,24 @@ const ( clusterID2 = "cluster-2" ) +func TestTimestampedZipWriter_SetsModifiedTimestamp(t *testing.T) { + var buf bytes.Buffer + before := time.Now().Add(-time.Minute) + zw := createNewZipWriter(&buf) + + w, err := zw.Create("test.csv") + require.NoError(t, err) + _, err = w.Write([]byte("data")) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + require.Len(t, reader.File, 1) + assert.False(t, reader.File[0].Modified.IsZero(), "ZIP entry Modified timestamp should not be zero") + assert.True(t, reader.File[0].Modified.After(before), "ZIP entry Modified timestamp should be recent") +} + func TestComplianceReportingFormatter(t *testing.T) { suite.Run(t, new(ComplianceReportingFormatterSuite)) } @@ -140,13 +162,15 @@ func (s *ComplianceReportingFormatterSuite) Test_FormatCSVReportWithPartialFaile // Add partial results results[clusterID2] = []*report.ResultRow{ { - ClusterName: "test_cluster-2", - CheckName: "test_check-2", - Profile: "test_profile-2", - ControlRef: "test_control_ref-2", - Description: "description-2", - Status: "Fail", - Remediation: "remediation-2", + ClusterName: "test_cluster-2", + CheckName: "test_check-2", + Profile: "test_profile-2", + ProfileType: "Profile", + ControlRef: "test_control_ref-2", + Description: "description-2", + Status: "Fail", + Remediation: "remediation-2", + AssessmentTime: "Wed, 07 May 2025 14:00:00 UTC", }, } @@ -311,22 +335,26 @@ func getFakeReportData() map[string][]*report.ResultRow { results := make(map[string][]*report.ResultRow) results[clusterID1] = []*report.ResultRow{ { - ClusterName: "test_cluster-1", - CheckName: "test_check-1", - Profile: "test_profile-1", - ControlRef: "test_control_ref-1", - Description: "description-1", - Status: "Pass", - Remediation: "remediation-1", + ClusterName: "test_cluster-1", + CheckName: "test_check-1", + Profile: "test_profile-1", + ProfileType: "Profile", + ControlRef: "test_control_ref-1", + Description: "description-1", + Status: "Pass", + Remediation: "remediation-1", + AssessmentTime: "Wed, 07 May 2025 12:00:00 UTC", }, { - ClusterName: "test_cluster-1", - CheckName: "test_check-2", - Profile: "test_profile-2", - ControlRef: "test_control_ref-2", - Description: "description-2", - Status: "Fail", - Remediation: "remediation-2", + ClusterName: "test_cluster-1", + CheckName: "test_check-2", + Profile: "test_profile-2", + ProfileType: "Tailored Profile", + ControlRef: "test_control_ref-2", + Description: "description-2", + Status: "Fail", + Remediation: "remediation-2", + AssessmentTime: "Wed, 07 May 2025 13:00:00 UTC", }, } return results @@ -384,7 +412,7 @@ func compareStringSlice(t *testing.T, actual []string, expected []string) bool { t.Logf("Expected slice %v but got %v", expected, actual) return false } - for i := 0; i < len(actual); i++ { + for i := range actual { if strings.Compare(actual[i], expected[i]) != 0 { t.Logf("Expected slice %v but got %v", expected, actual) return false diff --git a/central/complianceoperator/v2/report/manager/manager_impl.go b/central/complianceoperator/v2/report/manager/manager_impl.go index 8e1f1934d77d9..3d7ea399125fe 100644 --- a/central/complianceoperator/v2/report/manager/manager_impl.go +++ b/central/complianceoperator/v2/report/manager/manager_impl.go @@ -179,7 +179,7 @@ func (m *managerImpl) Start() { func (m *managerImpl) Stop() { m.metricsTicker.Stop() - if m.isStarted.Load() { + if !m.isStarted.Load() { log.Error("Compliance report manager not started") return } @@ -215,8 +215,8 @@ func (m *managerImpl) Stop() { } func (m *managerImpl) generateReportNoLock(req *reportRequest) { - clusterIds := []string{} - profiles := []string{} + clusterIds := make([]string, 0, len(req.scanConfig.GetClusters())) + profiles := make([]string, 0, len(req.scanConfig.GetProfiles())) for _, cluster := range req.scanConfig.GetClusters() { clusterIds = append(clusterIds, cluster.GetClusterId()) } @@ -364,11 +364,15 @@ func (m *managerImpl) HandleScan(sensorCtx context.Context, scan *storage.Compli log.Debug("The scan is missing the LastStartedTime field") return nil } - if errors.Is(err, watcher.ErrScanAlreadyHandled) { + if errors.Is(err, watcher.ErrScanAlreadyHandled) && !watcher.IsScanNotApplicable(scan) { log.Debugf("Scan %s was already handled", scan.GetScanName()) return nil } - return err + if !errors.Is(err, watcher.ErrScanAlreadyHandled) { + return err + } + log.Debugf("Scan %s was already handled but is NOT-APPLICABLE, allowing through", scan.GetScanName()) + id = fmt.Sprintf("%s:%s", scan.GetClusterId(), scan.GetId()) } numChecks, err := watcher.GetExpectedNumChecks(scan) if err != nil { @@ -491,8 +495,12 @@ func (m *managerImpl) handleReadyScan() { Observe(timeActive.Minutes()) delete(m.watchingScansStartTime, scanWatcherResult.WatcherID) }) - if err := watcher.DeleteOldResults(m.automaticReportingCtx, scanWatcherResult, m.checkResultDataStore); err != nil { - log.Errorf("unable to delete old CheckResults: %v", err) + if scanWatcherResult.Error == nil || errors.Is(scanWatcherResult.Error, watcher.ErrScanRemoved) { + if err := watcher.DeleteOldResults(m.automaticReportingCtx, scanWatcherResult, m.checkResultDataStore); err != nil { + log.Errorf("unable to delete old CheckResults: %v", err) + } + } else { + log.Debugf("Skipping DeleteOldResults for scan %s: %v", scanWatcherResult.Scan.GetScanName(), scanWatcherResult.Error) } if errors.Is(scanWatcherResult.Error, watcher.ErrScanRemoved) { log.Debugf("Scan %s was removed", scanWatcherResult.Scan.GetScanName()) @@ -542,7 +550,7 @@ func (m *managerImpl) getOrCreateScanConfigWatcher(ctx context.Context, results if err != nil { return nil, nil, false, errors.Wrap(err, "unable to retrieve snapshots from the store") } - if len(snapshot) > 0 { + if len(snapshot) > 0 && !watcher.IsScanNotApplicable(results.Scan) { // We already handled a scan newer than this one, we ignore this scanResults return nil, nil, false, watcher.ErrScanAlreadyHandled } @@ -599,13 +607,11 @@ func (m *managerImpl) handleReadyScanConfig() { return } for scanConfigWatcherResult := range m.scanConfigReadyQueue.Seq(m.stopper.LowLevel().GetStopRequestSignal()) { - log.Debugf("Scan Config %s done with %d scans and %d reports", scanConfigWatcherResult.ScanConfig.GetScanConfigName(), len(scanConfigWatcherResult.ScanResults), len(scanConfigWatcherResult.ReportSnapshot)) + log.Infof("Scan config %s completed with %d scans, %d reports, error: %v", + scanConfigWatcherResult.ScanConfig.GetScanConfigName(), len(scanConfigWatcherResult.ScanResults), len(scanConfigWatcherResult.ReportSnapshot), scanConfigWatcherResult.Error) concurrency.WithLock(&m.watchingScanConfigsLock, func() { delete(m.watchingScanConfigs, scanConfigWatcherResult.WatcherID) }) - if err := watcher.DeleteOldResultsFromMissingScans(m.automaticReportingCtx, scanConfigWatcherResult, m.profileDataStore, m.scanDataStore, m.checkResultDataStore); err != nil { - log.Errorf("unable to delete old CheckResults: %v", err) - } m.generateReportsFromWatcherResults(scanConfigWatcherResult) } } diff --git a/central/complianceoperator/v2/report/manager/manager_test.go b/central/complianceoperator/v2/report/manager/manager_test.go index c569327e4ba91..1707e4617c5e0 100644 --- a/central/complianceoperator/v2/report/manager/manager_test.go +++ b/central/complianceoperator/v2/report/manager/manager_test.go @@ -3,7 +3,6 @@ package manager import ( "context" "fmt" - "strings" "testing" "time" @@ -18,7 +17,6 @@ import ( scanMocks "github.com/stackrox/rox/central/complianceoperator/v2/scans/datastore/mocks" bindingsDS "github.com/stackrox/rox/central/complianceoperator/v2/scansettingbindings/datastore/mocks" suiteDS "github.com/stackrox/rox/central/complianceoperator/v2/suites/datastore/mocks" - v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/env" @@ -27,13 +25,10 @@ import ( "github.com/stackrox/rox/pkg/grpc/authn/mocks" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/sac" - "github.com/stackrox/rox/pkg/search" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" - "google.golang.org/protobuf/types/known/timestamppb" - "k8s.io/utils/strings/slices" ) type ManagerTestSuite struct { @@ -92,194 +87,350 @@ func (m *ManagerTestSuite) TearDownTest() { func (m *ManagerTestSuite) TestHandleReportRequest() { m.T().Setenv(env.ReportExecutionMaxConcurrency.EnvVar(), "1") - identity := mocks.NewMockIdentity(m.mockCtrl) - identity.EXPECT().UID().AnyTimes().Return("user-id") - identity.EXPECT().FullName().AnyTimes().Return("user-name") - identity.EXPECT().FriendlyName().AnyTimes().Return("user-friendly-name") - m.scanConfigDataStore.EXPECT().GetScanConfigClusterStatus(gomock.Any(), newGetScanConfigClusterStatusMatcher(getTestScanConfig())).AnyTimes().Return(getTestClusterStatusFromScanConfig(getTestScanConfig()), nil) - m.suiteDataStore.EXPECT().GetSuites(gomock.Any(), newGetSuitesMatcher(getTestScanConfig())).AnyTimes() - m.bindingsDataStore.EXPECT().GetScanSettingBindings(gomock.Any(), newGetBindingMatcher(getTestScanConfig())).AnyTimes() - ctx := authn.ContextWithIdentity(context.Background(), identity, m.T()) - - m.Run("Successful report, no watchers running", func() { - manager := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) - manager.Start() - scanConfig := getTestScanConfig() - wg := concurrency.NewWaitGroup(1) - m.snapshotDataStore.EXPECT().UpsertSnapshot(gomock.Any(), gomock.Any()).Times(1). - Return(nil) - m.reportGen.EXPECT().ProcessReportRequest(gomock.Any()).Times(1). - DoAndReturn(func(_ any) error { - wg.Add(-1) - return nil - }) - m.snapshotDataStore.EXPECT(). - GetLastSnapshotFromScanConfig(gomock.Any(), gomock.Eq(scanConfig.GetId())). + + scanConfig := getTestScanConfig() + + newIdentityCtx := func() context.Context { + identity := mocks.NewMockIdentity(m.mockCtrl) + identity.EXPECT().UID().AnyTimes().Return("user-id") + identity.EXPECT().FullName().AnyTimes().Return("user-name") + identity.EXPECT().FriendlyName().AnyTimes().Return("user-friendly-name") + return authn.ContextWithIdentity(context.Background(), identity, m.T()) + } + + setupReportDataMocks := func() { + m.scanConfigDataStore.EXPECT().GetScanConfigClusterStatus(gomock.Any(), gomock.Eq(scanConfig.GetId())). + Times(1).Return(getTestClusterStatusFromScanConfig(scanConfig), nil) + m.suiteDataStore.EXPECT().GetSuites(gomock.Any(), gomock.Any()).Times(1).Return(nil, nil) + m.bindingsDataStore.EXPECT().GetScanSettingBindings(gomock.Any(), gomock.Any()). + Times(len(scanConfig.GetClusters())).Return(nil, nil) + } + + m.Run("No watcher running", func() { + setupReportDataMocks() + m.snapshotDataStore.EXPECT().UpsertSnapshot(gomock.Any(), gomock.Any()).Times(1).Return(nil) + m.snapshotDataStore.EXPECT().GetLastSnapshotFromScanConfig(gomock.Any(), gomock.Eq(scanConfig.GetId())). Times(1).Return(nil, nil) - m.scanDataStore.EXPECT(). - SearchScans(gomock.Any(), gomock.Any()). + m.scanDataStore.EXPECT().SearchScans(gomock.Any(), gomock.Any()). Times(len(scanConfig.GetClusters())). Return(getScans(len(scanConfig.GetProfiles())), nil) - err := manager.SubmitReportRequest(ctx, getTestScanConfig(), storage.ComplianceOperatorReportStatus_EMAIL) + m.reportGen.EXPECT().ProcessReportRequest(gomock.Any()).Times(1).Return(nil) + + mgr := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) + impl := mgr.(*managerImpl) + req := &reportRequest{ + scanConfig: scanConfig, + ctx: newIdentityCtx(), + notificationMethod: storage.ComplianceOperatorReportStatus_EMAIL, + } + generated, err := impl.handleReportRequest(req) m.Require().NoError(err) - handleWaitGroup(m.T(), &wg, 10*time.Millisecond, "report generation") + m.Assert().True(generated) + m.Assert().NotEmpty(req.snapshotID) }) - m.Run("Error in the database", func() { - manager := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) - manager.Start() - wg := concurrency.NewWaitGroup(1) + m.Run("Upsert snapshot error", func() { + setupReportDataMocks() m.snapshotDataStore.EXPECT().UpsertSnapshot(gomock.Any(), gomock.Any()).Times(1). - DoAndReturn(func(_, _ any) error { - wg.Add(-1) - return errors.New("some error") - }) - err := manager.SubmitReportRequest(ctx, getTestScanConfig(), storage.ComplianceOperatorReportStatus_EMAIL) + Return(errors.New("db error")) + + mgr := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) + impl := mgr.(*managerImpl) + req := &reportRequest{ + scanConfig: scanConfig, + ctx: newIdentityCtx(), + notificationMethod: storage.ComplianceOperatorReportStatus_EMAIL, + } + generated, err := impl.handleReportRequest(req) + m.Require().Error(err) + m.Assert().False(generated) + m.Assert().Contains(err.Error(), "unable to upsert snapshot") + }) + + m.Run("Watcher running, subscribes and defers report", func() { + setupReportDataMocks() + stubWatcher := &stubScanConfigWatcher{} + m.snapshotDataStore.EXPECT().UpsertSnapshot(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + mgr := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) + impl := mgr.(*managerImpl) + concurrency.WithLock(&impl.watchingScanConfigsLock, func() { + impl.watchingScanConfigs[scanConfig.GetId()] = stubWatcher + }) + + req := &reportRequest{ + scanConfig: scanConfig, + ctx: newIdentityCtx(), + notificationMethod: storage.ComplianceOperatorReportStatus_EMAIL, + } + generated, err := impl.handleReportRequest(req) m.Require().NoError(err) - handleWaitGroup(m.T(), &wg, 10*time.Millisecond, "storage error") + m.Assert().False(generated, "report should be deferred when watcher is running") + m.Assert().Len(stubWatcher.subscribedSnapshots, 1) + m.Assert().Equal(storage.ComplianceOperatorReportStatus_WAITING, stubWatcher.subscribedSnapshots[0].GetReportStatus().GetRunState()) }) - m.Run("Successful report, with watcher running", func() { - m.T().Skip("ROX-30838: Enable again") - manager := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) - manager.Start() - wg := concurrency.NewWaitGroup(2) - now := protocompat.TimestampNow() - scanConfig := getTestScanConfig() - scans := getTestScansFromScanConfig(scanConfig, now) - scan := getTestScan(scans[0].GetId(), scanConfig.GetScanConfigName(), scans[0].GetClusterId(), now, true) - // Setup EXPECT calls - calls := m.setupExpectCallsFromScanConfig(scanConfig, now) - calls = append(calls, m.setupExpectCallsFromFinishScan(scan, scanConfig, now)...) - gomock.InOrder(calls...) - // Expect upsert Snapshot in the SubmitReportRequest call - calls = []any{ - m.snapshotDataStore.EXPECT(). - UpsertSnapshot(gomock.Any(), gomock.Any()). - Times(1).Return(nil), + m.Run("Missing identity returns error", func() { + mgr := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) + impl := mgr.(*managerImpl) + req := &reportRequest{ + scanConfig: scanConfig, + ctx: context.Background(), + notificationMethod: storage.ComplianceOperatorReportStatus_EMAIL, } - calls = append(calls, m.setupExpectCallsFromFinishAllScans(scanConfig, scans[1:], now, 2)...) - calls = append(calls, m.reportGen.EXPECT(). - ProcessReportRequest(gomock.Any()). - Times(2). + generated, err := impl.handleReportRequest(req) + m.Require().Error(err) + m.Assert().False(generated) + m.Assert().Contains(err.Error(), "could not determine user identity") + }) +} + +func (m *ManagerTestSuite) TestGenerateReportFromWatcherResults() { + m.T().Setenv(env.ReportExecutionMaxConcurrency.EnvVar(), "1") + scanConfig := getTestScanConfig() + + setupReportDataMocks := func() { + m.scanConfigDataStore.EXPECT().GetScanConfigClusterStatus(gomock.Any(), gomock.Eq(scanConfig.GetId())). + Times(1).Return(getTestClusterStatusFromScanConfig(scanConfig), nil) + m.suiteDataStore.EXPECT().GetSuites(gomock.Any(), gomock.Any()).Times(1).Return(nil, nil) + m.bindingsDataStore.EXPECT().GetScanSettingBindings(gomock.Any(), gomock.Any()). + Times(len(scanConfig.GetClusters())).Return(nil, nil) + } + + m.Run("Successful report generation", func() { + setupReportDataMocks() + m.complianceIntegrationDataStore.EXPECT(). + GetComplianceIntegrationByCluster(gomock.Any(), gomock.Any()).AnyTimes().Return(nil, nil) + m.scanDataStore.EXPECT().SearchScans(gomock.Any(), gomock.Any()). + Times(len(scanConfig.GetClusters())). + Return(getScans(len(scanConfig.GetProfiles())), nil) + done := make(chan struct{}) + m.reportGen.EXPECT().ProcessReportRequest(gomock.Any()).Times(1). DoAndReturn(func(_ any) error { - wg.Add(-1) + close(done) return nil - })) - gomock.InAnyOrder(calls) - - // Push the Resources - m.pushScansAndResults(manager, scanConfig, now) - m.finishFirstScan(manager, scan, scanConfig) - - m.Eventually(func() bool { - managerImp, ok := manager.(*managerImpl) - m.Require().True(ok) - return concurrency.WithLock1[bool](&managerImp.watchingScanConfigsLock, func() bool { - return len(managerImp.watchingScanConfigs) > 0 }) - }, 100*time.Millisecond, 10*time.Millisecond) - err := manager.SubmitReportRequest(ctx, scanConfig, storage.ComplianceOperatorReportStatus_EMAIL) + mgr := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) + impl := mgr.(*managerImpl) + result := &watcher.ScanConfigWatcherResults{ + WatcherID: scanConfig.GetId(), + ScanConfig: scanConfig, + ScanResults: map[string]*watcher.ScanWatcherResults{ + "cluster-1:scan-1": {Scan: &storage.ComplianceOperatorScanV2{Id: "scan-1", ClusterId: "cluster-1"}}, + "cluster-2:scan-2": {Scan: &storage.ComplianceOperatorScanV2{Id: "scan-2", ClusterId: "cluster-2"}}, + }, + } + snapshot := &storage.ComplianceOperatorReportSnapshotV2{ + ReportId: "report-1", + ReportStatus: &storage.ComplianceOperatorReportStatus{ + ReportNotificationMethod: storage.ComplianceOperatorReportStatus_EMAIL, + }, + } + err := impl.generateSingleReportFromWatcherResults(result, snapshot) m.Require().NoError(err) + m.Assert().Equal(storage.ComplianceOperatorReportStatus_PREPARING, snapshot.GetReportStatus().GetRunState()) - m.Eventually(func() bool { - managerImp, ok := manager.(*managerImpl) - m.Require().True(ok) - return concurrency.WithLock1[bool](&managerImp.mu, func() bool { - return len(managerImp.runningReportConfigs) > 0 + select { + case <-done: + case <-time.After(2 * time.Second): + m.FailNow("timeout waiting for ProcessReportRequest") + } + }) + + m.Run("Report generation with failed clusters", func() { + setupReportDataMocks() + m.complianceIntegrationDataStore.EXPECT(). + GetComplianceIntegrationByCluster(gomock.Any(), gomock.Any()).AnyTimes().Return(nil, nil) + m.scanDataStore.EXPECT().SearchScans(gomock.Any(), gomock.Any()). + Times(len(scanConfig.GetClusters())). + Return(getScans(len(scanConfig.GetProfiles())), nil) + m.snapshotDataStore.EXPECT().UpsertSnapshot(gomock.Any(), gomock.Any()).Times(1).Return(nil) + done := make(chan struct{}) + m.reportGen.EXPECT().ProcessReportRequest(gomock.Any()).Times(1). + DoAndReturn(func(_ any) error { + close(done) + return nil }) - }, 100*time.Millisecond, 10*time.Millisecond) - time.Sleep(100 * time.Millisecond) + mgr := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) + impl := mgr.(*managerImpl) + result := &watcher.ScanConfigWatcherResults{ + WatcherID: scanConfig.GetId(), + ScanConfig: scanConfig, + Error: watcher.ErrScanConfigTimeout, + ScanResults: map[string]*watcher.ScanWatcherResults{ + "cluster-1:scan-1": { + Scan: &storage.ComplianceOperatorScanV2{Id: "scan-1", ClusterId: "cluster-1"}, + Error: watcher.ErrScanTimeout, + }, + }, + } + snapshot := &storage.ComplianceOperatorReportSnapshotV2{ + ReportId: "report-1", + ReportStatus: &storage.ComplianceOperatorReportStatus{ + ReportNotificationMethod: storage.ComplianceOperatorReportStatus_EMAIL, + }, + } + err := impl.generateSingleReportFromWatcherResults(result, snapshot) + m.Require().NoError(err) + m.Assert().Equal(storage.ComplianceOperatorReportStatus_PREPARING, snapshot.GetReportStatus().GetRunState()) + m.Assert().NotEmpty(snapshot.GetReportStatus().GetErrorMsg()) - m.finishScans(manager, scans[1:]) + select { + case <-done: + case <-time.After(2 * time.Second): + m.FailNow("timeout waiting for ProcessReportRequest") + } + }) +} - m.Eventually(func() bool { - managerImp, ok := manager.(*managerImpl) - m.Require().True(ok) - return concurrency.WithLock1[bool](&managerImp.watchingScanConfigsLock, func() bool { - return len(managerImp.watchingScanConfigs) == 0 - }) - }, 100*time.Millisecond, 10*time.Millisecond) - handleWaitGroup(m.T(), &wg, 500*time.Millisecond, "reports to be generated") +func (m *ManagerTestSuite) TestCreateAutomaticSnapshotAndSubscribe() { + scanConfig := getTestScanConfig() + + m.Run("No notifiers returns error", func() { + noNotifierConfig := getTestScanConfig() + noNotifierConfig.Notifiers = nil + + mgr := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) + impl := mgr.(*managerImpl) + stubWatcher := &stubScanConfigWatcher{} + err := impl.createAutomaticSnapshotAndSubscribe(m.ctx, noNotifierConfig, stubWatcher) + m.Require().Error(err) + m.Assert().Len(stubWatcher.subscribedSnapshots, 0) }) + m.Run("With notifiers subscribes and upserts snapshot", func() { + m.scanConfigDataStore.EXPECT().GetScanConfigClusterStatus(gomock.Any(), gomock.Eq(scanConfig.GetId())). + Times(1).Return(getTestClusterStatusFromScanConfig(scanConfig), nil) + m.suiteDataStore.EXPECT().GetSuites(gomock.Any(), gomock.Any()).Times(1).Return(nil, nil) + m.bindingsDataStore.EXPECT().GetScanSettingBindings(gomock.Any(), gomock.Any()). + Times(len(scanConfig.GetClusters())).Return(nil, nil) + m.snapshotDataStore.EXPECT().UpsertSnapshot(gomock.Any(), gomock.Any()).Times(1).Return(nil) + + mgr := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) + impl := mgr.(*managerImpl) + stubWatcher := &stubScanConfigWatcher{} + err := impl.createAutomaticSnapshotAndSubscribe(m.ctx, scanConfig, stubWatcher) + m.Require().NoError(err) + m.Assert().Len(stubWatcher.subscribedSnapshots, 1) + m.Assert().Equal(storage.ComplianceOperatorReportStatus_WAITING, stubWatcher.subscribedSnapshots[0].GetReportStatus().GetRunState()) + m.Assert().Equal(storage.ComplianceOperatorReportStatus_SCHEDULED, stubWatcher.subscribedSnapshots[0].GetReportStatus().GetReportRequestType()) + }) } -func (m *ManagerTestSuite) TestFailedReportWithWatcherRunningAndNoNotifiers() { - m.T().Setenv(env.ReportExecutionMaxConcurrency.EnvVar(), "1") - identity := mocks.NewMockIdentity(m.mockCtrl) - identity.EXPECT().UID().AnyTimes().Return("user-id") - identity.EXPECT().FullName().AnyTimes().Return("user-name") - identity.EXPECT().FriendlyName().AnyTimes().Return("user-friendly-name") - m.scanConfigDataStore.EXPECT().GetScanConfigClusterStatus(gomock.Any(), newGetScanConfigClusterStatusMatcher(getTestScanConfig())).AnyTimes().Return(getTestClusterStatusFromScanConfig(getTestScanConfig()), nil) - m.suiteDataStore.EXPECT().GetSuites(gomock.Any(), newGetSuitesMatcher(getTestScanConfig())).AnyTimes() - m.bindingsDataStore.EXPECT().GetScanSettingBindings(gomock.Any(), newGetBindingMatcher(getTestScanConfig())).AnyTimes() - ctx := authn.ContextWithIdentity(context.Background(), identity, m.T()) - - // Set the timeouts to 2 so the scan watchers timeout fast - m.T().Setenv(env.ComplianceScanWatcherTimeout.EnvVar(), "2s") - // The scan config watcher should not timeout, if it does then the test is broken. - // We still set it to 20s to not have the test hagging for 10 minutes before the timeout. - m.T().Setenv(env.ComplianceScanScheduleWatcherTimeout.EnvVar(), "20s") - - manager := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) - manager.Start() +func (m *ManagerTestSuite) TestHandleReadyScanDeleteOldResultsGate() { now := protocompat.TimestampNow() - scanConfig := getTestScanConfig() - scanConfig.Notifiers = nil - scans := getTestScansFromScanConfig(scanConfig, now) - scan := getTestScan(scans[0].GetId(), scanConfig.GetScanConfigName(), scans[0].GetClusterId(), now, true) - wg := concurrency.NewWaitGroup(1) - // Setup EXPECT calls - calls := m.setupExpectCallsFromScanConfig(scanConfig, now) - calls = append(calls, m.setupExpectCallsFromFinishScan(scan, scanConfig, now)...) - gomock.InOrder(calls...) - calls = []any{ - m.snapshotDataStore.EXPECT(). - UpsertSnapshot(gomock.Any(), gomock.Any()). - Times(1).Return(nil), + scan := &storage.ComplianceOperatorScanV2{ + Id: "scan-1", + ClusterId: "cluster-1", + ScanConfigName: "test-scan", + ScanRefId: "ref-1", + LastStartedTime: now, + } + + tests := map[string]struct { + err error + expectFn func(done chan struct{}, checkResultDS *checkResultsMocks.MockDataStore, scanConfigDS *scanConfigurationDS.MockDataStore) + }{ + "success should call DeleteOldResults": { + err: nil, + expectFn: func(done chan struct{}, checkResultDS *checkResultsMocks.MockDataStore, scanConfigDS *scanConfigurationDS.MockDataStore) { + checkResultDS.EXPECT(). + DeleteOldResults(gomock.Any(), gomock.Eq(scan.GetLastStartedTime()), gomock.Eq(scan.GetScanRefId()), gomock.Eq(false)). + Times(1).Return(nil) + scanConfigDS.EXPECT(). + GetScanConfigurationByName(gomock.Any(), gomock.Eq(scan.GetScanConfigName())). + Times(1). + DoAndReturn(func(_ any, _ any) (*storage.ComplianceOperatorScanConfigurationV2, error) { + close(done) + return nil, errors.New("stop here") + }) + }, + }, + "ErrScanRemoved should call DeleteOldResults with includeCurrentResults": { + err: watcher.ErrScanRemoved, + expectFn: func(done chan struct{}, checkResultDS *checkResultsMocks.MockDataStore, scanConfigDS *scanConfigurationDS.MockDataStore) { + checkResultDS.EXPECT(). + DeleteOldResults(gomock.Any(), gomock.Eq(scan.GetLastStartedTime()), gomock.Eq(scan.GetScanRefId()), gomock.Eq(true)). + Times(1). + DoAndReturn(func(_, _, _, _ any) error { + close(done) + return nil + }) + }, + }, + "ErrScanTimeout should NOT call DeleteOldResults": { + err: watcher.ErrScanTimeout, + expectFn: func(done chan struct{}, checkResultDS *checkResultsMocks.MockDataStore, scanConfigDS *scanConfigurationDS.MockDataStore) { + scanConfigDS.EXPECT(). + GetScanConfigurationByName(gomock.Any(), gomock.Eq(scan.GetScanConfigName())). + Times(1). + DoAndReturn(func(_ any, _ any) (*storage.ComplianceOperatorScanConfigurationV2, error) { + close(done) + return nil, errors.New("stop here") + }) + }, + }, + "ErrScanContextCancelled should NOT call DeleteOldResults": { + err: watcher.ErrScanContextCancelled, + expectFn: func(done chan struct{}, checkResultDS *checkResultsMocks.MockDataStore, scanConfigDS *scanConfigurationDS.MockDataStore) { + scanConfigDS.EXPECT(). + GetScanConfigurationByName(gomock.Any(), gomock.Eq(scan.GetScanConfigName())). + Times(1). + DoAndReturn(func(_ any, _ any) (*storage.ComplianceOperatorScanConfigurationV2, error) { + close(done) + return nil, errors.New("stop here") + }) + }, + }, } - calls = append(calls, m.setupExpectCallsFromFailAllScans(scanConfig, scans[1:], now, 1)...) - calls = append(calls, m.reportGen.EXPECT(). - ProcessReportRequest(gomock.Any()). - Times(1). - DoAndReturn(func(_ any) error { - wg.Add(-1) - return nil - }), - ) - gomock.InAnyOrder(calls) - - // Push the resources - m.pushScansAndResults(manager, scanConfig, now) - m.finishFirstScan(manager, scan, scanConfig) - // The rest of the scan should time out after 1s - m.Eventually(func() bool { - managerImp, ok := manager.(*managerImpl) - m.Require().True(ok) - return concurrency.WithLock1[bool](&managerImp.watchingScanConfigsLock, func() bool { - return len(managerImp.watchingScanConfigs) > 0 - }) - }, 100*time.Millisecond, 10*time.Millisecond) - m.Require().NoError(manager.SubmitReportRequest(ctx, scanConfig, storage.ComplianceOperatorReportStatus_DOWNLOAD)) + for name, tc := range tests { + m.Run(name, func() { + ctrl := gomock.NewController(m.T()) + checkResultDS := checkResultsMocks.NewMockDataStore(ctrl) + scanConfigDS := scanConfigurationDS.NewMockDataStore(ctrl) + scanDS := scanMocks.NewMockDataStore(ctrl) + profileDS := profileMocks.NewMockDataStore(ctrl) + snapshotDS := snapshotMocks.NewMockDataStore(ctrl) + integrationDS := integrationMocks.NewMockDataStore(ctrl) + suiteStore := suiteDS.NewMockDataStore(ctrl) + bindingsStore := bindingsDS.NewMockDataStore(ctrl) + generator := reportGen.NewMockComplianceReportGenerator(ctrl) + generator.EXPECT().Stop().AnyTimes() + + manager := New(scanConfigDS, scanDS, profileDS, snapshotDS, integrationDS, suiteStore, bindingsStore, checkResultDS, generator) + manager.Start() + m.T().Cleanup(func() { + manager.Stop() + ctrl.Finish() + }) + managerImp := manager.(*managerImpl) - managerImp, ok := manager.(*managerImpl) - m.Require().True(ok) - m.Eventually(func() bool { - return concurrency.WithLock1[bool](&managerImp.watchingScanConfigsLock, func() bool { - return len(managerImp.watchingScanConfigs) == 0 - }) - }, 10*time.Second, 10*time.Millisecond) - // The runningReportConfigs should be empty at this point - m.Eventually(func() bool { - return concurrency.WithLock1[bool](&managerImp.mu, func() bool { - return len(managerImp.runningReportConfigs) == 0 + result := &watcher.ScanWatcherResults{ + WatcherID: "watcher-1", + Scan: scan, + Error: tc.err, + } + + done := make(chan struct{}) + tc.expectFn(done, checkResultDS, scanConfigDS) + + // Seed a start time entry so the metrics code doesn't panic. + concurrency.WithLock(&managerImp.watchingScansLock, func() { + managerImp.watchingScansStartTime[result.WatcherID] = time.Now() + }) + + managerImp.readyQueue.Push(result) + + select { + case <-done: + case <-time.After(2 * time.Second): + m.FailNow("timeout waiting for handleReadyScan to process the result") + } }) - }, 5*time.Second, 100*time.Millisecond) - handleWaitGroup(m.T(), &wg, 500*time.Millisecond, "reports to be generated") + } } func (m *ManagerTestSuite) TestHandleScan() { @@ -322,10 +473,12 @@ func (m *ManagerTestSuite) TestHandleScan() { } func (m *ManagerTestSuite) TestHandleScanRemove() { + m.reportGen.EXPECT().Stop().AnyTimes() manager := New(m.scanConfigDataStore, m.scanDataStore, m.profileDataStore, m.snapshotDataStore, m.complianceIntegrationDataStore, m.suiteDataStore, m.bindingsDataStore, m.checkResultDataStore, m.reportGen) managerImplementation, ok := manager.(*managerImpl) require.True(m.T(), ok) manager.Start() + defer manager.Stop() scanID := "scan-1" scan := &storage.ComplianceOperatorScanV2{ @@ -505,219 +658,6 @@ func getTestScanConfig() *storage.ComplianceOperatorScanConfigurationV2 { } } -func (m *ManagerTestSuite) pushScansAndResults(manager Manager, sc *storage.ComplianceOperatorScanConfigurationV2, timestamp *protocompat.Timestamp) { - ctx := context.Background() - for _, cluster := range sc.GetClusters() { - for _, profile := range sc.GetProfiles() { - scan := getTestScan(profile.GetProfileName(), sc.GetScanConfigName(), cluster.GetClusterId(), timestamp, false) - result := getTestResult(scan, timestamp) - err := manager.HandleScan(ctx, scan) - require.NoError(m.T(), err) - err = manager.HandleResult(ctx, result) - require.NoError(m.T(), err) - } - } -} - -func (m *ManagerTestSuite) finishFirstScan(manager Manager, scan *storage.ComplianceOperatorScanV2, sc *storage.ComplianceOperatorScanConfigurationV2) { - ctx := context.Background() - err := manager.HandleScan(ctx, scan) - require.NoError(m.T(), err) -} - -func (m *ManagerTestSuite) finishScans(manager Manager, scans []*storage.ComplianceOperatorScanV2) { - ctx := context.Background() - managerImp, ok := manager.(*managerImpl) - m.Require().True(ok) - numScanWatchers := concurrency.WithLock1[int](&managerImp.watchingScansLock, func() int { - return len(managerImp.watchingScans) - }) - for _, scan := range scans { - require.NoError(m.T(), manager.HandleScan(ctx, scan)) - m.Eventually(func() bool { - return concurrency.WithLock1[bool](&managerImp.watchingScansLock, func() bool { - return len(managerImp.watchingScans) == numScanWatchers-1 - }) - }, 100*time.Millisecond, 10*time.Millisecond) - numScanWatchers-- - } -} - -func getTestScansFromScanConfig(sc *storage.ComplianceOperatorScanConfigurationV2, timestamp *protocompat.Timestamp) []*storage.ComplianceOperatorScanV2 { - var ret []*storage.ComplianceOperatorScanV2 - for _, cluster := range sc.GetClusters() { - for _, profile := range sc.GetProfiles() { - ret = append(ret, getTestScan(profile.GetProfileName(), sc.GetScanConfigName(), cluster.GetClusterId(), timestamp, true)) - } - } - return ret -} - -func (m *ManagerTestSuite) setupExpectCallsFromScanConfig(sc *storage.ComplianceOperatorScanConfigurationV2, timestamp *protocompat.Timestamp) []any { - var expectedCalls []any - for _, cluster := range sc.GetClusters() { - for _, profile := range sc.GetProfiles() { - scan := getTestScan(profile.GetProfileName(), sc.GetScanConfigName(), cluster.GetClusterId(), timestamp, false) - calls := []any{ - m.scanConfigDataStore.EXPECT(). - GetScanConfigurations(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorScanConfigurationV2{sc}, nil), - m.snapshotDataStore.EXPECT(). - SearchSnapshots(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorReportSnapshotV2{}, nil), - m.scanDataStore.EXPECT().SearchScans(gomock.Any(), gomock.Any()). - Times(1).Return([]*storage.ComplianceOperatorScanV2{scan}, nil), - m.scanConfigDataStore.EXPECT(). - GetScanConfigurations(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorScanConfigurationV2{getTestScanConfig()}, nil), - m.snapshotDataStore.EXPECT(). - SearchSnapshots(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorReportSnapshotV2{}, nil), - } - expectedCalls = append(expectedCalls, calls...) - } - } - return expectedCalls -} - -func (m *ManagerTestSuite) setupExpectCallsFromFinishAllScans(sc *storage.ComplianceOperatorScanConfigurationV2, scans []*storage.ComplianceOperatorScanV2, timestamp *timestamppb.Timestamp, numSnapshots int) []any { - var expectedCalls []any - for _, scan := range scans { - calls := []any{ - m.scanConfigDataStore.EXPECT(). - GetScanConfigurations(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorScanConfigurationV2{sc}, nil), - m.snapshotDataStore.EXPECT(). - SearchSnapshots(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorReportSnapshotV2{}, nil), - m.checkResultDataStore.EXPECT(). - DeleteOldResults(gomock.Any(), gomock.Eq(scan.GetLastStartedTime()), gomock.Eq(scan.GetScanRefId()), gomock.Eq(false)). - Times(1).Return(nil), - m.scanConfigDataStore.EXPECT(). - GetScanConfigurationByName(gomock.Any(), gomock.Eq(sc.GetScanConfigName())). - Times(1).Return(sc, nil), - m.snapshotDataStore.EXPECT(). - UpsertSnapshot(gomock.Any(), gomock.Any()). - Times(numSnapshots). - Return(nil), - } - expectedCalls = append(expectedCalls, calls...) - } - allScans := getTestScansFromScanConfig(sc, timestamp) - calls := []any{ - // Delete Old Results of Missing Clusters - m.profileDataStore.EXPECT(). - SearchProfiles(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorProfileV2{{}}, nil), - m.scanDataStore.EXPECT(). - SearchScans(gomock.Any(), gomock.Any()). - Times(1).Return(allScans, nil), - m.scanDataStore.EXPECT(). - SearchScans(gomock.Any(), gomock.Any()). - Times(len(sc.GetClusters())*numSnapshots). - Return(scans, nil), - } - expectedCalls = append(expectedCalls, calls...) - return expectedCalls -} - -func (m *ManagerTestSuite) setupExpectCallsFromFailAllScans(sc *storage.ComplianceOperatorScanConfigurationV2, scans []*storage.ComplianceOperatorScanV2, timestamp *timestamppb.Timestamp, numSnapshots int) []any { - var expectedCalls []any - for _, scan := range scans { - calls := []any{ - m.scanConfigDataStore.EXPECT(). - GetScanConfigurationByName(gomock.Any(), gomock.Eq(sc.GetScanConfigName())). - Times(1).Return(sc, nil), - m.checkResultDataStore.EXPECT(). - DeleteOldResults(gomock.Any(), gomock.Eq(scan.GetLastStartedTime()), gomock.Eq(scan.GetScanRefId()), gomock.Eq(true)). - Times(1).Return(nil), - m.snapshotDataStore.EXPECT(). - UpsertSnapshot(gomock.Any(), gomock.Any()). - Times(numSnapshots).Return(nil), - } - expectedCalls = append(expectedCalls, calls...) - } - allScans := getTestScansFromScanConfig(sc, timestamp) - calls := []any{ - // Delete Old Results of Missing Clusters - m.profileDataStore.EXPECT(). - SearchProfiles(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorProfileV2{{}}, nil), - m.scanDataStore.EXPECT(). - SearchScans(gomock.Any(), gomock.Any()). - Times(1).Return(allScans, nil), - // Validate Results - m.complianceIntegrationDataStore.EXPECT(). - GetComplianceIntegrationByCluster(gomock.Any(), gomock.Any()). - Times(len(scans)).Return(nil, nil), - // Upsert Snapshots - m.snapshotDataStore.EXPECT(). - UpsertSnapshot(gomock.Any(), gomock.Cond[*storage.ComplianceOperatorReportSnapshotV2](func(target *storage.ComplianceOperatorReportSnapshotV2) bool { - return target.GetReportStatus().GetRunState() == storage.ComplianceOperatorReportStatus_PREPARING - - })). - Times(numSnapshots).Return(nil), - // GetClusterData - m.scanDataStore.EXPECT(). - SearchScans(gomock.Any(), gomock.Any()). - Times(len(sc.GetClusters())*numSnapshots). - Return(scans, nil), - } - expectedCalls = append(expectedCalls, calls...) - return expectedCalls -} - -func (m *ManagerTestSuite) setupExpectCallsFromFinishScan(scan *storage.ComplianceOperatorScanV2, sc *storage.ComplianceOperatorScanConfigurationV2, timestamp *timestamppb.Timestamp) []any { - scans := getTestScansFromScanConfig(sc, timestamp) - calls := []any{ - m.scanConfigDataStore.EXPECT(). - GetScanConfigurations(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorScanConfigurationV2{sc}, nil), - m.snapshotDataStore.EXPECT(). - SearchSnapshots(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorReportSnapshotV2{}, nil), - m.checkResultDataStore.EXPECT(). - DeleteOldResults(gomock.Any(), gomock.Eq(scan.GetLastStartedTime()), gomock.Eq(scan.GetScanRefId()), gomock.Eq(false)). - Times(1).Return(nil), - m.scanConfigDataStore.EXPECT(). - GetScanConfigurationByName(gomock.Any(), gomock.Eq(sc.GetScanConfigName())). - Times(1).Return(sc, nil), - m.snapshotDataStore.EXPECT(). - SearchSnapshots(gomock.Any(), gomock.Any()). - Times(1).Return(nil, nil), - } - if sc.GetNotifiers() != nil { - calls = append(calls, m.snapshotDataStore.EXPECT(). - UpsertSnapshot(gomock.Any(), gomock.Any()). - Times(1).Return(nil)) - } - calls = append(calls, []any{ - m.profileDataStore.EXPECT(). - SearchProfiles(gomock.Any(), gomock.Any()). - Times(1). - Return([]*storage.ComplianceOperatorProfileV2{{}}, nil), - m.scanDataStore.EXPECT(). - SearchScans(gomock.Any(), gomock.Any()). - Times(1).Return(scans, nil), - }...) - if sc.GetNotifiers() != nil { - calls = append(calls, m.snapshotDataStore.EXPECT(). - UpsertSnapshot(gomock.Any(), gomock.Any()). - Times(1).Return(nil)) - } - return calls -} - func getTestClusterStatusFromScanConfig(sc *storage.ComplianceOperatorScanConfigurationV2) []*storage.ComplianceOperatorClusterScanConfigStatus { ret := make([]*storage.ComplianceOperatorClusterScanConfigStatus, 0, len(sc.GetClusters())) for _, c := range sc.GetClusters() { @@ -729,42 +669,9 @@ func getTestClusterStatusFromScanConfig(sc *storage.ComplianceOperatorScanConfig return ret } -func getTestScan(scan, scanConfigName, cluster string, timestamp *timestamppb.Timestamp, done bool) *storage.ComplianceOperatorScanV2 { - ret := &storage.ComplianceOperatorScanV2{ - Id: scan, - ClusterId: cluster, - LastStartedTime: timestamp, - ScanConfigName: scanConfigName, - } - if done { - ret.Annotations = map[string]string{ - watcher.CheckCountAnnotationKey: "1", - } - } - return ret -} - -func getTestResult(scan *storage.ComplianceOperatorScanV2, timestamp *protocompat.Timestamp) *storage.ComplianceOperatorCheckResultV2 { - return &storage.ComplianceOperatorCheckResultV2{ - ScanRefId: scan.GetScanRefId(), - Annotations: map[string]string{ - watcher.LastScannedAnnotationKey: timestamp.AsTime().Format(time.RFC3339Nano), - }, - } -} - -func handleWaitGroup(t *testing.T, wg *concurrency.WaitGroup, timeout time.Duration, msg string) { - select { - case <-time.After(timeout): - t.Errorf("timeout waiting for %s", msg) - t.Fail() - case <-wg.Done(): - } -} - func getScans(numProfiles int) []*storage.ComplianceOperatorScanV2 { var ret []*storage.ComplianceOperatorScanV2 - for i := 0; i < numProfiles; i++ { + for i := range numProfiles { name := fmt.Sprintf("profile-%d", i) ret = append(ret, &storage.ComplianceOperatorScanV2{ ScanName: name, @@ -773,111 +680,26 @@ func getScans(numProfiles int) []*storage.ComplianceOperatorScanV2 { return ret } -func newGetScanConfigClusterStatusMatcher(sc *storage.ComplianceOperatorScanConfigurationV2) *getScanConfigClusterStatusMatcher { - return &getScanConfigClusterStatusMatcher{ - scanConfigID: sc.GetId(), - } -} - -type getScanConfigClusterStatusMatcher struct { - scanConfigID string - error string -} - -func (m *getScanConfigClusterStatusMatcher) Matches(target interface{}) bool { - scanConfigID, ok := target.(string) - if !ok { - m.error = "target is not of type string" - return false - } - m.error = fmt.Sprintf("expected field scan configuration ID %q", m.scanConfigID) - return m.scanConfigID == scanConfigID -} - -func (m *getScanConfigClusterStatusMatcher) String() string { - return m.error -} - -func newGetSuitesMatcher(sc *storage.ComplianceOperatorScanConfigurationV2) *getSuitesMatcher { - return &getSuitesMatcher{ - suiteName: sc.GetScanConfigName(), - } -} - -type getSuitesMatcher struct { - suiteName string - error string -} - -func (m *getSuitesMatcher) Matches(target interface{}) bool { - query, ok := target.(*v1.Query) - if !ok { - m.error = "target is not of type *v1.Query" - return false - } - m.error = fmt.Sprintf("expected field suite name %q", m.suiteName) - field := query.GetBaseQuery().GetMatchFieldQuery().GetField() - if field != search.ComplianceOperatorSuiteName.String() { - m.error = fmt.Sprintf("unexpected query field %s", field) - return false - } - value := strings.ReplaceAll(query.GetBaseQuery().GetMatchFieldQuery().GetValue(), "\"", "") - return value == m.suiteName +type stubScanConfigWatcher struct { + subscribedSnapshots []*storage.ComplianceOperatorReportSnapshotV2 + scans []*storage.ComplianceOperatorReportSnapshotV2_Scan } -func (m *getSuitesMatcher) String() string { - return m.error -} +func (s *stubScanConfigWatcher) PushScanResults(_ *watcher.ScanWatcherResults) error { return nil } -func newGetBindingMatcher(sc *storage.ComplianceOperatorScanConfigurationV2) *getBindingMatcher { - return &getBindingMatcher{ - scanConfigName: sc.GetScanConfigName(), - clusters: func() []string { - ret := make([]string, 0, len(sc.GetClusters())) - for _, c := range sc.GetClusters() { - ret = append(ret, c.GetClusterId()) - } - return ret - }(), - } +func (s *stubScanConfigWatcher) Subscribe(snapshot *storage.ComplianceOperatorReportSnapshotV2) error { + s.subscribedSnapshots = append(s.subscribedSnapshots, snapshot) + return nil } -type getBindingMatcher struct { - scanConfigName string - clusters []string - error string +func (s *stubScanConfigWatcher) GetScans() []*storage.ComplianceOperatorReportSnapshotV2_Scan { + return s.scans } -func (m *getBindingMatcher) Matches(target interface{}) bool { - query, ok := target.(*v1.Query) - if !ok { - m.error = "target is not of type *v1.Query" - return false - } - m.error = fmt.Sprintf("expected fields scan configuration name %q and clusters %v", m.scanConfigName, m.clusters) - scanConfigFound := false - clustersFound := false - for _, q := range query.GetConjunction().GetQueries() { - field := q.GetBaseQuery().GetMatchFieldQuery().GetField() - switch field { - case search.ComplianceOperatorScanConfigName.String(): - value := strings.ReplaceAll(q.GetBaseQuery().GetMatchFieldQuery().GetValue(), "\"", "") - if value == m.scanConfigName { - scanConfigFound = true - } - case search.ClusterID.String(): - value := strings.ReplaceAll(q.GetBaseQuery().GetMatchFieldQuery().GetValue(), "\"", "") - if slices.Contains(m.clusters, value) { - clustersFound = true - } - default: - m.error = fmt.Sprintf("unexpected query field %s", field) - return false - } - } - return scanConfigFound && clustersFound -} +func (s *stubScanConfigWatcher) Stop() {} -func (m *getBindingMatcher) String() string { - return m.error +func (s *stubScanConfigWatcher) Finished() concurrency.ReadOnlySignal { + sig := concurrency.NewSignal() + sig.Signal() + return &sig } diff --git a/central/complianceoperator/v2/report/manager/results/results_aggregator.go b/central/complianceoperator/v2/report/manager/results/results_aggregator.go index c0fae685bbf93..e7733febfdddd 100644 --- a/central/complianceoperator/v2/report/manager/results/results_aggregator.go +++ b/central/complianceoperator/v2/report/manager/results/results_aggregator.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" checkResults "github.com/stackrox/rox/central/complianceoperator/v2/checkresults/datastore" "github.com/stackrox/rox/central/complianceoperator/v2/checkresults/utils" @@ -14,13 +15,19 @@ import ( scanDS "github.com/stackrox/rox/central/complianceoperator/v2/scans/datastore" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/set" ) const ( DATA_NOT_AVAILABLE = "Data Not Available" - NO_REMEDIATION = "No Remediation Available" + // CONTROL_NOT_APPLICABLE is used when a control reference cannot be resolved for structural reasons + // (e.g. tailored profiles with no benchmark mapping, or profiles like E8 whose benchmark short name + // has no matching standard in rule annotations). DATA_NOT_AVAILABLE is used instead when the lookup + // fails due to errors or data-integrity issues. + CONTROL_NOT_APPLICABLE = "N/A" + NO_REMEDIATION = "No Remediation Available" ) var ( @@ -96,6 +103,13 @@ func (g *Aggregator) getReportDataForCluster(ctx context.Context, scanConfigID, } successfulScanNames := clusterData.ScanNames if clusterData.FailedInfo != nil { + if len(clusterData.FailedInfo.FailedScans) == 0 { + // The entire cluster failed to report (e.g. sensor disconnected, + // scan config watcher timed out without receiving any results from + // this cluster). Exclude all results to avoid including stale data + // from a previous scan cycle in the report. + return ret, statuses, nil + } allScansSet := set.NewStringSet(successfulScanNames...) failedScansSet := set.NewStringSet() for _, scan := range clusterData.FailedInfo.FailedScans { @@ -115,18 +129,20 @@ func (g *Aggregator) getReportDataForCluster(ctx context.Context, scanConfigID, func (g *Aggregator) AggregateResults(ctx context.Context, clusterID string, clusterResults *[]*report.ResultRow, checkStatus *checkStatus) checkResultWalkByQuery { return func(checkResult *storage.ComplianceOperatorCheckResultV2) error { row := &report.ResultRow{ - ClusterName: checkResult.GetClusterName(), - CheckName: checkResult.GetCheckName(), - Description: checkResult.GetDescription(), - Status: checkResult.GetStatus().String(), - Rationale: checkResult.GetRationale(), - Instructions: checkResult.GetInstructions(), + ClusterName: checkResult.GetClusterName(), + CheckName: checkResult.GetCheckName(), + Description: checkResult.GetDescription(), + Status: checkResult.GetStatus().String(), + Rationale: checkResult.GetRationale(), + Instructions: checkResult.GetInstructions(), + AssessmentTime: protocompat.ConvertTimestampToString(checkResult.GetLastStartedTime(), time.RFC1123), } - profileInfo, profileName, err := g.getProfileInfo(ctx, checkResult, clusterID) + profileInfo, profileName, profileType, err := g.getProfileInfo(ctx, checkResult, clusterID) if err != nil { return err } row.Profile = profileInfo + row.ProfileType = profileType remediationInfo, err := g.getRemediationInfo(ctx, checkResult, clusterID) if err != nil { return err @@ -143,19 +159,32 @@ func (g *Aggregator) AggregateResults(ctx context.Context, clusterID string, clu } } -func (g *Aggregator) getProfileInfo(ctx context.Context, checkResult *storage.ComplianceOperatorCheckResultV2, clusterID string) (string, string, error) { +func (g *Aggregator) getProfileInfo(ctx context.Context, checkResult *storage.ComplianceOperatorCheckResultV2, clusterID string) (string, string, string, error) { q := search.NewQueryBuilder(). AddExactMatches(search.ComplianceOperatorScanRef, checkResult.GetScanRefId()). ProtoQuery() profiles, err := g.profileDS.SearchProfiles(ctx, q) if err != nil { - return "", "", err + return "", "", "", err } if len(profiles) < 1 { log.Errorf("profile not found for cluster %s and check name %s", clusterID, checkResult.GetCheckName()) - return DATA_NOT_AVAILABLE, "", nil + return DATA_NOT_AVAILABLE, "", DATA_NOT_AVAILABLE, nil + } + profileInfo := fmt.Sprintf("%s %s", profiles[0].GetName(), profiles[0].GetProfileVersion()) + profileType := operatorKindToHumanReadable(profiles[0].GetOperatorKind()) + return profileInfo, profiles[0].GetName(), profileType, nil +} + +func operatorKindToHumanReadable(kind storage.ComplianceOperatorProfileV2_OperatorKind) string { + switch kind { + case storage.ComplianceOperatorProfileV2_PROFILE: + return "Profile" + case storage.ComplianceOperatorProfileV2_TAILORED_PROFILE: + return "Tailored Profile" + default: + return DATA_NOT_AVAILABLE } - return fmt.Sprintf("%s %s", profiles[0].GetName(), profiles[0].GetProfileVersion()), profiles[0].GetName(), nil } func (g *Aggregator) getRemediationInfo(ctx context.Context, checkResult *storage.ComplianceOperatorCheckResultV2, clusterID string) (string, error) { @@ -195,7 +224,7 @@ func (g *Aggregator) getControlsInfo(ctx context.Context, checkResult *storage.C return DATA_NOT_AVAILABLE, err } if len(controls) == 0 { - return DATA_NOT_AVAILABLE, nil + return CONTROL_NOT_APPLICABLE, nil } controlsList := make([]string, 0, len(controls)) for _, ctrl := range controls { diff --git a/central/complianceoperator/v2/report/manager/results/results_aggregator_test.go b/central/complianceoperator/v2/report/manager/results/results_aggregator_test.go index 73d888ce3e2fa..7d25f693efb5c 100644 --- a/central/complianceoperator/v2/report/manager/results/results_aggregator_test.go +++ b/central/complianceoperator/v2/report/manager/results/results_aggregator_test.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "testing" + "time" "github.com/pkg/errors" checkResultsMocks "github.com/stackrox/rox/central/complianceoperator/v2/checkresults/datastore/mocks" @@ -21,10 +22,12 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" ) const ( - scanConfigID = "scan-config-id" + scanConfigID = "scan-config-id" + expectedFormattedTimestamp = "Wed, 07 May 2025 12:00:00 UTC" ) func TestComplianceReportingDataGenerator(t *testing.T) { @@ -50,6 +53,7 @@ type getReportDataTestCase struct { numFailedChecksPerCluster int numMixedChecksPerCluster int numFailedClusters int + numFullyFailedClusters int expectedWalkByErr error } @@ -70,6 +74,14 @@ func (s *ComplianceResultsAggregatorSuite) Test_GetReportDataResultsGeneration() numMixedChecksPerCluster: 3, numFailedClusters: 1, }, + "generate report data with fully failed cluster excludes stale results": { + numClusters: 2, + numProfiles: 2, + numPassedChecksPerCluster: 2, + numFailedChecksPerCluster: 1, + numMixedChecksPerCluster: 3, + numFullyFailedClusters: 1, + }, "generate report walk by error": { numClusters: 3, numProfiles: 4, @@ -79,9 +91,11 @@ func (s *ComplianceResultsAggregatorSuite) Test_GetReportDataResultsGeneration() for tname, tcase := range cases { s.Run(tname, func() { ctx := context.Background() - req := getRequest(ctx, tcase.numClusters, tcase.numProfiles, tcase.numFailedClusters) + req := getRequest(ctx, tcase.numClusters, tcase.numProfiles, tcase.numFailedClusters, tcase.numFullyFailedClusters) + // Fully-failed clusters (empty FailedScans) return early without calling WalkByQuery + walkByTimes := tcase.numClusters + tcase.numFailedClusters s.checkResultsDS.EXPECT().WalkByQuery(gomock.Eq(ctx), gomock.Any(), gomock.Any()). - Times(tcase.numClusters + tcase.numFailedClusters). + Times(walkByTimes). DoAndReturn(fakeWalkByResponse( req.ClusterData, tcase.expectedWalkByErr, @@ -113,19 +127,19 @@ func fakeWalkByResponse( } } } - for i := 0; i < numPassedChecksPerCluster; i++ { + for i := range numPassedChecksPerCluster { _ = fn(&storage.ComplianceOperatorCheckResultV2{ CheckName: fmt.Sprintf("pass-check-%d", i), Status: storage.ComplianceOperatorCheckResultV2_PASS, }) } - for i := 0; i < numFailedChecksPerCluster; i++ { + for i := range numFailedChecksPerCluster { _ = fn(&storage.ComplianceOperatorCheckResultV2{ CheckName: fmt.Sprintf("fail-check-%d", i), Status: storage.ComplianceOperatorCheckResultV2_FAIL, }) } - for i := 0; i < numMixedChecksPerCluster; i++ { + for i := range numMixedChecksPerCluster { _ = fn(&storage.ComplianceOperatorCheckResultV2{ CheckName: fmt.Sprintf("mixed-check-%d", i), Status: storage.ComplianceOperatorCheckResultV2_INCONSISTENT, @@ -140,6 +154,7 @@ var ( { Name: "ocp4-cis", ProfileVersion: "1.7.0", + OperatorKind: storage.ComplianceOperatorProfileV2_PROFILE, }, } remediations = []*storage.ComplianceOperatorRemediationV2{ @@ -166,13 +181,14 @@ var ( ) type walkByQueryTestCase struct { - check *storage.ComplianceOperatorCheckResultV2 - expectedProfiles func() ([]*storage.ComplianceOperatorProfileV2, error) - expectedRemediations func() ([]*storage.ComplianceOperatorRemediationV2, error) - expectedRules func() ([]*storage.ComplianceOperatorRuleV2, error) - expectedBenchmarks func() ([]*storage.ComplianceOperatorBenchmarkV2, error) - expectedControls func() ([]*datastore.ControlResult, error) - expectError bool + check *storage.ComplianceOperatorCheckResultV2 + expectedProfiles func() ([]*storage.ComplianceOperatorProfileV2, error) + expectedRemediations func() ([]*storage.ComplianceOperatorRemediationV2, error) + expectedRules func() ([]*storage.ComplianceOperatorRuleV2, error) + expectedBenchmarks func() ([]*storage.ComplianceOperatorBenchmarkV2, error) + expectedControls func() ([]*datastore.ControlResult, error) + expectedAssessmentTime string + expectError bool } func (s *ComplianceResultsAggregatorSuite) Test_WalkByQuery() { @@ -195,6 +211,7 @@ func (s *ComplianceResultsAggregatorSuite) Test_WalkByQuery() { expectedControls: func() ([]*datastore.ControlResult, error) { return controls, nil }, + expectedAssessmentTime: expectedFormattedTimestamp, }, "fail check no error": { check: getCheckResult(storage.ComplianceOperatorCheckResultV2_FAIL), @@ -213,6 +230,7 @@ func (s *ComplianceResultsAggregatorSuite) Test_WalkByQuery() { expectedControls: func() ([]*datastore.ControlResult, error) { return controls, nil }, + expectedAssessmentTime: expectedFormattedTimestamp, }, "mixed check no error": { check: getCheckResult(storage.ComplianceOperatorCheckResultV2_INCONSISTENT), @@ -231,6 +249,7 @@ func (s *ComplianceResultsAggregatorSuite) Test_WalkByQuery() { expectedControls: func() ([]*datastore.ControlResult, error) { return controls, nil }, + expectedAssessmentTime: expectedFormattedTimestamp, }, "profile search error": { check: getCheckResult(storage.ComplianceOperatorCheckResultV2_PASS), @@ -253,6 +272,7 @@ func (s *ComplianceResultsAggregatorSuite) Test_WalkByQuery() { expectedBenchmarks: func() ([]*storage.ComplianceOperatorBenchmarkV2, error) { return benchmarks, nil }, + expectedAssessmentTime: expectedFormattedTimestamp, }, "remediation search error": { check: getCheckResult(storage.ComplianceOperatorCheckResultV2_PASS), @@ -281,6 +301,7 @@ func (s *ComplianceResultsAggregatorSuite) Test_WalkByQuery() { expectedControls: func() ([]*datastore.ControlResult, error) { return controls, nil }, + expectedAssessmentTime: expectedFormattedTimestamp, }, "rule search error": { check: getCheckResult(storage.ComplianceOperatorCheckResultV2_PASS), @@ -306,6 +327,7 @@ func (s *ComplianceResultsAggregatorSuite) Test_WalkByQuery() { expectedRules: func() ([]*storage.ComplianceOperatorRuleV2, error) { return []*storage.ComplianceOperatorRuleV2{}, nil }, + expectedAssessmentTime: expectedFormattedTimestamp, }, "benchmark not found": { check: getCheckResult(storage.ComplianceOperatorCheckResultV2_PASS), @@ -326,6 +348,7 @@ func (s *ComplianceResultsAggregatorSuite) Test_WalkByQuery() { expectedBenchmarks: func() ([]*storage.ComplianceOperatorBenchmarkV2, error) { return []*storage.ComplianceOperatorBenchmarkV2{}, nil }, + expectedAssessmentTime: expectedFormattedTimestamp, }, "control search error": { check: getCheckResult(storage.ComplianceOperatorCheckResultV2_PASS), @@ -363,6 +386,106 @@ func (s *ComplianceResultsAggregatorSuite) Test_WalkByQuery() { expectedControls: func() ([]*datastore.ControlResult, error) { return []*datastore.ControlResult{}, nil }, + expectedAssessmentTime: expectedFormattedTimestamp, + }, + "profile without benchmark mapping renders control reference as N/A": { + check: getCheckResult(storage.ComplianceOperatorCheckResultV2_PASS), + expectedProfiles: func() ([]*storage.ComplianceOperatorProfileV2, error) { + return []*storage.ComplianceOperatorProfileV2{ + { + Name: "my-custom-tailored-profile", + ProfileVersion: "1.0.0", + OperatorKind: storage.ComplianceOperatorProfileV2_TAILORED_PROFILE, + }, + }, nil + }, + expectedRemediations: func() ([]*storage.ComplianceOperatorRemediationV2, error) { + return remediations, nil + }, + expectedRules: func() ([]*storage.ComplianceOperatorRuleV2, error) { + return rules, nil + }, + expectedBenchmarks: func() ([]*storage.ComplianceOperatorBenchmarkV2, error) { + return []*storage.ComplianceOperatorBenchmarkV2{}, nil + }, + expectedAssessmentTime: expectedFormattedTimestamp, + }, + "profile with benchmark but no matching controls renders control reference as N/A": { + check: getCheckResult(storage.ComplianceOperatorCheckResultV2_PASS), + expectedProfiles: func() ([]*storage.ComplianceOperatorProfileV2, error) { + return []*storage.ComplianceOperatorProfileV2{ + { + Name: "ocp4-e8", + ProfileVersion: "1.0.0", + OperatorKind: storage.ComplianceOperatorProfileV2_PROFILE, + }, + }, nil + }, + expectedRemediations: func() ([]*storage.ComplianceOperatorRemediationV2, error) { + return remediations, nil + }, + expectedRules: func() ([]*storage.ComplianceOperatorRuleV2, error) { + return rules, nil + }, + expectedBenchmarks: func() ([]*storage.ComplianceOperatorBenchmarkV2, error) { + return benchmarks, nil + }, + expectedControls: func() ([]*datastore.ControlResult, error) { + return []*datastore.ControlResult{}, nil + }, + expectedAssessmentTime: expectedFormattedTimestamp, + }, + // getCheckResult not used here to test nil LastStartedTime + "nil last started time renders assessment time as N/A": { + check: &storage.ComplianceOperatorCheckResultV2{ + ClusterName: "cluster-1", + CheckName: "check", + Description: "description", + Status: storage.ComplianceOperatorCheckResultV2_PASS, + Rationale: "rationale", + Instructions: "instructions", + }, + expectedProfiles: func() ([]*storage.ComplianceOperatorProfileV2, error) { + return profiles, nil + }, + expectedRemediations: func() ([]*storage.ComplianceOperatorRemediationV2, error) { + return remediations, nil + }, + expectedRules: func() ([]*storage.ComplianceOperatorRuleV2, error) { + return rules, nil + }, + expectedBenchmarks: func() ([]*storage.ComplianceOperatorBenchmarkV2, error) { + return benchmarks, nil + }, + expectedControls: func() ([]*datastore.ControlResult, error) { + return controls, nil + }, + expectedAssessmentTime: "N/A", + }, + "profile matching benchmark regex resolves controls": { + check: getCheckResult(storage.ComplianceOperatorCheckResultV2_PASS), + expectedProfiles: func() ([]*storage.ComplianceOperatorProfileV2, error) { + return []*storage.ComplianceOperatorProfileV2{ + { + Name: "ocp4-cis", + ProfileVersion: "1.4.0", + OperatorKind: storage.ComplianceOperatorProfileV2_PROFILE, + }, + }, nil + }, + expectedRemediations: func() ([]*storage.ComplianceOperatorRemediationV2, error) { + return remediations, nil + }, + expectedRules: func() ([]*storage.ComplianceOperatorRuleV2, error) { + return rules, nil + }, + expectedBenchmarks: func() ([]*storage.ComplianceOperatorBenchmarkV2, error) { + return benchmarks, nil + }, + expectedControls: func() ([]*datastore.ControlResult, error) { + return controls, nil + }, + expectedAssessmentTime: expectedFormattedTimestamp, }, } for tname, tcase := range cases { @@ -405,7 +528,7 @@ func (s *ComplianceResultsAggregatorSuite) SetupTest() { s.aggregator = NewAggregator(s.checkResultsDS, s.scanDS, s.profileDS, s.remediationDS, s.ruleDS) } -func getRequest(ctx context.Context, numClusters, numProfiles, numFailedClusters int) *report.Request { +func getRequest(ctx context.Context, numClusters, numProfiles, numFailedClusters, numFullyFailedClusters int) *report.Request { ret := &report.Request{ Ctx: ctx, ScanConfigID: scanConfigID, @@ -413,10 +536,11 @@ func getRequest(ctx context.Context, numClusters, numProfiles, numFailedClusters Profiles: getNames("profile", numProfiles), } clusterData := make(map[string]*report.ClusterData) - for i := 0; i < numClusters+numFailedClusters; i++ { + totalExtra := numFailedClusters + numFullyFailedClusters + for i := 0; i < numClusters+totalExtra; i++ { id := fmt.Sprintf("cluster-%d", i) var profileNames []string - for j := 0; j < numProfiles; j++ { + for j := range numProfiles { profileNames = append(profileNames, fmt.Sprintf("profile-%d", j)) } clusterData[id] = &report.ClusterData{ @@ -448,13 +572,29 @@ func getRequest(ctx context.Context, numClusters, numProfiles, numFailedClusters } ret.NumFailedClusters = numFailedClusters } + // Fully-failed clusters: FailedInfo set but FailedScans empty (e.g. sensor + // disconnected, cluster never reported). Results aggregator should skip these + // entirely to avoid including stale data. + if numFullyFailedClusters > 0 { + offset := numClusters + numFailedClusters + for i := offset; i < offset+numFullyFailedClusters; i++ { + id := fmt.Sprintf("cluster-%d", i) + ret.ClusterIDs = append(ret.ClusterIDs, id) + clusterData[id].FailedInfo = &report.FailedCluster{ + ClusterId: id, + ClusterName: id, + Reasons: []string{"cluster did not report any results"}, + } + } + ret.NumFailedClusters += numFullyFailedClusters + } ret.ClusterData = clusterData return ret } func getNames(prefix string, num int) []string { ret := make([]string, 0, 2) - for i := 0; i < num; i++ { + for i := range num { ret = append(ret, fmt.Sprintf("%s-%d", prefix, i)) } return ret @@ -470,20 +610,22 @@ func mockWalkByQueryWrapper(_ context.Context, clusterID string, clusterResults func getRowFromCluster(check, clusterID string) *report.ResultRow { return &report.ResultRow{ - ClusterName: clusterID, - CheckName: fmt.Sprintf("check-%s-%s", clusterID, check), - Description: fmt.Sprintf("description-%s-%s", clusterID, check), - Status: fmt.Sprintf("status-%s-%s", clusterID, check), - Rationale: fmt.Sprintf("rationale-%s-%s", clusterID, check), - Instructions: fmt.Sprintf("instructions-%s-%s", clusterID, check), - Profile: fmt.Sprintf("profile-%s-%s", clusterID, check), - ControlRef: fmt.Sprintf("control-%s-%s", clusterID, check), - Remediation: fmt.Sprintf("remediation=%s-%s", clusterID, check), + ClusterName: clusterID, + CheckName: fmt.Sprintf("check-%s-%s", clusterID, check), + Description: fmt.Sprintf("description-%s-%s", clusterID, check), + Status: fmt.Sprintf("status-%s-%s", clusterID, check), + Rationale: fmt.Sprintf("rationale-%s-%s", clusterID, check), + Instructions: fmt.Sprintf("instructions-%s-%s", clusterID, check), + Profile: fmt.Sprintf("profile-%s-%s", clusterID, check), + ProfileType: fmt.Sprintf("profiletype-%s-%s", clusterID, check), + ControlRef: fmt.Sprintf("control-%s-%s", clusterID, check), + Remediation: fmt.Sprintf("remediation=%s-%s", clusterID, check), + AssessmentTime: "N/A", } } func assertResults(t *testing.T, tcase getReportDataTestCase, res *report.Results) { - assert.Equal(t, tcase.numClusters+tcase.numFailedClusters, res.Clusters) + assert.Equal(t, tcase.numClusters+tcase.numFailedClusters+tcase.numFullyFailedClusters, res.Clusters) assert.Equal(t, tcase.numProfiles, len(res.Profiles)) if tcase.expectedWalkByErr != nil { assert.Equal(t, 0, res.TotalPass) @@ -516,12 +658,13 @@ func assertResults(t *testing.T, tcase getReportDataTestCase, res *report.Result func getCheckResult(status storage.ComplianceOperatorCheckResultV2_CheckStatus) *storage.ComplianceOperatorCheckResultV2 { return &storage.ComplianceOperatorCheckResultV2{ - ClusterName: "cluster-1", - CheckName: "check", - Description: "description", - Status: status, - Rationale: "rationale", - Instructions: "instructions", + ClusterName: "cluster-1", + CheckName: "check", + Description: "description", + Status: status, + Rationale: "rationale", + Instructions: "instructions", + LastStartedTime: timestamppb.New(time.Date(2025, 5, 7, 12, 0, 0, 0, time.UTC)), } } @@ -549,13 +692,16 @@ func assertResult(t *testing.T, tcase walkByQueryTestCase, row *report.ResultRow assert.Equal(t, tcase.check.GetStatus().String(), row.Status) assert.Equal(t, tcase.check.GetRationale(), row.Rationale) assert.Equal(t, tcase.check.GetInstructions(), row.Instructions) + assert.Equal(t, tcase.expectedAssessmentTime, row.AssessmentTime) if tcase.expectedProfiles != nil { expProfiles, _ := tcase.expectedProfiles() if len(expProfiles) < 1 { assert.Equal(t, DATA_NOT_AVAILABLE, row.Profile) + assert.Equal(t, DATA_NOT_AVAILABLE, row.ProfileType) } else { require.Len(t, expProfiles, 1) assert.Equal(t, fmt.Sprintf("%s %s", expProfiles[0].GetName(), expProfiles[0].GetProfileVersion()), row.Profile) + assert.Equal(t, operatorKindToHumanReadable(expProfiles[0].GetOperatorKind()), row.ProfileType) } } if tcase.expectedRemediations != nil { @@ -584,7 +730,7 @@ func assertResult(t *testing.T, tcase walkByQueryTestCase, row *report.ResultRow } expBench, _ := tcase.expectedBenchmarks() if len(expBench) == 0 { - assert.Equal(t, DATA_NOT_AVAILABLE, row.ControlRef) + assert.Equal(t, CONTROL_NOT_APPLICABLE, row.ControlRef) return } if tcase.expectedControls == nil { @@ -592,7 +738,7 @@ func assertResult(t *testing.T, tcase walkByQueryTestCase, row *report.ResultRow } expControls, _ := tcase.expectedControls() if len(expControls) == 0 { - assert.Equal(t, DATA_NOT_AVAILABLE, row.ControlRef) + assert.Equal(t, CONTROL_NOT_APPLICABLE, row.ControlRef) return } expControlInfos := make([]string, 0, len(expControls)) diff --git a/central/complianceoperator/v2/report/manager/watcher/scanconfigwatcher.go b/central/complianceoperator/v2/report/manager/watcher/scanconfigwatcher.go index f3a8d7cf8456f..d6ad08b8f8794 100644 --- a/central/complianceoperator/v2/report/manager/watcher/scanconfigwatcher.go +++ b/central/complianceoperator/v2/report/manager/watcher/scanconfigwatcher.go @@ -3,10 +3,8 @@ package watcher import ( "context" "fmt" - "strings" "github.com/pkg/errors" - resultsDataStore "github.com/stackrox/rox/central/complianceoperator/v2/checkresults/datastore" profileDatastore "github.com/stackrox/rox/central/complianceoperator/v2/profiles/datastore" snapshotDataStore "github.com/stackrox/rox/central/complianceoperator/v2/report/datastore" scanConfigurationDS "github.com/stackrox/rox/central/complianceoperator/v2/scanconfigurations/datastore" @@ -31,40 +29,6 @@ func GetScanConfigFromScan(ctx context.Context, scan *storage.ComplianceOperator return scanConfigDS.GetScanConfigurationByName(ctx, scan.GetScanConfigName()) } -func DeleteOldResultsFromMissingScans(ctx context.Context, results *ScanConfigWatcherResults, profileDataStore profileDatastore.DataStore, scanDataStore scanDataStore.DataStore, resultsDataStore resultsDataStore.DataStore) error { - if results == nil { - return errors.New("unable to delete old CheckResults from an nil ScanConfigWatcherResults") - } - scans, err := GetScansFromScanConfiguration(ctx, results.ScanConfig, profileDataStore, scanDataStore) - if err != nil { - return err - } - for _, scanResults := range results.ScanResults { - scans.Remove(fmt.Sprintf("%s:%s", scanResults.Scan.GetClusterId(), scanResults.Scan.GetId())) - } - errList := errorhelpers.NewErrorList("delete old CheckResults from missing scans") - for scanWatcherID := range scans { - parts := strings.Split(scanWatcherID, ":") - if len(parts) != 2 { - errList.AddError(errors.Errorf("unable to parse ScanID from %q", scanWatcherID)) - continue - } - scan, found, err := scanDataStore.GetScan(ctx, parts[1]) - if err != nil { - errList.AddError(err) - continue - } - if !found { - errList.AddError(errors.Errorf("unable to find Scan with ID %q", parts[1])) - continue - } - if err := resultsDataStore.DeleteOldResults(ctx, scan.GetLastStartedTime(), scan.GetScanRefId(), true); err != nil { - errList.AddError(err) - } - } - return errList.ToError() -} - // ScanConfigWatcher determines if a ScanConfiguration has running scans or has completed type ScanConfigWatcher interface { PushScanResults(results *ScanWatcherResults) error @@ -180,6 +144,13 @@ func (w *scanConfigWatcherImpl) Finished() concurrency.ReadOnlySignal { return w.stopped } +func (w *scanConfigWatcherImpl) scanConfigName() string { + if w.scanConfigResults.ScanConfig != nil { + return w.scanConfigResults.ScanConfig.GetScanConfigName() + } + return w.scanConfigResults.WatcherID +} + func (w *scanConfigWatcherImpl) run(timer Timer) { defer func() { w.stopped.Signal() @@ -189,15 +160,17 @@ func (w *scanConfigWatcherImpl) run(timer Timer) { for { select { case <-w.ctx.Done(): - log.Infof("Stopping scan config watcher") concurrency.WithLock(&w.resultsLock, func() { + log.Infof("Stopping scan config watcher for %s. Received %d/%d scan results (watcher id: %s)", + w.scanConfigName(), len(w.scanConfigResults.ScanResults), w.totalResults, w.scanConfigResults.WatcherID) w.scanConfigResults.Error = ErrScanConfigContextCancelled w.readyQueue.Push(w.scanConfigResults) }) return case <-timer.C(): concurrency.WithLock(&w.resultsLock, func() { - log.Warnf("Timeout waiting for the ScanConfiguration %s's scans to finish", w.scanConfigResults.ScanConfig.GetScanConfigName()) + log.Warnf("Timeout waiting for the ScanConfiguration %s's scans to finish. Received %d/%d scan results (watcher id: %s, pending: %v)", + w.scanConfigName(), len(w.scanConfigResults.ScanResults), w.totalResults, w.scanConfigResults.WatcherID, w.scansToWait.AsSlice()) w.scanConfigResults.Error = ErrScanConfigTimeout w.readyQueue.Push(w.scanConfigResults) }) @@ -212,18 +185,41 @@ func (w *scanConfigWatcherImpl) run(timer Timer) { return } } - if concurrency.WithLock1[bool](&w.resultsLock, func() bool { - if w.totalResults != 0 && w.totalResults == len(w.scanConfigResults.ScanResults) { - w.readyQueue.Push(w.scanConfigResults) - return true - } - return false - }) { + if w.checkCompletion() { return } } } +// checkCompletion verifies all expected scans have been received. When scansToWait is empty, +// it re-queries the database to catch scan resources that were created after the initial query +// (e.g., the CO may still be creating scan resources when the first results arrive). +func (w *scanConfigWatcherImpl) checkCompletion() bool { + if w.totalResults == 0 || w.scansToWait.Cardinality() > 0 { + return false + } + newScans, err := GetScansFromScanConfiguration(w.ctx, w.scanConfigResults.ScanConfig, w.profileDS, w.scanDS) + if err != nil { + log.Warnf("Unable to re-query scans for completion check: %v", err) + return false + } + concurrency.WithLock(&w.resultsLock, func() { + for key := range w.scanConfigResults.ScanResults { + newScans.Remove(key) + } + }) + if newScans.Cardinality() > 0 { + log.Infof("Scan config %s: found %d additional scans after re-query, continuing to wait", w.scanConfigName(), newScans.Cardinality()) + w.scansToWait = newScans + w.totalResults += newScans.Cardinality() + return false + } + return concurrency.WithLock1[bool](&w.resultsLock, func() bool { + w.readyQueue.Push(w.scanConfigResults) + return true + }) +} + func (w *scanConfigWatcherImpl) handleScanResults(result *ScanWatcherResults) error { // Here we have the scan config id and the scan if w.totalResults == 0 { @@ -233,11 +229,11 @@ func (w *scanConfigWatcherImpl) handleScanResults(result *ScanWatcherResults) er } w.scansToWait = scans w.totalResults = len(w.scansToWait) - log.Debugf("Scan config %s needs to wait for %d scans", w.scanConfigResults.ScanConfig.GetScanConfigName(), w.totalResults) + log.Debugf("Scan config %s needs to wait for %d scans", w.scanConfigName(), w.totalResults) } log.Debugf("Scan to handle %s with id %s", result.Scan.GetScanName(), result.Scan.GetId()) scanResultKey := fmt.Sprintf("%s:%s", result.Scan.GetClusterId(), result.Scan.GetId()) - if found := w.scansToWait.Remove(fmt.Sprintf("%s:%s", result.Scan.GetClusterId(), result.Scan.GetId())); !found { + if found := w.scansToWait.Remove(scanResultKey); !found { newScanResult := result.Scan var timestampCmpResult int concurrency.WithLock(&w.resultsLock, func() { @@ -249,7 +245,22 @@ func (w *scanConfigWatcherImpl) handleScanResults(result *ScanWatcherResults) er // We already handled a newer scan, so we can ignore this scan. return nil } + // The scan ID is not in scansToWait — the CO may have recycled the scan resource + // with a new ID. Re-query to refresh scansToWait with current IDs from the DB. + if freshScans, err := GetScansFromScanConfiguration(w.ctx, w.scanConfigResults.ScanConfig, w.profileDS, w.scanDS); err == nil { + w.scansToWait = freshScans + w.totalResults = len(w.scansToWait) + w.scansToWait.Remove(scanResultKey) + concurrency.WithLock(&w.resultsLock, func() { + for key := range w.scanConfigResults.ScanResults { + w.scansToWait.Remove(key) + } + }) + } } + // Remove any stale result for the same scan name on the same cluster (CO may recycle scan + // resources with new IDs). This prevents inflating ScanResults and keeps scansToWait accurate. + w.removeStaleResult(result.Scan.GetClusterId(), result.Scan.GetScanName(), scanResultKey) concurrency.WithLock(&w.resultsLock, func() { w.scanConfigResults.ScanResults[scanResultKey] = result }) @@ -257,6 +268,18 @@ func (w *scanConfigWatcherImpl) handleScanResults(result *ScanWatcherResults) er return w.appendScanToSnapshots(w.ctx, result.Scan) } +func (w *scanConfigWatcherImpl) removeStaleResult(clusterID, scanName, currentKey string) { + concurrency.WithLock(&w.resultsLock, func() { + for key, existing := range w.scanConfigResults.ScanResults { + if key != currentKey && existing.Scan.GetClusterId() == clusterID && existing.Scan.GetScanName() == scanName { + log.Debugf("Removing stale scan result %s (replaced by %s)", key, currentKey) + delete(w.scanConfigResults.ScanResults, key) + w.scansToWait.Remove(key) + } + } + }) +} + func (w *scanConfigWatcherImpl) appendScanToSnapshots(ctx context.Context, scan *storage.ComplianceOperatorScanV2) error { errList := errorhelpers.NewErrorList("update snapshots' scans") concurrency.WithLock(&w.resultsLock, func() { @@ -278,11 +301,11 @@ func (w *scanConfigWatcherImpl) appendScanToSnapshots(ctx context.Context, scan // GetScansFromScanConfiguration returns the scans associated with a given ScanConfiguration func GetScansFromScanConfiguration(ctx context.Context, scanConfig *storage.ComplianceOperatorScanConfigurationV2, profileDataStore profileDatastore.DataStore, scanDataStore scanDataStore.DataStore) (set.StringSet, error) { ret := set.NewStringSet() - var profileNames []string + profileNames := make([]string, 0, len(scanConfig.GetProfiles())) for _, p := range scanConfig.GetProfiles() { profileNames = append(profileNames, p.GetProfileName()) } - var clusters []string + clusters := make([]string, 0, len(scanConfig.GetClusters())) for _, c := range scanConfig.GetClusters() { clusters = append(clusters, c.GetClusterId()) } diff --git a/central/complianceoperator/v2/report/manager/watcher/scanconfigwatcher_test.go b/central/complianceoperator/v2/report/manager/watcher/scanconfigwatcher_test.go index 421c340c952ac..45a9a3a6a8c60 100644 --- a/central/complianceoperator/v2/report/manager/watcher/scanconfigwatcher_test.go +++ b/central/complianceoperator/v2/report/manager/watcher/scanconfigwatcher_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/pkg/errors" - checkResultsMocks "github.com/stackrox/rox/central/complianceoperator/v2/checkresults/datastore/mocks" profileDatastore "github.com/stackrox/rox/central/complianceoperator/v2/profiles/datastore/mocks" snapshotMocks "github.com/stackrox/rox/central/complianceoperator/v2/report/datastore/mocks" scanMocks "github.com/stackrox/rox/central/complianceoperator/v2/scans/datastore/mocks" @@ -18,14 +17,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - "google.golang.org/protobuf/types/known/timestamppb" ) type scanConfigTestEvent func(*testing.T, ScanConfigWatcher) func handleInitialScanResults(id string, scanDS *scanMocks.MockDataStore, profileDS *profileDatastore.MockDataStore, numOfScans int) scanConfigTestEvent { return func(t *testing.T, watcher ScanConfigWatcher) { - profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(1). + profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(2). DoAndReturn(func(_, _ any) ([]*storage.ComplianceOperatorProfileV2, error) { return []*storage.ComplianceOperatorProfileV2{ { @@ -33,19 +31,21 @@ func handleInitialScanResults(id string, scanDS *scanMocks.MockDataStore, profil }, }, nil }) - scanDS.EXPECT().SearchScans(gomock.Any(), gomock.Any()).Times(1). + scanDS.EXPECT().SearchScans(gomock.Any(), gomock.Any()).Times(2). DoAndReturn(func(_, _ any) ([]*storage.ComplianceOperatorScanV2, error) { ret := make([]*storage.ComplianceOperatorScanV2, numOfScans) - for i := 0; i < numOfScans; i++ { + for i := range numOfScans { ret[i] = &storage.ComplianceOperatorScanV2{ - Id: fmt.Sprintf("scan-%d", i), + Id: fmt.Sprintf("scan-%d", i), + ScanName: fmt.Sprintf("scan-%d", i), } } return ret, nil }) err := watcher.PushScanResults(&ScanWatcherResults{ Scan: &storage.ComplianceOperatorScanV2{ - Id: id, + Id: id, + ScanName: id, }, }) require.NoError(t, err) @@ -56,7 +56,8 @@ func handleScanResults(id string) scanConfigTestEvent { return func(t *testing.T, watcher ScanConfigWatcher) { err := watcher.PushScanResults(&ScanWatcherResults{ Scan: &storage.ComplianceOperatorScanV2{ - Id: id, + Id: id, + ScanName: id, }, }) require.NoError(t, err) @@ -67,7 +68,8 @@ func handleScanResultsWithError(id string) scanConfigTestEvent { return func(t *testing.T, watcher ScanConfigWatcher) { err := watcher.PushScanResults(&ScanWatcherResults{ Scan: &storage.ComplianceOperatorScanV2{ - Id: id, + Id: id, + ScanName: id, }, Error: errors.New("some error in the scan"), }) @@ -131,8 +133,7 @@ func TestScanConfigWatcher(t *testing.T) { scanConfig := &storage.ComplianceOperatorScanConfigurationV2{ Id: watcherID, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() resultsQueue := queue.NewQueue[*ScanConfigWatcherResults]() scanConfigWatcher := NewScanConfigWatcher(ctx, ctx, watcherID, scanConfig, scanDS, profileDS, snapshotDS, resultsQueue) for _, id := range tCase.snapshotIDs { @@ -185,8 +186,10 @@ func TestScanConfigWatcherCancel(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) resultQueue := queue.NewQueue[*ScanConfigWatcherResults]() + profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorProfileV2{{Id: "p"}}, nil) + scanDS.EXPECT().SearchScans(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorScanV2{{Id: "scan-0"}, {Id: "scan-1"}}, nil) scanConfigWatcher := NewScanConfigWatcher(ctx, ctx, watcherID, scanConfig, scanDS, profileDS, snapshotDS, resultQueue) - handleInitialScanResults("scan-0", scanDS, profileDS, 2)(t, scanConfigWatcher) + handleScanResults("scan-0")(t, scanConfigWatcher) cancel() select { case <-scanConfigWatcher.Finished().Done(): @@ -214,8 +217,10 @@ func TestScanConfigWatcherStop(t *testing.T) { Id: watcherID, } resultQueue := queue.NewQueue[*ScanConfigWatcherResults]() + profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorProfileV2{{Id: "p"}}, nil) + scanDS.EXPECT().SearchScans(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorScanV2{{Id: "scan-0"}, {Id: "scan-1"}}, nil) scanConfigWatcher := NewScanConfigWatcher(context.Background(), context.Background(), watcherID, scanConfig, scanDS, profileDS, snapshotDS, resultQueue) - handleInitialScanResults("scan-0", scanDS, profileDS, 2)(t, scanConfigWatcher) + handleScanResults("scan-0")(t, scanConfigWatcher) scanConfigWatcher.Stop() select { case <-scanConfigWatcher.Finished().Done(): @@ -280,8 +285,10 @@ func TestScanConfigWatcherTimeout(t *testing.T) { readyQueue: resultQueue, scansToWait: set.NewStringSet(), } + profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorProfileV2{{Id: "p"}}, nil) + scanDS.EXPECT().SearchScans(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorScanV2{{Id: "scan-0"}, {Id: "scan-1"}}, nil) go scanConfigWatcher.run(timeout) - handleInitialScanResults("scan-0", scanDS, profileDS, 2)(t, scanConfigWatcher) + handleScanResults("scan-0")(t, scanConfigWatcher) timeoutC <- time.Now() select { case <-scanConfigWatcher.Finished().Done(): @@ -378,101 +385,3 @@ func TestScanConfigWatcherGetScans(t *testing.T) { scans = scanConfigWatcher.GetScans() require.Len(t, scans, 2) } - -func TestDeleteOldResultsFromMissingScans(t *testing.T) { - ctrl := gomock.NewController(t) - checkDS := checkResultsMocks.NewMockDataStore(ctrl) - profileDS := profileDatastore.NewMockDataStore(ctrl) - scanDS := scanMocks.NewMockDataStore(ctrl) - timeNow := timestamppb.Now() - scanID := "scan-id" - scanRefID := "ref-id" - t.Run("nil results should return an error", func(tt *testing.T) { - assert.Error(tt, DeleteOldResultsFromMissingScans(context.Background(), nil, profileDS, scanDS, checkDS)) - }) - t.Run("error retrieving profiles should return an error", func(tt *testing.T) { - results := &ScanConfigWatcherResults{ - ScanConfig: &storage.ComplianceOperatorScanConfigurationV2{}, - } - profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorProfileV2{}, errors.New("some error")) - assert.Error(tt, DeleteOldResultsFromMissingScans(context.Background(), results, profileDS, scanDS, checkDS)) - }) - t.Run("error retrieving scans set should return an error", func(tt *testing.T) { - results := &ScanConfigWatcherResults{ - ScanConfig: &storage.ComplianceOperatorScanConfigurationV2{}, - } - profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorProfileV2{ - {}, - }, nil) - scanDS.EXPECT().SearchScans(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorScanV2{}, errors.New("some error")) - assert.Error(tt, DeleteOldResultsFromMissingScans(context.Background(), results, profileDS, scanDS, checkDS)) - }) - t.Run("error retrieving scan should return an error", func(tt *testing.T) { - results := &ScanConfigWatcherResults{ - ScanConfig: &storage.ComplianceOperatorScanConfigurationV2{}, - } - profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorProfileV2{ - {}, - }, nil) - scanDS.EXPECT().SearchScans(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorScanV2{ - { - Id: scanID, - }, - }, nil) - scanDS.EXPECT().GetScan(gomock.Any(), gomock.Eq(scanID)).Times(1).Return(&storage.ComplianceOperatorScanV2{}, false, errors.New("some error")) - assert.Error(tt, DeleteOldResultsFromMissingScans(context.Background(), results, profileDS, scanDS, checkDS)) - }) - t.Run("scan not found should return an error", func(tt *testing.T) { - results := &ScanConfigWatcherResults{ - ScanConfig: &storage.ComplianceOperatorScanConfigurationV2{}, - } - profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorProfileV2{ - {}, - }, nil) - scanDS.EXPECT().SearchScans(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorScanV2{ - { - Id: scanID, - }, - }, nil) - scanDS.EXPECT().GetScan(gomock.Any(), gomock.Eq(scanID)).Times(1).Return(&storage.ComplianceOperatorScanV2{}, false, nil) - assert.Error(tt, DeleteOldResultsFromMissingScans(context.Background(), results, profileDS, scanDS, checkDS)) - }) - t.Run("delete old results error should return an error", func(tt *testing.T) { - results := &ScanConfigWatcherResults{ - ScanConfig: &storage.ComplianceOperatorScanConfigurationV2{}, - } - profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorProfileV2{ - {}, - }, nil) - scanDS.EXPECT().SearchScans(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorScanV2{ - { - Id: scanID, - }, - }, nil) - scanDS.EXPECT().GetScan(gomock.Any(), gomock.Eq(scanID)).Times(1).Return(&storage.ComplianceOperatorScanV2{ - ScanRefId: scanRefID, - LastStartedTime: timeNow, - }, true, nil) - checkDS.EXPECT().DeleteOldResults(gomock.Any(), gomock.Eq(timeNow), gomock.Eq(scanRefID), gomock.Eq(true)).Times(1).Return(errors.New("some error")) - assert.Error(tt, DeleteOldResultsFromMissingScans(context.Background(), results, profileDS, scanDS, checkDS)) - }) - t.Run("delete old results", func(tt *testing.T) { - results := &ScanConfigWatcherResults{ - ScanConfig: &storage.ComplianceOperatorScanConfigurationV2{}, - } - profileDS.EXPECT().SearchProfiles(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorProfileV2{ - {}, - }, nil) - scanDS.EXPECT().SearchScans(gomock.Any(), gomock.Any()).Times(1).Return([]*storage.ComplianceOperatorScanV2{ - { - Id: scanID, - }, - }, nil) - scanDS.EXPECT().GetScan(gomock.Any(), gomock.Eq(scanID)).Times(1).Return(&storage.ComplianceOperatorScanV2{ - ScanRefId: scanRefID, - LastStartedTime: timeNow, - }, true, nil) - checkDS.EXPECT().DeleteOldResults(gomock.Any(), gomock.Eq(timeNow), gomock.Eq(scanRefID), gomock.Eq(true)).Times(1).Return(nil) - assert.NoError(tt, DeleteOldResultsFromMissingScans(context.Background(), results, profileDS, scanDS, checkDS)) - }) -} diff --git a/central/complianceoperator/v2/report/manager/watcher/scanwatcher.go b/central/complianceoperator/v2/report/manager/watcher/scanwatcher.go index 69873c4ad13f1..202043edf226a 100644 --- a/central/complianceoperator/v2/report/manager/watcher/scanwatcher.go +++ b/central/complianceoperator/v2/report/manager/watcher/scanwatcher.go @@ -41,6 +41,7 @@ const ( minimumComplianceOperatorVersion = "v1.6.0" CheckCountAnnotationKey = "compliance.openshift.io/check-count" LastScannedAnnotationKey = "compliance.openshift.io/last-scanned-timestamp" + ScanResultNotApplicable = "NOT-APPLICABLE" defaultChanelSize = 100 ) @@ -261,6 +262,13 @@ func (s *scanWatcherImpl) Stop(err error) { s.cancel() } +func (s *scanWatcherImpl) scanName() string { + if s.scanResults.Scan != nil { + return s.scanResults.Scan.GetScanName() + } + return s.scanResults.WatcherID +} + func (s *scanWatcherImpl) run() { defer func() { s.stopped.Signal() @@ -271,20 +279,22 @@ func (s *scanWatcherImpl) run() { select { case <-s.ctx.Done(): concurrency.WithLock(&s.resultsLock, func() { - log.Infof("Stopping scan watcher for scan %s", s.scanResults.Scan.GetScanName()) + log.Infof("Stopping scan watcher for scan %s. Received %d/%d check results (watcher id: %s, error: %v)", + s.scanName(), len(s.scanResults.CheckResults), s.totalChecks, s.scanResults.WatcherID, s.scanResults.Error) if s.scanResults.Error == nil { s.scanResults.Error = ErrScanContextCancelled } }) s.readyQueue.Push(s.scanResults) - watcherFinishType.WithLabelValues(s.scanResults.Scan.GetScanName(), "stop requested").Inc() + watcherFinishType.WithLabelValues(s.scanName(), "stop requested").Inc() return case <-s.timeout.C(): concurrency.WithLock(&s.resultsLock, func() { - log.Warnf("Timeout waiting for the scan %s to finish", s.scanResults.Scan.GetScanName()) + log.Warnf("Timeout waiting for the scan %s to finish. Received %d/%d check results (watcher id: %s)", + s.scanName(), len(s.scanResults.CheckResults), s.totalChecks, s.scanResults.WatcherID) s.scanResults.Error = ErrScanTimeout }) - watcherFinishType.WithLabelValues(s.scanResults.Scan.GetScanName(), "timeout").Inc() + watcherFinishType.WithLabelValues(s.scanName(), "timeout").Inc() s.readyQueue.Push(s.scanResults) return case scan := <-s.scanC: @@ -301,15 +311,27 @@ func (s *scanWatcherImpl) run() { numCheckResults = len(s.scanResults.CheckResults) }) log.Debugf("Checking whether %s is finished. TotalChecks=%d, numCheckResults=%d (watcher id: %s)", - s.scanResults.Scan.GetScanName(), s.totalChecks, numCheckResults, s.scanResults.WatcherID) + s.scanName(), s.totalChecks, numCheckResults, s.scanResults.WatcherID) if s.totalChecks != 0 && s.totalChecks == numCheckResults { s.readyQueue.Push(s.scanResults) - watcherFinishType.WithLabelValues(s.scanResults.Scan.GetScanName(), "done").Inc() + watcherFinishType.WithLabelValues(s.scanName(), "done").Inc() + return + } + if s.totalChecks == 0 && numCheckResults == 0 && IsScanNotApplicable(s.scanResults.Scan) { + log.Infof("Scan %s completed as NOT-APPLICABLE with 0 check results (watcher id: %s)", s.scanName(), s.scanResults.WatcherID) + s.readyQueue.Push(s.scanResults) + watcherFinishType.WithLabelValues(s.scanName(), "not_applicable").Inc() return } } } +// IsScanNotApplicable returns true if the scan completed as NOT-APPLICABLE (e.g., a master-node +// scan on an HCP cluster with no master nodes). +func IsScanNotApplicable(scan *storage.ComplianceOperatorScanV2) bool { + return scan.GetStatus().GetResult() == ScanResultNotApplicable +} + func GetExpectedNumChecks(scan *storage.ComplianceOperatorScanV2) (int, error) { annotationsMap := scan.GetAnnotations() if len(annotationsMap) == 0 { @@ -332,20 +354,21 @@ func (s *scanWatcherImpl) handleScan(scan *storage.ComplianceOperatorScanV2) err if err != nil { return err } - s.totalChecks = numChecks s.resultsLock.Lock() defer s.resultsLock.Unlock() - if s.scanResults.Scan == nil { - s.scanResults.Scan = scan - } // If we received a newer timestamp we need to reset the watcher. if protocompat.CompareTimestamps(s.lastStartedTime, scan.GetLastStartedTime()) < 0 { + s.totalChecks = numChecks s.lastStartedTime = scan.GetLastStartedTime() s.scanResults.Scan = scan s.scanResults.CheckResults = set.NewStringSet() s.timeout.Reset() + } else if protocompat.CompareTimestamps(s.lastStartedTime, scan.GetLastStartedTime()) == 0 { + s.totalChecks = numChecks + // Same timestamp: update scan to pick up status changes (e.g., NOT-APPLICABLE). + s.scanResults.Scan = scan } return nil } diff --git a/central/complianceoperator/v2/report/manager/watcher/scanwatcher_test.go b/central/complianceoperator/v2/report/manager/watcher/scanwatcher_test.go index ca8543b152b57..9e4d21d79ef0d 100644 --- a/central/complianceoperator/v2/report/manager/watcher/scanwatcher_test.go +++ b/central/complianceoperator/v2/report/manager/watcher/scanwatcher_test.go @@ -51,6 +51,35 @@ func handleScanWithAnnotation(id, checkCount string, startTime *protocompat.Time } } +func handleScanNotApplicable(id string, startTime *protocompat.Timestamp) func(*testing.T, ScanWatcher) { + return func(t *testing.T, scanWatcher ScanWatcher) { + err := scanWatcher.PushScan(&storage.ComplianceOperatorScanV2{ + Id: id, + Annotations: map[string]string{CheckCountAnnotationKey: "0"}, + LastStartedTime: startTime, + Status: &storage.ScanStatus{ + Phase: "DONE", + Result: ScanResultNotApplicable, + }, + }) + require.NoError(t, err) + } +} + +func handleScanNotApplicableNoAnnotation(id string, startTime *protocompat.Timestamp) func(*testing.T, ScanWatcher) { + return func(t *testing.T, scanWatcher ScanWatcher) { + err := scanWatcher.PushScan(&storage.ComplianceOperatorScanV2{ + Id: id, + LastStartedTime: startTime, + Status: &storage.ScanStatus{ + Phase: "DONE", + Result: ScanResultNotApplicable, + }, + }) + require.NoError(t, err) + } +} + func handleResult(id string, startTime *protocompat.Timestamp) func(*testing.T, ScanWatcher) { return func(t *testing.T, scanWatcher ScanWatcher) { err := scanWatcher.PushCheckResult(&storage.ComplianceOperatorCheckResultV2{ @@ -140,12 +169,33 @@ func TestScanWatcher(t *testing.T) { assertScanID: "id-1", assertResultIDs: []string{"id-1", "id-2"}, }, + "scan not-applicable with zero check-count annotation": { + events: []testEvent{ + handleScanNotApplicable("id-1", timestampNow), + }, + assertScanID: "id-1", + assertResultIDs: []string{}, + }, + "scan not-applicable without check-count annotation": { + events: []testEvent{ + handleScanNotApplicableNoAnnotation("id-1", timestampNow), + }, + assertScanID: "id-1", + assertResultIDs: []string{}, + }, + "scan -> scan not-applicable (two-step update)": { + events: []testEvent{ + handleScan("id-1", timestampNow), + handleScanNotApplicable("id-1", timestampNow), + }, + assertScanID: "id-1", + assertResultIDs: []string{}, + }, } for tName, tCase := range cases { t.Run(tName, func(t *testing.T) { watcherID := "id" - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() resultQueue := queue.NewQueue[*ScanWatcherResults]() scanWatcher := NewScanWatcher(ctx, ctx, watcherID, resultQueue) for _, event := range tCase.events { @@ -156,7 +206,9 @@ func TestScanWatcher(t *testing.T) { }, 200*time.Millisecond, 10*time.Millisecond) result := resultQueue.Pull() require.NotNil(t, result) + assert.NoError(t, result.Error) assert.Equal(t, tCase.assertScanID, result.Scan.GetId()) + assert.Equal(t, len(tCase.assertResultIDs), len(result.CheckResults)) for _, checkID := range tCase.assertResultIDs { found := false for checkResult := range result.CheckResults { @@ -545,3 +597,36 @@ func TestDeleteOldResults(t *testing.T) { assert.NoError(tt, DeleteOldResults(context.Background(), results, ds)) }) } + +func TestIsScanNotApplicable(t *testing.T) { + cases := map[string]struct { + scan *storage.ComplianceOperatorScanV2 + expected bool + }{ + "nil scan": { + scan: nil, + expected: false, + }, + "nil status": { + scan: &storage.ComplianceOperatorScanV2{}, + expected: false, + }, + "empty result": { + scan: &storage.ComplianceOperatorScanV2{Status: &storage.ScanStatus{Result: ""}}, + expected: false, + }, + "NON-COMPLIANT result": { + scan: &storage.ComplianceOperatorScanV2{Status: &storage.ScanStatus{Result: "NON-COMPLIANT"}}, + expected: false, + }, + "NOT-APPLICABLE result": { + scan: &storage.ComplianceOperatorScanV2{Status: &storage.ScanStatus{Result: ScanResultNotApplicable}}, + expected: true, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, IsScanNotApplicable(tc.scan)) + }) + } +} diff --git a/central/complianceoperator/v2/report/manager/watcher/validator.go b/central/complianceoperator/v2/report/manager/watcher/validator.go index 6c8db050a7ac5..15dc0f12bcc85 100644 --- a/central/complianceoperator/v2/report/manager/watcher/validator.go +++ b/central/complianceoperator/v2/report/manager/watcher/validator.go @@ -17,7 +17,13 @@ func ValidateScanConfigResults(ctx context.Context, results *ScanConfigWatcherRe failedClusters := make(map[string]*report.FailedCluster) errList := errorhelpers.NewErrorList("failed clusters") clustersWithResults := set.NewStringSet() - for _, scanResult := range results.ScanResults { + clustersNotApplicable := set.NewStringSet() + for key, scanResult := range results.ScanResults { + if IsScanNotApplicable(scanResult.Scan) { + clustersNotApplicable.Add(scanResult.Scan.GetClusterId()) + delete(results.ScanResults, key) + continue + } clustersWithResults.Add(scanResult.Scan.GetClusterId()) failedClusterInfo, isInstallationError := ValidateScanResults(ctx, scanResult, integrationDataStore) if failedClusterInfo == nil { @@ -39,6 +45,14 @@ func ValidateScanConfigResults(ctx context.Context, results *ScanConfigWatcherRe if clustersWithResults.Contains(cluster.GetClusterId()) { continue } + if clustersNotApplicable.Contains(cluster.GetClusterId()) { + failedClusters[cluster.GetClusterId()] = &report.FailedCluster{ + ClusterId: cluster.GetClusterId(), + Reasons: []string{report.SCAN_NOT_APPLICABLE}, + } + errList.AddError(errors.New(fmt.Sprintf("cluster %s: all scans not applicable", cluster.GetClusterId()))) + continue + } clusterInfo := ValidateClusterHealth(ctx, cluster.GetClusterId(), integrationDataStore) errList.AddError(errors.New(fmt.Sprintf("cluster %s failed", clusterInfo.ClusterId))) if len(clusterInfo.Reasons) == 0 { diff --git a/central/complianceoperator/v2/report/manager/watcher/validator_test.go b/central/complianceoperator/v2/report/manager/watcher/validator_test.go index 7ff8787cd1728..aefb45383719b 100644 --- a/central/complianceoperator/v2/report/manager/watcher/validator_test.go +++ b/central/complianceoperator/v2/report/manager/watcher/validator_test.go @@ -30,20 +30,68 @@ func withExpectCall(fn func(*mocksComplianceIntegrationDS.MockDataStore)) func(* } func TestValidateScanConfigResults(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() cases := map[string]struct { - results *ScanConfigWatcherResults - expectFn func(*mocksComplianceIntegrationDS.MockDataStore) - expectedFailedClusters map[string]*report.FailedCluster - expectedError bool - expectedExactError error + results *ScanConfigWatcherResults + expectFn func(*mocksComplianceIntegrationDS.MockDataStore) + expectedFailedClusters map[string]*report.FailedCluster + expectedError bool + expectedExactError error + expectedScanResultsCount int + checkScanResultsCount bool }{ "no error": { results: getScanConfigResults(2, 0, 0, 1, nil), expectFn: withExpectCall(nil), expectedFailedClusters: make(map[string]*report.FailedCluster), }, + "not-applicable scans are removed from results": { + results: func() *ScanConfigWatcherResults { + r := getScanConfigResults(1, 0, 0, 1, nil) + r.ScanResults["cluster-0:not-applicable-scan"] = &ScanWatcherResults{ + Scan: &storage.ComplianceOperatorScanV2{ + ClusterId: "cluster-0", + ScanName: "not-applicable-scan", + Status: &storage.ScanStatus{Result: ScanResultNotApplicable}, + }, + } + return r + }(), + expectFn: withExpectCall(nil), + expectedFailedClusters: make(map[string]*report.FailedCluster), + checkScanResultsCount: true, + expectedScanResultsCount: 1, + }, + "cluster with only not-applicable scans is flagged": { + results: func() *ScanConfigWatcherResults { + return &ScanConfigWatcherResults{ + ScanResults: map[string]*ScanWatcherResults{ + "cluster-0:master-scan": { + Scan: &storage.ComplianceOperatorScanV2{ + ClusterId: "cluster-0", + ScanName: "master-scan", + Status: &storage.ScanStatus{Result: ScanResultNotApplicable}, + }, + }, + }, + ScanConfig: &storage.ComplianceOperatorScanConfigurationV2{ + Clusters: []*storage.ComplianceOperatorScanConfigurationV2_Cluster{ + {ClusterId: "cluster-0"}, + }, + }, + } + }(), + expectFn: withExpectCall(nil), + expectedFailedClusters: map[string]*report.FailedCluster{ + "cluster-0": { + ClusterId: "cluster-0", + Reasons: []string{report.SCAN_NOT_APPLICABLE}, + }, + }, + expectedError: true, + checkScanResultsCount: true, + expectedScanResultsCount: 0, + }, "two failed clusters": { results: getScanConfigResults(2, 2, 0, 1, nil), expectFn: withExpectCall(func(ds *mocksComplianceIntegrationDS.MockDataStore) { @@ -156,13 +204,15 @@ func TestValidateScanConfigResults(t *testing.T) { assert.ErrorIs(tt, err, tCase.expectedExactError) } } + if tCase.checkScanResultsCount { + assert.Equal(tt, tCase.expectedScanResultsCount, len(tCase.results.ScanResults)) + } }) } } func TestValidateScanResults(t *testing.T) { - ctx, validCtxCancelFn := context.WithCancel(context.Background()) - defer validCtxCancelFn() + ctx := t.Context() canceledCtx, canceledCtxCancelFn := context.WithCancel(context.Background()) canceledCtxCancelFn() cases := map[string]struct { @@ -327,8 +377,7 @@ func TestValidateScanResults(t *testing.T) { } func TestValidateClusterHealth(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() cases := map[string]struct { operatorStatus []*storage.ComplianceIntegration expectDSError error @@ -400,12 +449,12 @@ func getMockedIntegration(t *testing.T) *mocksComplianceIntegrationDS.MockDataSt func getScanConfigResults(numSuccessfulClusters, numFailedClusters, numMissingClusters, numScansPerCluster int, err error) *ScanConfigWatcherResults { scanResults := make(map[string]*ScanWatcherResults) var clusters []*storage.ComplianceOperatorScanConfigurationV2_Cluster - for i := 0; i < numSuccessfulClusters; i++ { + for i := range numSuccessfulClusters { id := fmt.Sprintf("cluster-%d", i) clusters = append(clusters, &storage.ComplianceOperatorScanConfigurationV2_Cluster{ ClusterId: id, }) - for j := 0; j < numScansPerCluster; j++ { + for j := range numScansPerCluster { resultsID := fmt.Sprintf("%s:scan-%d", id, j) scanResults[resultsID] = &ScanWatcherResults{ Scan: &storage.ComplianceOperatorScanV2{ @@ -419,7 +468,7 @@ func getScanConfigResults(numSuccessfulClusters, numFailedClusters, numMissingCl clusters = append(clusters, &storage.ComplianceOperatorScanConfigurationV2_Cluster{ ClusterId: id, }) - for j := 0; j < numScansPerCluster; j++ { + for j := range numScansPerCluster { resultsID := fmt.Sprintf("%s:scan-%d", id, j) scanResults[resultsID] = &ScanWatcherResults{ Scan: &storage.ComplianceOperatorScanV2{ @@ -456,7 +505,7 @@ func getFailedClusters(idx, numFailedClusters, numMissingClusters, numScans int) } ret[id] = failedCluster var reasons []string - for j := 0; j < numScans; j++ { + for range numScans { reasons = append(reasons, report.INTERNAL_ERROR) failedCluster.FailedScans = append(failedCluster.FailedScans, &storage.ComplianceOperatorScanV2{ ClusterId: id, diff --git a/central/complianceoperator/v2/report/result.go b/central/complianceoperator/v2/report/result.go index 3d79245867950..8fedcb149c66b 100644 --- a/central/complianceoperator/v2/report/result.go +++ b/central/complianceoperator/v2/report/result.go @@ -2,15 +2,17 @@ package report // ResultRow struct which hold all columns of a report row type ResultRow struct { - ClusterName string - CheckName string - Profile string - ControlRef string - Description string - Status string - Remediation string - Rationale string - Instructions string + ClusterName string + CheckName string + Profile string + ProfileType string + ControlRef string + Description string + Status string + Remediation string + Rationale string + Instructions string + AssessmentTime string } // Results struct which holds the results of a report. diff --git a/central/complianceoperator/v2/rules/datastore/datastore_impl.go b/central/complianceoperator/v2/rules/datastore/datastore_impl.go index e4e2bd6023bd6..0409ffc4f325c 100644 --- a/central/complianceoperator/v2/rules/datastore/datastore_impl.go +++ b/central/complianceoperator/v2/rules/datastore/datastore_impl.go @@ -71,7 +71,11 @@ func (d *datastoreImpl) GetControlsByRulesAndBenchmarks(ctx context.Context, rul ) query := builder.ProtoQuery() - results, err := pgSearch.RunSelectRequestForSchema[ControlResult](ctx, d.db, postgresSchema.ComplianceOperatorRuleV2Schema, query) + var results []*ControlResult + err := pgSearch.RunSelectRequestForSchemaFn[ControlResult](ctx, d.db, postgresSchema.ComplianceOperatorRuleV2Schema, query, func(r *ControlResult) error { + results = append(results, r) + return nil + }) if err != nil { return nil, err } diff --git a/central/complianceoperator/v2/scanconfigurations/datastore/datastore_impl.go b/central/complianceoperator/v2/scanconfigurations/datastore/datastore_impl.go index b24a935d4764b..7fd81c18d8070 100644 --- a/central/complianceoperator/v2/scanconfigurations/datastore/datastore_impl.go +++ b/central/complianceoperator/v2/scanconfigurations/datastore/datastore_impl.go @@ -71,7 +71,7 @@ func (ds *datastoreImpl) GetScanConfigurationByName(ctx context.Context, scanNam // ScanConfigurationProfileExists takes all the profiles being referenced by the scan configuration and checks if any cluster in the configuration is using it in any existing scan configurations. func (ds *datastoreImpl) ScanConfigurationProfileExists(ctx context.Context, id string, profiles []string, clusters []string) error { - for i := 0; i < len(profiles); i++ { + for i := range profiles { for j := i + 1; j < len(profiles); j++ { if strings.EqualFold(profiles[i], profiles[j]) { return errors.Errorf("the scan configuration contains duplicate profiles. Profile %q and profile %q", profiles[i], profiles[j]) @@ -317,18 +317,17 @@ func (ds *datastoreImpl) GetProfilesNames(ctx context.Context, q *v1.Query) ([]s clonedQuery.Pagination = q.GetPagination() - var results []*distinctProfileName - results, err = pgSearch.RunSelectRequestForSchema[distinctProfileName](ctx, ds.db, schema.ComplianceOperatorScanConfigurationV2Schema, clonedQuery) + var profileNames []string + err = pgSearch.RunSelectRequestForSchemaFn[distinctProfileName](ctx, ds.db, schema.ComplianceOperatorScanConfigurationV2Schema, clonedQuery, func(r *distinctProfileName) error { + profileNames = append(profileNames, r.ProfileName) + return nil + }) if err != nil { return nil, err } - if len(results) == 0 { + if len(profileNames) == 0 { return nil, nil } - profileNames := make([]string, 0, len(results)) - for _, result := range results { - profileNames = append(profileNames, result.ProfileName) - } return profileNames, err } @@ -354,14 +353,14 @@ func (ds *datastoreImpl) DistinctProfiles(ctx context.Context, q *v1.Query) (map }, } - result, err := pgSearch.RunSelectRequestForSchema[distinctProfileCount](ctx, ds.db, schema.ComplianceOperatorScanConfigurationV2Schema, withCountQuery(query, search.ComplianceOperatorConfigProfileName)) + countMap := make(map[string]int) + err = pgSearch.RunSelectRequestForSchemaFn[distinctProfileCount](ctx, ds.db, schema.ComplianceOperatorScanConfigurationV2Schema, withCountQuery(query, search.ComplianceOperatorConfigProfileName), func(dp *distinctProfileCount) error { + countMap[dp.Name] = dp.TotalCount + return nil + }) if err != nil { return nil, err } - countMap := make(map[string]int, len(result)) - for _, dp := range result { - countMap[dp.Name] = dp.TotalCount - } return countMap, nil } @@ -383,15 +382,65 @@ func withSACFilter(ctx context.Context, targetResource permissions.ResourceMetad func GatherProfiles(ds DataStore) phonehome.GatherFunc { return func(ctx context.Context) (map[string]any, error) { - telemetryCtx := sac.WithAllAccess(ctx) - counts, err := ds.DistinctProfiles(telemetryCtx, search.EmptyQuery()) + if ds == nil { + return map[string]any{}, nil + } + scanConfigs, err := ds.GetScanConfigurations(sac.WithAllAccess(ctx), search.EmptyQuery()) if err != nil { return nil, errors.Wrap(err, "gathering compliance operator profiles for telemetry") } - profiles := make(map[string]any, len(counts)) - for profile, count := range counts { - profiles["Compliance Operator Profile "+profile] = count + + profiles := make(map[string]int) + for _, sc := range scanConfigs { + refs := sc.GetProfileRefs() + // If ProfileRefs is empty, this scan configuration was created before adding Tailored Profiles support. + // Therefore, profiles are stored in the legacy "profiles" field, and all of them are non-tailored profiles. + if len(refs) == 0 { + for _, p := range sc.GetProfiles() { + profiles[p.GetProfileName()]++ + } + continue + } + for _, ref := range refs { + if ref.GetKind() == storage.ComplianceOperatorProfileV2_PROFILE { + profiles[ref.GetName()]++ + } + } } - return profiles, nil + + r := make(map[string]any) + for name, count := range profiles { + r["Compliance Operator Profile "+name] = count + } + return r, nil + } +} + +// GatherTailoredProfiles reports the total number of tailored profile references across all scan configurations +// (not the number of distinct tailored profiles - this mimics GatherProfiles where we count the same profile being +// present in several scan configs). Individual tailored profile names are not collected because they are user-defined +// (unlike standard Compliance Operator profiles) and would expose customer data. +func GatherTailoredProfiles(ds DataStore) phonehome.GatherFunc { + return func(ctx context.Context) (map[string]any, error) { + if ds == nil { + return map[string]any{}, nil + } + scanConfigs, err := ds.GetScanConfigurations(sac.WithAllAccess(ctx), search.EmptyQuery()) + if err != nil { + return nil, errors.Wrap(err, "gathering compliance operator tailored profiles for telemetry") + } + + tpCount := 0 + for _, sc := range scanConfigs { + for _, ref := range sc.GetProfileRefs() { + if ref.GetKind() == storage.ComplianceOperatorProfileV2_TAILORED_PROFILE { + tpCount++ + } + } + } + + return map[string]any{ + "Compliance Operator Tailored Profiles": tpCount, + }, nil } } diff --git a/central/complianceoperator/v2/scanconfigurations/datastore/telemetry_postgres_test.go b/central/complianceoperator/v2/scanconfigurations/datastore/telemetry_postgres_test.go new file mode 100644 index 0000000000000..a7bea60b9e3d7 --- /dev/null +++ b/central/complianceoperator/v2/scanconfigurations/datastore/telemetry_postgres_test.go @@ -0,0 +1,269 @@ +//go:build sql_integration + +package datastore + +import ( + "context" + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/sac/testconsts" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGatherProfilesNilDatastore(t *testing.T) { + props, err := GatherProfiles(nil)(context.Background()) + require.NoError(t, err) + assert.Empty(t, props) +} + +func TestGatherTailoredProfilesNilDatastore(t *testing.T) { + props, err := GatherTailoredProfiles(nil)(context.Background()) + require.NoError(t, err) + assert.Empty(t, props) +} + +func TestGatherProfiles(t *testing.T) { + t.Setenv(features.ComplianceEnhancements.EnvVar(), "true") + + pool := pgtest.ForT(t) + ds := GetTestPostgresDataStore(t, pool.DB) + + writeCtx := sac.WithGlobalAccessScopeChecker(context.Background(), + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS, storage.Access_READ_WRITE_ACCESS), + sac.ResourceScopeKeys(resources.Compliance, resources.Cluster), + ), + ) + + testCases := map[string]struct { + scanConfigs []*storage.ComplianceOperatorScanConfigurationV2 + expectedProps map[string]any + excludedKeys []string + }{ + "no scan configs": { + scanConfigs: nil, + expectedProps: map[string]any{}, + }, + "reports each regular profile name": { + scanConfigs: []*storage.ComplianceOperatorScanConfigurationV2{ + makeScanConfig("scan-1", []profileDef{ + {"ocp4-cis", storage.ComplianceOperatorProfileV2_PROFILE}, + {"ocp4-nist", storage.ComplianceOperatorProfileV2_PROFILE}, + }), + }, + expectedProps: map[string]any{ + "Compliance Operator Profile ocp4-cis": 1, + "Compliance Operator Profile ocp4-nist": 1, + }, + }, + "same profile in multiple scan configs counts each reference": { + scanConfigs: []*storage.ComplianceOperatorScanConfigurationV2{ + makeScanConfig("scan-a", []profileDef{ + {"ocp4-cis", storage.ComplianceOperatorProfileV2_PROFILE}, + }), + makeScanConfig("scan-b", []profileDef{ + {"ocp4-cis", storage.ComplianceOperatorProfileV2_PROFILE}, + }), + }, + expectedProps: map[string]any{ + "Compliance Operator Profile ocp4-cis": 2, + }, + }, + "excludes tailored profile names": { + scanConfigs: []*storage.ComplianceOperatorScanConfigurationV2{ + makeScanConfig("scan-2", []profileDef{ + {"ocp4-cis", storage.ComplianceOperatorProfileV2_PROFILE}, + {"my-tp", storage.ComplianceOperatorProfileV2_TAILORED_PROFILE}, + }), + }, + expectedProps: map[string]any{ + "Compliance Operator Profile ocp4-cis": 1, + }, + excludedKeys: []string{"Compliance Operator Profile my-tp"}, + }, + "excludes unspecified kind profiles": { + scanConfigs: []*storage.ComplianceOperatorScanConfigurationV2{ + makeScanConfig("scan-unspec", []profileDef{ + {"ocp4-cis", storage.ComplianceOperatorProfileV2_PROFILE}, + {"unknown-profile", storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED}, + }), + }, + expectedProps: map[string]any{ + "Compliance Operator Profile ocp4-cis": 1, + }, + excludedKeys: []string{"Compliance Operator Profile unknown-profile"}, + }, + "legacy scan config without profile_refs": { + scanConfigs: []*storage.ComplianceOperatorScanConfigurationV2{ + makeScanConfigNoRefs("scan-legacy", []string{"ocp4-cis"}), + }, + expectedProps: map[string]any{ + "Compliance Operator Profile ocp4-cis": 1, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + for _, sc := range tc.scanConfigs { + require.NoError(t, ds.UpsertScanConfiguration(writeCtx, sc)) + } + t.Cleanup(func() { + for _, sc := range tc.scanConfigs { + _, _ = ds.DeleteScanConfiguration(writeCtx, sc.GetId()) + } + }) + + props, err := GatherProfiles(ds)(context.Background()) + require.NoError(t, err) + + assert.Len(t, props, len(tc.expectedProps)) + for key, val := range tc.expectedProps { + assert.Equal(t, val, props[key], "key: %s", key) + } + for _, key := range tc.excludedKeys { + assert.NotContains(t, props, key) + } + }) + } +} + +func TestGatherTailoredProfiles(t *testing.T) { + t.Setenv(features.ComplianceEnhancements.EnvVar(), "true") + + pool := pgtest.ForT(t) + ds := GetTestPostgresDataStore(t, pool.DB) + + writeCtx := sac.WithGlobalAccessScopeChecker(context.Background(), + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS, storage.Access_READ_WRITE_ACCESS), + sac.ResourceScopeKeys(resources.Compliance, resources.Cluster), + ), + ) + + testCases := map[string]struct { + scanConfigs []*storage.ComplianceOperatorScanConfigurationV2 + expectedTailored int + }{ + "no scan configs": { + scanConfigs: nil, + expectedTailored: 0, + }, + "scan config with only regular profiles": { + scanConfigs: []*storage.ComplianceOperatorScanConfigurationV2{ + makeScanConfig("scan-regular", []profileDef{ + {"ocp4-cis", storage.ComplianceOperatorProfileV2_PROFILE}, + {"ocp4-nist", storage.ComplianceOperatorProfileV2_PROFILE}, + }), + }, + expectedTailored: 0, + }, + "scan config with tailored profiles": { + scanConfigs: []*storage.ComplianceOperatorScanConfigurationV2{ + makeScanConfig("scan-tailored", []profileDef{ + {"ocp4-cis", storage.ComplianceOperatorProfileV2_PROFILE}, + {"my-tailored-cis", storage.ComplianceOperatorProfileV2_TAILORED_PROFILE}, + }), + }, + expectedTailored: 1, + }, + "multiple scan configs with multiple tailored profiles": { + scanConfigs: []*storage.ComplianceOperatorScanConfigurationV2{ + makeScanConfig("scan-1", []profileDef{ + {"ocp4-cis", storage.ComplianceOperatorProfileV2_PROFILE}, + {"tp-cis", storage.ComplianceOperatorProfileV2_TAILORED_PROFILE}, + }), + makeScanConfig("scan-2", []profileDef{ + {"tp-nist", storage.ComplianceOperatorProfileV2_TAILORED_PROFILE}, + {"tp-pci", storage.ComplianceOperatorProfileV2_TAILORED_PROFILE}, + }), + }, + expectedTailored: 3, + }, + "scan config without profile_refs populated": { + scanConfigs: []*storage.ComplianceOperatorScanConfigurationV2{ + makeScanConfigNoRefs("scan-no-refs", []string{"ocp4-cis"}), + }, + expectedTailored: 0, + }, + "scan config with unspecified kind does not count as tailored": { + scanConfigs: []*storage.ComplianceOperatorScanConfigurationV2{ + makeScanConfig("scan-unspecified", []profileDef{ + {"ocp4-cis", storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED}, + {"ocp4-nist", storage.ComplianceOperatorProfileV2_PROFILE}, + {"tp-custom", storage.ComplianceOperatorProfileV2_TAILORED_PROFILE}, + }), + }, + expectedTailored: 1, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + for _, sc := range tc.scanConfigs { + require.NoError(t, ds.UpsertScanConfiguration(writeCtx, sc)) + } + t.Cleanup(func() { + for _, sc := range tc.scanConfigs { + _, _ = ds.DeleteScanConfiguration(writeCtx, sc.GetId()) + } + }) + + props, err := GatherTailoredProfiles(ds)(context.Background()) + require.NoError(t, err) + assert.Equal(t, tc.expectedTailored, props["Compliance Operator Tailored Profiles"]) + }) + } +} + +type profileDef struct { + name string + kind storage.ComplianceOperatorProfileV2_OperatorKind +} + +func makeScanConfig(scanName string, profiles []profileDef) *storage.ComplianceOperatorScanConfigurationV2 { + profileNames := make([]*storage.ComplianceOperatorScanConfigurationV2_ProfileName, 0, len(profiles)) + profileRefs := make([]*storage.ComplianceOperatorScanConfigurationV2_ProfileReference, 0, len(profiles)) + for _, p := range profiles { + profileNames = append(profileNames, &storage.ComplianceOperatorScanConfigurationV2_ProfileName{ + ProfileName: p.name, + }) + profileRefs = append(profileRefs, &storage.ComplianceOperatorScanConfigurationV2_ProfileReference{ + Name: p.name, + Kind: p.kind, + }) + } + return &storage.ComplianceOperatorScanConfigurationV2{ + Id: uuid.NewV4().String(), + ScanConfigName: scanName, + Profiles: profileNames, + ProfileRefs: profileRefs, + Clusters: []*storage.ComplianceOperatorScanConfigurationV2_Cluster{ + {ClusterId: testconsts.Cluster1}, + }, + } +} + +func makeScanConfigNoRefs(scanName string, profiles []string) *storage.ComplianceOperatorScanConfigurationV2 { + profileNames := make([]*storage.ComplianceOperatorScanConfigurationV2_ProfileName, 0, len(profiles)) + for _, p := range profiles { + profileNames = append(profileNames, &storage.ComplianceOperatorScanConfigurationV2_ProfileName{ + ProfileName: p, + }) + } + return &storage.ComplianceOperatorScanConfigurationV2{ + Id: uuid.NewV4().String(), + ScanConfigName: scanName, + Profiles: profileNames, + Clusters: []*storage.ComplianceOperatorScanConfigurationV2_Cluster{ + {ClusterId: testconsts.Cluster1}, + }, + } +} diff --git a/central/complianceoperator/v2/scanconfigurations/service/service_impl.go b/central/complianceoperator/v2/scanconfigurations/service/service_impl.go index a68532841977e..a9c4b0551922e 100644 --- a/central/complianceoperator/v2/scanconfigurations/service/service_impl.go +++ b/central/complianceoperator/v2/scanconfigurations/service/service_impl.go @@ -137,14 +137,10 @@ func (s *serviceImpl) CreateComplianceScanConfiguration(ctx context.Context, req // Convert to storage type scanConfig := convertV2ScanConfigToStorage(ctx, req) - // grab clusters - var clusterIDs []string - clusterIDs = append(clusterIDs, req.GetClusters()...) - // Process scan request, config may be updated in the event of errors from sensor. - scanConfig, err := s.manager.ProcessScanRequest(ctx, scanConfig, clusterIDs) + scanConfig, err := s.manager.ProcessScanRequest(ctx, scanConfig, req.GetClusters()) if err != nil { - return nil, errors.Wrapf(errox.InvalidArgs, "Unable to process scan config. %v", err) + return nil, errox.InvalidArgs.CausedBy(err) } return convertStorageScanConfigToV2(ctx, scanConfig, s.scanConfigDS) @@ -162,14 +158,10 @@ func (s *serviceImpl) UpdateComplianceScanConfiguration(ctx context.Context, req // Convert to storage type scanConfig := convertV2ScanConfigToStorage(ctx, req) - // grab clusters - var clusterIDs []string - clusterIDs = append(clusterIDs, req.GetClusters()...) - // Update scan request, config may be updated in the event of errors from sensor. - _, err := s.manager.UpdateScanRequest(ctx, scanConfig, clusterIDs) + _, err := s.manager.UpdateScanRequest(ctx, scanConfig, req.GetClusters()) if err != nil { - return nil, errors.Wrapf(errox.InvalidArgs, "Unable to process scan config. %v", err) + return nil, errox.InvalidArgs.CausedBy(err) } return &v2.Empty{}, nil @@ -212,7 +204,7 @@ func (s *serviceImpl) DeleteComplianceScanConfiguration(ctx context.Context, req err = s.manager.DeleteScan(ctx, req.GetId()) if err != nil { - return nil, errors.Wrapf(errox.InvalidArgs, "Unable to delete scan config: %v", err) + return nil, errox.InvalidArgs.CausedBy(err) } return &v2.Empty{}, nil @@ -551,6 +543,9 @@ func (s *serviceImpl) getProfiles(ctx context.Context, query *v1.Query, countQue if err != nil { return nil, 0, errors.Wrapf(errox.InvalidArgs, "Unable to retrieve scan configurations for query %v", query) } + if len(profileNames) == 0 { + return nil, 0, nil + } // Build query to get the filtered list by profile names profileQuery := search.NewQueryBuilder().AddSelectFields().AddExactMatches(search.ComplianceOperatorProfileName, profileNames...).ProtoQuery() diff --git a/central/complianceoperator/v2/scanconfigurations/service/service_impl_test.go b/central/complianceoperator/v2/scanconfigurations/service/service_impl_test.go index 56b24ac16c380..cd8ecb2d57655 100644 --- a/central/complianceoperator/v2/scanconfigurations/service/service_impl_test.go +++ b/central/complianceoperator/v2/scanconfigurations/service/service_impl_test.go @@ -277,7 +277,7 @@ func (s *ComplianceScanConfigServiceTestSuite) TestDeleteComplianceScanConfigura _, err = s.service.DeleteComplianceScanConfiguration(allAccessContext, &v2.ResourceByID{Id: failingID}) s.Require().Error(err) - s.Require().Contains(err.Error(), "Unable to delete scan config") + s.Require().Contains(err.Error(), "manager error") } func (s *ComplianceScanConfigServiceTestSuite) TestCreateComplianceScanConfigurationScanExists() { @@ -285,12 +285,12 @@ func (s *ComplianceScanConfigServiceTestSuite) TestCreateComplianceScanConfigura request := getTestAPIRec() storageRequest := convertV2ScanConfigToStorage(allAccessContext, request) - managerErr := errors.Errorf("Scan Configuration named %q already exists.", request.GetScanName()) + managerErr := errors.Errorf("Scan configuration named %q already exists.", request.GetScanName()) s.manager.EXPECT().ProcessScanRequest(gomock.Any(), storageRequest, []string{fixtureconsts.Cluster1}).Return(nil, managerErr).Times(1) - expectedErr := errors.Wrapf(errox.InvalidArgs, "Unable to process scan config. Scan Configuration named %q already exists.", request.GetScanName()) config, err := s.service.CreateComplianceScanConfiguration(allAccessContext, request) - s.Require().Equal(expectedErr.Error(), err.Error()) + s.Require().ErrorIs(err, errox.InvalidArgs) + s.Require().Contains(err.Error(), managerErr.Error()) s.Require().Nil(config) } @@ -1198,6 +1198,19 @@ func getTestAPIStatusRec(createdTime, lastUpdatedTime time.Time) *apiV2.Complian } } +// TestGetProfiles_NoMatch verifies that when GetProfilesNames returns nil (filter +// matches no scan configs), getProfiles short-circuits and returns an empty list +// rather than falling through to SearchProfiles with no WHERE clause (which would +// return every profile in the database). +func (s *ComplianceScanConfigServiceTestSuite) TestGetProfiles_NoMatch() { + s.scanConfigDatastore.EXPECT().GetProfilesNames(gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) + + profiles, count, err := s.service.(*serviceImpl).getProfiles(s.ctx, search.EmptyQuery(), search.EmptyQuery()) + s.Require().NoError(err) + s.Empty(profiles) + s.Zero(count) +} + func getTestAPIRec() *apiV2.ComplianceScanConfiguration { return &apiV2.ComplianceScanConfiguration{ ScanName: "test-scan", diff --git a/central/config/datastore/singleton.go b/central/config/datastore/singleton.go index 8718811507ef8..ecbcf73aef347 100644 --- a/central/config/datastore/singleton.go +++ b/central/config/datastore/singleton.go @@ -45,7 +45,7 @@ const ( // PlatformComponentLayeredProductsRuleName is the name of the system defined rule for matching workloads created by Red hat layered products PlatformComponentLayeredProductsRuleName = "red hat layered products" // PlatformComponentLayeredProductsDefaultRegex is the default regex for matching workloads created by Red hat layered products, this can be edited by users - PlatformComponentLayeredProductsDefaultRegex = `^aap$|^ack-system$|^aws-load-balancer-operator$|^catalogd-controller-manager$|^cert-manager$|^cert-manager-operator$|^cert-utils-operator$|^costmanagement-metrics-operator$|^external-dns-operator$|^metallb-system$|^mtr$|^multicluster-engine$|^multicluster-global-hub$|^node-observability-operator$|^open-cluster-management$|^openshift-adp$|^openshift-apiserver-operator$|^openshift-authentication$|^openshift-authentication-operator$|^openshift-builds$|^openshift-catalogd$|^openshift-cloud-controller-manager$|^openshift-cloud-controller-manager-operator$|^openshift-cloud-credential-operator$|^openshift-cloud-network-config-controller$|^openshift-cluster-csi-drivers$|^openshift-cluster-machine-approver$|^openshift-cluster-node-tuning-operator$|^openshift-cluster-observability-operator$|^openshift-cluster-olm-operator$|^openshift-cluster-samples-operator$|^openshift-cluster-storage-operator$|^openshift-cluster-version$|^openshift-cnv$|^openshift-compliance$|^openshift-config$|^openshift-config-managed$|^openshift-config-operator$|^openshift-console$|^openshift-console-operator$|^openshift-console-user-settings$|^openshift-controller-manager$|^openshift-controller-manager-operator$|^openshift-dbaas-operator$|^openshift-distributed-tracing$|^openshift-dns$|^openshift-dns-operator$|^openshift-dpu-network-operator$|^openshift-dr-system$|^openshift-etcd$|^openshift-etcd-operator$|^openshift-file-integrity$|^openshift-gitops$|^openshift-gitops-operator$|^openshift-host-network$|^openshift-image-registry$|^openshift-infra$|^openshift-ingress$|^openshift-ingress-canary$|^openshift-ingress-node-firewall$|^openshift-ingress-operator$|^openshift-insights$|^openshift-keda$|^openshift-kmm$|^openshift-kmm-hub$|^openshift-kni-infra$|^openshift-kube-apiserver$|^openshift-kube-apiserver-operator$|^openshift-kube-controller-manager$|^openshift-kube-controller-manager-operator$|^openshift-kube-scheduler$|^openshift-kube-scheduler-operator$|^openshift-kube-storage-version-migrator$|^openshift-kube-storage-version-migrator-operator$|^openshift-lifecycle-agent$|^openshift-local-storage$|^openshift-logging$|^openshift-machine-api$|^openshift-machine-config-operator$|^openshift-marketplace$|^openshift-migration$|^openshift-monitoring$|^openshift-mta$|^openshift-mtv$|^openshift-multus$|^openshift-netobserv-operator$|^openshift-network-diagnostics$|^openshift-network-node-identity$|^openshift-network-operator$|^openshift-nfd$|^openshift-nmstate$|^openshift-node$|^openshift-nutanix-infra$|^openshift-oauth-apiserver$|^openshift-openstack-infra$|^openshift-opentelemetry-operator$|^openshift-operator-controller$|^openshift-operator-lifecycle-manager$|^openshift-operators$|^openshift-operators-redhat$|^openshift-ovirt-infra$|^openshift-ovn-kubernetes$|^openshift-ptp$|^openshift-route-controller-manager$|^openshift-sandboxed-containers-operator$|^openshift-security-profiles$|^openshift-serverless$|^openshift-serverless-logic$|^openshift-service-ca$|^openshift-service-ca-operator$|^openshift-sriov-network-operator$|^openshift-storage$|^openshift-tempo-operator$|^openshift-update-service$|^openshift-user-workload-monitoring$|^openshift-vertical-pod-autoscaler$|^openshift-vsphere-infra$|^openshift-windows-machine-config-operator$|^openshift-workload-availability$|^redhat-ods-operator$|^rhdh-operator$|^service-telemetry$|^stackrox$|^submariner-operator$` + PlatformComponentLayeredProductsDefaultRegex = `^aap$|^ack-system$|^aws-load-balancer-operator$|^catalogd-controller-manager$|^cert-manager$|^cert-manager-operator$|^cert-utils-operator$|^costmanagement-metrics-operator$|^external-dns-operator$|^metallb-system$|^mtr$|^multicluster-engine$|^multicluster-global-hub$|^node-observability-operator$|^open-cluster-management$|^openshift-adp$|^openshift-apiserver-operator$|^openshift-authentication$|^openshift-authentication-operator$|^openshift-builds$|^openshift-catalogd$|^openshift-cloud-controller-manager$|^openshift-cloud-controller-manager-operator$|^openshift-cloud-credential-operator$|^openshift-cloud-network-config-controller$|^openshift-cluster-csi-drivers$|^openshift-cluster-machine-approver$|^openshift-cluster-node-tuning-operator$|^openshift-cluster-observability-operator$|^openshift-cluster-olm-operator$|^openshift-cluster-samples-operator$|^openshift-cluster-storage-operator$|^openshift-cluster-version$|^openshift-cnv$|^openshift-compliance$|^openshift-config$|^openshift-config-managed$|^openshift-config-operator$|^openshift-console$|^openshift-console-operator$|^openshift-console-user-settings$|^openshift-controller-manager$|^openshift-controller-manager-operator$|^openshift-dbaas-operator$|^openshift-distributed-tracing$|^openshift-dns$|^openshift-dns-operator$|^openshift-dpu-network-operator$|^openshift-dr-system$|^openshift-etcd$|^openshift-etcd-operator$|^openshift-file-integrity$|^openshift-gitops$|^openshift-gitops-operator$|^openshift-host-network$|^openshift-image-registry$|^openshift-infra$|^openshift-ingress$|^openshift-ingress-canary$|^openshift-ingress-node-firewall$|^openshift-ingress-operator$|^openshift-insights$|^openshift-keda$|^openshift-kmm$|^openshift-kmm-hub$|^openshift-kni-infra$|^openshift-kube-apiserver$|^openshift-kube-apiserver-operator$|^openshift-kube-controller-manager$|^openshift-kube-controller-manager-operator$|^openshift-kube-scheduler$|^openshift-kube-scheduler-operator$|^openshift-kube-storage-version-migrator$|^openshift-kube-storage-version-migrator-operator$|^openshift-lifecycle-agent$|^openshift-local-storage$|^openshift-logging$|^openshift-machine-api$|^openshift-machine-config-operator$|^openshift-marketplace$|^openshift-migration$|^openshift-monitoring$|^openshift-mta$|^openshift-mtv$|^openshift-multus$|^openshift-netobserv-operator$|^openshift-network-console$|^openshift-network-diagnostics$|^openshift-network-node-identity$|^openshift-network-operator$|^openshift-nfd$|^openshift-nmstate$|^openshift-node$|^openshift-nutanix-infra$|^openshift-oauth-apiserver$|^openshift-openstack-infra$|^openshift-opentelemetry-operator$|^openshift-operator-controller$|^openshift-operator-lifecycle-manager$|^openshift-operators$|^openshift-operators-redhat$|^openshift-ovirt-infra$|^openshift-ovn-kubernetes$|^openshift-ptp$|^openshift-route-controller-manager$|^openshift-sandboxed-containers-operator$|^openshift-security-profiles$|^openshift-serverless$|^openshift-serverless-logic$|^openshift-service-ca$|^openshift-service-ca-operator$|^openshift-sriov-network-operator$|^openshift-storage$|^openshift-tempo-operator$|^openshift-update-service$|^openshift-user-workload-monitoring$|^openshift-vertical-pod-autoscaler$|^openshift-vsphere-infra$|^openshift-windows-machine-config-operator$|^openshift-workload-availability$|^redhat-ods-operator$|^rhdh-operator$|^rhacs-operator$|^service-telemetry$|^stackrox$|^submariner-operator$` ) var ( diff --git a/central/config/datastore/singleton_test.go b/central/config/datastore/singleton_test.go index 6bed2f7a429a2..d917715bfda80 100644 --- a/central/config/datastore/singleton_test.go +++ b/central/config/datastore/singleton_test.go @@ -100,6 +100,7 @@ func TestPlatformComponentLayeredProductsRegex(t *testing.T) { "openshift-mtv", "openshift-multus", "openshift-netobserv-operator", + "openshift-network-console", "openshift-network-diagnostics", "openshift-network-node-identity", "openshift-network-operator", @@ -134,6 +135,7 @@ func TestPlatformComponentLayeredProductsRegex(t *testing.T) { "openshift-windows-machine-config-operator", "openshift-workload-availability", "redhat-ods-operator", + "rhacs-operator", "rhdh-operator", "service-telemetry", "stackrox", @@ -217,8 +219,8 @@ func TestPlatformComponentLayeredProductsRegex(t *testing.T) { t.Run("PatternCount", func(t *testing.T) { // Count the number of valid namespaces we've defined expectedCount := len(validNamespaces) - assert.Equal(t, 124, expectedCount, - "Expected 124 valid namespace patterns in the test array") + assert.Equal(t, 126, expectedCount, + "Expected 126 valid namespace patterns in the test array") }) } diff --git a/central/convert/internaltostorage/virtual_machine_test.go b/central/convert/internaltostorage/virtual_machine_test.go index 14234ff71fc89..4d9487aab8bee 100644 --- a/central/convert/internaltostorage/virtual_machine_test.go +++ b/central/convert/internaltostorage/virtual_machine_test.go @@ -7,6 +7,7 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/uuid" + "github.com/stackrox/rox/pkg/virtualmachine" "github.com/stretchr/testify/assert" ) @@ -29,7 +30,7 @@ func TestVirtualMachine(t *testing.T) { Name: "virtual-machine-name", ClusterId: uuid.NewTestUUID(1).String(), Facts: map[string]string{ - "Guest OS": "Red Hat Enterprise Linux", + virtualmachine.GuestOSKey: "Red Hat Enterprise Linux", }, VsockCid: 42, State: virtualMachineV1.VirtualMachine_RUNNING, @@ -40,7 +41,7 @@ func TestVirtualMachine(t *testing.T) { Name: "virtual-machine-name", ClusterId: uuid.NewTestUUID(1).String(), Facts: map[string]string{ - "Guest OS": "Red Hat Enterprise Linux", + virtualmachine.GuestOSKey: "Red Hat Enterprise Linux", }, VsockCid: 42, State: storage.VirtualMachine_RUNNING, @@ -54,10 +55,10 @@ func TestVirtualMachine(t *testing.T) { Name: "virtual-machine-name-2", ClusterId: uuid.NewTestUUID(2).String(), Facts: map[string]string{ - "Guest OS": "Red Hat Enterprise Linux", - "Node Name": "node-1", - "IP Address": "10.0.0.1", - "Description": "test description", + virtualmachine.GuestOSKey: "Red Hat Enterprise Linux", + virtualmachine.NodeNameKey: "node-1", + virtualmachine.IPAddressesKey: "10.0.0.1", + virtualmachine.DescriptionKey: "test description", }, VsockCid: 84, State: virtualMachineV1.VirtualMachine_STOPPED, @@ -68,10 +69,10 @@ func TestVirtualMachine(t *testing.T) { Name: "virtual-machine-name-2", ClusterId: uuid.NewTestUUID(2).String(), Facts: map[string]string{ - "Guest OS": "Red Hat Enterprise Linux", - "Node Name": "node-1", - "IP Address": "10.0.0.1", - "Description": "test description", + virtualmachine.GuestOSKey: "Red Hat Enterprise Linux", + virtualmachine.NodeNameKey: "node-1", + virtualmachine.IPAddressesKey: "10.0.0.1", + virtualmachine.DescriptionKey: "test description", }, VsockCid: 84, State: storage.VirtualMachine_STOPPED, diff --git a/central/convert/internaltostorage/virtual_machine_v2.go b/central/convert/internaltostorage/virtual_machine_v2.go new file mode 100644 index 0000000000000..34af48f85c62a --- /dev/null +++ b/central/convert/internaltostorage/virtual_machine_v2.go @@ -0,0 +1,43 @@ +package internaltostorage + +import ( + virtualMachineV1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" + "github.com/stackrox/rox/generated/storage" + pkgVM "github.com/stackrox/rox/pkg/virtualmachine" +) + +// VirtualMachineV2 converts a sensor-internal VirtualMachine to a storage VirtualMachineV2. +func VirtualMachineV2(vm *virtualMachineV1.VirtualMachine) *storage.VirtualMachineV2 { + if vm == nil { + return nil + } + + informerOS := vm.GetFacts()[pkgVM.GuestOSKey] + if informerOS == "" { + informerOS = pkgVM.UnknownGuestOS + } + + return &storage.VirtualMachineV2{ + Id: vm.GetId(), + Name: vm.GetName(), + Namespace: vm.GetNamespace(), + ClusterId: vm.GetClusterId(), + Facts: vm.GetFacts(), + GuestOs: pkgVM.DisplayGuestOS(vm.GetFacts(), informerOS), + State: convertVirtualMachineV2State(vm.GetState()), + VsockCid: vm.GetVsockCid(), + } +} + +func convertVirtualMachineV2State(state virtualMachineV1.VirtualMachine_State) storage.VirtualMachineV2_State { + switch state { + case virtualMachineV1.VirtualMachine_UNKNOWN: + return storage.VirtualMachineV2_UNKNOWN + case virtualMachineV1.VirtualMachine_STOPPED: + return storage.VirtualMachineV2_STOPPED + case virtualMachineV1.VirtualMachine_RUNNING: + return storage.VirtualMachineV2_RUNNING + default: + return storage.VirtualMachineV2_UNKNOWN + } +} diff --git a/central/convert/internaltostorage/virtual_machine_v2_test.go b/central/convert/internaltostorage/virtual_machine_v2_test.go new file mode 100644 index 0000000000000..63047a6b157d7 --- /dev/null +++ b/central/convert/internaltostorage/virtual_machine_v2_test.go @@ -0,0 +1,147 @@ +package internaltostorage + +import ( + "testing" + + virtualMachineV1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/uuid" + pkgVM "github.com/stackrox/rox/pkg/virtualmachine" + "github.com/stretchr/testify/assert" +) + +func TestVirtualMachineV2(t *testing.T) { + tests := []struct { + name string + input *virtualMachineV1.VirtualMachine + expected *storage.VirtualMachineV2 + }{ + { + name: "nil input", + input: nil, + expected: nil, + }, + { + name: "basic virtual machine", + input: &virtualMachineV1.VirtualMachine{ + Id: "VM-ID-1", + Namespace: "virtual-machine-namespace", + Name: "virtual-machine-name", + ClusterId: uuid.NewTestUUID(1).String(), + Facts: map[string]string{ + pkgVM.GuestOSKey: "Red Hat Enterprise Linux", + }, + VsockCid: 42, + State: virtualMachineV1.VirtualMachine_RUNNING, + }, + expected: &storage.VirtualMachineV2{ + Id: "VM-ID-1", + Namespace: "virtual-machine-namespace", + Name: "virtual-machine-name", + ClusterId: uuid.NewTestUUID(1).String(), + Facts: map[string]string{ + pkgVM.GuestOSKey: "Red Hat Enterprise Linux", + }, + GuestOs: "Red Hat Enterprise Linux", + VsockCid: 42, + State: storage.VirtualMachineV2_RUNNING, + }, + }, + { + name: "writes detected guest OS into GuestOs column", + input: &virtualMachineV1.VirtualMachine{ + Id: "VM-ID-3", + Namespace: "ns", + Name: "vm-agent-facts", + ClusterId: uuid.NewTestUUID(3).String(), + Facts: map[string]string{ + pkgVM.GuestOSKey: "Red Hat Enterprise Linux", + pkgVM.DetectedGuestOSKey: "Red Hat Enterprise Linux 9.2", + pkgVM.AgentVersionKey: "4.10.0", + }, + State: virtualMachineV1.VirtualMachine_RUNNING, + }, + expected: &storage.VirtualMachineV2{ + Id: "VM-ID-3", + Namespace: "ns", + Name: "vm-agent-facts", + ClusterId: uuid.NewTestUUID(3).String(), + Facts: map[string]string{ + pkgVM.GuestOSKey: "Red Hat Enterprise Linux", + pkgVM.DetectedGuestOSKey: "Red Hat Enterprise Linux 9.2", + pkgVM.AgentVersionKey: "4.10.0", + }, + GuestOs: "Red Hat Enterprise Linux 9.2", + State: storage.VirtualMachineV2_RUNNING, + }, + }, + { + name: "virtual machine without guestOS fact", + input: &virtualMachineV1.VirtualMachine{ + Id: "VM-ID-2", + Namespace: "ns", + Name: "vm-no-os", + ClusterId: uuid.NewTestUUID(2).String(), + Facts: map[string]string{ + pkgVM.NodeNameKey: "node-1", + }, + State: virtualMachineV1.VirtualMachine_STOPPED, + }, + expected: &storage.VirtualMachineV2{ + Id: "VM-ID-2", + Namespace: "ns", + Name: "vm-no-os", + ClusterId: uuid.NewTestUUID(2).String(), + Facts: map[string]string{ + pkgVM.NodeNameKey: "node-1", + }, + GuestOs: pkgVM.UnknownGuestOS, + State: storage.VirtualMachineV2_STOPPED, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(it *testing.T) { + converted := VirtualMachineV2(tt.input) + protoassert.Equal(it, tt.expected, converted) + }) + } +} + +func TestConvertVirtualMachineV2State(t *testing.T) { + tests := []struct { + name string + input virtualMachineV1.VirtualMachine_State + expected storage.VirtualMachineV2_State + }{ + { + name: "UNKNOWN", + input: virtualMachineV1.VirtualMachine_UNKNOWN, + expected: storage.VirtualMachineV2_UNKNOWN, + }, + { + name: "STOPPED", + input: virtualMachineV1.VirtualMachine_STOPPED, + expected: storage.VirtualMachineV2_STOPPED, + }, + { + name: "RUNNING", + input: virtualMachineV1.VirtualMachine_RUNNING, + expected: storage.VirtualMachineV2_RUNNING, + }, + { + name: "Other", + input: virtualMachineV1.VirtualMachine_State(-1), + expected: storage.VirtualMachineV2_UNKNOWN, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(it *testing.T) { + got := convertVirtualMachineV2State(tt.input) + assert.Equal(it, tt.expected, got) + }) + } +} diff --git a/central/convert/internaltov2storage/compliance_operator_profile.go b/central/convert/internaltov2storage/compliance_operator_profile.go index a7abf8bfb0de3..f40e87e99ba54 100644 --- a/central/convert/internaltov2storage/compliance_operator_profile.go +++ b/central/convert/internaltov2storage/compliance_operator_profile.go @@ -8,7 +8,7 @@ import ( // ComplianceOperatorProfileV2 converts internal api profiles to V2 storage profiles func ComplianceOperatorProfileV2(internalMsg *central.ComplianceOperatorProfileV2, clusterID string) *storage.ComplianceOperatorProfileV2 { - var rules []*storage.ComplianceOperatorProfileV2_Rule + rules := make([]*storage.ComplianceOperatorProfileV2_Rule, 0, len(internalMsg.GetRules())) for _, r := range internalMsg.GetRules() { rules = append(rules, &storage.ComplianceOperatorProfileV2_Rule{ RuleName: r.GetRuleName(), @@ -32,5 +32,68 @@ func ComplianceOperatorProfileV2(internalMsg *central.ComplianceOperatorProfileV Values: internalMsg.GetValues(), ClusterId: clusterID, ProfileRefId: BuildProfileRefID(clusterID, internalMsg.GetProfileId(), productType), + OperatorKind: centralToStorageProfileKind(internalMsg.GetOperatorKind()), + } +} + +// StorageToCentralProfileKind converts a storage profile OperatorKind to the internal API +// equivalent for sending to Sensor (e.g. when building scan config sync messages). +func StorageToCentralProfileKind(kind storage.ComplianceOperatorProfileV2_OperatorKind) central.ComplianceOperatorProfileV2_OperatorKind { + switch kind { + case storage.ComplianceOperatorProfileV2_PROFILE: + return central.ComplianceOperatorProfileV2_PROFILE + case storage.ComplianceOperatorProfileV2_TAILORED_PROFILE: + return central.ComplianceOperatorProfileV2_TAILORED_PROFILE + case storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED: + // ROX-31229: Profiles stored by older Central versions may have UNSPECIFIED kind. + // These are always regular profiles; treat as PROFILE. Mirrors centralToStorageProfileKind. + // This fallback can be removed once Central versions that pre-date kind tracking are not supported. + return central.ComplianceOperatorProfileV2_PROFILE + default: + return central.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED + } +} + +// ProfileV2ToScanConfigRefs extracts name and kind from a slice of stored profiles into +// the scan configuration's ProfileReference storage type. Used when persisting profile_refs +// alongside a scan config so the startup sync path has correct kinds on reconnect. +func ProfileV2ToScanConfigRefs(profiles []*storage.ComplianceOperatorProfileV2) []*storage.ComplianceOperatorScanConfigurationV2_ProfileReference { + refs := make([]*storage.ComplianceOperatorScanConfigurationV2_ProfileReference, 0, len(profiles)) + for _, p := range profiles { + refs = append(refs, &storage.ComplianceOperatorScanConfigurationV2_ProfileReference{ + Name: p.GetName(), + Kind: p.GetOperatorKind(), + }) + } + return refs +} + +// ScanConfigRefsToCentral converts storage scan config ProfileReferences to the internal API +// equivalent for scan config messages sent to Sensor. +func ScanConfigRefsToCentral(refs []*storage.ComplianceOperatorScanConfigurationV2_ProfileReference) []*central.ApplyComplianceScanConfigRequest_BaseScanSettings_ProfileReference { + centralRefs := make([]*central.ApplyComplianceScanConfigRequest_BaseScanSettings_ProfileReference, 0, len(refs)) + for _, ref := range refs { + centralRefs = append(centralRefs, ¢ral.ApplyComplianceScanConfigRequest_BaseScanSettings_ProfileReference{ + Name: ref.GetName(), + Kind: StorageToCentralProfileKind(ref.GetKind()), + }) + } + return centralRefs +} + +func centralToStorageProfileKind(kind central.ComplianceOperatorProfileV2_OperatorKind) storage.ComplianceOperatorProfileV2_OperatorKind { + switch kind { + case central.ComplianceOperatorProfileV2_PROFILE: + return storage.ComplianceOperatorProfileV2_PROFILE + case central.ComplianceOperatorProfileV2_TAILORED_PROFILE: + return storage.ComplianceOperatorProfileV2_TAILORED_PROFILE + case central.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED: + // ROX-31229: Older sensors do not set OperatorKind for regular (non-tailored) + // profiles, so UNSPECIFIED is treated as PROFILE. This fallback can be + // removed once versions that don't set OperatorKind (<= 4.10) are not supported. + return storage.ComplianceOperatorProfileV2_PROFILE + default: + log.Warnf("Unexpected profile operator kind %v", kind) + return storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED } } diff --git a/central/convert/internaltov2storage/compliance_operator_profile_test.go b/central/convert/internaltov2storage/compliance_operator_profile_test.go new file mode 100644 index 0000000000000..901241bcaf16b --- /dev/null +++ b/central/convert/internaltov2storage/compliance_operator_profile_test.go @@ -0,0 +1,45 @@ +package internaltov2storage + +import ( + "testing" + + "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" +) + +func TestStorageToCentralProfileKind(t *testing.T) { + testCases := []struct { + name string + input storage.ComplianceOperatorProfileV2_OperatorKind + expected central.ComplianceOperatorProfileV2_OperatorKind + }{ + { + name: "profile", + input: storage.ComplianceOperatorProfileV2_PROFILE, + expected: central.ComplianceOperatorProfileV2_PROFILE, + }, + { + name: "tailored profile", + input: storage.ComplianceOperatorProfileV2_TAILORED_PROFILE, + expected: central.ComplianceOperatorProfileV2_TAILORED_PROFILE, + }, + { + name: "unspecified falls back to profile for backward compatibility", + input: storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED, + expected: central.ComplianceOperatorProfileV2_PROFILE, + }, + { + name: "unknown kind falls back to unspecified", + input: storage.ComplianceOperatorProfileV2_OperatorKind(999), + expected: central.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := StorageToCentralProfileKind(tc.input) + assert.Equal(t, tc.expected, actual) + }) + } +} diff --git a/central/convert/internaltov2storage/compliance_rule.go b/central/convert/internaltov2storage/compliance_rule.go index 4b20296129c8a..ecc72ab3f01a9 100644 --- a/central/convert/internaltov2storage/compliance_rule.go +++ b/central/convert/internaltov2storage/compliance_rule.go @@ -45,10 +45,10 @@ func ComplianceOperatorRule(sensorData *central.ComplianceOperatorRuleV2, cluste standards := strings.Split(sensorData.GetAnnotations()[standardsKey], ",") controls := make([]*storage.RuleControls, 0, len(standards)) for _, standard := range standards { - controlAnnotationValues := strings.Split(sensorData.GetAnnotations()[controlAnnotationBase+standard], ";") + controlAnnotationValues := strings.SplitSeq(sensorData.GetAnnotations()[controlAnnotationBase+standard], ";") // Add a control entry for each Control + Standard. This data is intentionally denormalized for easier querying. - for _, controlValue := range controlAnnotationValues { + for controlValue := range controlAnnotationValues { controls = append(controls, &storage.RuleControls{ Standard: standard, Control: controlValue, @@ -56,7 +56,19 @@ func ComplianceOperatorRule(sensorData *central.ComplianceOperatorRuleV2, cluste } } + // parentRule is the DNS-friendly rule name shared between the rule and its check results, + // used as the input to BuildNameRefID (the join key between the two tables). + // + // For regular rules, CO sets compliance.openshift.io/rule on the Rule object to its own + // DNS-friendly name, so we read it from the annotation. + // + // For custom rules, CO does not set that annotation (CustomRule is a different CRD). + // CO derives the check result annotation from Spec.ID via IDToDNSFriendlyName, so we + // apply the same transformation here to produce a matching parentRule. parentRule := sensorData.GetAnnotations()[v1alpha1.RuleIDAnnotationKey] + if sensorData.GetOperatorKind() == central.ComplianceOperatorRuleV2_CUSTOM_RULE { + parentRule = idToDNSFriendlyName(sensorData.GetRuleId()) + } return &storage.ComplianceOperatorRuleV2{ Id: sensorData.GetId(), @@ -76,5 +88,23 @@ func ComplianceOperatorRule(sensorData *central.ComplianceOperatorRuleV2, cluste RuleRefId: BuildNameRefID(clusterID, parentRule), Instructions: sensorData.GetInstructions(), ParentRule: parentRule, + OperatorKind: centralToStorageRuleKind(sensorData.GetOperatorKind()), + } +} + +func centralToStorageRuleKind(kind central.ComplianceOperatorRuleV2_OperatorKind) storage.ComplianceOperatorRuleV2_OperatorKind { + switch kind { + case central.ComplianceOperatorRuleV2_RULE: + return storage.ComplianceOperatorRuleV2_RULE + case central.ComplianceOperatorRuleV2_CUSTOM_RULE: + return storage.ComplianceOperatorRuleV2_CUSTOM_RULE + case central.ComplianceOperatorRuleV2_OPERATOR_KIND_UNSPECIFIED: + // Older sensors do not set OperatorKind for regular (non-custom) rules, + // so UNSPECIFIED is treated as RULE. This fallback can be removed when + // versions that don't set OperatorKind (<= 4.10) are not supported. + return storage.ComplianceOperatorRuleV2_RULE + default: + log.Errorf("Unexpected rule operator kind %v", kind) + return storage.ComplianceOperatorRuleV2_OPERATOR_KIND_UNSPECIFIED } } diff --git a/central/convert/internaltov2storage/compliance_rule_test.go b/central/convert/internaltov2storage/compliance_rule_test.go new file mode 100644 index 0000000000000..ed1c4e0c1751b --- /dev/null +++ b/central/convert/internaltov2storage/compliance_rule_test.go @@ -0,0 +1,103 @@ +package internaltov2storage + +import ( + "testing" + + "github.com/ComplianceAsCode/compliance-operator/pkg/apis/compliance/v1alpha1" + "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" + "github.com/stretchr/testify/assert" +) + +func Test_idToDNSFriendlyName(t *testing.T) { + cases := map[string]struct { + input string + expected string + }{ + "ssgproject rule ID strips prefix": { + input: "xccdf_org.ssgproject.content_rule_kubelet_configure_tls_cert", + expected: "kubelet-configure-tls-cert", + }, + "ssgproject rule ID without content_ segment does not strip prefix": { + input: "xccdf_org.ssgproject.rule_abc", + expected: "xccdf-org.ssgproject.rule-abc", + }, + "custom rule ID with non-ssgproject prefix keeps prefix": { + input: "xccdf_org.example_rule_check_no_latest_tag", + expected: "xccdf-org.example-rule-check-no-latest-tag", + }, + "already lowercase, no underscores": { + input: "some-rule", + expected: "some-rule", + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, idToDNSFriendlyName(tc.input)) + }) + } +} + +func TestComplianceOperatorRule_ParentRuleAndRuleRefId(t *testing.T) { + const clusterID = fixtureconsts.Cluster1 + + t.Run("regular rule uses annotation for parentRule", func(t *testing.T) { + msg := ¢ral.ComplianceOperatorRuleV2{ + Id: "rule-uid", + RuleId: "xccdf_org.ssgproject.content_rule_some_other_rule", + Name: "kubelet-configure-tls-cert", + OperatorKind: central.ComplianceOperatorRuleV2_RULE, + Annotations: map[string]string{ + // The annotation value differs from idToDNSFriendlyName(RuleId) to confirm + // that parentRule is read from the annotation, not derived from RuleId. + v1alpha1.RuleIDAnnotationKey: "kubelet-configure-tls-cert", + }, + } + + result := ComplianceOperatorRule(msg, clusterID) + + assert.Equal(t, "kubelet-configure-tls-cert", result.GetParentRule()) + assert.Equal(t, BuildNameRefID(clusterID, "kubelet-configure-tls-cert"), result.GetRuleRefId()) + }) + + t.Run("unspecified OperatorKind is treated as RULE", func(t *testing.T) { + msg := ¢ral.ComplianceOperatorRuleV2{ + Id: "rule-uid", + RuleId: "xccdf_org.ssgproject.content_rule_some_rule", + Name: "some-rule", + Annotations: map[string]string{ + v1alpha1.RuleIDAnnotationKey: "some-rule", + }, + } + + result := ComplianceOperatorRule(msg, clusterID) + + assert.Equal(t, storage.ComplianceOperatorRuleV2_RULE, result.GetOperatorKind()) + assert.Equal(t, "some-rule", result.GetParentRule()) + assert.Equal(t, BuildNameRefID(clusterID, "some-rule"), result.GetRuleRefId()) + }) + + t.Run("custom rule derives parentRule from RuleId, ignoring RuleIDAnnotationKey annotation", func(t *testing.T) { + msg := ¢ral.ComplianceOperatorRuleV2{ + Id: "custom-rule-uid", + RuleId: "xccdf_org.example_rule_check_no_latest_tag", + Name: "check-no-latest-tag", + OperatorKind: central.ComplianceOperatorRuleV2_CUSTOM_RULE, + Annotations: map[string]string{ + // CustomRules do not have compliance.openshift.io/rule set by CO, but even + // if one were present it must be ignored: parentRule always comes from RuleId. + v1alpha1.RuleIDAnnotationKey: "some-ignored-annotation", + }, + } + + result := ComplianceOperatorRule(msg, clusterID) + + // CO sets compliance.openshift.io/rule on custom rule check results to + // IDToDNSFriendlyName(Spec.ID). parentRule must equal that value so that + // BuildNameRefID produces matching RuleRefIds on both sides of the join. + expectedParentRule := "xccdf-org.example-rule-check-no-latest-tag" + assert.Equal(t, expectedParentRule, result.GetParentRule()) + assert.Equal(t, BuildNameRefID(clusterID, expectedParentRule), result.GetRuleRefId()) + }) +} diff --git a/central/convert/internaltov2storage/utils.go b/central/convert/internaltov2storage/utils.go index 6f8c14891cdfa..d632ba6f2553b 100644 --- a/central/convert/internaltov2storage/utils.go +++ b/central/convert/internaltov2storage/utils.go @@ -1,6 +1,8 @@ package internaltov2storage import ( + "strings" + "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/uuid" ) @@ -25,6 +27,15 @@ func BuildNameRefID(clusterID string, name string) string { return buildDeterministicID(clusterID, name) } +// idToDNSFriendlyName mirrors compliance-operator's IDToDNSFriendlyName (https://github.com/ComplianceAsCode/compliance-operator/blob/ae5d84d57ee9abb1a6e6692fdf093d54283b739e/pkg/utils/nameutils.go#L43): +// it strips the standard ssgproject XCCDF prefix (if present) then replaces underscores with hyphens. +// Used to derive parentRule for CustomRule objects, which carry their identifier in Spec.ID +// rather than in the compliance.openshift.io/rule annotation. +func idToDNSFriendlyName(id string) string { + const ssgPrefix = "xccdf_org.ssgproject.content_rule_" + return strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(id, ssgPrefix), "_", "-")) +} + func buildDeterministicID(part1 string, part2 string) string { baseUUID, err := uuid.FromString(part1) if err != nil { diff --git a/central/convert/storagetov1/auth_m2m_config.go b/central/convert/storagetov1/auth_m2m_config.go index ac9f3d15ce018..3e1442af6a682 100644 --- a/central/convert/storagetov1/auth_m2m_config.go +++ b/central/convert/storagetov1/auth_m2m_config.go @@ -23,6 +23,7 @@ func AuthM2MConfig(config *storage.AuthMachineToMachineConfig) *v1.AuthMachineTo TokenExpirationDuration: config.GetTokenExpirationDuration(), Mappings: convertMappings(config.GetMappings()), Issuer: config.GetIssuer(), + Audience: config.GetAudience(), Traits: Traits(config.GetTraits()), } diff --git a/central/convert/storagetov1/auth_m2m_config_test.go b/central/convert/storagetov1/auth_m2m_config_test.go index 43bb0b5a3347b..ffea2ded2e5e6 100644 --- a/central/convert/storagetov1/auth_m2m_config_test.go +++ b/central/convert/storagetov1/auth_m2m_config_test.go @@ -17,6 +17,8 @@ func TestAuthM2MConfig(t *testing.T) { v1Config := AuthM2MConfig(config) + // Clear ExpiresAt since v1.Traits doesn't have this field (internal storage field). + config.GetTraits().ExpiresAt = nil convertTestUtils.AssertProtoMessageEqual(t, config, v1Config) assert.IsType(t, &v1.AuthMachineToMachineConfig{}, v1Config) } diff --git a/central/convert/storagetov1/traits_test.go b/central/convert/storagetov1/traits_test.go index db6a55c7122d5..8a6e1cb47f4f4 100644 --- a/central/convert/storagetov1/traits_test.go +++ b/central/convert/storagetov1/traits_test.go @@ -15,5 +15,7 @@ func TestTraits(t *testing.T) { v1Traits := Traits(traits) + // Clear ExpiresAt since v1.Traits doesn't have this field (internal storage field). + traits.ExpiresAt = nil convertTestUtils.AssertProtoMessageEqual(t, traits, v1Traits) } diff --git a/central/convert/storagetov2/compliance_profile_service.go b/central/convert/storagetov2/compliance_profile_service.go index 69b7b01257a7d..f8d3ba45a08d2 100644 --- a/central/convert/storagetov2/compliance_profile_service.go +++ b/central/convert/storagetov2/compliance_profile_service.go @@ -1,8 +1,10 @@ package storagetov2 import ( + "github.com/pkg/errors" v2 "github.com/stackrox/rox/generated/api/v2" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/utils" ) // ComplianceV2Profile converts V2 storage objects to V2 API objects @@ -36,6 +38,7 @@ func ComplianceV2Profile(incoming *storage.ComplianceOperatorProfileV2, benchmar Product: incoming.GetProduct(), Title: incoming.GetTitle(), Values: incoming.GetValues(), + OperatorKind: convertProfileOperatorKind(incoming.GetOperatorKind()), } } @@ -93,6 +96,7 @@ func ComplianceProfileSummary(incoming []*storage.ComplianceOperatorProfileV2, b RuleCount: int32(len(summary.GetRules())), ProfileVersion: summary.GetProfileVersion(), Standards: profileBenchmarkNameMap[summary.GetName()], + OperatorKind: convertProfileSummaryOperatorKind(summary.GetOperatorKind()), } orderedProfiles = append(orderedProfiles, summary.GetName()) } @@ -105,3 +109,37 @@ func ComplianceProfileSummary(incoming []*storage.ComplianceOperatorProfileV2, b return summaries } + +func convertProfileOperatorKind(kind storage.ComplianceOperatorProfileV2_OperatorKind) v2.ComplianceProfile_OperatorKind { + switch kind { + case storage.ComplianceOperatorProfileV2_PROFILE: + return v2.ComplianceProfile_PROFILE + case storage.ComplianceOperatorProfileV2_TAILORED_PROFILE: + return v2.ComplianceProfile_TAILORED_PROFILE + case storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED: + // Older centrals may have stored profiles without OperatorKind, + // so UNSPECIFIED is treated as PROFILE. This fallback can be removed when + // versions that don't set OperatorKind (<= 4.10) are not supported. + return v2.ComplianceProfile_PROFILE + default: + utils.Should(errors.Errorf("unhandled profile operator kind %s", kind)) + return v2.ComplianceProfile_OPERATOR_KIND_UNSPECIFIED + } +} + +func convertProfileSummaryOperatorKind(kind storage.ComplianceOperatorProfileV2_OperatorKind) v2.ComplianceProfileSummary_OperatorKind { + switch kind { + case storage.ComplianceOperatorProfileV2_PROFILE: + return v2.ComplianceProfileSummary_PROFILE + case storage.ComplianceOperatorProfileV2_TAILORED_PROFILE: + return v2.ComplianceProfileSummary_TAILORED_PROFILE + case storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED: + // Older centrals may have stored profiles without OperatorKind, + // so UNSPECIFIED is treated as PROFILE. This fallback can be removed when + // versions that don't set OperatorKind (<= 4.10) are not supported. + return v2.ComplianceProfileSummary_PROFILE + default: + utils.Should(errors.Errorf("unhandled profile summary operator kind %s", kind)) + return v2.ComplianceProfileSummary_OPERATOR_KIND_UNSPECIFIED + } +} diff --git a/central/convert/storagetov2/compliance_profile_service_test.go b/central/convert/storagetov2/compliance_profile_service_test.go new file mode 100644 index 0000000000000..83b6006d120a1 --- /dev/null +++ b/central/convert/storagetov2/compliance_profile_service_test.go @@ -0,0 +1,59 @@ +package storagetov2 + +import ( + "testing" + + v2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" +) + +func TestConvertProfileOperatorKind(t *testing.T) { + cases := map[string]struct { + input storage.ComplianceOperatorProfileV2_OperatorKind + expected v2.ComplianceProfile_OperatorKind + }{ + "PROFILE": { + input: storage.ComplianceOperatorProfileV2_PROFILE, + expected: v2.ComplianceProfile_PROFILE, + }, + "TAILORED_PROFILE": { + input: storage.ComplianceOperatorProfileV2_TAILORED_PROFILE, + expected: v2.ComplianceProfile_TAILORED_PROFILE, + }, + "UNSPECIFIED is treated as PROFILE": { + input: storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED, + expected: v2.ComplianceProfile_PROFILE, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, convertProfileOperatorKind(tc.input)) + }) + } +} + +func TestConvertProfileSummaryOperatorKind(t *testing.T) { + cases := map[string]struct { + input storage.ComplianceOperatorProfileV2_OperatorKind + expected v2.ComplianceProfileSummary_OperatorKind + }{ + "PROFILE": { + input: storage.ComplianceOperatorProfileV2_PROFILE, + expected: v2.ComplianceProfileSummary_PROFILE, + }, + "TAILORED_PROFILE": { + input: storage.ComplianceOperatorProfileV2_TAILORED_PROFILE, + expected: v2.ComplianceProfileSummary_TAILORED_PROFILE, + }, + "UNSPECIFIED is treated as PROFILE": { + input: storage.ComplianceOperatorProfileV2_OPERATOR_KIND_UNSPECIFIED, + expected: v2.ComplianceProfileSummary_PROFILE, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, convertProfileSummaryOperatorKind(tc.input)) + }) + } +} diff --git a/central/convert/storagetov2/compliance_results_service.go b/central/convert/storagetov2/compliance_results_service.go index e11d483237b99..ad82673bec7d6 100644 --- a/central/convert/storagetov2/compliance_results_service.go +++ b/central/convert/storagetov2/compliance_results_service.go @@ -8,19 +8,6 @@ import ( types "github.com/stackrox/rox/pkg/protocompat" ) -type checkResultKey struct { - scanConfigName string - scanConfigID string - profileName string - checkName string -} - -type scanResultKey struct { - scanConfigName string - scanConfigID string - profileName string -} - // ComplianceV2CheckResult converts a storage check result to a v2 check result func ComplianceV2CheckResult(incoming *storage.ComplianceOperatorCheckResultV2, lastScanTime *types.Timestamp, ruleName string, controlResults []*compRule.ControlResult) *v2.ComplianceClusterCheckStatus { converted := &v2.ComplianceClusterCheckStatus{ @@ -74,77 +61,9 @@ func ComplianceV2SpecificCheckResult(incoming []*storage.ComplianceOperatorCheck return converted } -// ComplianceV2ScanResults converts the storage check results to v2 scan results -func ComplianceV2ScanResults(incoming []*storage.ComplianceOperatorCheckResultV2, scanToScanID map[string]string) []*v2.ComplianceScanResult { - // Since a check result can hold the status for multiple clusters we need to build from - // bottom up. resultsByScanCheck holds check result based on the key combination of - // scanName, profileName, checkName. Then when that key is encountered again we - // add the cluster information to the check result. - resultsByScanCheck := make(map[checkResultKey]*v2.ComplianceClusterCheckStatus) - - // Used to maintain sort order from the query since maps are unordered. - var orderedKeys []checkResultKey - for _, result := range incoming { - key := checkResultKey{ - scanConfigName: result.GetScanConfigName(), - scanConfigID: scanToScanID[result.GetScanConfigName()], - profileName: "", - checkName: result.GetCheckName(), - } - workingResult, found := resultsByScanCheck[key] - // First time seeing this rule in the results. - if !found { - orderedKeys = append(orderedKeys, key) - resultsByScanCheck[key] = ComplianceV2CheckResult(result, nil, "", nil) - } else { - // Append the new cluster status to the v2 check result. - workingResult.Clusters = append(workingResult.Clusters, clusterStatus(result, nil)) - resultsByScanCheck[key] = workingResult - } - } - - // This builds the outer piece of the result. The key is simply - // scan name and profile name. The individual check results from - // resultsByScanCheck are appended to build the output - resultsByScan := make(map[scanResultKey][]*v2.ComplianceClusterCheckStatus) - var scanOrder []scanResultKey - var convertedResults []*v2.ComplianceScanResult - for _, key := range orderedKeys { - scanKey := scanResultKey{ - scanConfigName: key.scanConfigName, - scanConfigID: key.scanConfigID, - profileName: key.profileName, - } - result, resultFound := resultsByScanCheck[key] - if !resultFound { - continue - } - workingResult, found := resultsByScan[scanKey] - // First time seeing this rule in the results. - if !found { - scanOrder = append(scanOrder, scanKey) - resultsByScan[scanKey] = []*v2.ComplianceClusterCheckStatus{result} - } else { - workingResult = append(workingResult, result) - resultsByScan[scanKey] = workingResult - } - } - - for _, key := range scanOrder { - convertedResults = append(convertedResults, &v2.ComplianceScanResult{ - ScanName: key.scanConfigName, - ScanConfigId: key.scanConfigID, - ProfileName: key.profileName, - CheckResults: resultsByScan[key], - }) - } - - return convertedResults -} - // ComplianceV2ProfileResults converts the counts to the v2 stats func ComplianceV2ProfileResults(resultCounts []*datastore.ResourceResultsByProfile, controlResults []*compRule.ControlResult) []*v2.ComplianceCheckResultStatusCount { - var profileResults []*v2.ComplianceCheckResultStatusCount + profileResults := make([]*v2.ComplianceCheckResultStatusCount, 0, len(resultCounts)) for _, resultCount := range resultCounts { controls := GetControls(resultCount.RuleName, controlResults) diff --git a/central/convert/storagetov2/compliance_results_stats_service.go b/central/convert/storagetov2/compliance_results_stats_service.go index da4e96b767656..af3c10eeece4f 100644 --- a/central/convert/storagetov2/compliance_results_stats_service.go +++ b/central/convert/storagetov2/compliance_results_stats_service.go @@ -10,7 +10,7 @@ import ( // ComplianceV2ClusterStats converts the counts to the v2 stats func ComplianceV2ClusterStats(resultCounts []*datastore.ResourceResultCountByClusterScan, scanToScanID map[string]string) []*v2.ComplianceClusterScanStats { - var convertedResults []*v2.ComplianceClusterScanStats + convertedResults := make([]*v2.ComplianceClusterScanStats, 0, len(resultCounts)) for _, resultCount := range resultCounts { convertedResults = append(convertedResults, &v2.ComplianceClusterScanStats{ @@ -59,7 +59,7 @@ func ComplianceV2ClusterStats(resultCounts []*datastore.ResourceResultCountByClu // ComplianceV2ClusterOverallStats converts the counts to the v2 stats func ComplianceV2ClusterOverallStats(resultCounts []*datastore.ResultStatusCountByCluster, clusterErrors map[string][]string) []*v2.ComplianceClusterOverallStats { - var convertedResults []*v2.ComplianceClusterOverallStats + convertedResults := make([]*v2.ComplianceClusterOverallStats, 0, len(resultCounts)) for _, resultCount := range resultCounts { var lastScanTime *types.Timestamp @@ -110,7 +110,7 @@ func ComplianceV2ClusterOverallStats(resultCounts []*datastore.ResultStatusCount // ComplianceV2ProfileStats converts the counts to the v2 stats func ComplianceV2ProfileStats(resultCounts []*datastore.ResourceResultCountByProfile, profileMap map[string]*storage.ComplianceOperatorProfileV2, profileBenchmarkMap map[string][]*storage.ComplianceOperatorBenchmarkV2) []*v2.ComplianceProfileScanStats { - var convertedResults []*v2.ComplianceProfileScanStats + convertedResults := make([]*v2.ComplianceProfileScanStats, 0, len(resultCounts)) for _, resultCount := range resultCounts { convertedResults = append(convertedResults, &v2.ComplianceProfileScanStats{ @@ -154,7 +154,7 @@ func ComplianceV2ProfileStats(resultCounts []*datastore.ResourceResultCountByPro } func convertBenchmarks(incoming []*storage.ComplianceOperatorBenchmarkV2) []*v2.ComplianceBenchmark { - var convertedBenchmarks []*v2.ComplianceBenchmark + convertedBenchmarks := make([]*v2.ComplianceBenchmark, 0, len(incoming)) for _, benchmark := range incoming { convertedBenchmarks = append(convertedBenchmarks, &v2.ComplianceBenchmark{ Name: benchmark.GetName(), diff --git a/central/convert/storagetov2/compliance_rule_service.go b/central/convert/storagetov2/compliance_rule_service.go index e891281682491..da1e958f0b8ea 100644 --- a/central/convert/storagetov2/compliance_rule_service.go +++ b/central/convert/storagetov2/compliance_rule_service.go @@ -1,8 +1,10 @@ package storagetov2 import ( + "github.com/pkg/errors" v2 "github.com/stackrox/rox/generated/api/v2" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/utils" ) // ComplianceRule converts summary object to V2 API summary object @@ -28,5 +30,23 @@ func ComplianceRule(incoming *storage.ComplianceOperatorRuleV2) *v2.ComplianceRu Instructions: incoming.GetInstructions(), Warning: incoming.GetWarning(), ParentRule: incoming.GetParentRule(), + OperatorKind: convertRuleOperatorKind(incoming.GetOperatorKind()), + } +} + +func convertRuleOperatorKind(kind storage.ComplianceOperatorRuleV2_OperatorKind) v2.ComplianceRule_OperatorKind { + switch kind { + case storage.ComplianceOperatorRuleV2_RULE: + return v2.ComplianceRule_RULE + case storage.ComplianceOperatorRuleV2_CUSTOM_RULE: + return v2.ComplianceRule_CUSTOM_RULE + case storage.ComplianceOperatorRuleV2_OPERATOR_KIND_UNSPECIFIED: + // Older sensors do not set OperatorKind for regular (non-custom) rules, + // so UNSPECIFIED is treated as RULE. This fallback can be removed when + // versions that don't set OperatorKind (<= 4.10) are not supported. + return v2.ComplianceRule_RULE + default: + utils.Should(errors.Errorf("unhandled rule operator kind %s", kind)) + return v2.ComplianceRule_OPERATOR_KIND_UNSPECIFIED } } diff --git a/central/convert/storagetov2/compliance_rule_service_test.go b/central/convert/storagetov2/compliance_rule_service_test.go new file mode 100644 index 0000000000000..dbc0cb6bc931e --- /dev/null +++ b/central/convert/storagetov2/compliance_rule_service_test.go @@ -0,0 +1,34 @@ +package storagetov2 + +import ( + "testing" + + v2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" +) + +func TestConvertRuleOperatorKind(t *testing.T) { + cases := map[string]struct { + input storage.ComplianceOperatorRuleV2_OperatorKind + expected v2.ComplianceRule_OperatorKind + }{ + "RULE": { + input: storage.ComplianceOperatorRuleV2_RULE, + expected: v2.ComplianceRule_RULE, + }, + "CUSTOM_RULE": { + input: storage.ComplianceOperatorRuleV2_CUSTOM_RULE, + expected: v2.ComplianceRule_CUSTOM_RULE, + }, + "UNSPECIFIED is treated as RULE": { + input: storage.ComplianceOperatorRuleV2_OPERATOR_KIND_UNSPECIFIED, + expected: v2.ComplianceRule_RULE, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, convertRuleOperatorKind(tc.input)) + }) + } +} diff --git a/central/convert/storagetov2/virtual_machine_test.go b/central/convert/storagetov2/virtual_machine_test.go index ac057de4fb0cc..3d4bd0de69ece 100644 --- a/central/convert/storagetov2/virtual_machine_test.go +++ b/central/convert/storagetov2/virtual_machine_test.go @@ -8,6 +8,7 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/virtualmachine" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -42,9 +43,9 @@ func TestVirtualMachine(t *testing.T) { ClusterId: "cluster-456", ClusterName: "test-cluster", Facts: map[string]string{ - "Guest OS": "Red Hat Enterprise Linux", - "Node Name": "node-1", - "IP Addresses": "10.0.0.1, 10.0.0.2", + virtualmachine.GuestOSKey: "Red Hat Enterprise Linux", + virtualmachine.NodeNameKey: "node-1", + virtualmachine.IPAddressesKey: "10.0.0.1, 10.0.0.2", }, VsockCid: int32(42), State: storage.VirtualMachine_RUNNING, @@ -58,9 +59,9 @@ func TestVirtualMachine(t *testing.T) { ClusterId: "cluster-456", ClusterName: "test-cluster", Facts: map[string]string{ - "Guest OS": "Red Hat Enterprise Linux", - "Node Name": "node-1", - "IP Addresses": "10.0.0.1, 10.0.0.2", + virtualmachine.GuestOSKey: "Red Hat Enterprise Linux", + virtualmachine.NodeNameKey: "node-1", + virtualmachine.IPAddressesKey: "10.0.0.1, 10.0.0.2", }, VsockCid: int32(42), State: v2.VirtualMachine_RUNNING, diff --git a/central/convert/storagetov2/virtual_machine_v2.go b/central/convert/storagetov2/virtual_machine_v2.go new file mode 100644 index 0000000000000..c93dbcf03efbd --- /dev/null +++ b/central/convert/storagetov2/virtual_machine_v2.go @@ -0,0 +1,176 @@ +package storagetov2 + +import ( + "slices" + "time" + + "github.com/stackrox/rox/central/views/common" + v2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// VirtualMachineV2ToListItem converts a storage VM V2 to an API list item. +func VirtualMachineV2ToListItem(vm *storage.VirtualMachineV2) *v2.VMListItem { + if vm == nil { + return nil + } + return &v2.VMListItem{ + Id: vm.GetId(), + Name: vm.GetName(), + Namespace: vm.GetNamespace(), + ClusterId: vm.GetClusterId(), + ClusterName: vm.GetClusterName(), + GuestOs: vm.GetGuestOs(), + State: convertVirtualMachineV2State(vm.GetState()), + LastUpdated: vm.GetLastUpdated(), + } +} + +// VirtualMachineV2ToDetail converts a storage VM V2 to a detailed API response. +func VirtualMachineV2ToDetail(vm *storage.VirtualMachineV2) *v2.VMDetail { + if vm == nil { + return nil + } + notes := make([]v2.VMNote, 0, len(vm.GetNotes())) + for _, n := range vm.GetNotes() { + notes = append(notes, convertVMNote(n)) + } + return &v2.VMDetail{ + Id: vm.GetId(), + Name: vm.GetName(), + Namespace: vm.GetNamespace(), + ClusterId: vm.GetClusterId(), + ClusterName: vm.GetClusterName(), + GuestOs: vm.GetGuestOs(), + State: convertVirtualMachineV2State(vm.GetState()), + LastUpdated: vm.GetLastUpdated(), + Facts: vm.GetFacts(), + VsockCid: vm.GetVsockCid(), + Notes: notes, + } +} + +// AgentStatusFromLastContact maps scrape freshness onto the GetVM enum. +// A nil/invalid timestamp is UNKNOWN; age >= staleAfter is INACTIVE. +func AgentStatusFromLastContact(ts *timestamppb.Timestamp, now time.Time, staleAfter time.Duration) v2.AgentStatus { + if ts == nil || !ts.IsValid() { + return v2.AgentStatus_AGENT_STATUS_UNKNOWN + } + if now.Sub(ts.AsTime()) < staleAfter { + return v2.AgentStatus_AGENT_STATUS_ACTIVE + } + return v2.AgentStatus_AGENT_STATUS_INACTIVE +} + +func convertVMNote(note storage.VirtualMachineV2_Note) v2.VMNote { + switch note { + case storage.VirtualMachineV2_MISSING_SCAN_DATA: + return v2.VMNote_VM_NOTE_MISSING_SCAN_DATA + case storage.VirtualMachineV2_MISSING_SCANNER: + return v2.VMNote_VM_NOTE_MISSING_SCANNER + case storage.VirtualMachineV2_SCAN_FAILED: + return v2.VMNote_VM_NOTE_SCAN_FAILED + case storage.VirtualMachineV2_MISSING_SIGNATURE: + return v2.VMNote_VM_NOTE_MISSING_SIGNATURE + case storage.VirtualMachineV2_MISSING_SIGNATURE_VERIFICATION_DATA: + return v2.VMNote_VM_NOTE_MISSING_SIGNATURE_VERIFICATION_DATA + default: + return v2.VMNote_VM_NOTE_MISSING_METADATA + } +} + +func convertVirtualMachineV2State(state storage.VirtualMachineV2_State) v2.VirtualMachineV2State { + switch state { + case storage.VirtualMachineV2_STOPPED: + return v2.VirtualMachineV2State_VM_STATE_STOPPED + case storage.VirtualMachineV2_RUNNING: + return v2.VirtualMachineV2State_VM_STATE_RUNNING + default: + return v2.VirtualMachineV2State_VM_STATE_UNKNOWN + } +} + +// SeverityCountsToProto converts view severity counts to the API proto message. +func SeverityCountsToProto(counts common.ResourceCountByCVESeverity) *v2.VulnCountBySeverity { + if counts == nil { + return &v2.VulnCountBySeverity{} + } + return &v2.VulnCountBySeverity{ + Critical: fixabilityToProto(counts.GetCriticalSeverityCount()), + Important: fixabilityToProto(counts.GetImportantSeverityCount()), + Moderate: fixabilityToProto(counts.GetModerateSeverityCount()), + Low: fixabilityToProto(counts.GetLowSeverityCount()), + Unknown: fixabilityToProto(counts.GetUnknownSeverityCount()), + } +} + +// VirtualMachineCVEV2ToRow converts a storage CVE V2 to an API CVE row. +func VirtualMachineCVEV2ToRow(cve *storage.VirtualMachineCVEV2) *v2.VMCVERow { + if cve == nil { + return nil + } + return &v2.VMCVERow{ + Cve: cve.GetCveBaseInfo().GetCve(), + Severity: v2.VulnerabilitySeverity(cve.GetSeverity()), + IsFixable: cve.GetIsFixable(), + Cvss: cve.GetPreferredCvss(), + NvdCvss: cve.GetNvdcvss(), + EpssProbability: cve.GetEpssProbability(), + PublishedOn: cve.GetCveBaseInfo().GetPublishedOn(), + Summary: cve.GetCveBaseInfo().GetSummary(), + Link: cve.GetCveBaseInfo().GetLink(), + Advisory: advisoryToProto(cve.GetAdvisory()), + } +} + +// VirtualMachineComponentV2ToRow converts a storage component V2 to an API component row. +func VirtualMachineComponentV2ToRow(comp *storage.VirtualMachineComponentV2) *v2.VMComponentRow { + if comp == nil { + return nil + } + scanStatus := v2.ScanStatus_SCANNED + if slices.Contains(comp.GetNotes(), storage.VirtualMachineComponentV2_UNSCANNED) { + scanStatus = v2.ScanStatus_NOT_SCANNED + } + return &v2.VMComponentRow{ + Id: comp.GetId(), + Name: comp.GetName(), + Version: comp.GetVersion(), + Source: v2.SourceType(comp.GetSource()), + ScanStatus: scanStatus, + CveCount: comp.GetCveCount(), + } +} + +func advisoryToProto(adv *storage.Advisory) *v2.Advisory { + if adv == nil { + return nil + } + return &v2.Advisory{ + Name: adv.GetName(), + Link: adv.GetLink(), + } +} + +func fixabilityToProto(f common.ResourceCountByFixability) *v2.VulnFixableCount { + if f == nil { + return &v2.VulnFixableCount{} + } + return &v2.VulnFixableCount{ + Total: int32(f.GetTotal()), + Fixable: int32(f.GetFixable()), + } +} + +// ConvertScanNote converts a storage scan note to a v2 API scan note. +func ConvertScanNote(note storage.VirtualMachineScanV2_Note) v2.VMScanNote { + switch note { + case storage.VirtualMachineScanV2_OS_UNKNOWN: + return v2.VMScanNote_VM_SCAN_NOTE_OS_UNKNOWN + case storage.VirtualMachineScanV2_OS_UNSUPPORTED: + return v2.VMScanNote_VM_SCAN_NOTE_OS_UNSUPPORTED + default: + return v2.VMScanNote_VM_SCAN_NOTE_UNSET + } +} diff --git a/central/convert/storagetov2/virtual_machine_v2_test.go b/central/convert/storagetov2/virtual_machine_v2_test.go new file mode 100644 index 0000000000000..8381abddf867a --- /dev/null +++ b/central/convert/storagetov2/virtual_machine_v2_test.go @@ -0,0 +1,136 @@ +package storagetov2 + +import ( + "testing" + "time" + + v2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + pkgVM "github.com/stackrox/rox/pkg/virtualmachine" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestVirtualMachineV2ToDetail_Notes(t *testing.T) { + tests := map[string]struct { + note storage.VirtualMachineV2_Note + expected v2.VMNote + }{ + "should preserve legacy missing scan data note": { + note: storage.VirtualMachineV2_MISSING_SCAN_DATA, + expected: v2.VMNote_VM_NOTE_MISSING_SCAN_DATA, + }, + "should map missing scanner note": { + note: storage.VirtualMachineV2_MISSING_SCANNER, + expected: v2.VMNote_VM_NOTE_MISSING_SCANNER, + }, + "should map scan failed note": { + note: storage.VirtualMachineV2_SCAN_FAILED, + expected: v2.VMNote_VM_NOTE_SCAN_FAILED, + }, + "should preserve missing signature note": { + note: storage.VirtualMachineV2_MISSING_SIGNATURE, + expected: v2.VMNote_VM_NOTE_MISSING_SIGNATURE, + }, + "should preserve missing signature verification data note": { + note: storage.VirtualMachineV2_MISSING_SIGNATURE_VERIFICATION_DATA, + expected: v2.VMNote_VM_NOTE_MISSING_SIGNATURE_VERIFICATION_DATA, + }, + "should map missing metadata note": { + note: storage.VirtualMachineV2_MISSING_METADATA, + expected: v2.VMNote_VM_NOTE_MISSING_METADATA, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + detail := VirtualMachineV2ToDetail(&storage.VirtualMachineV2{ + Id: "vm-1", + Name: "vm-1", + Notes: []storage.VirtualMachineV2_Note{tc.note}, + }) + require.Equal(t, []v2.VMNote{tc.expected}, detail.GetNotes()) + }) + } +} + +func TestAgentStatusFromLastContact(t *testing.T) { + now := time.Unix(1_700_000_000, 0).UTC() + staleAfter := 12 * time.Hour + + tests := map[string]struct { + ts *timestamppb.Timestamp + expected v2.AgentStatus + }{ + "should return unknown when never scraped": { + ts: nil, + expected: v2.AgentStatus_AGENT_STATUS_UNKNOWN, + }, + "should return unknown when timestamp is invalid": { + ts: ×tamppb.Timestamp{Nanos: 2_000_000_000}, + expected: v2.AgentStatus_AGENT_STATUS_UNKNOWN, + }, + "should return active when last scrape is still inside the window": { + ts: timestamppb.New(now.Add(-time.Hour)), + expected: v2.AgentStatus_AGENT_STATUS_ACTIVE, + }, + "should return inactive when last scrape is exactly the window": { + ts: timestamppb.New(now.Add(-staleAfter)), + expected: v2.AgentStatus_AGENT_STATUS_INACTIVE, + }, + "should return inactive when last scrape is older than the window": { + ts: timestamppb.New(now.Add(-13 * time.Hour)), + expected: v2.AgentStatus_AGENT_STATUS_INACTIVE, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + require.Equal(t, tc.expected, AgentStatusFromLastContact(tc.ts, now, staleAfter)) + }) + } +} + +func TestVirtualMachineV2GuestOsDisplay(t *testing.T) { + tests := map[string]struct { + facts map[string]string + storedOS string + want string + }{ + "returns stored guest OS": { + facts: map[string]string{ + pkgVM.DetectedGuestOSKey: "Red Hat Enterprise Linux 9.2", + pkgVM.GuestOSKey: "Red Hat Enterprise Linux", + }, + storedOS: "Red Hat Enterprise Linux 9.2", + want: "Red Hat Enterprise Linux 9.2", + }, + "returns stored guest OS when detected is absent": { + facts: map[string]string{pkgVM.GuestOSKey: "Red Hat Enterprise Linux"}, + storedOS: "Red Hat Enterprise Linux", + want: "Red Hat Enterprise Linux", + }, + "does not overlay detected onto the stored column": { + facts: map[string]string{ + pkgVM.DetectedGuestOSKey: "Red Hat Enterprise Linux 9.2", + pkgVM.GuestOSKey: "Red Hat Enterprise Linux", + }, + storedOS: "Red Hat Enterprise Linux", + want: "Red Hat Enterprise Linux", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + vm := &storage.VirtualMachineV2{ + Id: "vm-1", + Facts: tt.facts, + GuestOs: tt.storedOS, + } + assert.Equal(t, tt.want, VirtualMachineV2ToDetail(vm).GetGuestOs()) + assert.Equal(t, tt.want, VirtualMachineV2ToListItem(vm).GetGuestOs()) + assert.Equal(t, tt.storedOS, vm.GetGuestOs()) + }) + } +} diff --git a/central/convert/testutils/compliance_profiles.go b/central/convert/testutils/compliance_profiles.go index b0054034149f9..10d650484ad9d 100644 --- a/central/convert/testutils/compliance_profiles.go +++ b/central/convert/testutils/compliance_profiles.go @@ -117,6 +117,7 @@ func GetProfileV2Storage(_ *testing.T) *storage.ComplianceOperatorProfileV2 { Values: values, ClusterId: fixtureconsts.Cluster1, ProfileRefId: internaltov2storage.BuildProfileRefID(fixtureconsts.Cluster1, profileID, ""), + OperatorKind: storage.ComplianceOperatorProfileV2_PROFILE, } } @@ -139,6 +140,7 @@ func GetProfilesV2Storage(_ *testing.T) []*storage.ComplianceOperatorProfileV2 { Values: values, ClusterId: fixtureconsts.Cluster1, ProfileRefId: internaltov2storage.BuildProfileRefID(fixtureconsts.Cluster1, profileID, ""), + OperatorKind: storage.ComplianceOperatorProfileV2_PROFILE, }, { Id: profileUID2, @@ -156,6 +158,7 @@ func GetProfilesV2Storage(_ *testing.T) []*storage.ComplianceOperatorProfileV2 { Values: values, ClusterId: fixtureconsts.Cluster1, ProfileRefId: internaltov2storage.BuildProfileRefID(fixtureconsts.Cluster1, profileID, ""), + OperatorKind: storage.ComplianceOperatorProfileV2_PROFILE, }, } } @@ -176,8 +179,9 @@ func GetProfileV2Api(_ *testing.T) *v2.ComplianceProfile { ShortName: "CIS-OCP", Version: "4.2", }}, - Product: "", - Values: values, + Product: "", + Values: values, + OperatorKind: v2.ComplianceProfile_PROFILE, } } @@ -198,8 +202,9 @@ func GetProfilesV2Api(_ *testing.T) []*v2.ComplianceProfile { ShortName: "CIS-OCP", Version: "4.2", }}, - Product: "", - Values: values, + Product: "", + Values: values, + OperatorKind: v2.ComplianceProfile_PROFILE, }, { Id: profileUID2, @@ -215,8 +220,9 @@ func GetProfilesV2Api(_ *testing.T) []*v2.ComplianceProfile { ShortName: "NIST-800-53", Version: "4.1.2", }}, - Product: "", - Values: values, + Product: "", + Values: values, + OperatorKind: v2.ComplianceProfile_PROFILE, }, } } diff --git a/central/convert/testutils/compliance_rules.go b/central/convert/testutils/compliance_rules.go index 27344c68840ac..660b702a6f0f3 100644 --- a/central/convert/testutils/compliance_rules.go +++ b/central/convert/testutils/compliance_rules.go @@ -29,18 +29,19 @@ func GetRuleV2SensorMsg(_ *testing.T) *central.ComplianceOperatorRuleV2 { } return ¢ral.ComplianceOperatorRuleV2{ - RuleId: ruleID, - Id: RuleUID, - Name: "ocp-cis", - RuleType: "node", - Severity: 0, - Labels: map[string]string{v1alpha1.SuiteLabel: "ocp-cis"}, - Annotations: getAnnotations(), - Title: "test rule", - Description: "test description", - Rationale: "test rationale", - Fixes: fixes, - Warning: "test warning", + RuleId: ruleID, + Id: RuleUID, + Name: "ocp-cis", + RuleType: "node", + Severity: 0, + Labels: map[string]string{v1alpha1.SuiteLabel: "ocp-cis"}, + Annotations: getAnnotations(), + Title: "test rule", + Description: "test description", + Rationale: "test rationale", + Fixes: fixes, + Warning: "test warning", + OperatorKind: central.ComplianceOperatorRuleV2_RULE, } } @@ -87,21 +88,22 @@ func GetRuleV2Storage(_ *testing.T) *storage.ComplianceOperatorRuleV2 { }, } return &storage.ComplianceOperatorRuleV2{ - Id: RuleUID, - RuleId: ruleID, - Name: "ocp-cis", - RuleType: "node", - Labels: map[string]string{v1alpha1.SuiteLabel: "ocp-cis"}, - Annotations: getAnnotations(), - Title: "test rule", - Description: "test description", - Rationale: "test rationale", - Fixes: fixes, - Warning: "test warning", - Controls: controls, - ClusterId: fixtureconsts.Cluster1, - RuleRefId: internaltov2storage.BuildNameRefID(fixtureconsts.Cluster1, "random-rule-name"), - ParentRule: "random-rule-name", + Id: RuleUID, + RuleId: ruleID, + Name: "ocp-cis", + RuleType: "node", + Labels: map[string]string{v1alpha1.SuiteLabel: "ocp-cis"}, + Annotations: getAnnotations(), + Title: "test rule", + Description: "test description", + Rationale: "test rationale", + Fixes: fixes, + Warning: "test warning", + Controls: controls, + ClusterId: fixtureconsts.Cluster1, + RuleRefId: internaltov2storage.BuildNameRefID(fixtureconsts.Cluster1, "random-rule-name"), + ParentRule: "random-rule-name", + OperatorKind: storage.ComplianceOperatorRuleV2_RULE, } } @@ -115,17 +117,18 @@ func GetRuleV2(_ *testing.T) *apiV2.ComplianceRule { } return &apiV2.ComplianceRule{ - Id: RuleUID, - RuleId: ruleID, - Name: "ocp-cis", - RuleType: "node", - Title: "test rule", - Description: "test description", - Rationale: "test rationale", - Warning: "test warning", - Severity: "UNSET_RULE_SEVERITY", - Fixes: fixes, - ParentRule: "random-rule-name", + Id: RuleUID, + RuleId: ruleID, + Name: "ocp-cis", + RuleType: "node", + Title: "test rule", + Description: "test description", + Rationale: "test rationale", + Warning: "test warning", + Severity: "UNSET_RULE_SEVERITY", + Fixes: fixes, + ParentRule: "random-rule-name", + OperatorKind: apiV2.ComplianceRule_RULE, } } diff --git a/central/convert/v1tostorage/auth_m2m_config.go b/central/convert/v1tostorage/auth_m2m_config.go index bd51541332ae8..a961a3e9b9f08 100644 --- a/central/convert/v1tostorage/auth_m2m_config.go +++ b/central/convert/v1tostorage/auth_m2m_config.go @@ -13,6 +13,7 @@ func AuthM2MConfig(config *v1.AuthMachineToMachineConfig) *storage.AuthMachineTo TokenExpirationDuration: config.GetTokenExpirationDuration(), Mappings: convertMappings(config.GetMappings()), Issuer: config.GetIssuer(), + Audience: config.GetAudience(), Traits: Traits(config.GetTraits()), } diff --git a/central/convert/v1tov2storage/vm_scan_parts.go b/central/convert/v1tov2storage/vm_scan_parts.go new file mode 100644 index 0000000000000..9f83ccb53dcce --- /dev/null +++ b/central/convert/v1tov2storage/vm_scan_parts.go @@ -0,0 +1,292 @@ +package v1tov2storage + +import ( + "strconv" + "strings" + + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/uuid" +) + +// ScanPartsFromV1Scan converts a v1 VirtualMachineScan into normalized v2 +// VMScanParts (scan, components, CVEs) ready for upsert into v2 datastores. +func ScanPartsFromV1Scan(vmID string, scan *storage.VirtualMachineScan) *common.VMScanParts { + if scan == nil { + return nil + } + + scanID := uuid.NewV7().String() + + var ( + components []*storage.VirtualMachineComponentV2 + cves []*storage.VirtualMachineCVEV2 + topCvss float32 + sourceComponents = scan.GetComponents() + ) + + for _, comp := range sourceComponents { + componentID := uuid.NewV7().String() + var compTopCvss float32 + var compCveCount int32 + + for _, vuln := range comp.GetVulnerabilities() { + cve := convertVulnerability(vmID, componentID, vuln) + cves = append(cves, cve) + compCveCount++ + if cve.GetPreferredCvss() > topCvss { + topCvss = cve.GetPreferredCvss() + } + if cve.GetPreferredCvss() > compTopCvss { + compTopCvss = cve.GetPreferredCvss() + } + } + + components = append(components, &storage.VirtualMachineComponentV2{ + Id: componentID, + VmScanId: scanID, + Name: comp.GetName(), + Version: comp.GetVersion(), + Source: comp.GetSource(), + OperatingSystem: scan.GetOperatingSystem(), + SetTopCvss: &storage.VirtualMachineComponentV2_TopCvss{ + TopCvss: compTopCvss, + }, + FixedBy: highestFixedBy(comp.GetVulnerabilities()), + CveCount: compCveCount, + Notes: convertComponentNotes(comp.GetNotes()), + }) + } + + scanNotes := convertScanNotes(scan.GetNotes()) + + return &common.VMScanParts{ + Scan: &storage.VirtualMachineScanV2{ + Id: scanID, + VmV2Id: vmID, + ScanOs: scan.GetOperatingSystem(), + ScanTime: scan.GetScanTime(), + TopCvss: topCvss, + Notes: scanNotes, + }, + Components: components, + CVEs: cves, + SourceComponents: sourceComponents, + } +} + +func convertVulnerability(vmID, componentID string, vuln *storage.VirtualMachineVulnerability) *storage.VirtualMachineCVEV2 { + info := vuln.GetCveBaseInfo() + + preferredCvss, preferredVersion := preferredCvssVersion(info) + nvdCvss, nvdVersion := nvdCvssScore(info) + impactScore := extractImpactScore(info) + + cve := &storage.VirtualMachineCVEV2{ + Id: uuid.NewV7().String(), + VmV2Id: vmID, + VmComponentId: componentID, + CveBaseInfo: convertCVEBaseInfo(info), + PreferredCvss: preferredCvss, + PreferredCvssVersion: preferredVersion, + Severity: vuln.GetSeverity(), + ImpactScore: impactScore, + Nvdcvss: nvdCvss, + NvdScoreVersion: nvdVersion, + } + + if fixedBy := vuln.GetFixedBy(); fixedBy != "" { + cve.IsFixable = true + cve.HasFixedBy = &storage.VirtualMachineCVEV2_FixedBy{FixedBy: fixedBy} + } + + if epss := info.GetEpss(); epss != nil { + cve.EpssProbability = epss.GetEpssProbability() + } + + if adv := info.GetAdvisory(); adv != nil { + cve.Advisory = &storage.Advisory{ + Name: adv.GetName(), + Link: adv.GetLink(), + } + } + + return cve +} + +func convertCVEBaseInfo(info *storage.VirtualMachineCVEInfo) *storage.CVEInfo { + if info == nil { + return nil + } + + var refs []*storage.CVEInfo_Reference + for _, r := range info.GetReferences() { + refs = append(refs, &storage.CVEInfo_Reference{ + URI: r.GetURI(), + Tags: r.GetTags(), + }) + } + + var epss *storage.EPSS + if e := info.GetEpss(); e != nil { + epss = &storage.EPSS{ + EpssProbability: e.GetEpssProbability(), + EpssPercentile: e.GetEpssPercentile(), + } + } + + return &storage.CVEInfo{ + Cve: info.GetCve(), + Summary: info.GetSummary(), + Link: info.GetLink(), + PublishedOn: info.GetPublishedOn(), + CreatedAt: info.GetCreatedAt(), + LastModified: info.GetLastModified(), + CvssMetrics: info.GetCvssMetrics(), + References: refs, + Epss: epss, + } +} + +func convertComponentNotes(notes []storage.EmbeddedVirtualMachineScanComponent_Note) []storage.VirtualMachineComponentV2_Note { + if len(notes) == 0 { + return nil + } + out := make([]storage.VirtualMachineComponentV2_Note, 0, len(notes)) + for _, n := range notes { + switch n { + case storage.EmbeddedVirtualMachineScanComponent_UNSCANNED: + out = append(out, storage.VirtualMachineComponentV2_UNSCANNED) + default: + out = append(out, storage.VirtualMachineComponentV2_UNSPECIFIED) + } + } + return out +} + +func convertScanNotes(notes []storage.VirtualMachineScan_Note) []storage.VirtualMachineScanV2_Note { + if len(notes) == 0 { + return nil + } + out := make([]storage.VirtualMachineScanV2_Note, 0, len(notes)) + for _, n := range notes { + switch n { + case storage.VirtualMachineScan_OS_UNKNOWN: + out = append(out, storage.VirtualMachineScanV2_OS_UNKNOWN) + case storage.VirtualMachineScan_OS_UNSUPPORTED: + out = append(out, storage.VirtualMachineScanV2_OS_UNSUPPORTED) + default: + out = append(out, storage.VirtualMachineScanV2_UNSET) + } + } + return out +} + +// preferredCvssVersion picks the preferred CVSS score and version (V3 > V2). +func preferredCvssVersion(info *storage.VirtualMachineCVEInfo) (float32, storage.CvssScoreVersion) { + if info == nil { + return 0, storage.CvssScoreVersion_UNKNOWN_VERSION + } + + for _, m := range info.GetCvssMetrics() { + if m.GetSource() == storage.Source_SOURCE_RED_HAT { + if v3 := m.GetCvssv3(); v3 != nil { + return v3.GetScore(), storage.CvssScoreVersion_V3 + } + if v2 := m.GetCvssv2(); v2 != nil { + return v2.GetScore(), storage.CvssScoreVersion_V2 + } + } + } + + // Fall back to any metric with a score. + for _, m := range info.GetCvssMetrics() { + if v3 := m.GetCvssv3(); v3 != nil { + return v3.GetScore(), storage.CvssScoreVersion_V3 + } + if v2 := m.GetCvssv2(); v2 != nil { + return v2.GetScore(), storage.CvssScoreVersion_V2 + } + } + + return 0, storage.CvssScoreVersion_UNKNOWN_VERSION +} + +// nvdCvssScore extracts the NVD-specific CVSS score from CVSS metrics. +func nvdCvssScore(info *storage.VirtualMachineCVEInfo) (float32, storage.CvssScoreVersion) { + if info == nil { + return 0, storage.CvssScoreVersion_UNKNOWN_VERSION + } + + for _, m := range info.GetCvssMetrics() { + if m.GetSource() == storage.Source_SOURCE_NVD { + if v3 := m.GetCvssv3(); v3 != nil { + return v3.GetScore(), storage.CvssScoreVersion_V3 + } + if v2 := m.GetCvssv2(); v2 != nil { + return v2.GetScore(), storage.CvssScoreVersion_V2 + } + } + } + + return 0, storage.CvssScoreVersion_UNKNOWN_VERSION +} + +// extractImpactScore extracts the impact score from the preferred CVSS V3 metric. +func extractImpactScore(info *storage.VirtualMachineCVEInfo) float32 { + if info == nil { + return 0 + } + + for _, m := range info.GetCvssMetrics() { + if m.GetSource() == storage.Source_SOURCE_RED_HAT { + if v3 := m.GetCvssv3(); v3 != nil { + return v3.GetImpactScore() + } + } + } + + for _, m := range info.GetCvssMetrics() { + if v3 := m.GetCvssv3(); v3 != nil { + return v3.GetImpactScore() + } + } + + return 0 +} + +// highestFixedBy finds the highest fixed-by version across all vulnerabilities. +func highestFixedBy(vulns []*storage.VirtualMachineVulnerability) string { + var highest string + for _, v := range vulns { + if fb := v.GetFixedBy(); fb != "" { + if highest == "" || compareVersionSegments(fb, highest) > 0 { + highest = fb + } + } + } + return highest +} + +// compareVersionSegments compares two dot-separated version strings numerically +// segment by segment. Returns negative if a < b, 0 if equal, positive if a > b. +func compareVersionSegments(a, b string) int { + aParts := strings.Split(a, ".") + bParts := strings.Split(b, ".") + + maxLen := max(len(bParts), len(aParts)) + + for i := range maxLen { + var aVal, bVal int + if i < len(aParts) { + aVal, _ = strconv.Atoi(aParts[i]) + } + if i < len(bParts) { + bVal, _ = strconv.Atoi(bParts[i]) + } + if aVal != bVal { + return aVal - bVal + } + } + return 0 +} diff --git a/central/convert/v1tov2storage/vm_scan_parts_test.go b/central/convert/v1tov2storage/vm_scan_parts_test.go new file mode 100644 index 0000000000000..7810642a9e9ee --- /dev/null +++ b/central/convert/v1tov2storage/vm_scan_parts_test.go @@ -0,0 +1,235 @@ +package v1tov2storage + +import ( + "testing" + + "github.com/stackrox/rox/central/convert/storagetov2" + v2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScanPartsFromV1Scan_NilScan(t *testing.T) { + result := ScanPartsFromV1Scan("vm-1", nil) + assert.Nil(t, result) +} + +func TestScanPartsFromV1Scan_EmptyScan(t *testing.T) { + result := ScanPartsFromV1Scan("vm-1", &storage.VirtualMachineScan{}) + require.NotNil(t, result) + assert.NotEmpty(t, result.Scan.GetId()) + assert.Equal(t, "vm-1", result.Scan.GetVmV2Id()) + assert.Empty(t, result.Components) + assert.Empty(t, result.CVEs) +} + +func TestScanPartsFromV1Scan_WithComponentsAndVulns(t *testing.T) { + scan := &storage.VirtualMachineScan{ + OperatingSystem: "rhel:9", + Notes: []storage.VirtualMachineScan_Note{storage.VirtualMachineScan_OS_UNKNOWN}, + Components: []*storage.EmbeddedVirtualMachineScanComponent{ + { + Name: "openssl", + Version: "1.1.1", + Source: storage.SourceType_OS, + Vulnerabilities: []*storage.VirtualMachineVulnerability{ + { + CveBaseInfo: &storage.VirtualMachineCVEInfo{ + Cve: "CVE-2023-0001", + Summary: "test vuln", + CvssMetrics: []*storage.CVSSScore{ + { + Source: storage.Source_SOURCE_RED_HAT, + CvssScore: &storage.CVSSScore_Cvssv3{ + Cvssv3: &storage.CVSSV3{ + Score: 7.5, + ImpactScore: 3.6, + }, + }, + }, + { + Source: storage.Source_SOURCE_NVD, + CvssScore: &storage.CVSSScore_Cvssv3{ + Cvssv3: &storage.CVSSV3{ + Score: 8.0, + ImpactScore: 4.0, + }, + }, + }, + }, + Epss: &storage.VirtualMachineEPSS{ + EpssProbability: 0.5, + }, + Advisory: &storage.VirtualMachineAdvisory{ + Name: "RHSA-2023:1234", + Link: "https://example.com/advisory", + }, + }, + Severity: storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, + SetFixedBy: &storage.VirtualMachineVulnerability_FixedBy{ + FixedBy: "1.1.2", + }, + Cvss: 7.5, + }, + }, + }, + }, + } + + result := ScanPartsFromV1Scan("vm-1", scan) + require.NotNil(t, result) + + // Scan + assert.Equal(t, "vm-1", result.Scan.GetVmV2Id()) + assert.Equal(t, "rhel:9", result.Scan.GetScanOs()) + assert.InDelta(t, float32(7.5), result.Scan.GetTopCvss(), 0.01) + assert.Equal(t, []storage.VirtualMachineScanV2_Note{storage.VirtualMachineScanV2_OS_UNKNOWN}, result.Scan.GetNotes()) + + // Components + require.Len(t, result.Components, 1) + comp := result.Components[0] + assert.Equal(t, "openssl", comp.GetName()) + assert.Equal(t, "1.1.1", comp.GetVersion()) + assert.Equal(t, storage.SourceType_OS, comp.GetSource()) + assert.Equal(t, "rhel:9", comp.GetOperatingSystem()) + assert.Equal(t, "1.1.2", comp.GetFixedBy()) + assert.Equal(t, int32(1), comp.GetCveCount()) + + // CVEs + require.Len(t, result.CVEs, 1) + cve := result.CVEs[0] + assert.Equal(t, "vm-1", cve.GetVmV2Id()) + assert.Equal(t, comp.GetId(), cve.GetVmComponentId()) + assert.Equal(t, "CVE-2023-0001", cve.GetCveBaseInfo().GetCve()) + assert.InDelta(t, float32(7.5), cve.GetPreferredCvss(), 0.01) + assert.Equal(t, storage.CvssScoreVersion_V3, cve.GetPreferredCvssVersion()) + assert.Equal(t, storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, cve.GetSeverity()) + assert.InDelta(t, float32(3.6), cve.GetImpactScore(), 0.01) + assert.InDelta(t, float32(8.0), cve.GetNvdcvss(), 0.01) + assert.Equal(t, storage.CvssScoreVersion_V3, cve.GetNvdScoreVersion()) + assert.True(t, cve.GetIsFixable()) + assert.Equal(t, "1.1.2", cve.GetFixedBy()) + assert.InDelta(t, float32(0.5), cve.GetEpssProbability(), 0.01) + assert.Equal(t, "RHSA-2023:1234", cve.GetAdvisory().GetName()) + + // Source components preserved + require.Len(t, result.SourceComponents, 1) + assert.Equal(t, "openssl", result.SourceComponents[0].GetName()) +} + +func TestScanPartsFromV1Scan_PreservesComponentNotes(t *testing.T) { + tests := map[string]struct { + notes []storage.EmbeddedVirtualMachineScanComponent_Note + wantNotes []storage.VirtualMachineComponentV2_Note + wantScanStatus v2.ScanStatus + }{ + "should map UNSCANNED notes to NOT_SCANNED on the V2 row": { + notes: []storage.EmbeddedVirtualMachineScanComponent_Note{ + storage.EmbeddedVirtualMachineScanComponent_UNSCANNED, + }, + wantNotes: []storage.VirtualMachineComponentV2_Note{ + storage.VirtualMachineComponentV2_UNSCANNED, + }, + wantScanStatus: v2.ScanStatus_NOT_SCANNED, + }, + "should treat a component with no notes as scanned": { + wantScanStatus: v2.ScanStatus_SCANNED, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + result := ScanPartsFromV1Scan("vm-1", &storage.VirtualMachineScan{ + OperatingSystem: "rhel:9", + Components: []*storage.EmbeddedVirtualMachineScanComponent{ + { + Name: "curl", + Version: "7.0", + Source: storage.SourceType_OS, + Notes: tt.notes, + }, + }, + }) + require.NotNil(t, result) + require.Len(t, result.Components, 1) + assert.Equal(t, tt.wantNotes, result.Components[0].GetNotes()) + + row := storagetov2.VirtualMachineComponentV2ToRow(result.Components[0]) + require.NotNil(t, row) + assert.Equal(t, tt.wantScanStatus, row.GetScanStatus()) + }) + } +} + +func TestCompareVersionSegments(t *testing.T) { + tests := []struct { + name string + a, b string + want int + }{ + {name: "equal", a: "1.2.3", b: "1.2.3", want: 0}, + {name: "a greater", a: "1.2.4", b: "1.2.3", want: 1}, + {name: "b greater", a: "1.2.3", b: "1.2.4", want: -1}, + {name: "different lengths a longer", a: "1.2.3.1", b: "1.2.3", want: 1}, + {name: "different lengths b longer", a: "1.2", b: "1.2.1", want: -1}, + {name: "major version diff", a: "2.0.0", b: "1.9.9", want: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(it *testing.T) { + got := compareVersionSegments(tt.a, tt.b) + if tt.want > 0 { + assert.Greater(it, got, 0) + } else if tt.want < 0 { + assert.Less(it, got, 0) + } else { + assert.Equal(it, 0, got) + } + }) + } +} + +func TestHighestFixedBy(t *testing.T) { + tests := []struct { + name string + vulns []*storage.VirtualMachineVulnerability + want string + }{ + { + name: "no vulns", + vulns: nil, + want: "", + }, + { + name: "no fixed by", + vulns: []*storage.VirtualMachineVulnerability{ + {CveBaseInfo: &storage.VirtualMachineCVEInfo{Cve: "CVE-1"}}, + }, + want: "", + }, + { + name: "single fixed by", + vulns: []*storage.VirtualMachineVulnerability{ + {SetFixedBy: &storage.VirtualMachineVulnerability_FixedBy{FixedBy: "1.0.1"}}, + }, + want: "1.0.1", + }, + { + name: "picks highest", + vulns: []*storage.VirtualMachineVulnerability{ + {SetFixedBy: &storage.VirtualMachineVulnerability_FixedBy{FixedBy: "1.0.1"}}, + {SetFixedBy: &storage.VirtualMachineVulnerability_FixedBy{FixedBy: "2.0.0"}}, + {SetFixedBy: &storage.VirtualMachineVulnerability_FixedBy{FixedBy: "1.5.3"}}, + }, + want: "2.0.0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(it *testing.T) { + got := highestFixedBy(tt.vulns) + assert.Equal(it, tt.want, got) + }) + } +} diff --git a/central/credentialexpiry/service/service_impl.go b/central/credentialexpiry/service/service_impl.go index bd5381be6bdda..05f86d009b8ad 100644 --- a/central/credentialexpiry/service/service_impl.go +++ b/central/credentialexpiry/service/service_impl.go @@ -310,7 +310,7 @@ func (s *serviceImpl) getScannerV4CertExpiry(ctx context.Context) (*v1.GetCertEx errorList := errorhelpers.NewErrorList("failed to determine Scanner V4 cert expiry") expiries := make([]*time.Time, 0, numEndpoints) - for i := 0; i < numEndpoints; i++ { + for range numEndpoints { select { case <-ctx.Done(): return nil, ctx.Err() diff --git a/central/cve/cluster/datastore/datastore_impl.go b/central/cve/cluster/datastore/datastore_impl.go index 7392c890bfa56..ad8a4eeff722e 100644 --- a/central/cve/cluster/datastore/datastore_impl.go +++ b/central/cve/cluster/datastore/datastore_impl.go @@ -260,7 +260,3 @@ func (c *ClusterCVESearchResultConverter) BuildLocation(result *pkgSearch.Result func (c *ClusterCVESearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_CLUSTER_VULNERABILITIES } - -func (c *ClusterCVESearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/cve/cluster/datastore/store/postgres/full_store.go b/central/cve/cluster/datastore/store/postgres/full_store.go index 987ba71cb6ba8..d3ba4ab2eb3b2 100644 --- a/central/cve/cluster/datastore/store/postgres/full_store.go +++ b/central/cve/cluster/datastore/store/postgres/full_store.go @@ -62,10 +62,16 @@ func (s *fullStoreImpl) DeleteClusterCVEsForCluster(ctx context.Context, cluster _, err = tx.Exec(ctx, "DELETE FROM "+clusterCVEEdgeTable+" WHERE clusterid = $1", uuid.FromStringOrNil(clusterID)) if err != nil { + if err := tx.Rollback(ctx); err != nil { + return err + } return err } _, err = tx.Exec(ctx, "DELETE FROM "+clusterCVEsTable+" WHERE not exists (select "+clusterCVEEdgeTable+".cveid from "+clusterCVEEdgeTable+" where "+clusterCVEEdgeTable+".cveid = "+clusterCVEsTable+".id)") if err != nil { + if err := tx.Rollback(ctx); err != nil { + return err + } return err } return tx.Commit(ctx) @@ -116,10 +122,7 @@ func copyFromCVEs(ctx context.Context, tx *postgres.Tx, iTime time.Time, objs .. return nil } - batchSize := pgSearch.MaxBatchSize - if len(objs) < batchSize { - batchSize = len(objs) - } + batchSize := min(len(objs), pgSearch.MaxBatchSize) inputRows := make([][]interface{}, 0, batchSize) var err error @@ -207,10 +210,7 @@ func copyFromClusterCVEEdges(ctx context.Context, tx *postgres.Tx, cveType stora return nil } - batchSize := pgSearch.MaxBatchSize - if len(objs) < batchSize { - batchSize = len(objs) - } + batchSize := min(len(objs), pgSearch.MaxBatchSize) inputRows := make([][]interface{}, 0, batchSize) var err error diff --git a/central/cve/cluster/datastore/store/postgres/store.go b/central/cve/cluster/datastore/store/postgres/store.go index f33040eef85e7..ef90c0d4b9e6b 100644 --- a/central/cve/cluster/datastore/store/postgres/store.go +++ b/central/cve/cluster/datastore/store/postgres/store.go @@ -105,6 +105,8 @@ func insertIntoClusterCves(batch *pgx.Batch, obj *storage.ClusterCVE) error { protocompat.NilOrTime(obj.GetCveBaseInfo().GetPublishedOn()), protocompat.NilOrTime(obj.GetCveBaseInfo().GetCreatedAt()), obj.GetCveBaseInfo().GetEpss().GetEpssProbability(), + obj.GetCveBaseInfo().GetCisaKev(), + obj.GetCveBaseInfo().GetKnownRansomwareCampaign(), obj.GetCvss(), obj.GetSeverity(), obj.GetImpactScore(), @@ -114,7 +116,7 @@ func insertIntoClusterCves(batch *pgx.Batch, obj *storage.ClusterCVE) error { serialized, } - finalStr := "INSERT INTO cluster_cves (Id, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, Cvss, Severity, ImpactScore, Snoozed, SnoozeExpiry, Type, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, Cvss = EXCLUDED.Cvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Snoozed = EXCLUDED.Snoozed, SnoozeExpiry = EXCLUDED.SnoozeExpiry, Type = EXCLUDED.Type, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO cluster_cves (Id, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, CveBaseInfo_CisaKev, CveBaseInfo_KnownRansomwareCampaign, Cvss, Severity, ImpactScore, Snoozed, SnoozeExpiry, Type, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, CveBaseInfo_CisaKev = EXCLUDED.CveBaseInfo_CisaKev, CveBaseInfo_KnownRansomwareCampaign = EXCLUDED.CveBaseInfo_KnownRansomwareCampaign, Cvss = EXCLUDED.Cvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Snoozed = EXCLUDED.Snoozed, SnoozeExpiry = EXCLUDED.SnoozeExpiry, Type = EXCLUDED.Type, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) return nil @@ -126,6 +128,8 @@ var copyColsClusterCves = []string{ "cvebaseinfo_publishedon", "cvebaseinfo_createdat", "cvebaseinfo_epss_epssprobability", + "cvebaseinfo_cisakev", + "cvebaseinfo_knownransomwarecampaign", "cvss", "severity", "impactscore", @@ -171,6 +175,8 @@ func copyFromClusterCves(ctx context.Context, s pgSearch.Deleter, tx *postgres.T protocompat.NilOrTime(obj.GetCveBaseInfo().GetPublishedOn()), protocompat.NilOrTime(obj.GetCveBaseInfo().GetCreatedAt()), obj.GetCveBaseInfo().GetEpss().GetEpssProbability(), + obj.GetCveBaseInfo().GetCisaKev(), + obj.GetCveBaseInfo().GetKnownRansomwareCampaign(), obj.GetCvss(), obj.GetSeverity(), obj.GetImpactScore(), diff --git a/central/cve/converter/utils/convert_utils.go b/central/cve/converter/utils/convert_utils.go index e26746faf73a8..a910783666165 100644 --- a/central/cve/converter/utils/convert_utils.go +++ b/central/cve/converter/utils/convert_utils.go @@ -213,6 +213,8 @@ func ImageCVEToEmbeddedVulnerability(vuln *storage.ImageCVE) *storage.EmbeddedVu CvssMetrics: vuln.GetCvssMetrics(), NvdCvss: vuln.GetNvdcvss(), Epss: vuln.GetCveBaseInfo().GetEpss(), + Exploit: vuln.GetCveBaseInfo().GetExploit(), + CisaKev: vuln.GetCveBaseInfo().GetCisaKev(), } if vuln.GetCveBaseInfo().GetCvssV3() != nil { embeddedCVE.ScoreVersion = storage.EmbeddedVulnerability_V3 @@ -290,15 +292,18 @@ func EmbeddedVulnerabilityToImageCVE(os string, from *storage.EmbeddedVulnerabil Id: cve.ID(from.GetCve(), os), OperatingSystem: os, CveBaseInfo: &storage.CVEInfo{ - Cve: from.GetCve(), - Summary: from.GetSummary(), - Link: from.GetLink(), - PublishedOn: from.GetPublishedOn(), - CreatedAt: from.GetFirstSystemOccurrence(), - LastModified: from.GetLastModified(), - CvssV2: from.GetCvssV2(), - CvssV3: from.GetCvssV3(), - Epss: from.GetEpss(), + Cve: from.GetCve(), + Summary: from.GetSummary(), + Link: from.GetLink(), + PublishedOn: from.GetPublishedOn(), + CreatedAt: from.GetFirstSystemOccurrence(), + LastModified: from.GetLastModified(), + CvssV2: from.GetCvssV2(), + CvssV3: from.GetCvssV3(), + Epss: from.GetEpss(), + Exploit: from.GetExploit(), + CisaKev: from.GetCisaKev(), + KnownRansomwareCampaign: from.GetExploit() != nil && from.GetExploit().GetKnownRansomwareCampaignUse() == "Known", }, Cvss: from.GetCvss(), Nvdcvss: nvdCvss, diff --git a/central/cve/converter/utils/convert_utils_v2.go b/central/cve/converter/utils/convert_utils_v2.go index 9379ec6e443e5..c02e2f9b3b4d2 100644 --- a/central/cve/converter/utils/convert_utils_v2.go +++ b/central/cve/converter/utils/convert_utils_v2.go @@ -24,14 +24,19 @@ func ImageCVEV2ToEmbeddedVulnerability(vuln *storage.ImageCVEV2) *storage.Embedd PublishedOn: vuln.GetCveBaseInfo().GetPublishedOn(), LastModified: vuln.GetCveBaseInfo().GetLastModified(), FirstSystemOccurrence: vuln.GetCveBaseInfo().GetCreatedAt(), + FixAvailableTimestamp: vuln.GetFixAvailableTimestamp(), Severity: vuln.GetSeverity(), CvssMetrics: vuln.GetCveBaseInfo().GetCvssMetrics(), NvdCvss: vuln.GetNvdcvss(), Epss: vuln.GetCveBaseInfo().GetEpss(), + Exploit: vuln.GetCveBaseInfo().GetExploit(), + CisaKev: vuln.GetCveBaseInfo().GetExploit() != nil, FirstImageOccurrence: vuln.GetFirstImageOccurrence(), VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, VulnerabilityTypes: []storage.EmbeddedVulnerability_VulnerabilityType{storage.EmbeddedVulnerability_IMAGE_VULNERABILITY}, State: vuln.GetState(), + Datasource: vuln.GetDatasource(), + Origin: vuln.GetOrigin(), } if vuln.GetIsFixable() { @@ -82,27 +87,33 @@ func EmbeddedVulnerabilityToImageCVEV2(imageID string, componentID string, index Id: cveID, ComponentId: componentID, CveBaseInfo: &storage.CVEInfo{ - Cve: from.GetCve(), - Summary: from.GetSummary(), - Link: from.GetLink(), - PublishedOn: from.GetPublishedOn(), - CreatedAt: from.GetFirstSystemOccurrence(), - LastModified: from.GetLastModified(), - CvssV2: from.GetCvssV2(), - CvssV3: from.GetCvssV3(), - CvssMetrics: from.GetCvssMetrics(), - Epss: from.GetEpss(), - ScoreVersion: scoreVersion, + Cve: from.GetCve(), + Summary: from.GetSummary(), + Link: from.GetLink(), + PublishedOn: from.GetPublishedOn(), + CreatedAt: from.GetFirstSystemOccurrence(), + LastModified: from.GetLastModified(), + CvssV2: from.GetCvssV2(), + CvssV3: from.GetCvssV3(), + CvssMetrics: from.GetCvssMetrics(), + Epss: from.GetEpss(), + Exploit: from.GetExploit(), + CisaKev: from.GetExploit() != nil, + KnownRansomwareCampaign: from.GetExploit() != nil && from.GetExploit().GetKnownRansomwareCampaignUse() == "Known", + ScoreVersion: scoreVersion, }, - Cvss: from.GetCvss(), - Nvdcvss: nvdCvss, - NvdScoreVersion: nvdVersion, - Severity: from.GetSeverity(), - FirstImageOccurrence: from.GetFirstImageOccurrence(), - State: from.GetState(), - IsFixable: from.GetFixedBy() != "", - ImpactScore: impactScore, - Advisory: from.GetAdvisory(), + Cvss: from.GetCvss(), + Nvdcvss: nvdCvss, + NvdScoreVersion: nvdVersion, + Severity: from.GetSeverity(), + FirstImageOccurrence: from.GetFirstImageOccurrence(), + FixAvailableTimestamp: from.GetFixAvailableTimestamp(), + State: from.GetState(), + IsFixable: from.GetFixedBy() != "", + ImpactScore: impactScore, + Advisory: from.GetAdvisory(), + Datasource: from.GetDatasource(), + Origin: from.GetOrigin(), } if !features.FlattenImageData.Enabled() { ret.ImageId = imageID diff --git a/central/cve/converter/utils/convert_utils_v2_test.go b/central/cve/converter/utils/convert_utils_v2_test.go index 0bf10ac6b1f02..d4cdc4c5b4348 100644 --- a/central/cve/converter/utils/convert_utils_v2_test.go +++ b/central/cve/converter/utils/convert_utils_v2_test.go @@ -5,6 +5,7 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/cve" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/scancomponent" @@ -91,6 +92,16 @@ var ( EpssProbability: 22, EpssPercentile: 98, }, + Exploit: &storage.Exploit{ + DateAdded: "2023-11-21", + ShortDescription: "Buffer overflow in glibc ld.so", + RequiredAction: "Apply mitigations per vendor instructions.", + DueDate: "2023-12-12", + KnownRansomwareCampaignUse: "Unknown", + }, + CisaKev: true, + Datasource: "test-ds", + Origin: storage.VulnOrigin_VULN_ORIGIN_RED_HAT, }, { Cve: "cve2", @@ -129,6 +140,8 @@ var ( CvssMetrics: nil, NvdCvss: 0, Epss: nil, + Exploit: nil, + CisaKev: false, }, } ) @@ -158,117 +171,136 @@ func getTestCVEID(testCVE *storage.EmbeddedVulnerability, componentID string, in } func getTestCVEs(t *testing.T) []*storage.ImageCVEV2 { - return []*storage.ImageCVEV2{ - { - Id: getTestCVEID(testVulns[0], getTestComponentID(0), 0), - ImageId: "sha", - CveBaseInfo: &storage.CVEInfo{ - Cve: "cve1", - Summary: "", - Link: "", - PublishedOn: ts, - CreatedAt: ts, - LastModified: ts, - ScoreVersion: 1, - CvssV2: nil, - CvssV3: &storage.CVSSV3{ - Vector: "testVector", - ExploitabilityScore: 1.0, - ImpactScore: 2.0, - AttackVector: storage.CVSSV3_ATTACK_NETWORK, - AttackComplexity: storage.CVSSV3_COMPLEXITY_HIGH, - PrivilegesRequired: storage.CVSSV3_PRIVILEGE_HIGH, - UserInteraction: storage.CVSSV3_UI_REQUIRED, - Scope: storage.CVSSV3_CHANGED, - Confidentiality: storage.CVSSV3_IMPACT_HIGH, - Integrity: storage.CVSSV3_IMPACT_HIGH, - Availability: storage.CVSSV3_IMPACT_HIGH, - Score: 11.0, - Severity: storage.CVSSV3_CRITICAL, - }, - References: nil, - CvssMetrics: []*storage.CVSSScore{ - { - Source: storage.Source_SOURCE_NVD, - Url: "blah.com", - CvssScore: &storage.CVSSScore_Cvssv3{ - Cvssv3: &storage.CVSSV3{ - Vector: "testVector", - ExploitabilityScore: 1.0, - ImpactScore: 2.0, - AttackVector: storage.CVSSV3_ATTACK_NETWORK, - AttackComplexity: storage.CVSSV3_COMPLEXITY_HIGH, - PrivilegesRequired: storage.CVSSV3_PRIVILEGE_HIGH, - UserInteraction: storage.CVSSV3_UI_REQUIRED, - Scope: storage.CVSSV3_CHANGED, - Confidentiality: storage.CVSSV3_IMPACT_HIGH, - Integrity: storage.CVSSV3_IMPACT_HIGH, - Availability: storage.CVSSV3_IMPACT_HIGH, - Score: 11.0, - Severity: storage.CVSSV3_CRITICAL, - }, + cve1 := &storage.ImageCVEV2{ + Id: getTestCVEID(testVulns[0], getTestComponentID(0), 0), + CveBaseInfo: &storage.CVEInfo{ + Cve: "cve1", + Summary: "", + Link: "", + PublishedOn: ts, + CreatedAt: ts, + LastModified: ts, + ScoreVersion: 1, + CvssV2: nil, + CvssV3: &storage.CVSSV3{ + Vector: "testVector", + ExploitabilityScore: 1.0, + ImpactScore: 2.0, + AttackVector: storage.CVSSV3_ATTACK_NETWORK, + AttackComplexity: storage.CVSSV3_COMPLEXITY_HIGH, + PrivilegesRequired: storage.CVSSV3_PRIVILEGE_HIGH, + UserInteraction: storage.CVSSV3_UI_REQUIRED, + Scope: storage.CVSSV3_CHANGED, + Confidentiality: storage.CVSSV3_IMPACT_HIGH, + Integrity: storage.CVSSV3_IMPACT_HIGH, + Availability: storage.CVSSV3_IMPACT_HIGH, + Score: 11.0, + Severity: storage.CVSSV3_CRITICAL, + }, + References: nil, + CvssMetrics: []*storage.CVSSScore{ + { + Source: storage.Source_SOURCE_NVD, + Url: "blah.com", + CvssScore: &storage.CVSSScore_Cvssv3{ + Cvssv3: &storage.CVSSV3{ + Vector: "testVector", + ExploitabilityScore: 1.0, + ImpactScore: 2.0, + AttackVector: storage.CVSSV3_ATTACK_NETWORK, + AttackComplexity: storage.CVSSV3_COMPLEXITY_HIGH, + PrivilegesRequired: storage.CVSSV3_PRIVILEGE_HIGH, + UserInteraction: storage.CVSSV3_UI_REQUIRED, + Scope: storage.CVSSV3_CHANGED, + Confidentiality: storage.CVSSV3_IMPACT_HIGH, + Integrity: storage.CVSSV3_IMPACT_HIGH, + Availability: storage.CVSSV3_IMPACT_HIGH, + Score: 11.0, + Severity: storage.CVSSV3_CRITICAL, }, }, }, - Epss: &storage.EPSS{ - EpssProbability: 22, - EpssPercentile: 98, - }, }, - Cvss: 0, - Severity: 0, - ImpactScore: 2.0, - Nvdcvss: 11, - NvdScoreVersion: storage.CvssScoreVersion_V3, - FirstImageOccurrence: ts, - State: 0, - IsFixable: false, - HasFixedBy: nil, - ComponentId: getTestComponentID(0), - }, - { - Id: getTestCVEID(testVulns[1], getTestComponentID(1), 1), - ImageId: "sha", - CveBaseInfo: &storage.CVEInfo{ - Cve: "cve2", - Summary: "", - Link: "", - PublishedOn: ts, - CreatedAt: ts, - LastModified: ts, - ScoreVersion: 0, - CvssV2: &storage.CVSSV2{ - Vector: "testVector2", - AttackVector: storage.CVSSV2_ATTACK_ADJACENT, - AccessComplexity: storage.CVSSV2_ACCESS_LOW, - Authentication: storage.CVSSV2_AUTH_NONE, - Confidentiality: storage.CVSSV2_IMPACT_COMPLETE, - Integrity: storage.CVSSV2_IMPACT_COMPLETE, - Availability: storage.CVSSV2_IMPACT_COMPLETE, - ExploitabilityScore: 22, - ImpactScore: 32, - Score: 43, - Severity: storage.CVSSV2_HIGH, - }, - CvssV3: nil, - References: nil, - CvssMetrics: nil, - Epss: nil, + Epss: &storage.EPSS{ + EpssProbability: 22, + EpssPercentile: 98, }, - Cvss: 0, - Severity: 0, - ImpactScore: 32, - Nvdcvss: 0, - NvdScoreVersion: storage.CvssScoreVersion_UNKNOWN_VERSION, - FirstImageOccurrence: ts, - State: 0, - IsFixable: true, - HasFixedBy: &storage.ImageCVEV2_FixedBy{ - FixedBy: "ver3", + Exploit: &storage.Exploit{ + DateAdded: "2023-11-21", + ShortDescription: "Buffer overflow in glibc ld.so", + RequiredAction: "Apply mitigations per vendor instructions.", + DueDate: "2023-12-12", + KnownRansomwareCampaignUse: "Unknown", + }, + CisaKev: true, + }, + Cvss: 0, + Severity: 0, + ImpactScore: 2.0, + Nvdcvss: 11, + NvdScoreVersion: storage.CvssScoreVersion_V3, + FirstImageOccurrence: ts, + State: 0, + IsFixable: false, + HasFixedBy: nil, + ComponentId: getTestComponentID(0), + Datasource: "test-ds", + Origin: storage.VulnOrigin_VULN_ORIGIN_RED_HAT, + } + + cve2 := &storage.ImageCVEV2{ + Id: getTestCVEID(testVulns[1], getTestComponentID(1), 1), + CveBaseInfo: &storage.CVEInfo{ + Cve: "cve2", + Summary: "", + Link: "", + PublishedOn: ts, + CreatedAt: ts, + LastModified: ts, + ScoreVersion: 0, + CvssV2: &storage.CVSSV2{ + Vector: "testVector2", + AttackVector: storage.CVSSV2_ATTACK_ADJACENT, + AccessComplexity: storage.CVSSV2_ACCESS_LOW, + Authentication: storage.CVSSV2_AUTH_NONE, + Confidentiality: storage.CVSSV2_IMPACT_COMPLETE, + Integrity: storage.CVSSV2_IMPACT_COMPLETE, + Availability: storage.CVSSV2_IMPACT_COMPLETE, + ExploitabilityScore: 22, + ImpactScore: 32, + Score: 43, + Severity: storage.CVSSV2_HIGH, }, - ComponentId: getTestComponentID(1), + CvssV3: nil, + References: nil, + CvssMetrics: nil, + Epss: nil, + Exploit: nil, }, + Cvss: 0, + Severity: 0, + ImpactScore: 32, + Nvdcvss: 0, + NvdScoreVersion: storage.CvssScoreVersion_UNKNOWN_VERSION, + FirstImageOccurrence: ts, + State: 0, + IsFixable: true, + HasFixedBy: &storage.ImageCVEV2_FixedBy{ + FixedBy: "ver3", + }, + ComponentId: getTestComponentID(1), + } + + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + cve1.ImageIdV2 = "sha" + cve2.ImageIdV2 = "sha" + } else { + cve1.ImageId = "sha" + cve2.ImageId = "sha" } + + return []*storage.ImageCVEV2{cve1, cve2} } func getComponentInfo(t *testing.T) []*componentPieces { diff --git a/central/cve/fetcher/manager.go b/central/cve/fetcher/manager.go index 863e83f316b4c..50d1daa580773 100644 --- a/central/cve/fetcher/manager.go +++ b/central/cve/fetcher/manager.go @@ -11,6 +11,7 @@ import ( cveMatcher "github.com/stackrox/rox/central/cve/matcher" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" pkgScanners "github.com/stackrox/rox/pkg/scanners" "github.com/stackrox/rox/pkg/scanners/clairify" "github.com/stackrox/rox/pkg/scanners/types" @@ -51,6 +52,11 @@ func NewOrchestratorIstioCVEManagerImpl( }, updateSignal: concurrency.NewSignal(), } + if !features.LegacyScanner.Enabled() { + log.Info("Orchestrator scanning is disabled: no orchestrator scanners are integrated") + return m, nil + } + clairifyName, clairifyCreator := clairify.OrchestratorScannerCreator() m.orchestratorCVEMgr.creators[clairifyName] = clairifyCreator diff --git a/central/cve/fetcher/manager_impl.go b/central/cve/fetcher/manager_impl.go index 27a5e07f64942..a0c242d4dd7c5 100644 --- a/central/cve/fetcher/manager_impl.go +++ b/central/cve/fetcher/manager_impl.go @@ -8,6 +8,7 @@ import ( cveMatcher "github.com/stackrox/rox/central/cve/matcher" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/throttle" @@ -31,6 +32,9 @@ func (m *orchestratorIstioCVEManagerImpl) initialize() { // Start begins the process to periodically scan orchestrator-level components, asynchronously. func (m *orchestratorIstioCVEManagerImpl) Start() { + if !features.LegacyScanner.Enabled() { + return + } go func() { ticker := time.NewTicker(env.OrchestratorVulnScanInterval.DurationSetting()) defer ticker.Stop() @@ -48,6 +52,9 @@ func (m *orchestratorIstioCVEManagerImpl) Start() { } func (m *orchestratorIstioCVEManagerImpl) HandleClusterConnection() { + if !features.LegacyScanner.Enabled() { + return + } connectionDropThrottle.Run(func() { m.updateSignal.Signal() }) diff --git a/central/cve/fetcher/manager_impl_postgres_test.go b/central/cve/fetcher/manager_impl_postgres_test.go index 5451634d74364..a9047ca007a64 100644 --- a/central/cve/fetcher/manager_impl_postgres_test.go +++ b/central/cve/fetcher/manager_impl_postgres_test.go @@ -17,6 +17,7 @@ import ( "github.com/stackrox/rox/central/cve/converter/v2" "github.com/stackrox/rox/central/cve/matcher" mockImageDataStore "github.com/stackrox/rox/central/image/datastore/mocks" + mockImageV2DataStore "github.com/stackrox/rox/central/imagev2/datastore/mocks" mockNSDataStore "github.com/stackrox/rox/central/namespace/datastore/mocks" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/cve" @@ -133,9 +134,10 @@ func TestReconcileIstioCVEsInPostgres(t *testing.T) { mockClusters := mockClusterDataStore.NewMockDataStore(ctrl) mockNamespaces := mockNSDataStore.NewMockDataStore(ctrl) mockImages := mockImageDataStore.NewMockDataStore(ctrl) + mockImagesV2 := mockImageV2DataStore.NewMockDataStore(ctrl) mockCVEs := mockCVEDataStore.NewMockDataStore(ctrl) - cveMatcher, err := matcher.NewCVEMatcher(mockClusters, mockNamespaces, mockImages) + cveMatcher, err := matcher.NewCVEMatcher(mockClusters, mockNamespaces, mockImages, mockImagesV2) require.NoError(t, err) cveManager := &orchestratorIstioCVEManagerImpl{ @@ -404,9 +406,10 @@ func TestReconcileCVEsInPostgres(t *testing.T) { mockClusters := mockClusterDataStore.NewMockDataStore(ctrl) mockNamespaces := mockNSDataStore.NewMockDataStore(ctrl) mockImages := mockImageDataStore.NewMockDataStore(ctrl) + mockImagesV2 := mockImageV2DataStore.NewMockDataStore(ctrl) mockCVEs := mockCVEDataStore.NewMockDataStore(ctrl) - cveMatcher, err := matcher.NewCVEMatcher(mockClusters, mockNamespaces, mockImages) + cveMatcher, err := matcher.NewCVEMatcher(mockClusters, mockNamespaces, mockImages, mockImagesV2) require.NoError(t, err) cveManager := &orchestratorIstioCVEManagerImpl{ @@ -436,6 +439,7 @@ type TestClusterCVEOpsInPostgresTestSuite struct { clusterCVEDatastore clusterCVEDataStore.DataStore mockNamespaces *mockNSDataStore.MockDataStore mockImages *mockImageDataStore.MockDataStore + mockImagesV2 *mockImageV2DataStore.MockDataStore cveManager *orchestratorCVEManager } @@ -447,6 +451,7 @@ func (s *TestClusterCVEOpsInPostgresTestSuite) SetupSuite() { // Create cluster datastore s.mockNamespaces = mockNSDataStore.NewMockDataStore(s.mockCtrl) s.mockImages = mockImageDataStore.NewMockDataStore(s.mockCtrl) + s.mockImagesV2 = mockImageV2DataStore.NewMockDataStore(s.mockCtrl) // Create cluster cve datastore clusterCVEDatastore, err := clusterCVEDataStore.GetTestPostgresDataStore(s.T(), s.testPostgres.DB) @@ -458,7 +463,7 @@ func (s *TestClusterCVEOpsInPostgresTestSuite) SetupSuite() { s.clusterDataStore = clusterDataStore // Create cve manager - cveMatcher, err := matcher.NewCVEMatcher(clusterDataStore, s.mockNamespaces, s.mockImages) + cveMatcher, err := matcher.NewCVEMatcher(clusterDataStore, s.mockNamespaces, s.mockImages, s.mockImagesV2) s.NoError(err) s.cveManager = &orchestratorCVEManager{ @@ -645,7 +650,7 @@ func (s *TestClusterCVEOpsInPostgresTestSuite) TestBasicOps() { func getTestClusterCVEParts(numCVEs int, clusters ...string) ([]*storage.EmbeddedVulnerability, map[string][]*storage.Cluster) { vulns := make([]*storage.EmbeddedVulnerability, 0, numCVEs) vulnToClustersMap := make(map[string][]*storage.Cluster) - for i := 0; i < numCVEs; i++ { + for i := range numCVEs { cve := fmt.Sprintf("cve-%d", i) vulns = append(vulns, &storage.EmbeddedVulnerability{ Cve: cve, diff --git a/central/cve/fetcher/manager_test.go b/central/cve/fetcher/manager_test.go new file mode 100644 index 0000000000000..210e0d6b05919 --- /dev/null +++ b/central/cve/fetcher/manager_test.go @@ -0,0 +1,117 @@ +package fetcher + +import ( + "context" + "testing" + + mockClusterDataStore "github.com/stackrox/rox/central/cluster/datastore/mocks" + mockClusterCVEEdgeDataStore "github.com/stackrox/rox/central/clustercveedge/datastore/mocks" + mockCVEDataStore "github.com/stackrox/rox/central/cve/cluster/datastore/mocks" + "github.com/stackrox/rox/central/cve/converter/utils" + "github.com/stackrox/rox/central/cve/matcher" + mockImageDataStore "github.com/stackrox/rox/central/image/datastore/mocks" + mockImageV2DataStore "github.com/stackrox/rox/central/imagev2/datastore/mocks" + mockNSDataStore "github.com/stackrox/rox/central/namespace/datastore/mocks" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestNewOrchestratorIstioCVEManagerImpl(t *testing.T) { + tests := map[string]struct { + legacyScannerEnabled bool + setUp func(t *testing.T, mockClusters *mockClusterDataStore.MockDataStore, mockCVEs *mockCVEDataStore.MockDataStore) + }{ + "when LegacyScanner is disabled then no creators registered and initialize not called": { + legacyScannerEnabled: false, + setUp: func(t *testing.T, mockClusters *mockClusterDataStore.MockDataStore, mockCVEs *mockCVEDataStore.MockDataStore) { + }, + }, + "when LegacyScanner is enabled then Clairify creator registered and initialize called": { + legacyScannerEnabled: true, + setUp: func(t *testing.T, mockClusters *mockClusterDataStore.MockDataStore, mockCVEs *mockCVEDataStore.MockDataStore) { + mockClusters.EXPECT().GetClusters(gomock.Any()).Return([]*storage.Cluster{}, nil).Times(1) + mockCVEs.EXPECT().UpsertClusterCVEsInternal(gomock.Any(), storage.CVE_K8S_CVE).Return(nil).Times(1) + mockCVEs.EXPECT().UpsertClusterCVEsInternal(gomock.Any(), storage.CVE_OPENSHIFT_CVE).Return(nil).Times(1) + mockCVEs.EXPECT().UpsertClusterCVEsInternal(gomock.Any(), storage.CVE_ISTIO_CVE).Return(nil).Times(1) + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + testutils.MustUpdateFeature(t, features.LegacyScanner, tt.legacyScannerEnabled) + + ctrl := gomock.NewController(t) + mockClusters := mockClusterDataStore.NewMockDataStore(ctrl) + mockCVEs := mockCVEDataStore.NewMockDataStore(ctrl) + mockEdges := mockClusterCVEEdgeDataStore.NewMockDataStore(ctrl) + mockNamespaces := mockNSDataStore.NewMockDataStore(ctrl) + mockImages := mockImageDataStore.NewMockDataStore(ctrl) + mockImagesV2 := mockImageV2DataStore.NewMockDataStore(ctrl) + + tt.setUp(t, mockClusters, mockCVEs) + + cveMatcher, err := matcher.NewCVEMatcher(mockClusters, mockNamespaces, mockImages, mockImagesV2) + require.NoError(t, err) + + mgr, err := NewOrchestratorIstioCVEManagerImpl(mockClusters, mockCVEs, mockEdges, cveMatcher) + assert.NoError(t, err) + assert.NotNil(t, mgr) + }) + } +} + +func TestInertManager_ConsumerPathsSafe(t *testing.T) { + testutils.MustUpdateFeature(t, features.LegacyScanner, false) + + ctrl := gomock.NewController(t) + mockClusters := mockClusterDataStore.NewMockDataStore(ctrl) + mockCVEs := mockCVEDataStore.NewMockDataStore(ctrl) + mockEdges := mockClusterCVEEdgeDataStore.NewMockDataStore(ctrl) + mockNamespaces := mockNSDataStore.NewMockDataStore(ctrl) + mockImages := mockImageDataStore.NewMockDataStore(ctrl) + mockImagesV2 := mockImageV2DataStore.NewMockDataStore(ctrl) + + cveMatcher, err := matcher.NewCVEMatcher(mockClusters, mockNamespaces, mockImages, mockImagesV2) + require.NoError(t, err) + + mgr, err := NewOrchestratorIstioCVEManagerImpl(mockClusters, mockCVEs, mockEdges, cveMatcher) + require.NoError(t, err) + + t.Run("GetAffectedClusters queries DB and returns results", func(t *testing.T) { + ctx := sac.WithAllAccess(context.Background()) + expected := []*storage.Cluster{{Id: "cluster-1", Name: "test-cluster"}} + mockClusters.EXPECT().SearchRawClusters(gomock.Any(), gomock.Any()).Return(expected, nil).Times(1) + + clusters, err := mgr.GetAffectedClusters(ctx, "CVE-2024-1234", utils.K8s, nil) + assert.NoError(t, err) + assert.Len(t, clusters, 1) + assert.Equal(t, "cluster-1", clusters[0].GetId()) + }) + + t.Run("HandleClusterConnection does not panic", func(t *testing.T) { + assert.NotPanics(t, func() { + mgr.HandleClusterConnection() + }) + }) + + t.Run("UpsertOrchestratorIntegration returns error for missing creator", func(t *testing.T) { + err := mgr.UpsertOrchestratorIntegration(&storage.OrchestratorIntegration{ + Id: "test-id", + Type: "clairify", + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "does not exist") + }) + + t.Run("RemoveIntegration does not panic", func(t *testing.T) { + assert.NotPanics(t, func() { + mgr.RemoveIntegration("nonexistent-id") + }) + }) +} diff --git a/central/cve/fetcher/orchestrator.go b/central/cve/fetcher/orchestrator.go index 2c62e1688c1e5..7b4e7f3d8e253 100644 --- a/central/cve/fetcher/orchestrator.go +++ b/central/cve/fetcher/orchestrator.go @@ -3,6 +3,7 @@ package fetcher import ( "context" "fmt" + "maps" "github.com/pkg/errors" clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" @@ -76,9 +77,7 @@ func (m *orchestratorCVEManager) Scan(version string, cveType utils.CVEType) ([] scanners := make(map[string]types.OrchestratorScanner) concurrency.WithLock(&m.mutex, func() { - for k, v := range m.scanners { - scanners[k] = v - } + maps.Copy(scanners, m.scanners) }) if len(scanners) == 0 { @@ -96,7 +95,7 @@ func (m *orchestratorCVEManager) Scan(version string, cveType utils.CVEType) ([] } func (m *orchestratorCVEManager) updateCVEs(embeddedCVEs []*storage.EmbeddedVulnerability, embeddedCVEToClusters map[string][]*storage.Cluster, cveType utils.CVEType) error { - var newCVEs []converterV2.ClusterCVEParts + newCVEs := make([]converterV2.ClusterCVEParts, 0, len(embeddedCVEs)) for _, embeddedCVE := range embeddedCVEs { cve := utils.EmbeddedVulnerabilityToClusterCVE(cveType.ToStorageCVEType(), embeddedCVE) newCVEs = append(newCVEs, converterV2.NewClusterCVEParts(cve, embeddedCVEToClusters[embeddedCVE.GetCve()], embeddedCVE.GetFixedBy())) diff --git a/central/cve/image/info/datastore/datastore.go b/central/cve/image/info/datastore/datastore.go index 669648c936f5a..77b2d838ca6df 100644 --- a/central/cve/image/info/datastore/datastore.go +++ b/central/cve/image/info/datastore/datastore.go @@ -6,7 +6,6 @@ import ( "github.com/stackrox/rox/central/cve/image/info/datastore/store" pgStore "github.com/stackrox/rox/central/cve/image/info/datastore/store/postgres" - v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/postgres" @@ -20,6 +19,7 @@ type DataStore interface { Get(ctx context.Context, id string) (*storage.ImageCVEInfo, bool, error) Count(ctx context.Context, q *v1.Query) (int, error) GetBatch(ctx context.Context, id []string) ([]*storage.ImageCVEInfo, error) + GetByCVENames(ctx context.Context, cveNames []string) ([]*storage.ImageCVEInfo, error) Upsert(ctx context.Context, cve *storage.ImageCVEInfo) error UpsertMany(ctx context.Context, cve []*storage.ImageCVEInfo) error } diff --git a/central/cve/image/info/datastore/datastore_impl.go b/central/cve/image/info/datastore/datastore_impl.go index 6fd45c3b082a3..6a1724905cae1 100644 --- a/central/cve/image/info/datastore/datastore_impl.go +++ b/central/cve/image/info/datastore/datastore_impl.go @@ -2,18 +2,21 @@ package datastore import ( "context" - "math" "github.com/stackrox/rox/central/cve/image/info/datastore/store" - "github.com/stackrox/rox/pkg/protocompat" - "github.com/stackrox/rox/pkg/sliceutils" - v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/sliceutils" + "github.com/stackrox/rox/pkg/sync" ) type datastoreImpl struct { storage store.Store + // upsertLock serializes Upsert and UpsertMany to make the + // read-modify-write of timestamp merging atomic. + upsertLock sync.Mutex } func (ds *datastoreImpl) SearchRawImageCVEInfos(ctx context.Context, q *v1.Query) ([]*storage.ImageCVEInfo, error) { @@ -42,7 +45,19 @@ func (ds *datastoreImpl) GetBatch(ctx context.Context, ids []string) ([]*storage return infos, err } +func (ds *datastoreImpl) GetByCVENames(ctx context.Context, cveNames []string) ([]*storage.ImageCVEInfo, error) { + if len(cveNames) == 0 { + return nil, nil + } + + q := search.NewQueryBuilder().AddExactMatches(search.CVE, cveNames...).ProtoQuery() + return ds.SearchRawImageCVEInfos(ctx, q) +} + func (ds *datastoreImpl) Upsert(ctx context.Context, info *storage.ImageCVEInfo) error { + ds.upsertLock.Lock() + defer ds.upsertLock.Unlock() + existing, found, err := ds.Get(ctx, info.GetId()) if err != nil { return err @@ -55,6 +70,9 @@ func (ds *datastoreImpl) Upsert(ctx context.Context, info *storage.ImageCVEInfo) } func (ds *datastoreImpl) UpsertMany(ctx context.Context, infos []*storage.ImageCVEInfo) error { + ds.upsertLock.Lock() + defer ds.upsertLock.Unlock() + // Create a list of ids to look up ids := sliceutils.Map[*storage.ImageCVEInfo, string](infos, func(info *storage.ImageCVEInfo) string { return info.GetId() @@ -65,17 +83,19 @@ func (ds *datastoreImpl) UpsertMany(ctx context.Context, infos []*storage.ImageC } newInfoMap := make(map[string]*storage.ImageCVEInfo) oldInfoMap := make(map[string]*storage.ImageCVEInfo) - // Populate both maps at the same time by looping through up to the length of the longer list to save time - for i := range int(math.Max(float64(len(infos)), float64(len(existing)))) { - // Check if this was the shorter list - if i < len(infos) { - newInfoMap[infos[i].GetId()] = infos[i] - } - // Same as above - if i < len(existing) { - oldInfoMap[infos[i].GetId()] = existing[i] + // Populate both maps separately - can't use index-based loop because + // existing may not be in the same order as infos + for _, info := range infos { + if prev, ok := newInfoMap[info.GetId()]; ok { + // if there are are multiple infos with same id, earlier timestamps take precedence + newInfoMap[info.GetId()] = updateTimestamps(prev, info) + } else { + newInfoMap[info.GetId()] = info } } + for _, info := range existing { + oldInfoMap[info.GetId()] = info + } // Create our list that we're going to actually upsert toUpsert := make([]*storage.ImageCVEInfo, 0) for k, v := range newInfoMap { diff --git a/central/cve/image/info/datastore/datastore_impl_test.go b/central/cve/image/info/datastore/datastore_impl_test.go index 26f163a0af891..2740c439c328e 100644 --- a/central/cve/image/info/datastore/datastore_impl_test.go +++ b/central/cve/image/info/datastore/datastore_impl_test.go @@ -44,6 +44,7 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpdateTimestamps_NilOld() { now := time.Now() newInfo := &storage.ImageCVEInfo{ Id: "test-id", + Cve: "test-cve", FirstSystemOccurrence: timestamppb.New(now), FixAvailableTimestamp: timestamppb.New(now), } @@ -62,10 +63,12 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpdateTimestamps_PreservesEarlierFirstS oldInfo := &storage.ImageCVEInfo{ Id: "test-id", + Cve: "test-cve", FirstSystemOccurrence: timestamppb.New(earlier), } newInfo := &storage.ImageCVEInfo{ Id: "test-id", + Cve: "test-cve", FirstSystemOccurrence: timestamppb.New(later), } @@ -82,10 +85,12 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpdateTimestamps_PreservesEarlierFixAva oldInfo := &storage.ImageCVEInfo{ Id: "test-id", + Cve: "test-cve", FixAvailableTimestamp: timestamppb.New(earlier), } newInfo := &storage.ImageCVEInfo{ Id: "test-id", + Cve: "test-cve", FixAvailableTimestamp: timestamppb.New(later), } @@ -101,10 +106,12 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpdateTimestamps_UsesNewWhenOldIsZero() oldInfo := &storage.ImageCVEInfo{ Id: "test-id", + Cve: "test-cve", FirstSystemOccurrence: nil, // Zero timestamp } newInfo := &storage.ImageCVEInfo{ Id: "test-id", + Cve: "test-cve", FirstSystemOccurrence: timestamppb.New(now), } @@ -120,10 +127,12 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpdateTimestamps_UsesOldWhenNewIsZero() oldInfo := &storage.ImageCVEInfo{ Id: "test-id", + Cve: "test-cve", FirstSystemOccurrence: timestamppb.New(earlier), } newInfo := &storage.ImageCVEInfo{ Id: "test-id", + Cve: "test-cve", FirstSystemOccurrence: nil, // Zero timestamp } @@ -141,6 +150,7 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpsert_PreservesTimestamps() { // First, insert an info with an earlier timestamp firstInfo := &storage.ImageCVEInfo{ Id: "test-cve#test-pkg#test-ds", + Cve: "test-cve", FirstSystemOccurrence: timestamppb.New(earlier), FixAvailableTimestamp: timestamppb.New(earlier), } @@ -150,6 +160,7 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpsert_PreservesTimestamps() { // Now upsert with a later timestamp secondInfo := &storage.ImageCVEInfo{ Id: "test-cve#test-pkg#test-ds", + Cve: "test-cve", FirstSystemOccurrence: timestamppb.New(later), FixAvailableTimestamp: timestamppb.New(later), } @@ -167,20 +178,30 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpsert_PreservesTimestamps() { // TestUpsertMany_PreservesTimestamps tests that UpsertMany preserves earlier timestamps. func (s *ImageCVEInfoDataStoreSuite) TestUpsertMany_PreservesTimestamps() { earlier := time.Now().Add(-24 * time.Hour) + earlier2 := time.Now().Add(-12 * time.Hour) later := time.Now() - // First, insert infos with earlier timestamps + // First, insert infos with earlier timestamps. + // If there are multiple infos with same id, earlier timestamps should be preserved. firstInfos := []*storage.ImageCVEInfo{ { Id: "test-cve-1#test-pkg#test-ds", + Cve: "test-cve-1", FirstSystemOccurrence: timestamppb.New(earlier), FixAvailableTimestamp: timestamppb.New(earlier), }, { Id: "test-cve-2#test-pkg#test-ds", + Cve: "test-cve-2", FirstSystemOccurrence: timestamppb.New(earlier), FixAvailableTimestamp: timestamppb.New(earlier), }, + { + Id: "test-cve-1#test-pkg#test-ds", + Cve: "test-cve-1", + FirstSystemOccurrence: nil, + FixAvailableTimestamp: timestamppb.New(earlier2), + }, } err := s.datastore.UpsertMany(s.ctx, firstInfos) s.NoError(err) @@ -189,11 +210,13 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpsertMany_PreservesTimestamps() { secondInfos := []*storage.ImageCVEInfo{ { Id: "test-cve-1#test-pkg#test-ds", + Cve: "test-cve-1", FirstSystemOccurrence: timestamppb.New(later), FixAvailableTimestamp: timestamppb.New(later), }, { Id: "test-cve-2#test-pkg#test-ds", + Cve: "test-cve-2", FirstSystemOccurrence: timestamppb.New(later), FixAvailableTimestamp: timestamppb.New(later), }, @@ -218,6 +241,7 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpsert_NewInfo() { info := &storage.ImageCVEInfo{ Id: "new-cve#new-pkg#new-ds", + Cve: "new-cve", FirstSystemOccurrence: timestamppb.New(now), FixAvailableTimestamp: timestamppb.New(now), } @@ -232,5 +256,66 @@ func (s *ImageCVEInfoDataStoreSuite) TestUpsert_NewInfo() { s.Equal(now.Unix(), result.GetFixAvailableTimestamp().AsTime().Unix()) } +// TestSearchRawImageCVEInfos tests that SearchRawImageCVEInfos can query by CVE name. +func (s *ImageCVEInfoDataStoreSuite) TestSearchRawImageCVEInfos() { + now := time.Now() + + // Insert test data with different CVE names + infos := []*storage.ImageCVEInfo{ + { + Id: "CVE-2024-1234#package-a#datasource-1", + Cve: "CVE-2024-1234", + FirstSystemOccurrence: timestamppb.New(now.Add(-24 * time.Hour)), + }, + { + Id: "CVE-2024-1234#package-b#datasource-2", + Cve: "CVE-2024-1234", + FirstSystemOccurrence: timestamppb.New(now.Add(-12 * time.Hour)), + }, + { + Id: "CVE-2024-5678#package-a#datasource-1", + Cve: "CVE-2024-5678", + FirstSystemOccurrence: timestamppb.New(now), + }, + { + Id: "CVE-2024-9999#package-c#datasource-3", + Cve: "CVE-2024-9999", + FirstSystemOccurrence: timestamppb.New(now.Add(-6 * time.Hour)), + }, + } + + err := s.datastore.UpsertMany(s.ctx, infos) + s.NoError(err) + + // Test 1: Query for single CVE name + results, err := s.datastore.GetByCVENames(s.ctx, []string{"CVE-2024-1234"}) + s.NoError(err) + s.Len(results, 2, "Should return 2 records for CVE-2024-1234") + for _, result := range results { + s.Equal("CVE-2024-1234", result.GetCve()) + } + + // Test 2: Query for multiple CVE names + results, err = s.datastore.GetByCVENames(s.ctx, []string{"CVE-2024-5678", "CVE-2024-9999"}) + s.NoError(err) + s.Len(results, 2, "Should return 2 records (one for each CVE)") + cveNames := map[string]bool{} + for _, result := range results { + cveNames[result.GetCve()] = true + } + s.True(cveNames["CVE-2024-5678"]) + s.True(cveNames["CVE-2024-9999"]) + + // Test 3: Query for non-existent CVE + results, err = s.datastore.GetByCVENames(s.ctx, []string{"CVE-2024-NONEXISTENT"}) + s.NoError(err) + s.Len(results, 0, "Should return empty for non-existent CVE") + + // Test 4: Query with empty list + results, err = s.datastore.GetByCVENames(s.ctx, []string{}) + s.NoError(err) + s.Len(results, 0, "Should return empty for empty query") +} + // Ensure protocompat is used (to satisfy import) var _ = protocompat.TimestampNow diff --git a/central/cve/image/info/datastore/store/postgres/gen.go b/central/cve/image/info/datastore/store/postgres/gen.go index 22f5743e25a80..d893bd2272428 100644 --- a/central/cve/image/info/datastore/store/postgres/gen.go +++ b/central/cve/image/info/datastore/store/postgres/gen.go @@ -1,3 +1,3 @@ package postgres -//go:generate pg-table-bindings-wrapper --type=storage.ImageCVEInfo --cached-store +//go:generate pg-table-bindings-wrapper --type=storage.ImageCVEInfo --cached-store --no-copy-from --search-category IMAGE_CVE_INFOS diff --git a/central/cve/image/info/datastore/store/postgres/store.go b/central/cve/image/info/datastore/store/postgres/store.go index 1eb0c84aa1029..c43dedce8823a 100644 --- a/central/cve/image/info/datastore/store/postgres/store.go +++ b/central/cve/image/info/datastore/store/postgres/store.go @@ -51,6 +51,9 @@ type Store interface { Search(ctx context.Context, q *v1.Query) ([]search.Result, error) Get(ctx context.Context, id string) (*storeType, bool, error) + // Deprecated: use GetByQueryFn instead + GetByQuery(ctx context.Context, query *v1.Query) ([]*storeType, error) + GetByQueryFn(ctx context.Context, query *v1.Query, fn callback) error GetMany(ctx context.Context, identifiers []string) ([]*storeType, []int, error) GetIDs(ctx context.Context) ([]string, error) @@ -68,7 +71,7 @@ func New(db postgres.DB) Store { schema, pkGetter, insertIntoImageCveInfos, - copyFromImageCveInfos, + nil, metricsSetAcquireDBConnDuration, metricsSetPostgresOperationDurationTime, metricsSetCacheOperationDurationTime, @@ -108,65 +111,14 @@ func insertIntoImageCveInfos(batch *pgx.Batch, obj *storage.ImageCVEInfo) error obj.GetId(), protocompat.NilOrTime(obj.GetFixAvailableTimestamp()), protocompat.NilOrTime(obj.GetFirstSystemOccurrence()), + obj.GetCve(), serialized, } - finalStr := "INSERT INTO image_cve_infos (Id, FixAvailableTimestamp, FirstSystemOccurrence, serialized) VALUES($1, $2, $3, $4) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, FixAvailableTimestamp = EXCLUDED.FixAvailableTimestamp, FirstSystemOccurrence = EXCLUDED.FirstSystemOccurrence, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO image_cve_infos (Id, FixAvailableTimestamp, FirstSystemOccurrence, Cve, serialized) VALUES($1, $2, $3, $4, $5) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, FixAvailableTimestamp = EXCLUDED.FixAvailableTimestamp, FirstSystemOccurrence = EXCLUDED.FirstSystemOccurrence, Cve = EXCLUDED.Cve, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) return nil } -var copyColsImageCveInfos = []string{ - "id", - "fixavailabletimestamp", - "firstsystemoccurrence", - "serialized", -} - -func copyFromImageCveInfos(ctx context.Context, s pgSearch.Deleter, tx *postgres.Tx, objs ...*storage.ImageCVEInfo) error { - if len(objs) == 0 { - return nil - } - - { - // CopyFrom does not upsert, so delete existing rows first to achieve upsert behavior. - // Parent deletion cascades to children, so only the top-level parent needs deletion. - deletes := make([]string, 0, len(objs)) - for _, obj := range objs { - deletes = append(deletes, obj.GetId()) - } - if err := s.DeleteMany(ctx, deletes); err != nil { - return err - } - } - - idx := 0 - inputRows := pgx.CopyFromFunc(func() ([]any, error) { - if idx >= len(objs) { - return nil, nil - } - obj := objs[idx] - idx++ - - serialized, marshalErr := obj.MarshalVT() - if marshalErr != nil { - return nil, marshalErr - } - - return []interface{}{ - obj.GetId(), - protocompat.NilOrTime(obj.GetFixAvailableTimestamp()), - protocompat.NilOrTime(obj.GetFirstSystemOccurrence()), - serialized, - }, nil - }) - - if _, err := tx.CopyFrom(ctx, pgx.Identifier{"image_cve_infos"}, copyColsImageCveInfos, inputRows); err != nil { - return err - } - - return nil -} - // endregion Helper functions diff --git a/central/cve/image/info/enricher/enricher.go b/central/cve/image/info/enricher/enricher.go new file mode 100644 index 0000000000000..13d257bf3c5df --- /dev/null +++ b/central/cve/image/info/enricher/enricher.go @@ -0,0 +1,161 @@ +package enricher + +import ( + "context" + + imageCVEInfoDS "github.com/stackrox/rox/central/cve/image/info/datastore" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/cve" + "github.com/stackrox/rox/pkg/features" + imageEnricher "github.com/stackrox/rox/pkg/images/enricher" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/set" +) + +type enricherImpl struct { + imageCVEInfoDS imageCVEInfoDS.DataStore +} + +// New creates a new CVEInfoEnricher. +func New(imageCVEInfoDS imageCVEInfoDS.DataStore) imageEnricher.CVEInfoEnricher { + return &enricherImpl{ + imageCVEInfoDS: imageCVEInfoDS, + } +} + +// EnrichImageWithCVEInfo enriches a V1 image's CVEs with timing metadata. +func (e *enricherImpl) EnrichImageWithCVEInfo(ctx context.Context, image *storage.Image) error { + if !features.CVEFixTimestampCriteria.Enabled() { + return nil + } + + scan := image.GetScan() + if scan == nil { + return nil + } + + // Populate the ImageCVEInfo table with CVE timing metadata + if err := e.upsertImageCVEInfos(ctx, scan); err != nil { + return err + } + + // Enrich the CVEs with accurate timestamps from lookup table + return e.enrichCVEsFromImageCVEInfo(ctx, scan) +} + +// EnrichImageV2WithCVEInfo enriches a V2 image's CVEs with timing metadata. +func (e *enricherImpl) EnrichImageV2WithCVEInfo(ctx context.Context, image *storage.ImageV2) error { + if !features.CVEFixTimestampCriteria.Enabled() { + return nil + } + + scan := image.GetScan() + if scan == nil { + return nil + } + + // Populate the ImageCVEInfo table with CVE timing metadata + if err := e.upsertImageCVEInfos(ctx, scan); err != nil { + return err + } + + // Enrich the CVEs with accurate timestamps from lookup table + return e.enrichCVEsFromImageCVEInfo(ctx, scan) +} + +// upsertImageCVEInfos populates the ImageCVEInfo lookup table with CVE timing metadata. +func (e *enricherImpl) upsertImageCVEInfos(ctx context.Context, scan *storage.ImageScan) error { + infos := make([]*storage.ImageCVEInfo, 0) + now := protocompat.TimestampNow() + + for _, component := range scan.GetComponents() { + for _, vuln := range component.GetVulns() { + // Determine fix available timestamp: use scanner-provided value if available, + // otherwise fabricate from scan time if the CVE is fixable (has a fix version). + // This handles non-Red Hat data sources that don't provide fix timestamps. + fixAvailableTimestamp := vuln.GetFixAvailableTimestamp() + if fixAvailableTimestamp == nil && vuln.GetFixedBy() != "" { + fixAvailableTimestamp = now + } + + // Use scanner-provided FirstSystemOccurrence if available, otherwise use current time. + // The datastore upsert will preserve the earliest timestamp across scans. + firstSystemOccurrenceTimestamp := vuln.GetFirstSystemOccurrence() + if firstSystemOccurrenceTimestamp == nil { + firstSystemOccurrenceTimestamp = now + } + + info := &storage.ImageCVEInfo{ + Id: cve.ImageCVEInfoID(vuln.GetCve(), component.GetName(), vuln.GetDatasource()), + Cve: vuln.GetCve(), + FixAvailableTimestamp: fixAvailableTimestamp, + FirstSystemOccurrence: firstSystemOccurrenceTimestamp, + } + infos = append(infos, info) + } + } + + if len(infos) == 0 { + return nil + } + + return e.imageCVEInfoDS.UpsertMany(sac.WithAllAccess(ctx), infos) +} + +// enrichCVEsFromImageCVEInfo enriches the image's CVEs with accurate timestamps from the lookup table. +func (e *enricherImpl) enrichCVEsFromImageCVEInfo(ctx context.Context, scan *storage.ImageScan) error { + cveNames := set.NewStringSet() + for _, component := range scan.GetComponents() { + for _, vuln := range component.GetVulns() { + cveNames.Add(vuln.GetCve()) + } + } + + if cveNames.Cardinality() == 0 { + return nil + } + + // Collect timestamps and aggregate the min timestamp from CVE infos with the same CVE name. + // Build two maps from the same result set: + // - CVE names -> Min timestamp -> FirstSystemOccurrence + // - CVE info IDs (in this scan)-> ImageCVEInfo -> FixAvailableTimestamp + allInfos, err := e.imageCVEInfoDS.GetByCVENames(sac.WithAllAccess(ctx), cveNames.AsSlice()) + if err != nil { + return err + } + + cveMinTimestamps := make(map[string]*protocompat.Timestamp) + infoMap := make(map[string]*storage.ImageCVEInfo) + + for _, info := range allInfos { + // Track MIN FirstSystemOccurrence per CVE name + cveName := info.GetCve() + timestamp := info.GetFirstSystemOccurrence() + if timestamp != nil { + if existing, ok := cveMinTimestamps[cveName]; !ok || protocompat.CompareTimestamps(timestamp, existing) < 0 { + cveMinTimestamps[cveName] = timestamp + } + } + infoMap[info.GetId()] = info + } + + // Enrich CVEs + for _, component := range scan.GetComponents() { + for _, vuln := range component.GetVulns() { + if minTimestamp, ok := cveMinTimestamps[vuln.GetCve()]; ok { + vuln.FirstSystemOccurrence = minTimestamp + } + + id := cve.ImageCVEInfoID(vuln.GetCve(), component.GetName(), vuln.GetDatasource()) + if info, ok := infoMap[id]; ok { + if vuln.GetFixAvailableTimestamp() == nil && vuln.GetFixedBy() != "" { + // Set the fix timestamp if it was not provided by the scanner + vuln.FixAvailableTimestamp = info.GetFixAvailableTimestamp() + } + } + } + } + + return nil +} diff --git a/central/cve/image/info/enricher/singleton.go b/central/cve/image/info/enricher/singleton.go new file mode 100644 index 0000000000000..ba5b452e54968 --- /dev/null +++ b/central/cve/image/info/enricher/singleton.go @@ -0,0 +1,23 @@ +package enricher + +import ( + imageCVEInfoDS "github.com/stackrox/rox/central/cve/image/info/datastore" + imageEnricher "github.com/stackrox/rox/pkg/images/enricher" + "github.com/stackrox/rox/pkg/sync" +) + +var ( + once sync.Once + + instance imageEnricher.CVEInfoEnricher +) + +func initialize() { + instance = New(imageCVEInfoDS.Singleton()) +} + +// Singleton returns a singleton instance of CVEInfoEnricher. +func Singleton() imageEnricher.CVEInfoEnricher { + once.Do(initialize) + return instance +} diff --git a/central/cve/image/v2/datastore/datastore_impl.go b/central/cve/image/v2/datastore/datastore_impl.go index c77fa4c39e014..efa3091af1bbf 100644 --- a/central/cve/image/v2/datastore/datastore_impl.go +++ b/central/cve/image/v2/datastore/datastore_impl.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/stackrox/rox/central/cve/image/v2/datastore/store" + imagev2common "github.com/stackrox/rox/central/imagev2/common" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" pkgSearch "github.com/stackrox/rox/pkg/search" @@ -15,6 +16,7 @@ type datastoreImpl struct { } func (ds *datastoreImpl) Search(ctx context.Context, q *v1.Query) ([]pkgSearch.Result, error) { + q = imagev2common.WithRowsFromImageV2Only(q) return ds.storage.Search(ctx, q) } @@ -22,6 +24,7 @@ func (ds *datastoreImpl) SearchImageCVEs(ctx context.Context, q *v1.Query) ([]*v if q == nil { q = pkgSearch.EmptyQuery() } + q = imagev2common.WithRowsFromImageV2Only(q) // Clone the query and add select fields for SearchResult construction clonedQuery := q.CloneVT() @@ -47,6 +50,7 @@ func (ds *datastoreImpl) SearchImageCVEs(ctx context.Context, q *v1.Query) ([]*v } func (ds *datastoreImpl) SearchRawImageCVEs(ctx context.Context, q *v1.Query) ([]*storage.ImageCVEV2, error) { + q = imagev2common.WithRowsFromImageV2Only(q) var cves []*storage.ImageCVEV2 err := ds.storage.GetByQueryFn(ctx, q, func(cve *storage.ImageCVEV2) error { cves = append(cves, cve) @@ -60,6 +64,7 @@ func (ds *datastoreImpl) SearchRawImageCVEs(ctx context.Context, q *v1.Query) ([ } func (ds *datastoreImpl) Count(ctx context.Context, q *v1.Query) (int, error) { + q = imagev2common.WithRowsFromImageV2Only(q) return ds.storage.Count(ctx, q) } @@ -100,7 +105,3 @@ func (c *ImageCVESearchResultConverter) BuildLocation(result *pkgSearch.Result) func (c *ImageCVESearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_IMAGE_VULNERABILITIES_V2 } - -func (c *ImageCVESearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/cve/image/v2/datastore/datastore_sac_test.go b/central/cve/image/v2/datastore/datastore_sac_test.go index 34d6a54147e23..fbc6f32916f9c 100644 --- a/central/cve/image/v2/datastore/datastore_sac_test.go +++ b/central/cve/image/v2/datastore/datastore_sac_test.go @@ -9,6 +9,7 @@ import ( graphDBTestUtils "github.com/stackrox/rox/central/graphdb/testutils" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/cve" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/sac" @@ -54,6 +55,22 @@ func getImageCVEID(vuln *storage.EmbeddedVulnerability, component *storage.Embed return cveID } +// TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. +func getSherlockHolmesImageID() string { + if features.FlattenImageData.Enabled() { + return fixtures.GetImageV2SherlockHolmes1().GetId() + } + return fixtures.GetImageSherlockHolmes1().GetId() +} + +// TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. +func getDoctorJekyllImageID() string { + if features.FlattenImageData.Enabled() { + return fixtures.GetImageV2DoctorJekyll2().GetId() + } + return fixtures.GetImageDoctorJekyll2().GetId() +} + func (s *cveV2DataStoreSACTestSuite) cleanImageToVulnerabilitiesGraph() { s.Require().NoError(s.testGraphDatastore.CleanImageToVulnerabilitiesGraph()) } @@ -63,246 +80,249 @@ type cveTestCase struct { expectedCVEFound map[string]bool } -var ( - imageCVETestCases = []cveTestCase{ +func getImageCVETestCases() []cveTestCase { + sherlockID := getSherlockHolmesImageID() + jekyllID := getDoctorJekyllImageID() + + return []cveTestCase{ { contextKey: sacTestUtils.UnrestrictedReadCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): true, }, }, { contextKey: sacTestUtils.UnrestrictedReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): true, }, }, { contextKey: sacTestUtils.Cluster1ReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { contextKey: sacTestUtils.Cluster1NamespaceAReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { contextKey: sacTestUtils.Cluster1NamespaceBReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { contextKey: sacTestUtils.Cluster1NamespacesABReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { contextKey: sacTestUtils.Cluster1NamespacesBCReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { contextKey: sacTestUtils.Cluster2ReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): true, }, }, { contextKey: sacTestUtils.Cluster2NamespaceAReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { contextKey: sacTestUtils.Cluster2NamespaceBReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): true, }, }, { contextKey: sacTestUtils.Cluster2NamespacesACReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { contextKey: sacTestUtils.Cluster2NamespacesBCReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): true, }, }, { contextKey: sacTestUtils.Cluster3ReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { contextKey: sacTestUtils.Cluster3NamespaceAReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { contextKey: sacTestUtils.Cluster3NamespaceBReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { contextKey: sacTestUtils.Cluster3NamespacesABReadWriteCtx, expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): false, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): false, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): false, }, }, { @@ -311,24 +331,24 @@ var ( // Therefore, it should see all vulnerabilities. // (images are in cluster1 namespaceA and cluster2 namespaceB). expectedCVEFound: map[string]bool{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageDoctorJekyll2().GetId(), 0, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 0): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1): true, - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), jekyllID, 0, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 0): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1): true, + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2): true, }, }, } -) +} func (s *cveV2DataStoreSACTestSuite) TestSACImageCVEExistsSingleScopeOnly() { // Inject the fixture graph, and test exists for CVE-1234-0001 - cveID := getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0) + cveID := getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), getSherlockHolmesImageID(), 0, 0) s.runImageTest("TestSACImageCVEExistsSingleScopeOnly", func(c cveTestCase) { testCtx := s.imageTestContexts[c.contextKey] exists, err := s.imageCVEStore.Exists(testCtx, cveID) @@ -341,7 +361,7 @@ func (s *cveV2DataStoreSACTestSuite) TestSACImageCVEGetSingleScopeOnly() { // Inject the fixture graph, and test retrieval for CVE-1234-0001 targetCVE := fixtures.GetEmbeddedImageCVE1234x0001() cveName := targetCVE.GetCve() - cveID := getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0) + cveID := getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), getSherlockHolmesImageID(), 0, 0) cvss := targetCVE.GetCvss() s.runImageTest("TestSACImageCVEGetSingleScopeOnly", func(c cveTestCase) { testCtx := s.imageTestContexts[c.contextKey] @@ -359,14 +379,16 @@ func (s *cveV2DataStoreSACTestSuite) TestSACImageCVEGetSingleScopeOnly() { } func (s *cveV2DataStoreSACTestSuite) TestSACImageCVEGetBatch() { + sherlockID := getSherlockHolmesImageID() + jekyllID := getDoctorJekyllImageID() batchCVEs := []string{ - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 0), - getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), fixtures.GetImageSherlockHolmes1().GetId(), 0, 1), - getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), fixtures.GetImageSherlockHolmes1().GetId(), 1, 0), - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 0), - getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), fixtures.GetImageSherlockHolmes1().GetId(), 2, 1), - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 1), - getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), fixtures.GetImageDoctorJekyll2().GetId(), 2, 2), + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0001(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 0), + getImageCVEID(fixtures.GetEmbeddedImageCVE4567x0002(), fixtures.GetEmbeddedImageComponent1x1(), sherlockID, 0, 1), + getImageCVEID(fixtures.GetEmbeddedImageCVE1234x0003(), fixtures.GetEmbeddedImageComponent1x2(), sherlockID, 1, 0), + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0004(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 0), + getImageCVEID(fixtures.GetEmbeddedImageCVE3456x0005(), fixtures.GetEmbeddedImageComponent1s2x3(), sherlockID, 2, 1), + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0006(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 1), + getImageCVEID(fixtures.GetEmbeddedImageCVE2345x0007(), fixtures.GetEmbeddedImageComponent2x5(), jekyllID, 2, 2), } s.runImageTest("TestSACImageCVEGetBatch", func(c cveTestCase) { @@ -467,7 +489,7 @@ func (s *cveV2DataStoreSACTestSuite) runImageTest(testName string, testFunc func ) failed := false - for _, c := range imageCVETestCases { + for _, c := range getImageCVETestCases() { caseSucceeded := s.Run(c.contextKey, func() { // When triggered in parallel, most tests fail. // TearDownTest is executed before the sub-tests. diff --git a/central/cve/image/v2/datastore/datastore_v1_filter_test.go b/central/cve/image/v2/datastore/datastore_v1_filter_test.go new file mode 100644 index 0000000000000..b75813d2f2248 --- /dev/null +++ b/central/cve/image/v2/datastore/datastore_v1_filter_test.go @@ -0,0 +1,104 @@ +//go:build sql_integration + +package datastore + +import ( + "context" + "testing" + + imageDS "github.com/stackrox/rox/central/image/datastore" + imageV2DS "github.com/stackrox/rox/central/imagev2/datastore" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/fixtures" + imageUtils "github.com/stackrox/rox/pkg/images/utils" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/set" + "github.com/stretchr/testify/suite" +) + +func TestImageCVEV2DataStoreV1Filter(t *testing.T) { + suite.Run(t, new(ImageCVEV2DataStoreV1FilterTestSuite)) +} + +type ImageCVEV2DataStoreV1FilterTestSuite struct { + suite.Suite + + testDB *pgtest.TestPostgres + cveStore DataStore + ctx context.Context + + v2Image *storage.ImageV2 +} + +func (s *ImageCVEV2DataStoreV1FilterTestSuite) SetupSuite() { + if !features.FlattenImageData.Enabled() { + s.T().Skip("Skipping test because FlattenImageData feature flag is disabled") + } + s.ctx = sac.WithAllAccess(context.Background()) + s.testDB = pgtest.ForT(s.T()) + + // Temporarily disable the flag to insert a V1 image. + // The V1 store's CVE converter sets ImageId (not ImageIdV2) only when the flag is disabled. + s.T().Setenv(features.FlattenImageData.EnvVar(), "false") + v1Store := imageDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + v1Image := fixtures.GetImageSherlockHolmes1() + s.Require().NoError(v1Store.UpsertImage(s.ctx, v1Image)) + + // Re-enable the flag for V2 image insertion and all subsequent queries. + s.T().Setenv(features.FlattenImageData.EnvVar(), "true") + v2Store := imageV2DS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.v2Image = imageUtils.ConvertToV2(fixtures.GetImageDoctorJekyll2()) + s.Require().NoError(v2Store.UpsertImage(s.ctx, s.v2Image)) + + s.cveStore = GetTestPostgresDataStore(s.T(), s.testDB.DB) +} + +func (s *ImageCVEV2DataStoreV1FilterTestSuite) expectedV2CVEs() set.StringSet { + cves := set.NewStringSet() + for _, comp := range s.v2Image.GetScan().GetComponents() { + for _, vuln := range comp.GetVulns() { + cves.Add(vuln.GetCve()) + } + } + return cves +} + +func (s *ImageCVEV2DataStoreV1FilterTestSuite) TestCountExcludesV1() { + expectedCVEs := s.expectedV2CVEs() + + count, err := s.cveStore.Count(s.ctx, search.EmptyQuery()) + s.Require().NoError(err) + s.Assert().Equal(expectedCVEs.Cardinality(), count) +} + +func (s *ImageCVEV2DataStoreV1FilterTestSuite) TestSearchExcludesV1() { + expectedCVEs := s.expectedV2CVEs() + + results, err := s.cveStore.Search(s.ctx, search.EmptyQuery()) + s.Require().NoError(err) + s.Assert().Equal(expectedCVEs.Cardinality(), len(results)) +} + +func (s *ImageCVEV2DataStoreV1FilterTestSuite) TestSearchRawImageCVEsExcludesV1() { + expectedCVEs := s.expectedV2CVEs() + + cves, err := s.cveStore.SearchRawImageCVEs(s.ctx, search.EmptyQuery()) + s.Require().NoError(err) + + returnedCVEs := set.NewStringSet() + for _, cve := range cves { + returnedCVEs.Add(cve.GetCveBaseInfo().GetCve()) + } + s.Assert().Equal(expectedCVEs, returnedCVEs) +} + +func (s *ImageCVEV2DataStoreV1FilterTestSuite) TestSearchImageCVEsExcludesV1() { + expectedCVEs := s.expectedV2CVEs() + + results, err := s.cveStore.SearchImageCVEs(s.ctx, search.EmptyQuery()) + s.Require().NoError(err) + s.Assert().Equal(expectedCVEs.Cardinality(), len(results)) +} diff --git a/central/cve/image/v2/datastore/store/postgres/store.go b/central/cve/image/v2/datastore/store/postgres/store.go index d0689c1074f50..d76c93ecd28f4 100644 --- a/central/cve/image/v2/datastore/store/postgres/store.go +++ b/central/cve/image/v2/datastore/store/postgres/store.go @@ -102,11 +102,13 @@ func insertIntoImageCvesV2(batch *pgx.Batch, obj *storage.ImageCVEV2) error { values := []interface{}{ // parent primary keys start obj.GetId(), - obj.GetImageId(), + pgutils.NilOrString(obj.GetImageId()), obj.GetCveBaseInfo().GetCve(), protocompat.NilOrTime(obj.GetCveBaseInfo().GetPublishedOn()), protocompat.NilOrTime(obj.GetCveBaseInfo().GetCreatedAt()), obj.GetCveBaseInfo().GetEpss().GetEpssProbability(), + obj.GetCveBaseInfo().GetCisaKev(), + obj.GetCveBaseInfo().GetKnownRansomwareCampaign(), obj.GetCvss(), obj.GetSeverity(), obj.GetImpactScore(), @@ -119,10 +121,12 @@ func insertIntoImageCvesV2(batch *pgx.Batch, obj *storage.ImageCVEV2) error { obj.GetAdvisory().GetName(), obj.GetAdvisory().GetLink(), pgutils.NilOrString(obj.GetImageIdV2()), + protocompat.NilOrTime(obj.GetFixAvailableTimestamp()), + obj.GetOrigin(), serialized, } - finalStr := "INSERT INTO image_cves_v2 (Id, ImageId, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, Cvss, Severity, ImpactScore, Nvdcvss, FirstImageOccurrence, State, IsFixable, FixedBy, ComponentId, Advisory_Name, Advisory_Link, ImageIdV2, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, ImageId = EXCLUDED.ImageId, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, Cvss = EXCLUDED.Cvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Nvdcvss = EXCLUDED.Nvdcvss, FirstImageOccurrence = EXCLUDED.FirstImageOccurrence, State = EXCLUDED.State, IsFixable = EXCLUDED.IsFixable, FixedBy = EXCLUDED.FixedBy, ComponentId = EXCLUDED.ComponentId, Advisory_Name = EXCLUDED.Advisory_Name, Advisory_Link = EXCLUDED.Advisory_Link, ImageIdV2 = EXCLUDED.ImageIdV2, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO image_cves_v2 (Id, ImageId, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, CveBaseInfo_CisaKev, CveBaseInfo_KnownRansomwareCampaign, Cvss, Severity, ImpactScore, Nvdcvss, FirstImageOccurrence, State, IsFixable, FixedBy, ComponentId, Advisory_Name, Advisory_Link, ImageIdV2, FixAvailableTimestamp, Origin, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, ImageId = EXCLUDED.ImageId, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, CveBaseInfo_CisaKev = EXCLUDED.CveBaseInfo_CisaKev, CveBaseInfo_KnownRansomwareCampaign = EXCLUDED.CveBaseInfo_KnownRansomwareCampaign, Cvss = EXCLUDED.Cvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Nvdcvss = EXCLUDED.Nvdcvss, FirstImageOccurrence = EXCLUDED.FirstImageOccurrence, State = EXCLUDED.State, IsFixable = EXCLUDED.IsFixable, FixedBy = EXCLUDED.FixedBy, ComponentId = EXCLUDED.ComponentId, Advisory_Name = EXCLUDED.Advisory_Name, Advisory_Link = EXCLUDED.Advisory_Link, ImageIdV2 = EXCLUDED.ImageIdV2, FixAvailableTimestamp = EXCLUDED.FixAvailableTimestamp, Origin = EXCLUDED.Origin, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) return nil @@ -135,6 +139,8 @@ var copyColsImageCvesV2 = []string{ "cvebaseinfo_publishedon", "cvebaseinfo_createdat", "cvebaseinfo_epss_epssprobability", + "cvebaseinfo_cisakev", + "cvebaseinfo_knownransomwarecampaign", "cvss", "severity", "impactscore", @@ -147,6 +153,8 @@ var copyColsImageCvesV2 = []string{ "advisory_name", "advisory_link", "imageidv2", + "fixavailabletimestamp", + "origin", "serialized", } @@ -182,11 +190,13 @@ func copyFromImageCvesV2(ctx context.Context, s pgSearch.Deleter, tx *postgres.T return []interface{}{ obj.GetId(), - obj.GetImageId(), + pgutils.NilOrString(obj.GetImageId()), obj.GetCveBaseInfo().GetCve(), protocompat.NilOrTime(obj.GetCveBaseInfo().GetPublishedOn()), protocompat.NilOrTime(obj.GetCveBaseInfo().GetCreatedAt()), obj.GetCveBaseInfo().GetEpss().GetEpssProbability(), + obj.GetCveBaseInfo().GetCisaKev(), + obj.GetCveBaseInfo().GetKnownRansomwareCampaign(), obj.GetCvss(), obj.GetSeverity(), obj.GetImpactScore(), @@ -199,6 +209,8 @@ func copyFromImageCvesV2(ctx context.Context, s pgSearch.Deleter, tx *postgres.T obj.GetAdvisory().GetName(), obj.GetAdvisory().GetLink(), pgutils.NilOrString(obj.GetImageIdV2()), + protocompat.NilOrTime(obj.GetFixAvailableTimestamp()), + obj.GetOrigin(), serialized, }, nil }) diff --git a/central/cve/matcher/matcher.go b/central/cve/matcher/matcher.go index 244fd43cd0e4d..24c6df0c593a4 100644 --- a/central/cve/matcher/matcher.go +++ b/central/cve/matcher/matcher.go @@ -12,9 +12,11 @@ import ( clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" "github.com/stackrox/rox/central/cve/converter/utils" imageDataStore "github.com/stackrox/rox/central/image/datastore" + imageV2DataStore "github.com/stackrox/rox/central/imagev2/datastore" nsDataStore "github.com/stackrox/rox/central/namespace/datastore" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/errorhelpers" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/set" @@ -33,14 +35,16 @@ type CVEMatcher struct { clusters clusterDataStore.DataStore namespaces nsDataStore.DataStore images imageDataStore.DataStore + imagesV2 imageV2DataStore.DataStore } // NewCVEMatcher returns new instance of CVEMatcher -func NewCVEMatcher(clusters clusterDataStore.DataStore, namespaces nsDataStore.DataStore, images imageDataStore.DataStore) (*CVEMatcher, error) { +func NewCVEMatcher(clusters clusterDataStore.DataStore, namespaces nsDataStore.DataStore, images imageDataStore.DataStore, imagesV2 imageV2DataStore.DataStore) (*CVEMatcher, error) { return &CVEMatcher{ clusters: clusters, namespaces: namespaces, images: images, + imagesV2: imagesV2, }, nil } @@ -135,7 +139,7 @@ func (m *CVEMatcher) IsClusterAffectedByIstioCVE(ctx context.Context, cluster *s return false, nil } for _, node := range cve.Configurations.Nodes { - for _, version := range versions.AsSlice() { + for version := range versions { matched, err := m.MatchVersions(node, version, utils.Istio) // If we could determine CVE impact from one of cpe string, we skip logging error if matched { @@ -183,12 +187,22 @@ func (m *CVEMatcher) getAllIstioComponentsVersionsInCluster(ctx context.Context, AddExactMatches(search.ImageRegistry, "docker.io"). AddStrings(search.ImageRemote, "istio"). ProtoQuery() - images, err := m.images.SearchRawImages(ctx, q) - if err != nil { - return set, err - } - for _, image := range images { - set.Add(image.GetName().GetTag()) + if features.FlattenImageData.Enabled() { + images, err := m.imagesV2.SearchRawImages(ctx, q) + if err != nil { + return set, err + } + for _, image := range images { + set.Add(image.GetName().GetTag()) + } + } else { + images, err := m.images.SearchRawImages(ctx, q) + if err != nil { + return set, err + } + for _, image := range images { + set.Add(image.GetName().GetTag()) + } } return set, nil } @@ -232,8 +246,8 @@ func (m *CVEMatcher) MatchVersions(node *schema.NVDCVEFeedJSON10DefNode, version // Note that cpeVersionAndUpdate can't be "*:*" in this case, since there is no info about start and end versions if stringutils.AllEmpty(cpeMatch.VersionStartIncluding, cpeMatch.VersionEndIncluding, cpeMatch.VersionEndExcluding) { // This means this version and all prelease, build versions of this version. For example 1.6.4:* - if strings.HasSuffix(cpeVersionAndUpdate, ":*") { - if match, err := matchBaseVersion(strings.TrimSuffix(cpeVersionAndUpdate, ":*"), versionToMatch); err != nil { + if before, ok := strings.CutSuffix(cpeVersionAndUpdate, ":*"); ok { + if match, err := matchBaseVersion(before, versionToMatch); err != nil { errList.AddError(errors.Wrapf(err, "could not compare base version %q with cluster version: %q", strings.TrimSuffix(cpeVersionAndUpdate, ":*"), versionToMatch)) } else if match { return true, errList.ToError() diff --git a/central/cve/matcher/matcher_test.go b/central/cve/matcher/matcher_test.go index 998f1cbadb937..52fbf0bb8b236 100644 --- a/central/cve/matcher/matcher_test.go +++ b/central/cve/matcher/matcher_test.go @@ -7,8 +7,10 @@ import ( "github.com/facebookincubator/nvdtools/cvefeed/nvd/schema" mockClusterDataStore "github.com/stackrox/rox/central/cluster/datastore/mocks" mockImagesDataStore "github.com/stackrox/rox/central/image/datastore/mocks" + mockImageV2DataStore "github.com/stackrox/rox/central/imagev2/datastore/mocks" mockNamespaceDataStore "github.com/stackrox/rox/central/namespace/datastore/mocks" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" @@ -30,6 +32,7 @@ type cveMatcherTestSuite struct { clusters *mockClusterDataStore.MockDataStore namespaces *mockNamespaceDataStore.MockDataStore images *mockImagesDataStore.MockDataStore + imagesV2 *mockImageV2DataStore.MockDataStore mockCtrl *gomock.Controller } @@ -48,9 +51,10 @@ func (s *cveMatcherTestSuite) SetupTest() { s.clusters = mockClusterDataStore.NewMockDataStore(s.mockCtrl) s.namespaces = mockNamespaceDataStore.NewMockDataStore(s.mockCtrl) s.images = mockImagesDataStore.NewMockDataStore(s.mockCtrl) + s.imagesV2 = mockImageV2DataStore.NewMockDataStore(s.mockCtrl) var err error - s.cveMatcher, err = NewCVEMatcher(s.clusters, s.namespaces, s.images) + s.cveMatcher, err = NewCVEMatcher(s.clusters, s.namespaces, s.images, s.imagesV2) s.Require().NoError(err) } @@ -746,7 +750,17 @@ func (s *cveMatcherTestSuite) TestIstioCVEImpactsCluster() { s.clusters.EXPECT().GetClusters(gomock.Any()).Return(clusters, nil).AnyTimes() s.namespaces.EXPECT().Search(gomock.Any(), gomock.Any()).Return(namespaces, nil).AnyTimes() - s.images.EXPECT().SearchRawImages(gomock.Any(), gomock.Any()).Return(images, nil).AnyTimes() + + // Matcher uses images or imagesV2 depending on FlattenImageData. + imagesV2 := make([]*storage.ImageV2, len(images)) + for i, img := range images { + imagesV2[i] = &storage.ImageV2{Id: img.GetId(), Name: img.GetName()} + } + if features.FlattenImageData.Enabled() { + s.imagesV2.EXPECT().SearchRawImages(gomock.Any(), gomock.Any()).Return(imagesV2, nil).AnyTimes() + } else { + s.images.EXPECT().SearchRawImages(gomock.Any(), gomock.Any()).Return(images, nil).AnyTimes() + } ok, err := s.cveMatcher.isIstioControlPlaneRunning(context.Background()) s.Nil(err) diff --git a/central/cve/matcher/singleton.go b/central/cve/matcher/singleton.go index a3aaa2f338852..2028840f1145d 100644 --- a/central/cve/matcher/singleton.go +++ b/central/cve/matcher/singleton.go @@ -3,6 +3,7 @@ package matcher import ( clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" imageDataStore "github.com/stackrox/rox/central/image/datastore" + imageV2DataStore "github.com/stackrox/rox/central/imagev2/datastore" nsDataStore "github.com/stackrox/rox/central/namespace/datastore" "github.com/stackrox/rox/pkg/sync" "github.com/stackrox/rox/pkg/utils" @@ -15,7 +16,7 @@ var ( func initialize() { var err error - cveMatcher, err = NewCVEMatcher(clusterDataStore.Singleton(), nsDataStore.Singleton(), imageDataStore.Singleton()) + cveMatcher, err = NewCVEMatcher(clusterDataStore.Singleton(), nsDataStore.Singleton(), imageDataStore.Singleton(), imageV2DataStore.Singleton()) utils.CrashOnError(err) } diff --git a/central/cve/node/datastore/datastore_impl.go b/central/cve/node/datastore/datastore_impl.go index b1ce20ba9145f..43f95f7cf2556 100644 --- a/central/cve/node/datastore/datastore_impl.go +++ b/central/cve/node/datastore/datastore_impl.go @@ -299,7 +299,3 @@ func (c *NodeCVESearchResultConverter) BuildLocation(result *pkgSearch.Result) s func (c *NodeCVESearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_NODE_VULNERABILITIES } - -func (c *NodeCVESearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/cve/node/datastore/store/postgres/store.go b/central/cve/node/datastore/store/postgres/store.go index 375e6f05a43a0..d60a5591f2700 100644 --- a/central/cve/node/datastore/store/postgres/store.go +++ b/central/cve/node/datastore/store/postgres/store.go @@ -105,6 +105,8 @@ func insertIntoNodeCves(batch *pgx.Batch, obj *storage.NodeCVE) error { protocompat.NilOrTime(obj.GetCveBaseInfo().GetPublishedOn()), protocompat.NilOrTime(obj.GetCveBaseInfo().GetCreatedAt()), obj.GetCveBaseInfo().GetEpss().GetEpssProbability(), + obj.GetCveBaseInfo().GetCisaKev(), + obj.GetCveBaseInfo().GetKnownRansomwareCampaign(), obj.GetOperatingSystem(), obj.GetCvss(), obj.GetSeverity(), @@ -116,7 +118,7 @@ func insertIntoNodeCves(batch *pgx.Batch, obj *storage.NodeCVE) error { serialized, } - finalStr := "INSERT INTO node_cves (Id, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, OperatingSystem, Cvss, Severity, ImpactScore, Snoozed, SnoozeExpiry, Orphaned, OrphanedTime, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, OperatingSystem = EXCLUDED.OperatingSystem, Cvss = EXCLUDED.Cvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Snoozed = EXCLUDED.Snoozed, SnoozeExpiry = EXCLUDED.SnoozeExpiry, Orphaned = EXCLUDED.Orphaned, OrphanedTime = EXCLUDED.OrphanedTime, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO node_cves (Id, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, CveBaseInfo_CisaKev, CveBaseInfo_KnownRansomwareCampaign, OperatingSystem, Cvss, Severity, ImpactScore, Snoozed, SnoozeExpiry, Orphaned, OrphanedTime, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, CveBaseInfo_CisaKev = EXCLUDED.CveBaseInfo_CisaKev, CveBaseInfo_KnownRansomwareCampaign = EXCLUDED.CveBaseInfo_KnownRansomwareCampaign, OperatingSystem = EXCLUDED.OperatingSystem, Cvss = EXCLUDED.Cvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Snoozed = EXCLUDED.Snoozed, SnoozeExpiry = EXCLUDED.SnoozeExpiry, Orphaned = EXCLUDED.Orphaned, OrphanedTime = EXCLUDED.OrphanedTime, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) return nil @@ -128,6 +130,8 @@ var copyColsNodeCves = []string{ "cvebaseinfo_publishedon", "cvebaseinfo_createdat", "cvebaseinfo_epss_epssprobability", + "cvebaseinfo_cisakev", + "cvebaseinfo_knownransomwarecampaign", "operatingsystem", "cvss", "severity", @@ -175,6 +179,8 @@ func copyFromNodeCves(ctx context.Context, s pgSearch.Deleter, tx *postgres.Tx, protocompat.NilOrTime(obj.GetCveBaseInfo().GetPublishedOn()), protocompat.NilOrTime(obj.GetCveBaseInfo().GetCreatedAt()), obj.GetCveBaseInfo().GetEpss().GetEpssProbability(), + obj.GetCveBaseInfo().GetCisaKev(), + obj.GetCveBaseInfo().GetKnownRansomwareCampaign(), obj.GetOperatingSystem(), obj.GetCvss(), obj.GetSeverity(), diff --git a/central/cve/utils/cve_types.go b/central/cve/utils/cve_types.go deleted file mode 100644 index 89303b9b1dfc1..0000000000000 --- a/central/cve/utils/cve_types.go +++ /dev/null @@ -1,24 +0,0 @@ -package utils - -import "github.com/stackrox/rox/generated/storage" - -// AddCVETypeIfAbsent adds the given CVE type to the given slice of CVE types if the slice does -// not already have the CVE type and returns a slice with the given type included. -func AddCVETypeIfAbsent(cveTypes []storage.CVE_CVEType, toAdd storage.CVE_CVEType) []storage.CVE_CVEType { - // New CVE's types slice will be nil/empty. - // Populate with the current CVE's. - addToCVETypes := true - // We simply loop through each type, as this slice is very small. - for _, cveType := range cveTypes { - if toAdd == cveType { - addToCVETypes = false - break - } - } - // Add the new CVE's type to the type slice if it's not already in it. - if addToCVETypes { - return append(cveTypes, toAdd) - } - - return cveTypes -} diff --git a/central/debug/service/db_diagnostics_test.go b/central/debug/service/db_diagnostics_test.go index 7140a1ecd4a30..83f63d7f34cce 100644 --- a/central/debug/service/db_diagnostics_test.go +++ b/central/debug/service/db_diagnostics_test.go @@ -8,6 +8,7 @@ import ( "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/postgres/stats" "github.com/stackrox/rox/pkg/sac" "github.com/stretchr/testify/suite" ) @@ -68,3 +69,18 @@ func (s *DBDiagnosticTestSuite) TestBuildDiagnosticData() { diagnosticData = buildDBDiagnosticData(s.ctx, nil, s.dbPool) s.Equal(diagnosticData.DatabaseConnectString, "") } + +func (s *DBDiagnosticTestSuite) TestGetPGStatActivities() { + activities := stats.GetPGStatActivities(s.ctx, s.dbPool, 100) + s.Empty(activities.Error) + s.True(len(activities.Activities) > 0) + + foundActive := false + for _, a := range activities.Activities { + if a.State != nil && *a.State == "active" { + foundActive = true + break + } + } + s.True(foundActive, "expected to find at least one active connection") +} diff --git a/central/debug/service/diagnostics_test.go b/central/debug/service/diagnostics_test.go index 040c832501e74..3b263c31c4279 100644 --- a/central/debug/service/diagnostics_test.go +++ b/central/debug/service/diagnostics_test.go @@ -75,7 +75,7 @@ func TestGetK8sDiagnostics(t *testing.T) { assert.NoError(t, err) assert.Len(t, zipReader.File, 2) - var zipFileNames []string + zipFileNames := make([]string, 0, len(zipReader.File)) for _, file := range zipReader.File { zipFileNames = append(zipFileNames, file.Name) } @@ -128,7 +128,7 @@ func TestPullSensorMetrics(t *testing.T) { assert.NoError(t, err) assert.Len(t, zipReader.File, 1) - var zipFileNames []string + zipFileNames := make([]string, 0, len(zipReader.File)) for _, file := range zipReader.File { zipFileNames = append(zipFileNames, file.Name) } diff --git a/central/debug/service/service.go b/central/debug/service/service.go index 5e50e29de4d57..b6028ced779bb 100644 --- a/central/debug/service/service.go +++ b/central/debug/service/service.go @@ -503,7 +503,25 @@ func getCentralDBData(ctx context.Context, zipWriter *zipWriter) error { if tuples.Error != "" { log.Errorw("error retrieving pg_stat_user_tables", logging.Err(errors.New(tuples.Error))) } - return addJSONToZip(zipWriter, "central-db-pg-tuples.json", tuples) + if err := addJSONToZip(zipWriter, "central-db-pg-tuples.json", tuples); err != nil { + return err + } + + // Get the active session stats + activities := stats.GetPGStatActivities(ctx, db, pgStatStatementsMax) + if activities.Error != "" { + log.Errorw("error retrieving pg_stat_activity", logging.Err(errors.New(activities.Error))) + } + if err := addJSONToZip(zipWriter, "central-db-pg-activity.json", activities); err != nil { + return err + } + + // Get the index health stats + indexStats := stats.GetPGIndexStats(ctx, db, pgStatStatementsMax) + if indexStats.Error != "" { + log.Errorw("error retrieving pg_stat_user_indexes", logging.Err(errors.New(indexStats.Error))) + } + return addJSONToZip(zipWriter, "central-db-pg-index-stats.json", indexStats) } func (s *serviceImpl) getLogImbue(ctx context.Context, zipWriter *zipWriter) error { @@ -549,7 +567,11 @@ func (m safeRawMessage) MarshalJSON() ([]byte, error) { func (s *serviceImpl) getAuthProviders(_ context.Context) (interface{}, error) { authProviders := s.authProviderRegistry.GetProviders(nil, nil) + // An empty slice and nil are marshaled differently. Therefore if keys is empty we want to keep result equal to nil var storageAuthProviders []*storage.AuthProvider + if len(authProviders) > 0 { + storageAuthProviders = make([]*storage.AuthProvider, 0, len(authProviders)) + } for _, authProvider := range authProviders { storageAuthProviders = append(storageAuthProviders, authProvider.StorageView()) } @@ -828,7 +850,9 @@ func (s *serviceImpl) writeZippedDebugDump(ctx context.Context, w http.ResponseW log.Info("Finished writing data to the diagnostic bundle") // Get logs last to also catch logs made during creation of diag bundle. - if (opts.withCentral || opts.withDBOnly) && (opts.logs == localLogs || failureDuringDiagnostics) { + if (opts.withCentral || opts.withDBOnly) && + (opts.logs == localLogs || failureDuringDiagnostics) && + !env.ManagedCentral.BooleanSetting() { if err := logging.ForEachRotation(logging.LoggingPath, func(logFileName string) error { return zipWriter.addFile(filepath.Base(logFileName), logFileName) }); err != nil { diff --git a/central/declarativeconfig/manager_impl.go b/central/declarativeconfig/manager_impl.go index ab0db50c96d51..133d90d7d61e9 100644 --- a/central/declarativeconfig/manager_impl.go +++ b/central/declarativeconfig/manager_impl.go @@ -304,7 +304,7 @@ func (m *managerImpl) doDeletion(transformedMessagesByHandler map[string]protoMe // Otherwise, the reason why the deletion failed will not be visible to users while the resource may still // exist. if err != nil { - log.Warnf("The following IDs failed deletion: [%s]", strings.Join(failedDeletionIDs, ",")) + log.Warnf("Failed to delete resources [%s]: %s", strings.Join(failedDeletionIDs, ","), err) allProtoIDsToSkip = append(allProtoIDsToSkip, failedDeletionIDs...) failureInDeletion = true } diff --git a/central/declarativeconfig/manager_impl_test.go b/central/declarativeconfig/manager_impl_test.go index 4087752f1f29f..5458a4c3a3ac2 100644 --- a/central/declarativeconfig/manager_impl_test.go +++ b/central/declarativeconfig/manager_impl_test.go @@ -308,7 +308,7 @@ func TestReconcileTransformedMessages_ErrorPropagatedToReporter(t *testing.T) { m.declarativeConfigHealthDS = mockHealthDS // We need to call this 3 times, only then the error will be propagated to the mockHealthDS. - for i := 0; i < consecutiveReconciliationErrorThreshold; i++ { + for range consecutiveReconciliationErrorThreshold { m.reconcileTransformedMessages(map[string]protoMessagesByType{ "test-handler-1": { types.PermissionSetType: []protocompat.Message{ diff --git a/central/declarativeconfig/types/accepted_types.go b/central/declarativeconfig/types/accepted_types.go index fb74c9f8731c3..f94b06e4246f6 100644 --- a/central/declarativeconfig/types/accepted_types.go +++ b/central/declarativeconfig/types/accepted_types.go @@ -8,19 +8,19 @@ import ( var ( // AuthProviderType reflects the type of storage.AuthProvider. - AuthProviderType = reflect.TypeOf((*storage.AuthProvider)(nil)) + AuthProviderType = reflect.TypeFor[*storage.AuthProvider]() // AccessScopeType reflects the type of storage.SimpleAccessScope. - AccessScopeType = reflect.TypeOf((*storage.SimpleAccessScope)(nil)) + AccessScopeType = reflect.TypeFor[*storage.SimpleAccessScope]() // GroupType reflects the type of storage.Group. - GroupType = reflect.TypeOf((*storage.Group)(nil)) + GroupType = reflect.TypeFor[*storage.Group]() // PermissionSetType reflects the type of storage.PermissionSet. - PermissionSetType = reflect.TypeOf((*storage.PermissionSet)(nil)) + PermissionSetType = reflect.TypeFor[*storage.PermissionSet]() // RoleType reflects the type of storage.Role. - RoleType = reflect.TypeOf((*storage.Role)(nil)) + RoleType = reflect.TypeFor[*storage.Role]() // NotifierType reflects the type of storage.Notifier. - NotifierType = reflect.TypeOf((*storage.Notifier)(nil)) + NotifierType = reflect.TypeFor[*storage.Notifier]() // AuthMachineToMachineConfigType reflects the type of storage.AuthMachineToMachineConfig. - AuthMachineToMachineConfigType = reflect.TypeOf((*storage.AuthMachineToMachineConfig)(nil)) + AuthMachineToMachineConfigType = reflect.TypeFor[*storage.AuthMachineToMachineConfig]() ) // GetSupportedProtobufTypesInProcessingOrder returns the list of protobuf types diff --git a/central/declarativeconfig/updater/role_updater_test.go b/central/declarativeconfig/updater/role_updater_test.go index f961b6c80133f..544450e72bc47 100644 --- a/central/declarativeconfig/updater/role_updater_test.go +++ b/central/declarativeconfig/updater/role_updater_test.go @@ -70,6 +70,7 @@ func (s *roleUpdaterTestSuite) SetupTest() { ads, tokenIssuerFactory, mapperFactory, + nil, ) s.Require().NoError(s.authProviderRegistry.RegisterBackendFactory( s.ctx, diff --git a/central/declarativeconfig/watch_handler.go b/central/declarativeconfig/watch_handler.go index b945c956c2708..0e4adbdfad9e0 100644 --- a/central/declarativeconfig/watch_handler.go +++ b/central/declarativeconfig/watch_handler.go @@ -151,7 +151,7 @@ func (w *watchHandler) checkForDeletedFiles(fileContents map[string][]byte) bool } newCachedFileHashes := maputil.ShallowClone(w.cachedFileHashes) - for _, removedFile := range removedFiles.AsSlice() { + for removedFile := range removedFiles { delete(newCachedFileHashes, removedFile) } @@ -160,12 +160,13 @@ func (w *watchHandler) checkForDeletedFiles(fileContents map[string][]byte) bool } func (w *watchHandler) logFileContents(contents map[string][]byte) { - logMessage := "Found declarative configuration file contents\n" + var logMessage strings.Builder + logMessage.WriteString("Found declarative configuration file contents\n") for fileName, fileContents := range contents { - logMessage += fmt.Sprintf("File %s: %s\n", fileName, fileContents) + logMessage.WriteString(fmt.Sprintf("File %s: %s\n", fileName, fileContents)) } - log.Debug(logMessage) + log.Debug(logMessage.String()) } // readDeclarativeConfigFile will read the file and additionally verify that the contents are valid YAML. diff --git a/central/delegatedregistryconfig/delegator/delegator.go b/central/delegatedregistryconfig/delegator/delegator.go index 00af344df8377..c1e128ad0ed97 100644 --- a/central/delegatedregistryconfig/delegator/delegator.go +++ b/central/delegatedregistryconfig/delegator/delegator.go @@ -6,6 +6,7 @@ import ( "time" "github.com/pkg/errors" + clusterDS "github.com/stackrox/rox/central/cluster/datastore" "github.com/stackrox/rox/central/delegatedregistryconfig/datastore" deleConnection "github.com/stackrox/rox/central/delegatedregistryconfig/util/connection" centralMetrics "github.com/stackrox/rox/central/metrics" @@ -31,13 +32,14 @@ var ( ) // New creates a new delegator. -func New(deleRegConfigDS datastore.DataStore, connManager connection.Manager, scanWaiterManager waiter.Manager[*storage.Image], scanWaiterManagerV2 waiter.Manager[*storage.ImageV2], namespaceSACHelper sachelper.ClusterNamespaceSacHelper) *delegatorImpl { +func New(deleRegConfigDS datastore.DataStore, connManager connection.Manager, scanWaiterManager waiter.Manager[*storage.Image], scanWaiterManagerV2 waiter.Manager[*storage.ImageV2], namespaceSACHelper sachelper.ClusterNamespaceSacHelper, clusterDataStore clusterDS.DataStore) *delegatorImpl { return &delegatorImpl{ deleRegConfigDS: deleRegConfigDS, connManager: connManager, scanWaiterManager: scanWaiterManager, scanWaiterManagerV2: scanWaiterManagerV2, namespaceSACHelper: namespaceSACHelper, + clusterDataStore: clusterDataStore, } } @@ -48,6 +50,10 @@ type delegatorImpl struct { // namespaceSACHelper for confirming namespace exists and user has access. namespaceSACHelper sachelper.ClusterNamespaceSacHelper + // clusterDataStore for resolving cluster names in user-facing messages within + // the requester's scope. + clusterDataStore clusterDS.DataStore + // connManager for sending scan requests to secured clusters and ensuring // clusters are valid for delegation. connManager connection.Manager @@ -79,7 +85,7 @@ func (d *delegatorImpl) GetDelegateClusterID(ctx context.Context, imgName *stora return "", true, delegatedregistry.ErrNoClusterSpecified } - if err := d.ValidateCluster(clusterID); err != nil { + if err := d.ValidateCluster(ctx, clusterID); err != nil { return "", true, err } @@ -264,19 +270,34 @@ func (d *delegatorImpl) shouldDelegate(imgName *storage.ImageName, config *stora } // ValidateCluster returns nil if a cluster is a valid target for delegation, otherwise returns an error. -func (d *delegatorImpl) ValidateCluster(clusterID string) error { +func (d *delegatorImpl) ValidateCluster(ctx context.Context, clusterID string) error { conn := d.connManager.GetConnection(clusterID) if conn == nil { - return errors.Errorf("no connection to %q", clusterID) + return errors.Errorf("no connection to cluster %q (%s), verify the cluster is healthy and connected", d.clusterName(ctx, clusterID), clusterID) } if !deleConnection.ValidForDelegation(conn) { - return errors.Errorf("cluster %q does not support delegated scanning", clusterID) + return errors.Errorf("cluster %q (%s) does not support delegated scanning", d.clusterName(ctx, clusterID), clusterID) } return nil } +// clusterName resolves a human-readable cluster name for user-facing messages, +// falling back to the cluster ID when the name cannot be resolved within the +// requester's scope. +func (d *delegatorImpl) clusterName(ctx context.Context, clusterID string) string { + // GetClusterName enforces a cluster-read SAC check against the passed context, so + // resolving under the requester's own (non-elevated) context reveals the name only + // to callers allowed to view this cluster. Everyone else keeps the opaque ID. + name, exists, err := d.clusterDataStore.GetClusterName(ctx, clusterID) + if err != nil || !exists { + return clusterID + } + + return name +} + // withAdminRead elevates a context to include admin read access. func withAdminRead(ctx context.Context) context.Context { elevatedCtx := sac.WithGlobalAccessScopeChecker( diff --git a/central/delegatedregistryconfig/delegator/delegator_test.go b/central/delegatedregistryconfig/delegator/delegator_test.go index 3f955add7c913..9532d6a952e9b 100644 --- a/central/delegatedregistryconfig/delegator/delegator_test.go +++ b/central/delegatedregistryconfig/delegator/delegator_test.go @@ -6,6 +6,7 @@ import ( "fmt" "testing" + clusterDSMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" deleDSMocks "github.com/stackrox/rox/central/delegatedregistryconfig/datastore/mocks" sacHelperMocks "github.com/stackrox/rox/central/role/sachelper/mocks" connMocks "github.com/stackrox/rox/central/sensor/service/connection/mocks" @@ -51,7 +52,9 @@ func TestGetDelegateClusterID(t *testing.T) { connMgr = connMocks.NewMockManager(ctrl) waiterMgr = waiterMocks.NewMockManager[*storage.Image](ctrl) deleClusterDS = deleDSMocks.NewMockDataStore(ctrl) - d = New(deleClusterDS, connMgr, waiterMgr, nil, nil) + clusterDS := clusterDSMocks.NewMockDataStore(ctrl) + clusterDS.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).Return("fake-cluster-name", true, nil).AnyTimes() + d = New(deleClusterDS, connMgr, waiterMgr, nil, nil, clusterDS) } t.Run("error get config", func(t *testing.T) { @@ -111,7 +114,7 @@ func TestGetDelegateClusterID(t *testing.T) { connMgr.EXPECT().GetConnection(gomock.Any()).Return(nil) _, shouldDelegate, err := d.GetDelegateClusterID(ctxBG, nil) assert.True(t, shouldDelegate) - assert.ErrorContains(t, err, "no connection") + assert.ErrorContains(t, err, "no connection to cluster") }) t.Run("all def cluster id conn no cap", func(t *testing.T) { @@ -263,7 +266,7 @@ func TestDelegateEnrichImage(t *testing.T) { namespaceSACHelper = sacHelperMocks.NewMockClusterNamespaceSacHelper(ctrl) waiter.EXPECT().ID().Return(fakeWaiterID).AnyTimes() - d = New(deleClusterDS, connMgr, waiterMgr, nil, namespaceSACHelper) + d = New(deleClusterDS, connMgr, waiterMgr, nil, namespaceSACHelper, nil) } t.Run("empty cluster id", func(t *testing.T) { @@ -357,7 +360,7 @@ func TestDelegateScanImageV2(t *testing.T) { namespaceSACHelper = sacHelperMocks.NewMockClusterNamespaceSacHelper(ctrl) waiter.EXPECT().ID().Return(fakeWaiterID).AnyTimes() - d = New(deleClusterDS, connMgr, nil, waiterMgr, namespaceSACHelper) + d = New(deleClusterDS, connMgr, nil, waiterMgr, namespaceSACHelper, nil) } t.Run("empty cluster id", func(t *testing.T) { @@ -440,7 +443,7 @@ func TestInferNamespace(t *testing.T) { ctrl := gomock.NewController(t) namespaceSACHelper = sacHelperMocks.NewMockClusterNamespaceSacHelper(ctrl) - d = New(nil, nil, nil, nil, namespaceSACHelper) + d = New(nil, nil, nil, nil, namespaceSACHelper, nil) } t.Run("no namespace on error", func(t *testing.T) { diff --git a/central/delegatedregistryconfig/util/imageintegration/imageintegration.go b/central/delegatedregistryconfig/util/imageintegration/imageintegration.go index 53b23167fffa9..10e260f60a360 100644 --- a/central/delegatedregistryconfig/util/imageintegration/imageintegration.go +++ b/central/delegatedregistryconfig/util/imageintegration/imageintegration.go @@ -1,6 +1,10 @@ package imageintegration -import "github.com/stackrox/rox/generated/storage" +import ( + "slices" + + "github.com/stackrox/rox/generated/storage" +) // ValidForSync returns true if the provided image integration is eligible // for syncing with secured clusters. @@ -9,11 +13,5 @@ func ValidForSync(ii *storage.ImageIntegration) bool { return false } - for _, cat := range ii.GetCategories() { - if cat == storage.ImageIntegrationCategory_REGISTRY { - return true - } - } - - return false + return slices.Contains(ii.GetCategories(), storage.ImageIntegrationCategory_REGISTRY) } diff --git a/central/deployment/datastore/datastore.go b/central/deployment/datastore/datastore.go index b8daec8bdf512..8b9bb033ff99c 100644 --- a/central/deployment/datastore/datastore.go +++ b/central/deployment/datastore/datastore.go @@ -34,6 +34,8 @@ type DataStore interface { ListDeployment(ctx context.Context, id string) (*storage.ListDeployment, bool, error) GetDeployment(ctx context.Context, id string) (*storage.Deployment, bool, error) + // DeploymentExists reports whether the deployment is stored, without fetching or copying it. + DeploymentExists(ctx context.Context, id string) (bool, error) GetDeployments(ctx context.Context, ids []string) ([]*storage.Deployment, error) CountDeployments(ctx context.Context) (int, error) // UpsertDeployment adds or updates a deployment. If the deployment exists, the tags in the deployment are taken from @@ -48,6 +50,8 @@ type DataStore interface { WalkByQuery(ctx context.Context, query *v1.Query, fn func(deployment *storage.Deployment) error) error GetContainerImageViews(ctx context.Context, q *v1.Query) ([]*views.ContainerImageView, error) + + GetV2ImageIDsForDigests(ctx context.Context, digests []string) (map[string][]string, error) } func newDataStore( diff --git a/central/deployment/datastore/datastore_bench_test.go b/central/deployment/datastore/datastore_bench_test.go index a12a8c10bab0f..fd5784016550a 100644 --- a/central/deployment/datastore/datastore_bench_test.go +++ b/central/deployment/datastore/datastore_bench_test.go @@ -23,7 +23,7 @@ func BenchmarkSearchAllDeployments(b *testing.B) { deploymentPrototype := fixtures.GetDeployment().CloneVT() const numDeployments = 1000 - for i := 0; i < numDeployments; i++ { + for i := range numDeployments { if i > 0 && i%100 == 0 { fmt.Println("Added", i, "deployments") } @@ -32,7 +32,7 @@ func BenchmarkSearchAllDeployments(b *testing.B) { } b.Run("SearchRetrievalList", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { deployments, err := deploymentsDatastore.SearchListDeployments(ctx, search2.EmptyQuery()) assert.NoError(b, err) assert.Len(b, deployments, numDeployments) @@ -40,7 +40,7 @@ func BenchmarkSearchAllDeployments(b *testing.B) { }) b.Run("SearchRetrievalFull", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { deployments, err := deploymentsDatastore.SearchRawDeployments(ctx, search2.EmptyQuery()) assert.NoError(b, err) assert.Len(b, deployments, numDeployments) diff --git a/central/deployment/datastore/datastore_impl.go b/central/deployment/datastore/datastore_impl.go index a513d311765a4..4c69aa96b9a0b 100644 --- a/central/deployment/datastore/datastore_impl.go +++ b/central/deployment/datastore/datastore_impl.go @@ -25,6 +25,7 @@ import ( "github.com/stackrox/rox/pkg/errorhelpers" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/images/types" + "github.com/stackrox/rox/pkg/images/utils" imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/kubernetes" "github.com/stackrox/rox/pkg/process/filter" @@ -157,13 +158,7 @@ func (ds *datastoreImpl) ListDeployment(ctx context.Context, id string) (*storag func (ds *datastoreImpl) SearchListDeployments(ctx context.Context, q *v1.Query) ([]*storage.ListDeployment, error) { defer metrics.SetDatastoreFunctionDuration(time.Now(), "Deployment", "SearchListDeployments") - results, err := ds.Search(ctx, q) - if err != nil { - return nil, err - } - - ids := pkgSearch.ResultsToIDs(results) - listDeployments, _, err := ds.deploymentStore.GetManyListDeployments(ctx, ids...) + listDeployments, err := ds.deploymentStore.SearchListDeployments(ctx, q) if err != nil { return nil, err } @@ -232,6 +227,11 @@ func (ds *datastoreImpl) GetDeployment(ctx context.Context, id string) (*storage return deployment, true, nil } +// DeploymentExists +func (ds *datastoreImpl) DeploymentExists(ctx context.Context, id string) (bool, error) { + return ds.deploymentStore.Exists(ctx, id) +} + // GetDeployments func (ds *datastoreImpl) GetDeployments(ctx context.Context, ids []string) ([]*storage.Deployment, error) { var deployments []*storage.Deployment @@ -282,11 +282,28 @@ func allImagesAreSpecifiedByDigest(d *storage.Deployment) bool { return true } -func (ds *datastoreImpl) mergeCronJobs(ctx context.Context, deployment *storage.Deployment) error { - if deployment.GetType() != kubernetes.CronJob { - return nil +func isMissingImageID(c *storage.Container) bool { + if features.FlattenImageData.Enabled() { + return c.GetImage().GetId() == "" || c.GetImage().GetIdV2() == "" + } + return c.GetImage().GetId() == "" +} + +func copyImageID(from, to *storage.Container) { + if from.GetImage().GetId() == "" { + return + } + if to.GetImage().GetName().GetFullName() != from.GetImage().GetName().GetFullName() { + return } - if allImagesAreSpecifiedByDigest(deployment) { + to.GetImage().Id = from.GetImage().GetId() + if features.FlattenImageData.Enabled() { + to.GetImage().IdV2 = utils.NewImageV2ID(to.GetImage().GetName(), to.GetImage().GetId()) + } +} + +func (ds *datastoreImpl) mergeCronJobs(ctx context.Context, deployment *storage.Deployment) error { + if deployment.GetType() != kubernetes.CronJob || allImagesAreSpecifiedByDigest(deployment) { return nil } oldDeployment, exists, err := ds.deploymentStore.Get(ctx, deployment.GetId()) @@ -296,22 +313,14 @@ func (ds *datastoreImpl) mergeCronJobs(ctx context.Context, deployment *storage. if !exists { return nil } - // Major changes to spec, just upsert - if len(oldDeployment.GetContainers()) != len(deployment.GetContainers()) { - return nil + oldContainersByName := make(map[string]*storage.Container, len(oldDeployment.GetContainers())) + for _, c := range oldDeployment.GetContainers() { + oldContainersByName[c.GetName()] = c } - for i, container := range deployment.GetContainers() { - if container.GetImage().GetId() != "" { - continue - } - oldContainer := oldDeployment.GetContainers()[i] - if oldContainer.GetImage().GetId() == "" { - continue - } - if container.GetImage().GetName().GetFullName() != oldContainer.GetImage().GetName().GetFullName() { - continue + for _, container := range deployment.GetContainers() { + if oldC, found := oldContainersByName[container.GetName()]; found && isMissingImageID(container) { + copyImageID(oldC, container) } - container.Image.Id = oldContainer.GetImage().GetId() } return nil } @@ -357,7 +366,8 @@ func (ds *datastoreImpl) RemoveDeployment(ctx context.Context, clusterID, id str return sac.ErrResourceAccessDenied } // Dedupe the removed deployments. This can happen because Pods have many completion states - // and we may receive multiple Remove calls + // and we may receive multiple Remove calls. Marking up front also stops concurrent UpsertFlows + // from writing flows for the deployment while it is being removed. if ds.deletedDeploymentCache != nil { if ds.deletedDeploymentCache.Contains(id) { return nil @@ -383,22 +393,20 @@ func (ds *datastoreImpl) RemoveDeployment(ctx context.Context, clusterID, id str errorList.AddError(err) } - flowStore, err := ds.networkFlows.GetFlowStore(deleteRelatedCtx, clusterID) - if err != nil { - errorList.AddError(err) - return errorList.ToError() - } - - if err := flowStore.RemoveFlowsForDeployment(deleteRelatedCtx, id); err != nil { - errorList.AddError(err) + // clusterID is nullable in the DB; guard against an empty value to avoid an error from flow store. + if clusterID != "" { + // A missing flow store must not block the delete, else deployments of a deleted cluster + // could never be pruned. + if flowStore, err := ds.networkFlows.GetFlowStore(deleteRelatedCtx, clusterID); err != nil { + errorList.AddError(err) + } else if err := flowStore.RemoveFlowsForDeployment(deleteRelatedCtx, id); err != nil { + errorList.AddError(err) + } } // Delete should be last to ensure that the above is always cleaned up even in the case of crash - err = ds.keyedMutex.DoStatusWithLock(id, func() error { - if err := ds.deploymentStore.Delete(ctx, id); err != nil { - return err - } - return nil + err := ds.keyedMutex.DoStatusWithLock(id, func() error { + return ds.deploymentStore.Delete(ctx, id) }) if err != nil { errorList.AddError(err) @@ -411,8 +419,14 @@ func (ds *datastoreImpl) RemoveDeployment(ctx context.Context, clusterID, id str func (ds *datastoreImpl) GetImagesForDeployment(ctx context.Context, deployment *storage.Deployment) ([]*storage.Image, error) { imageIDs := make([]string, 0, len(deployment.GetContainers())) for _, c := range deployment.GetContainers() { - if c.GetImage().GetId() != "" { - imageIDs = append(imageIDs, c.GetImage().GetId()) + if features.FlattenImageData.Enabled() { + if c.GetImage().GetIdV2() != "" { + imageIDs = append(imageIDs, c.GetImage().GetIdV2()) + } + } else { + if c.GetImage().GetId() != "" { + imageIDs = append(imageIDs, c.GetImage().GetId()) + } } } if features.FlattenImageData.Enabled() { @@ -478,6 +492,11 @@ func (ds *datastoreImpl) GetContainerImageViews(ctx context.Context, q *v1.Query return ds.deploymentStore.GetContainerImageViews(ctx, q) } +func (ds *datastoreImpl) GetV2ImageIDsForDigests(ctx context.Context, digests []string) (map[string][]string, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "Deployment", "GetV2ImageIDsForDigests") + return ds.deploymentStore.GetV2ImageIDsForDigests(ctx, digests) +} + type DeploymentSearchResultConverter struct{} func (c *DeploymentSearchResultConverter) BuildName(result *pkgSearch.Result) string { @@ -498,7 +517,3 @@ func (c *DeploymentSearchResultConverter) BuildLocation(result *pkgSearch.Result func (c *DeploymentSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_DEPLOYMENTS } - -func (c *DeploymentSearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/deployment/datastore/datastore_impl_postgres_test.go b/central/deployment/datastore/datastore_impl_postgres_test.go index 00911075e2361..38adfd4ff41d2 100644 --- a/central/deployment/datastore/datastore_impl_postgres_test.go +++ b/central/deployment/datastore/datastore_impl_postgres_test.go @@ -446,11 +446,121 @@ func TestSelectQueryOnDeployments(t *testing.T) { {2, "daemonset"}, {1, "replicaset"}, } - results, err := postgres.RunSelectRequestForSchema[deploymentCountByType](ctx, testDB.DB, schema.DeploymentsSchema, q) + var results []*deploymentCountByType + err = postgres.RunSelectRequestForSchemaFn[deploymentCountByType](ctx, testDB.DB, schema.DeploymentsSchema, q, func(r *deploymentCountByType) error { + results = append(results, r) + return nil + }) assert.NoError(t, err) assert.ElementsMatch(t, expected, results) } +func TestSearchListDeployments(t *testing.T) { + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(t) + + deploymentDS, err := GetTestPostgresDataStore(t, testDB.DB) + require.NoError(t, err) + + // Create test deployments + cluster1 := testconsts.Cluster1 + cluster2 := testconsts.Cluster2 + + dep1 := &storage.Deployment{ + Id: uuid.NewV4().String(), + Hash: 1234567890, + Name: "deployment-1", + ClusterId: cluster1, + ClusterName: "cluster-1-name", + Namespace: "namespace-1", + NamespaceId: cluster1 + "namespace-1", + } + dep2 := &storage.Deployment{ + Id: uuid.NewV4().String(), + Hash: 9876543210, + Name: "deployment-2", + ClusterId: cluster1, + ClusterName: "cluster-1-name", + Namespace: "namespace-2", + NamespaceId: cluster1 + "namespace-2", + } + dep3 := &storage.Deployment{ + Id: uuid.NewV4().String(), + Hash: 5555555555, + Name: "deployment-3", + ClusterId: cluster2, + ClusterName: "cluster-2-name", + Namespace: "namespace-1", + NamespaceId: cluster2 + "namespace-1", + } + + require.NoError(t, deploymentDS.UpsertDeployment(ctx, dep1)) + require.NoError(t, deploymentDS.UpsertDeployment(ctx, dep2)) + require.NoError(t, deploymentDS.UpsertDeployment(ctx, dep3)) + + testCases := []struct { + name string + query *v1.Query + expectedIDs []string + }{ + { + name: "Empty query returns all deployments", + query: pkgSearch.EmptyQuery(), + expectedIDs: []string{dep1.GetId(), dep2.GetId(), dep3.GetId()}, + }, + { + name: "Filter by deployment ID", + query: pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.DeploymentID, dep1.GetId()).ProtoQuery(), + expectedIDs: []string{dep1.GetId()}, + }, + { + name: "Filter by namespace", + query: pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.Namespace, "namespace-1").ProtoQuery(), + expectedIDs: []string{dep1.GetId(), dep3.GetId()}, + }, + { + name: "Filter by cluster", + query: pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.ClusterID, cluster2).ProtoQuery(), + expectedIDs: []string{dep3.GetId()}, + }, + { + name: "Filter by deployment name", + query: pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.DeploymentName, "deployment-2").ProtoQuery(), + expectedIDs: []string{dep2.GetId()}, + }, + { + name: "Non-matching query returns empty", + query: pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.DeploymentName, "nonexistent").ProtoQuery(), + expectedIDs: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + results, err := deploymentDS.SearchListDeployments(ctx, tc.query) + require.NoError(t, err) + require.Len(t, results, len(tc.expectedIDs)) + + actualIDs := make([]string, 0, len(results)) + for _, result := range results { + actualIDs = append(actualIDs, result.GetId()) + + // Verify ListDeployment fields are populated + assert.NotEmpty(t, result.GetId(), "ID should be populated") + assert.NotEmpty(t, result.GetName(), "Name should be populated") + assert.NotEmpty(t, result.GetCluster(), "Cluster should be populated") + assert.NotEmpty(t, result.GetClusterId(), "ClusterID should be populated") + assert.NotEmpty(t, result.GetNamespace(), "Namespace should be populated") + + // Verify priority is set (business logic from datastore layer) + assert.GreaterOrEqual(t, result.GetPriority(), int64(0), "Priority should be set") + } + + assert.ElementsMatch(t, tc.expectedIDs, actualIDs) + }) + } +} + func TestContainerImagesView(t *testing.T) { testutils.MustUpdateFeature(t, features.FlattenImageData, true) ctx := sac.WithAllAccess(context.Background()) @@ -594,12 +704,14 @@ func TestContainerImagesView(t *testing.T) { type imageInfo struct { digest string clusterIDs []string + imageName *storage.ImageName } responseMap := make(map[string]imageInfo) for _, resp := range responses { responseMap[resp.GetImageID()] = imageInfo{ digest: resp.GetImageDigest(), clusterIDs: resp.GetClusterIDs(), + imageName: resp.GetImageName(), } } @@ -609,6 +721,8 @@ func TestContainerImagesView(t *testing.T) { "sharedImageIDV2_1 should have correct digest") assert.ElementsMatch(t, []string{cluster1, cluster2}, responseMap[sharedImageIDV2_1].clusterIDs, "sharedImageIDV2_1 should be in both cluster1 and cluster2") + assert.Equal(t, "nginx:1.0", responseMap[sharedImageIDV2_1].imageName.GetFullName(), + "sharedImageIDV2_1 should have correct image name") // Verify sharedImageIDV2_2 is only in cluster2 with correct digest assert.Contains(t, responseMap, sharedImageIDV2_2) @@ -616,6 +730,8 @@ func TestContainerImagesView(t *testing.T) { "sharedImageIDV2_2 should have correct digest") assert.ElementsMatch(t, []string{cluster2}, responseMap[sharedImageIDV2_2].clusterIDs, "sharedImageIDV2_2 should be only in cluster2") + assert.Equal(t, "redis:latest", responseMap[sharedImageIDV2_2].imageName.GetFullName(), + "sharedImageIDV2_2 should have correct image name") // Verify uniqueImageIDV2 is only in cluster2 with correct digest assert.Contains(t, responseMap, uniqueImageIDV2) @@ -623,4 +739,6 @@ func TestContainerImagesView(t *testing.T) { "uniqueImageIDV2 should have correct digest") assert.ElementsMatch(t, []string{cluster2}, responseMap[uniqueImageIDV2].clusterIDs, "uniqueImageIDV2 should be only in cluster2") + assert.Equal(t, "postgres:15", responseMap[uniqueImageIDV2].imageName.GetFullName(), + "uniqueImageIDV2 should have correct image name") } diff --git a/central/deployment/datastore/datastore_impl_test.go b/central/deployment/datastore/datastore_impl_test.go index b5dc7ec039828..4440e5631294f 100644 --- a/central/deployment/datastore/datastore_impl_test.go +++ b/central/deployment/datastore/datastore_impl_test.go @@ -2,15 +2,19 @@ package datastore import ( "context" + "errors" "testing" storeMocks "github.com/stackrox/rox/central/deployment/datastore/internal/store/mocks" + flowMocks "github.com/stackrox/rox/central/networkgraph/flow/datastore/mocks" matcherMocks "github.com/stackrox/rox/central/platform/matcher/mocks" + baselineMocks "github.com/stackrox/rox/central/processbaseline/datastore/mocks" "github.com/stackrox/rox/central/ranking" riskMocks "github.com/stackrox/rox/central/risk/datastore/mocks" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/kubernetes" "github.com/stackrox/rox/pkg/process/filter" "github.com/stackrox/rox/pkg/protoassert" @@ -97,6 +101,50 @@ func (suite *DeploymentDataStoreTestSuite) TestInitializeRanker() { suite.Equal(int64(3), deploymentRanker.GetRankForID("3")) } +// Regression: a failed flow store lookup must not block the row delete, else +// deployments of an already-deleted cluster could never be pruned. +func (suite *DeploymentDataStoreTestSuite) TestRemoveDeploymentDeletesRowWhenFlowStoreLookupFails() { + baselines := baselineMocks.NewMockDataStore(suite.mockCtrl) + flows := flowMocks.NewMockClusterDataStore(suite.mockCtrl) + + ds := newDatastoreImpl(suite.storage, nil, nil, baselines, flows, suite.riskStore, nil, suite.filter, + ranking.NewRanker(), ranking.NewRanker(), ranking.NewRanker(), suite.matcher) + + const clusterID = "c1" + const deploymentID = "d1" + + suite.riskStore.EXPECT().RemoveRisk(gomock.Any(), deploymentID, gomock.Any()).Return(nil) + baselines.EXPECT().RemoveProcessBaselinesByDeployment(gomock.Any(), deploymentID).Return(nil) + flows.EXPECT().GetFlowStore(gomock.Any(), clusterID).Return(nil, errors.New("flow store gone")) + // The deployment row must still be deleted despite the flow-store failure. + suite.storage.EXPECT().Delete(gomock.Any(), deploymentID).Return(nil) + + err := ds.RemoveDeployment(suite.ctx, clusterID, deploymentID) + suite.Require().Error(err) + suite.ErrorContains(err, "flow store gone") +} + +// An empty clusterID (e.g. an orphaned deployment whose clusterid column is NULL) has no +// flow partition, so the flow store lookup must be skipped rather than dereferencing a nil +// store, while the row is still deleted. +func (suite *DeploymentDataStoreTestSuite) TestRemoveDeploymentSkipsFlowStoreWhenClusterIDEmpty() { + baselines := baselineMocks.NewMockDataStore(suite.mockCtrl) + flows := flowMocks.NewMockClusterDataStore(suite.mockCtrl) + + ds := newDatastoreImpl(suite.storage, nil, nil, baselines, flows, suite.riskStore, nil, suite.filter, + ranking.NewRanker(), ranking.NewRanker(), ranking.NewRanker(), suite.matcher) + + const deploymentID = "d1" + + suite.riskStore.EXPECT().RemoveRisk(gomock.Any(), deploymentID, gomock.Any()).Return(nil) + baselines.EXPECT().RemoveProcessBaselinesByDeployment(gomock.Any(), deploymentID).Return(nil) + // flows.GetFlowStore must not be called for an empty clusterID; the mock has no + // expectation set, so any call would fail the test. + suite.storage.EXPECT().Delete(gomock.Any(), deploymentID).Return(nil) + + suite.Require().NoError(ds.RemoveDeployment(suite.ctx, "", deploymentID)) +} + func walkMockFunc(deployments []*storage.Deployment) func(_ context.Context, _ *v1.Query, fn func(group *storage.Deployment) error) error { return func(_ context.Context, _ *v1.Query, fn func(deployment *storage.Deployment) error) error { for _, g := range deployments { @@ -123,11 +171,13 @@ func (suite *DeploymentDataStoreTestSuite) TestMergeCronJobs() { dep.Containers = []*storage.Container{ { + Name: "container-a", Image: &storage.ContainerImage{ Id: "abc", }, }, { + Name: "container-b", Image: &storage.ContainerImage{ Id: "def", }, @@ -146,30 +196,51 @@ func (suite *DeploymentDataStoreTestSuite) TestMergeCronJobs() { suite.NoError(ds.mergeCronJobs(ctx, dep)) protoassert.Equal(suite.T(), expectedDep, dep) - // Different numbers of containers for the CronJob so early exit with no changes + // Old deployment has fewer containers (e.g., no init containers yet), but matching containers still merge returnedDep := dep.CloneVT() returnedDep.Containers = returnedDep.GetContainers()[:1] + returnedDep.Containers[0].Image.Id = "abc" suite.storage.EXPECT().Get(ctx, "id").Return(returnedDep, true, nil) suite.NoError(ds.mergeCronJobs(ctx, dep)) + // container-b still has no ID since it wasn't in the old deployment protoassert.Equal(suite.T(), expectedDep, dep) - // Filled in for missing last container, but names do not match - returnedDep.Containers = append(returnedDep.Containers, dep.GetContainers()[1].CloneVT()) - returnedDep.Containers[1].Image.Id = "xyz" - returnedDep.Containers[1].Image.Name = &storage.ImageName{ - FullName: "fullname", + // Old container has matching name but different image full name, so no merge + returnedDep = &storage.Deployment{ + Id: "id", + Type: kubernetes.CronJob, + Containers: []*storage.Container{ + { + Name: "container-a", + Image: &storage.ContainerImage{ + Id: "abc", + }, + }, + { + Name: "container-b", + Image: &storage.ContainerImage{ + Id: "xyz", + Name: &storage.ImageName{ + FullName: "fullname", + }, + }, + }, + }, } suite.storage.EXPECT().Get(ctx, "id").Return(returnedDep, true, nil) suite.NoError(ds.mergeCronJobs(ctx, dep)) protoassert.Equal(suite.T(), expectedDep, dep) - // Fill in missing last container value since names match - dep.Containers[1].Image.Name = returnedDep.GetContainers()[1].GetImage().GetName() - expectedDep.Containers[1].Image.Name = returnedDep.GetContainers()[1].GetImage().GetName() + // Fill in missing last container value since image names match + dep.Containers[1].Image.Name = &storage.ImageName{FullName: "fullname"} + expectedDep.Containers[1].Image.Name = &storage.ImageName{FullName: "fullname"} expectedDep.Containers[1].Image.Id = "xyz" suite.storage.EXPECT().Get(ctx, "id").Return(returnedDep, true, nil) suite.NoError(ds.mergeCronJobs(ctx, dep)) + if features.FlattenImageData.Enabled() { + expectedDep.GetContainers()[1].GetImage().IdV2 = utils.NewImageV2ID(expectedDep.GetContainers()[1].GetImage().GetName(), expectedDep.GetContainers()[1].GetImage().GetId()) + } protoassert.Equal(suite.T(), expectedDep, dep) } diff --git a/central/deployment/datastore/internal/store/mocks/store.go b/central/deployment/datastore/internal/store/mocks/store.go index 000318c40be05..10a0a236b9c4c 100644 --- a/central/deployment/datastore/internal/store/mocks/store.go +++ b/central/deployment/datastore/internal/store/mocks/store.go @@ -73,6 +73,21 @@ func (mr *MockStoreMockRecorder) Delete(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockStore)(nil).Delete), ctx, id) } +// Exists mocks base method. +func (m *MockStore) Exists(ctx context.Context, id string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, id) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockStoreMockRecorder) Exists(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockStore)(nil).Exists), ctx, id) +} + // Get mocks base method. func (m *MockStore) Get(ctx context.Context, id string) (*storage.Deployment, bool, error) { m.ctrl.T.Helper() @@ -172,6 +187,21 @@ func (mr *MockStoreMockRecorder) GetManyListDeployments(ctx any, ids ...any) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetManyListDeployments", reflect.TypeOf((*MockStore)(nil).GetManyListDeployments), varargs...) } +// GetV2ImageIDsForDigests mocks base method. +func (m *MockStore) GetV2ImageIDsForDigests(ctx context.Context, digests []string) (map[string][]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetV2ImageIDsForDigests", ctx, digests) + ret0, _ := ret[0].(map[string][]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetV2ImageIDsForDigests indicates an expected call of GetV2ImageIDsForDigests. +func (mr *MockStoreMockRecorder) GetV2ImageIDsForDigests(ctx, digests any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetV2ImageIDsForDigests", reflect.TypeOf((*MockStore)(nil).GetV2ImageIDsForDigests), ctx, digests) +} + // Search mocks base method. func (m *MockStore) Search(ctx context.Context, q *v1.Query) ([]search.Result, error) { m.ctrl.T.Helper() @@ -187,6 +217,21 @@ func (mr *MockStoreMockRecorder) Search(ctx, q any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockStore)(nil).Search), ctx, q) } +// SearchListDeployments mocks base method. +func (m *MockStore) SearchListDeployments(ctx context.Context, q *v1.Query) ([]*storage.ListDeployment, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchListDeployments", ctx, q) + ret0, _ := ret[0].([]*storage.ListDeployment) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchListDeployments indicates an expected call of SearchListDeployments. +func (mr *MockStoreMockRecorder) SearchListDeployments(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchListDeployments", reflect.TypeOf((*MockStore)(nil).SearchListDeployments), ctx, q) +} + // Upsert mocks base method. func (m *MockStore) Upsert(ctx context.Context, deployment *storage.Deployment) error { m.ctrl.T.Helper() diff --git a/central/deployment/datastore/internal/store/postgres/full_store.go b/central/deployment/datastore/internal/store/postgres/full_store.go index d162412243bde..a5cc1f314be26 100644 --- a/central/deployment/datastore/internal/store/postgres/full_store.go +++ b/central/deployment/datastore/internal/store/postgres/full_store.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/stackrox/rox/central/deployment/datastore/internal/store" - "github.com/stackrox/rox/central/deployment/datastore/internal/store/types" "github.com/stackrox/rox/central/deployment/views" "github.com/stackrox/rox/central/views/common" v1 "github.com/stackrox/rox/generated/api/v1" @@ -14,7 +13,6 @@ import ( "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/postgres" pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" - "github.com/stackrox/rox/pkg/sac/resources" pkgSearch "github.com/stackrox/rox/pkg/search" pgSearch "github.com/stackrox/rox/pkg/search/postgres" "gorm.io/gorm" @@ -47,48 +45,113 @@ type fullStoreImpl struct { // GetListDeployment returns the list deployment of the passed ID. func (f *fullStoreImpl) GetListDeployment(ctx context.Context, id string) (*storage.ListDeployment, bool, error) { - dep, exists, err := f.Get(ctx, id) - if err != nil || !exists { + listDeployments, missingIndices, err := f.GetManyListDeployments(ctx, id) + if err != nil || len(missingIndices) > 0 || len(listDeployments) == 0 { return nil, false, err } - return types.ConvertDeploymentToDeploymentList(dep), true, nil + return listDeployments[0], true, nil } // GetManyListDeployments returns the list deployments as specified by the passed IDs. func (f *fullStoreImpl) GetManyListDeployments(ctx context.Context, ids ...string) ([]*storage.ListDeployment, []int, error) { - deployments, missing, err := f.GetMany(ctx, ids) + if len(ids) == 0 { + return nil, nil, nil + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + // Build query filtering by deployment IDs + query := pkgSearch.NewQueryBuilder(). + AddExactMatches(pkgSearch.DeploymentID, ids...). + ProtoQuery() + + // Set selects to only the columns needed for ListDeployment + query.Selects = views.ListDeploymentViewSelects() + + // Fetch results using the search framework + viewMap := make(map[string]*views.ListDeploymentView) + err := pgSearch.RunSelectRequestForSchemaFn(queryCtx, f.db, pkgSchema.DeploymentsSchema, query, + func(view *views.ListDeploymentView) error { + viewMap[view.ID] = view + return nil + }) if err != nil { return nil, nil, err } - listDeployments := make([]*storage.ListDeployment, 0, len(deployments)) - for _, d := range deployments { - listDeployments = append(listDeployments, types.ConvertDeploymentToDeploymentList(d)) + + // It is important that the results are populated in the same order as the input IDs + // slice, since some calling code relies on that to maintain order. + listDeployments := make([]*storage.ListDeployment, 0, len(ids)) + var missingIndices []int + for i, id := range ids { + if view, ok := viewMap[id]; ok { + listDeployments = append(listDeployments, view.ToListDeployment()) + } else { + missingIndices = append(missingIndices, i) + } } - return listDeployments, missing, nil + + return listDeployments, missingIndices, nil } -func (f *fullStoreImpl) GetContainerImageViews(ctx context.Context, q *v1.Query) ([]*views.ContainerImageView, error) { - if err := common.ValidateQuery(q); err != nil { - return nil, err +// SearchListDeployments returns list deployments matching the provided query. +func (f *fullStoreImpl) SearchListDeployments(ctx context.Context, q *v1.Query) ([]*storage.ListDeployment, error) { + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + // Clone the query to avoid mutating the original + query := q.CloneVT() + if query == nil { + query = pkgSearch.EmptyQuery() } - q, err := common.WithSACFilter(ctx, resources.Deployment, q) + + // Set selects to only the columns needed for ListDeployment + query.Selects = views.ListDeploymentViewSelects() + + var listDeployments []*storage.ListDeployment + err := pgSearch.RunSelectRequestForSchemaFn(queryCtx, f.db, pkgSchema.DeploymentsSchema, query, + func(view *views.ListDeploymentView) error { + listDeployments = append(listDeployments, view.ToListDeployment()) + return nil + }) if err != nil { return nil, err } - q.Selects = []*v1.QuerySelect{ + + return listDeployments, nil +} + +func (f *fullStoreImpl) GetContainerImageViews(ctx context.Context, q *v1.Query) ([]*views.ContainerImageView, error) { + if err := common.ValidateQuery(q); err != nil { + return nil, err + } + cloned := q.CloneVT() + cloned.Selects = []*v1.QuerySelect{ pkgSearch.NewQuerySelect(pkgSearch.ImageID).Proto(), pkgSearch.NewQuerySelect(pkgSearch.ImageSHA).Proto(), pkgSearch.NewQuerySelect(pkgSearch.ClusterID).Distinct().Proto(), + pkgSearch.NewQuerySelect(pkgSearch.ImageRegistry).Proto(), + pkgSearch.NewQuerySelect(pkgSearch.ImageRemote).Proto(), + pkgSearch.NewQuerySelect(pkgSearch.ImageTag).Proto(), + pkgSearch.NewQuerySelect(pkgSearch.ImageName).Proto(), } - q.GroupBy = &v1.QueryGroupBy{ - Fields: []string{pkgSearch.ImageID.String(), pkgSearch.ImageSHA.String()}, + cloned.GroupBy = &v1.QueryGroupBy{ + Fields: []string{ + pkgSearch.ImageID.String(), + pkgSearch.ImageSHA.String(), + pkgSearch.ImageRegistry.String(), + pkgSearch.ImageRemote.String(), + pkgSearch.ImageTag.String(), + pkgSearch.ImageName.String(), + }, } queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() var results []*views.ContainerImageView - err = pgSearch.RunSelectRequestForSchemaFn(queryCtx, f.db, pkgSchema.DeploymentsSchema, q, func(response *views.ContainerImageView) error { + err := pgSearch.RunSelectRequestForSchemaFn(queryCtx, f.db, pkgSchema.DeploymentsSchema, cloned, func(response *views.ContainerImageView) error { results = append(results, response) return nil }) @@ -99,6 +162,37 @@ func (f *fullStoreImpl) GetContainerImageViews(ctx context.Context, q *v1.Query) return results, nil } +type v2ImageIDsByDigestView struct { + ImageIDs []string `db:"image_id"` + ImageDigest string `db:"image_sha"` +} + +func (f *fullStoreImpl) GetV2ImageIDsForDigests(ctx context.Context, digests []string) (map[string][]string, error) { + if len(digests) == 0 { + return nil, nil + } + q := pkgSearch.NewQueryBuilder(). + AddExactMatches(pkgSearch.ImageSHA, digests...). + ProtoQuery() + q.Selects = []*v1.QuerySelect{ + pkgSearch.NewQuerySelect(pkgSearch.ImageID).Distinct().Proto(), + pkgSearch.NewQuerySelect(pkgSearch.ImageSHA).Proto(), + } + q.GroupBy = &v1.QueryGroupBy{ + Fields: []string{pkgSearch.ImageSHA.String()}, + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + result := make(map[string][]string, len(digests)) + err := pgSearch.RunSelectRequestForSchemaFn(queryCtx, f.db, pkgSchema.DeploymentsSchema, q, func(row *v2ImageIDsByDigestView) error { + result[row.ImageDigest] = row.ImageIDs + return nil + }) + return result, err +} + // NewFullTestStore is used for testing. func NewFullTestStore(ctx context.Context, _ testing.TB, store Store, db postgres.DB, gormDB *gorm.DB) store.Store { pkgSchema.ApplySchemaForTable(ctx, gormDB, baseTable) diff --git a/central/deployment/datastore/internal/store/postgres/full_store_test.go b/central/deployment/datastore/internal/store/postgres/full_store_test.go new file mode 100644 index 0000000000000..5f794b14ff714 --- /dev/null +++ b/central/deployment/datastore/internal/store/postgres/full_store_test.go @@ -0,0 +1,250 @@ +//go:build sql_integration + +package postgres + +import ( + "context" + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/images/types" + "github.com/stackrox/rox/pkg/images/utils" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/testutils" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestGetListDeployment tests the GetListDeployment method from the full store. +func TestGetListDeployment(t *testing.T) { + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(t) + fullStore := NewFullStore(testDB.DB) + + dep := &storage.Deployment{ + Id: uuid.NewV4().String(), + Hash: 1234567890, + Name: "test-deployment", + ClusterId: "cluster-id-1", + ClusterName: "test-cluster", + Namespace: "test-namespace", + NamespaceId: "cluster-id-1test-namespace", + } + require.NoError(t, testutils.FullInit(dep, testutils.SimpleInitializer(), testutils.JSONFieldsFilter)) + require.NoError(t, fullStore.Upsert(ctx, dep)) + + t.Run("Get existing deployment", func(t *testing.T) { + result, found, err := fullStore.GetListDeployment(ctx, dep.GetId()) + assert.NoError(t, err) + assert.True(t, found) + assert.NotNil(t, result) + assert.Equal(t, dep.GetId(), result.GetId()) + assert.Equal(t, dep.GetHash(), result.GetHash()) + assert.Equal(t, dep.GetName(), result.GetName()) + assert.Equal(t, dep.GetClusterName(), result.GetCluster()) + assert.Equal(t, dep.GetClusterId(), result.GetClusterId()) + assert.Equal(t, dep.GetNamespace(), result.GetNamespace()) + }) + + t.Run("Get non-existent deployment", func(t *testing.T) { + nonExistentID := uuid.NewV4().String() + result, found, err := fullStore.GetListDeployment(ctx, nonExistentID) + assert.NoError(t, err) + assert.False(t, found) + assert.Nil(t, result) + }) +} + +// TestGetManyListDeployments tests the GetManyListDeployments method from the full store. +func TestGetManyListDeployments(t *testing.T) { + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(t) + fullStore := NewFullStore(testDB.DB) + + // Create test deployments + dep1 := &storage.Deployment{ + Id: uuid.NewV4().String(), + Hash: 1111111111, + Name: "deployment-1", + ClusterId: "cluster-1", + ClusterName: "cluster-1-name", + Namespace: "namespace-1", + NamespaceId: "cluster-1namespace-1", + } + dep2 := &storage.Deployment{ + Id: uuid.NewV4().String(), + Hash: 2222222222, + Name: "deployment-2", + ClusterId: "cluster-1", + ClusterName: "cluster-1-name", + Namespace: "namespace-2", + NamespaceId: "cluster-1namespace-2", + } + dep3 := &storage.Deployment{ + Id: uuid.NewV4().String(), + Hash: 3333333333, + Name: "deployment-3", + ClusterId: "cluster-2", + ClusterName: "cluster-2-name", + Namespace: "namespace-1", + NamespaceId: "cluster-2namespace-1", + } + + require.NoError(t, testutils.FullInit(dep1, testutils.SimpleInitializer(), testutils.JSONFieldsFilter)) + require.NoError(t, testutils.FullInit(dep2, testutils.SimpleInitializer(), testutils.JSONFieldsFilter)) + require.NoError(t, testutils.FullInit(dep3, testutils.SimpleInitializer(), testutils.JSONFieldsFilter)) + + require.NoError(t, fullStore.Upsert(ctx, dep1)) + require.NoError(t, fullStore.Upsert(ctx, dep2)) + require.NoError(t, fullStore.Upsert(ctx, dep3)) + + t.Run("Get all deployments with valid IDs", func(t *testing.T) { + ids := []string{dep1.GetId(), dep2.GetId(), dep3.GetId()} + results, missingIndices, err := fullStore.GetManyListDeployments(ctx, ids...) + assert.NoError(t, err) + assert.Empty(t, missingIndices) + assert.Len(t, results, 3) + + // Results should be in same order as input IDs + assert.Equal(t, dep1.GetId(), results[0].GetId()) + assert.Equal(t, dep2.GetId(), results[1].GetId()) + assert.Equal(t, dep3.GetId(), results[2].GetId()) + }) + + t.Run("Get deployments with some missing IDs", func(t *testing.T) { + nonExistentID1 := uuid.NewV4().String() + nonExistentID2 := uuid.NewV4().String() + + ids := []string{dep1.GetId(), nonExistentID1, dep2.GetId(), nonExistentID2, dep3.GetId()} + results, missingIndices, err := fullStore.GetManyListDeployments(ctx, ids...) + assert.NoError(t, err) + assert.Len(t, results, 3) + assert.Equal(t, []int{1, 3}, missingIndices) + + // Results should preserve order + assert.Equal(t, dep1.GetId(), results[0].GetId()) + assert.Equal(t, dep2.GetId(), results[1].GetId()) + assert.Equal(t, dep3.GetId(), results[2].GetId()) + }) + + t.Run("Get deployments with all missing IDs", func(t *testing.T) { + nonExistentID1 := uuid.NewV4().String() + nonExistentID2 := uuid.NewV4().String() + + ids := []string{nonExistentID1, nonExistentID2} + results, missingIndices, err := fullStore.GetManyListDeployments(ctx, ids...) + assert.NoError(t, err) + assert.Empty(t, results) + assert.Equal(t, []int{0, 1}, missingIndices) + }) + + t.Run("Order preservation with different request order", func(t *testing.T) { + ids := []string{dep3.GetId(), dep1.GetId(), dep2.GetId()} + results, missingIndices, err := fullStore.GetManyListDeployments(ctx, ids...) + assert.NoError(t, err) + assert.Empty(t, missingIndices) + assert.Len(t, results, 3) + + // Results should match requested order + assert.Equal(t, dep3.GetId(), results[0].GetId()) + assert.Equal(t, dep1.GetId(), results[1].GetId()) + assert.Equal(t, dep2.GetId(), results[2].GetId()) + }) + + t.Run("Empty ID list", func(t *testing.T) { + results, missingIndices, err := fullStore.GetManyListDeployments(ctx) + assert.NoError(t, err) + assert.Nil(t, results) + assert.Nil(t, missingIndices) + }) + + t.Run("Verify ListDeployment fields are populated", func(t *testing.T) { + ids := []string{dep1.GetId()} + results, _, err := fullStore.GetManyListDeployments(ctx, ids...) + assert.NoError(t, err) + assert.Len(t, results, 1) + + result := results[0] + assert.Equal(t, dep1.GetId(), result.GetId()) + assert.Equal(t, dep1.GetHash(), result.GetHash()) + assert.Equal(t, dep1.GetName(), result.GetName()) + assert.Equal(t, dep1.GetClusterName(), result.GetCluster()) + assert.Equal(t, dep1.GetClusterId(), result.GetClusterId()) + assert.Equal(t, dep1.GetNamespace(), result.GetNamespace()) + }) +} + +func TestGetV2ImageIDsForDigests(t *testing.T) { + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(t) + fullStore := NewFullStore(testDB.DB) + + digest1 := types.NewDigest("digest-1").Digest() + digest2 := types.NewDigest("digest-2").Digest() + digest3 := types.NewDigest("digest-3").Digest() + + name1a := &storage.ImageName{FullName: "registry.io/repo/image1a:latest"} + name1b := &storage.ImageName{FullName: "registry.io/repo/image1b:latest"} + name2 := &storage.ImageName{FullName: "registry.io/repo/image2:latest"} + + v2ID1a := utils.NewImageV2ID(name1a, digest1) + v2ID1b := utils.NewImageV2ID(name1b, digest1) + v2ID2 := utils.NewImageV2ID(name2, digest2) + + // Two deployments reference digest1 with different V2 IDs (different image names). + dep1 := &storage.Deployment{ + Id: uuid.NewV4().String(), Name: "dep-1", Namespace: "ns", + ClusterId: "c1", ClusterName: "c1", NamespaceId: "c1ns", + Containers: []*storage.Container{{ + Image: &storage.ContainerImage{Id: digest1, IdV2: v2ID1a, Name: name1a}, + }}, + } + dep2 := &storage.Deployment{ + Id: uuid.NewV4().String(), Name: "dep-2", Namespace: "ns", + ClusterId: "c1", ClusterName: "c1", NamespaceId: "c1ns", + Containers: []*storage.Container{{ + Image: &storage.ContainerImage{Id: digest1, IdV2: v2ID1b, Name: name1b}, + }}, + } + // One deployment references digest2. + dep3 := &storage.Deployment{ + Id: uuid.NewV4().String(), Name: "dep-3", Namespace: "ns", + ClusterId: "c1", ClusterName: "c1", NamespaceId: "c1ns", + Containers: []*storage.Container{{ + Image: &storage.ContainerImage{Id: digest2, IdV2: v2ID2, Name: name2}, + }}, + } + + require.NoError(t, fullStore.Upsert(ctx, dep1)) + require.NoError(t, fullStore.Upsert(ctx, dep2)) + require.NoError(t, fullStore.Upsert(ctx, dep3)) + + t.Run("multiple digests with multiple V2 IDs", func(t *testing.T) { + result, err := fullStore.GetV2ImageIDsForDigests(ctx, []string{digest1, digest2}) + assert.NoError(t, err) + assert.Len(t, result, 2) + assert.ElementsMatch(t, []string{v2ID1a, v2ID1b}, result[digest1]) + assert.ElementsMatch(t, []string{v2ID2}, result[digest2]) + }) + + t.Run("digest not in any deployment", func(t *testing.T) { + result, err := fullStore.GetV2ImageIDsForDigests(ctx, []string{digest3}) + assert.NoError(t, err) + assert.Empty(t, result) + }) + + t.Run("empty input", func(t *testing.T) { + result, err := fullStore.GetV2ImageIDsForDigests(ctx, nil) + assert.NoError(t, err) + assert.Nil(t, result) + }) + + t.Run("mix of existing and non-existing digests", func(t *testing.T) { + result, err := fullStore.GetV2ImageIDsForDigests(ctx, []string{digest1, digest3}) + assert.NoError(t, err) + assert.Len(t, result, 1) + assert.ElementsMatch(t, []string{v2ID1a, v2ID1b}, result[digest1]) + }) +} diff --git a/central/deployment/datastore/internal/store/postgres/store.go b/central/deployment/datastore/internal/store/postgres/store.go index 18a1aaf676c98..c90082b0168c9 100644 --- a/central/deployment/datastore/internal/store/postgres/store.go +++ b/central/deployment/datastore/internal/store/postgres/store.go @@ -133,6 +133,7 @@ func insertIntoDeployments(batch *pgx.Batch, obj *storage.Deployment) error { // parent primary keys start pgutils.NilOrUUID(obj.GetId()), obj.GetName(), + obj.GetHash(), obj.GetType(), obj.GetNamespace(), pgutils.NilOrUUID(obj.GetNamespaceId()), @@ -152,7 +153,7 @@ func insertIntoDeployments(batch *pgx.Batch, obj *storage.Deployment) error { serialized, } - finalStr := "INSERT INTO deployments (Id, Name, Type, Namespace, NamespaceId, OrchestratorComponent, Labels, PodLabels, Created, ClusterId, ClusterName, Annotations, Priority, ImagePullSecrets, ServiceAccount, ServiceAccountPermissionLevel, RiskScore, PlatformComponent, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, Name = EXCLUDED.Name, Type = EXCLUDED.Type, Namespace = EXCLUDED.Namespace, NamespaceId = EXCLUDED.NamespaceId, OrchestratorComponent = EXCLUDED.OrchestratorComponent, Labels = EXCLUDED.Labels, PodLabels = EXCLUDED.PodLabels, Created = EXCLUDED.Created, ClusterId = EXCLUDED.ClusterId, ClusterName = EXCLUDED.ClusterName, Annotations = EXCLUDED.Annotations, Priority = EXCLUDED.Priority, ImagePullSecrets = EXCLUDED.ImagePullSecrets, ServiceAccount = EXCLUDED.ServiceAccount, ServiceAccountPermissionLevel = EXCLUDED.ServiceAccountPermissionLevel, RiskScore = EXCLUDED.RiskScore, PlatformComponent = EXCLUDED.PlatformComponent, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO deployments (Id, Name, Hash, Type, Namespace, NamespaceId, OrchestratorComponent, Labels, PodLabels, Created, ClusterId, ClusterName, Annotations, Priority, ImagePullSecrets, ServiceAccount, ServiceAccountPermissionLevel, RiskScore, PlatformComponent, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, Name = EXCLUDED.Name, Hash = EXCLUDED.Hash, Type = EXCLUDED.Type, Namespace = EXCLUDED.Namespace, NamespaceId = EXCLUDED.NamespaceId, OrchestratorComponent = EXCLUDED.OrchestratorComponent, Labels = EXCLUDED.Labels, PodLabels = EXCLUDED.PodLabels, Created = EXCLUDED.Created, ClusterId = EXCLUDED.ClusterId, ClusterName = EXCLUDED.ClusterName, Annotations = EXCLUDED.Annotations, Priority = EXCLUDED.Priority, ImagePullSecrets = EXCLUDED.ImagePullSecrets, ServiceAccount = EXCLUDED.ServiceAccount, ServiceAccountPermissionLevel = EXCLUDED.ServiceAccountPermissionLevel, RiskScore = EXCLUDED.RiskScore, PlatformComponent = EXCLUDED.PlatformComponent, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) var query string @@ -196,9 +197,10 @@ func insertIntoDeploymentsContainers(batch *pgx.Batch, obj *storage.Container, d obj.GetResources().GetCpuCoresLimit(), obj.GetResources().GetMemoryMbRequest(), obj.GetResources().GetMemoryMbLimit(), + obj.GetType(), } - finalStr := "INSERT INTO deployments_containers (deployments_Id, idx, Image_Id, Image_Name_Registry, Image_Name_Remote, Image_Name_Tag, Image_Name_FullName, Image_IdV2, SecurityContext_Privileged, SecurityContext_DropCapabilities, SecurityContext_AddCapabilities, SecurityContext_ReadOnlyRootFilesystem, Resources_CpuCoresRequest, Resources_CpuCoresLimit, Resources_MemoryMbRequest, Resources_MemoryMbLimit) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) ON CONFLICT(deployments_Id, idx) DO UPDATE SET deployments_Id = EXCLUDED.deployments_Id, idx = EXCLUDED.idx, Image_Id = EXCLUDED.Image_Id, Image_Name_Registry = EXCLUDED.Image_Name_Registry, Image_Name_Remote = EXCLUDED.Image_Name_Remote, Image_Name_Tag = EXCLUDED.Image_Name_Tag, Image_Name_FullName = EXCLUDED.Image_Name_FullName, Image_IdV2 = EXCLUDED.Image_IdV2, SecurityContext_Privileged = EXCLUDED.SecurityContext_Privileged, SecurityContext_DropCapabilities = EXCLUDED.SecurityContext_DropCapabilities, SecurityContext_AddCapabilities = EXCLUDED.SecurityContext_AddCapabilities, SecurityContext_ReadOnlyRootFilesystem = EXCLUDED.SecurityContext_ReadOnlyRootFilesystem, Resources_CpuCoresRequest = EXCLUDED.Resources_CpuCoresRequest, Resources_CpuCoresLimit = EXCLUDED.Resources_CpuCoresLimit, Resources_MemoryMbRequest = EXCLUDED.Resources_MemoryMbRequest, Resources_MemoryMbLimit = EXCLUDED.Resources_MemoryMbLimit" + finalStr := "INSERT INTO deployments_containers (deployments_Id, idx, Image_Id, Image_Name_Registry, Image_Name_Remote, Image_Name_Tag, Image_Name_FullName, Image_IdV2, SecurityContext_Privileged, SecurityContext_DropCapabilities, SecurityContext_AddCapabilities, SecurityContext_ReadOnlyRootFilesystem, Resources_CpuCoresRequest, Resources_CpuCoresLimit, Resources_MemoryMbRequest, Resources_MemoryMbLimit, Type) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) ON CONFLICT(deployments_Id, idx) DO UPDATE SET deployments_Id = EXCLUDED.deployments_Id, idx = EXCLUDED.idx, Image_Id = EXCLUDED.Image_Id, Image_Name_Registry = EXCLUDED.Image_Name_Registry, Image_Name_Remote = EXCLUDED.Image_Name_Remote, Image_Name_Tag = EXCLUDED.Image_Name_Tag, Image_Name_FullName = EXCLUDED.Image_Name_FullName, Image_IdV2 = EXCLUDED.Image_IdV2, SecurityContext_Privileged = EXCLUDED.SecurityContext_Privileged, SecurityContext_DropCapabilities = EXCLUDED.SecurityContext_DropCapabilities, SecurityContext_AddCapabilities = EXCLUDED.SecurityContext_AddCapabilities, SecurityContext_ReadOnlyRootFilesystem = EXCLUDED.SecurityContext_ReadOnlyRootFilesystem, Resources_CpuCoresRequest = EXCLUDED.Resources_CpuCoresRequest, Resources_CpuCoresLimit = EXCLUDED.Resources_CpuCoresLimit, Resources_MemoryMbRequest = EXCLUDED.Resources_MemoryMbRequest, Resources_MemoryMbLimit = EXCLUDED.Resources_MemoryMbLimit, Type = EXCLUDED.Type" batch.Queue(finalStr, values...) var query string @@ -336,6 +338,7 @@ func insertIntoDeploymentsPortsExposureInfos(batch *pgx.Batch, obj *storage.Port var copyColsDeployments = []string{ "id", "name", + "hash", "type", "namespace", "namespaceid", @@ -388,6 +391,7 @@ func copyFromDeployments(ctx context.Context, s pgSearch.Deleter, tx *postgres.T return []interface{}{ pgutils.NilOrUUID(obj.GetId()), obj.GetName(), + obj.GetHash(), obj.GetType(), obj.GetNamespace(), pgutils.NilOrUUID(obj.GetNamespaceId()), @@ -441,6 +445,7 @@ var copyColsDeploymentsContainers = []string{ "resources_cpucoreslimit", "resources_memorymbrequest", "resources_memorymblimit", + "type", } func copyFromDeploymentsContainers(ctx context.Context, s pgSearch.Deleter, tx *postgres.Tx, deploymentID string, objs ...*storage.Container) error { @@ -473,6 +478,7 @@ func copyFromDeploymentsContainers(ctx context.Context, s pgSearch.Deleter, tx * obj.GetResources().GetCpuCoresLimit(), obj.GetResources().GetMemoryMbRequest(), obj.GetResources().GetMemoryMbLimit(), + obj.GetType(), }, nil }) diff --git a/central/deployment/datastore/internal/store/store.go b/central/deployment/datastore/internal/store/store.go index e22d3e7ee559f..b2f155aa1cf6e 100644 --- a/central/deployment/datastore/internal/store/store.go +++ b/central/deployment/datastore/internal/store/store.go @@ -15,8 +15,10 @@ import ( type Store interface { GetListDeployment(ctx context.Context, id string) (*storage.ListDeployment, bool, error) GetManyListDeployments(ctx context.Context, ids ...string) ([]*storage.ListDeployment, []int, error) + SearchListDeployments(ctx context.Context, q *v1.Query) ([]*storage.ListDeployment, error) Get(ctx context.Context, id string) (*storage.Deployment, bool, error) + Exists(ctx context.Context, id string) (bool, error) GetMany(ctx context.Context, ids []string) ([]*storage.Deployment, []int, error) Walk(ctx context.Context, fn func(deployment *storage.Deployment) error) error WalkByQuery(ctx context.Context, query *v1.Query, fn func(deployment *storage.Deployment) error) error @@ -29,4 +31,6 @@ type Store interface { GetIDs(ctx context.Context) ([]string, error) GetContainerImageViews(ctx context.Context, q *v1.Query) ([]*views.ContainerImageView, error) + + GetV2ImageIDsForDigests(ctx context.Context, digests []string) (map[string][]string, error) } diff --git a/central/deployment/datastore/mocks/datastore.go b/central/deployment/datastore/mocks/datastore.go index bcf147d1cfa52..206d20c2f17fd 100644 --- a/central/deployment/datastore/mocks/datastore.go +++ b/central/deployment/datastore/mocks/datastore.go @@ -74,6 +74,21 @@ func (mr *MockDataStoreMockRecorder) CountDeployments(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountDeployments", reflect.TypeOf((*MockDataStore)(nil).CountDeployments), ctx) } +// DeploymentExists mocks base method. +func (m *MockDataStore) DeploymentExists(ctx context.Context, id string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeploymentExists", ctx, id) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeploymentExists indicates an expected call of DeploymentExists. +func (mr *MockDataStoreMockRecorder) DeploymentExists(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeploymentExists", reflect.TypeOf((*MockDataStore)(nil).DeploymentExists), ctx, id) +} + // GetContainerImageViews mocks base method. func (m *MockDataStore) GetContainerImageViews(ctx context.Context, q *v1.Query) ([]*views.ContainerImageView, error) { m.ctrl.T.Helper() @@ -150,6 +165,21 @@ func (mr *MockDataStoreMockRecorder) GetImagesForDeployment(ctx, deployment any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetImagesForDeployment", reflect.TypeOf((*MockDataStore)(nil).GetImagesForDeployment), ctx, deployment) } +// GetV2ImageIDsForDigests mocks base method. +func (m *MockDataStore) GetV2ImageIDsForDigests(ctx context.Context, digests []string) (map[string][]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetV2ImageIDsForDigests", ctx, digests) + ret0, _ := ret[0].(map[string][]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetV2ImageIDsForDigests indicates an expected call of GetV2ImageIDsForDigests. +func (mr *MockDataStoreMockRecorder) GetV2ImageIDsForDigests(ctx, digests any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetV2ImageIDsForDigests", reflect.TypeOf((*MockDataStore)(nil).GetV2ImageIDsForDigests), ctx, digests) +} + // ListDeployment mocks base method. func (m *MockDataStore) ListDeployment(ctx context.Context, id string) (*storage.ListDeployment, bool, error) { m.ctrl.T.Helper() diff --git a/central/deployment/datastore/search_comparison_test.go b/central/deployment/datastore/search_comparison_test.go index 9bbb217f69386..c0cd9e1b7a791 100644 --- a/central/deployment/datastore/search_comparison_test.go +++ b/central/deployment/datastore/search_comparison_test.go @@ -8,8 +8,10 @@ import ( "testing" imageDataStore "github.com/stackrox/rox/central/image/datastore" + imageV2DataStore "github.com/stackrox/rox/central/imagev2/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/postgres/pgtest" pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" @@ -34,6 +36,7 @@ type SearchComparisonTestSuite struct { testDB *pgtest.TestPostgres imageDatastore imageDataStore.DataStore + imageV2Datastore imageV2DataStore.DataStore deploymentDatastore DataStore optionsMap search.OptionsMap } @@ -45,11 +48,19 @@ func (s *SearchComparisonTestSuite) SetupSuite() { s.Require().NoError(err) s.deploymentDatastore = deploymentDS - imageDS := imageDataStore.GetTestPostgresDataStore(s.T(), s.testDB.DB) - s.imageDatastore = imageDS - s.optionsMap = pkgSchema.ImagesSchema.OptionsMap + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + imageV2DS := imageV2DataStore.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.imageV2Datastore = imageV2DS + s.optionsMap = pkgSchema.ImagesV2Schema.OptionsMap + } else { + imageDS := imageDataStore.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.imageDatastore = imageDS + s.optionsMap = pkgSchema.ImagesSchema.OptionsMap + } } +// TODO(ROX-30117): Remove when FlattenImageData feature flag is removed. func compareResults(t *testing.T, matches bool, predResult *search.Result, searchResults []search.Result) { assert.Equal(t, matches, len(searchResults) != 0) imageKeyMap := map[string]string{"image.scan.components.vulns.cve": "imagecvev2.cve_base_info.cve", "image.scan.components.vulns.cvss": "imagecvev2.cvss"} @@ -70,7 +81,36 @@ func compareResults(t *testing.T, matches bool, predResult *search.Result, searc } } +func compareResultsV2(t *testing.T, matches bool, predResult *search.Result, searchResults []search.Result) { + assert.Equal(t, matches, len(searchResults) != 0) + imageKeyMap := map[string]string{"imagev2.scan.components.vulns.cve": "imagecvev2.cve_base_info.cve", "imagev2.scan.components.vulns.cvss": "imagecvev2.cvss"} + + if matches && len(searchResults) > 0 { + for k := range predResult.Matches { + slices.Sort(predResult.Matches[k]) + newImageKey, ok := imageKeyMap[k] + // If the key exists + if ok { + slices.Sort(searchResults[0].Matches[newImageKey]) + assert.Equal(t, predResult.Matches[k], searchResults[0].Matches[newImageKey]) + } else { + slices.Sort(searchResults[0].Matches[k]) + assert.Equal(t, predResult.Matches[k], searchResults[0].Matches[k]) + } + } + } +} + func (s *SearchComparisonTestSuite) TestImageSearchResults() { + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + s.testImageSearchResultsV2() + } else { + s.testImageSearchResultsV1() + } +} + +func (s *SearchComparisonTestSuite) testImageSearchResultsV1() { cases := []struct { image *storage.Image query *v1.Query @@ -118,10 +158,10 @@ func (s *SearchComparisonTestSuite) TestImageSearchResults() { factory2 := factory.ForCustomOptionsMap(s.optionsMap) for _, c := range cases { s.T().Run("test", func(t *testing.T) { - predicate, err := factory2.GeneratePredicate(c.query) + pred, err := factory2.GeneratePredicate(c.query) require.NoError(t, err) - predResult, matches := predicate.Evaluate(c.image) + predResult, matches := pred.Evaluate(c.image) require.NoError(t, s.imageDatastore.UpsertImage(ctx, c.image)) searchResults, err := s.imageDatastore.Search(ctx, c.query) @@ -132,6 +172,68 @@ func (s *SearchComparisonTestSuite) TestImageSearchResults() { } } +func (s *SearchComparisonTestSuite) testImageSearchResultsV2() { + cases := []struct { + image *storage.ImageV2 + query *v1.Query + expectedResult *search.Result + }{ + { + image: fixtures.GetImageV2(), + query: search.NewQueryBuilder().AddStringsHighlighted(search.ImageTag, "latest").ProtoQuery(), + expectedResult: &search.Result{ + ID: "test", + Matches: map[string][]string{"imagecvev2.cve_base_info.cve": {"CVE-2014-6200", "CVE-2014-6201", "CVE-2014-6202", "CVE-2014-6203", "CVE-2014-6204"}, + "imagecvev2.cvss": {"5", "5", "5", "5", "5"}, + }, + }, + }, + { + image: fixtures.GetImageV2WithUniqueComponents(50), + query: search.NewQueryBuilder().AddLinkedFieldsHighlighted( + []search.FieldLabel{search.CVSS, search.CVE}, + []string{">=5", search.WildcardString}). + ProtoQuery(), + expectedResult: &search.Result{ + ID: "test", + Matches: map[string][]string{"imagecvev2.cve_base_info.cve": {"CVE-2014-6200", "CVE-2014-6201", "CVE-2014-6202", "CVE-2014-6203", "CVE-2014-6204"}, + "imagecvev2.cvss": {"5", "5", "5", "5", "5"}, + }, + }, + }, + { + image: fixtures.GetImageV2WithUniqueComponents(50), + query: search.NewQueryBuilder().AddLinkedFieldsHighlighted( + []search.FieldLabel{search.CVSS, search.CVE}, + []string{">2", "CVE-2014-620"}). + ProtoQuery(), + expectedResult: &search.Result{ + ID: "test", + Matches: map[string][]string{"imagecvev2.cve_base_info.cve": {"CVE-2014-6200", "CVE-2014-6201", "CVE-2014-6202", "CVE-2014-6203", "CVE-2014-6204"}, + "imagecvev2.cvss": {"5", "5", "5", "5", "5"}, + }, + }, + }, + } + + factory := predicate.NewFactory("imagev2", (*storage.ImageV2)(nil)) + factory2 := factory.ForCustomOptionsMap(s.optionsMap) + for _, c := range cases { + s.T().Run("test", func(t *testing.T) { + pred, err := factory2.GeneratePredicate(c.query) + require.NoError(t, err) + + predResult, matches := pred.Evaluate(c.image) + + require.NoError(t, s.imageV2Datastore.UpsertImage(ctx, c.image)) + searchResults, err := s.imageV2Datastore.Search(ctx, c.query) + require.NoError(t, err) + + compareResultsV2(t, matches, predResult, searchResults) + }) + } +} + func (s *SearchComparisonTestSuite) TestDeploymentSearchResults() { cases := []struct { deployment *storage.Deployment diff --git a/central/deployment/datastore/test/datastore_sac_test.go b/central/deployment/datastore/test/datastore_sac_test.go index d294948085719..c29bb67c667dc 100644 --- a/central/deployment/datastore/test/datastore_sac_test.go +++ b/central/deployment/datastore/test/datastore_sac_test.go @@ -4,15 +4,19 @@ package datastore import ( "context" + "slices" "testing" dDS "github.com/stackrox/rox/central/deployment/datastore" deploymentTypes "github.com/stackrox/rox/central/deployment/datastore/internal/store/types" imageDS "github.com/stackrox/rox/central/image/datastore" + imageV2DS "github.com/stackrox/rox/central/imagev2/datastore" nsDS "github.com/stackrox/rox/central/namespace/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/postgres/schema" @@ -48,6 +52,7 @@ type deploymentDatastoreSACSuite struct { datastore dDS.DataStore namespaceStore nsDS.DataStore imageStore imageDS.DataStore + imageV2Store imageV2DS.DataStore testContexts map[string]context.Context testContextsWithImageAccess map[string]context.Context @@ -70,6 +75,7 @@ func (s *deploymentDatastoreSACSuite) SetupSuite() { s.namespaceStore, err = nsDS.GetTestPostgresDataStore(s.T(), s.pool) s.Require().NoError(err) s.imageStore = imageDS.GetTestPostgresDataStore(s.T(), s.pool) + s.imageV2Store = imageV2DS.GetTestPostgresDataStore(s.T(), s.pool) s.optionsMap = schema.DeploymentsSchema.OptionsMap s.testContexts = testutils.GetNamespaceScopedTestContexts(context.Background(), s.T(), resources.Deployment) @@ -297,14 +303,7 @@ func (s *deploymentDatastoreSACSuite) TestGetDeployments() { ctx := s.testContexts[c.ScopeKey] expectedIDs := make([]string, 0, len(c.ExpectedDeploymentIDs)) for _, ID := range c.ExpectedDeploymentIDs { - found := false - for _, testID := range testIDs { - if ID == testID { - found = true - break - } - } - if found { + if slices.Contains(testIDs, ID) { expectedIDs = append(expectedIDs, ID) } } @@ -352,12 +351,20 @@ deployment object, or at the data pushed to the image datastore. func (s *deploymentDatastoreSACSuite) runGetImagesForDeploymentTest(testContexts map[string]context.Context, expectOnlyDeploymentData bool) { imageToStore := fixtures.LightweightDeploymentImage() - imageName := imageToStore.GetName() - imageName.Tag = "beta" - imageName.FullName = imageName.GetRegistry() + "/" + imageName.GetRemote() + ":" + imageName.GetTag() - imageToStore.Name = imageName - err := s.imageStore.UpsertImage(sac.WithAllAccess(context.Background()), imageToStore) - s.Require().NoError(err) + + // Slightly change the stored image so that it differs from the container image in the deployment + // This will help us check if GetImagesForDeployment returns the image from the deployment object, + // or the data from the image datastore. + imageToStore.IsClusterLocal = true + + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + err := s.imageV2Store.UpsertImage(sac.WithAllAccess(context.Background()), imageUtils.ConvertToV2(imageToStore)) + s.Require().NoError(err) + } else { + err := s.imageStore.UpsertImage(sac.WithAllAccess(context.Background()), imageToStore) + s.Require().NoError(err) + } deployment := s.pushDeploymentToStore(testconsts.Cluster2, testconsts.NamespaceB) imageFromDeployment := fixtures.LightweightDeploymentImage() @@ -371,8 +378,10 @@ func (s *deploymentDatastoreSACSuite) runGetImagesForDeploymentTest(testContexts s.Require().Equal(1, len(images)) if (!expectOnlyDeploymentData) && c.ExpectedFound { s.Equal(*imageToStore.GetName(), *images[0].GetName()) + s.Equal(imageToStore.GetIsClusterLocal(), images[0].GetIsClusterLocal()) } else { s.Equal(*imageFromDeployment.GetName(), *images[0].GetName()) + s.Equal(imageFromDeployment.GetIsClusterLocal(), images[0].GetIsClusterLocal()) } }) } diff --git a/central/deployment/service/ai_prompt.go b/central/deployment/service/ai_prompt.go new file mode 100644 index 0000000000000..35863fcc83529 --- /dev/null +++ b/central/deployment/service/ai_prompt.go @@ -0,0 +1,45 @@ +package service + +// This is the prompt for AI risk summary +const aiSummaryPrompt = `You are a Kubernetes security advisor embedded in Red Hat Advanced +Cluster Security. A security operator is investigating a deployment +flagged for review. + +AUDIENCE: Kubernetes cluster admin managing thousands of deployments. + +TONE: Brief incident report. Short declarative sentences. No filler. + +CONTEXT ALREADY VISIBLE TO THE USER: +The user already sees the deployment name, namespace, cluster, +risk score, container images, and a stat summary bar (policy +violations, CVE count, image age, component count) in the UI. +Do NOT restate any of that. Start with the insight. + +Use these exact section labels with no additional text: + +SUMMARY +2-3 sentences. Why the risk is high and the single most dangerous +finding. Name specific images, permission levels, and CVE counts +where relevant. Start with the insight, not the deployment metadata. + +RISK BREAKDOWN +Max 4 bullets, each on its own line starting with "- ". Top risk +factors ordered by score impact. One sentence per bullet, max 20 +words. Group related findings under one bullet (e.g., image age + +image CVEs). Skip factors scoring below 1.5. + +CONSTRAINTS: +- Plain text only. No markdown: no **, no backticks, no #. +- Refer to container images by name only or container name; + never include the exact tag or version. +- Report suspicious processes by executable name only; strip any + technical suffix (e.g. write "runc", not "runc:[1:CHILD]"). +- When a finding references a policy or compliance guideline + (e.g. CIS, NIST, BSI, PCI-DSS), describe the security risk it + represents, not the guideline name or code. +- Do NOT explain ACS, risk scoring, or how the system works. +- Do NOT hedge. Be direct. +- Do NOT echo these instructions or section descriptions in + your response. +- CLUSTER_ADMIN service accounts are always a top-priority finding + regardless of score.` diff --git a/central/deployment/service/risk_ai_summary_test.go b/central/deployment/service/risk_ai_summary_test.go new file mode 100644 index 0000000000000..2273fbc66c9f3 --- /dev/null +++ b/central/deployment/service/risk_ai_summary_test.go @@ -0,0 +1,631 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "testing" + + deploymentMocks "github.com/stackrox/rox/central/deployment/datastore/mocks" + olsClient "github.com/stackrox/rox/central/lightspeed/client" + processViews "github.com/stackrox/rox/central/processindicator/views" + riskMocks "github.com/stackrox/rox/central/risk/datastore/mocks" + deployment "github.com/stackrox/rox/central/risk/multipliers/deployment" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type mockOLSClient struct { + response *olsClient.QueryResponse + err error + captured *olsClient.QueryRequest +} + +func (m *mockOLSClient) Query(_ context.Context, req *olsClient.QueryRequest) (*olsClient.QueryResponse, error) { + m.captured = req + return m.response, m.err +} + +func (m *mockOLSClient) TestConnectivity() error { + return m.err +} + +func TestGetDeploymentRiskAISummary_Success(t *testing.T) { + ctrl := gomock.NewController(t) + mockDS := deploymentMocks.NewMockDataStore(ctrl) + mockRisks := riskMocks.NewMockDataStore(ctrl) + + deployment := &storage.Deployment{ + Id: "dep-1", + Name: "nginx-frontend", + Namespace: "production", + ClusterName: "cluster-east", + Type: "Deployment", + Created: timestamppb.Now(), + ServiceAccount: "default", + ServiceAccountPermissionLevel: storage.PermissionLevel_CLUSTER_ADMIN, + AutomountServiceAccountToken: true, + HostNetwork: false, + Containers: []*storage.Container{ + { + Name: "nginx", + Image: &storage.ContainerImage{ + Name: &storage.ImageName{FullName: "docker.io/nginx:1.25"}, + }, + SecurityContext: &storage.SecurityContext{ + Privileged: true, + AllowPrivilegeEscalation: true, + }, + Resources: &storage.Resources{ + CpuCoresRequest: 0.5, + CpuCoresLimit: 1.0, + MemoryMbRequest: 256, + MemoryMbLimit: 512, + }, + Config: &storage.ContainerConfig{ + Uid: 0, + Env: []*storage.ContainerConfig_EnvironmentConfig{ + {Key: "SECRET_KEY", Value: "super-secret-123"}, + {Key: "DB_URL", Value: "postgres://db.internal:5432/mydb"}, // #nosec G101 + }, + }, + LivenessProbe: &storage.LivenessProbe{Defined: true}, + ReadinessProbe: &storage.ReadinessProbe{Defined: false}, + Secrets: []*storage.EmbeddedSecret{ + {Name: "db-creds", Path: "/var/run/secrets/db"}, + }, + }, + }, + } + + risk := &storage.Risk{ + Id: "risk-1", + Score: 8.5, + Subject: &storage.RiskSubject{ + Id: "dep-1", + Namespace: "production", + ClusterId: "cluster-1", + Type: storage.RiskSubjectType_DEPLOYMENT, + }, + Results: []*storage.Risk_Result{ + { + Name: "Service Configuration", + Score: 3.5, + Factors: []*storage.Risk_Result_Factor{ + {Message: "Service account is cluster-admin"}, + {Message: "Automount SA token enabled"}, + }, + }, + { + Name: "Image Vulnerabilities", + Score: 2.0, + Factors: []*storage.Risk_Result_Factor{ + {Message: "Image has 42 CVEs", Url: "https://example.com/cves"}, + }, + }, + }, + } + + mockDS.EXPECT().GetDeployment(gomock.Any(), "dep-1").Return(deployment, true, nil) + mockRisks.EXPECT().GetRiskForDeployment(gomock.Any(), deployment).Return(risk, true, nil) + + olsMock := &mockOLSClient{ + response: &olsClient.QueryResponse{ + Response: "SUMMARY\nThis deployment runs as cluster-admin with a privileged container.", + }, + } + + svc := &serviceImpl{ + datastore: mockDS, + risks: mockRisks, + lightspeedClient: olsMock, + } + + resp, err := svc.GetDeploymentRiskAISummary(context.Background(), &v1.ResourceByID{Id: "dep-1"}) + require.NoError(t, err) + assert.Equal(t, "SUMMARY\nThis deployment runs as cluster-admin with a privileged container.", resp.GetSummary()) + + // Verify prompt and context were combined into the query. + assert.Contains(t, olsMock.captured.Query, aiSummaryPrompt) + assert.Contains(t, olsMock.captured.Query, "DEPLOYMENT AND RISK DATA:") + assert.Contains(t, olsMock.captured.Query, "nginx-frontend") +} + +func TestGetDeploymentRiskAISummary_SensitiveFieldsStripped(t *testing.T) { + ctrl := gomock.NewController(t) + mockDS := deploymentMocks.NewMockDataStore(ctrl) + mockRisks := riskMocks.NewMockDataStore(ctrl) + + deployment := &storage.Deployment{ + Id: "dep-2", + Name: "mid-server", + Namespace: "operations", + ClusterName: "cluster-west", + Type: "Deployment", + Containers: []*storage.Container{ + { + Name: "app", + Image: &storage.ContainerImage{ + Id: "sha256:abc123", + IdV2: "img-v2-id", + Name: &storage.ImageName{FullName: "registry.example.com/app:v2"}, + }, + Config: &storage.ContainerConfig{ + Uid: 1000, + Env: []*storage.ContainerConfig_EnvironmentConfig{ + {Key: "SERVICENOW_URL", Value: "https://company.service-now.com"}, + {Key: "SN_USER", Value: "admin"}, + {Key: "SN_SECRET_REF", Value: "vault:secret/data/sn-key"}, + }, + Command: []string{"/bin/app"}, + Args: []string{"--port=8080"}, + Directory: "/app", + }, + Secrets: []*storage.EmbeddedSecret{ + {Name: "sn-credentials", Path: "/var/run/secrets/servicenow"}, + }, + SecurityContext: &storage.SecurityContext{ + Privileged: false, + }, + }, + }, + } + + risk := &storage.Risk{ + Id: "risk-2", + Score: 6.0, + Subject: &storage.RiskSubject{ + Id: "dep-2", + Namespace: "operations", + }, + Results: []*storage.Risk_Result{ + { + Name: "Image Freshness", + Score: 2.0, + Factors: []*storage.Risk_Result_Factor{ + {Message: "Image is 180 days old", Url: ""}, + }, + }, + }, + } + + mockDS.EXPECT().GetDeployment(gomock.Any(), "dep-2").Return(deployment, true, nil) + mockRisks.EXPECT().GetRiskForDeployment(gomock.Any(), deployment).Return(risk, true, nil) + + olsMock := &mockOLSClient{ + response: &olsClient.QueryResponse{Response: "Low risk deployment."}, + } + + svc := &serviceImpl{ + datastore: mockDS, + risks: mockRisks, + lightspeedClient: olsMock, + } + + _, err := svc.GetDeploymentRiskAISummary(context.Background(), &v1.ResourceByID{Id: "dep-2"}) + require.NoError(t, err) + + capturedQuery := olsMock.captured.Query + + // Sensitive env vars must NOT appear in query sent to LLM. + assert.NotContains(t, capturedQuery, "SERVICENOW_URL") + assert.NotContains(t, capturedQuery, "https://company.service-now.com") + assert.NotContains(t, capturedQuery, "SN_USER") + assert.NotContains(t, capturedQuery, "SN_SECRET_REF") + assert.NotContains(t, capturedQuery, "vault:secret/data/sn-key") + + // Secret mount paths must NOT appear. + assert.NotContains(t, capturedQuery, "/var/run/secrets/servicenow") + assert.NotContains(t, capturedQuery, "sn-credentials") + + // Internal IDs must NOT appear. + assert.NotContains(t, capturedQuery, "sha256:abc123") + assert.NotContains(t, capturedQuery, "img-v2-id") + assert.NotContains(t, capturedQuery, "dep-2") // deployment ID + assert.NotContains(t, capturedQuery, "risk-2") // risk ID + + // Command, args, and directory must NOT appear. + assert.NotContains(t, capturedQuery, "/bin/app") + assert.NotContains(t, capturedQuery, "--port=8080") + + // Relevant fields MUST appear. + assert.Contains(t, capturedQuery, "mid-server") + assert.Contains(t, capturedQuery, "operations") + assert.Contains(t, capturedQuery, "cluster-west") + assert.Contains(t, capturedQuery, "registry.example.com/app:v2") + assert.Contains(t, capturedQuery, "Image is 180 days old") +} + +func TestGetDeploymentRiskAISummary_DeploymentNotFound(t *testing.T) { + ctrl := gomock.NewController(t) + mockDS := deploymentMocks.NewMockDataStore(ctrl) + + mockDS.EXPECT().GetDeployment(gomock.Any(), "nonexistent").Return(nil, false, nil) + + svc := &serviceImpl{ + datastore: mockDS, + } + + resp, err := svc.GetDeploymentRiskAISummary(context.Background(), &v1.ResourceByID{Id: "nonexistent"}) + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "does not exist") +} + +func TestGetDeploymentRiskAISummary_OLSError(t *testing.T) { + ctrl := gomock.NewController(t) + mockDS := deploymentMocks.NewMockDataStore(ctrl) + mockRisks := riskMocks.NewMockDataStore(ctrl) + + deployment := &storage.Deployment{ + Id: "dep-3", + Name: "test-app", + Namespace: "default", + ClusterName: "test-cluster", + Type: "Deployment", + } + risk := &storage.Risk{Score: 8.0} + + mockDS.EXPECT().GetDeployment(gomock.Any(), "dep-3").Return(deployment, true, nil) + mockRisks.EXPECT().GetRiskForDeployment(gomock.Any(), deployment).Return(risk, true, nil) + + olsMock := &mockOLSClient{ + err: errors.New("Lightspeed API returned HTTP 503"), + } + + svc := &serviceImpl{ + datastore: mockDS, + risks: mockRisks, + lightspeedClient: olsMock, + } + + resp, err := svc.GetDeploymentRiskAISummary(context.Background(), &v1.ResourceByID{Id: "dep-3"}) + assert.Nil(t, resp) + require.Error(t, err) + + st, ok := status.FromError(err) + require.True(t, ok, "expected a gRPC status error") + assert.Equal(t, codes.Unavailable, st.Code()) + assert.Equal(t, "AI service unavailable", st.Message()) +} + +func TestGetDeploymentRiskAISummary_NilRisk(t *testing.T) { + ctrl := gomock.NewController(t) + mockDS := deploymentMocks.NewMockDataStore(ctrl) + mockRisks := riskMocks.NewMockDataStore(ctrl) + + deployment := &storage.Deployment{ + Id: "dep-4", + Name: "new-app", + Namespace: "staging", + ClusterName: "staging-cluster", + Type: "Deployment", + } + + mockDS.EXPECT().GetDeployment(gomock.Any(), "dep-4").Return(deployment, true, nil) + mockRisks.EXPECT().GetRiskForDeployment(gomock.Any(), deployment).Return(nil, false, nil) + + // OLS client should NOT be called for nil/low risk - no mock needed + svc := &serviceImpl{ + datastore: mockDS, + risks: mockRisks, + } + + resp, err := svc.GetDeploymentRiskAISummary(context.Background(), &v1.ResourceByID{Id: "dep-4"}) + require.NoError(t, err) + // Nil risk returns a specific message without calling OLS + assert.Equal(t, "No risk data available for this deployment.", resp.GetSummary()) +} + +func TestGetDeploymentRiskAISummary_LowRiskScore(t *testing.T) { + ctrl := gomock.NewController(t) + mockDS := deploymentMocks.NewMockDataStore(ctrl) + mockRisks := riskMocks.NewMockDataStore(ctrl) + + deployment := &storage.Deployment{ + Id: "dep-5", + Name: "low-risk-app", + Namespace: "default", + ClusterName: "test-cluster", + Type: "Deployment", + } + + risk := &storage.Risk{Score: 3.5} // Below threshold of 5.0 + + mockDS.EXPECT().GetDeployment(gomock.Any(), "dep-5").Return(deployment, true, nil) + mockRisks.EXPECT().GetRiskForDeployment(gomock.Any(), deployment).Return(risk, true, nil) + + // OLS client should NOT be called for low risk scores + svc := &serviceImpl{ + datastore: mockDS, + risks: mockRisks, + } + + resp, err := svc.GetDeploymentRiskAISummary(context.Background(), &v1.ResourceByID{Id: "dep-5"}) + require.NoError(t, err) + assert.Equal(t, "Skipping AI summary since this is a low risk deployment with normalized risk score below 5.", resp.GetSummary()) +} + +func TestBuildSanitizedRiskContext_FieldSelection(t *testing.T) { + deployment := &storage.Deployment{ + Id: "id-should-be-stripped", + Hash: 12345, + Name: "my-deployment", + Namespace: "my-namespace", + NamespaceId: "ns-id-should-be-stripped", + ClusterId: "cluster-id-should-be-stripped", + ClusterName: "my-cluster", + Type: "Deployment", + Created: timestamppb.Now(), + ServiceAccount: "deployer-sa", + ServiceAccountPermissionLevel: storage.PermissionLevel_ELEVATED_IN_NAMESPACE, + AutomountServiceAccountToken: true, + HostNetwork: true, + HostPid: false, + HostIpc: false, + StateTimestamp: 999999, + ImagePullSecrets: []string{"registry-secret"}, + Tolerations: []*storage.Toleration{ + {Key: "node-role.kubernetes.io/master"}, + }, + Ports: []*storage.PortConfig{ + {Name: "http", ContainerPort: 8080}, + }, + Labels: map[string]string{"app": "web"}, + Annotations: map[string]string{"note": "test"}, + Containers: []*storage.Container{ + { + Id: "container-id-should-be-stripped", + Name: "web", + Image: &storage.ContainerImage{ + Id: "sha256:deadbeef", + IdV2: "idv2-stripped", + Name: &storage.ImageName{FullName: "quay.io/app:latest"}, + NotPullable: true, + IsClusterLocal: true, + }, + SecurityContext: &storage.SecurityContext{ + Privileged: true, + ReadOnlyRootFilesystem: true, + AllowPrivilegeEscalation: false, + DropCapabilities: []string{"ALL"}, + AddCapabilities: []string{"NET_BIND_SERVICE"}, + }, + Resources: &storage.Resources{ + CpuCoresRequest: 0.25, + CpuCoresLimit: 1.0, + MemoryMbRequest: 128, + MemoryMbLimit: 512, + }, + Config: &storage.ContainerConfig{ + Uid: 0, + Env: []*storage.ContainerConfig_EnvironmentConfig{ + {Key: "PASSWORD", Value: "secret123"}, + }, + Command: []string{"/entrypoint.sh"}, + Directory: "/app", + }, + LivenessProbe: &storage.LivenessProbe{Defined: true}, + ReadinessProbe: &storage.ReadinessProbe{Defined: true}, + Secrets: []*storage.EmbeddedSecret{ + {Name: "tls-cert", Path: "/etc/tls/certs"}, + }, + }, + }, + } + + risk := &storage.Risk{ + Id: "risk-id-stripped", + Score: 7.2, + Subject: &storage.RiskSubject{ + Id: "dep-subject", + Namespace: "my-namespace", + ClusterId: "cluster-1", + }, + Results: []*storage.Risk_Result{ + { + Name: "Policy Violations", + Score: 4.0, + Factors: []*storage.Risk_Result_Factor{ + {Message: "Privileged container detected", Url: "https://docs.example.com/priv"}, + {Message: "Running as root"}, + }, + }, + }, + } + + contextJSON, err := buildSanitizedRiskContext(deployment, risk) + require.NoError(t, err) + + // Parse to verify structure. + var parsed map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(contextJSON), &parsed)) + assert.Contains(t, string(parsed["deployment"]), "my-deployment") + assert.Contains(t, string(parsed["risk"]), "Policy Violations") + + // Fields that MUST be present. + assert.Contains(t, contextJSON, "my-deployment") + assert.Contains(t, contextJSON, "my-namespace") + assert.Contains(t, contextJSON, "my-cluster") + assert.Contains(t, contextJSON, "Deployment") + assert.Contains(t, contextJSON, "deployer-sa") + assert.Contains(t, contextJSON, "ELEVATED_IN_NAMESPACE") + assert.Contains(t, contextJSON, "quay.io/app:latest") + assert.Contains(t, contextJSON, "Privileged container detected") + assert.Contains(t, contextJSON, "Running as root") + + // Fields that MUST be stripped — IDs and metadata. + assert.NotContains(t, contextJSON, "id-should-be-stripped") + assert.NotContains(t, contextJSON, "ns-id-should-be-stripped") + assert.NotContains(t, contextJSON, "cluster-id-should-be-stripped") + assert.NotContains(t, contextJSON, "container-id-should-be-stripped") + assert.NotContains(t, contextJSON, "sha256:deadbeef") + assert.NotContains(t, contextJSON, "idv2-stripped") + assert.NotContains(t, contextJSON, "risk-id-stripped") + assert.NotContains(t, contextJSON, "dep-subject") + assert.NotContains(t, contextJSON, "999999") // stateTimestamp + + // Fields stripped for security. + assert.NotContains(t, contextJSON, "PASSWORD") + assert.NotContains(t, contextJSON, "secret123") + assert.NotContains(t, contextJSON, "/etc/tls/certs") + assert.NotContains(t, contextJSON, "tls-cert") + assert.NotContains(t, contextJSON, "/entrypoint.sh") + + // Wasteful fields stripped. + assert.NotContains(t, contextJSON, "registry-secret") // imagePullSecrets + assert.NotContains(t, contextJSON, "node-role.kubernetes.io/master") // tolerations + assert.NotContains(t, contextJSON, "https://docs.example.com/priv") // factor URLs + + // notPullable and isClusterLocal should not be included. + assert.NotContains(t, contextJSON, "notPullable") + assert.NotContains(t, contextJSON, "isClusterLocal") +} + +func TestBuildSanitizedRiskContext_ProcessArgsStripped(t *testing.T) { + dep := &storage.Deployment{ + Id: "dep-1", + Name: "test-app", + Namespace: "default", + ClusterName: "test-cluster", + Type: "Deployment", + } + + // Use FormatProcess to generate realistic messages. This ensures the test + // will catch any changes to FormatProcess that break our sanitization regex. + processWithSensitiveArgs1 := &processViews.ProcessIndicatorRiskView{ + SignalName: "/bin/bash", + SignalArgs: "-c export PASSWORD=secret123 && ./script.sh", + ContainerName: "app", + } + processWithSensitiveArgs2 := &processViews.ProcessIndicatorRiskView{ + SignalName: "/usr/bin/curl", + SignalArgs: "https://api.example.com?token=abc123&secret=xyz", + ContainerName: "app", + } + processWithoutArgs := &processViews.ProcessIndicatorRiskView{ + SignalName: "/bin/sh", + SignalArgs: "", // no args + ContainerName: "sidecar", + } + + risk := &storage.Risk{ + Id: "risk-1", + Score: 8.0, + Results: []*storage.Risk_Result{ + { + Name: deployment.ProcessBaselineHeading, + Score: 2.5, + Factors: []*storage.Risk_Result_Factor{ + // Generated using FormatProcess to stay in sync with the actual format. + {Message: deployment.FormatProcess(processWithSensitiveArgs1)}, + {Message: deployment.FormatProcess(processWithSensitiveArgs2)}, + {Message: deployment.FormatProcess(processWithoutArgs)}, + }, + }, + { + // Other risk results should NOT have args stripped. + Name: "Policy Violations", + Score: 1.5, + Factors: []*storage.Risk_Result_Factor{ + {Message: `Some message with args "should not be stripped"`}, + }, + }, + }, + } + + contextJSON, err := buildSanitizedRiskContext(dep, risk) + require.NoError(t, err) + + // Process names and container names SHOULD be present. + assert.Contains(t, contextJSON, "/bin/bash") + assert.Contains(t, contextJSON, "/usr/bin/curl") + assert.Contains(t, contextJSON, "/bin/sh") + assert.Contains(t, contextJSON, "container app") + assert.Contains(t, contextJSON, "container sidecar") + assert.Contains(t, contextJSON, deployment.ProcessBaselineHeading) + + // Process arguments MUST be redacted from process baseline results (may contain secrets). + assert.NotContains(t, contextJSON, "PASSWORD=secret123") + assert.NotContains(t, contextJSON, "token=abc123") + assert.NotContains(t, contextJSON, "secret=xyz") + assert.NotContains(t, contextJSON, "script.sh") + + // The ' with args "..."' pattern should be replaced with ''. + // Note: JSON encoding escapes < and > as \u003c and \u003e. + assert.NotContains(t, contextJSON, `with args "-c export`) + assert.NotContains(t, contextJSON, `with args "https://api`) + assert.Contains(t, contextJSON, `\u003credacted args\u003e`) + + // Message without args should remain intact (no added). + // Use the exact format from FormatProcess. + assert.Contains(t, contextJSON, `Detected execution of suspicious process \"/bin/sh\" in container sidecar`) + + // Other risk results should NOT have their messages modified. + assert.Contains(t, contextJSON, `with args \"should not be stripped\"`) +} + +func TestBuildSanitizedRiskContext_ProcessArgsWithEmbeddedQuotes(t *testing.T) { + // Regression test: strconv.Quote escapes embedded quotes as \". + // The regex must handle escaped quotes to avoid leaving sensitive data exposed. + // Using FormatProcess ensures this test will catch format changes. + dep := &storage.Deployment{ + Id: "dep-1", + Name: "test-app", + Namespace: "default", + ClusterName: "test-cluster", + Type: "Deployment", + } + + // Args with embedded quotes - FormatProcess uses strconv.Quote which escapes them. + processWithEmbeddedQuotes1 := &processViews.ProcessIndicatorRiskView{ + SignalName: "/bin/bash", + SignalArgs: `-c echo "password=secret123"`, + ContainerName: "app", + } + processWithEmbeddedQuotes2 := &processViews.ProcessIndicatorRiskView{ + SignalName: "/bin/sh", + SignalArgs: `cmd with "nested" quotes and token=abc123`, + ContainerName: "app", + } + + risk := &storage.Risk{ + Id: "risk-1", + Score: 8.0, + Results: []*storage.Risk_Result{ + { + Name: deployment.ProcessBaselineHeading, + Score: 2.0, + Factors: []*storage.Risk_Result_Factor{ + {Message: deployment.FormatProcess(processWithEmbeddedQuotes1)}, + {Message: deployment.FormatProcess(processWithEmbeddedQuotes2)}, + }, + }, + }, + } + + contextJSON, err := buildSanitizedRiskContext(dep, risk) + require.NoError(t, err) + + // Process names SHOULD be present. + assert.Contains(t, contextJSON, "/bin/bash") + assert.Contains(t, contextJSON, "/bin/sh") + + // Sensitive data with embedded quotes MUST be fully redacted. + assert.NotContains(t, contextJSON, "password=secret123") + assert.NotContains(t, contextJSON, "token=abc123") + assert.NotContains(t, contextJSON, "nested") + + // The entire args section should be replaced with . + assert.Contains(t, contextJSON, `\u003credacted args\u003e`) + + // Ensure no partial args remain (the old regex would stop at the first escaped quote). + assert.NotContains(t, contextJSON, `with args "-c echo`) + assert.NotContains(t, contextJSON, `with args "cmd with`) +} diff --git a/central/deployment/service/risk_sanitization.go b/central/deployment/service/risk_sanitization.go new file mode 100644 index 0000000000000..2a9c8ab2da26e --- /dev/null +++ b/central/deployment/service/risk_sanitization.go @@ -0,0 +1,152 @@ +package service + +import ( + "encoding/json" + "regexp" + + "github.com/stackrox/rox/central/risk/multipliers/deployment" + "github.com/stackrox/rox/generated/storage" +) + +// processArgsPattern matches ' with args "..."' in process baseline messages. +// Process arguments may contain sensitive data (passwords, tokens) and must be +// redacted before sending to an external LLM. +// The pattern handles escaped quotes (from strconv.Quote) to ensure the entire +// argument string is matched even when it contains embedded quotes. +var processArgsPattern = regexp.MustCompile(` with args "(\\.|[^"\\])*"`) + +// buildSanitizedRiskContext produces a minimal JSON representation of the +// deployment and risk data suitable for sending to an external LLM. It keeps +// only fields relevant to risk analysis and strips sensitive data (env vars, +// secret paths) and wasteful metadata (IDs, hashes, empty fields). +func buildSanitizedRiskContext(deployment *storage.Deployment, risk *storage.Risk) (string, error) { + ctx := sanitizedContext{ + Deployment: sanitizeDeployment(deployment), + Risk: sanitizeRisk(risk), + } + data, err := json.Marshal(ctx) + if err != nil { + return "", err + } + return string(data), nil +} + +type sanitizedContext struct { + Deployment sanitizedDeployment `json:"deployment"` + Risk sanitizedRisk `json:"risk"` +} + +type sanitizedDeployment struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + ClusterName string `json:"clusterName"` + Type string `json:"type"` + Created string `json:"created,omitempty"` + ServiceAccount string `json:"serviceAccount,omitempty"` + ServiceAccountPermissionLevel string `json:"serviceAccountPermissionLevel,omitempty"` + AutomountServiceAccountToken bool `json:"automountServiceAccountToken"` + HostNetwork bool `json:"hostNetwork"` + HostPid bool `json:"hostPid"` + HostIpc bool `json:"hostIpc"` + Containers []sanitizedContainer `json:"containers"` +} + +type sanitizedContainer struct { + Name string `json:"name"` + ImageFullName string `json:"imageFullName,omitempty"` + SecurityContext *storage.SecurityContext `json:"securityContext,omitempty"` + Resources *storage.Resources `json:"resources,omitempty"` + UID int64 `json:"uid,omitempty"` + LivenessProbe *storage.LivenessProbe `json:"livenessProbe,omitempty"` + ReadinessProbe *storage.ReadinessProbe `json:"readinessProbe,omitempty"` +} + +type sanitizedRisk struct { + Score float32 `json:"score"` + Results []sanitizedRiskResult `json:"results,omitempty"` +} + +type sanitizedRiskResult struct { + Name string `json:"name"` + Score float32 `json:"score"` + Factors []sanitizedRiskFactor `json:"factors,omitempty"` +} + +type sanitizedRiskFactor struct { + Message string `json:"message"` +} + +func sanitizeDeployment(d *storage.Deployment) sanitizedDeployment { + sd := sanitizedDeployment{ + Name: d.GetName(), + Namespace: d.GetNamespace(), + ClusterName: d.GetClusterName(), + Type: d.GetType(), + ServiceAccount: d.GetServiceAccount(), + AutomountServiceAccountToken: d.GetAutomountServiceAccountToken(), + HostNetwork: d.GetHostNetwork(), + HostPid: d.GetHostPid(), + HostIpc: d.GetHostIpc(), + } + + if d.GetServiceAccountPermissionLevel() != storage.PermissionLevel_UNSET { + sd.ServiceAccountPermissionLevel = d.GetServiceAccountPermissionLevel().String() + } + + if d.GetCreated() != nil { + sd.Created = d.GetCreated().AsTime().Format("2006-01-02T15:04:05Z") + } + + for _, c := range d.GetContainers() { + sc := sanitizedContainer{ + Name: c.GetName(), + SecurityContext: c.GetSecurityContext(), + UID: c.GetConfig().GetUid(), + } + + if img := c.GetImage(); img != nil && img.GetName() != nil { + sc.ImageFullName = img.GetName().GetFullName() + } + + if res := c.GetResources(); res != nil { + sc.Resources = res + } + + sc.LivenessProbe = c.GetLivenessProbe() + sc.ReadinessProbe = c.GetReadinessProbe() + + sd.Containers = append(sd.Containers, sc) + } + return sd +} + +func sanitizeRisk(r *storage.Risk) sanitizedRisk { + if r == nil { + return sanitizedRisk{} + } + sr := sanitizedRisk{ + Score: r.GetScore(), + } + for _, result := range r.GetResults() { + srr := sanitizedRiskResult{ + Name: result.GetName(), + Score: result.GetScore(), + } + for _, factor := range result.GetFactors() { + msg := factor.GetMessage() + if msg == "" { + continue + } + // Redact process arguments from process baseline violation messages + // as they may contain sensitive information (passwords, tokens, etc.). + if result.GetName() == deployment.ProcessBaselineHeading { + msg = processArgsPattern.ReplaceAllString(msg, "") + } + srr.Factors = append(srr.Factors, sanitizedRiskFactor{ + Message: msg, + }) + } + sr.Results = append(sr.Results, srr) + } + return sr +} diff --git a/central/deployment/service/service.go b/central/deployment/service/service.go index 85a5236c9de5f..9918f55fdf9f5 100644 --- a/central/deployment/service/service.go +++ b/central/deployment/service/service.go @@ -4,6 +4,7 @@ import ( "context" "github.com/stackrox/rox/central/deployment/datastore" + olsClient "github.com/stackrox/rox/central/lightspeed/client" processBaselineDataStore "github.com/stackrox/rox/central/processbaseline/datastore" processBaselineResultsStore "github.com/stackrox/rox/central/processbaselineresults/datastore" processIndicatorDataStore "github.com/stackrox/rox/central/processindicator/datastore" @@ -24,7 +25,7 @@ type Service interface { // New returns a new Service instance using the given DataStore. func New(datastore datastore.DataStore, processIndicators processIndicatorDataStore.DataStore, processBaselines processBaselineDataStore.DataStore, - processBaselineResults processBaselineResultsStore.DataStore, risks riskDataStore.DataStore, manager riskManager.Manager) Service { + processBaselineResults processBaselineResultsStore.DataStore, risks riskDataStore.DataStore, manager riskManager.Manager, lightspeed olsClient.Client) Service { return &serviceImpl{ datastore: datastore, processIndicators: processIndicators, @@ -32,5 +33,6 @@ func New(datastore datastore.DataStore, processIndicators processIndicatorDataSt processBaselineResults: processBaselineResults, risks: risks, manager: manager, + lightspeedClient: lightspeed, } } diff --git a/central/deployment/service/service_impl.go b/central/deployment/service/service_impl.go index 6cefc43bc7aef..3ba4f12a378f5 100644 --- a/central/deployment/service/service_impl.go +++ b/central/deployment/service/service_impl.go @@ -9,6 +9,7 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/pkg/errors" "github.com/stackrox/rox/central/deployment/datastore" + olsClient "github.com/stackrox/rox/central/lightspeed/client" processBaselineStore "github.com/stackrox/rox/central/processbaseline/datastore" processBaselineResultsStore "github.com/stackrox/rox/central/processbaselineresults/datastore" processIndicatorStore "github.com/stackrox/rox/central/processindicator/datastore" @@ -21,6 +22,7 @@ import ( "github.com/stackrox/rox/pkg/grpc/authz" "github.com/stackrox/rox/pkg/grpc/authz/perrpc" "github.com/stackrox/rox/pkg/grpc/authz/user" + "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/options/deployments" @@ -29,17 +31,23 @@ import ( "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/utils" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( maxDeploymentsReturned = 1000 + lowRiskThreshold = 5.0 ) +var log = logging.LoggerForModule() + var ( authorizer = perrpc.FromMap(map[authz.Authorizer][]string{ user.With(permissions.View(resources.Deployment)): { v1.DeploymentService_GetDeployment_FullMethodName, v1.DeploymentService_GetDeploymentWithRisk_FullMethodName, + v1.DeploymentService_GetDeploymentRiskAISummary_FullMethodName, v1.DeploymentService_CountDeployments_FullMethodName, v1.DeploymentService_ListDeployments_FullMethodName, v1.DeploymentService_GetLabels_FullMethodName, @@ -60,6 +68,7 @@ type serviceImpl struct { processBaselineResults processBaselineResultsStore.DataStore risks riskDataStore.DataStore manager manager.Manager + lightspeedClient olsClient.Client } func (s *serviceImpl) ExportDeployments(req *v1.ExportDeploymentRequest, srv v1.DeploymentService_ExportDeploymentsServer) error { @@ -279,3 +288,51 @@ func labelsMapFromSearchResults(results []search.Result) (map[string]*v1.Deploym return keyValuesMap, values } + +// GetDeploymentRiskAISummary returns an AI-generated risk summary for a deployment. +func (s *serviceImpl) GetDeploymentRiskAISummary(ctx context.Context, request *v1.ResourceByID) (*v1.DeploymentRiskAISummaryResponse, error) { + deployment, exists, err := s.datastore.GetDeployment(ctx, request.GetId()) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.Wrapf(errox.NotFound, "deployment with id '%s' does not exist", request.GetId()) + } + + risk, _, err := s.risks.GetRiskForDeployment(ctx, deployment) + if err != nil { + return nil, err + } + + // Skip LLM call for deployments without risk data. + if risk == nil { + return &v1.DeploymentRiskAISummaryResponse{ + Summary: "No risk data available for this deployment.", + }, nil + } + + // Skip LLM call for low-risk deployments. + if risk.GetScore() < lowRiskThreshold { + return &v1.DeploymentRiskAISummaryResponse{ + Summary: "Skipping AI summary since this is a low risk deployment with normalized risk score below 5.", + }, nil + } + + contextJSON, err := buildSanitizedRiskContext(deployment, risk) + if err != nil { + return nil, errors.Wrap(err, "building risk context for AI summary") + } + + query := aiSummaryPrompt + "\n\nDEPLOYMENT AND RISK DATA:\n" + contextJSON + olsResp, err := s.lightspeedClient.Query(ctx, &olsClient.QueryRequest{ + Query: query, + }) + if err != nil { + log.Errorf("Lightspeed query failed for deployment %s: %v", request.GetId(), err) + return nil, status.Error(codes.Unavailable, "AI service unavailable") + } + + return &v1.DeploymentRiskAISummaryResponse{ + Summary: olsResp.Response, + }, nil +} diff --git a/central/deployment/service/service_impl_benchmark_test.go b/central/deployment/service/service_impl_benchmark_test.go index 7d6af56a21247..a2daed471bb34 100644 --- a/central/deployment/service/service_impl_benchmark_test.go +++ b/central/deployment/service/service_impl_benchmark_test.go @@ -21,7 +21,7 @@ func BenchmarkService_Export(b *testing.B) { if err != nil { b.Error(err) } - svc := New(testHelper.Deployments, nil, nil, nil, nil, nil) + svc := New(testHelper.Deployments, nil, nil, nil, nil, nil, nil) benchmarkFunc := getExportServiceBenchmark(testHelper, svc) testHelper.InjectDataAndRunBenchmark(b, false, benchmarkFunc) } diff --git a/central/deployment/service/singleton.go b/central/deployment/service/singleton.go index 9453236863d01..e3d1042365d6d 100644 --- a/central/deployment/service/singleton.go +++ b/central/deployment/service/singleton.go @@ -2,6 +2,7 @@ package service import ( "github.com/stackrox/rox/central/deployment/datastore" + olsClient "github.com/stackrox/rox/central/lightspeed/client" processBaselineDataStore "github.com/stackrox/rox/central/processbaseline/datastore" processBaselineResultsStore "github.com/stackrox/rox/central/processbaselineresults/datastore" processIndicatorDataStore "github.com/stackrox/rox/central/processindicator/datastore" @@ -17,7 +18,7 @@ var ( ) func initialize() { - as = New(datastore.Singleton(), processIndicatorDataStore.Singleton(), processBaselineDataStore.Singleton(), processBaselineResultsStore.Singleton(), riskDataStore.Singleton(), manager.Singleton()) + as = New(datastore.Singleton(), processIndicatorDataStore.Singleton(), processBaselineDataStore.Singleton(), processBaselineResultsStore.Singleton(), riskDataStore.Singleton(), manager.Singleton(), olsClient.NewClient()) } // Singleton provides the instance of the Service interface to register. diff --git a/central/deployment/views/container_images.go b/central/deployment/views/container_images.go index f309c2d4011b7..109ffed328068 100644 --- a/central/deployment/views/container_images.go +++ b/central/deployment/views/container_images.go @@ -1,15 +1,18 @@ package views +import "github.com/stackrox/rox/generated/storage" + // ContainerImageView represents a container image with its associated cluster IDs. // This view is used to get distinct container images from active deployments // along with the clusters where they are deployed. type ContainerImageView struct { - ImageIDV2 string `db:"image_id"` - // Each imageIDV2 will map to a single digest, but SQL wants us to apply an aggreate func to all selected fields - // when grouing by imageIDV2. So we need to select the digest with a min aggregate, - // which will be a no-op for a single value. - ImageDigest string `db:"image_sha"` - ClusterIDs []string `db:"cluster_id"` + ImageIDV2 string `db:"image_id"` + ImageDigest string `db:"image_sha"` + ClusterIDs []string `db:"cluster_id"` + ImageNameRegistry string `db:"image_registry"` + ImageNameRemote string `db:"image_remote"` + ImageNameTag string `db:"image_tag"` + ImageNameFullName string `db:"image"` } // GetImageDigest returns the image digest. @@ -26,3 +29,13 @@ func (c *ContainerImageView) GetImageID() string { func (c *ContainerImageView) GetClusterIDs() []string { return c.ClusterIDs } + +// GetImageName returns the image name. +func (c *ContainerImageView) GetImageName() *storage.ImageName { + return &storage.ImageName{ + Registry: c.ImageNameRegistry, + Remote: c.ImageNameRemote, + Tag: c.ImageNameTag, + FullName: c.ImageNameFullName, + } +} diff --git a/central/deployment/views/list_deployment_view.go b/central/deployment/views/list_deployment_view.go new file mode 100644 index 0000000000000..b876ee035a8c2 --- /dev/null +++ b/central/deployment/views/list_deployment_view.go @@ -0,0 +1,51 @@ +package views + +import ( + "time" + + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/search" +) + +// ListDeploymentView represents deployment data for list responses. +// This view is used to populate ListDeployment protos from database queries. +// The db tags use search field labels (lowercase with underscores), not database column names. +type ListDeploymentView struct { + ID string `db:"deployment_id"` + Hash uint64 `db:"deployment_hash"` + Name string `db:"deployment"` + ClusterName string `db:"cluster"` + ClusterID string `db:"cluster_id"` + Namespace string `db:"namespace"` + Created *time.Time `db:"created"` + // Priority is NOT selected from DB - it's computed by the ranker +} + +// ToListDeployment converts the view to a storage.ListDeployment proto. +func (v *ListDeploymentView) ToListDeployment() *storage.ListDeployment { + return &storage.ListDeployment{ + Id: v.ID, + Hash: v.Hash, + Name: v.Name, + Cluster: v.ClusterName, + ClusterId: v.ClusterID, + Namespace: v.Namespace, + Created: protocompat.ConvertTimeToTimestampOrNil(v.Created), + // Priority is set by updateListDeploymentPriority in the datastore layer + } +} + +// ListDeploymentViewSelects returns the query selects needed for ListDeploymentView. +func ListDeploymentViewSelects() []*v1.QuerySelect { + return []*v1.QuerySelect{ + search.NewQuerySelect(search.DeploymentID).Proto(), + search.NewQuerySelect(search.DeploymentHash).Proto(), + search.NewQuerySelect(search.DeploymentName).Proto(), + search.NewQuerySelect(search.Cluster).Proto(), + search.NewQuerySelect(search.ClusterID).Proto(), + search.NewQuerySelect(search.Namespace).Proto(), + search.NewQuerySelect(search.Created).Proto(), + } +} diff --git a/central/detection/alertmanager/alert_manager_impl.go b/central/detection/alertmanager/alert_manager_impl.go index 79aa95f442d34..1891a4fa08e4a 100644 --- a/central/detection/alertmanager/alert_manager_impl.go +++ b/central/detection/alertmanager/alert_manager_impl.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" alertDataStore "github.com/stackrox/rox/central/alert/datastore" + alertviews "github.com/stackrox/rox/central/alert/views" "github.com/stackrox/rox/central/detection/runtime" "github.com/stackrox/rox/generated/storage" pkgAlert "github.com/stackrox/rox/pkg/alert" @@ -27,6 +28,11 @@ var ( log = logging.LoggerForModule() ) +type mergeCandidate struct { + incoming *storage.Alert + oldKeyID string +} + type alertManagerImpl struct { notifier notifierProcessor.Processor alerts alertDataStore.DataStore @@ -48,14 +54,18 @@ func getDeploymentIDsFromAlerts(alertSlices ...[]*storage.Alert) set.StringSet { // AlertAndNotify is the main function that implements the AlertManager interface func (d *alertManagerImpl) AlertAndNotify(ctx context.Context, currentAlerts []*storage.Alert, oldAlertFilters ...AlertFilterOption) (set.StringSet, error) { + defer observeDurationMs(alertAndNotifyDuration)() + alertAndNotifyIncomingCount.Observe(float64(len(currentAlerts))) + // Merge the old and the new alerts. - newAlerts, updatedAlerts, toBeResolvedAlerts, err := d.mergeManyAlerts(ctx, currentAlerts, oldAlertFilters...) + newAlerts, updatedAlerts, toBeResolvedIDs, resolvedDeploymentIDs, err := d.mergeManyAlerts(ctx, currentAlerts, oldAlertFilters...) if err != nil { return nil, err } // If any of the alerts are for a deployment, detect if the deployment itself is modified - modifiedDeployments := getDeploymentIDsFromAlerts(newAlerts, updatedAlerts, toBeResolvedAlerts) + modifiedDeployments := getDeploymentIDsFromAlerts(newAlerts, updatedAlerts) + modifiedDeployments = modifiedDeployments.Union(resolvedDeploymentIDs) // Mark any old alerts no longer generated as resolved, and insert new alerts. err = d.notifyAndUpdateBatch(ctx, newAlerts) @@ -66,7 +76,7 @@ func (d *alertManagerImpl) AlertAndNotify(ctx context.Context, currentAlerts []* if err != nil { return nil, err } - err = d.markAlertsResolved(ctx, toBeResolvedAlerts) + err = d.markAlertsResolvedByIDs(ctx, toBeResolvedIDs) if err != nil { return nil, err } @@ -101,16 +111,12 @@ func (d *alertManagerImpl) updateBatch(ctx context.Context, alertsToMark []*stor return errList.ToError() } -// markAlertsResolved marks all input alerts resolved in the input datastore. -func (d *alertManagerImpl) markAlertsResolved(ctx context.Context, alertsToMark []*storage.Alert) error { - if len(alertsToMark) == 0 { +// markAlertsResolvedByIDs marks alerts with the given IDs as resolved. +func (d *alertManagerImpl) markAlertsResolvedByIDs(ctx context.Context, ids []string) error { + if len(ids) == 0 { return nil } - ids := make([]string, 0, len(alertsToMark)) - for _, alert := range alertsToMark { - ids = append(ids, alert.GetId()) - } resolvedAlerts, err := d.alerts.MarkAlertsResolvedBatch(ctx, ids...) if err != nil { return err @@ -368,15 +374,27 @@ func mergeAlerts(old, newAlert *storage.Alert) *storage.Alert { } newAlert.FirstOccurred = old.GetFirstOccurred() + // Sensor's payload does not set Inactive; keep the stored flag on the merged alert. + if oldDep := old.GetDeployment(); oldDep != nil && oldDep.GetInactive() { + if newDep := newAlert.GetDeployment(); newDep != nil { + newDep.Inactive = true + } + } return newAlert } -// mergeManyAlerts merges two alerts. +// mergeManyAlerts uses a two-phase fetch to match incoming alerts against +// previous alerts. Phase 1 fetches lightweight AlertMatchKey structs (inline +// columns only, no TOAST I/O). Phase 2 fetches full alert blobs only for +// the small fraction that actually matched and need merging. func (d *alertManagerImpl) mergeManyAlerts( ctx context.Context, incomingAlerts []*storage.Alert, oldAlertFilters ...AlertFilterOption, -) (newAlerts, updatedAlerts, toBeResolvedAlerts []*storage.Alert, err error) { +) (newAlerts, updatedAlerts []*storage.Alert, toBeResolvedIDs []string, resolvedDeploymentIDs set.StringSet, err error) { + defer observeDurationMs(mergeManyAlertsDuration)() + resolvedDeploymentIDs = set.NewStringSet() + qb := search.NewQueryBuilder().AddExactMatches( search.ViolationState, storage.ViolationState_ACTIVE.String(), @@ -384,13 +402,18 @@ func (d *alertManagerImpl) mergeManyAlerts( for _, filter := range oldAlertFilters { filter.apply(qb) } - previousAlerts, err := d.alerts.SearchRawAlerts(ctx, qb.ProtoQuery(), true) + + // Phase 1: fetch lightweight match keys instead of full alert blobs. + previousKeys, err := d.alerts.SearchAlertMatchKeys(ctx, qb.ProtoQuery(), true) if err != nil { - err = errors.Wrapf(err, "couldn't load previous alerts (query was %s)", qb.Query()) + err = errors.Wrapf(err, "couldn't load previous alert keys (query was %s)", qb.Query()) return } + mergeManyAlertsPreviousCount.Observe(float64(len(previousKeys))) + + // Match incoming alerts against previous keys, collecting merge candidates. + var mergeCandidates []mergeCandidate - // Merge any alerts that have new and old alerts. for _, alert := range incomingAlerts { if pkgAlert.IsDeployTimeAttemptedAlert(alert) { // `alert.time` is the latest violation time. @@ -399,11 +422,8 @@ func (d *alertManagerImpl) mergeManyAlerts( continue } - if matchingOld := findAlert(alert, previousAlerts); matchingOld != nil { - mergedAlert := mergeAlerts(matchingOld, alert) - if mergedAlert != matchingOld && !mergedAlert.EqualVT(matchingOld) { - updatedAlerts = append(updatedAlerts, mergedAlert) - } + if matchingKey := findMatchingKey(alert, previousKeys); matchingKey != nil { + mergeCandidates = append(mergeCandidates, mergeCandidate{incoming: alert, oldKeyID: matchingKey.GetId()}) continue } @@ -412,7 +432,15 @@ func (d *alertManagerImpl) mergeManyAlerts( newAlerts = append(newAlerts, alert) } - // Get the deployments that are currently being removed as part of this alert update. + // Phase 2: fetch full alerts for matched keys and merge. + _, mergedNew, mergedUpdated, mergeErr := d.fetchAndMergeCandidates(ctx, mergeCandidates) + if mergeErr != nil { + err = mergeErr + return + } + newAlerts = append(newAlerts, mergedNew...) + updatedAlerts = append(updatedAlerts, mergedUpdated...) + deploymentsBeingRemoved := set.NewStringSet() for _, f := range oldAlertFilters { if depID := f.removedDeploymentID(); depID != "" { @@ -420,54 +448,161 @@ func (d *alertManagerImpl) mergeManyAlerts( } } - // Find any old alerts no longer being produced. - for _, previousAlert := range previousAlerts { - if d.shouldMarkAlertResolved(previousAlert, incomingAlerts, oldAlertFilters...) { - toBeResolvedAlerts = append(toBeResolvedAlerts, previousAlert) + for _, a := range newAlerts { + d.stampInactiveIfDeploymentGone(a, deploymentsBeingRemoved) + } + for _, a := range updatedAlerts { + d.stampInactiveIfDeploymentGone(a, deploymentsBeingRemoved) + } + + // IDs already in updatedAlerts were stamped in memory. Re-fetching them + // would upsert the stored blob and drop merged processes. + alreadyUpdated := set.NewStringSet() + for _, a := range updatedAlerts { + alreadyUpdated.Add(a.GetId()) + } + + var needInactiveIDs []string + for _, key := range previousKeys { + if d.shouldMarkAlertResolved(key, incomingAlerts, oldAlertFilters...) { + toBeResolvedIDs = append(toBeResolvedIDs, key.GetId()) + if key.HasDeployment() { + resolvedDeploymentIDs.Add(key.GetDeploymentId()) + } } - if previousAlert.GetLifecycleStage() == storage.LifecycleStage_RUNTIME || - previousAlert.GetState() == storage.ViolationState_ATTEMPTED { - // If we are in the context of a deployment removal, then mark the deployment as inactive without going to the - // store -- this is because the alert processing related to the deployment removal (ie, this code) runs - // _before_ the deployment is actually deleted, which means we will incorrectly mark the deployment as active - // if we check the store. - // If we're not in the context of a deployment removal, then just check whether the deployment is inactive - // in the store and use that. - if deployment := previousAlert.GetDeployment(); deployment != nil { - depID := deployment.GetId() + if key.GetLifecycleStage() == storage.LifecycleStage_RUNTIME || + key.GetState() == storage.ViolationState_ATTEMPTED { + if key.HasDeployment() && !key.IsDeploymentInactive() { + if alreadyUpdated.Contains(key.GetId()) { + continue + } + depID := key.GetDeploymentId() if deploymentsBeingRemoved.Contains(depID) || d.runtimeDetector.DeploymentInactive(depID) { - if deployment := previousAlert.GetDeployment(); deployment != nil && !deployment.GetInactive() { - deployment.Inactive = true - updatedAlerts = append(updatedAlerts, previousAlert) - } + needInactiveIDs = append(needInactiveIDs, key.GetId()) } } } } + + // Stored runtime alerts with no incoming write still need an inactive stamp. + inactiveUpdates, inactiveErr := d.markDeploymentsInactive(ctx, needInactiveIDs) + if inactiveErr != nil { + err = inactiveErr + return + } + updatedAlerts = append(updatedAlerts, inactiveUpdates...) + + recordAlertOutcomes(len(newAlerts), len(updatedAlerts), len(toBeResolvedIDs)) + return +} + +// fetchAndMergeCandidates fetches full alerts for merge candidates and merges +// them with incoming alerts. Returns merged IDs, alerts treated as new +// (deleted between phases), and updated alerts from merging. +func (d *alertManagerImpl) fetchAndMergeCandidates(ctx context.Context, candidates []mergeCandidate) (mergedIDs set.StringSet, newAlerts, updatedAlerts []*storage.Alert, err error) { + mergedIDs = set.NewStringSet() + if len(candidates) == 0 { + return + } + + uniqueIDs := set.NewStringSet() + for _, mc := range candidates { + uniqueIDs.Add(mc.oldKeyID) + } + mergeQuery := search.NewQueryBuilder(). + AddExactMatches(search.AlertID, uniqueIDs.AsSlice()...). + ProtoQuery() + fetched, fetchErr := d.alerts.SearchRawAlerts(ctx, mergeQuery, false) + if fetchErr != nil { + err = errors.Wrap(fetchErr, "failed to fetch alerts for merge") + return + } + oldAlertsByID := make(map[string]*storage.Alert, len(fetched)) + for _, a := range fetched { + oldAlertsByID[a.GetId()] = a + } + + for _, mc := range candidates { + matchingOld := oldAlertsByID[mc.oldKeyID] + if matchingOld == nil { + mc.incoming.FirstOccurred = mc.incoming.GetTime() + newAlerts = append(newAlerts, mc.incoming) + continue + } + mergedAlert := mergeAlerts(matchingOld, mc.incoming) + if !mergedAlert.EqualVT(matchingOld) { + updatedAlerts = append(updatedAlerts, mergedAlert) + } + mergedIDs.Add(mc.oldKeyID) + } return } -func (d *alertManagerImpl) shouldMarkAlertResolved(oldAlert *storage.Alert, incomingAlerts []*storage.Alert, oldAlertFilters ...AlertFilterOption) bool { - oldAndNew := []*storage.Alert{oldAlert} - oldAndNew = append(oldAndNew, incomingAlerts...) +// markDeploymentsInactive fetches alerts by ID and sets deployment.Inactive = true +// for any that have an active deployment. +func (d *alertManagerImpl) markDeploymentsInactive(ctx context.Context, alertIDs []string) ([]*storage.Alert, error) { + if len(alertIDs) == 0 { + return nil, nil + } + + inactiveQuery := search.NewQueryBuilder(). + AddExactMatches(search.AlertID, alertIDs...). + ProtoQuery() + inactiveAlerts, err := d.alerts.SearchRawAlerts(ctx, inactiveQuery, false) + if err != nil { + return nil, errors.Wrap(err, "failed to fetch alerts for inactive marking") + } + + var updated []*storage.Alert + for _, fullAlert := range inactiveAlerts { + if deployment := fullAlert.GetDeployment(); deployment != nil && !deployment.GetInactive() { + deployment.Inactive = true + updated = append(updated, fullAlert) + } + } + return updated, nil +} + +// stampInactiveIfDeploymentGone sets deployment.Inactive when the deployment is +// being removed or is already gone. Sensor can still send runtime alerts after that. +func (d *alertManagerImpl) stampInactiveIfDeploymentGone(alert *storage.Alert, deploymentsBeingRemoved set.StringSet) { + if alert.GetLifecycleStage() != storage.LifecycleStage_RUNTIME && + alert.GetState() != storage.ViolationState_ATTEMPTED { + return + } + dep := alert.GetDeployment() + if dep == nil || dep.GetInactive() { + return + } + if deploymentsBeingRemoved.Contains(dep.GetId()) || d.runtimeDetector.DeploymentInactive(dep.GetId()) { + dep.Inactive = true + } +} + +func (d *alertManagerImpl) shouldMarkAlertResolved(old alertviews.AlertMatcher, incomingAlerts []*storage.Alert, oldAlertFilters ...AlertFilterOption) bool { // Do not mark any attempted alerts as stale. All attempted alerts must be resolved by users. - if pkgAlert.AnyAttemptedAlert(oldAndNew...) { + if old.GetState() == storage.ViolationState_ATTEMPTED { return false } + for _, incoming := range incomingAlerts { + if incoming.GetState() == storage.ViolationState_ATTEMPTED { + return false + } + } // If the alert is still being produced, don't mark it stale. - if matchingNew := findAlert(oldAlert, incomingAlerts); matchingNew != nil { + if matchingNew := findMatchingAlert(old, incomingAlerts); matchingNew != nil { return false } // Only runtime alerts should not be marked stale when they are no longer produced. // (Deploy time alerts should disappear along with deployments, for example.) - if oldAlert.GetLifecycleStage() != storage.LifecycleStage_RUNTIME { + if old.GetLifecycleStage() != storage.LifecycleStage_RUNTIME { return true } - if !d.runtimeDetector.PolicySet().Exists(oldAlert.GetPolicy().GetId()) { + if !d.runtimeDetector.PolicySet().Exists(old.GetPolicyId()) { return true } @@ -484,47 +619,91 @@ func (d *alertManagerImpl) shouldMarkAlertResolved(oldAlert *storage.Alert, inco } // Some other policies were updated, we don't want to mark this alert stale in response. - if !specifiedPolicyIDs.Contains(oldAlert.GetPolicy().GetId()) { + if !specifiedPolicyIDs.Contains(old.GetPolicyId()) { return false } // If the deployment is excluded from the scope of the policy now, we should mark the alert stale, otherwise we will keep it around. - return d.runtimeDetector.DeploymentWhitelistedForPolicy(oldAlert.GetDeployment().GetId(), oldAlert.GetPolicy().GetId()) + return d.runtimeDetector.DeploymentWhitelistedForPolicy(old.GetDeploymentId(), old.GetPolicyId()) +} + +// alertAdapter wraps *storage.Alert to satisfy the AlertMatcher interface. +type alertAdapter struct{ a *storage.Alert } + +func (w alertAdapter) GetId() string { return w.a.GetId() } +func (w alertAdapter) GetPolicyId() string { return w.a.GetPolicy().GetId() } +func (w alertAdapter) GetState() storage.ViolationState { return w.a.GetState() } +func (w alertAdapter) GetLifecycleStage() storage.LifecycleStage { return w.a.GetLifecycleStage() } +func (w alertAdapter) HasDeployment() bool { return w.a.GetDeployment() != nil } +func (w alertAdapter) GetDeploymentId() string { return w.a.GetDeployment().GetId() } +func (w alertAdapter) IsDeploymentInactive() bool { return w.a.GetDeployment().GetInactive() } +func (w alertAdapter) HasResource() bool { return w.a.GetResource() != nil } +func (w alertAdapter) GetResourceType() storage.Alert_Resource_ResourceType { + return w.a.GetResource().GetResourceType() +} +func (w alertAdapter) GetResourceName() string { return w.a.GetResource().GetName() } +func (w alertAdapter) HasNode() bool { return w.a.GetNode() != nil } +func (w alertAdapter) GetNodeId() string { return w.a.GetNode().GetId() } +func (w alertAdapter) GetNodeName() string { return w.a.GetNode().GetName() } + +// GetClusterId returns the entity-specific cluster ID to match the original +// comparison behavior. Resource and node alerts store the cluster ID on their +// sub-objects; the top-level Alert.ClusterId is a denormalized copy that may +// not be set in unit test fixtures. +func (w alertAdapter) GetClusterId() string { + if r := w.a.GetResource(); r != nil { + return r.GetClusterId() + } + if n := w.a.GetNode(); n != nil { + return n.GetClusterId() + } + return w.a.GetClusterId() +} + +// GetNamespace returns the entity-specific namespace. Resource alerts store the +// namespace on Alert_Resource; deployment alerts use the top-level field. +func (w alertAdapter) GetNamespace() string { + if r := w.a.GetResource(); r != nil { + return r.GetNamespace() + } + return w.a.GetNamespace() } -func findAlert(toFind *storage.Alert, alerts []*storage.Alert) *storage.Alert { +func findMatchingAlert(toFind alertviews.AlertMatcher, alerts []*storage.Alert) *storage.Alert { for _, alert := range alerts { - if alertsAreForSamePolicyAndEntity(alert, toFind) { + if alertsAreForSamePolicyAndEntity(toFind, alertAdapter{alert}) { return alert } } return nil } -func alertsAreForSamePolicyAndEntity(a1, a2 *storage.Alert) bool { - if a1.GetPolicy().GetId() != a2.GetPolicy().GetId() || a1.GetState() != a2.GetState() { +func findMatchingKey(toFind *storage.Alert, keys []*alertviews.AlertMatchKey) *alertviews.AlertMatchKey { + wrapper := alertAdapter{toFind} + for _, key := range keys { + if alertsAreForSamePolicyAndEntity(wrapper, key) { + return key + } + } + return nil +} + +func alertsAreForSamePolicyAndEntity(a1, a2 alertviews.AlertMatcher) bool { + if a1.GetPolicyId() != a2.GetPolicyId() || a1.GetState() != a2.GetState() { return false } - if a1.GetDeployment() != nil && a2.GetDeployment() != nil { - return a1.GetDeployment().GetId() == a2.GetDeployment().GetId() - } else if a1.GetResource() != nil && a2.GetResource() != nil { - return alertsAreForSameResource(a1.GetResource(), a2.GetResource()) - } else if a1.GetNode() != nil && a2.GetNode() != nil { - return alertsAreForSameNode(a1.GetNode(), a2.GetNode()) + if a1.HasDeployment() && a2.HasDeployment() { + return a1.GetDeploymentId() == a2.GetDeploymentId() + } else if a1.HasResource() && a2.HasResource() { + return a1.GetResourceType() == a2.GetResourceType() && + a1.GetResourceName() == a2.GetResourceName() && + a1.GetClusterId() == a2.GetClusterId() && + a1.GetNamespace() == a2.GetNamespace() + } else if a1.HasNode() && a2.HasNode() { + return a1.GetNodeId() == a2.GetNodeId() && + a1.GetNodeName() == a2.GetNodeName() && + a1.GetClusterId() == a2.GetClusterId() } return false } - -func alertsAreForSameResource(a1, a2 *storage.Alert_Resource) bool { - return a1.GetResourceType() == a2.GetResourceType() && - a1.GetName() == a2.GetName() && - a1.GetClusterId() == a2.GetClusterId() && - a1.GetNamespace() == a2.GetNamespace() -} - -func alertsAreForSameNode(a1, a2 *storage.Alert_Node) bool { - return a1.GetId() == a2.GetId() && - a1.GetName() == a2.GetName() && - a1.GetClusterId() == a2.GetClusterId() -} diff --git a/central/detection/alertmanager/alert_manager_impl_test.go b/central/detection/alertmanager/alert_manager_impl_test.go index 4af007f537c59..aecff5c1961c8 100644 --- a/central/detection/alertmanager/alert_manager_impl_test.go +++ b/central/detection/alertmanager/alert_manager_impl_test.go @@ -8,6 +8,7 @@ import ( "time" alertMocks "github.com/stackrox/rox/central/alert/datastore/mocks" + alertviews "github.com/stackrox/rox/central/alert/views" "github.com/stackrox/rox/central/detection" runtimeDetectorMocks "github.com/stackrox/rox/central/detection/runtime/mocks" policyMocks "github.com/stackrox/rox/central/policy/datastore/mocks" @@ -49,6 +50,61 @@ var ( twoDaysAgoFileAccessViolation = getFileAccess(twoDaysAgo) ) +// alertToMatchKey converts a *storage.Alert to an *alertviews.AlertMatchKey, +// extracting the fields the same way the alertAdapter does in the impl file. +func alertToMatchKey(a *storage.Alert) *alertviews.AlertMatchKey { + key := &alertviews.AlertMatchKey{ + ID: a.GetId(), + PolicyID: a.GetPolicy().GetId(), + State: int(a.GetState()), + LifecycleStage: int(a.GetLifecycleStage()), + } + + // Deployment fields + if dep := a.GetDeployment(); dep != nil { + key.DeploymentID = new(dep.GetId()) + key.DeploymentInactive = new(dep.GetInactive()) + } + + // Resource fields + if res := a.GetResource(); res != nil { + key.ResourceType = new(int(res.GetResourceType())) + key.ResourceName = new(res.GetName()) + key.ClusterID = new(res.GetClusterId()) + key.Namespace = new(res.GetNamespace()) + } + + // Node fields + if node := a.GetNode(); node != nil { + key.NodeID = new(node.GetId()) + key.NodeName = new(node.GetName()) + if key.ClusterID == nil || *key.ClusterID == "" { + key.ClusterID = new(node.GetClusterId()) + } + } + + // Top-level cluster ID fallback (for deployment alerts) + if key.ClusterID == nil || *key.ClusterID == "" { + key.ClusterID = new(a.GetClusterId()) + } + + // Top-level namespace fallback (for deployment alerts) + if key.Namespace == nil || *key.Namespace == "" { + key.Namespace = new(a.GetNamespace()) + } + + return key +} + +// alertsToMatchKeys converts a slice of *storage.Alert to a slice of *alertviews.AlertMatchKey. +func alertsToMatchKeys(alerts []*storage.Alert) []*alertviews.AlertMatchKey { + keys := make([]*alertviews.AlertMatchKey, len(alerts)) + for i, alert := range alerts { + keys[i] = alertToMatchKey(alert) + } + return keys +} + func getKubeEventViolation(msg string, violationTime time.Time) *storage.Alert_Violation { return &storage.Alert_Violation{ Message: msg, @@ -96,7 +152,7 @@ func getFileAccess(accessTime time.Time) *storage.FileAccess { } func getFakeFileAccessAlert(accesses ...*storage.FileAccess) *storage.Alert { - var violations []*storage.Alert_Violation + violations := make([]*storage.Alert_Violation, 0, len(accesses)) for _, access := range accesses { violations = append(violations, printer.GenerateFileAccessViolation(access)) } @@ -115,6 +171,26 @@ func getFakeRuntimeAlert(indicators ...*storage.ProcessIndicator) *storage.Alert } } +func runtimeDeploymentAlert(id, depID string, inactive bool, process *storage.ProcessIndicator) *storage.Alert { + v := &storage.Alert_ProcessViolation{Processes: []*storage.ProcessIndicator{process}} + printer.UpdateProcessAlertViolationMessage(v) + return &storage.Alert{ + Id: id, + Policy: getPolicies()[0], + Entity: &storage.Alert_Deployment_{ + Deployment: &storage.Alert_Deployment{ + Id: depID, + Name: depID, + Inactive: inactive, + }, + }, + LifecycleStage: storage.LifecycleStage_RUNTIME, + State: storage.ViolationState_ACTIVE, + Time: protocompat.GetProtoTimestampFromSeconds(100), + ProcessViolation: v, + } +} + func getFakeResourceRuntimeAlert(resourceType storage.Alert_Resource_ResourceType, resourceName, clusterID, namespaceID, namespace string) *storage.Alert { return &storage.Alert{ LifecycleStage: storage.LifecycleStage_RUNTIME, @@ -160,7 +236,7 @@ func (suite *AlertManagerTestSuite) SetupTest() { suite.alertsMock = alertMocks.NewMockDataStore(suite.mockCtrl) suite.notifierMock = notifierMocks.NewMockProcessor(suite.mockCtrl) suite.runtimeDetectorMock = runtimeDetectorMocks.NewMockDetector(suite.mockCtrl) - suite.policySet = detection.NewPolicySet(policyMocks.NewMockDataStore(suite.mockCtrl)) + suite.policySet = detection.NewPolicySet(policyMocks.NewMockDataStore(suite.mockCtrl), nil, nil) suite.alertManager = New(suite.notifierMock, suite.alertsMock, suite.runtimeDetectorMock) } @@ -229,7 +305,7 @@ func (suite *AlertManagerTestSuite) TestNotifyAndUpdateBatch() { } func (suite *AlertManagerTestSuite) TestGetAlertsByPolicy() { - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, testutils.PredMatcher("query for violation state, policy", queryHasFields(search.ViolationState, search.PolicyID)), true).Return(([]*storage.Alert)(nil), nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, testutils.PredMatcher("query for violation state, policy", queryHasFields(search.ViolationState, search.PolicyID)), true).Return(([]*alertviews.AlertMatchKey)(nil), nil) modified, err := suite.alertManager.AlertAndNotify(suite.ctx, nil, WithPolicyID("pid")) suite.False(modified.Cardinality() > 0) @@ -237,7 +313,7 @@ func (suite *AlertManagerTestSuite) TestGetAlertsByPolicy() { } func (suite *AlertManagerTestSuite) TestGetAlertsByDeployment() { - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, testutils.PredMatcher("query for violation state, deployment", queryHasFields(search.ViolationState, search.DeploymentID)), true).Return(([]*storage.Alert)(nil), nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, testutils.PredMatcher("query for violation state, deployment", queryHasFields(search.ViolationState, search.DeploymentID)), true).Return(([]*alertviews.AlertMatchKey)(nil), nil) modified, err := suite.alertManager.AlertAndNotify(suite.ctx, nil, WithDeploymentID("did", false)) suite.False(modified.Cardinality() > 0) @@ -245,9 +321,9 @@ func (suite *AlertManagerTestSuite) TestGetAlertsByDeployment() { } func (suite *AlertManagerTestSuite) TestGetAlertsByClusterAndResource() { - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, testutils.PredMatcher("query for violation state, cluster id and resource type", queryHasFields(search.ViolationState, search.ClusterID, search.ResourceType)), true, - ).Return(([]*storage.Alert)(nil), nil) + ).Return(([]*alertviews.AlertMatchKey)(nil), nil) modified, err := suite.alertManager.AlertAndNotify(suite.ctx, nil, WithLifecycleStage(storage.LifecycleStage_RUNTIME), WithClusterID("cid"), WithNamespace("nn"), WithResource("rn", storage.Alert_Resource_SECRETS)) suite.False(modified.Cardinality() > 0) @@ -257,8 +333,11 @@ func (suite *AlertManagerTestSuite) TestGetAlertsByClusterAndResource() { func (suite *AlertManagerTestSuite) TestOnUpdatesWhenAlertsDoNotChange() { alerts := getAlerts() - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) - // No updates should be attempted + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) + + // Phase 2 batch-fetches all matching alerts (they match exactly so mergeAlerts returns the old alert unchanged) + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return(alerts, nil) + // No updates should be attempted because the merged alerts equal the old alerts modified, err := suite.alertManager.AlertAndNotify(suite.ctx, alerts) suite.False(modified.Cardinality() > 0) @@ -268,12 +347,15 @@ func (suite *AlertManagerTestSuite) TestOnUpdatesWhenAlertsDoNotChange() { func (suite *AlertManagerTestSuite) TestMarksOldAlertsResolved() { alerts := getAlerts() - suite.alertsMock.EXPECT().MarkAlertsResolvedBatch(suite.ctx, alerts[0].GetId()).Return([]*storage.Alert{alerts[0]}, nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) - // Unchanged alerts should not be updated. + // Phase 2 batch-fetches the matched alerts (alerts[1] and alerts[2]) + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return(alerts[1:], nil) - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) - // We should get a notification for the new alert. + // Unchanged alerts should not be updated (they match exactly so mergeAlerts returns old == new). + + // Alert 0 should be marked as resolved and we should get a notification for it. + suite.alertsMock.EXPECT().MarkAlertsResolvedBatch(suite.ctx, alerts[0].GetId()).Return([]*storage.Alert{alerts[0]}, nil) suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), alerts[0]).Return() // Make one of the alerts not appear in the current alerts. @@ -285,14 +367,15 @@ func (suite *AlertManagerTestSuite) TestMarksOldAlertsResolved() { func (suite *AlertManagerTestSuite) TestSendsNotificationsForNewAlerts() { alerts := getAlerts() - // Only the new alert will be updated. - suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, alerts[0]).Return(nil) + // Make one of the alerts not appear in the previous alerts. + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts[1:]), nil) - // We should get a notification for the new alert. - suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), alerts[0]).Return() + // Phase 2 batch-fetches the matched alerts (alerts[1] and alerts[2]) + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return(alerts[1:], nil) - // Make one of the alerts not appear in the previous alerts. - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts[1:], nil) + // Only the new alert (alerts[0]) will be notified and upserted. + suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), alerts[0]).Return() + suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, alerts[0]).Return(nil) modified, err := suite.alertManager.AlertAndNotify(suite.ctx, alerts) suite.True(modified.Cardinality() > 0) @@ -309,7 +392,7 @@ func (suite *AlertManagerTestSuite) TestNewResourceAlertIsAdded() { // We should get a notification for the new alert. suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), newAlert).Return() - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) // Add all the policies from the old alerts so that they aren't marked as stale for _, a := range alerts { @@ -330,13 +413,16 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlerts() { expectedMergedAlert := newAlert.CloneVT() expectedMergedAlert.Violations = append(expectedMergedAlert.Violations, alerts[0].GetViolations()...) + // Phase 2 batch-fetches the matched alert for merging + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return([]*storage.Alert{alerts[0]}, nil) + // Only the merged alert will be updated. suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, protomock.GoMockMatcherEqualMessage(expectedMergedAlert)).Return(nil) // Updated alert should notify suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), newAlert).Return() - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) // Add all the policies from the old alerts so that they aren't marked as stale for _, a := range alerts { @@ -358,12 +444,15 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlertsNoNotify() { expectedMergedAlert := newAlert.CloneVT() expectedMergedAlert.Violations = append(expectedMergedAlert.Violations, alerts[0].GetViolations()...) + // Phase 2 batch-fetches the matched alert for merging + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return([]*storage.Alert{alerts[0]}, nil) + // Only the merged alert will be updated. suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, protomock.GoMockMatcherEqualMessage(expectedMergedAlert)).Return(nil) // Updated alert should not notify - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) // Add all the policies from the old alerts so that they aren't marked as stale for _, a := range alerts { @@ -383,6 +472,9 @@ func (suite *AlertManagerTestSuite) TestMergeMultipleResourceAlerts() { newAlert2 := alerts[0].CloneVT() newAlert2.Violations[0].Message = "new-violation-2" + // Phase 2 batch-fetches the matched alert for merging (one unique ID) + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return([]*storage.Alert{alerts[0]}, nil) + // There will be two calls to Upsert suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, gomock.Any()).Return(nil) suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, gomock.Any()).Return(nil) @@ -391,7 +483,7 @@ func (suite *AlertManagerTestSuite) TestMergeMultipleResourceAlerts() { suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), newAlert).Return() suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), newAlert2).Return() - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) // Add all the policies from the old alerts so that they aren't marked as stale for _, a := range alerts { @@ -408,7 +500,7 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlertsKeepsNewViolationsIfM alerts := getResourceAlerts() newAlert := alerts[0].CloneVT() newAlert.Violations = make([]*storage.Alert_Violation, maxRunTimeViolationsPerAlert) - for i := 0; i < maxRunTimeViolationsPerAlert; i++ { + for i := range maxRunTimeViolationsPerAlert { newAlert.Violations[i] = &storage.Alert_Violation{Message: fmt.Sprintf("new-violation-%d", i), Type: storage.Alert_Violation_K8S_EVENT} } @@ -416,6 +508,9 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlertsKeepsNewViolationsIfM expectedMergedAlert.Violations = append(expectedMergedAlert.Violations, alerts[0].GetViolations()...) expectedMergedAlert.Violations = expectedMergedAlert.GetViolations()[:maxRunTimeViolationsPerAlert] + // Phase 2 batch-fetches the matched alert for merging + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return([]*storage.Alert{alerts[0]}, nil) + // Only the merged alert will be updated. suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, protomock.GoMockMatcherEqualMessage(expectedMergedAlert)).Return(nil) @@ -424,7 +519,7 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlertsKeepsNewViolationsIfM suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), newAlert).Return() } - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) // Add all the policies from the old alerts so that they aren't marked as stale for _, a := range alerts { @@ -442,7 +537,7 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlertsKeepsNewViolationsIfM alerts := getResourceAlerts() newAlert := alerts[0].CloneVT() newAlert.Violations = make([]*storage.Alert_Violation, maxRunTimeViolationsPerAlert) - for i := 0; i < maxRunTimeViolationsPerAlert; i++ { + for i := range maxRunTimeViolationsPerAlert { newAlert.Violations[i] = &storage.Alert_Violation{Message: fmt.Sprintf("new-violation-%d", i), Type: storage.Alert_Violation_K8S_EVENT} } @@ -450,12 +545,15 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlertsKeepsNewViolationsIfM expectedMergedAlert.Violations = append(expectedMergedAlert.Violations, alerts[0].GetViolations()...) expectedMergedAlert.Violations = expectedMergedAlert.GetViolations()[:maxRunTimeViolationsPerAlert] + // Phase 2 batch-fetches the matched alert for merging + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return([]*storage.Alert{alerts[0]}, nil) + // Only the merged alert will be updated. suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, protomock.GoMockMatcherEqualMessage(expectedMergedAlert)).Return(nil) // Updated alert should not notify - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) // Add all the policies from the old alerts so that they aren't marked as stale for _, a := range alerts { @@ -471,7 +569,7 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlertsKeepsNewViolationsIfM func (suite *AlertManagerTestSuite) TestMergeResourceAlertsOnlyKeepsMaxViolations() { alerts := getResourceAlerts() alerts[0].Violations = make([]*storage.Alert_Violation, maxRunTimeViolationsPerAlert) - for i := 0; i < maxRunTimeViolationsPerAlert; i++ { + for i := range maxRunTimeViolationsPerAlert { alerts[0].Violations[i] = &storage.Alert_Violation{Message: fmt.Sprintf("old-violation-%d", i), Type: storage.Alert_Violation_K8S_EVENT} } newAlert := alerts[0].CloneVT() @@ -479,13 +577,16 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlertsOnlyKeepsMaxViolation expectedMergedAlert := newAlert.CloneVT() + // Phase 2 batch-fetches the matched alert for merging + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return([]*storage.Alert{alerts[0]}, nil) + // Only the merged alert will be updated. suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, protomock.GoMockMatcherEqualMessage(expectedMergedAlert)).Return(nil) // Updated alert should notify if set to suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), newAlert).Return() - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) // Add all the policies from the old alerts so that they aren't marked as stale for _, a := range alerts { @@ -502,7 +603,7 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlertsOnlyKeepsMaxViolation suite.T().Setenv("NOTIFY_EVERY_RUNTIME_EVENT", "false") alerts := getResourceAlerts() alerts[0].Violations = make([]*storage.Alert_Violation, maxRunTimeViolationsPerAlert) - for i := 0; i < maxRunTimeViolationsPerAlert; i++ { + for i := range maxRunTimeViolationsPerAlert { alerts[0].Violations[i] = &storage.Alert_Violation{Message: fmt.Sprintf("old-violation-%d", i), Type: storage.Alert_Violation_K8S_EVENT} } newAlert := alerts[0].CloneVT() @@ -510,12 +611,15 @@ func (suite *AlertManagerTestSuite) TestMergeResourceAlertsOnlyKeepsMaxViolation expectedMergedAlert := newAlert.CloneVT() + // Phase 2 batch-fetches the matched alert for merging + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return([]*storage.Alert{alerts[0]}, nil) + // Only the merged alert will be updated. suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, protomock.GoMockMatcherEqualMessage(expectedMergedAlert)).Return(nil) // Updated alert should not notify - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) // Add all the policies from the old alerts so that they aren't marked as stale for _, a := range alerts { @@ -538,7 +642,7 @@ func (suite *AlertManagerTestSuite) TestOldResourceAlertAreMarkedAsResolvedWhenP // We should get a notifications for new alert suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), newAlert).Return() - suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), true).Return(alerts, nil) + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts), nil) // Don't add any policies to simulate policies being deleted suite.runtimeDetectorMock.EXPECT().PolicySet().Return(suite.policySet).AnyTimes() @@ -560,6 +664,227 @@ func (suite *AlertManagerTestSuite) TestOldResourceAlertAreMarkedAsResolvedWhenP suite.NoError(err, "update should succeed") } +func (suite *AlertManagerTestSuite) TestAlertDeletedBetweenPhases() { + alerts := getAlerts() + incoming := []*storage.Alert{alerts[0]} + + // Phase 1 returns a key that matches the incoming alert. + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(alertsToMatchKeys(alerts[:1]), nil) + + // Phase 2: the alert was deleted between phases — SearchRawAlerts returns empty. + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return(nil, nil) + + // Since the alert is gone, it should be treated as new. + suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, gomock.Any()).DoAndReturn(func(_ context.Context, a *storage.Alert) error { + suite.Equal(alerts[0].GetId(), a.GetId()) + suite.NotNil(a.GetFirstOccurred(), "FirstOccurred should be set for new alerts") + return nil + }) + suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), gomock.Any()).Return() + + modified, err := suite.alertManager.AlertAndNotify(suite.ctx, incoming) + suite.NoError(err) + suite.True(modified.Cardinality() > 0) +} + +func (suite *AlertManagerTestSuite) TestDeploymentMarkedInactiveOnRemoval() { + dep := &storage.Alert_Deployment{ + Id: "dep-to-remove", + Name: "dep-to-remove", + } + previousAlert := &storage.Alert{ + Id: "runtime-alert-1", + Policy: getPolicies()[0], + Entity: &storage.Alert_Deployment_{Deployment: dep}, + LifecycleStage: storage.LifecycleStage_RUNTIME, + State: storage.ViolationState_ACTIVE, + Time: protocompat.GetProtoTimestampFromSeconds(100), + } + + // Phase 1 returns the key for the previous runtime alert. + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true). + Return(alertsToMatchKeys([]*storage.Alert{previousAlert}), nil) + + // The runtime detector says the deployment is being removed. + suite.runtimeDetectorMock.EXPECT().DeploymentInactive("dep-to-remove").Return(true).AnyTimes() + suite.runtimeDetectorMock.EXPECT().PolicySet().Return(suite.policySet).AnyTimes() + + // Batch-fetch full alert for inactive marking. + fullAlert := previousAlert.CloneVT() + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return([]*storage.Alert{fullAlert}, nil) + + // The alert should be updated with deployment.Inactive = true. + suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, gomock.Any()).DoAndReturn(func(_ context.Context, a *storage.Alert) error { + suite.True(a.GetDeployment().GetInactive(), "deployment should be marked inactive") + return nil + }) + + // The alert should also be resolved (no incoming alerts match it). + suite.alertsMock.EXPECT().MarkAlertsResolvedBatch(suite.ctx, "runtime-alert-1").Return([]*storage.Alert{previousAlert}, nil) + // Two notifications: one for the resolved alert, one from notifyUpdatedRuntimeAlerts. + suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), gomock.Any()).Return().Times(2) + + modified, err := suite.alertManager.AlertAndNotify(suite.ctx, nil, + WithDeploymentID("dep-to-remove", true)) + suite.NoError(err) + suite.True(modified.Contains("dep-to-remove"), "removed deployment should appear in modified set") +} + +func (suite *AlertManagerTestSuite) TestRepeatedDeploymentRemovalPreservesRuntimeAlert() { + stored := &storage.Alert{ + Id: "runtime-alert-1", + Policy: fixtures.GetNetworkFlowPolicy(), + Entity: &storage.Alert_Deployment_{Deployment: &storage.Alert_Deployment{Id: "dep-to-remove"}}, + LifecycleStage: storage.LifecycleStage_RUNTIME, + State: storage.ViolationState_ACTIVE, + Time: protocompat.GetProtoTimestampFromSeconds(100), + } + suite.Require().NoError(suite.policySet.UpsertPolicy(stored.GetPolicy())) + suite.runtimeDetectorMock.EXPECT().PolicySet().Return(suite.policySet).AnyTimes() + + // Each removal reads the current persisted state. Only the first should write. + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true). + DoAndReturn(func(context.Context, *v1.Query, bool) ([]*alertviews.AlertMatchKey, error) { + return alertsToMatchKeys([]*storage.Alert{stored}), nil + }).Times(2) + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false). + Return([]*storage.Alert{stored.CloneVT()}, nil) + suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, gomock.Any()). + DoAndReturn(func(_ context.Context, a *storage.Alert) error { + suite.True(a.GetDeployment().GetInactive()) + suite.Equal(storage.ViolationState_ACTIVE, a.GetState()) + stored = a.CloneVT() + return nil + }) + suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), gomock.Any()).Return() + + modified, err := suite.alertManager.AlertAndNotify(suite.ctx, nil, + WithDeploymentID("dep-to-remove", true)) + suite.Require().NoError(err) + suite.True(modified.Contains("dep-to-remove")) + + modified, err = suite.alertManager.AlertAndNotify(suite.ctx, nil, + WithDeploymentID("dep-to-remove", true)) + suite.Require().NoError(err) + suite.Empty(modified.AsSlice()) + suite.True(stored.GetDeployment().GetInactive()) + suite.Equal(storage.ViolationState_ACTIVE, stored.GetState()) +} + +// TestNewRuntimeAlertMarkedInactiveWhenDeploymentGone covers a first runtime +// alert arriving after the deployment is already gone. +func (suite *AlertManagerTestSuite) TestNewRuntimeAlertMarkedInactiveWhenDeploymentGone() { + incoming := runtimeDeploymentAlert("runtime-alert-new", "dep-gone", false, nowProcess) + + suite.runtimeDetectorMock.EXPECT().DeploymentInactive("dep-gone").Return(true).AnyTimes() + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true).Return(nil, nil) + suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, gomock.Any()).DoAndReturn(func(_ context.Context, a *storage.Alert) error { + suite.True(a.GetDeployment().GetInactive(), "new runtime alert should be marked inactive") + return nil + }) + suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), gomock.Any()).Return() + + modified, err := suite.alertManager.AlertAndNotify(suite.ctx, []*storage.Alert{incoming}) + suite.NoError(err) + suite.True(modified.Contains("dep-gone")) +} + +// TestMergedRuntimeAlertMarkedInactiveWhenDeploymentGone covers a later runtime +// alert merging into a stored one after the deployment is already gone. +func (suite *AlertManagerTestSuite) TestMergedRuntimeAlertMarkedInactiveWhenDeploymentGone() { + previous := runtimeDeploymentAlert("runtime-alert-1", "dep-gone", false, yesterdayProcess) + incoming := runtimeDeploymentAlert("runtime-alert-incoming", "dep-gone", false, nowProcess) + + suite.runtimeDetectorMock.EXPECT().DeploymentInactive("dep-gone").Return(true).AnyTimes() + suite.runtimeDetectorMock.EXPECT().PolicySet().Return(suite.policySet).AnyTimes() + + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true). + Return(alertsToMatchKeys([]*storage.Alert{previous}), nil) + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return([]*storage.Alert{previous.CloneVT()}, nil) + + suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, gomock.Any()).DoAndReturn(func(_ context.Context, a *storage.Alert) error { + suite.True(a.GetDeployment().GetInactive(), "merged runtime alert should be marked inactive") + suite.Len(a.GetProcessViolation().GetProcesses(), 2, "inactive stamp must not replace the merged processes") + return nil + }) + suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), gomock.Any()).Return() + + modified, err := suite.alertManager.AlertAndNotify(suite.ctx, []*storage.Alert{incoming}) + suite.NoError(err) + suite.True(modified.Contains("dep-gone")) +} + +func (suite *AlertManagerTestSuite) TestUnchangedRuntimeAlertMarkedInactiveWhenDeploymentGone() { + previous := runtimeDeploymentAlert("runtime-alert-1", "dep-gone", false, yesterdayProcess) + incoming := runtimeDeploymentAlert("runtime-alert-incoming", "dep-gone", false, yesterdayProcess) + + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true). + Return(alertsToMatchKeys([]*storage.Alert{previous}), nil) + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false). + Return([]*storage.Alert{previous.CloneVT()}, nil) + suite.runtimeDetectorMock.EXPECT().DeploymentInactive("dep-gone").Return(true) + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false). + Return([]*storage.Alert{previous.CloneVT()}, nil) + suite.alertsMock.EXPECT().UpsertAlert(suite.ctx, gomock.Any()).DoAndReturn(func(_ context.Context, stored *storage.Alert) error { + suite.Equal(previous.GetId(), stored.GetId()) + suite.True(stored.GetDeployment().GetInactive(), "stored runtime alert should be marked inactive") + return nil + }) + suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), gomock.Any()).Return() + + modified, err := suite.alertManager.AlertAndNotify(suite.ctx, []*storage.Alert{incoming}) + suite.NoError(err) + suite.True(modified.Contains("dep-gone")) +} + +func (suite *AlertManagerTestSuite) TestResolvedDeploymentAlertReturnsDeploymentID() { + alerts := getAlerts() + + // Phase 1: return all 3 deployment alerts. + suite.alertsMock.EXPECT().SearchAlertMatchKeys(suite.ctx, gomock.Any(), true). + Return(alertsToMatchKeys(alerts), nil) + + // Incoming has alerts[1] and alerts[2] but not alerts[0]. + // alerts[1] and alerts[2] match their keys, so Phase 2 batch-fetches them. + suite.alertsMock.EXPECT().SearchRawAlerts(suite.ctx, gomock.Any(), false).Return(alerts[1:], nil) + + // alerts[0] should be resolved. + suite.alertsMock.EXPECT().MarkAlertsResolvedBatch(suite.ctx, alerts[0].GetId()).Return([]*storage.Alert{alerts[0]}, nil) + suite.notifierMock.EXPECT().ProcessAlert(gomock.Any(), alerts[0]).Return() + + modified, err := suite.alertManager.AlertAndNotify(suite.ctx, alerts[1:]) + suite.NoError(err) + suite.True(modified.Contains(alerts[0].GetDeployment().GetId()), + "resolved deployment alert's deployment ID should appear in modified set") +} + +func TestMergeAlertsPreservesDeploymentInactive(t *testing.T) { + cases := map[string]struct { + oldInactive bool + newInactive bool + want bool + }{ + "stored inactive is kept when Sensor sends a new process": { + oldInactive: true, + newInactive: false, + want: true, + }, + "active stays active when both sides are active": { + oldInactive: false, + newInactive: false, + want: false, + }, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + old := runtimeDeploymentAlert("alert-1", "dep", c.oldInactive, yesterdayProcess) + newAlert := runtimeDeploymentAlert("alert-incoming", "dep", c.newInactive, nowProcess) + merged := mergeAlerts(old, newAlert) + assert.Equal(t, c.want, merged.GetDeployment().GetInactive()) + }) + } +} + func TestMergeProcessesFromOldIntoNew(t *testing.T) { for _, c := range []struct { desc string @@ -657,14 +982,14 @@ func TestMergeFileAccessAlerts(t *testing.T) { desc: "New has many that exceed max", old: func() *storage.Alert { accesses := make([]*storage.FileAccess, 30) - for i := 0; i < 30; i++ { + for i := range 30 { accesses[i] = getFileAccess(twoDaysAgo.Add(time.Duration(i) * time.Minute)) } return getFakeFileAccessAlert(accesses...) }(), new: func() *storage.Alert { accesses := make([]*storage.FileAccess, 20) - for i := 0; i < 20; i++ { + for i := range 20 { accesses[i] = getFileAccess(yesterday.Add(time.Duration(i) * time.Minute)) } return getFakeFileAccessAlert(accesses...) @@ -676,11 +1001,11 @@ func TestMergeFileAccessAlerts(t *testing.T) { // This is old[10:30] (20 old accesses) + new[0:20] (20 new accesses) accesses := make([]*storage.FileAccess, maxRunTimeViolationsPerAlert) // 20 from old (indices 10-29, the most recent old ones) - for i := 0; i < 20; i++ { + for i := range 20 { accesses[i] = getFileAccess(twoDaysAgo.Add(time.Duration(10+i) * time.Minute)) } // All 20 from new - for i := 0; i < 20; i++ { + for i := range 20 { accesses[20+i] = getFileAccess(yesterday.Add(time.Duration(i) * time.Minute)) } return getFakeFileAccessAlert(accesses...) @@ -691,7 +1016,7 @@ func TestMergeFileAccessAlerts(t *testing.T) { desc: "Old at max; new access", old: func() *storage.Alert { accesses := make([]*storage.FileAccess, maxRunTimeViolationsPerAlert) - for i := 0; i < maxRunTimeViolationsPerAlert; i++ { + for i := range maxRunTimeViolationsPerAlert { accesses[i] = getFileAccess(twoDaysAgo.Add(time.Duration(i) * time.Minute)) } return getFakeFileAccessAlert(accesses...) @@ -981,7 +1306,7 @@ func TestFindAlert(t *testing.T) { }, } { t.Run(c.desc, func(t *testing.T) { - found := findAlert(c.toFind, c.alerts) + found := findMatchingAlert(alertAdapter{c.toFind}, c.alerts) protoassert.Equal(t, c.expected, found) }) } @@ -1019,6 +1344,7 @@ func getAlerts() []*storage.Alert { func getDeployments() []*storage.Alert_Deployment { return []*storage.Alert_Deployment{ { + Id: "deployment1", Name: "deployment1", Containers: []*storage.Alert_Deployment_Container{ { @@ -1032,6 +1358,7 @@ func getDeployments() []*storage.Alert_Deployment { }, }, { + Id: "deployment2", Name: "deployment2", Containers: []*storage.Alert_Deployment_Container{ { @@ -1045,6 +1372,7 @@ func getDeployments() []*storage.Alert_Deployment { }, }, { + Id: "deployment3", Name: "deployment3", Containers: []*storage.Alert_Deployment_Container{ { diff --git a/central/detection/alertmanager/bench_postgres_test.go b/central/detection/alertmanager/bench_postgres_test.go new file mode 100644 index 0000000000000..3e199733b58ac --- /dev/null +++ b/central/detection/alertmanager/bench_postgres_test.go @@ -0,0 +1,170 @@ +//go:build sql_integration + +package alertmanager + +import ( + "context" + "fmt" + "math/rand" + "testing" + + alertDataStore "github.com/stackrox/rox/central/alert/datastore" + runtimeDetectorMocks "github.com/stackrox/rox/central/detection/runtime/mocks" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/fixtures" + "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" + notifierMocks "github.com/stackrox/rox/pkg/notifier/mocks" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +var defaultPolicyIDs = []struct { + id string + name string + severity storage.Severity +}{ + {"2e90874a-3521-44de-85c6-5720f519a701", "Latest tag", storage.Severity_LOW_SEVERITY}, + {"fe9de18b-86db-44d5-a7c4-74173ccffe2e", "Privileged Container", storage.Severity_MEDIUM_SEVERITY}, + {"886c3c94-3a6a-4f2b-82fc-d6bf5a310840", "No CPU request or memory limit specified", storage.Severity_MEDIUM_SEVERITY}, + {"f09f8da1-6111-4ca0-8f49-294a76c65115", "Fixable CVSS >= 7", storage.Severity_HIGH_SEVERITY}, + {"cf80fb33-c7d0-4490-b6f4-e56e1f27b4e4", "Log4Shell: log4j Remote Code Execution vulnerability", storage.Severity_CRITICAL_SEVERITY}, +} + +type benchFixture struct { + ctx context.Context + alertManager AlertManager + datastore alertDataStore.DataStore + mockCtrl *gomock.Controller +} + +func setupBench(b *testing.B, totalAlerts int) *benchFixture { + b.Helper() + + testDB := pgtest.ForT(b) + ctx := sac.WithAllAccess(context.Background()) + ds := alertDataStore.GetTestPostgresDataStore(b, testDB.DB) + + mockCtrl := gomock.NewController(b) + notifier := notifierMocks.NewMockProcessor(mockCtrl) + runtimeDetector := runtimeDetectorMocks.NewMockDetector(mockCtrl) + + notifier.EXPECT().ProcessAlert(gomock.Any(), gomock.Any()).AnyTimes() + runtimeDetector.EXPECT().DeploymentInactive(gomock.Any()).Return(false).AnyTimes() + + am := New(notifier, ds, runtimeDetector) + + for range totalAlerts { + a := fixtures.GetAlertWithID(uuid.NewV4().String()) + + // Distribute across 3 deployments with 60/25/15 ratio + randVal := rand.Intn(100) + switch { + case randVal < 60: + a.GetDeployment().Id = fixtureconsts.Deployment1 + case randVal < 85: + a.GetDeployment().Id = fixtureconsts.Deployment2 + default: + a.GetDeployment().Id = fixtureconsts.Deployment3 + } + + policyInfo := defaultPolicyIDs[rand.Intn(len(defaultPolicyIDs))] + a.Policy = fixtures.GetPolicy() + a.Policy.Id = policyInfo.id + a.Policy.Name = policyInfo.name + a.Policy.Severity = policyInfo.severity + + lifecycleStages := []storage.LifecycleStage{ + storage.LifecycleStage_DEPLOY, + storage.LifecycleStage_RUNTIME, + } + a.LifecycleStage = lifecycleStages[rand.Intn(len(lifecycleStages))] + + states := []storage.ViolationState{ + storage.ViolationState_ACTIVE, + storage.ViolationState_ATTEMPTED, + } + a.State = states[rand.Intn(len(states))] + + require.NoError(b, ds.UpsertAlert(ctx, a)) + } + + return &benchFixture{ + ctx: ctx, + alertManager: am, + datastore: ds, + mockCtrl: mockCtrl, + } +} + +// makeIncomingAlerts creates incoming alerts for a specific deployment with +// the given lifecycle stage. These simulate what the detection pipeline +// produces before calling AlertAndNotify. +func makeIncomingAlerts(count int, deploymentID string, stage storage.LifecycleStage) []*storage.Alert { + alerts := make([]*storage.Alert, count) + for i := range alerts { + a := fixtures.GetAlertWithID(uuid.NewV4().String()) + a.GetDeployment().Id = deploymentID + + policyInfo := defaultPolicyIDs[rand.Intn(len(defaultPolicyIDs))] + a.Policy = fixtures.GetPolicy() + a.Policy.Id = policyInfo.id + a.Policy.Name = policyInfo.name + a.Policy.Severity = policyInfo.severity + a.LifecycleStage = stage + a.State = storage.ViolationState_ACTIVE + + alerts[i] = a + } + return alerts +} + +func BenchmarkMergeManyAlerts(b *testing.B) { + alertCounts := []int{100, 500, 1000} + + for _, totalAlerts := range alertCounts { + b.Run(fmt.Sprintf("previousAlerts=%d", totalAlerts), func(b *testing.B) { + fix := setupBench(b, totalAlerts) + impl := fix.alertManager.(*alertManagerImpl) + + // Simulate HandleDeploymentAlerts: incoming alerts for one deployment + incoming := makeIncomingAlerts(5, fixtureconsts.Deployment1, storage.LifecycleStage_DEPLOY) + filters := []AlertFilterOption{ + WithLifecycleStage(storage.LifecycleStage_DEPLOY), + WithDeploymentID(fixtureconsts.Deployment1, false), + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, _, _, err := impl.mergeManyAlerts(fix.ctx, incoming, filters...) + require.NoError(b, err) + } + }) + } +} + +func BenchmarkAlertAndNotify(b *testing.B) { + alertCounts := []int{100, 500, 1000} + + for _, totalAlerts := range alertCounts { + b.Run(fmt.Sprintf("previousAlerts=%d", totalAlerts), func(b *testing.B) { + fix := setupBench(b, totalAlerts) + + incoming := makeIncomingAlerts(5, fixtureconsts.Deployment1, storage.LifecycleStage_DEPLOY) + filters := []AlertFilterOption{ + WithLifecycleStage(storage.LifecycleStage_DEPLOY), + WithDeploymentID(fixtureconsts.Deployment1, false), + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := fix.alertManager.AlertAndNotify(fix.ctx, incoming, filters...) + require.NoError(b, err) + } + }) + } +} diff --git a/central/detection/alertmanager/metrics.go b/central/detection/alertmanager/metrics.go new file mode 100644 index 0000000000000..eb58d2c23230f --- /dev/null +++ b/central/detection/alertmanager/metrics.go @@ -0,0 +1,77 @@ +package alertmanager + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stackrox/rox/pkg/metrics" +) + +// observeDurationMs returns a function that, when called, observes the elapsed +// time in milliseconds since observeDurationMs was invoked. Intended for use +// with defer: +// +// defer observeDurationMs(myHistogram)() +func observeDurationMs(h prometheus.Histogram) func() { + start := time.Now() + return func() { + h.Observe(float64(time.Since(start).Milliseconds())) + } +} + +var ( + alertAndNotifyDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "alert_and_notify_duration_ms", + Help: "End-to-end duration of AlertAndNotify in milliseconds", + Buckets: prometheus.ExponentialBuckets(4, 2, 12), + }) + + alertAndNotifyIncomingCount = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "alert_and_notify_incoming_count", + Help: "Number of incoming alerts per AlertAndNotify call", + Buckets: prometheus.ExponentialBuckets(1, 2, 14), + }) + + mergeManyAlertsDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "merge_many_alerts_duration_ms", + Help: "Duration of mergeManyAlerts in milliseconds", + Buckets: prometheus.ExponentialBuckets(4, 2, 12), + }) + + mergeManyAlertsPreviousCount = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "merge_many_alerts_previous_count", + Help: "Number of previous alerts fetched from DB per mergeManyAlerts call", + Buckets: prometheus.ExponentialBuckets(1, 2, 14), + }) + + alertOutcomeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "alert_outcome_total", + Help: "Cumulative count of alert outcomes from mergeManyAlerts", + }, []string{"outcome"}) +) + +func recordAlertOutcomes(newCount, updatedCount, resolvedCount int) { + alertOutcomeTotal.WithLabelValues("new").Add(float64(newCount)) + alertOutcomeTotal.WithLabelValues("updated").Add(float64(updatedCount)) + alertOutcomeTotal.WithLabelValues("resolved").Add(float64(resolvedCount)) +} + +func init() { + metrics.EmplaceCollector( + alertAndNotifyDuration, + alertAndNotifyIncomingCount, + mergeManyAlertsDuration, + mergeManyAlertsPreviousCount, + alertOutcomeTotal, + ) +} diff --git a/central/detection/buildtime/detector.go b/central/detection/buildtime/detector.go index 2b2abd0089af5..6e25147fe9fd7 100644 --- a/central/detection/buildtime/detector.go +++ b/central/detection/buildtime/detector.go @@ -1,6 +1,8 @@ package buildtime import ( + "context" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/detection" ) @@ -9,7 +11,7 @@ import ( type Detector interface { PolicySet() detection.PolicySet - Detect(image *storage.Image, policyFilters ...detection.FilterOption) ([]*storage.Alert, error) + Detect(ctx context.Context, image *storage.Image, policyFilters ...detection.FilterOption) ([]*storage.Alert, error) } // NewDetector returns a new instance of a Detector. diff --git a/central/detection/buildtime/detector_impl.go b/central/detection/buildtime/detector_impl.go index 9318dce736462..afbd4fa531713 100644 --- a/central/detection/buildtime/detector_impl.go +++ b/central/detection/buildtime/detector_impl.go @@ -1,6 +1,8 @@ package buildtime import ( + "context" + "github.com/pkg/errors" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/booleanpolicy" @@ -21,7 +23,7 @@ func (d *detectorImpl) PolicySet() detection.PolicySet { // Detect runs detection on an image, returning any generated alerts. If policy categories are specified, we will only // run policies with the specified categories -func (d *detectorImpl) Detect(image *storage.Image, policyFilters ...detection.FilterOption) ([]*storage.Alert, error) { +func (d *detectorImpl) Detect(ctx context.Context, image *storage.Image, policyFilters ...detection.FilterOption) ([]*storage.Alert, error) { if image == nil { return nil, errors.New("cannot detect on a nil image") } @@ -37,7 +39,7 @@ func (d *detectorImpl) Detect(image *storage.Image, policyFilters ...detection.F return nil } } - if !compiled.AppliesTo(image) { + if !compiled.AppliesTo(ctx, image) { return nil } violations, err := compiled.MatchAgainstImage(&cacheReceptacle, image) diff --git a/central/detection/buildtime/detector_impl_test.go b/central/detection/buildtime/detector_impl_test.go index 10ab09eec76e0..430101d9725df 100644 --- a/central/detection/buildtime/detector_impl_test.go +++ b/central/detection/buildtime/detector_impl_test.go @@ -1,6 +1,7 @@ package buildtime import ( + "context" "testing" "github.com/stackrox/rox/central/detection" @@ -25,7 +26,7 @@ func getPolicy(defaultPolicies []*storage.Policy, name string, t *testing.T) *st func TestDetector(t *testing.T) { controller := gomock.NewController(t) - policySet := detection.NewPolicySet(mocks.NewMockDataStore(controller)) + policySet := detection.NewPolicySet(mocks.NewMockDataStore(controller), nil, nil) detector := NewDetector(policySet) defaultPolicies, err := policies.DefaultPolicies() @@ -70,7 +71,7 @@ func TestDetector(t *testing.T) { } { t.Run(protocompat.MarshalTextString(testCase.image), func(t *testing.T) { filter, getUnusedCategories := detection.MakeCategoryFilter(testCase.allowedCategories) - alerts, err := detector.Detect(testCase.image, filter) + alerts, err := detector.Detect(context.Background(), testCase.image, filter) require.NoError(t, err) require.ElementsMatch(t, testCase.expectedUnusedCategories, getUnusedCategories()) assert.Len(t, alerts, testCase.expectedAlerts) diff --git a/central/detection/buildtime/singleton.go b/central/detection/buildtime/singleton.go index 9cb6d3714411b..28892e50b7d97 100644 --- a/central/detection/buildtime/singleton.go +++ b/central/detection/buildtime/singleton.go @@ -3,7 +3,9 @@ package buildtime import ( "context" + clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" "github.com/stackrox/rox/central/detection" + namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" policyDataStore "github.com/stackrox/rox/central/policy/datastore" "github.com/stackrox/rox/generated/storage" policyUtils "github.com/stackrox/rox/pkg/policies" @@ -36,7 +38,7 @@ func SingletonPolicySet() detection.PolicySet { } func initialize() { - policySet = detection.NewPolicySet(policyDataStore.Singleton()) + policySet = detection.NewPolicySet(policyDataStore.Singleton(), clusterDataStore.Singleton(), namespaceDataStore.Singleton()) policies, err := policyDataStore.Singleton().GetAllPolicies(policyCtx) utils.CrashOnError(err) diff --git a/central/detection/deploytime/default_policies_bench_test.go b/central/detection/deploytime/default_policies_bench_test.go index e47b70fc171bc..3e554c2baa6ff 100644 --- a/central/detection/deploytime/default_policies_bench_test.go +++ b/central/detection/deploytime/default_policies_bench_test.go @@ -1,21 +1,20 @@ package deploytime import ( + "context" "testing" "github.com/stackrox/rox/central/detection" "github.com/stackrox/rox/pkg/booleanpolicy" "github.com/stackrox/rox/pkg/defaults/policies" - "github.com/stackrox/rox/pkg/detection/deploytime" "github.com/stackrox/rox/pkg/fixtures" pkgPolicies "github.com/stackrox/rox/pkg/policies" "github.com/stretchr/testify/require" ) func BenchmarkDefaultPolicies(b *testing.B) { - b.StopTimer() - policySet = detection.NewPolicySet(nil) + policySet = detection.NewPolicySet(nil, nil, nil) policies, err := policies.DefaultPolicies() require.NoError(b, err) @@ -31,9 +30,8 @@ func BenchmarkDefaultPolicies(b *testing.B) { dep := fixtures.GetDeployment() images := fixtures.DeploymentImages() - b.StartTimer() - for i := 0; i < b.N; i++ { - _, err := detection.Detect(deploytime.DetectionContext{}, booleanpolicy.EnhancedDeployment{ + for b.Loop() { + _, err := detection.Detect(context.Background(), booleanpolicy.EnhancedDeployment{ Deployment: dep, Images: images, }) diff --git a/central/detection/deploytime/detector.go b/central/detection/deploytime/detector.go index 88548c65e0259..05cc4c2287671 100644 --- a/central/detection/deploytime/detector.go +++ b/central/detection/deploytime/detector.go @@ -1,6 +1,8 @@ package deploytime import ( + "context" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/booleanpolicy" "github.com/stackrox/rox/pkg/detection" @@ -11,7 +13,7 @@ import ( type Detector interface { PolicySet() detection.PolicySet - Detect(ctx deploytime.DetectionContext, enhancedDeployment booleanpolicy.EnhancedDeployment, policyFilters ...detection.FilterOption) ([]*storage.Alert, error) + Detect(ctx context.Context, enhancedDeployment booleanpolicy.EnhancedDeployment, opts ...deploytime.DetectOption) ([]*storage.Alert, error) } // NewDetector returns a new instance of a Detector. diff --git a/central/detection/deploytime/detector_impl.go b/central/detection/deploytime/detector_impl.go index fa1b211f5680a..637e4776cebc2 100644 --- a/central/detection/deploytime/detector_impl.go +++ b/central/detection/deploytime/detector_impl.go @@ -1,6 +1,8 @@ package deploytime import ( + "context" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/booleanpolicy" "github.com/stackrox/rox/pkg/detection" @@ -18,11 +20,6 @@ func (d *detectorImpl) PolicySet() detection.PolicySet { } // Detect runs detection on a deployment, returning any generated alerts. -func (d *detectorImpl) Detect(ctx deploytime.DetectionContext, enhancedDeployment booleanpolicy.EnhancedDeployment, policyFilters ...detection.FilterOption) ([]*storage.Alert, error) { - alerts, err := d.singleDetector.Detect(ctx, enhancedDeployment, policyFilters...) - if err != nil { - return nil, err - } - - return alerts, nil +func (d *detectorImpl) Detect(ctx context.Context, enhancedDeployment booleanpolicy.EnhancedDeployment, opts ...deploytime.DetectOption) ([]*storage.Alert, error) { + return d.singleDetector.Detect(ctx, enhancedDeployment, opts...) } diff --git a/central/detection/deploytime/singleton.go b/central/detection/deploytime/singleton.go index 49047a5ee9185..dbc7a12a05620 100644 --- a/central/detection/deploytime/singleton.go +++ b/central/detection/deploytime/singleton.go @@ -1,7 +1,9 @@ package deploytime import ( + clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" "github.com/stackrox/rox/central/detection" + namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" policyDataStore "github.com/stackrox/rox/central/policy/datastore" "github.com/stackrox/rox/pkg/sync" ) @@ -26,6 +28,6 @@ func SingletonPolicySet() detection.PolicySet { } func initialize() { - policySet = detection.NewPolicySet(policyDataStore.Singleton()) + policySet = detection.NewPolicySet(policyDataStore.Singleton(), clusterDataStore.Singleton(), namespaceDataStore.Singleton()) detector = NewDetector(policySet) } diff --git a/central/detection/lifecycle/indicator_filter_benchmark_test.go b/central/detection/lifecycle/indicator_filter_benchmark_test.go index ea187b624e795..5585c4c79c5c2 100644 --- a/central/detection/lifecycle/indicator_filter_benchmark_test.go +++ b/central/detection/lifecycle/indicator_filter_benchmark_test.go @@ -199,7 +199,7 @@ func BenchmarkBuildIndicatorFilterMemory(b *testing.B) { manager.buildIndicatorFilter() // Force multiple garbage collections to get accurate heap profile - for i := 0; i < 10; i++ { + for range 10 { runtime.GC() time.Sleep(500 * time.Millisecond) } diff --git a/central/detection/lifecycle/manager_impl.go b/central/detection/lifecycle/manager_impl.go index 2d68fcbb6f04d..10900a38ce7da 100644 --- a/central/detection/lifecycle/manager_impl.go +++ b/central/detection/lifecycle/manager_impl.go @@ -23,6 +23,7 @@ import ( "github.com/stackrox/rox/central/sensor/service/connection" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + clusterPkg "github.com/stackrox/rox/pkg/cluster" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/features" @@ -127,7 +128,7 @@ func (m *managerImpl) buildIndicatorFilter() { } log.Infof("Cleaning up %d processes as a part of building process filter", len(processesToRemove)) - if err := m.processesDataStore.RemoveProcessIndicators(ctx, processesToRemove); err != nil { + if err := m.processesDataStore.RemoveProcessIndicators(ctx, processesToRemove, processIndicatorDatastore.RemovalReasonProcessFilter); err != nil { utils.Should(errors.Wrap(err, "error removing process indicators")) } log.Infof("Successfully cleaned up those %d processes", len(processesToRemove)) @@ -208,7 +209,8 @@ func (m *managerImpl) autoLockProcessBaselines(baselines []*storage.ProcessBasel } } -// Perhaps the cluster config should be kept in memory and calling the database should not be needed +// Lifecycle manager uses cachedStorage for cluster datastore, +// thus repeated calls are memoized func (m *managerImpl) isAutoLockEnabledForCluster(clusterId string) bool { if !features.AutoLockProcessBaselines.Enabled() { return false @@ -226,11 +228,7 @@ func (m *managerImpl) isAutoLockEnabledForCluster(clusterId string) bool { return false } - if cluster.GetManagedBy() == storage.ManagerType_MANAGER_TYPE_MANUAL || cluster.GetManagedBy() == storage.ManagerType_MANAGER_TYPE_UNKNOWN { - return cluster.GetDynamicConfig().GetAutoLockProcessBaselinesConfig().GetEnabled() - } - - return cluster.GetHelmConfig().GetDynamicConfig().GetAutoLockProcessBaselinesConfig().GetEnabled() + return clusterPkg.GetAutoLockProcessBaselinesEnabled(cluster) } func (m *managerImpl) flushIndicatorQueue() { @@ -255,7 +253,23 @@ func (m *managerImpl) flushIndicatorQueue() { if m.deletedDeploymentsCache.Contains(indicator.GetDeploymentId()) { continue } - indicatorSlice = append(indicatorSlice, indicator) + + match, err := m.clusterDataStore.MatchProcessIndicator(lifecycleMgrCtx, indicator) + + if err != nil || !match { + // Add the indicator if not matching, or in the presence of an error + // (we consider any errors from MatchProcessIndicator as non-blocking). + // + // The only scenario when the indicator is not persisted is when it + // matches and we got no errors. + if err != nil { + log.Errorf("Cannot match indicator %s: %v", indicator.GetId(), err) + } + + indicatorSlice = append(indicatorSlice, indicator) + } else { + processIndicatorDatastore.RecordProcessIndicatorNotPersisted(indicator) + } } // Index the process indicators in batch @@ -282,15 +296,21 @@ func (m *managerImpl) addToIndicatorQueue(indicator *storage.ProcessIndicator) { func (m *managerImpl) addBaseline(deploymentID string) []*storage.ProcessBaseline { defer centralMetrics.SetFunctionSegmentDuration(time.Now(), "AddBaseline") - // Simply use search to find the process indicators for the deployment - indicatorSlice, _ := m.processesDataStore.SearchRawProcessIndicators( - lifecycleMgrCtx, - search.NewQueryBuilder(). - AddExactMatches(search.DeploymentID, deploymentID). - ProtoQuery(), - ) + // Group the processes into particular baseline segments + baselineMap := make(map[processBaselineKey][]*storage.ProcessIndicator) - return m.buildMapAndCheckBaseline(indicatorSlice) + fn := func(indicator *storage.ProcessIndicator) error { + key := indicatorToBaselineKey(indicator) + baselineMap[key] = append(baselineMap[key], indicator) + return nil + } + + query := search.NewQueryBuilder(). + AddExactMatches(search.DeploymentID, deploymentID). + ProtoQuery() + utils.Should(m.processesDataStore.GetByQueryFn(lifecycleMgrCtx, query, fn)) + + return m.processBaselineMap(baselineMap) } func (m *managerImpl) buildMapAndCheckBaseline(indicatorSlice []*storage.ProcessIndicator) []*storage.ProcessBaseline { @@ -301,7 +321,11 @@ func (m *managerImpl) buildMapAndCheckBaseline(indicatorSlice []*storage.Process baselineMap[key] = append(baselineMap[key], indicator) } - baselines := make([]*storage.ProcessBaseline, 0) + return m.processBaselineMap(baselineMap) +} + +func (m *managerImpl) processBaselineMap(baselineMap map[processBaselineKey][]*storage.ProcessIndicator) []*storage.ProcessBaseline { + baselines := make([]*storage.ProcessBaseline, 0, len(baselineMap)) for key, indicators := range baselineMap { if baseline, _, err := m.checkAndUpdateBaseline(key, indicators); err != nil { @@ -500,14 +524,15 @@ func (m *managerImpl) HandleNodeAlerts(clusterID string, alerts []*storage.Alert return nil } -func (m *managerImpl) UpsertPolicy(policy *storage.Policy) error { +// registerPolicy updates detector policy sets and the disabled policy tracker under the lock. +// Returns true if the policy applies at runtime and alert reconciliation is needed. +func (m *managerImpl) registerPolicy(policy *storage.Policy) (needsAlertReconcile bool, err error) { m.policyAlertsLock.Lock() defer m.policyAlertsLock.Unlock() - // Add policy to set. if policies.AppliesAtBuildTime(policy) { if err := m.buildTimeDetector.PolicySet().UpsertPolicy(policy); err != nil { - return errors.Wrapf(err, "adding policy %s to build time detector", policy.GetName()) + return false, errors.Wrapf(err, "adding policy %s to build time detector", policy.GetName()) } } else { m.buildTimeDetector.PolicySet().RemovePolicy(policy.GetId()) @@ -515,7 +540,7 @@ func (m *managerImpl) UpsertPolicy(policy *storage.Policy) error { if policies.AppliesAtDeployTime(policy) { if err := m.deployTimeDetector.PolicySet().UpsertPolicy(policy); err != nil { - return errors.Wrapf(err, "adding policy %s to deploy time detector", policy.GetName()) + return false, errors.Wrapf(err, "adding policy %s to deploy time detector", policy.GetName()) } } else { m.deployTimeDetector.PolicySet().RemovePolicy(policy.GetId()) @@ -523,18 +548,9 @@ func (m *managerImpl) UpsertPolicy(policy *storage.Policy) error { if policies.AppliesAtRunTime(policy) { if err := m.runtimeDetector.PolicySet().UpsertPolicy(policy); err != nil { - return errors.Wrapf(err, "adding policy %s to runtime detector", policy.GetName()) + return false, errors.Wrapf(err, "adding policy %s to runtime detector", policy.GetName()) } - // Perform notifications and update DB. - modifiedDeployments, err := m.alertManager.AlertAndNotify(lifecycleMgrCtx, nil, - alertmanager.WithPolicyID(policy.GetId())) - if err != nil { - return err - } - if modifiedDeployments.Cardinality() > 0 { - defer m.reprocessor.ReprocessRiskForDeployments(modifiedDeployments.AsSlice()...) - } - + needsAlertReconcile = true } else { m.runtimeDetector.PolicySet().RemovePolicy(policy.GetId()) } @@ -544,6 +560,30 @@ func (m *managerImpl) UpsertPolicy(policy *storage.Policy) error { } else { m.removedOrDisabledPolicies.Remove(policy.GetId()) } + return needsAlertReconcile, nil +} + +// reconcileAlertsForPolicy reconciles alerts and reprocesses risk for a policy outside of any lock. +func (m *managerImpl) reconcileAlertsForPolicy(policyID string) error { + modifiedDeployments, err := m.alertManager.AlertAndNotify(lifecycleMgrCtx, nil, + alertmanager.WithPolicyID(policyID)) + if err != nil { + return err + } + if modifiedDeployments.Cardinality() > 0 { + m.reprocessor.ReprocessRiskForDeployments(modifiedDeployments.AsSlice()...) + } + return nil +} + +func (m *managerImpl) UpsertPolicy(policy *storage.Policy) error { + needsReconcile, err := m.registerPolicy(policy) + if err != nil { + return err + } + if needsReconcile { + return m.reconcileAlertsForPolicy(policy.GetId()) + } return nil } @@ -559,12 +599,13 @@ func (m *managerImpl) RemoveDeploymentFromObservation(deploymentID string) { m.deploymentObservationQueue.RemoveFromObservation(deploymentID) } -func (m *managerImpl) RemovePolicy(policyID string) error { +// deregisterPolicy removes a policy from all detectors under the lock. +// Returns true if a runtime policy was removed and alert reconciliation is needed. +func (m *managerImpl) deregisterPolicy(policyID string) bool { m.policyAlertsLock.Lock() defer m.policyAlertsLock.Unlock() m.buildTimeDetector.PolicySet().RemovePolicy(policyID) - m.deployTimeDetector.PolicySet().RemovePolicy(policyID) numRuntimePolicies := len(m.runtimeDetector.PolicySet().GetCompiledPolicies()) @@ -572,16 +613,13 @@ func (m *managerImpl) RemovePolicy(policyID string) error { runtimePolicyRemoved := numRuntimePolicies-len(m.runtimeDetector.PolicySet().GetCompiledPolicies()) > 0 m.removedOrDisabledPolicies.Add(policyID) + return runtimePolicyRemoved +} - // Runtime alerts need to be explicitly marked resolved as their updates are not synced from sensors +func (m *managerImpl) RemovePolicy(policyID string) error { + runtimePolicyRemoved := m.deregisterPolicy(policyID) if runtimePolicyRemoved { - modifiedDeployments, err := m.alertManager.AlertAndNotify(lifecycleMgrCtx, nil, alertmanager.WithPolicyID(policyID)) - if err != nil { - return err - } - if modifiedDeployments.Cardinality() > 0 { - m.reprocessor.ReprocessRiskForDeployments(modifiedDeployments.AsSlice()...) - } + return m.reconcileAlertsForPolicy(policyID) } return nil } diff --git a/central/detection/lifecycle/manager_impl_test.go b/central/detection/lifecycle/manager_impl_test.go index d26c97363d828..4952b53670c23 100644 --- a/central/detection/lifecycle/manager_impl_test.go +++ b/central/detection/lifecycle/manager_impl_test.go @@ -7,9 +7,12 @@ import ( "github.com/pkg/errors" clusterDataStoreMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" + "github.com/stackrox/rox/central/deployment/cache" queueMocks "github.com/stackrox/rox/central/deployment/queue/mocks" alertManagerMocks "github.com/stackrox/rox/central/detection/alertmanager/mocks" processBaselineDataStoreMocks "github.com/stackrox/rox/central/processbaseline/datastore/mocks" + processIndicatorsDataStoreMocks "github.com/stackrox/rox/central/processindicator/datastore/mocks" + piFilter "github.com/stackrox/rox/central/processindicator/filter" reprocessorMocks "github.com/stackrox/rox/central/reprocessor/mocks" connectionMocks "github.com/stackrox/rox/central/sensor/service/connection/mocks" "github.com/stackrox/rox/generated/storage" @@ -17,6 +20,7 @@ import ( "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" + "github.com/stackrox/rox/pkg/process/filter" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/set" @@ -82,6 +86,8 @@ type ManagerTestSuite struct { mockCtrl *gomock.Controller connectionManager *connectionMocks.MockManager cluster *clusterDataStoreMocks.MockDataStore + indicators *processIndicatorsDataStoreMocks.MockDataStore + filter filter.Filter } func (suite *ManagerTestSuite) SetupTest() { @@ -93,6 +99,8 @@ func (suite *ManagerTestSuite) SetupTest() { suite.deploymentObservationQueue = queueMocks.NewMockDeploymentObservationQueue(suite.mockCtrl) suite.connectionManager = connectionMocks.NewMockManager(suite.mockCtrl) suite.cluster = clusterDataStoreMocks.NewMockDataStore(suite.mockCtrl) + suite.filter = piFilter.Singleton() + suite.indicators = processIndicatorsDataStoreMocks.NewMockDataStore(suite.mockCtrl) suite.manager = &managerImpl{ baselines: suite.baselines, @@ -101,11 +109,21 @@ func (suite *ManagerTestSuite) SetupTest() { deploymentObservationQueue: suite.deploymentObservationQueue, connectionManager: suite.connectionManager, clusterDataStore: suite.cluster, + processFilter: suite.filter, + processesDataStore: suite.indicators, + queuedIndicators: make(map[string]*storage.ProcessIndicator), + deletedDeploymentsCache: cache.DeletedDeploymentsSingleton(), } } func (suite *ManagerTestSuite) TearDownTest() { suite.mockCtrl.Finish() + + // reset the state + suite.filter = piFilter.Singleton() + suite.manager.processFilter = suite.filter + suite.manager.queuedIndicators = make(map[string]*storage.ProcessIndicator) + suite.manager.deletedDeploymentsCache = cache.DeletedDeploymentsSingleton() } func makeIndicator() (*storage.ProcessBaselineKey, *storage.ProcessIndicator) { @@ -331,3 +349,46 @@ func (suite *ManagerTestSuite) TestAutoLockProcessBaselinesNoCluster() { enabled := suite.manager.isAutoLockEnabledForCluster(clusterId) suite.False(enabled) } + +func (suite *ManagerTestSuite) TestFlushIndicators() { + _, indicator1 := makeIndicator() + _, indicator2 := makeIndicator() + + // Make first indicator to match and be filtered out + suite.cluster.EXPECT().MatchProcessIndicator(gomock.Any(), indicator1). + Return(true, nil) + + // The second indicator should pass through + suite.cluster.EXPECT().MatchProcessIndicator(gomock.Any(), indicator2). + Return(false, nil) + + // The second indicator will cause reading of the corresponding baseline... + suite.baselines.EXPECT().GetProcessBaseline(gomock.Any(), gomock.Any()). + Return(nil, false, nil) + + // ... as well as new deployment in the observation queue ... + suite.deploymentObservationQueue.EXPECT(). + InObservation(gomock.Any()) + + // ... and update of the corresponding baseline + suite.baselines.EXPECT().UpsertProcessBaseline( + gomock.Any(), gomock.Any(), gomock.Any(), true, true). + Return(nil, nil) + + // Only the second indicator is getting stored + suite.indicators.EXPECT(). + AddProcessIndicators(gomock.Any(), indicator2). + Return(nil) + + // Unfortunately it's not easy to test new indicators in the lifecycle + // manager at the higher level, since flushIndicatorQueue is executed in a + // separate go routine, which makes mock expectation checking complicated + // (we have to somehow wait for this routine to end). Thus we test at + // flushIndicatorQueue level, and manually prepare the indicator queue. + // + // TODO: Is it possible to incorporate testing/synctest here on top of testify? + suite.manager.queuedIndicators[indicator1.GetId()] = indicator1 + suite.manager.queuedIndicators[indicator2.GetId()] = indicator2 + + suite.manager.flushIndicatorQueue() +} diff --git a/central/detection/policy_set.go b/central/detection/policy_set.go index 0130b86d29f5a..56a48177f7d8c 100644 --- a/central/detection/policy_set.go +++ b/central/detection/policy_set.go @@ -3,6 +3,7 @@ package detection import ( policyDatastore "github.com/stackrox/rox/central/policy/datastore" "github.com/stackrox/rox/pkg/detection" + "github.com/stackrox/rox/pkg/scopecomp" ) // PolicySet is a set of policies. @@ -13,9 +14,9 @@ type PolicySet interface { } // NewPolicySet returns a new instance of a PolicySet. -func NewPolicySet(store policyDatastore.DataStore) PolicySet { +func NewPolicySet(store policyDatastore.DataStore, clusterProvider scopecomp.ClusterLabelProvider, namespaceProvider scopecomp.NamespaceLabelProvider) PolicySet { return &setImpl{ - PolicySet: detection.NewPolicySet(), + PolicySet: detection.NewPolicySet(clusterProvider, namespaceProvider), policyStore: store, } } diff --git a/central/detection/policy_set_impl.go b/central/detection/policy_set_impl.go index da751a6bae548..f4691624ddd45 100644 --- a/central/detection/policy_set_impl.go +++ b/central/detection/policy_set_impl.go @@ -2,15 +2,20 @@ package detection import ( "context" + "errors" policyDatastore "github.com/stackrox/rox/central/policy/datastore" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/detection" + "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" ) var ( + log = logging.LoggerForModule() + policyCtx = sac.WithGlobalAccessScopeChecker(context.Background(), sac.AllowFixedScopes(sac.AccessModeScopeKeys(storage.Access_READ_WRITE_ACCESS), sac.ResourceScopeKeys(resources.WorkflowAdministration))) @@ -45,6 +50,13 @@ func (p *setImpl) RemoveNotifier(notifierID string) error { err := p.policyStore.UpdatePolicy(policyCtx, policy) if err != nil { + // Policy may have been deleted concurrently (e.g., config-controller removing a declarative policy). + // Update of categories will fail because of foreign key constraint. + if errors.Is(err, errox.ReferencedObjectNotFound) { + log.Warnf("Skipping notifier removal from policy %s: %v", policy.GetId(), err) + continue + } + return err } } diff --git a/central/detection/policy_set_impl_test.go b/central/detection/policy_set_impl_test.go index 7af4634ccd6c0..da6eabb78c642 100644 --- a/central/detection/policy_set_impl_test.go +++ b/central/detection/policy_set_impl_test.go @@ -4,11 +4,18 @@ import ( "context" "testing" + clusterMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" + namespaceMocks "github.com/stackrox/rox/central/namespace/datastore/mocks" mocks2 "github.com/stackrox/rox/central/policy/datastore/mocks" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/booleanpolicy/fieldnames" "github.com/stackrox/rox/pkg/detection" "github.com/stackrox/rox/pkg/detection/mocks" + "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/testutils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) @@ -85,3 +92,227 @@ func TestPolicySet_RemoveNotifier(t *testing.T) { protoassert.ElementsMatch(t, expectedUpdates, updatedPolicies) } + +func TestPolicySet_WithLabelProviders(t *testing.T) { + testutils.MustUpdateFeature(t, features.LabelBasedPolicyScoping, true) + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + // Create mock datastores + clusterDS := clusterMocks.NewMockDataStore(mockCtrl) + namespaceDS := namespaceMocks.NewMockDataStore(mockCtrl) + policyDS := mocks2.NewMockDataStore(mockCtrl) + + // Set up test data + testClusterID := "test-cluster-123" + testNamespaceID := "test-namespace-456" + + testCluster := &storage.Cluster{ + Id: testClusterID, + Name: "production-cluster", + Labels: map[string]string{ + "env": "prod", + "region": "us-east-1", + }, + } + + testNamespace := &storage.NamespaceMetadata{ + Id: testNamespaceID, + Name: "backend-services", + ClusterId: testClusterID, + ClusterName: "production-cluster", + Labels: map[string]string{ + "team": "backend", + "tier": "production", + }, + } + + // Configure mock expectations + clusterDS.EXPECT(). + GetCluster(gomock.Any(), testClusterID). + Return(testCluster, true, nil). + AnyTimes() + + namespaceDS.EXPECT(). + GetNamespace(gomock.Any(), testNamespaceID). + Return(testNamespace, true, nil). + AnyTimes() + + // Create PolicySet - datastores implement the provider interfaces directly + policySet := NewPolicySet(policyDS, clusterDS, namespaceDS) + + // Test 1: Policy with cluster_label scope + policyWithClusterLabel := &storage.Policy{ + Id: "policy-cluster-label", + Name: "Production Cluster Policy", + Severity: storage.Severity_HIGH_SEVERITY, + Scope: []*storage.Scope{ + { + ClusterLabel: &storage.Scope_Label{ + Key: "env", + Value: "prod", + }, + }, + }, + PolicySections: []*storage.PolicySection{ + { + PolicyGroups: []*storage.PolicyGroup{ + { + FieldName: fieldnames.ImageTag, + Values: []*storage.PolicyValue{ + {Value: "latest"}, + }, + }, + }, + }, + }, + LifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_DEPLOY}, + PolicyVersion: "1.1", + } + + err := policySet.UpsertPolicy(policyWithClusterLabel) + require.NoError(t, err, "Policy with cluster_label should compile successfully") + + // Verify the policy was added + assert.True(t, policySet.Exists("policy-cluster-label")) + + // Test 2: Policy with namespace_label scope + policyWithNamespaceLabel := &storage.Policy{ + Id: "policy-namespace-label", + Name: "Backend Team Policy", + Severity: storage.Severity_MEDIUM_SEVERITY, + Scope: []*storage.Scope{ + { + NamespaceLabel: &storage.Scope_Label{ + Key: "team", + Value: "backend", + }, + }, + }, + PolicySections: []*storage.PolicySection{ + { + PolicyGroups: []*storage.PolicyGroup{ + { + FieldName: fieldnames.ImageTag, + Values: []*storage.PolicyValue{ + {Value: ".*"}, + }, + }, + }, + }, + }, + LifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_DEPLOY}, + PolicyVersion: "1.1", + } + + err = policySet.UpsertPolicy(policyWithNamespaceLabel) + require.NoError(t, err, "Policy with namespace_label should compile successfully") + + assert.True(t, policySet.Exists("policy-namespace-label")) + + // Test 3: Policy with both cluster_label and namespace_label + policyWithBothLabels := &storage.Policy{ + Id: "policy-both-labels", + Name: "Prod Backend Policy", + Severity: storage.Severity_CRITICAL_SEVERITY, + Scope: []*storage.Scope{ + { + ClusterLabel: &storage.Scope_Label{ + Key: "env", + Value: "prod", + }, + NamespaceLabel: &storage.Scope_Label{ + Key: "team", + Value: "backend", + }, + }, + }, + PolicySections: []*storage.PolicySection{ + { + PolicyGroups: []*storage.PolicyGroup{ + { + FieldName: fieldnames.PrivilegedContainer, + Values: []*storage.PolicyValue{ + {Value: "true"}, + }, + }, + }, + }, + }, + LifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_DEPLOY}, + PolicyVersion: "1.1", + } + + err = policySet.UpsertPolicy(policyWithBothLabels) + require.NoError(t, err, "Policy with both cluster_label and namespace_label should compile successfully") + + assert.True(t, policySet.Exists("policy-both-labels")) + + // Verify all three policies are in the set + compiledPolicies := policySet.GetCompiledPolicies() + assert.Len(t, compiledPolicies, 3, "PolicySet should contain all three policies") + + // Verify we can iterate over policies + policyCount := 0 + err = policySet.ForEach(func(compiled detection.CompiledPolicy) error { + policyCount++ + assert.NotNil(t, compiled.Policy()) + return nil + }) + require.NoError(t, err) + assert.Equal(t, 3, policyCount, "ForEach should iterate over all policies") +} + +func TestPolicySet_RemoveNotifier_ConcurrentPolicyDeletion(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + policySetMock := mocks.NewMockPolicySet(mockCtrl) + policyDatastoreMock := mocks2.NewMockDataStore(mockCtrl) + + policySet := &setImpl{ + PolicySet: policySetMock, + policyStore: policyDatastoreMock, + } + + policySetMock.EXPECT().GetCompiledPolicies().Return(map[string]detection.CompiledPolicy{ + "policy1": wrapPolicy(&storage.Policy{ + Id: "policy1", + Notifiers: []string{"notifier1", "notifier2"}, + }), + "deleted-policy": wrapPolicy(&storage.Policy{ + Id: "deleted-policy", + Notifiers: []string{"notifier1", "notifier2", "notifier3"}, + }), + "policy3": wrapPolicy(&storage.Policy{ + Id: "policy3", + Notifiers: []string{"notifier2", "notifier3"}, + }), + }) + + var updatedPolicies []*storage.Policy + policyDatastoreMock.EXPECT().UpdatePolicy(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(func(_ context.Context, policy *storage.Policy) error { + if policy.GetId() == "deleted-policy" { + // Test assumes that toErrox() is called and DB error is mapped. + return errox.ReferencedObjectNotFound + } + updatedPolicies = append(updatedPolicies, policy) + return nil + }) + + require.NoError(t, policySet.RemoveNotifier("notifier2")) + + expectedUpdates := []*storage.Policy{ + { + Id: "policy1", + Notifiers: []string{"notifier1"}, + }, + { + Id: "policy3", + Notifiers: []string{"notifier3"}, + }, + } + + protoassert.ElementsMatch(t, expectedUpdates, updatedPolicies) +} diff --git a/central/detection/runtime/detector_impl.go b/central/detection/runtime/detector_impl.go index 1529e5b599583..5746c5927b238 100644 --- a/central/detection/runtime/detector_impl.go +++ b/central/detection/runtime/detector_impl.go @@ -40,7 +40,7 @@ func (d *detectorImpl) DeploymentWhitelistedForPolicy(deploymentID, policyID str result = false return nil } - result = !compiled.AppliesTo(dep) + result = !compiled.AppliesTo(detectorCtx, dep) return nil }) if err != nil { @@ -50,7 +50,7 @@ func (d *detectorImpl) DeploymentWhitelistedForPolicy(deploymentID, policyID str } func (d *detectorImpl) DeploymentInactive(deploymentID string) bool { - _, exists, err := d.deployments.ListDeployment(detectorCtx, deploymentID) + exists, err := d.deployments.DeploymentExists(detectorCtx, deploymentID) if err != nil { log.Errorf("Couldn't determine inactive state of deployment %q: %v", deploymentID, err) return false diff --git a/central/detection/runtime/detector_impl_test.go b/central/detection/runtime/detector_impl_test.go new file mode 100644 index 0000000000000..53cd5d5f228d2 --- /dev/null +++ b/central/detection/runtime/detector_impl_test.go @@ -0,0 +1,42 @@ +package runtime + +import ( + "errors" + "testing" + + "github.com/stackrox/rox/central/deployment/datastore/mocks" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" +) + +func TestDeploymentInactive(t *testing.T) { + tests := map[string]struct { + exists bool + err error + inactive bool + }{ + "present deployment": { + exists: true, + inactive: false, + }, + "missing deployment": { + exists: false, + inactive: true, + }, + // A failed lookup must not mark live deployments' alerts inactive. + "datastore error": { + err: errors.New("database unavailable"), + inactive: false, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + deployments := mocks.NewMockDataStore(gomock.NewController(t)) + deployments.EXPECT().DeploymentExists(gomock.Any(), "deployment-id").Return(test.exists, test.err) + + detector := NewDetector(nil, deployments) + assert.Equal(t, test.inactive, detector.DeploymentInactive("deployment-id")) + }) + } +} diff --git a/central/detection/runtime/singleton.go b/central/detection/runtime/singleton.go index ef7bb0497aa13..eee3a01fb35ea 100644 --- a/central/detection/runtime/singleton.go +++ b/central/detection/runtime/singleton.go @@ -1,8 +1,10 @@ package runtime import ( + clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" "github.com/stackrox/rox/central/deployment/datastore" "github.com/stackrox/rox/central/detection" + namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" policyDataStore "github.com/stackrox/rox/central/policy/datastore" "github.com/stackrox/rox/pkg/sync" ) @@ -27,6 +29,6 @@ func SingletonPolicySet() detection.PolicySet { } func initialize() { - policySet = detection.NewPolicySet(policyDataStore.Singleton()) + policySet = detection.NewPolicySet(policyDataStore.Singleton(), clusterDataStore.Singleton(), namespaceDataStore.Singleton()) detector = NewDetector(policySet, datastore.Singleton()) } diff --git a/central/detection/service/service.go b/central/detection/service/service.go index 30808bd2988d7..209428ea6e0f1 100644 --- a/central/detection/service/service.go +++ b/central/detection/service/service.go @@ -9,6 +9,7 @@ import ( "github.com/stackrox/rox/central/detection/deploytime" "github.com/stackrox/rox/central/enrichment" imageDatastore "github.com/stackrox/rox/central/image/datastore" + namespaceDatastore "github.com/stackrox/rox/central/namespace/datastore" networkPolicyDS "github.com/stackrox/rox/central/networkpolicies/datastore" "github.com/stackrox/rox/central/risk/manager" "github.com/stackrox/rox/central/role/sachelper" @@ -32,6 +33,7 @@ type Service interface { // New returns a new Service instance using the given DataStore. func New( clusters clusterDatastore.DataStore, + namespaces namespaceDatastore.DataStore, imageEnricher enricher.ImageEnricher, imageEnricherV2 enricher.ImageEnricherV2, imageDatastore imageDatastore.DataStore, @@ -48,6 +50,7 @@ func New( ) Service { return &serviceImpl{ clusters: clusters, + namespaces: namespaces, imageEnricher: imageEnricher, imageEnricherV2: imageEnricherV2, imageDatastore: imageDatastore, diff --git a/central/detection/service/service_impl.go b/central/detection/service/service_impl.go index a58b5da6b37a6..b33eb1123d741 100644 --- a/central/detection/service/service_impl.go +++ b/central/detection/service/service_impl.go @@ -20,6 +20,7 @@ import ( "github.com/stackrox/rox/central/detection/deploytime" "github.com/stackrox/rox/central/enrichment" imageDatastore "github.com/stackrox/rox/central/image/datastore" + namespaceDatastore "github.com/stackrox/rox/central/namespace/datastore" networkPolicyDS "github.com/stackrox/rox/central/networkpolicies/datastore" "github.com/stackrox/rox/central/risk/manager" "github.com/stackrox/rox/central/role/sachelper" @@ -106,6 +107,7 @@ type serviceImpl struct { deploymentEnricher enrichment.Enricher buildTimeDetector buildtime.Detector clusters clusterDatastore.DataStore + namespaces namespaceDatastore.DataStore connManager connection.Manager netpols networkPolicyDS.DataStore @@ -235,7 +237,7 @@ func (s *serviceImpl) DetectBuildTime(ctx context.Context, req *apiV1.BuildDetec } filter, getUnusedCategories := centralDetection.MakeCategoryFilter(req.GetPolicyCategories()) - alerts, err := s.buildTimeDetector.Detect(img, filter) + alerts, err := s.buildTimeDetector.Detect(ctx, img, filter) if err != nil { return nil, err } @@ -292,10 +294,6 @@ func (s *serviceImpl) enrichAndDetect(ctx context.Context, enrichmentContext enr } } - detectionCtx := deploytimePkg.DetectionContext{ - EnforcementOnly: enrichmentContext.EnforcementOnly, - } - var appliedNetpols *augmentedobjs.NetworkPoliciesApplied if enrichmentContext.ClusterID != "" { var err error @@ -308,11 +306,16 @@ func (s *serviceImpl) enrichAndDetect(ctx context.Context, enrichmentContext enr } filter, getUnusedCategories := centralDetection.MakeCategoryFilter(policyCategories) - alerts, err := s.detector.Detect(detectionCtx, booleanpolicy.EnhancedDeployment{ + var opts []deploytimePkg.DetectOption + if enrichmentContext.EnforcementOnly { + opts = append(opts, deploytimePkg.WithEnforcementOnly()) + } + opts = append(opts, deploytimePkg.WithPolicyFilters(filter)) + alerts, err := s.detector.Detect(ctx, booleanpolicy.EnhancedDeployment{ Deployment: deployment, Images: images, NetworkPoliciesApplied: appliedNetpols, - }, filter) + }, opts...) if err != nil { return nil, err } @@ -480,6 +483,13 @@ func (s *serviceImpl) DetectDeployTimeFromYAML(ctx context.Context, req *apiV1.D log.Warnf("Deployment YAMLs failed to parse: %v", errs) } + // Populate cluster ID on all deployments for label-based scope matching + for _, d := range deployments { + if eCtx.ClusterID != "" { + d.ClusterId = eCtx.ClusterID + } + } + // If a cluster is provided and Sensor has the capability, enhance deployments with additional info from Sensor if eCtx.ClusterID != "" { conn := s.connManager.GetConnection(eCtx.ClusterID) diff --git a/central/detection/service/singleton.go b/central/detection/service/singleton.go index 498249c688008..dacecd0a8017f 100644 --- a/central/detection/service/singleton.go +++ b/central/detection/service/singleton.go @@ -6,6 +6,7 @@ import ( "github.com/stackrox/rox/central/detection/deploytime" "github.com/stackrox/rox/central/enrichment" imageDatastore "github.com/stackrox/rox/central/image/datastore" + namespaceDatastore "github.com/stackrox/rox/central/namespace/datastore" networkpolicyDatastore "github.com/stackrox/rox/central/networkpolicies/datastore" "github.com/stackrox/rox/central/notifier/processor" "github.com/stackrox/rox/central/risk/manager" @@ -26,6 +27,7 @@ func initialize() { as = New( clusterDS, + namespaceDatastore.Singleton(), enrichment.ImageEnricherSingleton(), enrichment.ImageEnricherV2Singleton(), imageDatastore.Singleton(), diff --git a/central/discoveredclusters/service/service_impl_postgres_test.go b/central/discoveredclusters/service/service_impl_postgres_test.go index af3ad0d5f0336..d44c701f28b0d 100644 --- a/central/discoveredclusters/service/service_impl_postgres_test.go +++ b/central/discoveredclusters/service/service_impl_postgres_test.go @@ -173,7 +173,7 @@ func (s *servicePostgresTestSuite) TestListDiscoveredClusters() { func (s *servicePostgresTestSuite) addDiscoveredClusters(num int) []*v1.DiscoveredCluster { fakeClusters := fixtures.GetManyDiscoveredClusters(num) s.Require().NoError(s.datastore.UpsertDiscoveredClusters(s.writeCtx, fakeClusters...)) - v1Clusters := []*v1.DiscoveredCluster{} + v1Clusters := make([]*v1.DiscoveredCluster, 0, len(fakeClusters)) for _, dc := range fakeClusters { storageCluster := typetostorage.DiscoveredCluster(dc) v1Clusters = append(v1Clusters, storagetov1.DiscoveredCluster(storageCluster)) diff --git a/central/endpoints/instantiate.go b/central/endpoints/instantiate.go index 8b1964ef9bad5..4a7507a2d5852 100644 --- a/central/endpoints/instantiate.go +++ b/central/endpoints/instantiate.go @@ -42,6 +42,8 @@ func (c *TLSConfig) Instantiate(tlsMgr TLSConfigurerProvider) (verifier.TLSConfi opts.ServerCerts = append(opts.ServerCerts, tlsconfig.DefaultTLSCertSource) case "service": opts.ServerCerts = append(opts.ServerCerts, tlsconfig.ServiceCertSource) + case "openshift": + opts.ServerCerts = append(opts.ServerCerts, tlsconfig.OpenShiftTLSCertSource) default: return nil, errors.Errorf("unknown server certificate setting %q", src) } diff --git a/central/endpoints/instantiate_all.go b/central/endpoints/instantiate_all.go index ea69b3277d3e3..a1f3e62e518bb 100644 --- a/central/endpoints/instantiate_all.go +++ b/central/endpoints/instantiate_all.go @@ -1,6 +1,8 @@ package endpoints import ( + "slices" + "github.com/pkg/errors" "github.com/stackrox/rox/central/tlsconfig" "github.com/stackrox/rox/pkg/env" @@ -31,7 +33,16 @@ func loadAllConfigs() ([]EndpointConfig, error) { var allEndpointCfgs []EndpointConfig if !cfg.DisableDefault { - allEndpointCfgs = append(allEndpointCfgs, defaultEndpoint) + ep := defaultEndpoint + if tlsconfig.OpenShiftTLSConfigured() { + // Clone TLS config so we do not mutate package-level defaultTLSConfig shared + // with ROX_SECURE_ENDPOINTS. Add the OpenShift service-serving cert source used + // by the central-ocp Service (SNI: central-ocp..svc). + tlsCfg := *ep.TLS + tlsCfg.ServerCerts = append(slices.Clone(ep.TLS.ServerCerts), "openshift") + ep.TLS = &tlsCfg + } + allEndpointCfgs = append(allEndpointCfgs, ep) } allEndpointCfgs = append(allEndpointCfgs, ParseLegacySpec(env.PlaintextEndpoints.Setting(), &plaintextTLSConfig)...) diff --git a/central/enrichment/enricher_impl.go b/central/enrichment/enricher_impl.go index fbaabfda3fc4d..721bcee1f1588 100644 --- a/central/enrichment/enricher_impl.go +++ b/central/enrichment/enricher_impl.go @@ -78,7 +78,7 @@ func (e *enricherImpl) EnrichDeploymentV2(ctx context.Context, enrichCtx enriche var imgToProcess *storage.ImageV2 if (enrichCtx.FetchOnlyIfMetadataEmpty() || enrichCtx.FetchOnlyIfScanEmpty()) && c.GetImage().GetIdV2() != "" { var img *storage.ImageV2 - img, _, err = e.imagesV2.GetImage(getImageContext, c.GetImage().GetId()) + img, _, err = e.imagesV2.GetImage(getImageContext, c.GetImage().GetIdV2()) if err != nil { return } diff --git a/central/enrichment/enricher_impl_test.go b/central/enrichment/enricher_impl_test.go new file mode 100644 index 0000000000000..18f1ac2328469 --- /dev/null +++ b/central/enrichment/enricher_impl_test.go @@ -0,0 +1,165 @@ +package enrichment + +import ( + "context" + "testing" + + imageV2Mocks "github.com/stackrox/rox/central/imagev2/datastore/mocks" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/images/enricher" + enricherMocks "github.com/stackrox/rox/pkg/images/enricher/mocks" + "github.com/stackrox/rox/pkg/images/utils" + "github.com/stackrox/rox/pkg/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestEnrichDeploymentV2(t *testing.T) { + testutils.MustUpdateFeature(t, features.FlattenImageData, true) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockImagesV2 := imageV2Mocks.NewMockDataStore(ctrl) + mockImageEnricherV2 := enricherMocks.NewMockImageEnricherV2(ctrl) + + e := &enricherImpl{ + images: nil, + imagesV2: mockImagesV2, + imageEnricher: nil, + imageEnricherV2: mockImageEnricherV2, + } + + ctx := context.Background() + enrichCtx := enricher.EnrichmentContext{FetchOpt: enricher.ForceRefetch} + + t.Run("empty deployment returns empty images", func(t *testing.T) { + deployment := &storage.Deployment{ + Containers: []*storage.Container{}, + } + images, updatedIndices, pending, err := e.EnrichDeploymentV2(ctx, enrichCtx, deployment) + require.NoError(t, err) + assert.Empty(t, images) + assert.Empty(t, updatedIndices) + assert.False(t, pending) + }) + + t.Run("container with non existing image enriches", func(t *testing.T) { + idV2 := utils.NewImageV2ID( + &storage.ImageName{Registry: "reg", Remote: "img", Tag: "latest", FullName: "reg/img:latest"}, + "sha256:abc", + ) + deployment := &storage.Deployment{ + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{ + Name: &storage.ImageName{Registry: "reg", Remote: "img", Tag: "latest", FullName: "reg/img:latest"}, + Id: "sha256:abc", + IdV2: idV2, + }, + }, + }, + } + useCacheCtx := enricher.EnrichmentContext{FetchOpt: enricher.UseCachesIfPossible} + mockImagesV2.EXPECT(). + GetImage(gomock.Any(), idV2). + Return(nil, false, nil) + mockImageEnricherV2.EXPECT(). + EnrichImage(gomock.Any(), useCacheCtx, gomock.Any()). + DoAndReturn(func(_ context.Context, _ enricher.EnrichmentContext, img *storage.ImageV2) (enricher.EnrichmentResult, error) { + assert.Equal(t, idV2, img.GetId()) + assert.Equal(t, "sha256:abc", img.GetDigest()) + return enricher.EnrichmentResult{ImageUpdated: true, ScanResult: enricher.ScanSucceeded}, nil + }) + images, updatedIndices, pending, err := e.EnrichDeploymentV2(ctx, useCacheCtx, deployment) + require.NoError(t, err) + require.Len(t, images, 1) + assert.Equal(t, []int{0}, updatedIndices) + assert.False(t, pending) + }) + + t.Run("container with IdV2 fetches from datastore when fetch opts allow", func(t *testing.T) { + idV2 := utils.NewImageV2ID( + &storage.ImageName{Registry: "reg", Remote: "img", Tag: "latest", FullName: "reg/img:latest"}, + "sha256:abc", + ) + cachedImage := &storage.ImageV2{ + Id: idV2, + Digest: "sha256:abc", + Name: &storage.ImageName{Registry: "reg", Remote: "img", Tag: "latest", FullName: "reg/img:latest"}, + } + deployment := &storage.Deployment{ + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{ + Name: &storage.ImageName{Registry: "reg", Remote: "img", Tag: "latest", FullName: "reg/img:latest"}, + Id: "sha256:abc", + IdV2: idV2, + }, + }, + }, + } + useCacheCtx := enricher.EnrichmentContext{FetchOpt: enricher.UseCachesIfPossible} + mockImagesV2.EXPECT(). + GetImage(gomock.Any(), idV2). + Return(cachedImage, true, nil) + mockImageEnricherV2.EXPECT(). + EnrichImage(gomock.Any(), useCacheCtx, gomock.Any()). + Return(enricher.EnrichmentResult{ImageUpdated: false, ScanResult: enricher.ScanSucceeded}, nil) + + images, updatedIndices, pending, err := e.EnrichDeploymentV2(ctx, useCacheCtx, deployment) + require.NoError(t, err) + require.Len(t, images, 1) + assert.Equal(t, idV2, images[0].GetId()) + assert.Empty(t, updatedIndices) + assert.False(t, pending) + }) + + t.Run("not pullable image is not enriched", func(t *testing.T) { + idV2 := utils.NewImageV2ID( + &storage.ImageName{Registry: "reg", Remote: "img", Tag: "latest", FullName: "reg/img:latest"}, + "sha256:abc", + ) + deployment := &storage.Deployment{ + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{ + Name: &storage.ImageName{Registry: "reg", Remote: "img", Tag: "latest", FullName: "reg/img:latest"}, + Id: "sha256:abc", + IdV2: idV2, + NotPullable: true, + }, + }, + }, + } + // GetImage may or may not be called depending on fetch opts; with ForceRefetch we don't use cache + // so we go to ToImageV2. With NotPullable, we skip EnrichImage. So no EnrichImage call. + images, updatedIndices, pending, err := e.EnrichDeploymentV2(ctx, enrichCtx, deployment) + require.NoError(t, err) + require.Len(t, images, 1) + assert.True(t, images[0].GetNotPullable()) + assert.Empty(t, updatedIndices) + assert.False(t, pending) + }) + + t.Run("enricher ScanTriggered sets pendingEnrichment", func(t *testing.T) { + deployment := &storage.Deployment{ + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{ + Name: &storage.ImageName{Registry: "reg", Remote: "img", Tag: "latest", FullName: "reg/img:latest"}, + Id: "sha256:abc", + }, + }, + }, + } + mockImageEnricherV2.EXPECT(). + EnrichImage(gomock.Any(), enrichCtx, gomock.Any()). + Return(enricher.EnrichmentResult{ImageUpdated: true, ScanResult: enricher.ScanTriggered}, nil) + _, _, pending, err := e.EnrichDeploymentV2(ctx, enrichCtx, deployment) + require.NoError(t, err) + assert.True(t, pending) + }) +} diff --git a/central/enrichment/integration.go b/central/enrichment/integration.go index 3c87a2f76c647..20b69b48e2143 100644 --- a/central/enrichment/integration.go +++ b/central/enrichment/integration.go @@ -1,6 +1,8 @@ package enrichment import ( + "slices" + "github.com/pkg/errors" "github.com/stackrox/rox/central/cve/fetcher" "github.com/stackrox/rox/generated/storage" @@ -34,12 +36,7 @@ type managerImpl struct { // isNodeIntegration returns "true" if the image integration is also a node integration. // It loops through the categories, which is a very small slice. func isNodeIntegration(integration *storage.ImageIntegration) bool { - for _, category := range integration.GetCategories() { - if category == storage.ImageIntegrationCategory_NODE_SCANNER { - return true - } - } - return false + return slices.Contains(integration.GetCategories(), storage.ImageIntegrationCategory_NODE_SCANNER) } // ImageIntegrationToNodeIntegration converts the given image integration into a node integration. diff --git a/central/enrichment/singleton.go b/central/enrichment/singleton.go index cb28eb5b95aca..598731bd43634 100644 --- a/central/enrichment/singleton.go +++ b/central/enrichment/singleton.go @@ -7,6 +7,7 @@ import ( baseImageMatcher "github.com/stackrox/rox/central/baseimage/matcher" clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" "github.com/stackrox/rox/central/cve/fetcher" + cveInfoEnricher "github.com/stackrox/rox/central/cve/image/info/enricher" nodeCVEDataStore "github.com/stackrox/rox/central/cve/node/datastore" delegatedRegistryConfigDS "github.com/stackrox/rox/central/delegatedregistryconfig/datastore" "github.com/stackrox/rox/central/delegatedregistryconfig/delegator" @@ -56,14 +57,15 @@ func initialize() { scanwaiterv2.Singleton(), sachelper.NewClusterNamespaceSacHelper(clusterDataStore.Singleton(), namespaceDataStore.Singleton()), + clusterDataStore.Singleton(), ) if !features.FlattenImageData.Enabled() { - imgEnricher = imageEnricher.New(suppressor.Singleton(), imageintegration.Set(), + imgEnricher = imageEnricher.New(suppressor.Singleton(), cveInfoEnricher.Singleton(), imageintegration.Set(), metrics.CentralSubsystem, cache.ImageMetadataCacheSingleton(), baseImageMatcher.Singleton().MatchWithBaseImages, datastore.Singleton().GetImage, reporter.Singleton(), signatureIntegrationDataStore.Singleton().GetAllSignatureIntegrations, scanDelegator) } else { - imgEnricherV2 = imageEnricher.NewV2(suppressor.Singleton(), imageintegration.Set(), + imgEnricherV2 = imageEnricher.NewV2(suppressor.Singleton(), cveInfoEnricher.Singleton(), imageintegration.Set(), metrics.CentralSubsystem, cache.ImageMetadataCacheSingleton(), baseImageMatcher.Singleton().MatchWithBaseImages, imageV2DataStore.Singleton().GetImage, reporter.Singleton(), signatureIntegrationDataStore.Singleton().GetAllSignatureIntegrations, scanDelegator) } diff --git a/central/externalbackups/plugins/s3/common/s3common.go b/central/externalbackups/plugins/s3/common/s3common.go index 0bfcee6b2f6c2..c06277312f353 100644 --- a/central/externalbackups/plugins/s3/common/s3common.go +++ b/central/externalbackups/plugins/s3/common/s3common.go @@ -139,9 +139,15 @@ func NewS3Client(cfg ConfigWrapper) (types.ExternalBackup, error) { now: time.Now, clientWrapper: &s3WrapperImpl{ - bucket: cfg.GetBucket(), - client: awsClient, - uploader: manager.NewUploader(awsClient), + bucket: cfg.GetBucket(), + client: awsClient, + // manager.NewUploader defaults RequestChecksumCalculation to + // WhenSupported, ignoring the config/client setting. Propagate + // it explicitly so S3-compatible services aren't sent checksums + // they don't support. + uploader: manager.NewUploader(awsClient, func(u *manager.Uploader) { + u.RequestChecksumCalculation = awsCfg.RequestChecksumCalculation + }), }, }, nil } diff --git a/central/externalbackups/plugins/s3/common/s3common_test.go b/central/externalbackups/plugins/s3/common/s3common_test.go index a347cfbdb652b..44415c014a486 100644 --- a/central/externalbackups/plugins/s3/common/s3common_test.go +++ b/central/externalbackups/plugins/s3/common/s3common_test.go @@ -157,7 +157,7 @@ func TestNewS3Client(t *testing.T) { func TestPruneBackupsIfNecessary(t *testing.T) { var keys []string var objects []types.Object - for i := 0; i < 5; i++ { + for i := range 5 { key := fmt.Sprintf("Backup %d", i+1) var lastModified *time.Time if i == 1 { diff --git a/central/globaldb/metrics/metrics.go b/central/globaldb/metrics/metrics.go index 6a5153cc686f1..a25cdb54b3690 100644 --- a/central/globaldb/metrics/metrics.go +++ b/central/globaldb/metrics/metrics.go @@ -20,6 +20,7 @@ func init() { PostgresMaximumConnections, PostgresTableLiveTuples, PostgresTableDeadTuples, + PostgresInvalidIndexes, ) } @@ -115,4 +116,11 @@ var ( Name: "postgres_dead_tuples", Help: "dead tuples for the table", }, []string{"table"}) + + PostgresInvalidIndexes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "postgres_invalid_indexes", + Help: "invalid indexes in the database (1 per invalid index)", + }, []string{"index_name", "table_name"}) ) diff --git a/central/globaldb/postgres.go b/central/globaldb/postgres.go index 5c113ae7f8480..a1a1005f02565 100644 --- a/central/globaldb/postgres.go +++ b/central/globaldb/postgres.go @@ -2,11 +2,16 @@ package globaldb import ( "context" + "fmt" + "strings" "testing" "time" "github.com/prometheus/client_golang/prometheus" "github.com/stackrox/rox/central/globaldb/metrics" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/administration/events" + "github.com/stackrox/rox/pkg/administration/events/stream" "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/postgres" @@ -89,6 +94,11 @@ SELECT maxConnectionQuery = `SELECT current_setting('max_connections')::int;` + invalidIndexQuery = `SELECT s.relname, s.indexrelname + FROM pg_stat_user_indexes s + JOIN pg_index ix ON ix.indexrelid = s.indexrelid + WHERE ix.indisvalid = false` + pgStatStatementsMax = 1000 ) @@ -124,6 +134,12 @@ func SetPostgresTest(t *testing.T, db postgres.DB) postgres.DB { // InitializePostgres creates and returns returns a global database instance. func InitializePostgres(ctx context.Context) postgres.DB { + return InitializePostgresWithPoolSize(ctx, 0) +} + +// InitializePostgresWithPoolSize creates a global database instance with an overridden +// connection pool size. If maxConns is 0, the default from the DSN is used. +func InitializePostgresWithPoolSize(ctx context.Context, maxConns int32) postgres.DB { pgSync.Do(func() { _, dbConfig, err := pgconfig.GetPostgresConfig() if err != nil { @@ -131,13 +147,14 @@ func InitializePostgres(ctx context.Context) postgres.DB { } if !pgconfig.IsExternalDatabase() { - // Get the active database name for the connection activeDB := pgconfig.GetActiveDB() - - // Set the connection to be the active database. dbConfig.ConnConfig.Database = activeDB } + if maxConns > 0 { + dbConfig.MaxConns = maxConns + } + if err := retry.WithRetry(func() error { postgresDB, err = postgres.New(ctx, dbConfig) return err @@ -378,6 +395,15 @@ func getMaxConnections(ctx context.Context, db postgres.DB) { } func startMonitoringPostgres(ctx context.Context, db postgres.DB, postgresConfig *postgres.Config) { + go func() { + t := time.NewTicker(1 * time.Hour) + defer t.Stop() + CollectPostgresIndexStats(ctx, db) + for range t.C { + CollectPostgresIndexStats(ctx, db) + } + }() + t := time.NewTicker(1 * time.Minute) defer t.Stop() for range t.C { @@ -387,3 +413,64 @@ func startMonitoringPostgres(ctx context.Context, db postgres.DB, postgresConfig CollectPostgresTupleStats(ctx, db) } } + +// CollectPostgresIndexStats checks for invalid indexes, exports a Prometheus metric, +// and produces an administration event if any are found. +func CollectPostgresIndexStats(ctx context.Context, db postgres.DB) { + ctx, cancel := context.WithTimeout(ctx, PostgresQueryTimeout) + defer cancel() + + rows, err := db.Query(ctx, invalidIndexQuery) + if err != nil { + log.Errorf("error checking for invalid indexes: %v", err) + metrics.PostgresInvalidIndexes.Reset() + return + } + defer rows.Close() + + type invalidIndex struct { + tableName string + indexName string + } + var invalidIndexes []invalidIndex + + for rows.Next() { + var idx invalidIndex + if err := rows.Scan(&idx.tableName, &idx.indexName); err != nil { + log.Errorf("error scanning row for invalid index data: %v", err) + return + } + invalidIndexes = append(invalidIndexes, idx) + } + if err := rows.Err(); err != nil { + log.Errorf("error reading invalid index rows: %v", err) + } + + metrics.PostgresInvalidIndexes.Reset() + for _, idx := range invalidIndexes { + metrics.PostgresInvalidIndexes.With(prometheus.Labels{ + "index_name": idx.indexName, + "table_name": idx.tableName, + }).Set(1) + } + + if len(invalidIndexes) > 0 { + var details strings.Builder + for i, idx := range invalidIndexes { + if i > 0 { + details.WriteString(", ") + } + fmt.Fprintf(&details, "%s (table: %s)", idx.indexName, idx.tableName) + } + + stream.Singleton().Produce(&events.AdministrationEvent{ + Type: storage.AdministrationEventType_ADMINISTRATION_EVENT_TYPE_GENERIC, + Level: storage.AdministrationEventLevel_ADMINISTRATION_EVENT_LEVEL_WARNING, + Domain: events.DefaultDomain, + Message: fmt.Sprintf("Detected %d invalid PostgreSQL index(es): %s", len(invalidIndexes), details.String()), + ResourceType: "Database", + ResourceName: "central-db", + Hint: "Invalid indexes can degrade query performance and may indicate a failed index creation or migration. To fix, run: REINDEX INDEX CONCURRENTLY ; for each invalid index. If the problem persists, contact Red Hat support.", + }) + } +} diff --git a/central/globaldb/postgres_test.go b/central/globaldb/postgres_test.go index f5455063583c3..d8a676afc2b01 100644 --- a/central/globaldb/postgres_test.go +++ b/central/globaldb/postgres_test.go @@ -7,6 +7,9 @@ import ( "testing" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stackrox/rox/central/globaldb/metrics" "github.com/stackrox/rox/pkg/postgres/pgconfig" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/sac" @@ -98,3 +101,37 @@ func (s *PostgresUtilitySuite) TestCollectPostgresStats() { s.NotNil(stats) s.Equal(false, stats.DatabaseAvailable) } + +func (s *PostgresUtilitySuite) TestCollectPostgresIndexStats() { + ctx := sac.WithAllAccess(context.Background()) + tp := pgtest.ForT(s.T()) + defer tp.Close() + + // On a clean test DB there should be no invalid indexes. + CollectPostgresIndexStats(ctx, tp.DB) + s.Equal(0, testutil.CollectAndCount(metrics.PostgresInvalidIndexes)) + + // Create a table and index, then mark the index invalid. + _, err := tp.DB.Exec(ctx, "CREATE TABLE test_invalid_idx_check (id int)") + s.Require().NoError(err) + _, err = tp.DB.Exec(ctx, "CREATE INDEX test_idx ON test_invalid_idx_check (id)") + s.Require().NoError(err) + _, err = tp.DB.Exec(ctx, "UPDATE pg_index SET indisvalid = false WHERE indexrelid = 'test_idx'::regclass") + s.Require().NoError(err) + + CollectPostgresIndexStats(ctx, tp.DB) + s.Equal(1, testutil.CollectAndCount(metrics.PostgresInvalidIndexes)) + s.Equal(1.0, testutil.ToFloat64(metrics.PostgresInvalidIndexes.With(prometheus.Labels{ + "index_name": "test_idx", + "table_name": "test_invalid_idx_check", + }))) + + // Clean up: restore validity so DROP works cleanly. + _, err = tp.DB.Exec(ctx, "UPDATE pg_index SET indisvalid = true WHERE indexrelid = 'test_idx'::regclass") + s.Require().NoError(err) + _, err = tp.DB.Exec(ctx, "DROP TABLE test_invalid_idx_check") + s.Require().NoError(err) + + CollectPostgresIndexStats(ctx, tp.DB) + s.Equal(0, testutil.CollectAndCount(metrics.PostgresInvalidIndexes)) +} diff --git a/central/globaldb/telemetry.go b/central/globaldb/telemetry.go index a57edbfd8a6e9..9111ac1797359 100644 --- a/central/globaldb/telemetry.go +++ b/central/globaldb/telemetry.go @@ -30,9 +30,14 @@ var Gather phonehome.GatherFunc = func(ctx context.Context) (map[string]any, err return nil, errors.Wrap(err, "failed to get postgres config") } - dbSize, err := pgadmin.GetDatabaseSize(config, pgconfig.GetActiveDB()) + dbName := pgconfig.GetActiveDB() + if pgconfig.IsExternalDatabase() { + dbName = config.ConnConfig.Database + } + + dbSize, err := pgadmin.GetDatabaseSize(config, dbName) if err != nil { - return nil, errors.Wrap(err, "failed to get databaze size") + return nil, errors.Wrap(err, "failed to get database size") } db := GetPostgres() diff --git a/central/globaldb/v2backuprestore/backup/generators/dbs/postgres.go b/central/globaldb/v2backuprestore/backup/generators/dbs/postgres.go index 1be778abcf89b..bb2b8894ac9a5 100644 --- a/central/globaldb/v2backuprestore/backup/generators/dbs/postgres.go +++ b/central/globaldb/v2backuprestore/backup/generators/dbs/postgres.go @@ -36,16 +36,18 @@ func (bu *PostgresBackup) WriteTo(_ context.Context, out io.Writer) error { return err } + // Get the common DB connection info + connOpts := pgadmin.GetConnectionOptions(config) + // Set the options for pg_dump from the connection config - options := []string{ + options := make([]string, 0, 4+len(connOpts)) + options = append(options, "-d", pgconfig.GetActiveDB(), "-Fc", // Custom format, compressed hopefully supports stdin to restore "-v", - } - - // Get the common DB connection info - options = append(options, pgadmin.GetConnectionOptions(config)...) + ) + options = append(options, connOpts...) cmd := exec.Command("pg_dump", options...) diff --git a/central/globaldb/v2backuprestore/backup/generators/dbs/postgres_size.go b/central/globaldb/v2backuprestore/backup/generators/dbs/postgres_size.go index dd0b042fd5a12..9758058020683 100644 --- a/central/globaldb/v2backuprestore/backup/generators/dbs/postgres_size.go +++ b/central/globaldb/v2backuprestore/backup/generators/dbs/postgres_size.go @@ -37,7 +37,7 @@ func (ps *PostgresSize) WriteTo(_ context.Context, out io.Writer) error { return err } - _, err = out.Write([]byte(fmt.Sprintf("%d", size))) + _, err = out.Write(fmt.Appendf(nil, "%d", size)) if err != nil { return err } diff --git a/central/globaldb/v2backuprestore/manager/restore_process.go b/central/globaldb/v2backuprestore/manager/restore_process.go index b0693475e1fba..5105a73a8adc1 100644 --- a/central/globaldb/v2backuprestore/manager/restore_process.go +++ b/central/globaldb/v2backuprestore/manager/restore_process.go @@ -71,7 +71,7 @@ type restoreProcess struct { // For statistics (atomic access; approximate only) bytesRead int64 - filesProcessed int64 + filesProcessed atomic.Int64 // Informs whether processing a RocksDB or Postgres backup bundle postgresBundle bool @@ -416,7 +416,7 @@ func (p *restoreProcess) processSingleFile(ctx *restoreProcessContext, file *res return errors.New("not all bytes in file chunk were read") } - atomic.AddInt64(&p.filesProcessed, 1) + p.filesProcessed.Add(1) return nil } @@ -425,7 +425,7 @@ func (p *restoreProcess) ProtoStatus() *v1.DBRestoreProcessStatus { status := &v1.DBRestoreProcessStatus{ Metadata: p.Metadata(), BytesRead: atomic.LoadInt64(&p.bytesRead), - FilesProcessed: atomic.LoadInt64(&p.filesProcessed), + FilesProcessed: p.filesProcessed.Load(), } if !p.started.Get() { diff --git a/central/globaldb/v2backuprestore/restore/postgres.go b/central/globaldb/v2backuprestore/restore/postgres.go index bfc6ffa362be4..183b6c422c1de 100644 --- a/central/globaldb/v2backuprestore/restore/postgres.go +++ b/central/globaldb/v2backuprestore/restore/postgres.go @@ -48,8 +48,12 @@ func LoadRestoreStream(fileReader io.Reader) error { } func runRestoreStream(fileReader io.Reader, sourceMap map[string]string, config *postgres.Config, restoreDB string) error { - // Set the options for pg_dump from the connection config - options := []string{ + // Get the common DB connection info + connOpts := pgadmin.GetConnectionOptions(config) + + // Set the options for pg_restore from the connection config + options := make([]string, 0, 9+len(connOpts)) + options = append(options, "-d", restoreDB, "--no-owner", @@ -59,10 +63,8 @@ func runRestoreStream(fileReader io.Reader, sourceMap map[string]string, config "-Fc", "-vvv", "--single-transaction", - } - - // Get the common DB connection info - options = append(options, pgadmin.GetConnectionOptions(config)...) + ) + options = append(options, connOpts...) cmd := exec.Command("pg_restore", options...) diff --git a/central/globalindex/mapping/mapping.go b/central/globalindex/mapping/mapping.go index 4caed3c831f4d..a62952f6a41e6 100644 --- a/central/globalindex/mapping/mapping.go +++ b/central/globalindex/mapping/mapping.go @@ -8,6 +8,7 @@ import ( "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/postgres/schema" "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/policycategory" ) // GetEntityOptionsMap is a mapping from search categories to the options @@ -52,6 +53,11 @@ func GetEntityOptionsMap() map[v1.SearchCategory]search.OptionsMap { schema.RoleBindingsSchema.OptionsMap, ) + policiesSearchOptions := search.CombineOptionsMaps( + policycategory.CategorySearchOptions, + schema.PoliciesSchema.OptionsMap, + ) + // EntityOptionsMap is a mapping from search categories to the options map for that category. // search document maps are also built off this map entityOptionsMap := map[v1.SearchCategory]search.OptionsMap{ @@ -69,7 +75,7 @@ func GetEntityOptionsMap() map[v1.SearchCategory]search.OptionsMap { v1.SearchCategory_NODE_VULNERABILITIES: nodeToVulnerabilitySearchOptions, v1.SearchCategory_NODES: nodeToVulnerabilitySearchOptions, v1.SearchCategory_PODS: schema.PodsSchema.OptionsMap, - v1.SearchCategory_POLICIES: schema.PoliciesSchema.OptionsMap, + v1.SearchCategory_POLICIES: policiesSearchOptions, v1.SearchCategory_POLICY_CATEGORIES: schema.PolicyCategoriesSchema.OptionsMap, v1.SearchCategory_PROCESS_BASELINES: schema.ProcessBaselinesSchema.OptionsMap, v1.SearchCategory_PROCESS_INDICATORS: schema.ProcessIndicatorsSchema.OptionsMap, @@ -95,6 +101,7 @@ func GetEntityOptionsMap() map[v1.SearchCategory]search.OptionsMap { entityOptionsMap[v1.SearchCategory_IMAGE_VULNERABILITIES_V2] = imageToVulnerabilityV2SearchOptions if features.FlattenImageData.Enabled() { entityOptionsMap[v1.SearchCategory_IMAGES_V2] = imageToVulnerabilityV2SearchOptions + entityOptionsMap[v1.SearchCategory_IMAGES] = imageToVulnerabilityV2SearchOptions } else { entityOptionsMap[v1.SearchCategory_IMAGES] = imageToVulnerabilityV2SearchOptions } diff --git a/central/graphdb/testutils/datastore.go b/central/graphdb/testutils/datastore.go index fda32e355feb0..a15852ba9119a 100644 --- a/central/graphdb/testutils/datastore.go +++ b/central/graphdb/testutils/datastore.go @@ -10,12 +10,15 @@ import ( deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" imageDataStore "github.com/stackrox/rox/central/image/datastore" imagePostgresV2 "github.com/stackrox/rox/central/image/datastore/store/v2/postgres" + imageV2DataStore "github.com/stackrox/rox/central/imagev2/datastore" + imageV2Postgres "github.com/stackrox/rox/central/imagev2/datastore/store/postgres" namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" nodeDataStore "github.com/stackrox/rox/central/node/datastore" "github.com/stackrox/rox/central/ranking" riskDS "github.com/stackrox/rox/central/risk/datastore" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgtest" @@ -50,6 +53,7 @@ type testGraphDataStoreImpl struct { namespaceStore namespaceDataStore.DataStore deploymentStore deploymentDataStore.DataStore imageStore imageDataStore.DataStore + imageV2Store imageV2DataStore.DataStore nodeStore nodeDataStore.DataStore clusterStore clusterDataStore.DataStore clusterCVEStore clusterCVEDataStore.DataStore @@ -168,8 +172,6 @@ func (s *testGraphDataStoreImpl) PushImageToVulnerabilitiesGraph() (err error) { ctx := sac.WithAllAccess(context.Background()) testNamespace1 := fixtures.GetNamespace(testconsts.Cluster1, testconsts.Cluster1, testconsts.NamespaceA) testNamespace2 := fixtures.GetNamespace(testconsts.Cluster2, testconsts.Cluster2, testconsts.NamespaceB) - testImage1 := fixtures.GetImageSherlockHolmes1() - testImage2 := fixtures.GetImageDoctorJekyll2() testDeployment1 := fixtures.GetDeploymentSherlockHolmes1(uuid.NewV4().String(), testNamespace1) testDeployment2 := fixtures.GetDeploymentDoctorJekyll2(uuid.NewV4().String(), testNamespace2) err = s.namespaceStore.AddNamespace(ctx, testNamespace1) @@ -182,16 +184,36 @@ func (s *testGraphDataStoreImpl) PushImageToVulnerabilitiesGraph() (err error) { return err } s.storedNamespaces = append(s.storedNamespaces, testNamespace2.GetId()) - err = s.imageStore.UpsertImage(ctx, testImage1) - if err != nil { - return err - } - s.storedImages = append(s.storedImages, testImage1.GetId()) - err = s.imageStore.UpsertImage(ctx, testImage2) - if err != nil { - return err + + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + testImage1 := fixtures.GetImageV2SherlockHolmes1() + testImage2 := fixtures.GetImageV2DoctorJekyll2() + err = s.imageV2Store.UpsertImage(ctx, testImage1) + if err != nil { + return err + } + s.storedImages = append(s.storedImages, testImage1.GetId()) + err = s.imageV2Store.UpsertImage(ctx, testImage2) + if err != nil { + return err + } + s.storedImages = append(s.storedImages, testImage2.GetId()) + } else { + testImage1 := fixtures.GetImageSherlockHolmes1() + testImage2 := fixtures.GetImageDoctorJekyll2() + err = s.imageStore.UpsertImage(ctx, testImage1) + if err != nil { + return err + } + s.storedImages = append(s.storedImages, testImage1.GetId()) + err = s.imageStore.UpsertImage(ctx, testImage2) + if err != nil { + return err + } + s.storedImages = append(s.storedImages, testImage2.GetId()) } - s.storedImages = append(s.storedImages, testImage2.GetId()) + err = s.deploymentStore.UpsertDeployment(ctx, testDeployment1) if err != nil { return err @@ -268,7 +290,12 @@ func (s *testGraphDataStoreImpl) CleanImageToVulnerabilitiesGraph() (err error) s.storedDeployments = s.storedDeployments[:0] storedImages := s.storedImages for _, imageID := range storedImages { - err = s.imageStore.DeleteImages(ctx, imageID) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + err = s.imageV2Store.DeleteImages(ctx, imageID) + } else { + err = s.imageStore.DeleteImages(ctx, imageID) + } if err != nil { return err } @@ -307,9 +334,16 @@ func NewTestGraphDataStore(t *testing.T) (TestGraphDataStore, error) { s.pgtestbase = pgtest.ForT(t) s.nodeStore = nodeDataStore.GetTestPostgresDataStore(t, s.GetPostgresPool()) + riskStore := riskDS.GetTestPostgresDataStore(t, s.GetPostgresPool()) s.imageStore = imageDataStore.NewWithPostgres( imagePostgresV2.New(s.GetPostgresPool(), false, concurrency.NewKeyFence()), - riskDS.GetTestPostgresDataStore(t, s.GetPostgresPool()), + riskStore, + ranking.NewRanker(), + ranking.NewRanker(), + ) + s.imageV2Store = imageV2DataStore.NewWithPostgres( + imageV2Postgres.New(s.GetPostgresPool(), false, concurrency.NewKeyFence()), + riskStore, ranking.NewRanker(), ranking.NewRanker(), ) diff --git a/central/graphdb/testutils/graphdump.go b/central/graphdb/testutils/graphdump.go index ca55e28c5dcfe..f0cf152be08a7 100644 --- a/central/graphdb/testutils/graphdump.go +++ b/central/graphdb/testutils/graphdump.go @@ -419,7 +419,7 @@ func populateListFromDB[T any](ctx context.Context, _ *testing.T, db postgres.DB func getIndent(offset int) string { var indent strings.Builder - for i := 0; i < offset; i++ { + for range offset { indent.WriteString(" ") } return indent.String() diff --git a/central/graphql/generator/codegen/codegen.go b/central/graphql/generator/codegen/codegen.go index 89af57b4d998a..9710b1701dc6b 100644 --- a/central/graphql/generator/codegen/codegen.go +++ b/central/graphql/generator/codegen/codegen.go @@ -31,6 +31,7 @@ var templates = map[string]string{ "raw": `value`, "rawslice": `value`, "time": `protocompat.ConvertTimestampToGraphqlTimeOrError(value)`, + "bytes": `base64.StdEncoding.EncodeToString(value)`, } func listName(td typeData) string { @@ -75,7 +76,7 @@ func getFieldTransform(fd fieldData) (templateName string, returnType string) { if fd.Type.Key().Kind() == reflect.String && fd.Type.Elem().Kind() == reflect.String { return "label", "labels" } - case reflect.Ptr: + case reflect.Pointer: if fd.Type == protocompat.TimestampPtrType { return "time", "(*graphql.Time, error)" } @@ -87,6 +88,10 @@ func getFieldTransform(fd fieldData) (templateName string, returnType string) { return "pointer", fmt.Sprintf("(*%sResolver, error)", lower(fd.Type.Elem().Name())) } case reflect.Slice: + // Handle []byte (protobuf bytes) as base64-encoded strings + if fd.Type.Elem().Kind() == reflect.Uint8 { + return "bytes", "string" + } template, ret := getFieldTransform(fieldData{Name: fd.Name, Type: fd.Type.Elem()}) if len(ret) > 0 && ret[0] == '(' { // this converts (*fooResolver, error) into ([]*fooResolver, error) diff --git a/central/graphql/generator/codegen/codegen.go.tpl b/central/graphql/generator/codegen/codegen.go.tpl index dc41bbab819ee..5700b04321ab8 100644 --- a/central/graphql/generator/codegen/codegen.go.tpl +++ b/central/graphql/generator/codegen/codegen.go.tpl @@ -4,6 +4,7 @@ package resolvers import ( "context" + "encoding/base64" "reflect" "github.com/graph-gophers/graphql-go" @@ -34,6 +35,9 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { {{- end }}{{ end }} {{- range $td.Data.UnionData }} "{{lower .Name }}: {{ $td.Data.Name }}{{.Name}}", +{{- end }} +{{- if and (not $td.Data.FieldData) (not $td.Data.UnionData) }} + "unused: String!", {{- end }} })) {{- range $ud := $td.Data.UnionData }} @@ -163,4 +167,9 @@ func (resolver *{{lower $td.Data.Name}}{{$ud.Name}}Resolver) To{{$ut.Type.Elem.N res, ok := resolver.resolver.(*{{lower $ut.Type.Elem.Name}}Resolver) return res, ok } -{{end}}{{end}}{{end}}{{- end -}} +{{end}}{{end}} +{{- if and (not $td.Data.FieldData) (not $td.Data.UnionData) }} +func (resolver *{{lower $td.Data.Name}}Resolver) Unused(ctx context.Context) string { + return "" +} +{{ end -}}{{end}}{{- end -}} diff --git a/central/graphql/generator/codegen/schema.go b/central/graphql/generator/codegen/schema.go index 67bdb979f5e9a..af8e223c97608 100644 --- a/central/graphql/generator/codegen/schema.go +++ b/central/graphql/generator/codegen/schema.go @@ -19,7 +19,7 @@ func isListType(p reflect.Type) bool { return false } name := p.Name() - if p.Kind() == reflect.Ptr { + if p.Kind() == reflect.Pointer { name = p.Elem().Name() } return isProto(p) && len(name) > 4 && name[0:4] == "List" @@ -91,6 +91,10 @@ func schemaExpand(p reflect.Type) string { case reflect.Bool: return "Boolean!" case reflect.Slice: + // Handle []byte (protobuf bytes) as base64-encoded strings + if p.Elem().Kind() == reflect.Uint8 { + return "String!" + } inner := schemaExpand(p.Elem()) if inner != "" { return fmt.Sprintf("[%s]!", inner) @@ -100,7 +104,7 @@ func schemaExpand(p reflect.Type) string { if p.Key().Kind() == reflect.String && p.Elem().Kind() == reflect.String { return "[Label!]!" } - case reflect.Ptr: + case reflect.Pointer: if p == protocompat.TimestampPtrType { return "Time" } diff --git a/central/graphql/generator/codegen/typewalk.go b/central/graphql/generator/codegen/typewalk.go index 5b62765b3a5e6..b1a1bcb84d58b 100644 --- a/central/graphql/generator/codegen/typewalk.go +++ b/central/graphql/generator/codegen/typewalk.go @@ -2,6 +2,7 @@ package codegen import ( "reflect" + "slices" "sort" "strings" @@ -95,7 +96,7 @@ func (ctx *walkState) walkType(typeDesc typeDescriptor) { if ty.Implements(messageType) { unions = ctx.walkUnions(ty) } - for ty.Kind() == reflect.Ptr { + for ty.Kind() == reflect.Pointer { ty = ty.Elem() } if _, ok := ctx.typeData[ty]; ok { @@ -109,8 +110,8 @@ func (ctx *walkState) walkType(typeDesc typeDescriptor) { td.IsInputType = true } if ty.Kind() == reflect.Struct { - for i := 0; i < ty.NumField(); i++ { - ctx.walkField(&td, ty, ty.Field(i)) + for field := range ty.Fields() { + ctx.walkField(&td, ty, field) } sort.Slice(td.FieldData, func(i, j int) bool { return td.FieldData[i].Name < td.FieldData[j].Name @@ -125,6 +126,9 @@ func (ctx *walkState) walkField(td *typeData, p reflect.Type, sf reflect.StructF if !td.IsInputType && protoreflect.IsInternalGeneratorField(sf) { return } + if _, ok := sf.Tag.Lookup("protobuf_oneof"); ok { + return + } if strings.HasPrefix(sf.Name, "DEPRECATED") { return } @@ -138,12 +142,7 @@ func (ctx *walkState) walkField(td *typeData, p reflect.Type, sf reflect.StructF } func rejectedType(p reflect.Type, blacklist []reflect.Type) bool { - for _, t := range blacklist { - if t == p { - return true - } - } - return false + return slices.Contains(blacklist, p) } func rejectedField(parentType reflect.Type, field reflect.StructField, blacklist []generator2.TypeAndField) bool { diff --git a/central/graphql/generator/codegen/utils.go b/central/graphql/generator/codegen/utils.go index c43eef62c2fdf..d818caba0fc71 100644 --- a/central/graphql/generator/codegen/utils.go +++ b/central/graphql/generator/codegen/utils.go @@ -9,18 +9,18 @@ import ( ) var ( - messageType = reflect.TypeOf((*protocompat.Message)(nil)).Elem() + messageType = reflect.TypeFor[protocompat.Message]() ) func isProto(p reflect.Type) bool { if p == nil { return false } - if p.Kind() == reflect.Ptr { + if p.Kind() == reflect.Pointer { return p.Implements(messageType) } if p.Kind() == reflect.Struct { - p = reflect.PtrTo(p) + p = reflect.PointerTo(p) return p.Implements(messageType) } return false diff --git a/central/graphql/generator/schema2.go b/central/graphql/generator/schema2.go index ef3ea7996f562..10bcd32c537a1 100644 --- a/central/graphql/generator/schema2.go +++ b/central/graphql/generator/schema2.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "reflect" + "slices" "sort" "strings" "text/template" @@ -101,9 +102,7 @@ func (t *typeEntry) InputFields() []string { } func (t *typeEntry) ExtraFields() []string { - sort.Slice(t.extraResolvers, func(i, j int) bool { - return t.extraResolvers[i] < t.extraResolvers[j] - }) + slices.Sort(t.extraResolvers) return t.extraResolvers } diff --git a/central/graphql/handler/deployments_test.go b/central/graphql/handler/deployments_test.go index 2b1b724fb8b4d..a202312786524 100644 --- a/central/graphql/handler/deployments_test.go +++ b/central/graphql/handler/deployments_test.go @@ -35,7 +35,7 @@ func TestGetDeployment(t *testing.T) { func TestGetDeploymentsFlattenedCVEData(t *testing.T) { mocks := mockResolver(t) - results := make([]deploymentsView.DeploymentCore, 0) + results := make([]deploymentsView.DeploymentCore, 0, 2) core1 := deploymentsViewMocks.NewMockDeploymentCore(mocks.ctrl) core1.EXPECT().GetDeploymentID().Return("one") results = append(results, core1) diff --git a/central/graphql/handler/handler.go b/central/graphql/handler/handler.go index 4d8d5edc00db5..3bac80e63fa89 100644 --- a/central/graphql/handler/handler.go +++ b/central/graphql/handler/handler.go @@ -23,6 +23,7 @@ var ( queryTracerEnabled = env.PostgresQueryTracer.BooleanSetting() graphQLQueryThreshold = env.PostgresQueryTracerGraphQLThreshold.DurationSetting() + graphQLQueryMaxDepth = env.GraphQLQueryMaxDepth.IntegerSetting() ) type logger struct { @@ -94,7 +95,10 @@ func (h *relayHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Handler returns an HTTP handler for the graphql api endpoint func Handler() http.Handler { - opts := []graphql.SchemaOpt{graphql.Logger(&logger{})} + opts := []graphql.SchemaOpt{ + graphql.Logger(&logger{}), + graphql.MaxDepth(graphQLQueryMaxDepth), + } s := resolvers.Schema() ourSchema, err := graphql.ParseSchema(s, resolvers.New(), opts...) if err != nil { diff --git a/central/graphql/handler/handler_benchmark_test.go b/central/graphql/handler/handler_benchmark_test.go index a3075b17a9bc7..83a8bf8e9ac97 100644 --- a/central/graphql/handler/handler_benchmark_test.go +++ b/central/graphql/handler/handler_benchmark_test.go @@ -157,11 +157,11 @@ func getGraphQLServer(testHelper *testutils.ExportServicePostgresTestHelper) (*h deploymentFactory := func() interface{} { return loaders.NewDeploymentLoader(resolver.DeploymentDataStore, testHelper.DeploymentView) } - loaders.RegisterTypeFactory(reflect.TypeOf(storage.Deployment{}), deploymentFactory) + loaders.RegisterTypeFactory(reflect.TypeFor[storage.Deployment](), deploymentFactory) imageFactory := func() interface{} { return loaders.NewImageLoader(resolver.ImageDataStore, testHelper.ImageView) } - loaders.RegisterTypeFactory(reflect.TypeOf(storage.Image{}), imageFactory) + loaders.RegisterTypeFactory(reflect.TypeFor[storage.Image](), imageFactory) ourSchema, err := graphql.ParseSchema(schema, resolver) if err != nil { return nil, err diff --git a/central/graphql/handler/handler_test.go b/central/graphql/handler/handler_test.go index 6971844a5fe00..e73950761afe5 100644 --- a/central/graphql/handler/handler_test.go +++ b/central/graphql/handler/handler_test.go @@ -82,7 +82,7 @@ func mockResolver(t *testing.T) mocks { SecretsDataStore: secret, NetworkPoliciesStore: nps, } - loaders.RegisterTypeFactory(reflect.TypeOf(storage.Deployment{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.Deployment](), func() interface{} { return loaders.NewDeploymentLoader(deployment, deploymentView) }) diff --git a/central/graphql/handler/namespace_test.go b/central/graphql/handler/namespace_test.go index 3c70faaf48fdf..2df63f64771d0 100644 --- a/central/graphql/handler/namespace_test.go +++ b/central/graphql/handler/namespace_test.go @@ -14,7 +14,7 @@ import ( func TestGetNamespaces(t *testing.T) { mocks := mockResolver(t) - loaders.RegisterTypeFactory(reflect.TypeOf(storage.NamespaceMetadata{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.NamespaceMetadata](), func() interface{} { return loaders.NewNamespaceLoader(mocks.namespace) }) mocks.namespace.EXPECT().Search(gomock.Any(), emptyPaginatedQuery()).Return([]search.Result{ diff --git a/central/graphql/resolvers/cluster_vulnerabilities.go b/central/graphql/resolvers/cluster_vulnerabilities.go index 89ce553b46107..d4b463e85add1 100644 --- a/central/graphql/resolvers/cluster_vulnerabilities.go +++ b/central/graphql/resolvers/cluster_vulnerabilities.go @@ -55,6 +55,8 @@ type ClusterVulnerabilityResolver interface { VulnerabilityTypes() []string } +var _ ClusterVulnerabilityResolver = (*clusterCVEResolver)(nil) + // ClusterVulnerability returns a vulnerability of the given id func (resolver *Resolver) ClusterVulnerability(ctx context.Context, args IDQuery) (ClusterVulnerabilityResolver, error) { defer metrics.SetGraphQLOperationDurationTime(time.Now(), pkgMetrics.Root, "ClusterVulnerability") diff --git a/central/graphql/resolvers/cluster_vulnerabilities_test.go b/central/graphql/resolvers/cluster_vulnerabilities_test.go new file mode 100644 index 0000000000000..f052e4366b74f --- /dev/null +++ b/central/graphql/resolvers/cluster_vulnerabilities_test.go @@ -0,0 +1,25 @@ +package resolvers + +import ( + "reflect" + "testing" +) + +func TestClusterVulnerabilityResolverType(t *testing.T) { + // The implementation type has four more methods than the interface it implements. + // + // 17 - SnoozeExpiry + // 18 - SnoozeStart + // 19 - Snoozed + // 24 - Type + // + // This can break graphQL queries to the system until + // https://github.com/graph-gophers/graphql-go/issues/763 is fixed. + // + // TODO(ROX-35654): Unskip this tests. + t.Skip("Interface and implementation types do not have aligned method indices.") + resolverInterface := reflect.TypeFor[ClusterVulnerabilityResolver]() + resolverImplType := reflect.TypeFor[*clusterCVEResolver]() + + validateAlignedMethodIndex(t, resolverInterface, resolverImplType) +} diff --git a/central/graphql/resolvers/clusters.go b/central/graphql/resolvers/clusters.go index 9e8d1c7c8d81e..4679ef99fb37b 100644 --- a/central/graphql/resolvers/clusters.go +++ b/central/graphql/resolvers/clusters.go @@ -252,11 +252,11 @@ func (resolver *clusterResolver) FailingPolicyCounter(ctx context.Context, args return nil, err } - alerts, err := resolver.root.ViolationsDataStore.SearchListAlerts(ctx, q, true) + counts, err := resolver.root.ViolationsDataStore.SearchAlertPolicySeverityCounts(ctx, q, true) if err != nil { return nil, nil } - return mapListAlertsToPolicySeverityCount(alerts), nil + return policySeverityCountsToResolver(counts), nil } // Deployments returns GraphQL resolvers for all deployments in this cluster @@ -696,7 +696,7 @@ func (resolver *clusterResolver) getApplicablePolicies(ctx context.Context, q *v return nil, err } - applicable, _ := matcher.NewClusterMatcher(resolver.data, namespaces).FilterApplicablePolicies(policies) + applicable, _ := matcher.NewClusterMatcher(resolver.data, namespaces).FilterApplicablePolicies(ctx, policies) return applicable, nil } diff --git a/central/graphql/resolvers/clusters_postgres_test.go b/central/graphql/resolvers/clusters_postgres_test.go index f5767c81d2c18..cee6f8f36749f 100644 --- a/central/graphql/resolvers/clusters_postgres_test.go +++ b/central/graphql/resolvers/clusters_postgres_test.go @@ -43,7 +43,7 @@ func (s *graphQLClusterTestSuite) SetupSuite() { s.resolver = &Resolver{ClusterDataStore: clusterDataStore} ctx := sac.WithAllAccess(context.Background()) - for i := 0; i < 6; i++ { + for i := range 6 { cluster := fixtures.GetCluster(fmt.Sprintf("Test cluster %d", i+1)) id, addErr := clusterDataStore.AddCluster(ctx, cluster) s.Require().NoError(addErr) diff --git a/central/graphql/resolvers/deploymentctx/deployment_context.go b/central/graphql/resolvers/deploymentctx/deployment_context.go deleted file mode 100644 index ea29b4988e973..0000000000000 --- a/central/graphql/resolvers/deploymentctx/deployment_context.go +++ /dev/null @@ -1,37 +0,0 @@ -package deploymentctx - -import ( - "context" -) - -// distroContextKey is the key for the distribution value in the context. -type deploymentContextKey struct{} - -// deploymentContextValue holds the value of the distro in the context. -type deploymentContextValue struct { - deployment string -} - -// Context returns a new context with the scope attached. -func Context(ctx context.Context, deploymentID string) context.Context { - return context.WithValue(ctx, deploymentContextKey{}, &deploymentContextValue{ - deployment: deploymentID, - }) -} - -// IsDeploymentScoped returns a boolean if a distro is set -func IsDeploymentScoped(ctx context.Context) bool { - return FromContext(ctx) != "" -} - -// FromContext returns the deployment from the input context. -func FromContext(context context.Context) string { - if context == nil { - return "" - } - deploymentCtxValue := context.Value(deploymentContextKey{}) - if deploymentCtxValue == nil { - return "" - } - return deploymentCtxValue.(*deploymentContextValue).deployment -} diff --git a/central/graphql/resolvers/deploymentevents.go b/central/graphql/resolvers/deploymentevents.go index 6f61aacd4357f..f0c99eaa9c930 100644 --- a/central/graphql/resolvers/deploymentevents.go +++ b/central/graphql/resolvers/deploymentevents.go @@ -216,17 +216,13 @@ func (resolver *Resolver) getProcessActivityEvents(ctx context.Context, query *v return nil, err } - indicators, err := resolver.ProcessIndicatorStore.SearchRawProcessIndicators(ctx, query) - if err != nil { - return nil, errors.Wrap(err, "retrieving process indicators from search") - } - - processEvents := make([]*ProcessActivityEventResolver, 0, len(indicators)) + var processEvents []*ProcessActivityEventResolver baselines := make(map[string]*set.StringSet) // This determines if we should read baseline information. // nil means we can. canReadBaseline := readDeploymentExtensions(ctx) == nil - for _, indicator := range indicators { + + fn := func(indicator *storage.ProcessIndicator) error { var keyStr, procName string if canReadBaseline { key := &storage.ProcessBaselineKey{ @@ -242,10 +238,10 @@ func (resolver *Resolver) getProcessActivityEvents(ctx context.Context, query *v baseline, exists, err := resolver.BaselineDataStore.GetProcessBaseline(ctx, key) if err != nil { log.Error(errors.Wrapf(err, "retrieving baseline data for process %s", indicator.GetSignal().GetName())) - continue + return nil } if !exists { - continue + return nil } baselines[keyStr] = processbaseline.Processes(baseline, processbaseline.RoxOrUserLocked) @@ -256,7 +252,7 @@ func (resolver *Resolver) getProcessActivityEvents(ctx context.Context, query *v timestamp, err := protocompat.ConvertTimestampToTimeOrError(indicator.GetSignal().GetTime()) if err != nil { logTimestampConversionError(indicator.GetSignal().GetName(), "indicator", err) - continue + return nil } // -1 indicates we do not have parent UID information (either no parent exists or we do not know its UID). var parentName *string @@ -282,7 +278,14 @@ func (resolver *Resolver) getProcessActivityEvents(ctx context.Context, query *v canReadBaseline: canReadBaseline, inBaseline: baselines[keyStr] == nil || baselines[keyStr].Contains(procName), }) + return nil } + + err := resolver.ProcessIndicatorStore.GetByQueryFn(ctx, query, fn) + if err != nil { + return nil, errors.Wrap(err, "retrieving process indicators from search") + } + return processEvents, nil } diff --git a/central/graphql/resolvers/deployments.go b/central/graphql/resolvers/deployments.go index 00b70dc320841..cd1e3c5ceb491 100644 --- a/central/graphql/resolvers/deployments.go +++ b/central/graphql/resolvers/deployments.go @@ -267,7 +267,7 @@ func (resolver *deploymentResolver) getApplicablePolicies(ctx context.Context, q return nil, err } - applicable, _ := matcher.NewDeploymentMatcher(resolver.data).FilterApplicablePolicies(policies) + applicable, _ := matcher.NewDeploymentMatcher(resolver.data, resolver.root.ClusterDataStore, resolver.root.NamespaceDataStore).FilterApplicablePolicies(ctx, policies) return applicable, nil } @@ -381,11 +381,11 @@ func (resolver *deploymentResolver) FailingPolicyCounter(ctx context.Context, ar return nil, err } - alerts, err := resolver.root.ViolationsDataStore.SearchListAlerts(ctx, q, true) + counts, err := resolver.root.ViolationsDataStore.SearchAlertPolicySeverityCounts(ctx, q, true) if err != nil { return nil, nil } - return mapListAlertsToPolicySeverityCount(alerts), nil + return policySeverityCountsToResolver(counts), nil } // Secrets returns the total number of secrets for this deployment diff --git a/central/graphql/resolvers/deployments_test.go b/central/graphql/resolvers/deployments_test.go index 8ff04cb1facb9..a52a4aab78fe2 100644 --- a/central/graphql/resolvers/deployments_test.go +++ b/central/graphql/resolvers/deployments_test.go @@ -13,8 +13,9 @@ import ( "github.com/stackrox/rox/central/views/imagecveflat" imagesView "github.com/stackrox/rox/central/views/images" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/grpc/authz/allow" - "github.com/stackrox/rox/pkg/pointers" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/set" @@ -42,16 +43,31 @@ func (s *DeploymentResolversTestSuite) SetupSuite() { s.ctx = loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) mockCtrl := gomock.NewController(s.T()) s.testDB = SetupTestPostgresConn(s.T()) - imgDataStore := CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl) - resolver, _ := SetupTestResolver(s.T(), - CreateTestDeploymentDatastore(s.T(), s.testDB, mockCtrl, imgDataStore), - deploymentsView.NewDeploymentView(s.testDB.DB), - imagesView.NewImageView(s.testDB.DB), - imgDataStore, - CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), - imagecve.NewCVEView(s.testDB.DB), - imagecveflat.NewCVEFlatView(s.testDB.DB), - ) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + var resolver *Resolver + if features.FlattenImageData.Enabled() { + imgV2DataStore := CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl) + resolver, _ = SetupTestResolver(s.T(), + CreateTestDeploymentDatastoreWithImageV2(s.T(), s.testDB, mockCtrl, imgV2DataStore), + deploymentsView.NewDeploymentView(s.testDB.DB), + imagesView.NewImageView(s.testDB.DB), + imgV2DataStore, + CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), + imagecve.NewCVEView(s.testDB.DB), + imagecveflat.NewCVEFlatView(s.testDB.DB), + ) + } else { + imgDataStore := CreateTestImageDatastore(s.T(), s.testDB, mockCtrl) + resolver, _ = SetupTestResolver(s.T(), + CreateTestDeploymentDatastore(s.T(), s.testDB, mockCtrl, imgDataStore), + deploymentsView.NewDeploymentView(s.testDB.DB), + imagesView.NewImageView(s.testDB.DB), + imgDataStore, + CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), + imagecve.NewCVEView(s.testDB.DB), + imagecveflat.NewCVEFlatView(s.testDB.DB), + ) + } s.resolver = resolver // Add Test Data. @@ -60,8 +76,15 @@ func (s *DeploymentResolversTestSuite) SetupSuite() { s.NoError(s.resolver.DeploymentDataStore.UpsertDeployment(s.ctx, deployment)) } s.testImages = testImages() - for _, image := range testImages() { - s.NoError(s.resolver.ImageDataStore.UpsertImage(s.ctx, image)) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + for _, image := range s.testImages { + s.NoError(s.resolver.ImageV2DataStore.UpsertImage(s.ctx, imageUtils.ConvertToV2(image))) + } + } else { + for _, image := range s.testImages { + s.NoError(s.resolver.ImageDataStore.UpsertImage(s.ctx, image)) + } } } @@ -84,7 +107,7 @@ func (s *DeploymentResolversTestSuite) TestDeployments() { }, { desc: "filter by namespace", - q: PaginatedQuery{Query: pointers.String("Namespace:namespace1name")}, + q: PaginatedQuery{Query: new("Namespace:namespace1name")}, deploymentFiler: func(d *storage.Deployment) bool { return strings.HasPrefix(d.GetNamespace(), "namespace1name") }, @@ -93,7 +116,7 @@ func (s *DeploymentResolversTestSuite) TestDeployments() { }, { desc: "filter by image", - q: PaginatedQuery{Query: pointers.String("Image:reg1/img1")}, + q: PaginatedQuery{Query: new("Image:reg1/img1")}, deploymentFiler: func(d *storage.Deployment) bool { return true }, imageFilter: func(img *storage.Image) bool { return strings.HasPrefix(img.GetName().GetFullName(), "reg1/img1") @@ -102,7 +125,7 @@ func (s *DeploymentResolversTestSuite) TestDeployments() { }, { desc: "filter by cve", - q: PaginatedQuery{Query: pointers.String("CVE:cve-2019-2")}, + q: PaginatedQuery{Query: new("CVE:cve-2019-2")}, deploymentFiler: func(d *storage.Deployment) bool { return true }, imageFilter: func(img *storage.Image) bool { for _, component := range img.GetScan().GetComponents() { @@ -120,7 +143,7 @@ func (s *DeploymentResolversTestSuite) TestDeployments() { }, { desc: "filter by deployment+cve", - q: PaginatedQuery{Query: pointers.String("Deployment:dep2name+CVE:cve-2019-2")}, + q: PaginatedQuery{Query: new("Deployment:dep2name+CVE:cve-2019-2")}, deploymentFiler: func(d *storage.Deployment) bool { return strings.HasPrefix(d.GetName(), "dep2name") }, @@ -140,7 +163,7 @@ func (s *DeploymentResolversTestSuite) TestDeployments() { }, { desc: "filter by severity", - q: PaginatedQuery{Query: pointers.String("Severity:CRITICAL_VULNERABILITY_SEVERITY")}, + q: PaginatedQuery{Query: new("Severity:CRITICAL_VULNERABILITY_SEVERITY")}, deploymentFiler: func(d *storage.Deployment) bool { return true }, imageFilter: func(img *storage.Image) bool { for _, component := range img.GetScan().GetComponents() { @@ -158,7 +181,7 @@ func (s *DeploymentResolversTestSuite) TestDeployments() { }, { desc: "filter by severity+fixable", - q: PaginatedQuery{Query: pointers.String("Severity:UNSET_VULNERABILITY_SEVERITY+Fixable:true")}, + q: PaginatedQuery{Query: new("Severity:UNSET_VULNERABILITY_SEVERITY+Fixable:true")}, deploymentFiler: func(d *storage.Deployment) bool { return true }, imageFilter: func(img *storage.Image) bool { for _, component := range img.GetScan().GetComponents() { diff --git a/central/graphql/resolvers/distroctx/distro_context.go b/central/graphql/resolvers/distroctx/distro_context.go deleted file mode 100644 index cd2e5550f8df0..0000000000000 --- a/central/graphql/resolvers/distroctx/distro_context.go +++ /dev/null @@ -1,38 +0,0 @@ -package distroctx - -import ( - "context" -) - -// distroContextKey is the key for the distribution value in the context. -type distroContextKey struct{} - -// distroContextValue holds the value of the distro in the context. -type distroContextValue struct { - distro string -} - -// Context returns a new context with the scope attached. -func Context(ctx context.Context, distro string) context.Context { - return context.WithValue(ctx, distroContextKey{}, &distroContextValue{ - distro: distro, - }) -} - -// IsImageScoped returns a boolean if a distro is set -func IsImageScoped(ctx context.Context) bool { - return FromContext(ctx) != "" -} - -// FromContext returns the distro from the input context -func FromContext(hasDistroContext context.Context) string { - if hasDistroContext == nil { - return "" - } - inter := hasDistroContext.Value(distroContextKey{}) - if inter == nil { - return "" - } - s := inter.(*distroContextValue) - return s.distro -} diff --git a/central/graphql/resolvers/embeddedobjs/component_context.go b/central/graphql/resolvers/embeddedobjs/component_context.go deleted file mode 100644 index c9377020cfad8..0000000000000 --- a/central/graphql/resolvers/embeddedobjs/component_context.go +++ /dev/null @@ -1,64 +0,0 @@ -package embeddedobjs - -import ( - "context" - "time" - - "github.com/stackrox/rox/generated/storage" -) - -// componentContextKey is the key for the *storage.EmbeddedImageScanComponent value in the context. -// All data is scoped to the embedding image. -type componentContextKey struct{} - -// componentContextValue holds the value of the distro in the context. -type componentContextValue struct { - component *storage.EmbeddedImageScanComponent - os string - lastScanned *time.Time -} - -// ComponentContext returns a new context with the component attached. -func ComponentContext(ctx context.Context, os string, lastScanned *time.Time, component *storage.EmbeddedImageScanComponent) context.Context { - return context.WithValue(ctx, componentContextKey{}, &componentContextValue{ - component: component, - os: os, - lastScanned: lastScanned, - }) -} - -// ComponentFromContext returns the component from the input context. -func ComponentFromContext(context context.Context) *storage.EmbeddedImageScanComponent { - if context == nil { - return nil - } - value := context.Value(componentContextKey{}) - if value == nil { - return nil - } - return value.(*componentContextValue).component -} - -// LastScannedFromContext returns the last scanned time of the component, scoped to embedding image, from the input context. -func LastScannedFromContext(context context.Context) *time.Time { - if context == nil { - return nil - } - value := context.Value(componentContextKey{}) - if value == nil { - return nil - } - return value.(*componentContextValue).lastScanned -} - -// OSFromContext returns the operating system of the component, scoped to embedding image, from the input context. -func OSFromContext(context context.Context) string { - if context == nil { - return "" - } - value := context.Value(componentContextKey{}) - if value == nil { - return "" - } - return value.(*componentContextValue).os -} diff --git a/central/graphql/resolvers/embeddedobjs/vuln_context.go b/central/graphql/resolvers/embeddedobjs/vuln_context.go deleted file mode 100644 index e2ce76dcdc071..0000000000000 --- a/central/graphql/resolvers/embeddedobjs/vuln_context.go +++ /dev/null @@ -1,34 +0,0 @@ -package embeddedobjs - -import ( - "context" - - "github.com/stackrox/rox/generated/storage" -) - -// vulnContextKey is the key for the *storage.EmbeddedVulnerability value in the context. -type vulnContextKey struct{} - -// vulnContextValue holds the value of the distro in the context. -type vulnContextValue struct { - vuln *storage.EmbeddedVulnerability -} - -// VulnContext returns a new context with the vuln attached. -func VulnContext(ctx context.Context, vuln *storage.EmbeddedVulnerability) context.Context { - return context.WithValue(ctx, vulnContextKey{}, &vulnContextValue{ - vuln: vuln, - }) -} - -// VulnFromContext returns the vuln from the input context. -func VulnFromContext(context context.Context) *storage.EmbeddedVulnerability { - if context == nil { - return nil - } - value := context.Value(vulnContextKey{}) - if value == nil { - return nil - } - return value.(*vulnContextValue).vuln -} diff --git a/central/graphql/resolvers/gen/main.go b/central/graphql/resolvers/gen/main.go index 29ac8258bac0b..da2fd5ca9664d 100644 --- a/central/graphql/resolvers/gen/main.go +++ b/central/graphql/resolvers/gen/main.go @@ -17,118 +17,140 @@ import ( var ( walkParameters = generator.TypeWalkParameters{ IncludedTypes: []reflect.Type{ - reflect.TypeOf((*storage.Alert)(nil)), - reflect.TypeOf((*storage.Cluster)(nil)), - reflect.TypeOf((*storage.ClusterCVE)(nil)), - reflect.TypeOf((*storage.ComplianceAggregation_Response)(nil)), - reflect.TypeOf((*storage.ComplianceControlResult)(nil)), - reflect.TypeOf((*storage.CVE)(nil)), - reflect.TypeOf((*storage.Deployment)(nil)), - reflect.TypeOf((*storage.FalsePositiveRequest)(nil)), - reflect.TypeOf((*storage.Group)(nil)), - reflect.TypeOf((*storage.Image)(nil)), - reflect.TypeOf((*storage.ImageV2)(nil)), - reflect.TypeOf((*storage.K8SRole)(nil)), - reflect.TypeOf((*storage.K8SRoleBinding)(nil)), - reflect.TypeOf((*storage.ListAlert)(nil)), - reflect.TypeOf((*storage.ListDeployment)(nil)), - reflect.TypeOf((*storage.ListImage)(nil)), - reflect.TypeOf((*storage.ListImageV2)(nil)), - reflect.TypeOf((*storage.ListSecret)(nil)), - reflect.TypeOf((*storage.MitreAttackVector)(nil)), - reflect.TypeOf((*storage.NetworkFlow)(nil)), - reflect.TypeOf((*storage.Node)(nil)), - reflect.TypeOf((*storage.NodeComponent)(nil)), - reflect.TypeOf((*storage.NodeCVE)(nil)), - reflect.TypeOf((*storage.Notifier)(nil)), - reflect.TypeOf((*storage.PermissionSet)(nil)), - reflect.TypeOf((*storage.Pod)(nil)), - reflect.TypeOf((*storage.RequestComment)(nil)), - reflect.TypeOf((*storage.Risk)(nil)), - reflect.TypeOf((*storage.Role)(nil)), - reflect.TypeOf((*storage.Secret)(nil)), - reflect.TypeOf((*storage.ServiceAccount)(nil)), - reflect.TypeOf((*storage.SimpleAccessScope)(nil)), - reflect.TypeOf((*storage.SlimUser)(nil)), - reflect.TypeOf((*storage.Subject)(nil)), - reflect.TypeOf((*storage.TokenMetadata)(nil)), - reflect.TypeOf((*storage.VulnerabilityRequest_Scope)(nil)), - reflect.TypeOf((*storage.VulnerabilityRequest_CVEs)(nil)), - reflect.TypeOf((*storage.ComplianceDomain_Cluster)(nil)), - reflect.TypeOf((*storage.ComplianceDomain_Deployment)(nil)), - reflect.TypeOf((*storage.ComplianceDomain_Node)(nil)), + reflect.TypeFor[*storage.Alert](), + reflect.TypeFor[*storage.Cluster](), + reflect.TypeFor[*storage.ClusterCVE](), + reflect.TypeFor[*storage.ComplianceAggregation_Response](), + reflect.TypeFor[*storage.ComplianceControlResult](), + reflect.TypeFor[*storage.CVE](), + reflect.TypeFor[*storage.Deployment](), + reflect.TypeFor[*storage.FalsePositiveRequest](), + reflect.TypeFor[*storage.Group](), + reflect.TypeFor[*storage.Image](), + reflect.TypeFor[*storage.ImageV2](), + reflect.TypeFor[*storage.K8SRole](), + reflect.TypeFor[*storage.K8SRoleBinding](), + reflect.TypeFor[*storage.ListAlert](), + reflect.TypeFor[*storage.ListDeployment](), + reflect.TypeFor[*storage.ListImage](), + reflect.TypeFor[*storage.ListImageV2](), + reflect.TypeFor[*storage.ListSecret](), + reflect.TypeFor[*storage.MitreAttackVector](), + reflect.TypeFor[*storage.NetworkFlow](), + reflect.TypeFor[*storage.Node](), + reflect.TypeFor[*storage.NodeComponent](), + reflect.TypeFor[*storage.NodeCVE](), + reflect.TypeFor[*storage.Notifier](), + reflect.TypeFor[*storage.PermissionSet](), + reflect.TypeFor[*storage.Pod](), + reflect.TypeFor[*storage.RequestComment](), + reflect.TypeFor[*storage.Risk](), + reflect.TypeFor[*storage.Role](), + reflect.TypeFor[*storage.Secret](), + reflect.TypeFor[*storage.ServiceAccount](), + reflect.TypeFor[*storage.SimpleAccessScope](), + reflect.TypeFor[*storage.SlimUser](), + reflect.TypeFor[*storage.Subject](), + reflect.TypeFor[*storage.TokenMetadata](), + reflect.TypeFor[*storage.VulnerabilityRequest_Scope](), + reflect.TypeFor[*storage.VulnerabilityRequest_CVEs](), + reflect.TypeFor[*storage.ComplianceDomain_Cluster](), + reflect.TypeFor[*storage.ComplianceDomain_Deployment](), + reflect.TypeFor[*storage.ComplianceDomain_Node](), - reflect.TypeOf((*v1.ComplianceStandard)(nil)), - reflect.TypeOf((*v1.GenerateTokenResponse)(nil)), - reflect.TypeOf((*v1.GetComplianceRunStatusesResponse)(nil)), - reflect.TypeOf((*v1.GetPermissionsResponse)(nil)), - reflect.TypeOf((*v1.Metadata)(nil)), - reflect.TypeOf((*v1.Namespace)(nil)), - reflect.TypeOf((*v1.ProcessNameGroup)(nil)), - reflect.TypeOf((*v1.ScopeObject)(nil)), - reflect.TypeOf((*v1.SearchResult)(nil)), + reflect.TypeFor[*v1.ComplianceStandard](), + reflect.TypeFor[*v1.GenerateTokenResponse](), + reflect.TypeFor[*v1.GetComplianceRunStatusesResponse](), + reflect.TypeFor[*v1.GetPermissionsResponse](), + reflect.TypeFor[*v1.Metadata](), + reflect.TypeFor[*v1.Namespace](), + reflect.TypeFor[*v1.ProcessNameGroup](), + reflect.TypeFor[*v1.ScopeObject](), + reflect.TypeFor[*v1.SearchResult](), }, SkipResolvers: []reflect.Type{ - reflect.TypeOf(storage.EmbeddedVulnerability{}), - reflect.TypeOf(storage.EmbeddedImageScanComponent{}), - reflect.TypeOf(storage.EmbeddedNodeScanComponent{}), + reflect.TypeFor[storage.BaseImageInfo](), + reflect.TypeFor[storage.EmbeddedVulnerability](), + reflect.TypeFor[storage.EmbeddedImageScanComponent](), + reflect.TypeFor[storage.EmbeddedNodeScanComponent](), protocompat.TimestampType, - reflect.TypeOf(storage.NodeVulnerability{}), - reflect.TypeOf((*storage.ImageCVEV2)(nil)), - reflect.TypeOf((*storage.ImageComponentV2)(nil)), + reflect.TypeFor[storage.NodeVulnerability](), + reflect.TypeFor[*storage.ImageCVEV2](), + reflect.TypeFor[*storage.ImageComponentV2](), + reflect.TypeFor[storage.EvaluationFilter](), }, SkipFields: []generator.TypeAndField{ { - ParentType: reflect.TypeOf(storage.Image{}), + ParentType: reflect.TypeFor[storage.Image](), FieldName: "Scan", }, { - ParentType: reflect.TypeOf(storage.ImageV2{}), + ParentType: reflect.TypeFor[storage.Image](), + FieldName: "BaseImageInfo", + }, + { + ParentType: reflect.TypeFor[storage.ImageV2](), FieldName: "Scan", }, { - ParentType: reflect.TypeOf(storage.ImageV2{}), + ParentType: reflect.TypeFor[storage.ImageV2](), FieldName: "ScanStats", }, { - ParentType: reflect.TypeOf(storage.ImageScan{}), + ParentType: reflect.TypeFor[storage.ImageV2](), + FieldName: "BaseImageInfo", + }, + { + ParentType: reflect.TypeFor[storage.ImageScan](), FieldName: "Components", }, { - ParentType: reflect.TypeOf(storage.NodeScan{}), + ParentType: reflect.TypeFor[storage.NodeScan](), FieldName: "Components", }, { - ParentType: reflect.TypeOf(storage.Node{}), + ParentType: reflect.TypeFor[storage.Node](), FieldName: "Scan", }, { - ParentType: reflect.TypeOf(storage.CVE{}), + ParentType: reflect.TypeFor[storage.CVE](), FieldName: "Cvss", }, { - ParentType: reflect.TypeOf(storage.CVE{}), + ParentType: reflect.TypeFor[storage.CVE](), FieldName: "CvssV2", }, { - ParentType: reflect.TypeOf(storage.CVE{}), + ParentType: reflect.TypeFor[storage.CVE](), FieldName: "CvssV3", }, { - ParentType: reflect.TypeOf(storage.ImageComponentV2{}), + ParentType: reflect.TypeFor[storage.ImageComponentV2](), FieldName: "Location", }, + { + ParentType: reflect.TypeFor[storage.ImageSignatureVerificationResult](), + FieldName: "VerifierName", + }, + { + ParentType: reflect.TypeFor[storage.Policy](), + FieldName: "EvaluationFilter", + }, + { + ParentType: reflect.TypeFor[storage.ListPolicy](), + FieldName: "EvaluationFilter", + }, }, InputTypes: []reflect.Type{ - reflect.TypeOf((*inputtypes.FalsePositiveVulnRequest)(nil)), - reflect.TypeOf((*inputtypes.AggregateBy)(nil)), - reflect.TypeOf((*inputtypes.SortOption)(nil)), - reflect.TypeOf((*[]*inputtypes.SortOption)(nil)), - reflect.TypeOf((*inputtypes.Pagination)(nil)), - reflect.TypeOf((*inputtypes.VulnReqGlobalScope)(nil)), - reflect.TypeOf((*inputtypes.VulnReqImageScope)(nil)), - reflect.TypeOf((*inputtypes.VulnReqScope)(nil)), - reflect.TypeOf((*analystnotes.ProcessNoteKey)(nil)), + reflect.TypeFor[*inputtypes.FalsePositiveVulnRequest](), + reflect.TypeFor[*inputtypes.AggregateBy](), + reflect.TypeFor[*inputtypes.SortOption](), + reflect.TypeFor[*[]*inputtypes.SortOption](), + reflect.TypeFor[*inputtypes.Pagination](), + reflect.TypeFor[*inputtypes.VulnReqGlobalScope](), + reflect.TypeFor[*inputtypes.VulnReqImageScope](), + reflect.TypeFor[*inputtypes.VulnReqScope](), + reflect.TypeFor[*analystnotes.ProcessNoteKey](), }, } ) diff --git a/central/graphql/resolvers/generated.go b/central/graphql/resolvers/generated.go index b5e51cf34a994..1e9ccedb56b07 100644 --- a/central/graphql/resolvers/generated.go +++ b/central/graphql/resolvers/generated.go @@ -4,6 +4,7 @@ package resolvers import ( "context" + "encoding/base64" "reflect" "github.com/graph-gophers/graphql-go" @@ -30,6 +31,13 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "stsEnabled: Boolean!", })) generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.Access(0))) + utils.Must(builder.AddType("AclEntry", []string{ + "id: Int!", + "perm: Int!", + "tag: AclEntry_AclTag!", + })) + generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.AclEntry_AclTag(0))) + generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.AclType(0))) utils.Must(builder.AddType("AdmissionControlHealthInfo", []string{ "statusErrors: [String!]!", })) @@ -53,6 +61,7 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "clusterName: String!", "deployment: Alert_Deployment", "enforcement: Alert_Enforcement", + "enforcementCount: Int!", "entityType: Alert_EntityType!", "firstOccurred: Time", "id: ID!", @@ -158,12 +167,6 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { utils.Must(builder.AddType("AzureProviderMetadata", []string{ "subscriptionId: String!", })) - utils.Must(builder.AddType("BaseImageInfo", []string{ - "baseImageDigest: String!", - "baseImageFullName: String!", - "baseImageId: String!", - "created: Time", - })) generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.BooleanOperator(0))) utils.Must(builder.AddType("CSCC", []string{ "serviceAccount: String!", @@ -188,12 +191,15 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "types: [CVE_CVEType!]!", })) utils.Must(builder.AddType("CVEInfo", []string{ + "cisaKev: Boolean!", "createdAt: Time", "cve: String!", "cvssMetrics: [CVSSScore]!", "cvssV2: CVSSV2", "cvssV3: CVSSV3", "epss: EPSS", + "exploit: Exploit", + "knownRansomwareCampaign: Boolean!", "lastModified: Time", "link: String!", "publishedOn: Time", @@ -348,6 +354,7 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "orchestratorMetadata: OrchestratorMetadata", "providerMetadata: ProviderMetadata", "sensorVersion: String!", + "sensorVersionCompatibility: SensorVersionCompatibility!", "upgradeStatus: ClusterUpgradeStatus", })) generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.ClusterType(0))) @@ -525,6 +532,7 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "resources: Resources", "secrets: [EmbeddedSecret]!", "securityContext: SecurityContext", + "type: ContainerType!", "volumes: [Volume]!", })) utils.Must(builder.AddType("ContainerConfig", []string{ @@ -570,8 +578,17 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "type: ContainerRuntime!", "version: String!", })) + generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.ContainerType(0))) utils.Must(builder.AddType("CosignSignature", []string{ + "certChainPem: String!", + "certPem: String!", + "rawSignature: String!", + "rekorBundle: String!", + "signatureFormat: CosignSignature_SignatureFormat!", + "signaturePayload: String!", + "sigstoreBundle: String!", })) + generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.CosignSignature_SignatureFormat(0))) utils.Must(builder.AddType("DataSource", []string{ "id: ID!", "mirror: String!", @@ -613,8 +630,14 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "admissionControllerConfig: AdmissionControllerConfig", "autoLockProcessBaselinesConfig: AutoLockProcessBaselinesConfig", "disableAuditLogs: Boolean!", + "processIndicators: DynamicClusterConfig_ProcessIndicatorsConfig", "registryOverride: String!", })) + utils.Must(builder.AddType("DynamicClusterConfig_ProcessIndicatorsConfig", []string{ + "excludeNamespaceFilter: String!", + "excludeOpenshiftNs: Boolean!", + "noPersistence: Boolean!", + })) utils.Must(builder.AddType("EPSS", []string{ "epssPercentile: Float!", "epssProbability: Float!", @@ -646,18 +669,36 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.EventSource(0))) utils.Must(builder.AddType("Exclusion", []string{ "deployment: Exclusion_Deployment", + "excludeByKind: Exclusion_ExcludeByKind", "expiration: Time", "image: Exclusion_Image", "name: String!", + "matcher: ExclusionMatcher", + })) + utils.Must(builder.AddUnionType("ExclusionMatcher", []string{ + "Exclusion_Deployment", + "Exclusion_ExcludeByKind", })) utils.Must(builder.AddType("Exclusion_Deployment", []string{ "name: String!", "scope: Scope", })) + utils.Must(builder.AddType("Exclusion_ExcludeByKind", []string{ + "kinds: [Exclusion_WorkloadKind!]!", + })) utils.Must(builder.AddType("Exclusion_Image", []string{ "name: String!", })) + generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.Exclusion_WorkloadKind(0))) + utils.Must(builder.AddType("Exploit", []string{ + "dateAdded: String!", + "dueDate: String!", + "knownRansomwareCampaignUse: String!", + "requiredAction: String!", + "shortDescription: String!", + })) utils.Must(builder.AddType("FalsePositiveRequest", []string{ + "unused: String!", })) utils.Must(builder.AddInput("FalsePositiveVulnRequest", []string{ "comment: String", @@ -678,11 +719,14 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "meta: FileAccess_FileMetadata", })) utils.Must(builder.AddType("FileAccess_FileMetadata", []string{ + "aclEntries: [AclEntry]!", + "aclType: AclType!", "gid: Int!", "group: String!", "mode: Int!", "uid: Int!", "username: String!", + "xattrName: String!", })) generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.FileAccess_Operation(0))) utils.Must(builder.AddType("GenerateTokenResponse", []string{ @@ -721,7 +765,6 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "value: String!", })) utils.Must(builder.AddType("Image", []string{ - "baseImageInfo: [BaseImageInfo]!", "id: ID!", "isClusterLocal: Boolean!", "lastUpdated: Time", @@ -785,7 +828,6 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { })) generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.ImageSignatureVerificationResult_Status(0))) utils.Must(builder.AddType("ImageV2", []string{ - "baseImageInfo: [BaseImageInfo]!", "digest: String!", "id: ID!", "isClusterLocal: Boolean!", @@ -883,6 +925,7 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.ManagerType(0))) utils.Must(builder.AddType("Metadata", []string{ "buildFlavor: String!", + "compatibleSensorVersions: [String!]!", "licenseStatus: Metadata_LicenseStatus!", "releaseBuild: Boolean!", "version: String!", @@ -1278,8 +1321,10 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { })) utils.Must(builder.AddType("Scope", []string{ "cluster: String!", + "clusterLabel: Scope_Label", "label: Scope_Label", "namespace: String!", + "namespaceLabel: Scope_Label", })) utils.Must(builder.AddType("ScopeObject", []string{ "id: ID!", @@ -1363,6 +1408,7 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "k8SNodeName: String!", "systemNamespaceId: String!", })) + generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.SensorVersionCompatibility(0))) utils.Must(builder.AddType("ServiceAccount", []string{ "annotations: [Label!]!", "automountToken: Boolean!", @@ -1402,11 +1448,13 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { })) utils.Must(builder.AddType("SimpleAccessScope_Rules", []string{ "clusterLabelSelectors: [SetBasedLabelSelector]!", + "includedClusterIds: [String!]!", "includedClusters: [String!]!", "includedNamespaces: [SimpleAccessScope_Rules_Namespace]!", "namespaceLabelSelectors: [SetBasedLabelSelector]!", })) utils.Must(builder.AddType("SimpleAccessScope_Rules_Namespace", []string{ + "clusterId: String!", "clusterName: String!", "namespaceName: String!", })) @@ -1500,6 +1548,7 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "disabled: Boolean!", })) utils.Must(builder.AddType("Traits", []string{ + "expiresAt: Time", "mutabilityMode: Traits_MutabilityMode!", "origin: Traits_Origin!", "visibility: Traits_Visibility!", @@ -1537,6 +1586,7 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "type: String!", })) generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.Volume_MountPropagation(0))) + generator.RegisterProtoEnum(builder, reflect.TypeOf(storage.VulnOrigin(0))) utils.Must(builder.AddInput("VulnReqGlobalScope", []string{ "images: VulnReqImageScope", })) @@ -1562,6 +1612,7 @@ func registerGeneratedTypes(builder generator.SchemaBuilder) { "VulnerabilityRequest_Scope_Global", })) utils.Must(builder.AddType("VulnerabilityRequest_Scope_Global", []string{ + "unused: String!", })) utils.Must(builder.AddType("VulnerabilityRequest_Scope_Image", []string{ "registry: String!", @@ -1751,6 +1802,99 @@ func toAccesses(values *[]string) []storage.Access { return output } +type aclEntryResolver struct { + ctx context.Context + root *Resolver + data *storage.AclEntry +} + +func (resolver *Resolver) wrapAclEntry(value *storage.AclEntry, ok bool, err error) (*aclEntryResolver, error) { + if !ok || err != nil || value == nil { + return nil, err + } + return &aclEntryResolver{root: resolver, data: value}, nil +} + +func (resolver *Resolver) wrapAclEntries(values []*storage.AclEntry, err error) ([]*aclEntryResolver, error) { + if err != nil || len(values) == 0 { + return nil, err + } + output := make([]*aclEntryResolver, len(values)) + for i, v := range values { + output[i] = &aclEntryResolver{root: resolver, data: v} + } + return output, nil +} + +func (resolver *Resolver) wrapAclEntryWithContext(ctx context.Context, value *storage.AclEntry, ok bool, err error) (*aclEntryResolver, error) { + if !ok || err != nil || value == nil { + return nil, err + } + return &aclEntryResolver{ctx: ctx, root: resolver, data: value}, nil +} + +func (resolver *Resolver) wrapAclEntriesWithContext(ctx context.Context, values []*storage.AclEntry, err error) ([]*aclEntryResolver, error) { + if err != nil || len(values) == 0 { + return nil, err + } + output := make([]*aclEntryResolver, len(values)) + for i, v := range values { + output[i] = &aclEntryResolver{ctx: ctx, root: resolver, data: v} + } + return output, nil +} + +func (resolver *aclEntryResolver) Id(ctx context.Context) int32 { + value := resolver.data.GetId() + return int32(value) +} + +func (resolver *aclEntryResolver) Perm(ctx context.Context) int32 { + value := resolver.data.GetPerm() + return int32(value) +} + +func (resolver *aclEntryResolver) Tag(ctx context.Context) string { + value := resolver.data.GetTag() + return value.String() +} + +func toAclEntry_AclTag(value *string) storage.AclEntry_AclTag { + if value != nil { + return storage.AclEntry_AclTag(storage.AclEntry_AclTag_value[*value]) + } + return storage.AclEntry_AclTag(0) +} + +func toAclEntry_AclTags(values *[]string) []storage.AclEntry_AclTag { + if values == nil { + return nil + } + output := make([]storage.AclEntry_AclTag, len(*values)) + for i, v := range *values { + output[i] = toAclEntry_AclTag(&v) + } + return output +} + +func toAclType(value *string) storage.AclType { + if value != nil { + return storage.AclType(storage.AclType_value[*value]) + } + return storage.AclType(0) +} + +func toAclTypes(values *[]string) []storage.AclType { + if values == nil { + return nil + } + output := make([]storage.AclType, len(*values)) + for i, v := range *values { + output[i] = toAclType(&v) + } + return output +} + type admissionControlHealthInfoResolver struct { ctx context.Context root *Resolver @@ -2001,6 +2145,14 @@ func (resolver *alertResolver) Enforcement(ctx context.Context) (*alert_Enforcem return resolver.root.wrapAlert_Enforcement(value, true, nil) } +func (resolver *alertResolver) EnforcementCount(ctx context.Context) int32 { + value := resolver.data.GetEnforcementCount() + if resolver.data == nil { + value = resolver.list.GetEnforcementCount() + } + return value +} + func (resolver *alertResolver) EntityType(ctx context.Context) string { resolver.ensureData(ctx) value := resolver.data.GetEntityType() @@ -3021,68 +3173,6 @@ func (resolver *azureProviderMetadataResolver) SubscriptionId(ctx context.Contex return value } -type baseImageInfoResolver struct { - ctx context.Context - root *Resolver - data *storage.BaseImageInfo -} - -func (resolver *Resolver) wrapBaseImageInfo(value *storage.BaseImageInfo, ok bool, err error) (*baseImageInfoResolver, error) { - if !ok || err != nil || value == nil { - return nil, err - } - return &baseImageInfoResolver{root: resolver, data: value}, nil -} - -func (resolver *Resolver) wrapBaseImageInfos(values []*storage.BaseImageInfo, err error) ([]*baseImageInfoResolver, error) { - if err != nil || len(values) == 0 { - return nil, err - } - output := make([]*baseImageInfoResolver, len(values)) - for i, v := range values { - output[i] = &baseImageInfoResolver{root: resolver, data: v} - } - return output, nil -} - -func (resolver *Resolver) wrapBaseImageInfoWithContext(ctx context.Context, value *storage.BaseImageInfo, ok bool, err error) (*baseImageInfoResolver, error) { - if !ok || err != nil || value == nil { - return nil, err - } - return &baseImageInfoResolver{ctx: ctx, root: resolver, data: value}, nil -} - -func (resolver *Resolver) wrapBaseImageInfosWithContext(ctx context.Context, values []*storage.BaseImageInfo, err error) ([]*baseImageInfoResolver, error) { - if err != nil || len(values) == 0 { - return nil, err - } - output := make([]*baseImageInfoResolver, len(values)) - for i, v := range values { - output[i] = &baseImageInfoResolver{ctx: ctx, root: resolver, data: v} - } - return output, nil -} - -func (resolver *baseImageInfoResolver) BaseImageDigest(ctx context.Context) string { - value := resolver.data.GetBaseImageDigest() - return value -} - -func (resolver *baseImageInfoResolver) BaseImageFullName(ctx context.Context) string { - value := resolver.data.GetBaseImageFullName() - return value -} - -func (resolver *baseImageInfoResolver) BaseImageId(ctx context.Context) string { - value := resolver.data.GetBaseImageId() - return value -} - -func (resolver *baseImageInfoResolver) Created(ctx context.Context) (*graphql.Time, error) { - value := resolver.data.GetCreated() - return protocompat.ConvertTimestampToGraphqlTimeOrError(value) -} - func toBooleanOperator(value *string) storage.BooleanOperator { if value != nil { return storage.BooleanOperator(storage.BooleanOperator_value[*value]) @@ -3317,6 +3407,11 @@ func (resolver *Resolver) wrapCVEInfosWithContext(ctx context.Context, values [] return output, nil } +func (resolver *cVEInfoResolver) CisaKev(ctx context.Context) bool { + value := resolver.data.GetCisaKev() + return value +} + func (resolver *cVEInfoResolver) CreatedAt(ctx context.Context) (*graphql.Time, error) { value := resolver.data.GetCreatedAt() return protocompat.ConvertTimestampToGraphqlTimeOrError(value) @@ -3347,6 +3442,16 @@ func (resolver *cVEInfoResolver) Epss(ctx context.Context) (*ePSSResolver, error return resolver.root.wrapEPSS(value, true, nil) } +func (resolver *cVEInfoResolver) Exploit(ctx context.Context) (*exploitResolver, error) { + value := resolver.data.GetExploit() + return resolver.root.wrapExploit(value, true, nil) +} + +func (resolver *cVEInfoResolver) KnownRansomwareCampaign(ctx context.Context) bool { + value := resolver.data.GetKnownRansomwareCampaign() + return value +} + func (resolver *cVEInfoResolver) LastModified(ctx context.Context) (*graphql.Time, error) { value := resolver.data.GetLastModified() return protocompat.ConvertTimestampToGraphqlTimeOrError(value) @@ -4757,6 +4862,11 @@ func (resolver *clusterStatusResolver) SensorVersion(ctx context.Context) string return value } +func (resolver *clusterStatusResolver) SensorVersionCompatibility(ctx context.Context) string { + value := resolver.data.GetSensorVersionCompatibility() + return value.String() +} + func (resolver *clusterStatusResolver) UpgradeStatus(ctx context.Context) (*clusterUpgradeStatusResolver, error) { value := resolver.data.GetUpgradeStatus() return resolver.root.wrapClusterUpgradeStatus(value, true, nil) @@ -6559,6 +6669,11 @@ func (resolver *containerResolver) SecurityContext(ctx context.Context) (*securi return resolver.root.wrapSecurityContext(value, true, nil) } +func (resolver *containerResolver) Type(ctx context.Context) string { + value := resolver.data.GetType() + return value.String() +} + func (resolver *containerResolver) Volumes(ctx context.Context) ([]*volumeResolver, error) { value := resolver.data.GetVolumes() return resolver.root.wrapVolumes(value, nil) @@ -6997,6 +7112,24 @@ func (resolver *containerRuntimeInfoResolver) Version(ctx context.Context) strin return value } +func toContainerType(value *string) storage.ContainerType { + if value != nil { + return storage.ContainerType(storage.ContainerType_value[*value]) + } + return storage.ContainerType(0) +} + +func toContainerTypes(values *[]string) []storage.ContainerType { + if values == nil { + return nil + } + output := make([]storage.ContainerType, len(*values)) + for i, v := range *values { + output[i] = toContainerType(&v) + } + return output +} + type cosignSignatureResolver struct { ctx context.Context root *Resolver @@ -7039,29 +7172,57 @@ func (resolver *Resolver) wrapCosignSignaturesWithContext(ctx context.Context, v return output, nil } -func (resolver *cosignSignatureResolver) CertChainPem(ctx context.Context) []byte { +func (resolver *cosignSignatureResolver) CertChainPem(ctx context.Context) string { value := resolver.data.GetCertChainPem() - return value + return base64.StdEncoding.EncodeToString(value) } -func (resolver *cosignSignatureResolver) CertPem(ctx context.Context) []byte { +func (resolver *cosignSignatureResolver) CertPem(ctx context.Context) string { value := resolver.data.GetCertPem() - return value + return base64.StdEncoding.EncodeToString(value) } -func (resolver *cosignSignatureResolver) RawSignature(ctx context.Context) []byte { +func (resolver *cosignSignatureResolver) RawSignature(ctx context.Context) string { value := resolver.data.GetRawSignature() - return value + return base64.StdEncoding.EncodeToString(value) } -func (resolver *cosignSignatureResolver) RekorBundle(ctx context.Context) []byte { +func (resolver *cosignSignatureResolver) RekorBundle(ctx context.Context) string { value := resolver.data.GetRekorBundle() - return value + return base64.StdEncoding.EncodeToString(value) +} + +func (resolver *cosignSignatureResolver) SignatureFormat(ctx context.Context) string { + value := resolver.data.GetSignatureFormat() + return value.String() } -func (resolver *cosignSignatureResolver) SignaturePayload(ctx context.Context) []byte { +func (resolver *cosignSignatureResolver) SignaturePayload(ctx context.Context) string { value := resolver.data.GetSignaturePayload() - return value + return base64.StdEncoding.EncodeToString(value) +} + +func (resolver *cosignSignatureResolver) SigstoreBundle(ctx context.Context) string { + value := resolver.data.GetSigstoreBundle() + return base64.StdEncoding.EncodeToString(value) +} + +func toCosignSignature_SignatureFormat(value *string) storage.CosignSignature_SignatureFormat { + if value != nil { + return storage.CosignSignature_SignatureFormat(storage.CosignSignature_SignatureFormat_value[*value]) + } + return storage.CosignSignature_SignatureFormat(0) +} + +func toCosignSignature_SignatureFormats(values *[]string) []storage.CosignSignature_SignatureFormat { + if values == nil { + return nil + } + output := make([]storage.CosignSignature_SignatureFormat, len(*values)) + for i, v := range *values { + output[i] = toCosignSignature_SignatureFormat(&v) + } + return output } type dataSourceResolver struct { @@ -7430,11 +7591,73 @@ func (resolver *dynamicClusterConfigResolver) DisableAuditLogs(ctx context.Conte return value } +func (resolver *dynamicClusterConfigResolver) ProcessIndicators(ctx context.Context) (*dynamicClusterConfig_ProcessIndicatorsConfigResolver, error) { + value := resolver.data.GetProcessIndicators() + return resolver.root.wrapDynamicClusterConfig_ProcessIndicatorsConfig(value, true, nil) +} + func (resolver *dynamicClusterConfigResolver) RegistryOverride(ctx context.Context) string { value := resolver.data.GetRegistryOverride() return value } +type dynamicClusterConfig_ProcessIndicatorsConfigResolver struct { + ctx context.Context + root *Resolver + data *storage.DynamicClusterConfig_ProcessIndicatorsConfig +} + +func (resolver *Resolver) wrapDynamicClusterConfig_ProcessIndicatorsConfig(value *storage.DynamicClusterConfig_ProcessIndicatorsConfig, ok bool, err error) (*dynamicClusterConfig_ProcessIndicatorsConfigResolver, error) { + if !ok || err != nil || value == nil { + return nil, err + } + return &dynamicClusterConfig_ProcessIndicatorsConfigResolver{root: resolver, data: value}, nil +} + +func (resolver *Resolver) wrapDynamicClusterConfig_ProcessIndicatorsConfigs(values []*storage.DynamicClusterConfig_ProcessIndicatorsConfig, err error) ([]*dynamicClusterConfig_ProcessIndicatorsConfigResolver, error) { + if err != nil || len(values) == 0 { + return nil, err + } + output := make([]*dynamicClusterConfig_ProcessIndicatorsConfigResolver, len(values)) + for i, v := range values { + output[i] = &dynamicClusterConfig_ProcessIndicatorsConfigResolver{root: resolver, data: v} + } + return output, nil +} + +func (resolver *Resolver) wrapDynamicClusterConfig_ProcessIndicatorsConfigWithContext(ctx context.Context, value *storage.DynamicClusterConfig_ProcessIndicatorsConfig, ok bool, err error) (*dynamicClusterConfig_ProcessIndicatorsConfigResolver, error) { + if !ok || err != nil || value == nil { + return nil, err + } + return &dynamicClusterConfig_ProcessIndicatorsConfigResolver{ctx: ctx, root: resolver, data: value}, nil +} + +func (resolver *Resolver) wrapDynamicClusterConfig_ProcessIndicatorsConfigsWithContext(ctx context.Context, values []*storage.DynamicClusterConfig_ProcessIndicatorsConfig, err error) ([]*dynamicClusterConfig_ProcessIndicatorsConfigResolver, error) { + if err != nil || len(values) == 0 { + return nil, err + } + output := make([]*dynamicClusterConfig_ProcessIndicatorsConfigResolver, len(values)) + for i, v := range values { + output[i] = &dynamicClusterConfig_ProcessIndicatorsConfigResolver{ctx: ctx, root: resolver, data: v} + } + return output, nil +} + +func (resolver *dynamicClusterConfig_ProcessIndicatorsConfigResolver) ExcludeNamespaceFilter(ctx context.Context) string { + value := resolver.data.GetExcludeNamespaceFilter() + return value +} + +func (resolver *dynamicClusterConfig_ProcessIndicatorsConfigResolver) ExcludeOpenshiftNs(ctx context.Context) bool { + value := resolver.data.GetExcludeOpenshiftNs() + return value +} + +func (resolver *dynamicClusterConfig_ProcessIndicatorsConfigResolver) NoPersistence(ctx context.Context) bool { + value := resolver.data.GetNoPersistence() + return value +} + type ePSSResolver struct { ctx context.Context root *Resolver @@ -7820,6 +8043,11 @@ func (resolver *exclusionResolver) Deployment(ctx context.Context) (*exclusion_D return resolver.root.wrapExclusion_Deployment(value, true, nil) } +func (resolver *exclusionResolver) ExcludeByKind(ctx context.Context) (*exclusion_ExcludeByKindResolver, error) { + value := resolver.data.GetExcludeByKind() + return resolver.root.wrapExclusion_ExcludeByKind(value, true, nil) +} + func (resolver *exclusionResolver) Expiration(ctx context.Context) (*graphql.Time, error) { value := resolver.data.GetExpiration() return protocompat.ConvertTimestampToGraphqlTimeOrError(value) @@ -7835,6 +8063,34 @@ func (resolver *exclusionResolver) Name(ctx context.Context) string { return value } +type exclusionMatcherResolver struct { + resolver interface{} +} + +func (resolver *exclusionResolver) Matcher() *exclusionMatcherResolver { + if val := resolver.data.GetDeployment(); val != nil { + return &exclusionMatcherResolver{ + resolver: &exclusion_DeploymentResolver{root: resolver.root, data: val}, + } + } + if val := resolver.data.GetExcludeByKind(); val != nil { + return &exclusionMatcherResolver{ + resolver: &exclusion_ExcludeByKindResolver{root: resolver.root, data: val}, + } + } + return nil +} + +func (resolver *exclusionMatcherResolver) ToExclusion_Deployment() (*exclusion_DeploymentResolver, bool) { + res, ok := resolver.resolver.(*exclusion_DeploymentResolver) + return res, ok +} + +func (resolver *exclusionMatcherResolver) ToExclusion_ExcludeByKind() (*exclusion_ExcludeByKindResolver, bool) { + res, ok := resolver.resolver.(*exclusion_ExcludeByKindResolver) + return res, ok +} + type exclusion_DeploymentResolver struct { ctx context.Context root *Resolver @@ -7887,6 +8143,53 @@ func (resolver *exclusion_DeploymentResolver) Scope(ctx context.Context) (*scope return resolver.root.wrapScope(value, true, nil) } +type exclusion_ExcludeByKindResolver struct { + ctx context.Context + root *Resolver + data *storage.Exclusion_ExcludeByKind +} + +func (resolver *Resolver) wrapExclusion_ExcludeByKind(value *storage.Exclusion_ExcludeByKind, ok bool, err error) (*exclusion_ExcludeByKindResolver, error) { + if !ok || err != nil || value == nil { + return nil, err + } + return &exclusion_ExcludeByKindResolver{root: resolver, data: value}, nil +} + +func (resolver *Resolver) wrapExclusion_ExcludeByKinds(values []*storage.Exclusion_ExcludeByKind, err error) ([]*exclusion_ExcludeByKindResolver, error) { + if err != nil || len(values) == 0 { + return nil, err + } + output := make([]*exclusion_ExcludeByKindResolver, len(values)) + for i, v := range values { + output[i] = &exclusion_ExcludeByKindResolver{root: resolver, data: v} + } + return output, nil +} + +func (resolver *Resolver) wrapExclusion_ExcludeByKindWithContext(ctx context.Context, value *storage.Exclusion_ExcludeByKind, ok bool, err error) (*exclusion_ExcludeByKindResolver, error) { + if !ok || err != nil || value == nil { + return nil, err + } + return &exclusion_ExcludeByKindResolver{ctx: ctx, root: resolver, data: value}, nil +} + +func (resolver *Resolver) wrapExclusion_ExcludeByKindsWithContext(ctx context.Context, values []*storage.Exclusion_ExcludeByKind, err error) ([]*exclusion_ExcludeByKindResolver, error) { + if err != nil || len(values) == 0 { + return nil, err + } + output := make([]*exclusion_ExcludeByKindResolver, len(values)) + for i, v := range values { + output[i] = &exclusion_ExcludeByKindResolver{ctx: ctx, root: resolver, data: v} + } + return output, nil +} + +func (resolver *exclusion_ExcludeByKindResolver) Kinds(ctx context.Context) []string { + value := resolver.data.GetKinds() + return stringSlice(value) +} + type exclusion_ImageResolver struct { ctx context.Context root *Resolver @@ -7934,6 +8237,91 @@ func (resolver *exclusion_ImageResolver) Name(ctx context.Context) string { return value } +func toExclusion_WorkloadKind(value *string) storage.Exclusion_WorkloadKind { + if value != nil { + return storage.Exclusion_WorkloadKind(storage.Exclusion_WorkloadKind_value[*value]) + } + return storage.Exclusion_WorkloadKind(0) +} + +func toExclusion_WorkloadKinds(values *[]string) []storage.Exclusion_WorkloadKind { + if values == nil { + return nil + } + output := make([]storage.Exclusion_WorkloadKind, len(*values)) + for i, v := range *values { + output[i] = toExclusion_WorkloadKind(&v) + } + return output +} + +type exploitResolver struct { + ctx context.Context + root *Resolver + data *storage.Exploit +} + +func (resolver *Resolver) wrapExploit(value *storage.Exploit, ok bool, err error) (*exploitResolver, error) { + if !ok || err != nil || value == nil { + return nil, err + } + return &exploitResolver{root: resolver, data: value}, nil +} + +func (resolver *Resolver) wrapExploits(values []*storage.Exploit, err error) ([]*exploitResolver, error) { + if err != nil || len(values) == 0 { + return nil, err + } + output := make([]*exploitResolver, len(values)) + for i, v := range values { + output[i] = &exploitResolver{root: resolver, data: v} + } + return output, nil +} + +func (resolver *Resolver) wrapExploitWithContext(ctx context.Context, value *storage.Exploit, ok bool, err error) (*exploitResolver, error) { + if !ok || err != nil || value == nil { + return nil, err + } + return &exploitResolver{ctx: ctx, root: resolver, data: value}, nil +} + +func (resolver *Resolver) wrapExploitsWithContext(ctx context.Context, values []*storage.Exploit, err error) ([]*exploitResolver, error) { + if err != nil || len(values) == 0 { + return nil, err + } + output := make([]*exploitResolver, len(values)) + for i, v := range values { + output[i] = &exploitResolver{ctx: ctx, root: resolver, data: v} + } + return output, nil +} + +func (resolver *exploitResolver) DateAdded(ctx context.Context) string { + value := resolver.data.GetDateAdded() + return value +} + +func (resolver *exploitResolver) DueDate(ctx context.Context) string { + value := resolver.data.GetDueDate() + return value +} + +func (resolver *exploitResolver) KnownRansomwareCampaignUse(ctx context.Context) string { + value := resolver.data.GetKnownRansomwareCampaignUse() + return value +} + +func (resolver *exploitResolver) RequiredAction(ctx context.Context) string { + value := resolver.data.GetRequiredAction() + return value +} + +func (resolver *exploitResolver) ShortDescription(ctx context.Context) string { + value := resolver.data.GetShortDescription() + return value +} + type falsePositiveRequestResolver struct { ctx context.Context root *Resolver @@ -7976,6 +8364,10 @@ func (resolver *Resolver) wrapFalsePositiveRequestsWithContext(ctx context.Conte return output, nil } +func (resolver *falsePositiveRequestResolver) Unused(ctx context.Context) string { + return "" +} + type fileAccessResolver struct { ctx context.Context root *Resolver @@ -8147,6 +8539,16 @@ func (resolver *Resolver) wrapFileAccess_FileMetadatasWithContext(ctx context.Co return output, nil } +func (resolver *fileAccess_FileMetadataResolver) AclEntries(ctx context.Context) ([]*aclEntryResolver, error) { + value := resolver.data.GetAclEntries() + return resolver.root.wrapAclEntries(value, nil) +} + +func (resolver *fileAccess_FileMetadataResolver) AclType(ctx context.Context) string { + value := resolver.data.GetAclType() + return value.String() +} + func (resolver *fileAccess_FileMetadataResolver) Gid(ctx context.Context) int32 { value := resolver.data.GetGid() return int32(value) @@ -8172,6 +8574,11 @@ func (resolver *fileAccess_FileMetadataResolver) Username(ctx context.Context) s return value } +func (resolver *fileAccess_FileMetadataResolver) XattrName(ctx context.Context) string { + value := resolver.data.GetXattrName() + return value +} + func toFileAccess_Operation(value *string) storage.FileAccess_Operation { if value != nil { return storage.FileAccess_Operation(storage.FileAccess_Operation_value[*value]) @@ -8649,12 +9056,6 @@ func (resolver *imageResolver) ensureData(ctx context.Context) { } } -func (resolver *imageResolver) BaseImageInfo(ctx context.Context) ([]*baseImageInfoResolver, error) { - resolver.ensureData(ctx) - value := resolver.data.GetBaseImageInfo() - return resolver.root.wrapBaseImageInfos(value, nil) -} - func (resolver *imageResolver) Id(ctx context.Context) graphql.ID { value := resolver.data.GetId() if resolver.data == nil { @@ -9352,12 +9753,6 @@ func (resolver *imageV2Resolver) ensureData(ctx context.Context) { } } -func (resolver *imageV2Resolver) BaseImageInfo(ctx context.Context) ([]*baseImageInfoResolver, error) { - resolver.ensureData(ctx) - value := resolver.data.GetBaseImageInfo() - return resolver.root.wrapBaseImageInfos(value, nil) -} - func (resolver *imageV2Resolver) Digest(ctx context.Context) string { resolver.ensureData(ctx) value := resolver.data.GetDigest() @@ -10305,6 +10700,11 @@ func (resolver *metadataResolver) BuildFlavor(ctx context.Context) string { return value } +func (resolver *metadataResolver) CompatibleSensorVersions(ctx context.Context) []string { + value := resolver.data.GetCompatibleSensorVersions() + return value +} + func (resolver *metadataResolver) LicenseStatus(ctx context.Context) string { value := resolver.data.GetLicenseStatus() return value.String() @@ -13927,6 +14327,11 @@ func (resolver *scopeResolver) Cluster(ctx context.Context) string { return value } +func (resolver *scopeResolver) ClusterLabel(ctx context.Context) (*scope_LabelResolver, error) { + value := resolver.data.GetClusterLabel() + return resolver.root.wrapScope_Label(value, true, nil) +} + func (resolver *scopeResolver) Label(ctx context.Context) (*scope_LabelResolver, error) { value := resolver.data.GetLabel() return resolver.root.wrapScope_Label(value, true, nil) @@ -13937,6 +14342,11 @@ func (resolver *scopeResolver) Namespace(ctx context.Context) string { return value } +func (resolver *scopeResolver) NamespaceLabel(ctx context.Context) (*scope_LabelResolver, error) { + value := resolver.data.GetNamespaceLabel() + return resolver.root.wrapScope_Label(value, true, nil) +} + type scopeObjectResolver struct { ctx context.Context root *Resolver @@ -14253,8 +14663,10 @@ func (resolver *secretResolver) Namespace(ctx context.Context) string { } func (resolver *secretResolver) Relationship(ctx context.Context) (*secretRelationshipResolver, error) { - resolver.ensureData(ctx) value := resolver.data.GetRelationship() + if resolver.data == nil { + value = resolver.list.GetRelationship() + } return resolver.root.wrapSecretRelationship(value, true, nil) } @@ -14814,6 +15226,24 @@ func (resolver *sensorDeploymentIdentificationResolver) SystemNamespaceId(ctx co return value } +func toSensorVersionCompatibility(value *string) storage.SensorVersionCompatibility { + if value != nil { + return storage.SensorVersionCompatibility(storage.SensorVersionCompatibility_value[*value]) + } + return storage.SensorVersionCompatibility(0) +} + +func toSensorVersionCompatibilities(values *[]string) []storage.SensorVersionCompatibility { + if values == nil { + return nil + } + output := make([]storage.SensorVersionCompatibility, len(*values)) + for i, v := range *values { + output[i] = toSensorVersionCompatibility(&v) + } + return output +} + type serviceAccountResolver struct { ctx context.Context root *Resolver @@ -15230,6 +15660,11 @@ func (resolver *simpleAccessScope_RulesResolver) ClusterLabelSelectors(ctx conte return resolver.root.wrapSetBasedLabelSelectors(value, nil) } +func (resolver *simpleAccessScope_RulesResolver) IncludedClusterIds(ctx context.Context) []string { + value := resolver.data.GetIncludedClusterIds() + return value +} + func (resolver *simpleAccessScope_RulesResolver) IncludedClusters(ctx context.Context) []string { value := resolver.data.GetIncludedClusters() return value @@ -15287,6 +15722,11 @@ func (resolver *Resolver) wrapSimpleAccessScope_Rules_NamespacesWithContext(ctx return output, nil } +func (resolver *simpleAccessScope_Rules_NamespaceResolver) ClusterId(ctx context.Context) string { + value := resolver.data.GetClusterId() + return value +} + func (resolver *simpleAccessScope_Rules_NamespaceResolver) ClusterName(ctx context.Context) string { value := resolver.data.GetClusterName() return value @@ -16200,6 +16640,11 @@ func (resolver *Resolver) wrapTraitsesWithContext(ctx context.Context, values [] return output, nil } +func (resolver *traitsResolver) ExpiresAt(ctx context.Context) (*graphql.Time, error) { + value := resolver.data.GetExpiresAt() + return protocompat.ConvertTimestampToGraphqlTimeOrError(value) +} + func (resolver *traitsResolver) MutabilityMode(ctx context.Context) string { value := resolver.data.GetMutabilityMode() return value.String() @@ -16586,6 +17031,24 @@ func toVolume_MountPropagations(values *[]string) []storage.Volume_MountPropagat return output } +func toVulnOrigin(value *string) storage.VulnOrigin { + if value != nil { + return storage.VulnOrigin(storage.VulnOrigin_value[*value]) + } + return storage.VulnOrigin(0) +} + +func toVulnOrigins(values *[]string) []storage.VulnOrigin { + if values == nil { + return nil + } + output := make([]storage.VulnOrigin, len(*values)) + for i, v := range *values { + output[i] = toVulnOrigin(&v) + } + return output +} + type vulnerabilityRequest_CVEsResolver struct { ctx context.Context root *Resolver @@ -16755,6 +17218,10 @@ func (resolver *Resolver) wrapVulnerabilityRequest_Scope_GlobalsWithContext(ctx return output, nil } +func (resolver *vulnerabilityRequest_Scope_GlobalResolver) Unused(ctx context.Context) string { + return "" +} + type vulnerabilityRequest_Scope_ImageResolver struct { ctx context.Context root *Resolver diff --git a/central/graphql/resolvers/image_components.go b/central/graphql/resolvers/image_components.go index 6b6e14890ec50..d7c6482e5c05c 100644 --- a/central/graphql/resolvers/image_components.go +++ b/central/graphql/resolvers/image_components.go @@ -6,7 +6,6 @@ import ( "github.com/graph-gophers/graphql-go" "github.com/pkg/errors" - "github.com/stackrox/rox/central/graphql/resolvers/embeddedobjs" "github.com/stackrox/rox/central/graphql/resolvers/loaders" "github.com/stackrox/rox/central/metrics" v1 "github.com/stackrox/rox/generated/api/v1" @@ -104,6 +103,8 @@ type ImageComponentResolver interface { FixedIn(ctx context.Context) string } +var _ ImageComponentResolver = (*imageComponentV2Resolver)(nil) + // ImageComponent returns an image component based on an input id (name:version) func (resolver *Resolver) ImageComponent(ctx context.Context, args IDQuery) (ImageComponentResolver, error) { defer metrics.SetGraphQLOperationDurationTime(time.Now(), pkgMetrics.Root, "ImageComponent") @@ -292,11 +293,6 @@ func (resolver *imageComponentV2Resolver) LastScanned(ctx context.Context) (*gra resolver.ctx = ctx } - // Short path. Full image is embedded when image scan resolver is called. - if scanTime := embeddedobjs.LastScannedFromContext(resolver.ctx); scanTime != nil { - return &graphql.Time{Time: *scanTime}, nil - } - if features.FlattenImageData.Enabled() { imageLoader, err := loaders.GetImageV2Loader(resolver.ctx) if err != nil { diff --git a/central/graphql/resolvers/image_components_mocked_test.go b/central/graphql/resolvers/image_components_mocked_test.go new file mode 100644 index 0000000000000..f4080d2970467 --- /dev/null +++ b/central/graphql/resolvers/image_components_mocked_test.go @@ -0,0 +1,427 @@ +package resolvers + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/graph-gophers/graphql-go" + imageCVEV2Mocks "github.com/stackrox/rox/central/cve/image/v2/datastore/mocks" + "github.com/stackrox/rox/central/graphql/resolvers/loaders" + imageComponentV2Mocks "github.com/stackrox/rox/central/imagecomponent/v2/datastore/mocks" + "github.com/stackrox/rox/central/views/imagecomponentflat" + imageComponentFlatViewMocks "github.com/stackrox/rox/central/views/imagecomponentflat/mocks" + "github.com/stackrox/rox/central/views/imagecveflat" + imageCVEFlatViewMocks "github.com/stackrox/rox/central/views/imagecveflat/mocks" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/grpc/authz/allow" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/sac" + searchPkg "github.com/stackrox/rox/pkg/search" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/mock/gomock" +) + +// These tests were created to investigate the failures that occurred +// during the upgrade of graphql-go from 1.5.0 to the next version +// (eventually carried to 1.10.2). + +const ( + // GraphQL query matching the user's request + getFixableCVEsForEntityQuery = ` + query getFixableCvesForEntity($id: ID!, $scopeQuery: String, $vulnQuery: String) { + result: imageComponent(id: $id) { + vulnerabilities: imageVulnerabilities( + query: $vulnQuery + scopeQuery: $scopeQuery + ) { + cve + cvss + severity + fixedByVersion + } + } + } + ` + + findComponentID = ` + query findComponent($query: String) { + components: imageComponents(query: $query) { + id + } + } + ` +) + +// This test was extracted from TestGetImageComponentVulnerabilitiesGraphQL / +// TestGetFixableCVEsForEntityWithGraphQLEngine / query_without_filters +// for detailed tracing purpose. +func TestImageComponentGraphQLQuery(t *testing.T) { + testCtx := loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockContainer := getImageComponentCVEMocks(mockCtrl) + resolver := getImageComponentCVEResolver(t, mockContainer) + + schema, err := graphql.ParseSchema(Schema(), resolver) + require.NoError(t, err) + + ctx := SetAuthorizerOverride(testCtx, allow.Anonymous()) + + componentID := "systemd#0#4cd5259a-d1fc-5c81-ab1a-92484311441e" + + setupImageComponentMocks(mockCtrl, mockContainer) + setupImageComponentCVEMocks(mockCtrl, mockContainer) + + response := schema.Exec(ctx, getFixableCVEsForEntityQuery, "getFixableCvesForEntity", + map[string]interface{}{ + "id": componentID, + "vulnQuery": nil, + "scopeQuery": nil, + }) + + // Check for GraphQL errors + validateResponseHasNoError(t, response) + // Validate the response + validateResponseContainsCVE2023x7008(t, response) +} + +func TestGetImageComponentVulnerabilitiesGraphQL(t *testing.T) { + suite.Run(t, new(ImageComponentVulnerabilitiesGraphQLTestSuite)) +} + +type ImageComponentVulnerabilitiesGraphQLTestSuite struct { + suite.Suite + + ctx context.Context + resolver *Resolver + schema *graphql.Schema + + mockCtrl *gomock.Controller + mockContainer *imageComponentCVEMocks +} + +func (s *ImageComponentVulnerabilitiesGraphQLTestSuite) SetupTest() { + s.ctx = loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) + s.mockCtrl = gomock.NewController(s.T()) + s.mockContainer = getImageComponentCVEMocks(s.mockCtrl) + s.resolver = getImageComponentCVEResolver(s.T(), s.mockContainer) + + // Parse the GraphQL schema + var err error + s.schema, err = graphql.ParseSchema(Schema(), s.resolver) + s.Require().NoError(err) +} + +func (s *ImageComponentVulnerabilitiesGraphQLTestSuite) TearDownTest() { + s.mockCtrl.Finish() +} + +// TestGetFixableCVEsForEntityWithGraphQLEngine validates the GraphQL query by executing it +// through the GraphQL engine (graph-gophers/graphql-go) rather than calling resolver functions directly. +// This tests the full query execution path including parsing, validation, and execution. +func (s *ImageComponentVulnerabilitiesGraphQLTestSuite) TestGetFixableCVEsForEntityWithGraphQLEngine() { + ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) + + setupImageComponentMocks(s.mockCtrl, s.mockContainer) + + // Step 1: Find the systemd component ID using a separate query + findResponse := s.schema.Exec(ctx, findComponentID, "findComponent", + map[string]interface{}{ + "query": "Component:systemd+Component Version:249.11-0ubuntu3.11", + }) + + validateResponseHasNoError(s.T(), findResponse) + componentIDs := validateComponentIDResponse(s.T(), findResponse) + require.NotEmpty(s.T(), componentIDs) + componentID := componentIDs[0] + s.T().Logf("Found systemd component with ID: %s", componentID) + + s.T().Run("query without filters", func(t *testing.T) { + // Execute the GraphQL query without vulnerability filters + setupImageComponentCVEMocks(s.mockCtrl, s.mockContainer) + + response := s.schema.Exec(ctx, getFixableCVEsForEntityQuery, "getFixableCvesForEntity", + map[string]interface{}{ + "id": componentID, + "vulnQuery": nil, + "scopeQuery": nil, + }) + + // Check for GraphQL errors + validateResponseHasNoError(t, response) + // Validate the response + validateResponseContainsCVE2023x7008(t, response) + }) + + s.T().Run("query with vulnQuery filter", func(t *testing.T) { + // Execute the GraphQL query with CVE filter + setupImageComponentCVEMocks(s.mockCtrl, s.mockContainer) + + response := s.schema.Exec(ctx, getFixableCVEsForEntityQuery, "getFixableCvesForEntity", + map[string]interface{}{ + "id": componentID, + "vulnQuery": "CVE:CVE-2023-7008", + "scopeQuery": nil, + }) + + validateResponseHasNoError(t, response) + // Should find exactly one CVE when filtered + validateSingleVulnerabilityResponse(t, response) + validateResponseContainsCVE2023x7008(t, response) + }) + + s.T().Run("query with fixable filter", func(t *testing.T) { + // Execute the GraphQL query with fixable filter + setupImageComponentCVEMocks(s.mockCtrl, s.mockContainer) + + response := s.schema.Exec(ctx, getFixableCVEsForEntityQuery, "getFixableCvesForEntity", + map[string]interface{}{ + "id": componentID, + "vulnQuery": "Fixable:true", + "scopeQuery": nil, + }) + + validateResponseHasNoError(t, response) + // Validate the response + validateResponseContainsCVE2023x7008(t, response) + }) + + s.T().Run("query with combined filters", func(t *testing.T) { + // Execute the GraphQL query with combined CVE and fixable filters + setupImageComponentCVEMocks(s.mockCtrl, s.mockContainer) + + response := s.schema.Exec(ctx, getFixableCVEsForEntityQuery, "getFixableCvesForEntity", + map[string]interface{}{ + "id": componentID, + "vulnQuery": "CVE:CVE-2023-7008+Fixable:true", + "scopeQuery": nil, + }) + + // Validate the response + validateResponseHasNoError(t, response) + validateSingleVulnerabilityResponse(t, response) + validateResponseContainsCVE2023x7008(t, response) + }) + + s.T().Run("query with invalid component ID", func(t *testing.T) { + // Test error handling with invalid component ID + response := s.schema.Exec(ctx, getFixableCVEsForEntityQuery, "getFixableCvesForEntity", + map[string]interface{}{ + "id": "invalid-component-id", + "vulnQuery": nil, + "scopeQuery": nil, + }) + + // Should have errors for invalid component + assert.NotEmpty(t, response.Errors, "Query with invalid ID should produce errors") + + if len(response.Errors) > 0 { + assert.ErrorContains(t, response.Errors[0].Unwrap(), "not all components could be found: invalid-component-id") + } + }) +} + +// region test helper functions + +func getImageComponentCVEMocks(mockCtrl *gomock.Controller) *imageComponentCVEMocks { + return &imageComponentCVEMocks{ + imageComponentDS: imageComponentV2Mocks.NewMockDataStore(mockCtrl), + imageComponentFlatView: imageComponentFlatViewMocks.NewMockComponentFlatView(mockCtrl), + imageCVEDS: imageCVEV2Mocks.NewMockDataStore(mockCtrl), + imageCVEFlatView: imageCVEFlatViewMocks.NewMockCveFlatView(mockCtrl), + } +} + +func getImageComponentCVEResolver(t *testing.T, mockContainer *imageComponentCVEMocks) *Resolver { + resolver, _ := SetupTestResolver( + t, + mockContainer.imageComponentDS, + mockContainer.imageCVEDS, + mockContainer.imageCVEFlatView, + mockContainer.imageComponentFlatView, + ) + return resolver +} + +func setupImageComponentMocks(mockCtrl *gomock.Controller, mockContainer *imageComponentCVEMocks) { + systemdFlatComponent := imageComponentFlatViewMocks.NewMockComponentFlat(mockCtrl) + systemdFlatComponent.EXPECT().GetComponent().AnyTimes().Return("systemd") + systemdFlatComponent.EXPECT().GetComponentIDs().AnyTimes().Return([]string{"systemd#0#4cd5259a-d1fc-5c81-ab1a-92484311441e"}) + systemdFlatComponent.EXPECT().GetVersion().AnyTimes().Return("249.11-0ubuntu3.11") + systemdFlatComponent.EXPECT().GetTopCVSS().AnyTimes().Return(float32(5.5)) + systemdFlatComponent.EXPECT().GetRiskScore().AnyTimes().Return(float32(0)) + systemdFlatComponent.EXPECT().GetOperatingSystem().AnyTimes().Return("ubuntu:22.04") + systemdStorageComponent := &storage.ImageComponentV2{ + Id: "systemd#0#4cd5259a-d1fc-5c81-ab1a-92484311441e", + Name: "systemd", + Version: "249.11-0ubuntu3.11", + Priority: 1, + Source: storage.SourceType_OS, + RiskScore: 0, + SetTopCvss: &storage.ImageComponentV2_TopCvss{ + TopCvss: 5.5, + }, + OperatingSystem: "ubuntu:22.04", + ImageIdV2: "4cd5259a-d1fc-5c81-ab1a-92484311441e", + FromBaseImage: false, + LayerType: storage.LayerType_APPLICATION, + } + mockContainer.imageComponentFlatView.EXPECT(). + Get(gomock.Any(), gomock.Any()). + AnyTimes().Return([]imagecomponentflat.ComponentFlat{systemdFlatComponent}, nil) + mockContainer.imageComponentDS.EXPECT(). + SearchRawImageComponents(gomock.Any(), gomock.Any()). + AnyTimes().Return([]*storage.ImageComponentV2{systemdStorageComponent}, nil) + mockContainer.imageComponentDS.EXPECT(). + GetBatch(gomock.Any(), gomock.Any()). + AnyTimes().Return([]*storage.ImageComponentV2{systemdStorageComponent}, nil) +} + +func setupImageComponentCVEMocks(mockCtrl *gomock.Controller, mockContainer *imageComponentCVEMocks) { + now := new(time.Now()) + moderateVuln := new(storage.VulnerabilitySeverity_MODERATE_VULNERABILITY_SEVERITY) + observedState := new(storage.VulnerabilityState_OBSERVED) + + flatCVE2023x7008 := imageCVEFlatViewMocks.NewMockCveFlat(mockCtrl) + flatCVE2023x7008.EXPECT().GetCVE().AnyTimes().Return("CVE-2023-7008") + flatCVE2023x7008.EXPECT().GetCVEIDs().AnyTimes().Return([]string{"CVE-2023-7008#0#systemd#0#4cd5259a-d1fc-5c81-ab1a-92484311441e"}) + flatCVE2023x7008.EXPECT().GetSeverity().AnyTimes().Return(moderateVuln) + flatCVE2023x7008.EXPECT().GetTopCVSS().AnyTimes().Return(float32(5.5)) + flatCVE2023x7008.EXPECT().GetTopNVDCVSS().AnyTimes().Return(float32(5.4)) + flatCVE2023x7008.EXPECT().GetEPSSProbability().AnyTimes().Return(float32(0.45)) + flatCVE2023x7008.EXPECT().GetAffectedImageCount().AnyTimes().Return(1) + flatCVE2023x7008.EXPECT().GetFirstDiscoveredInSystem().AnyTimes().Return(now) + flatCVE2023x7008.EXPECT().GetPublishDate().AnyTimes().Return(now) + flatCVE2023x7008.EXPECT().GetFirstImageOccurrence().AnyTimes().Return(now) + flatCVE2023x7008.EXPECT().GetState().AnyTimes().Return(observedState) + cve2023x7008 := &storage.ImageCVEV2{ + Id: "CVE-2023-7008#0#systemd#0#4cd5259a-d1fc-5c81-ab1a-92484311441e", + CveBaseInfo: &storage.CVEInfo{ + Cve: "CVE-2023-7008", + CreatedAt: protocompat.TimestampNow(), + ScoreVersion: storage.CVEInfo_V2, + }, + Cvss: 5.5, + Severity: storage.VulnerabilitySeverity_MODERATE_VULNERABILITY_SEVERITY, + NvdScoreVersion: storage.CvssScoreVersion_UNKNOWN_VERSION, + FirstImageOccurrence: protocompat.TimestampNow(), + State: storage.VulnerabilityState_OBSERVED, + IsFixable: true, + HasFixedBy: &storage.ImageCVEV2_FixedBy{ + FixedBy: "249.11-0ubuntu3.12", + }, + ComponentId: "systemd#0#4cd5259a-d1fc-5c81-ab1a-92484311441e", + ImageIdV2: "4cd5259a-d1fc-5c81-ab1a-92484311441e", + } + mockContainer.imageCVEDS.EXPECT(). + Search(gomock.Any(), gomock.Any()). + AnyTimes().Return([]searchPkg.Result{{ID: cve2023x7008.GetId()}}, nil) + mockContainer.imageCVEDS.EXPECT(). + SearchRawImageCVEs(gomock.Any(), gomock.Any()). + AnyTimes().Return([]*storage.ImageCVEV2{cve2023x7008}, nil) + mockContainer.imageCVEFlatView.EXPECT(). + Get(gomock.Any(), gomock.Any(), gomock.Any()). + AnyTimes().Return([]imagecveflat.CveFlat{flatCVE2023x7008}, nil) + mockContainer.imageCVEDS.EXPECT(). + GetBatch(gomock.Any(), gomock.Any()). + AnyTimes().Return([]*storage.ImageCVEV2{cve2023x7008}, nil) + +} + +func validateResponseHasNoError(t testing.TB, response *graphql.Response) { + t.Helper() + + if len(response.Errors) > 0 { + for _, err := range response.Errors { + t.Logf("GraphQL Error: %s", err.Error()) + } + } + assert.Empty(t, response.Errors, "Query should not produce errors") +} + +func validateComponentIDResponse(t testing.TB, response *graphql.Response) []string { + t.Helper() + + var findResult struct { + Components []struct { + ID string `json:"id"` + } `json:"components"` + } + require.NoError(t, json.Unmarshal(response.Data, &findResult)) + require.NotEmpty(t, findResult.Components, "Should find systemd component") + + res := make([]string, 0, len(findResult.Components)) + for _, component := range findResult.Components { + res = append(res, component.ID) + } + return res +} + +func validateSingleVulnerabilityResponse(t testing.TB, response *graphql.Response) { + t.Helper() + + var result testQueryResponse + require.NoError(t, json.Unmarshal(response.Data, &result)) + + require.Len(t, result.Result.Vulnerabilities, 1, "Should find exactly one fixable CVE-2023-7008") +} + +func validateResponseContainsCVE2023x7008(t testing.TB, response *graphql.Response) { + t.Helper() + + // Parse the response + var result testQueryResponse + require.NoError(t, json.Unmarshal(response.Data, &result)) + + // Validate we got results + assert.NotEmpty(t, result.Result.Vulnerabilities, "Should have at least one vulnerability") + + // Validate CVE-2023-7008 is present + foundCVE := false + for _, vuln := range result.Result.Vulnerabilities { + t.Logf("Found CVE: %s, CVSS: %f, Severity: %s, Fixed: %s", + vuln.CVE, vuln.CVSS, vuln.Severity, vuln.FixedByVersion) + + if vuln.CVE == "CVE-2023-7008" { + foundCVE = true + assert.Equal(t, 5.5, vuln.CVSS, "CVSS should match") + assert.Equal(t, "MODERATE_VULNERABILITY_SEVERITY", vuln.Severity, "Severity should match") + assert.Equal(t, "249.11-0ubuntu3.12", vuln.FixedByVersion, "Fix version should match") + } + } + assert.True(t, foundCVE, "CVE-2023-7008 should be found in results") +} + +// endregion test helper functions + +// region helper test types + +type imageComponentCVEMocks struct { + imageComponentDS *imageComponentV2Mocks.MockDataStore + imageComponentFlatView *imageComponentFlatViewMocks.MockComponentFlatView + imageCVEDS *imageCVEV2Mocks.MockDataStore + imageCVEFlatView *imageCVEFlatViewMocks.MockCveFlatView +} + +// Response structure matching the GraphQL query +type testCVEResponse struct { + CVE string `json:"cve"` + CVSS float64 `json:"cvss"` + Severity string `json:"severity"` + FixedByVersion string `json:"fixedByVersion"` +} + +type testImageComponentResponse struct { + Vulnerabilities []testCVEResponse `json:"vulnerabilities"` +} + +type testQueryResponse struct { + Result testImageComponentResponse `json:"result"` +} + +// endregion helper test types diff --git a/central/graphql/resolvers/image_components_test.go b/central/graphql/resolvers/image_components_test.go new file mode 100644 index 0000000000000..72b2c65c8c955 --- /dev/null +++ b/central/graphql/resolvers/image_components_test.go @@ -0,0 +1,23 @@ +package resolvers + +import ( + "reflect" + "testing" +) + +func TestImageComponentResolverType(t *testing.T) { + // The implementation type has two more methods than the interface it implements. + // + // 0 - Architecture + // 7 - ImageId + // + // This can break graphQL queries to the system until + // https://github.com/graph-gophers/graphql-go/issues/763 is fixed. + // + // TODO(ROX-35654): Unskip this tests. + t.Skip("Interface and implementation types do not have aligned method indices.") + resolverInterface := reflect.TypeFor[ImageComponentResolver]() + resolverImplType := reflect.TypeFor[*imageComponentV2Resolver]() + + validateAlignedMethodIndex(t, resolverInterface, resolverImplType) +} diff --git a/central/graphql/resolvers/image_components_utilities.go b/central/graphql/resolvers/image_components_utilities.go index e9db5ed1e5ffd..57a95b10c0077 100644 --- a/central/graphql/resolvers/image_components_utilities.go +++ b/central/graphql/resolvers/image_components_utilities.go @@ -22,42 +22,6 @@ type imageComponentV2Resolver struct { flatData imagecomponentflat.ComponentFlat } -func (resolver *Resolver) wrapImageComponentV2(value *storage.ImageComponentV2, ok bool, err error) (*imageComponentV2Resolver, error) { - if !ok || err != nil || value == nil { - return nil, err - } - return &imageComponentV2Resolver{root: resolver, data: value}, nil -} - -func (resolver *Resolver) wrapImageComponentV2s(values []*storage.ImageComponentV2, err error) ([]*imageComponentV2Resolver, error) { - if err != nil || len(values) == 0 { - return nil, err - } - output := make([]*imageComponentV2Resolver, len(values)) - for i, v := range values { - output[i] = &imageComponentV2Resolver{root: resolver, data: v} - } - return output, nil -} - -func (resolver *Resolver) wrapImageComponentV2WithContext(ctx context.Context, value *storage.ImageComponentV2, ok bool, err error) (*imageComponentV2Resolver, error) { - if !ok || err != nil || value == nil { - return nil, err - } - return &imageComponentV2Resolver{ctx: ctx, root: resolver, data: value}, nil -} - -func (resolver *Resolver) wrapImageComponentV2sWithContext(ctx context.Context, values []*storage.ImageComponentV2, err error) ([]*imageComponentV2Resolver, error) { - if err != nil || len(values) == 0 { - return nil, err - } - output := make([]*imageComponentV2Resolver, len(values)) - for i, v := range values { - output[i] = &imageComponentV2Resolver{ctx: ctx, root: resolver, data: v} - } - return output, nil -} - func (resolver *Resolver) wrapImageComponentV2FlatWithContext(ctx context.Context, value *storage.ImageComponentV2, flatData imagecomponentflat.ComponentFlat, ok bool, err error) (*imageComponentV2Resolver, error) { if !ok || err != nil || value == nil { return nil, err @@ -130,5 +94,5 @@ func (resolver *imageComponentV2Resolver) Version(_ context.Context) string { } func (resolver *imageComponentV2Resolver) InBaseImageLayer(ctx context.Context) bool { - return resolver.data.GetFromBaseImage() + return resolver.data.GetLayerType() == storage.LayerType_BASE_IMAGE } diff --git a/central/graphql/resolvers/image_components_v2_postgres_test.go b/central/graphql/resolvers/image_components_v2_postgres_test.go index d31eab72f83f2..0af4fdd85ed42 100644 --- a/central/graphql/resolvers/image_components_v2_postgres_test.go +++ b/central/graphql/resolvers/image_components_v2_postgres_test.go @@ -14,8 +14,10 @@ import ( "github.com/stackrox/rox/central/views/imagecveflat" imagesView "github.com/stackrox/rox/central/views/images" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" "github.com/stackrox/rox/pkg/grpc/authz/allow" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/sac" "github.com/stretchr/testify/assert" @@ -54,6 +56,7 @@ type GraphQLImageComponentV2TestSuite struct { ctx context.Context testDB *pgtest.TestPostgres resolver *Resolver + testImages []*storage.Image componentIDMap map[string]string } @@ -61,17 +64,33 @@ func (s *GraphQLImageComponentV2TestSuite) SetupSuite() { s.ctx = loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) mockCtrl := gomock.NewController(s.T()) s.testDB = pgtest.ForT(s.T()) - imageDataStore := CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl) - resolver, _ := SetupTestResolver(s.T(), - imagesView.NewImageView(s.testDB.DB), - imageDataStore, - CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), - CreateTestImageCVEV2Datastore(s.T(), s.testDB), - CreateTestDeploymentDatastore(s.T(), s.testDB, mockCtrl, imageDataStore), - deploymentsView.NewDeploymentView(s.testDB.DB), - imagecveflat.NewCVEFlatView(s.testDB.DB), - imagecomponentflat.NewComponentFlatView(s.testDB.DB), - ) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + var resolver *Resolver + if features.FlattenImageData.Enabled() { + imgV2DataStore := CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl) + resolver, _ = SetupTestResolver(s.T(), + imagesView.NewImageView(s.testDB.DB), + imgV2DataStore, + CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), + CreateTestImageCVEV2Datastore(s.T(), s.testDB), + CreateTestDeploymentDatastoreWithImageV2(s.T(), s.testDB, mockCtrl, imgV2DataStore), + deploymentsView.NewDeploymentView(s.testDB.DB), + imagecveflat.NewCVEFlatView(s.testDB.DB), + imagecomponentflat.NewComponentFlatView(s.testDB.DB), + ) + } else { + imageDataStore := CreateTestImageDatastore(s.T(), s.testDB, mockCtrl) + resolver, _ = SetupTestResolver(s.T(), + imagesView.NewImageView(s.testDB.DB), + imageDataStore, + CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), + CreateTestImageCVEV2Datastore(s.T(), s.testDB), + CreateTestDeploymentDatastore(s.T(), s.testDB, mockCtrl, imageDataStore), + deploymentsView.NewDeploymentView(s.testDB.DB), + imagecveflat.NewCVEFlatView(s.testDB.DB), + imagecomponentflat.NewComponentFlatView(s.testDB.DB), + ) + } s.resolver = resolver // Add Test Data to DataStores @@ -81,13 +100,21 @@ func (s *GraphQLImageComponentV2TestSuite) SetupSuite() { s.NoError(err) } - testImages := testImagesWithOperatingSystems() - for _, image := range testImages { - err := s.resolver.ImageDataStore.UpsertImage(s.ctx, image) - s.NoError(err) + s.testImages = testImagesWithOperatingSystems() + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + for _, image := range s.testImages { + err := s.resolver.ImageV2DataStore.UpsertImage(s.ctx, imageUtils.ConvertToV2(image)) + s.NoError(err) + } + } else { + for _, image := range s.testImages { + err := s.resolver.ImageDataStore.UpsertImage(s.ctx, image) + s.NoError(err) + } } - s.componentIDMap = s.getComponentIDMap() + s.componentIDMap = s.getComponentIDMap(s.testImages) } func (s *GraphQLImageComponentV2TestSuite) TestUnauthorizedImageComponentEndpoint() { @@ -128,7 +155,7 @@ func (s *GraphQLImageComponentV2TestSuite) TestImageComponents() { } inBaseImageLayerMap := map[string]bool{ s.componentIDMap[comp11]: false, - s.componentIDMap[comp12]: true, + s.componentIDMap[comp12]: false, // comp1 on image2 has no LayerIndex, so we can't determine origin s.componentIDMap[comp21]: false, s.componentIDMap[comp31]: false, s.componentIDMap[comp32]: true, @@ -159,8 +186,8 @@ func (s *GraphQLImageComponentV2TestSuite) TestImageComponentsScoped() { expectedComponentIDs []string }{ { - "sha1", - "sha1", + "image1", + s.getImageID(s.testImages[0]), []string{ s.componentIDMap[comp11], s.componentIDMap[comp21], @@ -168,8 +195,8 @@ func (s *GraphQLImageComponentV2TestSuite) TestImageComponentsScoped() { }, }, { - "sha2", - "sha2", + "image2", + s.getImageID(s.testImages[1]), []string{ s.componentIDMap[comp12], s.componentIDMap[comp32], @@ -208,8 +235,8 @@ func (s *GraphQLImageComponentV2TestSuite) TestImageComponentsScopeTree() { cveToExpectedComponentIDs map[string][]string }{ { - "sha1", - "sha1", + "image1", + s.getImageID(s.testImages[0]), map[string][]string{ "cve-2018-1": { s.componentIDMap[comp11], @@ -224,8 +251,8 @@ func (s *GraphQLImageComponentV2TestSuite) TestImageComponentsScopeTree() { }, }, { - "sha2", - "sha2", + "image2", + s.getImageID(s.testImages[1]), map[string][]string{ "cve-2018-1": { s.componentIDMap[comp12], @@ -296,6 +323,9 @@ func (s *GraphQLImageComponentV2TestSuite) TestImageComponentHit() { func (s *GraphQLImageComponentV2TestSuite) TestImageComponentImages() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) + imageID1 := s.getImageID(s.testImages[0]) + imageID2 := s.getImageID(s.testImages[1]) + imageCompTests := []struct { name string id string @@ -304,32 +334,32 @@ func (s *GraphQLImageComponentV2TestSuite) TestImageComponentImages() { { "comp1image1", s.componentIDMap[comp11], - []string{"sha1"}, + []string{imageID1}, }, { "comp1image2", s.componentIDMap[comp12], - []string{"sha2"}, + []string{imageID2}, }, { "comp2image1", s.componentIDMap[comp21], - []string{"sha1"}, + []string{imageID1}, }, { "comp3image1", s.componentIDMap[comp31], - []string{"sha1"}, + []string{imageID1}, }, { "comp3image2", s.componentIDMap[comp32], - []string{"sha2"}, + []string{imageID2}, }, { "comp4image2", s.componentIDMap[comp42], - []string{"sha2"}, + []string{imageID2}, }, } @@ -592,14 +622,24 @@ func (s *GraphQLImageComponentV2TestSuite) getImageComponentResolver(ctx context return vuln } -func (s *GraphQLImageComponentV2TestSuite) getComponentIDMap() map[string]string { +// TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. +func (s *GraphQLImageComponentV2TestSuite) getImageID(image *storage.Image) string { + if features.FlattenImageData.Enabled() { + return imageUtils.ConvertToV2(image).GetId() + } + return image.GetId() +} + +func (s *GraphQLImageComponentV2TestSuite) getComponentIDMap(images []*storage.Image) map[string]string { + imageID1 := s.getImageID(images[0]) + imageID2 := s.getImageID(images[1]) return map[string]string{ - comp11: getTestComponentID(testImages()[0].GetScan().GetComponents()[0], "sha1", 0), - comp12: getTestComponentID(testImages()[1].GetScan().GetComponents()[0], "sha2", 0), - comp21: getTestComponentID(testImages()[0].GetScan().GetComponents()[1], "sha1", 1), - comp31: getTestComponentID(testImages()[0].GetScan().GetComponents()[2], "sha1", 2), - comp32: getTestComponentID(testImages()[1].GetScan().GetComponents()[1], "sha2", 1), - comp42: getTestComponentID(testImages()[1].GetScan().GetComponents()[2], "sha2", 2), + comp11: getTestComponentID(images[0].GetScan().GetComponents()[0], imageID1, 0), + comp12: getTestComponentID(images[1].GetScan().GetComponents()[0], imageID2, 0), + comp21: getTestComponentID(images[0].GetScan().GetComponents()[1], imageID1, 1), + comp31: getTestComponentID(images[0].GetScan().GetComponents()[2], imageID1, 2), + comp32: getTestComponentID(images[1].GetScan().GetComponents()[1], imageID2, 1), + comp42: getTestComponentID(images[1].GetScan().GetComponents()[2], imageID2, 2), } } diff --git a/central/graphql/resolvers/image_cve_core.go b/central/graphql/resolvers/image_cve_core.go index 0c79acd591c40..6762b283093d2 100644 --- a/central/graphql/resolvers/image_cve_core.go +++ b/central/graphql/resolvers/image_cve_core.go @@ -15,7 +15,6 @@ import ( "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/features" pkgMetrics "github.com/stackrox/rox/pkg/metrics" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/paginated" "github.com/stackrox/rox/pkg/utils" @@ -167,7 +166,7 @@ func (resolver *imageCVECoreResolver) Deployments(ctx context.Context, args stru depQ := search.NewQueryBuilder().AddExactMatches(search.DeploymentID, deploymentIDs...).Query() return resolver.root.Deployments(ctx, PaginatedQuery{ - Query: pointers.String(depQ), + Query: new(depQ), Pagination: args.Pagination, }) } @@ -288,7 +287,7 @@ func (resolver *imageCVECoreResolver) Images(ctx context.Context, args struct{ P } imageQ := search.NewQueryBuilder().AddExactMatches(searchField, imageIDs...).Query() return resolver.root.Images(ctx, PaginatedQuery{ - Query: pointers.String(imageQ), + Query: new(imageQ), Pagination: args.Pagination, }) } diff --git a/central/graphql/resolvers/image_cve_core_test.go b/central/graphql/resolvers/image_cve_core_test.go index ec25a7aac0be4..0b008dbbbabee 100644 --- a/central/graphql/resolvers/image_cve_core_test.go +++ b/central/graphql/resolvers/image_cve_core_test.go @@ -9,7 +9,6 @@ import ( "github.com/stackrox/rox/central/views" "github.com/stackrox/rox/central/views/imagecve" imageCVEViewMock "github.com/stackrox/rox/central/views/imagecve/mocks" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" "github.com/stretchr/testify/suite" @@ -77,7 +76,7 @@ func (s *ImageCVECoreResolverTestSuite) TestGetImageCVEsNonEmpty() { func (s *ImageCVECoreResolverTestSuite) TestGetImageCVEsWithQuery() { q := &PaginatedQuery{ - Query: pointers.String("CVE:cve-2022-xyz"), + Query: new("CVE:cve-2022-xyz"), } expectedQ := search.NewQueryBuilder().AddStrings(search.CVE, "cve-2022-xyz"). WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery() @@ -98,9 +97,9 @@ func (s *ImageCVECoreResolverTestSuite) TestImageCVEsWithPaginatedQuery() { q := &PaginatedQuery{ Pagination: &inputtypes.Pagination{ SortOption: &inputtypes.SortOption{ - Field: pointers.String(search.CVSS.String()), + Field: new(search.CVSS.String()), AggregateBy: &inputtypes.AggregateBy{ - AggregateFunc: pointers.String(aggregatefunc.Max.Name()), + AggregateFunc: new(aggregatefunc.Max.Name()), }, }, }, @@ -141,7 +140,7 @@ func (s *ImageCVECoreResolverTestSuite) TestImageCVECount() { func (s *ImageCVECoreResolverTestSuite) TestImageCVECountWithQuery() { q := &RawQuery{ - Query: pointers.String("Image:image"), + Query: new("Image:image"), } expectedQ := search.NewQueryBuilder().AddStrings(search.ImageName, "image").ProtoQuery() @@ -172,7 +171,7 @@ func (s *ImageCVECoreResolverTestSuite) TestGetImageCVENonEmpty() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), + Cve: new("cve-xyz"), }, ) s.NoError(err) @@ -192,8 +191,8 @@ func (s *ImageCVECoreResolverTestSuite) TestGetImageCVENonEmpty() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), - SubfieldScopeQuery: pointers.String("Fixable:true"), + Cve: new("cve-xyz"), + SubfieldScopeQuery: new("Fixable:true"), }, ) s.NoError(err) @@ -213,8 +212,8 @@ func (s *ImageCVECoreResolverTestSuite) TestGetImageCVENonEmpty() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), - SubfieldScopeQuery: pointers.String("Namespace:n1"), + Cve: new("cve-xyz"), + SubfieldScopeQuery: new("Namespace:n1"), }, ) s.NoError(err) diff --git a/central/graphql/resolvers/image_cve_v2_core_test.go b/central/graphql/resolvers/image_cve_v2_core_test.go index 05a0d88cb5586..b047f904909cc 100644 --- a/central/graphql/resolvers/image_cve_v2_core_test.go +++ b/central/graphql/resolvers/image_cve_v2_core_test.go @@ -9,7 +9,6 @@ import ( "github.com/stackrox/rox/central/views" "github.com/stackrox/rox/central/views/imagecve" imageCVEViewMock "github.com/stackrox/rox/central/views/imagecve/mocks" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" "github.com/stretchr/testify/suite" @@ -77,7 +76,7 @@ func (s *ImageCVEV2CoreResolverTestSuite) TestGetImageCVEsNonEmpty() { func (s *ImageCVEV2CoreResolverTestSuite) TestGetImageCVEsWithQuery() { q := &PaginatedQuery{ - Query: pointers.String("CVE:cve-2022-xyz"), + Query: new("CVE:cve-2022-xyz"), } expectedQ := search.NewQueryBuilder().AddStrings(search.CVE, "cve-2022-xyz"). WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery() @@ -98,9 +97,9 @@ func (s *ImageCVEV2CoreResolverTestSuite) TestImageCVEsWithPaginatedQuery() { q := &PaginatedQuery{ Pagination: &inputtypes.Pagination{ SortOption: &inputtypes.SortOption{ - Field: pointers.String(search.CVSS.String()), + Field: new(search.CVSS.String()), AggregateBy: &inputtypes.AggregateBy{ - AggregateFunc: pointers.String(aggregatefunc.Max.Name()), + AggregateFunc: new(aggregatefunc.Max.Name()), }, }, }, @@ -141,7 +140,7 @@ func (s *ImageCVEV2CoreResolverTestSuite) TestImageCVECount() { func (s *ImageCVEV2CoreResolverTestSuite) TestImageCVECountWithQuery() { q := &RawQuery{ - Query: pointers.String("Image:image"), + Query: new("Image:image"), } expectedQ := search.NewQueryBuilder().AddStrings(search.ImageName, "image").ProtoQuery() @@ -172,7 +171,7 @@ func (s *ImageCVEV2CoreResolverTestSuite) TestGetImageCVENonEmpty() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), + Cve: new("cve-xyz"), }, ) s.NoError(err) @@ -192,8 +191,8 @@ func (s *ImageCVEV2CoreResolverTestSuite) TestGetImageCVENonEmpty() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), - SubfieldScopeQuery: pointers.String("Fixable:true"), + Cve: new("cve-xyz"), + SubfieldScopeQuery: new("Fixable:true"), }, ) s.NoError(err) @@ -213,8 +212,8 @@ func (s *ImageCVEV2CoreResolverTestSuite) TestGetImageCVENonEmpty() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), - SubfieldScopeQuery: pointers.String("Namespace:n1"), + Cve: new("cve-xyz"), + SubfieldScopeQuery: new("Namespace:n1"), }, ) s.NoError(err) diff --git a/central/graphql/resolvers/image_resolver_test.go b/central/graphql/resolvers/image_resolver_test.go new file mode 100644 index 0000000000000..8df40ac7f4cfb --- /dev/null +++ b/central/graphql/resolvers/image_resolver_test.go @@ -0,0 +1,20 @@ +package resolvers + +import ( + "reflect" + "testing" +) + +func TestImageResolverType(t *testing.T) { + resolverInterface := reflect.TypeFor[ImageResolver]() + resolverImplType := reflect.TypeFor[*imageResolver]() + + validateAlignedMethodIndex(t, resolverInterface, resolverImplType) +} + +func TestImageV2ResolverType(t *testing.T) { + resolverInterface := reflect.TypeFor[ImageResolver]() + resolverImplType := reflect.TypeFor[*imageV2Resolver]() + + validateAlignedMethodIndex(t, resolverInterface, resolverImplType) +} diff --git a/central/graphql/resolvers/image_scan_benchmark_test.go b/central/graphql/resolvers/image_scan_benchmark_test.go index 0d02d0fd3da74..8c0714cc8572b 100644 --- a/central/graphql/resolvers/image_scan_benchmark_test.go +++ b/central/graphql/resolvers/image_scan_benchmark_test.go @@ -3,9 +3,11 @@ package resolvers import ( "testing" + graphql "github.com/graph-gophers/graphql-go" "github.com/stackrox/rox/central/views/imagecomponentflat" "github.com/stackrox/rox/central/views/imagecveflat" imagesView "github.com/stackrox/rox/central/views/images" + "github.com/stackrox/rox/pkg/features" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) @@ -75,23 +77,40 @@ const ( func BenchmarkImageResolver(b *testing.B) { mockCtrl := gomock.NewController(b) testDB := SetupTestPostgresConn(b) - resolver, schema := SetupTestResolver(b, - imagesView.NewImageView(testDB.DB), - CreateTestImageComponentV2Datastore(b, testDB, mockCtrl), - CreateTestImageCVEV2Datastore(b, testDB), - CreateTestImageV2Datastore(b, testDB, mockCtrl), - imagecveflat.NewCVEFlatView(testDB.DB), - imagecomponentflat.NewComponentFlatView(testDB.DB), - ) ctx := contextWithImagePerm(b, mockCtrl) - images := getTestImages(100) - for _, image := range images { - require.NoError(b, resolver.ImageDataStore.UpsertImage(ctx, image)) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + var schema *graphql.Schema + if features.FlattenImageData.Enabled() { + resolver, s := SetupTestResolver(b, + imagesView.NewImageView(testDB.DB), + CreateTestImageComponentV2Datastore(b, testDB, mockCtrl), + CreateTestImageCVEV2Datastore(b, testDB), + CreateTestImageV2Datastore(b, testDB, mockCtrl), + imagecveflat.NewCVEFlatView(testDB.DB), + imagecomponentflat.NewComponentFlatView(testDB.DB), + ) + schema = s + for _, image := range getTestImagesV2(100) { + require.NoError(b, resolver.ImageV2DataStore.UpsertImage(ctx, image)) + } + } else { + resolver, s := SetupTestResolver(b, + imagesView.NewImageView(testDB.DB), + CreateTestImageComponentV2Datastore(b, testDB, mockCtrl), + CreateTestImageCVEV2Datastore(b, testDB), + CreateTestImageDatastore(b, testDB, mockCtrl), + imagecveflat.NewCVEFlatView(testDB.DB), + imagecomponentflat.NewComponentFlatView(testDB.DB), + ) + schema = s + for _, image := range getTestImages(100) { + require.NoError(b, resolver.ImageDataStore.UpsertImage(ctx, image)) + } } b.Run("GetImageComponentsInImageScanResolver", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, imageWithScanQuery, "getImages", @@ -106,7 +125,7 @@ func BenchmarkImageResolver(b *testing.B) { }) b.Run("GetImageComponentsWithoutImageScanResolver", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, imageWithoutScanQuery, "getImages", @@ -121,7 +140,7 @@ func BenchmarkImageResolver(b *testing.B) { }) b.Run("GetImageComponentsDerivedFieldsWithImageScanResolver", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, imageWithScanLongQuery, "getImages", @@ -136,7 +155,7 @@ func BenchmarkImageResolver(b *testing.B) { }) b.Run("GetImageComponentsDerivedWithoutImageScanResolver", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, imageWithoutScanLongQuery, "getImages", @@ -151,7 +170,7 @@ func BenchmarkImageResolver(b *testing.B) { }) b.Run("GetImageOnly", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, imageOnlyQuery, "getImages", @@ -166,7 +185,7 @@ func BenchmarkImageResolver(b *testing.B) { }) b.Run("GetImageWithCounts", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, imageWithCountsQuery, "getImages", @@ -181,7 +200,7 @@ func BenchmarkImageResolver(b *testing.B) { }) b.Run("GetImageScanTimeTopLevel", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, imageWithTopLevelScanTimeQuery, "getImages", @@ -196,7 +215,7 @@ func BenchmarkImageResolver(b *testing.B) { }) b.Run("GetImageScanTimeNested", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, imageWithNestedScanTimeQuery, "getImages", diff --git a/central/graphql/resolvers/image_scan_test.go b/central/graphql/resolvers/image_scan_test.go index c0da7579c634e..fd4c1f8871993 100644 --- a/central/graphql/resolvers/image_scan_test.go +++ b/central/graphql/resolvers/image_scan_test.go @@ -7,10 +7,12 @@ import ( "github.com/graph-gophers/graphql-go" imageDSMocks "github.com/stackrox/rox/central/image/datastore/mocks" imageComponentsDSV2Mocks "github.com/stackrox/rox/central/imagecomponent/v2/datastore/mocks" + imageV2DSMocks "github.com/stackrox/rox/central/imagev2/datastore/mocks" imagesComponentViewMocks "github.com/stackrox/rox/central/views/imagecomponentflat/mocks" imagesView "github.com/stackrox/rox/central/views/images" imagesViewMocks "github.com/stackrox/rox/central/views/images/mocks" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" @@ -19,7 +21,7 @@ import ( const ( imageWithScanQuery = ` query getImages($query: String, $pagination: Pagination) { - images(query: $query, pagination: $pagination) { + images(query: $query, pagination: $pagination) { id scan { imageComponents { @@ -33,7 +35,7 @@ const ( imageWithoutScanQuery = ` query getImages($query: String, $pagination: Pagination) { - images(query: $query, pagination: $pagination) { + images(query: $query, pagination: $pagination) { id imageComponents { name @@ -44,6 +46,7 @@ const ( }}` ) +// TODO(ROX-30117): Remove ImageScanResolverTestSuite when FlattenImageData feature flag is removed. func TestImageScanResolver(t *testing.T) { suite.Run(t, new(ImageScanResolverTestSuite)) } @@ -84,6 +87,9 @@ func (s *ImageScanResolverTestSuite) TearDownSuite() { } func (s *ImageScanResolverTestSuite) TestGetImagesWithScan() { + if features.FlattenImageData.Enabled() { + s.T().Skip("Skipping test - FlattenImageData is enabled") + } // Verify that full image is fetched. img := fixtures.GetImageWithUniqueComponents(5) imageCore := imagesViewMocks.NewMockImageCore(s.mockCtrl) @@ -104,6 +110,9 @@ func (s *ImageScanResolverTestSuite) TestGetImagesWithScan() { } func (s *ImageScanResolverTestSuite) TestGetImagesWithoutScan() { + if features.FlattenImageData.Enabled() { + s.T().Skip("Skipping test - FlattenImageData is enabled") + } // Verify that full image is not fetched but rather image component and vuln stores are queried. img := fixtures.GetImageWithUniqueComponents(5) imageCore := imagesViewMocks.NewMockImageCore(s.mockCtrl) @@ -121,3 +130,88 @@ func (s *ImageScanResolverTestSuite) TestGetImagesWithoutScan() { response := s.schema.Exec(s.ctx, imageWithoutScanQuery, "getImages", nil) s.Len(response.Errors, 0) } + +// ImageScanResolverV2TestSuite tests image scan resolver when FlattenImageData feature is enabled. +func TestImageScanResolverV2(t *testing.T) { + suite.Run(t, new(ImageScanResolverV2TestSuite)) +} + +type ImageScanResolverV2TestSuite struct { + suite.Suite + + ctx context.Context + mockCtrl *gomock.Controller + + imageV2DataStore *imageV2DSMocks.MockDataStore + imageView *imagesViewMocks.MockImageView + imageComponentDataStoreV2 *imageComponentsDSV2Mocks.MockDataStore + imageComponentFlatView *imagesComponentViewMocks.MockComponentFlatView + + resolver *Resolver + schema *graphql.Schema +} + +func (s *ImageScanResolverV2TestSuite) SetupTest() { + s.mockCtrl = gomock.NewController(s.T()) + s.ctx = contextWithImagePerm(s.T(), s.mockCtrl) + + s.imageV2DataStore = imageV2DSMocks.NewMockDataStore(s.mockCtrl) + s.imageView = imagesViewMocks.NewMockImageView(s.mockCtrl) + s.imageComponentFlatView = imagesComponentViewMocks.NewMockComponentFlatView(s.mockCtrl) + + s.imageComponentDataStoreV2 = imageComponentsDSV2Mocks.NewMockDataStore(s.mockCtrl) + + s.resolver, s.schema = SetupTestResolver(s.T(), s.imageV2DataStore, s.imageView, s.imageComponentDataStoreV2, s.imageComponentFlatView) +} + +func (s *ImageScanResolverV2TestSuite) TearDownTest() { + s.mockCtrl.Finish() +} + +func (s *ImageScanResolverV2TestSuite) TearDownSuite() { +} + +func (s *ImageScanResolverV2TestSuite) TestGetImagesWithScan() { + if !features.FlattenImageData.Enabled() { + s.T().Skip("Skipping test - FlattenImageData is disabled") + } + // Verify that full image is fetched. + img := fixtures.GetImageV2WithUniqueComponents(5) + imageCore := imagesViewMocks.NewMockImageCore(s.mockCtrl) + imageCore.EXPECT().GetImageID().Return(img.GetId()) + s.imageView.EXPECT().Get(gomock.Any(), gomock.Any()). + Return([]imagesView.ImageCore{imageCore}, nil) + cloned := img.CloneVT() + cloned.Scan.Components = nil + s.imageV2DataStore.EXPECT().GetManyImageMetadata(gomock.Any(), gomock.Any()). + Return([]*storage.ImageV2{cloned}, nil) + s.imageV2DataStore.EXPECT().GetImagesBatch(gomock.Any(), gomock.Any()). + Return([]*storage.ImageV2{img}, nil) + s.imageComponentFlatView.EXPECT().Get(gomock.Any(), gomock.Any()).Return(nil, nil) + s.imageComponentDataStoreV2.EXPECT().SearchRawImageComponents(gomock.Any(), gomock.Any()). + Return(nil, nil) + response := s.schema.Exec(s.ctx, imageWithScanQuery, "getImages", nil) + s.Len(response.Errors, 0) +} + +func (s *ImageScanResolverV2TestSuite) TestGetImagesWithoutScan() { + if !features.FlattenImageData.Enabled() { + s.T().Skip("Skipping test - FlattenImageData is disabled") + } + // Verify that full image is not fetched but rather image component and vuln stores are queried. + img := fixtures.GetImageV2WithUniqueComponents(5) + imageCore := imagesViewMocks.NewMockImageCore(s.mockCtrl) + imageCore.EXPECT().GetImageID().Return(img.GetId()) + s.imageView.EXPECT().Get(gomock.Any(), gomock.Any()). + Return([]imagesView.ImageCore{imageCore}, nil) + + cloned := img.CloneVT() + cloned.Scan.Components = nil + s.imageV2DataStore.EXPECT().GetManyImageMetadata(gomock.Any(), gomock.Any()). + Return([]*storage.ImageV2{cloned}, nil) + s.imageComponentFlatView.EXPECT().Get(gomock.Any(), gomock.Any()).Return(nil, nil) + s.imageComponentDataStoreV2.EXPECT().SearchRawImageComponents(gomock.Any(), gomock.Any()). + Return(nil, nil) + response := s.schema.Exec(s.ctx, imageWithoutScanQuery, "getImages", nil) + s.Len(response.Errors, 0) +} diff --git a/central/graphql/resolvers/image_signature_verification.go b/central/graphql/resolvers/image_signature_verification.go new file mode 100644 index 0000000000000..626b38875ab49 --- /dev/null +++ b/central/graphql/resolvers/image_signature_verification.go @@ -0,0 +1,30 @@ +package resolvers + +import ( + "context" + + "github.com/stackrox/rox/pkg/signatureintegration" + "github.com/stackrox/rox/pkg/utils" +) + +func init() { + schema := getBuilder() + utils.Must( + schema.AddExtraResolver("ImageSignatureVerificationResult", "verifierName: String!"), + ) +} + +// VerifierName returns the signature integration name. If the name was +// pre-populated by the service, it is returned directly. Otherwise, a lookup +// is performed from the SignatureIntegrationDataStore. +func (resolver *imageSignatureVerificationResultResolver) VerifierName(ctx context.Context) (string, error) { + // Short-circuit if the name was already populated by the service. + if name := resolver.data.GetVerifierName(); name != "" { + return name, nil + } + return signatureintegration.GetVerifierName( + ctx, + resolver.root.SignatureIntegrationDataStore, + resolver.data.GetVerifierId(), + ) +} diff --git a/central/graphql/resolvers/image_vulnerabilities.go b/central/graphql/resolvers/image_vulnerabilities.go index fa5030ead2517..6d81452f5e2da 100644 --- a/central/graphql/resolvers/image_vulnerabilities.go +++ b/central/graphql/resolvers/image_vulnerabilities.go @@ -8,7 +8,6 @@ import ( "github.com/graph-gophers/graphql-go" "github.com/pkg/errors" "github.com/stackrox/rox/central/graphql/generator" - "github.com/stackrox/rox/central/graphql/resolvers/embeddedobjs" "github.com/stackrox/rox/central/graphql/resolvers/loaders" "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/central/views" @@ -27,7 +26,7 @@ import ( func init() { schema := getBuilder() - generator.RegisterProtoEnum(schema, reflect.TypeOf(storage.CvssScoreVersion(0))) + generator.RegisterProtoEnum(schema, reflect.TypeFor[storage.CvssScoreVersion]()) utils.Must(schema.AddType("ImageCVEV2", []string{ "componentId: String!", @@ -60,6 +59,7 @@ func init() { "images(query: String, pagination: Pagination): [Image!]!", "operatingSystem: String!", "vulnerabilityState: String!", + "origin: String!", "nvdCvss: Float!", "nvdScoreVersion: String!", )), @@ -87,10 +87,13 @@ type ImageVulnerabilityResolver interface { Images(ctx context.Context, args PaginatedQuery) ([]ImageResolver, error) OperatingSystem(ctx context.Context) string VulnerabilityState(ctx context.Context) string + Origin(ctx context.Context) string Nvdcvss(ctx context.Context) float64 NvdScoreVersion(ctx context.Context) string } +var _ ImageVulnerabilityResolver = (*imageCVEV2Resolver)(nil) + // ImageVulnerability returns a vulnerability of the given id func (resolver *Resolver) ImageVulnerability(ctx context.Context, args IDQuery) (ImageVulnerabilityResolver, error) { defer metrics.SetGraphQLOperationDurationTime(time.Now(), pkgMetrics.Root, "ImageVulnerability") @@ -412,11 +415,6 @@ func (resolver *imageCVEV2Resolver) FixedByVersion(ctx context.Context) (string, resolver.ctx = ctx } - // Short path. Full image is embedded when image scan resolver is called. - if embeddedVuln := embeddedobjs.VulnFromContext(resolver.ctx); embeddedVuln != nil { - return embeddedVuln.GetFixedBy(), nil - } - scope, hasScope := scoped.GetScope(resolver.ctx) if !hasScope { return "", nil @@ -441,11 +439,6 @@ func (resolver *imageCVEV2Resolver) IsFixable(ctx context.Context, _ RawQuery) ( resolver.ctx = ctx } - // Short path. Full image is embedded when image scan resolver is called. - if embeddedVuln := embeddedobjs.VulnFromContext(resolver.ctx); embeddedVuln != nil { - return embeddedVuln.GetFixedBy() != "", nil - } - query := search.NewQueryBuilder(). AddExactMatches(search.CVEID, resolver.flatData.GetCVEIDs()...). AddBools(search.Fixable, true). @@ -460,11 +453,6 @@ func (resolver *imageCVEV2Resolver) IsFixable(ctx context.Context, _ RawQuery) ( func (resolver *imageCVEV2Resolver) LastScanned(ctx context.Context) (*graphql.Time, error) { defer metrics.SetGraphQLOperationDurationTime(time.Now(), pkgMetrics.ImageCVEs, "LastScanned") - // Short path. Full image is embedded when image scan resolver is called. - if scanTime := embeddedobjs.LastScannedFromContext(resolver.ctx); scanTime != nil { - return &graphql.Time{Time: *scanTime}, nil - } - if features.FlattenImageData.Enabled() { imageLoader, err := loaders.GetImageV2Loader(resolver.ctx) if err != nil { @@ -674,11 +662,6 @@ func (resolver *imageCVEV2Resolver) Advisory(ctx context.Context) (*advisoryReso resolver.ctx = ctx } - // Short path. Full image is embedded when image scan resolver is called. - if embeddedVuln := embeddedobjs.VulnFromContext(resolver.ctx); embeddedVuln != nil { - return resolver.root.wrapAdvisory(embeddedVuln.GetAdvisory(), true, nil) - } - scope, hasScope := scoped.GetScope(resolver.ctx) if !hasScope { return nil, nil diff --git a/central/graphql/resolvers/image_vulnerabilities_benchmark_test.go b/central/graphql/resolvers/image_vulnerabilities_benchmark_test.go new file mode 100644 index 0000000000000..f2affc26f26ef --- /dev/null +++ b/central/graphql/resolvers/image_vulnerabilities_benchmark_test.go @@ -0,0 +1,120 @@ +//go:build sql_integration + +package resolvers + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/stackrox/rox/central/graphql/resolvers/inputtypes" + "github.com/stackrox/rox/central/graphql/resolvers/loaders" + imageDS "github.com/stackrox/rox/central/image/datastore" + imageV2DS "github.com/stackrox/rox/central/imagev2/datastore" + "github.com/stackrox/rox/central/views/imagecomponentflat" + "github.com/stackrox/rox/central/views/imagecveflat" + imagesView "github.com/stackrox/rox/central/views/images" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/fixtures" + "github.com/stackrox/rox/pkg/grpc/authz/allow" + imageUtils "github.com/stackrox/rox/pkg/images/utils" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func BenchmarkImageVulnerabilities(b *testing.B) { + if !features.FlattenImageData.Enabled() { + b.Skip("only applicable when FlattenImageData is enabled") + } + + mockCtrl := gomock.NewController(b) + testDB := SetupTestPostgresConn(b) + + ctx := SetAuthorizerOverride( + loaders.WithLoaderContext(sac.WithAllAccess(context.Background())), + allow.Anonymous(), + ) + + v1Store := CreateTestImageDatastore(b, testDB, mockCtrl) + v2Store := CreateTestImageV2Datastore(b, testDB, mockCtrl) + + resolver, _ := SetupTestResolver(b, + imagesView.NewImageView(testDB.DB), + CreateTestImageComponentV2Datastore(b, testDB, mockCtrl), + CreateTestImageCVEV2Datastore(b, testDB), + v2Store, + imagecveflat.NewCVEFlatView(testDB.DB), + imagecomponentflat.NewComponentFlatView(testDB.DB), + ) + + paginatedQuery := PaginatedQuery{ + Pagination: &inputtypes.Pagination{ + Limit: new(int32(20000)), + }, + } + + imageCount := 100 + + seedV1AndV2Images(b, ctx, v1Store, v2Store, imageCount) + + b.Run(fmt.Sprintf("ImageVulnerabilities/images=%d", imageCount), func(b *testing.B) { + for range b.N { + vulns, err := resolver.ImageVulnerabilities(ctx, paginatedQuery) + require.NoError(b, err) + require.NotEmpty(b, vulns) + } + }) + + truncateImageBenchData(b, ctx, testDB) +} + +func seedV1AndV2Images( + b *testing.B, + ctx context.Context, + v1Store imageDS.DataStore, + v2Store imageV2DS.DataStore, + imageCount int, +) { + b.Helper() + + v1Images := make([]*storage.Image, 0, imageCount) + for i := range imageCount { + v1Image := fixtures.GetImageWithUniqueComponents(100) + v1Image.Id = fmt.Sprintf("sha256:%s", uuid.NewV4().String()) + v1Image.Name = &storage.ImageName{ + Registry: "bench.io", + Remote: fmt.Sprintf("bench/img-%d", i), + Tag: "latest", + FullName: fmt.Sprintf("bench.io/bench/img-%d:latest", i), + } + for _, comp := range v1Image.GetScan().GetComponents() { + comp.Name = fmt.Sprintf("img%d-%s", i, comp.GetName()) + for _, vuln := range comp.GetVulns() { + vuln.Cve = fmt.Sprintf("img%d-%s", i, vuln.GetCve()) + } + } + v1Images = append(v1Images, v1Image) + } + + require.NoError(b, os.Setenv(features.FlattenImageData.EnvVar(), "false")) + for _, v1Image := range v1Images { + require.NoError(b, v1Store.UpsertImage(ctx, v1Image)) + } + + require.NoError(b, os.Setenv(features.FlattenImageData.EnvVar(), "true")) + for _, v1Image := range v1Images { + v2Image := imageUtils.ConvertToV2(v1Image) + require.NoError(b, v2Store.UpsertImage(ctx, v2Image)) + } +} + +func truncateImageBenchData(b *testing.B, ctx context.Context, testDB *pgtest.TestPostgres) { + b.Helper() + _, err := testDB.DB.Exec(ctx, "TRUNCATE images, images_v2, image_cves_v2, image_component_v2 CASCADE") + require.NoError(b, err) +} diff --git a/central/graphql/resolvers/image_vulnerabilities_test.go b/central/graphql/resolvers/image_vulnerabilities_test.go new file mode 100644 index 0000000000000..f0f4abd10613b --- /dev/null +++ b/central/graphql/resolvers/image_vulnerabilities_test.go @@ -0,0 +1,26 @@ +package resolvers + +import ( + "reflect" + "testing" +) + +func TestImageVulnerabilityResolverType(t *testing.T) { + // The implementation type has five more methods than the interface it implements. + // + // 2 - ComponentId + // 12 - FirstImageOccurrence + // 14 - ID + // 19 - ImageId + // 32 - State + // + // This can break graphQL queries to the system until + // https://github.com/graph-gophers/graphql-go/issues/763 is fixed. + // + // TODO(ROX-35654): Unskip this tests. + t.Skip("Interface and implementation types do not have aligned method indices.") + resolverInterface := reflect.TypeFor[ImageVulnerabilityResolver]() + resolverImplType := reflect.TypeFor[*imageCVEV2Resolver]() + + validateAlignedMethodIndex(t, resolverInterface, resolverImplType) +} diff --git a/central/graphql/resolvers/image_vulnerabilities_utilities.go b/central/graphql/resolvers/image_vulnerabilities_utilities.go index 55118dfcc0116..63f3dec694cfc 100644 --- a/central/graphql/resolvers/image_vulnerabilities_utilities.go +++ b/central/graphql/resolvers/image_vulnerabilities_utilities.go @@ -17,24 +17,6 @@ type imageCVEV2Resolver struct { flatData imagecveflat.CveFlat } -func (resolver *Resolver) wrapImageCVEV2(value *storage.ImageCVEV2, ok bool, err error) (*imageCVEV2Resolver, error) { - if !ok || err != nil || value == nil { - return nil, err - } - return &imageCVEV2Resolver{root: resolver, data: value}, nil -} - -func (resolver *Resolver) wrapImageCVEV2s(values []*storage.ImageCVEV2, err error) ([]*imageCVEV2Resolver, error) { - if err != nil || len(values) == 0 { - return nil, err - } - output := make([]*imageCVEV2Resolver, len(values)) - for i, v := range values { - output[i] = &imageCVEV2Resolver{root: resolver, data: v} - } - return output, nil -} - func (resolver *Resolver) wrapImageCVEV2WithContext(ctx context.Context, value *storage.ImageCVEV2, ok bool, err error) (*imageCVEV2Resolver, error) { if !ok || err != nil || value == nil { return nil, err @@ -49,17 +31,6 @@ func (resolver *Resolver) wrapImageCVEV2FlatWithContext(ctx context.Context, val return &imageCVEV2Resolver{ctx: ctx, root: resolver, data: value, flatData: flatData}, nil } -func (resolver *Resolver) wrapImageCVEV2sWithContext(ctx context.Context, values []*storage.ImageCVEV2, err error) ([]*imageCVEV2Resolver, error) { - if err != nil || len(values) == 0 { - return nil, err - } - output := make([]*imageCVEV2Resolver, len(values)) - for i, v := range values { - output[i] = &imageCVEV2Resolver{ctx: ctx, root: resolver, data: v} - } - return output, nil -} - func (resolver *Resolver) wrapImageCVEV2sFlatWithContext(ctx context.Context, values []*storage.ImageCVEV2, flatData []imagecveflat.CveFlat, err error) ([]*imageCVEV2Resolver, error) { if err != nil || len(values) == 0 { return nil, err @@ -88,8 +59,10 @@ func (resolver *imageCVEV2Resolver) CveBaseInfo(ctx context.Context) (*cVEInfoRe } func (resolver *imageCVEV2Resolver) Cvss(ctx context.Context) float64 { - value := resolver.data.GetCvss() - return float64(value) + if resolver.flatData != nil { + return float64(resolver.flatData.GetTopCVSS()) + } + return float64(resolver.data.GetCvss()) } func (resolver *imageCVEV2Resolver) FirstImageOccurrence(ctx context.Context) (*graphql.Time, error) { @@ -118,16 +91,24 @@ func (resolver *imageCVEV2Resolver) NvdScoreVersion(ctx context.Context) string } func (resolver *imageCVEV2Resolver) Nvdcvss(ctx context.Context) float64 { - value := resolver.data.GetNvdcvss() - return float64(value) + if resolver.flatData != nil { + return float64(resolver.flatData.GetTopNVDCVSS()) + } + return float64(resolver.data.GetNvdcvss()) } func (resolver *imageCVEV2Resolver) Severity(ctx context.Context) string { - value := resolver.data.GetSeverity() - return value.String() + if resolver.flatData != nil { + return resolver.flatData.GetSeverity().String() + } + return resolver.data.GetSeverity().String() } func (resolver *imageCVEV2Resolver) State(ctx context.Context) string { value := resolver.data.GetState() return value.String() } + +func (resolver *imageCVEV2Resolver) Origin(_ context.Context) string { + return resolver.data.GetOrigin().String() +} diff --git a/central/graphql/resolvers/image_vulnerabilities_utilities_test.go b/central/graphql/resolvers/image_vulnerabilities_utilities_test.go new file mode 100644 index 0000000000000..0396318e80960 --- /dev/null +++ b/central/graphql/resolvers/image_vulnerabilities_utilities_test.go @@ -0,0 +1,65 @@ +package resolvers + +import ( + "context" + "testing" + + cveFlatMocks "github.com/stackrox/rox/central/views/imagecveflat/mocks" + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" +) + +func TestImageCVEV2Resolver_PrefersFlatData(t *testing.T) { + ctrl := gomock.NewController(t) + + critical := storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY + important := storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY + + flatData := cveFlatMocks.NewMockCveFlat(ctrl) + flatData.EXPECT().GetSeverity().Return(&critical).AnyTimes() + flatData.EXPECT().GetTopCVSS().Return(float32(9.8)).AnyTimes() + flatData.EXPECT().GetTopNVDCVSS().Return(float32(9.1)).AnyTimes() + + cases := map[string]struct { + resolver *imageCVEV2Resolver + expectedSeverity string + expectedCvss float64 + expectedNvdCvss float64 + }{ + "with flatData, aggregated values are returned": { + resolver: &imageCVEV2Resolver{ + data: &storage.ImageCVEV2{ + Severity: important, + Cvss: 7.5, + Nvdcvss: 6.9, + }, + flatData: flatData, + }, + expectedSeverity: critical.String(), + expectedCvss: 9.8, + expectedNvdCvss: 9.1, + }, + "without flatData, denormalized values are returned": { + resolver: &imageCVEV2Resolver{ + data: &storage.ImageCVEV2{ + Severity: important, + Cvss: 7.5, + Nvdcvss: 6.9, + }, + }, + expectedSeverity: important.String(), + expectedCvss: 7.5, + expectedNvdCvss: 6.9, + }, + } + + ctx := context.Background() + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expectedSeverity, tc.resolver.Severity(ctx)) + assert.InDelta(t, tc.expectedCvss, tc.resolver.Cvss(ctx), 0.001) + assert.InDelta(t, tc.expectedNvdCvss, tc.resolver.Nvdcvss(ctx), 0.001) + }) + } +} diff --git a/central/graphql/resolvers/image_vulnerabilities_v2_test.go b/central/graphql/resolvers/image_vulnerabilities_v2_test.go index a2aa7e526c26f..b0c047809923b 100644 --- a/central/graphql/resolvers/image_vulnerabilities_v2_test.go +++ b/central/graphql/resolvers/image_vulnerabilities_v2_test.go @@ -13,9 +13,10 @@ import ( imagesView "github.com/stackrox/rox/central/views/images" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/grpc/authz/allow" - "github.com/stackrox/rox/pkg/pointers" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/search/scoped" @@ -65,6 +66,7 @@ type GraphQLImageVulnerabilityV2TestSuite struct { testDB *pgtest.TestPostgres resolver *Resolver + testImages []*storage.Image cveIDMap map[string]string componentIDMap map[string]string } @@ -75,22 +77,44 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) SetupSuite() { s.testDB = pgtest.ForT(s.T()) vulnReqDatastore, err := TestVulnReqDatastore(s.T(), s.testDB) s.Require().NoError(err) - resolver, _ := SetupTestResolver(s.T(), - imagesView.NewImageView(s.testDB.DB), - CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl), - CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), - CreateTestImageCVEV2Datastore(s.T(), s.testDB), - vulnReqDatastore, - imagecveflat.NewCVEFlatView(s.testDB.DB), - imagecomponentflat.NewComponentFlatView(s.testDB.DB), - ) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + var resolver *Resolver + if features.FlattenImageData.Enabled() { + resolver, _ = SetupTestResolver(s.T(), + imagesView.NewImageView(s.testDB.DB), + CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl), + CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), + CreateTestImageCVEV2Datastore(s.T(), s.testDB), + vulnReqDatastore, + imagecveflat.NewCVEFlatView(s.testDB.DB), + imagecomponentflat.NewComponentFlatView(s.testDB.DB), + ) + } else { + resolver, _ = SetupTestResolver(s.T(), + imagesView.NewImageView(s.testDB.DB), + CreateTestImageDatastore(s.T(), s.testDB, mockCtrl), + CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), + CreateTestImageCVEV2Datastore(s.T(), s.testDB), + vulnReqDatastore, + imagecveflat.NewCVEFlatView(s.testDB.DB), + imagecomponentflat.NewComponentFlatView(s.testDB.DB), + ) + } s.resolver = resolver // Add Test Data to DataStores - testImages := testImages() - for _, image := range testImages { - err := s.resolver.ImageDataStore.UpsertImage(s.ctx, image) - s.NoError(err) + s.testImages = testImages() + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + for _, image := range s.testImages { + err := s.resolver.ImageV2DataStore.UpsertImage(s.ctx, imageUtils.ConvertToV2(image)) + s.NoError(err) + } + } else { + for _, image := range s.testImages { + err := s.resolver.ImageDataStore.UpsertImage(s.ctx, image) + s.NoError(err) + } } // Add test vulnerability exceptions @@ -273,7 +297,10 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilitiesFixedByVe func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilitiesScoped() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) - image := s.getImageResolver(ctx, "sha1") + imageID1 := s.getImageID(s.testImages[0]) + imageID2 := s.getImageID(s.testImages[1]) + + image := s.getImageResolver(ctx, imageID1) expected := int32(3) expectedCVEs := []string{"cve-2019-1", "cve-2019-2", "cve-2018-1"} @@ -292,7 +319,7 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilitiesScoped() s.NoError(err) checkVulnerabilityCounter(s.T(), counter, expected, 1, 1, 0, 1, 1) - image = s.getImageResolver(ctx, "sha2") + image = s.getImageResolver(ctx, imageID2) expected = int32(5) expectedCVEs = []string{"cve-2019-1", "cve-2019-2", "cve-2018-1", "cve-2017-1", "cve-2017-2"} @@ -340,7 +367,8 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestTopImageVulnerabilityUnscoped func (s *GraphQLImageVulnerabilityV2TestSuite) TestTopImageVulnerability() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) - image := s.getImageResolver(ctx, "sha1") + imageID1 := s.getImageID(s.testImages[0]) + image := s.getImageResolver(ctx, imageID1) expected := graphql.ID(s.cveIDMap[cve231]) topVuln, err := image.TopImageVulnerability(ctx, RawQuery{}) @@ -351,13 +379,16 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestTopImageVulnerability() { func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityImages() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) + imageID1 := s.getImageID(s.testImages[0]) + imageID2 := s.getImageID(s.testImages[1]) + vuln := s.getImageVulnerabilityResolver(ctx, s.cveIDMap[cve111]) images, err := vuln.Images(ctx, PaginatedQuery{}) s.NoError(err) s.Equal(2, len(images)) idList := getIDList(ctx, images) - s.ElementsMatch([]string{"sha1", "sha2"}, idList) + s.ElementsMatch([]string{imageID1, imageID2}, idList) count, err := vuln.ImageCount(ctx, RawQuery{}) s.NoError(err) @@ -369,7 +400,7 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityImages() { s.NoError(err) s.Equal(1, len(images)) idList = getIDList(ctx, images) - s.ElementsMatch([]string{"sha2"}, idList) + s.ElementsMatch([]string{imageID2}, idList) count, err = vuln.ImageCount(ctx, RawQuery{}) s.NoError(err) @@ -436,7 +467,7 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCo func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCountPending() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) - status := []*string{pointers.String(storage.RequestStatus_PENDING.String())} + status := []*string{new(storage.RequestStatus_PENDING.String())} args := struct { RequestStatus *[]*string }{ @@ -468,7 +499,7 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCo func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCountApproved() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) - status := []*string{pointers.String(storage.RequestStatus_APPROVED.String())} + status := []*string{new(storage.RequestStatus_APPROVED.String())} args := struct { RequestStatus *[]*string }{ @@ -496,7 +527,7 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCo func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCountPendingUpdate() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) - status := []*string{pointers.String(storage.RequestStatus_APPROVED_PENDING_UPDATE.String())} + status := []*string{new(storage.RequestStatus_APPROVED_PENDING_UPDATE.String())} args := struct { RequestStatus *[]*string }{ @@ -523,9 +554,10 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCo func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCountAllWithImageScope() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) + imageID1 := s.getImageID(s.testImages[0]) ctx = scoped.Context(ctx, scoped.Scope{ - IDs: []string{"sha1"}, - Level: v1.SearchCategory_IMAGES, + IDs: []string{imageID1}, + Level: s.getImagesSearchCategory(), }) args := struct { RequestStatus *[]*string @@ -555,10 +587,11 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCo func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCountPendingWithImageScope() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) - status := []*string{pointers.String(storage.RequestStatus_PENDING.String())} + status := []*string{new(storage.RequestStatus_PENDING.String())} + imageID1 := s.getImageID(s.testImages[0]) ctx = scoped.Context(ctx, scoped.Scope{ - IDs: []string{"sha1"}, - Level: v1.SearchCategory_IMAGES, + IDs: []string{imageID1}, + Level: s.getImagesSearchCategory(), }) args := struct { RequestStatus *[]*string @@ -590,10 +623,11 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCo func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCountApprovedWithImageScope() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) - status := []*string{pointers.String(storage.RequestStatus_APPROVED.String())} + status := []*string{new(storage.RequestStatus_APPROVED.String())} + imageID1 := s.getImageID(s.testImages[0]) ctx = scoped.Context(ctx, scoped.Scope{ - IDs: []string{"sha1"}, - Level: v1.SearchCategory_IMAGES, + IDs: []string{imageID1}, + Level: s.getImagesSearchCategory(), }) args := struct { RequestStatus *[]*string @@ -618,10 +652,11 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCo func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCountPendingUpdateWithImageScope() { ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) - status := []*string{pointers.String(storage.RequestStatus_APPROVED_PENDING_UPDATE.String())} + status := []*string{new(storage.RequestStatus_APPROVED_PENDING_UPDATE.String())} + imageID2 := s.getImageID(s.testImages[1]) ctx = scoped.Context(ctx, scoped.Scope{ - IDs: []string{"sha2"}, - Level: v1.SearchCategory_IMAGES, + IDs: []string{imageID2}, + Level: s.getImagesSearchCategory(), }) args := struct { RequestStatus *[]*string @@ -639,15 +674,29 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCo func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCountTagless() { s.T().Skip("Need to figure this one out.") - taglessImage := testImages()[1] + taglessImage := s.testImages[1].CloneVT() taglessImage.Id = "sha3" taglessImage.Name.Tag = "" - err := s.resolver.ImageDataStore.UpsertImage(s.ctx, taglessImage) - s.NoError(err) - // Revert the upsert so that other tests are not affected. - defer func() { - s.NoError(s.resolver.ImageDataStore.DeleteImages(s.ctx, "sha3")) - }() + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + var taglessImageID string + if features.FlattenImageData.Enabled() { + taglessImageV2 := imageUtils.ConvertToV2(taglessImage) + taglessImageID = taglessImageV2.GetId() + err := s.resolver.ImageV2DataStore.UpsertImage(s.ctx, taglessImageV2) + s.NoError(err) + defer func() { + s.NoError(s.resolver.ImageV2DataStore.DeleteImages(s.ctx, taglessImageID)) + }() + } else { + taglessImageID = taglessImage.GetId() + err := s.resolver.ImageDataStore.UpsertImage(s.ctx, taglessImage) + s.NoError(err) + defer func() { + s.NoError(s.resolver.ImageDataStore.DeleteImages(s.ctx, taglessImageID)) + }() + } + + imageID1 := s.getImageID(s.testImages[0]) ctx := SetAuthorizerOverride(s.ctx, allow.Anonymous()) args := struct { @@ -662,8 +711,8 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCo s.Equal(int32(1), count) ctx = scoped.Context(ctx, scoped.Scope{ - IDs: []string{"sha1"}, - Level: v1.SearchCategory_IMAGES, + IDs: []string{imageID1}, + Level: s.getImagesSearchCategory(), }) vuln = s.getImageVulnerabilityResolver(ctx, "cve-2017-1#") @@ -673,8 +722,8 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) TestImageVulnerabilityExceptionCo ctx = SetAuthorizerOverride(s.ctx, allow.Anonymous()) ctx = scoped.Context(ctx, scoped.Scope{ - IDs: []string{"sha3"}, - Level: v1.SearchCategory_IMAGES, + IDs: []string{taglessImageID}, + Level: s.getImagesSearchCategory(), }) // Deferral: @@ -756,13 +805,31 @@ func (s *GraphQLImageVulnerabilityV2TestSuite) getIDMap() map[string]string { } } +// TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. +func (s *GraphQLImageVulnerabilityV2TestSuite) getImageID(image *storage.Image) string { + if features.FlattenImageData.Enabled() { + return imageUtils.ConvertToV2(image).GetId() + } + return image.GetId() +} + +// TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. +func (s *GraphQLImageVulnerabilityV2TestSuite) getImagesSearchCategory() v1.SearchCategory { + if features.FlattenImageData.Enabled() { + return v1.SearchCategory_IMAGES_V2 + } + return v1.SearchCategory_IMAGES +} + func (s *GraphQLImageVulnerabilityV2TestSuite) getComponentIDMap() map[string]string { + imageID1 := s.getImageID(s.testImages[0]) + imageID2 := s.getImageID(s.testImages[1]) return map[string]string{ - comp11: getTestComponentID(testImages()[0].GetScan().GetComponents()[0], "sha1", 0), - comp12: getTestComponentID(testImages()[1].GetScan().GetComponents()[0], "sha2", 0), - comp21: getTestComponentID(testImages()[0].GetScan().GetComponents()[1], "sha1", 1), - comp31: getTestComponentID(testImages()[0].GetScan().GetComponents()[2], "sha1", 2), - comp32: getTestComponentID(testImages()[1].GetScan().GetComponents()[1], "sha2", 1), - comp42: getTestComponentID(testImages()[1].GetScan().GetComponents()[2], "sha2", 2), + comp11: getTestComponentID(s.testImages[0].GetScan().GetComponents()[0], imageID1, 0), + comp12: getTestComponentID(s.testImages[1].GetScan().GetComponents()[0], imageID2, 0), + comp21: getTestComponentID(s.testImages[0].GetScan().GetComponents()[1], imageID1, 1), + comp31: getTestComponentID(s.testImages[0].GetScan().GetComponents()[2], imageID1, 2), + comp32: getTestComponentID(s.testImages[1].GetScan().GetComponents()[1], imageID2, 1), + comp42: getTestComponentID(s.testImages[1].GetScan().GetComponents()[2], imageID2, 2), } } diff --git a/central/graphql/resolvers/images.go b/central/graphql/resolvers/images.go index afdb3a04b9659..e7c775ee04e68 100644 --- a/central/graphql/resolvers/images.go +++ b/central/graphql/resolvers/images.go @@ -2,6 +2,7 @@ package resolvers import ( "context" + "slices" "time" "github.com/graph-gophers/graphql-go" @@ -48,9 +49,9 @@ type ImageResolver interface { Signature(ctx context.Context) (*imageSignatureResolver, error) SignatureVerificationData(ctx context.Context) (*imageSignatureVerificationDataResolver, error) TopCvss(ctx context.Context) float64 - BaseImageInfo(ctx context.Context) ([]*baseImageInfoResolver, error) UnknownCveCount(ctx context.Context) int32 + BaseImage(ctx context.Context) (*baseImageResolver, error) Deployments(ctx context.Context, args PaginatedQuery) ([]*deploymentResolver, error) DeploymentCount(ctx context.Context, args RawQuery) (int32, error) TopImageVulnerability(ctx context.Context, args RawQuery) (ImageVulnerabilityResolver, error) @@ -72,6 +73,8 @@ type ImageResolver interface { SignatureCount(ctx context.Context) (int32, error) } +var _ ImageResolver = (*imageResolver)(nil) + var ( imageWatchStatuses []string @@ -88,8 +91,14 @@ func registerImageWatchStatus(s string) string { func init() { schema := getBuilder() utils.Must( + schema.AddType("BaseImage", []string{ + "imageSha: String!", + "names: [String!]!", + "created: Time", + }), // NOTE: This list is and should remain alphabetically ordered schema.AddExtraResolvers("Image", []string{ + "baseImage: BaseImage", "deploymentCount(query: String): Int!", "deployments(query: String, pagination: Pagination): [Deployment!]!", "imageComponentCount(query: String): Int!", @@ -111,6 +120,10 @@ func init() { "scanTime: Time", "scannerVersion: String!", + // Image digest (SHA). For V1 images this equals the image ID; + // for V2 images this is the separate SHA digest. + "digest: String!", + // Image signature-related fields. "signatureCount: Int!", }), @@ -272,7 +285,7 @@ func (resolver *imageResolver) ImageComponents(ctx context.Context, args Paginat log.Error(err) return nil, err } - return imgResolver, err + return imgResolver, nil } func (resolver *imageResolver) ImageComponentCount(ctx context.Context, args RawQuery) (int32, error) { @@ -515,3 +528,66 @@ func (resolver *imageResolver) TopCvss(_ context.Context) float64 { } return float64(value) } + +func (resolver *imageResolver) BaseImage(ctx context.Context) (*baseImageResolver, error) { + resolver.ensureData(ctx) + baseImageInfos := resolver.data.GetBaseImageInfo() + return resolver.root.wrapBaseImage(baseImageInfos) +} + +// baseImageResolver resolves base image information +type baseImageResolver struct { + root *Resolver + data *baseImageData +} + +type baseImageData struct { + imageSha string + names []string + created *graphql.Time +} + +func (resolver *Resolver) wrapBaseImage(baseImageInfos []*storage.BaseImageInfo) (*baseImageResolver, error) { + if len(baseImageInfos) == 0 { + return nil, nil + } + + // All entries should have the same digest and create time, take the first one + imageSha := baseImageInfos[0].GetBaseImageDigest() + createTimestamp := baseImageInfos[0].GetCreated() + created, err := protocompat.ConvertTimestampToGraphqlTimeOrError(createTimestamp) + if err != nil { + return nil, err + } + + // Collect all full names + names := make([]string, 0, len(baseImageInfos)) + for _, info := range baseImageInfos { + names = append(names, info.GetBaseImageFullName()) + } + + // Stablize the names + slices.Sort(names) + data := &baseImageData{ + imageSha: imageSha, + names: names, + created: created, + } + + return &baseImageResolver{ + root: resolver, + data: data, + }, nil +} + +func (resolver *baseImageResolver) ImageSha(_ context.Context) string { + return resolver.data.imageSha +} + +func (resolver *baseImageResolver) Names(_ context.Context) []string { + return resolver.data.names +} + +func (resolver *baseImageResolver) Created(_ context.Context) (*graphql.Time, error) { + return resolver.data.created, nil +} diff --git a/central/graphql/resolvers/images_test.go b/central/graphql/resolvers/images_test.go index 66d4d68689c9d..3a1d55068a16e 100644 --- a/central/graphql/resolvers/images_test.go +++ b/central/graphql/resolvers/images_test.go @@ -6,6 +6,7 @@ import ( "context" "strings" "testing" + "time" "github.com/graph-gophers/graphql-go" "github.com/stackrox/rox/central/graphql/resolvers/loaders" @@ -14,10 +15,12 @@ import ( "github.com/stackrox/rox/central/views/imagecveflat" imagesView "github.com/stackrox/rox/central/views/images" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" "github.com/stackrox/rox/pkg/grpc/authz/allow" - "github.com/stackrox/rox/pkg/pointers" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stretchr/testify/assert" @@ -46,17 +49,34 @@ func (s *ImageResolversTestSuite) SetupSuite() { mockCtrl := gomock.NewController(s.T()) s.testDB = SetupTestPostgresConn(s.T()) - imgDataStore := CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl) - s.resolver, _ = SetupTestResolver(s.T(), - CreateTestDeploymentDatastore(s.T(), s.testDB, mockCtrl, imgDataStore), - deploymentsView.NewDeploymentView(s.testDB.DB), - imagesView.NewImageView(s.testDB.DB), - imgDataStore, - CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), - CreateTestImageCVEV2Datastore(s.T(), s.testDB), - imagecve.NewCVEView(s.testDB.DB), - imagecveflat.NewCVEFlatView(s.testDB.DB), - ) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + var resolver *Resolver + if features.FlattenImageData.Enabled() { + imgV2DataStore := CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl) + resolver, _ = SetupTestResolver(s.T(), + CreateTestDeploymentDatastoreWithImageV2(s.T(), s.testDB, mockCtrl, imgV2DataStore), + deploymentsView.NewDeploymentView(s.testDB.DB), + imagesView.NewImageView(s.testDB.DB), + imgV2DataStore, + CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), + CreateTestImageCVEV2Datastore(s.T(), s.testDB), + imagecve.NewCVEView(s.testDB.DB), + imagecveflat.NewCVEFlatView(s.testDB.DB), + ) + } else { + imgDataStore := CreateTestImageDatastore(s.T(), s.testDB, mockCtrl) + resolver, _ = SetupTestResolver(s.T(), + CreateTestDeploymentDatastore(s.T(), s.testDB, mockCtrl, imgDataStore), + deploymentsView.NewDeploymentView(s.testDB.DB), + imagesView.NewImageView(s.testDB.DB), + imgDataStore, + CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), + CreateTestImageCVEV2Datastore(s.T(), s.testDB), + imagecve.NewCVEView(s.testDB.DB), + imagecveflat.NewCVEFlatView(s.testDB.DB), + ) + } + s.resolver = resolver // Add Test Data. s.testDeployments = testDeployments() @@ -64,8 +84,15 @@ func (s *ImageResolversTestSuite) SetupSuite() { s.NoError(s.resolver.DeploymentDataStore.UpsertDeployment(s.ctx, deployment)) } s.testImages = testImages() - for _, image := range testImages() { - s.NoError(s.resolver.ImageDataStore.UpsertImage(s.ctx, image)) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + for _, image := range s.testImages { + s.NoError(s.resolver.ImageV2DataStore.UpsertImage(s.ctx, imageUtils.ConvertToV2(image))) + } + } else { + for _, image := range s.testImages { + s.NoError(s.resolver.ImageDataStore.UpsertImage(s.ctx, image)) + } } } @@ -104,7 +131,7 @@ func (s *ImageResolversTestSuite) TestDeployments() { { desc: "filter by namespace", ctx: ctx, - q: PaginatedQuery{Query: pointers.String("Namespace:namespace1name")}, + q: PaginatedQuery{Query: new("Namespace:namespace1name")}, deploymentFiler: func(d *storage.Deployment) bool { return strings.HasPrefix(d.GetNamespace(), "namespace1name") }, @@ -114,7 +141,7 @@ func (s *ImageResolversTestSuite) TestDeployments() { { desc: "filter by deployment", ctx: ctx, - q: PaginatedQuery{Query: pointers.String("Deployment:" + dep1name)}, + q: PaginatedQuery{Query: new("Deployment:" + dep1name)}, deploymentFiler: func(d *storage.Deployment) bool { return d.GetName() == dep1name }, @@ -124,7 +151,7 @@ func (s *ImageResolversTestSuite) TestDeployments() { { desc: "filter by image", ctx: ctx, - q: PaginatedQuery{Query: pointers.String("Image:reg1/img1")}, + q: PaginatedQuery{Query: new("Image:reg1/img1")}, deploymentFiler: func(d *storage.Deployment) bool { return true }, imageFilter: func(img *storage.Image) bool { return strings.HasPrefix(img.GetName().GetFullName(), "reg1/img1") @@ -134,7 +161,7 @@ func (s *ImageResolversTestSuite) TestDeployments() { { desc: "filter by cve", ctx: ctx, - q: PaginatedQuery{Query: pointers.String("CVE:cve-2019-2")}, + q: PaginatedQuery{Query: new("CVE:cve-2019-2")}, deploymentFiler: func(d *storage.Deployment) bool { return true }, imageFilter: func(img *storage.Image) bool { for _, component := range img.GetScan().GetComponents() { @@ -153,7 +180,7 @@ func (s *ImageResolversTestSuite) TestDeployments() { { desc: "filter by deployment+cve", ctx: ctx, - q: PaginatedQuery{Query: pointers.String("Deployment:dep2name+CVE:cve-2019-2")}, + q: PaginatedQuery{Query: new("Deployment:dep2name+CVE:cve-2019-2")}, deploymentFiler: func(d *storage.Deployment) bool { return strings.HasPrefix(d.GetName(), "dep2name") }, @@ -174,7 +201,7 @@ func (s *ImageResolversTestSuite) TestDeployments() { { desc: "filter by severity", ctx: ctx, - q: PaginatedQuery{Query: pointers.String("Severity:CRITICAL_VULNERABILITY_SEVERITY")}, + q: PaginatedQuery{Query: new("Severity:CRITICAL_VULNERABILITY_SEVERITY")}, deploymentFiler: func(d *storage.Deployment) bool { return true }, imageFilter: func(img *storage.Image) bool { for _, component := range img.GetScan().GetComponents() { @@ -193,7 +220,7 @@ func (s *ImageResolversTestSuite) TestDeployments() { { desc: "filter by severity+fixable", ctx: ctx, - q: PaginatedQuery{Query: pointers.String("Severity:UNSET_VULNERABILITY_SEVERITY+Fixable:true")}, + q: PaginatedQuery{Query: new("Severity:UNSET_VULNERABILITY_SEVERITY+Fixable:true")}, deploymentFiler: func(d *storage.Deployment) bool { return true }, imageFilter: func(img *storage.Image) bool { for _, component := range img.GetScan().GetComponents() { @@ -235,8 +262,8 @@ func (s *ImageResolversTestSuite) TestDeployments() { actualImages, err := s.resolver.Images(testCtx, paginatedQ) require.NoError(t, err) var expectedIDs []string - for _, dep := range expectedImages { - expectedIDs = append(expectedIDs, dep.GetId()) + for imgID := range expectedImages { + expectedIDs = append(expectedIDs, imgID) } assert.ElementsMatch(t, expectedIDs, getIDList(testCtx, actualImages)) @@ -267,16 +294,38 @@ func (s *ImageResolversTestSuite) TestDeployments() { assert.Equal(t, int32(expectedCVESevCount.moderate), moderate.Total(testCtx)) assert.Equal(t, int32(expectedCVESevCount.low), low.Total(testCtx)) - // Test BaseImageInfo field for each image resolver. - expectedImage := expectedImages[imageID] - actualBaseImageInfo, err := image.BaseImageInfo(testCtx) + // Test BaseImage field + actualBaseImage, err := image.BaseImage(testCtx) assert.NoError(t, err) - assert.Len(t, actualBaseImageInfo, len(expectedImage.GetBaseImageInfo())) - for i, baseInfo := range actualBaseImageInfo { - expectedBaseInfo := expectedImage.GetBaseImageInfo()[i] - assert.Equal(t, expectedBaseInfo.GetBaseImageId(), baseInfo.BaseImageId(testCtx)) - assert.Equal(t, expectedBaseInfo.GetBaseImageFullName(), baseInfo.BaseImageFullName(testCtx)) - assert.Equal(t, expectedBaseInfo.GetBaseImageDigest(), baseInfo.BaseImageDigest(testCtx)) + + expectedImage, exists := expectedImages[imageID] + require.True(t, exists, "Expected image %s not found in expectedImages map", imageID) + baseImageInfos := expectedImage.GetBaseImageInfo() + if len(baseImageInfos) == 0 { + assert.Nil(t, actualBaseImage) + } else { + require.NotNil(t, actualBaseImage) + + // Test imageSha (should be the digest from the first base image info) + expectedSha := baseImageInfos[0].GetBaseImageDigest() + assert.Equal(t, expectedSha, actualBaseImage.ImageSha(testCtx)) + + // Test name array + expectedNames := []string{ + baseImageInfos[1].GetBaseImageFullName(), + baseImageInfos[0].GetBaseImageFullName(), + } + assert.Equal(t, expectedNames, actualBaseImage.Names(testCtx)) + + // Test created timestamp + actualCreated, err := actualBaseImage.Created(testCtx) + assert.NoError(t, err) + assert.NotNil(t, actualCreated) + expectedTimestamp, err := protocompat.ConvertTimeToTimestampOrError(time.Unix(0, 3000)) + assert.NoError(t, err) + expectedCreated, err := protocompat.ConvertTimestampToGraphqlTimeOrError(expectedTimestamp) + assert.NoError(t, err) + assert.Equal(t, expectedCreated, actualCreated) } // Test image -> deployments -> images @@ -315,6 +364,14 @@ func (s *ImageResolversTestSuite) TestDeployments() { } } +// TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. +func getImageIDForTest(image *storage.Image) string { + if features.FlattenImageData.Enabled() { + return imageUtils.ConvertToV2(image).GetId() + } + return image.GetId() +} + func compileExpectedForImageGraphQL(deployments []*storage.Deployment, images []*storage.Image, deploymentFilter func(d *storage.Deployment) bool, imageFilter func(d *storage.Image) bool) (map[string]*storage.Image, map[string][]*storage.Deployment) { @@ -333,6 +390,7 @@ func compileExpectedForImageGraphQL(deployments []*storage.Deployment, images [] continue } + imageID := getImageIDForTest(image) var deploymentFilterPassed bool for _, deployment := range imageToDeploymentsMap[image.GetName().GetFullName()] { if deployment == nil { @@ -342,11 +400,11 @@ func compileExpectedForImageGraphQL(deployments []*storage.Deployment, images [] continue } deploymentFilterPassed = true - matchedDeploymentsPerImage[image.GetId()] = append(matchedDeploymentsPerImage[image.GetId()], deployment) + matchedDeploymentsPerImage[imageID] = append(matchedDeploymentsPerImage[imageID], deployment) } if deploymentFilterPassed { - matchedImages[image.GetId()] = image + matchedImages[imageID] = image } } diff --git a/central/graphql/resolvers/images_v2.go b/central/graphql/resolvers/images_v2.go index 94dbe92ecda64..2ab0c69f0855e 100644 --- a/central/graphql/resolvers/images_v2.go +++ b/central/graphql/resolvers/images_v2.go @@ -23,6 +23,7 @@ func init() { schema := getBuilder() utils.Must( schema.AddExtraResolvers("ImageV2", []string{ + "baseImage: BaseImage", "deploymentCount(query: String): Int!", "deployments(query: String, pagination: Pagination): [Deployment!]!", "imageComponentCount(query: String): Int!", @@ -55,6 +56,8 @@ func init() { ) } +var _ ImageResolver = (*imageV2Resolver)(nil) + // ImagesV2 returns GraphQL resolvers for all images using the ImageV2 model func (resolver *Resolver) ImageV2s(ctx context.Context, args PaginatedQuery) ([]ImageResolver, error) { defer metrics.SetGraphQLOperationDurationTime(time.Now(), pkgMetrics.Root, "ImageV2s") @@ -393,3 +396,9 @@ func (resolver *imageV2Resolver) FixableLowCveCount(ctx context.Context) int32 { resolver.ensureData(ctx) return resolver.data.GetScanStats().GetFixableLowCveCount() } + +func (resolver *imageV2Resolver) BaseImage(ctx context.Context) (*baseImageResolver, error) { + resolver.ensureData(ctx) + baseImageInfos := resolver.data.GetBaseImageInfo() + return resolver.root.wrapBaseImage(baseImageInfos) +} diff --git a/central/graphql/resolvers/inputtypes/vuln_req_test.go b/central/graphql/resolvers/inputtypes/vuln_req_test.go index 3a582f52f31d9..4208e2f292fe4 100644 --- a/central/graphql/resolvers/inputtypes/vuln_req_test.go +++ b/central/graphql/resolvers/inputtypes/vuln_req_test.go @@ -28,7 +28,7 @@ func (s *VulnReqInputResolversTestSuite) TestAsRequestExpiry() { { name: "Expiring at time", input: &VulnReqExpiry{ - ExpiresWhenFixed: boolPtr(false), + ExpiresWhenFixed: new(false), ExpiresOn: &graphql.Time{Time: now}, }, expectedExpiry: &storage.RequestExpiry{ @@ -49,7 +49,7 @@ func (s *VulnReqInputResolversTestSuite) TestAsRequestExpiry() { { name: "Expiring when fixed with some value in ExpiresOn", input: &VulnReqExpiry{ - ExpiresWhenFixed: boolPtr(true), + ExpiresWhenFixed: new(true), ExpiresOn: &graphql.Time{Time: now}, }, expectedExpiry: &storage.RequestExpiry{ @@ -60,7 +60,7 @@ func (s *VulnReqInputResolversTestSuite) TestAsRequestExpiry() { }, { name: "Expiring when fixed with nil ExpiresOn", - input: &VulnReqExpiry{ExpiresWhenFixed: boolPtr(true)}, + input: &VulnReqExpiry{ExpiresWhenFixed: new(true)}, expectedExpiry: &storage.RequestExpiry{ Expiry: &storage.RequestExpiry_ExpiresWhenFixed{ ExpiresWhenFixed: true, @@ -70,7 +70,7 @@ func (s *VulnReqInputResolversTestSuite) TestAsRequestExpiry() { { name: "Never expiring with nil ExpiresOn", input: &VulnReqExpiry{ - ExpiresWhenFixed: boolPtr(false), + ExpiresWhenFixed: new(false), ExpiresOn: nil, }, expectedExpiry: &storage.RequestExpiry{}, @@ -88,7 +88,7 @@ func (s *VulnReqInputResolversTestSuite) TestAsRequestExpiry() { { name: "Expiring at zero time", input: &VulnReqExpiry{ - ExpiresWhenFixed: boolPtr(false), + ExpiresWhenFixed: new(false), ExpiresOn: &graphql.Time{Time: time.Time{}}, }, expectedExpiry: &storage.RequestExpiry{ @@ -105,7 +105,3 @@ func (s *VulnReqInputResolversTestSuite) TestAsRequestExpiry() { }) } } - -func boolPtr(b bool) *bool { - return &b -} diff --git a/central/graphql/resolvers/loaders/cluster_cves.go b/central/graphql/resolvers/loaders/cluster_cves.go index a4c53b6467008..c5eabe6830b4f 100644 --- a/central/graphql/resolvers/loaders/cluster_cves.go +++ b/central/graphql/resolvers/loaders/cluster_cves.go @@ -13,10 +13,10 @@ import ( "github.com/stackrox/rox/pkg/sync" ) -var clusterCveLoaderType = reflect.TypeOf(storage.ClusterCVE{}) +var clusterCveLoaderType = reflect.TypeFor[storage.ClusterCVE]() func init() { - RegisterTypeFactory(reflect.TypeOf(storage.ClusterCVE{}), func() interface{} { + RegisterTypeFactory(reflect.TypeFor[storage.ClusterCVE](), func() interface{} { return NewClusterCVELoader(clusterCVEDataStore.Singleton()) }) } @@ -45,7 +45,6 @@ type ClusterCVELoader interface { FromQuery(ctx context.Context, query *v1.Query) ([]*storage.ClusterCVE, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // clusterCveLoaderImpl implements the ClusterCVELoader interface. @@ -91,11 +90,6 @@ func (idl *clusterCveLoaderImpl) CountFromQuery(ctx context.Context, query *v1.Q return int32(count), nil } -func (idl *clusterCveLoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := idl.ds.Count(ctx, search.EmptyQuery()) - return int32(count), err -} - func (idl *clusterCveLoaderImpl) load(ctx context.Context, ids []string) ([]*storage.ClusterCVE, error) { cves, missing := idl.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/loaders/componentsV2.go b/central/graphql/resolvers/loaders/componentsV2.go index 54af9ed470e99..ea95c54ef2cd8 100644 --- a/central/graphql/resolvers/loaders/componentsV2.go +++ b/central/graphql/resolvers/loaders/componentsV2.go @@ -9,12 +9,11 @@ import ( "github.com/stackrox/rox/central/imagecomponent/v2/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/sync" ) var ( - componentV2LoaderType = reflect.TypeOf(storage.ImageComponentV2{}) + componentV2LoaderType = reflect.TypeFor[storage.ImageComponentV2]() ) func init() { @@ -47,7 +46,6 @@ type ComponentV2Loader interface { FromQuery(ctx context.Context, query *v1.Query) ([]*storage.ImageComponentV2, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // componentV2LoaderImpl implements the ComponentDataLoader interface. @@ -89,11 +87,6 @@ func (idl *componentV2LoaderImpl) CountFromQuery(ctx context.Context, query *v1. return int32(numResults), nil } -func (idl *componentV2LoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := idl.ds.Count(ctx, search.EmptyQuery()) - return int32(count), err -} - func (idl *componentV2LoaderImpl) load(ctx context.Context, ids []string) ([]*storage.ImageComponentV2, error) { components, missing := idl.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/loaders/deployments.go b/central/graphql/resolvers/loaders/deployments.go index dcf658aed635e..6bddb97f21481 100644 --- a/central/graphql/resolvers/loaders/deployments.go +++ b/central/graphql/resolvers/loaders/deployments.go @@ -13,10 +13,10 @@ import ( "github.com/stackrox/rox/pkg/sync" ) -var deploymentLoaderType = reflect.TypeOf(storage.Deployment{}) +var deploymentLoaderType = reflect.TypeFor[storage.Deployment]() func init() { - RegisterTypeFactory(reflect.TypeOf(storage.Deployment{}), func() interface{} { + RegisterTypeFactory(reflect.TypeFor[storage.Deployment](), func() interface{} { return NewDeploymentLoader(datastore.Singleton(), deploymentsView.Singleton()) }) } @@ -46,7 +46,6 @@ type DeploymentLoader interface { FromQuery(ctx context.Context, query *v1.Query) ([]*storage.Deployment, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // deploymentLoaderImpl implements the DeploymentDataLoader interface. @@ -94,12 +93,6 @@ func (idl *deploymentLoaderImpl) CountFromQuery(ctx context.Context, query *v1.Q return int32(count), nil } -// CountFromQuery returns the total number of deployments. -func (idl *deploymentLoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := idl.ds.CountDeployments(ctx) - return int32(count), err -} - func (idl *deploymentLoaderImpl) load(ctx context.Context, ids []string) ([]*storage.Deployment, error) { deployments, missing := idl.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/loaders/deployments_test.go b/central/graphql/resolvers/loaders/deployments_test.go index f378e5d928d73..a840a59907005 100644 --- a/central/graphql/resolvers/loaders/deployments_test.go +++ b/central/graphql/resolvers/loaders/deployments_test.go @@ -131,7 +131,7 @@ func (suite *DeploymentLoaderTestSuite) TestFromQuery() { } query := &v1.Query{} - results := make([]deploymentsView.DeploymentCore, 0) + results := make([]deploymentsView.DeploymentCore, 0, 2) core1 := deploymentsViewMocks.NewMockDeploymentCore(suite.mockCtrl) core1.EXPECT().GetDeploymentID().Return(dep1) results = append(results, core1) @@ -150,7 +150,7 @@ func (suite *DeploymentLoaderTestSuite) TestFromQuery() { }, deployments) // Get a non-preloaded deployment from id. - results = make([]deploymentsView.DeploymentCore, 0) + results = make([]deploymentsView.DeploymentCore, 0, 3) core1 = deploymentsViewMocks.NewMockDeploymentCore(suite.mockCtrl) core1.EXPECT().GetDeploymentID().Return(dep1) results = append(results, core1) @@ -178,7 +178,7 @@ func (suite *DeploymentLoaderTestSuite) TestFromQuery() { }, deployments) // Above call should now be preloaded. - results = make([]deploymentsView.DeploymentCore, 0) + results = make([]deploymentsView.DeploymentCore, 0, 3) core1 = deploymentsViewMocks.NewMockDeploymentCore(suite.mockCtrl) core1.EXPECT().GetDeploymentID().Return(dep1) results = append(results, core1) diff --git a/central/graphql/resolvers/loaders/image_cves_v2.go b/central/graphql/resolvers/loaders/image_cves_v2.go index ed4b837fe0d38..2f0533dec7562 100644 --- a/central/graphql/resolvers/loaders/image_cves_v2.go +++ b/central/graphql/resolvers/loaders/image_cves_v2.go @@ -13,7 +13,7 @@ import ( "github.com/stackrox/rox/pkg/sync" ) -var imageCveV2LoaderType = reflect.TypeOf(storage.ImageCVEV2{}) +var imageCveV2LoaderType = reflect.TypeFor[storage.ImageCVEV2]() func init() { RegisterTypeFactory(imageCveV2LoaderType, func() interface{} { @@ -45,7 +45,6 @@ type ImageCVEV2Loader interface { FromQuery(ctx context.Context, query *v1.Query) ([]*storage.ImageCVEV2, error) GetIDs(ctx context.Context, query *v1.Query) ([]string, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // imageCveV2LoaderImpl implements the ImageCVELoader interface. @@ -99,11 +98,6 @@ func (idl *imageCveV2LoaderImpl) CountFromQuery(ctx context.Context, query *v1.Q return int32(count), nil } -func (idl *imageCveV2LoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := idl.ds.Count(ctx, search.EmptyQuery()) - return int32(count), err -} - func (idl *imageCveV2LoaderImpl) load(ctx context.Context, ids []string) ([]*storage.ImageCVEV2, error) { cves, missing := idl.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/loaders/images.go b/central/graphql/resolvers/loaders/images.go index 27b5e57fd49e8..a224029e0eb25 100644 --- a/central/graphql/resolvers/loaders/images.go +++ b/central/graphql/resolvers/loaders/images.go @@ -14,10 +14,10 @@ import ( "github.com/stackrox/rox/pkg/sync" ) -var imageLoaderType = reflect.TypeOf(storage.Image{}) +var imageLoaderType = reflect.TypeFor[storage.Image]() func init() { - RegisterTypeFactory(reflect.TypeOf(storage.Image{}), func() interface{} { + RegisterTypeFactory(reflect.TypeFor[storage.Image](), func() interface{} { return NewImageLoader(datastore.Singleton(), imagesView.Singleton()) }) } @@ -48,7 +48,6 @@ type ImageLoader interface { FullImageWithID(ctx context.Context, id string) (*storage.Image, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // imageLoaderImpl implements the ImageDataLoader interface. @@ -123,11 +122,6 @@ func (idl *imageLoaderImpl) CountFromQuery(ctx context.Context, query *v1.Query) return int32(count), nil } -func (idl *imageLoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := idl.ds.CountImages(ctx) - return int32(count), err -} - func (idl *imageLoaderImpl) load(ctx context.Context, ids []string, pullFullObject bool) ([]*storage.Image, error) { images, missing := idl.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/loaders/images_test.go b/central/graphql/resolvers/loaders/images_test.go index 5bc7683901306..fff050e5cb66c 100644 --- a/central/graphql/resolvers/loaders/images_test.go +++ b/central/graphql/resolvers/loaders/images_test.go @@ -172,7 +172,7 @@ func (suite *ImageLoaderTestSuite) TestFromQuery() { query := &v1.Query{} // Get a preloaded image from id. - results := make([]imagesView.ImageCore, 0) + results := make([]imagesView.ImageCore, 0, 2) core1 := imagesViewMocks.NewMockImageCore(suite.mockCtrl) core1.EXPECT().GetImageID().Return(sha1) results = append(results, core1) @@ -191,7 +191,7 @@ func (suite *ImageLoaderTestSuite) TestFromQuery() { }, images) // Get a non-preloaded image from id. - results = make([]imagesView.ImageCore, 0) + results = make([]imagesView.ImageCore, 0, 3) core1 = imagesViewMocks.NewMockImageCore(suite.mockCtrl) core1.EXPECT().GetImageID().Return(sha1) results = append(results, core1) @@ -219,7 +219,7 @@ func (suite *ImageLoaderTestSuite) TestFromQuery() { }, images) // Above call should now be preloaded. - results = make([]imagesView.ImageCore, 0) + results = make([]imagesView.ImageCore, 0, 3) core1 = imagesViewMocks.NewMockImageCore(suite.mockCtrl) core1.EXPECT().GetImageID().Return(sha1) results = append(results, core1) diff --git a/central/graphql/resolvers/loaders/images_v2.go b/central/graphql/resolvers/loaders/images_v2.go index 8b0a252a3ed50..0c4273a37e516 100644 --- a/central/graphql/resolvers/loaders/images_v2.go +++ b/central/graphql/resolvers/loaders/images_v2.go @@ -10,14 +10,13 @@ import ( v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" - searchPkg "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/sync" ) -var imageV2LoaderType = reflect.TypeOf(storage.ImageV2{}) +var imageV2LoaderType = reflect.TypeFor[storage.ImageV2]() func init() { - RegisterTypeFactory(reflect.TypeOf(storage.ImageV2{}), func() interface{} { + RegisterTypeFactory(reflect.TypeFor[storage.ImageV2](), func() interface{} { return NewImageV2Loader(datastore.Singleton(), imagesView.Singleton()) }) } @@ -48,7 +47,6 @@ type ImageV2Loader interface { FullImageWithID(ctx context.Context, id string) (*storage.ImageV2, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // imageV2LoaderImpl implements the ImageDataLoader interface. @@ -123,11 +121,6 @@ func (idl *imageV2LoaderImpl) CountFromQuery(ctx context.Context, query *v1.Quer return int32(count), nil } -func (idl *imageV2LoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := idl.ds.Count(ctx, searchPkg.EmptyQuery()) - return int32(count), err -} - func (idl *imageV2LoaderImpl) load(ctx context.Context, ids []string, pullFullObject bool) ([]*storage.ImageV2, error) { images, missing := idl.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/loaders/images_v2_test.go b/central/graphql/resolvers/loaders/images_v2_test.go index d1ef530ac0866..f0242aba1180e 100644 --- a/central/graphql/resolvers/loaders/images_v2_test.go +++ b/central/graphql/resolvers/loaders/images_v2_test.go @@ -168,7 +168,7 @@ func (suite *ImageV2LoaderTestSuite) TestFromQuery() { query := &v1.Query{} // Get a preloaded image from id. - results := make([]imagesView.ImageCore, 0) + results := make([]imagesView.ImageCore, 0, 2) core1 := imagesViewMocks.NewMockImageCore(suite.mockCtrl) core1.EXPECT().GetImageID().Return(sha1) results = append(results, core1) @@ -187,7 +187,7 @@ func (suite *ImageV2LoaderTestSuite) TestFromQuery() { }, images) // Get a non-preloaded image from id. - results = make([]imagesView.ImageCore, 0) + results = make([]imagesView.ImageCore, 0, 3) core1 = imagesViewMocks.NewMockImageCore(suite.mockCtrl) core1.EXPECT().GetImageID().Return(sha1) results = append(results, core1) @@ -215,7 +215,7 @@ func (suite *ImageV2LoaderTestSuite) TestFromQuery() { }, images) // Above call should now be preloaded. - results = make([]imagesView.ImageCore, 0) + results = make([]imagesView.ImageCore, 0, 3) core1 = imagesViewMocks.NewMockImageCore(suite.mockCtrl) core1.EXPECT().GetImageID().Return(sha1) results = append(results, core1) diff --git a/central/graphql/resolvers/loaders/list_deployments.go b/central/graphql/resolvers/loaders/list_deployments.go index 49eed624b87e1..251f4d0ec2b5b 100644 --- a/central/graphql/resolvers/loaders/list_deployments.go +++ b/central/graphql/resolvers/loaders/list_deployments.go @@ -13,10 +13,10 @@ import ( "github.com/stackrox/rox/pkg/sync" ) -var listDeploymentLoaderType = reflect.TypeOf(storage.ListDeployment{}) +var listDeploymentLoaderType = reflect.TypeFor[storage.ListDeployment]() func init() { - RegisterTypeFactory(reflect.TypeOf(storage.ListDeployment{}), func() interface{} { + RegisterTypeFactory(reflect.TypeFor[storage.ListDeployment](), func() interface{} { return NewListDeploymentLoader(datastore.Singleton()) }) } diff --git a/central/graphql/resolvers/loaders/namespaces.go b/central/graphql/resolvers/loaders/namespaces.go index ef4dfac67a009..09b27d98b58da 100644 --- a/central/graphql/resolvers/loaders/namespaces.go +++ b/central/graphql/resolvers/loaders/namespaces.go @@ -13,10 +13,10 @@ import ( "github.com/stackrox/rox/pkg/sync" ) -var namespaceLoaderType = reflect.TypeOf(storage.NamespaceMetadata{}) +var namespaceLoaderType = reflect.TypeFor[storage.NamespaceMetadata]() func init() { - RegisterTypeFactory(reflect.TypeOf(storage.NamespaceMetadata{}), func() interface{} { + RegisterTypeFactory(reflect.TypeFor[storage.NamespaceMetadata](), func() interface{} { return NewNamespaceLoader(datastore.Singleton()) }) } @@ -45,7 +45,6 @@ type NamespaceLoader interface { FromQuery(ctx context.Context, query *v1.Query) ([]*storage.NamespaceMetadata, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // namespaceLoaderImpl implements the NamespaceLoader interface. @@ -92,12 +91,6 @@ func (nsldr *namespaceLoaderImpl) CountFromQuery(ctx context.Context, query *v1. return int32(numResults), nil } -// CountAll returns number of all namespaces -func (nsldr *namespaceLoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := nsldr.ds.Count(ctx, search.EmptyQuery()) - return int32(count), err -} - func (nsldr *namespaceLoaderImpl) load(ctx context.Context, ids []string) ([]*storage.NamespaceMetadata, error) { namespaces, missing := nsldr.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/loaders/node_components.go b/central/graphql/resolvers/loaders/node_components.go index 02f264f770323..53bd833761760 100644 --- a/central/graphql/resolvers/loaders/node_components.go +++ b/central/graphql/resolvers/loaders/node_components.go @@ -13,10 +13,10 @@ import ( "github.com/stackrox/rox/pkg/sync" ) -var nodeComponentLoaderType = reflect.TypeOf(storage.NodeComponent{}) +var nodeComponentLoaderType = reflect.TypeFor[storage.NodeComponent]() func init() { - RegisterTypeFactory(reflect.TypeOf(storage.NodeComponent{}), func() interface{} { + RegisterTypeFactory(reflect.TypeFor[storage.NodeComponent](), func() interface{} { return NewNodeComponentLoader(datastore.Singleton()) }) } @@ -45,7 +45,6 @@ type NodeComponentLoader interface { FromQuery(ctx context.Context, query *v1.Query) ([]*storage.NodeComponent, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // nodeComponentLoaderImpl implements the NodeComponentLoader interface. @@ -88,11 +87,6 @@ func (idl *nodeComponentLoaderImpl) CountFromQuery(ctx context.Context, query *v return int32(numResults), nil } -func (idl *nodeComponentLoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := idl.ds.Count(ctx, search.EmptyQuery()) - return int32(count), err -} - func (idl *nodeComponentLoaderImpl) load(ctx context.Context, ids []string) ([]*storage.NodeComponent, error) { components, missing := idl.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/loaders/node_cves.go b/central/graphql/resolvers/loaders/node_cves.go index d25e577b5824e..c34fa2e5ddcd6 100644 --- a/central/graphql/resolvers/loaders/node_cves.go +++ b/central/graphql/resolvers/loaders/node_cves.go @@ -13,10 +13,10 @@ import ( "github.com/stackrox/rox/pkg/sync" ) -var nodeCveLoaderType = reflect.TypeOf(storage.NodeCVE{}) +var nodeCveLoaderType = reflect.TypeFor[storage.NodeCVE]() func init() { - RegisterTypeFactory(reflect.TypeOf(storage.NodeCVE{}), func() interface{} { + RegisterTypeFactory(reflect.TypeFor[storage.NodeCVE](), func() interface{} { return NewNodeCVELoader(nodeCVEDataStore.Singleton()) }) } @@ -45,7 +45,6 @@ type NodeCVELoader interface { FromQuery(ctx context.Context, query *v1.Query) ([]*storage.NodeCVE, error) GetIDs(ctx context.Context, query *v1.Query) ([]string, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // nodeCVELoaderImpl implements the NodeCVELoader interface. @@ -99,11 +98,6 @@ func (idl *nodeCVELoaderImpl) CountFromQuery(ctx context.Context, query *v1.Quer return int32(count), nil } -func (idl *nodeCVELoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := idl.ds.Count(ctx, search.EmptyQuery()) - return int32(count), err -} - func (idl *nodeCVELoaderImpl) load(ctx context.Context, ids []string) ([]*storage.NodeCVE, error) { nodeCves, missing := idl.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/loaders/nodes.go b/central/graphql/resolvers/loaders/nodes.go index c79672c94e994..e846700e8cf40 100644 --- a/central/graphql/resolvers/loaders/nodes.go +++ b/central/graphql/resolvers/loaders/nodes.go @@ -14,7 +14,7 @@ import ( "github.com/stackrox/rox/pkg/sync" ) -var nodeLoaderType = reflect.TypeOf(storage.Node{}) +var nodeLoaderType = reflect.TypeFor[storage.Node]() func init() { RegisterTypeFactory(nodeLoaderType, func() interface{} { @@ -47,7 +47,6 @@ type NodeLoader interface { FullNodeWithID(ctx context.Context, id string) (*storage.Node, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // nodeLoaderImpl implements the NodeLoader interface. @@ -116,12 +115,6 @@ func (ndl *nodeLoaderImpl) CountFromQuery(ctx context.Context, query *v1.Query) return int32(numResults), nil } -// CountAll returns the total number of nodes. -func (ndl *nodeLoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := ndl.ds.CountNodes(ctx) - return int32(count), err -} - func (ndl *nodeLoaderImpl) load(ctx context.Context, ids []string, pullFullObject bool) ([]*storage.Node, error) { nodes, missing := ndl.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/loaders/policies.go b/central/graphql/resolvers/loaders/policies.go index 28bdfa0ddd1ae..29e778e07cc5c 100644 --- a/central/graphql/resolvers/loaders/policies.go +++ b/central/graphql/resolvers/loaders/policies.go @@ -13,10 +13,10 @@ import ( "github.com/stackrox/rox/pkg/sync" ) -var policyLoaderType = reflect.TypeOf(storage.Policy{}) +var policyLoaderType = reflect.TypeFor[storage.Policy]() func init() { - RegisterTypeFactory(reflect.TypeOf(storage.Policy{}), func() interface{} { + RegisterTypeFactory(reflect.TypeFor[storage.Policy](), func() interface{} { return NewPolicyLoader(policyDataStore.Singleton()) }) } @@ -45,7 +45,6 @@ type PolicyLoader interface { FromQuery(ctx context.Context, query *v1.Query) ([]*storage.Policy, error) CountFromQuery(ctx context.Context, query *v1.Query) (int32, error) - CountAll(ctx context.Context) (int32, error) } // policyLoaderImpl implements the PolicyDataLoader interface. @@ -93,12 +92,6 @@ func (idl *policyLoaderImpl) CountFromQuery(ctx context.Context, query *v1.Query return int32(numResults), nil } -// CountAll returns the total number of policies. -func (idl *policyLoaderImpl) CountAll(ctx context.Context) (int32, error) { - count, err := idl.CountFromQuery(ctx, search.EmptyQuery()) - return count, err -} - func (idl *policyLoaderImpl) load(ctx context.Context, ids []string) ([]*storage.Policy, error) { policies, missing := idl.readAll(ids) if len(missing) > 0 { diff --git a/central/graphql/resolvers/namespaces.go b/central/graphql/resolvers/namespaces.go index dab78461e4ebc..cbde5703ff071 100644 --- a/central/graphql/resolvers/namespaces.go +++ b/central/graphql/resolvers/namespaces.go @@ -325,7 +325,7 @@ func (resolver *namespaceResolver) getApplicablePolicies(ctx context.Context, q return nil, err } - applicable, _ := matcher.NewNamespaceMatcher(resolver.data.GetMetadata()).FilterApplicablePolicies(policies) + applicable, _ := matcher.NewNamespaceMatcher(resolver.data.GetMetadata()).FilterApplicablePolicies(ctx, policies) return applicable, nil } @@ -396,11 +396,11 @@ func (resolver *namespaceResolver) FailingPolicyCounter(ctx context.Context, arg return nil, err } - alerts, err := resolver.root.ViolationsDataStore.SearchListAlerts(ctx, q, true) + counts, err := resolver.root.ViolationsDataStore.SearchAlertPolicySeverityCounts(ctx, q, true) if err != nil { return nil, nil } - return mapListAlertsToPolicySeverityCount(alerts), nil + return policySeverityCountsToResolver(counts), nil } // PolicyStatus returns true if there is no policy violation for this namespace diff --git a/central/graphql/resolvers/node_components.go b/central/graphql/resolvers/node_components.go index f54f8404851ab..ebb33ec6058f7 100644 --- a/central/graphql/resolvers/node_components.go +++ b/central/graphql/resolvers/node_components.go @@ -76,6 +76,8 @@ type NodeComponentResolver interface { Version(ctx context.Context) string } +var _ NodeComponentResolver = (*nodeComponentResolver)(nil) + // NodeComponent returns a node component based on an input id (name:version) func (resolver *Resolver) NodeComponent(ctx context.Context, args IDQuery) (NodeComponentResolver, error) { defer metrics.SetGraphQLOperationDurationTime(time.Now(), pkgMetrics.Root, "NodeComponent") diff --git a/central/graphql/resolvers/node_components_test.go b/central/graphql/resolvers/node_components_test.go new file mode 100644 index 0000000000000..3a71785b22ea6 --- /dev/null +++ b/central/graphql/resolvers/node_components_test.go @@ -0,0 +1,13 @@ +package resolvers + +import ( + "reflect" + "testing" +) + +func TestNodeComponentResolverType(t *testing.T) { + resolverInterface := reflect.TypeFor[NodeComponentResolver]() + resolverImplType := reflect.TypeFor[*nodeComponentResolver]() + + validateAlignedMethodIndex(t, resolverInterface, resolverImplType) +} diff --git a/central/graphql/resolvers/node_cve_core.go b/central/graphql/resolvers/node_cve_core.go index ca8092891113f..5708d24d94baf 100644 --- a/central/graphql/resolvers/node_cve_core.go +++ b/central/graphql/resolvers/node_cve_core.go @@ -12,7 +12,6 @@ import ( "github.com/stackrox/rox/central/views/nodecve" v1 "github.com/stackrox/rox/generated/api/v1" pkgMetrics "github.com/stackrox/rox/pkg/metrics" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/paginated" "github.com/stackrox/rox/pkg/utils" @@ -91,7 +90,7 @@ func (resolver *nodeCVECoreResolver) DistroTuples(ctx context.Context) ([]NodeVu // obtained via SQF. So, the auto removal of snoozed CVEs is unintentional here. Hence, we add explicit filter with // CVESuppressed == true OR false q := PaginatedQuery{ - Query: pointers.String(search.NewQueryBuilder().AddExactMatches(search.CVEID, resolver.data.GetCVEIDs()...). + Query: new(search.NewQueryBuilder().AddExactMatches(search.CVEID, resolver.data.GetCVEIDs()...). AddBools(search.CVESuppressed, true, false). Query()), } diff --git a/central/graphql/resolvers/node_cve_core_test.go b/central/graphql/resolvers/node_cve_core_test.go index 2eea19b0d87f5..88956b3c0ba9f 100644 --- a/central/graphql/resolvers/node_cve_core_test.go +++ b/central/graphql/resolvers/node_cve_core_test.go @@ -15,7 +15,6 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" "github.com/stretchr/testify/suite" @@ -84,7 +83,7 @@ func (s *NodeCVECoreResolverTestSuite) TestNodeCVEsNonEmpty() { func (s *NodeCVECoreResolverTestSuite) TestNodeCVEsWithQuery() { q := &PaginatedQuery{ - Query: pointers.String("CVE:cve-2022-xyz"), + Query: new("CVE:cve-2022-xyz"), } expectedQ := search.NewQueryBuilder().AddStrings(search.CVE, "cve-2022-xyz"). WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery() @@ -107,9 +106,9 @@ func (s *NodeCVECoreResolverTestSuite) TestNodeCVEsWithPaginatedQuery() { q := &PaginatedQuery{ Pagination: &inputtypes.Pagination{ SortOption: &inputtypes.SortOption{ - Field: pointers.String(search.NodeTopCVSS.String()), + Field: new(search.NodeTopCVSS.String()), AggregateBy: &inputtypes.AggregateBy{ - AggregateFunc: pointers.String(aggregatefunc.Max.Name()), + AggregateFunc: new(aggregatefunc.Max.Name()), }, }, }, @@ -160,7 +159,7 @@ func (s *NodeCVECoreResolverTestSuite) TestNodeCVECount() { func (s *NodeCVECoreResolverTestSuite) TestNodeCVECountWithQuery() { q := &RawQuery{ - Query: pointers.String("Node:node"), + Query: new("Node:node"), } expectedQ := search.NewQueryBuilder().AddStrings(search.Node, "node").ProtoQuery() expectedQ = tryUnsuppressedQuery(expectedQ) @@ -200,7 +199,7 @@ func (s *NodeCVECoreResolverTestSuite) TestNodeCVENoNodePerm() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), + Cve: new("cve-xyz"), }, ) s.Error(err) @@ -221,7 +220,7 @@ func (s *NodeCVECoreResolverTestSuite) TestNodeCVENonEmpty() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), + Cve: new("cve-xyz"), }, ) s.NoError(err) @@ -242,8 +241,8 @@ func (s *NodeCVECoreResolverTestSuite) TestNodeCVENonEmpty() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), - SubfieldScopeQuery: pointers.String("Fixable:true"), + Cve: new("cve-xyz"), + SubfieldScopeQuery: new("Fixable:true"), }, ) s.NoError(err) @@ -264,8 +263,8 @@ func (s *NodeCVECoreResolverTestSuite) TestNodeCVENonEmpty() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), - SubfieldScopeQuery: pointers.String("Namespace:n1"), + Cve: new("cve-xyz"), + SubfieldScopeQuery: new("Namespace:n1"), }, ) s.NoError(err) @@ -307,7 +306,7 @@ func (s *NodeCVECoreResolverTestSuite) TestNodeCVECountBySeverity() { func (s *NodeCVECoreResolverTestSuite) TestNodeCVECountBySeverityWithQuery() { q := &RawQuery{ - Query: pointers.String("Node:node"), + Query: new("Node:node"), } expectedQ := search.NewQueryBuilder().AddStrings(search.Node, "node").ProtoQuery() expectedQ = tryUnsuppressedQuery(expectedQ) @@ -374,7 +373,7 @@ func (s *NodeCVECoreResolverTestSuite) TestNodeCVESubResolvers() { Cve *string SubfieldScopeQuery *string }{ - Cve: pointers.String("cve-xyz"), + Cve: new("cve-xyz"), }, ) s.NoError(err) diff --git a/central/graphql/resolvers/node_scan_benchmark_test.go b/central/graphql/resolvers/node_scan_benchmark_test.go index 7ac5bb4eab9d1..f58034ca3d6d9 100644 --- a/central/graphql/resolvers/node_scan_benchmark_test.go +++ b/central/graphql/resolvers/node_scan_benchmark_test.go @@ -73,7 +73,7 @@ func BenchmarkNodeResolver(b *testing.B) { } b.Run("GetNodeComponentsInNodeScanResolver", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, nodeWithScanQuery, "getNodes", @@ -88,7 +88,7 @@ func BenchmarkNodeResolver(b *testing.B) { }) b.Run("GetNodeComponentsWithoutNodeScanResolver", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, nodeWithoutScanQuery, "getNodes", @@ -103,7 +103,7 @@ func BenchmarkNodeResolver(b *testing.B) { }) b.Run("GetNodeComponentsDerivedFieldsWithNodeScanResolver", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, nodeWithScanLongQuery, "getNodes", @@ -118,7 +118,7 @@ func BenchmarkNodeResolver(b *testing.B) { }) b.Run("GetNodeComponentsDerivedWithoutNodeScanResolver", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, nodeWithoutScanLongQuery, "getNodes", @@ -133,7 +133,7 @@ func BenchmarkNodeResolver(b *testing.B) { }) b.Run("GetNodeOnly", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, nodeOnlyQuery, "getNodes", @@ -148,7 +148,7 @@ func BenchmarkNodeResolver(b *testing.B) { }) b.Run("GetNodeWithCounts", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { response := schema.Exec(ctx, nodeWithCountsQuery, "getNodes", diff --git a/central/graphql/resolvers/node_vulnerabilities.go b/central/graphql/resolvers/node_vulnerabilities.go index 0e4ef66d3b0c9..d8b0a063b9f28 100644 --- a/central/graphql/resolvers/node_vulnerabilities.go +++ b/central/graphql/resolvers/node_vulnerabilities.go @@ -50,6 +50,8 @@ type NodeVulnerabilityResolver interface { OperatingSystem(ctx context.Context) string } +var _ NodeVulnerabilityResolver = (*nodeCVEResolver)(nil) + // NodeVulnerability resolves a single vulnerability based on an id func (resolver *Resolver) NodeVulnerability(ctx context.Context, args IDQuery) (NodeVulnerabilityResolver, error) { defer metrics.SetGraphQLOperationDurationTime(time.Now(), pkgMetrics.Root, "NodeVulnerability") diff --git a/central/graphql/resolvers/node_vulnerabilities_test.go b/central/graphql/resolvers/node_vulnerabilities_test.go new file mode 100644 index 0000000000000..25d0ce5421753 --- /dev/null +++ b/central/graphql/resolvers/node_vulnerabilities_test.go @@ -0,0 +1,27 @@ +package resolvers + +import ( + "reflect" + "testing" +) + +func TestNodeVulnerabilityResolverType(t *testing.T) { + // The implementation type has six more methods than the interface it implements. + // + // 6 - ID + // 18 - Orphaned + // 19 - OrphanedTime + // 23 - SnoozeExpiry + // 24 - SnoozeStart + // 25 - Snoozed + // + // This can break graphQL queries to the system until + // https://github.com/graph-gophers/graphql-go/issues/763 is fixed. + // + // TODO(ROX-35654): Unskip this tests. + t.Skip("Interface and implementation types do not have aligned method indices.") + resolverInterface := reflect.TypeFor[NodeVulnerabilityResolver]() + resolverImplType := reflect.TypeFor[*nodeCVEResolver]() + + validateAlignedMethodIndex(t, resolverInterface, resolverImplType) +} diff --git a/central/graphql/resolvers/platform_cve_core_test.go b/central/graphql/resolvers/platform_cve_core_test.go index 2dabd865c6ad1..1a87d297195dc 100644 --- a/central/graphql/resolvers/platform_cve_core_test.go +++ b/central/graphql/resolvers/platform_cve_core_test.go @@ -9,7 +9,6 @@ import ( "github.com/stackrox/rox/central/views/platformcve" platformCVEViewMock "github.com/stackrox/rox/central/views/platformcve/mocks" "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" "github.com/stretchr/testify/suite" @@ -79,7 +78,7 @@ func (s *PlatformCVEResolverTestSuite) TestGetPlatformCVEsNonEmpty() { func (s *PlatformCVEResolverTestSuite) TestGetPlatformCVEsWithQuery() { q := &PaginatedQuery{ - Query: pointers.String("CVE:cve-2022-xyz"), + Query: new("CVE:cve-2022-xyz"), } expectedQ := search.NewQueryBuilder().AddStrings(search.CVE, "cve-2022-xyz"). WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery() @@ -101,9 +100,9 @@ func (s *PlatformCVEResolverTestSuite) TestPlatformCVEsCVEsWithPaginatedQuery() q := &PaginatedQuery{ Pagination: &inputtypes.Pagination{ SortOption: &inputtypes.SortOption{ - Field: pointers.String(search.CVSS.String()), + Field: new(search.CVSS.String()), AggregateBy: &inputtypes.AggregateBy{ - AggregateFunc: pointers.String(aggregatefunc.Max.Name()), + AggregateFunc: new(aggregatefunc.Max.Name()), }, }, }, @@ -140,7 +139,7 @@ func (s *PlatformCVEResolverTestSuite) TestPlatformCVECount() { func (s *PlatformCVEResolverTestSuite) TestPlatformCVECountWithQuery() { q := &RawQuery{ - Query: pointers.String("Cluster:c1"), + Query: new("Cluster:c1"), } expectedQ := search.NewQueryBuilder().AddStrings(search.Cluster, "c1").ProtoQuery() expectedQ = tryUnsuppressedQuery(expectedQ) @@ -172,7 +171,7 @@ func (s *PlatformCVEResolverTestSuite) TestGetPlatformCVENonEmpty() { CveID *string SubfieldScopeQuery *string }{ - CveID: pointers.String("cve-xyz#K8S_CVE"), + CveID: new("cve-xyz#K8S_CVE"), }, ) s.NoError(err) @@ -192,8 +191,8 @@ func (s *PlatformCVEResolverTestSuite) TestGetPlatformCVENonEmpty() { CveID *string SubfieldScopeQuery *string }{ - CveID: pointers.String("cve-xyz#K8S_CVE"), - SubfieldScopeQuery: pointers.String("Cluster:c1"), + CveID: new("cve-xyz#K8S_CVE"), + SubfieldScopeQuery: new("Cluster:c1"), }, ) s.NoError(err) @@ -213,8 +212,8 @@ func (s *PlatformCVEResolverTestSuite) TestGetPlatformCVENonEmpty() { CveID *string SubfieldScopeQuery *string }{ - CveID: pointers.String("cve-xyz#K8S_CVE"), - SubfieldScopeQuery: pointers.String("Cluster Platform Type:KUBERNETES_CLUSTER"), + CveID: new("cve-xyz#K8S_CVE"), + SubfieldScopeQuery: new("Cluster Platform Type:KUBERNETES_CLUSTER"), }, ) s.NoError(err) diff --git a/central/graphql/resolvers/policies.go b/central/graphql/resolvers/policies.go index 6e6bcd700ab65..1bea72a663a26 100644 --- a/central/graphql/resolvers/policies.go +++ b/central/graphql/resolvers/policies.go @@ -13,7 +13,6 @@ import ( mitreUtils "github.com/stackrox/rox/pkg/mitre/utils" "github.com/stackrox/rox/pkg/policyutils" "github.com/stackrox/rox/pkg/search" - "github.com/stackrox/rox/pkg/search/paginated" "github.com/stackrox/rox/pkg/search/scoped" "github.com/stackrox/rox/pkg/utils" ) @@ -184,17 +183,11 @@ func (resolver *policyResolver) failingDeployments(ctx context.Context, q *v1.Qu alertsQuery := search.ConjunctionQuery(resolver.getPolicyQuery(), search.NewQueryBuilder().AddExactMatches(search.ViolationState, storage.ViolationState_ACTIVE.String()).ProtoQuery()) - alertsQuery = paginated.FillDefaultSortOption(alertsQuery, paginated.GetViolationTimeSortOption()) - listAlerts, err := resolver.root.ViolationsDataStore.SearchListAlerts(ctx, alertsQuery, true) + deploymentIDs, err := resolver.root.ViolationsDataStore.SearchAlertDeploymentIDs(ctx, alertsQuery, true) if err != nil { return nil, err } - deploymentIDs := make([]string, 0, len(listAlerts)) - for _, alert := range listAlerts { - deploymentIDs = append(deploymentIDs, alert.GetDeployment().GetId()) - } - deploymentQuery := search.ConjunctionQuery(q, search.NewQueryBuilder().AddDocIDs(deploymentIDs...).ProtoQuery()) deploymentQuery.Pagination = q.GetPagination() diff --git a/central/graphql/resolvers/policy_counter.go b/central/graphql/resolvers/policy_counter.go index ad88a1907cf86..a53d26ffb24f0 100644 --- a/central/graphql/resolvers/policy_counter.go +++ b/central/graphql/resolvers/policy_counter.go @@ -3,6 +3,7 @@ package resolvers import ( "context" + alertviews "github.com/stackrox/rox/central/alert/views" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/utils" @@ -58,14 +59,10 @@ func (evr *PolicyCounterResolver) Critical(_ context.Context) int32 { // Static helpers. ////////////////// -func mapListAlertsToPolicySeverityCount(alerts []*storage.ListAlert) *PolicyCounterResolver { +func mapListAlertPoliciesToPolicySeverityCount(policies []*storage.ListAlertPolicy) *PolicyCounterResolver { counter := &PolicyCounterResolver{} policyIDs := set.NewStringSet() - for _, alert := range alerts { - if alert.GetState() != storage.ViolationState_ACTIVE { - continue - } - policy := alert.GetPolicy() + for _, policy := range policies { if !policyIDs.Add(policy.GetId()) { continue } @@ -74,16 +71,14 @@ func mapListAlertsToPolicySeverityCount(alerts []*storage.ListAlert) *PolicyCoun return counter } -func mapListAlertPoliciesToPolicySeverityCount(policies []*storage.ListAlertPolicy) *PolicyCounterResolver { - counter := &PolicyCounterResolver{} - policyIDs := set.NewStringSet() - for _, policy := range policies { - if !policyIDs.Add(policy.GetId()) { - continue - } - incPolicyCounter(counter, policy.GetSeverity()) +func policySeverityCountsToResolver(counts *alertviews.PolicySeverityCounts) *PolicyCounterResolver { + return &PolicyCounterResolver{ + total: int32(counts.LowCount + counts.MediumCount + counts.HighCount + counts.CriticalCount), + low: int32(counts.LowCount), + medium: int32(counts.MediumCount), + high: int32(counts.HighCount), + critical: int32(counts.CriticalCount), } - return counter } func incPolicyCounter(counter *PolicyCounterResolver, severity storage.Severity) { diff --git a/central/graphql/resolvers/policy_counter_test.go b/central/graphql/resolvers/policy_counter_test.go index 9026efafe858c..851236764df01 100644 --- a/central/graphql/resolvers/policy_counter_test.go +++ b/central/graphql/resolvers/policy_counter_test.go @@ -3,53 +3,56 @@ package resolvers import ( "testing" - "github.com/stackrox/rox/generated/storage" + alertviews "github.com/stackrox/rox/central/alert/views" "github.com/stretchr/testify/assert" ) -func TestMapListAlertsToPolicyCounterResolver(t *testing.T) { - alerts := []*storage.ListAlert{ +func TestPolicySeverityCountsToResolver(t *testing.T) { + cases := []struct { + name string + counts *alertviews.PolicySeverityCounts + total int32 + low int32 + medium int32 + high int32 + critical int32 + }{ { - State: storage.ViolationState_ACTIVE, - Policy: &storage.ListAlertPolicy{ - Id: "id1", - Severity: storage.Severity_LOW_SEVERITY, - }, - }, - { - State: storage.ViolationState_ACTIVE, - Policy: &storage.ListAlertPolicy{ - Id: "id2", - Severity: storage.Severity_HIGH_SEVERITY, - }, + name: "all zeros", + counts: &alertviews.PolicySeverityCounts{}, }, { - State: storage.ViolationState_RESOLVED, - Policy: &storage.ListAlertPolicy{ - Id: "id3", - Severity: storage.Severity_CRITICAL_SEVERITY, + name: "mixed counts", + counts: &alertviews.PolicySeverityCounts{ + LowCount: 2, + MediumCount: 3, + HighCount: 1, + CriticalCount: 4, }, + total: 10, + low: 2, + medium: 3, + high: 1, + critical: 4, }, { - State: storage.ViolationState_ACTIVE, - Policy: &storage.ListAlertPolicy{ - Id: "id1", - Severity: storage.Severity_LOW_SEVERITY, - }, - }, - { - State: storage.ViolationState_ACTIVE, - Policy: &storage.ListAlertPolicy{ - Id: "id3", - Severity: storage.Severity_LOW_SEVERITY, + name: "single severity", + counts: &alertviews.PolicySeverityCounts{ + CriticalCount: 7, }, + total: 7, + critical: 7, }, } - counterResolver := mapListAlertsToPolicySeverityCount(alerts) - assert.Equal(t, int32(3), counterResolver.total) - assert.Equal(t, int32(2), counterResolver.low) - assert.Equal(t, int32(0), counterResolver.medium) - assert.Equal(t, int32(1), counterResolver.high) - assert.Equal(t, int32(0), counterResolver.critical) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + resolver := policySeverityCountsToResolver(tc.counts) + assert.Equal(t, tc.total, resolver.total) + assert.Equal(t, tc.low, resolver.low) + assert.Equal(t, tc.medium, resolver.medium) + assert.Equal(t, tc.high, resolver.high) + assert.Equal(t, tc.critical, resolver.critical) + }) + } } diff --git a/central/graphql/resolvers/resolver.go b/central/graphql/resolvers/resolver.go index 6671e78dccf92..7d90f9e275efd 100644 --- a/central/graphql/resolvers/resolver.go +++ b/central/graphql/resolvers/resolver.go @@ -49,6 +49,7 @@ import ( roleDataStore "github.com/stackrox/rox/central/role/datastore" secretDataStore "github.com/stackrox/rox/central/secret/datastore" serviceAccountDataStore "github.com/stackrox/rox/central/serviceaccount/datastore" + signatureIntegrationDataStore "github.com/stackrox/rox/central/signatureintegration/datastore" "github.com/stackrox/rox/central/views/imagecomponentflat" "github.com/stackrox/rox/central/views/imagecve" "github.com/stackrox/rox/central/views/imagecveflat" @@ -106,6 +107,7 @@ type Resolver struct { ViolationsDataStore violationsDatastore.DataStore BaselineDataStore baselineStore.DataStore WatchedImageDataStore watchedImageDataStore.DataStore + SignatureIntegrationDataStore signatureIntegrationDataStore.DataStore orchestratorIstioCVEManager fetcher.OrchestratorIstioCVEManager cveMatcher *cveMatcher.CVEMatcher manager complianceOperatorManager.Manager @@ -156,6 +158,7 @@ func New() *Resolver { ViolationsDataStore: violationsDatastore.Singleton(), BaselineDataStore: baselineStore.Singleton(), WatchedImageDataStore: watchedImageDataStore.Singleton(), + SignatureIntegrationDataStore: signatureIntegrationDataStore.Singleton(), orchestratorIstioCVEManager: fetcher.SingletonManager(), cveMatcher: cveMatcher.Singleton(), manager: complianceOperatorManager.Singleton(), diff --git a/central/graphql/resolvers/search.go b/central/graphql/resolvers/search.go index 21e36a2d2eff5..07ebbce742d43 100644 --- a/central/graphql/resolvers/search.go +++ b/central/graphql/resolvers/search.go @@ -19,6 +19,15 @@ import ( func init() { schema := getBuilder() utils.Must( + schema.AddInput("PaginatedQuery", []string{ + "query: String", + "scopeQuery: String", + "pagination: Pagination", + }), + schema.AddInput("RawQuery", []string{ + "query: String", + "scopeQuery: String", + }), schema.AddQuery("searchOptions(categories: [SearchCategory!]): [String!]!"), schema.AddQuery("globalSearch(categories: [SearchCategory!], query: String!): [SearchResult!]!"), schema.AddQuery("searchAutocomplete(categories: [SearchCategory!], query: String!): [String!]!"), diff --git a/central/graphql/resolvers/search_test.go b/central/graphql/resolvers/search_test.go index 3e22e67cb086e..2e16406fe4455 100644 --- a/central/graphql/resolvers/search_test.go +++ b/central/graphql/resolvers/search_test.go @@ -19,6 +19,7 @@ import ( imageDS "github.com/stackrox/rox/central/image/datastore" imageMocks "github.com/stackrox/rox/central/image/datastore/mocks" imageComponentV2Mocks "github.com/stackrox/rox/central/imagecomponent/v2/datastore/mocks" + imageV2DS "github.com/stackrox/rox/central/imagev2/datastore" imageV2Mocks "github.com/stackrox/rox/central/imagev2/datastore/mocks" namespaceMocks "github.com/stackrox/rox/central/namespace/datastore/mocks" npsMocks "github.com/stackrox/rox/central/networkpolicies/datastore/mocks" @@ -34,9 +35,11 @@ import ( serviceAccountMocks "github.com/stackrox/rox/central/serviceaccount/datastore/mocks" imagesView "github.com/stackrox/rox/central/views/images" v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/grpc/authz/allow" - "github.com/stackrox/rox/pkg/pointers" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/sac" @@ -112,7 +115,7 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "simple query", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc"), + Query: new("CVE:abc"), }, expectedQ: search.NewQueryBuilder().AddStrings(search.CVE, "abc"). WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery(), @@ -120,7 +123,7 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "simple query w/ plus in value", arg: PaginatedQuery{ - Query: pointers.String("CVE:ab+c"), + Query: new("CVE:ab+c"), }, expectedQ: search.NewQueryBuilder().AddStrings(search.CVE, "ab+c"). WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery(), @@ -128,7 +131,7 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "exact query", arg: PaginatedQuery{ - Query: pointers.String("CVE:\"abc\""), + Query: new("CVE:\"abc\""), }, expectedQ: search.NewQueryBuilder().AddExactMatches(search.CVE, "abc"). WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery(), @@ -136,7 +139,7 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "exact query w/ plus in value", arg: PaginatedQuery{ - Query: pointers.String("CVE:\"ab+c\""), + Query: new("CVE:\"ab+c\""), }, expectedQ: search.NewQueryBuilder().AddExactMatches(search.CVE, "ab+c"). WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery(), @@ -144,7 +147,7 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "conjunction query", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc+Image:xyz"), + Query: new("CVE:abc+Image:xyz"), }, expectedQ: search.NewQueryBuilder(). AddStrings(search.CVE, "abc").AddStrings(search.ImageName, "xyz"). @@ -153,7 +156,7 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "disjunction query", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc,xyz"), + Query: new("CVE:abc,xyz"), }, expectedQ: search.NewQueryBuilder().AddStrings(search.CVE, "abc", "xyz"). WithPagination(search.NewPagination().Limit(math.MaxInt32)).ProtoQuery(), @@ -161,7 +164,7 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "conjunction & disjunctions", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc,xyz+Image:img1,img2"), + Query: new("CVE:abc,xyz+Image:img1,img2"), }, expectedQ: search.NewQueryBuilder(). AddStrings(search.CVE, "abc", "xyz").AddStrings(search.ImageName, "img1", "img2"). @@ -170,10 +173,10 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "query + sort", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc"), + Query: new("CVE:abc"), Pagination: &inputtypes.Pagination{ SortOption: &inputtypes.SortOption{ - Field: pointers.String("Image"), + Field: new("Image"), }, }, }, @@ -184,12 +187,12 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "query + sort + limit", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc"), + Query: new("CVE:abc"), Pagination: &inputtypes.Pagination{ SortOption: &inputtypes.SortOption{ - Field: pointers.String("Image"), + Field: new("Image"), }, - Limit: pointers.Int32(10), + Limit: new(int32(10)), }, }, expectedQ: search.NewQueryBuilder().AddStrings(search.CVE, "abc").WithPagination( @@ -199,15 +202,15 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "query + sort + aggregate + limit", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc"), + Query: new("CVE:abc"), Pagination: &inputtypes.Pagination{ SortOption: &inputtypes.SortOption{ - Field: pointers.String("Image"), + Field: new("Image"), AggregateBy: &inputtypes.AggregateBy{ - AggregateFunc: pointers.String("count"), + AggregateFunc: new("count"), }, }, - Limit: pointers.Int32(10), + Limit: new(int32(10)), }, }, expectedQ: search.NewQueryBuilder().AddStrings(search.CVE, "abc").WithPagination( @@ -219,17 +222,17 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "query + primary sort + secondary sort + limit", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc"), + Query: new("CVE:abc"), Pagination: &inputtypes.Pagination{ SortOptions: &[]*inputtypes.SortOption{ { - Field: pointers.String("Image"), + Field: new("Image"), }, { - Field: pointers.String("Component"), + Field: new("Component"), }, }, - Limit: pointers.Int32(10), + Limit: new(int32(10)), }, }, expectedQ: search.NewQueryBuilder().AddStrings(search.CVE, "abc").WithPagination( @@ -242,20 +245,20 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "query + primary sort w/ aggregate + secondary sort + limit", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc"), + Query: new("CVE:abc"), Pagination: &inputtypes.Pagination{ SortOptions: &[]*inputtypes.SortOption{ { - Field: pointers.String("Image"), + Field: new("Image"), AggregateBy: &inputtypes.AggregateBy{ - AggregateFunc: pointers.String("count"), + AggregateFunc: new("count"), }, }, { - Field: pointers.String("Component"), + Field: new("Component"), }, }, - Limit: pointers.Int32(10), + Limit: new(int32(10)), }, }, expectedQ: search.NewQueryBuilder().AddStrings(search.CVE, "abc").WithPagination( @@ -268,20 +271,20 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "query + primary sort + secondary sort w/ aggregate + limit", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc"), + Query: new("CVE:abc"), Pagination: &inputtypes.Pagination{ SortOptions: &[]*inputtypes.SortOption{ { - Field: pointers.String("Image"), + Field: new("Image"), }, { - Field: pointers.String("Component"), + Field: new("Component"), AggregateBy: &inputtypes.AggregateBy{ - AggregateFunc: pointers.String("count"), + AggregateFunc: new("count"), }, }, }, - Limit: pointers.Int32(10), + Limit: new(int32(10)), }, }, expectedQ: search.NewQueryBuilder().AddStrings(search.CVE, "abc").WithPagination( @@ -294,7 +297,7 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "query + nil sort", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc"), + Query: new("CVE:abc"), Pagination: &inputtypes.Pagination{ SortOptions: &[]*inputtypes.SortOption{nil}, }, @@ -305,7 +308,7 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "query + empty sorts", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc"), + Query: new("CVE:abc"), Pagination: &inputtypes.Pagination{ SortOptions: &[]*inputtypes.SortOption{ {}, @@ -322,20 +325,20 @@ func TestAsV1QueryOrEmpty(t *testing.T) { { desc: "query + primary sort + secondary sort w/ invalid aggregate + limit", arg: PaginatedQuery{ - Query: pointers.String("CVE:abc"), + Query: new("CVE:abc"), Pagination: &inputtypes.Pagination{ SortOptions: &[]*inputtypes.SortOption{ { - Field: pointers.String("Image"), + Field: new("Image"), }, { - Field: pointers.String("Component"), + Field: new("Component"), AggregateBy: &inputtypes.AggregateBy{ - AggregateFunc: pointers.String("trinity"), + AggregateFunc: new("trinity"), }, }, }, - Limit: pointers.Int32(10), + Limit: new(int32(10)), }, }, expectedQ: search.NewQueryBuilder().AddStrings(search.CVE, "abc").WithPagination( @@ -440,17 +443,32 @@ func TestImageLabelAutoCompleteSearch(t *testing.T) { testGormDB := testDB.GetGormDB(t) defer pgtest.CloseGormDB(t, testGormDB) - imageDatastore := imageDS.GetTestPostgresDataStore(t, testDB.DB) ctx := loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) - resolver, _ := SetupTestResolver(t, imageDatastore, imagesView.NewImageView(testDB.DB)) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + var resolver *Resolver + if features.FlattenImageData.Enabled() { + imageV2Datastore := imageV2DS.GetTestPostgresDataStore(t, testDB.DB) + resolver, _ = SetupTestResolver(t, imageV2Datastore, imagesView.NewImageView(testDB.DB)) + } else { + imageDatastore := imageDS.GetTestPostgresDataStore(t, testDB.DB) + resolver, _ = SetupTestResolver(t, imageDatastore, imagesView.NewImageView(testDB.DB)) + } + allowAllCtx := SetAuthorizerOverride(ctx, allow.Anonymous()) + upsertImage := func(img *storage.Image) error { + if features.FlattenImageData.Enabled() { + return resolver.ImageV2DataStore.UpsertImage(ctx, imageUtils.ConvertToV2(img)) + } + return resolver.ImageDataStore.UpsertImage(ctx, img) + } + // Case: nil labels image := fixtures.GetImage() image.GetMetadata().GetV1().Labels = nil - require.NoError(t, imageDatastore.UpsertImage(ctx, image)) + require.NoError(t, upsertImage(image)) request := searchRequest{ Query: "Image Label:", @@ -466,7 +484,7 @@ func TestImageLabelAutoCompleteSearch(t *testing.T) { "k2": "v2", } - require.NoError(t, imageDatastore.UpsertImage(ctx, image)) + require.NoError(t, upsertImage(image)) request = searchRequest{ Query: "Image Label:", @@ -570,7 +588,7 @@ func TestSubjectGlobalSearch(t *testing.T) { } func getResultIDs(results []*searchResultResolver) []string { - var ids []string + ids := make([]string, 0, len(results)) for _, r := range results { ids = append(ids, r.data.GetId()) } diff --git a/central/graphql/resolvers/test_setup_utils.go b/central/graphql/resolvers/test_setup_utils.go index 56b485481dd8c..d34ad58c09ae7 100644 --- a/central/graphql/resolvers/test_setup_utils.go +++ b/central/graphql/resolvers/test_setup_utils.go @@ -140,8 +140,9 @@ func SetupTestResolver(t testing.TB, datastores ...interface{}) (*Resolver, *gra return resolver, schema } -// CreateTestImageV2Datastore creates image datastore for testing -func CreateTestImageV2Datastore(_ testing.TB, testDB *pgtest.TestPostgres, ctrl *gomock.Controller) imageDS.DataStore { +// CreateTestImageDatastore creates image datastore for testing +// TODO(ROX-30117): Remove when FlattenImageData feature flag is removed. +func CreateTestImageDatastore(_ testing.TB, testDB *pgtest.TestPostgres, ctrl *gomock.Controller) imageDS.DataStore { risks := mockRisks.NewMockDataStore(ctrl) risks.EXPECT().RemoveRisk(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() return imageDS.NewWithPostgres( @@ -152,8 +153,8 @@ func CreateTestImageV2Datastore(_ testing.TB, testDB *pgtest.TestPostgres, ctrl ) } -// CreateTestImageV2V2Datastore creates image datastore for testing -func CreateTestImageV2V2Datastore(_ testing.TB, testDB *pgtest.TestPostgres, ctrl *gomock.Controller) imageV2DS.DataStore { +// CreateTestImageV2Datastore creates imageV2 datastore for testing +func CreateTestImageV2Datastore(_ testing.TB, testDB *pgtest.TestPostgres, ctrl *gomock.Controller) imageV2DS.DataStore { risks := mockRisks.NewMockDataStore(ctrl) risks.EXPECT().RemoveRisk(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() return imageV2DS.NewWithPostgres( @@ -169,7 +170,7 @@ func CreateTestImageComponentV2Datastore(_ testing.TB, testDB *pgtest.TestPostgr mockRisk := mockRisks.NewMockDataStore(ctrl) storage := imageComponentV2Postgres.New(testDB.DB) - return imageComponentV2DS.New(storage, mockRisk, ranking.NewRanker()) + return imageComponentV2DS.New(testDB.DB, storage, mockRisk, ranking.NewRanker()) } // CreateTestImageCVEV2Datastore creates imageCVE datastore for testing @@ -181,6 +182,7 @@ func CreateTestImageCVEV2Datastore(_ testing.TB, testDB *pgtest.TestPostgres) im } // CreateTestDeploymentDatastore creates deployment datastore for testing +// TODO(ROX-30117): Remove when FlattenImageData feature flag is removed. func CreateTestDeploymentDatastore(t testing.TB, testDB *pgtest.TestPostgres, ctrl *gomock.Controller, imageDatastore imageDS.DataStore) deploymentDatastore.DataStore { mockRisk := mockRisks.NewMockDataStore(ctrl) ds, err := deploymentDatastore.NewTestDataStore( @@ -254,7 +256,8 @@ func CreateTestClusterDatastore(t testing.TB, testDB *pgtest.TestPostgres, ctrl connMgr.EXPECT().GetConnection(gomock.Any()).AnyTimes() datastore, err := clusterDataStore.New(storage, healthStorage, clusterCVEDS, nil, nil, namespaceDS, nil, nodeDataStore, nil, nil, - netFlows, netEntities, nil, nil, nil, connMgr, nil, ranking.ClusterRanker(), nil, nil) + netFlows, netEntities, nil, nil, nil, connMgr, nil, ranking.ClusterRanker(), nil, nil, nil, + nil, nil) assert.NoError(t, err, "failed to create cluster datastore") return datastore } @@ -293,55 +296,55 @@ func TestVulnReqDatastore(t testing.TB, testDB *pgtest.TestPostgres) (vulnReqDat } func registerImageLoader(_ testing.TB, ds imageDS.DataStore, view imagesView.ImageView) { - loaders.RegisterTypeFactory(reflect.TypeOf(storage.Image{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.Image](), func() interface{} { return loaders.NewImageLoader(ds, view) }) } func registerImageV2Loader(_ testing.TB, ds imageV2DS.DataStore, view imagesView.ImageView) { - loaders.RegisterTypeFactory(reflect.TypeOf(storage.ImageV2{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.ImageV2](), func() interface{} { return loaders.NewImageV2Loader(ds, view) }) } func registerImageComponentV2Loader(_ testing.TB, ds imageComponentV2DS.DataStore) { - loaders.RegisterTypeFactory(reflect.TypeOf(storage.ImageComponentV2{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.ImageComponentV2](), func() interface{} { return loaders.NewComponentV2Loader(ds) }) } func registerImageCveV2Loader(_ testing.TB, ds imageCVEV2DS.DataStore) { - loaders.RegisterTypeFactory(reflect.TypeOf(storage.ImageCVEV2{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.ImageCVEV2](), func() interface{} { return loaders.NewImageCVEV2Loader(ds) }) } func registerClusterCveLoader(_ testing.TB, ds clusterCVEDataStore.DataStore) { - loaders.RegisterTypeFactory(reflect.TypeOf(storage.ClusterCVE{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.ClusterCVE](), func() interface{} { return loaders.NewClusterCVELoader(ds) }) } func registerNodeLoader(_ testing.TB, ds nodeDS.DataStore) { - loaders.RegisterTypeFactory(reflect.TypeOf(storage.Node{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.Node](), func() interface{} { return loaders.NewNodeLoader(ds) }) } func registerNodeComponentLoader(_ testing.TB, ds nodeComponentDataStore.DataStore) { - loaders.RegisterTypeFactory(reflect.TypeOf(storage.NodeComponent{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.NodeComponent](), func() interface{} { return loaders.NewNodeComponentLoader(ds) }) } func registerNodeCVELoader(_ testing.TB, ds nodeCVEDataStore.DataStore) { - loaders.RegisterTypeFactory(reflect.TypeOf(storage.NodeCVE{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.NodeCVE](), func() interface{} { return loaders.NewNodeCVELoader(ds) }) } func registerDeploymentLoader(_ testing.TB, ds deploymentDatastore.DataStore, view deploymentsView.DeploymentView) { - loaders.RegisterTypeFactory(reflect.TypeOf(storage.Deployment{}), func() interface{} { + loaders.RegisterTypeFactory(reflect.TypeFor[storage.Deployment](), func() interface{} { return loaders.NewDeploymentLoader(ds, view) }) } diff --git a/central/graphql/resolvers/test_utils.go b/central/graphql/resolvers/test_utils.go index bb113742a1e0a..4211e85e0711c 100644 --- a/central/graphql/resolvers/test_utils.go +++ b/central/graphql/resolvers/test_utils.go @@ -96,6 +96,8 @@ func testImages() []*storage.Image { utils.CrashOnError(err) t2, err := protocompat.ConvertTimeToTimestampOrError(time.Unix(0, 2000)) utils.CrashOnError(err) + t3, err := protocompat.ConvertTimeToTimestampOrError(time.Unix(0, 3000)) + utils.CrashOnError(err) return []*storage.Image{ { Id: "sha1", @@ -233,13 +235,17 @@ func testImages() []*storage.Image { BaseImageInfo: []*storage.BaseImageInfo{ { BaseImageId: "base-sha2", - BaseImageFullName: "alpine:3.12", + BaseImageFullName: "busybox:latest", BaseImageDigest: "sha256:alpine312", + Created: t3, + MaxLayerIndex: 10, }, { BaseImageId: "base-sha3", - BaseImageFullName: "busybox:latest", + BaseImageFullName: "alpine:3.12", BaseImageDigest: "sha256:busybox1", + Created: t3, + MaxLayerIndex: 10, }, }, }, @@ -726,7 +732,7 @@ func getNodeVulnerabilityResolver(ctx context.Context, t *testing.T, resolver *R func getTestImages(imageCount int) []*storage.Image { images := make([]*storage.Image, 0, imageCount) - for i := 0; i < imageCount; i++ { + for i := range imageCount { img := fixtures.GetImageWithUniqueComponents(100) id := fmt.Sprintf("%d", i) img.Id = id @@ -735,6 +741,17 @@ func getTestImages(imageCount int) []*storage.Image { return images } +func getTestImagesV2(imageCount int) []*storage.ImageV2 { + images := make([]*storage.ImageV2, 0, imageCount) + for i := range imageCount { + img := fixtures.GetImageV2WithUniqueComponents(100) + img.Digest = fmt.Sprintf("%d", i) + img.Id = uuid.NewV5FromNonUUIDs(img.GetName().GetFullName(), img.GetDigest()).String() + images = append(images, img) + } + return images +} + func contextWithImagePerm(t testing.TB, ctrl *gomock.Controller) context.Context { id := mockIdentity.NewMockIdentity(ctrl) id.EXPECT().Permissions().Return(map[string]storage.Access{"Image": storage.Access_READ_ACCESS}).AnyTimes() @@ -743,7 +760,7 @@ func contextWithImagePerm(t testing.TB, ctrl *gomock.Controller) context.Context func getTestNodes(nodeCount int) []*storage.Node { nodes := make([]*storage.Node, 0, nodeCount) - for i := 0; i < nodeCount; i++ { + for range nodeCount { node := fixtures.GetNodeWithUniqueComponents(100, 5) nodeConverter.MoveNodeVulnsToNewField(node) id := uuid.NewV4().String() diff --git a/central/graphql/resolvers/test_utils_test.go b/central/graphql/resolvers/test_utils_test.go new file mode 100644 index 0000000000000..8313d6ffd1521 --- /dev/null +++ b/central/graphql/resolvers/test_utils_test.go @@ -0,0 +1,80 @@ +package resolvers + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func validateAlignedMethodIndex(t testing.TB, type1 reflect.Type, type2 reflect.Type) { + t.Helper() + + if !assert.Equalf(t, type1.NumMethod(), type2.NumMethod(), + "Types %s and %s should have the same number of methods", + type1.String(), type2.String(), + ) { + dumpTypeMethods(t, type1, type1.String(), false) + dumpTypeMethods(t, type2, type2.String(), false) + return + } + for i := range type1.NumMethod() { + m1 := type1.Method(i) + m2 := type2.Method(i) + assert.Equal(t, m1.Name, m2.Name) + } +} + +// dumpTypeMethods uses reflection to dump the index and names of methods for a given type. +// It accepts any resolver value and outputs method information to the test logger. +// +// Example usage: +// +// // After creating a resolver instance in a test: +// componentResolver, err := resolver.ImageComponent(ctx, struct{ ID *graphql.ID }{ID: &componentID}) +// require.NoError(t, err) +// dumpTypeMethods(t, reflect.TypeOf(componentResolver), "imageComponentV2Resolver") +// +// Output includes: +// - Index of each method (0-based) +// - Method name +// - Full method signature +// - Input parameter types +// - Output parameter types +func dumpTypeMethods(t testing.TB, typ reflect.Type, typeName string, verbose bool) { + t.Logf("\n=== Methods of %s ===", typeName) + t.Logf("Type: %s", typ.String()) + t.Logf("Kind: %s", typ.Kind()) + t.Logf("Number of methods: %d\n", typ.NumMethod()) + + for i := 0; i < typ.NumMethod(); i++ { + method := typ.Method(i) + t.Logf("[%d] %s", i, method.Name) + + if verbose { + // Also log the method signature + methodType := method.Type + t.Logf(" Signature: %s", methodType.String()) + + // Log input parameters + numIn := methodType.NumIn() + if numIn > 0 { + t.Logf(" Inputs (%d):", numIn) + for j := range numIn { + t.Logf(" [%d] %s", j, methodType.In(j).String()) + } + } + + // Log return parameters + numOut := methodType.NumOut() + if numOut > 0 { + t.Logf(" Outputs (%d):", numOut) + for j := range numOut { + t.Logf(" [%d] %s", j, methodType.Out(j).String()) + } + } + t.Logf("") + } + } + t.Logf("=== End of methods for %s ===\n", typeName) +} diff --git a/central/graphql/resolvers/utils.go b/central/graphql/resolvers/utils.go index fd6876f339bcf..566bc2c495e44 100644 --- a/central/graphql/resolvers/utils.go +++ b/central/graphql/resolvers/utils.go @@ -11,7 +11,6 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/grpc/authz/interceptor" "github.com/stackrox/rox/pkg/k8srbac" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/set" ) @@ -296,17 +295,17 @@ func paginate[T any](pv *v1.QueryPagination, slice []T, err error) ([]T, error) // V1RawQueryAsResolverQuery parse v1.RawQuery into inputtypes.RawQuery and inputtypes.Pagination queries used by graphQL resolvers. func V1RawQueryAsResolverQuery(rQ *v1.RawQuery) (RawQuery, PaginatedQuery) { if rQ.GetPagination() == nil { - return RawQuery{pointers.String(rQ.GetQuery()), nil}, PaginatedQuery{Query: pointers.String(rQ.GetQuery())} + return RawQuery{new(rQ.GetQuery()), nil}, PaginatedQuery{Query: new(rQ.GetQuery())} } - return RawQuery{pointers.String(rQ.GetQuery()), nil}, PaginatedQuery{ - Query: pointers.String(rQ.GetQuery()), + return RawQuery{new(rQ.GetQuery()), nil}, PaginatedQuery{ + Query: new(rQ.GetQuery()), Pagination: &inputtypes.Pagination{ - Limit: pointers.Int32(rQ.GetPagination().GetLimit()), - Offset: pointers.Int32(rQ.GetPagination().GetOffset()), + Limit: new(rQ.GetPagination().GetLimit()), + Offset: new(rQ.GetPagination().GetOffset()), SortOption: &inputtypes.SortOption{ - Field: pointers.String(rQ.GetPagination().GetSortOption().GetField()), - Reversed: pointers.Bool(rQ.GetPagination().GetSortOption().GetReversed()), + Field: new(rQ.GetPagination().GetSortOption().GetField()), + Reversed: new(rQ.GetPagination().GetSortOption().GetReversed()), }, }, } @@ -325,7 +324,7 @@ func logErrorOnQueryContainingField(query *v1.Query, label search.FieldLabel, re // FilterFieldFromRawQuery removes the given field from RawQuery func FilterFieldFromRawQuery(rq RawQuery, label search.FieldLabel) RawQuery { return RawQuery{ - Query: pointers.String(search.FilterFields(rq.String(), func(field string) bool { + Query: new(search.FilterFields(rq.String(), func(field string) bool { return label.String() != field })), ScopeQuery: rq.ScopeQuery, diff --git a/central/graphql/resolvers/vulnerabilities_v1_test.go b/central/graphql/resolvers/vulnerabilities_v1_test.go index 2f0be883eeaed..60d7f92e625e1 100644 --- a/central/graphql/resolvers/vulnerabilities_v1_test.go +++ b/central/graphql/resolvers/vulnerabilities_v1_test.go @@ -9,6 +9,7 @@ import ( "github.com/stackrox/rox/central/cve/converter/utils" "github.com/stackrox/rox/central/cve/matcher" imageMocks "github.com/stackrox/rox/central/image/datastore/mocks" + imageV2Mocks "github.com/stackrox/rox/central/imagev2/datastore/mocks" nsMocks "github.com/stackrox/rox/central/namespace/datastore/mocks" "github.com/stackrox/rox/generated/storage" "github.com/stretchr/testify/assert" @@ -71,7 +72,8 @@ func TestK8sCVEEnvImpact(t *testing.T) { clusterDataStore := clusterMocks.NewMockDataStore(ctrl) nsDataStore := nsMocks.NewMockDataStore(ctrl) imageDataStore := imageMocks.NewMockDataStore(ctrl) - cveMatcher, err := matcher.NewCVEMatcher(clusterDataStore, nsDataStore, imageDataStore) + imageV2DataStore := imageV2Mocks.NewMockDataStore(ctrl) + cveMatcher, err := matcher.NewCVEMatcher(clusterDataStore, nsDataStore, imageDataStore, imageV2DataStore) require.NoError(t, err) clusterDataStore.EXPECT().GetClusters(gomock.Any()).Return(clusters, nil).AnyTimes() diff --git a/central/hash/manager/metrics.go b/central/hash/manager/metrics.go index a783ef6731b58..937adace71116 100644 --- a/central/hash/manager/metrics.go +++ b/central/hash/manager/metrics.go @@ -14,13 +14,13 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "deduping_hash_size", - Help: "Number of hashes in the deduping hashes", + Help: "Number of persisted deduplication hashes for a cluster at last flush", }, []string{"cluster"}) dedupingHashCounterVec = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "deduping_hash_count", - Help: "Number of operations against the hashes by cluster, resource type and operation", + Help: "Counts add/remove operations on deduplication hashes by cluster and resource type", }, []string{"cluster", "ResourceType", "Operation"}) ) diff --git a/central/image/datastore/datastore.go b/central/image/datastore/datastore.go index 5d4fee9f0987e..b3aac9d1f2fff 100644 --- a/central/image/datastore/datastore.go +++ b/central/image/datastore/datastore.go @@ -29,6 +29,7 @@ type DataStore interface { GetManyImageMetadata(ctx context.Context, ids []string) ([]*storage.Image, error) GetImagesBatch(ctx context.Context, shas []string) ([]*storage.Image, error) WalkByQuery(ctx context.Context, q *v1.Query, fn func(image *storage.Image) error) error + WalkMetadataByQuery(ctx context.Context, q *v1.Query, fn func(image *storage.Image) error) error UpsertImage(ctx context.Context, image *storage.Image) error UpdateVulnerabilityState(ctx context.Context, cve string, images []string, state storage.VulnerabilityState) error diff --git a/central/image/datastore/datastore_bench_postgres_test.go b/central/image/datastore/datastore_bench_postgres_test.go index 123dfb8c5337c..90885b14cde8f 100644 --- a/central/image/datastore/datastore_bench_postgres_test.go +++ b/central/image/datastore/datastore_bench_postgres_test.go @@ -14,6 +14,7 @@ import ( "github.com/stackrox/rox/central/ranking" mockRisks "github.com/stackrox/rox/central/risk/datastore/mocks" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/sac" @@ -21,7 +22,11 @@ import ( "go.uber.org/mock/gomock" ) +// TODO(ROX-30117): Remove this benchmark when FlattenImageData feature flag is removed. func BenchmarkImageGetMany(b *testing.B) { + if features.FlattenImageData.Enabled() { + b.Skip("Skipping benchmark - FlattenImageData is enabled.") + } ctx := sac.WithAllAccess(context.Background()) testDB := pgtest.ForT(b) @@ -33,7 +38,7 @@ func BenchmarkImageGetMany(b *testing.B) { ids := make([]string, 0, 100) images := make([]*storage.Image, 0, 100) - for i := 0; i < 100; i++ { + for i := range 100 { img := fixtures.GetImageWithUniqueComponents(5) id := fmt.Sprintf("%d", i) ids = append(ids, id) @@ -60,7 +65,11 @@ func BenchmarkImageGetMany(b *testing.B) { }) } +// TODO(ROX-30117): Remove this benchmark when FlattenImageData feature flag is removed. func BenchmarkImageUpsert(b *testing.B) { + if features.FlattenImageData.Enabled() { + b.Skip("Skipping benchmark - FlattenImageData is enabled.") + } ctx := sac.WithAllAccess(context.Background()) testDB := pgtest.ForT(b) @@ -71,7 +80,7 @@ func BenchmarkImageUpsert(b *testing.B) { datastore := NewWithPostgres(pgStoreV2.New(db, false, keyfence.ImageKeyFenceSingleton()), mockRisk, ranking.NewRanker(), ranking.NewRanker()) images := make([]*storage.Image, 0, 100) - for i := 0; i < 100; i++ { + for range 100 { img := fixtures.GetImageWithUniqueComponents(50) data := make([]byte, 10) if _, err := rand.Read(data); err == nil { diff --git a/central/image/datastore/datastore_impl.go b/central/image/datastore/datastore_impl.go index 7bf178ff7f2c7..443f30ef90bdb 100644 --- a/central/image/datastore/datastore_impl.go +++ b/central/image/datastore/datastore_impl.go @@ -6,7 +6,6 @@ import ( "time" "github.com/pkg/errors" - "github.com/stackrox/rox/central/globaldb" "github.com/stackrox/rox/central/image/datastore/store" "github.com/stackrox/rox/central/image/views" "github.com/stackrox/rox/central/metrics" @@ -14,7 +13,6 @@ import ( riskDS "github.com/stackrox/rox/central/risk/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/errorhelpers" "github.com/stackrox/rox/pkg/images/enricher" imageTypes "github.com/stackrox/rox/pkg/images/types" @@ -32,8 +30,6 @@ var ( ) type datastoreImpl struct { - keyedMutex *concurrency.KeyedMutex - storage store.Store risks riskDS.DataStore @@ -51,8 +47,6 @@ func newDatastoreImpl(storage store.Store, risks riskDS.DataStore, imageRanker: imageRanker, imageComponentRanker: imageComponentRanker, - - keyedMutex: concurrency.NewKeyedMutex(globaldb.DefaultDataStorePoolSize), } return ds } @@ -81,7 +75,7 @@ func (ds *datastoreImpl) SearchImages(ctx context.Context, q *v1.Query) ([]*v1.S clonedQuery := q.CloneVT() clonedQuery.Selects = append(q.GetSelects(), pkgSearch.NewQuerySelect(pkgSearch.ImageName).Proto()) - results, err := ds.Search(ctx, q) + results, err := ds.Search(ctx, clonedQuery) if err != nil { return nil, err } @@ -155,6 +149,10 @@ func (ds *datastoreImpl) WalkByQuery(ctx context.Context, q *v1.Query, fn func(i return ds.storage.WalkByQuery(ctx, q, wrappedFn) } +func (ds *datastoreImpl) WalkMetadataByQuery(ctx context.Context, q *v1.Query, fn func(image *storage.Image) error) error { + return ds.storage.WalkMetadataByQuery(ctx, q, fn) +} + // CountImages delegates to the underlying store. func (ds *datastoreImpl) CountImages(ctx context.Context) (int, error) { if _, err := imagesSAC.ReadAllowed(ctx); err != nil { @@ -262,9 +260,6 @@ func (ds *datastoreImpl) UpsertImage(ctx context.Context, image *storage.Image) return sac.ErrResourceAccessDenied } - ds.keyedMutex.Lock(image.GetId()) - defer ds.keyedMutex.Unlock(image.GetId()) - ds.updateComponentRisk(image) enricher.FillScanStats(image) @@ -298,7 +293,7 @@ func (ds *datastoreImpl) DeleteImages(ctx context.Context, ids ...string) error return err } } - // removing component risk handled by pruning + // Component risks are not stored in the risk datastore (only embedded in components). return errorList.ToError() } @@ -386,7 +381,3 @@ func (c *ImageSearchResultConverter) BuildLocation(result *pkgSearch.Result) str func (c *ImageSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_IMAGES } - -func (c *ImageSearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/image/datastore/datastore_impl_flat_postgres_test.go b/central/image/datastore/datastore_impl_flat_postgres_test.go index 99e08af4a9c0e..fe544cfb1fb6e 100644 --- a/central/image/datastore/datastore_impl_flat_postgres_test.go +++ b/central/image/datastore/datastore_impl_flat_postgres_test.go @@ -7,7 +7,11 @@ import ( "fmt" "sort" "testing" + "time" + imageCVEInfoDS "github.com/stackrox/rox/central/cve/image/info/datastore" + imageCVEInfoPostgres "github.com/stackrox/rox/central/cve/image/info/datastore/store/postgres" + cveInfoEnricher "github.com/stackrox/rox/central/cve/image/info/enricher" imageCVEDS "github.com/stackrox/rox/central/cve/image/v2/datastore" imageCVEPostgres "github.com/stackrox/rox/central/cve/image/v2/datastore/store/postgres" "github.com/stackrox/rox/central/image/datastore/keyfence" @@ -19,7 +23,9 @@ import ( v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" pkgCVE "github.com/stackrox/rox/pkg/cve" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" + imageEnricher "github.com/stackrox/rox/pkg/images/enricher" imageTypes "github.com/stackrox/rox/pkg/images/types" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgtest" @@ -34,9 +40,14 @@ import ( "github.com/stackrox/rox/pkg/set" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" ) +// TODO(ROX-30117): Remove this test when FlattenImageData feature flag is removed. func TestImageFlatDataStoreWithPostgres(t *testing.T) { + if features.FlattenImageData.Enabled() { + t.Skip("Skipping test - FlattenImageData is enabled. Equivalent tests exist in imageV2 datastore.") + } suite.Run(t, new(ImageFlatPostgresDataStoreTestSuite)) } @@ -50,6 +61,8 @@ type ImageFlatPostgresDataStoreTestSuite struct { mockRisk *mockRisks.MockDataStore componentDataStore imageComponentDS.DataStore cveDataStore imageCVEDS.DataStore + cveInfoDataStore imageCVEInfoDS.DataStore + cveInfoEnricher imageEnricher.CVEInfoEnricher } func (s *ImageFlatPostgresDataStoreTestSuite) SetupSuite() { @@ -64,11 +77,15 @@ func (s *ImageFlatPostgresDataStoreTestSuite) SetupTest() { s.datastore = NewWithPostgres(dbStore, s.mockRisk, ranking.ImageRanker(), ranking.ComponentRanker()) componentStorage := imageComponentPostgres.New(s.db) - s.componentDataStore = imageComponentDS.New(componentStorage, s.mockRisk, ranking.NewRanker()) + s.componentDataStore = imageComponentDS.New(s.db, componentStorage, s.mockRisk, ranking.NewRanker()) cveStorage := imageCVEPostgres.New(s.db) cveDataStore := imageCVEDS.New(cveStorage) s.cveDataStore = cveDataStore + + cveInfoStorage := imageCVEInfoPostgres.New(s.db) + s.cveInfoDataStore = imageCVEInfoDS.New(cveInfoStorage) + s.cveInfoEnricher = cveInfoEnricher.New(s.cveInfoDataStore) } func (s *ImageFlatPostgresDataStoreTestSuite) TearDownTest() { @@ -76,6 +93,7 @@ func (s *ImageFlatPostgresDataStoreTestSuite) TearDownTest() { s.truncateTable(postgresSchema.ImagesTableName) s.truncateTable(postgresSchema.ImageComponentV2TableName) s.truncateTable(postgresSchema.ImageCvesV2TableName) + s.truncateTable(postgresSchema.ImageCveInfosTableName) } func (s *ImageFlatPostgresDataStoreTestSuite) TestSearchWithPostgres() { @@ -102,6 +120,7 @@ func (s *ImageFlatPostgresDataStoreTestSuite) TestSearchWithPostgres() { s.NoError(errRes) s.Len(searchRes, 1) s.Equal(imageTypes.NewDigest(image.GetId()).Digest(), searchRes[0].GetId()) + s.Equal(image.GetName().GetFullName(), searchRes[0].GetName()) // Sort by impact score q = pkgSearch.EmptyQuery() @@ -191,6 +210,23 @@ func (s *ImageFlatPostgresDataStoreTestSuite) TestSearchWithPostgres() { s.NoError(err) s.Len(results, 1) s.Equal(newImage.GetId(), results[0].ID) + + // Search by Dockerfile Instruction Keyword (from child table images_layers). + q = pkgSearch.NewQueryBuilder().AddStrings(pkgSearch.DockerfileInstructionKeyword, "RUN").ProtoQuery() + results, err = s.datastore.Search(ctx, q) + s.NoError(err) + s.Len(results, 2) + + // Count by Dockerfile Instruction Keyword. + count, err := s.datastore.Count(ctx, q) + s.NoError(err) + s.Equal(2, count) + + // Search by Dockerfile Instruction Value. + q = pkgSearch.NewQueryBuilder().AddStrings(pkgSearch.DockerfileInstructionValue, "apt-get").ProtoQuery() + results, err = s.datastore.Search(ctx, q) + s.NoError(err) + s.Len(results, 2) } func (s *ImageFlatPostgresDataStoreTestSuite) TestFixableWithPostgres() { @@ -348,9 +384,14 @@ func (s *ImageFlatPostgresDataStoreTestSuite) TestImageDeletes() { storedImage, found, err := s.datastore.GetImage(ctx, testImage.GetId()) s.NoError(err) s.True(found) - for _, component := range testImage.GetScan().GetComponents() { - for _, cve := range component.GetVulns() { - cve.FirstSystemOccurrence = storedImage.GetLastUpdated() + for compI, component := range testImage.GetScan().GetComponents() { + for cveI, cve := range component.GetVulns() { + if features.CVEFixTimestampCriteria.Enabled() { + cve.FirstSystemOccurrence = storedImage.GetScan().GetComponents()[compI].GetVulns()[cveI].GetFirstSystemOccurrence() + cve.FixAvailableTimestamp = storedImage.GetScan().GetComponents()[compI].GetVulns()[cveI].GetFixAvailableTimestamp() + } else { + cve.FirstSystemOccurrence = storedImage.GetLastUpdated() + } cve.FirstImageOccurrence = storedImage.GetLastUpdated() cve.VulnerabilityTypes = []storage.EmbeddedVulnerability_VulnerabilityType{storage.EmbeddedVulnerability_IMAGE_VULNERABILITY} } @@ -402,7 +443,7 @@ func (s *ImageFlatPostgresDataStoreTestSuite) TestImageDeletes() { s.True(found) for _, component := range testImage2.GetScan().GetComponents() { for _, cve := range component.GetVulns() { - // System Occurrence remains unchanged. + // System Occurrence and fix available times remain unchanged. cve.FirstImageOccurrence = storedImage.GetLastUpdated() cve.VulnerabilityTypes = []storage.EmbeddedVulnerability_VulnerabilityType{storage.EmbeddedVulnerability_IMAGE_VULNERABILITY} } @@ -436,10 +477,15 @@ func (s *ImageFlatPostgresDataStoreTestSuite) TestImageDeletes() { storedImage, found, err = s.datastore.GetImage(ctx, testImage2.GetId()) s.NoError(err) s.True(found) - for _, component := range testImage2.GetScan().GetComponents() { + for compI, component := range testImage2.GetScan().GetComponents() { // Components and Vulns are deduped, therefore, update testImage structure. - for _, cve := range component.GetVulns() { - cve.FirstSystemOccurrence = storedImage.GetLastUpdated() + for cveI, cve := range component.GetVulns() { + if features.CVEFixTimestampCriteria.Enabled() { + cve.FirstSystemOccurrence = storedImage.GetScan().GetComponents()[compI].GetVulns()[cveI].GetFirstSystemOccurrence() + cve.FixAvailableTimestamp = storedImage.GetScan().GetComponents()[compI].GetVulns()[cveI].GetFixAvailableTimestamp() + } else { + cve.FirstSystemOccurrence = storedImage.GetLastUpdated() + } cve.FirstImageOccurrence = storedImage.GetLastUpdated() } } @@ -537,6 +583,325 @@ func (s *ImageFlatPostgresDataStoreTestSuite) TestGetManyImageMetadata() { protoassert.ElementsMatch(s.T(), []*storage.Image{testImage1, testImage2, testImage3}, storedImages) } +func (s *ImageFlatPostgresDataStoreTestSuite) TestCVETimestampPersistence() { + s.T().Setenv(features.CVEFixTimestampCriteria.EnvVar(), "true") + if !features.CVEFixTimestampCriteria.Enabled() { + s.T().Skip("CVEFixTimestampCriteria feature must be enabled for this test") + } + + ctx := sac.WithAllAccess(context.Background()) + + // Scanner-provided timestamp for when the shared CVE was first discovered + cveDiscoverTimestamp := timestamppb.New(time.Now().Add(-24 * time.Hour)) + + sharedCVEID := "CVE-2024-1234" + datasource := "alpine:v3.18" + image0ID := "image0-sha" + + // Three images share a CVE but each also has unique CVEs. + // The shared CVE in the second image has an earlier FirstSystemOccurrence timestamp. + images := []*storage.Image{ + { + Id: image0ID, + Name: &storage.ImageName{ + FullName: "registry.io/image0:v0", + }, + Scan: &storage.ImageScan{ + OperatingSystem: "alpine", + ScanTime: timestamppb.Now(), + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "shared-component", + Version: "1.0.0", + Source: storage.SourceType_OS, + Vulns: []*storage.EmbeddedVulnerability{ + { + Cve: sharedCVEID, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, + FirstSystemOccurrence: timestamppb.Now(), + }, + { + Cve: "CVE-2024-1111", + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + FirstSystemOccurrence: timestamppb.Now(), + }, + }, + }, + }, + }, + }, + { + Id: "image1-sha", + Name: &storage.ImageName{ + FullName: "registry.io/image1:v1", + }, + Scan: &storage.ImageScan{ + OperatingSystem: "alpine", + ScanTime: timestamppb.Now(), + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "shared-component", + Version: "1.0.0", + Source: storage.SourceType_OS, + Vulns: []*storage.EmbeddedVulnerability{ + { + Cve: sharedCVEID, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, + FirstSystemOccurrence: cveDiscoverTimestamp, // Earlier time stamp + }, + { + Cve: "CVE-2024-5678", + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + FirstSystemOccurrence: timestamppb.Now(), + }, + }, + }, + }, + }, + }, + { + Id: "image2-sha", + Name: &storage.ImageName{ + FullName: "registry.io/image2:v2", + }, + Scan: &storage.ImageScan{ + OperatingSystem: "alpine", + ScanTime: timestamppb.Now(), + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "shared-component", + Version: "1.0.0", + Source: storage.SourceType_OS, + Vulns: []*storage.EmbeddedVulnerability{ + { + Cve: sharedCVEID, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, + FirstSystemOccurrence: timestamppb.Now(), + }, + { + Cve: "CVE-2024-9999", + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + FirstSystemOccurrence: timestamppb.Now(), + }, + }, + }, + }, + }, + }, + } + + // First pass: process all images in order. + // With the new CVE-name-based aggregation, images processed after image1 will be enriched + // with the earliest timestamp across ALL records for that CVE name. + for _, img := range images { + imgClone := img.CloneVT() + s.NoError(s.cveInfoEnricher.EnrichImageWithCVEInfo(ctx, imgClone)) + s.NoError(s.datastore.UpsertImage(ctx, imgClone)) + } + + // Verify ImageCVEInfo persisted the earliest timestamp for the specific composite ID + cveInfoID := pkgCVE.ImageCVEInfoID(sharedCVEID, "shared-component", datasource) + cveInfo, found, err := s.cveInfoDataStore.Get(ctx, cveInfoID) + s.NoError(err) + s.True(found) + s.Equal(cveDiscoverTimestamp, cveInfo.GetFirstSystemOccurrence()) + + // Verify first pass results: + // With CVE-name-based aggregation: + // - image0: shared CVE should NOT have cveDiscoverTimestamp (processed before image1) + // - image1: shared CVE has cveDiscoverTimestamp (source of early timestamp) + // - image2: shared CVE should have cveDiscoverTimestamp (enriched from MIN across all CVE-2024-1234 records) + // - All unique CVEs should have their own timestamps + for i, img := range images { + stored, found, err := s.datastore.GetImage(ctx, img.GetId()) + s.NoError(err) + s.True(found) + + components := stored.GetScan().GetComponents() + s.Require().Len(components, 1) + s.Require().Len(components[0].GetVulns(), 2) + + for _, vuln := range components[0].GetVulns() { + if vuln.GetCve() != sharedCVEID { + s.NotEqual(cveDiscoverTimestamp, vuln.GetFirstSystemOccurrence(), + "Unique CVE should not have the shared timestamp (image %d)", i) + } else if img.GetId() == image0ID { + s.NotEqual(cveDiscoverTimestamp, vuln.GetFirstSystemOccurrence(), + "image0 was processed before image1, should not have the earlier timestamp yet") + } else { + s.Equal(cveDiscoverTimestamp, vuln.GetFirstSystemOccurrence(), + "Shared CVE should have the earlier timestamp from image1 (image %d)", i) + } + } + } + + // Second pass: rescan all images. + // All images should now have the preserved earliest timestamp for the shared CVE, + // since the enricher queries by CVE name and returns the MIN across all records. + for _, img := range images { + imgClone := img.CloneVT() + imgClone.GetScan().ScanTime = timestamppb.Now() + s.NoError(s.cveInfoEnricher.EnrichImageWithCVEInfo(ctx, imgClone)) + s.NoError(s.datastore.UpsertImage(ctx, imgClone)) + } + + // Verify all images now have the preserved earliest timestamp + // The CVE-name-based aggregation ensures all images get the MIN timestamp for CVE-2024-1234 + for i, img := range images { + stored, found, err := s.datastore.GetImage(ctx, img.GetId()) + s.NoError(err) + s.True(found) + + components := stored.GetScan().GetComponents() + s.Require().Len(components, 1) + s.Require().Len(components[0].GetVulns(), 2) + + for _, vuln := range components[0].GetVulns() { + if vuln.GetCve() == sharedCVEID { + s.Equal(cveDiscoverTimestamp, vuln.GetFirstSystemOccurrence(), + "After second pass, all images should have the preserved earliest timestamp for shared CVE (image %d)", i) + } else { + s.NotEqual(cveDiscoverTimestamp, vuln.GetFirstSystemOccurrence(), + "Unique CVE should not have the shared timestamp (image %d)", i) + } + } + } +} + +func (s *ImageFlatPostgresDataStoreTestSuite) TestCVETimestampAggregation() { + s.T().Setenv(features.CVEFixTimestampCriteria.EnvVar(), "true") + if !features.CVEFixTimestampCriteria.Enabled() { + s.T().Skip("CVEFixTimestampCriteria feature must be enabled for this test") + } + + ctx := sac.WithAllAccess(context.Background()) + + // Two images with the same CVE name but different components and operating systems + sharedCVEName := "CVE-2024-SHARED" + earlierTimestamp := timestamppb.New(time.Now().Add(-48 * time.Hour)) + laterTimestamp := timestamppb.New(time.Now().Add(-24 * time.Hour)) + + // Image 1: Ubuntu with package-a, has the earlier timestamp + image1 := &storage.Image{ + Id: "image1-sha", + Name: &storage.ImageName{ + FullName: "registry.io/ubuntu-app:v1", + }, + Scan: &storage.ImageScan{ + OperatingSystem: "ubuntu", + ScanTime: timestamppb.Now(), + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "package-a", + Version: "1.0.0", + Source: storage.SourceType_OS, + Vulns: []*storage.EmbeddedVulnerability{ + { + Cve: sharedCVEName, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: "ubuntu-updater::ubuntu:20.04", + Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, + FirstSystemOccurrence: earlierTimestamp, + }, + }, + }, + }, + }, + } + + // Image 2: Alpine with package-b (different component and OS), has the later timestamp + image2 := &storage.Image{ + Id: "image2-sha", + Name: &storage.ImageName{ + FullName: "registry.io/alpine-app:v1", + }, + Scan: &storage.ImageScan{ + OperatingSystem: "alpine", + ScanTime: timestamppb.Now(), + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "package-b", + Version: "2.0.0", + Source: storage.SourceType_OS, + Vulns: []*storage.EmbeddedVulnerability{ + { + Cve: sharedCVEName, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: "alpine-updater::alpine:3.18", + Severity: storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, + FirstSystemOccurrence: laterTimestamp, + }, + }, + }, + }, + }, + } + + // Process image1 first (has earlier timestamp) + img1Clone := image1.CloneVT() + s.NoError(s.cveInfoEnricher.EnrichImageWithCVEInfo(ctx, img1Clone)) + s.NoError(s.datastore.UpsertImage(ctx, img1Clone)) + + // Process image2 (has later timestamp) + img2Clone := image2.CloneVT() + s.NoError(s.cveInfoEnricher.EnrichImageWithCVEInfo(ctx, img2Clone)) + s.NoError(s.datastore.UpsertImage(ctx, img2Clone)) + + // Verify that two separate ImageCVEInfo records exist (different composite IDs) + cveInfo1ID := pkgCVE.ImageCVEInfoID(sharedCVEName, "package-a", "ubuntu-updater::ubuntu:20.04") + cveInfo2ID := pkgCVE.ImageCVEInfoID(sharedCVEName, "package-b", "alpine-updater::alpine:3.18") + + cveInfo1, found, err := s.cveInfoDataStore.Get(ctx, cveInfo1ID) + s.NoError(err) + s.True(found, "Should have ImageCVEInfo for image1's composite ID") + s.Equal(earlierTimestamp, cveInfo1.GetFirstSystemOccurrence()) + + cveInfo2, found, err := s.cveInfoDataStore.Get(ctx, cveInfo2ID) + s.NoError(err) + s.True(found, "Should have ImageCVEInfo for image2's composite ID") + s.Equal(laterTimestamp, cveInfo2.GetFirstSystemOccurrence()) + + // Verify both images have the MIN timestamp (from image1) after enrichment + storedImage1, found, err := s.datastore.GetImage(ctx, "image1-sha") + s.NoError(err) + s.True(found) + s.Require().Len(storedImage1.GetScan().GetComponents(), 1) + s.Require().Len(storedImage1.GetScan().GetComponents()[0].GetVulns(), 1) + s.Equal(earlierTimestamp, storedImage1.GetScan().GetComponents()[0].GetVulns()[0].GetFirstSystemOccurrence(), + "Image1 should have the earlier timestamp") + + storedImage2, found, err := s.datastore.GetImage(ctx, "image2-sha") + s.NoError(err) + s.True(found) + s.Require().Len(storedImage2.GetScan().GetComponents(), 1) + s.Require().Len(storedImage2.GetScan().GetComponents()[0].GetVulns(), 1) + s.Equal(earlierTimestamp, storedImage2.GetScan().GetComponents()[0].GetVulns()[0].GetFirstSystemOccurrence(), + "Image2 should also have the earlier timestamp from MIN aggregation across all CVE records") + + // Rescan image1 to verify the MIN timestamp is still preserved + img1Rescan := image1.CloneVT() + img1Rescan.GetScan().ScanTime = timestamppb.Now() + s.NoError(s.cveInfoEnricher.EnrichImageWithCVEInfo(ctx, img1Rescan)) + s.NoError(s.datastore.UpsertImage(ctx, img1Rescan)) + + storedImage1After, found, err := s.datastore.GetImage(ctx, "image1-sha") + s.NoError(err) + s.True(found) + s.Require().Len(storedImage1After.GetScan().GetComponents(), 1) + s.Require().Len(storedImage1After.GetScan().GetComponents()[0].GetVulns(), 1) + s.Equal(earlierTimestamp, storedImage1After.GetScan().GetComponents()[0].GetVulns()[0].GetFirstSystemOccurrence(), + "After rescan, image1 should still have the MIN timestamp") +} + func (s *ImageFlatPostgresDataStoreTestSuite) truncateTable(name string) { sql := fmt.Sprintf("TRUNCATE %s CASCADE", name) _, err := s.testDB.Exec(s.ctx, sql) @@ -610,16 +975,66 @@ func getTestImage(id string) *storage.Image { }, }, }, + Metadata: &storage.ImageMetadata{ + V1: &storage.V1Metadata{ + Layers: []*storage.ImageLayer{ + {Instruction: "ADD", Value: "file:abc123 in /"}, + {Instruction: "RUN", Value: "apt-get update && apt-get install -y curl"}, + }, + }, + }, RiskScore: 30, Priority: 1, } } +// TestComponentRiskScorePersistedToDB verifies that when the component ranker +// contains a risk score for a component, UpsertImage writes that score to the +// database. This is a regression test for the bug where the ranker was not +// being updated after risk calculation, causing updateComponentRisk to +// overwrite calculated scores with 0. +func (s *ImageFlatPostgresDataStoreTestSuite) TestComponentRiskScorePersistedToDB() { + ctx := sac.WithAllAccess(context.Background()) + + componentRanker := ranking.NewRanker() + ds := NewWithPostgres( + pgStoreV2.New(s.db, false, keyfence.ImageKeyFenceSingleton()), + s.mockRisk, + ranking.NewRanker(), + componentRanker, + ) + + img := fixtures.GetImageWithUniqueComponents(3) + + // Pre-populate the ranker with known scores for each component, + // simulating what the risk manager does after calculating risk. + expectedScores := []float32{1.5, 2.7, 4.2} + for index, component := range img.GetScan().GetComponents() { + componentID := scancomponent.ComponentIDV2(component, img.GetId(), index) + componentRanker.Add(componentID, expectedScores[index]) + } + + s.NoError(ds.UpsertImage(ctx, img)) + + // Read back from DB — the risk scores should match what was in the ranker. + got, found, err := ds.GetImage(ctx, img.GetId()) + s.NoError(err) + s.True(found) + + gotComponents := got.GetScan().GetComponents() + s.Require().Len(gotComponents, 3) + for i, component := range gotComponents { + s.InDelta(expectedScores[i], component.GetRiskScore(), 0.001, + "component %q risk score should be persisted to DB", component.GetName()) + } +} + func cloneAndUpdateRiskPriority(image *storage.Image) *storage.Image { cloned := image.CloneVT() cloned.Priority = 1 for _, component := range cloned.GetScan().GetComponents() { component.Priority = 1 } + cloned.LastUpdated = image.GetLastUpdated() return cloned } diff --git a/central/image/datastore/fixable_flat_test.go b/central/image/datastore/fixable_flat_test.go index 3d9e8d86f62f1..5fcc41c39ad16 100644 --- a/central/image/datastore/fixable_flat_test.go +++ b/central/image/datastore/fixable_flat_test.go @@ -11,6 +11,7 @@ import ( v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/cve" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/search" @@ -23,7 +24,11 @@ type normalizedImageComponent struct { version string } +// TODO(ROX-30117): Remove this test when FlattenImageData feature flag is removed. func TestFixableFlatSearch(t *testing.T) { + if features.FlattenImageData.Enabled() { + t.Skip("Skipping test - FlattenImageData is enabled. Equivalent tests exist in imageV2 datastore.") + } suite.Run(t, new(FixableFlatSearchTestSuite)) } diff --git a/central/image/datastore/mocks/datastore.go b/central/image/datastore/mocks/datastore.go index fe28488fb626e..90e0fe6f61efb 100644 --- a/central/image/datastore/mocks/datastore.go +++ b/central/image/datastore/mocks/datastore.go @@ -286,3 +286,17 @@ func (mr *MockDataStoreMockRecorder) WalkByQuery(ctx, q, fn any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WalkByQuery", reflect.TypeOf((*MockDataStore)(nil).WalkByQuery), ctx, q, fn) } + +// WalkMetadataByQuery mocks base method. +func (m *MockDataStore) WalkMetadataByQuery(ctx context.Context, q *v1.Query, fn func(*storage.Image) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WalkMetadataByQuery", ctx, q, fn) + ret0, _ := ret[0].(error) + return ret0 +} + +// WalkMetadataByQuery indicates an expected call of WalkMetadataByQuery. +func (mr *MockDataStoreMockRecorder) WalkMetadataByQuery(ctx, q, fn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WalkMetadataByQuery", reflect.TypeOf((*MockDataStore)(nil).WalkMetadataByQuery), ctx, q, fn) +} diff --git a/central/image/datastore/store/common/v2/parts_test.go b/central/image/datastore/store/common/v2/parts_test.go index 692c241aa0de5..47f086799e629 100644 --- a/central/image/datastore/store/common/v2/parts_test.go +++ b/central/image/datastore/store/common/v2/parts_test.go @@ -5,6 +5,7 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/cve" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/scancomponent" @@ -152,7 +153,11 @@ var ( } ) +// TODO(ROX-30117): Remove this test when FlattenImageData feature flag is removed. func TestSplitAndMergeImage(t *testing.T) { + if features.FlattenImageData.Enabled() { + t.Skip("Skipping test - FlattenImageData is enabled") + } image := &storage.Image{ Id: "sha", Name: &storage.ImageName{ @@ -162,11 +167,13 @@ func TestSplitAndMergeImage(t *testing.T) { BaseImageId: "some-id", BaseImageFullName: "registry.example.com/ns/base:tag", BaseImageDigest: "sha256:...", + MaxLayerIndex: 3, }, { BaseImageId: "another-id", BaseImageFullName: "registry.example.com/ns/other:tag", BaseImageDigest: "sha256:...", + MaxLayerIndex: 3, }, }, Metadata: &storage.ImageMetadata{ @@ -302,11 +309,13 @@ func TestSplitAndMergeImage(t *testing.T) { BaseImageId: "some-id", BaseImageFullName: "registry.example.com/ns/base:tag", BaseImageDigest: "sha256:...", + MaxLayerIndex: 3, }, { BaseImageId: "another-id", BaseImageFullName: "registry.example.com/ns/other:tag", BaseImageDigest: "sha256:...", + MaxLayerIndex: 3, }, }, Scan: &storage.ImageScan{ @@ -332,7 +341,7 @@ func TestSplitAndMergeImage(t *testing.T) { HasLayerIndex: &storage.ImageComponentV2_LayerIndex{ LayerIndex: 1, }, - FromBaseImage: true, + LayerType: storage.LayerType_BASE_IMAGE, }, Children: []CVEParts{}, }, @@ -345,7 +354,7 @@ func TestSplitAndMergeImage(t *testing.T) { HasLayerIndex: &storage.ImageComponentV2_LayerIndex{ LayerIndex: 3, }, - FromBaseImage: true, + LayerType: storage.LayerType_BASE_IMAGE, }, Children: []CVEParts{ { @@ -406,7 +415,7 @@ func TestSplitAndMergeImage(t *testing.T) { HasLayerIndex: &storage.ImageComponentV2_LayerIndex{ LayerIndex: 2, }, - FromBaseImage: true, + LayerType: storage.LayerType_BASE_IMAGE, }, Children: []CVEParts{ { @@ -450,7 +459,7 @@ func TestSplitAndMergeImage(t *testing.T) { HasLayerIndex: &storage.ImageComponentV2_LayerIndex{ LayerIndex: 2, }, - FromBaseImage: true, + LayerType: storage.LayerType_BASE_IMAGE, }, Children: []CVEParts{ { diff --git a/central/image/datastore/store/common/v2/split_v2.go b/central/image/datastore/store/common/v2/split_v2.go index 19a92a7c43b4b..1489bf96cc346 100644 --- a/central/image/datastore/store/common/v2/split_v2.go +++ b/central/image/datastore/store/common/v2/split_v2.go @@ -96,7 +96,12 @@ func GenerateImageComponentV2(os string, image *storage.Image, index int, from * LayerIndex: from.GetLayerIndex(), } } - // TODO ROX-31847 compute image component base image layer type - ret.FromBaseImage = len(image.GetBaseImageInfo()) > 0 + + ret.LayerType = storage.LayerType_APPLICATION + if len(image.GetBaseImageInfo()) > 0 && from.GetHasLayerIndex() != nil { + if from.GetLayerIndex() <= image.GetBaseImageInfo()[0].GetMaxLayerIndex() { + ret.LayerType = storage.LayerType_BASE_IMAGE + } + } return ret, nil } diff --git a/central/image/datastore/store/v2/postgres/gen.go b/central/image/datastore/store/v2/postgres/gen.go index 8fb781f3ebc24..da6c0c0b55e3f 100644 --- a/central/image/datastore/store/v2/postgres/gen.go +++ b/central/image/datastore/store/v2/postgres/gen.go @@ -1,3 +1,3 @@ package postgres -//go:generate pg-table-bindings-wrapper --type=storage.Image --search-category IMAGES --schema-only --search-scope IMAGE_VULNERABILITIES_V2,IMAGE_COMPONENTS_V2,DEPLOYMENTS,NAMESPACES,CLUSTERS +//go:generate pg-table-bindings-wrapper --type=storage.Image --search-category IMAGES --schema-only --search-scope IMAGE_VULNERABILITIES_V2,IMAGE_COMPONENTS_V2,IMAGES,DEPLOYMENTS,NAMESPACES,CLUSTERS diff --git a/central/image/datastore/store/v2/postgres/store.go b/central/image/datastore/store/v2/postgres/store.go index 32525eed5e387..d9abd944a4c44 100644 --- a/central/image/datastore/store/v2/postgres/store.go +++ b/central/image/datastore/store/v2/postgres/store.go @@ -16,7 +16,9 @@ import ( "github.com/stackrox/rox/central/metrics" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/baseimage" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/logging" ops "github.com/stackrox/rox/pkg/metrics" "github.com/stackrox/rox/pkg/postgres" @@ -252,10 +254,7 @@ func (s *storeImpl) copyFromImageComponentsV2(ctx context.Context, tx *postgres. if err := s.deleteImageComponents(ctx, tx, imageID); err != nil { return err } - batchSize := pgSearch.MaxBatchSize - if len(objs) < batchSize { - batchSize = len(objs) - } + batchSize := min(len(objs), pgSearch.MaxBatchSize) inputRows := make([][]interface{}, 0, batchSize) copyCols := []string{ @@ -269,6 +268,7 @@ func (s *storeImpl) copyFromImageComponentsV2(ctx context.Context, tx *postgres. "operatingsystem", "imageid", "location", + "layertype", "serialized", } @@ -289,6 +289,7 @@ func (s *storeImpl) copyFromImageComponentsV2(ctx context.Context, tx *postgres. obj.GetOperatingSystem(), obj.GetImageId(), obj.GetLocation(), + obj.GetLayerType(), serialized, }) @@ -306,10 +307,7 @@ func (s *storeImpl) copyFromImageComponentsV2(ctx context.Context, tx *postgres. } func copyFromImageComponentV2Cves(ctx context.Context, tx *postgres.Tx, iTime time.Time, cveTimeMap map[string]*timeFields, objs ...*storage.ImageCVEV2) error { - batchSize := pgSearch.MaxBatchSize - if len(objs) < batchSize { - batchSize = len(objs) - } + batchSize := min(len(objs), pgSearch.MaxBatchSize) inputRows := make([][]interface{}, 0, batchSize) copyCols := []string{ @@ -330,6 +328,8 @@ func copyFromImageComponentV2Cves(ctx context.Context, tx *postgres.Tx, iTime ti "componentid", "advisory_name", "advisory_link", + "fixavailabletimestamp", + "origin", "serialized", } @@ -370,6 +370,8 @@ func copyFromImageComponentV2Cves(ctx context.Context, tx *postgres.Tx, iTime ti obj.GetComponentId(), obj.GetAdvisory().GetName(), obj.GetAdvisory().GetLink(), + protocompat.NilOrTime(obj.GetFixAvailableTimestamp()), + obj.GetOrigin(), serialized, }) @@ -421,10 +423,17 @@ func (s *storeImpl) isUpdated(oldImage, image *storage.Image) (bool, bool, error type hashWrapper struct { Components []*storage.EmbeddedImageScanComponent `hash:"set"` + Count int + VulnCount int } func populateImageScanHash(scan *storage.ImageScan) error { - hash, err := hashstructure.Hash(hashWrapper{scan.GetComponents()}, &hashstructure.HashOptions{ZeroNil: true}) + components := scan.GetComponents() + vulnCount := 0 + for _, c := range components { + vulnCount += len(c.GetVulns()) + } + hash, err := hashstructure.Hash(hashWrapper{Components: components, Count: len(components), VulnCount: vulnCount}, &hashstructure.HashOptions{ZeroNil: true}) if err != nil { return errors.Wrap(err, "calculating hash for image scan") } @@ -471,6 +480,13 @@ func (s *storeImpl) upsert(ctx context.Context, obj *storage.Image) error { scanUpdated = scanUpdated || componentsEmpty + if features.BaseImageDetection.Enabled() { + // Re-verify base images when base image detection is enabled: + // 1. Legacy images may lack base image info if the feature was enabled after they were scanned. + // 2. User-provided base images may change over time. + scanUpdated = scanUpdated || baseimage.BaseImagesUpdated(oldImage.GetBaseImageInfo(), obj.GetBaseImageInfo()) + } + splitParts, err := common.SplitV2(obj, scanUpdated) if err != nil { return err @@ -568,29 +584,25 @@ func (s *storeImpl) populateImage(ctx context.Context, tx *postgres.Tx, image *s return err } + cvesByComponent, err := getAllImageComponentCVEs(ctx, tx, image.GetId()) + if err != nil { + return err + } + imageParts := common.ImageParts{ Image: image, - Children: []common.ComponentParts{}, + Children: make([]common.ComponentParts, 0, len(components)), } for _, component := range components { - cves, err := getImageComponentCVEs(ctx, tx, component.GetId()) - if err != nil { - return err - } - + cves := cvesByComponent[component.GetId()] cveParts := make([]common.CVEParts, 0, len(cves)) for _, cve := range cves { - cvePart := common.CVEParts{ - CVEV2: cve, - } - cveParts = append(cveParts, cvePart) + cveParts = append(cveParts, common.CVEParts{CVEV2: cve}) } - - child := common.ComponentParts{ + imageParts.Children = append(imageParts.Children, common.ComponentParts{ ComponentV2: component, Children: cveParts, - } - imageParts.Children = append(imageParts.Children, child) + }) } common.MergeV2(imageParts) return nil @@ -630,16 +642,24 @@ func getImageComponents(ctx context.Context, tx *postgres.Tx, imageID string) ([ return pgutils.ScanRows[storage.ImageComponentV2, *storage.ImageComponentV2](rows) } -func getImageComponentCVEs(ctx context.Context, tx *postgres.Tx, componentID string) ([]*storage.ImageCVEV2, error) { +func getAllImageComponentCVEs(ctx context.Context, tx *postgres.Tx, imageID string) (map[string][]*storage.ImageCVEV2, error) { defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Get, "ImageCVEsV2") - // Using this method instead of accessing the component store to ensure the query is in the same transaction as - // the updates. That may prove to not matter, but for now doing it this way. - rows, err := tx.Query(ctx, "SELECT serialized FROM "+imageComponentsV2CVEsTable+" WHERE componentid = $1", componentID) + rows, err := tx.Query(ctx, "SELECT serialized FROM "+imageComponentsV2CVEsTable+" WHERE imageid = $1", imageID) if err != nil { return nil, err } - return pgutils.ScanRows[storage.ImageCVEV2, *storage.ImageCVEV2](rows) + cves, err := pgutils.ScanRows[storage.ImageCVEV2, *storage.ImageCVEV2](rows) + if err != nil { + return nil, err + } + + result := make(map[string][]*storage.ImageCVEV2) + for _, cve := range cves { + compID := cve.GetComponentId() + result[compID] = append(result[compID], cve) + } + return result, nil } func getImageCVEs(ctx context.Context, tx *postgres.Tx, imageID string) ([]*storage.EmbeddedVulnerability, error) { @@ -839,7 +859,7 @@ func (s *storeImpl) WalkByQuery(ctx context.Context, q *v1.Query, fn func(image } return nil } - err = pgSearch.RunCursorQueryForSchemaFn(ctx, pkgSchema.ImagesSchema, q, s.db, callback) + err = pgSearch.RunCursorQueryForSchemaFn(ctx, pkgSchema.ImagesSchema, q, s.db, "WalkByQuery", callback) if err != nil { return errors.Wrap(err, "cursor by query") } @@ -851,7 +871,7 @@ func (s *storeImpl) WalkMetadataByQuery(ctx context.Context, q *v1.Query, fn fun q = applyDefaultSort(q) - err := pgSearch.RunCursorQueryForSchemaFn(ctx, pkgSchema.ImagesSchema, q, s.db, fn) + err := pgSearch.RunCursorQueryForSchemaFn(ctx, pkgSchema.ImagesSchema, q, s.db, "WalkMetadataByQuery", fn) if err != nil { return errors.Wrap(err, "cursor by query") } @@ -892,8 +912,11 @@ func (s *storeImpl) retryableGetManyImageMetadata(ctx context.Context, ids []str // GetImagesRiskView retrieves an image id and risk score to initialize rankers func (s *storeImpl) GetImagesRiskView(ctx context.Context, q *v1.Query) ([]*views.ImageRiskView, error) { // The entire image is not needed to initialize the ranker. We only need the image id and risk score. - var results []*views.ImageRiskView - results, err := pgSearch.RunSelectRequestForSchema[views.ImageRiskView](ctx, s.db, pkgSchema.ImagesSchema, q) + results := make([]*views.ImageRiskView, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[views.ImageRiskView](ctx, s.db, pkgSchema.ImagesSchema, q, func(r *views.ImageRiskView) error { + results = append(results, r) + return nil + }) if err != nil { log.Errorf("unable to initialize image ranking: %v", err) } @@ -995,10 +1018,13 @@ func (s *storeImpl) insertIntoImageComponentV2Cves(batch *pgx.Batch, obj *storag obj.GetFixedBy(), obj.GetComponentId(), obj.GetAdvisory().GetName(), + obj.GetAdvisory().GetLink(), + protocompat.NilOrTime(obj.GetFixAvailableTimestamp()), + obj.GetOrigin(), serialized, } - finalStr := "INSERT INTO image_cves_v2 (Id, ImageId, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, Cvss, Severity, ImpactScore, Nvdcvss, FirstImageOccurrence, State, IsFixable, FixedBy, ComponentId, advisory_name, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16,$17) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, ImageId = EXCLUDED.ImageId, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, Cvss = EXCLUDED.Cvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Nvdcvss = EXCLUDED.Nvdcvss, FirstImageOccurrence = EXCLUDED.FirstImageOccurrence, State = EXCLUDED.State, IsFixable = EXCLUDED.IsFixable, FixedBy = EXCLUDED.FixedBy, ComponentId = EXCLUDED.ComponentId, advisory_name = EXCLUDED.advisory_name, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO image_cves_v2 (Id, ImageId, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, Cvss, Severity, ImpactScore, Nvdcvss, FirstImageOccurrence, State, IsFixable, FixedBy, ComponentId, advisory_name, advisory_link, FixAvailableTimestamp, Origin, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, ImageId = EXCLUDED.ImageId, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, Cvss = EXCLUDED.Cvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Nvdcvss = EXCLUDED.Nvdcvss, FirstImageOccurrence = EXCLUDED.FirstImageOccurrence, State = EXCLUDED.State, IsFixable = EXCLUDED.IsFixable, FixedBy = EXCLUDED.FixedBy, ComponentId = EXCLUDED.ComponentId, advisory_name = EXCLUDED.advisory_name, advisory_link = EXCLUDED.advisory_link, FixAvailableTimestamp = EXCLUDED.FixAvailableTimestamp, Origin = EXCLUDED.Origin, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) return nil diff --git a/central/image/datastore/store/v2/postgres/store_bench_test.go b/central/image/datastore/store/v2/postgres/store_bench_test.go index f20a44306e7ab..1844d1b56f561 100644 --- a/central/image/datastore/store/v2/postgres/store_bench_test.go +++ b/central/image/datastore/store/v2/postgres/store_bench_test.go @@ -9,6 +9,7 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/sac" @@ -16,8 +17,77 @@ import ( "github.com/stretchr/testify/require" ) +// BenchmarkRealisticPopulateImage benchmarks image retrieval with realistic component/CVE counts. +// 50 images x 100 components x 10 CVEs = 50,000 CVEs across 5,000 components. +func BenchmarkRealisticPopulateImage(b *testing.B) { + if features.FlattenImageData.Enabled() { + b.Skip("Skipping benchmark - FlattenImageData is enabled.") + } + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(b) + + store := New(testDB.DB, false, concurrency.NewKeyFence()) + + numImages := 50 + numComponents := 100 + numCVEs := 10 + images := make([]*storage.Image, 0, numImages) + for i := range numImages { + components := make([]*storage.EmbeddedImageScanComponent, 0, numComponents) + for j := range numComponents { + vulns := make([]*storage.EmbeddedVulnerability, 0, numCVEs) + for k := range numCVEs { + vulns = append(vulns, &storage.EmbeddedVulnerability{ + Cve: fmt.Sprintf("CVE-2024-%d%d%d", i, j, k), + Cvss: 5, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Severity: storage.VulnerabilitySeverity_MODERATE_VULNERABILITY_SEVERITY, + SetFixedBy: &storage.EmbeddedVulnerability_FixedBy{FixedBy: "1.0.0"}, + }) + } + components = append(components, &storage.EmbeddedImageScanComponent{ + Name: fmt.Sprintf("pkg-%d-%d", i, j), + Version: fmt.Sprintf("%d.0.0", j), + Vulns: vulns, + }) + } + img := fixtures.GetImageWithUniqueComponents(0) + img.Id = fmt.Sprintf("sha256:realistic-%d", i) + img.Scan = &storage.ImageScan{ + Components: components, + } + images = append(images, img) + } + + for _, image := range images { + require.NoError(b, store.Upsert(ctx, image)) + } + + b.Run("WalkByQuery", func(b *testing.B) { + for b.Loop() { + count := 0 + err := store.WalkByQuery(ctx, search.EmptyQuery(), func(image *storage.Image) error { + count++ + return nil + }) + require.NoError(b, err) + } + }) + + b.Run("GetSingleImage", func(b *testing.B) { + for b.Loop() { + _, _, err := store.Get(ctx, "sha256:realistic-0") + require.NoError(b, err) + } + }) +} + +// TODO(ROX-30117): Remove this benchmark when FlattenImageData feature flag is removed. // BenchmarkWalkComparison benchmarks both Walk functions for comparison func BenchmarkWalkComparison(b *testing.B) { + if features.FlattenImageData.Enabled() { + b.Skip("Skipping benchmark - FlattenImageData is enabled.") + } ctx := sac.WithAllAccess(context.Background()) testDB := pgtest.ForT(b) @@ -26,7 +96,7 @@ func BenchmarkWalkComparison(b *testing.B) { // Setup: Insert test images numImages := 100 images := make([]*storage.Image, 0, numImages) - for i := 0; i < numImages; i++ { + for i := range numImages { img := fixtures.GetImageWithUniqueComponents(5) img.Id = fmt.Sprintf("%d", i) images = append(images, img) diff --git a/central/image/datastore/store/v2/postgres/store_hash_test.go b/central/image/datastore/store/v2/postgres/store_hash_test.go new file mode 100644 index 0000000000000..568bb7f32398f --- /dev/null +++ b/central/image/datastore/store/v2/postgres/store_hash_test.go @@ -0,0 +1,108 @@ +package postgres + +import ( + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPopulateImageScanHash_DuplicateComponentsChangeHash(t *testing.T) { + component := &storage.EmbeddedImageScanComponent{ + Name: "ubi10/ubi-micro", + Version: "1784668691", + } + + scanWith1 := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{component}, + } + scanWith3 := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{component, component, component}, + } + + require.NoError(t, populateImageScanHash(scanWith1)) + require.NoError(t, populateImageScanHash(scanWith3)) + + assert.NotEqual(t, scanWith1.GetHash(), scanWith3.GetHash(), + "removing duplicate components must produce a different hash") +} + +func TestPopulateImageScanHash_DuplicateVulnsChangeHash(t *testing.T) { + vuln := &storage.EmbeddedVulnerability{ + Cve: "CVE-2025-1234", + Severity: storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, + } + + scanWith1Vuln := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{ + {Name: "pkg", Version: "1.0", Vulns: []*storage.EmbeddedVulnerability{vuln}}, + }, + } + scanWith3Vulns := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{ + {Name: "pkg", Version: "1.0", Vulns: []*storage.EmbeddedVulnerability{vuln, vuln, vuln}}, + }, + } + + require.NoError(t, populateImageScanHash(scanWith1Vuln)) + require.NoError(t, populateImageScanHash(scanWith3Vulns)) + + assert.NotEqual(t, scanWith1Vuln.GetHash(), scanWith3Vulns.GetHash(), + "removing duplicate vulns must produce a different hash") +} + +func TestPopulateImageScanHash_DistinctChangesDetected(t *testing.T) { + scanA := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{ + {Name: "pkg", Version: "1.0", Vulns: []*storage.EmbeddedVulnerability{ + {Cve: "CVE-2025-1111"}, + }}, + }, + } + scanB := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{ + {Name: "pkg", Version: "1.0", Vulns: []*storage.EmbeddedVulnerability{ + {Cve: "CVE-2025-2222"}, + }}, + }, + } + + require.NoError(t, populateImageScanHash(scanA)) + require.NoError(t, populateImageScanHash(scanB)) + + assert.NotEqual(t, scanA.GetHash(), scanB.GetHash(), + "different CVEs must produce a different hash") +} + +func TestPopulateImageScanHash_IdenticalScansMatch(t *testing.T) { + scan1 := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{ + {Name: "pkg-a", Version: "1.0", Vulns: []*storage.EmbeddedVulnerability{ + {Cve: "CVE-2025-1111"}, + }}, + {Name: "pkg-b", Version: "2.0"}, + }, + } + scan2 := scan1.CloneVT() + + require.NoError(t, populateImageScanHash(scan1)) + require.NoError(t, populateImageScanHash(scan2)) + + assert.Equal(t, scan1.GetHash(), scan2.GetHash(), + "identical scans must produce the same hash") +} + +func TestPopulateImageScanHash_OrderIndependent(t *testing.T) { + compA := &storage.EmbeddedImageScanComponent{Name: "aaa", Version: "1.0"} + compB := &storage.EmbeddedImageScanComponent{Name: "bbb", Version: "2.0"} + + scanAB := &storage.ImageScan{Components: []*storage.EmbeddedImageScanComponent{compA, compB}} + scanBA := &storage.ImageScan{Components: []*storage.EmbeddedImageScanComponent{compB, compA}} + + require.NoError(t, populateImageScanHash(scanAB)) + require.NoError(t, populateImageScanHash(scanBA)) + + assert.Equal(t, scanAB.GetHash(), scanBA.GetHash(), + "component order must not affect the hash") +} diff --git a/central/image/datastore/store/v2/postgres/store_test.go b/central/image/datastore/store/v2/postgres/store_test.go index 593d6a360a838..a7d12334ec17f 100644 --- a/central/image/datastore/store/v2/postgres/store_test.go +++ b/central/image/datastore/store/v2/postgres/store_test.go @@ -12,6 +12,7 @@ import ( "github.com/stackrox/rox/central/image/datastore/store" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgtest" @@ -41,7 +42,11 @@ type ImagesStoreSuite struct { cvePgStore cveStore.Store } +// TODO(ROX-30117): Remove this test when FlattenImageData feature flag is removed. func TestImagesStore(t *testing.T) { + if features.FlattenImageData.Enabled() { + t.Skip("Skipping test - FlattenImageData is enabled. Equivalent tests exist in imageV2 datastore.") + } suite.Run(t, new(ImagesStoreSuite)) } diff --git a/central/image/datastoretest/datastore_flat_sac_test.go b/central/image/datastoretest/datastore_flat_sac_test.go index de28e5a61c3bc..44af15262e1b7 100644 --- a/central/image/datastoretest/datastore_flat_sac_test.go +++ b/central/image/datastoretest/datastore_flat_sac_test.go @@ -12,6 +12,7 @@ import ( "github.com/stackrox/rox/central/image/datastore" namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/images/types" "github.com/stackrox/rox/pkg/logging" @@ -30,7 +31,11 @@ var ( log = logging.LoggerForModule() ) +// TODO(ROX-30117): Remove this test when FlattenImageData feature flag is removed. func TestImageDataStoreFlatSAC(t *testing.T) { + if features.FlattenImageData.Enabled() { + t.Skip("Skipping test - FlattenImageData is enabled. Equivalent tests exist in imageV2 datastore.") + } suite.Run(t, new(imageDatastoreFlatSACSuite)) } diff --git a/central/image/service/http_handler.go b/central/image/service/http_handler.go index ddb13ab8da39c..c00ac2a7477d3 100644 --- a/central/image/service/http_handler.go +++ b/central/image/service/http_handler.go @@ -24,12 +24,13 @@ import ( "github.com/stackrox/rox/pkg/images/types" "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/scanners" scannerTypes "github.com/stackrox/rox/pkg/scanners/types" "github.com/stackrox/rox/pkg/zip" "google.golang.org/grpc/codes" ) -type sbomHttpHandler struct { +type sbomGenHttpHandler struct { integration integration.Set enricher enricher.ImageEnricher enricherV2 enricher.ImageEnricherV2 @@ -37,11 +38,11 @@ type sbomHttpHandler struct { riskManager manager.Manager } -var _ http.Handler = (*sbomHttpHandler)(nil) +var _ http.Handler = (*sbomGenHttpHandler)(nil) // SBOMHandler returns a handler for get sbom http request. -func SBOMHandler(integration integration.Set, enricher enricher.ImageEnricher, enricherV2 enricher.ImageEnricherV2, clusterSACHelper sachelper.ClusterSacHelper, riskManager manager.Manager) http.Handler { - return sbomHttpHandler{ +func SBOMGenHandler(integration integration.Set, enricher enricher.ImageEnricher, enricherV2 enricher.ImageEnricherV2, clusterSACHelper sachelper.ClusterSacHelper, riskManager manager.Manager) http.Handler { + return sbomGenHttpHandler{ integration: integration, enricher: enricher, enricherV2: enricherV2, @@ -50,7 +51,7 @@ func SBOMHandler(integration integration.Set, enricher enricher.ImageEnricher, e } } -func (h sbomHttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (h sbomGenHttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { w.WriteHeader(http.StatusMethodNotAllowed) return @@ -94,7 +95,7 @@ func (h sbomHttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // enrichWithModelSwitch enriches an image by name, returning both V1 and V2 models based on the feature flag. -func (h sbomHttpHandler) enrichWithModelSwitch( +func (h sbomGenHttpHandler) enrichWithModelSwitch( ctx context.Context, enrichmentCtx enricher.EnrichmentContext, ci *storage.ContainerImage, @@ -121,7 +122,7 @@ func (h sbomHttpHandler) enrichWithModelSwitch( } // enrichImage enriches the image with the given name and based on the given enrichment context. -func (h sbomHttpHandler) enrichImage(ctx context.Context, enrichmentCtx enricher.EnrichmentContext, ci *storage.ContainerImage) (*storage.Image, bool, error) { +func (h sbomGenHttpHandler) enrichImage(ctx context.Context, enrichmentCtx enricher.EnrichmentContext, ci *storage.ContainerImage) (*storage.Image, bool, error) { // forcedEnrichment is set to true when enrichImage forces an enrichment. forcedEnrichment := false @@ -159,7 +160,7 @@ func errorOrNotScanned(enrichmentResult enricher.EnrichmentResult, err error) er return err } - if !enrichmentResult.ImageUpdated || (enrichmentResult.ScanResult != enricher.ScanSucceeded) { + if !enrichmentResult.ScanResult.HasScanData() { return errors.New("scan could not be completed, please check that an applicable registry is integrated") } @@ -167,7 +168,7 @@ func errorOrNotScanned(enrichmentResult enricher.EnrichmentResult, err error) er } // getSBOM generates an SBOM for the specified parameters. -func (h sbomHttpHandler) getSBOM(ctx context.Context, params apiparams.SBOMRequestBody) ([]byte, error) { +func (h sbomGenHttpHandler) getSBOM(ctx context.Context, params apiparams.SBOMRequestBody) ([]byte, error) { enrichmentCtx := enricher.EnrichmentContext{ Delegable: true, FetchOpt: enricher.UseCachesIfPossible, @@ -268,25 +269,18 @@ func addForceToEnrichmentContext(enrichmentCtx *enricher.EnrichmentContext) { } // getScannerV4SBOMIntegration returns the SBOM interface of Scanner V4. -func (h sbomHttpHandler) getScannerV4SBOMIntegration() (scannerTypes.SBOMer, error) { - scanners := h.integration.ScannerSet() - for _, scanner := range scanners.GetAll() { - if scanner.GetScanner().Type() == scannerTypes.ScannerV4 { - if scannerv4, ok := scanner.GetScanner().(scannerTypes.SBOMer); ok { - return scannerv4, nil - } - } - } - return nil, errors.New("Scanner V4 integration not found") +func (h sbomGenHttpHandler) getScannerV4SBOMIntegration() (scannerTypes.SBOMer, error) { + sbomer, _, err := getScannerV4SBOMIntegration(h.integration.ScannerSet()) + return sbomer, err } // scannedByScannerV4 checks if image is scanned by Scanner V4. -func (h sbomHttpHandler) scannedByScannerV4(scan *storage.ImageScan) bool { +func (h sbomGenHttpHandler) scannedByScannerV4(scan *storage.ImageScan) bool { return scan.GetDataSource().GetId() == iiStore.DefaultScannerV4Integration.GetId() } // saveImage saves the image to Central's database. -func (h sbomHttpHandler) saveImage(img *storage.Image, imgV2 *storage.ImageV2) error { +func (h sbomGenHttpHandler) saveImage(img *storage.Image, imgV2 *storage.ImageV2) error { if features.FlattenImageData.Enabled() { return h.saveImageV2(imgV2) } @@ -294,7 +288,7 @@ func (h sbomHttpHandler) saveImage(img *storage.Image, imgV2 *storage.ImageV2) e } // saveImageV1 saves an Image V1 to Central's database. -func (h sbomHttpHandler) saveImageV1(img *storage.Image) error { +func (h sbomGenHttpHandler) saveImageV1(img *storage.Image) error { img.Id = utils.GetSHA(img) if img.GetId() == "" { return nil @@ -308,7 +302,7 @@ func (h sbomHttpHandler) saveImageV1(img *storage.Image) error { } // saveImageV2 saves an Image V2 to Central's database. -func (h sbomHttpHandler) saveImageV2(imgV2 *storage.ImageV2) error { +func (h sbomGenHttpHandler) saveImageV2(imgV2 *storage.ImageV2) error { if imgV2 == nil { return errors.New("nil images cannot be saved") } @@ -328,3 +322,14 @@ func (h sbomHttpHandler) saveImageV2(imgV2 *storage.ImageV2) error { } return nil } + +func getScannerV4SBOMIntegration(scanners scanners.Set) (scannerTypes.SBOMer, *storage.DataSource, error) { + for _, scanner := range scanners.GetAll() { + if scanner.GetScanner().Type() == scannerTypes.ScannerV4 { + if scannerv4, ok := scanner.GetScanner().(scannerTypes.SBOMer); ok { + return scannerv4, scanner.DataSource(), nil + } + } + } + return nil, nil, errors.New("Scanner V4 integration not found") +} diff --git a/central/image/service/http_handler_test.go b/central/image/service/http_handler_test.go index aa3664cf315d9..cebbdee7cb677 100644 --- a/central/image/service/http_handler_test.go +++ b/central/image/service/http_handler_test.go @@ -56,7 +56,7 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/sbom", nil) recorder := httptest.NewRecorder() - handler := SBOMHandler(imageintegration.Set(), nil, nil, nil, nil) + handler := SBOMGenHandler(imageintegration.Set(), nil, nil, nil, nil) handler.ServeHTTP(recorder, req) res := recorder.Result() @@ -99,7 +99,7 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/sbom", bytes.NewReader(reqJson)) recorder := httptest.NewRecorder() - handler := SBOMHandler(imageintegration.Set(), mockEnricher, mockEnricherV2, nil, nil) + handler := SBOMGenHandler(imageintegration.Set(), mockEnricher, mockEnricherV2, nil, nil) handler.ServeHTTP(recorder, req) res := recorder.Result() @@ -115,9 +115,13 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { testcases := []struct { name string flattenImageDataEnabled bool + scanResult enricher.ScanResult + imageUpdated bool }{ - {name: "with flatten image data disabled", flattenImageDataEnabled: false}, - {name: "with flatten image data enabled", flattenImageDataEnabled: true}, + {name: "with flatten image data disabled", flattenImageDataEnabled: false, scanResult: enricher.ScanSucceeded, imageUpdated: true}, + {name: "with flatten image data enabled", flattenImageDataEnabled: true, scanResult: enricher.ScanSucceeded, imageUpdated: true}, + {name: "with reused scan", flattenImageDataEnabled: false, scanResult: enricher.ScanReused, imageUpdated: false}, + {name: "with reused scan and flatten image data enabled", flattenImageDataEnabled: true, scanResult: enricher.ScanReused, imageUpdated: false}, } for _, tc := range testcases { @@ -136,12 +140,13 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { scanner := scannerTypesMocks.NewMockScannerSBOMer(ctrl) fsr := scannerTypesMocks.NewMockImageScannerWithDataSource(ctrl) - mockEnricher.EXPECT().EnrichImage(gomock.Any(), gomock.Any(), gomock.Any()).Return(enricher.EnrichmentResult{ImageUpdated: true, ScanResult: enricher.ScanSucceeded}, nil).AnyTimes() - mockEnricherV2.EXPECT().EnrichImage(gomock.Any(), gomock.Any(), gomock.Any()).Return(enricher.EnrichmentResult{ImageUpdated: true, ScanResult: enricher.ScanSucceeded}, nil).AnyTimes() + mockEnricher.EXPECT().EnrichImage(gomock.Any(), gomock.Any(), gomock.Any()).Return(enricher.EnrichmentResult{ImageUpdated: tc.imageUpdated, ScanResult: tc.scanResult}, nil).AnyTimes() + mockEnricherV2.EXPECT().EnrichImage(gomock.Any(), gomock.Any(), gomock.Any()).Return(enricher.EnrichmentResult{ImageUpdated: tc.imageUpdated, ScanResult: tc.scanResult}, nil).AnyTimes() scanner.EXPECT().Type().Return(scannerTypes.ScannerV4).AnyTimes() scanner.EXPECT().GetSBOM(gomock.Any()).DoAndReturn(getFakeSBOM).AnyTimes() set.EXPECT().ScannerSet().Return(scannerSet).AnyTimes() fsr.EXPECT().GetScanner().Return(scanner).AnyTimes() + fsr.EXPECT().DataSource().Return(nil).AnyTimes() scannerSet.EXPECT().GetAll().Return([]scannerTypes.ImageScannerWithDataSource{fsr}).AnyTimes() reqBody := &apiparams.SBOMRequestBody{ @@ -154,7 +159,7 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/sbom", bytes.NewReader(reqJson)) recorder := httptest.NewRecorder() - handler := SBOMHandler(set, mockEnricher, mockEnricherV2, nil, nil) + handler := SBOMGenHandler(set, mockEnricher, mockEnricherV2, nil, nil) handler.ServeHTTP(recorder, req) res := recorder.Result() @@ -172,7 +177,7 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/sbom", bytes.NewBufferString(invalidJson)) recorder := httptest.NewRecorder() - handler := SBOMHandler(imageintegration.Set(), nil, nil, nil, nil) + handler := SBOMGenHandler(imageintegration.Set(), nil, nil, nil, nil) handler.ServeHTTP(recorder, req) res := recorder.Result() @@ -189,7 +194,7 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/sbom", bytes.NewReader(reqBody)) recorder := httptest.NewRecorder() - handler := SBOMHandler(imageintegration.Set(), nil, nil, nil, nil) + handler := SBOMGenHandler(imageintegration.Set(), nil, nil, nil, nil) handler.ServeHTTP(recorder, req) res := recorder.Result() @@ -205,7 +210,7 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/sbom", nil) recorder := httptest.NewRecorder() - handler := SBOMHandler(imageintegration.Set(), nil, nil, nil, nil) + handler := SBOMGenHandler(imageintegration.Set(), nil, nil, nil, nil) handler.ServeHTTP(recorder, req) res := recorder.Result() @@ -222,7 +227,7 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/sbom", bytes.NewReader(largeRequestBody)) recorder := httptest.NewRecorder() - handler := SBOMHandler(imageintegration.Set(), nil, nil, nil, nil) + handler := SBOMGenHandler(imageintegration.Set(), nil, nil, nil, nil) handler.ServeHTTP(recorder, req) res := recorder.Result() @@ -280,6 +285,7 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { mockImageScannerWithDS := scannerTypesMocks.NewMockImageScannerWithDataSource(ctrl) mockImageScannerWithDS.EXPECT().GetScanner().Return(mockScanner).AnyTimes() + mockImageScannerWithDS.EXPECT().DataSource().Return(nil).AnyTimes() mockScannerSet := scannerMocks.NewMockSet(ctrl) mockScannerSet.EXPECT().GetAll().Return([]scannerTypes.ImageScannerWithDataSource{mockImageScannerWithDS}).AnyTimes() @@ -312,7 +318,7 @@ func TestHttpHandler_ServeHTTP(t *testing.T) { assert.NoError(t, err) req := httptest.NewRequest(http.MethodPost, "/sbom", bytes.NewReader(reqJson)) recorder := httptest.NewRecorder() - handler := SBOMHandler(mockIntegrationSet, mockEnricher, mockEnricherV2, nil, mockRiskManager) + handler := SBOMGenHandler(mockIntegrationSet, mockEnricher, mockEnricherV2, nil, mockRiskManager) // Make the SBOM generation request. handler.ServeHTTP(recorder, req) diff --git a/central/image/service/sbom_scan_http_handler.go b/central/image/service/sbom_scan_http_handler.go new file mode 100644 index 0000000000000..4402acd0cde3f --- /dev/null +++ b/central/image/service/sbom_scan_http_handler.go @@ -0,0 +1,149 @@ +package service + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + "github.com/pkg/errors" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/httputil" + "github.com/stackrox/rox/pkg/images/integration" + "github.com/stackrox/rox/pkg/ioutils" + scannerTypes "github.com/stackrox/rox/pkg/scanners/types" + "github.com/stackrox/rox/pkg/set" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/encoding/protojson" +) + +var ( + supportedMediaTypes = set.NewFrozenStringSet( + "text/spdx+json", // Used by Sigstore/Cosign, not IANA registered. + "application/spdx+json", // IANA registered type for SPDX JSON. + ) +) + +type sbomScanHttpHandler struct { + integrations integration.Set +} + +var _ http.Handler = (*sbomScanHttpHandler)(nil) + +func SBOMScanHandler(integrations integration.Set) http.Handler { + return sbomScanHttpHandler{ + integrations: integrations, + } +} + +func (s sbomScanHttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Verify Scanner V4 is enabled. + if !features.ScannerV4.Enabled() { + httputil.WriteGRPCStyleError(w, codes.Unimplemented, errors.New("Scanner V4 is disabled.")) + return + } + + if !features.SBOMScanning.Enabled() { + httputil.WriteGRPCStyleError(w, codes.Unimplemented, errors.New("SBOM Scanning is disabled.")) + return + } + + // Only POST requests are supported. + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + // Validate the media type is supported. + contentType := r.Header.Get("Content-Type") + err := s.validateMediaType(contentType) + if err != nil { + httputil.WriteGRPCStyleError(w, codes.InvalidArgument, fmt.Errorf("validating media type: %w", err)) + return + } + + // Enforce maximum uncompressed request size to prevent excessive memory usage. + // MaxBytesReader returns an error if the request body exceeds the limit. + maxReqSizeBytes := env.SBOMScanMaxReqSizeBytes.IntegerSetting() + limitedBody := http.MaxBytesReader(w, r.Body, int64(maxReqSizeBytes)) + + // Add cancellation safety to prevent partial/corrupted data on interruption. + // InterruptibleReader: Ensures clean termination without partial reads. + body, interrupt := ioutils.NewInterruptibleReader(limitedBody) + defer interrupt() + + // ContextBoundReader: Ensures reads fail fast when request context is canceled. + // This prevents hanging reads on connection interruption + readCtx, cancel := context.WithCancel(r.Context()) + defer cancel() + body = ioutils.NewContextBoundReader(readCtx, body) + + sbomScanResponse, err := s.scanSBOM(readCtx, body, contentType) + if err != nil { + // Check if error is due to request body exceeding size limit. + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + httputil.WriteGRPCStyleError(w, codes.InvalidArgument, fmt.Errorf("request body exceeds maximum size of %d bytes", maxBytesErr.Limit)) + return + } + httputil.WriteGRPCStyleError(w, codes.Internal, fmt.Errorf("scanning SBOM: %w", err)) + return + } + + // Serialize the scan result to JSON using protojson for proper protobuf handling. + // protojson handles protobuf-specific types (enums, oneof, etc.) correctly. + jsonBytes, err := protojson.MarshalOptions{Multiline: true}.Marshal(sbomScanResponse) + if err != nil { + httputil.WriteGRPCStyleError(w, codes.Internal, fmt.Errorf("serializing SBOM scan response: %w", err)) + return + } + + // Set response headers and write JSON response. + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(jsonBytes); err != nil { + log.Warnw("writing SBOM scan response: %v", err) + return + } +} + +// scanSBOM will request a scan of the SBOM from Scanner V4. +func (s sbomScanHttpHandler) scanSBOM(ctx context.Context, limitedReader io.Reader, contentType string) (*v1.SBOMScanResponse, error) { + // Get reference to Scanner V4. + scannerV4, dataSource, err := s.getScannerV4Integration() + if err != nil { + return nil, fmt.Errorf("getting Scanner V4 integration: %w", err) + } + + // Scan the SBOM. + sbomScanResponse, err := scannerV4.ScanSBOM(ctx, limitedReader, contentType) + if err != nil { + return nil, fmt.Errorf("scanning sbom: %w", err) + } + // Set the scan DataSource used to do the scan. + if sbomScanResponse.GetScan() != nil { + sbomScanResponse.GetScan().DataSource = dataSource + } + + return sbomScanResponse, nil +} + +// getScannerV4Integration returns the SBOM interface of Scanner V4. +func (s sbomScanHttpHandler) getScannerV4Integration() (scannerTypes.SBOMer, *storage.DataSource, error) { + sbomer, dataSource, err := getScannerV4SBOMIntegration(s.integrations.ScannerSet()) + return sbomer, dataSource, err +} + +// validateMediaType validates the media type from the content type header is supported. +func (s sbomScanHttpHandler) validateMediaType(contentType string) error { + // Strip any parameters (e.g., charset) from the media type + mediaType := strings.TrimSpace(strings.Split(contentType, ";")[0]) + if !supportedMediaTypes.Contains(mediaType) { + return fmt.Errorf("unsupported media type %q, supported types %v", mediaType, supportedMediaTypes.AsSlice()) + } + + return nil +} diff --git a/central/image/service/sbom_scan_http_handler_test.go b/central/image/service/sbom_scan_http_handler_test.go new file mode 100644 index 0000000000000..869d9c7b2815b --- /dev/null +++ b/central/image/service/sbom_scan_http_handler_test.go @@ -0,0 +1,168 @@ +package service + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/pkg/errors" + "github.com/stackrox/rox/central/imageintegration" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + intergrationMocks "github.com/stackrox/rox/pkg/images/integration/mocks" + scannerMocks "github.com/stackrox/rox/pkg/scanners/mocks" + scannerTypes "github.com/stackrox/rox/pkg/scanners/types" + scannerTypesMocks "github.com/stackrox/rox/pkg/scanners/types/mocks" + "github.com/stackrox/rox/pkg/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestScanSBOMHttpHandler_ServeHTTP(t *testing.T) { + t.Run("scanner v4 disabled", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.ScannerV4, false) + + req := httptest.NewRequest(http.MethodGet, "/sbom", nil) + recorder := httptest.NewRecorder() + + handler := SBOMScanHandler(imageintegration.Set()) + handler.ServeHTTP(recorder, req) + + res := recorder.Result() + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + err = res.Body.Close() + assert.NoError(t, err) + assert.Equal(t, http.StatusNotImplemented, res.StatusCode) + assert.Contains(t, string(body), "Scanner V4 is disabled") + }) + + t.Run("invalid request method", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.ScannerV4, true) + testutils.MustUpdateFeature(t, features.SBOMScanning, true) + + req := httptest.NewRequest(http.MethodGet, "/sbom", nil) + recorder := httptest.NewRecorder() + + handler := SBOMScanHandler(imageintegration.Set()) + handler.ServeHTTP(recorder, req) + + res := recorder.Result() + err := res.Body.Close() + assert.NoError(t, err) + assert.Equal(t, http.StatusMethodNotAllowed, res.StatusCode) + }) + + t.Run("invalid media type", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.ScannerV4, true) + testutils.MustUpdateFeature(t, features.SBOMScanning, true) + + req := httptest.NewRequest(http.MethodPost, "/sbom", nil) + req.Header.Add("Content-Type", "wrong") + recorder := httptest.NewRecorder() + + handler := SBOMScanHandler(imageintegration.Set()) + handler.ServeHTTP(recorder, req) + + res := recorder.Result() + err := res.Body.Close() + assert.NoError(t, err) + assert.Equal(t, http.StatusBadRequest, res.StatusCode) + }) + + t.Run("scanner v4 integration missing", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.ScannerV4, true) + testutils.MustUpdateFeature(t, features.SBOMScanning, true) + + req := httptest.NewRequest(http.MethodPost, "/sbom", nil) + req.Header.Add("Content-Type", supportedMediaTypes.AsSlice()[0]) + recorder := httptest.NewRecorder() + + handler := SBOMScanHandler(imageintegration.Set()) + handler.ServeHTTP(recorder, req) + + res := recorder.Result() + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + err = res.Body.Close() + assert.NoError(t, err) + assert.Equal(t, http.StatusInternalServerError, res.StatusCode) + assert.Contains(t, string(body), "integration") + }) + + t.Run("scanner v4 scan error", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.ScannerV4, true) + testutils.MustUpdateFeature(t, features.SBOMScanning, true) + + req := httptest.NewRequest(http.MethodPost, "/sbom", nil) + req.Header.Add("Content-Type", supportedMediaTypes.AsSlice()[0]) + recorder := httptest.NewRecorder() + + ctrl := gomock.NewController(t) + + mockSBOMScanner := scannerTypesMocks.NewMockScannerSBOMer(ctrl) + mockSBOMScanner.EXPECT().ScanSBOM(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("fake error")) + mockSBOMScanner.EXPECT().Type().Return(scannerTypes.ScannerV4) + + mockImageScannerWithDS := scannerTypesMocks.NewMockImageScannerWithDataSource(ctrl) + mockImageScannerWithDS.EXPECT().GetScanner().Return(mockSBOMScanner).AnyTimes() + mockImageScannerWithDS.EXPECT().DataSource().Return(&storage.DataSource{}).AnyTimes() + + mockScannerSet := scannerMocks.NewMockSet(ctrl) + mockScannerSet.EXPECT().GetAll().Return([]scannerTypes.ImageScannerWithDataSource{mockImageScannerWithDS}) + + mockIntegrationSet := intergrationMocks.NewMockSet(ctrl) + mockIntegrationSet.EXPECT().ScannerSet().Return(mockScannerSet) + + handler := SBOMScanHandler(mockIntegrationSet) + handler.ServeHTTP(recorder, req) + + res := recorder.Result() + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + err = res.Body.Close() + assert.NoError(t, err) + assert.Equal(t, http.StatusInternalServerError, res.StatusCode) + assert.Contains(t, string(body), "scanning sbom") + }) + + t.Run("valid scan", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.ScannerV4, true) + testutils.MustUpdateFeature(t, features.SBOMScanning, true) + + req := httptest.NewRequest(http.MethodPost, "/sbom", nil) + req.Header.Add("Content-Type", supportedMediaTypes.AsSlice()[0]) + recorder := httptest.NewRecorder() + + ctrl := gomock.NewController(t) + + mockSBOMScanner := scannerTypesMocks.NewMockScannerSBOMer(ctrl) + mockSBOMScanner.EXPECT().ScanSBOM(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.SBOMScanResponse{Id: "fake-sbom-id"}, nil) + mockSBOMScanner.EXPECT().Type().Return(scannerTypes.ScannerV4) + + mockImageScannerWithDS := scannerTypesMocks.NewMockImageScannerWithDataSource(ctrl) + mockImageScannerWithDS.EXPECT().GetScanner().Return(mockSBOMScanner).AnyTimes() + mockImageScannerWithDS.EXPECT().DataSource().Return(&storage.DataSource{}).AnyTimes() + + mockScannerSet := scannerMocks.NewMockSet(ctrl) + mockScannerSet.EXPECT().GetAll().Return([]scannerTypes.ImageScannerWithDataSource{mockImageScannerWithDS}) + + mockIntegrationSet := intergrationMocks.NewMockSet(ctrl) + mockIntegrationSet.EXPECT().ScannerSet().Return(mockScannerSet) + + handler := SBOMScanHandler(mockIntegrationSet) + handler.ServeHTTP(recorder, req) + + res := recorder.Result() + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + err = res.Body.Close() + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Contains(t, string(body), "fake-sbom-id") + }) + +} diff --git a/central/image/service/service.go b/central/image/service/service.go index 24b30c1f2bf0b..2666b07eab8e3 100644 --- a/central/image/service/service.go +++ b/central/image/service/service.go @@ -9,6 +9,7 @@ import ( "github.com/stackrox/rox/central/risk/manager" "github.com/stackrox/rox/central/role/sachelper" "github.com/stackrox/rox/central/sensor/service/connection" + signatureIntegrationDS "github.com/stackrox/rox/central/signatureintegration/datastore" watchedImageDataStore "github.com/stackrox/rox/central/watchedimage/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" @@ -49,21 +50,23 @@ func New( scanWaiterManager waiter.Manager[*storage.Image], scanWaiterManagerV2 waiter.Manager[*storage.ImageV2], clusterSACHelper sachelper.ClusterSacHelper, + signatureIntegrationDataStore signatureIntegrationDS.DataStore, ) Service { images.SetCentralScanSemaphoreLimit(float64(env.MaxParallelImageScanInternal.IntegerSetting())) return &serviceImpl{ - datastore: datastore, - datastoreV2: datastoreV2, - mappingDatastore: mappingDatastore, - watchedImages: watchedImages, - riskManager: riskManager, - enricher: enricher, - enricherV2: enricherV2, - metadataCache: metadataCache, - connManager: connManager, - scanWaiterManager: scanWaiterManager, - scanWaiterManagerV2: scanWaiterManagerV2, - internalScanSemaphore: semaphore.NewWeighted(int64(env.MaxParallelImageScanInternal.IntegerSetting())), - clusterSACHelper: clusterSACHelper, + datastore: datastore, + datastoreV2: datastoreV2, + mappingDatastore: mappingDatastore, + watchedImages: watchedImages, + riskManager: riskManager, + enricher: enricher, + enricherV2: enricherV2, + metadataCache: metadataCache, + connManager: connManager, + scanWaiterManager: scanWaiterManager, + scanWaiterManagerV2: scanWaiterManagerV2, + internalScanSemaphore: semaphore.NewWeighted(int64(env.MaxParallelImageScanInternal.IntegerSetting())), + clusterSACHelper: clusterSACHelper, + signatureIntegrationDataStore: signatureIntegrationDataStore, } } diff --git a/central/image/service/service_impl.go b/central/image/service/service_impl.go index 6dc7997147909..a98dd40cf79ca 100644 --- a/central/image/service/service_impl.go +++ b/central/image/service/service_impl.go @@ -17,6 +17,7 @@ import ( "github.com/stackrox/rox/central/risk/manager" "github.com/stackrox/rox/central/role/sachelper" "github.com/stackrox/rox/central/sensor/service/connection" + signatureIntegrationDS "github.com/stackrox/rox/central/signatureintegration/datastore" watchedImageDataStore "github.com/stackrox/rox/central/watchedimage/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/internalapi/central" @@ -43,8 +44,10 @@ import ( "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/paginated" "github.com/stackrox/rox/pkg/set" + "github.com/stackrox/rox/pkg/signatureintegration" "github.com/stackrox/rox/pkg/timestamp" pkgUtils "github.com/stackrox/rox/pkg/utils" + "github.com/stackrox/rox/pkg/uuid" "github.com/stackrox/rox/pkg/waiter" "golang.org/x/sync/semaphore" "google.golang.org/grpc" @@ -68,6 +71,7 @@ var ( }, or.SensorOr(idcheck.AdmissionControlOnly()): { v1.ImageService_ScanImageInternal_FullMethodName, + v1.ImageService_ScanImageInternalForAdmission_FullMethodName, }, idcheck.SensorsOnly(): { v1.ImageService_GetImageVulnerabilitiesInternal_FullMethodName, @@ -95,7 +99,8 @@ var ( imageScanMetricsLabel = prometheus.Labels{ "subsystem": "central", "entity": "central-image-scan-service", - "requestedFrom": "n/a"} + "requestedFrom": "n/a", + } ) // serviceImpl provides APIs for alerts. @@ -123,6 +128,8 @@ type serviceImpl struct { scanWaiterManagerV2 waiter.Manager[*storage.ImageV2] clusterSACHelper sachelper.ClusterSacHelper + + signatureIntegrationDataStore signatureIntegrationDS.DataStore } // RegisterServiceServer registers this service with the given gRPC Server. @@ -146,7 +153,13 @@ func (s *serviceImpl) GetImage(ctx context.Context, request *v1.GetImageRequest) return nil, errors.Wrap(errox.InvalidArgs, "id must be specified") } - id := types.NewDigest(request.GetId()).Digest() + id := request.GetId() + // When FlattenImageData is enabled, the ID may be a UUID (ImageV2 ID). + // Only normalize as a digest if it's not a UUID, to avoid mangling it + // with an incorrect "sha256:" prefix. + if _, err := uuid.FromString(id); err != nil { + id = types.NewDigest(id).Digest() + } image, exists, err := s.mappingDatastore.GetImage(ctx, id) if err != nil { @@ -164,6 +177,10 @@ func (s *serviceImpl) GetImage(ctx context.Context, request *v1.GetImageRequest) // This modifies the image object utils.StripCVEDescriptionsNoClone(image) } + utils.StripDatasourceNoClone(image.GetScan()) + signatureintegration.EnrichVerificationResults(ctx, + s.signatureIntegrationDataStore, image.GetSignatureVerificationData().GetResults(), + ) return image, nil } @@ -216,6 +233,10 @@ func (s *serviceImpl) ExportImages(req *v1.ExportImageRequest, srv v1.ImageServi defer cancel() } return s.mappingDatastore.WalkByQuery(ctx, parsedQuery, func(image *storage.Image) error { + utils.StripDatasourceNoClone(image.GetScan()) + signatureintegration.EnrichVerificationResults(ctx, + s.signatureIntegrationDataStore, image.GetSignatureVerificationData().GetResults(), + ) if err := srv.Send(&v1.ExportImageResponse{Image: image}); err != nil { return err } @@ -282,8 +303,24 @@ func (s *serviceImpl) saveImageV2(img *storage.ImageV2) error { // ScanImageInternal handles an image request from Sensor and Admission Controller. func (s *serviceImpl) ScanImageInternal(ctx context.Context, request *v1.ScanImageInternalRequest) (*v1.ScanImageInternalResponse, error) { + return s.scanImageInternalWithOpt(ctx, request, enricher.ForceRefetch) +} + +// ScanImageInternalForAdmission handles admission-controller image requests. +// For tag-only images it refreshes registry metadata but reuses an existing +// scan from Central's DB when the resolved digest is already known, avoiding +// a redundant Scanner round-trip. For digest-based images it behaves +// identically to ScanImageInternal. +func (s *serviceImpl) ScanImageInternalForAdmission(ctx context.Context, request *v1.ScanImageInternalRequest) (*v1.ScanImageInternalResponse, error) { + return s.scanImageInternalWithOpt(ctx, request, enricher.ForceRefetchMetadataOnly) +} + +// scanImageInternalWithOpt is the shared implementation for ScanImageInternal +// and ScanImageInternalForAdmission. tagOnlyFetchOpt controls the enricher +// FetchOption used when the request has no digest (tag-only image). +func (s *serviceImpl) scanImageInternalWithOpt(ctx context.Context, request *v1.ScanImageInternalRequest, tagOnlyFetchOpt enricher.FetchOption) (*v1.ScanImageInternalResponse, error) { if features.FlattenImageData.Enabled() { - return s.scanImageV2Internal(ctx, request) + return s.scanImageV2InternalWithOpt(ctx, request, tagOnlyFetchOpt) } if err := s.acquireScanSemaphore(ctx); err != nil { @@ -360,8 +397,8 @@ func (s *serviceImpl) ScanImageInternal(ctx context.Context, request *v1.ScanIma fetchOpt = enricher.UseCachesIfPossible if request.GetCachedOnly() { fetchOpt = enricher.NoExternalMetadata - } else if imgID == "" { // If no ID, then don't use caches as they could return stale data. - fetchOpt = enricher.ForceRefetch + } else if imgID == "" { + fetchOpt = tagOnlyFetchOpt } img = types.ToImage(request.GetImage()) } @@ -380,7 +417,7 @@ func (s *serviceImpl) ScanImageInternal(ctx context.Context, request *v1.ScanIma return internalScanRespFromImage(img), nil } -func (s *serviceImpl) scanImageV2Internal(ctx context.Context, request *v1.ScanImageInternalRequest) (*v1.ScanImageInternalResponse, error) { +func (s *serviceImpl) scanImageV2InternalWithOpt(ctx context.Context, request *v1.ScanImageInternalRequest, tagOnlyFetchOpt enricher.FetchOption) (*v1.ScanImageInternalResponse, error) { imgID := request.GetImage().GetIdV2() if imgID == "" { imgID = utils.NewImageV2ID(request.GetImage().GetName(), request.GetImage().GetId()) @@ -432,6 +469,10 @@ func (s *serviceImpl) scanImageV2Internal(ctx context.Context, request *v1.ScanI logging.Bool("cluster_local_scan_expired", clusterLocalScanExpired), ) + if clusterLocalScanExpired { + fetchOpt = enricher.IgnoreExistingImages + } + imgV2 = existingImgV2 imgExists = true } @@ -441,8 +482,8 @@ func (s *serviceImpl) scanImageV2Internal(ctx context.Context, request *v1.ScanI fetchOpt = enricher.UseCachesIfPossible if request.GetCachedOnly() { fetchOpt = enricher.NoExternalMetadata - } else if imgID == "" { // If no ID, then don't use caches as they could return stale data. - fetchOpt = enricher.ForceRefetch + } else if imgID == "" { + fetchOpt = tagOnlyFetchOpt } imgV2 = types.ToImageV2(request.GetImage()) if imgV2.GetId() == "" { @@ -622,8 +663,13 @@ func (s *serviceImpl) ScanImage(ctx context.Context, request *v1.ScanImageReques if !request.GetIncludeSnoozed() { utils.FilterSuppressedCVEsNoCloneV2(imgV2) } + utils.StripDatasourceNoClone(imgV2.GetScan()) - return utils.ConvertToV1(imgV2), nil + img := utils.ConvertToV1(imgV2) + signatureintegration.EnrichVerificationResults(ctx, + s.signatureIntegrationDataStore, img.GetSignatureVerificationData().GetResults(), + ) + return img, nil } img, err := enricher.EnrichImageByName(ctx, s.enricher, enrichmentCtx, request.GetImageName()) @@ -648,7 +694,10 @@ func (s *serviceImpl) ScanImage(ctx context.Context, request *v1.ScanImageReques if !request.GetIncludeSnoozed() { utils.FilterSuppressedCVEsNoClone(img) } - + utils.StripDatasourceNoClone(img.GetScan()) + signatureintegration.EnrichVerificationResults(ctx, + s.signatureIntegrationDataStore, img.GetSignatureVerificationData().GetResults(), + ) return img, nil } @@ -1236,11 +1285,11 @@ func (s *serviceImpl) DeleteImages(ctx context.Context, request *v1.DeleteImages if features.FlattenImageData.Enabled() { // Extract ImageV2 ID and SHA from search results - keys := make([]*central.InvalidateImageCache_ImageKey, 0, len(results)) + keys := make([]*central.ImageKey, 0, len(results)) for _, res := range results { if res.FieldValues != nil { // Set both ImageId (SHA for old sensors) and ImageIdV2 (ImageV2 ID for new sensors) - keys = append(keys, ¢ral.InvalidateImageCache_ImageKey{ + keys = append(keys, ¢ral.ImageKey{ ImageId: res.FieldValues[strings.ToLower(search.ImageSHA.String())], // SHA for backward compatibility ImageIdV2: res.ID, // ImageV2 ID for new sensors ImageFullName: res.FieldValues[strings.ToLower(search.ImageName.String())], @@ -1258,9 +1307,9 @@ func (s *serviceImpl) DeleteImages(ctx context.Context, request *v1.DeleteImages }) } } else { - keys := make([]*central.InvalidateImageCache_ImageKey, 0, len(idSlice)) + keys := make([]*central.ImageKey, 0, len(idSlice)) for _, id := range idSlice { - keys = append(keys, ¢ral.InvalidateImageCache_ImageKey{ + keys = append(keys, ¢ral.ImageKey{ ImageId: id, }) } @@ -1323,7 +1372,7 @@ func (s *serviceImpl) WatchImage(ctx context.Context, request *v1.WatchImageRequ }, nil } - if !enrichmentResult.ImageUpdated || (enrichmentResult.ScanResult != enricher.ScanSucceeded) { + if !enrichmentResult.ImageUpdated || !enrichmentResult.ScanResult.HasScanData() { return &v1.WatchImageResponse{ ErrorMessage: "scan could not be completed, due to no applicable registry/scanner integration", ErrorType: v1.WatchImageResponse_NO_VALID_INTEGRATION, @@ -1372,7 +1421,7 @@ func (s *serviceImpl) watchImageV2(ctx context.Context, containerImage *storage. }, nil } - if !enrichmentResult.ImageUpdated || (enrichmentResult.ScanResult != enricher.ScanSucceeded) { + if !enrichmentResult.ImageUpdated || !enrichmentResult.ScanResult.HasScanData() { return &v1.WatchImageResponse{ ErrorMessage: "scan could not be completed, due to no applicable registry/scanner integration", ErrorType: v1.WatchImageResponse_NO_VALID_INTEGRATION, diff --git a/central/image/service/service_impl_benchmark_test.go b/central/image/service/service_impl_benchmark_test.go index 92125cce518cc..83d0c5e0093de 100644 --- a/central/image/service/service_impl_benchmark_test.go +++ b/central/image/service/service_impl_benchmark_test.go @@ -22,7 +22,7 @@ func BenchmarkService_Export(b *testing.B) { b.Error(err) } - svc := New(testHelper.Images, testHelper.ImagesV2, testHelper.Images, nil, nil, nil, nil, nil, nil, nil, nil, nil) + svc := New(testHelper.Images, testHelper.ImagesV2, testHelper.Images, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) benchmarkFunc := getExportServiceBenchmark(testHelper, svc) testHelper.InjectDataAndRunBenchmark(b, true, benchmarkFunc) } diff --git a/central/image/service/service_impl_postgres_test.go b/central/image/service/service_impl_postgres_test.go new file mode 100644 index 0000000000000..0cd042900fca5 --- /dev/null +++ b/central/image/service/service_impl_postgres_test.go @@ -0,0 +1,230 @@ +//go:build sql_integration + +package service + +import ( + "context" + "errors" + "io" + "testing" + + imageDataStore "github.com/stackrox/rox/central/image/datastore" + policyDataStoreMock "github.com/stackrox/rox/central/policy/datastore/mocks" + signatureIntegrationDS "github.com/stackrox/rox/central/signatureintegration/datastore" + signatureIntegrationPostgres "github.com/stackrox/rox/central/signatureintegration/store/postgres" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + pkgGRPC "github.com/stackrox/rox/pkg/grpc" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stretchr/testify/suite" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" +) + +func TestImageServicePostgres(t *testing.T) { + suite.Run(t, new(imageServicePostgresTestSuite)) +} + +type imageServicePostgresTestSuite struct { + suite.Suite + + pool *pgtest.TestPostgres + imageDS imageDataStore.DataStore + sigIntegrationDS signatureIntegrationDS.DataStore + service Service + + ctx context.Context +} + +func (s *imageServicePostgresTestSuite) SetupTest() { + s.pool = pgtest.ForT(s.T()) + + s.imageDS = imageDataStore.GetTestPostgresDataStore(s.T(), s.pool) + + sigStore := signatureIntegrationPostgres.New(s.pool) + policyMock := policyDataStoreMock.NewMockDataStore(gomock.NewController(s.T())) + s.sigIntegrationDS = signatureIntegrationDS.New(sigStore, policyMock) + + s.service = New(s.imageDS, nil, s.imageDS, nil, nil, nil, nil, nil, nil, nil, nil, nil, s.sigIntegrationDS) + + s.ctx = sac.WithAllAccess(context.Background()) +} + +func (s *imageServicePostgresTestSuite) TestGetImageEnrichesVerifierName() { + // Create a signature integration with a known name. + integration := newTestSignatureIntegration("my-cosign-verifier") + savedIntegration, err := s.sigIntegrationDS.AddSignatureIntegration(s.ctx, integration) + s.Require().NoError(err) + s.Require().NotNil(savedIntegration) + + // Create an image with a verification result referencing this integration. + image := newTestImageWithVerificationResults("sha256:aaa111", []*storage.ImageSignatureVerificationResult{ + { + VerifierId: savedIntegration.GetId(), + Status: storage.ImageSignatureVerificationResult_VERIFIED, + }, + }) + s.Require().NoError(s.imageDS.UpsertImage(s.ctx, image)) + + // Call GetImage and verify that VerifierName is populated. + resp, err := s.service.GetImage(s.ctx, &v1.GetImageRequest{Id: image.GetId()}) + s.Require().NoError(err) + s.Require().NotNil(resp) + + results := resp.GetSignatureVerificationData().GetResults() + s.Require().Len(results, 1) + s.Equal("my-cosign-verifier", results[0].GetVerifierName()) +} + +func (s *imageServicePostgresTestSuite) TestGetImageWithUnknownVerifierId() { + // Create an image with a verification result pointing to a non-existent integration. + image := newTestImageWithVerificationResults("sha256:bbb222", []*storage.ImageSignatureVerificationResult{ + { + VerifierId: "io.stackrox.signatureintegration.non-existent-id", + Status: storage.ImageSignatureVerificationResult_VERIFIED, + }, + }) + s.Require().NoError(s.imageDS.UpsertImage(s.ctx, image)) + + resp, err := s.service.GetImage(s.ctx, &v1.GetImageRequest{Id: image.GetId()}) + s.Require().NoError(err) + s.Require().NotNil(resp) + + results := resp.GetSignatureVerificationData().GetResults() + s.Require().Len(results, 1) + s.Empty(results[0].GetVerifierName(), "VerifierName should be empty for unknown integration ID") +} + +func (s *imageServicePostgresTestSuite) TestGetImageWithEmptyVerifierId() { + // Create an image with a verification result where VerifierId is empty. + image := newTestImageWithVerificationResults("sha256:ccc333", []*storage.ImageSignatureVerificationResult{ + { + VerifierId: "", + Status: storage.ImageSignatureVerificationResult_VERIFIED, + }, + }) + s.Require().NoError(s.imageDS.UpsertImage(s.ctx, image)) + + resp, err := s.service.GetImage(s.ctx, &v1.GetImageRequest{Id: image.GetId()}) + s.Require().NoError(err) + s.Require().NotNil(resp) + + results := resp.GetSignatureVerificationData().GetResults() + s.Require().Len(results, 1) + s.Empty(results[0].GetVerifierName(), "VerifierName should be empty when VerifierId is empty") +} + +func (s *imageServicePostgresTestSuite) TestGetImageWithMultipleVerificationResults() { + // Create two signature integrations with different names. + integration1 := newTestSignatureIntegration("verifier-alpha") + saved1, err := s.sigIntegrationDS.AddSignatureIntegration(s.ctx, integration1) + s.Require().NoError(err) + + integration2 := newTestSignatureIntegration("verifier-beta") + saved2, err := s.sigIntegrationDS.AddSignatureIntegration(s.ctx, integration2) + s.Require().NoError(err) + + // Create an image with multiple verification results. + image := newTestImageWithVerificationResults("sha256:ddd444", []*storage.ImageSignatureVerificationResult{ + { + VerifierId: saved1.GetId(), + Status: storage.ImageSignatureVerificationResult_VERIFIED, + }, + { + VerifierId: saved2.GetId(), + Status: storage.ImageSignatureVerificationResult_VERIFIED, + }, + }) + s.Require().NoError(s.imageDS.UpsertImage(s.ctx, image)) + + resp, err := s.service.GetImage(s.ctx, &v1.GetImageRequest{Id: image.GetId()}) + s.Require().NoError(err) + s.Require().NotNil(resp) + + results := resp.GetSignatureVerificationData().GetResults() + s.Require().Len(results, 2) + + // Build a map of verifier ID to verifier name for order-independent assertion. + nameByID := make(map[string]string, len(results)) + for _, r := range results { + nameByID[r.GetVerifierId()] = r.GetVerifierName() + } + s.Equal("verifier-alpha", nameByID[saved1.GetId()]) + s.Equal("verifier-beta", nameByID[saved2.GetId()]) +} + +func (s *imageServicePostgresTestSuite) TestExportImagesEnrichesVerifierName() { + // Create a signature integration. + integration := newTestSignatureIntegration("export-verifier") + saved, err := s.sigIntegrationDS.AddSignatureIntegration(s.ctx, integration) + s.Require().NoError(err) + + // Create an image with a verification result referencing this integration. + image := newTestImageWithVerificationResults("sha256:eee555", []*storage.ImageSignatureVerificationResult{ + { + VerifierId: saved.GetId(), + Status: storage.ImageSignatureVerificationResult_VERIFIED, + }, + }) + s.Require().NoError(s.imageDS.UpsertImage(s.ctx, image)) + + // Set up a gRPC streaming server. + conn, closeFunc, err := pkgGRPC.CreateTestGRPCStreamingService( + s.ctx, + s.T(), + func(registrar grpc.ServiceRegistrar) { + v1.RegisterImageServiceServer(registrar, s.service) + }, + ) + s.Require().NoError(err) + defer closeFunc() + + client := v1.NewImageServiceClient(conn) + stream, err := client.ExportImages(s.ctx, &v1.ExportImageRequest{Timeout: 60}) + s.Require().NoError(err) + + var exported []*storage.Image + for { + resp, recvErr := stream.Recv() + if errors.Is(recvErr, io.EOF) { + break + } + s.Require().NoError(recvErr) + exported = append(exported, resp.GetImage()) + } + + s.Require().Len(exported, 1) + results := exported[0].GetSignatureVerificationData().GetResults() + s.Require().Len(results, 1) + s.Equal("export-verifier", results[0].GetVerifierName()) +} + +// newTestSignatureIntegration creates a minimal SignatureIntegration for testing. +func newTestSignatureIntegration(name string) *storage.SignatureIntegration { + return &storage.SignatureIntegration{ + Name: name, + Cosign: &storage.CosignPublicKeyVerification{ + PublicKeys: []*storage.CosignPublicKeyVerification_PublicKey{ + { + Name: "key1", + PublicKeyPemEnc: "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAryQICCl6NZ5gDKrnSztO\n3Hy8PEUcuyvg/ikC+VcIo2SFFSf18a3IMYldIugqqqZCs4/4uVW3sbdLs/6PfgdX\n7O9D22ZiFWHPYA2k2N744MNiCD1UE+tJyllUhSblK48bn+v1oZHCM0nYQ2NqUkvS\nj+hwUU3RiWl7x3D2s9wSdNt7XUtW05a/FXehsPSiJfKvHJJnGOX0BgTvkLnkAOTd\nOrUZ/wK69Dzu4IvrN4vs9Nes8vbwPa/ddZEzGR0cQMt0JBkhk9kU/qwqUseP1QRJ\n5I1jR4g8aYPL/ke9K35PxZWuDp3U0UPAZ3PjFAh+5T+fc7gzCs9dPzSHloruU+gl\nFQIDAQAB\n-----END PUBLIC KEY-----", + }, + }, + }, + } +} + +// newTestImageWithVerificationResults creates a minimal Image with the given +// signature verification results. +func newTestImageWithVerificationResults(id string, results []*storage.ImageSignatureVerificationResult) *storage.Image { + return &storage.Image{ + Id: id, + Name: &storage.ImageName{ + FullName: "docker.io/library/test:" + id, + }, + SignatureVerificationData: &storage.ImageSignatureVerificationData{ + Results: results, + }, + } +} diff --git a/central/image/service/service_impl_test.go b/central/image/service/service_impl_test.go index 7e39e908a88a1..023538816f75d 100644 --- a/central/image/service/service_impl_test.go +++ b/central/image/service/service_impl_test.go @@ -293,6 +293,7 @@ func TestResetClusterLocal(t *testing.T) { tcs := []struct { desc string existingImg *storage.Image + existingImgV2 *storage.ImageV2 expectedFetchOpt enricher.FetchOption enrichErr error finalClusterLocal bool @@ -300,31 +301,37 @@ func TestResetClusterLocal(t *testing.T) { { "do not reset flag when scan not expired", &storage.Image{IsClusterLocal: true, Name: name, Names: names, Scan: curScan}, + &storage.ImageV2{IsClusterLocal: true, Name: name, Scan: curScan}, enricher.UseCachesIfPossible, nil, true, }, { "reset flag when scan expired", &storage.Image{IsClusterLocal: true, Name: name, Names: names, Scan: expScan}, + &storage.ImageV2{IsClusterLocal: true, Name: name, Scan: expScan}, enricher.IgnoreExistingImages, nil, false, }, { "do not reset flag when scan not expired and existing name not found", &storage.Image{IsClusterLocal: true, Name: name, Names: nil, Scan: curScan}, + &storage.ImageV2{IsClusterLocal: true, Name: name, Scan: curScan}, enricher.ForceRefetchSignaturesOnly, nil, true, }, { "reset flag when scan expired and existing name not found", &storage.Image{IsClusterLocal: true, Name: name, Names: nil, Scan: expScan}, + &storage.ImageV2{IsClusterLocal: true, Name: name, Scan: expScan}, enricher.IgnoreExistingImages, nil, false, }, { "do not reset flag when scan not expired and new scan fails", &storage.Image{IsClusterLocal: true, Name: name, Names: names, Scan: curScan}, + &storage.ImageV2{IsClusterLocal: true, Name: name, Scan: curScan}, enricher.IgnoreExistingImages, errors.New("broken"), true, }, { "reset flag when scan expired and new scan fails", &storage.Image{IsClusterLocal: true, Name: name, Names: names, Scan: expScan}, + &storage.ImageV2{IsClusterLocal: true, Name: name, Scan: expScan}, enricher.IgnoreExistingImages, errors.New("broken"), false, }, } @@ -335,26 +342,60 @@ func TestResetClusterLocal(t *testing.T) { fetchOptMatcher := gomock.Cond(func(eCtx enricher.EnrichmentContext) bool { return eCtx.FetchOpt == tc.expectedFetchOpt }) - imageEnricherMock := enricherMocks.NewMockImageEnricher(ctrl) - imageEnricherMock.EXPECT(). - EnrichImage(gomock.Any(), fetchOptMatcher, gomock.Any()). - Return(enricher.EnrichmentResult{}, tc.enrichErr).AnyTimes() riskManagerMock := riskManagerMocks.NewMockManager(ctrl) - riskManagerMock.EXPECT(). - CalculateRiskAndUpsertImage(gomock.Any()). - Return(nil).AnyTimes() - imageDSMock := imageDSMocks.NewMockDataStore(ctrl) - imageDSMock.EXPECT(). - GetImage(gomock.Any(), gomock.Any()). - Return(tc.existingImg, tc.existingImg != nil, nil).AnyTimes() - - s := &serviceImpl{ - internalScanSemaphore: semaphore.NewWeighted(int64(env.MaxParallelImageScanInternal.IntegerSetting())), - enricher: imageEnricherMock, - datastore: imageDSMock, - riskManager: riskManagerMock, + var s *serviceImpl + if features.FlattenImageData.Enabled() { + imageEnricherV2Mock := enricherMocks.NewMockImageEnricherV2(ctrl) + imageEnricherV2Mock.EXPECT(). + EnrichImage(gomock.Any(), fetchOptMatcher, gomock.Any()). + Return(enricher.EnrichmentResult{}, tc.enrichErr).AnyTimes() + + riskManagerMock.EXPECT(). + CalculateRiskAndUpsertImageV2(gomock.Any()). + Return(nil).AnyTimes() + + imageV2DSMock := imageV2DSMocks.NewMockDataStore(ctrl) + imageV2DSMock.EXPECT(). + GetImage(gomock.Any(), gomock.Any()). + Return(tc.existingImgV2, tc.existingImgV2 != nil, nil).AnyTimes() + imageV2DSMock.EXPECT(). + GetImageNames(gomock.Any(), gomock.Any()). + Return(names, nil).AnyTimes() + + connMgrMock := connMgrMocks.NewMockManager(ctrl) + connMgrMock.EXPECT().AllSensorsHaveCapability(gomock.Any()).AnyTimes().Return(false) + + s = &serviceImpl{ + internalScanSemaphore: semaphore.NewWeighted(int64(env.MaxParallelImageScanInternal.IntegerSetting())), + enricherV2: imageEnricherV2Mock, + datastoreV2: imageV2DSMock, + riskManager: riskManagerMock, + connManager: connMgrMock, + } + } else { + // TODO(ROX-30117): Remove this block when FlattenImageData feature flag is removed. + imageEnricherMock := enricherMocks.NewMockImageEnricher(ctrl) + imageEnricherMock.EXPECT(). + EnrichImage(gomock.Any(), fetchOptMatcher, gomock.Any()). + Return(enricher.EnrichmentResult{}, tc.enrichErr).AnyTimes() + + riskManagerMock.EXPECT(). + CalculateRiskAndUpsertImage(gomock.Any()). + Return(nil).AnyTimes() + + imageDSMock := imageDSMocks.NewMockDataStore(ctrl) + imageDSMock.EXPECT(). + GetImage(gomock.Any(), gomock.Any()). + Return(tc.existingImg, tc.existingImg != nil, nil).AnyTimes() + + s = &serviceImpl{ + internalScanSemaphore: semaphore.NewWeighted(int64(env.MaxParallelImageScanInternal.IntegerSetting())), + enricher: imageEnricherMock, + datastore: imageDSMock, + riskManager: riskManagerMock, + } } resp, err := s.ScanImageInternal(context.Background(), scanReq) @@ -364,10 +405,198 @@ func TestResetClusterLocal(t *testing.T) { } } +// TestScanImageInternalForAdmission_FetchOpt verifies that the admission RPC +// uses ForceRefetchMetadataOnly for tag-only requests and behaves identically +// to ScanImageInternal for digest-based requests. +func TestScanImageInternalForAdmission_FetchOpt(t *testing.T) { + name, _, err := utils.GenerateImageNameFromString("reg.invalid/some/image:latest") + require.NoError(t, err) + + names := []*storage.ImageName{name} + + tcs := map[string]struct { + request *v1.ScanImageInternalRequest + existingImg *storage.Image + existingImgV2 *storage.ImageV2 + expectedFetchOpt enricher.FetchOption + expectEnrich bool + }{ + "tag-only uses ForceRefetchMetadataOnly": { + request: &v1.ScanImageInternalRequest{ + Image: &storage.ContainerImage{Name: name}, + }, + expectedFetchOpt: enricher.ForceRefetchMetadataOnly, + expectEnrich: true, + }, + "digest request, image in DB, skips enricher entirely": { + request: &v1.ScanImageInternalRequest{ + Image: &storage.ContainerImage{Id: "sha256:abc", Name: name}, + }, + existingImg: &storage.Image{ + Id: "sha256:abc", Name: name, Names: names, + Scan: &storage.ImageScan{}, + }, + existingImgV2: &storage.ImageV2{ + Id: utils.NewImageV2ID(name, "sha256:abc"), Digest: "sha256:abc", + Name: name, Scan: &storage.ImageScan{}, + }, + expectEnrich: false, + }, + "digest request, image NOT in DB, uses UseCachesIfPossible": { + request: &v1.ScanImageInternalRequest{ + Image: &storage.ContainerImage{Id: "sha256:abc", Name: name}, + }, + expectedFetchOpt: enricher.UseCachesIfPossible, + expectEnrich: true, + }, + } + + for desc, tc := range tcs { + t.Run(desc, func(t *testing.T) { + ctrl := gomock.NewController(t) + riskManagerMock := riskManagerMocks.NewMockManager(ctrl) + + var s *serviceImpl + if features.FlattenImageData.Enabled() { + imageEnricherV2Mock := enricherMocks.NewMockImageEnricherV2(ctrl) + if tc.expectEnrich { + fetchOptMatcher := gomock.Cond(func(eCtx enricher.EnrichmentContext) bool { + return eCtx.FetchOpt == tc.expectedFetchOpt + }) + imageEnricherV2Mock.EXPECT(). + EnrichImage(gomock.Any(), fetchOptMatcher, gomock.Any()). + Return(enricher.EnrichmentResult{}, nil).Times(1) + } + + riskManagerMock.EXPECT(). + CalculateRiskAndUpsertImageV2(gomock.Any()). + Return(nil).AnyTimes() + + imageV2DSMock := imageV2DSMocks.NewMockDataStore(ctrl) + imageV2DSMock.EXPECT(). + GetImage(gomock.Any(), gomock.Any()). + Return(tc.existingImgV2, tc.existingImgV2 != nil, nil).AnyTimes() + imageV2DSMock.EXPECT(). + GetImageNames(gomock.Any(), gomock.Any()). + Return(names, nil).AnyTimes() + + connMgrMock := connMgrMocks.NewMockManager(ctrl) + connMgrMock.EXPECT().AllSensorsHaveCapability(gomock.Any()).AnyTimes().Return(false) + + s = &serviceImpl{ + internalScanSemaphore: semaphore.NewWeighted(int64(env.MaxParallelImageScanInternal.IntegerSetting())), + enricherV2: imageEnricherV2Mock, + datastoreV2: imageV2DSMock, + riskManager: riskManagerMock, + connManager: connMgrMock, + } + } else { + imageEnricherMock := enricherMocks.NewMockImageEnricher(ctrl) + if tc.expectEnrich { + fetchOptMatcher := gomock.Cond(func(eCtx enricher.EnrichmentContext) bool { + return eCtx.FetchOpt == tc.expectedFetchOpt + }) + imageEnricherMock.EXPECT(). + EnrichImage(gomock.Any(), fetchOptMatcher, gomock.Any()). + Return(enricher.EnrichmentResult{}, nil).Times(1) + } + + riskManagerMock.EXPECT(). + CalculateRiskAndUpsertImage(gomock.Any()). + Return(nil).AnyTimes() + + imageDSMock := imageDSMocks.NewMockDataStore(ctrl) + imageDSMock.EXPECT(). + GetImage(gomock.Any(), gomock.Any()). + Return(tc.existingImg, tc.existingImg != nil, nil).AnyTimes() + + s = &serviceImpl{ + internalScanSemaphore: semaphore.NewWeighted(int64(env.MaxParallelImageScanInternal.IntegerSetting())), + enricher: imageEnricherMock, + datastore: imageDSMock, + riskManager: riskManagerMock, + } + } + + resp, err := s.ScanImageInternalForAdmission(context.Background(), tc.request) + require.NoError(t, err) + assert.NotNil(t, resp) + }) + } +} + +// TestScanImageInternalForAdmission_DigestParity ensures the admission RPC +// produces the same result as ScanImageInternal for digest-based requests. +func TestScanImageInternalForAdmission_DigestParity(t *testing.T) { + name, _, err := utils.GenerateImageNameFromString("reg.invalid/some/image:latest") + require.NoError(t, err) + + names := []*storage.ImageName{name} + req := &v1.ScanImageInternalRequest{ + Image: &storage.ContainerImage{Id: "sha256:abc", Name: name}, + } + + curScan := &storage.ImageScan{ScanTime: timestamppb.New(time.Now())} + + buildService := func(ctrl *gomock.Controller) *serviceImpl { + riskManagerMock := riskManagerMocks.NewMockManager(ctrl) + if features.FlattenImageData.Enabled() { + existingImgV2 := &storage.ImageV2{ + Id: utils.NewImageV2ID(name, "sha256:abc"), Digest: "sha256:abc", + Name: name, Scan: curScan, + } + imageV2DSMock := imageV2DSMocks.NewMockDataStore(ctrl) + imageV2DSMock.EXPECT(). + GetImage(gomock.Any(), gomock.Any()). + Return(existingImgV2, true, nil).AnyTimes() + imageV2DSMock.EXPECT(). + GetImageNames(gomock.Any(), gomock.Any()). + Return(names, nil).AnyTimes() + + connMgrMock := connMgrMocks.NewMockManager(ctrl) + connMgrMock.EXPECT().AllSensorsHaveCapability(gomock.Any()).AnyTimes().Return(false) + + return &serviceImpl{ + internalScanSemaphore: semaphore.NewWeighted(int64(env.MaxParallelImageScanInternal.IntegerSetting())), + datastoreV2: imageV2DSMock, + riskManager: riskManagerMock, + connManager: connMgrMock, + } + } + existingImg := &storage.Image{ + Id: "sha256:abc", Name: name, Names: names, Scan: curScan, + } + imageDSMock := imageDSMocks.NewMockDataStore(ctrl) + imageDSMock.EXPECT(). + GetImage(gomock.Any(), gomock.Any()). + Return(existingImg, true, nil).AnyTimes() + + return &serviceImpl{ + internalScanSemaphore: semaphore.NewWeighted(int64(env.MaxParallelImageScanInternal.IntegerSetting())), + datastore: imageDSMock, + riskManager: riskManagerMock, + } + } + + ctrl1 := gomock.NewController(t) + ctrl2 := gomock.NewController(t) + + s1 := buildService(ctrl1) + s2 := buildService(ctrl2) + + resp1, err1 := s1.ScanImageInternal(context.Background(), req) + resp2, err2 := s2.ScanImageInternalForAdmission(context.Background(), req) + + require.NoError(t, err1) + require.NoError(t, err2) + protoassert.Equal(t, resp1.GetImage(), resp2.GetImage()) +} + // TestEnrichLocalImageInternal_ImageNames ensures that image names are // populated from the existing image in Central DB when the image // requires re-enrichment. (ie: when scan has expired) func TestEnrichLocalImageInternal_ImageNames(t *testing.T) { + pkgTestUtils.MustUpdateFeature(t, features.FlattenImageData, false) ctx := context.Background() ctrl := gomock.NewController(t) @@ -476,6 +705,44 @@ func TestEnrichLocalImageV2Internal_ImageNames(t *testing.T) { require.Len(t, resp.GetImage().GetNames(), 3) } +// TestGetImage_IDNormalization ensures that GetImage passes UUID-based IDs through +// unchanged, while still normalizing plain SHA hashes with the "sha256:" prefix. +func TestGetImage_IDNormalization(t *testing.T) { + cases := map[string]struct { + requestID string + expectedID string + }{ + "UUID is not mangled with sha256 prefix": { + requestID: "faa743bd-0365-54ad-9898-9650fa51b808", + expectedID: "faa743bd-0365-54ad-9898-9650fa51b808", + }, + "plain SHA hash is normalized with sha256 prefix": { + requestID: "72daaf46f11cc753c4eab981cbf869919bd1fee3d2170a2adeac12400f494728", + expectedID: "sha256:72daaf46f11cc753c4eab981cbf869919bd1fee3d2170a2adeac12400f494728", + }, + "sha256-prefixed digest is unchanged": { + requestID: "sha256:72daaf46f11cc753c4eab981cbf869919bd1fee3d2170a2adeac12400f494728", + expectedID: "sha256:72daaf46f11cc753c4eab981cbf869919bd1fee3d2170a2adeac12400f494728", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + imageDSMock := imageDSMocks.NewMockDataStore(ctrl) + imageDSMock.EXPECT(). + GetImage(gomock.Any(), tc.expectedID). + Return(&storage.Image{Id: tc.expectedID}, true, nil) + + s := &serviceImpl{mappingDatastore: imageDSMock} + + resp, err := s.GetImage(context.Background(), &v1.GetImageRequest{Id: tc.requestID}) + require.NoError(t, err) + assert.Equal(t, tc.expectedID, resp.GetId()) + }) + } +} + // TestDeleteImages_V2 ensures that DeleteImages correctly queries for ImageID and SHA // and broadcasts InvalidateImageCache messages with both fields populated. func TestDeleteImages_V2(t *testing.T) { @@ -521,7 +788,7 @@ func TestDeleteImages_V2(t *testing.T) { expectedMsg := ¢ral.MsgToSensor{ Msg: ¢ral.MsgToSensor_InvalidateImageCache{ InvalidateImageCache: ¢ral.InvalidateImageCache{ - ImageKeys: []*central.InvalidateImageCache_ImageKey{ + ImageKeys: []*central.ImageKey{ { ImageId: "sha256:abc123", ImageIdV2: "image-uuid-1", diff --git a/central/image/service/singleton.go b/central/image/service/singleton.go index fd497522049f0..2eb117c289460 100644 --- a/central/image/service/singleton.go +++ b/central/image/service/singleton.go @@ -11,6 +11,7 @@ import ( "github.com/stackrox/rox/central/risk/manager" "github.com/stackrox/rox/central/role/sachelper" "github.com/stackrox/rox/central/sensor/service/connection" + signatureIntegrationDS "github.com/stackrox/rox/central/signatureintegration/datastore" watchedImageDataStore "github.com/stackrox/rox/central/watchedimage/datastore" "github.com/stackrox/rox/pkg/images/cache" "github.com/stackrox/rox/pkg/sync" @@ -36,6 +37,7 @@ func initialize() { scanwaiter.Singleton(), scanwaiterv2.Singleton(), sachelper.NewClusterSacHelper(clusterDataStore.Singleton()), + signatureIntegrationDS.Singleton(), ) } diff --git a/central/imagecomponent/v2/datastore/datastore.go b/central/imagecomponent/v2/datastore/datastore.go index 19428e5496d64..dc63df11642a5 100644 --- a/central/imagecomponent/v2/datastore/datastore.go +++ b/central/imagecomponent/v2/datastore/datastore.go @@ -28,9 +28,10 @@ type DataStore interface { } // New returns a new instance of a DataStore. -func New(storage pgStore.Store, risks riskDataStore.DataStore, ranker *ranking.Ranker) DataStore { +func New(db postgres.DB, storage pgStore.Store, risks riskDataStore.DataStore, ranker *ranking.Ranker) DataStore { ds := &datastoreImpl{ storage: storage, + db: db, risks: risks, imageComponentRanker: ranker, } @@ -43,5 +44,5 @@ func New(storage pgStore.Store, risks riskDataStore.DataStore, ranker *ranking.R func GetTestPostgresDataStore(t testing.TB, pool postgres.DB) DataStore { dbstore := pgStore.New(pool) riskStore := riskDataStore.GetTestPostgresDataStore(t, pool) - return New(dbstore, riskStore, ranking.ComponentRanker()) + return New(pool, dbstore, riskStore, ranking.ComponentRanker()) } diff --git a/central/imagecomponent/v2/datastore/datastore_bench_postgres_test.go b/central/imagecomponent/v2/datastore/datastore_bench_postgres_test.go new file mode 100644 index 0000000000000..0fa799744d938 --- /dev/null +++ b/central/imagecomponent/v2/datastore/datastore_bench_postgres_test.go @@ -0,0 +1,102 @@ +//go:build sql_integration + +package datastore + +import ( + "context" + "fmt" + "testing" + + pgStore "github.com/stackrox/rox/central/imagecomponent/v2/datastore/store/postgres" + "github.com/stackrox/rox/central/imagecomponent/v2/views" + "github.com/stackrox/rox/central/ranking" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres/pgtest" + pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/search" + pgSearch "github.com/stackrox/rox/pkg/search/postgres" + "github.com/stretchr/testify/require" +) + +// BenchmarkInitializeRankers compares the current Walk-based approach for +// initializing component rankers against a lightweight select query that +// fetches only the component ID and risk score. +func BenchmarkInitializeRankers(b *testing.B) { + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(b) + store := pgStore.New(testDB.DB) + + // Insert a parent image row to satisfy the foreign key constraint. + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + _, err := testDB.DB.Exec(ctx, "INSERT INTO images_v2 (Id, serialized) VALUES ($1, $2) ON CONFLICT DO NOTHING", "bench-image", []byte{}) + require.NoError(b, err) + } else { + _, err := testDB.DB.Exec(ctx, "INSERT INTO images (Id, serialized) VALUES ($1, $2) ON CONFLICT DO NOTHING", "bench-image", []byte{}) + require.NoError(b, err) + } + + // Insert test components with realistic serialized blob sizes. + numComponents := 500 + for i := range numComponents { + component := &storage.ImageComponentV2{ + Id: fmt.Sprintf("component-%d", i), + Name: fmt.Sprintf("pkg-%d", i), + Version: fmt.Sprintf("%d.0.0", i), + RiskScore: float32(i) / float32(numComponents), + Source: storage.SourceType_OS, + OperatingSystem: "linux", + Location: fmt.Sprintf("/usr/lib/pkg-%d", i), + } + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + component.ImageIdV2 = "bench-image" + } else { + component.ImageId = "bench-image" + } + require.NoError(b, store.Upsert(ctx, component)) + } + + b.Run("Walk", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + ranker := ranking.NewRanker() + count := 0 + err := store.Walk(ctx, func(component *storage.ImageComponentV2) error { + ranker.Add(component.GetId(), component.GetRiskScore()) + count++ + return nil + }) + require.NoError(b, err) + require.Equal(b, numComponents, count) + } + }) + + b.Run("SelectRiskView", func(b *testing.B) { + b.ReportAllocs() + selects := []*v1.QuerySelect{ + search.NewQuerySelect(search.ComponentID).Proto(), + search.NewQuerySelect(search.ComponentRiskScore).Proto(), + } + query := search.EmptyQuery() + query.Selects = selects + + for b.Loop() { + ranker := ranking.NewRanker() + count := 0 + err := pgSearch.RunSelectRequestForSchemaFn[views.ComponentRiskView]( + ctx, testDB.DB, pkgSchema.ImageComponentV2Schema, query, + func(r *views.ComponentRiskView) error { + ranker.Add(r.ComponentID, r.ComponentRiskScore) + count++ + return nil + }, + ) + require.NoError(b, err) + require.Equal(b, numComponents, count) + } + }) +} diff --git a/central/imagecomponent/v2/datastore/datastore_impl.go b/central/imagecomponent/v2/datastore/datastore_impl.go index c6ddd1dccbb8a..01cf2ab270be3 100644 --- a/central/imagecomponent/v2/datastore/datastore_impl.go +++ b/central/imagecomponent/v2/datastore/datastore_impl.go @@ -4,15 +4,20 @@ import ( "context" "strings" - pgStore "github.com/stackrox/rox/central/imagecomponent/v2/datastore/store/postgres" + "github.com/stackrox/rox/central/imagecomponent/v2/datastore/store/postgres" + "github.com/stackrox/rox/central/imagecomponent/v2/views" + imagev2common "github.com/stackrox/rox/central/imagev2/common" "github.com/stackrox/rox/central/ranking" riskDataStore "github.com/stackrox/rox/central/risk/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/logging" + pgPkg "github.com/stackrox/rox/pkg/postgres" + pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" pkgSearch "github.com/stackrox/rox/pkg/search" + pgSearch "github.com/stackrox/rox/pkg/search/postgres" ) var ( @@ -20,18 +25,21 @@ var ( ) type datastoreImpl struct { - storage pgStore.Store + storage postgres.Store + db pgPkg.DB risks riskDataStore.DataStore imageComponentRanker *ranking.Ranker } func (ds *datastoreImpl) Search(ctx context.Context, q *v1.Query) ([]pkgSearch.Result, error) { + q = imagev2common.WithRowsFromImageV2Only(q) return ds.storage.Search(ctx, q) } // Count returns the number of search results from the query func (ds *datastoreImpl) Count(ctx context.Context, q *v1.Query) (int, error) { + q = imagev2common.WithRowsFromImageV2Only(q) return ds.storage.Count(ctx, q) } @@ -63,6 +71,7 @@ func (ds *datastoreImpl) SearchImageComponents(ctx context.Context, q *v1.Query) } func (ds *datastoreImpl) SearchRawImageComponents(ctx context.Context, q *v1.Query) ([]*storage.ImageComponentV2, error) { + q = imagev2common.WithRowsFromImageV2Only(q) var components []*storage.ImageComponentV2 err := ds.storage.GetByQueryFn(ctx, q, func(component *storage.ImageComponentV2) error { components = append(components, component) @@ -109,16 +118,29 @@ func (ds *datastoreImpl) initializeRankers() { sac.AllowFixedScopes( sac.AccessModeScopeKeys(storage.Access_READ_ACCESS), sac.ResourceScopeKeys(resources.Image, resources.Node))) - err := ds.storage.Walk(readCtx, func(component *storage.ImageComponentV2) error { - ds.imageComponentRanker.Add(component.GetId(), component.GetRiskScore()) - return nil - }) + // The entire component is not needed to initialize the ranker. We only need the component ID and risk score. + selects := []*v1.QuerySelect{ + pkgSearch.NewQuerySelect(pkgSearch.ComponentID).Proto(), + pkgSearch.NewQuerySelect(pkgSearch.ComponentRiskScore).Proto(), + } + query := pkgSearch.EmptyQuery() + query.Selects = selects + + count := 0 + err := pgSearch.RunSelectRequestForSchemaFn[views.ComponentRiskView]( + readCtx, ds.db, pkgSchema.ImageComponentV2Schema, query, + func(r *views.ComponentRiskView) error { + ds.imageComponentRanker.Add(r.ComponentID, r.ComponentRiskScore) + count++ + return nil + }, + ) if err != nil { log.Errorf("unable to initialize image component ranking: %v", err) return } - log.Info("Initialized image component ranking") + log.Infof("Initialized image component ranking with %d components", count) } func (ds *datastoreImpl) updateImageComponentPriority(ics ...*storage.ImageComponentV2) { @@ -142,7 +164,3 @@ func (c *ImageComponentSearchResultConverter) BuildLocation(result *pkgSearch.Re func (c *ImageComponentSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_IMAGE_COMPONENTS_V2 } - -func (c *ImageComponentSearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/imagecomponent/v2/datastore/datastore_sac_test.go b/central/imagecomponent/v2/datastore/datastore_sac_test.go index 36c86f9d95446..4867167b230fa 100644 --- a/central/imagecomponent/v2/datastore/datastore_sac_test.go +++ b/central/imagecomponent/v2/datastore/datastore_sac_test.go @@ -9,6 +9,7 @@ import ( graphDBTestUtils "github.com/stackrox/rox/central/graphdb/testutils" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" @@ -43,6 +44,22 @@ func getImageComponentID(component *storage.EmbeddedImageScanComponent, imageID return scancomponent.ComponentIDV2(component, imageID, index) } +// TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. +func getSherlockHolmesImageID() string { + if features.FlattenImageData.Enabled() { + return fixtures.GetImageV2SherlockHolmes1().GetId() + } + return fixtures.GetImageSherlockHolmes1().GetId() +} + +// TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. +func getDoctorJekyllImageID() string { + if features.FlattenImageData.Enabled() { + return fixtures.GetImageV2DoctorJekyll2().GetId() + } + return fixtures.GetImageDoctorJekyll2().GetId() +} + func (s *componentV2DataStoreSACTestSuite) cleanImageToVulnerabilitiesGraph() { s.Require().NoError(s.testGraphDatastore.CleanImageToVulnerabilitiesGraph()) } @@ -58,12 +75,12 @@ var ( imageComponent1s2x3 = fixtures.GetEmbeddedImageComponent1s2x3() imageComponent2x4 = fixtures.GetEmbeddedImageComponent2x4() imageComponent2x5 = fixtures.GetEmbeddedImageComponent2x5() - imageComponentID1x1 = getImageComponentID(imageComponent1x1, fixtures.GetImageSherlockHolmes1().GetId(), 0) - imageComponentID1x2 = getImageComponentID(imageComponent1x2, fixtures.GetImageSherlockHolmes1().GetId(), 1) - imageComponentID1s2x3i1 = getImageComponentID(imageComponent1s2x3, fixtures.GetImageSherlockHolmes1().GetId(), 2) - imageComponentID1s2x3i2 = getImageComponentID(imageComponent1s2x3, fixtures.GetImageDoctorJekyll2().GetId(), 0) - imageComponentID2x4 = getImageComponentID(imageComponent2x4, fixtures.GetImageDoctorJekyll2().GetId(), 1) - imageComponentID2x5 = getImageComponentID(imageComponent2x5, fixtures.GetImageDoctorJekyll2().GetId(), 2) + imageComponentID1x1 = getImageComponentID(imageComponent1x1, getSherlockHolmesImageID(), 0) + imageComponentID1x2 = getImageComponentID(imageComponent1x2, getSherlockHolmesImageID(), 1) + imageComponentID1s2x3i1 = getImageComponentID(imageComponent1s2x3, getSherlockHolmesImageID(), 2) + imageComponentID1s2x3i2 = getImageComponentID(imageComponent1s2x3, getDoctorJekyllImageID(), 0) + imageComponentID2x4 = getImageComponentID(imageComponent2x4, getDoctorJekyllImageID(), 1) + imageComponentID2x5 = getImageComponentID(imageComponent2x5, getDoctorJekyllImageID(), 2) imageComponentTestCases = []componentTestCase{ { diff --git a/central/imagecomponent/v2/datastore/datastore_v1_filter_test.go b/central/imagecomponent/v2/datastore/datastore_v1_filter_test.go new file mode 100644 index 0000000000000..80ce21cd9b6a6 --- /dev/null +++ b/central/imagecomponent/v2/datastore/datastore_v1_filter_test.go @@ -0,0 +1,99 @@ +//go:build sql_integration + +package datastore + +import ( + "context" + "testing" + + imageDS "github.com/stackrox/rox/central/image/datastore" + imageV2DS "github.com/stackrox/rox/central/imagev2/datastore" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/fixtures" + imageUtils "github.com/stackrox/rox/pkg/images/utils" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/set" + "github.com/stretchr/testify/suite" +) + +func TestImageComponentV2DataStoreV1Filter(t *testing.T) { + suite.Run(t, new(ImageComponentV2DataStoreV1FilterTestSuite)) +} + +type ImageComponentV2DataStoreV1FilterTestSuite struct { + suite.Suite + + testDB *pgtest.TestPostgres + componentStore DataStore + ctx context.Context + + v2Image *storage.ImageV2 +} + +func (s *ImageComponentV2DataStoreV1FilterTestSuite) SetupSuite() { + if !features.FlattenImageData.Enabled() { + s.T().Skip("Skipping test because FlattenImageData feature flag is disabled") + } + s.ctx = sac.WithAllAccess(context.Background()) + s.testDB = pgtest.ForT(s.T()) + + // The CVE converter sets ImageId vs ImageIdV2 based on the FlattenImageData flag, so temporarily disable it to insert a V1 image. + s.T().Setenv(features.FlattenImageData.EnvVar(), "false") + v1Store := imageDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(v1Store.UpsertImage(s.ctx, fixtures.GetImageSherlockHolmes1())) + + s.T().Setenv(features.FlattenImageData.EnvVar(), "true") + v2Store := imageV2DS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.v2Image = imageUtils.ConvertToV2(fixtures.GetImageDoctorJekyll2()) + s.Require().NoError(v2Store.UpsertImage(s.ctx, s.v2Image)) + + s.componentStore = GetTestPostgresDataStore(s.T(), s.testDB.DB) +} + +func (s *ImageComponentV2DataStoreV1FilterTestSuite) expectedV2Components() set.StringSet { + components := set.NewStringSet() + for _, comp := range s.v2Image.GetScan().GetComponents() { + components.Add(comp.GetName()) + } + return components +} + +func (s *ImageComponentV2DataStoreV1FilterTestSuite) TestCountExcludesV1() { + expectedComponents := s.expectedV2Components() + + count, err := s.componentStore.Count(s.ctx, search.EmptyQuery()) + s.Require().NoError(err) + s.Assert().Equal(expectedComponents.Cardinality(), count) +} + +func (s *ImageComponentV2DataStoreV1FilterTestSuite) TestSearchExcludesV1() { + expectedComponents := s.expectedV2Components() + + results, err := s.componentStore.Search(s.ctx, search.EmptyQuery()) + s.Require().NoError(err) + s.Assert().Equal(expectedComponents.Cardinality(), len(results)) +} + +func (s *ImageComponentV2DataStoreV1FilterTestSuite) TestSearchRawImageComponentsExcludesV1() { + expectedComponents := s.expectedV2Components() + + components, err := s.componentStore.SearchRawImageComponents(s.ctx, search.EmptyQuery()) + s.Require().NoError(err) + + returnedComponents := set.NewStringSet() + for _, comp := range components { + returnedComponents.Add(comp.GetName()) + } + s.Assert().Equal(expectedComponents, returnedComponents) +} + +func (s *ImageComponentV2DataStoreV1FilterTestSuite) TestSearchImageComponentsExcludesV1() { + expectedComponents := s.expectedV2Components() + + results, err := s.componentStore.SearchImageComponents(s.ctx, search.EmptyQuery()) + s.Require().NoError(err) + s.Assert().Equal(expectedComponents.Cardinality(), len(results)) +} diff --git a/central/imagecomponent/v2/datastore/singleton.go b/central/imagecomponent/v2/datastore/singleton.go index 8ec83bfbd1828..f971aadb4b37c 100644 --- a/central/imagecomponent/v2/datastore/singleton.go +++ b/central/imagecomponent/v2/datastore/singleton.go @@ -15,8 +15,9 @@ var ( ) func initialize() { - storage := pgStore.New(globaldb.GetPostgres()) - ad = New(storage, riskDataStore.Singleton(), ranking.ComponentRanker()) + db := globaldb.GetPostgres() + storage := pgStore.New(db) + ad = New(db, storage, riskDataStore.Singleton(), ranking.ComponentRanker()) } // Singleton provides the interface for non-service external interaction. diff --git a/central/imagecomponent/v2/datastore/store/postgres/store.go b/central/imagecomponent/v2/datastore/store/postgres/store.go index 0faadd18f8b83..ada2d8b3ae40c 100644 --- a/central/imagecomponent/v2/datastore/store/postgres/store.go +++ b/central/imagecomponent/v2/datastore/store/postgres/store.go @@ -108,14 +108,14 @@ func insertIntoImageComponentV2(batch *pgx.Batch, obj *storage.ImageComponentV2) obj.GetRiskScore(), obj.GetTopCvss(), obj.GetOperatingSystem(), - obj.GetImageId(), + pgutils.NilOrString(obj.GetImageId()), obj.GetLocation(), pgutils.NilOrString(obj.GetImageIdV2()), - obj.GetFromBaseImage(), + obj.GetLayerType(), serialized, } - finalStr := "INSERT INTO image_component_v2 (Id, Name, Version, Priority, Source, RiskScore, TopCvss, OperatingSystem, ImageId, Location, ImageIdV2, FromBaseImage, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, Name = EXCLUDED.Name, Version = EXCLUDED.Version, Priority = EXCLUDED.Priority, Source = EXCLUDED.Source, RiskScore = EXCLUDED.RiskScore, TopCvss = EXCLUDED.TopCvss, OperatingSystem = EXCLUDED.OperatingSystem, ImageId = EXCLUDED.ImageId, Location = EXCLUDED.Location, ImageIdV2 = EXCLUDED.ImageIdV2, FromBaseImage = EXCLUDED.FromBaseImage, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO image_component_v2 (Id, Name, Version, Priority, Source, RiskScore, TopCvss, OperatingSystem, ImageId, Location, ImageIdV2, LayerType, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, Name = EXCLUDED.Name, Version = EXCLUDED.Version, Priority = EXCLUDED.Priority, Source = EXCLUDED.Source, RiskScore = EXCLUDED.RiskScore, TopCvss = EXCLUDED.TopCvss, OperatingSystem = EXCLUDED.OperatingSystem, ImageId = EXCLUDED.ImageId, Location = EXCLUDED.Location, ImageIdV2 = EXCLUDED.ImageIdV2, LayerType = EXCLUDED.LayerType, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) return nil @@ -133,7 +133,7 @@ var copyColsImageComponentV2 = []string{ "imageid", "location", "imageidv2", - "frombaseimage", + "layertype", "serialized", } @@ -176,10 +176,10 @@ func copyFromImageComponentV2(ctx context.Context, s pgSearch.Deleter, tx *postg obj.GetRiskScore(), obj.GetTopCvss(), obj.GetOperatingSystem(), - obj.GetImageId(), + pgutils.NilOrString(obj.GetImageId()), obj.GetLocation(), pgutils.NilOrString(obj.GetImageIdV2()), - obj.GetFromBaseImage(), + obj.GetLayerType(), serialized, }, nil }) diff --git a/central/imagecomponent/v2/views/component_risk_view.go b/central/imagecomponent/v2/views/component_risk_view.go new file mode 100644 index 0000000000000..a2386f60d401d --- /dev/null +++ b/central/imagecomponent/v2/views/component_risk_view.go @@ -0,0 +1,8 @@ +package views + +// ComponentRiskView is a lightweight view struct for initializing component rankers. +// It fetches only the component ID and risk score, avoiding full protobuf deserialization. +type ComponentRiskView struct { + ComponentID string `db:"component_id"` + ComponentRiskScore float32 `db:"component_risk_score"` +} diff --git a/central/imageintegration/datastore/datastore_impl.go b/central/imageintegration/datastore/datastore_impl.go index 1a7ae675f2670..7f6e719ce80d5 100644 --- a/central/imageintegration/datastore/datastore_impl.go +++ b/central/imageintegration/datastore/datastore_impl.go @@ -152,7 +152,3 @@ func (c *ImageIntegrationSearchResultConverter) BuildLocation(result *searchPkg. func (c *ImageIntegrationSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_IMAGE_INTEGRATIONS } - -func (c *ImageIntegrationSearchResultConverter) GetScore(result *searchPkg.Result) float64 { - return result.Score -} diff --git a/central/imageintegration/datastore/datastore_impl_test.go b/central/imageintegration/datastore/datastore_impl_test.go index bf3353d56f806..36c4020ddbeec 100644 --- a/central/imageintegration/datastore/datastore_impl_test.go +++ b/central/imageintegration/datastore/datastore_impl_test.go @@ -941,8 +941,8 @@ func (suite *ImageIntegrationDataStoreTestSuite) TestSearchImageIntegrationsFiel suite.NoError(err) suite.Len(searchResults, 2) - var actualIds []string - var actualNames []string + actualIds := make([]string, 0, len(searchResults)) + actualNames := make([]string, 0, len(searchResults)) for _, result := range searchResults { actualIds = append(actualIds, result.GetId()) actualNames = append(actualNames, result.GetName()) diff --git a/central/imageintegration/datastore/singleton.go b/central/imageintegration/datastore/singleton.go index 79027486d359e..f54a5acc8fd93 100644 --- a/central/imageintegration/datastore/singleton.go +++ b/central/imageintegration/datastore/singleton.go @@ -2,13 +2,16 @@ package datastore import ( "context" + "slices" "github.com/pkg/errors" "github.com/stackrox/rox/central/globaldb" "github.com/stackrox/rox/central/imageintegration/store" pgStore "github.com/stackrox/rox/central/imageintegration/store/postgres" + integrationHealthDS "github.com/stackrox/rox/central/integrationhealth/datastore" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/openshift" "github.com/stackrox/rox/pkg/sac" @@ -40,6 +43,8 @@ func initializeIntegrations(iiStore store.Store) { deleteAutogeneratedRegistries(ctx, iiStore, iis) setupScannerV4Integration(ctx, iiStore, iis) + + setupLegacyScannerIntegration(ctx, iiStore, integrationHealthDS.Singleton(), iis) } func deleteAutogeneratedRegistries(ctx context.Context, iiStore store.Store, iis []*storage.ImageIntegration) { @@ -128,19 +133,46 @@ func setupScannerV4Integration(ctx context.Context, iiStore store.Store, iis []* } // createDefaultScannerV4Integration will create the default Scanner V4 integration if it does -// not currently exist. +// not currently exist, or reconcile its categories if they differ from the default. func createDefaultScannerV4Integration(ctx context.Context, iiStore store.Store) { - if _, exists, err := iiStore.Get(ctx, store.DefaultScannerV4Integration.GetId()); err != nil { + defaultID := store.DefaultScannerV4Integration.GetId() + existing, exists, err := iiStore.Get(ctx, defaultID) + if err != nil { utils.Should(errors.Wrap(err, "unable to detect if default Scanner V4 integration exists")) return - } else if exists { - // Nothing to do, integration exists. + } + + if !exists { + log.Infof("Upserting default Scanner V4 integration %q (%s)", store.DefaultScannerV4Integration.GetName(), defaultID) + err := iiStore.Upsert(ctx, store.DefaultScannerV4Integration) + utils.Should(errors.Wrap(err, "unable to upsert default ScannerV4 integration")) + return + } + + // Restore default categories on records that predate them. The API rejects category changes, + // so this only repairs older data. Preserve other settings. + if categoriesMatch(existing.GetCategories(), store.DefaultScannerV4Integration.GetCategories()) { return } - log.Infof("Upserting default Scanner V4 integration %q (%s)", store.DefaultScannerV4Integration.GetName(), store.DefaultScannerV4Integration.GetId()) - err := iiStore.Upsert(ctx, store.DefaultScannerV4Integration) - utils.Should(errors.Wrap(err, "unable to upsert default ScannerV4 integration")) + log.Warnf("Reconciling categories for default Scanner V4 integration %q (%s): updating from %v to %v", + existing.GetName(), defaultID, existing.GetCategories(), store.DefaultScannerV4Integration.GetCategories()) + + existing.Categories = store.DefaultScannerV4Integration.GetCategories() + err = iiStore.Upsert(ctx, existing) + utils.Should(errors.Wrap(err, "unable to reconcile default ScannerV4 integration categories")) +} + +// categoriesMatch returns true if two category slices contain the same elements (order-independent). +func categoriesMatch(a, b []storage.ImageIntegrationCategory) bool { + if len(a) != len(b) { + return false + } + a = slices.Clone(a) + b = slices.Clone(b) + slices.Sort(a) + slices.Sort(b) + return slices.Equal(a, b) } // deleteScannerV4Integrations will delete all Scanner V4 integrations except for @@ -172,6 +204,43 @@ func deleteScannerV4Integrations(ctx context.Context, iiStore store.Store, iis [ log.Infof("Deleted %d Scanner V4 integrations", len(idsToDelete)) } +// setupLegacyScannerIntegration will delete all Clairify (legacy scanner) integrations +// and their health records if the legacy scanner feature is disabled. +func setupLegacyScannerIntegration(ctx context.Context, iiStore store.Store, healthDS integrationHealthDS.DataStore, iis []*storage.ImageIntegration) { + if features.LegacyScanner.Enabled() { + // Legacy scanner integration setup happens elsewhere. + return + } + deleteClairifyIntegrations(ctx, iiStore, healthDS, iis) +} + +func deleteClairifyIntegrations(ctx context.Context, iiStore store.Store, healthDS integrationHealthDS.DataStore, iis []*storage.ImageIntegration) { + idsToDelete := make([]string, 0, len(iis)) + for _, integration := range iis { + if integration.GetType() != scannerTypes.Clairify { + continue + } + idsToDelete = append(idsToDelete, integration.GetId()) + } + + if len(idsToDelete) == 0 { + log.Debug("No Clairify integrations to delete") + return + } + + if err := iiStore.PruneMany(ctx, idsToDelete); err != nil { + utils.Should(errors.Wrap(err, "failed to delete Clairify integrations")) + } + + for _, id := range idsToDelete { + if err := healthDS.RemoveIntegrationHealth(ctx, id); err != nil && !errors.Is(err, errox.NotFound) { + utils.Should(errors.Wrapf(err, "failed to remove health for Clairify integration %s", id)) + } + } + + log.Infof("Deleted %d Clairify integration(s) and associated health records", len(idsToDelete)) +} + func initialize() { // Create underlying store and datastore. storage := pgStore.New(globaldb.GetPostgres()) diff --git a/central/imageintegration/datastore/singleton_test.go b/central/imageintegration/datastore/singleton_test.go index 4f0d3fe81f694..94b79b4e8b4d2 100644 --- a/central/imageintegration/datastore/singleton_test.go +++ b/central/imageintegration/datastore/singleton_test.go @@ -8,9 +8,11 @@ import ( "github.com/stackrox/rox/central/imageintegration/store" mockIIStore "github.com/stackrox/rox/central/imageintegration/store/mocks" + mockHealthDS "github.com/stackrox/rox/central/integrationhealth/datastore/mocks" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/buildinfo" "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/openshift" scannerTypes "github.com/stackrox/rox/pkg/scanners/types" @@ -147,12 +149,252 @@ func TestSetupScannerV4Integration(t *testing.T) { } }) - t.Run("do not try upsert if default integration exists", func(t *testing.T) { + t.Run("do not try upsert if default integration exists with correct categories", func(t *testing.T) { testutils.MustUpdateFeature(t, features.ScannerV4, true) s := mockIIStore.NewMockStore(gomock.NewController(t)) - s.EXPECT().Get(ctx, defID).Return(nil, true, nil) + existingCorrect := &storage.ImageIntegration{ + Id: defID, + Name: "Scanner V4", + Type: scannerTypes.ScannerV4, + Categories: []storage.ImageIntegrationCategory{ + storage.ImageIntegrationCategory_SCANNER, + storage.ImageIntegrationCategory_NODE_SCANNER, + }, + IntegrationConfig: &storage.ImageIntegration_ScannerV4{ + ScannerV4: &storage.ScannerV4Config{}, + }, + } + s.EXPECT().Get(ctx, defID).Return(existingCorrect, true, nil) s.EXPECT().PruneMany(ctx, []string{"ID-Other-ScannerV4"}) setupScannerV4Integration(ctx, s, iis) }) } + +func TestSetupLegacyScannerIntegration(t *testing.T) { + ctx := context.Background() + + iis := []*storage.ImageIntegration{ + {Id: "ID-ScannerV4", Type: scannerTypes.ScannerV4}, + {Id: "ID-Clairify-1", Type: scannerTypes.Clairify}, + {Id: "ID-Clairify-2", Type: scannerTypes.Clairify}, + } + + t.Run("do nothing when LegacyScanner is enabled", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) + ctrl := gomock.NewController(t) + s := mockIIStore.NewMockStore(ctrl) + h := mockHealthDS.NewMockDataStore(ctrl) + + setupLegacyScannerIntegration(ctx, s, h, iis) + }) + + t.Run("do nothing if no Clairify integrations", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.LegacyScanner, false) + ctrl := gomock.NewController(t) + s := mockIIStore.NewMockStore(ctrl) + h := mockHealthDS.NewMockDataStore(ctrl) + + setupLegacyScannerIntegration(ctx, s, h, iis[:1]) + }) + + t.Run("delete Clairify integrations and health when feature disabled", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.LegacyScanner, false) + ctrl := gomock.NewController(t) + s := mockIIStore.NewMockStore(ctrl) + h := mockHealthDS.NewMockDataStore(ctrl) + + s.EXPECT().PruneMany(ctx, []string{"ID-Clairify-1", "ID-Clairify-2"}) + h.EXPECT().RemoveIntegrationHealth(ctx, "ID-Clairify-1").Return(nil) + h.EXPECT().RemoveIntegrationHealth(ctx, "ID-Clairify-2").Return(nil) + + setupLegacyScannerIntegration(ctx, s, h, iis) + }) + + t.Run("ignore NotFound when health record does not exist", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.LegacyScanner, false) + ctrl := gomock.NewController(t) + s := mockIIStore.NewMockStore(ctrl) + h := mockHealthDS.NewMockDataStore(ctrl) + + s.EXPECT().PruneMany(ctx, []string{"ID-Clairify-1", "ID-Clairify-2"}) + h.EXPECT().RemoveIntegrationHealth(ctx, "ID-Clairify-1").Return(errox.NotFound.New("not found")) + h.EXPECT().RemoveIntegrationHealth(ctx, "ID-Clairify-2").Return(nil) + + setupLegacyScannerIntegration(ctx, s, h, iis) + }) +} + +func TestCreateDefaultScannerV4Integration(t *testing.T) { + ctx := context.Background() + defaultID := store.DefaultScannerV4Integration.GetId() + + t.Run("create default integration when it does not exist", func(t *testing.T) { + s := mockIIStore.NewMockStore(gomock.NewController(t)) + + s.EXPECT().Get(ctx, defaultID).Return(nil, false, nil) + s.EXPECT().Upsert(ctx, store.DefaultScannerV4Integration).Return(nil) + + createDefaultScannerV4Integration(ctx, s) + }) + + t.Run("reconcile categories when existing integration has SCANNER only", func(t *testing.T) { + s := mockIIStore.NewMockStore(gomock.NewController(t)) + + existingScannerOnly := &storage.ImageIntegration{ + Id: defaultID, + Name: "Scanner V4", + Type: scannerTypes.ScannerV4, + Categories: []storage.ImageIntegrationCategory{ + storage.ImageIntegrationCategory_SCANNER, + }, + IntegrationConfig: &storage.ImageIntegration_ScannerV4{ + ScannerV4: &storage.ScannerV4Config{ + IndexerEndpoint: "custom-indexer:8443", + MatcherEndpoint: "custom-matcher:8443", + }, + }, + } + + s.EXPECT().Get(ctx, defaultID).Return(existingScannerOnly, true, nil) + s.EXPECT().Upsert(ctx, gomock.Any()).DoAndReturn(func(_ context.Context, ii *storage.ImageIntegration) error { + assert.ElementsMatch(t, store.DefaultScannerV4Integration.GetCategories(), ii.GetCategories()) + assert.Equal(t, "custom-indexer:8443", ii.GetScannerV4().GetIndexerEndpoint()) + assert.Equal(t, "custom-matcher:8443", ii.GetScannerV4().GetMatcherEndpoint()) + return nil + }) + + createDefaultScannerV4Integration(ctx, s) + }) + + t.Run("reconcile categories when existing integration has wrong categories", func(t *testing.T) { + s := mockIIStore.NewMockStore(gomock.NewController(t)) + + existingNodeOnly := &storage.ImageIntegration{ + Id: defaultID, + Name: "Scanner V4", + Type: scannerTypes.ScannerV4, + Categories: []storage.ImageIntegrationCategory{ + storage.ImageIntegrationCategory_NODE_SCANNER, + }, + IntegrationConfig: &storage.ImageIntegration_ScannerV4{ + ScannerV4: &storage.ScannerV4Config{ + IndexerEndpoint: "indexer:8080", + MatcherEndpoint: "matcher:8080", + }, + }, + } + + s.EXPECT().Get(ctx, defaultID).Return(existingNodeOnly, true, nil) + s.EXPECT().Upsert(ctx, gomock.Any()).DoAndReturn(func(_ context.Context, ii *storage.ImageIntegration) error { + assert.ElementsMatch(t, store.DefaultScannerV4Integration.GetCategories(), ii.GetCategories()) + assert.Equal(t, "indexer:8080", ii.GetScannerV4().GetIndexerEndpoint()) + return nil + }) + + createDefaultScannerV4Integration(ctx, s) + }) + + t.Run("reconcile categories when existing integration has nil categories", func(t *testing.T) { + s := mockIIStore.NewMockStore(gomock.NewController(t)) + + existingNilCategories := &storage.ImageIntegration{ + Id: defaultID, + Name: "Scanner V4", + Type: scannerTypes.ScannerV4, + Categories: nil, + IntegrationConfig: &storage.ImageIntegration_ScannerV4{ + ScannerV4: &storage.ScannerV4Config{ + IndexerEndpoint: "legacy-indexer:8443", + }, + }, + } + + s.EXPECT().Get(ctx, defaultID).Return(existingNilCategories, true, nil) + s.EXPECT().Upsert(ctx, gomock.Any()).DoAndReturn(func(_ context.Context, ii *storage.ImageIntegration) error { + assert.ElementsMatch(t, store.DefaultScannerV4Integration.GetCategories(), ii.GetCategories()) + assert.Equal(t, "legacy-indexer:8443", ii.GetScannerV4().GetIndexerEndpoint()) + return nil + }) + + createDefaultScannerV4Integration(ctx, s) + }) + + t.Run("do nothing when Get returns error", func(t *testing.T) { + s := mockIIStore.NewMockStore(gomock.NewController(t)) + + s.EXPECT().Get(ctx, defaultID).Return(nil, false, errors.New("database error")) + + if buildinfo.ReleaseBuild { + createDefaultScannerV4Integration(ctx, s) + } else { + assert.Panics(t, func() { createDefaultScannerV4Integration(ctx, s) }) + } + }) +} + +func TestCategoriesMatch(t *testing.T) { + testCases := map[string]struct { + a []storage.ImageIntegrationCategory + b []storage.ImageIntegrationCategory + expected bool + }{ + "empty slices match": { + []storage.ImageIntegrationCategory{}, + []storage.ImageIntegrationCategory{}, + true, + }, + "nil and empty slice match": { + nil, + []storage.ImageIntegrationCategory{}, + true, + }, + "same single element": { + []storage.ImageIntegrationCategory{storage.ImageIntegrationCategory_SCANNER}, + []storage.ImageIntegrationCategory{storage.ImageIntegrationCategory_SCANNER}, + true, + }, + "same multiple elements in same order": { + []storage.ImageIntegrationCategory{ + storage.ImageIntegrationCategory_SCANNER, + storage.ImageIntegrationCategory_NODE_SCANNER, + }, + []storage.ImageIntegrationCategory{ + storage.ImageIntegrationCategory_SCANNER, + storage.ImageIntegrationCategory_NODE_SCANNER, + }, + true, + }, + "same multiple elements in different order": { + []storage.ImageIntegrationCategory{ + storage.ImageIntegrationCategory_NODE_SCANNER, + storage.ImageIntegrationCategory_SCANNER, + }, + []storage.ImageIntegrationCategory{ + storage.ImageIntegrationCategory_SCANNER, + storage.ImageIntegrationCategory_NODE_SCANNER, + }, + true, + }, + "different lengths do not match": { + []storage.ImageIntegrationCategory{storage.ImageIntegrationCategory_SCANNER}, + []storage.ImageIntegrationCategory{ + storage.ImageIntegrationCategory_SCANNER, + storage.ImageIntegrationCategory_NODE_SCANNER, + }, + false, + }, + "different elements do not match": { + []storage.ImageIntegrationCategory{storage.ImageIntegrationCategory_SCANNER}, + []storage.ImageIntegrationCategory{storage.ImageIntegrationCategory_NODE_SCANNER}, + false, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + result := categoriesMatch(tc.a, tc.b) + assert.Equal(t, tc.expected, result) + }) + } +} diff --git a/central/imageintegration/service/service_impl.go b/central/imageintegration/service/service_impl.go index 40cc1c3148cb7..57eccf6e553c7 100644 --- a/central/imageintegration/service/service_impl.go +++ b/central/imageintegration/service/service_impl.go @@ -2,6 +2,7 @@ package service import ( "context" + "slices" "sort" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" @@ -131,6 +132,18 @@ func sortCategories(categories []storage.ImageIntegrationCategory) { }) } +// sameCategories returns true if both slices contain the same categories, ignoring order. +func sameCategories(a, b []storage.ImageIntegrationCategory) bool { + if len(a) != len(b) { + return false + } + a = slices.Clone(a) + b = slices.Clone(b) + slices.Sort(a) + slices.Sort(b) + return slices.Equal(a, b) +} + func (s *serviceImpl) validateTestAndNormalize(ctx context.Context, request *storage.ImageIntegration) error { if err := s.validateIntegration(ctx, request); err != nil { return err @@ -162,6 +175,12 @@ func (s *serviceImpl) PostImageIntegration(ctx context.Context, request *storage return nil, errors.Wrap(errox.InvalidArgs, "scanner V4 integration cannot be manually created") } + if request.GetType() == types.GoogleType { + return nil, errors.Wrap(errox.InvalidArgs, + "Google Container Registry (GCR) has been deprecated by Google and new integrations cannot be created. "+ + "Use Google Artifact Registry instead") + } + if err := s.validateTestAndNormalize(ctx, request); err != nil { return nil, errors.Wrap(errox.InvalidArgs, err.Error()) } @@ -398,6 +417,9 @@ func (s *serviceImpl) reconcileUpdateImageIntegrationRequest(ctx context.Context if newType != oldType && (newType == scannerTypes.ScannerV4 || oldType == scannerTypes.ScannerV4) { return errors.Wrap(errox.InvalidArgs, "cannot change integration type to/from scanner V4") } + if oldType == scannerTypes.ScannerV4 && !sameCategories(integration.GetCategories(), updateRequest.GetConfig().GetCategories()) { + return errors.Wrap(errox.InvalidArgs, "cannot change categories of scanner V4 integration") + } // Note that integrations of type "azure" support both `DockerConfig` (deprecated in 4.7) and `AzureConfig`. // Here we migrate requests with type "azure" and `DockerConfig` on the fly to type `AzureConfig`. diff --git a/central/imageintegration/service/service_impl_test.go b/central/imageintegration/service/service_impl_test.go index b61e32c26c604..7d930eb0e97a8 100644 --- a/central/imageintegration/service/service_impl_test.go +++ b/central/imageintegration/service/service_impl_test.go @@ -513,6 +513,66 @@ func TestScannerV4Restrictions(t *testing.T) { _, err := s.UpdateImageIntegration(context.Background(), &v1.UpdateImageIntegrationRequest{Config: iiNew}) assert.ErrorContains(t, err, "scanner V4") }) + + t.Run("prevent category change on scannerv4", func(t *testing.T) { + ctrl := gomock.NewController(t) + iiDS := integrationMocks.NewMockDataStore(ctrl) + s := &serviceImpl{datastore: iiDS} + + iiNew := ii.CloneVT() + iiNew.Categories = append(iiNew.Categories, storage.ImageIntegrationCategory_NODE_SCANNER) + + iiDS.EXPECT().GetImageIntegrations(gomock.Any(), gomock.Any()).Return([]*storage.ImageIntegration{ii}, nil) + iiDS.EXPECT().GetImageIntegration(gomock.Any(), gomock.Any()).Return(ii, true, nil) + + _, err := s.UpdateImageIntegration(context.Background(), &v1.UpdateImageIntegrationRequest{Config: iiNew}) + assert.ErrorIs(t, err, errox.InvalidArgs) + assert.ErrorContains(t, err, "categories of scanner V4") + }) +} + +func TestSameCategories(t *testing.T) { + scanner := storage.ImageIntegrationCategory_SCANNER + nodeScanner := storage.ImageIntegrationCategory_NODE_SCANNER + + cases := map[string]struct { + a, b []storage.ImageIntegrationCategory + expected bool + }{ + "both empty": {expected: true}, + "same order": {a: []storage.ImageIntegrationCategory{scanner, nodeScanner}, b: []storage.ImageIntegrationCategory{scanner, nodeScanner}, expected: true}, + "different order": {a: []storage.ImageIntegrationCategory{nodeScanner, scanner}, b: []storage.ImageIntegrationCategory{scanner, nodeScanner}, expected: true}, + "category added": {a: []storage.ImageIntegrationCategory{scanner}, b: []storage.ImageIntegrationCategory{scanner, nodeScanner}, expected: false}, + "category removed": {a: []storage.ImageIntegrationCategory{scanner, nodeScanner}, b: []storage.ImageIntegrationCategory{scanner}, expected: false}, + "category swapped": {a: []storage.ImageIntegrationCategory{scanner}, b: []storage.ImageIntegrationCategory{nodeScanner}, expected: false}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + assert.Equal(t, c.expected, sameCategories(c.a, c.b)) + }) + } +} + +func TestGCRCreationBlocked(t *testing.T) { + s := &serviceImpl{} + + ii := &storage.ImageIntegration{ + Name: "my-gcr", + Type: registryTypes.GoogleType, + Categories: []storage.ImageIntegrationCategory{ + storage.ImageIntegrationCategory_REGISTRY, + }, + IntegrationConfig: &storage.ImageIntegration_Google{ + Google: &storage.GoogleConfig{ + Endpoint: "gcr.io", + }, + }, + } + + _, err := s.PostImageIntegration(context.Background(), ii) + assert.ErrorIs(t, err, errox.InvalidArgs) + assert.ErrorContains(t, err, "Google Container Registry") + assert.ErrorContains(t, err, "deprecated") } func TestIAMRoleCapabilityValidation(t *testing.T) { @@ -552,6 +612,7 @@ func TestIAMRoleCapabilityValidation(t *testing.T) { storage.ImageIntegrationCategory_REGISTRY, }, IntegrationConfig: &storage.ImageIntegration_Ecr{ + // #nosec G101 -- test data Ecr: &storage.ECRConfig{ RegistryId: "123456789012", Region: "us-east-1", diff --git a/central/imageintegration/store/defaults.go b/central/imageintegration/store/defaults.go index 50d2e7dc1a485..410f78f5e4ffa 100644 --- a/central/imageintegration/store/defaults.go +++ b/central/imageintegration/store/defaults.go @@ -2,13 +2,16 @@ package store import ( "fmt" + "io" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" registryTypes "github.com/stackrox/rox/pkg/registries/types" "github.com/stackrox/rox/pkg/scanners" "github.com/stackrox/rox/pkg/scanners/clairify" "github.com/stackrox/rox/pkg/scanners/scannerv4" scannerTypes "github.com/stackrox/rox/pkg/scanners/types" + "github.com/stackrox/rox/pkg/utils" ) // DefaultImageIntegrations are the default public registries @@ -159,6 +162,9 @@ func makeDelayedIntegration(imageIntegration *storage.ImageIntegration, creatorF if err != nil { return false } + if closer, ok := scanner.(io.Closer); ok { + defer utils.IgnoreError(closer.Close) + } return scanner.Test() == nil }, } @@ -179,12 +185,19 @@ var ( }, }, } +) + +// GetDelayedIntegrations returns default integrations to be added only when +// the trigger function returns true. +func GetDelayedIntegrations() []DelayedIntegration { + if !features.LegacyScanner.Enabled() { + return nil + } - // DelayedIntegrations are default integrations to be added only when the trigger function returns true - DelayedIntegrations = []DelayedIntegration{ + return []DelayedIntegration{ makeDelayedIntegration(defaultScanner, func() scanners.Creator { _, creator := clairify.Creator(nil) return creator }), } -) +} diff --git a/central/imageintegration/store/defaults_test.go b/central/imageintegration/store/defaults_test.go new file mode 100644 index 0000000000000..b5f814a4f511e --- /dev/null +++ b/central/imageintegration/store/defaults_test.go @@ -0,0 +1,133 @@ +package store + +import ( + "errors" + "testing" + + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/scanners" + scannerTypes "github.com/stackrox/rox/pkg/scanners/types" + "github.com/stackrox/rox/pkg/testutils" + "github.com/stretchr/testify/assert" + "golang.org/x/sync/semaphore" +) + +func TestGetDelayedIntegrations(t *testing.T) { + tests := map[string]struct { + legacyScannerEnabled bool + expectCount int + }{ + "when LegacyScanner is enabled, returns Clairify scanner": { + legacyScannerEnabled: true, + expectCount: 1, + }, + "when LegacyScanner is disabled, returns empty list": { + legacyScannerEnabled: false, + expectCount: 0, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + testutils.MustUpdateFeature(t, features.LegacyScanner, tc.legacyScannerEnabled) + + integrations := GetDelayedIntegrations() + + if tc.expectCount == 0 { + assert.Empty(t, integrations) + } else { + assert.Len(t, integrations, tc.expectCount) + assert.Equal(t, defaultScanner.GetName(), integrations[0].Integration.GetName()) + } + }) + } +} + +type mockScanner struct { + testErr error +} + +func (m *mockScanner) MaxConcurrentScanSemaphore() *semaphore.Weighted { return nil } +func (m *mockScanner) GetScan(_ *storage.Image) (*storage.ImageScan, error) { + return nil, nil +} +func (m *mockScanner) Match(_ *storage.ImageName) bool { return false } +func (m *mockScanner) Test() error { return m.testErr } +func (m *mockScanner) Type() string { return "mock" } +func (m *mockScanner) Name() string { return "mock" } +func (m *mockScanner) GetVulnDefinitionsInfo() (*v1.VulnDefinitionsInfo, error) { + return nil, nil +} + +type mockClosableScanner struct { + mockScanner + closeCalled bool +} + +func (m *mockClosableScanner) Close() error { + m.closeCalled = true + return nil +} + +func TestMakeDelayedIntegration(t *testing.T) { + integration := &storage.ImageIntegration{Id: "test-id", Name: "test"} + + tests := map[string]struct { + creatorErr error + testErr error + closable bool + expectTrigger bool + expectClosed bool + }{ + "scanner test succeeds and Close is called": { + closable: true, + expectTrigger: true, + expectClosed: true, + }, + "scanner test fails and Close is called": { + testErr: errors.New("connection refused"), + closable: true, + expectTrigger: false, + expectClosed: true, + }, + "creator returns error so no scanner to close": { + creatorErr: errors.New("bad config"), + expectTrigger: false, + expectClosed: false, + }, + "scanner without io.Closer works without panic": { + closable: false, + expectTrigger: true, + expectClosed: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + var closable *mockClosableScanner + + creatorFactory := func() scanners.Creator { + return func(_ *storage.ImageIntegration) (scannerTypes.Scanner, error) { + if tc.creatorErr != nil { + return nil, tc.creatorErr + } + if tc.closable { + closable = &mockClosableScanner{mockScanner: mockScanner{testErr: tc.testErr}} + return closable, nil + } + return &mockScanner{testErr: tc.testErr}, nil + } + } + + di := makeDelayedIntegration(integration, creatorFactory) + result := di.Trigger() + + assert.Equal(t, tc.expectTrigger, result) + if tc.expectClosed { + assert.True(t, closable.closeCalled, "Close() should have been called") + } + }) + } +} diff --git a/central/imagev2/common/query.go b/central/imagev2/common/query.go new file mode 100644 index 0000000000000..92ca0ec3b7e9d --- /dev/null +++ b/central/imagev2/common/query.go @@ -0,0 +1,26 @@ +package common + +import ( + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/search" +) + +// WithRowsFromImageV2Only decorates the query to exclude V1 image CVE and component rows when FlattenImageData is enabled. +func WithRowsFromImageV2Only(q *v1.Query) *v1.Query { + if !features.FlattenImageData.Enabled() { + return q + } + // V1 rows have NULL imagev2id, so this query will exclude them. + v2Filter := search.NewQueryBuilder(). + AddStrings(search.ImageID, search.WildcardString). + ProtoQuery() + pagination := q.GetPagination() + selects := q.GetSelects() + groupBy := q.GetGroupBy() + query := search.ConjunctionQuery(v2Filter, q) + query.Pagination = pagination + query.Selects = selects + query.GroupBy = groupBy + return query +} diff --git a/central/imagev2/datastore/datastore.go b/central/imagev2/datastore/datastore.go index 822590f67a77d..9bb673d692c26 100644 --- a/central/imagev2/datastore/datastore.go +++ b/central/imagev2/datastore/datastore.go @@ -4,7 +4,6 @@ import ( "context" "github.com/stackrox/rox/central/imagev2/datastore/store" - "github.com/stackrox/rox/central/imagev2/views" "github.com/stackrox/rox/central/ranking" riskDS "github.com/stackrox/rox/central/risk/datastore" v1 "github.com/stackrox/rox/generated/api/v1" @@ -20,6 +19,7 @@ type DataStore interface { Count(ctx context.Context, q *v1.Query) (int, error) SearchImages(ctx context.Context, q *v1.Query) ([]*v1.SearchResult, error) SearchRawImages(ctx context.Context, q *v1.Query) ([]*storage.ImageV2, error) + SearchListImages(ctx context.Context, q *v1.Query) ([]*storage.ListImage, error) GetImage(ctx context.Context, id string) (*storage.ImageV2, bool, error) GetImageMetadata(ctx context.Context, id string) (*storage.ImageV2, bool, error) @@ -33,8 +33,6 @@ type DataStore interface { DeleteImages(ctx context.Context, ids ...string) error Exists(ctx context.Context, id string) (bool, error) - - GetImageIDsAndDigests(ctx context.Context, q *v1.Query) ([]*views.ImageIDAndDigestView, error) } // NewWithPostgres returns a new instance of DataStore using the input store, and searcher. diff --git a/central/imagev2/datastore/datastore_bench_postgres_test.go b/central/imagev2/datastore/datastore_bench_postgres_test.go index db7d1248ca418..2777b90089cfa 100644 --- a/central/imagev2/datastore/datastore_bench_postgres_test.go +++ b/central/imagev2/datastore/datastore_bench_postgres_test.go @@ -36,7 +36,7 @@ func BenchmarkImageGetMany(b *testing.B) { ids := make([]string, 0, 100) images := make([]*storage.ImageV2, 0, 100) - for i := 0; i < 100; i++ { + for i := range 100 { img := fixtures.GetImageV2WithUniqueComponents(5) id := fmt.Sprintf("%d", i) ids = append(ids, id) @@ -77,7 +77,7 @@ func BenchmarkImageUpsert(b *testing.B) { datastore := NewWithPostgres(pgStore.New(db, false, keyfence.ImageKeyFenceSingleton()), mockRisk, ranking.NewRanker(), ranking.NewRanker()) images := make([]*storage.ImageV2, 0, 100) - for i := 0; i < 100; i++ { + for range 100 { img := fixtures.GetImageV2WithUniqueComponents(5) img.Id = uuid.NewV4().String() images = append(images, img) diff --git a/central/imagev2/datastore/datastore_impl.go b/central/imagev2/datastore/datastore_impl.go index 461fb17c2bb27..b9e4238669ded 100644 --- a/central/imagev2/datastore/datastore_impl.go +++ b/central/imagev2/datastore/datastore_impl.go @@ -6,7 +6,6 @@ import ( "time" "github.com/pkg/errors" - "github.com/stackrox/rox/central/globaldb" "github.com/stackrox/rox/central/imagev2/datastore/store" "github.com/stackrox/rox/central/imagev2/views" "github.com/stackrox/rox/central/metrics" @@ -14,7 +13,6 @@ import ( riskDS "github.com/stackrox/rox/central/risk/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/errorhelpers" "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/logging" @@ -32,8 +30,6 @@ var ( ) type datastoreImpl struct { - keyedMutex *concurrency.KeyedMutex - storage store.Store risks riskDS.DataStore @@ -51,8 +47,6 @@ func newDatastoreImpl(storage store.Store, risks riskDS.DataStore, imageRanker: imageRanker, imageComponentRanker: imageComponentRanker, - - keyedMutex: concurrency.NewKeyedMutex(globaldb.DefaultDataStorePoolSize), } return ds } @@ -107,6 +101,24 @@ func (ds *datastoreImpl) SearchImages(ctx context.Context, q *v1.Query) ([]*v1.S return search.ResultsToSearchResultProtos(results, &ImageSearchResultConverter{}), nil } +func (ds *datastoreImpl) SearchListImages(ctx context.Context, q *v1.Query) ([]*storage.ListImage, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "ImageV2", "SearchListImages") + + results, err := ds.storage.GetListImagesView(ctx, q) + if err != nil { + return nil, err + } + + imgs := make([]*storage.ListImage, 0, len(results)) + for _, r := range results { + img := r.ToListImage() + img.Priority = ds.imageRanker.GetRankForID(r.Digest) + imgs = append(imgs, img) + } + + return imgs, nil +} + // TODO(ROX-29943): Eliminate unnecessary 2 pass database queries // SearchRawImages delegates to the underlying searcher. func (ds *datastoreImpl) SearchRawImages(ctx context.Context, q *v1.Query) ([]*storage.ImageV2, error) { @@ -268,9 +280,6 @@ func (ds *datastoreImpl) UpsertImage(ctx context.Context, image *storage.ImageV2 return sac.ErrResourceAccessDenied } - ds.keyedMutex.Lock(image.GetId()) - defer ds.keyedMutex.Unlock(image.GetId()) - ds.updateComponentRisk(image) utils.FillScanStatsV2(image) @@ -304,7 +313,7 @@ func (ds *datastoreImpl) DeleteImages(ctx context.Context, ids ...string) error return err } } - // removing component risk handled by pruning + // Component risks are not stored in the risk datastore (only embedded in components). return errorList.ToError() } @@ -354,11 +363,6 @@ func (ds *datastoreImpl) initializeRankers() { log.Infof("Initialized image ranking with %d images", len(results)) } -func (ds *datastoreImpl) GetImageIDsAndDigests(ctx context.Context, q *v1.Query) ([]*views.ImageIDAndDigestView, error) { - defer metrics.SetDatastoreFunctionDuration(time.Now(), "ImageV2", "GetImageIDsAndDigest") - return ds.storage.GetImagesIdAndDigestView(ctx, q) -} - func (ds *datastoreImpl) updateImagePriority(images ...*storage.ImageV2) { for _, image := range images { image.Priority = ds.imageRanker.GetRankForID(image.GetId()) diff --git a/central/imagev2/datastore/mapper/datastore/datastore_impl.go b/central/imagev2/datastore/mapper/datastore/datastore_impl.go index 69eb86475149a..31b63d3812be7 100644 --- a/central/imagev2/datastore/mapper/datastore/datastore_impl.go +++ b/central/imagev2/datastore/mapper/datastore/datastore_impl.go @@ -3,6 +3,7 @@ package datastore import ( "context" + "github.com/pkg/errors" imageDatastore "github.com/stackrox/rox/central/image/datastore" imageV2Datastore "github.com/stackrox/rox/central/imagev2/datastore" v1 "github.com/stackrox/rox/generated/api/v1" @@ -36,44 +37,7 @@ func (ds *datastoreImpl) SearchListImages(ctx context.Context, q *v1.Query) ([]* return ds.imageDataStore.SearchListImages(ctx, q) } - // Get image IDs from search (fast) - results, err := ds.imageV2DataStore.Search(ctx, q) - if err != nil { - return nil, err - } - - // Get image metadata in bulk (much faster than GetByIDs with full scan data) - ids := make([]string, 0, len(results)) - for _, result := range results { - ids = append(ids, result.ID) - } - - images, err := ds.imageV2DataStore.GetManyImageMetadata(ctx, ids) - if err != nil { - return nil, err - } - - // Build a map for O(1) lookup when reordering - imageByID := make(map[string]*storage.ImageV2, len(images)) - for _, image := range images { - imageByID[image.GetId()] = image - } - - // Convert to list images, preserving the original search order from results - listImages := make([]*storage.ListImage, 0, len(ids)) - for _, id := range ids { - image, exists := imageByID[id] - if !exists { - // Image may have been deleted between search and lookup, skip it - continue - } - // Convert v2 to v1 - v1Image := imageUtils.ConvertToV1(image) - // Convert v1 to ListImage - listImage := types.ConvertImageToListImage(v1Image) - listImages = append(listImages, listImage) - } - return listImages, nil + return ds.imageV2DataStore.SearchListImages(ctx, q) } func (ds *datastoreImpl) ListImage(ctx context.Context, sha string) (*storage.ListImage, bool, error) { @@ -152,7 +116,11 @@ func (ds *datastoreImpl) GetImage(ctx context.Context, sha string) (*storage.Ima if len(images) == 0 { return nil, false, nil } - return imageUtils.ConvertToV1(images[0]), true, nil + allNames, err := ds.imageV2DataStore.GetImageNames(ctx, sha) + if err != nil { + return nil, false, errors.Wrap(err, "getting image names") + } + return imageUtils.ConvertToV1(images[0], allNames...), true, nil } func (ds *datastoreImpl) GetImageMetadata(ctx context.Context, id string) (*storage.Image, bool, error) { @@ -218,6 +186,10 @@ func (ds *datastoreImpl) WalkByQuery(ctx context.Context, q *v1.Query, fn func(i }) } +func (ds *datastoreImpl) WalkMetadataByQuery(_ context.Context, _ *v1.Query, _ func(image *storage.Image) error) error { + return errors.New("WalkMetadataByQuery is not implemented for mapper datastore") +} + func (ds *datastoreImpl) UpsertImage(ctx context.Context, image *storage.Image) error { if !ds.flattenImageData { return ds.imageDataStore.UpsertImage(ctx, image) diff --git a/central/imagev2/datastore/mapper/datastore/datastore_impl_test.go b/central/imagev2/datastore/mapper/datastore/datastore_impl_test.go new file mode 100644 index 0000000000000..87f4a6a7fbf98 --- /dev/null +++ b/central/imagev2/datastore/mapper/datastore/datastore_impl_test.go @@ -0,0 +1,59 @@ +//go:build sql_integration + +package datastore + +import ( + "context" + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetImage_MergesAllNamesWhenMultipleV2ImagesShareDigest(t *testing.T) { + if !features.FlattenImageData.Enabled() { + t.Setenv("ROX_FLATTEN_IMAGE_DATA", "true") + } + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(t) + ds := GetTestPostgresDataStore(t, testDB.DB).(*datastoreImpl) + + digest := "sha256:abcdef1234567890" + + imageA := &storage.Image{ + Id: digest, + Name: &storage.ImageName{ + Registry: "quay.io", + Remote: "repo-a/image", + Tag: "latest", + FullName: "quay.io/repo-a/image:latest", + }, + } + imageB := &storage.Image{ + Id: digest, + Name: &storage.ImageName{ + Registry: "quay.io", + Remote: "repo-b/image", + Tag: "v1", + FullName: "quay.io/repo-b/image:v1", + }, + } + + require.NoError(t, ds.UpsertImage(ctx, imageA)) + require.NoError(t, ds.UpsertImage(ctx, imageB)) + + result, found, err := ds.GetImage(ctx, digest) + require.NoError(t, err) + require.True(t, found) + + fullNames := make([]string, 0, len(result.GetNames())) + for _, n := range result.GetNames() { + fullNames = append(fullNames, n.GetFullName()) + } + assert.Contains(t, fullNames, imageA.GetName().GetFullName()) + assert.Contains(t, fullNames, imageB.GetName().GetFullName()) +} diff --git a/central/imagev2/datastore/mapper/datastore/datastore_test_constructors.go b/central/imagev2/datastore/mapper/datastore/datastore_test_constructors.go new file mode 100644 index 0000000000000..5b3d3e2c26f42 --- /dev/null +++ b/central/imagev2/datastore/mapper/datastore/datastore_test_constructors.go @@ -0,0 +1,16 @@ +package datastore + +import ( + "testing" + + imageDatastore "github.com/stackrox/rox/central/image/datastore" + imageV2Datastore "github.com/stackrox/rox/central/imagev2/datastore" + "github.com/stackrox/rox/pkg/postgres" +) + +// GetTestPostgresDataStore provides a datastore connected to postgres for testing purposes. +func GetTestPostgresDataStore(t testing.TB, pool postgres.DB) imageDatastore.DataStore { + v1Store := imageDatastore.GetTestPostgresDataStore(t, pool) + v2Store := imageV2Datastore.GetTestPostgresDataStore(t, pool) + return New(v1Store, v2Store) +} diff --git a/central/imagev2/datastore/mocks/datastore.go b/central/imagev2/datastore/mocks/datastore.go index f5e1da8a66a98..3623815bbaf4c 100644 --- a/central/imagev2/datastore/mocks/datastore.go +++ b/central/imagev2/datastore/mocks/datastore.go @@ -13,7 +13,6 @@ import ( context "context" reflect "reflect" - views "github.com/stackrox/rox/central/imagev2/views" v1 "github.com/stackrox/rox/generated/api/v1" storage "github.com/stackrox/rox/generated/storage" search "github.com/stackrox/rox/pkg/search" @@ -109,21 +108,6 @@ func (mr *MockDataStoreMockRecorder) GetImage(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetImage", reflect.TypeOf((*MockDataStore)(nil).GetImage), ctx, id) } -// GetImageIDsAndDigests mocks base method. -func (m *MockDataStore) GetImageIDsAndDigests(ctx context.Context, q *v1.Query) ([]*views.ImageIDAndDigestView, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetImageIDsAndDigests", ctx, q) - ret0, _ := ret[0].([]*views.ImageIDAndDigestView) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetImageIDsAndDigests indicates an expected call of GetImageIDsAndDigests. -func (mr *MockDataStoreMockRecorder) GetImageIDsAndDigests(ctx, q any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetImageIDsAndDigests", reflect.TypeOf((*MockDataStore)(nil).GetImageIDsAndDigests), ctx, q) -} - // GetImageMetadata mocks base method. func (m *MockDataStore) GetImageMetadata(ctx context.Context, id string) (*storage.ImageV2, bool, error) { m.ctrl.T.Helper() @@ -215,6 +199,21 @@ func (mr *MockDataStoreMockRecorder) SearchImages(ctx, q any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchImages", reflect.TypeOf((*MockDataStore)(nil).SearchImages), ctx, q) } +// SearchListImages mocks base method. +func (m *MockDataStore) SearchListImages(ctx context.Context, q *v1.Query) ([]*storage.ListImage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchListImages", ctx, q) + ret0, _ := ret[0].([]*storage.ListImage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchListImages indicates an expected call of SearchListImages. +func (mr *MockDataStoreMockRecorder) SearchListImages(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchListImages", reflect.TypeOf((*MockDataStore)(nil).SearchListImages), ctx, q) +} + // SearchRawImages mocks base method. func (m *MockDataStore) SearchRawImages(ctx context.Context, q *v1.Query) ([]*storage.ImageV2, error) { m.ctrl.T.Helper() diff --git a/central/imagev2/datastore/store/common/parts_test.go b/central/imagev2/datastore/store/common/parts_test.go index 6a404295e7dba..3d8f59ac2f926 100644 --- a/central/imagev2/datastore/store/common/parts_test.go +++ b/central/imagev2/datastore/store/common/parts_test.go @@ -175,11 +175,13 @@ func TestSplitAndMergeImageV2(t *testing.T) { BaseImageId: "some-id", BaseImageFullName: "registry.example.com/ns/base:tag", BaseImageDigest: "sha256:...", + MaxLayerIndex: 3, }, { BaseImageId: "another-id", BaseImageFullName: "registry.example.com/ns/other:tag", BaseImageDigest: "sha256:...", + MaxLayerIndex: 3, }, }, Metadata: &storage.ImageMetadata{ @@ -323,11 +325,13 @@ func TestSplitAndMergeImageV2(t *testing.T) { { BaseImageId: "some-id", BaseImageFullName: "registry.example.com/ns/base:tag", + MaxLayerIndex: 3, BaseImageDigest: "sha256:...", }, { BaseImageId: "another-id", BaseImageFullName: "registry.example.com/ns/other:tag", + MaxLayerIndex: 3, BaseImageDigest: "sha256:...", }, }, @@ -342,7 +346,7 @@ func TestSplitAndMergeImageV2(t *testing.T) { HasLayerIndex: &storage.ImageComponentV2_LayerIndex{ LayerIndex: 1, }, - FromBaseImage: true, + LayerType: storage.LayerType_BASE_IMAGE, }, Children: []CVEPartsV2{}, }, @@ -355,7 +359,7 @@ func TestSplitAndMergeImageV2(t *testing.T) { HasLayerIndex: &storage.ImageComponentV2_LayerIndex{ LayerIndex: 3, }, - FromBaseImage: true, + LayerType: storage.LayerType_BASE_IMAGE, }, Children: []CVEPartsV2{ { @@ -416,7 +420,7 @@ func TestSplitAndMergeImageV2(t *testing.T) { HasLayerIndex: &storage.ImageComponentV2_LayerIndex{ LayerIndex: 2, }, - FromBaseImage: true, + LayerType: storage.LayerType_BASE_IMAGE, }, Children: []CVEPartsV2{ { @@ -460,7 +464,7 @@ func TestSplitAndMergeImageV2(t *testing.T) { HasLayerIndex: &storage.ImageComponentV2_LayerIndex{ LayerIndex: 2, }, - FromBaseImage: true, + LayerType: storage.LayerType_BASE_IMAGE, }, Children: []CVEPartsV2{ { diff --git a/central/imagev2/datastore/store/common/split.go b/central/imagev2/datastore/store/common/split.go index b581958775dea..60d4829f87145 100644 --- a/central/imagev2/datastore/store/common/split.go +++ b/central/imagev2/datastore/store/common/split.go @@ -100,7 +100,12 @@ func GenerateImageComponentV2(os string, image *storage.ImageV2, index int, from LayerIndex: from.GetLayerIndex(), } } - // TODO ROX-31847 compute image component base image layer type - ret.FromBaseImage = len(image.GetBaseImageInfo()) > 0 + + ret.LayerType = storage.LayerType_APPLICATION + if len(image.GetBaseImageInfo()) > 0 && from.GetHasLayerIndex() != nil { + if from.GetLayerIndex() <= image.GetBaseImageInfo()[0].GetMaxLayerIndex() { + ret.LayerType = storage.LayerType_BASE_IMAGE + } + } return ret, nil } diff --git a/central/imagev2/datastore/store/mocks/store.go b/central/imagev2/datastore/store/mocks/store.go index 28b878c939b56..18a71859b34e0 100644 --- a/central/imagev2/datastore/store/mocks/store.go +++ b/central/imagev2/datastore/store/mocks/store.go @@ -135,34 +135,34 @@ func (mr *MockStoreMockRecorder) GetImageMetadata(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetImageMetadata", reflect.TypeOf((*MockStore)(nil).GetImageMetadata), ctx, id) } -// GetImagesIdAndDigestView mocks base method. -func (m *MockStore) GetImagesIdAndDigestView(ctx context.Context, q *v1.Query) ([]*views.ImageIDAndDigestView, error) { +// GetImagesRiskView mocks base method. +func (m *MockStore) GetImagesRiskView(ctx context.Context, q *v1.Query) ([]*views.ImageV2RiskView, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetImagesIdAndDigestView", ctx, q) - ret0, _ := ret[0].([]*views.ImageIDAndDigestView) + ret := m.ctrl.Call(m, "GetImagesRiskView", ctx, q) + ret0, _ := ret[0].([]*views.ImageV2RiskView) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetImagesIdAndDigestView indicates an expected call of GetImagesIdAndDigestView. -func (mr *MockStoreMockRecorder) GetImagesIdAndDigestView(ctx, q any) *gomock.Call { +// GetImagesRiskView indicates an expected call of GetImagesRiskView. +func (mr *MockStoreMockRecorder) GetImagesRiskView(ctx, q any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetImagesIdAndDigestView", reflect.TypeOf((*MockStore)(nil).GetImagesIdAndDigestView), ctx, q) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetImagesRiskView", reflect.TypeOf((*MockStore)(nil).GetImagesRiskView), ctx, q) } -// GetImagesRiskView mocks base method. -func (m *MockStore) GetImagesRiskView(ctx context.Context, q *v1.Query) ([]*views.ImageV2RiskView, error) { +// GetListImagesView mocks base method. +func (m *MockStore) GetListImagesView(ctx context.Context, q *v1.Query) ([]*views.ListImageV2View, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetImagesRiskView", ctx, q) - ret0, _ := ret[0].([]*views.ImageV2RiskView) + ret := m.ctrl.Call(m, "GetListImagesView", ctx, q) + ret0, _ := ret[0].([]*views.ListImageV2View) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetImagesRiskView indicates an expected call of GetImagesRiskView. -func (mr *MockStoreMockRecorder) GetImagesRiskView(ctx, q any) *gomock.Call { +// GetListImagesView indicates an expected call of GetListImagesView. +func (mr *MockStoreMockRecorder) GetListImagesView(ctx, q any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetImagesRiskView", reflect.TypeOf((*MockStore)(nil).GetImagesRiskView), ctx, q) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetListImagesView", reflect.TypeOf((*MockStore)(nil).GetListImagesView), ctx, q) } // GetManyImageMetadata mocks base method. @@ -236,3 +236,17 @@ func (mr *MockStoreMockRecorder) WalkByQuery(ctx, q, fn any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WalkByQuery", reflect.TypeOf((*MockStore)(nil).WalkByQuery), ctx, q, fn) } + +// WalkMetadataByQuery mocks base method. +func (m *MockStore) WalkMetadataByQuery(ctx context.Context, q *v1.Query, fn func(*storage.ImageV2) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WalkMetadataByQuery", ctx, q, fn) + ret0, _ := ret[0].(error) + return ret0 +} + +// WalkMetadataByQuery indicates an expected call of WalkMetadataByQuery. +func (mr *MockStoreMockRecorder) WalkMetadataByQuery(ctx, q, fn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WalkMetadataByQuery", reflect.TypeOf((*MockStore)(nil).WalkMetadataByQuery), ctx, q, fn) +} diff --git a/central/imagev2/datastore/store/postgres/first_occurrence_bench_test.go b/central/imagev2/datastore/store/postgres/first_occurrence_bench_test.go new file mode 100644 index 0000000000000..e9c7f0c0026cd --- /dev/null +++ b/central/imagev2/datastore/store/postgres/first_occurrence_bench_test.go @@ -0,0 +1,186 @@ +//go:build sql_integration + +package postgres + +import ( + "context" + "fmt" + "testing" + "time" + + convertutils "github.com/stackrox/rox/central/cve/converter/utils" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/postgres/pgutils" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/protoutils" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// BenchmarkFirstImageOccurrence compares the selective column read +// (getImageCVETimestamps) against the previous full-deserialization approach +// for preserving FirstImageOccurrence timestamps during image rescans. +// +// Three cases exercise realistic overlap patterns: +// - 250 unique CVEs, no overlap (each component has distinct CVEs) +// - 100 unique CVEs across 350 rows (~3.5 rows per CVE, partial overlap) +// - 500 unique CVEs across 750 rows (~1.5 rows per CVE, moderate overlap) +func BenchmarkFirstImageOccurrence(b *testing.B) { + if !features.FlattenImageData.Enabled() { + b.Setenv("ROX_FLATTEN_IMAGE_DATA", "true") + } + + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(b) + s := New(testDB.DB, false, concurrency.NewKeyFence()) + storeImpl := s.(*storeImpl) + + for _, tc := range []struct { + name string + numComponents int + vulnsPerComp int + cvePoolSize int + }{ + // No overlap: 50 components × 5 CVEs, pool of 250 → each CVE appears once + {"250rows_no_overlap", 50, 5, 250}, + // Partial overlap: 50 components × 7 CVEs, pool of 100 → each CVE appears ~3.5 times + {"350rows_100cves", 50, 7, 100}, + // Moderate overlap at higher scale: 75 components × 10 CVEs, pool of 500 + {"750rows_500cves", 75, 10, 500}, + } { + image := buildBenchImage(tc.numComponents, tc.vulnsPerComp, tc.cvePoolSize) + require.NoError(b, s.Upsert(ctx, image)) + + // Count actual unique CVEs to validate the expected GROUP BY reduction. + tx, txCtx, err := storeImpl.begin(ctx) + require.NoError(b, err) + actualUnique, err := getImageCVETimestamps(txCtx, tx, image.GetId(), NewImageIDField) + require.NoError(b, err) + require.NoError(b, tx.Rollback(ctx)) + uniqueCount := len(actualUnique) + + var totalRows int + tx, txCtx, err = storeImpl.begin(ctx) + require.NoError(b, err) + require.NoError(b, tx.QueryRow(txCtx, + "SELECT COUNT(*) FROM "+imageComponentsV2CVEsTable+" WHERE imageidv2 = $1", + image.GetId()).Scan(&totalRows)) + require.NoError(b, tx.Rollback(ctx)) + + b.Logf("%s: %d total rows, %d unique CVEs (%.1fx reduction from GROUP BY)", + tc.name, totalRows, uniqueCount, float64(totalRows)/float64(uniqueCount)) + + b.Run(tc.name+"/SelectiveColumnRead", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + tx, txCtx, err := storeImpl.begin(ctx) + require.NoError(b, err) + result, err := getImageCVETimestamps(txCtx, tx, image.GetId(), NewImageIDField) + require.NoError(b, err) + require.Len(b, result, uniqueCount) + require.NoError(b, tx.Rollback(ctx)) + } + }) + + b.Run(tc.name+"/FullDeserialization", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + tx, txCtx, err := storeImpl.begin(ctx) + require.NoError(b, err) + result, err := getImageCVETimestampsFull(txCtx, tx, image.GetId(), NewImageIDField) + require.NoError(b, err) + require.Len(b, result, uniqueCount) + require.NoError(b, tx.Rollback(ctx)) + } + }) + } +} + +// getImageCVETimestampsFull reproduces the old approach: SELECT serialized, +// deserialize every row into a full ImageCVEV2 proto, convert to +// EmbeddedVulnerability, then extract timestamps. This is the baseline. +func getImageCVETimestampsFull(ctx context.Context, tx *postgres.Tx, imageID string, imageIDField string) (map[string]*timestamppb.Timestamp, error) { + rows, err := tx.Query(ctx, + "SELECT serialized FROM "+imageComponentsV2CVEsTable+" WHERE "+imageIDField+" = $1", + imageID) + if err != nil { + return nil, err + } + + imageCVEs, err := pgutils.ScanRows[storage.ImageCVEV2, *storage.ImageCVEV2](rows) + if err != nil { + return nil, err + } + + result := make(map[string]*timestamppb.Timestamp, len(imageCVEs)) + for _, cve := range imageCVEs { + vuln := convertutils.ImageCVEV2ToEmbeddedVulnerability(cve) + ts := vuln.GetFirstImageOccurrence() + if ts == nil { + continue + } + if existing, ok := result[vuln.GetCve()]; !ok || protoutils.After(existing, ts) { + result[vuln.GetCve()] = ts + } + } + return result, nil +} + +// buildBenchImage creates an image with partial CVE overlap across components. +// Each component draws vulnsPerComponent CVEs from a shared pool of cvePoolSize +// unique CVEs using a sliding window. This produces realistic overlap: adjacent +// components share some CVEs (like packages depending on the same library), +// while distant components have mostly distinct CVEs. +func buildBenchImage(numComponents, vulnsPerComponent, cvePoolSize int) *storage.ImageV2 { + now := time.Now() + stride := max(1, cvePoolSize/numComponents) + + components := make([]*storage.EmbeddedImageScanComponent, 0, numComponents) + for i := range numComponents { + vulns := make([]*storage.EmbeddedVulnerability, 0, vulnsPerComponent) + for j := range vulnsPerComponent { + cveIdx := (i*stride + j) % cvePoolSize + ts := now.Add(-time.Duration(cveIdx) * time.Hour) + vulns = append(vulns, &storage.EmbeddedVulnerability{ + Cve: fmt.Sprintf("CVE-2024-%05d", cveIdx), + Cvss: 5.0 + float32(j), + Severity: storage.VulnerabilitySeverity(1 + int32(j%4)), + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + VulnerabilityTypes: []storage.EmbeddedVulnerability_VulnerabilityType{storage.EmbeddedVulnerability_IMAGE_VULNERABILITY}, + Summary: fmt.Sprintf("A vulnerability in component %d affecting libfoo", i), + Link: fmt.Sprintf("https://nvd.nist.gov/vuln/detail/CVE-2024-%05d", cveIdx), + SetFixedBy: &storage.EmbeddedVulnerability_FixedBy{FixedBy: "1.0.1"}, + FirstImageOccurrence: protocompat.ConvertTimeToTimestampOrNil(&ts), + FirstSystemOccurrence: protocompat.ConvertTimeToTimestampOrNil(&ts), + }) + } + components = append(components, &storage.EmbeddedImageScanComponent{ + Name: fmt.Sprintf("pkg-%d", i), + Version: fmt.Sprintf("%d.0.0", i), + Vulns: vulns, + }) + } + + id := uuid.NewV4().String() + return &storage.ImageV2{ + Id: id, + Digest: "sha256:bench" + id[:8], + Name: &storage.ImageName{ + Registry: "registry.example.com", + Remote: "bench/image", + Tag: "latest", + FullName: "registry.example.com/bench/image:latest", + }, + Scan: &storage.ImageScan{ + ScanTime: protocompat.TimestampNow(), + OperatingSystem: "rhel:9", + Components: components, + }, + } +} diff --git a/central/imagev2/datastore/store/postgres/store.go b/central/imagev2/datastore/store/postgres/store.go index 9a43a71a66587..80d94e4117421 100644 --- a/central/imagev2/datastore/store/postgres/store.go +++ b/central/imagev2/datastore/store/postgres/store.go @@ -15,13 +15,16 @@ import ( "github.com/stackrox/rox/central/metrics" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/baseimage" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/logging" ops "github.com/stackrox/rox/pkg/metrics" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgutils" pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/protoutils" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/paginated" pgSearch "github.com/stackrox/rox/pkg/search/postgres" @@ -58,11 +61,6 @@ type imagePartsAsSlice struct { cvesV2 []*storage.ImageCVEV2 } -type timeFields struct { - createdAt time.Time - firstImageOccurrence time.Time -} - // TODO(ROX-28222): Refactor logic operating on other tables out and up to the datastore layer. // New returns a new Store instance using the provided sql instance. @@ -86,74 +84,8 @@ func (s *storeImpl) insertIntoImages( ctx context.Context, tx *postgres.Tx, parts *imagePartsAsSlice, metadataUpdated, scanUpdated bool, - iTime time.Time, + iTimestamp *timestamppb.Timestamp, ) error { - // First Image Occurrence and Created At are set based on the CVE itself, not the CVE ID - // within the image. Since a CVE can occur multiple times within an image we can grab - // those times for the incoming data and set the times appropriately. We will later go through the - // existing CVEs to make further adjustments if necessary to make sure we do not overwrite - // the times of previous occurrences. - cveTimeMap := make(map[string]*timeFields) - for _, cve := range parts.cvesV2 { - if val, ok := cveTimeMap[cve.GetCveBaseInfo().GetCve()]; ok { - if cve.GetCveBaseInfo().GetCreatedAt() != nil && val.createdAt.After(cve.GetCveBaseInfo().GetCreatedAt().AsTime()) { - val.createdAt = cve.GetCveBaseInfo().GetCreatedAt().AsTime() - } - if cve.GetFirstImageOccurrence() != nil && val.firstImageOccurrence.After(cve.GetFirstImageOccurrence().AsTime()) { - val.firstImageOccurrence = cve.GetFirstImageOccurrence().AsTime() - } - } else { - if cve.GetFirstImageOccurrence() == nil { - cve.FirstImageOccurrence = timestamppb.New(iTime) - } - if cve.GetCveBaseInfo().GetCreatedAt() == nil { - cve.GetCveBaseInfo().CreatedAt = timestamppb.New(iTime) - } - cveTimeMap[cve.GetCveBaseInfo().GetCve()] = &timeFields{ - createdAt: cve.GetCveBaseInfo().GetCreatedAt().AsTime(), - firstImageOccurrence: cve.GetFirstImageOccurrence().AsTime(), - } - } - } - - // Grab all CVEs for the image. - existingCVEs, err := getImageCVEs(ctx, tx, parts.image.GetId(), NewImageIDField) - if err != nil { - return err - } - - if len(existingCVEs) == 0 { - // If migrating from a version that already has the new CVE model, we try to get the existing CVEs from the new CVE model using the legacy image ID. - // This should only run the first time an image is scanned after migrating to the new image model. - existingCVEs, err = getImageCVEs(ctx, tx, parts.image.GetDigest(), LegacyImageIDField) - if err != nil { - return err - } - } - - if len(existingCVEs) == 0 { - // If migrating from a version that did not even have the new CVE model, we try to get the existing CVEs from the legacy CVE model. - // This should only run the first time an image is scanned after migrating to the new image model. - existingCVEs, err = getLegacyImageCVEs(ctx, tx, parts.image.GetDigest()) - if err != nil { - return err - } - } - - // Update CVE times based on existing CVEs - for _, cve := range existingCVEs { - // If the existing CVE is not already in the map that implies it no longer exists for this image and - // the CVE will be removed. - if val, ok := cveTimeMap[cve.GetCve()]; ok { - if cve.GetFirstSystemOccurrence() != nil && val.createdAt.After(cve.GetFirstSystemOccurrence().AsTime()) { - val.createdAt = cve.GetFirstSystemOccurrence().AsTime() - } - if cve.GetFirstImageOccurrence() != nil && val.firstImageOccurrence.After(cve.GetFirstImageOccurrence().AsTime()) { - val.firstImageOccurrence = cve.GetFirstImageOccurrence().AsTime() - } - } - } - cloned := parts.image // Since we are converting the component and CVE data embedded within the Image.Scan, we // need to clear that data out so that it is not stored with Image thus greatly duplicating data. @@ -203,12 +135,10 @@ func (s *storeImpl) insertIntoImages( } finalStr := "INSERT INTO " + imagesV2Table + " (Id, Digest, Name_Registry, Name_Remote, Name_Tag, Name_FullName, Metadata_V1_Created, Metadata_V1_User, Metadata_V1_Command, Metadata_V1_Entrypoint, Metadata_V1_Volumes, Metadata_V1_Labels, Scan_ScanTime, Scan_OperatingSystem, Signature_Fetched, ScanStats_ComponentCount, ScanStats_CveCount, ScanStats_FixableCveCount, ScanStats_UnknownCveCount, ScanStats_FixableUnknownCveCount, ScanStats_CriticalCveCount, ScanStats_FixableCriticalCveCount, ScanStats_ImportantCveCount, ScanStats_FixableImportantCveCount, ScanStats_ModerateCveCount, ScanStats_FixableModerateCveCount, ScanStats_LowCveCount, ScanStats_FixableLowCveCount, LastUpdated, Priority, RiskScore, TopCvss, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, Digest = EXCLUDED.Digest, Name_Registry = EXCLUDED.Name_Registry, Name_Remote = EXCLUDED.Name_Remote, Name_Tag = EXCLUDED.Name_Tag, Name_FullName = EXCLUDED.Name_FullName, Metadata_V1_Created = EXCLUDED.Metadata_V1_Created, Metadata_V1_User = EXCLUDED.Metadata_V1_User, Metadata_V1_Command = EXCLUDED.Metadata_V1_Command, Metadata_V1_Entrypoint = EXCLUDED.Metadata_V1_Entrypoint, Metadata_V1_Volumes = EXCLUDED.Metadata_V1_Volumes, Metadata_V1_Labels = EXCLUDED.Metadata_V1_Labels, Scan_ScanTime = EXCLUDED.Scan_ScanTime, Scan_OperatingSystem = EXCLUDED.Scan_OperatingSystem, Signature_Fetched = EXCLUDED.Signature_Fetched, ScanStats_ComponentCount = EXCLUDED.ScanStats_ComponentCount, ScanStats_CveCount = EXCLUDED.ScanStats_CveCount, ScanStats_FixableCveCount = EXCLUDED.ScanStats_FixableCveCount, ScanStats_UnknownCveCount = EXCLUDED.ScanStats_UnknownCveCount, ScanStats_FixableUnknownCveCount = EXCLUDED.ScanStats_FixableUnknownCveCount, ScanStats_CriticalCveCount = EXCLUDED.ScanStats_CriticalCveCount, ScanStats_FixableCriticalCveCount = EXCLUDED.ScanStats_FixableCriticalCveCount, ScanStats_ImportantCveCount = EXCLUDED.ScanStats_ImportantCveCount, ScanStats_FixableImportantCveCount = EXCLUDED.ScanStats_FixableImportantCveCount, ScanStats_ModerateCveCount = EXCLUDED.ScanStats_ModerateCveCount, ScanStats_FixableModerateCveCount = EXCLUDED.ScanStats_FixableModerateCveCount, ScanStats_LowCveCount = EXCLUDED.ScanStats_LowCveCount, ScanStats_FixableLowCveCount = EXCLUDED.ScanStats_FixableLowCveCount, LastUpdated = EXCLUDED.LastUpdated, Priority = EXCLUDED.Priority, RiskScore = EXCLUDED.RiskScore, TopCvss = EXCLUDED.TopCvss, serialized = EXCLUDED.serialized" - _, err = tx.Exec(ctx, finalStr, values...) - if err != nil { + if _, err := tx.Exec(ctx, finalStr, values...); err != nil { return err } - var query string if metadataUpdated { for childIdx, child := range cloned.GetMetadata().GetV1().GetLayers() { if err := insertIntoImagesLayers(ctx, tx, child, cloned.GetId(), childIdx); err != nil { @@ -216,9 +146,8 @@ func (s *storeImpl) insertIntoImages( } } - query = "DELETE FROM images_v2_Layers WHERE images_v2_Id = $1 AND idx >= $2" - _, err = tx.Exec(ctx, query, cloned.GetId(), len(cloned.GetMetadata().GetV1().GetLayers())) - if err != nil { + query := "DELETE FROM images_v2_Layers WHERE images_v2_Id = $1 AND idx >= $2" + if _, err := tx.Exec(ctx, query, cloned.GetId(), len(cloned.GetMetadata().GetV1().GetLayers())); err != nil { return err } } @@ -231,12 +160,62 @@ func (s *storeImpl) insertIntoImages( } common.SensorEventsDeduperCounter.With(prometheus.Labels{"status": "passed"}).Inc() + cveTimeMap := make(map[string]*timestamppb.Timestamp) + for _, cve := range parts.cvesV2 { + if cve.GetFirstImageOccurrence() == nil { + cve.FirstImageOccurrence = iTimestamp + } + if val, ok := cveTimeMap[cve.GetCveBaseInfo().GetCve()]; !ok || protoutils.After(val, cve.GetFirstImageOccurrence()) { + cveTimeMap[cve.GetCveBaseInfo().GetCve()] = cve.GetFirstImageOccurrence() + } + } + + // Grab existing FirstImageOccurrence timestamps for this image's CVEs. + // Uses a selective column read (CVE name + MIN timestamp) instead of + // deserializing full protobuf objects. + existingTimestamps, err := getImageCVETimestamps(ctx, tx, parts.image.GetId(), NewImageIDField) + if err != nil { + return err + } + + if len(existingTimestamps) == 0 { + existingTimestamps, err = getImageCVETimestamps(ctx, tx, parts.image.GetDigest(), LegacyImageIDField) + if err != nil { + return err + } + } + + if len(existingTimestamps) == 0 { + existingCVEs, legacyErr := getLegacyImageCVEs(ctx, tx, parts.image.GetDigest()) + if legacyErr != nil { + return legacyErr + } + existingTimestamps = make(map[string]*timestamppb.Timestamp, len(existingCVEs)) + for _, cve := range existingCVEs { + ts := cve.GetFirstImageOccurrence() + if ts == nil { + continue + } + if existing, ok := existingTimestamps[cve.GetCve()]; !ok || protoutils.After(existing, ts) { + existingTimestamps[cve.GetCve()] = ts + } + } + } + + for cveName, existingTS := range existingTimestamps { + if val, ok := cveTimeMap[cveName]; ok { + if existingTS != nil && protoutils.After(val, existingTS) { + cveTimeMap[cveName] = existingTS + } + } + } + err = s.copyFromImageComponentsV2(ctx, tx, parts.image.GetId(), parts.componentsV2...) if err != nil { return err } - return copyFromImageComponentV2Cves(ctx, tx, iTime, cveTimeMap, parts.cvesV2...) + return copyFromImageComponentV2Cves(ctx, tx, iTimestamp, cveTimeMap, parts.cvesV2...) } func getPartsAsSlice(parts common.ImagePartsV2) *imagePartsAsSlice { @@ -278,10 +257,7 @@ func (s *storeImpl) copyFromImageComponentsV2(ctx context.Context, tx *postgres. if err := s.deleteImageComponents(ctx, tx, imageID); err != nil { return err } - batchSize := pgSearch.MaxBatchSize - if len(objs) < batchSize { - batchSize = len(objs) - } + batchSize := min(len(objs), pgSearch.MaxBatchSize) inputRows := make([][]interface{}, 0, batchSize) copyCols := []string{ @@ -295,6 +271,7 @@ func (s *storeImpl) copyFromImageComponentsV2(ctx context.Context, tx *postgres. "operatingsystem", "imageidv2", "location", + "layertype", "serialized", } @@ -315,6 +292,7 @@ func (s *storeImpl) copyFromImageComponentsV2(ctx context.Context, tx *postgres. obj.GetOperatingSystem(), obj.GetImageIdV2(), obj.GetLocation(), + obj.GetLayerType(), serialized, }) @@ -331,11 +309,8 @@ func (s *storeImpl) copyFromImageComponentsV2(ctx context.Context, tx *postgres. return nil } -func copyFromImageComponentV2Cves(ctx context.Context, tx *postgres.Tx, iTime time.Time, cveTimeMap map[string]*timeFields, objs ...*storage.ImageCVEV2) error { - batchSize := pgSearch.MaxBatchSize - if len(objs) < batchSize { - batchSize = len(objs) - } +func copyFromImageComponentV2Cves(ctx context.Context, tx *postgres.Tx, iTimestamp *timestamppb.Timestamp, cveTimeMap map[string]*timestamppb.Timestamp, objs ...*storage.ImageCVEV2) error { + batchSize := min(len(objs), pgSearch.MaxBatchSize) inputRows := make([][]interface{}, 0, batchSize) copyCols := []string{ @@ -345,6 +320,8 @@ func copyFromImageComponentV2Cves(ctx context.Context, tx *postgres.Tx, iTime ti "cvebaseinfo_publishedon", "cvebaseinfo_createdat", "cvebaseinfo_epss_epssprobability", + "cvebaseinfo_cisakev", + "cvebaseinfo_knownransomwarecampaign", "cvss", "severity", "impactscore", @@ -356,21 +333,23 @@ func copyFromImageComponentV2Cves(ctx context.Context, tx *postgres.Tx, iTime ti "componentid", "advisory_name", "advisory_link", + "fixavailabletimestamp", + "origin", "serialized", } for idx, obj := range objs { - // If we have seen this CVE in the image already, set the times consistently. - if cveTimes := cveTimeMap[obj.GetCveBaseInfo().GetCve()]; cveTimes != nil { - obj.CveBaseInfo.CreatedAt = protocompat.ConvertTimeToTimestampOrNil(&cveTimes.createdAt) - obj.FirstImageOccurrence = protocompat.ConvertTimeToTimestampOrNil(&cveTimes.firstImageOccurrence) - } else { - if obj.GetCveBaseInfo().GetCreatedAt() == nil { - obj.CveBaseInfo.CreatedAt = protocompat.ConvertTimeToTimestampOrNil(&iTime) - } - if obj.GetFirstImageOccurrence() == nil { - obj.FirstImageOccurrence = protocompat.ConvertTimeToTimestampOrNil(&iTime) - } + // If we have seen this CVE in the image already, preserve the earliest FirstImageOccurrence. + if cveTime, ok := cveTimeMap[obj.GetCveBaseInfo().GetCve()]; ok { + obj.FirstImageOccurrence = cveTime + } + + // Final sanity Check + if obj.GetCveBaseInfo().GetCreatedAt() == nil { + obj.CveBaseInfo.CreatedAt = iTimestamp + } + if obj.GetFirstImageOccurrence() == nil { + obj.FirstImageOccurrence = iTimestamp } serialized, marshalErr := obj.MarshalVT() @@ -385,6 +364,8 @@ func copyFromImageComponentV2Cves(ctx context.Context, tx *postgres.Tx, iTime ti protocompat.NilOrTime(obj.GetCveBaseInfo().GetPublishedOn()), protocompat.NilOrTime(obj.GetCveBaseInfo().GetCreatedAt()), obj.GetCveBaseInfo().GetEpss().GetEpssProbability(), + obj.GetCveBaseInfo().GetCisaKev(), + obj.GetCveBaseInfo().GetKnownRansomwareCampaign(), obj.GetCvss(), obj.GetSeverity(), obj.GetImpactScore(), @@ -396,6 +377,8 @@ func copyFromImageComponentV2Cves(ctx context.Context, tx *postgres.Tx, iTime ti obj.GetComponentId(), obj.GetAdvisory().GetName(), obj.GetAdvisory().GetLink(), + protocompat.NilOrTime(obj.GetFixAvailableTimestamp()), + obj.GetOrigin(), serialized, }) @@ -439,10 +422,17 @@ func (s *storeImpl) isUpdated(oldImage, image *storage.ImageV2) (bool, bool, err type hashWrapper struct { Components []*storage.EmbeddedImageScanComponent `hash:"set"` + Count int + VulnCount int } func populateImageScanHash(scan *storage.ImageScan) error { - hash, err := hashstructure.Hash(hashWrapper{scan.GetComponents()}, &hashstructure.HashOptions{ZeroNil: true}) + components := scan.GetComponents() + vulnCount := 0 + for _, c := range components { + vulnCount += len(c.GetVulns()) + } + hash, err := hashstructure.Hash(hashWrapper{Components: components, Count: len(components), VulnCount: vulnCount}, &hashstructure.HashOptions{ZeroNil: true}) if err != nil { return errors.Wrap(err, "calculating hash for image scan") } @@ -474,10 +464,10 @@ func fillScanStatsFromExistingImage(oldImage *storage.ImageV2, image *storage.Im } func (s *storeImpl) upsert(ctx context.Context, obj *storage.ImageV2) error { - iTime := time.Now() + iTimestamp := protocompat.TimestampNow() if !s.noUpdateTimestamps { - obj.LastUpdated = protocompat.ConvertTimeToTimestampOrNil(&iTime) + obj.LastUpdated = iTimestamp } oldImage, _, err := s.GetImageMetadata(ctx, obj.GetId()) @@ -516,6 +506,13 @@ func (s *storeImpl) upsert(ctx context.Context, obj *storage.ImageV2) error { scanUpdated = scanUpdated || componentsEmpty + if features.BaseImageDetection.Enabled() { + // Re-verify base images when base image detection is enabled: + // 1. Legacy images may lack base image info if the feature was enabled after they were scanned. + // 2. User-provided base images may change over time. + scanUpdated = scanUpdated || baseimage.BaseImagesUpdated(oldImage.GetBaseImageInfo(), obj.GetBaseImageInfo()) + } + splitParts, err := common.Split(obj, scanUpdated) if err != nil { return err @@ -529,7 +526,7 @@ func (s *storeImpl) upsert(ctx context.Context, obj *storage.ImageV2) error { return err } - if err := s.insertIntoImages(ctx, tx, imageParts, metadataUpdated, scanUpdated, iTime); err != nil { + if err := s.insertIntoImages(ctx, tx, imageParts, metadataUpdated, scanUpdated, iTimestamp); err != nil { if errTx := tx.Rollback(ctx); errTx != nil { return errors.Wrapf(errTx, "rolling back transaction due to: %v", err) } @@ -612,29 +609,25 @@ func (s *storeImpl) populateImage(ctx context.Context, tx *postgres.Tx, image *s return err } + cvesByComponent, err := getAllImageComponentCVEs(ctx, tx, image.GetId()) + if err != nil { + return err + } + imageParts := common.ImagePartsV2{ Image: image, - Children: []common.ComponentPartsV2{}, + Children: make([]common.ComponentPartsV2, 0, len(components)), } for _, component := range components { - cves, err := getImageComponentCVEs(ctx, tx, component.GetId()) - if err != nil { - return err - } - + cves := cvesByComponent[component.GetId()] cveParts := make([]common.CVEPartsV2, 0, len(cves)) for _, cve := range cves { - cvePart := common.CVEPartsV2{ - CVEV2: cve, - } - cveParts = append(cveParts, cvePart) + cveParts = append(cveParts, common.CVEPartsV2{CVEV2: cve}) } - - child := common.ComponentPartsV2{ + imageParts.Children = append(imageParts.Children, common.ComponentPartsV2{ ComponentV2: component, Children: cveParts, - } - imageParts.Children = append(imageParts.Children, child) + }) } common.Merge(imageParts) return nil @@ -674,45 +667,59 @@ func getImageComponents(ctx context.Context, tx *postgres.Tx, imageID string) ([ return pgutils.ScanRows[storage.ImageComponentV2, *storage.ImageComponentV2](rows) } -func getImageComponentCVEs(ctx context.Context, tx *postgres.Tx, componentID string) ([]*storage.ImageCVEV2, error) { +func getAllImageComponentCVEs(ctx context.Context, tx *postgres.Tx, imageID string) (map[string][]*storage.ImageCVEV2, error) { defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Get, "ImageCVEsV2") - // Using this method instead of accessing the component store to ensure the query is in the same transaction as - // the updates. That may prove to not matter, but for now doing it this way. - rows, err := tx.Query(ctx, "SELECT serialized FROM "+imageComponentsV2CVEsTable+" WHERE componentid = $1", componentID) + rows, err := tx.Query(ctx, "SELECT serialized FROM "+imageComponentsV2CVEsTable+" WHERE imageidv2 = $1", imageID) if err != nil { return nil, err } - return pgutils.ScanRows[storage.ImageCVEV2, *storage.ImageCVEV2](rows) -} - -func getImageCVEs(ctx context.Context, tx *postgres.Tx, imageID string, imageIDField string) ([]*storage.EmbeddedVulnerability, error) { - defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Get, "ImageCVEsV2") - - // Using this method instead of accessing the component store to ensure the query is in the same transaction as - // the updates. That may prove to not matter, but for now doing it this way. - rows, err := tx.Query(ctx, "SELECT serialized FROM "+imageComponentsV2CVEsTable+" WHERE "+imageIDField+" = $1", imageID) + cves, err := pgutils.ScanRows[storage.ImageCVEV2, *storage.ImageCVEV2](rows) if err != nil { return nil, err } - var imageCVEs []*storage.ImageCVEV2 - imageCVEs, err = pgutils.ScanRows[storage.ImageCVEV2, *storage.ImageCVEV2](rows) + result := make(map[string][]*storage.ImageCVEV2) + for _, cve := range cves { + compID := cve.GetComponentId() + result[compID] = append(result[compID], cve) + } + return result, nil +} + +// getImageCVETimestamps returns the earliest FirstImageOccurrence per CVE name +// for the given image. It reads only the two columns needed (CVE name and +// timestamp) and aggregates with MIN in SQL, avoiding full protobuf +// deserialization. +func getImageCVETimestamps(ctx context.Context, tx *postgres.Tx, imageID string, imageIDField string) (map[string]*timestamppb.Timestamp, error) { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Get, "ImageCVEsV2") + + rows, err := tx.Query(ctx, + "SELECT cvebaseinfo_cve, MIN(firstimageoccurrence) FROM "+imageComponentsV2CVEsTable+ + " WHERE "+imageIDField+" = $1 GROUP BY cvebaseinfo_cve", + imageID) if err != nil { return nil, err } + defer rows.Close() - vulns := make([]*storage.EmbeddedVulnerability, 0, len(imageCVEs)) - for _, cve := range imageCVEs { - vulns = append(vulns, convertutils.ImageCVEV2ToEmbeddedVulnerability(cve)) + result := make(map[string]*timestamppb.Timestamp) + for rows.Next() { + var cveName string + var ts *time.Time + if err := rows.Scan(&cveName, &ts); err != nil { + return nil, err + } + if ts != nil { + result[cveName] = timestamppb.New(*ts) + } } - - return vulns, nil + return result, rows.Err() } -// The purpose of this function is to get legacy CVEs for the given imageID so that we can migrate the CVE created and -// first image occurrence timestamps to the new CVE data model. So we do not populate the fixedBy and vulnerability state -// in the returned vulns as that information is not necessary for migrating the timestamps. +// The purpose of this function is to get legacy CVEs for the given imageID so that we can migrate the +// FirstImageOccurrence timestamp to the new CVE data model. So we do not populate the fixedBy and vulnerability state +// in the returned vulns as that information is not necessary for migrating the timestamp. func getLegacyImageCVEs(ctx context.Context, tx *postgres.Tx, imageSha string) ([]*storage.EmbeddedVulnerability, error) { defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Get, "ImageCVEs") @@ -877,7 +884,19 @@ func (s *storeImpl) WalkByQuery(ctx context.Context, q *v1.Query, fn func(image } return nil } - err = pgSearch.RunCursorQueryForSchemaFn(ctx, pkgSchema.ImagesV2Schema, q, s.db, callback) + err = pgSearch.RunCursorQueryForSchemaFn(ctx, pkgSchema.ImagesV2Schema, q, s.db, "WalkByQuery", callback) + if err != nil { + return errors.Wrap(err, "cursor by query") + } + return nil +} + +func (s *storeImpl) WalkMetadataByQuery(ctx context.Context, q *v1.Query, fn func(image *storage.ImageV2) error) error { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.WalkMetadataByQuery, "Image") + + q = s.applyDefaultSort(q) + + err := pgSearch.RunCursorQueryForSchemaFn(ctx, pkgSchema.ImagesV2Schema, q, s.db, "WalkMetadataByQuery", fn) if err != nil { return errors.Wrap(err, "cursor by query") } @@ -924,8 +943,11 @@ func (s *storeImpl) retryableGetManyImageMetadata(ctx context.Context, ids []str // GetImagesRiskView retrieves an image id and risk score to initialize rankers func (s *storeImpl) GetImagesRiskView(ctx context.Context, q *v1.Query) ([]*views.ImageV2RiskView, error) { // The entire image is not needed to initialize the ranker. We only need the image id and risk score. - var results []*views.ImageV2RiskView - results, err := pgSearch.RunSelectRequestForSchema[views.ImageV2RiskView](ctx, s.db, pkgSchema.ImagesV2Schema, q) + results := make([]*views.ImageV2RiskView, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[views.ImageV2RiskView](ctx, s.db, pkgSchema.ImagesV2Schema, q, func(r *views.ImageV2RiskView) error { + results = append(results, r) + return nil + }) if err != nil { log.Errorf("unable to initialize image ranking: %v", err) } @@ -933,21 +955,34 @@ func (s *storeImpl) GetImagesRiskView(ctx context.Context, q *v1.Query) ([]*view return results, err } -// GetImagesIdAndDigestView retrieves an image id and digest for pruning purposes -func (s *storeImpl) GetImagesIdAndDigestView(ctx context.Context, q *v1.Query) ([]*views.ImageIDAndDigestView, error) { +// GetListImagesView retrieves the fields needed for ListImage responses. +func (s *storeImpl) GetListImagesView(ctx context.Context, q *v1.Query) ([]*views.ListImageV2View, error) { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Search, "ListImagesView") + + q = s.applyDefaultSort(q) + selects := []*v1.QuerySelect{ - search.NewQuerySelect(search.ImageID).Proto(), search.NewQuerySelect(search.ImageSHA).Proto(), + search.NewQuerySelect(search.ImageName).Proto(), + search.NewQuerySelect(search.ImageComponentCount).Proto(), + search.NewQuerySelect(search.ImageCVECount).Proto(), + search.NewQuerySelect(search.FixableCVECount).Proto(), + search.NewQuerySelect(search.ImageCreatedTime).Proto(), + search.NewQuerySelect(search.LastUpdatedTime).Proto(), + } + cloned := q.CloneVT() + cloned.Selects = selects + + // Add GROUP BY on the primary key to deduplicate results when joining with other tables. + cloned.GroupBy = &v1.QueryGroupBy{ + Fields: []string{search.ImageID.String()}, } - q.Selects = selects - var results []*views.ImageIDAndDigestView - err := pgSearch.RunSelectRequestForSchemaFn[views.ImageIDAndDigestView](ctx, s.db, pkgSchema.ImagesV2Schema, q, func(row *views.ImageIDAndDigestView) error { + + var results []*views.ListImageV2View + err := pgSearch.RunSelectRequestForSchemaFn[views.ListImageV2View](ctx, s.db, pkgSchema.ImagesV2Schema, cloned, func(row *views.ListImageV2View) error { results = append(results, row) return nil }) - if err != nil { - log.Errorf("unable to retrieve image id and digests: %v", err) - } return results, err } @@ -1046,10 +1081,13 @@ func (s *storeImpl) insertIntoImageComponentV2Cves(batch *pgx.Batch, obj *storag obj.GetFixedBy(), obj.GetComponentId(), obj.GetAdvisory().GetName(), + obj.GetAdvisory().GetLink(), + protocompat.NilOrTime(obj.GetFixAvailableTimestamp()), + obj.GetOrigin(), serialized, } - finalStr := "INSERT INTO image_cves_v2 (Id, ImageIdV2, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, Cvss, Severity, ImpactScore, Nvdcvss, FirstImageOccurrence, State, IsFixable, FixedBy, ComponentId, advisory_name, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16,$17) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, ImageIdV2 = EXCLUDED.ImageIdV2, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, Cvss = EXCLUDED.Cvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Nvdcvss = EXCLUDED.Nvdcvss, FirstImageOccurrence = EXCLUDED.FirstImageOccurrence, State = EXCLUDED.State, IsFixable = EXCLUDED.IsFixable, FixedBy = EXCLUDED.FixedBy, ComponentId = EXCLUDED.ComponentId, advisory_name = EXCLUDED.advisory_name, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO image_cves_v2 (Id, ImageIdV2, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, Cvss, Severity, ImpactScore, Nvdcvss, FirstImageOccurrence, State, IsFixable, FixedBy, ComponentId, advisory_name, advisory_link, FixAvailableTimestamp, Origin, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, ImageIdV2 = EXCLUDED.ImageIdV2, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, Cvss = EXCLUDED.Cvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Nvdcvss = EXCLUDED.Nvdcvss, FirstImageOccurrence = EXCLUDED.FirstImageOccurrence, State = EXCLUDED.State, IsFixable = EXCLUDED.IsFixable, FixedBy = EXCLUDED.FixedBy, ComponentId = EXCLUDED.ComponentId, advisory_name = EXCLUDED.advisory_name, advisory_link = EXCLUDED.advisory_link, FixAvailableTimestamp = EXCLUDED.FixAvailableTimestamp, Origin = EXCLUDED.Origin, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) return nil diff --git a/central/imagev2/datastore/store/postgres/store_bench_test.go b/central/imagev2/datastore/store/postgres/store_bench_test.go new file mode 100644 index 0000000000000..d3afa4a63865f --- /dev/null +++ b/central/imagev2/datastore/store/postgres/store_bench_test.go @@ -0,0 +1,129 @@ +//go:build sql_integration + +package postgres + +import ( + "context" + "fmt" + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/fixtures" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/search" + "github.com/stretchr/testify/require" +) + +// BenchmarkRealisticPopulateImage benchmarks image retrieval with realistic component/CVE counts. +// 50 images x 100 components x 10 CVEs = 50,000 CVEs across 5,000 components. +func BenchmarkRealisticPopulateImage(b *testing.B) { + if !features.FlattenImageData.Enabled() { + b.Skip("Skipping benchmark - FlattenImageData is disabled.") + } + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(b) + + store := New(testDB.DB, false, concurrency.NewKeyFence()) + + numImages := 50 + numComponents := 100 + numCVEs := 10 + images := make([]*storage.ImageV2, 0, numImages) + for i := range numImages { + components := make([]*storage.EmbeddedImageScanComponent, 0, numComponents) + for j := range numComponents { + vulns := make([]*storage.EmbeddedVulnerability, 0, numCVEs) + for k := range numCVEs { + vulns = append(vulns, &storage.EmbeddedVulnerability{ + Cve: fmt.Sprintf("CVE-2024-%d%d%d", i, j, k), + Cvss: 5, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Severity: storage.VulnerabilitySeverity_MODERATE_VULNERABILITY_SEVERITY, + SetFixedBy: &storage.EmbeddedVulnerability_FixedBy{FixedBy: "1.0.0"}, + }) + } + components = append(components, &storage.EmbeddedImageScanComponent{ + Name: fmt.Sprintf("pkg-%d-%d", i, j), + Version: fmt.Sprintf("%d.0.0", j), + Vulns: vulns, + }) + } + img := fixtures.GetImageV2WithUniqueComponents(0) + img.Id = fmt.Sprintf("sha256:realistic-%d", i) + img.Scan = &storage.ImageScan{ + Components: components, + } + images = append(images, img) + } + + for _, image := range images { + require.NoError(b, store.Upsert(ctx, image)) + } + + b.Run("WalkByQuery", func(b *testing.B) { + for b.Loop() { + count := 0 + err := store.WalkByQuery(ctx, search.EmptyQuery(), func(image *storage.ImageV2) error { + count++ + return nil + }) + require.NoError(b, err) + } + }) + + b.Run("GetSingleImage", func(b *testing.B) { + for b.Loop() { + _, _, err := store.Get(ctx, "sha256:realistic-0") + require.NoError(b, err) + } + }) +} + +// BenchmarkWalkComparison benchmarks both Walk functions for comparison +func BenchmarkWalkComparison(b *testing.B) { + if !features.FlattenImageData.Enabled() { + b.Skip("Skipping benchmark - FlattenImageData is disabled.") + } + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(b) + + store := New(testDB.DB, false, concurrency.NewKeyFence()) + + // Setup: Insert test images + numImages := 100 + images := make([]*storage.ImageV2, 0, numImages) + for i := range numImages { + img := fixtures.GetImageV2WithUniqueComponents(5) + img.Id = fmt.Sprintf("%d", i) + images = append(images, img) + } + + for _, image := range images { + require.NoError(b, store.Upsert(ctx, image)) + } + + b.Run("WalkByQuery", func(b *testing.B) { + for b.Loop() { + count := 0 + err := store.WalkByQuery(ctx, search.EmptyQuery(), func(image *storage.ImageV2) error { + count++ + return nil + }) + require.NoError(b, err) + } + }) + + b.Run("WalkMetadataByQuery", func(b *testing.B) { + for b.Loop() { + count := 0 + err := store.WalkMetadataByQuery(ctx, search.EmptyQuery(), func(image *storage.ImageV2) error { + count++ + return nil + }) + require.NoError(b, err) + } + }) +} diff --git a/central/imagev2/datastore/store/postgres/store_hash_test.go b/central/imagev2/datastore/store/postgres/store_hash_test.go new file mode 100644 index 0000000000000..568bb7f32398f --- /dev/null +++ b/central/imagev2/datastore/store/postgres/store_hash_test.go @@ -0,0 +1,108 @@ +package postgres + +import ( + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPopulateImageScanHash_DuplicateComponentsChangeHash(t *testing.T) { + component := &storage.EmbeddedImageScanComponent{ + Name: "ubi10/ubi-micro", + Version: "1784668691", + } + + scanWith1 := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{component}, + } + scanWith3 := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{component, component, component}, + } + + require.NoError(t, populateImageScanHash(scanWith1)) + require.NoError(t, populateImageScanHash(scanWith3)) + + assert.NotEqual(t, scanWith1.GetHash(), scanWith3.GetHash(), + "removing duplicate components must produce a different hash") +} + +func TestPopulateImageScanHash_DuplicateVulnsChangeHash(t *testing.T) { + vuln := &storage.EmbeddedVulnerability{ + Cve: "CVE-2025-1234", + Severity: storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, + } + + scanWith1Vuln := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{ + {Name: "pkg", Version: "1.0", Vulns: []*storage.EmbeddedVulnerability{vuln}}, + }, + } + scanWith3Vulns := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{ + {Name: "pkg", Version: "1.0", Vulns: []*storage.EmbeddedVulnerability{vuln, vuln, vuln}}, + }, + } + + require.NoError(t, populateImageScanHash(scanWith1Vuln)) + require.NoError(t, populateImageScanHash(scanWith3Vulns)) + + assert.NotEqual(t, scanWith1Vuln.GetHash(), scanWith3Vulns.GetHash(), + "removing duplicate vulns must produce a different hash") +} + +func TestPopulateImageScanHash_DistinctChangesDetected(t *testing.T) { + scanA := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{ + {Name: "pkg", Version: "1.0", Vulns: []*storage.EmbeddedVulnerability{ + {Cve: "CVE-2025-1111"}, + }}, + }, + } + scanB := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{ + {Name: "pkg", Version: "1.0", Vulns: []*storage.EmbeddedVulnerability{ + {Cve: "CVE-2025-2222"}, + }}, + }, + } + + require.NoError(t, populateImageScanHash(scanA)) + require.NoError(t, populateImageScanHash(scanB)) + + assert.NotEqual(t, scanA.GetHash(), scanB.GetHash(), + "different CVEs must produce a different hash") +} + +func TestPopulateImageScanHash_IdenticalScansMatch(t *testing.T) { + scan1 := &storage.ImageScan{ + Components: []*storage.EmbeddedImageScanComponent{ + {Name: "pkg-a", Version: "1.0", Vulns: []*storage.EmbeddedVulnerability{ + {Cve: "CVE-2025-1111"}, + }}, + {Name: "pkg-b", Version: "2.0"}, + }, + } + scan2 := scan1.CloneVT() + + require.NoError(t, populateImageScanHash(scan1)) + require.NoError(t, populateImageScanHash(scan2)) + + assert.Equal(t, scan1.GetHash(), scan2.GetHash(), + "identical scans must produce the same hash") +} + +func TestPopulateImageScanHash_OrderIndependent(t *testing.T) { + compA := &storage.EmbeddedImageScanComponent{Name: "aaa", Version: "1.0"} + compB := &storage.EmbeddedImageScanComponent{Name: "bbb", Version: "2.0"} + + scanAB := &storage.ImageScan{Components: []*storage.EmbeddedImageScanComponent{compA, compB}} + scanBA := &storage.ImageScan{Components: []*storage.EmbeddedImageScanComponent{compB, compA}} + + require.NoError(t, populateImageScanHash(scanAB)) + require.NoError(t, populateImageScanHash(scanBA)) + + assert.Equal(t, scanAB.GetHash(), scanBA.GetHash(), + "component order must not affect the hash") +} diff --git a/central/imagev2/datastore/store/postgres/store_test.go b/central/imagev2/datastore/store/postgres/store_test.go index fc606c78835cd..a118bae2e5dcf 100644 --- a/central/imagev2/datastore/store/postgres/store_test.go +++ b/central/imagev2/datastore/store/postgres/store_test.go @@ -30,9 +30,12 @@ import ( // TODO(ROX-31640): Add tests for using old legacy times back in unless ROX-29911 gets implemented // TODO(ROX-29911): add tests. var ( - lastWeek = time.Now().Add(-7 * 24 * time.Hour) - yesterday = time.Now().Add(-24 * time.Hour) - nextWeek = time.Now().Add(7 * 24 * time.Hour) + // Truncate to microsecond precision to match PostgreSQL's timestamp storage. + // This avoids sub-microsecond rounding mismatches when timestamps round-trip + // through the database column. + lastWeek = time.Now().Add(-7 * 24 * time.Hour).Truncate(time.Microsecond) + yesterday = time.Now().Add(-24 * time.Hour).Truncate(time.Microsecond) + nextWeek = time.Now().Add(7 * 24 * time.Hour).Truncate(time.Microsecond) ) type ImagesV2StoreSuite struct { @@ -45,6 +48,11 @@ type ImagesV2StoreSuite struct { cvePgStore cveStore.Store } +type timeFields struct { + createdAt *protocompat.Timestamp + firstImageOccurrence *protocompat.Timestamp +} + func TestImagesV2Store(t *testing.T) { suite.Run(t, new(ImagesV2StoreSuite)) } @@ -98,6 +106,8 @@ func (s *ImagesV2StoreSuite) TestStore() { vuln.SuppressExpiry = nil vuln.Advisory = nil vuln.FixAvailableTimestamp = nil + vuln.Exploit = nil + vuln.CisaKev = false } comp.License = nil } @@ -219,9 +229,11 @@ func (s *ImagesV2StoreSuite) TestUpsert_MigrationFromNewCVEModel() { for _, comp := range foundImageV1.GetScan().GetComponents() { for _, vuln := range comp.GetVulns() { if _, ok := expectedTimestamps[vuln.GetCve()]; !ok { + // Round to microsecond precision to match what the upsert path stores. + fio := roundToMicroseconds(vuln.GetFirstImageOccurrence()) expectedTimestamps[vuln.GetCve()] = &timeFields{ - createdAt: vuln.GetFirstSystemOccurrence().AsTime(), - firstImageOccurrence: vuln.GetFirstImageOccurrence().AsTime(), + createdAt: protocompat.ConvertTimeToTimestampOrNil(&nextWeek), + firstImageOccurrence: fio, } } } @@ -232,14 +244,14 @@ func (s *ImagesV2StoreSuite) TestUpsert_MigrationFromNewCVEModel() { for _, vuln := range comp.GetVulns() { if _, ok := actualTimestamps[vuln.GetCve()]; !ok { actualTimestamps[vuln.GetCve()] = &timeFields{ - createdAt: vuln.GetFirstSystemOccurrence().AsTime(), - firstImageOccurrence: vuln.GetFirstImageOccurrence().AsTime(), + createdAt: vuln.GetFirstSystemOccurrence(), + firstImageOccurrence: vuln.GetFirstImageOccurrence(), } } } } - // Created at and first image occurrence timestamps should not have changed to the future ones. + // FirstImageOccurrence should be preserved from V1, but CreatedAt should use the new value. s.Assert().Equal(expectedTimestamps, actualTimestamps) } @@ -261,16 +273,16 @@ func (s *ImagesV2StoreSuite) TestUpsert() { protoassert.Equal(s.T(), cloned, foundImage) // Add a new component with "cve1" that has new times - // Ensure old times are associated with the CVE in the new component. + // Ensure FirstImageOccurrence is preserved for the CVE in the new component. image.Scan.Components = append(image.Scan.Components, getComponent3()) s.NoError(s.store.Upsert(s.ctx, image)) foundImage, exists, err = s.store.Get(s.ctx, image.GetId()) s.NoError(err) s.True(exists) - // Should pull the old CVE times for CVE1 even though it just appeared in - // the component. The CVE has still existed in the image even though it is - // new to the component. + // Should pull the old FirstImageOccurrence for CVE1 even though it just appeared in + // the component. The CVE has still existed in the image even though it is + // new to the component. FirstSystemOccurrence will use the new timestamp. cloned.LastUpdated = foundImage.GetLastUpdated() cloned.Scan.Hashoneof = &storage.ImageScan_Hash{ Hash: foundImage.GetScan().GetHash(), @@ -287,9 +299,6 @@ func (s *ImagesV2StoreSuite) TestUpsert() { s.True(exists) cloned = image.CloneVT() - // Should pull the old CVE times for CVE1 even though it just appeared in - // the component. The CVE has still existed in the image even though it is - // new to the component. cloned.LastUpdated = foundImage.GetLastUpdated() cloned.Scan.Hashoneof = &storage.ImageScan_Hash{ Hash: foundImage.GetScan().GetHash(), @@ -564,7 +573,7 @@ func getTestImageComponentsVerify() []*storage.EmbeddedImageScanComponent { ScoreVersion: storage.EmbeddedVulnerability_V3, PublishedOn: protocompat.ConvertTimeToTimestampOrNil(&yesterday), FirstImageOccurrence: protocompat.ConvertTimeToTimestampOrNil(&lastWeek), - FirstSystemOccurrence: protocompat.ConvertTimeToTimestampOrNil(&lastWeek), + FirstSystemOccurrence: protocompat.ConvertTimeToTimestampOrNil(&yesterday), }, }, }, @@ -658,7 +667,7 @@ func getComponent3Verify() *storage.EmbeddedImageScanComponent { ScoreVersion: storage.EmbeddedVulnerability_V3, PublishedOn: protocompat.ConvertTimeToTimestampOrNil(&yesterday), FirstImageOccurrence: protocompat.ConvertTimeToTimestampOrNil(&lastWeek), - FirstSystemOccurrence: protocompat.ConvertTimeToTimestampOrNil(&lastWeek), + FirstSystemOccurrence: protocompat.ConvertTimeToTimestampOrNil(&yesterday), }, { Cve: "cve3", @@ -676,6 +685,14 @@ func getComponent3Verify() *storage.EmbeddedImageScanComponent { } } +func roundToMicroseconds(ts *protocompat.Timestamp) *protocompat.Timestamp { + if ts == nil { + return nil + } + t := protocompat.NilOrTime(ts) + return protocompat.ConvertTimeToTimestampOrNil(t) +} + func getTestImageComponentsFixedCVE1() []*storage.EmbeddedImageScanComponent { return []*storage.EmbeddedImageScanComponent{ { diff --git a/central/imagev2/datastore/store/store.go b/central/imagev2/datastore/store/store.go index 8e62886f4b480..4350aa318d955 100644 --- a/central/imagev2/datastore/store/store.go +++ b/central/imagev2/datastore/store/store.go @@ -25,6 +25,7 @@ type Store interface { GetImageMetadata(ctx context.Context, id string) (*storage.ImageV2, bool, error) GetManyImageMetadata(ctx context.Context, id []string) ([]*storage.ImageV2, error) WalkByQuery(ctx context.Context, q *v1.Query, fn func(img *storage.ImageV2) error) error + WalkMetadataByQuery(ctx context.Context, q *v1.Query, fn func(img *storage.ImageV2) error) error Upsert(ctx context.Context, image *storage.ImageV2) error Delete(ctx context.Context, id string) error @@ -33,6 +34,6 @@ type Store interface { // GetImagesRiskView retrieves an image id and risk score to initialize rankers GetImagesRiskView(ctx context.Context, q *v1.Query) ([]*views.ImageV2RiskView, error) - // GetImagesIdAndDigestView retrieves an image id and digest for pruning purposes - GetImagesIdAndDigestView(ctx context.Context, q *v1.Query) ([]*views.ImageIDAndDigestView, error) + // GetListImagesView retrieves the fields needed for ListImage responses + GetListImagesView(ctx context.Context, q *v1.Query) ([]*views.ListImageV2View, error) } diff --git a/central/imagev2/datastoretest/datastore_impl_test.go b/central/imagev2/datastoretest/datastore_impl_test.go index 3ed6c2147ca99..e9fbfdd2cd29c 100644 --- a/central/imagev2/datastoretest/datastore_impl_test.go +++ b/central/imagev2/datastoretest/datastore_impl_test.go @@ -7,9 +7,14 @@ import ( "fmt" "sort" "testing" + "time" + imageCVEInfoDS "github.com/stackrox/rox/central/cve/image/info/datastore" + imageCVEInfoPostgres "github.com/stackrox/rox/central/cve/image/info/datastore/store/postgres" + cveInfoEnricher "github.com/stackrox/rox/central/cve/image/info/enricher" imageCVEDS "github.com/stackrox/rox/central/cve/image/v2/datastore" imageCVEPostgres "github.com/stackrox/rox/central/cve/image/v2/datastore/store/postgres" + deploymentDS "github.com/stackrox/rox/central/deployment/datastore" imageComponentDS "github.com/stackrox/rox/central/imagecomponent/v2/datastore" imageComponentPostgres "github.com/stackrox/rox/central/imagecomponent/v2/datastore/store/postgres" imageDataStoreV2 "github.com/stackrox/rox/central/imagev2/datastore" @@ -22,6 +27,8 @@ import ( pkgCVE "github.com/stackrox/rox/pkg/cve" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" + imageEnricher "github.com/stackrox/rox/pkg/images/enricher" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres/pgtest" postgresSchema "github.com/stackrox/rox/pkg/postgres/schema" "github.com/stackrox/rox/pkg/protoassert" @@ -35,6 +42,7 @@ import ( "github.com/stackrox/rox/pkg/uuid" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" ) func TestImageV2DataStore(t *testing.T) { @@ -47,12 +55,15 @@ func TestImageV2DataStore(t *testing.T) { type ImageV2DataStoreTestSuite struct { suite.Suite - ctx context.Context - testDB *pgtest.TestPostgres - datastore imageDataStoreV2.DataStore - mockRisk *mockRisks.MockDataStore - componentDataStore imageComponentDS.DataStore - cveDataStore imageCVEDS.DataStore + ctx context.Context + testDB *pgtest.TestPostgres + datastore imageDataStoreV2.DataStore + mockRisk *mockRisks.MockDataStore + componentDataStore imageComponentDS.DataStore + cveDataStore imageCVEDS.DataStore + deploymentDataStore deploymentDS.DataStore + cveInfoDataStore imageCVEInfoDS.DataStore + cveInfoEnricher imageEnricher.CVEInfoEnricher } func (s *ImageV2DataStoreTestSuite) SetupSuite() { @@ -66,16 +77,26 @@ func (s *ImageV2DataStoreTestSuite) SetupTest() { s.datastore = imageDataStoreV2.NewWithPostgres(dbStore, s.mockRisk, ranking.NewRanker(), ranking.NewRanker()) componentStorage := imageComponentPostgres.New(s.testDB.DB) - s.componentDataStore = imageComponentDS.New(componentStorage, s.mockRisk, ranking.NewRanker()) + s.componentDataStore = imageComponentDS.New(s.testDB.DB, componentStorage, s.mockRisk, ranking.NewRanker()) cveStorage := imageCVEPostgres.New(s.testDB.DB) s.cveDataStore = imageCVEDS.New(cveStorage) + + cveInfoStorage := imageCVEInfoPostgres.New(s.testDB.DB) + s.cveInfoDataStore = imageCVEInfoDS.New(cveInfoStorage) + s.cveInfoEnricher = cveInfoEnricher.New(s.cveInfoDataStore) + + var err error + s.deploymentDataStore, err = deploymentDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(err) } func (s *ImageV2DataStoreTestSuite) TearDownTest() { + s.truncateTable(postgresSchema.DeploymentsTableName) s.truncateTable(postgresSchema.ImagesV2TableName) s.truncateTable(postgresSchema.ImageComponentV2TableName) s.truncateTable(postgresSchema.ImageCvesV2TableName) + s.truncateTable(postgresSchema.ImageCveInfosTableName) } func (s *ImageV2DataStoreTestSuite) TestSearch() { @@ -306,6 +327,12 @@ func (s *ImageV2DataStoreTestSuite) TestImageDeletes() { } } expectedImage := cloneAndUpdateRiskPriority(testImage) + // PostgreSQL timestamps have microsecond precision, so round both + // sides to microseconds before comparing. + roundFirstImageOccurrence(expectedImage) + roundFirstImageOccurrence(storedImage) + roundFirstImageOccurrence(expectedImage) + roundFirstImageOccurrence(storedImage) protoassert.Equal(s.T(), expectedImage, storedImage) // Verify that new scan with less components cleans up the old relations correctly. @@ -326,6 +353,8 @@ func (s *ImageV2DataStoreTestSuite) TestImageDeletes() { s.NoError(err) s.True(found) expectedImage = cloneAndUpdateRiskPriority(testImage) + roundFirstImageOccurrence(expectedImage) + roundFirstImageOccurrence(storedImage) protoassert.Equal(s.T(), expectedImage, storedImage) // Verify orphaned image components are removed. @@ -359,6 +388,8 @@ func (s *ImageV2DataStoreTestSuite) TestImageDeletes() { } } expectedImage = cloneAndUpdateRiskPriority(testImage2) + roundFirstImageOccurrence(expectedImage) + roundFirstImageOccurrence(storedImage) protoassert.Equal(s.T(), expectedImage, storedImage) s.mockRisk.EXPECT().RemoveRisk(gomock.Any(), testImage.GetId(), gomock.Any()).Return(nil) @@ -369,6 +400,8 @@ func (s *ImageV2DataStoreTestSuite) TestImageDeletes() { s.NoError(err) s.True(found) expectedImage = cloneAndUpdateRiskPriority(testImage2) + roundFirstImageOccurrence(expectedImage) + roundFirstImageOccurrence(storedImage) protoassert.Equal(s.T(), expectedImage, storedImage) // Set all components to contain same cve. @@ -395,6 +428,8 @@ func (s *ImageV2DataStoreTestSuite) TestImageDeletes() { } } expectedImage = cloneAndUpdateRiskPriority(testImage2) + roundFirstImageOccurrence(expectedImage) + roundFirstImageOccurrence(storedImage) protoassert.Equal(s.T(), expectedImage, storedImage) // Verify orphaned image components are removed. @@ -418,6 +453,8 @@ func (s *ImageV2DataStoreTestSuite) TestImageDeletes() { s.NoError(err) s.True(found) expectedImage = cloneAndUpdateRiskPriority(testImage2) + roundFirstImageOccurrence(expectedImage) + roundFirstImageOccurrence(storedImage) protoassert.Equal(s.T(), expectedImage, storedImage) // Verify orphaned image components are removed. @@ -440,6 +477,8 @@ func (s *ImageV2DataStoreTestSuite) TestImageDeletes() { s.NoError(err) s.True(found) expectedImage = cloneAndUpdateRiskPriority(testImage2) + roundFirstImageOccurrence(expectedImage) + roundFirstImageOccurrence(storedImage) protoassert.Equal(s.T(), expectedImage, storedImage) // Verify no components exist. @@ -490,17 +529,6 @@ func (s *ImageV2DataStoreTestSuite) TestGetManyImageMetadata() { protoassert.ElementsMatch(s.T(), []*storage.ImageV2{testImage1, testImage2, testImage3}, storedImages) } -func (s *ImageV2DataStoreTestSuite) TestGetImageIdsAndDigest() { - ctx := sac.WithAllAccess(context.Background()) - testImage1 := fixtures.GetImageV2() - s.NoError(s.datastore.UpsertImage(ctx, testImage1)) - results, err := s.datastore.GetImageIDsAndDigests(ctx, pkgSearch.EmptyQuery()) - s.NoError(err) - s.Len(results, 1) - s.Equal(testImage1.GetId(), results[0].ImageID) - s.Equal(testImage1.GetDigest(), results[0].Digest) -} - func (s *ImageV2DataStoreTestSuite) TestGetImageNames() { ctx := sac.WithAllAccess(context.Background()) img1 := getTestImageV2("img1") @@ -552,6 +580,337 @@ func (s *ImageV2DataStoreTestSuite) TestGetImageNames() { protoassert.ElementsMatch(s.T(), expectedImageNames, imageNames) } +func (s *ImageV2DataStoreTestSuite) TestCVETimestampPersistence() { + s.T().Setenv(features.CVEFixTimestampCriteria.EnvVar(), "true") + if !features.CVEFixTimestampCriteria.Enabled() { + s.T().Skip("CVEFixTimestampCriteria feature must be enabled for this test") + } + + ctx := sac.WithAllAccess(context.Background()) + + // Scanner-provided timestamp for when the shared CVE was first discovered + cveDiscoverTimestamp := timestamppb.New(time.Now().Add(-24 * time.Hour)) + + sharedCVEID := "CVE-2024-1234" + datasource := "alpine:v3.18" + + image0Name := &storage.ImageName{FullName: "registry.io/image0:v0"} + image0Digest := "sha256:image0digest" + image0ID := imageUtils.NewImageV2ID(image0Name, image0Digest) + + image1Name := &storage.ImageName{FullName: "registry.io/image1:v1"} + image1Digest := "sha256:image1digest" + + image2Name := &storage.ImageName{FullName: "registry.io/image2:v2"} + image2Digest := "sha256:image2digest" + + // Three images share a CVE but each also has unique CVEs. + // The shared CVE in the second image has an earlier FirstSystemOccurrence timestamp. + images := []*storage.ImageV2{ + { + Id: image0ID, + Digest: image0Digest, + Name: image0Name, + Scan: &storage.ImageScan{ + OperatingSystem: "alpine", + ScanTime: timestamppb.Now(), + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "shared-component", + Version: "1.0.0", + Source: storage.SourceType_OS, + Vulns: []*storage.EmbeddedVulnerability{ + { + Cve: sharedCVEID, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, + FirstSystemOccurrence: timestamppb.Now(), + }, + { + Cve: "CVE-2024-1111", + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + FirstSystemOccurrence: timestamppb.Now(), + }, + }, + }, + }, + }, + }, + { + Id: imageUtils.NewImageV2ID(image1Name, image1Digest), + Digest: image1Digest, + Name: image1Name, + Scan: &storage.ImageScan{ + OperatingSystem: "alpine", + ScanTime: timestamppb.Now(), + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "shared-component", + Version: "1.0.0", + Source: storage.SourceType_OS, + Vulns: []*storage.EmbeddedVulnerability{ + { + Cve: sharedCVEID, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, + FirstSystemOccurrence: cveDiscoverTimestamp, // Earlier timestamp + }, + { + Cve: "CVE-2024-5678", + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + FirstSystemOccurrence: timestamppb.Now(), + }, + }, + }, + }, + }, + }, + { + Id: imageUtils.NewImageV2ID(image2Name, image2Digest), + Digest: image2Digest, + Name: image2Name, + Scan: &storage.ImageScan{ + OperatingSystem: "alpine", + ScanTime: timestamppb.Now(), + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "shared-component", + Version: "1.0.0", + Source: storage.SourceType_OS, + Vulns: []*storage.EmbeddedVulnerability{ + { + Cve: sharedCVEID, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, + FirstSystemOccurrence: timestamppb.Now(), + }, + { + Cve: "CVE-2024-9999", + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: datasource, + FirstSystemOccurrence: timestamppb.Now(), + }, + }, + }, + }, + }, + }, + } + + // First pass: process all images in order. + // With the new CVE-name-based aggregation, images processed after image1 will be enriched + // with the earliest timestamp across ALL records for that CVE name. + for _, img := range images { + imgClone := img.CloneVT() + s.NoError(s.cveInfoEnricher.EnrichImageV2WithCVEInfo(ctx, imgClone)) + s.NoError(s.datastore.UpsertImage(ctx, imgClone)) + } + + // Verify ImageCVEInfo persisted the earliest timestamp for the specific composite ID + cveInfoID := pkgCVE.ImageCVEInfoID(sharedCVEID, "shared-component", datasource) + cveInfo, found, err := s.cveInfoDataStore.Get(ctx, cveInfoID) + s.NoError(err) + s.True(found) + s.Equal(cveDiscoverTimestamp, cveInfo.GetFirstSystemOccurrence()) + + // Verify first pass results: + // With CVE-name-based aggregation: + // - image0: shared CVE should NOT have cveDiscoverTimestamp (processed before image1) + // - image1: shared CVE has cveDiscoverTimestamp (source of early timestamp) + // - image2: shared CVE should have cveDiscoverTimestamp (enriched from MIN across all CVE-2024-1234 records) + // - All unique CVEs should have their own timestamps + for i, img := range images { + stored, found, err := s.datastore.GetImage(ctx, img.GetId()) + s.NoError(err) + s.True(found) + + components := stored.GetScan().GetComponents() + s.Require().Len(components, 1) + s.Require().Len(components[0].GetVulns(), 2) + + for _, vuln := range components[0].GetVulns() { + if vuln.GetCve() != sharedCVEID { + s.NotEqual(cveDiscoverTimestamp, vuln.GetFirstSystemOccurrence(), + "Unique CVE should not have the shared timestamp (image %d)", i) + } else if img.GetId() == image0ID { + s.NotEqual(cveDiscoverTimestamp, vuln.GetFirstSystemOccurrence(), + "image0 was processed before image1, should not have the earlier timestamp yet") + } else { + s.Equal(cveDiscoverTimestamp, vuln.GetFirstSystemOccurrence(), + "Shared CVE should have the earlier timestamp from image1 (image %d)", i) + } + } + } + + // Second pass: rescan all images. + // All images should now have the preserved earliest timestamp for the shared CVE, + // since the enricher queries by CVE name and returns the MIN across all records. + for _, img := range images { + imgClone := img.CloneVT() + imgClone.GetScan().ScanTime = timestamppb.Now() + s.NoError(s.cveInfoEnricher.EnrichImageV2WithCVEInfo(ctx, imgClone)) + s.NoError(s.datastore.UpsertImage(ctx, imgClone)) + } + + // Verify all images now have the preserved earliest timestamp. + // The CVE-name-based aggregation ensures all images get the MIN timestamp for CVE-2024-1234. + for i, img := range images { + stored, found, err := s.datastore.GetImage(ctx, img.GetId()) + s.NoError(err) + s.True(found) + + components := stored.GetScan().GetComponents() + s.Require().Len(components, 1) + s.Require().Len(components[0].GetVulns(), 2) + + for _, vuln := range components[0].GetVulns() { + if vuln.GetCve() == sharedCVEID { + s.Equal(cveDiscoverTimestamp, vuln.GetFirstSystemOccurrence(), + "After second pass, all images should have the preserved earliest timestamp for shared CVE (image %d)", i) + } else { + s.NotEqual(cveDiscoverTimestamp, vuln.GetFirstSystemOccurrence(), + "Unique CVE should not have the shared timestamp (image %d)", i) + } + } + } +} + +func (s *ImageV2DataStoreTestSuite) TestCVETimestampAggregation() { + s.T().Setenv(features.CVEFixTimestampCriteria.EnvVar(), "true") + if !features.CVEFixTimestampCriteria.Enabled() { + s.T().Skip("CVEFixTimestampCriteria feature must be enabled for this test") + } + + ctx := sac.WithAllAccess(context.Background()) + + // Two images with the same CVE name but different components and operating systems + sharedCVEName := "CVE-2024-SHARED" + earlierTimestamp := timestamppb.New(time.Now().Add(-48 * time.Hour)) + laterTimestamp := timestamppb.New(time.Now().Add(-24 * time.Hour)) + + img1Name := &storage.ImageName{FullName: "registry.io/ubuntu-app:v1"} + img1Digest := "sha256:image1digest" + img1ID := imageUtils.NewImageV2ID(img1Name, img1Digest) + + img2Name := &storage.ImageName{FullName: "registry.io/alpine-app:v1"} + img2Digest := "sha256:image2digest" + img2ID := imageUtils.NewImageV2ID(img2Name, img2Digest) + + // Image 1: Ubuntu with package-a, has the earlier timestamp + image1 := &storage.ImageV2{ + Id: img1ID, + Digest: img1Digest, + Name: img1Name, + Scan: &storage.ImageScan{ + OperatingSystem: "ubuntu", + ScanTime: timestamppb.Now(), + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "package-a", + Version: "1.0.0", + Source: storage.SourceType_OS, + Vulns: []*storage.EmbeddedVulnerability{ + { + Cve: sharedCVEName, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: "ubuntu-updater::ubuntu:20.04", + Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, + FirstSystemOccurrence: earlierTimestamp, + }, + }, + }, + }, + }, + } + + // Image 2: Alpine with package-b (different component and OS), has the later timestamp + image2 := &storage.ImageV2{ + Id: img2ID, + Digest: img2Digest, + Name: img2Name, + Scan: &storage.ImageScan{ + OperatingSystem: "alpine", + ScanTime: timestamppb.Now(), + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "package-b", + Version: "2.0.0", + Source: storage.SourceType_OS, + Vulns: []*storage.EmbeddedVulnerability{ + { + Cve: sharedCVEName, + VulnerabilityType: storage.EmbeddedVulnerability_IMAGE_VULNERABILITY, + Datasource: "alpine-updater::alpine:3.18", + Severity: storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, + FirstSystemOccurrence: laterTimestamp, + }, + }, + }, + }, + }, + } + + // Process image1 first (has earlier timestamp) + img1Clone := image1.CloneVT() + s.NoError(s.cveInfoEnricher.EnrichImageV2WithCVEInfo(ctx, img1Clone)) + s.NoError(s.datastore.UpsertImage(ctx, img1Clone)) + + // Process image2 (has later timestamp) + img2Clone := image2.CloneVT() + s.NoError(s.cveInfoEnricher.EnrichImageV2WithCVEInfo(ctx, img2Clone)) + s.NoError(s.datastore.UpsertImage(ctx, img2Clone)) + + // Verify that two separate ImageCVEInfo records exist (different composite IDs) + cveInfo1ID := pkgCVE.ImageCVEInfoID(sharedCVEName, "package-a", "ubuntu-updater::ubuntu:20.04") + cveInfo2ID := pkgCVE.ImageCVEInfoID(sharedCVEName, "package-b", "alpine-updater::alpine:3.18") + + cveInfo1, found, err := s.cveInfoDataStore.Get(ctx, cveInfo1ID) + s.NoError(err) + s.True(found, "Should have ImageCVEInfo for image1's composite ID") + s.Equal(earlierTimestamp, cveInfo1.GetFirstSystemOccurrence()) + + cveInfo2, found, err := s.cveInfoDataStore.Get(ctx, cveInfo2ID) + s.NoError(err) + s.True(found, "Should have ImageCVEInfo for image2's composite ID") + s.Equal(laterTimestamp, cveInfo2.GetFirstSystemOccurrence()) + + // Verify both images have the MIN timestamp (from image1) after enrichment + storedImage1, found, err := s.datastore.GetImage(ctx, img1ID) + s.NoError(err) + s.True(found) + s.Require().Len(storedImage1.GetScan().GetComponents(), 1) + s.Require().Len(storedImage1.GetScan().GetComponents()[0].GetVulns(), 1) + s.Equal(earlierTimestamp, storedImage1.GetScan().GetComponents()[0].GetVulns()[0].GetFirstSystemOccurrence(), + "Image1 should have the earlier timestamp") + + storedImage2, found, err := s.datastore.GetImage(ctx, img2ID) + s.NoError(err) + s.True(found) + s.Require().Len(storedImage2.GetScan().GetComponents(), 1) + s.Require().Len(storedImage2.GetScan().GetComponents()[0].GetVulns(), 1) + s.Equal(earlierTimestamp, storedImage2.GetScan().GetComponents()[0].GetVulns()[0].GetFirstSystemOccurrence(), + "Image2 should also have the earlier timestamp from MIN aggregation across all CVE records") + + // Rescan image1 to verify the MIN timestamp is still preserved + img1Rescan := image1.CloneVT() + img1Rescan.GetScan().ScanTime = timestamppb.Now() + s.NoError(s.cveInfoEnricher.EnrichImageV2WithCVEInfo(ctx, img1Rescan)) + s.NoError(s.datastore.UpsertImage(ctx, img1Rescan)) + + storedImage1After, found, err := s.datastore.GetImage(ctx, img1ID) + s.NoError(err) + s.True(found) + s.Require().Len(storedImage1After.GetScan().GetComponents(), 1) + s.Require().Len(storedImage1After.GetScan().GetComponents()[0].GetVulns(), 1) + s.Equal(earlierTimestamp, storedImage1After.GetScan().GetComponents()[0].GetVulns()[0].GetFirstSystemOccurrence(), + "After rescan, image1 should still have the MIN timestamp") +} + func (s *ImageV2DataStoreTestSuite) truncateTable(name string) { sql := fmt.Sprintf("TRUNCATE %s CASCADE", name) _, err := s.testDB.Exec(s.ctx, sql) @@ -638,6 +997,21 @@ func getTestImageV2(name string) *storage.ImageV2 { } } +// roundFirstImageOccurrence truncates all FirstImageOccurrence timestamps on +// an image to microsecond precision. PostgreSQL timestamp columns have +// microsecond precision, so both expected and actual must be truncated before +// comparison. +func roundFirstImageOccurrence(image *storage.ImageV2) { + for _, comp := range image.GetScan().GetComponents() { + for _, vuln := range comp.GetVulns() { + if ts := vuln.GetFirstImageOccurrence(); ts != nil { + t := ts.AsTime().Truncate(time.Microsecond) + vuln.FirstImageOccurrence = timestamppb.New(t) + } + } + } +} + func cloneAndUpdateRiskPriority(image *storage.ImageV2) *storage.ImageV2 { cloned := image.CloneVT() cloned.Priority = 1 @@ -646,3 +1020,198 @@ func cloneAndUpdateRiskPriority(image *storage.ImageV2) *storage.ImageV2 { } return cloned } + +func (s *ImageV2DataStoreTestSuite) TestSearchListImages() { + ctx := sac.WithAllAccess(context.Background()) + + // Create and upsert test images + img1 := getTestImageV2("img1") + img2 := getTestImageV2("img2") + img3 := getTestImageV2("img3") + + s.NoError(s.datastore.UpsertImage(ctx, img1)) + s.NoError(s.datastore.UpsertImage(ctx, img2)) + s.NoError(s.datastore.UpsertImage(ctx, img3)) + + // Test 1: Search all images with empty query + listImages, err := s.datastore.SearchListImages(ctx, pkgSearch.EmptyQuery()) + s.NoError(err) + s.Len(listImages, 3) + + // Verify that all images are present + // Note: In V2->V1 conversion, Image.Id is set to the digest (SHA), not the UUID + imageDigests := set.NewStringSet() + for _, img := range listImages { + imageDigests.Add(img.GetId()) + // Verify priority is set by the ranker + s.NotZero(img.GetPriority()) + // Verify ListImage has expected fields + s.NotEmpty(img.GetId()) + s.NotEmpty(img.GetName()) + // LastUpdated should be set + s.NotNil(img.GetLastUpdated()) + } + // Verify all image digests are present + s.True(imageDigests.Contains(img1.GetDigest())) + s.True(imageDigests.Contains(img2.GetDigest())) + s.True(imageDigests.Contains(img3.GetDigest())) + + // Test 2: Search with specific image name + q := pkgSearch.NewQueryBuilder().AddStrings(pkgSearch.ImageName, "registry.test.io/img1:latest").ProtoQuery() + listImages, err = s.datastore.SearchListImages(ctx, q) + s.NoError(err) + s.Len(listImages, 1) + s.Equal(img1.GetDigest(), listImages[0].GetId()) // ListImage ID is the digest in V1 format + s.Equal("registry.test.io/img1:latest", listImages[0].GetName()) + + // Test 3: Search with image UUID (V2 ID) + q = pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.ImageID, img2.GetId()).ProtoQuery() + listImages, err = s.datastore.SearchListImages(ctx, q) + s.NoError(err) + s.Len(listImages, 1) + s.Equal(img2.GetDigest(), listImages[0].GetId()) + + // Test 4: Search with SHA + q = pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.ImageSHA, img3.GetDigest()).ProtoQuery() + listImages, err = s.datastore.SearchListImages(ctx, q) + s.NoError(err) + s.Len(listImages, 1) + s.Equal(img3.GetDigest(), listImages[0].GetId()) + + // Test 5: Search with pagination + q = pkgSearch.EmptyQuery() + q.Pagination = &v1.QueryPagination{ + Limit: 2, + Offset: 0, + } + listImages, err = s.datastore.SearchListImages(ctx, q) + s.NoError(err) + s.Len(listImages, 2) + + // Test 6: Search with no results + q = pkgSearch.NewQueryBuilder().AddStrings(pkgSearch.ImageName, "nonexistent").ProtoQuery() + listImages, err = s.datastore.SearchListImages(ctx, q) + s.NoError(err) + s.Len(listImages, 0) + + // Test 7: Verify ListImage contains component and CVE counts from ScanStats + // Create an image with scan stats populated + imgWithStats := fixtures.GetImageV2WithUniqueComponents(3) + s.NoError(s.datastore.UpsertImage(ctx, imgWithStats)) + + q = pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.ImageID, imgWithStats.GetId()).ProtoQuery() + listImages, err = s.datastore.SearchListImages(ctx, q) + s.NoError(err) + s.Len(listImages, 1) + s.Equal(imgWithStats.GetDigest(), listImages[0].GetId()) + + // Verify the component count comes from ScanStats (fixture sets ComponentCount to 3) + s.Equal(int32(3), listImages[0].GetComponents(), "Expected component count to match fixture") + + // CVE and fixable CVE counts may be populated by the datastore during upsert + // We just verify they are non-negative (could be 0 if not populated) + s.GreaterOrEqual(listImages[0].GetCves(), int32(0)) + s.GreaterOrEqual(listImages[0].GetFixableCves(), int32(0)) + + // Test 8: Test with access control - no access context + noAccessCtx := sac.WithNoAccess(context.Background()) + listImages, err = s.datastore.SearchListImages(noAccessCtx, pkgSearch.EmptyQuery()) + s.NoError(err) + s.Len(listImages, 0) + + // Test 9: Verify sorting works correctly + q = pkgSearch.EmptyQuery() + q.Pagination = &v1.QueryPagination{ + SortOptions: []*v1.QuerySortOption{ + { + Field: pkgSearch.ImageName.String(), + }, + }, + } + listImages, err = s.datastore.SearchListImages(ctx, q) + s.NoError(err) + s.Greater(len(listImages), 1) + // Verify images are sorted by name + for i := 1; i < len(listImages); i++ { + s.LessOrEqual(listImages[i-1].GetName(), listImages[i].GetName(), "Images should be sorted by name") + } + + // Test 10: Verify distinct results when joining with other tables (ROX-33514) + // Create 2 deployments that reference all 3 images + dep1 := fixtures.LightweightDeployment() + dep1.Id = uuid.NewV4().String() + dep1.Name = "deployment1" + dep1.Containers = []*storage.Container{ + {Name: "container1", Image: &storage.ContainerImage{Id: img1.GetDigest(), IdV2: img1.GetId(), Name: img1.GetName()}}, + {Name: "container2", Image: &storage.ContainerImage{Id: img2.GetDigest(), IdV2: img2.GetId(), Name: img2.GetName()}}, + {Name: "container3", Image: &storage.ContainerImage{Id: img3.GetDigest(), IdV2: img3.GetId(), Name: img3.GetName()}}, + } + + dep2 := fixtures.LightweightDeployment() + dep2.Id = uuid.NewV4().String() + dep2.Name = "deployment2" + dep2.Containers = []*storage.Container{ + {Name: "container4", Image: &storage.ContainerImage{Id: img1.GetDigest(), IdV2: img1.GetId(), Name: img1.GetName()}}, + {Name: "container5", Image: &storage.ContainerImage{Id: img2.GetDigest(), IdV2: img2.GetId(), Name: img2.GetName()}}, + {Name: "container6", Image: &storage.ContainerImage{Id: img3.GetDigest(), IdV2: img3.GetId(), Name: img3.GetName()}}, + } + + s.NoError(s.deploymentDataStore.UpsertDeployment(ctx, dep1)) + s.NoError(s.deploymentDataStore.UpsertDeployment(ctx, dep2)) + + // Search for images with deployment filter - should return 3 unique images, not 6 + q = pkgSearch.NewQueryBuilder().AddRegexes(pkgSearch.DeploymentName, ".*").ProtoQuery() + listImages, err = s.datastore.SearchListImages(ctx, q) + s.NoError(err) + + // Verify we get exactly 3 unique images despite having 2 deployments referencing all 3 + s.Len(listImages, 3, "Expected 3 unique images, not duplicates from multiple deployments") + imageDigests = set.NewStringSet() + for _, img := range listImages { + imageDigests.Add(img.GetId()) + } + s.True(imageDigests.Contains(img1.GetDigest())) + s.True(imageDigests.Contains(img2.GetDigest())) + s.True(imageDigests.Contains(img3.GetDigest())) +} + +// TestComponentRiskScorePersistedToDB verifies that when the component ranker +// contains a risk score for a component, UpsertImage writes that score to the +// database. This is a regression test for the bug where the ranker was not +// being updated after risk calculation, causing updateComponentRisk to +// overwrite calculated scores with 0. +func (s *ImageV2DataStoreTestSuite) TestComponentRiskScorePersistedToDB() { + ctx := sac.WithAllAccess(context.Background()) + + componentRanker := ranking.NewRanker() + ds := imageDataStoreV2.NewWithPostgres( + pgStore.New(s.testDB.DB, false, keyfence.ImageKeyFenceSingleton()), + s.mockRisk, + ranking.NewRanker(), + componentRanker, + ) + + img := fixtures.GetImageV2WithUniqueComponents(3) + + // Pre-populate the ranker with known scores for each component, + // simulating what the risk manager does after calculating risk. + expectedScores := []float32{1.5, 2.7, 4.2} + for index, component := range img.GetScan().GetComponents() { + componentID := scancomponent.ComponentIDV2(component, img.GetId(), index) + componentRanker.Add(componentID, expectedScores[index]) + } + + s.NoError(ds.UpsertImage(ctx, img)) + + // Read back from DB — the risk scores should match what was in the ranker. + got, found, err := ds.GetImage(ctx, img.GetId()) + s.NoError(err) + s.True(found) + + gotComponents := got.GetScan().GetComponents() + s.Require().Len(gotComponents, 3) + for i, component := range gotComponents { + s.InDelta(expectedScores[i], component.GetRiskScore(), 0.001, + "component %q risk score should be persisted to DB", component.GetName()) + } +} diff --git a/central/imagev2/views/image_id_and_digest_view.go b/central/imagev2/views/image_id_and_digest_view.go deleted file mode 100644 index ddd9bf5ffd411..0000000000000 --- a/central/imagev2/views/image_id_and_digest_view.go +++ /dev/null @@ -1,6 +0,0 @@ -package views - -type ImageIDAndDigestView struct { - ImageID string `db:"image_id"` - Digest string `db:"image_sha"` -} diff --git a/central/imagev2/views/list_image_view.go b/central/imagev2/views/list_image_view.go new file mode 100644 index 0000000000000..2be005fd8e474 --- /dev/null +++ b/central/imagev2/views/list_image_view.go @@ -0,0 +1,55 @@ +package views + +import ( + "time" + + "github.com/stackrox/rox/generated/storage" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// ListImageV2View holds the fields needed for ListImage responses, using view-based +// column selection instead of full protobuf deserialization. +type ListImageV2View struct { + Digest string `db:"image_sha"` + Name string `db:"image"` + ComponentCount int32 `db:"image_component_count"` + CVECount int32 `db:"image_cve_count"` + FixableCVECount int32 `db:"fixable_cve_count"` + Created *time.Time `db:"image_created_time"` + LastUpdated *time.Time `db:"last_updated"` +} + +// GetCreated returns the Created timestamp as a protobuf Timestamp, or nil if not set. +func (v *ListImageV2View) GetCreated() *timestamppb.Timestamp { + if v.Created == nil { + return nil + } + return timestamppb.New(*v.Created) +} + +// GetLastUpdated returns the LastUpdated timestamp as a protobuf Timestamp, or nil if not set. +func (v *ListImageV2View) GetLastUpdated() *timestamppb.Timestamp { + if v.LastUpdated == nil { + return nil + } + return timestamppb.New(*v.LastUpdated) +} + +// ToListImage converts a ListImageV2View to a storage.ListImage proto. +func (v *ListImageV2View) ToListImage() *storage.ListImage { + return &storage.ListImage{ + Id: v.Digest, + Name: v.Name, + SetComponents: &storage.ListImage_Components{ + Components: v.ComponentCount, + }, + SetCves: &storage.ListImage_Cves{ + Cves: v.CVECount, + }, + SetFixable: &storage.ListImage_FixableCves{ + FixableCves: v.FixableCVECount, + }, + Created: v.GetCreated(), + LastUpdated: v.GetLastUpdated(), + } +} diff --git a/central/internal/server.go b/central/internal/server.go index 2cfd375897313..6a39e8d772376 100644 --- a/central/internal/server.go +++ b/central/internal/server.go @@ -1,6 +1,7 @@ package internal import ( + "crypto/tls" "log" "net/http" @@ -8,6 +9,7 @@ import ( "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/grpc/metrics" "github.com/stackrox/rox/pkg/mtls" + "github.com/stackrox/rox/pkg/tlsprofile" ) // HTTPServer is a HTTP server to serve functionality available only within the cluster. @@ -45,6 +47,10 @@ func (s *HTTPServer) RunForever() { httpServer := &http.Server{ Addr: s.Address, Handler: s.mux, + TLSConfig: &tls.Config{ + MinVersion: tlsprofile.MinVersion(), + CipherSuites: tlsprofile.CipherSuites(), + }, } go runForever(httpServer) } diff --git a/central/jwt/jwt_test.go b/central/jwt/jwt_test.go index 8e21ba55aada6..7f6de28d326ac 100644 --- a/central/jwt/jwt_test.go +++ b/central/jwt/jwt_test.go @@ -31,9 +31,10 @@ func (v mockRoxValidator) Validate(ctx context.Context, token string) (*tokens.T func tokenWithIssuer(issuer string) string { claims := fmt.Sprintf(`{"iss":"%s"}`, issuer) - return fmt.Sprintf("%s.%s.signature", + return fmt.Sprintf("%s.%s.%s", base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"HS256","typ":"JWT"}`)), - base64.RawURLEncoding.EncodeToString([]byte(claims))) + base64.RawURLEncoding.EncodeToString([]byte(claims)), + base64.RawURLEncoding.EncodeToString([]byte("fake-signature"))) } func TestM2MValidator(t *testing.T) { diff --git a/central/lightspeed/client/client.go b/central/lightspeed/client/client.go new file mode 100644 index 0000000000000..867f400ffe27d --- /dev/null +++ b/central/lightspeed/client/client.go @@ -0,0 +1,120 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/pkg/errors" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/httputil/proxy" + "github.com/stackrox/rox/pkg/satoken" + "github.com/stackrox/rox/pkg/tlsutils" + "github.com/stackrox/rox/pkg/utils" +) + +const ( + serviceOperatorCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" + defaultRequestTimeout = 120 * time.Second +) + +var ( + // LightspeedEndpoint is the URL of the OpenShift Lightspeed API. URL will be populated from integration in a follow up PR + LightspeedEndpoint = env.RegisterSetting("ROX_LIGHTSPEED_ENDPOINT", + env.WithDefault("https://lightspeed-app-server.openshift-lightspeed.svc:8443")) +) + +// QueryRequest is the request payload for the OLS query API. +type QueryRequest struct { + Query string `json:"query"` +} + +// QueryResponse is the response payload from the OLS query API. +type QueryResponse struct { + Response string `json:"response"` +} + +// Client calls the OpenShift Lightspeed API. +type Client interface { + Query(ctx context.Context, req *QueryRequest) (*QueryResponse, error) + TestConnectivity() error +} + +// NewClient creates a Client that authenticates with the OLS API using the +// pod's Kubernetes service account token and trusts the service CA. +func NewClient() Client { + transport := tlsutils.TransportWithServiceCA() + transport.Proxy = proxy.FromConfig() + + return &clientImpl{ + loadToken: satoken.LoadTokenFromFile, + endpoint: LightspeedEndpoint.Setting(), + httpClient: &http.Client{ + Transport: transport, + Timeout: defaultRequestTimeout, + }, + } +} + +type clientImpl struct { + loadToken func() (string, error) + endpoint string + httpClient *http.Client +} + +func (c *clientImpl) Query(ctx context.Context, req *QueryRequest) (*QueryResponse, error) { + token, err := c.loadToken() + if err != nil { + return nil, errors.Wrap(err, "loading service account token") + } + + body, err := json.Marshal(req) + if err != nil { + return nil, errors.Wrap(err, "marshaling request") + } + + url := fmt.Sprintf("%s/v1/query", c.endpoint) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, errors.Wrap(err, "building HTTP request") + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return nil, errors.Wrap(err, "sending request to Lightspeed") + } + defer utils.IgnoreError(resp.Body.Close) + + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("Lightspeed API returned HTTP %d", resp.StatusCode) + } + + var result QueryResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, errors.Wrap(err, "decoding Lightspeed response") + } + + return &result, nil +} + +// TestConnectivity checks if the Lightspeed service is reachable. +func (c *clientImpl) TestConnectivity() error { + healthURL := fmt.Sprintf("%s/readiness", c.endpoint) + resp, err := c.httpClient.Get(healthURL) + if err != nil { + return errors.Wrap(err, "failed to connect to AI service") + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("AI service returned status %d", resp.StatusCode) + } + + return nil +} diff --git a/central/main.go b/central/main.go old mode 100644 new mode 100755 index b002a8d387a2c..cb3bb5d8a53a4 --- a/central/main.go +++ b/central/main.go @@ -24,11 +24,13 @@ import ( "github.com/stackrox/rox/central/audit" authDS "github.com/stackrox/rox/central/auth/datastore" internalTokenAuthService "github.com/stackrox/rox/central/auth/internaltokens/service" + openshiftAuth "github.com/stackrox/rox/central/auth/openshift" authService "github.com/stackrox/rox/central/auth/service" "github.com/stackrox/rox/central/auth/userpass" authProviderRegistry "github.com/stackrox/rox/central/authprovider/registry" authProviderSvc "github.com/stackrox/rox/central/authprovider/service" authProviderTelemetry "github.com/stackrox/rox/central/authprovider/telemetry" + backgroundmigrations "github.com/stackrox/rox/central/backgroundmigrations/runner" baseImageService "github.com/stackrox/rox/central/baseimage/service" baseImageWatcher "github.com/stackrox/rox/central/baseimage/watcher" centralHealthService "github.com/stackrox/rox/central/centralhealth/service" @@ -43,6 +45,7 @@ import ( clusterService "github.com/stackrox/rox/central/cluster/service" "github.com/stackrox/rox/central/clusterinit/backend" clusterInitService "github.com/stackrox/rox/central/clusterinit/service" + clusterInitStore "github.com/stackrox/rox/central/clusterinit/store/singleton" clustersHelmConfig "github.com/stackrox/rox/central/clusters/helmconfig" clustersZip "github.com/stackrox/rox/central/clusters/zip" complianceDatastore "github.com/stackrox/rox/central/compliance/datastore" @@ -104,6 +107,7 @@ import ( imageintegrationsDS "github.com/stackrox/rox/central/imageintegration/datastore" iiService "github.com/stackrox/rox/central/imageintegration/service" iiStore "github.com/stackrox/rox/central/imageintegration/store" + imageV2Datastore "github.com/stackrox/rox/central/imagev2/datastore" installationStore "github.com/stackrox/rox/central/installation/store" integrationHealthService "github.com/stackrox/rox/central/integrationhealth/service" "github.com/stackrox/rox/central/internal" @@ -141,7 +145,7 @@ import ( rbacService "github.com/stackrox/rox/central/rbac/service" vulnReportV2Scheduler "github.com/stackrox/rox/central/reports/scheduler/v2" reportServiceV2 "github.com/stackrox/rox/central/reports/service/v2" - v2Service "github.com/stackrox/rox/central/reports/service/v2" + nodeReportServiceV2 "github.com/stackrox/rox/central/reports/service/v2/node" "github.com/stackrox/rox/central/reprocessor" collectionService "github.com/stackrox/rox/central/resourcecollection/service" "github.com/stackrox/rox/central/risk/handlers/timeline" @@ -174,9 +178,11 @@ import ( userService "github.com/stackrox/rox/central/user/service" "github.com/stackrox/rox/central/version" vStore "github.com/stackrox/rox/central/version/store" - versionUtils "github.com/stackrox/rox/central/version/utils" virtualMachineDS "github.com/stackrox/rox/central/virtualmachine/datastore" + virtualMachineScanV2DS "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore" virtualmachineService "github.com/stackrox/rox/central/virtualmachine/service" + virtualMachineV2DS "github.com/stackrox/rox/central/virtualmachine/v2/datastore" + virtualmachineV2Service "github.com/stackrox/rox/central/virtualmachine/v2/service" vulnMgmtService "github.com/stackrox/rox/central/vulnmgmt/service" vulnRequestManager "github.com/stackrox/rox/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr" vulnRequestServiceV2 "github.com/stackrox/rox/central/vulnmgmt/vulnerabilityrequest/service/v2" @@ -213,6 +219,7 @@ import ( "github.com/stackrox/rox/pkg/grpc/errors" "github.com/stackrox/rox/pkg/grpc/ratelimit" "github.com/stackrox/rox/pkg/grpc/routes" + "github.com/stackrox/rox/pkg/grpc/versionheader" "github.com/stackrox/rox/pkg/httputil/proxy" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/memlimit" @@ -317,7 +324,6 @@ func main() { log.Errorf("Failed to remove backup DB: %v", err) } } - versionUtils.SetCurrentVersionPostgres(globaldb.GetPostgres()) features.LogFeatureFlags() @@ -348,8 +354,8 @@ func startTelemetryServer() { // * kubectl exec into central pod // * modifying central service definition func clusterInternalRoutes() []*internal.Route { - result := make([]*internal.Route, 0) debugRoutes := debugRoutes() + result := make([]*internal.Route, 0, len(debugRoutes)+1) for _, r := range debugRoutes { result = append(result, &internal.Route{ Route: r.Route, @@ -393,11 +399,20 @@ func startServices() { platformReprocessor.Singleton().Start() } - go registerDelayedIntegrations(iiStore.DelayedIntegrations) + // If we have no delayed integrations, we can skip spawning the goroutine + delayedIntegrations := iiStore.GetDelayedIntegrations() + log.Debug("Registering delayed integrations", "numIntegrations", len(delayedIntegrations)) + if len(delayedIntegrations) > 0 { + go registerDelayedIntegrations(delayedIntegrations) + } if env.DeclarativeConfiguration.BooleanSetting() { declarativeconfig.ManagerSingleton().ReconcileDeclarativeConfigurations() } + + if features.BackgroundMigration.Enabled() { + backgroundmigrations.Singleton().Start() + } } func servicesToRegister() []pkgGRPC.APIService { @@ -455,7 +470,7 @@ func servicesToRegister() []pkgGRPC.APIService { roleService.Singleton(), searchService.Singleton(), secretService.Singleton(), - sensorService.New(connection.ManagerSingleton(), all.Singleton(), clusterDataStore.Singleton(), installationStore.Singleton()), + sensorService.New(connection.ManagerSingleton(), all.Singleton(), clusterDataStore.Singleton(), installationStore.Singleton(), clusterInitStore.Singleton()), sensorUpgradeControlService.Singleton(), sensorUpgradeService.Singleton(), serviceAccountService.Singleton(), @@ -476,6 +491,10 @@ func servicesToRegister() []pkgGRPC.APIService { servicesToRegister = append(servicesToRegister, reportServiceV2.Singleton()) + if features.NodeVulnerabilityReports.Enabled() { + servicesToRegister = append(servicesToRegister, nodeReportServiceV2.Singleton()) + } + if features.ComplianceEnhancements.Enabled() { servicesToRegister = append(servicesToRegister, complianceOperatorIntegrationService.Singleton()) servicesToRegister = append(servicesToRegister, complianceScanSettings.Singleton()) @@ -486,7 +505,12 @@ func servicesToRegister() []pkgGRPC.APIService { } if features.VirtualMachines.Enabled() { - servicesToRegister = append(servicesToRegister, virtualmachineService.Singleton()) + if features.VirtualMachinesEnhancedDataModel.Enabled() { + // V2 service replaces V1 to avoid route conflicts on /v2/virtualmachines/{id}. + servicesToRegister = append(servicesToRegister, virtualmachineV2Service.Singleton()) + } else { + servicesToRegister = append(servicesToRegister, virtualmachineService.Singleton()) + } } if features.BaseImageDetection.Enabled() { @@ -513,7 +537,9 @@ func servicesToRegister() []pkgGRPC.APIService { } // Start cluster-level (Kubernetes, OpenShift, Istio) vulnerability data fetcher. - fetcher.SingletonManager().Start() + if features.LegacyScanner.Enabled() { + fetcher.SingletonManager().Start() + } if devbuild.IsEnabled() { servicesToRegister = append(servicesToRegister, developmentService.Singleton()) @@ -557,6 +583,9 @@ func startGRPCServer() { // is the case, we can be setting up an auth providers which won't work. if env.EnableOpenShiftAuth.BooleanSetting() { authProviderBackendFactories[openshift.TypeName] = openshift.NewFactory + if features.ACMAccessControlDelegation.Enabled() { + authProviderBackendFactories[openshift.TypeNameWithACMAccessControlDelegation] = openshift.NewFactoryWithACMAccessControlDelegation + } } for typeName, factoryCreator := range authProviderBackendFactories { @@ -575,6 +604,8 @@ func startGRPCServer() { basicAuthProvider := userpass.RegisterAuthProviderOrPanic(authProviderRegisteringCtx, basicAuthMgr, registry) + openshiftAuth.SeedMetricsAuthProvider(authProviderRegisteringCtx, registry, roleDataStore.Singleton(), groupDataStore.Singleton()) + clusterInitBackend := backend.Singleton() serviceMTLSExtractor, err := service.NewExtractorWithCertValidation(clusterInitBackend) if err != nil { @@ -634,6 +665,8 @@ func startGRPCServer() { centralSAC.GetEnricher().GetPreAuthContextEnricher(authzTraceSink), ) + config.UnaryInterceptors = append(config.UnaryInterceptors, versionheader.CentralVersionServerInterceptor()) + // Telemetry client has to add interceptors before starting the server. c := phonehomeClient.Singleton() config.HTTPInterceptors = append(config.HTTPInterceptors, c.GetHTTPInterceptor()) @@ -681,7 +714,8 @@ func addCentralIdentityGatherers(c *phonehomeClient.CentralClient) { add(authProviderTelemetry.Gather) add(cloudSourcesDS.Gather(cloudSourcesDS.Singleton())) add(clusterDataStore.Gather) - add(virtualMachineDS.Gather(virtualMachineDS.Singleton())) + add(complianceScanDS.GatherProfiles(complianceScanDS.Singleton())) + add(complianceScanDS.GatherTailoredProfiles(complianceScanDS.Singleton())) add(declarativeconfig.ManagerSingleton().Gather()) add(delegatedRegistryConfigDS.Gather(delegatedRegistryConfigDS.Singleton())) add(externalbackupsDS.Gather) @@ -689,16 +723,27 @@ func addCentralIdentityGatherers(c *phonehomeClient.CentralClient) { add(globaldb.Gather) add(imageintegrationsDS.Gather) add(notifierDS.Gather) + add(policyDataStore.Gather) add(roleDataStore.Gather) add(signatureIntegrationDS.Gather) - add(complianceScanDS.GatherProfiles(complianceScanDS.Singleton())) - add(policyDataStore.Gather) + add(virtualMachineDS.Gather(virtualMachineDS.Singleton())) + add(virtualMachineDS.GatherV2( + virtualMachineV2DS.Singleton(), + virtualMachineScanV2DS.Singleton(), + )) } func registerDelayedIntegrations(integrationsInput []iiStore.DelayedIntegration) { + numIntegrations := len(integrationsInput) + if numIntegrations == 0 { + // we shouldn't get here, but in case we do let's just bail out here + log.Debug("No delayed integrations to register") + return + } + integrationManager := enrichment.ManagerSingleton() - integrations := make(map[int]iiStore.DelayedIntegration, len(integrationsInput)) + integrations := make(map[int]iiStore.DelayedIntegration, numIntegrations) for k, v := range integrationsInput { integrations[k] = v } @@ -807,13 +852,19 @@ func customRoutes() (customRoutes []routes.CustomRoute) { { Route: "/api/v1/images/sbom", Authorizer: user.With(permissions.Modify(resources.Image)), - ServerHandler: imageService.SBOMHandler(imageintegration.Set(), enrichment.ImageEnricherSingleton(), enrichment.ImageEnricherV2Singleton(), sachelper.NewClusterSacHelper(clusterDataStore.Singleton()), riskManager.Singleton()), + ServerHandler: imageService.SBOMGenHandler(imageintegration.Set(), enrichment.ImageEnricherSingleton(), enrichment.ImageEnricherV2Singleton(), sachelper.NewClusterSacHelper(clusterDataStore.Singleton()), riskManager.Singleton()), + Compression: true, + }, + { + Route: "/api/v1/sboms/scan", + Authorizer: user.With(permissions.Modify(resources.Image)), + ServerHandler: imageService.SBOMScanHandler(imageintegration.Set()), Compression: true, }, { Route: "/api/splunk/ta/vulnmgmt", Authorizer: user.With(permissions.View(resources.Image), permissions.View(resources.Deployment)), - ServerHandler: splunk.NewVulnMgmtHandler(deploymentDatastore.Singleton(), imageDatastore.Singleton()), + ServerHandler: splunk.NewVulnMgmtHandler(deploymentDatastore.Singleton(), imageDatastore.Singleton(), imageV2Datastore.Singleton()), Compression: true, }, { @@ -950,11 +1001,20 @@ func customRoutes() (customRoutes []routes.CustomRoute) { // Append report custom routes customRoutes = append(customRoutes, routes.CustomRoute{ Route: "/api/reports/jobs/download", - Authorizer: user.With(permissions.Modify(resources.WorkflowAdministration), permissions.View(resources.Image)), - ServerHandler: v2Service.NewDownloadHandler(), + Authorizer: user.With(permissions.View(resources.Image)), + ServerHandler: reportServiceV2.NewDownloadHandler(), Compression: true, }) + if features.NodeVulnerabilityReports.Enabled() { + customRoutes = append(customRoutes, routes.CustomRoute{ + Route: "/api/reports/node/jobs/download", + Authorizer: user.With(permissions.View(resources.Node), permissions.View(resources.Cluster)), + ServerHandler: reportServiceV2.NewNodeDownloadHandler(), + Compression: true, + }) + } + if features.ComplianceEnhancements.Enabled() && features.ComplianceReporting.Enabled() && features.ScanScheduleReportJobs.Enabled() { customRoutes = append(customRoutes, routes.CustomRoute{ Route: "/v2/compliance/scan/configurations/reports/download", @@ -970,6 +1030,10 @@ func customRoutes() (customRoutes []routes.CustomRoute) { } func debugRoutes() []routes.CustomRoute { + if env.ContinuousProfiling.BooleanSetting() { + return []routes.CustomRoute{} + } + customRoutes := make([]routes.CustomRoute, 0, len(routes.DebugRoutes)) for r, h := range routes.DebugRoutes { @@ -1001,7 +1065,9 @@ func waitForTerminationSignal() { stoppables := []stoppableWithName{ {reprocessor.Singleton(), "reprocessor loop"}, {suppress.Singleton(), "cve unsuppress loop"}, - {pruning.Singleton(), "garbage collector"}, + } + stoppables = append(stoppables, stoppableWithName{pruning.Singleton(), "garbage collector"}) + stoppables = append(stoppables, []stoppableWithName{ {gatherer.Singleton(), "network graph default external sources gatherer"}, {vulnRequestManager.Singleton(), "vuln deferral requests expiry loop"}, {phonehomeClient.Singleton().Gatherer(), "telemetry gatherer"}, @@ -1011,14 +1077,16 @@ func waitForTerminationSignal() { {gcp.Singleton(), "GCP cloud credentials manager"}, {cloudSourcesManager.Singleton(), "cloud sources manager"}, {administrationEventHandler.Singleton(), "administration events handler"}, - } + }...) if baseImageWatcher.Enabled() { stoppables = append(stoppables, stoppableWithName{baseImageWatcher.Singleton(), "base image watcher"}) } - stoppables = append(stoppables, - stoppableWithName{vulnReportV2Scheduler.Singleton(), "vuln reports v2 scheduler"}) + if !env.CentralWorkerEnabled.BooleanSetting() { + stoppables = append(stoppables, + stoppableWithName{vulnReportV2Scheduler.Singleton(), "vuln reports v2 scheduler"}) + } if features.ComplianceReporting.Enabled() { stoppables = append(stoppables, @@ -1030,6 +1098,18 @@ func waitForTerminationSignal() { stoppableWithName{platformReprocessor.Singleton(), "platform components reprocessor"}) } + if features.BackgroundMigration.Enabled() { + stoppables = append(stoppables, + stoppableWithName{backgroundmigrations.Singleton(), "background migrations"}) + } + + if w := signatureIntegrationDS.KeyBundleWatcher(); w != nil { + stoppables = append(stoppables, stoppableWithName{w, "Red Hat signing key bundle watcher"}) + } + if u := signatureIntegrationDS.KeyBundleUpdater(); u != nil { + stoppables = append(stoppables, stoppableWithName{u, "Red Hat signing key bundle updater"}) + } + var wg sync.WaitGroup for _, stoppable := range stoppables { wg.Add(1) diff --git a/central/metadata/service/service.go b/central/metadata/service/service.go index c1e18a07c8fd6..ab17f66d74b27 100644 --- a/central/metadata/service/service.go +++ b/central/metadata/service/service.go @@ -4,6 +4,7 @@ import ( "context" "github.com/stackrox/rox/central/globaldb" + olsClient "github.com/stackrox/rox/central/lightspeed/client" systemInfoStorage "github.com/stackrox/rox/central/systeminfo/store/postgres" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/pkg/grpc" @@ -24,14 +25,15 @@ type Service interface { // New returns a new instance of service. func New() Service { - return NewWithCertificateProvider(&defaultCertificateProvider{}) + return NewWithCertificateProvider(&defaultCertificateProvider{}, olsClient.NewClient()) } // NewWithCertificateProvider returns a new instance of service with a custom certificate provider. -func NewWithCertificateProvider(certProvider CertificateProvider) Service { +func NewWithCertificateProvider(certProvider CertificateProvider, lightspeedClient olsClient.Client) Service { return &serviceImpl{ - db: globaldb.GetPostgres(), - systemInfoStore: systemInfoStorage.Singleton(), - certProvider: certProvider, + db: globaldb.GetPostgres(), + systemInfoStore: systemInfoStorage.Singleton(), + certProvider: certProvider, + lightspeedClient: lightspeedClient, } } diff --git a/central/metadata/service/service_impl.go b/central/metadata/service/service_impl.go index 851e9ec01b2d2..3f88b23ae1037 100644 --- a/central/metadata/service/service_impl.go +++ b/central/metadata/service/service_impl.go @@ -5,11 +5,14 @@ import ( "crypto/tls" "crypto/x509" "encoding/base64" + "sync/atomic" + "time" cTLS "github.com/google/certificate-transparency-go/tls" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/pkg/errors" "github.com/stackrox/rox/central/globaldb" + olsClient "github.com/stackrox/rox/central/lightspeed/client" "github.com/stackrox/rox/central/metadata/centralcapabilities" systemInfoStorage "github.com/stackrox/rox/central/systeminfo/store/postgres" "github.com/stackrox/rox/central/tlsconfig" @@ -25,10 +28,12 @@ import ( "github.com/stackrox/rox/pkg/grpc/authz/perrpc" "github.com/stackrox/rox/pkg/grpc/authz/user" "github.com/stackrox/rox/pkg/mtls" + "github.com/stackrox/rox/pkg/mtls/certwatch" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgconfig" "github.com/stackrox/rox/pkg/sync" "github.com/stackrox/rox/pkg/version" + "github.com/stackrox/rox/pkg/version/versioncompatibility" "google.golang.org/grpc" ) @@ -38,6 +43,7 @@ var ( v1.MetadataService_GetDatabaseStatus_FullMethodName, v1.MetadataService_GetDatabaseBackupStatus_FullMethodName, v1.MetadataService_GetCentralCapabilities_FullMethodName, + v1.MetadataService_GetLightspeedStatus_FullMethodName, }, // When this endpoint was public, Sensor relied on it to check Central's // availability. While Sensor might not do so today, we need to ensure @@ -50,15 +56,15 @@ var ( }, }) - // Primary leaf certificate caching + primaryLeafCert atomic.Pointer[tls.Certificate] primaryLeafCertOnce sync.Once - primaryLeafCert tls.Certificate - primaryLeafCertErr error - // Secondary CA leaf certificate caching - secondaryCALeafCertOnce sync.Once - secondaryCALeafCert tls.Certificate - secondaryCALeafCertErr error + // secondaryCALeafCert holds a short-lived leaf cert signed by the secondary CA, + // used only to sign TLSChallenge responses for Sensors that trust the secondary CA. + // Re-issued automatically when closer than secondaryLeafCertRenewalBuf to expiry. + secondaryCALeafCert atomic.Pointer[tls.Certificate] + secondaryCALeafCertIssueMu sync.Mutex + secondaryLeafCertRenewalBuf = 1 * time.Hour ) // CertificateProvider provides certificates for TLS challenge operations @@ -80,21 +86,26 @@ type CertificateProvider interface { // defaultCertificateProvider implements CertificateProvider using global mtls functions type defaultCertificateProvider struct{} +func loadAndWatchPrimaryLeafCert() { + primaryLeafCertOnce.Do(func() { + certwatch.WatchCertDir("internal service", mtls.CertsPrefix, tlsconfig.LoadInternalCertificateFromDirectory, func(cert *tls.Certificate) { + if cert != nil { + primaryLeafCert.Store(cert) + } + }, certwatch.WithVerify(false)) + }) +} + func (p *defaultCertificateProvider) GetPrimaryCACert() (*x509.Certificate, []byte, error) { return mtls.CACert() } func (p *defaultCertificateProvider) GetPrimaryLeafCert() (tls.Certificate, error) { - primaryLeafCertOnce.Do(func() { - cert, err := mtls.LeafCertificateFromFile() - if err != nil { - primaryLeafCertErr = err - return - } - primaryLeafCert = cert - }) - - return primaryLeafCert, primaryLeafCertErr + loadAndWatchPrimaryLeafCert() + if cert := primaryLeafCert.Load(); cert != nil { + return *cert, nil + } + return mtls.LeafCertificateFromFile() } func (p *defaultCertificateProvider) GetSecondaryCAForSigning() (mtls.CA, error) { @@ -105,21 +116,39 @@ func (p *defaultCertificateProvider) GetSecondaryCACert() (*x509.Certificate, [] return mtls.SecondaryCACert() } +func loadSecondaryLeafCertIfValid() *tls.Certificate { + cert := secondaryCALeafCert.Load() + if cert == nil || cert.Leaf == nil { + return nil + } + stillInRenewalBuffer := time.Now().Add(secondaryLeafCertRenewalBuf).Before(cert.Leaf.NotAfter) + if stillInRenewalBuffer { + return cert + } + return nil +} + func (p *defaultCertificateProvider) GetSecondaryLeafCert() (tls.Certificate, error) { - secondaryCALeafCertOnce.Do(func() { - leafCert, err := issueSecondaryCALeafCert(p) - if err != nil { - secondaryCALeafCertErr = err - return - } - secondaryCALeafCert = leafCert - }) + if cert := loadSecondaryLeafCertIfValid(); cert != nil { + return *cert, nil + } + return p.ensureSecondaryLeafCert() +} + +func (p *defaultCertificateProvider) ensureSecondaryLeafCert() (tls.Certificate, error) { + secondaryCALeafCertIssueMu.Lock() + defer secondaryCALeafCertIssueMu.Unlock() - if secondaryCALeafCertErr != nil { - return tls.Certificate{}, secondaryCALeafCertErr + if cert := loadSecondaryLeafCertIfValid(); cert != nil { + return *cert, nil } - return secondaryCALeafCert, nil + leafCert, err := issueSecondaryCALeafCert(p) + if err != nil { + return tls.Certificate{}, err + } + secondaryCALeafCert.Store(&leafCert) + return leafCert, nil } func issueSecondaryCALeafCert(certProvider CertificateProvider) (tls.Certificate, error) { @@ -128,7 +157,7 @@ func issueSecondaryCALeafCert(certProvider CertificateProvider) (tls.Certificate return tls.Certificate{}, errors.Wrap(err, "failed to load secondary CA for signing") } - issuedCert, issueErr := secondaryCA.IssueCertForSubject(mtls.CentralSubject) + issuedCert, issueErr := secondaryCA.IssueCertForSubject(mtls.CentralSubject, mtls.WithValidityExpiringInHours()) if issueErr != nil { return tls.Certificate{}, errors.Wrap(issueErr, "failed to issue leaf certificate from secondary CA") } @@ -138,6 +167,11 @@ func issueSecondaryCALeafCert(certProvider CertificateProvider) (tls.Certificate return tls.Certificate{}, errors.Wrap(err, "failed to load X509 key pair for temporary leaf cert from secondary CA") } + leafCert.Leaf, err = x509.ParseCertificate(leafCert.Certificate[0]) + if err != nil { + return tls.Certificate{}, errors.Wrap(err, "parsing secondary leaf certificate") + } + return leafCert, nil } @@ -145,9 +179,10 @@ func issueSecondaryCALeafCert(certProvider CertificateProvider) (tls.Certificate type serviceImpl struct { v1.UnimplementedMetadataServiceServer - db postgres.DB - systemInfoStore systemInfoStorage.Store - certProvider CertificateProvider + db postgres.DB + systemInfoStore systemInfoStorage.Store + certProvider CertificateProvider + lightspeedClient olsClient.Client } // RegisterServiceServer registers this service with the given gRPC Server. @@ -172,9 +207,14 @@ func (s *serviceImpl) GetMetadata(ctx context.Context, _ *v1.Empty) (*v1.Metadat ReleaseBuild: buildinfo.ReleaseBuild, LicenseStatus: v1.Metadata_VALID, } - // Only return the version to logged in users, not anonymous users. + // Only return version information to logged-in users, not anonymous users. if authn.IdentityFromContextOrNil(ctx) != nil { metadata.Version = version.GetMainVersion() + if versions, err := versioncompatibility.CompatibleVersions(); err == nil { + for _, xy := range versions { + metadata.CompatibleSensorVersions = append(metadata.CompatibleSensorVersions, xy.String()) + } + } } return metadata, nil } @@ -331,3 +371,25 @@ func (s *serviceImpl) GetDatabaseBackupStatus(ctx context.Context, _ *v1.Empty) func (s *serviceImpl) GetCentralCapabilities(_ context.Context, _ *v1.Empty) (*v1.CentralServicesCapabilities, error) { return centralcapabilities.GetCentralCapabilities(), nil } + +// GetLightspeedStatus returns the availability status of the Lightspeed AI service. +func (s *serviceImpl) GetLightspeedStatus(_ context.Context, _ *v1.Empty) (*v1.LightspeedStatusResponse, error) { + if s.lightspeedClient == nil { + return &v1.LightspeedStatusResponse{ + Available: false, + Message: "Lightspeed client not configured", + }, nil + } + + if err := s.lightspeedClient.TestConnectivity(); err != nil { + return &v1.LightspeedStatusResponse{ + Available: false, + Message: errors.Wrap(err, "Lightspeed AI service connectivity test failed").Error(), + }, nil + } + + return &v1.LightspeedStatusResponse{ + Available: true, + Message: "Lightspeed AI service is available", + }, nil +} diff --git a/central/metadata/service/service_impl_test.go b/central/metadata/service/service_impl_test.go index e6452674110fa..e9601870b4acd 100644 --- a/central/metadata/service/service_impl_test.go +++ b/central/metadata/service/service_impl_test.go @@ -12,6 +12,7 @@ import ( "path" "path/filepath" "testing" + "time" "github.com/cloudflare/cfssl/csr" "github.com/cloudflare/cfssl/initca" @@ -266,6 +267,63 @@ func (s *serviceImplTestSuite) TestIssueSecondaryCALeafCert() { s.Contains(err.Error(), "failed to load secondary CA for signing") s.Contains(err.Error(), "secondary CA file not found") }) + + s.Run("issued cert has short validity", func() { + mockProvider := &testCertificateProvider{ + secondaryCA: secondaryCA, + shouldFailSecondary: false, + } + + cert, err := issueSecondaryCALeafCert(mockProvider) + s.Require().NoError(err) + s.Require().NotNil(cert.Leaf) + + validity := cert.Leaf.NotAfter.Sub(cert.Leaf.NotBefore) + s.Less(validity, 4*time.Hour, "secondary leaf cert should have short validity (<=3 hours)") + }) +} + +func (s *serviceImplTestSuite) TestLoadSecondaryLeafCertIfValid() { + s.Run("returns nil when no cert is cached", func() { + secondaryCALeafCert.Store(nil) + s.Nil(loadSecondaryLeafCertIfValid()) + }) + + s.Run("returns cert when valid", func() { + secondaryCA := s.createTestCA("Test Secondary CA", "17520h") + mockProvider := &testCertificateProvider{ + secondaryCA: secondaryCA, + shouldFailSecondary: false, + } + cert, err := issueSecondaryCALeafCert(mockProvider) + s.Require().NoError(err) + + secondaryCALeafCert.Store(&cert) + s.NotNil(loadSecondaryLeafCertIfValid()) + secondaryCALeafCert.Store(nil) + }) + + s.Run("returns nil when cert is expired", func() { + expiredCert := &tls.Certificate{ + Leaf: &x509.Certificate{ + NotAfter: time.Now().Add(-1 * time.Hour), + }, + } + secondaryCALeafCert.Store(expiredCert) + s.Nil(loadSecondaryLeafCertIfValid()) + secondaryCALeafCert.Store(nil) + }) + + s.Run("returns nil when cert is within renewal buffer", func() { + nearExpiryCert := &tls.Certificate{ + Leaf: &x509.Certificate{ + NotAfter: time.Now().Add(30 * time.Minute), + }, + } + secondaryCALeafCert.Store(nearExpiryCert) + s.Nil(loadSecondaryLeafCertIfValid(), "cert expiring within renewal buffer should trigger re-issue") + secondaryCALeafCert.Store(nil) + }) } func (s *serviceImplTestSuite) createTestCA(commonName, expiry string) mtls.CA { diff --git a/central/metadata/service/service_metadata_test.go b/central/metadata/service/service_metadata_test.go new file mode 100644 index 0000000000000..546ef6fbf882b --- /dev/null +++ b/central/metadata/service/service_metadata_test.go @@ -0,0 +1,37 @@ +package service + +import ( + "context" + "testing" + + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/pkg/grpc/authn" + mockIdentity "github.com/stackrox/rox/pkg/grpc/authn/mocks" + "github.com/stackrox/rox/pkg/version/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestGetMetadata(t *testing.T) { + testutils.SetMainVersion(t, "4.11.0-testing") + svc := &serviceImpl{} + + t.Run("logged in user gets version and compatible sensors", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + mockID := mockIdentity.NewMockIdentity(mockCtrl) + ctx := authn.ContextWithIdentity(context.Background(), mockID, t) + + metadata, err := svc.GetMetadata(ctx, &v1.Empty{}) + require.NoError(t, err) + assert.Equal(t, "4.11.0-testing", metadata.GetVersion()) + assert.NotEmpty(t, metadata.GetCompatibleSensorVersions()) + }) + + t.Run("anonymous user gets no version", func(t *testing.T) { + metadata, err := svc.GetMetadata(context.Background(), &v1.Empty{}) + require.NoError(t, err) + assert.Empty(t, metadata.GetVersion()) + assert.Empty(t, metadata.GetCompatibleSensorVersions()) + }) +} diff --git a/central/metrics/central.go b/central/metrics/central.go index 248e18e2aac60..c52b146564af8 100644 --- a/central/metrics/central.go +++ b/central/metrics/central.go @@ -16,7 +16,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "index_op_duration", - Help: "Time taken to perform an index operation", + Help: "Time spent performing an index operation in milliseconds", // We care more about precision at lower latencies, or outliers at higher latencies. Buckets: prometheus.ExponentialBuckets(4, 2, 8), }, []string{"Operation", "Type"}) @@ -25,7 +25,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "cache_op_duration", - Help: "Time taken to perform a cache operation", + Help: "Time spent performing a cache operation in milliseconds", // We care more about precision at lower latencies, or outliers at higher latencies. Buckets: prometheus.ExponentialBuckets(4, 2, 8), }, []string{"Operation", "Type"}) @@ -34,7 +34,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "prune_duration", - Help: "Time to perform a pruning operation", + Help: "Time to perform a pruning operation in milliseconds", // We care more about precision at lower latencies, or outliers at higher latencies. Buckets: prometheus.ExponentialBuckets(4, 2, 8), }, []string{"Type"}) @@ -43,7 +43,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "postgres_op_duration", - Help: "Time taken to perform a postgres operation", + Help: "Time spent performing a postgres operation in milliseconds", // We care more about precision at lower latencies, or outliers at higher latencies. Buckets: prometheus.ExponentialBuckets(4, 2, 8), }, []string{"Operation", "Type"}) @@ -52,7 +52,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "postgres_acquire_conn_op_duration", - Help: "Time taken to acquire a Postgres connection", + Help: "Time spent acquiring a Postgres connection in milliseconds", // We care more about precision at lower latencies, or outliers at higher latencies. Buckets: prometheus.ExponentialBuckets(4, 2, 8), }, []string{"Operation", "Type"}) @@ -61,7 +61,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "graphql_op_duration", - Help: "Time taken to run a single graphql sub resolver/sub query", + Help: "Time spent running a single graphql sub resolver/sub query in milliseconds", // We care more about precision at lower latencies, or outliers at higher latencies. Buckets: prometheus.ExponentialBuckets(4, 2, 8), }, []string{"Resolver", "Operation"}) @@ -70,7 +70,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "graphql_query_duration", - Help: "Time taken to run a single graphql query", + Help: "Time spent running a single graphql query in milliseconds", // We care more about precision at lower latencies, or outliers at higher latencies. Buckets: prometheus.ExponentialBuckets(4, 2, 8), }, []string{"Query"}) @@ -79,7 +79,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "sensor_event_duration", - Help: "Time taken to perform an process a sensor event operation", + Help: "Time spent performing a sensor event operation in milliseconds", // We care more about precision at lower latencies, or outliers at higher latencies. Buckets: prometheus.ExponentialBuckets(4, 2, 13), }, []string{"Type", "Action"}) @@ -88,14 +88,14 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "sensor_event_queue", - Help: "Number of elements in removed from the queue", + Help: "Number of enqueue and dequeue operations on Central's event deduping queues for messages arriving from Sensor", }, []string{"Operation", "Type"}) resourceProcessedCounterVec = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "resource_processed_count", - Help: "Number of elements received and processed", + Help: "Number of sensor event resources successfully processed by Central pipelines", }, []string{"Operation", "Resource"}) totalNetworkFlowsReceivedCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ @@ -112,18 +112,26 @@ var ( Help: "A counter of the total number of network endpoints received by Central from Sensor", }, []string{"ClusterID"}) + // totalExternalPoliciesGauge is deprecated due to naming confusion (vector vs. gauge), use currentExternalPolicies instead. totalExternalPoliciesGauge = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "total_external_policies_count", - Help: "A gauge of the total number of policy as code CRs that have been accepted by Central from Config Controller", + Help: "Number of policy-as-code CRs accepted by Central from Config Controller", + }) + // currentExternalPolicies replaces the totalExternalPoliciesGauge + currentExternalPolicies = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "number_of_external_policies_current", + Help: "Number of policy-as-code CRs accepted by Central from Config Controller", }) riskProcessingHistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "risk_processing_duration", - Help: "Histogram of how long risk processing takes", + Help: "Time in milliseconds spent recomputing and persisting risk scores in Central's risk manager for deployments, nodes, images, and their components", Buckets: prometheus.ExponentialBuckets(4, 2, 8), }, []string{"Risk_Reprocessor"}) @@ -131,7 +139,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "datastore_function_duration", - Help: "Histogram of how long a datastore function takes", + Help: "Histogram of how long a datastore function takes in milliseconds", Buckets: prometheus.ExponentialBuckets(4, 2, 8), }, []string{"Type", "Function"}) @@ -139,7 +147,7 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "function_segment_duration", - Help: "Histogram of how long a particular segment within a function takes", + Help: "Histogram of how long a particular segment within a function takes in milliseconds", Buckets: prometheus.ExponentialBuckets(4, 2, 8), }, []string{"Segment"}) @@ -169,34 +177,35 @@ var ( Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "orphaned_plop_total", - Help: "A counter of the total number of PLOP objects without a reference to a ProcessIndicator", + Help: "Count of process-listening-on-port records that arrived without a matching process indicator", }, []string{"ClusterID"}) processQueueLengthGauge = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "process_queue_length", - Help: "A gauge that indicates the current number of processes that have not been flushed", + Help: "Current number of process indicators queued for baseline evaluation and persistence", }) sensorEventsDeduperCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "sensor_event_deduper", - Help: "A counter that tracks objects that has passed the sensor event deduper in the connection stream", + Help: "Counts sensor events skipped by the deduper vs processed as new", }, []string{"status", "type"}) pipelinePanicCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "pipeline_panics", - Help: "A counter that tracks the number of panics that have occurred in the processing pipelines", + Help: "Count of panics recovered in Central processing pipelines", }, []string{"resource"}) sensorConnectedCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "sensor_connected", + Help: "Count of sensor connections observed by Central", }, []string{"ClusterID", "connection_state"}) grpcLastMessageSizeReceived = prometheus.NewGaugeVec(prometheus.GaugeOpts{ @@ -421,10 +430,12 @@ func IncrementTotalNetworkEndpointsReceivedCounter(clusterID string, numberOfEnd func IncrementTotalExternalPoliciesGauge() { totalExternalPoliciesGauge.Inc() + currentExternalPolicies.Inc() } func DecrementTotalExternalPoliciesGauge() { totalExternalPoliciesGauge.Dec() + currentExternalPolicies.Dec() } // ObserveRiskProcessingDuration adds an observation for risk processing duration. diff --git a/central/metrics/custom/administrative_events/labels.go b/central/metrics/custom/administrative_events/labels.go new file mode 100644 index 0000000000000..d8eabab03a53a --- /dev/null +++ b/central/metrics/custom/administrative_events/labels.go @@ -0,0 +1,24 @@ +package administrative_events + +import ( + "github.com/stackrox/rox/central/metrics/custom/tracker" + "github.com/stackrox/rox/generated/storage" +) + +var LazyLabels = tracker.LazyLabelGetters[*finding]{ + "Type": func(f *finding) string { return f.GetType().String() }, + "Level": func(f *finding) string { return f.GetLevel().String() }, + "Domain": func(f *finding) string { return f.GetDomain() }, + "ResourceType": func(f *finding) string { return f.GetResource().GetType() }, + "ResourceName": func(f *finding) string { return f.GetResource().GetName() }, +} + +type finding struct { + *storage.AdministrationEvent +} + +func (f *finding) GetIncrement() int { + return int(f.GetNumOccurrences()) +} + +var _ tracker.WithIncrement = (*finding)(nil) diff --git a/central/metrics/custom/administrative_events/tracker.go b/central/metrics/custom/administrative_events/tracker.go new file mode 100644 index 0000000000000..8a383ff745142 --- /dev/null +++ b/central/metrics/custom/administrative_events/tracker.go @@ -0,0 +1,40 @@ +package administrative_events + +import ( + "context" + + adminEventDS "github.com/stackrox/rox/central/administration/events/datastore" + "github.com/stackrox/rox/central/metrics" + "github.com/stackrox/rox/central/metrics/custom/tracker" + "github.com/stackrox/rox/pkg/search" +) + +func New(ds adminEventDS.DataStore) *tracker.TrackerBase[*finding] { + return tracker.MakeGlobalTrackerBase( + metrics.AdminEvents, + "administrative events", + LazyLabels, + func(ctx context.Context, _ tracker.MetricDescriptors) tracker.FindingErrorSequence[*finding] { + return track(ctx, ds) + }, + ) +} + +func track(ctx context.Context, ds adminEventDS.DataStore) tracker.FindingErrorSequence[*finding] { + return func(yield func(*finding, error) bool) { + if ds == nil { + return + } + collector := tracker.NewFindingCollector(yield) + events, err := ds.ListEvents(ctx, search.EmptyQuery()) + if err != nil { + collector.Error(err) + return + } + for _, event := range events { + if err := collector.Yield(&finding{event}); err != nil { + return + } + } + } +} diff --git a/central/metrics/custom/administrative_events/tracker_test.go b/central/metrics/custom/administrative_events/tracker_test.go new file mode 100644 index 0000000000000..4ff0fe68359e2 --- /dev/null +++ b/central/metrics/custom/administrative_events/tracker_test.go @@ -0,0 +1,69 @@ +package administrative_events + +import ( + "context" + "testing" + + adminEventDS "github.com/stackrox/rox/central/administration/events/datastore/mocks" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" +) + +func Test_track(t *testing.T) { + ctrl := gomock.NewController(t) + ds := adminEventDS.NewMockDataStore(ctrl) + + ds.EXPECT().ListEvents(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ *v1.Query) ([]*storage.AdministrationEvent, error) { + return []*storage.AdministrationEvent{ + { + Type: storage.AdministrationEventType_ADMINISTRATION_EVENT_TYPE_LOG_MESSAGE, + Level: storage.AdministrationEventLevel_ADMINISTRATION_EVENT_LEVEL_ERROR, + Domain: "AuthProvider", + Resource: &storage.AdministrationEvent_Resource{ + Type: "AuthProvider", + Name: "LDAP", + }, + NumOccurrences: 5, + }, + { + Type: storage.AdministrationEventType_ADMINISTRATION_EVENT_TYPE_GENERIC, + Level: storage.AdministrationEventLevel_ADMINISTRATION_EVENT_LEVEL_WARNING, + Domain: "PolicyManagement", + Resource: &storage.AdministrationEvent_Resource{ + Type: "Policy", + Name: "No bash", + }, + NumOccurrences: 3, + }, + }, nil + }) + + var findings []*finding + for f := range track(context.Background(), ds) { + findings = append(findings, f) + } + + assert.Len(t, findings, 2) + + assert.Equal(t, "ADMINISTRATION_EVENT_TYPE_LOG_MESSAGE", LazyLabels["Type"](findings[0])) + assert.Equal(t, "ADMINISTRATION_EVENT_LEVEL_ERROR", LazyLabels["Level"](findings[0])) + assert.Equal(t, "AuthProvider", LazyLabels["Domain"](findings[0])) + assert.Equal(t, "AuthProvider", LazyLabels["ResourceType"](findings[0])) + assert.Equal(t, "LDAP", LazyLabels["ResourceName"](findings[0])) + assert.Equal(t, 5, findings[0].GetIncrement()) + + assert.Equal(t, "ADMINISTRATION_EVENT_TYPE_GENERIC", LazyLabels["Type"](findings[1])) + assert.Equal(t, "PolicyManagement", LazyLabels["Domain"](findings[1])) + assert.Equal(t, 3, findings[1].GetIncrement()) +} + +func Test_track_nil_ds(t *testing.T) { + var count int + for range track(context.Background(), nil) { + count++ + } + assert.Zero(t, count) +} diff --git a/central/metrics/custom/clusters/tracker.go b/central/metrics/custom/clusters/tracker.go index 13468130ac655..c791f0a750337 100644 --- a/central/metrics/custom/clusters/tracker.go +++ b/central/metrics/custom/clusters/tracker.go @@ -4,12 +4,13 @@ import ( "context" clusterDS "github.com/stackrox/rox/central/cluster/datastore" + "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/central/metrics/custom/tracker" ) func New(ds clusterDS.DataStore) *tracker.TrackerBase[*finding] { return tracker.MakeTrackerBase( - "health", + metrics.Health, "clusters", LazyLabels, func(ctx context.Context, _ tracker.MetricDescriptors) tracker.FindingErrorSequence[*finding] { diff --git a/central/metrics/custom/expiry/tracker.go b/central/metrics/custom/expiry/tracker.go index e587f127effd7..30b7d17ca5977 100644 --- a/central/metrics/custom/expiry/tracker.go +++ b/central/metrics/custom/expiry/tracker.go @@ -5,14 +5,16 @@ import ( "time" "github.com/stackrox/rox/central/credentialexpiry/service" + "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/central/metrics/custom/tracker" v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/pkg/logging" ) func New(s service.Service) *tracker.TrackerBase[*finding] { - return tracker.MakeTrackerBase( - "cert_exp", - "certificate expiry", + return tracker.MakeGlobalTrackerBase( + metrics.Expiry, + "hours before certificate expires", LazyLabels, func(ctx context.Context, _ tracker.MetricDescriptors) tracker.FindingErrorSequence[*finding] { return track(ctx, s) @@ -33,11 +35,18 @@ func track(ctx context.Context, s service.Service) tracker.FindingErrorSequence[ result, err := s.GetCertExpiry(ctx, &v1.GetCertExpiry_Request{ Component: v1.GetCertExpiry_Component(i), }) + if err != nil { + // Ignore particular component errors, as they do not affect + // other components metrics. + logging.LoggerForModule().Errorw("Failed to get certificate expiry", + logging.String("component", component), logging.Err(err)) + continue + } f.component = component if result != nil { f.hoursUntilExpiration = int(time.Until(result.GetExpiry().AsTime()).Hours()) } - if !yield(&f, err) { + if !yield(&f, nil) { return } } diff --git a/central/metrics/custom/expiry/tracker_test.go b/central/metrics/custom/expiry/tracker_test.go new file mode 100644 index 0000000000000..2af2506b6c759 --- /dev/null +++ b/central/metrics/custom/expiry/tracker_test.go @@ -0,0 +1,49 @@ +package expiry + +import ( + "context" + "errors" + "testing" + + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/stackrox/rox/central/credentialexpiry/service" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc" +) + +var errTest = errors.New("test") + +type mockService struct{} + +func (ms *mockService) AuthFuncOverride(context.Context, string) (context.Context, error) { + panic("unimplemented") +} + +func (ms *mockService) GetCertExpiry(_ context.Context, req *v1.GetCertExpiry_Request) (*v1.GetCertExpiry_Response, error) { + if req.GetComponent() == v1.GetCertExpiry_SCANNER_V4 { + return nil, errTest + } + return nil, nil +} + +func (ms *mockService) RegisterServiceHandler(context.Context, *runtime.ServeMux, *grpc.ClientConn) error { + panic("unimplemented") +} + +func (ms *mockService) RegisterServiceServer(*grpc.Server) { + panic("unimplemented") +} + +var _ service.Service = (*mockService)(nil) + +func Test_track(t *testing.T) { + var s mockService + + components := make([]string, 0, len(v1.GetCertExpiry_Component_name)) + for f := range track(context.Background(), &s) { + components = append(components, f.component) + } + assert.ElementsMatch(t, []string{"SCANNER", "CENTRAL_DB", "CENTRAL"}, components, + "should have no UNKNOWN and SCANNER_V4") +} diff --git a/central/metrics/custom/image_vulnerabilities/labels.go b/central/metrics/custom/image_vulnerabilities/labels.go index b0e7a39f19153..5745307489f81 100644 --- a/central/metrics/custom/image_vulnerabilities/labels.go +++ b/central/metrics/custom/image_vulnerabilities/labels.go @@ -14,6 +14,7 @@ var lazyLabels = tracker.LazyLabelGetters[*finding]{ "Deployment": func(f *finding) string { return f.deployment.GetName() }, "Type": func(f *finding) string { return f.deployment.GetType() }, "IsActive": func(f *finding) string { return strconv.FormatBool(!f.deployment.GetInactive()) }, + "IsSuppressed": func(f *finding) string { return strconv.FormatBool(f.vuln.GetSuppressed()) }, "IsPlatformWorkload": isPlatformWorkload, "ImageID": func(f *finding) string { return f.image.GetId() }, @@ -33,7 +34,14 @@ var lazyLabels = tracker.LazyLabelGetters[*finding]{ "EPSSPercentile": func(f *finding) string { return strconv.FormatFloat(float64(f.vuln.GetEpss().GetEpssPercentile()), 'f', 1, 32) }, - "IsFixable": func(f *finding) string { return strconv.FormatBool(f.vuln.GetFixedBy() != "") }, + "Origin": func(f *finding) string { return f.vuln.GetOrigin().String() }, + + "IsFixable": func(f *finding) string { return strconv.FormatBool(f.vuln.GetFixedBy() != "") }, + "FixAvailable": func(f *finding) string { return strconv.FormatInt(f.vuln.GetFixAvailableTimestamp().GetSeconds(), 10) }, + "HasCisaKev": func(f *finding) string { return strconv.FormatBool(f.vuln.GetCisaKev()) }, + "Published": func(f *finding) string { return strconv.FormatInt(f.vuln.GetPublishedOn().GetSeconds(), 10) }, + "FirstImageOccurrence": func(f *finding) string { return strconv.FormatInt(f.vuln.GetFirstImageOccurrence().GetSeconds(), 10) }, + "FirstSystemOccurrence": func(f *finding) string { return strconv.FormatInt(f.vuln.GetFirstSystemOccurrence().GetSeconds(), 10) }, } // finding holds all information for computing any label in this category. diff --git a/central/metrics/custom/image_vulnerabilities/tracker.go b/central/metrics/custom/image_vulnerabilities/tracker.go index e7ebaf6f4bf14..442a3d2a643a5 100644 --- a/central/metrics/custom/image_vulnerabilities/tracker.go +++ b/central/metrics/custom/image_vulnerabilities/tracker.go @@ -4,6 +4,7 @@ import ( "context" deploymentDS "github.com/stackrox/rox/central/deployment/datastore" + "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/central/metrics/custom/tracker" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/search" @@ -11,7 +12,7 @@ import ( func New(ds deploymentDS.DataStore) *tracker.TrackerBase[*finding] { return tracker.MakeTrackerBase( - "image_vuln", + metrics.ImageVulns, "image vulnerabilities", lazyLabels, func(ctx context.Context, md tracker.MetricDescriptors) tracker.FindingErrorSequence[*finding] { diff --git a/central/metrics/custom/node_vulnerabilities/labels.go b/central/metrics/custom/node_vulnerabilities/labels.go index f2e7afd9d5fc2..d25fb35066427 100644 --- a/central/metrics/custom/node_vulnerabilities/labels.go +++ b/central/metrics/custom/node_vulnerabilities/labels.go @@ -19,7 +19,6 @@ var lazyLabels = tracker.LazyLabelGetters[*finding]{ "CVE": func(f *finding) string { return f.vulnerability.GetCveBaseInfo().GetCve() }, "CVSS": func(f *finding) string { return strconv.FormatFloat(float64(f.vulnerability.GetCvss()), 'f', 1, 32) }, "Severity": func(f *finding) string { return f.vulnerability.GetSeverity().String() }, - "IsFixable": func(f *finding) string { return strconv.FormatBool(f.vulnerability.GetFixedBy() != "") }, "IsSnoozed": func(f *finding) string { return strconv.FormatBool(f.vulnerability.GetSnoozed()) }, "EPSSPercentile": func(f *finding) string { return strconv.FormatFloat(float64(f.vulnerability.GetCveBaseInfo().GetEpss().GetEpssPercentile()), 'f', 1, 32) @@ -27,6 +26,12 @@ var lazyLabels = tracker.LazyLabelGetters[*finding]{ "EPSSProbability": func(f *finding) string { return strconv.FormatFloat(float64(f.vulnerability.GetCveBaseInfo().GetEpss().GetEpssProbability()), 'f', 1, 32) }, + + "IsFixable": func(f *finding) string { return strconv.FormatBool(f.vulnerability.GetFixedBy() != "") }, + "HasCisaKev": func(f *finding) string { return strconv.FormatBool(f.vulnerability.GetCveBaseInfo().GetCisaKev()) }, + "Published": func(f *finding) string { + return strconv.FormatInt(f.vulnerability.GetCveBaseInfo().GetPublishedOn().GetSeconds(), 10) + }, } type finding struct { diff --git a/central/metrics/custom/node_vulnerabilities/tracker.go b/central/metrics/custom/node_vulnerabilities/tracker.go index 0de90ecbae728..da169f6bca6af 100644 --- a/central/metrics/custom/node_vulnerabilities/tracker.go +++ b/central/metrics/custom/node_vulnerabilities/tracker.go @@ -3,6 +3,7 @@ package node_vulnerabilities import ( "context" + "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/central/metrics/custom/tracker" nodeDS "github.com/stackrox/rox/central/node/datastore" "github.com/stackrox/rox/generated/storage" @@ -11,7 +12,7 @@ import ( func New(nodes nodeDS.DataStore) *tracker.TrackerBase[*finding] { return tracker.MakeTrackerBase( - "node_vuln", + metrics.NodeVulns, "node CVEs", lazyLabels, func(ctx context.Context, _ tracker.MetricDescriptors) tracker.FindingErrorSequence[*finding] { diff --git a/central/metrics/custom/policies/tracker.go b/central/metrics/custom/policies/tracker.go index bc7a85f31f3a0..3b5604b55e674 100644 --- a/central/metrics/custom/policies/tracker.go +++ b/central/metrics/custom/policies/tracker.go @@ -3,14 +3,15 @@ package policies import ( "context" + "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/central/metrics/custom/tracker" policyDS "github.com/stackrox/rox/central/policy/datastore" "github.com/stackrox/rox/pkg/search" ) func New(ds policyDS.DataStore) *tracker.TrackerBase[*finding] { - return tracker.MakeTrackerBase( - "cfg", + return tracker.MakeGlobalTrackerBase( + metrics.Configuration, "policies", LazyLabels, func(ctx context.Context, _ tracker.MetricDescriptors) tracker.FindingErrorSequence[*finding] { diff --git a/central/metrics/custom/policy_violations/tracker.go b/central/metrics/custom/policy_violations/tracker.go index 6c9807b9cff48..8141a59774a73 100644 --- a/central/metrics/custom/policy_violations/tracker.go +++ b/central/metrics/custom/policy_violations/tracker.go @@ -4,6 +4,7 @@ import ( "context" alertDS "github.com/stackrox/rox/central/alert/datastore" + "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/central/metrics/custom/tracker" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/search" @@ -11,7 +12,7 @@ import ( func New(ds alertDS.DataStore) *tracker.TrackerBase[*finding] { return tracker.MakeTrackerBase( - "policy_violation", + metrics.Violations, "policy violations", lazyLabels, func(ctx context.Context, _ tracker.MetricDescriptors) tracker.FindingErrorSequence[*finding] { diff --git a/central/metrics/custom/refresh/interface.go b/central/metrics/custom/refresh/interface.go new file mode 100644 index 0000000000000..0172fac1a0ba7 --- /dev/null +++ b/central/metrics/custom/refresh/interface.go @@ -0,0 +1,31 @@ +// This package resolves import cycle between datastores and the runner. +package refresh + +import ( + "github.com/stackrox/rox/pkg/sync" +) + +type Refresher interface { + RefreshTracker(prefix string) +} + +var ( + singleton Refresher + mu sync.RWMutex +) + +// SetSingleton initializes the refresher singleton object. +func SetSingleton(r Refresher) { + mu.Lock() + defer mu.Unlock() + singleton = r +} + +// RefreshTracker refreshes the tracker, identified by the metric prefix. +func RefreshTracker(prefix string) { + mu.RLock() + defer mu.RUnlock() + if singleton != nil { + singleton.RefreshTracker(prefix) + } +} diff --git a/central/metrics/custom/runner.go b/central/metrics/custom/runner.go index a0d6f77831c2c..2806ee9b6f3a6 100644 --- a/central/metrics/custom/runner.go +++ b/central/metrics/custom/runner.go @@ -4,12 +4,16 @@ import ( "context" "net/http" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + adminEventDS "github.com/stackrox/rox/central/administration/events/datastore" alertDS "github.com/stackrox/rox/central/alert/datastore" clusterDS "github.com/stackrox/rox/central/cluster/datastore" configDS "github.com/stackrox/rox/central/config/datastore" expiryS "github.com/stackrox/rox/central/credentialexpiry/service" deploymentDS "github.com/stackrox/rox/central/deployment/datastore" "github.com/stackrox/rox/central/metrics" + "github.com/stackrox/rox/central/metrics/custom/administrative_events" "github.com/stackrox/rox/central/metrics/custom/clusters" "github.com/stackrox/rox/central/metrics/custom/expiry" "github.com/stackrox/rox/central/metrics/custom/image_vulnerabilities" @@ -48,6 +52,7 @@ type runnerDatastores struct { clusters clusterDS.DataStore policies policyDS.DataStore expiry expiryS.Service + adminEvents adminEventDS.DataStore } func withHardcodedConfiguration(period uint32, descriptors map[string][]string) func(*storage.PrometheusMetrics) *storage.PrometheusMetrics_Group { @@ -95,10 +100,24 @@ func makeRunner(ds *runnerDatastores) trackerRunner { // rox_central_cert_exp_hours "hours": expiry.LazyLabels.GetLabels(), }), + }, { + administrative_events.New(ds.adminEvents), + (*storage.PrometheusMetrics).GetAdministrativeEvents, }, } } +// RefreshTracker implements the refresh.Refresher interface. +func (tr trackerRunner) RefreshTracker(prefix string) { + for _, t := range tr { + if t.Tracker.GetPrefix() == prefix { + go t.Refresh() + // do not return, as there could be several trackers with the same + // prefix. E.g., cfg. + } + } +} + func (tr trackerRunner) initialize(cds configDS.DataStore) { ctx := sac.WithAllAccess(context.Background()) systemPrivateConfig, err := cds.GetPrivateConfig(ctx) @@ -130,7 +149,8 @@ func (tr trackerRunner) ValidateConfiguration(cfg *storage.PrometheusMetrics) (R return runnerConfig, nil } -// Reconfigure applies the provided configuration. +// Reconfigure applies the provided configuration and schedules gatherers to run +// after next scrape request. // Non-nil runner will panic on nil cfg. Don't pass nil. func (tr trackerRunner) Reconfigure(cfg RunnerConfiguration) { if tr == nil { @@ -162,14 +182,28 @@ func (tr trackerRunner) ServeHTTP(w http.ResponseWriter, req *http.Request) { go tracker.Gather(newCtx) } } - registry, err := metrics.GetCustomRegistry(userID) + + userRegistry, err := metrics.GetCustomRegistry(userID) + if err != nil { + httputil.WriteError(w, err) + return + } + + globalRegistry, err := metrics.GetGlobalRegistry() if err != nil { httputil.WriteError(w, err) return } - registry.Lock() - defer registry.Unlock() - registry.ServeHTTP(w, req) + + userRegistry.Lock() + defer userRegistry.Unlock() + globalRegistry.Lock() + defer globalRegistry.Unlock() + + promhttp.HandlerFor( + prometheus.Gatherers{userRegistry, globalRegistry}, + promhttp.HandlerOpts{}).ServeHTTP(w, req) + go phonehome() } diff --git a/central/metrics/custom/singleton.go b/central/metrics/custom/singleton.go index 40cb859c1931e..a1a86a89ffb13 100644 --- a/central/metrics/custom/singleton.go +++ b/central/metrics/custom/singleton.go @@ -3,11 +3,13 @@ package custom import ( "net/http" + adminEventDS "github.com/stackrox/rox/central/administration/events/datastore" alertDS "github.com/stackrox/rox/central/alert/datastore" clusterDS "github.com/stackrox/rox/central/cluster/datastore" configDS "github.com/stackrox/rox/central/config/datastore" expiryS "github.com/stackrox/rox/central/credentialexpiry/service" deploymentDS "github.com/stackrox/rox/central/deployment/datastore" + "github.com/stackrox/rox/central/metrics/custom/refresh" nodeDS "github.com/stackrox/rox/central/node/datastore" policyDS "github.com/stackrox/rox/central/policy/datastore" "github.com/stackrox/rox/generated/storage" @@ -26,6 +28,7 @@ type Runner interface { http.Handler ValidateConfiguration(*storage.PrometheusMetrics) (RunnerConfiguration, error) Reconfigure(RunnerConfiguration) + RefreshTracker(prefix string) } // Singleton returns a runner, or nil if there were errors during @@ -39,7 +42,11 @@ func Singleton() Runner { clusterDS.Singleton(), policyDS.Singleton(), expiryS.Singleton(), + adminEventDS.Singleton(), }) + // Datastores may use the singleton to refresh the trackers. + // This abstraction resolves the import loop. + refresh.SetSingleton(runner) go runner.initialize(configDS.Singleton()) }) return runner diff --git a/central/metrics/custom/tracker/configuration.go b/central/metrics/custom/tracker/configuration.go index f411d8c91f6c1..0900f7e60ceca 100644 --- a/central/metrics/custom/tracker/configuration.go +++ b/central/metrics/custom/tracker/configuration.go @@ -41,3 +41,7 @@ type Configuration struct { toDelete []MetricName period time.Duration } + +func (cfg *Configuration) isEnabled() bool { + return cfg != nil && len(cfg.metrics) > 0 && cfg.period > 0 +} diff --git a/central/metrics/custom/tracker/tracker_base.go b/central/metrics/custom/tracker/tracker_base.go index f55a353ed3b04..ac38d7343df74 100644 --- a/central/metrics/custom/tracker/tracker_base.go +++ b/central/metrics/custom/tracker/tracker_base.go @@ -20,9 +20,11 @@ import ( "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/sync" "github.com/stackrox/rox/pkg/telemetry/phonehome/telemeter" + "github.com/stackrox/rox/pkg/utils" ) const inactiveGathererTTL = 2 * 24 * time.Hour +const globalScopeID = "" var ( log = logging.CreateLogger(logging.ModuleForName("central_metrics"), 1) @@ -55,6 +57,11 @@ type Tracker interface { NewConfiguration(*storage.PrometheusMetrics_Group) (*Configuration, error) // Reconfigure the tracker with the provided tracker configuration. Reconfigure(*Configuration) + // GetPrefix returns the metric prefix. + GetPrefix() string + // Refresh the data on the next scrape request without waiting for the next + // gathering period. + Refresh() } // FindingErrorSequence is a sequence of pairs of findings and errors. @@ -67,19 +74,49 @@ type gatherer[F Finding] struct { http.Handler lastGather time.Time running atomic.Bool + refreshing atomic.Bool registry metrics.CustomRegistry aggregator *aggregator[F] config *Configuration } +// updateMetrics aggregates the fetched findings and updates the gauges. +func (g *gatherer[F]) updateMetrics(generator iter.Seq2[F, error]) error { + g.aggregator.reset() + for finding, err := range generator { + if err != nil { + return err + } + g.aggregator.count(finding) + } + g.registry.Lock() + defer g.registry.Unlock() + for metric, records := range g.aggregator.result { + g.registry.Reset(string(metric)) + for _, rec := range records { + g.registry.SetTotal(string(metric), rec.labels, rec.total) + } + } + return nil +} + +func (g *gatherer[F]) trySetRunning() bool { + return g.running.CompareAndSwap(false, true) +} + // TrackerBase implements a generic finding tracker. // Configured with a finding generator and other arguments, it runs a goroutine // that periodically aggregates gathered values and updates the gauge values. +// +// A tracker can be scoped, in which case it creates a separate registry for +// each scrape user ID, or global, where it reuses the same global registry for +// all users. type TrackerBase[F Finding] struct { metricPrefix string description string getters LazyLabelGetters[F] generator FindingGenerator[F] + scoped bool // metricsConfig can be changed with an API call. config *Configuration @@ -91,20 +128,46 @@ type TrackerBase[F Finding] struct { registryFactory func(userID string) (metrics.CustomRegistry, error) // for mocking in tests. } -// MakeTrackerBase initializes a tracker without any period or metrics +// MakeTrackerBase initializes a scoped tracker without any period or metrics // configuration. Call Reconfigure to configure the period and the metrics. func MakeTrackerBase[F Finding](metricPrefix, description string, getters LazyLabelGetters[F], generator FindingGenerator[F], ) *TrackerBase[F] { + return makeTrackerBase(metricPrefix, description, true, getters, generator) +} + +// MakeGlobalTrackerBase creates a global, i.e. non-scoped tracker. +func MakeGlobalTrackerBase[F Finding](metricPrefix, description string, + getters LazyLabelGetters[F], generator FindingGenerator[F], +) *TrackerBase[F] { + return makeTrackerBase(metricPrefix, description, false, getters, generator) +} + +func globalRegistryFactory(string) (metrics.CustomRegistry, error) { + return metrics.GetGlobalRegistry() +} + +func makeTrackerBase[F Finding](metricPrefix, description string, scoped bool, + getters LazyLabelGetters[F], generator FindingGenerator[F], +) *TrackerBase[F] { + registryFactory := globalRegistryFactory + if scoped { + registryFactory = metrics.GetCustomRegistry + } return &TrackerBase[F]{ metricPrefix: metricPrefix, description: description, getters: getters, generator: generator, - registryFactory: metrics.GetCustomRegistry, + scoped: scoped, + registryFactory: registryFactory, } } +func (tracker *TrackerBase[F]) GetPrefix() string { + return tracker.metricPrefix +} + // NewConfiguration does not apply the configuration. func (tracker *TrackerBase[F]) NewConfiguration(cfg *storage.PrometheusMetrics_Group) (*Configuration, error) { current := tracker.getConfiguration() @@ -139,7 +202,7 @@ func (tracker *TrackerBase[F]) Reconfigure(cfg *Configuration) { } previous := tracker.setConfiguration(cfg) if previous != nil { - if cfg.period == 0 { + if !cfg.isEnabled() { log.Debugf("Metrics collection has been disabled for %s", tracker.description) tracker.unregisterMetrics(slices.Collect(maps.Keys(previous.metrics))) return @@ -147,8 +210,17 @@ func (tracker *TrackerBase[F]) Reconfigure(cfg *Configuration) { tracker.unregisterMetrics(cfg.toDelete) } tracker.registerMetrics(cfg, cfg.toAdd) - // Note: aggregators are recreated lazily in getGatherer() when config - // changes, to avoid race conditions with running gatherers. + // Aggregators are recreated and gather is forced lazily in + // getGatherer() when config changes, to avoid race conditions with + // running gatherers. +} + +// Refresh marks all gatherers for re-gathering on the next scrape request. +func (tracker *TrackerBase[F]) Refresh() { + tracker.gatherers.Range(func(_, gv any) bool { + gv.(*gatherer[F]).refreshing.Store(true) + return true + }) } func labelsAsStrings(labels []Label) []string { @@ -235,6 +307,8 @@ func (tracker *TrackerBase[F]) getConfiguration() *Configuration { return tracker.config } +// setConfiguration updates the tracker configuration and returns the previous +// one. func (tracker *TrackerBase[F]) setConfiguration(config *Configuration) *Configuration { tracker.metricsConfigMux.Lock() defer tracker.metricsConfigMux.Unlock() @@ -243,43 +317,23 @@ func (tracker *TrackerBase[F]) setConfiguration(config *Configuration) *Configur return previous } -// track aggregates the fetched findings and updates the gauges. -func (tracker *TrackerBase[F]) track(ctx context.Context, gatherer *gatherer[F], cfg *Configuration) error { - if len(cfg.metrics) == 0 { - return nil - } - aggregator := gatherer.aggregator - registry := gatherer.registry - aggregator.reset() - for finding, err := range tracker.generator(ctx, cfg.metrics) { - if err != nil { - return err - } - aggregator.count(finding) - } - registry.Lock() - defer registry.Unlock() - for metric, records := range aggregator.result { - registry.Reset(string(metric)) - for _, rec := range records { - registry.SetTotal(string(metric), rec.labels, rec.total) - } - } - return nil -} - // Gather the data not more often then maxAge. func (tracker *TrackerBase[F]) Gather(ctx context.Context) { - id, err := authn.IdentityFromContext(ctx) - if err != nil { - return - } cfg := tracker.getConfiguration() - if cfg == nil { + if !cfg.isEnabled() { return } + id := globalScopeID + if tracker.scoped { + userID, err := authn.IdentityFromContext(ctx) + if err != nil { + utils.Should(err) + return + } + id = userID.UID() + } // Pass the cfg so that the same configuration is used there and here. - gatherer := tracker.getGatherer(id.UID(), cfg) + gatherer := tracker.getGatherer(id, cfg) // getGatherer() returns nil if the gatherer is still running. if gatherer == nil { return @@ -287,11 +341,11 @@ func (tracker *TrackerBase[F]) Gather(ctx context.Context) { defer tracker.cleanupInactiveGatherers() defer gatherer.running.Store(false) - if cfg.period == 0 || time.Since(gatherer.lastGather) < cfg.period { + if !gatherer.refreshing.Swap(false) && time.Since(gatherer.lastGather) < cfg.period { return } begin := time.Now() - if err := tracker.track(ctx, gatherer, cfg); err != nil { + if err := gatherer.updateMetrics(tracker.generator(ctx, cfg.metrics)); err != nil { log.Errorf("Failed to gather %s metrics: %v", tracker.description, err) } end := time.Now() @@ -355,25 +409,20 @@ func (tracker *TrackerBase[F]) getGatherer(userID string, cfg *Configuration) *g if !gr.trySetRunning() { return nil } - // Recreate aggregator if config has changed since last run. + // Recreate aggregator and force immediate gather if config has changed. if gr.config != cfg { gr.aggregator = makeAggregator(cfg.metrics, cfg.includeFilters, cfg.excludeFilters, tracker.getters) gr.config = cfg + gr.refreshing.Store(true) } } return gr } -func (g *gatherer[F]) trySetRunning() bool { - return g.running.CompareAndSwap(false, true) -} - // cleanupInactiveGatherers frees the registries for the userIDs, that haven't // shown up for inactiveGathererTTL. func (tracker *TrackerBase[F]) cleanupInactiveGatherers() { - tracker.cleanupWG.Add(1) - go func() { - defer tracker.cleanupWG.Done() + tracker.cleanupWG.Go(func() { tracker.gatherers.Range(func(userID, gv any) bool { g := gv.(*gatherer[F]) // Try to make it running to not interfere with the normal gathering @@ -393,5 +442,5 @@ func (tracker *TrackerBase[F]) cleanupInactiveGatherers() { } return true }) - }() + }) } diff --git a/central/metrics/custom/tracker/tracker_base_test.go b/central/metrics/custom/tracker/tracker_base_test.go index 62282eec5a218..f8967beae17e2 100644 --- a/central/metrics/custom/tracker/tracker_base_test.go +++ b/central/metrics/custom/tracker/tracker_base_test.go @@ -3,6 +3,8 @@ package tracker import ( "context" "maps" + "net/http" + "net/http/httptest" "regexp" "slices" "strings" @@ -13,6 +15,7 @@ import ( "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/central/metrics/mocks" "github.com/stackrox/rox/pkg/auth/authproviders" + "github.com/stackrox/rox/pkg/defaults/accesscontrol" "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/grpc/authn" "github.com/stackrox/rox/pkg/grpc/authn/basic" @@ -44,6 +47,14 @@ func TestMakeTrackerBase(t *testing.T) { tracker := MakeTrackerBase("test", "Test", testLabelGetters, nilGatherFunc) assert.NotNil(t, tracker) assert.Nil(t, tracker.getConfiguration()) + assert.True(t, tracker.scoped) +} + +func Test_makeTrackerBase(t *testing.T) { + tracker := makeTrackerBase("test", "Test", false, testLabelGetters, nilGatherFunc) + assert.NotNil(t, tracker) + assert.Nil(t, tracker.getConfiguration()) + assert.False(t, tracker.scoped) } func TestTrackerBase_Reconfigure(t *testing.T) { @@ -137,24 +148,9 @@ func TestTrackerBase_Reconfigure(t *testing.T) { assert.Empty(t, registered) assert.ElementsMatch(t, cfg1.toDelete, unregistered) - // Less than period since last Gather, gathering ignored: - tracker.Gather(ctx) - tracker.cleanupWG.Wait() - assert.Empty(t, trackedMetricNames) - - { // Reset lastGather - identity, _ := authn.IdentityFromContext(ctx) - gRaw, ok := tracker.gatherers.Load(identity.UID()) - require.True(t, ok) - g := gRaw.(*gatherer[testFinding]) - // Make it temporarily running to avoid data race on lastGather. - require.Eventually(t, g.trySetRunning, 5*time.Second, 10*time.Millisecond) - g.lastGather = time.Time{} - g.running.Store(false) - } + // Gatherers will run after Reconfigure shifted the last gathering: tracker.Gather(ctx) tracker.cleanupWG.Wait() - assert.ElementsMatch(t, slices.Compact(trackedMetricNames), metricNames[1:]) // Stop and unregister everything: @@ -197,7 +193,7 @@ func TestTrackerBase_Track(t *testing.T) { registry: rf, aggregator: makeAggregator(tracker.config.metrics, tracker.config.includeFilters, tracker.config.excludeFilters, tracker.getters), } - assert.NoError(t, tracker.track(context.Background(), testGatherer, tracker.config)) + assert.NoError(t, testGatherer.updateMetrics(tracker.generator(context.Background(), tracker.config.metrics))) if assert.Len(t, result, 2) && assert.Contains(t, result, "test_TestTrackerBase_Track_metric1") && @@ -261,7 +257,7 @@ func TestTrackerBase_error(t *testing.T) { registry: rf, aggregator: makeAggregator(tracker.config.metrics, tracker.config.includeFilters, tracker.config.excludeFilters, tracker.getters), } - assert.ErrorIs(t, tracker.track(context.Background(), testGatherer, tracker.config), + assert.ErrorIs(t, testGatherer.updateMetrics(tracker.generator(context.Background(), tracker.config.metrics)), errox.InvariantViolation) } @@ -588,3 +584,192 @@ func Test_formatMetricsHelp(t *testing.T) { period: time.Hour, }, "metric1")) } + +func makeScopedGatherFunc(ctx context.Context, _ MetricDescriptors) FindingErrorSequence[testFinding] { + return func(yield func(testFinding, error) bool) { + identity, err := authn.IdentityFromContext(ctx) + if err != nil { + return + } + + username := identity.UID() + var finding testFinding + for idx := range testData { + cluster := testData[idx]["Cluster"] + + shouldYield := false + switch username { + case accesscontrol.Admin: + shouldYield = true + case "No Access": // see basic.ContextWithNoAccessIdentity code. + shouldYield = cluster == "cluster 1" + case accesscontrol.None: + shouldYield = false + } + + if shouldYield && !yield(finding, nil) { + return + } + finding++ + } + } +} + +func makeTestCtxIdentity(t *testing.T, provider authproviders.Provider, accessCtxFunc func(*testing.T, authproviders.Provider) context.Context) (context.Context, authn.Identity) { + ctx := accessCtxFunc(t, provider) + id, _ := authn.IdentityFromContext(ctx) + return ctx, id +} + +func Test_scope(t *testing.T) { + t.Run("scoped access", func(t *testing.T) { + tracker := MakeTrackerBase("test", "Test", + testLabelGetters, + makeScopedGatherFunc) + + md := makeTestMetricDescriptors(t) + tracker.Reconfigure(&Configuration{ + metrics: md, + toAdd: slices.Collect(maps.Keys(md)), + period: time.Hour, + }) + + provider, _ := authproviders.NewProvider( + authproviders.WithEnabled(true), + authproviders.WithID(uuid.NewV4().String()), + authproviders.WithName("test"), + ) + + adminCtx, adminIdentity := makeTestCtxIdentity(t, provider, + basic.ContextWithAdminIdentity) + cluster1Ctx, cluster1Identity := makeTestCtxIdentity(t, provider, + basic.ContextWithNoAccessIdentity) + noAccessCtx, noAccessIdentity := makeTestCtxIdentity(t, provider, + basic.ContextWithNoneIdentity) + + tracker.Gather(adminCtx) + tracker.Gather(cluster1Ctx) + tracker.Gather(noAccessCtx) + tracker.cleanupWG.Wait() + + adminGatherer, _ := tracker.gatherers.Load(adminIdentity.UID()) + cluster1Gatherer, _ := tracker.gatherers.Load(cluster1Identity.UID()) + noAccessGatherer, _ := tracker.gatherers.Load(noAccessIdentity.UID()) + + adminRegistry := adminGatherer.(*gatherer[testFinding]).registry + cluster1Registry := cluster1Gatherer.(*gatherer[testFinding]).registry + noAccessRegistry := noAccessGatherer.(*gatherer[testFinding]).registry + + adminMetrics := readMetrics(adminRegistry) + cluster1Metrics := readMetrics(cluster1Registry) + noAccessMetrics := readMetrics(noAccessRegistry) + + const expectedAdminMetrics = `# HELP rox_central_test_Test_scope_scoped_access_metric1 The total number of Test aggregated by Cluster, Severity, and gathered every 1h0m0s +# TYPE rox_central_test_Test_scope_scoped_access_metric1 gauge +rox_central_test_Test_scope_scoped_access_metric1{Cluster="cluster 1",Severity="CRITICAL"} 2 +rox_central_test_Test_scope_scoped_access_metric1{Cluster="cluster 2",Severity="HIGH"} 1 +rox_central_test_Test_scope_scoped_access_metric1{Cluster="cluster 3",Severity="LOW"} 1 +rox_central_test_Test_scope_scoped_access_metric1{Cluster="cluster 5",Severity="LOW"} 1 +# HELP rox_central_test_Test_scope_scoped_access_metric2 The total number of Test aggregated by Namespace, and gathered every 1h0m0s +# TYPE rox_central_test_Test_scope_scoped_access_metric2 gauge +rox_central_test_Test_scope_scoped_access_metric2{Namespace="ns 1"} 1 +rox_central_test_Test_scope_scoped_access_metric2{Namespace="ns 2"} 1 +rox_central_test_Test_scope_scoped_access_metric2{Namespace="ns 3"} 3 +` + + const expectedCluster1Metrics = `# HELP rox_central_test_Test_scope_scoped_access_metric1 The total number of Test aggregated by Cluster, Severity, and gathered every 1h0m0s +# TYPE rox_central_test_Test_scope_scoped_access_metric1 gauge +rox_central_test_Test_scope_scoped_access_metric1{Cluster="cluster 1",Severity="CRITICAL"} 2 +# HELP rox_central_test_Test_scope_scoped_access_metric2 The total number of Test aggregated by Namespace, and gathered every 1h0m0s +# TYPE rox_central_test_Test_scope_scoped_access_metric2 gauge +rox_central_test_Test_scope_scoped_access_metric2{Namespace="ns 1"} 1 +rox_central_test_Test_scope_scoped_access_metric2{Namespace="ns 3"} 1 +` + + assert.Equal(t, expectedAdminMetrics, adminMetrics) + assert.Equal(t, expectedCluster1Metrics, cluster1Metrics) + assert.Empty(t, noAccessMetrics) + + t.Cleanup(func() { + metrics.DeleteCustomRegistry(adminIdentity.UID()) + metrics.DeleteCustomRegistry(cluster1Identity.UID()) + metrics.DeleteCustomRegistry(noAccessIdentity.UID()) + }) + }) + + t.Run("global access", func(t *testing.T) { + tracker := MakeGlobalTrackerBase("test", "Test", + testLabelGetters, + makeTestGatherFunc(testData)) + + md := makeTestMetricDescriptors(t) + tracker.Reconfigure(&Configuration{ + metrics: md, + toAdd: slices.Collect(maps.Keys(md)), + period: time.Hour, + }) + + provider, _ := authproviders.NewProvider( + authproviders.WithEnabled(true), + authproviders.WithID(uuid.NewV4().String()), + authproviders.WithName("test"), + ) + + adminCtx, _ := makeTestCtxIdentity(t, provider, + basic.ContextWithAdminIdentity) + cluster1Ctx, _ := makeTestCtxIdentity(t, provider, + basic.ContextWithNoAccessIdentity) + noAccessCtx, _ := makeTestCtxIdentity(t, provider, + basic.ContextWithNoneIdentity) + + tracker.Gather(adminCtx) + tracker.Gather(cluster1Ctx) + tracker.Gather(noAccessCtx) + tracker.cleanupWG.Wait() + + globalRegistry, err := metrics.GetGlobalRegistry() + require.NoError(t, err) + + const expectedMetrics = `# HELP rox_central_test_Test_scope_global_access_metric1 The total number of Test aggregated by Cluster, Severity, and gathered every 1h0m0s +# TYPE rox_central_test_Test_scope_global_access_metric1 gauge +rox_central_test_Test_scope_global_access_metric1{Cluster="cluster 1",Severity="CRITICAL"} 2 +rox_central_test_Test_scope_global_access_metric1{Cluster="cluster 2",Severity="HIGH"} 1 +rox_central_test_Test_scope_global_access_metric1{Cluster="cluster 3",Severity="LOW"} 1 +rox_central_test_Test_scope_global_access_metric1{Cluster="cluster 5",Severity="LOW"} 1 +# HELP rox_central_test_Test_scope_global_access_metric2 The total number of Test aggregated by Namespace, and gathered every 1h0m0s +# TYPE rox_central_test_Test_scope_global_access_metric2 gauge +rox_central_test_Test_scope_global_access_metric2{Namespace="ns 1"} 1 +rox_central_test_Test_scope_global_access_metric2{Namespace="ns 2"} 1 +rox_central_test_Test_scope_global_access_metric2{Namespace="ns 3"} 3 +` + + // All users should see the same global metrics. + globalMetrics := readMetrics(globalRegistry) + assert.Equal(t, expectedMetrics, globalMetrics) + }) +} + +func readMetrics(registry metrics.CustomRegistry) string { + rec := httptest.NewRecorder() + registry.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + return rec.Body.String() +} + +func TestTrackerBase_Refresh(t *testing.T) { + tracker := MakeTrackerBase("test", "telemetry test", + testLabelGetters, + makeTestGatherFunc(testData)) + + md := makeTestMetricDescriptors(t) + cfg := &Configuration{ + metrics: md, + toAdd: slices.Collect(maps.Keys(md)), + period: time.Hour, + } + tracker.setConfiguration(cfg) + g := tracker.getGatherer("test-id", cfg) + g.running.Store(false) + assert.False(t, g.refreshing.Load()) + tracker.Refresh() + assert.True(t, g.refreshing.Load()) +} diff --git a/central/metrics/custom_registry.go b/central/metrics/custom_registry.go index 5cc6a05d466b2..ba50b57d38a7b 100644 --- a/central/metrics/custom_registry.go +++ b/central/metrics/custom_registry.go @@ -45,12 +45,13 @@ type customRegistry struct { var ( userRegistries map[string]*customRegistry = make(map[string]*customRegistry) + globalRegistry *customRegistry registriesMux sync.Mutex ErrTooMany = errox.ResourceExhausted.New("too many custom registries") ) // GetCustomRegistry is a CustomRegistry factory that returns the existing or -// a new registry for the user. +// a new registry for the user. This is used for scoped (per-user) metrics. func GetCustomRegistry(userID string) (CustomRegistry, error) { registriesMux.Lock() defer registriesMux.Unlock() @@ -68,6 +69,21 @@ func GetCustomRegistry(userID string) (CustomRegistry, error) { return registry, nil } +// GetGlobalRegistry returns the shared global registry for non-scoped metrics. +// This registry is shared across all users and contains metrics that are not +// user-specific (e.g., API request counters). +func GetGlobalRegistry() (CustomRegistry, error) { + registriesMux.Lock() + defer registriesMux.Unlock() + if globalRegistry == nil { + globalRegistry = &customRegistry{ + Registry: prometheus.NewRegistry(), + } + globalRegistry.Handler = promhttp.HandlerFor(globalRegistry, promhttp.HandlerOpts{}) + } + return globalRegistry, nil +} + // DeleteCustomRegistry unregisters all metrics and deletes a registry for the // given userID. func DeleteCustomRegistry(userID string) { diff --git a/central/metrics/custom_registry_test.go b/central/metrics/custom_registry_test.go index a9f1dd89be9c2..a9c3ca49421ee 100644 --- a/central/metrics/custom_registry_test.go +++ b/central/metrics/custom_registry_test.go @@ -50,6 +50,16 @@ func TestMakeCustomRegistry(t *testing.T) { assert.True(t, cr2.UnregisterMetric("test")) } +func TestGetGlobalRegistry(t *testing.T) { + n := GetCustomRegistriesCount() + r0, err0 := GetGlobalRegistry() + assert.NoError(t, err0) + r1, err1 := GetGlobalRegistry() + assert.NoError(t, err1) + assert.Equal(t, r0, r1) + assert.Equal(t, n, GetCustomRegistriesCount()) +} + func TestCustomRegistry_Reset(t *testing.T) { cr, err := GetCustomRegistry("user1") assert.NoError(t, err) diff --git a/central/metrics/init.go b/central/metrics/init.go index 7f2f8e9b79f4b..d62706ebf4836 100644 --- a/central/metrics/init.go +++ b/central/metrics/init.go @@ -17,6 +17,7 @@ func init() { totalNetworkFlowsReceivedCounter, totalNetworkEndpointsReceivedCounter, totalExternalPoliciesGauge, + currentExternalPolicies, sensorEventDurationHistogramVec, riskProcessingHistogramVec, datastoreFunctionDurationHistogramVec, diff --git a/central/metrics/prefixes.go b/central/metrics/prefixes.go new file mode 100644 index 0000000000000..7b839544870b4 --- /dev/null +++ b/central/metrics/prefixes.go @@ -0,0 +1,12 @@ +package metrics + +const ( + // These are prefixes for custom metrics. + AdminEvents = "admin_event" + Configuration = "cfg" + Expiry = "cert_exp" + Health = "health" + ImageVulns = "image_vuln" + NodeVulns = "node_vuln" + Violations = "policy_violation" +) diff --git a/central/namespace/datastore/datastore.go b/central/namespace/datastore/datastore.go index 5cdd33be84b83..3fa1df81a8b8d 100644 --- a/central/namespace/datastore/datastore.go +++ b/central/namespace/datastore/datastore.go @@ -4,10 +4,12 @@ import ( "context" "fmt" "strings" + "time" "github.com/pkg/errors" "github.com/stackrox/rox/central/convert/storagetoeffectiveaccessscope" deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" + "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/central/namespace/datastore/internal/store" "github.com/stackrox/rox/central/ranking" v1 "github.com/stackrox/rox/generated/api/v1" @@ -19,6 +21,7 @@ import ( "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/paginated" "github.com/stackrox/rox/pkg/search/sorted" + "github.com/stackrox/rox/pkg/utils" ) //go:generate mockgen-wrapper @@ -29,6 +32,7 @@ type DataStore interface { GetAllNamespaces(ctx context.Context) ([]*storage.NamespaceMetadata, error) GetNamespacesForSAC() ([]effectiveaccessscope.Namespace, error) GetManyNamespaces(ctx context.Context, id []string) ([]*storage.NamespaceMetadata, error) + GetNamespaceLabels(ctx context.Context, clusterID string, namespaceName string) (map[string]string, error) AddNamespace(context.Context, *storage.NamespaceMetadata) error UpdateNamespace(context.Context, *storage.NamespaceMetadata) error @@ -271,6 +275,29 @@ func (b *datastoreImpl) updateNamespacePriority(nss ...*storage.NamespaceMetadat } } +// GetNamespaceLabels returns the labels for the specified namespace. +// TODO: Evaluate metric to see if we need to optimize this function at all +func (b *datastoreImpl) GetNamespaceLabels(ctx context.Context, clusterID string, namespaceName string) (map[string]string, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "Namespace", "GetNamespaceLabels") + + q := search.NewQueryBuilder(). + AddExactMatches(search.Namespace, namespaceName). + AddExactMatches(search.ClusterID, clusterID). + ProtoQuery() + + namespaces, err := b.SearchNamespaces(ctx, q) + if err != nil { + return nil, err + } + if len(namespaces) == 0 { + return nil, nil + } + if len(namespaces) > 1 { + utils.Should(errors.Errorf("found %d namespaces for cluster %q and name %q", len(namespaces), clusterID, namespaceName)) + } + return namespaces[0].GetLabels(), nil +} + // NamespaceSearchResultConverter implements search.SearchResultConverter for namespace search results. // This enables single-pass query construction for SearchResult protos. type NamespaceSearchResultConverter struct{} diff --git a/central/namespace/datastore/datastore_sac_test.go b/central/namespace/datastore/datastore_sac_test.go index 2732e16269d5e..1a47ed23c1a39 100644 --- a/central/namespace/datastore/datastore_sac_test.go +++ b/central/namespace/datastore/datastore_sac_test.go @@ -123,6 +123,38 @@ func (s *namespaceDatastoreSACSuite) TestGetNamespace() { } } +func (s *namespaceDatastoreSACSuite) TestGetNamespaceLabels() { + // Remove data injected in SetupTest to avoid collisions with (cluster, namespace) lookup + for _, id := range s.testNamespaceIDs { + s.deleteNamespace(id) + } + s.testNamespaceIDs = s.testNamespaceIDs[:0] + + testLabels := map[string]string{"env": "test", "team": "backend"} + namespace := fixtures.GetScopedNamespace(uuid.NewV4().String(), testconsts.Cluster2, + testconsts.NamespaceB) + namespace.Labels = testLabels + err := s.datastore.AddNamespace(s.testContexts[testutils.UnrestrictedReadWriteCtx], namespace) + s.Require().NoError(err) + s.testNamespaceIDs = append(s.testNamespaceIDs, namespace.GetId()) + + cases := testutils.GenericNamespaceSACGetTestCases(s.T()) + + for name, c := range cases { + s.Run(name, func() { + ctx := s.testContexts[c.ScopeKey] + labels, err := s.datastore.GetNamespaceLabels(ctx, namespace.GetClusterId(), namespace.GetName()) + s.Require().NoError(err) + if c.ExpectedFound { + s.Require().NotNil(labels) + s.Equal(testLabels, labels) + } else { + s.Nil(labels) + } + }) + } +} + func (s *namespaceDatastoreSACSuite) TestGetNamespaces() { // Remove data injected in SetupTest. for _, id := range s.testNamespaceIDs { diff --git a/central/namespace/datastore/datastore_test.go b/central/namespace/datastore/datastore_test.go index 45d5ad299e996..b8bb599afdc03 100644 --- a/central/namespace/datastore/datastore_test.go +++ b/central/namespace/datastore/datastore_test.go @@ -5,7 +5,6 @@ package datastore import ( "context" "fmt" - "sync" "testing" "time" @@ -18,6 +17,7 @@ import ( "github.com/stackrox/rox/pkg/sac/testconsts" "github.com/stackrox/rox/pkg/sac/testutils" searchPkg "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/sync" "github.com/stackrox/rox/pkg/uuid" "github.com/stretchr/testify/suite" ) @@ -405,7 +405,7 @@ func (s *namespaceDatastoreComprehensiveSuite) TestPerformanceWithLargeDataset() namespaces := make([]*storage.NamespaceMetadata, 0, numNamespaces) // Create many namespaces - for i := 0; i < numNamespaces; i++ { + for i := range numNamespaces { ns := fixtures.GetScopedNamespace( uuid.NewV4().String(), testconsts.Cluster1, @@ -511,12 +511,12 @@ func (s *namespaceDatastoreComprehensiveSuite) TestConcurrentOperations() { var wg sync.WaitGroup errors := make(chan error, numGoroutines*numOperationsPerGoroutine) - for i := 0; i < numGoroutines; i++ { + for i := range numGoroutines { wg.Add(1) go func(goroutineID int) { defer wg.Done() - for j := 0; j < numOperationsPerGoroutine; j++ { + for j := range numOperationsPerGoroutine { ns := fixtures.GetScopedNamespace( uuid.NewV4().String(), testconsts.Cluster1, @@ -612,3 +612,70 @@ func (s *namespaceDatastoreComprehensiveSuite) TestGetNamespacesForSACEdgeCases( }) } } + +// TestGetNamespaceLabels tests the GetNamespaceLabels method +func (s *namespaceDatastoreComprehensiveSuite) TestGetNamespaceLabels() { + testCases := []struct { + name string + namespaceLabels map[string]string + namespaceExists bool + expectedLabels map[string]string + expectError bool + }{ + { + name: "namespace exists with labels", + namespaceLabels: map[string]string{"env": "staging", "owner": "team-a"}, + namespaceExists: true, + expectedLabels: map[string]string{"env": "staging", "owner": "team-a"}, + expectError: false, + }, + { + name: "namespace exists with no labels", + namespaceLabels: nil, + namespaceExists: true, + expectedLabels: nil, + expectError: false, + }, + { + name: "namespace does not exist", + namespaceExists: false, + expectedLabels: nil, + expectError: false, + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + var clusterID, namespaceName string + ctx := s.testContexts[testutils.UnrestrictedReadWriteCtx] + + if tc.namespaceExists { + // Create a namespace with the specified labels + // Use unique namespace name to avoid test isolation issues + uniqueNamespaceName := fmt.Sprintf("test-ns-%s", uuid.NewV4().String()) + ns := fixtures.GetScopedNamespace(uuid.NewV4().String(), testconsts.Cluster1, uniqueNamespaceName) + ns.Labels = tc.namespaceLabels + err := s.datastore.AddNamespace(ctx, ns) + s.Require().NoError(err) + clusterID = ns.GetClusterId() + namespaceName = ns.GetName() + s.testNamespaceIDs = append(s.testNamespaceIDs, ns.GetId()) + } else { + // Use non-existent cluster and namespace + clusterID = uuid.NewV4().String() + namespaceName = "non-existent-namespace" + } + + // Call GetNamespaceLabels + labels, err := s.datastore.GetNamespaceLabels(ctx, clusterID, namespaceName) + + // Verify results + if tc.expectError { + s.Error(err) + } else { + s.NoError(err) + s.Equal(tc.expectedLabels, labels) + } + }) + } +} diff --git a/central/namespace/datastore/internal/store/store.go b/central/namespace/datastore/internal/store/store.go index 52cb213300606..6d9f7e9aec3d8 100644 --- a/central/namespace/datastore/internal/store/store.go +++ b/central/namespace/datastore/internal/store/store.go @@ -19,5 +19,6 @@ type Store interface { Upsert(context.Context, *storage.NamespaceMetadata) error Delete(ctx context.Context, id string) error GetMany(ctx context.Context, ids []string) ([]*storage.NamespaceMetadata, []int, error) + // Deprecated: It will not clone the object so it should be used only for SAC. GetAllFromCacheForSAC() []*storage.NamespaceMetadata } diff --git a/central/namespace/datastore/mocks/datastore.go b/central/namespace/datastore/mocks/datastore.go index 6dc7ee23d9c5f..fb1dab50f609a 100644 --- a/central/namespace/datastore/mocks/datastore.go +++ b/central/namespace/datastore/mocks/datastore.go @@ -119,6 +119,21 @@ func (mr *MockDataStoreMockRecorder) GetNamespace(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNamespace", reflect.TypeOf((*MockDataStore)(nil).GetNamespace), ctx, id) } +// GetNamespaceLabels mocks base method. +func (m *MockDataStore) GetNamespaceLabels(ctx context.Context, clusterID, namespaceName string) (map[string]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNamespaceLabels", ctx, clusterID, namespaceName) + ret0, _ := ret[0].(map[string]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNamespaceLabels indicates an expected call of GetNamespaceLabels. +func (mr *MockDataStoreMockRecorder) GetNamespaceLabels(ctx, clusterID, namespaceName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNamespaceLabels", reflect.TypeOf((*MockDataStore)(nil).GetNamespaceLabels), ctx, clusterID, namespaceName) +} + // GetNamespacesForSAC mocks base method. func (m *MockDataStore) GetNamespacesForSAC() ([]effectiveaccessscope.Namespace, error) { m.ctrl.T.Helper() diff --git a/central/networkbaseline/datastore/datastore_impl_test.go b/central/networkbaseline/datastore/datastore_impl_test.go index f4ef4590573d6..f329292442f1c 100644 --- a/central/networkbaseline/datastore/datastore_impl_test.go +++ b/central/networkbaseline/datastore/datastore_impl_test.go @@ -181,7 +181,7 @@ func (suite *NetworkBaselineDataStoreTestSuite) TestSAC() { // Test Delete Multiple { nbs := fixtures.GetSACTestNetworkBaseline() - var ids []string + ids := make([]string, 0, len(nbs)) for _, nb := range nbs { ids = append(ids, nb.GetDeploymentId()) } diff --git a/central/networkbaseline/datastore/datastore_sac_test.go b/central/networkbaseline/datastore/datastore_sac_test.go index 91d74d06224e3..75345f32b4de1 100644 --- a/central/networkbaseline/datastore/datastore_sac_test.go +++ b/central/networkbaseline/datastore/datastore_sac_test.go @@ -228,7 +228,7 @@ func (s *networkBaselineDatastoreSACTestSuite) TestDeleteNetworkBaselines() { // Upsert resources with various scopes. s.NoError(s.datastore.UpsertNetworkBaselines(s.testContexts[testutils.UnrestrictedReadWriteCtx], nbs)) var ( - ids []string + ids = make([]string, 0, len(nbs)) cluster1ids []string ) for _, nb := range nbs { diff --git a/central/networkbaseline/manager/manager_impl_bench_test.go b/central/networkbaseline/manager/manager_impl_bench_test.go index 8e42c0c77b425..519a6ef3a100f 100644 --- a/central/networkbaseline/manager/manager_impl_bench_test.go +++ b/central/networkbaseline/manager/manager_impl_bench_test.go @@ -23,7 +23,7 @@ import ( func generateBaselines(b *testing.B) []*storage.NetworkBaseline { var networkBaselines []*storage.NetworkBaseline - for i := 0; i < 2000; i++ { + for range 2000 { networkBaseline := &storage.NetworkBaseline{} require.NoError(b, testutils.FullInit(networkBaseline, testutils.UniqueInitializer(), testutils.JSONFieldsFilter)) networkBaseline.Peers = nil @@ -56,7 +56,7 @@ func BenchmarkInitFromStore(b *testing.B) { // load it up require.NoError(b, nbStore.UpsertNetworkBaselines(ctx, generateBaselines(b))) - for i := 0; i < 2000; i++ { + for range 2000 { networkPolicy := &storage.NetworkPolicy{} require.NoError(b, testutils.FullInit(networkPolicy, testutils.UniqueInitializer(), testutils.JSONFieldsFilter)) require.NoError(b, npStore.UpsertNetworkPolicy(ctx, networkPolicy)) diff --git a/central/networkbaseline/service/service_impl_sql_test.go b/central/networkbaseline/service/service_impl_sql_test.go index 4e7c64cd5497a..19c09016b2985 100644 --- a/central/networkbaseline/service/service_impl_sql_test.go +++ b/central/networkbaseline/service/service_impl_sql_test.go @@ -148,7 +148,7 @@ func (s *networkBaselineServiceSuite) setupTablesExternalFlows() { deployment := fixtures.LightweightDeployment() s.NoError(s.deploymentDataStore.UpsertDeployment(allAllowedCtx, deployment)) - var entities []*storage.NetworkEntity + entities := make([]*storage.NetworkEntity, 0, len(externalIps)) for _, ip := range externalIps { cidr := fmt.Sprintf("%s/32", ip) @@ -168,7 +168,7 @@ func (s *networkBaselineServiceSuite) setupTablesExternalFlows() { _, err := s.entityDataStore.CreateExtNetworkEntitiesForCluster(allAllowedCtx, fixtureconsts.Cluster1, entities...) s.NoError(err) - var flows []*storage.NetworkFlow + flows := make([]*storage.NetworkFlow, 0, 2*len(entities)) ts := time.Now().Add(-10 * time.Minute) diff --git a/central/networkgraph/entity/datastore/datastore_bench_test.go b/central/networkgraph/entity/datastore/datastore_bench_test.go index 3a79285aaf6f2..12114c5e40052 100644 --- a/central/networkgraph/entity/datastore/datastore_bench_test.go +++ b/central/networkgraph/entity/datastore/datastore_bench_test.go @@ -46,9 +46,7 @@ func BenchmarkNetEntityCreates(b *testing.B) { entities, err := testutils.GenRandomExtSrcNetworkEntity(pkgNet.IPv4, b.N, testconsts.Cluster1) require.NoError(b, err) - b.ResetTimer() - - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { require.NoError(b, ds.CreateExternalNetworkEntity(globalAccessCtx, entities[i], true)) } } @@ -68,9 +66,7 @@ func BenchmarkNetEntityCreateBatch(b *testing.B) { entities, err := testutils.GenRandomExtSrcNetworkEntity(pkgNet.IPv4, 10000, testconsts.Cluster1) require.NoError(b, err) - b.ResetTimer() - - for i := 0; i < b.N; i++ { + for b.Loop() { _, err = ds.CreateExtNetworkEntitiesForCluster(globalAccessCtx, testconsts.Cluster1, entities...) require.NoError(b, err) @@ -98,9 +94,7 @@ func BenchmarkNetEntityUpdates(b *testing.B) { require.NoError(b, ds.CreateExternalNetworkEntity(globalAccessCtx, e, true)) } - b.ResetTimer() - - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { require.NoError(b, ds.UpdateExternalNetworkEntity(globalAccessCtx, entities[i], true)) } } diff --git a/central/networkgraph/entity/networktree/manager_bench_test.go b/central/networkgraph/entity/networktree/manager_bench_test.go index 8c6f3e1bba18f..5b56cec55d96b 100644 --- a/central/networkgraph/entity/networktree/manager_bench_test.go +++ b/central/networkgraph/entity/networktree/manager_bench_test.go @@ -38,14 +38,14 @@ func BenchmarkCreateNetworkTree(b *testing.B) { // hence resulting in comparison with every node for each new entry. b.Run("initialize", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { mgr := initialize(b, entitiesByCluster) require.NotNil(b, mgr) } }) b.Run("insertIntoNetworkTree", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { b.StopTimer() mgr := initialize(b, entitiesByCluster) t := mgr.CreateNetworkTree(context.Background(), "c2") diff --git a/central/networkgraph/entity/networktree/manager_impl.go b/central/networkgraph/entity/networktree/manager_impl.go index 7b3257e20f1a9..2666ce7059497 100644 --- a/central/networkgraph/entity/networktree/manager_impl.go +++ b/central/networkgraph/entity/networktree/manager_impl.go @@ -30,6 +30,9 @@ type managerImpl struct { func (f *managerImpl) Initialize(entitiesByCluster map[string][]*storage.NetworkEntityInfo) error { defer f.initializedSig.Signal() + f.lock.Lock() + defer f.lock.Unlock() + for cluster, entities := range entitiesByCluster { if _, err := f.createNoLock(cluster, entities...); err != nil { return err diff --git a/central/networkgraph/flow/datastore/datastore_benchmark_test.go b/central/networkgraph/flow/datastore/datastore_benchmark_test.go index 1a0919f780013..d564542481ede 100644 --- a/central/networkgraph/flow/datastore/datastore_benchmark_test.go +++ b/central/networkgraph/flow/datastore/datastore_benchmark_test.go @@ -50,7 +50,7 @@ func BenchmarkDatastoreUpsertFlows(b *testing.B) { func benchmarkDatastoreUpsertFlows(ds *flowDataStoreImpl, numFlows uint32) func(*testing.B) { return func(b *testing.B) { flows := make([]*storage.NetworkFlow, 0, numFlows) - for i := uint32(0); i < numFlows; i++ { + for i := range numFlows { id, err := testutils.ExtIdFromIPv4(fixtureconsts.Cluster1, i) require.NoError(b, err) flows = append(flows, testutils.ExtFlow(fixtureconsts.Deployment1, id.String())) diff --git a/central/networkgraph/flow/datastore/internal/store/common/common.go b/central/networkgraph/flow/datastore/internal/store/common/common.go index 6be3525bee554..83b1daae385c2 100644 --- a/central/networkgraph/flow/datastore/internal/store/common/common.go +++ b/central/networkgraph/flow/datastore/internal/store/common/common.go @@ -21,7 +21,7 @@ var ( // FlowStoreKeyPrefix returns the prefix for a specific flow store func FlowStoreKeyPrefix(clusterID string) []byte { - return []byte(fmt.Sprintf("%s\x00%s\x00", GlobalPrefix, clusterID)) + return fmt.Appendf(nil, "%s\x00%s\x00", GlobalPrefix, clusterID) } // GetDeploymentIDsFromKey take in an id []byte and return the deployments in the id @@ -32,7 +32,7 @@ func GetDeploymentIDsFromKey(id []byte) ([]byte, []byte) { // GetID converts *storage.NetworkFlowProperties into a []byte func GetID(props *storage.NetworkFlowProperties) []byte { - return []byte(fmt.Sprintf("%x:%s:%x:%s:%x:%x", int32(props.GetSrcEntity().GetType()), props.GetSrcEntity().GetId(), int32(props.GetDstEntity().GetType()), props.GetDstEntity().GetId(), props.GetDstPort(), int32(props.GetL4Protocol()))) + return fmt.Appendf(nil, "%x:%s:%x:%s:%x:%x", int32(props.GetSrcEntity().GetType()), props.GetSrcEntity().GetId(), int32(props.GetDstEntity().GetType()), props.GetDstEntity().GetId(), props.GetDstPort(), int32(props.GetL4Protocol())) } // ParseID parses the bytes into a storage.NetworkFlowProperties struct diff --git a/central/networkgraph/flow/datastore/internal/store/postgres/store.go b/central/networkgraph/flow/datastore/internal/store/postgres/store.go index 63ee10bfd2dd5..36041952aecc9 100644 --- a/central/networkgraph/flow/datastore/internal/store/postgres/store.go +++ b/central/networkgraph/flow/datastore/internal/store/postgres/store.go @@ -196,10 +196,7 @@ func (s *flowStoreImpl) insertIntoNetworkflow(ctx context.Context, tx *postgres. } func (s *flowStoreImpl) copyFromNetworkflow(ctx context.Context, tx *postgres.Tx, lastUpdateTS timestamp.MicroTS, objs ...*storage.NetworkFlow) error { - batchSize := pgSearch.MaxBatchSize - if len(objs) < batchSize { - batchSize = len(objs) - } + batchSize := min(len(objs), pgSearch.MaxBatchSize) inputRows := make([][]interface{}, 0, batchSize) var err error diff --git a/central/networkgraph/flow/datastore/internal/store/postgres/tests/flow_impl_benchmark_test.go b/central/networkgraph/flow/datastore/internal/store/postgres/tests/flow_impl_benchmark_test.go index 0bd5fcd49561a..9d77fef89e75e 100644 --- a/central/networkgraph/flow/datastore/internal/store/postgres/tests/flow_impl_benchmark_test.go +++ b/central/networkgraph/flow/datastore/internal/store/postgres/tests/flow_impl_benchmark_test.go @@ -66,7 +66,7 @@ func BenchmarkUpsertFlows(b *testing.B) { func benchmarkUpsertFlows(flowStore store.FlowStore, numFlows uint32) func(*testing.B) { return func(b *testing.B) { flows := make([]*storage.NetworkFlow, 0, numFlows) - for i := uint32(0); i < numFlows; i++ { + for i := range numFlows { id, err := testutils.ExtIdFromIPv4(fixtureconsts.Cluster1, i) require.NoError(b, err) flows = append(flows, testutils.ExtFlow(fixtureconsts.Deployment1, id.String())) @@ -125,7 +125,7 @@ func BenchmarkGetFlowsForDeployment(b *testing.B) { func setupDeploymentFlows(b *testing.B, flowStore store.FlowStore, fromId string, toId string, numFlows int) { flows := make([]*storage.NetworkFlow, 0, numFlows) - for i := 0; i < numFlows; i++ { + for range numFlows { flows = append(flows, testutils.DepFlow(toId, fromId)) } @@ -135,7 +135,7 @@ func setupDeploymentFlows(b *testing.B, flowStore store.FlowStore, fromId string func setupExternalIngressFlows(b *testing.B, flowStore store.FlowStore, deploymentId string, numFlows uint32) { flows := make([]*storage.NetworkFlow, 0, numFlows) - for i := uint32(0); i < numFlows; i++ { + for i := range numFlows { id, err := testutils.ExtIdFromIPv4(fixtureconsts.Cluster1, i) require.NoError(b, err) flows = append(flows, testutils.ExtFlow(deploymentId, id.String())) @@ -147,7 +147,7 @@ func setupExternalIngressFlows(b *testing.B, flowStore store.FlowStore, deployme func setupExternalEgressFlows(b *testing.B, flowStore store.FlowStore, deploymentId string, numFlows uint32) { flows := make([]*storage.NetworkFlow, 0, numFlows) - for i := uint32(0); i < numFlows; i++ { + for i := range numFlows { id, err := testutils.ExtIdFromIPv4(fixtureconsts.Cluster1, i) require.NoError(b, err) diff --git a/central/networkgraph/flow/datastore/internal/store/postgres/tests/store_test.go b/central/networkgraph/flow/datastore/internal/store/postgres/tests/store_test.go index cfc0c0cc71467..f3364607d2395 100644 --- a/central/networkgraph/flow/datastore/internal/store/postgres/tests/store_test.go +++ b/central/networkgraph/flow/datastore/internal/store/postgres/tests/store_test.go @@ -127,7 +127,7 @@ func (s *NetworkflowStoreSuite) TestStore() { var networkFlows []*storage.NetworkFlow flowCount := 100 - for i := 0; i < flowCount; i++ { + for range flowCount { networkFlow := &storage.NetworkFlow{} s.NoError(testutils.FullInit(networkFlow, testutils.UniqueInitializer(), testutils.JSONFieldsFilter)) networkFlows = append(networkFlows, networkFlow) diff --git a/central/networkgraph/service/service_impl_postgres_benchmark_test.go b/central/networkgraph/service/service_impl_postgres_benchmark_test.go index a940c2e896c6d..3a4d981ab0a04 100644 --- a/central/networkgraph/service/service_impl_postgres_benchmark_test.go +++ b/central/networkgraph/service/service_impl_postgres_benchmark_test.go @@ -151,7 +151,7 @@ func (s *networkGraphServiceBenchmarks) createFlows(b *testing.B, numDeployments chunkSize := len(entities) / numDeployments ts := time.Now() - for i := 0; i < numDeployments; i++ { + for i := range numDeployments { name := fmt.Sprintf("deployment-%d", i) deployment := &storage.Deployment{ Name: name, diff --git a/central/networkgraph/service/service_impl_postgres_test.go b/central/networkgraph/service/service_impl_postgres_test.go index 1d320641edac3..d6ebec694f60b 100644 --- a/central/networkgraph/service/service_impl_postgres_test.go +++ b/central/networkgraph/service/service_impl_postgres_test.go @@ -445,7 +445,8 @@ func (s *networkGraphServiceSuite) TestGetExternalNetworkFlows() { externalFlow(deployments[1], entities[2], false, nil), } - allFlows := []*storage.NetworkFlow{singleEntityFlow} + allFlows := make([]*storage.NetworkFlow, 0, 1+len(multiEntityFlows)) + allFlows = append(allFlows, singleEntityFlow) allFlows = append(allFlows, multiEntityFlows...) flowStore, err := s.flowDataStore.CreateFlowStore(globalWriteAccessCtx, testCluster) diff --git a/central/networkgraph/service/service_impl_test.go b/central/networkgraph/service/service_impl_test.go index 32f48043a78f0..98e4917af163f 100644 --- a/central/networkgraph/service/service_impl_test.go +++ b/central/networkgraph/service/service_impl_test.go @@ -926,7 +926,7 @@ func (s *NetworkGraphServiceTestSuite) TestGenerateNetworkGraphWithSACDeterminis s.Require().NoError(err) var flowStrings []string - var nodeIDs []string + nodeIDs := make([]string, 0, len(graph.GetNodes())) for _, node := range graph.GetNodes() { nodeIDs = append(nodeIDs, node.GetEntity().GetId()) for succIdx := range node.GetOutEdges() { @@ -970,7 +970,7 @@ func (s *NetworkGraphServiceTestSuite) TestGenerateNetworkGraphWithSACDeterminis s.Require().NoError(err) var flowStringsSecondPass []string - var nodeIDsSecondPass []string + nodeIDsSecondPass := make([]string, 0, len(graph.GetNodes())) for _, node := range graph.GetNodes() { nodeIDsSecondPass = append(nodeIDsSecondPass, node.GetEntity().GetId()) for succIdx := range node.GetOutEdges() { diff --git a/central/networkpolicies/deployment/matcher.go b/central/networkpolicies/deployment/matcher.go index db4fd85ca8eb1..256c0274b5c86 100644 --- a/central/networkpolicies/deployment/matcher.go +++ b/central/networkpolicies/deployment/matcher.go @@ -2,6 +2,7 @@ package deployment import ( "context" + "slices" "github.com/pkg/errors" networkPolicyDS "github.com/stackrox/rox/central/networkpolicies/datastore" @@ -121,18 +122,9 @@ func (m *policyMatcherImpl) GetIsolationDetails(resource LabeledResource) Isolat } func hasEgress(types []storage.NetworkPolicyType) bool { - return hasPolicyType(types, storage.NetworkPolicyType_EGRESS_NETWORK_POLICY_TYPE) + return slices.Contains(types, storage.NetworkPolicyType_EGRESS_NETWORK_POLICY_TYPE) } func hasIngress(types []storage.NetworkPolicyType) bool { - return len(types) == 0 || hasPolicyType(types, storage.NetworkPolicyType_INGRESS_NETWORK_POLICY_TYPE) -} - -func hasPolicyType(types []storage.NetworkPolicyType, t storage.NetworkPolicyType) bool { - for _, pType := range types { - if pType == t { - return true - } - } - return false + return len(types) == 0 || slices.Contains(types, storage.NetworkPolicyType_INGRESS_NETWORK_POLICY_TYPE) } diff --git a/central/networkpolicies/generator/ingress.go b/central/networkpolicies/generator/ingress.go index 547c2948aba4b..a1dbfe49107f0 100644 --- a/central/networkpolicies/generator/ingress.go +++ b/central/networkpolicies/generator/ingress.go @@ -18,7 +18,7 @@ var allowAllPodsAllNS = &storage.NetworkPolicyIngressRule{ } func generateIngressRules(node *node, namespacesByName map[string]*storage.NamespaceMetadata) []*storage.NetworkPolicyIngressRule { - var rules []*storage.NetworkPolicyIngressRule + rules := make([]*storage.NetworkPolicyIngressRule, 0, len(node.incoming)) for port := range node.incoming { rules = append(rules, generateIngressRule(node, port, namespacesByName)) diff --git a/central/networkpolicies/generator/network_policy_util.go b/central/networkpolicies/generator/network_policy_util.go index 652281685df6b..e5ef777fde341 100644 --- a/central/networkpolicies/generator/network_policy_util.go +++ b/central/networkpolicies/generator/network_policy_util.go @@ -1,17 +1,14 @@ package generator import ( + "slices" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/labels" ) func checkPolicyType(policy *storage.NetworkPolicy, policyType storage.NetworkPolicyType) bool { - for _, ty := range policy.GetSpec().GetPolicyTypes() { - if ty == policyType { - return true - } - } - return false + return slices.Contains(policy.GetSpec().GetPolicyTypes(), policyType) } func groupNetworkPolicies(policies []*storage.NetworkPolicy) (ingressPolicies, egressPolicies map[string][]*storage.NetworkPolicy) { diff --git a/central/networkpolicies/graph/bench_test.go b/central/networkpolicies/graph/bench_test.go index 6c074797e35d6..01e74906e016e 100644 --- a/central/networkpolicies/graph/bench_test.go +++ b/central/networkpolicies/graph/bench_test.go @@ -92,18 +92,18 @@ func matchPolicyToRandomDeps(networkPolicies []*storage.NetworkPolicy, deploymen func benchmarkEvaluateCluster(b *testing.B, numDeployments, numNetworkPolicies, numPoliciesApplyTo, ingressMatches, egressMatches int) { m := newMockGraphEvaluator() deployments := make([]*storage.Deployment, 0, numDeployments) - for i := 0; i < numDeployments; i++ { + for i := range numDeployments { deployments = append(deployments, getMockDeployment(strconv.Itoa(i))) } networkPolicies := make([]*storage.NetworkPolicy, 0, numDeployments) - for i := 0; i < numNetworkPolicies; i++ { + for i := range numNetworkPolicies { networkPolicies = append(networkPolicies, getMockNetworkPolicy(fmt.Sprintf("%d", i))) } applyPolicies(networkPolicies, deployments, numPoliciesApplyTo) matchIngressRules(networkPolicies, deployments, ingressMatches) matchEgressRules(networkPolicies, deployments, egressMatches) - b.ResetTimer() - for i := 0; i < b.N; i++ { + + for b.Loop() { m.GetGraph("", nil, deployments, nil, networkPolicies, false) } } diff --git a/central/networkpolicies/graph/evaluator_test.go b/central/networkpolicies/graph/evaluator_test.go index c0786e2f478ca..9c6c671e9df6e 100644 --- a/central/networkpolicies/graph/evaluator_test.go +++ b/central/networkpolicies/graph/evaluator_test.go @@ -652,7 +652,7 @@ type testEdge struct { } func egressEdges(src string, dsts ...string) []testEdge { - var edges []testEdge + edges := make([]testEdge, 0, len(dsts)) for _, d := range dsts { edges = append(edges, testEdge{Source: src, Target: d}) } @@ -660,7 +660,7 @@ func egressEdges(src string, dsts ...string) []testEdge { } func egressEdgesWithPorts(src string, pds portDescs, dsts ...string) []testEdge { - var edges []testEdge + edges := make([]testEdge, 0, len(dsts)) for _, d := range dsts { edges = append(edges, testEdge{Source: src, Target: d, Ports: pds}) } @@ -668,7 +668,7 @@ func egressEdgesWithPorts(src string, pds portDescs, dsts ...string) []testEdge } func ingressEdges(dst string, srcs ...string) []testEdge { - var edges []testEdge + edges := make([]testEdge, 0, len(srcs)) for _, s := range srcs { edges = append(edges, testEdge{Source: s, Target: dst}) } @@ -676,7 +676,7 @@ func ingressEdges(dst string, srcs ...string) []testEdge { } func ingressEdgesWithPort(dst string, pds portDescs, srcs ...string) []testEdge { - var edges []testEdge + edges := make([]testEdge, 0, len(srcs)) for _, s := range srcs { edges = append(edges, testEdge{Source: s, Target: dst, Ports: pds}) } diff --git a/central/networkpolicies/graph/graph_builder_test.go b/central/networkpolicies/graph/graph_builder_test.go index 1aa48a2a0c3b8..2aec60e2c8446 100644 --- a/central/networkpolicies/graph/graph_builder_test.go +++ b/central/networkpolicies/graph/graph_builder_test.go @@ -432,7 +432,7 @@ func TestMatchPolicyPeer(t *testing.T) { b := newGraphBuilder(nil, c.deployments, c.networkTree, namespacesByID) // Run the test multiple times to make sure that the graph builder is not put into a bad state the first time, // and is capable of returning consistent results. - for i := 0; i < 2; i++ { + for range 2 { matches := b.evaluatePeer(namespacesByID[c.policyNamespace], c.peer) formattedMatches := make([]expectedMatch, 0, len(matches)) for _, match := range matches { diff --git a/central/networkpolicies/graph/netpol_util.go b/central/networkpolicies/graph/netpol_util.go index daeade8441bf2..0998486872de7 100644 --- a/central/networkpolicies/graph/netpol_util.go +++ b/central/networkpolicies/graph/netpol_util.go @@ -1,23 +1,18 @@ package graph -import "github.com/stackrox/rox/generated/storage" +import ( + "slices" + + "github.com/stackrox/rox/generated/storage" +) func hasEgress(types []storage.NetworkPolicyType) bool { - return hasPolicyType(types, storage.NetworkPolicyType_EGRESS_NETWORK_POLICY_TYPE) + return slices.Contains(types, storage.NetworkPolicyType_EGRESS_NETWORK_POLICY_TYPE) } func hasIngress(types []storage.NetworkPolicyType) bool { if len(types) == 0 { return true } - return hasPolicyType(types, storage.NetworkPolicyType_INGRESS_NETWORK_POLICY_TYPE) -} - -func hasPolicyType(types []storage.NetworkPolicyType, t storage.NetworkPolicyType) bool { - for _, pType := range types { - if pType == t { - return true - } - } - return false + return slices.Contains(types, storage.NetworkPolicyType_INGRESS_NETWORK_POLICY_TYPE) } diff --git a/central/networkpolicies/service/service_impl_test.go b/central/networkpolicies/service/service_impl_test.go index a7df55bd2efd5..11fef97878a90 100644 --- a/central/networkpolicies/service/service_impl_test.go +++ b/central/networkpolicies/service/service_impl_test.go @@ -601,7 +601,7 @@ func (suite *ServiceTestSuite) TestGetAllowedPeersFromCurrentPolicyForDeployment // Prepare deployment001 - deployment004 numDeployments := 4 deps := make([]*storage.Deployment, 0, numDeployments) - for i := 0; i < numDeployments; i++ { + for i := range numDeployments { deps = append(deps, &storage.Deployment{ Id: fmt.Sprintf("deployment%03d", i), Name: fmt.Sprintf("deployment%03d", i), diff --git a/central/node/datastore/datastore_bench_postgres_test.go b/central/node/datastore/datastore_bench_postgres_test.go index 1a9830d1adff8..2e0088a87b69b 100644 --- a/central/node/datastore/datastore_bench_postgres_test.go +++ b/central/node/datastore/datastore_bench_postgres_test.go @@ -34,7 +34,7 @@ func BenchmarkGetManyNodes(b *testing.B) { ids := make([]string, 0, 100) nodes := make([]*storage.Node, 0, 100) - for i := 0; i < 100; i++ { + for range 100 { node := fixtures.GetNodeWithUniqueComponents(5, 5) converter.MoveNodeVulnsToNewField(node) id := uuid.NewV4().String() diff --git a/central/node/datastore/datastore_bench_test.go b/central/node/datastore/datastore_bench_test.go index 2598a6fabdd4c..bd076a79ad360 100644 --- a/central/node/datastore/datastore_bench_test.go +++ b/central/node/datastore/datastore_bench_test.go @@ -27,7 +27,7 @@ func BenchmarkNodes(b *testing.B) { fakeNode := fixtures.GetNodeWithUniqueComponents(100, 100) nodes := make([]*storage.Node, 100) - for i := 0; i < 100; i++ { + for i := range 100 { fakeNode.Id = uuid.NewV4().String() fakeNode.ClusterId = fakeNode.GetId() fakeNode.ClusterName = fmt.Sprintf("c-%d", i) @@ -39,14 +39,14 @@ func BenchmarkNodes(b *testing.B) { // Stored node is read because it contains new scan. b.Run("upsertNodeWithOldScan", func(b *testing.B) { fakeNode.Scan.ScanTime.Seconds = fakeNode.GetScan().GetScanTime().GetSeconds() - 500 - for i := 0; i < b.N; i++ { + for b.Loop() { require.NoError(b, nodeDS.UpsertNode(ctx, fakeNode)) } }) b.Run("upsertNodeWithNewScan", func(b *testing.B) { fakeNode.Scan.ScanTime.Seconds = fakeNode.GetScan().GetScanTime().GetSeconds() + 500 - for i := 0; i < b.N; i++ { + for b.Loop() { require.NoError(b, nodeDS.UpsertNode(ctx, fakeNode)) } }) @@ -64,7 +64,7 @@ func BenchmarkNodes(b *testing.B) { }) b.Run("deleteForClusters", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for i := 0; b.Loop(); i++ { idx := i % len(nodes) err := nodeDS.DeleteAllNodesForCluster(ctx, nodes[idx].GetClusterId()) require.NoError(b, err) diff --git a/central/node/datastore/datastore_impl.go b/central/node/datastore/datastore_impl.go index 97fba26ab90d1..4ee8231e664ba 100644 --- a/central/node/datastore/datastore_impl.go +++ b/central/node/datastore/datastore_impl.go @@ -331,7 +331,3 @@ func (c *NodeSearchResultConverter) BuildLocation(result *pkgSearch.Result) stri func (c *NodeSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_NODES } - -func (c *NodeSearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/node/datastore/datastore_impl_postgres_test.go b/central/node/datastore/datastore_impl_postgres_test.go index 37c4fed5b63dc..34c364b82a20d 100644 --- a/central/node/datastore/datastore_impl_postgres_test.go +++ b/central/node/datastore/datastore_impl_postgres_test.go @@ -4,8 +4,11 @@ package datastore import ( "context" + "fmt" "sort" + "sync/atomic" "testing" + "time" nodeCVEDS "github.com/stackrox/rox/central/cve/node/datastore" nodeCVEPostgres "github.com/stackrox/rox/central/cve/node/datastore/store/postgres" @@ -31,6 +34,7 @@ import ( pkgSearch "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/scoped" "github.com/stackrox/rox/pkg/set" + "github.com/stackrox/rox/pkg/sync" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" ) @@ -78,6 +82,99 @@ func (suite *NodePostgresDataStoreTestSuite) TearDownTest() { suite.db.Close() } +// TestConcurrentUpsertNode_KeyedMutexPreventsRegression is the datastore-layer counterpart of +// store_test.go's TestStore_ConcurrentUpsert_BypassingOuterLock. That test proves the store layer alone has +// a check-then-act race in isUpdated(): the read-old-scan-and-decide step isn't covered by the store's own +// keyFence lock. This test drives the same workload through UpsertNode, the production call path, which +// wraps the entire read-decide-write sequence in a per-node-ID keyedMutex (see UpsertNode in +// datastore_impl.go). Every caller in this codebase reaches the store exclusively through UpsertNode, so +// this path determines whether concurrent upserts can regress scan_time; the test asserts scan_time never +// regresses even under heavy concurrency. +func (suite *NodePostgresDataStoreTestSuite) TestConcurrentUpsertNode_KeyedMutexPreventsRegression() { + allowAllCtx := sac.WithAllAccess(context.Background()) + + node := getTestNodeForPostgres(fixtureconsts.Node1, "concurrent-node") + baseTime := time.Now().Add(-24 * time.Hour) + node.Scan.ScanTime = protocompat.ConvertTimeToTimestampOrNil(&baseTime) + suite.NoError(suite.datastore.UpsertNode(allowAllCtx, node)) + + const iterations = 500 + const freshWriters = 4 + const metaWriters = 4 + var wg sync.WaitGroup + // A writer that fails leaves the intended workload incomplete, so a zero regression count would be + // meaningless. Track upsert failures and assert none happened. + var upsertFailures atomic.Int64 + + // Simulates the nodeindex pipeline: always a fresh, monotonically increasing ScanTime. + for w := range freshWriters { + wg.Go(func() { + for i := range iterations { + fresh := node.CloneVT() + t := baseTime.Add(time.Duration(w*iterations+i+1) * time.Millisecond) + fresh.Scan.ScanTime = protocompat.ConvertTimeToTimestampOrNil(&t) + if err := suite.datastore.UpsertNode(allowAllCtx, fresh); err != nil { + upsertFailures.Add(1) + } + } + }) + } + + // Simulates the nodes pipeline: frequent metadata-only churn, Scan explicitly nil. + for w := range metaWriters { + wg.Go(func() { + for i := range iterations { + metaOnly := node.CloneVT() + metaOnly.Scan = nil + metaOnly.Labels = map[string]string{"writer": fmt.Sprintf("%d-%d", w, i)} + if err := suite.datastore.UpsertNode(allowAllCtx, metaOnly); err != nil { + upsertFailures.Add(1) + } + } + }) + } + + var regressions atomic.Int64 + stop := make(chan struct{}) + var pollWG sync.WaitGroup + for range 2 { + pollWG.Go(func() { + lastSeen := baseTime + for { + select { + case <-stop: + return + default: + } + current, exists, err := suite.datastore.GetNode(allowAllCtx, node.GetId()) + if err == nil && exists { + st := current.GetScan().GetScanTime().AsTime() + if st.Before(lastSeen) { + regressions.Add(1) + } else { + lastSeen = st + } + } + // Yield between reads so the pollers don't hammer the DB in a tight busy-loop. + time.Sleep(time.Millisecond) + } + }) + } + + wg.Wait() + close(stop) + pollWG.Wait() + + suite.Zero(upsertFailures.Load(), + "every concurrent UpsertNode must succeed; a failure means the intended workload did not complete, "+ + "which would make the regression assertion below meaningless") + suite.Zero(regressions.Load(), + "scan_time must never regress through UpsertNode, since every real caller (nodeindex and nodes "+ + "pipelines alike) goes through the per-node-ID keyedMutex that wraps the entire read-decide-write "+ + "sequence; a non-zero count here would mean the keyedMutex itself has a bug, not just the store "+ + "layer's already-known gap") +} + func (suite *NodePostgresDataStoreTestSuite) TestBasicOps() { node := getTestNodeForPostgres(fixtureconsts.Node1, "name1") allowAllCtx := sac.WithAllAccess(context.Background()) diff --git a/central/node/datastore/datastore_sac_test.go b/central/node/datastore/datastore_sac_test.go index c3aa5ee3ae5b7..cceb21b0069ee 100644 --- a/central/node/datastore/datastore_sac_test.go +++ b/central/node/datastore/datastore_sac_test.go @@ -85,7 +85,7 @@ func (s *nodeDatastoreSACSuite) initTestResourceSet() { const numberOfNodes = 3 for _, clusterID := range clusters { s.testNodeIDs[clusterID] = make([]string, 0, numberOfNodes) - for i := 0; i < numberOfNodes; i++ { + for range numberOfNodes { s.addTestNode(clusterID) } } @@ -514,7 +514,7 @@ func (s *nodeDatastoreSACSuite) TestDeleteNodesMulti() { targetClusterID := testconsts.Cluster2 var delNodeIDs []string - for i := 0; i < 3; i++ { + for range 3 { testNodeID := s.addTestNode(targetClusterID) delNodeIDs = append(delNodeIDs, testNodeID) diff --git a/central/node/datastore/store/postgres/store.go b/central/node/datastore/store/postgres/store.go index f8202f3472433..9f8a4ddbf51f3 100644 --- a/central/node/datastore/store/postgres/store.go +++ b/central/node/datastore/store/postgres/store.go @@ -529,6 +529,17 @@ func (s *storeImpl) isUpdated(ctx context.Context, node *storage.Node) (bool, er // We skip rewriting components and vulnerabilities if the node scan is older. scanUpdated := protocompat.CompareTimestamps(oldNode.GetScan().GetScanTime(), node.GetScan().GetScanTime()) <= 0 if !scanUpdated { + // Metadata-only upserts (e.g. from the informer-driven nodes pipeline) intentionally send no scan at + // all, relying on this method to preserve the stored one; only warn when a real, incoming scan with + // its own scan_time was rejected as not newer than what's stored. + if node.GetScan().GetScanTime() != nil { + log.Warnf("Rejecting incoming node scan for node %s (%s) in cluster %s (%s) as not newer than the "+ + "stored one: stored scan_time=%s, incoming scan_time=%s. The incoming scan and its "+ + "component/CVE data will be discarded and the previously stored scan will be kept.", + node.GetId(), node.GetName(), node.GetClusterName(), node.GetClusterId(), + protocompat.ConvertTimestampToString(oldNode.GetScan().GetScanTime(), time.RFC3339Nano), + protocompat.ConvertTimestampToString(node.GetScan().GetScanTime(), time.RFC3339Nano)) + } node.Scan = oldNode.GetScan() node.RiskScore = oldNode.GetRiskScore() node.SetComponents = oldNode.GetSetComponents() @@ -900,7 +911,7 @@ func (s *storeImpl) WalkByQuery(ctx context.Context, q *v1.Query, fn func(node * return nil } - err = pgSearch.RunCursorQueryForSchemaFn(ctx, pkgSchema.NodesSchema, q, s.db, callback) + err = pgSearch.RunCursorQueryForSchemaFn(ctx, pkgSchema.NodesSchema, q, s.db, "WalkByQuery", callback) if err != nil { return errors.Wrap(err, "cursor by query") } diff --git a/central/node/datastore/store/postgres/store_test.go b/central/node/datastore/store/postgres/store_test.go index de012c53d363e..8c3ca0de4cc41 100644 --- a/central/node/datastore/store/postgres/store_test.go +++ b/central/node/datastore/store/postgres/store_test.go @@ -12,6 +12,7 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protoassert" @@ -21,9 +22,22 @@ import ( "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/testutils" "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) +const rejectionWarnSnippet = "Rejecting incoming node scan for node" + +func swapObservedLogger(t interface{ Cleanup(func()) }) *observer.ObservedLogs { + core, logs := observer.New(zap.WarnLevel) + orig := log + log = &logging.LoggerImpl{InnerLogger: zap.New(core).Sugar()} + t.Cleanup(func() { log = orig }) + return logs +} + type NodesStoreSuite struct { suite.Suite ctx context.Context @@ -151,6 +165,73 @@ func (s *NodesStoreSuite) TestStore_UpsertWithoutScan() { protoassert.Equal(s.T(), foundNode, newNode) } +// TestStore_FreshScanAlwaysWinsSequential covers the sequential happy path of the isUpdated() freshness +// guard: upsert an old scan, then upsert a strictly newer one for the same node ID, and confirm the newer +// one is persisted (both scan_time and payload advance). This isolates single-threaded correctness of the +// guard from the concurrency behavior exercised elsewhere. +func (s *NodesStoreSuite) TestStore_FreshScanAlwaysWinsSequential() { + store := New(s.pool, false, concurrency.NewKeyFence()) + + node := &storage.Node{} + s.NoError(testutils.FullInit(node, testutils.UniqueInitializer(), testutils.JSONFieldsFilter)) + for _, comp := range node.GetScan().GetComponents() { + comp.Vulns = nil + } + + oldTime := time.Now().Add(-60 * 24 * time.Hour) + node.Scan.ScanTime = protocompat.ConvertTimeToTimestampOrNil(&oldTime) + s.NoError(store.Upsert(s.ctx, node)) + + fresh := node.CloneVT() + freshTime := time.Now() + fresh.Scan.ScanTime = protocompat.ConvertTimeToTimestampOrNil(&freshTime) + fresh.Scan.Components = fresh.GetScan().GetComponents()[:1] // Change the payload shape too, not just the timestamp. + s.NoError(store.Upsert(s.ctx, fresh)) + + found, exists, err := store.Get(s.ctx, node.GetId()) + s.NoError(err) + s.True(exists) + s.Equal(freshTime.Unix(), found.GetScan().GetScanTime().AsTime().Unix(), + "a strictly newer scan must always replace an older one when applied sequentially") + s.Len(found.GetScan().GetComponents(), 1) +} + +// TestStore_RejectionWarningFiresOnlyForRealStaleScan validates the rejection warning in isUpdated(): +// a real, non-nil incoming scan that is older than the stored one is rejected AND warned about, while a +// metadata-only upsert (Scan == nil) - which is also "not newer" - is rejected silently (the gate). +func (s *NodesStoreSuite) TestStore_RejectionWarningFiresOnlyForRealStaleScan() { + store := New(s.pool, false, concurrency.NewKeyFence()) + logs := swapObservedLogger(s.T()) + + node := &storage.Node{} + s.NoError(testutils.FullInit(node, testutils.UniqueInitializer(), testutils.JSONFieldsFilter)) + for _, comp := range node.GetScan().GetComponents() { + comp.Vulns = nil + } + newer := time.Now() + node.Scan.ScanTime = protocompat.ConvertTimeToTimestampOrNil(&newer) + s.NoError(store.Upsert(s.ctx, node)) + + // A real, non-nil, older scan must be rejected and warned about. + stale := node.CloneVT() + older := newer.Add(-time.Hour) + stale.Scan.ScanTime = protocompat.ConvertTimeToTimestampOrNil(&older) + s.NoError(store.Upsert(s.ctx, stale)) + + found, exists, err := store.Get(s.ctx, node.GetId()) + s.NoError(err) + s.True(exists) + s.Equal(newer.Unix(), found.GetScan().GetScanTime().AsTime().Unix(), "stale scan must be rejected") + require.Len(s.T(), logs.FilterMessageSnippet(rejectionWarnSnippet).All(), 1) + + // A metadata-only upsert (Scan == nil) is also "not newer" but must NOT emit the warning. + metaOnly := found.CloneVT() + metaOnly.Scan = nil + s.NoError(store.Upsert(s.ctx, metaOnly)) + require.Len(s.T(), logs.FilterMessageSnippet(rejectionWarnSnippet).All(), 1, + "metadata-only upserts must not emit the rejection warning") +} + func (s *NodesStoreSuite) TestStore_OrphanedCVEs() { s.T().Setenv(env.OrphanedCVEsKeepAlive.EnvVar(), "true") if !env.OrphanedCVEsKeepAlive.BooleanSetting() { @@ -203,7 +284,7 @@ func (s *NodesStoreSuite) TestStore_OrphanedCVEs() { s.NoError(err) s.NotEmpty(orphanedCVEs) for _, cve := range orphanedCVEs { - s.NotNil(cve.OrphanedTime) + s.NotNil(cve.GetOrphanedTime()) s.True(vulnNames.Contains(cve.GetCveBaseInfo().GetCve())) } @@ -230,8 +311,8 @@ func (s *NodesStoreSuite) TestStore_OrphanedCVEs() { s.NoError(err) s.NotEmpty(nodeCVEs) for _, cve := range nodeCVEs { - s.False(cve.Orphaned) - s.Nil(cve.OrphanedTime) + s.False(cve.GetOrphaned()) + s.Nil(cve.GetOrphanedTime()) val, ok := orphanedCveIDToCve[cve.GetId()] s.True(ok) s.Equal(val.GetCveBaseInfo().GetCreatedAt(), cve.GetCveBaseInfo().GetCreatedAt()) diff --git a/central/nodecomponent/datastore/datastore_impl.go b/central/nodecomponent/datastore/datastore_impl.go index 9f71320692c93..4bfa62e1e50e9 100644 --- a/central/nodecomponent/datastore/datastore_impl.go +++ b/central/nodecomponent/datastore/datastore_impl.go @@ -138,7 +138,3 @@ func (c *NodeComponentSearchResultConverter) BuildLocation(result *pkgSearch.Res func (c *NodeComponentSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_NODE_COMPONENTS } - -func (c *NodeComponentSearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/nodecomponentcveedge/datastore/datastore_impl.go b/central/nodecomponentcveedge/datastore/datastore_impl.go index 9d75f74e62077..b07d236f26e05 100644 --- a/central/nodecomponentcveedge/datastore/datastore_impl.go +++ b/central/nodecomponentcveedge/datastore/datastore_impl.go @@ -83,7 +83,3 @@ func (c *NodeComponentCVEEdgeSearchResultConverter) BuildLocation(result *search func (c *NodeComponentCVEEdgeSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_NODE_COMPONENT_CVE_EDGE } - -func (c *NodeComponentCVEEdgeSearchResultConverter) GetScore(result *searchPkg.Result) float64 { - return result.Score -} diff --git a/central/nodecomponentedge/datastore/datastore_impl.go b/central/nodecomponentedge/datastore/datastore_impl.go index 0c47e4a860a22..662aa359fe29f 100644 --- a/central/nodecomponentedge/datastore/datastore_impl.go +++ b/central/nodecomponentedge/datastore/datastore_impl.go @@ -88,7 +88,3 @@ func (c *NodeComponentEdgeSearchResultConverter) BuildLocation(result *searchPkg func (c *NodeComponentEdgeSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_NODE_COMPONENT_EDGE } - -func (c *NodeComponentEdgeSearchResultConverter) GetScore(result *searchPkg.Result) float64 { - return result.Score -} diff --git a/central/notifier/service/service.go b/central/notifier/service/service.go index 1b1d93c9c9fc0..da712971b88ec 100644 --- a/central/notifier/service/service.go +++ b/central/notifier/service/service.go @@ -10,6 +10,7 @@ import ( "github.com/stackrox/rox/pkg/integrationhealth" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/notifier" + "github.com/stackrox/rox/pkg/postgres" ) var ( @@ -30,12 +31,14 @@ func New(storage datastore.DataStore, processor notifier.Processor, policyCleaner policycleaner.PolicyCleaner, reporter integrationhealth.Reporter, - cryptoKey string) Service { + cryptoKey string, + db postgres.DB) Service { return &serviceImpl{ storage: storage, processor: processor, policyCleaner: policyCleaner, reporter: reporter, cryptoKey: cryptoKey, + db: db, } } diff --git a/central/notifier/service/service_impl.go b/central/notifier/service/service_impl.go index 186832a06d008..e720e0b3a2f97 100644 --- a/central/notifier/service/service_impl.go +++ b/central/notifier/service/service_impl.go @@ -24,6 +24,8 @@ import ( "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/notifier" pkgNotifiers "github.com/stackrox/rox/pkg/notifiers" + "github.com/stackrox/rox/pkg/postgres" + pgNotify "github.com/stackrox/rox/pkg/postgres/notify" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/secrets" "google.golang.org/grpc" @@ -58,6 +60,7 @@ type serviceImpl struct { processor notifier.Processor reporter integrationhealth.Reporter cryptoKey string + db postgres.DB policyCleaner policycleaner.PolicyCleaner } @@ -164,6 +167,7 @@ func (s *serviceImpl) UpdateNotifier(ctx context.Context, request *v1.UpdateNoti return nil, err } s.processor.UpdateNotifier(ctx, notifier) + s.notifyWorker(ctx, request.GetNotifier().GetId()) return &v1.Empty{}, nil } @@ -195,6 +199,7 @@ func (s *serviceImpl) PostNotifier(ctx context.Context, request *storage.Notifie } request.Id = id s.processor.UpdateNotifier(ctx, notifier) + s.notifyWorker(ctx, id) if err = s.reporter.Register(request.GetId(), request.GetName(), storage.IntegrationHealth_NOTIFIER); err != nil { return nil, err @@ -266,12 +271,22 @@ func (s *serviceImpl) DeleteNotifier(ctx context.Context, request *v1.DeleteNoti } s.processor.RemoveNotifier(ctx, request.GetId()) + s.notifyWorker(ctx, request.GetId()) if err := s.reporter.RemoveIntegrationHealth(request.GetId()); err != nil { return nil, err } return &v1.Empty{}, nil } +func (s *serviceImpl) notifyWorker(ctx context.Context, notifierID string) { + if s.db == nil { + return + } + if err := pgNotify.Notify(ctx, s.db, pgNotify.NotifierChanged, notifierID); err != nil { + log.Errorf("Failed to notify worker of notifier change %s: %v", notifierID, err) + } +} + func (s *serviceImpl) reconcileUpdateNotifierRequest(ctx context.Context, updateRequest *v1.UpdateNotifierRequest) error { if updateRequest.GetUpdatePassword() { return nil diff --git a/central/notifier/service/singleton.go b/central/notifier/service/singleton.go index 8e8e9ce009172..a95e071cff2a5 100644 --- a/central/notifier/service/singleton.go +++ b/central/notifier/service/singleton.go @@ -1,6 +1,7 @@ package service import ( + "github.com/stackrox/rox/central/globaldb" "github.com/stackrox/rox/central/integrationhealth/reporter" "github.com/stackrox/rox/central/notifier/datastore" "github.com/stackrox/rox/central/notifier/policycleaner" @@ -33,6 +34,7 @@ func initialize() { policycleaner.Singleton(), reporter.Singleton(), cryptoKey, + globaldb.GetPostgres(), ) } diff --git a/central/notifiers/all/all_test.go b/central/notifiers/all/all_test.go index fe8b7edbd8503..89da5925d1728 100644 --- a/central/notifiers/all/all_test.go +++ b/central/notifiers/all/all_test.go @@ -1,7 +1,6 @@ package all import ( - "go/ast" "go/parser" "go/token" "os" @@ -35,14 +34,11 @@ func TestAllPackagesAreImported(t *testing.T) { existingNotifiers.Add(baseName) } - var allImports []*ast.ImportSpec f, err := parser.ParseFile(token.NewFileSet(), "all.go", nil, parser.ImportsOnly) require.NoError(t, err, "failed to parse all.go") - allImports = append(allImports, f.Imports...) - importedNotifiers := set.NewStringSet() - for _, imp := range allImports { + for _, imp := range f.Imports { pkgName := strings.TrimSuffix(strings.TrimPrefix(imp.Path.Value, `"`), `"`) pkgBaseName := path.Base(pkgName) importedNotifiers.Add(pkgBaseName) diff --git a/central/notifiers/email/email.go b/central/notifiers/email/email.go index 2b7e84885381c..a485895bba98c 100644 --- a/central/notifiers/email/email.go +++ b/central/notifiers/email/email.go @@ -128,7 +128,7 @@ type smtpServer struct { } func (s *smtpServer) endpoint() string { - return fmt.Sprintf("%v:%v", s.host, s.port) + return net.JoinHostPort(s.host, strconv.Itoa(s.port)) } // Validate Email notifier diff --git a/central/notifiers/jira/jira.go b/central/notifiers/jira/jira.go index 198e5b26cd0b8..0d956b57eec3a 100644 --- a/central/notifiers/jira/jira.go +++ b/central/notifiers/jira/jira.go @@ -259,7 +259,7 @@ func (j *jira) getAlertDescription(alert *storage.Alert) (string, error) { opString = " AND " } - var valueStrings []string + valueStrings := make([]string, 0, len(values)) for _, value := range values { valueStrings = append(valueStrings, value.GetValue()) } diff --git a/central/notifiers/jira/jira_fake_test.go b/central/notifiers/jira/jira_fake_test.go index e212ed49c27a6..f44652fa8a0fc 100644 --- a/central/notifiers/jira/jira_fake_test.go +++ b/central/notifiers/jira/jira_fake_test.go @@ -49,7 +49,7 @@ func (j *fakeJira) Handler() http.Handler { return mux } - basicAuthHeader := fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", j.username, j.password)))) + basicAuthHeader := fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "%s:%s", j.username, j.password))) tokenAuthHeader := fmt.Sprintf("Bearer %s", j.token) return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Header.Get("Authorization") != basicAuthHeader && req.Header.Get("Authorization") != tokenAuthHeader { @@ -310,4 +310,12 @@ func testWithFakeJira(t *testing.T, cloud bool) { assert.Contains(t, issue.Fields.Description, "myDeploymentID") assert.Contains(t, issue.Fields.Description, "Fake policy") assert.Equal(t, "P1", issue.Fields.Priority.Name) + + assert.NoError(t, j.NetworkPolicyYAMLNotify(context.Background(), "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy", "test-cluster")) + require.Len(t, fj.createdIssues, 3) + + issue = fj.createdIssues[2] + assert.Contains(t, issue.Fields.Summary, "test-cluster") + assert.Contains(t, issue.Fields.Description, "NetworkPolicy") + assert.Equal(t, "P2", issue.Fields.Priority.Name) } diff --git a/central/notifiers/jira/jira_integration_test.go b/central/notifiers/jira/jira_integration_test.go deleted file mode 100644 index 892c5c242d429..0000000000000 --- a/central/notifiers/jira/jira_integration_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// build +integration - -package jira - -import ( - "context" - "os" - "testing" - - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/cryptoutils/cryptocodec" - "github.com/stackrox/rox/pkg/fixtures" - mitreMocks "github.com/stackrox/rox/pkg/mitre/datastore/mocks" - notifierMocks "github.com/stackrox/rox/pkg/notifiers/mocks" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" -) - -const testJiraPassword = "JIRA_PASSWORD" -const testJiraUser = "JIRA_EMAIL" - -func skip(t *testing.T) (string, string) { - user := os.Getenv(testJiraUser) - if user == "" { - t.Skipf("Skipping jira integration test because %v is not defined", testJiraUser) - } - password := os.Getenv(testJiraPassword) - if password == "" { - t.Skipf("Skipping jira integration test because %v is not defined", testJiraPassword) - } - return user, password -} - -func getJira(t *testing.T) (*jira, *gomock.Controller) { - mockCtrl := gomock.NewController(t) - metadataGetter := notifierMocks.NewMockMetadataGetter(mockCtrl) - metadataGetter.EXPECT().GetAnnotationValue(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("").AnyTimes() - mitreStore := mitreMocks.NewMockAttackReadOnlyDataStore(mockCtrl) - mitreStore.EXPECT().Get(gomock.Any()).Return(&storage.MitreAttackVector{}, nil).AnyTimes() - - user, password := skip(t) - notifier := &storage.Notifier{ - UiEndpoint: "http://google.com", - Config: &storage.Notifier_Jira{ - Jira: &storage.Jira{ - Username: user, - Password: password, - IssueType: "Bug", - Url: "https://stack-rox.atlassian.net/", - }, - }, - LabelDefault: "AJIT", - } - - j, err := newJira(notifier, metadataGetter, mitreStore, cryptocodec.Singleton(), "stackrox") - require.NoError(t, err) - return j, mockCtrl -} - -func TestJiraAlertNotify(t *testing.T) { - j, mockCtrl := getJira(t) - defer mockCtrl.Finish() - - assert.NoError(t, j.AlertNotify(context.Background(), fixtures.GetAlert())) -} - -func TestJiraNetworkPolicyYAMLNotify(t *testing.T) { - j, mockCtrl := getJira(t) - defer mockCtrl.Finish() - - assert.NoError(t, j.NetworkPolicyYAMLNotify(context.Background(), fixtures.GetYAML(), "test-cluster")) -} - -func TestJiraTest(t *testing.T) { - j, mockCtrl := getJira(t) - defer mockCtrl.Finish() - - assert.NoError(t, j.Test(context.Background())) -} diff --git a/central/notifiers/slack/slack.go b/central/notifiers/slack/slack.go index 31ebfa4b064fc..2052dac3ad57b 100644 --- a/central/notifiers/slack/slack.go +++ b/central/notifiers/slack/slack.go @@ -16,9 +16,11 @@ import ( "github.com/stackrox/rox/pkg/administration/events/codes" "github.com/stackrox/rox/pkg/administration/events/option" "github.com/stackrox/rox/pkg/httputil/proxy" + "github.com/stackrox/rox/pkg/images/types" "github.com/stackrox/rox/pkg/logging" mitreDS "github.com/stackrox/rox/pkg/mitre/datastore" "github.com/stackrox/rox/pkg/notifiers" + "github.com/stackrox/rox/pkg/protoconv" "github.com/stackrox/rox/pkg/retry" "github.com/stackrox/rox/pkg/urlfmt" "github.com/stackrox/rox/pkg/utils" @@ -43,10 +45,26 @@ type slack struct { mitreStore mitreDS.AttackReadOnlyDataStore } -// notification json struct for richly-formatted notifications +// notification json struct for richly-formatted notifications. +// Flat top-level fields are included alongside attachments so that Slack +// workflow webhooks (which only support flat key-value pairs) can extract +// individual alert properties as workflow variables. type notification struct { - Attachments []attachment `json:"attachments"` + Attachments []attachment `json:"attachments,omitempty"` Text string `json:"text"` + + Summary string `json:"summary,omitempty"` + AlertID string `json:"alert_id,omitempty"` + AlertURL string `json:"alert_url,omitempty"` + Severity string `json:"severity,omitempty"` + Time string `json:"time,omitempty"` + Policy string `json:"policy,omitempty"` + Cluster string `json:"cluster,omitempty"` + Namespace string `json:"namespace,omitempty"` + Entity string `json:"entity,omitempty"` + EntityType string `json:"entity_type,omitempty"` + Violations string `json:"violations,omitempty"` + Description string `json:"description,omitempty"` } // attachment json struct for attachments @@ -98,7 +116,7 @@ func (s *slack) getDescription(alert *storage.Alert) (string, error) { opString = " AND " } - var valueStrings []string + valueStrings := make([]string, 0, len(values)) for _, value := range values { codeString := fmt.Sprintf("`%s`", value.GetValue()) valueStrings = append(valueStrings, codeString) @@ -122,23 +140,89 @@ func (*slack) Close(_ context.Context) error { return nil } +type entityMetadata struct { + name string + entityType string + cluster string + namespace string +} + +func getEntityMetadata(alert *storage.Alert) entityMetadata { + switch entity := alert.GetEntity().(type) { + case *storage.Alert_Deployment_: + return entityMetadata{ + name: entity.Deployment.GetName(), + entityType: "Deployment", + cluster: entity.Deployment.GetClusterName(), + namespace: entity.Deployment.GetNamespace(), + } + case *storage.Alert_Image: + return entityMetadata{ + name: types.Wrapper{GenericImage: entity.Image}.FullName(), + entityType: "Image", + } + case *storage.Alert_Resource_: + return entityMetadata{ + name: entity.Resource.GetName(), + entityType: "Resource", + cluster: entity.Resource.GetClusterName(), + namespace: entity.Resource.GetNamespace(), + } + case *storage.Alert_Node_: + return entityMetadata{ + name: entity.Node.GetName(), + entityType: "Node", + cluster: entity.Node.GetClusterName(), + } + } + return entityMetadata{} +} + +func violationMessages(alert *storage.Alert) string { + var msgs []string + for _, v := range alert.GetViolations() { + msgs = append(msgs, v.GetMessage()) + } + if pv := alert.GetProcessViolation(); pv != nil { + msgs = append(msgs, pv.GetMessage()) + } + return strings.Join(msgs, "\n") +} + // AlertNotify takes in an alert and generates the Slack message func (s *slack) AlertNotify(ctx context.Context, alert *storage.Alert) error { body, err := s.getDescription(alert) if err != nil { return err } + summary := notifiers.SummaryForAlert(alert) attachments := []attachment{ { FallBack: body, Color: notifiers.GetAttachmentColor(alert.GetPolicy().GetSeverity()), - Pretext: fmt.Sprintf("*%s*", notifiers.SummaryForAlert(alert)), + Pretext: fmt.Sprintf("*%s*", summary), Text: body, MarkDownFields: []string{"pretext", "text", "fields"}, }, } + + meta := getEntityMetadata(alert) + alertLink := notifiers.AlertLink(s.Notifier.GetUiEndpoint(), alert) + notification := notification{ Attachments: attachments, + Summary: summary, + AlertID: alert.GetId(), + AlertURL: alertLink, + Severity: notifiers.SeverityString(alert.GetPolicy().GetSeverity()), + Time: protoconv.ReadableTime(alert.GetTime()), + Policy: alert.GetPolicy().GetName(), + Cluster: meta.cluster, + Namespace: meta.namespace, + Entity: meta.name, + EntityType: meta.entityType, + Violations: violationMessages(alert), + Description: body, } jsonPayload, err := json.Marshal(¬ification) if err != nil { @@ -194,6 +278,8 @@ func (s *slack) NetworkPolicyYAMLNotify(ctx context.Context, yaml string, cluste } notification := notification{ Attachments: attachments, + Cluster: clusterName, + Description: body, } jsonPayload, err := json.Marshal(¬ification) if err != nil { diff --git a/central/notifiers/slack/slack_test.go b/central/notifiers/slack/slack_test.go index 7e043528315af..d0952b0aed7f3 100644 --- a/central/notifiers/slack/slack_test.go +++ b/central/notifiers/slack/slack_test.go @@ -4,6 +4,7 @@ package slack import ( "context" + "encoding/json" "os" "testing" @@ -12,6 +13,7 @@ import ( mitreMocks "github.com/stackrox/rox/pkg/mitre/datastore/mocks" notifierMocks "github.com/stackrox/rox/pkg/notifiers/mocks" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) @@ -69,3 +71,105 @@ func TestSlackTest(t *testing.T) { assert.NoError(t, s.Test(context.Background())) } + +func TestNotificationContainsFlatFields(t *testing.T) { + alert := fixtures.GetAlert() + s, mockCtrl := getSlackWithMock(t, &storage.Notifier{ + UiEndpoint: "https://stackrox.example.com", + LabelDefault: "https://hooks.slack.com/test", + }) + defer mockCtrl.Finish() + + body, err := s.getDescription(alert) + require.NoError(t, err) + + meta := getEntityMetadata(alert) + n := notification{ + Attachments: []attachment{{Text: body}}, + Summary: "test summary", + AlertID: alert.GetId(), + Severity: "High", + Policy: alert.GetPolicy().GetName(), + Cluster: meta.cluster, + Namespace: meta.namespace, + Entity: meta.name, + EntityType: meta.entityType, + Violations: violationMessages(alert), + Description: body, + } + + data, err := json.Marshal(&n) + require.NoError(t, err) + + var raw map[string]interface{} + require.NoError(t, json.Unmarshal(data, &raw)) + + assert.Equal(t, alert.GetId(), raw["alert_id"]) + assert.Equal(t, "High", raw["severity"]) + assert.Equal(t, alert.GetPolicy().GetName(), raw["policy"]) + assert.Equal(t, "prod cluster", raw["cluster"]) + assert.Equal(t, "stackrox", raw["namespace"]) + assert.Equal(t, "nginx_server", raw["entity"]) + assert.Equal(t, "Deployment", raw["entity_type"]) + assert.Contains(t, raw["violations"], "CVE-2017-15804") + assert.Contains(t, raw["violations"], "This is a process violation") + assert.NotEmpty(t, raw["description"]) + assert.NotNil(t, raw["attachments"], "attachments should still be present for legacy webhook compatibility") +} + +func TestGetEntityMetadata(t *testing.T) { + cases := map[string]struct { + alert *storage.Alert + expectedName string + expectedEntityType string + expectedCluster string + expectedNamespace string + }{ + "deployment": { + alert: fixtures.GetAlert(), + expectedName: "nginx_server", + expectedEntityType: "Deployment", + expectedCluster: "prod cluster", + expectedNamespace: "stackrox", + }, + "resource": { + alert: fixtures.GetResourceAlert(), + expectedName: "my-secret", + expectedEntityType: "Resource", + expectedCluster: "prod cluster", + expectedNamespace: "stackrox", + }, + "image": { + alert: fixtures.GetImageAlert(), + expectedName: "stackrox.io/srox/mongo:latest", + expectedEntityType: "Image", + expectedCluster: "", + expectedNamespace: "", + }, + "node": { + alert: fixtures.GetNodeAlert(), + expectedName: "bbaaaaaa-bbbb-4011-0000-111111111111", + expectedEntityType: "Node", + expectedCluster: "Cluster1", + expectedNamespace: "", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + meta := getEntityMetadata(tc.alert) + assert.Equal(t, tc.expectedName, meta.name) + assert.Equal(t, tc.expectedEntityType, meta.entityType) + assert.Equal(t, tc.expectedCluster, meta.cluster) + assert.Equal(t, tc.expectedNamespace, meta.namespace) + }) + } +} + +func TestViolationMessages(t *testing.T) { + alert := fixtures.GetAlert() + msgs := violationMessages(alert) + assert.Contains(t, msgs, "CVE-2017-15804") + assert.Contains(t, msgs, "CVE-2017-15670") + assert.Contains(t, msgs, "This is a process violation") +} diff --git a/central/notifiers/syslog/syslog_test.go b/central/notifiers/syslog/syslog_test.go index 2266e432eb80c..9ce125623a610 100644 --- a/central/notifiers/syslog/syslog_test.go +++ b/central/notifiers/syslog/syslog_test.go @@ -312,6 +312,14 @@ func (s *SyslogNotifierTestSuite) TestValidateRemoteConfig() { } _, errURLValidIP := validateRemoteConfig(tcpConfigValidIP) s.NoError(errURLValidIP) + + tcpConfigIPv6 := &storage.Syslog_TCPConfig{ + Hostname: "2001:db8::1", + Port: 514, + } + hostname, errIPv6 := validateRemoteConfig(tcpConfigIPv6) + s.NoError(errIPv6) + s.Equal("[2001:db8::1]:514", hostname) } func (s *SyslogNotifierTestSuite) TestHeaderFormat() { diff --git a/central/notifiers/syslog/tcp_sender.go b/central/notifiers/syslog/tcp_sender.go index 6f8ca2d8da810..9286aa8c4ed66 100644 --- a/central/notifiers/syslog/tcp_sender.go +++ b/central/notifiers/syslog/tcp_sender.go @@ -10,7 +10,7 @@ import ( "sync/atomic" "time" - "github.com/cenkalti/backoff/v3" + "github.com/cenkalti/backoff/v4" "github.com/pkg/errors" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" @@ -87,7 +87,7 @@ func validateRemoteConfig(endpointConfig *storage.Syslog_TCPConfig) (string, err return "", errors.Errorf("invalid port number %d must be between 1 and 65535", port) } - hostName := fmt.Sprintf("%s:%d", endpointConfig.GetHostname(), endpointConfig.GetPort()) + hostName := net.JoinHostPort(endpointConfig.GetHostname(), strconv.Itoa(int(endpointConfig.GetPort()))) sysURL := fmt.Sprintf("tcp://%s", hostName) diff --git a/central/notifiers/teams/teams.go b/central/notifiers/teams/teams.go index c416821c78c7a..e304bc0737dad 100644 --- a/central/notifiers/teams/teams.go +++ b/central/notifiers/teams/teams.go @@ -211,7 +211,7 @@ func (t *teams) getViolationSection(alert *storage.Alert) (section, error) { } func (t *teams) getSectionFacts(policySections []*storage.PolicySection) []fact { - var facts []fact + facts := make([]fact, 0, len(policySections)) for _, section := range policySections { sectionName := "Section " if section.GetSectionName() != "" { @@ -229,7 +229,7 @@ func (t *teams) getSectionFacts(policySections []*storage.PolicySection) []fact } func groupsToString(groups []*storage.PolicyGroup) string { - var groupStrings []string + groupStrings := make([]string, 0, len(groups)) for _, group := range groups { var op string if group.GetBooleanOperator() == storage.BooleanOperator_OR { @@ -247,7 +247,7 @@ func groupsToString(groups []*storage.PolicyGroup) string { } func valueListToString(values []*storage.PolicyValue, opString string) string { - var valueList []string + valueList := make([]string, 0, len(values)) for _, value := range values { valueList = append(valueList, value.GetValue()) } diff --git a/central/notifiers/teams/teams_test.go b/central/notifiers/teams/teams_test.go index c7131d3b68ccb..0b2e5b3af98fb 100644 --- a/central/notifiers/teams/teams_test.go +++ b/central/notifiers/teams/teams_test.go @@ -103,7 +103,7 @@ func TestPolicySeverityEnumConverter(t *testing.T) { for k := range storage.Severity_value { actual, err := notifiers.GetNotifiersCompatiblePolicySeverity(k) assert.Nil(t, err) - prefix := strings.Split(k, "_")[0] + prefix, _, _ := strings.Cut(k, "_") expected := strings.Title(strings.ToLower(prefix)) assert.Equal(t, actual, expected) } diff --git a/central/platform/matcher/matcher.go b/central/platform/matcher/matcher.go index a59ad3c850357..b2869a0db5ebd 100644 --- a/central/platform/matcher/matcher.go +++ b/central/platform/matcher/matcher.go @@ -22,8 +22,8 @@ type PlatformMatcher interface { func New(configDatastore configDS.DataStore) PlatformMatcher { allAccessCtx := sac.WithAllAccess(context.Background()) - regexes := []*regexp.Regexp{} config, _, _ := configDatastore.GetPlatformComponentConfig(allAccessCtx) + regexes := make([]*regexp.Regexp, 0, len(config.GetRules())) for _, rule := range config.GetRules() { regex, _ := regexp.Compile(rule.GetNamespaceRule().GetRegex()) regexes = append(regexes, regex) diff --git a/central/pod/datastore/datastore.go b/central/pod/datastore/datastore.go index 78174d285c03e..0421b122e9da0 100644 --- a/central/pod/datastore/datastore.go +++ b/central/pod/datastore/datastore.go @@ -22,6 +22,7 @@ type DataStore interface { SearchRawPods(ctx context.Context, q *v1.Query) ([]*storage.Pod, error) GetPod(ctx context.Context, id string) (*storage.Pod, bool, error) + GetDeploymentIDsByDigest(ctx context.Context, digest string) ([]string, error) WalkByQuery(ctx context.Context, q *v1.Query, fn func(p *storage.Pod) error) error UpsertPod(ctx context.Context, pod *storage.Pod) error diff --git a/central/pod/datastore/datastore_bench_test.go b/central/pod/datastore/datastore_bench_test.go index fbc673773d0af..21fe543bb7553 100644 --- a/central/pod/datastore/datastore_bench_test.go +++ b/central/pod/datastore/datastore_bench_test.go @@ -35,7 +35,7 @@ func BenchmarkSearchAllPods(b *testing.B) { podPrototype := fixtures.GetPod().CloneVT() const numPods = 1000 - for i := 0; i < numPods; i++ { + for range numPods { podPrototype.Id = uuid.NewV4().String() require.NoError(b, podsDatastore.UpsertPod(ctx, podPrototype)) } diff --git a/central/pod/datastore/datastore_impl.go b/central/pod/datastore/datastore_impl.go index 57ab915ad8a30..e50a11192ada6 100644 --- a/central/pod/datastore/datastore_impl.go +++ b/central/pod/datastore/datastore_impl.go @@ -2,6 +2,7 @@ package datastore import ( "context" + "strings" "time" "github.com/pkg/errors" @@ -85,6 +86,35 @@ func (ds *datastoreImpl) GetPod(ctx context.Context, id string) (*storage.Pod, b return pod, true, nil } +func (ds *datastoreImpl) GetDeploymentIDsByDigest(ctx context.Context, digest string) ([]string, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), resourceType, "GetDeploymentIDsByDigest") + + if digest == "" { + return nil, nil + } + + // Filter pods with the given digest and a valid (non-NULL) deployment. + query := pkgSearch.NewQueryBuilder(). + AddExactMatches(pkgSearch.ContainerImageDigest, digest). + AddStrings(pkgSearch.DeploymentID, pkgSearch.WildcardString). + ForSearchResults(pkgSearch.DeploymentID). + ProtoQuery() + results, err := ds.Search(ctx, query) + if err != nil { + return nil, err + } + + deploymentIDs := make([]string, 0, len(results)) + for _, res := range results { + if res.FieldValues != nil { + if depID, ok := res.FieldValues[strings.ToLower(pkgSearch.DeploymentID.String())]; ok && depID != "" { + deploymentIDs = append(deploymentIDs, depID) + } + } + } + return deploymentIDs, nil +} + func (ds *datastoreImpl) WalkByQuery(ctx context.Context, q *v1.Query, fn func(p *storage.Pod) error) error { return ds.podStore.WalkByQuery(ctx, q, fn) } @@ -93,12 +123,6 @@ func (ds *datastoreImpl) WalkByQuery(ctx context.Context, q *v1.Query, fn func(p func (ds *datastoreImpl) UpsertPod(ctx context.Context, pod *storage.Pod) error { defer metrics.SetDatastoreFunctionDuration(time.Now(), resourceType, "Upsert") - if ok, err := podsSAC.WriteAllowed(ctx); err != nil { - return err - } else if !ok { - return sac.ErrResourceAccessDenied - } - ds.processFilter.UpdateByPod(pod) err := ds.keyedMutex.DoStatusWithLock(pod.GetId(), func() error { @@ -162,18 +186,10 @@ func mergeContainerInstances(newPod *storage.Pod, oldPod *storage.Pod) { func (ds *datastoreImpl) RemovePod(ctx context.Context, id string) error { defer metrics.SetDatastoreFunctionDuration(time.Now(), resourceType, "Delete") - if ok, err := podsSAC.WriteAllowed(ctx); err != nil { - return err - } else if !ok { - return sac.ErrResourceAccessDenied - } - pod, found, err := ds.podStore.Get(ctx, id) if err != nil || !found { return err } - ds.processFilter.DeleteByPod(pod) - err = ds.keyedMutex.DoStatusWithLock(id, func() error { return ds.podStore.Delete(ctx, id) }) @@ -181,6 +197,8 @@ func (ds *datastoreImpl) RemovePod(ctx context.Context, id string) error { return err } + ds.processFilter.DeleteByPod(pod) + deleteIndicatorsCtx := sac.WithGlobalAccessScopeChecker(ctx, sac.AllowFixedScopes( sac.AccessModeScopeKeys(storage.Access_READ_ACCESS, storage.Access_READ_WRITE_ACCESS), diff --git a/central/pod/datastore/datastore_impl_real_test.go b/central/pod/datastore/datastore_impl_real_test.go index 5c626958361de..4748e1a62c241 100644 --- a/central/pod/datastore/datastore_impl_real_test.go +++ b/central/pod/datastore/datastore_impl_real_test.go @@ -83,6 +83,77 @@ func (s *PodDatastoreSuite) getProcessIndicatorsFromDB() []*storage.ProcessIndic return indicatorsFromDB } +func (s *PodDatastoreSuite) TestGetDeploymentIDsByDigest() { + deploymentDS, err := deploymentStore.GetTestPostgresDataStore(s.T(), s.postgres.DB) + s.Require().NoError(err) + + dep1 := fixtureconsts.Deployment1 + dep2 := fixtureconsts.Deployment2 + + s.NoError(deploymentDS.UpsertDeployment(s.ctx, &storage.Deployment{ + Id: dep1, Namespace: fixtureconsts.Namespace1, ClusterId: fixtureconsts.Cluster1, + })) + s.NoError(deploymentDS.UpsertDeployment(s.ctx, &storage.Deployment{ + Id: dep2, Namespace: fixtureconsts.Namespace1, ClusterId: fixtureconsts.Cluster1, + })) + + digest1 := "sha256:aaaa" + digest2 := "sha256:bbbb" + + s.NoError(s.datastore.UpsertPod(s.ctx, &storage.Pod{ + Id: fixtureconsts.PodUID1, + Name: "pod1", + DeploymentId: dep1, + ClusterId: fixtureconsts.Cluster1, + Namespace: fixtureconsts.Namespace1, + LiveInstances: []*storage.ContainerInstance{{ + InstanceId: &storage.ContainerInstanceID{Id: "c1"}, + ContainerName: "container1", + ImageDigest: digest1, + }}, + })) + s.NoError(s.datastore.UpsertPod(s.ctx, &storage.Pod{ + Id: fixtureconsts.PodUID2, + Name: "pod2", + DeploymentId: dep2, + ClusterId: fixtureconsts.Cluster1, + Namespace: fixtureconsts.Namespace1, + LiveInstances: []*storage.ContainerInstance{{ + InstanceId: &storage.ContainerInstanceID{Id: "c2"}, + ContainerName: "container2", + ImageDigest: digest1, + }}, + })) + s.NoError(s.datastore.UpsertPod(s.ctx, &storage.Pod{ + Id: fixtureconsts.PodUID3, + Name: "pod3", + DeploymentId: dep2, + ClusterId: fixtureconsts.Cluster1, + Namespace: fixtureconsts.Namespace1, + LiveInstances: []*storage.ContainerInstance{{ + InstanceId: &storage.ContainerInstanceID{Id: "c3"}, + ContainerName: "container3", + ImageDigest: digest2, + }}, + })) + + ids, err := s.datastore.GetDeploymentIDsByDigest(s.ctx, digest1) + s.NoError(err) + s.ElementsMatch([]string{dep1, dep2}, ids) + + ids, err = s.datastore.GetDeploymentIDsByDigest(s.ctx, digest2) + s.NoError(err) + s.ElementsMatch([]string{dep2}, ids) + + ids, err = s.datastore.GetDeploymentIDsByDigest(s.ctx, "sha256:nonexistent") + s.NoError(err) + s.Empty(ids) + + ids, err = s.datastore.GetDeploymentIDsByDigest(s.ctx, "") + s.NoError(err) + s.Nil(ids) +} + // Add plops, process indicators, pods. Delete one of the pods. // Check that the correct pod, process indicators, and plops are deleted. func (s *PodDatastoreSuite) TestRemovePod() { diff --git a/central/pod/datastore/datastore_impl_test.go b/central/pod/datastore/datastore_impl_test.go index bc1a299e1a5bc..15a1e48fa87df 100644 --- a/central/pod/datastore/datastore_impl_test.go +++ b/central/pod/datastore/datastore_impl_test.go @@ -62,9 +62,9 @@ func (suite *PodDataStoreTestSuite) TestNoAccessAllowed() { _, ok, _ := suite.datastore.GetPod(ctx, expectedPod.GetId()) suite.False(ok) - suite.Error(suite.datastore.UpsertPod(ctx, expectedPod), "permission denied") - - suite.Error(suite.datastore.RemovePod(ctx, expectedPod.GetId()), "permission denied") + // The datastore delegates the access control checks to the storage layer + // for UpsertPod and RemovePod. The "end-to-end" behaviour of the datastore + // for these functions is now tested in datastore_sac_test.go } func (suite *PodDataStoreTestSuite) TestGetPod() { diff --git a/central/pod/datastore/datastore_sac_test.go b/central/pod/datastore/datastore_sac_test.go index 138e66505af46..50b13a4b2f881 100644 --- a/central/pod/datastore/datastore_sac_test.go +++ b/central/pod/datastore/datastore_sac_test.go @@ -74,7 +74,7 @@ func (s *podDatastoreSACSuite) deletePod(id string) { } func (s *podDatastoreSACSuite) TestUpsertPod() { - cases := testutils.GenericGlobalSACUpsertTestCases(s.T(), testutils.VerbUpsert) + cases := testutils.GenericNamespaceSACUpsertTestCases(s.T(), testutils.VerbUpsert) for name, c := range cases { s.Run(name, func() { @@ -118,7 +118,7 @@ func (s *podDatastoreSACSuite) TestGetPod() { } func (s *podDatastoreSACSuite) TestRemovePod() { - cases := testutils.GenericGlobalSACDeleteTestCases(s.T()) + cases := testutils.GenericNamespaceSACDeleteTestCases(s.T()) for name, c := range cases { s.Run(name, func() { @@ -131,11 +131,18 @@ func (s *podDatastoreSACSuite) TestRemovePod() { defer s.deletePod(pod.GetId()) err = s.datastore.RemovePod(ctx, pod.GetId()) - if c.ExpectError { - s.Require().Error(err) - s.ErrorIs(err, c.ExpectedError) + s.NoError(err) + fetched, found, fetchErr := s.datastore.GetPod( + s.testContexts[testutils.UnrestrictedReadWriteCtx], + pod.GetId(), + ) + s.NoError(fetchErr) + if c.ExpectedFound { + s.True(found) + protoassert.Equal(s.T(), pod, fetched) } else { - s.NoError(err) + s.False(found) + s.Nil(fetched) } }) } diff --git a/central/pod/datastore/mocks/datastore.go b/central/pod/datastore/mocks/datastore.go index 86c1734f0d405..c14dcf7f8c73f 100644 --- a/central/pod/datastore/mocks/datastore.go +++ b/central/pod/datastore/mocks/datastore.go @@ -58,6 +58,21 @@ func (mr *MockDataStoreMockRecorder) Count(ctx, q any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockDataStore)(nil).Count), ctx, q) } +// GetDeploymentIDsByDigest mocks base method. +func (m *MockDataStore) GetDeploymentIDsByDigest(ctx context.Context, digest string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDeploymentIDsByDigest", ctx, digest) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDeploymentIDsByDigest indicates an expected call of GetDeploymentIDsByDigest. +func (mr *MockDataStoreMockRecorder) GetDeploymentIDsByDigest(ctx, digest any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeploymentIDsByDigest", reflect.TypeOf((*MockDataStore)(nil).GetDeploymentIDsByDigest), ctx, digest) +} + // GetPod mocks base method. func (m *MockDataStore) GetPod(ctx context.Context, id string) (*storage.Pod, bool, error) { m.ctrl.T.Helper() diff --git a/central/policy/customresource/custom_resource_test.go b/central/policy/customresource/custom_resource_test.go index a2da8636955ff..b5462769f16bb 100644 --- a/central/policy/customresource/custom_resource_test.go +++ b/central/policy/customresource/custom_resource_test.go @@ -24,6 +24,29 @@ func TestConvertToCR(t *testing.T) { assert.YAMLEq(t, templateFile, converted) } +func TestConvertExcludeByKind(t *testing.T) { + policy := fixtures.GetPolicy() + policy.Exclusions = []*storage.Exclusion{ + { + Name: "skip-batch", + Matcher: &storage.Exclusion_ExcludeByKind_{ + ExcludeByKind: &storage.Exclusion_ExcludeByKind{ + Kinds: []storage.Exclusion_WorkloadKind{ + storage.Exclusion_CRON_JOB, + storage.Exclusion_JOB, + }, + }, + }, + }, + } + cr := ConvertPolicyToCustomResource(policy) + require.Len(t, cr.SecurityPolicySpec.Exclusions, 1) + exclusion := cr.SecurityPolicySpec.Exclusions[0] + require.NotNil(t, exclusion.ExcludeByKind) + assert.Equal(t, []string{"CRON_JOB", "JOB"}, exclusion.ExcludeByKind.Kinds) + assert.Nil(t, exclusion.Deployment) +} + func getTestPolicy() *storage.Policy { p := fixtures.GetPolicy() p.Notifiers = []string{ @@ -45,8 +68,7 @@ func getTestPolicy() *storage.Policy { } p.Exclusions = []*storage.Exclusion{ { - Name: "exclusionName1", - Deployment: &storage.Exclusion_Deployment{ + Name: "exclusionName1", Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ Name: "deployment1", Scope: &storage.Scope{ Cluster: "cluster1", @@ -56,12 +78,10 @@ func getTestPolicy() *storage.Policy { Value: "value1", }, }, - }, - Expiration: protocompat.GetProtoTimestampFromSeconds(2334221123), + }}, Expiration: protocompat.GetProtoTimestampFromSeconds(2334221123), }, { - Name: "exclusionName2", - Deployment: &storage.Exclusion_Deployment{ + Name: "exclusionName2", Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ Name: "deployment2", Scope: &storage.Scope{ Cluster: "cluster2", @@ -71,9 +91,16 @@ func getTestPolicy() *storage.Policy { Value: "value2", }, }, - }, + }}, }, } + p.Scope = append(p.Scope, &storage.Scope{ + ClusterLabel: &storage.Scope_Label{Key: "env", Value: "prod"}, + NamespaceLabel: &storage.Scope_Label{Key: "team", Value: "platform"}, + }) + p.EvaluationFilter = &storage.EvaluationFilter{ + SkipContainerTypes: []storage.ContainerType{storage.ContainerType_REGULAR, storage.ContainerType_INIT}, + } return p } diff --git a/central/policy/customresource/policy.go b/central/policy/customresource/policy.go index e0f2211cf9dfd..e4f6dae68366d 100644 --- a/central/policy/customresource/policy.go +++ b/central/policy/customresource/policy.go @@ -9,9 +9,11 @@ import ( // Scope represents storage.Scope in the Custom Resource. type Scope struct { - Cluster string `yaml:",omitempty"` - Namespace string `yaml:",omitempty"` - Label *storage.Scope_Label `yaml:",omitempty"` + Cluster string `yaml:",omitempty"` + Namespace string `yaml:",omitempty"` + Label *storage.Scope_Label `yaml:",omitempty"` + ClusterLabel *storage.Scope_Label `yaml:"clusterLabel,omitempty"` + NamespaceLabel *storage.Scope_Label `yaml:"namespaceLabel,omitempty"` } // convertScope Converts storage.Scope to *Scope @@ -21,9 +23,11 @@ func convertScope(p *storage.Scope) *Scope { } return &Scope{ - Cluster: p.Cluster, - Namespace: p.Namespace, - Label: p.Label, + Cluster: p.GetCluster(), + Namespace: p.GetNamespace(), + Label: p.GetLabel(), + ClusterLabel: p.GetClusterLabel(), + NamespaceLabel: p.GetNamespaceLabel(), } } @@ -40,17 +44,34 @@ func convertExclusion_Deployment(p *storage.Exclusion_Deployment) *Exclusion_Dep } return &Exclusion_Deployment{ - Name: p.Name, - Scope: convertScope(p.Scope), + Name: p.GetName(), + Scope: convertScope(p.GetScope()), + } +} + +// Exclusion_ExcludeByKind represents storage.Exclusion_ExcludeByKind in the Custom Resource. +type Exclusion_ExcludeByKind struct { + Kinds []string `yaml:",omitempty"` +} + +// convertExclusion_ExcludeByKind Converts storage.Exclusion_ExcludeByKind to *Exclusion_ExcludeByKind +func convertExclusion_ExcludeByKind(p *storage.Exclusion_ExcludeByKind) *Exclusion_ExcludeByKind { + if p == nil { + return nil + } + + return &Exclusion_ExcludeByKind{ + Kinds: sliceutils.StringSlice(p.GetKinds()...), } } // Exclusion represents storage.Exclusion in the Custom Resource. type Exclusion struct { - Name string `yaml:",omitempty"` - Deployment *Exclusion_Deployment `yaml:",omitempty"` - Image *storage.Exclusion_Image `yaml:",omitempty"` - Expiration string `yaml:",omitempty"` + Name string `yaml:",omitempty"` + Image *storage.Exclusion_Image `yaml:",omitempty"` + Expiration string `yaml:",omitempty"` + Deployment *Exclusion_Deployment `yaml:",omitempty"` + ExcludeByKind *Exclusion_ExcludeByKind `yaml:"excludeByKind,omitempty"` } // convertExclusion Converts storage.Exclusion to *Exclusion @@ -60,10 +81,11 @@ func convertExclusion(p *storage.Exclusion) *Exclusion { } return &Exclusion{ - Name: p.Name, - Deployment: convertExclusion_Deployment(p.Deployment), - Image: p.Image, - Expiration: timestampToFormatRFC3339(p.Expiration), + Name: p.GetName(), + Image: p.GetImage(), + Expiration: timestampToFormatRFC3339(p.GetExpiration()), + Deployment: convertExclusion_Deployment(p.GetDeployment()), + ExcludeByKind: convertExclusion_ExcludeByKind(p.GetExcludeByKind()), } } @@ -82,10 +104,10 @@ func convertPolicyGroup(p *storage.PolicyGroup) *PolicyGroup { } return &PolicyGroup{ - FieldName: p.FieldName, - BooleanOperator: p.BooleanOperator.String(), - Negate: p.Negate, - Values: p.Values, + FieldName: p.GetFieldName(), + BooleanOperator: p.GetBooleanOperator().String(), + Negate: p.GetNegate(), + Values: p.GetValues(), } } @@ -102,8 +124,24 @@ func convertPolicySection(p *storage.PolicySection) *PolicySection { } return &PolicySection{ - SectionName: p.SectionName, - PolicyGroups: sliceutils.ConvertSlice(p.PolicyGroups, convertPolicyGroup), + SectionName: p.GetSectionName(), + PolicyGroups: sliceutils.ConvertSlice(p.GetPolicyGroups(), convertPolicyGroup), + } +} + +// EvaluationFilter represents storage.EvaluationFilter in the Custom Resource. +type EvaluationFilter struct { + SkipContainerTypes []string `yaml:"skipContainerTypes,omitempty"` +} + +// convertEvaluationFilter Converts storage.EvaluationFilter to *EvaluationFilter +func convertEvaluationFilter(p *storage.EvaluationFilter) *EvaluationFilter { + if p == nil { + return nil + } + + return &EvaluationFilter{ + SkipContainerTypes: sliceutils.StringSlice(p.GetSkipContainerTypes()...), } } @@ -127,6 +165,7 @@ type Policy struct { CriteriaLocked bool `yaml:"criteriaLocked"` MitreVectorsLocked bool `yaml:"mitreVectorsLocked"` IsDefault bool `yaml:"isDefault"` + EvaluationFilter *EvaluationFilter `yaml:"evaluationFilter,omitempty"` } // convertPolicy Converts storage.Policy to *Policy @@ -136,24 +175,25 @@ func convertPolicy(p *storage.Policy) *Policy { } return &Policy{ - Name: p.Name, - Description: p.Description, - Rationale: p.Rationale, - Remediation: p.Remediation, - Disabled: p.Disabled, - Categories: p.Categories, - LifecycleStages: sliceutils.StringSlice(p.LifecycleStages...), - EventSource: p.EventSource.String(), - Exclusions: sliceutils.ConvertSlice(p.Exclusions, convertExclusion), - Scope: sliceutils.ConvertSlice(p.Scope, convertScope), - Severity: p.Severity.String(), - EnforcementActions: sliceutils.StringSlice(p.EnforcementActions...), - Notifiers: p.Notifiers, - PolicySections: sliceutils.ConvertSlice(p.PolicySections, convertPolicySection), - MitreAttackVectors: p.MitreAttackVectors, - CriteriaLocked: p.CriteriaLocked, - MitreVectorsLocked: p.MitreVectorsLocked, - IsDefault: p.IsDefault, + Name: p.GetName(), + Description: p.GetDescription(), + Rationale: p.GetRationale(), + Remediation: p.GetRemediation(), + Disabled: p.GetDisabled(), + Categories: p.GetCategories(), + LifecycleStages: sliceutils.StringSlice(p.GetLifecycleStages()...), + EventSource: p.GetEventSource().String(), + Exclusions: sliceutils.ConvertSlice(p.GetExclusions(), convertExclusion), + Scope: sliceutils.ConvertSlice(p.GetScope(), convertScope), + Severity: p.GetSeverity().String(), + EnforcementActions: sliceutils.StringSlice(p.GetEnforcementActions()...), + Notifiers: p.GetNotifiers(), + PolicySections: sliceutils.ConvertSlice(p.GetPolicySections(), convertPolicySection), + MitreAttackVectors: p.GetMitreAttackVectors(), + CriteriaLocked: p.GetCriteriaLocked(), + MitreVectorsLocked: p.GetMitreVectorsLocked(), + IsDefault: p.GetIsDefault(), + EvaluationFilter: convertEvaluationFilter(p.GetEvaluationFilter()), } } diff --git a/central/policy/customresource/testdata/custom_resource.yaml b/central/policy/customresource/testdata/custom_resource.yaml index ceac71f4f2acb..bce167fd2d230 100644 --- a/central/policy/customresource/testdata/custom_resource.yaml +++ b/central/policy/customresource/testdata/custom_resource.yaml @@ -42,6 +42,12 @@ spec: label: key: com.docker.stack.namespace value: prevent + - clusterLabel: + key: env + value: prod + namespaceLabel: + key: team + value: platform severity: LOW_SEVERITY enforcementActions: - FAIL_BUILD_ENFORCEMENT @@ -141,3 +147,7 @@ spec: criteriaLocked: false mitreVectorsLocked: false isDefault: false + evaluationFilter: + skipContainerTypes: + - INIT + - REGULAR diff --git a/central/policy/datastore/datastore_impl.go b/central/policy/datastore/datastore_impl.go index ff23dca458514..b766f21746d2c 100644 --- a/central/policy/datastore/datastore_impl.go +++ b/central/policy/datastore/datastore_impl.go @@ -9,6 +9,7 @@ import ( errorsPkg "github.com/pkg/errors" clusterDS "github.com/stackrox/rox/central/cluster/datastore" "github.com/stackrox/rox/central/metrics" + "github.com/stackrox/rox/central/metrics/custom/refresh" notifierDS "github.com/stackrox/rox/central/notifier/datastore" "github.com/stackrox/rox/central/policy/store" categoriesDataStore "github.com/stackrox/rox/central/policycategory/datastore" @@ -268,7 +269,8 @@ func (ds *datastoreImpl) AddPolicy(ctx context.Context, policy *storage.Policy) } if findPolicyWithSameName(policyNameToPolicyMap, policy.GetName()) != nil { - return "", fmt.Errorf("Could not add policy due to name validation, policy with name %s already exists", policy.GetName()) + nameError := fmt.Errorf("Could not add policy due to name validation, policy with name %s already exists", policy.GetName()) + return "", ds.wrapWithRollback(ctx, tx, nameError) } policyutils.FillSortHelperFields(policy) // Any policy added after startup must be marked custom policy. @@ -298,6 +300,7 @@ func (ds *datastoreImpl) AddPolicy(ctx context.Context, policy *storage.Policy) metrics.IncrementTotalExternalPoliciesGauge() } + refresh.RefreshTracker(metrics.Configuration) return clonedPolicy.GetId(), nil } @@ -336,6 +339,7 @@ func (ds *datastoreImpl) UpdatePolicy(ctx context.Context, policy *storage.Polic if err = ds.storage.Upsert(ctx, clonedPolicy); err != nil { return ds.wrapWithRollback(ctx, tx, err) } + defer refresh.RefreshTracker(metrics.Configuration) return tx.Commit(ctx) } @@ -352,10 +356,12 @@ func (ds *datastoreImpl) RemovePolicy(ctx context.Context, policy *storage.Polic err := ds.removePolicyNoLock(ctx, policy.GetId()) - if err == nil && policy.GetSource() == storage.PolicySource_DECLARATIVE { - metrics.DecrementTotalExternalPoliciesGauge() + if err == nil { + if policy.GetSource() == storage.PolicySource_DECLARATIVE { + metrics.DecrementTotalExternalPoliciesGauge() + } + refresh.RefreshTracker(metrics.Configuration) } - return err } @@ -409,7 +415,7 @@ func (ds *datastoreImpl) ImportPolicies(ctx context.Context, importPolicies []*s responses[i] = response } - + refresh.RefreshTracker(metrics.Configuration) return responses, allSucceeded, nil } @@ -548,8 +554,7 @@ func (ds *datastoreImpl) importOverwrite(ctx context.Context, policy *storage.Po } func getImportErrorsFromError(err error) []*v1.ImportPolicyError { - var policyError *PolicyStoreErrorList - if errors.As(err, &policyError) { + if policyError, converted := errors.AsType[*PolicyStoreErrorList](err); converted { return handlePolicyStoreErrorList(policyError) } @@ -564,8 +569,7 @@ func getImportErrorsFromError(err error) []*v1.ImportPolicyError { func handlePolicyStoreErrorList(policyError *PolicyStoreErrorList) []*v1.ImportPolicyError { var errList []*v1.ImportPolicyError for _, err := range policyError.Errors { - var nameErr *NameConflictError - if errors.As(err, &nameErr) { + if nameErr, converted := errors.AsType[*NameConflictError](err); converted { errList = append(errList, &v1.ImportPolicyError{ Message: nameErr.ErrString, Type: policiesPkg.ErrImportDuplicateName, @@ -576,8 +580,7 @@ func handlePolicyStoreErrorList(policyError *PolicyStoreErrorList) []*v1.ImportP continue } - var idError *IDConflictError - if errors.As(err, &idError) { + if idError, converted := errors.AsType[*IDConflictError](err); converted { errList = append(errList, &v1.ImportPolicyError{ Message: idError.ErrString, Type: policiesPkg.ErrImportDuplicateID, @@ -689,7 +692,3 @@ func (c *PolicySearchResultConverter) BuildLocation(result *searchPkg.Result) st func (c *PolicySearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_POLICIES } - -func (c *PolicySearchResultConverter) GetScore(result *searchPkg.Result) float64 { - return result.Score -} diff --git a/central/policy/datastore/datastore_impl_test.go b/central/policy/datastore/datastore_impl_test.go index 607c4fb1f32fc..e8008bf94966c 100644 --- a/central/policy/datastore/datastore_impl_test.go +++ b/central/policy/datastore/datastore_impl_test.go @@ -318,12 +318,11 @@ func (s *PolicyDatastoreTestSuite) TestRemoveScopesAndNotifiers() { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Cluster: clusterName, - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Cluster: clusterName, }, + }}, }, }, Notifiers: []string{notifierName}, @@ -369,12 +368,11 @@ func (s *PolicyDatastoreTestSuite) TestDoesNotRemoveScopesAndNotifiers() { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Cluster: clusterID, - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Cluster: clusterID, }, + }}, }, }, Notifiers: []string{notifierName}, diff --git a/central/policy/handlers/handler.go b/central/policy/handlers/handler.go index 548db03171318..9eaa9b0389716 100644 --- a/central/policy/handlers/handler.go +++ b/central/policy/handlers/handler.go @@ -17,6 +17,7 @@ import ( v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/apiparams" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/httputil" "github.com/stackrox/rox/pkg/jsonutil" "github.com/stackrox/rox/pkg/logging" @@ -101,6 +102,9 @@ func (h httpHandler) saveAsCustomResources(ctx context.Context, request *apipara writer.Header().Set("Content-Type", "application/zip") names := set.NewStringSet() for _, policy := range policyList { + if !features.EvaluationFilter.Enabled() { + policy.EvaluationFilter = nil + } cr := customresource.ConvertPolicyToCustomResource(policy) // Switch notifier IDs to names notifierNames := make([]string, 0, len(cr.SecurityPolicySpec.Notifiers)) diff --git a/central/policy/matcher/cluster.go b/central/policy/matcher/cluster.go index d255a020be8c5..7196d41589c9b 100644 --- a/central/policy/matcher/cluster.go +++ b/central/policy/matcher/cluster.go @@ -1,6 +1,9 @@ package matcher import ( + "context" + "slices" + "github.com/pkg/errors" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/scopecomp" @@ -21,12 +24,12 @@ func NewClusterMatcher(cluster *storage.Cluster, namespaces []*storage.Namespace } // FilterApplicablePolicies filters incoming policies into policies that apply to cluster and policies that do not apply to cluster -func (m *clusterMatcher) FilterApplicablePolicies(policies []*storage.Policy) ([]*storage.Policy, []*storage.Policy) { +func (m *clusterMatcher) FilterApplicablePolicies(ctx context.Context, policies []*storage.Policy) ([]*storage.Policy, []*storage.Policy) { applicable := make([]*storage.Policy, 0, len(policies)/2) notApplicable := make([]*storage.Policy, 0, len(policies)/2) for _, policy := range policies { - if m.IsPolicyApplicable(policy) { + if m.IsPolicyApplicable(ctx, policy) { applicable = append(applicable, policy) } else { notApplicable = append(notApplicable, policy) @@ -36,21 +39,22 @@ func (m *clusterMatcher) FilterApplicablePolicies(policies []*storage.Policy) ([ } // IsPolicyApplicable returns true if the policy is applicable to cluster -func (m *clusterMatcher) IsPolicyApplicable(policy *storage.Policy) bool { +func (m *clusterMatcher) IsPolicyApplicable(_ context.Context, policy *storage.Policy) bool { return !policy.GetDisabled() && !m.anyExclusionMatches(policy.GetExclusions()) && m.anyScopeMatches(policy.GetScope()) } func (m *clusterMatcher) anyExclusionMatches(exclusions []*storage.Exclusion) bool { - for _, exclusion := range exclusions { - if m.exclusionMatches(exclusion) { - return true - } - } - return false + return slices.ContainsFunc(exclusions, m.exclusionMatches) } func (m *clusterMatcher) exclusionMatches(exclusion *storage.Exclusion) bool { - cs, err := scopecomp.CompileScope(exclusion.GetDeployment().GetScope()) + // A kind exclusion does not take the policy off the cluster. Image-only exclusions have no + // deployment scope, and a nil scope would otherwise match every cluster. + if exclusion.GetDeployment() == nil { + return false + } + + cs, err := scopecomp.CompileScope(exclusion.GetDeployment().GetScope(), nil, nil) if err != nil { utils.Should(errors.Wrap(err, "could not compile excluded scopes")) return false @@ -73,20 +77,11 @@ func (m *clusterMatcher) exclusionMatches(exclusion *storage.Exclusion) bool { } func (m *clusterMatcher) anyScopeMatches(scopes []*storage.Scope) bool { - if len(scopes) == 0 { - return true - } - - for _, scope := range scopes { - if m.scopeMatches(scope) { - return true - } - } - return false + return len(scopes) == 0 || slices.ContainsFunc(scopes, m.scopeMatches) } func (m *clusterMatcher) scopeMatches(scope *storage.Scope) bool { - cs, err := scopecomp.CompileScope(scope) + cs, err := scopecomp.CompileScope(scope, nil, nil) if err != nil { utils.Should(errors.Wrap(err, "could not compile scope")) return false diff --git a/central/policy/matcher/cluster_test.go b/central/policy/matcher/cluster_test.go index 9ebe3a1e9bb87..e0b5532d19e4e 100644 --- a/central/policy/matcher/cluster_test.go +++ b/central/policy/matcher/cluster_test.go @@ -1,6 +1,7 @@ package matcher import ( + "context" "testing" "github.com/stackrox/rox/generated/storage" @@ -108,7 +109,7 @@ func TestClusterMatcher(t *testing.T) { } for _, c := range cases { - actual := NewClusterMatcher(c.cluster, c.namespaces).IsPolicyApplicable(c.policy) + actual := NewClusterMatcher(c.cluster, c.namespaces).IsPolicyApplicable(context.Background(), c.policy) assert.Equal(t, c.matches, actual) } } @@ -137,12 +138,11 @@ func TestClusterMatcherWithExclusion(t *testing.T) { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Namespace: "ns.*", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Namespace: "ns.*", }, + }}, }, }, }, @@ -166,12 +166,11 @@ func TestClusterMatcherWithExclusion(t *testing.T) { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Namespace: "ns.*", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Namespace: "ns.*", }, + }}, }, }, }, @@ -194,12 +193,11 @@ func TestClusterMatcherWithExclusion(t *testing.T) { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Cluster: "cluster1", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Cluster: "cluster1", }, + }}, }, }, }, @@ -217,12 +215,11 @@ func TestClusterMatcherWithExclusion(t *testing.T) { }, policy: &storage.Policy{ Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Namespace: "ns2.*", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Namespace: "ns2.*", }, + }}, }, }, }, @@ -231,7 +228,7 @@ func TestClusterMatcherWithExclusion(t *testing.T) { } for _, c := range cases { - actual := NewClusterMatcher(c.cluster, c.namespaces).IsPolicyApplicable(c.policy) + actual := NewClusterMatcher(c.cluster, c.namespaces).IsPolicyApplicable(context.Background(), c.policy) assert.Equal(t, c.matches, actual) } } diff --git a/central/policy/matcher/deployment.go b/central/policy/matcher/deployment.go index ff4256112862c..86e035585a65f 100644 --- a/central/policy/matcher/deployment.go +++ b/central/policy/matcher/deployment.go @@ -1,30 +1,38 @@ package matcher import ( + "context" + "github.com/pkg/errors" + clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" + namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/scopecomp" "github.com/stackrox/rox/pkg/utils" ) type deploymentMatcher struct { - deployment *storage.Deployment + deployment *storage.Deployment + clusterProvider scopecomp.ClusterLabelProvider + namespaceProvider scopecomp.NamespaceLabelProvider } // NewDeploymentMatcher creates a new policy matcher for deployment data. -func NewDeploymentMatcher(deployment *storage.Deployment) Matcher { +func NewDeploymentMatcher(deployment *storage.Deployment, clusterDS clusterDataStore.DataStore, namespaceDS namespaceDataStore.DataStore) Matcher { return &deploymentMatcher{ - deployment: deployment, + deployment: deployment, + clusterProvider: clusterDS, + namespaceProvider: namespaceDS, } } // FilterApplicablePolicies filters incoming policies into policies that apply to deployment and policies that do not apply to deployment -func (m *deploymentMatcher) FilterApplicablePolicies(policies []*storage.Policy) ([]*storage.Policy, []*storage.Policy) { +func (m *deploymentMatcher) FilterApplicablePolicies(ctx context.Context, policies []*storage.Policy) ([]*storage.Policy, []*storage.Policy) { applicable := make([]*storage.Policy, 0, len(policies)/2) notApplicable := make([]*storage.Policy, 0, len(policies)/2) for _, policy := range policies { - if m.IsPolicyApplicable(policy) { + if m.IsPolicyApplicable(ctx, policy) { applicable = append(applicable, policy) } else { notApplicable = append(notApplicable, policy) @@ -34,22 +42,31 @@ func (m *deploymentMatcher) FilterApplicablePolicies(policies []*storage.Policy) } // IsPolicyApplicable returns true if the policy is applicable to deployment -func (m *deploymentMatcher) IsPolicyApplicable(policy *storage.Policy) bool { - return !policy.GetDisabled() && !m.anyExclusionMatches(policy.GetExclusions()) && m.anyScopeMatches(policy.GetScope()) +func (m *deploymentMatcher) IsPolicyApplicable(ctx context.Context, policy *storage.Policy) bool { + return !policy.GetDisabled() && !m.anyExclusionMatches(ctx, policy.GetExclusions()) && m.anyScopeMatches(ctx, policy.GetScope()) } -func (m *deploymentMatcher) anyExclusionMatches(exclusions []*storage.Exclusion) bool { +func (m *deploymentMatcher) anyExclusionMatches(ctx context.Context, exclusions []*storage.Exclusion) bool { for _, exclusion := range exclusions { - if m.exclusionMatches(exclusion) { + if m.exclusionMatches(ctx, exclusion) { return true } } return false } -func (m *deploymentMatcher) exclusionMatches(exclusion *storage.Exclusion) bool { +func (m *deploymentMatcher) exclusionMatches(ctx context.Context, exclusion *storage.Exclusion) bool { + if !appliesToDeployments(exclusion) { + return false + } + // Kind exclusions have no deployment scope. Falling through would treat the nil scope as + // matching every deployment. + if exclusion.GetExcludeByKind() != nil { + return kindExclusionMatches(exclusion, m.deployment.GetType()) + } + // If excluded scope does not match the deployment then no need to check for deployment name - if !m.scopeMatches(exclusion.GetDeployment().GetScope()) { + if !m.scopeMatches(ctx, exclusion.GetDeployment().GetScope()) { return false } @@ -60,25 +77,25 @@ func (m *deploymentMatcher) exclusionMatches(exclusion *storage.Exclusion) bool exclusion.GetDeployment().GetName() == m.deployment.GetName() } -func (m *deploymentMatcher) anyScopeMatches(scopes []*storage.Scope) bool { +func (m *deploymentMatcher) anyScopeMatches(ctx context.Context, scopes []*storage.Scope) bool { if len(scopes) == 0 { return true } for _, scope := range scopes { - if m.scopeMatches(scope) { + if m.scopeMatches(ctx, scope) { return true } } return false } -func (m *deploymentMatcher) scopeMatches(scope *storage.Scope) bool { - cs, err := scopecomp.CompileScope(scope) +func (m *deploymentMatcher) scopeMatches(ctx context.Context, scope *storage.Scope) bool { + cs, err := scopecomp.CompileScope(scope, m.clusterProvider, m.namespaceProvider) if err != nil { utils.Should(errors.Wrap(err, "could not compile scope")) return false } - return cs.MatchesDeployment(m.deployment) + return cs.MatchesDeployment(ctx, m.deployment) } diff --git a/central/policy/matcher/deployment_test.go b/central/policy/matcher/deployment_test.go index a1761a14fbf7b..1221d49a70e1b 100644 --- a/central/policy/matcher/deployment_test.go +++ b/central/policy/matcher/deployment_test.go @@ -1,10 +1,16 @@ package matcher import ( + "context" "testing" + clusterMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" + namespaceMocks "github.com/stackrox/rox/central/namespace/datastore/mocks" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/testutils" "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" ) func TestDeploymentMatcher(t *testing.T) { @@ -71,7 +77,7 @@ func TestDeploymentMatcher(t *testing.T) { } for _, c := range cases { - actual := NewDeploymentMatcher(c.deployment).IsPolicyApplicable(c.policy) + actual := NewDeploymentMatcher(c.deployment, nil, nil).IsPolicyApplicable(context.Background(), c.policy) assert.Equal(t, c.matches, actual) } } @@ -95,12 +101,11 @@ func TestDeploymentWithExclusion(t *testing.T) { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Namespace: "ns.*", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Namespace: "ns.*", }, + }}, }, }, }, @@ -120,13 +125,12 @@ func TestDeploymentWithExclusion(t *testing.T) { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Name: "deployment2", - Scope: &storage.Scope{ - Namespace: "ns.*", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Name: "deployment2", + Scope: &storage.Scope{ + Namespace: "ns.*", }, + }}, }, }, }, @@ -145,18 +149,16 @@ func TestDeploymentWithExclusion(t *testing.T) { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Name: "deployment2", - Scope: &storage.Scope{ - Namespace: "ns1", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Name: "deployment2", + Scope: &storage.Scope{ + Namespace: "ns1", }, + }}, }, - { - Deployment: &storage.Exclusion_Deployment{ - Name: "deployment1", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Name: "deployment1", + }}, }, }, }, @@ -175,20 +177,18 @@ func TestDeploymentWithExclusion(t *testing.T) { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Name: "deployment2", - Scope: &storage.Scope{ - Namespace: "ns1", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Name: "deployment2", + Scope: &storage.Scope{ + Namespace: "ns1", }, + }}, }, - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Namespace: "ns1", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Namespace: "ns1", }, + }}, }, }, }, @@ -206,7 +206,138 @@ func TestDeploymentWithExclusion(t *testing.T) { } for _, c := range cases { - actual := NewDeploymentMatcher(c.deployment).IsPolicyApplicable(c.policy) + actual := NewDeploymentMatcher(c.deployment, nil, nil).IsPolicyApplicable(context.Background(), c.policy) assert.Equal(t, c.matches, actual) } } + +func TestDeploymentMatcher_WithLabelProviders(t *testing.T) { + testutils.MustUpdateFeature(t, features.LabelBasedPolicyScoping, true) + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + clusterDS := clusterMocks.NewMockDataStore(mockCtrl) + namespaceDS := namespaceMocks.NewMockDataStore(mockCtrl) + + testCluster := &storage.Cluster{ + Id: "cluster1", + Name: "prod-cluster", + Labels: map[string]string{ + "env": "prod", + "region": "us-east-1", + }, + } + + testNamespace := &storage.NamespaceMetadata{ + Id: "ns1", + Name: "kube-system", + ClusterId: "cluster1", + ClusterName: "prod-cluster", + Labels: map[string]string{ + "tier": "system", + }, + } + + // Configure mock expectations + clusterDS.EXPECT().GetCluster(gomock.Any(), "cluster1").Return(testCluster, true, nil).AnyTimes() + clusterDS.EXPECT().GetClusterLabels(gomock.Any(), "cluster1").Return(testCluster.GetLabels(), nil).AnyTimes() + namespaceDS.EXPECT().GetNamespace(gomock.Any(), "ns1").Return(testNamespace, true, nil).AnyTimes() + namespaceDS.EXPECT().GetNamespaceLabels(gomock.Any(), "cluster1", "kube-system").Return(testNamespace.GetLabels(), nil).AnyTimes() + + deployment := &storage.Deployment{ + Name: "test-deployment", + ClusterId: "cluster1", + Namespace: "kube-system", + NamespaceId: "ns1", + } + + cases := []struct { + name string + policy *storage.Policy + matches bool + }{ + { + name: "policy with matching cluster label", + policy: &storage.Policy{ + Scope: []*storage.Scope{ + { + ClusterLabel: &storage.Scope_Label{ + Key: "env", + Value: "prod", + }, + }, + }, + }, + matches: true, + }, + { + name: "policy with non-matching cluster label", + policy: &storage.Policy{ + Scope: []*storage.Scope{ + { + ClusterLabel: &storage.Scope_Label{ + Key: "env", + Value: "dev", + }, + }, + }, + }, + matches: false, + }, + { + name: "policy with matching namespace label", + policy: &storage.Policy{ + Scope: []*storage.Scope{ + { + NamespaceLabel: &storage.Scope_Label{ + Key: "tier", + Value: "system", + }, + }, + }, + }, + matches: true, + }, + { + name: "policy with non-matching namespace label", + policy: &storage.Policy{ + Scope: []*storage.Scope{ + { + NamespaceLabel: &storage.Scope_Label{ + Key: "tier", + Value: "app", + }, + }, + }, + }, + matches: false, + }, + { + name: "policy with both matching cluster and namespace labels", + policy: &storage.Policy{ + Scope: []*storage.Scope{ + { + ClusterLabel: &storage.Scope_Label{ + Key: "region", + Value: "us-east-1", + }, + NamespaceLabel: &storage.Scope_Label{ + Key: "tier", + Value: "system", + }, + }, + }, + }, + matches: true, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + matcher := NewDeploymentMatcher(deployment, clusterDS, namespaceDS) + actual := matcher.IsPolicyApplicable(context.Background(), c.policy) + assert.Equal(t, c.matches, actual) + }) + } +} diff --git a/central/policy/matcher/matcher.go b/central/policy/matcher/matcher.go index 9668ddbd93b31..c94097cd287c7 100644 --- a/central/policy/matcher/matcher.go +++ b/central/policy/matcher/matcher.go @@ -1,9 +1,44 @@ package matcher -import "github.com/stackrox/rox/generated/storage" +import ( + "context" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/kubernetes" +) // Matcher provides functionality to evaluate whether or not policies are applicable to an entity. type Matcher interface { - FilterApplicablePolicies(policies []*storage.Policy) (applicable []*storage.Policy, notApplicable []*storage.Policy) - IsPolicyApplicable(policy *storage.Policy) bool + FilterApplicablePolicies(ctx context.Context, policies []*storage.Policy) (applicable []*storage.Policy, notApplicable []*storage.Policy) + IsPolicyApplicable(ctx context.Context, policy *storage.Policy) bool +} + +// appliesToDeployments reports whether the exclusion can take a deployment out of policy scope. +// A deployment name/scope exclusion and a workload-kind exclusion both can. An image-only exclusion +// cannot: its nil deployment scope matches every deployment and would make the policy look +// inapplicable everywhere. +func appliesToDeployments(exclusion *storage.Exclusion) bool { + return exclusion.GetDeployment() != nil || exclusion.GetExcludeByKind() != nil +} + +// kindExclusionMatches reports whether a kind exclusion excludes this deployment. +// The feature flag gates it. A disabled flag leaves the policy applicable, matching detection. +func kindExclusionMatches(exclusion *storage.Exclusion, deploymentType string) bool { + if !features.PolicyWorkloadKindExclusion.Enabled() { + return false + } + for _, kind := range exclusion.GetExcludeByKind().GetKinds() { + switch kind { + case storage.Exclusion_CRON_JOB: + if deploymentType == kubernetes.CronJob { + return true + } + case storage.Exclusion_JOB: + if deploymentType == kubernetes.Job { + return true + } + } + } + return false } diff --git a/central/policy/matcher/matcher_test.go b/central/policy/matcher/matcher_test.go new file mode 100644 index 0000000000000..0642a5308378c --- /dev/null +++ b/central/policy/matcher/matcher_test.go @@ -0,0 +1,79 @@ +package matcher + +import ( + "context" + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/kubernetes" + "github.com/stackrox/rox/pkg/testutils" + "github.com/stretchr/testify/assert" +) + +// TestImageOnlyExclusionDoesNotExcludeEntities checks that an image-only exclusion leaves the policy +// applicable to every deployment, namespace, and cluster, while a deployment exclusion still excludes. +func TestImageOnlyExclusionDoesNotExcludeEntities(t *testing.T) { + ctx := context.Background() + deployment := &storage.Deployment{Name: "payments", ClusterId: "cluster1", Namespace: "shop"} + namespace := &storage.NamespaceMetadata{Name: "shop", ClusterId: "cluster1"} + cluster := &storage.Cluster{Id: "cluster1"} + + matchers := map[string]Matcher{ + "deployment": NewDeploymentMatcher(deployment, nil, nil), + "namespace": NewNamespaceMatcher(namespace), + "cluster": NewClusterMatcher(cluster, []*storage.NamespaceMetadata{namespace}), + } + + imageOnly := &storage.Policy{Exclusions: []*storage.Exclusion{ + {Image: &storage.Exclusion_Image{Name: "docker.io/library/nginx"}}, + }} + clusterExclusion := &storage.Policy{Exclusions: []*storage.Exclusion{ + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{Scope: &storage.Scope{Cluster: "cluster1"}}}}, + }} + + for name, m := range matchers { + t.Run(name, func(t *testing.T) { + assert.True(t, m.IsPolicyApplicable(ctx, imageOnly), "image-only exclusion should not exclude") + assert.False(t, m.IsPolicyApplicable(ctx, clusterExclusion), "cluster-scoped exclusion should exclude") + }) + } +} + +func TestKindExclusionExcludesMatchingDeploymentsOnly(t *testing.T) { + testutils.MustUpdateFeature(t, features.PolicyWorkloadKindExclusion, true) + ctx := context.Background() + namespace := &storage.NamespaceMetadata{Name: "shop", ClusterId: "cluster1"} + cluster := &storage.Cluster{Id: "cluster1"} + policy := &storage.Policy{Exclusions: []*storage.Exclusion{{ + Matcher: &storage.Exclusion_ExcludeByKind_{ + ExcludeByKind: &storage.Exclusion_ExcludeByKind{ + Kinds: []storage.Exclusion_WorkloadKind{storage.Exclusion_JOB, storage.Exclusion_CRON_JOB}, + }, + }, + }}} + + job := &storage.Deployment{Name: "batch", Type: kubernetes.Job, ClusterId: "cluster1", Namespace: "shop"} + cronJob := &storage.Deployment{Name: "nightly", Type: kubernetes.CronJob, ClusterId: "cluster1", Namespace: "shop"} + deploy := &storage.Deployment{Name: "web", Type: kubernetes.Deployment, ClusterId: "cluster1", Namespace: "shop"} + + assert.False(t, NewDeploymentMatcher(job, nil, nil).IsPolicyApplicable(ctx, policy)) + assert.False(t, NewDeploymentMatcher(cronJob, nil, nil).IsPolicyApplicable(ctx, policy)) + assert.True(t, NewDeploymentMatcher(deploy, nil, nil).IsPolicyApplicable(ctx, policy)) + assert.True(t, NewNamespaceMatcher(namespace).IsPolicyApplicable(ctx, policy)) + assert.True(t, NewClusterMatcher(cluster, []*storage.NamespaceMetadata{namespace}).IsPolicyApplicable(ctx, policy)) +} + +func TestKindExclusionFlagOffLeavesPolicyApplicable(t *testing.T) { + testutils.MustUpdateFeature(t, features.PolicyWorkloadKindExclusion, false) + job := &storage.Deployment{Name: "batch", Type: kubernetes.Job, ClusterId: "cluster1", Namespace: "shop"} + policy := &storage.Policy{Exclusions: []*storage.Exclusion{{ + Matcher: &storage.Exclusion_ExcludeByKind_{ + ExcludeByKind: &storage.Exclusion_ExcludeByKind{ + Kinds: []storage.Exclusion_WorkloadKind{storage.Exclusion_JOB}, + }, + }, + }}} + + assert.True(t, NewDeploymentMatcher(job, nil, nil).IsPolicyApplicable(context.Background(), policy)) +} diff --git a/central/policy/matcher/namespace.go b/central/policy/matcher/namespace.go index 9bf96711fae45..6bd6ee7e7f2aa 100644 --- a/central/policy/matcher/namespace.go +++ b/central/policy/matcher/namespace.go @@ -1,6 +1,9 @@ package matcher import ( + "context" + "slices" + "github.com/pkg/errors" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/scopecomp" @@ -19,12 +22,12 @@ func NewNamespaceMatcher(namespace *storage.NamespaceMetadata) Matcher { } // FilterApplicablePolicies filters incoming policies into policies that apply to namespace and policies that do not apply to namespace -func (m *namespaceMatcher) FilterApplicablePolicies(policies []*storage.Policy) ([]*storage.Policy, []*storage.Policy) { +func (m *namespaceMatcher) FilterApplicablePolicies(ctx context.Context, policies []*storage.Policy) ([]*storage.Policy, []*storage.Policy) { applicable := make([]*storage.Policy, 0, len(policies)/2) notApplicable := make([]*storage.Policy, 0, len(policies)/2) for _, policy := range policies { - if m.IsPolicyApplicable(policy) { + if m.IsPolicyApplicable(ctx, policy) { applicable = append(applicable, policy) } else { notApplicable = append(notApplicable, policy) @@ -34,38 +37,30 @@ func (m *namespaceMatcher) FilterApplicablePolicies(policies []*storage.Policy) } // IsPolicyApplicable returns true if the policy is applicable to namespace -func (m *namespaceMatcher) IsPolicyApplicable(policy *storage.Policy) bool { +func (m *namespaceMatcher) IsPolicyApplicable(_ context.Context, policy *storage.Policy) bool { return !policy.GetDisabled() && !m.anyExclusionMatches(policy.GetExclusions()) && m.anyScopeMatches(policy.GetScope()) } func (m *namespaceMatcher) anyExclusionMatches(exclusions []*storage.Exclusion) bool { - for _, exclusion := range exclusions { - if m.exclusionMatches(exclusion) { - return true - } - } - return false + return slices.ContainsFunc(exclusions, m.exclusionMatches) } func (m *namespaceMatcher) exclusionMatches(exclusion *storage.Exclusion) bool { + // A kind exclusion does not take the policy off the namespace: other workload kinds there + // are still in scope. Image-only exclusions have no deployment scope, and a nil scope would + // otherwise match every namespace. + if exclusion.GetDeployment() == nil { + return false + } return m.scopeMatches(exclusion.GetDeployment().GetScope()) } func (m *namespaceMatcher) anyScopeMatches(scopes []*storage.Scope) bool { - if len(scopes) == 0 { - return true - } - - for _, scope := range scopes { - if m.scopeMatches(scope) { - return true - } - } - return false + return len(scopes) == 0 || slices.ContainsFunc(scopes, m.scopeMatches) } func (m *namespaceMatcher) scopeMatches(scope *storage.Scope) bool { - cs, err := scopecomp.CompileScope(scope) + cs, err := scopecomp.CompileScope(scope, nil, nil) if err != nil { utils.Should(errors.Wrap(err, "could not compiled scope")) return false diff --git a/central/policy/matcher/namespace_test.go b/central/policy/matcher/namespace_test.go index befa4cb4c15f3..23bc5edb1b26a 100644 --- a/central/policy/matcher/namespace_test.go +++ b/central/policy/matcher/namespace_test.go @@ -1,6 +1,7 @@ package matcher import ( + "context" "testing" "github.com/stackrox/rox/generated/storage" @@ -74,7 +75,7 @@ func TestNamespaceMatcher(t *testing.T) { } for _, c := range cases { - actual := NewNamespaceMatcher(c.namespace).IsPolicyApplicable(c.policy) + actual := NewNamespaceMatcher(c.namespace).IsPolicyApplicable(context.Background(), c.policy) assert.Equal(t, c.matches, actual) } } @@ -113,12 +114,11 @@ func TestNamespaceMatcherWithWhitespace(t *testing.T) { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Namespace: "ns.*", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Namespace: "ns.*", }, + }}, }, }, }, @@ -136,13 +136,12 @@ func TestNamespaceMatcherWithWhitespace(t *testing.T) { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Cluster: "cluster1", - Namespace: "ns1", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Cluster: "cluster1", + Namespace: "ns1", }, + }}, }, }, }, @@ -160,12 +159,11 @@ func TestNamespaceMatcherWithWhitespace(t *testing.T) { }, }, Exclusions: []*storage.Exclusion{ - { - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Cluster: "cluster1", - }, + {Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Cluster: "cluster1", }, + }}, }, }, }, @@ -174,7 +172,7 @@ func TestNamespaceMatcherWithWhitespace(t *testing.T) { } for _, c := range cases { - actual := NewNamespaceMatcher(c.namespace).IsPolicyApplicable(c.policy) + actual := NewNamespaceMatcher(c.namespace).IsPolicyApplicable(context.Background(), c.policy) assert.Equal(t, c.matches, actual) } } diff --git a/central/policy/service/service.go b/central/policy/service/service.go index 19fcdea16c9b1..d2251a4418acd 100644 --- a/central/policy/service/service.go +++ b/central/policy/service/service.go @@ -6,6 +6,7 @@ import ( clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" "github.com/stackrox/rox/central/detection/lifecycle" + namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" networkPolicyDS "github.com/stackrox/rox/central/networkpolicies/datastore" notifierDataStore "github.com/stackrox/rox/central/notifier/datastore" "github.com/stackrox/rox/central/policy/datastore" @@ -31,6 +32,7 @@ type Service interface { // New returns a new Service instance using the given DataStore. func New(policies datastore.DataStore, clusters clusterDataStore.DataStore, + namespaces namespaceDataStore.DataStore, deployments deploymentDataStore.DataStore, networkPolicies networkPolicyDS.DataStore, notifiers notifierDataStore.DataStore, @@ -45,6 +47,7 @@ func New(policies datastore.DataStore, return &serviceImpl{ policies: policies, clusters: clusters, + namespaces: namespaces, deployments: deployments, reprocessor: reprocessor, notifiers: notifiers, diff --git a/central/policy/service/service_impl.go b/central/policy/service/service_impl.go index a2c42f8c7ee55..a21a570ff9efb 100644 --- a/central/policy/service/service_impl.go +++ b/central/policy/service/service_impl.go @@ -12,6 +12,7 @@ import ( clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" "github.com/stackrox/rox/central/detection/lifecycle" + namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" networkPolicyDS "github.com/stackrox/rox/central/networkpolicies/datastore" notifierDataStore "github.com/stackrox/rox/central/notifier/datastore" "github.com/stackrox/rox/central/policy/datastore" @@ -28,9 +29,11 @@ import ( "github.com/stackrox/rox/pkg/booleanpolicy/policyversion" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/contextutil" - "github.com/stackrox/rox/pkg/detection" + accesscontrol "github.com/stackrox/rox/pkg/defaults/accesscontrol" + pkgDetection "github.com/stackrox/rox/pkg/detection" "github.com/stackrox/rox/pkg/errorhelpers" "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/grpc/authn" "github.com/stackrox/rox/pkg/grpc/authz" "github.com/stackrox/rox/pkg/grpc/authz/perrpc" @@ -105,6 +108,7 @@ type serviceImpl struct { policies datastore.DataStore clusters clusterDataStore.DataStore + namespaces namespaceDataStore.DataStore deployments deploymentDataStore.DataStore networkPolicies networkPolicyDS.DataStore notifiers notifierDataStore.DataStore @@ -154,13 +158,20 @@ func (s *serviceImpl) getPolicy(ctx context.Context, id string) (*storage.Policy if len(policy.GetCategories()) == 0 { policy.Categories = []string{uncategorizedCategory} } + stripEvaluationFilterIfDisabled(policy) return policy, nil } +func stripEvaluationFilterIfDisabled(p *storage.Policy) { + if !features.EvaluationFilter.Enabled() { + p.EvaluationFilter = nil + } +} + func convertPoliciesToListPolicies(policies []*storage.Policy) []*storage.ListPolicy { listPolicies := make([]*storage.ListPolicy, 0, len(policies)) for _, p := range policies { - listPolicies = append(listPolicies, &storage.ListPolicy{ + lp := &storage.ListPolicy{ Id: p.GetId(), Name: p.GetName(), Description: p.GetDescription(), @@ -172,7 +183,11 @@ func convertPoliciesToListPolicies(policies []*storage.Policy) []*storage.ListPo EventSource: p.GetEventSource(), IsDefault: p.GetIsDefault(), Source: p.GetSource(), - }) + } + if features.EvaluationFilter.Enabled() { + lp.EvaluationFilter = p.GetEvaluationFilter() + } + listPolicies = append(listPolicies, lp) } return listPolicies } @@ -336,6 +351,25 @@ func (s *serviceImpl) DeletePolicy(ctx context.Context, request *v1.ResourceByID return nil, errors.Wrap(errox.InvalidArgs, "A default policy cannot be deleted. (You can disable a default policy, but not delete it.)") } + // Declarative policies can only be deleted by the config-controller via its finalizer. + if policy.GetSource() == storage.PolicySource_DECLARATIVE { + identity, err := authn.IdentityFromContext(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to determine caller identity") + } + isConfigController := false + for _, role := range identity.Roles() { + if role.GetRoleName() == accesscontrol.ConfigController { + isConfigController = true + break + } + } + if !isConfigController { + return nil, errors.Wrap(errox.NotAuthorized, + "An externally managed policy can only be deleted by deleting the corresponding SecurityPolicy custom resource.") + } + } + if err := s.policies.RemovePolicy(ctx, policy); err != nil { return nil, err } @@ -443,7 +477,11 @@ func (s *serviceImpl) predicateBasedDryRunPolicy(ctx context.Context, cancelCtx return &resp, nil } - compiledPolicy, err := detection.CompilePolicy(request) + // Create providers for label-based scope matching + clusterProvider := s.clusters + namespaceProvider := s.namespaces + + compiledPolicy, err := pkgDetection.CompilePolicy(request, clusterProvider, namespaceProvider) if err != nil { return nil, errors.Wrapf(errox.InvalidArgs, "invalid policy: %v", err) } @@ -492,7 +530,7 @@ func (s *serviceImpl) predicateBasedDryRunPolicy(ctx context.Context, cancelCtx return } - if !compiledPolicy.AppliesTo(deployment) { + if !compiledPolicy.AppliesTo(ctx, deployment) { return } @@ -739,6 +777,7 @@ func (s *serviceImpl) ExportPolicies(ctx context.Context, request *v1.ExportPoli for _, policy := range policyList { removeInternal(policy) + stripEvaluationFilterIfDisabled(policy) } return &storage.ExportPoliciesResponse{ Policies: policyList, @@ -1099,9 +1138,9 @@ func combineStrings(toCombine [][]string) []string { for _, category := range toCombine { maxIterations *= len(category) } - for i := 0; i < maxIterations; i++ { + for range maxIterations { combination := "" - for i := 0; i < len(indices); i++ { + for i := range indices { if i > 0 { combination = combination + "=" } @@ -1109,7 +1148,7 @@ func combineStrings(toCombine [][]string) []string { } combinations = append(combinations, combination) - for index := len(indices) - 1; index >= 0; index-- { + for index := range slices.Backward(indices) { indices[index]++ if indices[index] < len(toCombine[index]) { break diff --git a/central/policy/service/service_impl_postgres_test.go b/central/policy/service/service_impl_postgres_test.go index 72df8a9d9f885..e7a001e679492 100644 --- a/central/policy/service/service_impl_postgres_test.go +++ b/central/policy/service/service_impl_postgres_test.go @@ -75,7 +75,7 @@ func (s *PolicyServicePostgresSuite) SetupSuite() { s.connectionManager = connectionMocks.NewMockManager(s.mockCtrl) - s.tested = New(s.policies, s.clusters, nil, nil, notifierDS, s.mitreVectorStore, nil, s.lifecycleManager, nil, nil, s.connectionManager) + s.tested = New(s.policies, s.clusters, nil, nil, nil, notifierDS, s.mitreVectorStore, nil, s.lifecycleManager, nil, nil, s.connectionManager) } // TestPostPolicy tests posting and then immediately after putting the same policy, as this discovered a bug in the diff --git a/central/policy/service/service_impl_test.go b/central/policy/service/service_impl_test.go index c3dc1207095f5..5524ee15a576b 100644 --- a/central/policy/service/service_impl_test.go +++ b/central/policy/service/service_impl_test.go @@ -3,22 +3,28 @@ package service import ( "context" _ "embed" + "errors" "fmt" "reflect" "strings" "testing" - "github.com/pkg/errors" + pkgErrors "github.com/pkg/errors" clusterMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" lifecycleMocks "github.com/stackrox/rox/central/detection/lifecycle/mocks" "github.com/stackrox/rox/central/policy/datastore/mocks" connectionMocks "github.com/stackrox/rox/central/sensor/service/connection/mocks" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/auth/permissions" + permissionsMocks "github.com/stackrox/rox/pkg/auth/permissions/mocks" "github.com/stackrox/rox/pkg/booleanpolicy/fieldnames" "github.com/stackrox/rox/pkg/booleanpolicy/policyversion" + accesscontrol "github.com/stackrox/rox/pkg/defaults/accesscontrol" "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/fixtures" + "github.com/stackrox/rox/pkg/grpc/authn" + authnMocks "github.com/stackrox/rox/pkg/grpc/authn/mocks" mitreMocks "github.com/stackrox/rox/pkg/mitre/datastore/mocks" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/search" @@ -68,14 +74,15 @@ func (s *PolicyServiceTestSuite) SetupTest() { s.tested = New( s.policies, s.clusters, - nil, - nil, - nil, + nil, // namespaces + nil, // deployments + nil, // networkPolicies + nil, // notifiers s.mitreVectorStore, - nil, + nil, // reprocessor s.mockLifecycleManager, - nil, - nil, + nil, // processor + nil, // metadataCache s.mockConnectionManager, ) } @@ -180,10 +187,9 @@ func (s *PolicyServiceTestSuite) TestExportedPolicyHasNoSortFields() { func (s *PolicyServiceTestSuite) TestPoliciesHaveNoUnexpectedSORTFields() { expectedSORTFields := set.NewStringSet("SORTLifecycleStage", "SORTEnforcement", "SORTName") - var policy *storage.Policy - policyType := reflect.TypeOf(policy).Elem() + policyType := reflect.TypeFor[storage.Policy]() numFields := policyType.NumField() - for i := 0; i < numFields; i++ { + for i := range numFields { fieldName := policyType.Field(i).Name if strings.HasPrefix(fieldName, "SORT") { s.Contains(expectedSORTFields, fieldName, "Found unexpected SORT field %s, SORT fields must be cleared in exported policies in removeInternal()", fieldName) @@ -234,7 +240,7 @@ func (s *PolicyServiceTestSuite) TestListPoliciesHandlesQueryAndPagination() { ctx := context.Background() basePolicy := fixtures.GetPolicy() policies := make([]*storage.Policy, 4) - for i := 0; i < 4; i++ { + for i := range 4 { p := basePolicy.CloneVT() p.Id = fmt.Sprintf("policy-%d", i) policies = append(policies, p) @@ -910,7 +916,7 @@ func (s *PolicyServiceTestSuite) TestDeletingDefaultPolicyIsBlocked() { IsDefault: true, } s.policies.EXPECT().GetPolicy(ctx, mockPolicy.GetId()).Return(mockPolicy, true, nil) - expectedErr := errors.Wrap(errox.InvalidArgs, "A default policy cannot be deleted. (You can disable a default policy, but not delete it.)") + expectedErr := pkgErrors.Wrap(errox.InvalidArgs, "A default policy cannot be deleted. (You can disable a default policy, but not delete it.)") // act fakeResourceByIDRequest := &v1.ResourceByID{Id: mockPolicy.GetId()} @@ -921,6 +927,60 @@ func (s *PolicyServiceTestSuite) TestDeletingDefaultPolicyIsBlocked() { s.Require().Nil(resp) } +func (s *PolicyServiceTestSuite) TestDeletingDeclarativePolicyIsBlocked() { + // arrange + mockRole := permissionsMocks.NewMockResolvedRole(s.mockCtrl) + mockRole.EXPECT().GetRoleName().Return(accesscontrol.Admin) + + mockIdentity := authnMocks.NewMockIdentity(s.mockCtrl) + mockIdentity.EXPECT().Roles().Return([]permissions.ResolvedRole{mockRole}) + ctx := authn.ContextWithIdentity(context.Background(), mockIdentity, s.T()) + + mockPolicy := &storage.Policy{ + Id: mockRequestOneID.GetPolicyIds()[0], + Source: storage.PolicySource_DECLARATIVE, + } + s.policies.EXPECT().GetPolicy(ctx, mockPolicy.GetId()).Return(mockPolicy, true, nil) + + // act + fakeResourceByIDRequest := &v1.ResourceByID{Id: mockPolicy.GetId()} + resp, err := s.tested.DeletePolicy(ctx, fakeResourceByIDRequest) + + // assert + s.Require().Error(err) + s.True(errors.Is(err, errox.NotAuthorized)) + s.Require().Nil(resp) + s.Contains(err.Error(), "externally managed policy") +} + +func (s *PolicyServiceTestSuite) TestDeletingDeclarativePolicyByConfigControllerSucceeds() { + // arrange + mockRole := permissionsMocks.NewMockResolvedRole(s.mockCtrl) + mockRole.EXPECT().GetRoleName().Return(accesscontrol.ConfigController) + + mockIdentity := authnMocks.NewMockIdentity(s.mockCtrl) + mockIdentity.EXPECT().Roles().Return([]permissions.ResolvedRole{mockRole}) + ctx := authn.ContextWithIdentity(context.Background(), mockIdentity, s.T()) + + mockPolicy := &storage.Policy{ + Id: mockRequestOneID.GetPolicyIds()[0], + Source: storage.PolicySource_DECLARATIVE, + } + s.policies.EXPECT().GetPolicy(ctx, mockPolicy.GetId()).Return(mockPolicy, true, nil) + s.policies.EXPECT().RemovePolicy(ctx, mockPolicy).Return(nil) + s.mockLifecycleManager.EXPECT().RemovePolicy(mockPolicy.GetId()).Return(nil) + s.policies.EXPECT().GetAllPolicies(gomock.Any()).Return(nil, nil) + s.mockConnectionManager.EXPECT().PreparePoliciesAndBroadcast(gomock.Any()) + + // act + fakeResourceByIDRequest := &v1.ResourceByID{Id: mockPolicy.GetId()} + resp, err := s.tested.DeletePolicy(ctx, fakeResourceByIDRequest) + + // assert + s.NoError(err) + s.NotNil(resp) +} + func (s *PolicyServiceTestSuite) TestDeletingNonExistentPolicyDoesNothing() { ctx := context.Background() @@ -942,9 +1002,9 @@ func (s *PolicyServiceTestSuite) TestDeletingPolicyErrOnDbError() { // arrange mockPolicyID := mockRequestOneID.GetPolicyIds()[0] // used only for the ID - dbErr := errors.New("the deebee has failed you") + dbErr := pkgErrors.New("the deebee has failed you") s.policies.EXPECT().GetPolicy(ctx, mockPolicyID).Return(nil, true, dbErr) - expectedErr := errors.Wrap(dbErr, "DB error while trying to delete policy") + expectedErr := pkgErrors.Wrap(dbErr, "DB error while trying to delete policy") // act fakeResourceByIDRequest := &v1.ResourceByID{Id: mockPolicyID} diff --git a/central/policy/service/singleton.go b/central/policy/service/singleton.go index 1b8b48bc2a8a6..f846d27a42b08 100644 --- a/central/policy/service/singleton.go +++ b/central/policy/service/singleton.go @@ -4,6 +4,7 @@ import ( clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" "github.com/stackrox/rox/central/detection/lifecycle" + namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" networkPolicyDS "github.com/stackrox/rox/central/networkpolicies/datastore" notifierDataStore "github.com/stackrox/rox/central/notifier/datastore" notifierProcessor "github.com/stackrox/rox/central/notifier/processor" @@ -24,6 +25,7 @@ var ( func initialize() { as = New(datastore.Singleton(), clusterDataStore.Singleton(), + namespaceDataStore.Singleton(), deploymentDataStore.Singleton(), networkPolicyDS.Singleton(), notifierDataStore.Singleton(), diff --git a/central/policy/service/validator.go b/central/policy/service/validator.go index de2af721b1128..38c93dec4bb9a 100644 --- a/central/policy/service/validator.go +++ b/central/policy/service/validator.go @@ -96,6 +96,7 @@ func (s *policyValidator) internalValidate(policy *storage.Policy, additionalVal errorList.AddError(s.validateCapabilities(policy)) errorList.AddError(s.validateEventSource(policy)) errorList.AddError(s.validateEnforcement(policy)) + errorList.AddError(s.validateEvaluationFilter(policy)) for _, validator := range additionalValidators { errorList.AddError(validator(policy)) @@ -285,7 +286,7 @@ func (s *policyValidator) validateExclusions(policy *storage.Policy) error { } func (s *policyValidator) validateExclusion(policy *storage.Policy, exclusion *storage.Exclusion) error { - if exclusion.GetDeployment() == nil && exclusion.GetImage() == nil { + if exclusion.GetDeployment() == nil && exclusion.GetImage() == nil && exclusion.GetExcludeByKind() == nil { return errors.New("all excluded scopes must have some criteria to match on") } if exclusion.GetDeployment() != nil { @@ -296,6 +297,17 @@ func (s *policyValidator) validateExclusion(policy *storage.Policy, exclusion *s return err } } + if exclusion.GetExcludeByKind() != nil { + if !features.PolicyWorkloadKindExclusion.Enabled() { + return errors.New("exclude_by_kind requires feature flag ROX_POLICY_WORKLOAD_KIND_EXCLUSION to be enabled") + } + if !policies.AppliesAtDeployTime(policy) && !policies.AppliesAtRunTime(policy) { + return errors.New("excluding by workload kind is only valid during the DEPLOY and RUNTIME lifecycles") + } + if err := validateExcludeByKind(exclusion.GetExcludeByKind()); err != nil { + return err + } + } if exclusion.GetImage() != nil { if !policies.AppliesAtBuildTime(policy) { return errors.New("excluding an image is only valid during the BUILD lifecycle") @@ -307,13 +319,38 @@ func (s *policyValidator) validateExclusion(policy *storage.Policy, exclusion *s return nil } +func validateExcludeByKind(excludeByKind *storage.Exclusion_ExcludeByKind) error { + kinds := excludeByKind.GetKinds() + if len(kinds) == 0 { + return errors.New("exclude_by_kind must specify at least one workload kind") + } + for _, kind := range kinds { + if kind == storage.Exclusion_WORKLOAD_KIND_UNSPECIFIED { + return errors.New("exclude_by_kind cannot include WORKLOAD_KIND_UNSPECIFIED") + } + if kind != storage.Exclusion_CRON_JOB && kind != storage.Exclusion_JOB { + return errors.Errorf("exclude_by_kind contains unknown workload kind %s", kind) + } + } + return nil +} + func (s *policyValidator) validateDeploymentExclusion(exclusion *storage.Exclusion) error { deployment := exclusion.GetDeployment() if deployment.GetScope() == nil && deployment.GetName() == "" { return errors.New("at least one field of deployment exclusion scope must be defined") } if deployment.GetScope() != nil { - if err := s.validateScope(deployment.GetScope()); err != nil { + // No feature flag check — exclusion scopes do not + // support cluster and namespace labels. + scope := deployment.GetScope() + if scope.GetClusterLabel() != nil { + return errors.New("exclusion scopes do not support cluster labels") + } + if scope.GetNamespaceLabel() != nil { + return errors.New("exclusion scopes do not support namespace labels") + } + if err := s.validateScope(scope); err != nil { return errors.Wrap(err, "deployment exclusion scope is invalid") } } @@ -321,10 +358,27 @@ func (s *policyValidator) validateDeploymentExclusion(exclusion *storage.Exclusi } func (s *policyValidator) validateScope(scope *storage.Scope) error { - if scope.GetCluster() == "" && scope.GetNamespace() == "" && scope.GetLabel() == nil { + if scope.GetCluster() == "" && scope.GetNamespace() == "" && scope.GetLabel() == nil && scope.GetClusterLabel() == nil && scope.GetNamespaceLabel() == nil { return errors.New("scope must have at least one field populated") } - if _, err := scopecomp.CompileScope(scope); err != nil { + // Reject cluster_label and namespace_label if feature flag is disabled + if !features.LabelBasedPolicyScoping.Enabled() { + if scope.GetClusterLabel() != nil { + return errors.New("cluster_label field requires feature flag ROX_LABEL_BASED_POLICY_SCOPING to be enabled") + } + if scope.GetNamespaceLabel() != nil { + return errors.New("namespace_label field requires feature flag ROX_LABEL_BASED_POLICY_SCOPING to be enabled") + } + } + // Cluster and cluster_label are mutually exclusive + if scope.GetCluster() != "" && scope.GetClusterLabel() != nil { + return errors.New("scope cannot have both 'cluster' and 'cluster_label' fields populated") + } + // Namespace and namespace_label are mutually exclusive + if scope.GetNamespace() != "" && scope.GetNamespaceLabel() != nil { + return errors.New("scope cannot have both 'namespace' and 'namespace_label' fields populated") + } + if _, err := scopecomp.CompileScope(scope, nil, nil); err != nil { return errors.Wrap(err, "could not compile scope") } return nil @@ -355,8 +409,8 @@ func (s *policyValidator) compilesForRunTime(policy *storage.Policy, options ... return errors.New("A runtime policy must contain at least one policy criterion from process, network flow, audit log events, or Kubernetes events criteria categories") } - if !booleanpolicy.ContainsDiscreteRuntimeFieldCategorySections(policy) { - return errors.New("A runtime policy section must contain only one criterion from process, network flow, audit log events, or Kubernetes events criteria categories") + if !booleanpolicy.ContainsValidRuntimeFieldCategorySections(policy) { + return errors.New("A runtime policy must not mix incompatible runtime criteria: process and file criteria must be in the same section, and cannot coexist with network flow or Kubernetes events criteria") } if err := s.validateNodeEventPolicy(policy); err != nil { @@ -457,6 +511,13 @@ func (s *policyValidator) validateNodeEventPolicy(policy *storage.Policy) error return errors.New("Node event policies must contain only node fields") } + for _, section := range policy.GetPolicySections() { + if booleanpolicy.SectionContainsFieldOfType(section, booleanpolicy.Process) && + !booleanpolicy.SectionContainsFieldOfType(section, booleanpolicy.FileAccess) { + return errors.New("Node event policies with process criteria must include file access criteria in the same section") + } + } + return nil } @@ -471,3 +532,20 @@ func (s *policyValidator) eventSourceError() error { return fmt.Errorf("event source must be %s for runtime policies", strings.Join(eventSources, " or ")) } + +func (s *policyValidator) validateEvaluationFilter(policy *storage.Policy) error { + if len(policy.GetEvaluationFilter().GetSkipContainerTypes()) == 0 { + return nil + } + if !containerTypeFilterApplicable(policy) { + return errors.New("container type filters apply only to deploy-time policies and runtime policies with a deployment event source") + } + return nil +} + +func containerTypeFilterApplicable(policy *storage.Policy) bool { + if policies.AppliesAtDeployTime(policy) { + return true + } + return policies.AppliesAtRunTime(policy) && policy.GetEventSource() == storage.EventSource_DEPLOYMENT_EVENT +} diff --git a/central/policy/service/validator_test.go b/central/policy/service/validator_test.go index 0b3a0e81a3f46..784601206355f 100644 --- a/central/policy/service/validator_test.go +++ b/central/policy/service/validator_test.go @@ -614,9 +614,7 @@ func (s *PolicyValidatorTestSuite) TestValidateExclusions() { deployment := &storage.Exclusion_Deployment{ Name: "that phat cluster", } - deploymentExclusion := &storage.Exclusion{ - Deployment: deployment, - } + deploymentExclusion := &storage.Exclusion{Matcher: &storage.Exclusion_Deployment_{Deployment: deployment}} policy = &storage.Policy{ LifecycleStages: []storage.LifecycleStage{ storage.LifecycleStage_DEPLOY, @@ -661,14 +659,13 @@ func (s *PolicyValidatorTestSuite) TestValidateExclusions() { err = s.validator.validateExclusions(policy) s.Error(err, "excluded scope requires either container or deployment configuration") - emptyLabelExclusion := &storage.Exclusion{ - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Label: &storage.Scope_Label{ - Key: "", - }, + emptyLabelExclusion := &storage.Exclusion{Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Label: &storage.Scope_Label{ + Key: "", }, }, + }}, } policy = &storage.Policy{ Exclusions: []*storage.Exclusion{ @@ -678,15 +675,14 @@ func (s *PolicyValidatorTestSuite) TestValidateExclusions() { err = s.validator.validateExclusions(policy) s.Error(err, "label regex in excluded scope, if not nil, must be non-empty") - anyKeyLabelExclusion := &storage.Exclusion{ - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Label: &storage.Scope_Label{ - Key: ".*", - Value: "", - }, + anyKeyLabelExclusion := &storage.Exclusion{Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Label: &storage.Scope_Label{ + Key: ".*", + Value: "", }, }, + }}, } policy = &storage.Policy{ LifecycleStages: []storage.LifecycleStage{ @@ -698,15 +694,14 @@ func (s *PolicyValidatorTestSuite) TestValidateExclusions() { } s.NoError(s.validator.validateExclusions(policy)) - anyLabelExclusion := &storage.Exclusion{ - Deployment: &storage.Exclusion_Deployment{ - Scope: &storage.Scope{ - Label: &storage.Scope_Label{ - Key: ".*", - Value: ".*", - }, + anyLabelExclusion := &storage.Exclusion{Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Label: &storage.Scope_Label{ + Key: ".*", + Value: ".*", }, }, + }}, } policy = &storage.Policy{ LifecycleStages: []storage.LifecycleStage{ @@ -719,6 +714,139 @@ func (s *PolicyValidatorTestSuite) TestValidateExclusions() { s.NoError(s.validator.validateExclusions(policy)) } +func (s *PolicyValidatorTestSuite) TestValidateExcludeByKind() { + kindExclusion := func(kinds ...storage.Exclusion_WorkloadKind) *storage.Exclusion { + return &storage.Exclusion{ + Matcher: &storage.Exclusion_ExcludeByKind_{ + ExcludeByKind: &storage.Exclusion_ExcludeByKind{Kinds: kinds}, + }, + } + } + deployPolicy := func(exclusion *storage.Exclusion) *storage.Policy { + return &storage.Policy{ + LifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_DEPLOY}, + Exclusions: []*storage.Exclusion{exclusion}, + } + } + + s.ErrorContains(s.validator.validateExclusions(deployPolicy(kindExclusion(storage.Exclusion_JOB))), "ROX_POLICY_WORKLOAD_KIND_EXCLUSION") + + testutils.MustUpdateFeature(s.T(), features.PolicyWorkloadKindExclusion, true) + + for name, tc := range map[string]struct { + policy *storage.Policy + errContains string + }{ + "JOB is valid": { + policy: deployPolicy(kindExclusion(storage.Exclusion_JOB)), + }, + "CRON_JOB is valid": { + policy: deployPolicy(kindExclusion(storage.Exclusion_CRON_JOB)), + }, + "JOB and CRON_JOB are valid": { + policy: deployPolicy(kindExclusion(storage.Exclusion_CRON_JOB, storage.Exclusion_JOB)), + }, + "empty kind list is rejected": { + policy: deployPolicy(kindExclusion()), + errContains: "at least one workload kind", + }, + "UNSPECIFIED is rejected": { + policy: deployPolicy(kindExclusion(storage.Exclusion_WORKLOAD_KIND_UNSPECIFIED)), + errContains: "WORKLOAD_KIND_UNSPECIFIED", + }, + "unknown kind is rejected": { + policy: deployPolicy(kindExclusion(storage.Exclusion_WorkloadKind(99))), + errContains: "unknown workload kind", + }, + "mixed valid and unknown kinds are rejected": { + policy: deployPolicy(kindExclusion(storage.Exclusion_JOB, storage.Exclusion_WorkloadKind(99))), + errContains: "unknown workload kind", + }, + "RUNTIME lifecycle is valid": { + policy: &storage.Policy{ + LifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_RUNTIME}, + Exclusions: []*storage.Exclusion{kindExclusion(storage.Exclusion_JOB)}, + }, + }, + "BUILD lifecycle is rejected": { + policy: &storage.Policy{ + LifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_BUILD}, + Exclusions: []*storage.Exclusion{kindExclusion(storage.Exclusion_JOB)}, + }, + errContains: "DEPLOY and RUNTIME", + }, + } { + s.T().Run(name, func(t *testing.T) { + err := s.validator.validateExclusions(tc.policy) + if tc.errContains == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tc.errContains) + } + }) + } +} + +func (s *PolicyValidatorTestSuite) TestValidateExclusionRejectsLabels() { + for name, tc := range map[string]struct { + exclusion *storage.Exclusion + errExpected bool + errContains string + }{ + "cluster_label on exclusion scope is rejected": { + exclusion: &storage.Exclusion{Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + ClusterLabel: &storage.Scope_Label{Key: "env", Value: "prod"}, + }, + }}, + }, + errExpected: true, + errContains: "cluster labels", + }, + "namespace_label on exclusion scope is rejected": { + exclusion: &storage.Exclusion{Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + NamespaceLabel: &storage.Scope_Label{Key: "team", Value: "backend"}, + }, + }}, + }, + errExpected: true, + errContains: "namespace labels", + }, + "deployment label on exclusion scope is allowed": { + exclusion: &storage.Exclusion{Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Label: &storage.Scope_Label{Key: "app", Value: "nginx"}, + }, + }}, + }, + errExpected: false, + }, + "cluster ID on exclusion scope is allowed": { + exclusion: &storage.Exclusion{Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ + Scope: &storage.Scope{ + Cluster: "cluster-1", + }, + }}, + }, + errExpected: false, + }, + } { + s.T().Run(name, func(t *testing.T) { + policy := &storage.Policy{ + LifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_DEPLOY}, + Exclusions: []*storage.Exclusion{tc.exclusion}, + } + err := s.validator.validateExclusions(policy) + if tc.errExpected { + assert.ErrorContains(t, err, tc.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + func (s *PolicyValidatorTestSuite) TestAllDefaultPoliciesValidate() { defaultPolicies, err := policies.DefaultPolicies() s.Require().NoError(err) @@ -992,37 +1120,29 @@ func (s *PolicyValidatorTestSuite) TestValidateEnforcement() { } } -func (s *PolicyValidatorTestSuite) TestValidateEffectivePathEventSource() { +func (s *PolicyValidatorTestSuite) TestValidateDeploymentFileActivityEventSource() { testCases := []struct { description string p *storage.Policy errExpected bool }{ { - description: "Deployment policy with valid Effective Path field", + description: "Deployment policy with valid File Path field", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_DEPLOYMENT_EVENT, map[string]string{ - fieldnames.EffectivePath: "/etc/passwd", + fieldnames.FilePath: "/etc/passwd", }), }, { - description: "Deployment policy with Effective Path and FileOperation", + description: "Deployment policy with File Path and File Operation", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_DEPLOYMENT_EVENT, map[string]string{ - fieldnames.EffectivePath: "/etc/shadow", + fieldnames.FilePath: "/etc/shadow", fieldnames.FileOperation: "open", }), }, { - description: "Node policy with Effective Path (should be invalid)", - p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_NODE_EVENT, - map[string]string{ - fieldnames.EffectivePath: "/etc/passwd", - }), - errExpected: true, - }, - { - description: "Deployment policy with FileOperation but no file path", + description: "Deployment policy with File Operation but no file path", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_DEPLOYMENT_EVENT, map[string]string{ fieldnames.FileOperation: "open", @@ -1030,26 +1150,26 @@ func (s *PolicyValidatorTestSuite) TestValidateEffectivePathEventSource() { errExpected: true, }, { - description: "Deployment policy with invalid Effective Path", + description: "Deployment policy with invalid File Path", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_DEPLOYMENT_EVENT, map[string]string{ - fieldnames.EffectivePath: "relative/path.sh", + fieldnames.FilePath: "relative/path.sh", }), errExpected: true, }, { - description: "Deployment policy with Effective Path in wrong lifecycle stage (build)", + description: "Deployment policy with File Path in wrong lifecycle stage (build)", p: booleanPolicyWithFields(storage.LifecycleStage_BUILD, storage.EventSource_DEPLOYMENT_EVENT, map[string]string{ - fieldnames.EffectivePath: "/etc/hosts", + fieldnames.FilePath: "/etc/hosts", }), errExpected: true, }, { - description: "Deployment policy with Effective Path in wrong lifecycle stage (deploy)", + description: "Deployment policy with File Path in wrong lifecycle stage (deploy)", p: booleanPolicyWithFields(storage.LifecycleStage_DEPLOY, storage.EventSource_DEPLOYMENT_EVENT, map[string]string{ - fieldnames.EffectivePath: "/etc/passwd", + fieldnames.FilePath: "/etc/passwd", }), errExpected: true, }, @@ -1084,10 +1204,10 @@ func (s *PolicyValidatorTestSuite) TestValidateNodeEventSource() { errExpected bool }{ { - description: "Node policy with valid Actual Path field", + description: "Node policy with valid File Path field", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_NODE_EVENT, map[string]string{ - fieldnames.ActualPath: "/etc/passwd", + fieldnames.FilePath: "/etc/passwd", }), }, { @@ -1123,49 +1243,48 @@ func (s *PolicyValidatorTestSuite) TestValidateNodeEventSource() { errExpected: true, }, { - description: "Node policy with Actual Path and invalid process fields", + description: "Node policy with File Path and invalid process fields", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_NODE_EVENT, map[string]string{ - fieldnames.ActualPath: "/var/log/audit.log", + fieldnames.FilePath: "/var/log/audit.log", fieldnames.ProcessName: "suspicious-binary", }), - errExpected: true, }, { - description: "Node policy with Actual Path and invalid container fields", + description: "Node policy with File Path and invalid container fields", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_NODE_EVENT, map[string]string{ - fieldnames.ActualPath: "/etc/shadow", + fieldnames.FilePath: "/etc/shadow", fieldnames.ContainerName: "malicious-container", }), errExpected: true, }, { - description: "Node policy with Actual Path in wrong lifecycle stage (build)", + description: "Node policy with File Path in wrong lifecycle stage (build)", p: booleanPolicyWithFields(storage.LifecycleStage_BUILD, storage.EventSource_NODE_EVENT, map[string]string{ - fieldnames.ActualPath: "/etc/hosts", + fieldnames.FilePath: "/etc/hosts", }), errExpected: true, }, { - description: "Node policy with Actual Path in wrong lifecycle stage (deploy)", + description: "Node policy with File Path in wrong lifecycle stage (deploy)", p: booleanPolicyWithFields(storage.LifecycleStage_DEPLOY, storage.EventSource_NODE_EVENT, map[string]string{ - fieldnames.ActualPath: "/tmp/malicious.sh", + fieldnames.FilePath: "/tmp/malicious.sh", }), errExpected: true, }, { - description: "Node policy invalid Actual Path", + description: "Node policy invalid File Path", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_NODE_EVENT, map[string]string{ - fieldnames.ActualPath: "relative/path.sh", + fieldnames.FilePath: "relative/path.sh", }), errExpected: true, }, { - description: "Node policy with valid FileOperation field but no file path", + description: "Node policy with valid File Operation field but no File Path", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_NODE_EVENT, map[string]string{ fieldnames.FileOperation: "open", @@ -1173,15 +1292,15 @@ func (s *PolicyValidatorTestSuite) TestValidateNodeEventSource() { errExpected: true, }, { - description: "Node policy with FileOperation and valid Actual Path field", + description: "Node policy with File Operation and valid File Path field", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_NODE_EVENT, map[string]string{ fieldnames.FileOperation: "open", - fieldnames.ActualPath: "/etc/passwd", + fieldnames.FilePath: "/etc/passwd", }), }, { - description: "Node policy with invalid FileOperation", + description: "Node policy with invalid File Operation", p: booleanPolicyWithFields(storage.LifecycleStage_RUNTIME, storage.EventSource_NODE_EVENT, map[string]string{ fieldnames.FileOperation: "execute", @@ -1189,7 +1308,7 @@ func (s *PolicyValidatorTestSuite) TestValidateNodeEventSource() { errExpected: true, }, { - description: "Node policy with FileOperation in wrong lifecycle stage (build)", + description: "Node policy with File Operation in wrong lifecycle stage (build)", p: booleanPolicyWithFields(storage.LifecycleStage_BUILD, storage.EventSource_NODE_EVENT, map[string]string{ fieldnames.FileOperation: "open", @@ -1197,7 +1316,7 @@ func (s *PolicyValidatorTestSuite) TestValidateNodeEventSource() { errExpected: true, }, { - description: "Node policy with FileOperation in wrong lifecycle stage (deploy)", + description: "Node policy with File Operation in wrong lifecycle stage (deploy)", p: booleanPolicyWithFields(storage.LifecycleStage_DEPLOY, storage.EventSource_NODE_EVENT, map[string]string{ fieldnames.FileOperation: "open", @@ -1227,3 +1346,177 @@ func (s *PolicyValidatorTestSuite) TestValidateNodeEventSource() { }) } } + +func (s *PolicyValidatorTestSuite) TestValidateScope() { + testutils.MustUpdateFeature(s.T(), features.LabelBasedPolicyScoping, true) + defer testutils.MustUpdateFeature(s.T(), features.LabelBasedPolicyScoping, false) + + testCases := []struct { + description string + scope *storage.Scope + errExpected bool + }{ + { + description: "cluster and cluster_label are mutually exclusive", + scope: &storage.Scope{ + Cluster: "cluster1", + ClusterLabel: &storage.Scope_Label{ + Key: "env", + Value: "prod", + }, + }, + errExpected: true, + }, + { + description: "namespace and namespace_label are mutually exclusive", + scope: &storage.Scope{ + Namespace: "default", + NamespaceLabel: &storage.Scope_Label{ + Key: "team", + Value: "backend", + }, + }, + errExpected: true, + }, + { + description: "cluster_label alone is valid", + scope: &storage.Scope{ + ClusterLabel: &storage.Scope_Label{ + Key: "env", + Value: "prod", + }, + }, + errExpected: false, + }, + { + description: "namespace_label alone is valid", + scope: &storage.Scope{ + NamespaceLabel: &storage.Scope_Label{ + Key: "team", + Value: "backend", + }, + }, + errExpected: false, + }, + { + description: "cluster and namespace_label together is valid", + scope: &storage.Scope{ + Cluster: "cluster1", + NamespaceLabel: &storage.Scope_Label{ + Key: "team", + Value: "backend", + }, + }, + errExpected: false, + }, + { + description: "cluster_label and namespace together is valid", + scope: &storage.Scope{ + ClusterLabel: &storage.Scope_Label{ + Key: "env", + Value: "prod", + }, + Namespace: "default", + }, + errExpected: false, + }, + { + description: "empty scope is invalid", + scope: &storage.Scope{}, + errExpected: true, + }, + } + + for _, tc := range testCases { + s.T().Run(tc.description, func(t *testing.T) { + err := s.validator.validateScope(tc.scope) + if tc.errExpected { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func (s *PolicyValidatorTestSuite) TestValidateEvaluationFilter() { + tests := map[string]struct { + lifecycleStages []storage.LifecycleStage + eventSource storage.EventSource + skipContainerTypes []storage.ContainerType + expectError bool + }{ + "build-only with skip init is rejected": { + lifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_BUILD}, + skipContainerTypes: []storage.ContainerType{storage.ContainerType_INIT}, + expectError: true, + }, + "deploy with skip init is allowed": { + lifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_DEPLOY}, + skipContainerTypes: []storage.ContainerType{storage.ContainerType_INIT}, + }, + "deploy with skip regular is allowed": { + lifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_DEPLOY}, + skipContainerTypes: []storage.ContainerType{storage.ContainerType_REGULAR}, + }, + "build + deploy with skip init is allowed": { + lifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_BUILD, storage.LifecycleStage_DEPLOY}, + skipContainerTypes: []storage.ContainerType{storage.ContainerType_INIT}, + }, + "build-only with no filter is allowed": { + lifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_BUILD}, + }, + "runtime without event source with skip init is rejected": { + lifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_RUNTIME}, + skipContainerTypes: []storage.ContainerType{storage.ContainerType_INIT}, + expectError: true, + }, + "no lifecycle stages with skip init is rejected": { + skipContainerTypes: []storage.ContainerType{storage.ContainerType_INIT}, + expectError: true, + }, + "audit log event with skip init is rejected": { + lifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_RUNTIME}, + eventSource: storage.EventSource_AUDIT_LOG_EVENT, + skipContainerTypes: []storage.ContainerType{storage.ContainerType_INIT}, + expectError: true, + }, + "node event with skip init is rejected": { + lifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_RUNTIME}, + eventSource: storage.EventSource_NODE_EVENT, + skipContainerTypes: []storage.ContainerType{storage.ContainerType_INIT}, + expectError: true, + }, + "deployment event with skip init is allowed": { + lifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_RUNTIME}, + eventSource: storage.EventSource_DEPLOYMENT_EVENT, + skipContainerTypes: []storage.ContainerType{storage.ContainerType_INIT}, + }, + "deployment event with skip regular is allowed": { + lifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_RUNTIME}, + eventSource: storage.EventSource_DEPLOYMENT_EVENT, + skipContainerTypes: []storage.ContainerType{storage.ContainerType_REGULAR}, + }, + } + + for name, tc := range tests { + s.Run(name, func() { + policy := &storage.Policy{ + LifecycleStages: tc.lifecycleStages, + EventSource: tc.eventSource, + } + if tc.skipContainerTypes != nil { + policy.EvaluationFilter = &storage.EvaluationFilter{ + SkipContainerTypes: tc.skipContainerTypes, + } + } + + err := s.validator.validateEvaluationFilter(policy) + if tc.expectError { + s.Error(err) + } else { + s.NoError(err) + } + }) + } +} diff --git a/central/policycategory/datastore/datastore_impl.go b/central/policycategory/datastore/datastore_impl.go index 8023a9f730f32..9f7bead1f7c9e 100644 --- a/central/policycategory/datastore/datastore_impl.go +++ b/central/policycategory/datastore/datastore_impl.go @@ -335,7 +335,3 @@ func (c *PolicyCategorySearchResultConverter) BuildLocation(result *searchPkg.Re func (c *PolicyCategorySearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_POLICY_CATEGORIES } - -func (c *PolicyCategorySearchResultConverter) GetScore(result *searchPkg.Result) float64 { - return result.Score -} diff --git a/central/policycategoryedge/datastore/datastore_impl.go b/central/policycategoryedge/datastore/datastore_impl.go index 273d2e481da2a..aa2b9efd6bea5 100644 --- a/central/policycategoryedge/datastore/datastore_impl.go +++ b/central/policycategoryedge/datastore/datastore_impl.go @@ -162,7 +162,3 @@ func (c *PolicyCategoryEdgeSearchResultConverter) BuildLocation(result *searchPk func (c *PolicyCategoryEdgeSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_POLICY_CATEGORY_EDGE } - -func (c *PolicyCategoryEdgeSearchResultConverter) GetScore(result *searchPkg.Result) float64 { - return result.Score -} diff --git a/central/postgres/pruning.go b/central/postgres/pruning.go index 6e02b194b9874..2bb614e068468 100644 --- a/central/postgres/pruning.go +++ b/central/postgres/pruning.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/jackc/pgx/v5" "github.com/pkg/errors" "github.com/stackrox/rox/pkg/contextutil" "github.com/stackrox/rox/pkg/env" @@ -28,6 +29,9 @@ const ( getAllOrphanedNodes = `SELECT id FROM nodes WHERE NOT EXISTS (SELECT 1 FROM clusters WHERE nodes.clusterid = clusters.Id)` + getAllOrphanedDeployments = `SELECT id, clusterid FROM deployments WHERE NOT EXISTS + (SELECT 1 FROM clusters WHERE deployments.clusterid = clusters.Id)` + getOrphanedProcessesByDeployment = `SELECT id FROM process_indicators pi WHERE NOT EXISTS (SELECT 1 FROM deployments WHERE pi.deploymentid = deployments.Id) AND (signal_time < now() AT time zone 'utc' - INTERVAL '%d MINUTES' OR signal_time IS NULL)` @@ -120,6 +124,14 @@ const ( revoked = TRUE OR expiration < now() at time zone 'utc' - INTERVAL '%d MINUTES' )` // #nosec G101 + + inactiveImageIdentifiersQuery = `SELECT img.id, img.digest, img.name_fullname + FROM images_v2 img + WHERE img.lastupdated < now() AT TIME ZONE 'utc' - INTERVAL '%d DAYS' + AND NOT EXISTS ( + SELECT 1 FROM deployments_containers dc + WHERE dc.image_idv2 = img.id + )` ) var ( @@ -180,6 +192,37 @@ func GetOrphanedNodeIDs(ctx context.Context, pool postgres.DB) ([]string, error) }) } +// OrphanedDeployment holds a deployment id with its clusterid +type OrphanedDeployment struct { + ID string + ClusterID string +} + +// GetOrphanedDeploymentIDs returns the deployments that have a cluster that has been removed. +func GetOrphanedDeploymentIDs(ctx context.Context, pool postgres.DB) ([]OrphanedDeployment, error) { + return pgutils.Retry2(ctx, func() ([]OrphanedDeployment, error) { + ctx, cancel := context.WithTimeout(ctx, orphanedQueryTimeout) + defer cancel() + + rows, err := pool.Query(ctx, getAllOrphanedDeployments) + if err != nil { + return nil, errors.Wrap(err, "failed to get orphaned deployments") + } + return pgx.CollectRows(rows, func(r pgx.CollectableRow) (OrphanedDeployment, error) { + var d OrphanedDeployment + // clusterid is nullable; scan into a pointer so a NULL does not abort the pruning cycle. + var clusterID *string + if err := r.Scan(&d.ID, &clusterID); err != nil { + return OrphanedDeployment{}, errors.Wrap(err, "scanning orphaned deployment") + } + if clusterID != nil { + d.ClusterID = *clusterID + } + return d, nil + }) + }) +} + // GetOrphanedProcessIDsByDeployment returns the process ids that have a deployment that has been removed. func GetOrphanedProcessIDsByDeployment(ctx context.Context, pool postgres.DB, orphanWindow time.Duration) ([]string, error) { return pgutils.Retry2(ctx, func() ([]string, error) { @@ -269,3 +312,32 @@ func PruneInvalidAPITokens(ctx context.Context, pool postgres.DB, retentionDurat log.Errorf("failed to prune invalid api tokens: %v", err) } } + +// ImageIdentifier holds the id, digest, and full name of an image. +type ImageIdentifier struct { + ID string + Digest string + FullName string +} + +// GetInactiveImageIdentifiers returns images that are older than retentionDays and have no +// deployment container referencing them. +func GetInactiveImageIdentifiers(ctx context.Context, pool postgres.DB, retentionDays int) ([]*ImageIdentifier, error) { + return pgutils.Retry2(ctx, func() ([]*ImageIdentifier, error) { + ctx, cancel := context.WithTimeout(ctx, orphanedQueryTimeout) + defer cancel() + + query := fmt.Sprintf(inactiveImageIdentifiersQuery, retentionDays) + rows, err := pool.Query(ctx, query) + if err != nil { + return nil, errors.Wrap(err, "failed to get inactive image identifiers") + } + return pgx.CollectRows(rows, func(row pgx.CollectableRow) (*ImageIdentifier, error) { + var img ImageIdentifier + if err := row.Scan(&img.ID, &img.Digest, &img.FullName); err != nil { + return nil, errors.Wrap(err, "scanning image identifier") + } + return &img, nil + }) + }) +} diff --git a/central/postgres/pruning_test.go b/central/postgres/pruning_test.go index 9a919ff80521d..ade6dd5c22796 100644 --- a/central/postgres/pruning_test.go +++ b/central/postgres/pruning_test.go @@ -233,6 +233,68 @@ func (s *PostgresPruningSuite) TestGetOrphanedPodIDs() { s.Equal(len(idsToPrune), cluster2PodCount) } +func (s *PostgresPruningSuite) TestGetOrphanedDeploymentIDs() { + deploymentDS, err := deploymentStore.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(err) + + clusterDS, err := clusterStore.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(err) + + liveClusterID, err := clusterDS.AddCluster(s.ctx, &storage.Cluster{Name: "testOrphanDeploymentCluster", MainImage: "docker.io/stackrox/rox:latest"}) + s.Require().NoError(err) + + // Deployment tied to a live cluster - never orphaned. + liveDeploymentID := uuid.NewV4().String() + s.Require().NoError(deploymentDS.UpsertDeployment(s.ctx, &storage.Deployment{Id: liveDeploymentID, ClusterId: liveClusterID})) + + // Deployments whose clusterid has no matching cluster row - orphaned. This mirrors + // the field scenario where a cluster is hard-deleted but its deployments survive. + orphanClusterID := fixtureconsts.Cluster1 + orphanedIDs := set.NewStringSet() + for range 3 { + id := uuid.NewV4().String() + orphanedIDs.Add(id) + s.Require().NoError(deploymentDS.UpsertDeployment(s.ctx, &storage.Deployment{Id: id, ClusterId: orphanClusterID})) + } + + orphaned, err := GetOrphanedDeploymentIDs(s.ctx, s.testDB.DB) + s.Require().NoError(err) + + got := set.NewStringSet() + for _, d := range orphaned { + got.Add(d.ID) + s.Equal(orphanClusterID, d.ClusterID) + } + s.Equal(orphanedIDs, got) + s.NotContains(got.AsSlice(), liveDeploymentID) +} + +// A deployment with a NULL clusterid must not abort the sweep; it is orphaned (no matching +// cluster row) and must be returned with an empty ClusterID. +func (s *PostgresPruningSuite) TestGetOrphanedDeploymentIDsHandlesNullClusterID() { + deploymentDS, err := deploymentStore.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(err) + + // Upsert with a non-matching cluster id, then null out the collumn, to verify + // the pruning works in the unlikely case deployments without clusterid made it to the DB + nullClusterDeploymentID := uuid.NewV4().String() + s.Require().NoError(deploymentDS.UpsertDeployment(s.ctx, &storage.Deployment{Id: nullClusterDeploymentID, ClusterId: fixtureconsts.Cluster1})) + _, err = s.testDB.DB.Exec(s.ctx, "UPDATE deployments SET clusterid = NULL WHERE id = $1", nullClusterDeploymentID) + s.Require().NoError(err) + + orphaned, err := GetOrphanedDeploymentIDs(s.ctx, s.testDB.DB) + s.Require().NoError(err) + + got := set.NewStringSet() + for _, d := range orphaned { + got.Add(d.ID) + if d.ID == nullClusterDeploymentID { + s.Empty(d.ClusterID) + } + } + s.Contains(got.AsSlice(), nullClusterDeploymentID) +} + func (s *PostgresPruningSuite) TestRemoveOrphanedProcesses() { cases := []struct { name string @@ -344,12 +406,12 @@ func (s *PostgresPruningSuite) TestRemoveOrphanedProcesses() { // Add deployments if necessary deploymentDS, err := deploymentStore.GetTestPostgresDataStore(s.T(), s.testDB.DB) s.Require().NoError(err) - for _, deploymentID := range c.deployments.AsSlice() { + for deploymentID := range c.deployments.All() { s.Require().NoError(deploymentDS.UpsertDeployment(s.ctx, &storage.Deployment{Id: deploymentID, ClusterId: fixtureconsts.Cluster1})) } podDS := podStore.GetTestPostgresDataStore(s.T(), s.testDB.DB) - for _, podID := range c.pods.AsSlice() { + for podID := range c.pods.All() { err := podDS.UpsertPod(s.ctx, &storage.Pod{Id: podID, ClusterId: fixtureconsts.Cluster1}) s.Require().NoError(err) } @@ -373,13 +435,13 @@ func (s *PostgresPruningSuite) TestRemoveOrphanedProcesses() { for _, process := range c.initialProcesses { cleanupIDs = append(cleanupIDs, process.GetId()) } - s.Require().NoError(processDatastore.RemoveProcessIndicators(s.ctx, cleanupIDs)) + s.Require().NoError(processDatastore.RemoveProcessIndicators(s.ctx, cleanupIDs, processIndicatorDatastore.RemovalReasonProcessFilter)) - for _, deploymentID := range c.deployments.AsSlice() { + for deploymentID := range c.deployments.All() { s.Require().NoError(deploymentDS.RemoveDeployment(s.ctx, fixtureconsts.Cluster1, deploymentID)) } - for _, podID := range c.pods.AsSlice() { + for podID := range c.pods.All() { s.Require().NoError(podDS.RemovePod(s.ctx, podID)) } }) @@ -511,7 +573,7 @@ func (s *PostgresPruningSuite) TestPruneAPITokens() { // Helper functions. func (s *PostgresPruningSuite) addSomePods(podDS podStore.DataStore, clusterID string, numberPods int) { - for i := 0; i < numberPods; i++ { + for range numberPods { pod := &storage.Pod{ Id: uuid.NewV4().String(), ClusterId: clusterID, diff --git a/central/processbaseline/baselineutils.go b/central/processbaseline/baselineutils.go index be87e245f8f0a..ac19c2f628574 100644 --- a/central/processbaseline/baselineutils.go +++ b/central/processbaseline/baselineutils.go @@ -92,9 +92,7 @@ func IsStartupProcessView(process *views.ProcessIndicatorRiskView) bool { return false } // TODO(ROX-31107): Determine if nil SignalTime should be considered startup task. By this logic it is. - durationBetweenProcessAndContainerStart := protoutils.Sub(protocompat.ConvertTimeToTimestampOrNil(process.SignalTime), - protocompat.ConvertTimeToTimestampOrNil(process.ContainerStartTime)) - return durationBetweenProcessAndContainerStart < ContainerStartupDuration + return process.SignalTime == nil || process.SignalTime.Sub(*process.ContainerStartTime) < ContainerStartupDuration } // BaselineItemFromProcessView returns what we baseline for a given process. diff --git a/central/processbaseline/datastore/datastore_bench_test.go b/central/processbaseline/datastore/datastore_bench_test.go index e4fba9e36d94e..279a651d2c108 100644 --- a/central/processbaseline/datastore/datastore_bench_test.go +++ b/central/processbaseline/datastore/datastore_bench_test.go @@ -74,7 +74,7 @@ func generateProcessBaselines(numBaselines int, elementsPerBaseline int) []*stor // Create baselines with varied characteristics // Each deployment can have multiple containers - for i := 0; i < numBaselines; i++ { + for i := range numBaselines { deploymentID := deploymentIDs[i%len(deploymentIDs)] clusterID := clusterIDs[i%len(clusterIDs)] namespace := fmt.Sprintf("namespace-%d", i%20) // 20 different namespaces @@ -89,7 +89,7 @@ func generateProcessBaselines(numBaselines int, elementsPerBaseline int) []*stor // Create elements for this baseline elements := make([]*storage.BaselineElement, 0, elementsPerBaseline) - for j := 0; j < elementsPerBaseline; j++ { + for j := range elementsPerBaseline { // Mix common processes with unique ones var processName string if j < len(processTemplates) { diff --git a/central/processbaseline/datastore/datastore_impl.go b/central/processbaseline/datastore/datastore_impl.go index 2fdb1f36c99be..c97a9629cea26 100644 --- a/central/processbaseline/datastore/datastore_impl.go +++ b/central/processbaseline/datastore/datastore_impl.go @@ -367,21 +367,26 @@ func (ds *datastoreImpl) CreateUnlockedProcessBaseline(ctx context.Context, key return baseline, nil } - // Get the list of processes - baselineList, err := ds.getProcessList(ctx, key) - if err != nil { - return nil, err - } - // Build the de-duped elements for the baseline - elements := make(map[string]*storage.BaselineItem, len(baselineList)) + elements := make(map[string]*storage.BaselineItem) - for _, indicator := range baselineList { + fn := func(indicator *storage.ProcessIndicator) error { baselineItem := processBaselinePkg.BaselineItemFromProcess(indicator) insertableElement := &storage.BaselineItem{Item: &storage.BaselineItem_ProcessName{ProcessName: baselineItem}} elements[baselineItem] = insertableElement + return nil + } + + query := pkgSearch.NewQueryBuilder(). + AddExactMatches(pkgSearch.DeploymentID, key.GetDeploymentId()). + AddExactMatches(pkgSearch.ContainerName, key.GetContainerName()). + ProtoQuery() + + err = ds.processesDataStore.GetByQueryFn(ctx, query, fn) + if err != nil { + return nil, err } baseElements := make([]*storage.BaselineElement, 0, len(elements)) @@ -405,17 +410,6 @@ func (ds *datastoreImpl) CreateUnlockedProcessBaseline(ctx context.Context, key return baseline, err } -func (ds *datastoreImpl) getProcessList(ctx context.Context, key *storage.ProcessBaselineKey) ([]*storage.ProcessIndicator, error) { - // Simply use search to find the process indicators for the baseline key - return ds.processesDataStore.SearchRawProcessIndicators( - ctx, - pkgSearch.NewQueryBuilder(). - AddExactMatches(pkgSearch.DeploymentID, key.GetDeploymentId()). - AddExactMatches(pkgSearch.ContainerName, key.GetContainerName()). - ProtoQuery(), - ) -} - func (ds *datastoreImpl) WalkAll(ctx context.Context, fn func(baseline *storage.ProcessBaseline) error) error { if ok, err := deploymentExtensionSAC.ReadAllowed(ctx); err != nil { return err diff --git a/central/processbaseline/datastore/datastore_impl_test.go b/central/processbaseline/datastore/datastore_impl_test.go index 429fb991ab018..5ffedccd19d44 100644 --- a/central/processbaseline/datastore/datastore_impl_test.go +++ b/central/processbaseline/datastore/datastore_impl_test.go @@ -342,7 +342,15 @@ func (suite *ProcessBaselineDataStoreTestSuite) TestBuildUnlockedProcessBaseline }, } - suite.indicatorMockStore.EXPECT().SearchRawProcessIndicators(suite.requestContext, gomock.Any()).Return(indicators, nil) + suite.indicatorMockStore.EXPECT().GetByQueryFn(suite.requestContext, gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, query *v1.Query, fn func(*storage.ProcessIndicator) error) error { + for _, indicator := range indicators { + if err := fn(indicator); err != nil { + return err + } + } + return nil + }) baseline, err := suite.datastore.CreateUnlockedProcessBaseline(suite.requestContext, key) suite.NoError(err) @@ -395,7 +403,15 @@ func (suite *ProcessBaselineDataStoreTestSuite) TestBuildUnlockedProcessBaseline }, } - suite.indicatorMockStore.EXPECT().SearchRawProcessIndicators(suite.requestContext, gomock.Any()).Return(indicators, nil) + suite.indicatorMockStore.EXPECT().GetByQueryFn(suite.requestContext, gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, query *v1.Query, fn func(*storage.ProcessIndicator) error) error { + for _, indicator := range indicators { + if err := fn(indicator); err != nil { + return err + } + } + return nil + }) baseline, err := suite.datastore.CreateUnlockedProcessBaseline(suite.requestContext, key) suite.NoError(err) @@ -411,7 +427,7 @@ func (suite *ProcessBaselineDataStoreTestSuite) TestBuildUnlockedProcessBaseline func (suite *ProcessBaselineDataStoreTestSuite) TestBuildUnlockedProcessBaselineNoProcesses() { key := fixtures.GetBaselineKey() - suite.indicatorMockStore.EXPECT().SearchRawProcessIndicators(suite.requestContext, gomock.Any()) + suite.indicatorMockStore.EXPECT().GetByQueryFn(suite.requestContext, gomock.Any(), gomock.Any()).Return(nil) baseline, err := suite.datastore.CreateUnlockedProcessBaseline(suite.requestContext, key) suite.NoError(err) diff --git a/central/processbaseline/evaluator/bench_test.go b/central/processbaseline/evaluator/bench_test.go index 6bf7a054c8f8d..3c35fbfe6de87 100644 --- a/central/processbaseline/evaluator/bench_test.go +++ b/central/processbaseline/evaluator/bench_test.go @@ -71,7 +71,7 @@ func generateProcessIndicators(numProcesses int, deploymentID string, containers containerStartTimes[containerName] = protoconv.MustConvertTimeToTimestamp(startTime) } - for i := 0; i < numProcesses; i++ { + for i := range numProcesses { template := processTemplates[i%len(processTemplates)] containerName := containers[i%len(containers)] @@ -227,7 +227,7 @@ func BenchmarkEvaluateBaselinesAndPersistResult(b *testing.B) { wg := sync.WaitGroup{} const parallelism = 10 wg.Add(parallelism) - for j := 0; j < parallelism; j++ { + for range parallelism { go func() { defer wg.Done() violatingProcesses, err := evaluator.EvaluateBaselinesAndPersistResult(deployment) diff --git a/central/processbaseline/evaluator/evaluator_integration_test.go b/central/processbaseline/evaluator/evaluator_integration_test.go index b206509330c25..9be1b58294d6e 100644 --- a/central/processbaseline/evaluator/evaluator_integration_test.go +++ b/central/processbaseline/evaluator/evaluator_integration_test.go @@ -13,7 +13,6 @@ import ( "github.com/stackrox/rox/central/processindicator/views" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/fixtures" - "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protoconv" @@ -24,8 +23,6 @@ import ( "github.com/stretchr/testify/suite" ) -var log = logging.LoggerForModule() - func TestProcessBaselineEvaluatorIntegration(t *testing.T) { suite.Run(t, new(ProcessBaselineEvaluatorIntegrationTestSuite)) } @@ -195,8 +192,6 @@ func (suite *ProcessBaselineEvaluatorIntegrationTestSuite) TestLockedProcessBase suite.NoError(err) suite.NotNil(persistedResult) suite.Len(persistedResult.GetBaselineStatuses(), 2) - log.Infof("SHREWS -- %v", persistedResult) - for _, status := range persistedResult.GetBaselineStatuses() { // We only locked the first container if status.GetContainerName() == containerName { @@ -761,7 +756,7 @@ func (suite *ProcessBaselineEvaluatorIntegrationTestSuite) TestComplexWorkflow() suite.True(persistedResult.GetBaselineStatuses()[0].GetAnomalousProcessesExecuted()) // Remove the anomalous process - err = suite.indicatorsDatastore.RemoveProcessIndicators(suite.ctx, []string{processIndicator.GetId()}) + err = suite.indicatorsDatastore.RemoveProcessIndicators(suite.ctx, []string{processIndicator.GetId()}, indicatorDatastore.RemovalReasonProcessFilter) suite.NoError(err) // Should now show no anomalous processes diff --git a/central/processbaseline/service/service_impl.go b/central/processbaseline/service/service_impl.go index 2a5428c142fe3..28703a66714af 100644 --- a/central/processbaseline/service/service_impl.go +++ b/central/processbaseline/service/service_impl.go @@ -299,7 +299,7 @@ func (s *serviceImpl) DeleteProcessBaselines(ctx context.Context, request *v1.De } func (s *serviceImpl) reprocessUpdatedBaselines(resp **v1.UpdateProcessBaselinesResponse) { - var keys []*storage.ProcessBaselineKey + keys := make([]*storage.ProcessBaselineKey, 0, len((*resp).GetBaselines())) for _, pb := range (*resp).GetBaselines() { keys = append(keys, pb.GetKey()) } diff --git a/central/processbaseline/service/service_impl_test.go b/central/processbaseline/service/service_impl_test.go index e669be56433c6..f4f66ee66bffc 100644 --- a/central/processbaseline/service/service_impl_test.go +++ b/central/processbaseline/service/service_impl_test.go @@ -197,7 +197,15 @@ func (suite *ProcessBaselineServiceTestSuite) TestGetLoadProcessBaseline() { requestByKey := &v1.GetProcessBaselineRequest{Key: knownBaseline.GetKey()} suite.deployments.EXPECT().GetDeployment(hasWriteCtx, gomock.Any()).Return(nil, true, nil) - suite.indicatorMockStore.EXPECT().SearchRawProcessIndicators(hasWriteCtx, gomock.Any()).Return(indicators, nil) + suite.indicatorMockStore.EXPECT().GetByQueryFn(hasWriteCtx, gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, query *v1.Query, fn func(*storage.ProcessIndicator) error) error { + for _, indicator := range indicators { + if err := fn(indicator); err != nil { + return err + } + } + return nil + }) baseline, _ := suite.service.GetProcessBaseline(hasWriteCtx, requestByKey) diff --git a/central/processindicator/datastore/bench_test.go b/central/processindicator/datastore/bench_test.go index f1a63a93633cc..c419fb0acc159 100644 --- a/central/processindicator/datastore/bench_test.go +++ b/central/processindicator/datastore/bench_test.go @@ -19,7 +19,7 @@ import ( func BenchmarkAddIndicator(b *testing.B) { var indicators []*storage.ProcessIndicator - for i := 0; i < 100000; i++ { + for range 100000 { pi := fixtures.GetProcessIndicator() pi.Id = uuid.NewV4().String() indicators = append(indicators, pi) @@ -32,8 +32,8 @@ func BenchmarkAddIndicator(b *testing.B) { datastore := New(db, store, plopStore, nil) ctx := sac.WithAllAccess(context.Background()) - b.ResetTimer() - for i := 0; i < b.N; i++ { + + for b.Loop() { err := datastore.AddProcessIndicators(ctx, indicators...) require.NoError(b, err) } @@ -61,7 +61,7 @@ func BenchmarkProcessIndicators(b *testing.B) { // D1PodID1: 3,025 (55%) // D1PodID2: 1,375 (25%) // D1PodID3: 1,100 (20%) - for i := 0; i < 5500; i++ { + for i := range 5500 { pi := fixtures.GetProcessIndicator() pi.Id = uuid.NewV4().String() pi.DeploymentId = fixtureconsts.Deployment1 @@ -79,7 +79,7 @@ func BenchmarkProcessIndicators(b *testing.B) { // D2PodID1: 1,375 (55%) // D2PodID2: 625 (25%) // D2PodID3: 500 (20%) - for i := 0; i < 2500; i++ { + for i := range 2500 { pi := fixtures.GetProcessIndicator() pi.Id = uuid.NewV4().String() pi.DeploymentId = fixtureconsts.Deployment2 @@ -97,7 +97,7 @@ func BenchmarkProcessIndicators(b *testing.B) { // D3PodID1: 1,100 (55%) // D3PodID2: 500 (25%) // D3PodID3: 400 (20%) - for i := 0; i < 2000; i++ { + for i := range 2000 { pi := fixtures.GetProcessIndicator() pi.Id = uuid.NewV4().String() pi.DeploymentId = fixtureconsts.Deployment3 @@ -131,8 +131,7 @@ func BenchmarkProcessIndicators(b *testing.B) { // Search benchmarks - non-destructive, can run multiple times b.Run("Search/ByDeployment1", func(b *testing.B) { query := search.NewQueryBuilder().AddExactMatches(search.DeploymentID, fixtureconsts.Deployment1).ProtoQuery() - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { results, err := datastore.SearchRawProcessIndicators(ctx, query) require.NoError(b, err) require.True(b, len(results) > 0) @@ -141,8 +140,7 @@ func BenchmarkProcessIndicators(b *testing.B) { b.Run("Search/ByDeployment2", func(b *testing.B) { query := search.NewQueryBuilder().AddExactMatches(search.DeploymentID, fixtureconsts.Deployment2).ProtoQuery() - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { results, err := datastore.SearchRawProcessIndicators(ctx, query) require.NoError(b, err) require.True(b, len(results) > 0) @@ -151,8 +149,7 @@ func BenchmarkProcessIndicators(b *testing.B) { b.Run("Search/ByDeployment3", func(b *testing.B) { query := search.NewQueryBuilder().AddExactMatches(search.DeploymentID, fixtureconsts.Deployment3).ProtoQuery() - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { results, err := datastore.SearchRawProcessIndicators(ctx, query) require.NoError(b, err) require.True(b, len(results) > 0) @@ -161,8 +158,7 @@ func BenchmarkProcessIndicators(b *testing.B) { b.Run("Search/ByD1PodID1", func(b *testing.B) { query := search.NewQueryBuilder().AddExactMatches(search.PodUID, d1PodID1).ProtoQuery() - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { results, err := datastore.SearchRawProcessIndicators(ctx, query) require.NoError(b, err) require.True(b, len(results) > 0) @@ -171,8 +167,7 @@ func BenchmarkProcessIndicators(b *testing.B) { b.Run("Search/ByD2PodID1", func(b *testing.B) { query := search.NewQueryBuilder().AddExactMatches(search.PodUID, d2PodID1).ProtoQuery() - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { results, err := datastore.SearchRawProcessIndicators(ctx, query) require.NoError(b, err) require.True(b, len(results) > 0) @@ -181,8 +176,7 @@ func BenchmarkProcessIndicators(b *testing.B) { b.Run("Search/ByD3PodID1", func(b *testing.B) { query := search.NewQueryBuilder().AddExactMatches(search.PodUID, d3PodID1).ProtoQuery() - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { results, err := datastore.SearchRawProcessIndicators(ctx, query) require.NoError(b, err) require.True(b, len(results) > 0) @@ -217,8 +211,7 @@ func BenchmarkProcessIndicators(b *testing.B) { } b.Run("Delete/ByDeployment1", func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { b.StopTimer() // Re-add before each iteration err := datastore.AddProcessIndicators(ctx, d1Results...) @@ -226,14 +219,13 @@ func BenchmarkProcessIndicators(b *testing.B) { b.StartTimer() // Delete - err = datastore.RemoveProcessIndicators(ctx, d1DeleteIDs) + err = datastore.RemoveProcessIndicators(ctx, d1DeleteIDs, RemovalReasonProcessFilter) require.NoError(b, err) } }) b.Run("Delete/ByD1PodID2", func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { b.StopTimer() // Re-add before each iteration err := datastore.AddProcessIndicators(ctx, d1PodID2Results...) @@ -241,7 +233,7 @@ func BenchmarkProcessIndicators(b *testing.B) { b.StartTimer() // Delete - err = datastore.RemoveProcessIndicators(ctx, d1PodID2DeleteIDs) + err = datastore.RemoveProcessIndicators(ctx, d1PodID2DeleteIDs, RemovalReasonProcessFilter) require.NoError(b, err) } }) diff --git a/central/processindicator/datastore/datastore.go b/central/processindicator/datastore/datastore.go index 516cb08758030..ef05845d12224 100644 --- a/central/processindicator/datastore/datastore.go +++ b/central/processindicator/datastore/datastore.go @@ -27,13 +27,14 @@ type DataStore interface { Search(ctx context.Context, q *v1.Query) ([]pkgSearch.Result, error) SearchRawProcessIndicators(ctx context.Context, q *v1.Query) ([]*storage.ProcessIndicator, error) + GetByQueryFn(ctx context.Context, query *v1.Query, fn func(obj *storage.ProcessIndicator) error) error GetProcessIndicator(ctx context.Context, id string) (*storage.ProcessIndicator, bool, error) GetProcessIndicators(ctx context.Context, ids []string) ([]*storage.ProcessIndicator, bool, error) AddProcessIndicators(context.Context, ...*storage.ProcessIndicator) error RemoveProcessIndicatorsByPod(ctx context.Context, id string) error - RemoveProcessIndicators(ctx context.Context, ids []string) error - PruneProcessIndicators(ctx context.Context, ids []string) (int, error) + RemoveProcessIndicators(ctx context.Context, ids []string, reason string) error + PruneProcessIndicators(ctx context.Context, ids []string, reason string) (int, error) WalkByQuery(ctx context.Context, query *v1.Query, fn func(obj *storage.ProcessIndicator) error) error diff --git a/central/processindicator/datastore/datastore_impl.go b/central/processindicator/datastore/datastore_impl.go index 887a4294d34d2..ed9d1757df857 100644 --- a/central/processindicator/datastore/datastore_impl.go +++ b/central/processindicator/datastore/datastore_impl.go @@ -60,6 +60,10 @@ func (ds *datastoreImpl) SearchRawProcessIndicators(ctx context.Context, q *v1.Q return ds.storage.GetByQuery(ctx, q) } +func (ds *datastoreImpl) GetByQueryFn(ctx context.Context, query *v1.Query, fn func(obj *storage.ProcessIndicator) error) error { + return ds.storage.GetByQueryFn(ctx, query, fn) +} + func (ds *datastoreImpl) GetProcessIndicator(ctx context.Context, id string) (*storage.ProcessIndicator, bool, error) { indicator, exists, err := ds.storage.Get(ctx, id) if err != nil || !exists { @@ -98,6 +102,7 @@ func (ds *datastoreImpl) AddProcessIndicators(ctx context.Context, indicators .. return err } } else { + recordProcessIndicatorsBatchAdded(identifierBatch) log.Debugf("successfully added a batch of %d process indicators", len(identifierBatch)) } } @@ -109,7 +114,7 @@ func (ds *datastoreImpl) WalkByQuery(ctx context.Context, q *v1.Query, fn func(p return ds.storage.WalkByQuery(ctx, q, fn) } -func (ds *datastoreImpl) RemoveProcessIndicators(ctx context.Context, ids []string) error { +func (ds *datastoreImpl) RemoveProcessIndicators(ctx context.Context, ids []string, reason string) error { if len(ids) == 0 { return nil } @@ -117,14 +122,15 @@ func (ds *datastoreImpl) RemoveProcessIndicators(ctx context.Context, ids []stri return err } + recordProcessIndicatorsRemoved(len(ids), reason) return nil } -func (ds *datastoreImpl) PruneProcessIndicators(ctx context.Context, ids []string) (int, error) { - return ds.pruneIndicators(ctx, ids), nil +func (ds *datastoreImpl) PruneProcessIndicators(ctx context.Context, ids []string, reason string) (int, error) { + return ds.pruneIndicators(ctx, ids, reason), nil } -func (ds *datastoreImpl) pruneIndicators(ctx context.Context, ids []string) int { +func (ds *datastoreImpl) pruneIndicators(ctx context.Context, ids []string, reason string) int { // Previously this used removeIndicators and would call "DeleteMany". The issue // with that is "DeleteMany" wraps the entire delete into a transaction making it an // all or nothing proposition. For pruning, if a batch fails it shouldn't fail them all. @@ -165,12 +171,22 @@ func (ds *datastoreImpl) pruneIndicators(ctx context.Context, ids []string) int } log.Infof("successfully pruned %d out of %d indicators", successfullyPruned, initialSize) + recordProcessIndicatorsRemoved(successfullyPruned, reason) return successfullyPruned } func (ds *datastoreImpl) RemoveProcessIndicatorsByPod(ctx context.Context, id string) error { q := pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.PodUID, id).ProtoQuery() - return ds.storage.DeleteByQuery(ctx, q) + + deletedIDs, err := ds.storage.DeleteByQueryWithIDs(ctx, q) + if err != nil { + return err + } + + if len(deletedIDs) > 0 { + recordProcessIndicatorsRemoved(len(deletedIDs), RemovalReasonPodDeletion) + } + return nil } // IterateOverProcessIndicatorsRiskView iterates over minimal fields from process indicator for risk evaluation @@ -259,8 +275,7 @@ func (ds *datastoreImpl) prune(ctx context.Context) { idsToRemove := pruner.Prune(args) var successfullyPruned int if len(idsToRemove) > 0 { - successfullyPruned = ds.pruneIndicators(ctx, idsToRemove) - incrementPrunedProcessesMetric(successfullyPruned) + successfullyPruned = ds.pruneIndicators(ctx, idsToRemove, PruneReasonSimilarity) } ds.prunedArgsLengthCache[processInfo] = numArgsReceived - successfullyPruned } diff --git a/central/processindicator/datastore/datastore_impl_test.go b/central/processindicator/datastore/datastore_impl_test.go index b24a8efe634e6..3e586cbe3b977 100644 --- a/central/processindicator/datastore/datastore_impl_test.go +++ b/central/processindicator/datastore/datastore_impl_test.go @@ -204,7 +204,7 @@ func (suite *IndicatorDataStoreTestSuite) TestIndicatorAddOneByOne() { } func (suite *IndicatorDataStoreTestSuite) generateIndicatorsWithPods(podIDs []string, containerIDs []string) []*storage.ProcessIndicator { - var indicators []*storage.ProcessIndicator + indicators := make([]*storage.ProcessIndicator, 0, len(containerIDs)*len(podIDs)) for _, p := range podIDs { for _, c := range containerIDs { indicators = append(indicators, &storage.ProcessIndicator{ @@ -265,7 +265,7 @@ func (suite *IndicatorDataStoreTestSuite) TestIndicatorRemovalBatch() { } // Try to remove where pod id does not exist in indicators - suite.NoError(suite.datastore.RemoveProcessIndicators(suite.hasWriteCtx, ids)) + suite.NoError(suite.datastore.RemoveProcessIndicators(suite.hasWriteCtx, ids, RemovalReasonProcessFilter)) suite.verifyIndicatorsAre(indicators[0]) } @@ -335,13 +335,15 @@ func (suite *IndicatorDataStoreTestSuite) TestPruning() { suite.NoError(suite.datastore.AddProcessIndicators(suite.hasWriteCtx, extraIndicator)) // Allow the next prune to go through; this time, prune something. - expectedIndicators := []*storage.ProcessIndicator{extraIndicator} + expectedIndicators := make([]*storage.ProcessIndicator, 0, 1+len(indicators)) + expectedIndicators = append(expectedIndicators, extraIndicator) expectedIndicators = append(expectedIndicators, indicators...) mockPruner.EXPECT().Prune(matcher(expectedIndicators...)).Return([]string{indicators[0].GetId()}) prunedSignal.Reset() pruneTurnstile.AllowOne() - suite.True(concurrency.WaitWithTimeout(&prunedSignal, 3*prunePeriod)) - expectedIndicators = []*storage.ProcessIndicator{extraIndicator} + suite.True(concurrency.WaitWithTimeout(&prunedSignal, 5*prunePeriod)) + expectedIndicators = make([]*storage.ProcessIndicator, 0, len(indicators)) + expectedIndicators = append(expectedIndicators, extraIndicator) expectedIndicators = append(expectedIndicators, indicators[1:]...) suite.verifyIndicatorsAre(expectedIndicators...) @@ -414,7 +416,7 @@ func (suite *IndicatorDataStoreTestSuite) TestAllowsAddMany() { func (suite *IndicatorDataStoreTestSuite) TestAllowsRemoveByPod() { storeMock := suite.setupDataStoreWithMocks() - storeMock.EXPECT().DeleteByQuery(gomock.Any(), gomock.Any()).Return(nil) + storeMock.EXPECT().DeleteByQueryWithIDs(gomock.Any(), gomock.Any()).Return([]string{}, nil) err := suite.datastore.RemoveProcessIndicatorsByPod(suite.hasWriteCtx, uuid.NewDummy().String()) suite.NoError(err, "expected no error trying to write with permissions") @@ -435,7 +437,7 @@ func (suite *IndicatorDataStoreTestSuite) TestIndicatorPruneBatch() { ids := suite.buildIDsToPrune(batchSize) // Try to remove indicators by id - indicatorCount, err := suite.datastore.PruneProcessIndicators(suite.hasWriteCtx, ids) + indicatorCount, err := suite.datastore.PruneProcessIndicators(suite.hasWriteCtx, ids, PruneReasonSimilarity) suite.Require().NoError(err) suite.Require().Equal(batchSize, indicatorCount) } @@ -449,7 +451,7 @@ func (suite *IndicatorDataStoreTestSuite) buildIDsToPrune(count int) []string { ids := make([]string, 0, count) pruneIndicators := make([]*storage.ProcessIndicator, 0, count) - for i := 0; i < count; i++ { + for range count { id := uuid.NewV4().String() newIndicator := indicators[0].CloneVT() newIndicator.Id = id diff --git a/central/processindicator/datastore/datastore_sac_test.go b/central/processindicator/datastore/datastore_sac_test.go index fe1fed6d45184..3e285158a74c6 100644 --- a/central/processindicator/datastore/datastore_sac_test.go +++ b/central/processindicator/datastore/datastore_sac_test.go @@ -67,13 +67,13 @@ func (s *processIndicatorDatastoreSACSuite) SetupTest() { func (s *processIndicatorDatastoreSACSuite) TearDownTest() { err := s.datastore.RemoveProcessIndicators(s.testContexts[sacTestUtils.UnrestrictedReadWriteCtx], - s.testProcessIndicatorIDs) + s.testProcessIndicatorIDs, RemovalReasonProcessFilter) s.Require().NoError(err) } func (s *processIndicatorDatastoreSACSuite) deleteProcessIndicator(id string) { s.Require().NoError(s.datastore.RemoveProcessIndicators(s.testContexts[sacTestUtils.UnrestrictedReadWriteCtx], - []string{id})) + []string{id}, RemovalReasonProcessFilter)) } func (s *processIndicatorDatastoreSACSuite) TestAddProcessIndicators() { @@ -162,7 +162,7 @@ func (s *processIndicatorDatastoreSACSuite) TestRemoveProcessIndicators() { s.Require().NoError(err) defer s.deleteProcessIndicator(processIndicator.GetId()) - err = s.datastore.RemoveProcessIndicators(ctx, []string{processIndicator.GetId()}) + err = s.datastore.RemoveProcessIndicators(ctx, []string{processIndicator.GetId()}, RemovalReasonProcessFilter) s.NoError(err) fetchedProcess, found, err := s.datastore.GetProcessIndicator( @@ -195,7 +195,7 @@ func (s *processIndicatorDatastoreSACSuite) TestPruneProcessIndicators() { s.Require().NoError(err) defer s.deleteProcessIndicator(processIndicator.GetId()) - prunedCount, err := s.datastore.PruneProcessIndicators(ctx, []string{processIndicator.GetId()}) + prunedCount, err := s.datastore.PruneProcessIndicators(ctx, []string{processIndicator.GetId()}, PruneReasonSimilarity) s.NoError(err) fetchedProcess, found, err := s.datastore.GetProcessIndicator( diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index 601c2ed978273..4a7b2b0eb7e37 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -2,17 +2,25 @@ package datastore import ( "github.com/prometheus/client_golang/prometheus" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/metrics" ) -var ( - prunedProcesses = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.CentralSubsystem.String(), - Name: "pruned_process_indicators", - Help: "Number of process indicators removed by pruning", - }) +const ( + // PruneReasonSimilarity represents pruning based on Jaccard similarity algorithm + PruneReasonSimilarity = "similarity" + // PruneReasonOrphanedByDeployment represents pruning of indicators orphaned by deleted deployments + PruneReasonOrphanedByDeployment = "orphaned_deployment" + // PruneReasonOrphanedByPod represents pruning of indicators orphaned by deleted pods + PruneReasonOrphanedByPod = "orphaned_pod" + // RemovalReasonProcessFilter represents removal during process filter cleanup + RemovalReasonProcessFilter = "process_filter" + // RemovalReasonPodDeletion represents removal when a pod is deleted + RemovalReasonPodDeletion = "pod_deletion" +) + +var ( processPruningCacheHits = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), @@ -26,10 +34,106 @@ var ( Name: "process_pruning_cache_misses", Help: "Number of times we miss the cache, and have to evaluate, when trying to prune processes", }) + + processUpsertedArgsSizeHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_upserted_args_size", + Help: "Distribution of process argument sizes in bytes for upserted indicators", + Buckets: []float64{0, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, + }) + + processUpsertedArgsSizeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_upserted_args_size_total", + Help: "Total upserted process argument sizes in bytes by cluster and namespace", + }, []string{"cluster", "namespace"}) + + processUpsertedCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_upserted_count", + Help: "Number of process indicators upserted by cluster and namespace", + }, []string{"cluster", "namespace"}) + + processIndicatorsRemoved = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_indicators_removed", + Help: "Number of process indicators removed from the database, broken down by reason", + }, []string{"reason"}) + + processIndicatorsRemovedTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_indicators_removed_total", + Help: "Total number of process indicators removed from the database across all reasons", + }) + + processUpsertedLineageSizeHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_upserted_lineage_size", + Help: "Distribution of process lineage sizes in bytes for upserted indicators", + Buckets: []float64{0, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, + }) + + processUpsertedLineageSizeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_upserted_lineage_size_total", + Help: "Total upserted process lineage sizes in bytes by cluster and namespace", + }, []string{"cluster", "namespace"}) + + processNotPersistedTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_indicators_not_persisted", + Help: "Number of process indicators filtered out and not persisted", + }) + + processNotPersistedArgsSizeHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_not_persisted_args_size", + Help: "Distribution of process argument sizes in bytes for indicators not persisted", + Buckets: []float64{0, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, + }) + + processNotPersistedArgsSizeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_not_persisted_args_size_total", + Help: "Total not-persisted process argument sizes in bytes by cluster and namespace", + }, []string{"cluster", "namespace"}) + + processNotPersistedCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_not_persisted_count", + Help: "Number of process indicators not persisted by cluster and namespace", + }, []string{"cluster", "namespace"}) + + processNotPersistedLineageSizeHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_not_persisted_lineage_size", + Help: "Distribution of process lineage sizes in bytes for indicators not persisted", + Buckets: []float64{0, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, + }) + + processNotPersistedLineageSizeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_not_persisted_lineage_size_total", + Help: "Total not-persisted process lineage sizes in bytes by cluster and namespace", + }, []string{"cluster", "namespace"}) ) -func incrementPrunedProcessesMetric(num int) { - prunedProcesses.Add(float64(num)) +func recordProcessIndicatorsRemoved(num int, reason string) { + processIndicatorsRemoved.WithLabelValues(reason).Add(float64(num)) + processIndicatorsRemovedTotal.Add(float64(num)) } func incrementProcessPruningCacheHitsMetrics() { @@ -40,10 +144,80 @@ func incrementProcessPruningCacheMissesMetric() { processPruningCacheMisses.Inc() } +func getProcessArgsSizeBytes(indicator *storage.ProcessIndicator) int { + if indicator == nil || indicator.GetSignal() == nil { + return 0 + } + return len(indicator.GetSignal().GetArgs()) +} + +func getProcessLineageSizeBytes(indicator *storage.ProcessIndicator) int { + if indicator == nil || indicator.GetSignal() == nil { + return 0 + } + + lineageInfo := indicator.GetSignal().GetLineageInfo() + if len(lineageInfo) == 0 { + return 0 + } + + totalBytes := 0 + for _, info := range lineageInfo { + if info != nil { + totalBytes += len(info.GetParentExecFilePath()) + } + } + + return totalBytes +} + +// recordProcessIndicatorsBatchAdded records metrics for a batch of process indicators successfully written to DB. +func recordProcessIndicatorsBatchAdded(indicators []*storage.ProcessIndicator) { + for _, indicator := range indicators { + argsSizeBytes := getProcessArgsSizeBytes(indicator) + lineageSizeBytes := getProcessLineageSizeBytes(indicator) + clusterID := indicator.GetClusterId() + namespace := indicator.GetNamespace() + + processUpsertedArgsSizeHistogram.Observe(float64(argsSizeBytes)) + processUpsertedArgsSizeTotal.WithLabelValues(clusterID, namespace).Add(float64(argsSizeBytes)) + processUpsertedCount.WithLabelValues(clusterID, namespace).Inc() + processUpsertedLineageSizeHistogram.Observe(float64(lineageSizeBytes)) + processUpsertedLineageSizeTotal.WithLabelValues(clusterID, namespace).Add(float64(lineageSizeBytes)) + } +} + +// RecordProcessIndicatorNotPersisted records metrics for a process indicator that was filtered out and not persisted. +func RecordProcessIndicatorNotPersisted(indicator *storage.ProcessIndicator) { + argsSizeBytes := getProcessArgsSizeBytes(indicator) + lineageSizeBytes := getProcessLineageSizeBytes(indicator) + clusterID := indicator.GetClusterId() + namespace := indicator.GetNamespace() + + processNotPersistedTotal.Inc() + processNotPersistedArgsSizeHistogram.Observe(float64(argsSizeBytes)) + processNotPersistedArgsSizeTotal.WithLabelValues(clusterID, namespace).Add(float64(argsSizeBytes)) + processNotPersistedCount.WithLabelValues(clusterID, namespace).Inc() + processNotPersistedLineageSizeHistogram.Observe(float64(lineageSizeBytes)) + processNotPersistedLineageSizeTotal.WithLabelValues(clusterID, namespace).Add(float64(lineageSizeBytes)) +} + func init() { prometheus.MustRegister( - prunedProcesses, processPruningCacheHits, processPruningCacheMisses, + processUpsertedArgsSizeHistogram, + processUpsertedArgsSizeTotal, + processUpsertedCount, + processIndicatorsRemoved, + processIndicatorsRemovedTotal, + processUpsertedLineageSizeHistogram, + processUpsertedLineageSizeTotal, + processNotPersistedTotal, + processNotPersistedArgsSizeHistogram, + processNotPersistedArgsSizeTotal, + processNotPersistedCount, + processNotPersistedLineageSizeHistogram, + processNotPersistedLineageSizeTotal, ) } diff --git a/central/processindicator/datastore/mocks/datastore.go b/central/processindicator/datastore/mocks/datastore.go index 55b8c9f491b6f..ea793c75545f9 100644 --- a/central/processindicator/datastore/mocks/datastore.go +++ b/central/processindicator/datastore/mocks/datastore.go @@ -79,6 +79,20 @@ func (mr *MockDataStoreMockRecorder) Count(ctx, q any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockDataStore)(nil).Count), ctx, q) } +// GetByQueryFn mocks base method. +func (m *MockDataStore) GetByQueryFn(ctx context.Context, query *v1.Query, fn func(*storage.ProcessIndicator) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetByQueryFn", ctx, query, fn) + ret0, _ := ret[0].(error) + return ret0 +} + +// GetByQueryFn indicates an expected call of GetByQueryFn. +func (mr *MockDataStoreMockRecorder) GetByQueryFn(ctx, query, fn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByQueryFn", reflect.TypeOf((*MockDataStore)(nil).GetByQueryFn), ctx, query, fn) +} + // GetProcessIndicator mocks base method. func (m *MockDataStore) GetProcessIndicator(ctx context.Context, id string) (*storage.ProcessIndicator, bool, error) { m.ctrl.T.Helper() @@ -126,32 +140,32 @@ func (mr *MockDataStoreMockRecorder) IterateOverProcessIndicatorsRiskView(ctx, q } // PruneProcessIndicators mocks base method. -func (m *MockDataStore) PruneProcessIndicators(ctx context.Context, ids []string) (int, error) { +func (m *MockDataStore) PruneProcessIndicators(ctx context.Context, ids []string, reason string) (int, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "PruneProcessIndicators", ctx, ids) + ret := m.ctrl.Call(m, "PruneProcessIndicators", ctx, ids, reason) ret0, _ := ret[0].(int) ret1, _ := ret[1].(error) return ret0, ret1 } // PruneProcessIndicators indicates an expected call of PruneProcessIndicators. -func (mr *MockDataStoreMockRecorder) PruneProcessIndicators(ctx, ids any) *gomock.Call { +func (mr *MockDataStoreMockRecorder) PruneProcessIndicators(ctx, ids, reason any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PruneProcessIndicators", reflect.TypeOf((*MockDataStore)(nil).PruneProcessIndicators), ctx, ids) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PruneProcessIndicators", reflect.TypeOf((*MockDataStore)(nil).PruneProcessIndicators), ctx, ids, reason) } // RemoveProcessIndicators mocks base method. -func (m *MockDataStore) RemoveProcessIndicators(ctx context.Context, ids []string) error { +func (m *MockDataStore) RemoveProcessIndicators(ctx context.Context, ids []string, reason string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RemoveProcessIndicators", ctx, ids) + ret := m.ctrl.Call(m, "RemoveProcessIndicators", ctx, ids, reason) ret0, _ := ret[0].(error) return ret0 } // RemoveProcessIndicators indicates an expected call of RemoveProcessIndicators. -func (mr *MockDataStoreMockRecorder) RemoveProcessIndicators(ctx, ids any) *gomock.Call { +func (mr *MockDataStoreMockRecorder) RemoveProcessIndicators(ctx, ids, reason any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveProcessIndicators", reflect.TypeOf((*MockDataStore)(nil).RemoveProcessIndicators), ctx, ids) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveProcessIndicators", reflect.TypeOf((*MockDataStore)(nil).RemoveProcessIndicators), ctx, ids, reason) } // RemoveProcessIndicatorsByPod mocks base method. diff --git a/central/processindicator/filter/singleton.go b/central/processindicator/filter/singleton.go index 706620942dcbe..8af5434a3b062 100644 --- a/central/processindicator/filter/singleton.go +++ b/central/processindicator/filter/singleton.go @@ -1,6 +1,8 @@ package filter import ( + "fmt" + "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/process/filter" @@ -18,14 +20,21 @@ var ( // Singleton returns a global, threadsafe process filter func Singleton() filter.Filter { singletonInstance.Do(func() { - maxExactPathMatches := env.ProcessFilterMaxExactPathMatches.IntegerSetting() - maxUniqueProcesses := env.ProcessFilterMaxProcessPaths.IntegerSetting() - fanOutLevels, fanOutLevelsWarning := env.ProcessFilterFanOutLevels.IntegerArraySetting() + // Get effective configuration respecting both mode presets and individual overrides + config, mode, err := env.GetEffectiveProcessFilterConfig() + + if err != nil { + log.Warn(err) + } - if fanOutLevelsWarning != "" { - log.Warn(fanOutLevelsWarning) + modeStr := "" + if mode != "" { + modeStr = fmt.Sprintf("mode=%s, ", mode) } - singletonFilter = filter.NewFilter(maxExactPathMatches, maxUniqueProcesses, fanOutLevels) + log.Infof("Process filter configuration: %smaxExactPathMatches=%d, fanOutLevels=%v, maxProcessPaths=%d", + modeStr, config.MaxExactPathMatches, config.FanOutLevels, config.MaxProcessPaths) + + singletonFilter = filter.NewFilter(config.MaxExactPathMatches, config.MaxProcessPaths, config.FanOutLevels) }) return singletonFilter } diff --git a/central/processindicator/pruner/pruner_impl.go b/central/processindicator/pruner/pruner_impl.go index 24af28f21e47b..8aacbe694d3a8 100644 --- a/central/processindicator/pruner/pruner_impl.go +++ b/central/processindicator/pruner/pruner_impl.go @@ -4,7 +4,7 @@ import ( "strings" "time" - "github.com/RoaringBitmap/roaring" + "github.com/RoaringBitmap/roaring/v2" "github.com/stackrox/rox/central/processindicator" ) diff --git a/central/processindicator/pruner/pruner_test.go b/central/processindicator/pruner/pruner_test.go index 23e56a3c23f03..5bde0cfead5e0 100644 --- a/central/processindicator/pruner/pruner_test.go +++ b/central/processindicator/pruner/pruner_test.go @@ -66,7 +66,7 @@ func processToIDAndArgs(process *storage.ProcessIndicator) processindicator.IDAn func TestRabbitMQPruning(t *testing.T) { var processes []processindicator.IDAndArgs processes = append(processes, processToIDAndArgs(deterministicRabbitMQProcess)) - for i := 0; i < 1000; i++ { + for range 1000 { processes = append(processes, processToIDAndArgs(rabbitMQBeamSMPProcess())) } pruner := NewFactory(1, time.Second).StartPruning() @@ -79,10 +79,10 @@ func TestRabbitMQPruning(t *testing.T) { func BenchmarkRabbitMQPruning(b *testing.B) { var processes []processindicator.IDAndArgs processes = append(processes, processToIDAndArgs(deterministicRabbitMQProcess)) - for i := 0; i < 1000000; i++ { + for range 1000000 { processes = append(processes, processToIDAndArgs(rabbitMQBeamSMPProcess())) } - for i := 0; i < b.N; i++ { + for b.Loop() { pruner := NewFactory(1, time.Second).StartPruning() pruner.Prune(processes) pruner.Finish() diff --git a/central/processindicator/store/mocks/store.go b/central/processindicator/store/mocks/store.go index 17d0894672c04..91c996bec9d1b 100644 --- a/central/processindicator/store/mocks/store.go +++ b/central/processindicator/store/mocks/store.go @@ -72,6 +72,21 @@ func (mr *MockStoreMockRecorder) DeleteByQuery(ctx, query any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteByQuery", reflect.TypeOf((*MockStore)(nil).DeleteByQuery), ctx, query) } +// DeleteByQueryWithIDs mocks base method. +func (m *MockStore) DeleteByQueryWithIDs(ctx context.Context, query *v1.Query) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteByQueryWithIDs", ctx, query) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteByQueryWithIDs indicates an expected call of DeleteByQueryWithIDs. +func (mr *MockStoreMockRecorder) DeleteByQueryWithIDs(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteByQueryWithIDs", reflect.TypeOf((*MockStore)(nil).DeleteByQueryWithIDs), ctx, query) +} + // DeleteMany mocks base method. func (m *MockStore) DeleteMany(ctx context.Context, id []string) error { m.ctrl.T.Helper() @@ -117,6 +132,20 @@ func (mr *MockStoreMockRecorder) GetByQuery(ctx, q any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByQuery", reflect.TypeOf((*MockStore)(nil).GetByQuery), ctx, q) } +// GetByQueryFn mocks base method. +func (m *MockStore) GetByQueryFn(ctx context.Context, query *v1.Query, fn func(*storage.ProcessIndicator) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetByQueryFn", ctx, query, fn) + ret0, _ := ret[0].(error) + return ret0 +} + +// GetByQueryFn indicates an expected call of GetByQueryFn. +func (mr *MockStoreMockRecorder) GetByQueryFn(ctx, query, fn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByQueryFn", reflect.TypeOf((*MockStore)(nil).GetByQueryFn), ctx, query, fn) +} + // GetMany mocks base method. func (m *MockStore) GetMany(ctx context.Context, ids []string) ([]*storage.ProcessIndicator, []int, error) { m.ctrl.T.Helper() diff --git a/central/processindicator/store/store.go b/central/processindicator/store/store.go index 17ef4d6bbfef2..f14895c56ca46 100644 --- a/central/processindicator/store/store.go +++ b/central/processindicator/store/store.go @@ -17,6 +17,7 @@ type Store interface { Get(ctx context.Context, id string) (*storage.ProcessIndicator, bool, error) GetByQuery(ctx context.Context, q *v1.Query) ([]*storage.ProcessIndicator, error) + GetByQueryFn(ctx context.Context, query *v1.Query, fn func(obj *storage.ProcessIndicator) error) error GetMany(ctx context.Context, ids []string) ([]*storage.ProcessIndicator, []int, error) UpsertMany(context.Context, []*storage.ProcessIndicator) error @@ -25,4 +26,5 @@ type Store interface { Walk(context.Context, func(pi *storage.ProcessIndicator) error) error WalkByQuery(context.Context, *v1.Query, func(pi *storage.ProcessIndicator) error) error DeleteByQuery(ctx context.Context, query *v1.Query) error + DeleteByQueryWithIDs(ctx context.Context, query *v1.Query) ([]string, error) } diff --git a/central/processlisteningonport/datastore/datastore.go b/central/processlisteningonport/datastore/datastore.go index d7744f88b2b32..f7308a5a418ed 100644 --- a/central/processlisteningonport/datastore/datastore.go +++ b/central/processlisteningonport/datastore/datastore.go @@ -29,12 +29,9 @@ type DataStore interface { deployment string, ) ([]*storage.ProcessListeningOnPort, error) WalkAll(ctx context.Context, fn WalkFn) error - RemoveProcessListeningOnPort(ctx context.Context, ids []string) error RemovePlopsByPod(ctx context.Context, id string) error PruneOrphanedPLOPs(ctx context.Context, orphanWindow time.Duration) int64 - PruneOrphanedPLOPsByProcessIndicators(ctx context.Context, orphanWindow time.Duration) RemovePLOPsWithoutProcessIndicatorOrProcessInfo(ctx context.Context) (int64, error) - RemovePLOPsWithoutPodUID(ctx context.Context) (int64, error) } // New creates a data store object to access the database. Since some diff --git a/central/processlisteningonport/datastore/datastore_impl.go b/central/processlisteningonport/datastore/datastore_impl.go index cd8219320a726..3783a97139f7b 100644 --- a/central/processlisteningonport/datastore/datastore_impl.go +++ b/central/processlisteningonport/datastore/datastore_impl.go @@ -306,25 +306,6 @@ func (ds *datastoreImpl) WalkAll(ctx context.Context, fn WalkFn) error { return ds.storage.Walk(ctx, fn) } -func (ds *datastoreImpl) RemoveProcessListeningOnPort(ctx context.Context, ids []string) error { - if ok, err := plopSAC.WriteAllowed(ctx); err != nil { - return err - } else if !ok { - return sac.ErrResourceAccessDenied - } - - return ds.removePLOP(ctx, ids) -} - -func (ds *datastoreImpl) removePLOP(ctx context.Context, ids []string) error { - - if len(ids) == 0 { - return nil - } - - return ds.storage.DeleteMany(ctx, ids) -} - func (ds *datastoreImpl) fetchExistingPLOPs( ctx context.Context, indicatorIds []string, @@ -596,27 +577,6 @@ func (ds *datastoreImpl) PruneOrphanedPLOPs(ctx context.Context, orphanWindow ti return commandTag.RowsAffected() } -// PruneOrphanedPLOPsByProcessIndicators prunes PLOPs that match process indicators without pods -func (ds *datastoreImpl) PruneOrphanedPLOPsByProcessIndicators(ctx context.Context, orphanWindow time.Duration) { - // TODO(ROX-22443): Once it is guaranteed that all listening endpoints have PodUIDs, remove this function - ds.mutex.Lock() - defer ds.mutex.Unlock() - - // Delete processes listening on ports orphaned because process indicators are orphaned due to - // missing deployments - query := fmt.Sprintf(deleteOrphanedPLOPDeploymentsAndPI, int(orphanWindow.Minutes())) - if _, err := ds.pool.Exec(ctx, query); err != nil { - log.Errorf("failed to prune process listening on ports by deployment: %v", err) - } - - // Delete processes listening on ports orphaned because process indicators are orphaned due to - // missing pods. - query = fmt.Sprintf(deleteOrphanedPLOPPods, int(orphanWindow.Minutes())) - if _, err := ds.pool.Exec(ctx, query); err != nil { - log.Errorf("failed to prune process listening on ports by pods: %v", err) - } -} - func (ds *datastoreImpl) readRowsToFindPLOPsWithNoProcessInformation(rows pgx.Rows) ([]string, error) { var ids []string @@ -672,89 +632,3 @@ func (ds *datastoreImpl) RemovePLOPsWithoutProcessIndicatorOrProcessInfo(ctx con return int64(len(plopsToDelete)), nil } - -// Removes PLOPs without poduids between a range of ids. -func (ds *datastoreImpl) removePLOPsWithoutPodUIDOnePage(ctx context.Context, prevId string, nextId string) (int64, error) { - ds.mutex.Lock() - defer ds.mutex.Unlock() - - query := fmt.Sprintf(deletePLOPsWithoutPoduidInPage, prevId, nextId) - commandTag, err := ds.pool.Exec(ctx, query) - - if err != nil { - return 0, err - } - - return commandTag.RowsAffected(), nil -} - -// Given a set of rows with ids, returns the id of the last row. This is useful for pagination. -func (ds *datastoreImpl) getLastIdFromRows(ctx context.Context, rows pgx.Rows) (string, error) { - id := "" - - for rows.Next() { - if err := rows.Scan(&id); err != nil { - return "", pgutils.ErrNilIfNoRows(err) - } - } - - return id, rows.Err() -} - -// Given an id and a limit, returns the id a limit number of rows after the given id. This is useful -// for efficient pagination. -func (ds *datastoreImpl) getNextPageId(ctx context.Context, prevId string, limit int) (string, error) { - ds.mutex.Lock() - defer ds.mutex.Unlock() - - query := fmt.Sprintf(getLastIdFromPage, prevId, limit) - rows, err := ds.pool.Query(ctx, query) - - if err != nil { - // Do not be alarmed if the error is simply NoRows - err = pgutils.ErrNilIfNoRows(err) - if err != nil { - log.Warnf("%s: %s", query, err) - } - return "", err - } - defer rows.Close() - - nextId, err := ds.getLastIdFromRows(ctx, rows) - - if err != nil { - return "", err - } - - return nextId, nil -} - -func (ds *datastoreImpl) retryableRemovePLOPsWithoutPodUID(ctx context.Context) (int64, error) { - limit := 10000 - totalRows := int64(0) - - prevId := "00000000-0000-0000-0000-000000000000" - for { - nextId, err := ds.getNextPageId(ctx, prevId, limit) - if err != nil { - return totalRows, err - } - if nextId == "" { - break - } - nrows, err := ds.removePLOPsWithoutPodUIDOnePage(ctx, prevId, nextId) - if err != nil { - return totalRows, err - } - totalRows += nrows - prevId = nextId - } - - return totalRows, nil -} - -func (ds *datastoreImpl) RemovePLOPsWithoutPodUID(ctx context.Context) (int64, error) { - return pgutils.Retry2(ctx, func() (int64, error) { - return ds.retryableRemovePLOPsWithoutPodUID(ctx) - }) -} diff --git a/central/processlisteningonport/datastore/datastore_impl_test.go b/central/processlisteningonport/datastore/datastore_impl_test.go index fc2116c543457..66d5619a04dac 100644 --- a/central/processlisteningonport/datastore/datastore_impl_test.go +++ b/central/processlisteningonport/datastore/datastore_impl_test.go @@ -6,8 +6,6 @@ import ( "context" "fmt" "math/rand" - "slices" - "sync" "testing" "time" @@ -27,6 +25,7 @@ import ( "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/set" + "github.com/stackrox/rox/pkg/sync" "github.com/stackrox/rox/pkg/uuid" "github.com/stretchr/testify/suite" ) @@ -1556,7 +1555,7 @@ func (suite *PLOPDataStoreTestSuite) TestPLOPDeleteAndCreateDeployment() { // Delete the indicator suite.NoError(suite.indicatorDataStore.RemoveProcessIndicators( - suite.hasWriteCtx, idsToDelete)) + suite.hasWriteCtx, idsToDelete, processIndicatorDataStore.RemovalReasonProcessFilter)) _, err := suite.datastore.RemovePLOPsWithoutProcessIndicatorOrProcessInfo(suite.hasWriteCtx) suite.NoError(err) @@ -2294,12 +2293,12 @@ func makeRandomPlops(nport int, nprocess int, npod int, deployment string) []*st nplops := 2 * nprocess * npod * nport plops := make([]*storage.ProcessListeningOnPortFromSensor, nplops) - for podIdx := 0; podIdx < npod; podIdx++ { + for range npod { podID := makeRandomString(10) podUid := uuid.NewV4().String() - for processIdx := 0; processIdx < nprocess; processIdx++ { + for range nprocess { execFilePath := makeRandomString(10) - for port := 0; port < nport; port++ { + for port := range nport { plopTCP := &storage.ProcessListeningOnPortFromSensor{ Port: uint32(port), @@ -2380,12 +2379,10 @@ func (suite *PLOPDataStoreTestSuite) TestDeletePods() { // It is done async so that the UpsertMany in AddProcessListeningOnPort // runs at the same time as PLOPs are deleted by pod var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { suite.NoError(suite.datastore.AddProcessListeningOnPort( suite.hasWriteCtx, fixtureconsts.Cluster1, plopObjects...)) - }() + }) // The pods are deleted and all PLOPs are deleted by pod for podUID := range podUids { @@ -2470,11 +2467,11 @@ func (suite *PLOPDataStoreTestSuite) TestRemoveOrphanedPLOPs() { // Add deployments if necessary deploymentDS, err := deploymentStore.GetTestPostgresDataStore(suite.T(), suite.postgres.DB) suite.Nil(err) - for _, deploymentID := range c.deployments.AsSlice() { + for deploymentID := range c.deployments.All() { suite.NoError(deploymentDS.UpsertDeployment(suite.hasAllCtx, &storage.Deployment{Id: deploymentID, ClusterId: fixtureconsts.Cluster1})) } - for _, podID := range c.pods.AsSlice() { + for podID := range c.pods.All() { insertPod := fmt.Sprintf("INSERT INTO pods (id, clusterid) VALUES ('%s', '%s')", podID, fixtureconsts.Cluster1) _, err := suite.postgres.DB.Exec(suite.hasWriteCtx, insertPod) suite.Nil(err) @@ -2502,174 +2499,6 @@ func (suite *PLOPDataStoreTestSuite) TestRemoveOrphanedPLOPs() { } } -func newIndicatorWithDeployment(id string, age time.Duration, deploymentID string) *storage.ProcessIndicator { - return &storage.ProcessIndicator{ - Id: id, - DeploymentId: deploymentID, - ContainerName: "", - PodId: "", - Signal: &storage.ProcessSignal{ - Time: protoconv.NowMinus(age), - }, - } -} - -func newIndicatorWithDeploymentAndPod(id string, age time.Duration, deploymentID, podUID string) *storage.ProcessIndicator { - indicator := newIndicatorWithDeployment(id, age, deploymentID) - indicator.PodUid = podUID - return indicator -} - -func (suite *PLOPDataStoreTestSuite) TestRemoveOrphanedPLOPsByProcesses() { - orphanWindow := 30 * time.Minute - - cases := []struct { - name string - initialProcesses []*storage.ProcessIndicator - initialPlops []*storage.ProcessListeningOnPortStorage - deployments set.FrozenStringSet - pods set.FrozenStringSet - expectedPlopDeletions []string - }{ - { - name: "no deployments nor pods - remove all old indicators", - initialProcesses: []*storage.ProcessIndicator{ - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID1, 1*time.Hour, fixtureconsts.Deployment6, fixtureconsts.PodUID1), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID2, 1*time.Hour, fixtureconsts.Deployment5, fixtureconsts.PodUID2), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID3, 1*time.Hour, fixtureconsts.Deployment3, fixtureconsts.PodUID3), - }, - initialPlops: []*storage.ProcessListeningOnPortStorage{ - fixtures.GetPlopStorage1(), - fixtures.GetPlopStorage2(), - fixtures.GetPlopStorage3(), - fixtures.GetPlopStorage4(), - fixtures.GetPlopStorage5(), - fixtures.GetPlopStorage6(), - }, - deployments: set.NewFrozenStringSet(), - pods: set.NewFrozenStringSet(), - expectedPlopDeletions: []string{ - fixtureconsts.PlopUID1, - fixtureconsts.PlopUID2, - fixtureconsts.PlopUID3, - fixtureconsts.PlopUID4, - fixtureconsts.PlopUID5, - fixtureconsts.PlopUID6, - }, - }, - { - name: "no deployments nor pods - remove no new orphaned indicators", - initialProcesses: []*storage.ProcessIndicator{ - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID1, 20*time.Minute, fixtureconsts.Deployment6, fixtureconsts.PodUID1), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID2, 20*time.Minute, fixtureconsts.Deployment5, fixtureconsts.PodUID2), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID3, 20*time.Minute, fixtureconsts.Deployment3, fixtureconsts.PodUID3), - }, - initialPlops: []*storage.ProcessListeningOnPortStorage{ - fixtures.GetPlopStorage1(), - fixtures.GetPlopStorage2(), - fixtures.GetPlopStorage3(), - }, - deployments: set.NewFrozenStringSet(), - pods: set.NewFrozenStringSet(), - expectedPlopDeletions: nil, - }, - { - name: "all pods separate deployments - remove no indicators", - initialProcesses: []*storage.ProcessIndicator{ - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID1, 1*time.Hour, fixtureconsts.Deployment6, fixtureconsts.PodUID1), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID2, 1*time.Hour, fixtureconsts.Deployment5, fixtureconsts.PodUID2), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID3, 1*time.Hour, fixtureconsts.Deployment3, fixtureconsts.PodUID3), - }, - initialPlops: []*storage.ProcessListeningOnPortStorage{ - fixtures.GetPlopStorage1(), - fixtures.GetPlopStorage2(), - fixtures.GetPlopStorage3(), - }, - deployments: set.NewFrozenStringSet(fixtureconsts.Deployment6, fixtureconsts.Deployment5, fixtureconsts.Deployment3), - pods: set.NewFrozenStringSet(fixtureconsts.PodUID1, fixtureconsts.PodUID2, fixtureconsts.PodUID3), - expectedPlopDeletions: nil, - }, - { - name: "all pods same deployment - remove no indicators", - initialProcesses: []*storage.ProcessIndicator{ - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID1, 1*time.Hour, fixtureconsts.Deployment6, fixtureconsts.PodUID1), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID2, 1*time.Hour, fixtureconsts.Deployment6, fixtureconsts.PodUID2), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID3, 1*time.Hour, fixtureconsts.Deployment6, fixtureconsts.PodUID3), - }, - deployments: set.NewFrozenStringSet(fixtureconsts.Deployment6), - pods: set.NewFrozenStringSet(fixtureconsts.PodUID1, fixtureconsts.PodUID2, fixtureconsts.PodUID3), - expectedPlopDeletions: nil, - }, - { - name: "some pods separate deployments - remove some indicators", - initialProcesses: []*storage.ProcessIndicator{ - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID1, 1*time.Hour, fixtureconsts.Deployment6, fixtureconsts.PodUID1), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID2, 20*time.Minute, fixtureconsts.Deployment5, fixtureconsts.PodUID2), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID3, 1*time.Hour, fixtureconsts.Deployment3, fixtureconsts.PodUID3), - }, - initialPlops: []*storage.ProcessListeningOnPortStorage{ - fixtures.GetPlopStorage1(), - fixtures.GetPlopStorage2(), - fixtures.GetPlopStorage3(), - }, - deployments: set.NewFrozenStringSet(fixtureconsts.Deployment3), - pods: set.NewFrozenStringSet(fixtureconsts.PodUID3), - expectedPlopDeletions: []string{fixtureconsts.PlopUID1}, - }, - { - name: "some pods same deployment - remove some indicators", - initialProcesses: []*storage.ProcessIndicator{ - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID1, 1*time.Hour, fixtureconsts.Deployment6, fixtureconsts.PodUID1), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID2, 20*time.Minute, fixtureconsts.Deployment6, fixtureconsts.PodUID2), - newIndicatorWithDeploymentAndPod(fixtureconsts.ProcessIndicatorID3, 1*time.Hour, fixtureconsts.Deployment6, fixtureconsts.PodUID3), - }, - initialPlops: []*storage.ProcessListeningOnPortStorage{ - fixtures.GetPlopStorage1(), - fixtures.GetPlopStorage2(), - fixtures.GetPlopStorage3(), - }, - deployments: set.NewFrozenStringSet(fixtureconsts.Deployment6), - pods: set.NewFrozenStringSet(fixtureconsts.PodUID3), - expectedPlopDeletions: []string{fixtureconsts.PlopUID1}, - }, - } - for _, c := range cases { - suite.T().Run(c.name, func(t *testing.T) { - suite.SetupTest() - // Add deployments if necessary - deploymentDS, err := deploymentStore.GetTestPostgresDataStore(suite.T(), suite.postgres.DB) - suite.Nil(err) - for _, deploymentID := range c.deployments.AsSlice() { - suite.NoError(deploymentDS.UpsertDeployment(suite.hasAllCtx, &storage.Deployment{Id: deploymentID, ClusterId: fixtureconsts.Cluster1})) - } - - for _, podID := range c.pods.AsSlice() { - insertPod := fmt.Sprintf("INSERT INTO pods (id, clusterid) VALUES ('%s', '%s')", podID, fixtureconsts.Cluster1) - _, err := suite.postgres.DB.Exec(suite.hasWriteCtx, insertPod) - suite.Nil(err) - } - - suite.NoError(suite.indicatorDataStore.AddProcessIndicators(suite.hasWriteCtx, c.initialProcesses...)) - countFromDB, err := suite.indicatorDataStore.Count(suite.hasAllCtx, nil) - suite.NoError(err) - suite.Equal(len(c.initialProcesses), countFromDB) - - err = suite.store.UpsertMany(suite.hasWriteCtx, c.initialPlops) - suite.NoError(err) - plopCount, err := suite.store.Count(suite.hasReadCtx, search.EmptyQuery()) - suite.NoError(err) - suite.Equal(len(c.initialPlops), plopCount) - - suite.datastore.PruneOrphanedPLOPsByProcessIndicators(suite.hasAllCtx, orphanWindow) - - plopCount, err = suite.store.Count(suite.hasReadCtx, search.EmptyQuery()) - suite.NoError(err) - suite.Equal(len(c.initialPlops)-len(c.expectedPlopDeletions), plopCount) - - }) - } -} - // RemovePLOPsWithoutProcessIndicatorOrProcessInfo Adds a PLOP with a matching process indicator. // The process indicator is then deleted. This means that the PLOP has no matching // process indicator and does not have any process information. A pruning function @@ -2677,7 +2506,7 @@ func (suite *PLOPDataStoreTestSuite) TestRemoveOrphanedPLOPsByProcesses() { func (suite *PLOPDataStoreTestSuite) RemovePLOPsWithoutProcessIndicatorOrProcessInfo() { indicators := getIndicators() - var indicatorIds []string + indicatorIds := make([]string, 0, len(indicators)) for _, indicator := range indicators { indicatorIds = append(indicatorIds, indicator.GetId()) @@ -2696,7 +2525,7 @@ func (suite *PLOPDataStoreTestSuite) RemovePLOPsWithoutProcessIndicatorOrProcess suite.hasWriteCtx, fixtureconsts.Cluster1, plopObjects...)) suite.NoError(suite.indicatorDataStore.RemoveProcessIndicators( - suite.hasWriteCtx, indicatorIds)) + suite.hasWriteCtx, indicatorIds, processIndicatorDataStore.RemovalReasonProcessFilter)) _, err := suite.datastore.RemovePLOPsWithoutProcessIndicatorOrProcessInfo(suite.hasWriteCtx) suite.NoError(err) @@ -2704,191 +2533,3 @@ func (suite *PLOPDataStoreTestSuite) RemovePLOPsWithoutProcessIndicatorOrProcess newPlopsFromDB := suite.getPlopsFromDB() suite.Len(newPlopsFromDB, 0) } - -func (suite *PLOPDataStoreTestSuite) TestRemovePLOPsWithoutPodUID() { - initialPlops := []*storage.ProcessListeningOnPortStorage{ - fixtures.GetPlopStorage1(), - fixtures.GetPlopStorage2(), - fixtures.GetPlopStorage3(), - fixtures.GetPlopStorage4(), - fixtures.GetPlopStorage5(), - fixtures.GetPlopStorage6(), - } - expectedPlopDeletions := []string{fixtureconsts.PlopUID1, fixtureconsts.PlopUID2, fixtureconsts.PlopUID3} - - err := suite.store.UpsertMany(suite.hasWriteCtx, initialPlops) - suite.NoError(err) - plopCount, err := suite.store.Count(suite.hasReadCtx, search.EmptyQuery()) - suite.NoError(err) - suite.Equal(len(initialPlops), plopCount) - - prunedCount, err := suite.datastore.RemovePLOPsWithoutPodUID(suite.hasWriteCtx) - suite.NoError(err) - suite.Equal(int64(3), prunedCount) - - plopCount, err = suite.store.Count(suite.hasReadCtx, search.EmptyQuery()) - suite.NoError(err) - suite.Equal(len(initialPlops)-len(expectedPlopDeletions), plopCount) - - ids, err := suite.store.GetIDs(suite.hasReadCtx) - suite.NoError(err) - for id := range ids { - suite.NotContains(expectedPlopDeletions, id) - } -} - -func (suite *PLOPDataStoreTestSuite) addTooMany(plops []*storage.ProcessListeningOnPortFromSensor) { - batchSize := 30000 - - for plopBatch := range slices.Chunk(plops, batchSize) { - err := suite.datastore.AddProcessListeningOnPort(suite.hasWriteCtx, fixtureconsts.Cluster1, plopBatch...) - suite.NoError(err) - } -} - -func (suite *PLOPDataStoreTestSuite) RemovePLOPsWithoutPodUIDScale(nport int, nprocess int, npod int) { - plopObjects := makeRandomPlops(nport, nprocess, npod, fixtureconsts.Deployment1) - - plopsWithoutPodUids := 0 - for _, plop := range plopObjects { - p := rand.Float32() - if p < 0.5 { - plop.PodUid = "" - plopsWithoutPodUids++ - } - } - - // Add the PLOPs - suite.addTooMany(plopObjects) - - plopCount, err := suite.store.Count(suite.hasReadCtx, search.EmptyQuery()) - suite.Equal(plopCount, 2*nport*nprocess*npod) - suite.NoError(err) - - start := time.Now() - prunedCount, err := suite.datastore.RemovePLOPsWithoutPodUID(suite.hasWriteCtx) - suite.Equal(int64(plopsWithoutPodUids), prunedCount) - duration := time.Since(start) - suite.NoError(err) - log.Infof("Pruning %d plops took %s", prunedCount, duration) - _, err = suite.datastore.RemovePLOPsWithoutPodUID(suite.hasWriteCtx) - suite.NoError(err) -} - -func (suite *PLOPDataStoreTestSuite) TestRemovePLOPsWithoutPodUIDScale27K() { - nport := 30 - nprocess := 30 - npod := 30 - - suite.RemovePLOPsWithoutPodUIDScale(nport, nprocess, npod) -} - -func (suite *PLOPDataStoreTestSuite) TestRemovePLOPsWithoutPodUIDScale125K() { - nport := 50 - nprocess := 50 - npod := 50 - - suite.RemovePLOPsWithoutPodUIDScale(nport, nprocess, npod) -} - -func (suite *PLOPDataStoreTestSuite) TestRemovePLOPsWithoutPodUIDScaleRaceCondition() { - var mutex sync.RWMutex - var wg sync.WaitGroup - wg.Add(1) - - running := true - plopsWithoutPodUids := 0 - go func() { - defer wg.Done() - iterations := 3 - for i := 0; i < iterations; i++ { - nport := 30 - nprocess := 30 - npod := 30 - plopObjects := makeRandomPlops(nport, nprocess, npod, fixtureconsts.Deployment1) - - for _, plop := range plopObjects { - p := rand.Float32() - if p < 0.5 { - plop.PodUid = "" - plopsWithoutPodUids += 1 - } else { - plop.PodUid = uuid.NewV4().String() - } - } - - // Add the open PLOPs - suite.addTooMany(plopObjects) - - // Close the PLOPs - // This is so that UpsertMany will trigger deletes - for _, plop := range plopObjects { - plop.CloseTimestamp = protoconv.ConvertTimeToTimestamp(time.Now()) - } - - // Add the closed PLOPs - suite.addTooMany(plopObjects) - } - }() - - totalPrunedCount := 0 - - var wgPrune sync.WaitGroup - wgPrune.Add(1) - go func() { - defer wgPrune.Done() - for { - mutex.Lock() - stop := !running - mutex.Unlock() - - if stop { - break - } - - prunedCount, err := suite.datastore.RemovePLOPsWithoutPodUID(suite.hasWriteCtx) - suite.NoError(err) - totalPrunedCount += int(prunedCount) - } - }() - - wg.Wait() - - mutex.Lock() - running = false - mutex.Unlock() - - wgPrune.Wait() - - prunedCount, err := suite.datastore.RemovePLOPsWithoutPodUID(suite.hasWriteCtx) - suite.NoError(err) - totalPrunedCount += int(prunedCount) - - // Each PLOP is opened and closed. If a PLOP is opened then pruned, then closed, - // and pruned again, then it will be pruned twice. If a PLOP is opened, then closed, - // and pruned, then it will be pruned once. Therefore the number of rows pruned will - // be between the number of PLOPs that don't have poduids that were added and twice - // that number. - suite.GreaterOrEqual(plopsWithoutPodUids, totalPrunedCount/2) - suite.LessOrEqual(plopsWithoutPodUids, totalPrunedCount) -} - -func (suite *PLOPDataStoreTestSuite) TestSortMany() { - nport := 50 - nprocess := 50 - npod := 50 - - plops := makeRandomPlops(nport, nprocess, npod, fixtureconsts.Deployment1) - suite.addTooMany(plops) - - suite.addDeployments() - - startTime := time.Now() - newPlops, err := suite.datastore.GetProcessListeningOnPort( - suite.hasWriteCtx, fixtureconsts.Deployment1) - suite.NoError(err) - duration := time.Since(startTime) - - fmt.Printf("Fetching %d plops took %s\n", len(newPlops), duration) - -} diff --git a/central/processlisteningonport/datastore/mocks/datastore.go b/central/processlisteningonport/datastore/mocks/datastore.go index bc71a23f3157e..053977e9f8ed7 100644 --- a/central/processlisteningonport/datastore/mocks/datastore.go +++ b/central/processlisteningonport/datastore/mocks/datastore.go @@ -91,33 +91,6 @@ func (mr *MockDataStoreMockRecorder) PruneOrphanedPLOPs(ctx, orphanWindow any) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PruneOrphanedPLOPs", reflect.TypeOf((*MockDataStore)(nil).PruneOrphanedPLOPs), ctx, orphanWindow) } -// PruneOrphanedPLOPsByProcessIndicators mocks base method. -func (m *MockDataStore) PruneOrphanedPLOPsByProcessIndicators(ctx context.Context, orphanWindow time.Duration) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "PruneOrphanedPLOPsByProcessIndicators", ctx, orphanWindow) -} - -// PruneOrphanedPLOPsByProcessIndicators indicates an expected call of PruneOrphanedPLOPsByProcessIndicators. -func (mr *MockDataStoreMockRecorder) PruneOrphanedPLOPsByProcessIndicators(ctx, orphanWindow any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PruneOrphanedPLOPsByProcessIndicators", reflect.TypeOf((*MockDataStore)(nil).PruneOrphanedPLOPsByProcessIndicators), ctx, orphanWindow) -} - -// RemovePLOPsWithoutPodUID mocks base method. -func (m *MockDataStore) RemovePLOPsWithoutPodUID(ctx context.Context) (int64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RemovePLOPsWithoutPodUID", ctx) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// RemovePLOPsWithoutPodUID indicates an expected call of RemovePLOPsWithoutPodUID. -func (mr *MockDataStoreMockRecorder) RemovePLOPsWithoutPodUID(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePLOPsWithoutPodUID", reflect.TypeOf((*MockDataStore)(nil).RemovePLOPsWithoutPodUID), ctx) -} - // RemovePLOPsWithoutProcessIndicatorOrProcessInfo mocks base method. func (m *MockDataStore) RemovePLOPsWithoutProcessIndicatorOrProcessInfo(ctx context.Context) (int64, error) { m.ctrl.T.Helper() @@ -147,20 +120,6 @@ func (mr *MockDataStoreMockRecorder) RemovePlopsByPod(ctx, id any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePlopsByPod", reflect.TypeOf((*MockDataStore)(nil).RemovePlopsByPod), ctx, id) } -// RemoveProcessListeningOnPort mocks base method. -func (m *MockDataStore) RemoveProcessListeningOnPort(ctx context.Context, ids []string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RemoveProcessListeningOnPort", ctx, ids) - ret0, _ := ret[0].(error) - return ret0 -} - -// RemoveProcessListeningOnPort indicates an expected call of RemoveProcessListeningOnPort. -func (mr *MockDataStoreMockRecorder) RemoveProcessListeningOnPort(ctx, ids any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveProcessListeningOnPort", reflect.TypeOf((*MockDataStore)(nil).RemoveProcessListeningOnPort), ctx, ids) -} - // WalkAll mocks base method. func (m *MockDataStore) WalkAll(ctx context.Context, fn datastore.WalkFn) error { m.ctrl.T.Helper() diff --git a/central/processlisteningonport/datastore/pruning.go b/central/processlisteningonport/datastore/pruning.go index 1d4da65fcbc26..0b9a484590e9e 100644 --- a/central/processlisteningonport/datastore/pruning.go +++ b/central/processlisteningonport/datastore/pruning.go @@ -1,15 +1,6 @@ package datastore const ( - // Explain Analyze indicated that 2 statements for PLOP is faster than one. - deleteOrphanedPLOPDeploymentsAndPI = `DELETE FROM listening_endpoints WHERE processindicatorid in (SELECT id from process_indicators pi WHERE NOT EXISTS - (SELECT 1 FROM deployments WHERE pi.deploymentid = deployments.Id) AND - (signal_time < now() at time zone 'utc' - INTERVAL '%d MINUTES' OR signal_time is NULL))` - - deleteOrphanedPLOPPods = `DELETE FROM listening_endpoints WHERE processindicatorid in (SELECT id from process_indicators pi WHERE NOT EXISTS - (SELECT 1 FROM pods WHERE pi.poduid = pods.Id) AND - (signal_time < now() at time zone 'utc' - INTERVAL '%d MINUTES' OR signal_time is NULL))` - // Unfortunately if a listening endpoint is marked as being open there is no indication of how old it is. // This leads to a possible race condition where a listening endpoint reaches the database before the deployment, // and the pruning job happens to run before the deployment information arrives in the database. @@ -31,17 +22,4 @@ const ( // if the serialized data has process information. PLOPs without process information are then deleted. getPotentiallyOrphanedPLOPs = `SELECT plop.serialized FROM listening_endpoints plop where NOT EXISTS (select 1 FROM process_indicators proc where plop.processindicatorid = proc.id)` - - // This is used to make pagination more efficient compared to using offset. The ids obtained using this query are used for deleting - // PLOPs without poduids in batches. See the query below this query. - getLastIdFromPage = `WITH tmp as ( - SELECT id FROM listening_endpoints WHERE id > '%s' ORDER BY id LIMIT %d - ) - SELECT id FROM tmp ORDER BY id DESC LIMIT 1` - - // Deletes PLOPs without poduids in batches according to id, which is more efficient than using offset. - // It is possible that new rows may be inserted between the starting and ending ids, after the two ids - // are obtained. Meaning that the number of rows in the page may be different than what is expected for - // the page. This should not cause a problem. - deletePLOPsWithoutPoduidInPage = "DELETE FROM listening_endpoints WHERE poduid is null AND id >= '%s' AND id <= '%s'" ) diff --git a/central/processlisteningonport/store/postgres/full_store_benchmark_test.go b/central/processlisteningonport/store/postgres/full_store_benchmark_test.go index dbe13ff1a2fbd..b98dab7e59b1f 100644 --- a/central/processlisteningonport/store/postgres/full_store_benchmark_test.go +++ b/central/processlisteningonport/store/postgres/full_store_benchmark_test.go @@ -23,12 +23,12 @@ func makeRandomPlops(nport int, nprocess int, npod int) []*storage.ProcessListen count := 0 plops := make([]*storage.ProcessListeningOnPort, 2*nport*nprocess*npod) - for podIdx := 0; podIdx < npod; podIdx++ { + for range npod { podID := makeRandomString(10) podUID := makeRandomString(10) - for processIdx := 0; processIdx < nprocess; processIdx++ { + for range nprocess { execFilePath := makeRandomString(10) - for port := 0; port < nport; port++ { + for port := range nport { plopTCP := &storage.ProcessListeningOnPort{ Endpoint: &storage.ProcessListeningOnPort_Endpoint{ diff --git a/central/pruning/image_pruning_bench_test.go b/central/pruning/image_pruning_bench_test.go new file mode 100644 index 0000000000000..7e18bdd665524 --- /dev/null +++ b/central/pruning/image_pruning_bench_test.go @@ -0,0 +1,209 @@ +//go:build sql_integration + +package pruning + +import ( + "context" + "fmt" + "testing" + "time" + + configDSMocks "github.com/stackrox/rox/central/config/datastore/mocks" + deploymentDS "github.com/stackrox/rox/central/deployment/datastore" + imageV2DS "github.com/stackrox/rox/central/imagev2/datastore" + imageV2Postgres "github.com/stackrox/rox/central/imagev2/datastore/store/postgres" + platformmatcher "github.com/stackrox/rox/central/platform/matcher" + podDS "github.com/stackrox/rox/central/pod/datastore" + processBaselineDSMocks "github.com/stackrox/rox/central/processbaseline/datastore/mocks" + processIndicatorDSMocks "github.com/stackrox/rox/central/processindicator/datastore/mocks" + plopDSMocks "github.com/stackrox/rox/central/processlisteningonport/datastore/mocks" + "github.com/stackrox/rox/central/ranking" + riskDSMocks "github.com/stackrox/rox/central/risk/datastore/mocks" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/images/types" + "github.com/stackrox/rox/pkg/images/utils" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/pgtest" + filterMocks "github.com/stackrox/rox/pkg/process/filter/mocks" + "github.com/stackrox/rox/pkg/protoconv" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func newImage(prefix string, index int, timestamp *timestamppb.Timestamp) *storage.ImageV2 { + digest := types.NewDigest(fmt.Sprintf("%s-%d", prefix, index)).Digest() + name := &storage.ImageName{FullName: fmt.Sprintf("ghcr.io/bench/%s-%d:latest@%s", prefix, index, digest)} + return &storage.ImageV2{ + Id: utils.NewImageV2ID(name, digest), + Digest: digest, + Name: name, + LastUpdated: timestamp, + } +} + +func setupBenchmarkTest(b *testing.B, pool postgres.DB) ( + imageV2DS.DataStore, deploymentDS.DataStore, podDS.DataStore, *garbageCollectorImpl, +) { + + ctrl := gomock.NewController(b) + any := gomock.Any() + + mockRiskDS := riskDSMocks.NewMockDataStore(ctrl) + mockRiskDS.EXPECT().RemoveRisk(any, any, any).AnyTimes() + + mockProcessIndicatorDS := processIndicatorDSMocks.NewMockDataStore(ctrl) + mockProcessIndicatorDS.EXPECT().RemoveProcessIndicatorsByPod(any, any).AnyTimes().Return(nil) + + mockPlopDS := plopDSMocks.NewMockDataStore(ctrl) + mockPlopDS.EXPECT().RemovePlopsByPod(any, any).AnyTimes().Return(nil) + + mockConfigDS := configDSMocks.NewMockDataStore(ctrl) + mockConfigDS.EXPECT().GetPrivateConfig(any).AnyTimes().Return(testConfig.GetPrivateConfig(), nil) + + mockFilter := filterMocks.NewMockFilter(ctrl) + mockFilter.EXPECT().UpdateByPod(any).AnyTimes() + mockFilter.EXPECT().DeleteByPod(any).AnyTimes() + + deployments, err := deploymentDS.New( + pool, nil, nil, + processBaselineDSMocks.NewMockDataStore(ctrl), + nil, mockRiskDS, nil, mockFilter, + ranking.NewRanker(), ranking.NewRanker(), ranking.NewRanker(), + platformmatcher.GetTestPlatformMatcherWithDefaultPlatformComponentConfig(ctrl), + ) + require.NoError(b, err) + + imagesV2 := imageV2DS.NewWithPostgres( + imageV2Postgres.New(pool, true, concurrency.NewKeyFence()), + mockRiskDS, + ranking.ImageRanker(), + ranking.ComponentRanker(), + ) + + pods := podDS.NewPostgresDB(pool, mockProcessIndicatorDS, mockPlopDS, mockFilter) + + gc := newGarbageCollector(nil, nil, nil, imagesV2, nil, deployments, pods, + nil, nil, nil, mockConfigDS, nil, + nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil).(*garbageCollectorImpl) + gc.postgres = pool + + return imagesV2, deployments, pods, gc +} + +func BenchmarkImagePruning(b *testing.B) { + if !features.FlattenImageData.Enabled() { + b.Skip("only applicable when FlattenImageData is enabled") + } + + ctx := sac.WithAllAccess(context.Background()) + testingDB := pgtest.ForT(b) + defer testingDB.DB.Close() + pool := testingDB.DB + + imagesV2, deployments, pods, gc := setupBenchmarkTest(b, pool) + + retentionDays := testConfig.GetPrivateConfig().GetImageRetentionDurationDays() + privateConfig := testConfig.GetPrivateConfig() + + for _, imageCount := range []int{50, 200, 500} { + b.Run(fmt.Sprintf("images=%d", imageCount), func(b *testing.B) { + for range b.N { + b.StopTimer() + cleanupBenchmarkData(b, ctx, pool) + seedBenchmarkData(b, ctx, imagesV2, deployments, pods, imageCount, retentionDays) + b.StartTimer() + + gc.collectImages(privateConfig) + } + b.StopTimer() + cleanupBenchmarkData(b, ctx, pool) + }) + } +} + +// seedBenchmarkData populates the database with test images, deployments, and pods. +// +// Distribution: +// - 50% of images: referenced by a deployment -- should NOT be pruned +// - 20% of images: inactive with old timestamp (past retention) -- candidates for pruning +// - 20% of images: inactive with new timestamp (within retention) -- should NOT be pruned +// - 10% of images: inactive with old timestamp but have a stuck pod (deployment updated +// to different digest, pod still running old digest) -- should NOT be pruned +func seedBenchmarkData( + b *testing.B, + ctx context.Context, + imagesV2 imageV2DS.DataStore, + deployments deploymentDS.DataStore, + pods podDS.DataStore, + imageCount int, + retentionDays int32, +) { + oldTimestamp := protoconv.ConvertTimeToTimestamp( + time.Now().Add(-24 * time.Duration(retentionDays+1) * time.Hour), + ) + newTimestamp := protoconv.ConvertTimeToTimestamp(time.Now().Add(-1 * time.Hour)) + + deployedCount := imageCount * 50 / 100 // active images referenced by a deployment + inactiveOldCount := imageCount * 20 / 100 // inactive images past retention window + inactiveNewCount := imageCount * 20 / 100 // inactive images within retention window + stuckPodCount := imageCount - deployedCount - inactiveOldCount - inactiveNewCount // inactive images (old timestamp) with a stuck pod + + // Group 1: Images referenced by deployments + for i := range deployedCount { + img := newImage("deployed", i, oldTimestamp) + require.NoError(b, imagesV2.UpsertImage(ctx, img)) + + dep := &storage.Deployment{ + Id: uuid.NewV4().String(), + Name: fmt.Sprintf("dep-%d", i), + Namespace: "bench", + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{ + Id: img.GetDigest(), + IdV2: img.GetId(), + Name: img.GetName(), + }, + }, + }, + } + require.NoError(b, deployments.UpsertDeployment(ctx, dep)) + } + + // Group 2: Inactive images with old timestamp (past retention, no references) + for i := range inactiveOldCount { + require.NoError(b, imagesV2.UpsertImage(ctx, newImage("inactive-old", i, oldTimestamp))) + } + + // Group 3: Inactive images with new timestamp (within retention, no references) + for i := range inactiveNewCount { + require.NoError(b, imagesV2.UpsertImage(ctx, newImage("inactive-new", i, newTimestamp))) + } + + // Group 4: Stuck-pod images. The pod is still running the old image digest. + // The image should NOT be pruned. + for i := range stuckPodCount { + img := newImage("stuck", i, oldTimestamp) + require.NoError(b, imagesV2.UpsertImage(ctx, img)) + + pod := &storage.Pod{ + Id: uuid.NewV4().String(), + DeploymentId: uuid.NewV4().String(), + LiveInstances: []*storage.ContainerInstance{ + {ImageDigest: img.GetDigest()}, + }, + } + require.NoError(b, pods.UpsertPod(ctx, pod)) + } +} + +func cleanupBenchmarkData(b *testing.B, ctx context.Context, pool postgres.DB) { + _, err := pool.Exec(ctx, "TRUNCATE images_v2, deployments, pods CASCADE") + require.NoError(b, err) +} diff --git a/central/pruning/pruning.go b/central/pruning/pruning.go index 9e72ec93029ef..a1b82548091de 100644 --- a/central/pruning/pruning.go +++ b/central/pruning/pruning.go @@ -2,6 +2,8 @@ package pruning import ( "context" + "sync/atomic" + "testing" "time" "github.com/pkg/errors" @@ -13,7 +15,6 @@ import ( deploymentDatastore "github.com/stackrox/rox/central/deployment/datastore" "github.com/stackrox/rox/central/globaldb" imageDatastore "github.com/stackrox/rox/central/image/datastore" - imageComponentV2Datastore "github.com/stackrox/rox/central/imagecomponent/v2/datastore" imageV2Datastore "github.com/stackrox/rox/central/imagev2/datastore" logimbueDataStore "github.com/stackrox/rox/central/logimbue/store" "github.com/stackrox/rox/central/metrics" @@ -29,14 +30,18 @@ import ( "github.com/stackrox/rox/central/reports/common" snapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" riskDataStore "github.com/stackrox/rox/central/risk/datastore" + roleDataStore "github.com/stackrox/rox/central/role/datastore" serviceAccountDataStore "github.com/stackrox/rox/central/serviceaccount/datastore" vulnReqDataStore "github.com/stackrox/rox/central/vulnmgmt/vulnerabilityrequest/datastore" + watchedImageDatastore "github.com/stackrox/rox/central/watchedimage/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/contextutil" + "github.com/stackrox/rox/pkg/dblock" "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/features" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/maputil" pgPkg "github.com/stackrox/rox/pkg/postgres" @@ -56,8 +61,7 @@ const ( logImbueGCFreq = 24 * time.Hour logImbueWindow = 24 * 7 * time.Hour - alertQueryTimeout = 10 * time.Minute - alertDeleteBatchSize = 5000 + alertQueryTimeout = 10 * time.Minute flowsSemaphoreWeight = 5 @@ -69,17 +73,34 @@ const ( ) var ( - log = logging.LoggerForModule() - pruningCtx = sac.WithAllAccess(context.Background()) - lastClusterPruneTime time.Time - lastLogImbuePruneTime time.Time - pruningTimeout = env.PostgresDefaultPruningStatementTimeout.DurationSetting() - prunedPLOPsWithoutPodUIDs = false + log = logging.LoggerForModule() + pruningCtx = sac.WithAllAccess(context.Background()) + lastClusterPruneTime time.Time + lastLogImbuePruneTime time.Time + lastV1ImagePruneTime time.Time + lastPrunedV1ImageID string + pruningTimeout = env.PostgresDefaultPruningStatementTimeout.DurationSetting() pruneInterval = env.PruneInterval.DurationSetting() orphanWindow = env.PruneOrphanedWindow.DurationSetting() + + // dynamicRBACPruningEnabled controls whether expired dynamic RBAC objects should be pruned. + // It is disabled by default and enabled on the first call to the internaltoken API service. + dynamicRBACPruningEnabled atomic.Bool ) +// EnableDynamicRBACPruning enables pruning of expired dynamic RBAC objects. +// This should be called on requests to the internaltoken API service. +func EnableDynamicRBACPruning() { + dynamicRBACPruningEnabled.Store(true) +} + +// disableDynamicRBACPruningForTest disables pruning of expired dynamic RBAC objects. +// This is for testing purposes only. +func disableDynamicRBACPruningForTest(*testing.T) { + dynamicRBACPruningEnabled.Store(false) +} + // GarbageCollector implements a generic garbage collection mechanism. type GarbageCollector interface { Start() @@ -97,7 +118,6 @@ func newGarbageCollector(alerts alertDatastore.DataStore, processbaseline processBaselineDatastore.DataStore, networkflows networkFlowDatastore.ClusterDataStore, config configDatastore.DataStore, - imageComponentsV2 imageComponentV2Datastore.DataStore, risks riskDataStore.DataStore, vulnReqs vulnReqDataStore.DataStore, serviceAccts serviceAccountDataStore.DataStore, @@ -108,61 +128,65 @@ func newGarbageCollector(alerts alertDatastore.DataStore, plops plopDataStore.DataStore, blobStore blobDatastore.Datastore, nodeCVEStore nodeCVEDS.DataStore, + roleStore roleDataStore.DataStore, + watchedImages watchedImageDatastore.DataStore, ) GarbageCollector { return &garbageCollectorImpl{ - alerts: alerts, - clusters: clusters, - nodes: nodes, - images: images, - imagesV2: imagesV2, - imageComponentsV2: imageComponentsV2, - deployments: deployments, - pods: pods, - processes: processes, - processbaseline: processbaseline, - networkflows: networkflows, - config: config, - risks: risks, - vulnReqs: vulnReqs, - serviceAccts: serviceAccts, - k8sRoles: k8sRoles, - k8sRoleBindings: k8sRoleBindings, - logimbueStore: logimbueStore, - stopper: concurrency.NewStopper(), - postgres: globaldb.GetPostgres(), - reportSnapshot: reportSnapshotDS, - plops: plops, - blobStore: blobStore, - nodeCVEStore: nodeCVEStore, + alerts: alerts, + clusters: clusters, + nodes: nodes, + images: images, + imagesV2: imagesV2, + deployments: deployments, + pods: pods, + processes: processes, + processbaseline: processbaseline, + networkflows: networkflows, + config: config, + risks: risks, + vulnReqs: vulnReqs, + serviceAccts: serviceAccts, + k8sRoles: k8sRoles, + k8sRoleBindings: k8sRoleBindings, + logimbueStore: logimbueStore, + stopper: concurrency.NewStopper(), + postgres: globaldb.GetPostgres(), + reportSnapshot: reportSnapshotDS, + plops: plops, + blobStore: blobStore, + nodeCVEStore: nodeCVEStore, + roleStore: roleStore, + watchedImages: watchedImages, } } type garbageCollectorImpl struct { postgres pgPkg.DB - alerts alertDatastore.DataStore - clusters clusterDatastore.DataStore - nodes nodeDatastore.DataStore - images imageDatastore.DataStore - imagesV2 imageV2Datastore.DataStore - imageComponentsV2 imageComponentV2Datastore.DataStore - deployments deploymentDatastore.DataStore - pods podDatastore.DataStore - processes processDatastore.DataStore - processbaseline processBaselineDatastore.DataStore - networkflows networkFlowDatastore.ClusterDataStore - config configDatastore.DataStore - risks riskDataStore.DataStore - vulnReqs vulnReqDataStore.DataStore - serviceAccts serviceAccountDataStore.DataStore - k8sRoles k8sRoleDataStore.DataStore - k8sRoleBindings roleBindingDataStore.DataStore - logimbueStore logimbueDataStore.Store - stopper concurrency.Stopper - reportSnapshot snapshotDS.DataStore - plops plopDataStore.DataStore - blobStore blobDatastore.Datastore - nodeCVEStore nodeCVEDS.DataStore + alerts alertDatastore.DataStore + clusters clusterDatastore.DataStore + nodes nodeDatastore.DataStore + images imageDatastore.DataStore + imagesV2 imageV2Datastore.DataStore + deployments deploymentDatastore.DataStore + pods podDatastore.DataStore + processes processDatastore.DataStore + processbaseline processBaselineDatastore.DataStore + networkflows networkFlowDatastore.ClusterDataStore + config configDatastore.DataStore + risks riskDataStore.DataStore + vulnReqs vulnReqDataStore.DataStore + serviceAccts serviceAccountDataStore.DataStore + k8sRoles k8sRoleDataStore.DataStore + k8sRoleBindings roleBindingDataStore.DataStore + logimbueStore logimbueDataStore.Store + stopper concurrency.Stopper + reportSnapshot snapshotDS.DataStore + plops plopDataStore.DataStore + blobStore blobDatastore.Datastore + nodeCVEStore nodeCVEDS.DataStore + roleStore roleDataStore.DataStore + watchedImages watchedImageDatastore.DataStore } func (g *garbageCollectorImpl) Start() { @@ -170,6 +194,17 @@ func (g *garbageCollectorImpl) Start() { } func (g *garbageCollectorImpl) pruneBasedOnConfig() { + acquired, release, err := dblock.TryAcquireAdvisoryLock(pruningCtx, g.postgres, dblock.PruningGCLockID) + if err != nil { + log.Errorf("[Pruning] Failed to acquire advisory lock: %v", err) + return + } + if !acquired { + log.Info("[Pruning] Skipping cycle: advisory lock held by another process") + return + } + defer release() + pvtConfig, err := g.config.GetPrivateConfig(pruningCtx) if err != nil { log.Error(err) @@ -192,12 +227,16 @@ func (g *garbageCollectorImpl) pruneBasedOnConfig() { g.removeExpiredAdministrationEvents(pvtConfig) g.removeExpiredDiscoveredClusters() g.removeInvalidAPITokens() + g.removeExpiredDynamicRBACObjects() postgres.PruneClusterHealthStatuses(pruningCtx, g.postgres) g.pruneLogImbues() if env.OrphanedCVEsKeepAlive.BooleanSetting() { g.pruneOrphanedNodeCVEs() } + if features.FlattenImageData.Enabled() && env.PruneV1Images.BooleanSetting() { + g.pruneImageV1s() + } log.Info("[Pruning] Finished garbage collection cycle") } @@ -207,6 +246,7 @@ func (g *garbageCollectorImpl) runGC() { lastClusterPruneTime = time.Now().Add(-24 * time.Hour) lastLogImbuePruneTime = time.Now().Add(-24 * time.Hour) + lastV1ImagePruneTime = time.Now().Add(-env.V1ImagePruneInterval.DurationSetting()) g.pruneBasedOnConfig() t := time.NewTicker(pruneInterval) @@ -220,6 +260,212 @@ func (g *garbageCollectorImpl) runGC() { } } +func (g *garbageCollectorImpl) pruneImageV1s() { + if time.Since(lastV1ImagePruneTime) < env.V1ImagePruneInterval.DurationSetting() { + return + } + defer func() { lastV1ImagePruneTime = time.Now() }() + defer metrics.SetPruningDuration(time.Now(), "V1Images") + + log.Info("[Pruning] Pruning V1 images with V2 twins") + g.pruneImageV1Batch() +} + +// pruneImageV1Batch prunes V1 images from the old images table whose CVE data +// has been migrated to V2. +func (g *garbageCollectorImpl) pruneImageV1Batch() { + // Algorithm: + // 1. Fetch a batch of V1 images ordered by SHA, using cursor-based pagination. + // 2. Delete images with 0 CVEs (no scan data to preserve). + // 3. For images with CVEs, look up V2 twins from deployment containers; + // for watched (undeployed) images, generate V2 IDs from watched names. + // 4. Delete undeployed, unwatched images (no V2 twin to check). + // 5. Delete images whose V2 twins are all enriched (have >0 CVEs). + // + // We check V2 twin enrichment rather than per-CVE migration because reprocessing + // already replaces the full scan — a partial scan loses first-discovered timestamps + // regardless of image model, so this doesn't make things worse. + batchSize := env.V1ImagePruneBatchSize.IntegerSetting() + + sortOpt := search.NewSortOption(search.ImageSHA) + // Cursor-based pagination: fetch images with SHA > last processed SHA. + if lastPrunedV1ImageID != "" { + sortOpt = sortOpt.SearchAfter(lastPrunedV1ImageID) + } + q := search.NewQueryBuilder(). + WithPagination( + search.NewPagination(). + AddSortOption(sortOpt). + Limit(int32(batchSize)), + ).ProtoQuery() + + // Fetch a batch of V1 images + var v1Images []*storage.Image + err := g.images.WalkMetadataByQuery(pruningCtx, q, func(img *storage.Image) error { + v1Images = append(v1Images, img) + return nil + }) + if err != nil { + log.Errorf("[Pruning] Error fetching V1 images: %v", err) + return + } + if len(v1Images) == 0 { + lastPrunedV1ImageID = "" + return + } + + deleteIDs := g.filterV1ImageIDsToPrune(v1Images) + + if len(deleteIDs) > 0 { + if err := g.images.DeleteImages(pruningCtx, deleteIDs...); err != nil { + log.Errorf("[Pruning] Error deleting V1 images: %v", err) + return + } + log.Infof("[Pruning] Deleted %d V1 images", len(deleteIDs)) + } + + if len(v1Images) < batchSize { + lastPrunedV1ImageID = "" + } else { + lastPrunedV1ImageID = v1Images[len(v1Images)-1].GetId() + } +} + +func (g *garbageCollectorImpl) filterV1ImageIDsToPrune(v1Images []*storage.Image) []string { + var deleteIDs []string + var v1ImagesWithCVEs []*storage.Image + + for _, img := range v1Images { + // V1 images with 0 CVEs have no scan data worth preserving, safe to delete. + if img.GetCves() == 0 { + deleteIDs = append(deleteIDs, img.GetId()) + } else { + // V1 images with CVEs need to be checked against V2 twins. If all V2 twins have been enriched, we can delete the V1 image. + v1ImagesWithCVEs = append(v1ImagesWithCVEs, img) + } + } + + if len(v1ImagesWithCVEs) == 0 { + return deleteIDs + } + + var v1Digests []string + for _, img := range v1ImagesWithCVEs { + v1Digests = append(v1Digests, img.GetId()) + } + + // Image V2 twin IDs for the V1 image digests + digestToV2IDs, err := g.deployments.GetV2ImageIDsForDigests(pruningCtx, v1Digests) + if err != nil { + log.Errorf("[Pruning] Error fetching V2 image IDs for digests: %v", err) + return deleteIDs + } + + watchedNames, err := g.getWatchedNameSet() + if err != nil { + log.Errorf("[Pruning] Error fetching watched images: %v", err) + return deleteIDs + } + + for _, img := range v1ImagesWithCVEs { + digest := img.GetId() + v2IDs := digestToV2IDs[digest] + + if len(v2IDs) == 0 { + // V2 twins of a digest can also be generated from watched images. So check if the image name is watched. + watchedV2IDs := g.watchedV2IDsForImage(img, watchedNames) + if len(watchedV2IDs) == 0 { + deleteIDs = append(deleteIDs, digest) + continue + } + digestToV2IDs[digest] = watchedV2IDs + } + } + + // Get Image V2 twins from the IDs gathered above. + v2ImageMap := g.buildV2ImageMap(digestToV2IDs) + if v2ImageMap == nil { + return deleteIDs + } + + for _, img := range v1ImagesWithCVEs { + digest := img.GetId() + v2IDs := digestToV2IDs[digest] + if len(v2IDs) == 0 { + continue + } + // If all V2 twins of a v1 image are enriched, we can delete the v1 image. + if g.allV2TwinsEnriched(v2IDs, v2ImageMap) { + deleteIDs = append(deleteIDs, digest) + } + } + + return deleteIDs +} + +func (g *garbageCollectorImpl) getWatchedNameSet() (set.StringSet, error) { + if g.watchedImages == nil { + return set.NewStringSet(), nil + } + watched, err := g.watchedImages.GetAllWatchedImages(pruningCtx) + if err != nil { + return nil, err + } + names := set.NewStringSet() + for _, w := range watched { + names.Add(w.GetName()) + } + return names, nil +} + +func (g *garbageCollectorImpl) watchedV2IDsForImage(img *storage.Image, watchedNames set.StringSet) []string { + var v2IDs []string + for _, name := range img.GetNames() { + if watchedNames.Contains(name.GetFullName()) { + v2IDs = append(v2IDs, imageUtils.NewImageV2ID(name, img.GetId())) + } + } + return v2IDs +} + +func (g *garbageCollectorImpl) buildV2ImageMap(digestToV2IDs map[string][]string) map[string]*storage.ImageV2 { + var allV2IDs []string + for _, v2IDs := range digestToV2IDs { + allV2IDs = append(allV2IDs, v2IDs...) + } + if len(allV2IDs) == 0 { + return make(map[string]*storage.ImageV2) + } + + v2Images, err := g.imagesV2.GetManyImageMetadata(pruningCtx, allV2IDs) + if err != nil { + log.Errorf("[Pruning] Error fetching V2 image metadata: %v", err) + return nil + } + + v2Map := make(map[string]*storage.ImageV2, len(v2Images)) + for _, img := range v2Images { + v2Map[img.GetId()] = img + } + return v2Map +} + +// allV2TwinsEnriched returns true only if every V2 twin exists and has CVEs. +// A V2 twin with 0 CVEs (remember that corresponging V1 image has CVEs) indicates it may not have been enriched yet. +func (g *garbageCollectorImpl) allV2TwinsEnriched(v2IDs []string, v2ImageMap map[string]*storage.ImageV2) bool { + for _, v2ID := range v2IDs { + v2Img, exists := v2ImageMap[v2ID] + // It is enough to just check if the V2 twin has any CVEs. + // We don't check per-CVE migration because reprocessing already replaces the full + // scan result — a partial scan loses first-discovered timestamps regardless of + // image model, so this doesn't make things worse than they already are. + if !exists || v2Img.GetScanStats().GetCveCount() == 0 { + return false + } + } + return true +} + // Remove vulnerability requests that have expired and past the retention period. func (g *garbageCollectorImpl) removeExpiredVulnRequests() { defer metrics.SetPruningDuration(time.Now(), "VulnRequests") @@ -267,6 +513,30 @@ func (g *garbageCollectorImpl) removeOrphanedPods() { } } +// Remove deployments whose cluster has been deleted: fire-and-forget cluster-deletion cleanup can orphan them. No orphan window needed - cluster deletion is a hard delete that +// never re-adopts its deployments. +func (g *garbageCollectorImpl) removeOrphanedDeployments() { + defer metrics.SetPruningDuration(time.Now(), "Deployments") + deploymentsToRemove, err := postgres.GetOrphanedDeploymentIDs(pruningCtx, g.postgres) + if err != nil { + log.Errorf("Error finding orphaned deployments: %v", err) + return + } + + if len(deploymentsToRemove) == 0 { + log.Info("[Pruning] Found no orphaned deployments...") + return + } + log.Infof("[Pruning] Found %d orphaned deployments (from formerly deleted clusters). Deleting...", + len(deploymentsToRemove)) + + for _, d := range deploymentsToRemove { + if err := g.deployments.RemoveDeployment(pruningCtx, d.ClusterID, d.ID); err != nil { + log.Errorf("Failed to remove deployment with id %s: %v", d.ID, err) + } + } +} + // Remove nodes where the cluster has been deleted. func (g *garbageCollectorImpl) removeOrphanedNodes() { defer metrics.SetPruningDuration(time.Now(), "Nodes") @@ -340,6 +610,10 @@ func (g *garbageCollectorImpl) removeOrphanedResources() { } clusterIDSet := set.NewFrozenStringSet(clusterIDs...) + // Before the deploymentSet snapshot and alert/process/risk sweeps so they observe the + // deletions this cycle. Child rows (deployments_containers, etc.) cascade. + g.removeOrphanedDeployments() + deploymentIDs, err := g.deployments.GetDeploymentIDs(pruningCtx) if err != nil { log.Error(errors.Wrap(err, "unable to fetch deployment IDs in pruning")) @@ -395,9 +669,6 @@ func clusterIDsToNegationQuery(clusterIDSet set.FrozenStringSet) *v1.Query { func (g *garbageCollectorImpl) removeOrphanedProcesses() { defer metrics.SetPruningDuration(time.Now(), "Processes") - g.plops.PruneOrphanedPLOPsByProcessIndicators(pruningCtx, orphanWindow) - - log.Info("[PLOP pruning by processes] Pruning of orphaned PLOPs by processes complete") // Prune processes in chunks. First get the ones orphaned by deployments and then go back and // do the same for those orphaned by pod @@ -407,7 +678,7 @@ func (g *garbageCollectorImpl) removeOrphanedProcesses() { return } - deploymentIndicatorCount, err := g.removeProcesses(processesToRemove, "deployment") + deploymentIndicatorCount, err := g.removeProcesses(processesToRemove, processDatastore.PruneReasonOrphanedByDeployment, "deployment") if err != nil { log.Errorf("[Pruning] Error removing processes orphaned by deployment: %v", err) } @@ -418,7 +689,7 @@ func (g *garbageCollectorImpl) removeOrphanedProcesses() { return } - podIndicatorCount, err := g.removeProcesses(processesToRemove, "pod") + podIndicatorCount, err := g.removeProcesses(processesToRemove, processDatastore.PruneReasonOrphanedByPod, "pod") if err != nil { log.Errorf("[Pruning] Error removing processes orphaned by pod: %v", err) } @@ -427,18 +698,18 @@ func (g *garbageCollectorImpl) removeOrphanedProcesses() { deploymentIndicatorCount+podIndicatorCount, deploymentIndicatorCount, podIndicatorCount) } -func (g *garbageCollectorImpl) removeProcesses(processesToRemove []string, processParent string) (int, error) { +func (g *garbageCollectorImpl) removeProcesses(processesToRemove []string, reason, displayName string) (int, error) { if len(processesToRemove) == 0 { - log.Infof("[Pruning] Found no processes orphaned by %s...", processParent) + log.Infof("[Pruning] Found no processes orphaned by %s...", displayName) return 0, nil } log.Infof("[Pruning] Found %d orphaned processes (from formerly deleted %s). Deleting...", - len(processesToRemove), processParent) + len(processesToRemove), displayName) pruneCtxWithTimeout, cancel := contextutil.ContextWithTimeoutIfNotExists(pruningCtx, pruningTimeout) defer cancel() - return g.processes.PruneProcessIndicators(pruneCtxWithTimeout, processesToRemove) + return g.processes.PruneProcessIndicators(pruneCtxWithTimeout, processesToRemove, reason) } func (g *garbageCollectorImpl) removeOrphanedProcessBaselines(deployments set.FrozenStringSet) { @@ -514,15 +785,6 @@ func (g *garbageCollectorImpl) removeOrphanedPLOPs() { log.Errorf("error removing PLOPs with no matching process indicator or process information: %v", err) } log.Infof("[PLOP pruning] Pruning of %d orphaned PLOPs with no matching process indicator or process information complete", prunedCount) - - // Only run once since we don't expect any new PLOPs without poduids. - if !prunedPLOPsWithoutPodUIDs { - prunedCount, err = g.plops.RemovePLOPsWithoutPodUID(pruningCtx) - if err != nil { - log.Errorf("error removing PLOPs without poduid: %v", err) - } - log.Infof("[PLOP pruning] Prunned %d orphaned PLOPs with no poduid", prunedCount) - } } func (g *garbageCollectorImpl) removeExpiredAdministrationEvents(config *storage.PrivateConfig) { @@ -568,7 +830,7 @@ func (g *garbageCollectorImpl) removeOrphanedNetworkFlows(clusters set.FrozenStr // Each cluster has a separate store thus we can take advantage of doing these deletions concurrently. If we don't // the entire prune job will be stuck waiting on processing the network flows deletions in cluster sequence. - for _, c := range clusters.AsSlice() { + for c := range clusters.All() { if err := sema.Acquire(pruningCtx, 1); err != nil { log.Errorf("context cancelled via stop: %v", err) return @@ -614,37 +876,30 @@ func (g *garbageCollectorImpl) collectImages(config *storage.PrivateConfig) { log.Info("[Image Pruning] pruning is disabled.") return } - qb := search.NewQueryBuilder().AddDays(search.LastUpdatedTime, int64(pruneImageAfterDays)).ProtoQuery() if features.FlattenImageData.Enabled() { - imageResults, err := g.imagesV2.GetImageIDsAndDigests(pruningCtx, qb) + imageResults, err := postgres.GetInactiveImageIdentifiers(pruningCtx, g.postgres, int(pruneImageAfterDays)) if err != nil { log.Error(err) return } - log.Infof("[Image pruning] Found %d image search results", len(imageResults)) + log.Infof("[Image pruning] Found %d inactive images", len(imageResults)) imagesToPrune := make([]string, 0, len(imageResults)) for _, result := range imageResults { - q1 := search.NewQueryBuilder().AddExactMatches(search.ImageID, result.ImageID).ProtoQuery() - deploymentResults, err := g.deployments.Search(pruningCtx, q1) - if err != nil { - log.Errorf("[Image pruning] searching deployments: %v", err) - continue - } - if len(deploymentResults) != 0 { + // TODO(ROX-34650): This would be simpler if we stored image full name and IdV2 in + // pods_live_instances. Then we could just look up pods running the same image IdV2. + // But we don't have that right now, so this is a temporary workaround. + // + // Check if the image is still running in any terminating pods. This can happen if the deployment was updated + // to run a different image but the pods running the old image are not yet terminated. + // We should keep the image around in this case. There is an issue with this appraoch too. + // If there is one stuck image, then all images with the same digest will be kept. But it is still + // better than not pruning an image at all when there are pods running an image with the + // same digest - even though none are stuck. + if g.anyTerminatingPodsWithDigest(result) { continue } - - q2 := search.NewQueryBuilder().AddExactMatches(search.ContainerImageDigest, result.Digest).ProtoQuery() - podResults, err := g.pods.Search(pruningCtx, q2) - if err != nil { - log.Errorf("[Image pruning] searching pods: %v", err) - continue - } - if len(podResults) != 0 { - continue - } - imagesToPrune = append(imagesToPrune, result.ImageID) + imagesToPrune = append(imagesToPrune, result.ID) } if len(imagesToPrune) > 0 { log.Infof("[Image Pruning] Removing %d images", len(imagesToPrune)) @@ -654,6 +909,7 @@ func (g *garbageCollectorImpl) collectImages(config *storage.PrivateConfig) { } } } else { + qb := search.NewQueryBuilder().AddDays(search.LastUpdatedTime, int64(pruneImageAfterDays)).ProtoQuery() imageResults, err := g.images.Search(pruningCtx, qb) if err != nil { log.Error(err) @@ -694,6 +950,42 @@ func (g *garbageCollectorImpl) collectImages(config *storage.PrivateConfig) { } } +// anyTerminatingPodsWithDigest checks if a pod is still running an image with this digest while its +// deployment has moved on to a different image. Returns true if a stuck/terminating pod is found, +// meaning the image should be kept until the pod terminates. +// +// TODO(ROX-34650): Rename this to isImageActiveInPods and use IDV2 to look up pods directly. +func (g *garbageCollectorImpl) anyTerminatingPodsWithDigest(img *postgres.ImageIdentifier) bool { + // Step 1: Get deployment IDs from pods running images with this digest. + depIDsFromPods, err := g.pods.GetDeploymentIDsByDigest(pruningCtx, img.Digest) + if err != nil { + log.Errorf("[Image pruning] getting deployment IDs by digest: %v", err) + return true + } + if len(depIDsFromPods) == 0 { + return false + } + + // Step 2: Get deployment IDs whose container specs reference this digest + // but with a different image name. This will give us deploymentIDs running an image with the + // same digest but a different name. If there is a stuck pod whose deployment spec is updated with a + // different image, then this will not return that deployment ID. + depQ := search.NewQueryBuilder(). + AddExactMatches(search.ImageSHA, img.Digest). + AddStrings(search.ImageName, search.NegateQueryString(search.ExactMatchString(img.FullName))). + ProtoQuery() + depResults, err := g.deployments.Search(pruningCtx, depQ) + if err != nil { + log.Errorf("[Image pruning] searching deployments by digest: %v", err) + return true + } + depIDsWithDigest := search.ResultsToIDSet(depResults) + + // Step 3: If there is a stuck pod, its deploymentID will not be in depIDsWithDigest. + podDepIDSet := set.NewStringSet(depIDsFromPods...) + return !podDepIDSet.IsSubsetOf(depIDsWithDigest) +} + func (g *garbageCollectorImpl) removeOldReportHistory(config *storage.PrivateConfig) { defer metrics.SetPruningDuration(time.Now(), "ReportHistory") reportHistoryRetentionConfig := config.GetReportRetentionConfig().GetHistoryRetentionDurationDays() @@ -1008,7 +1300,6 @@ func (g *garbageCollectorImpl) getAlertsToPrune(query *v1.Query) ([]string, erro func (g *garbageCollectorImpl) removeOrphanedRisks() { g.removeOrphanedDeploymentRisks() g.removeOrphanedImageRisks() - g.removeOrphanedImageComponentRisks() g.removeOrphanedNodeRisks() } @@ -1046,24 +1337,6 @@ func (g *garbageCollectorImpl) removeOrphanedImageRisks() { g.removeRisks(storage.RiskSubjectType_IMAGE, prunable...) } -func (g *garbageCollectorImpl) removeOrphanedImageComponentRisks() { - defer metrics.SetPruningDuration(time.Now(), "ImageCompositionRisks") - var prunable []string - var results []search.Result - var err error - componentsWithRisk := g.getRisks(storage.RiskSubjectType_IMAGE_COMPONENT) - - results, err = g.imageComponentsV2.Search(pruningCtx, search.EmptyQuery()) - if err != nil { - log.Errorf("[Risk pruning] Searching image components: %v", err) - return - } - - prunable = componentsWithRisk.Difference(search.ResultsToIDSet(results)).AsSlice() - log.Infof("[Risk pruning] Removing %d image component risks", len(prunable)) - g.removeRisks(storage.RiskSubjectType_IMAGE_COMPONENT, prunable...) -} - func (g *garbageCollectorImpl) removeOrphanedNodeRisks() { defer metrics.SetPruningDuration(time.Now(), "NodeRisks") nodesWithRisk := g.getRisks(storage.RiskSubjectType_NODE) @@ -1149,6 +1422,65 @@ func (g *garbageCollectorImpl) pruneOrphanedNodeCVEs() { } } +// traitsHolder is an interface for objects that have traits. +type traitsHolder interface { + GetTraits() *storage.Traits +} + +// withTraitsFilter creates a filter function that applies a traits-based predicate to objects. +func withTraitsFilter[T traitsHolder](traitsPredicate func(*storage.Traits) bool) func(T) bool { + return func(obj T) bool { + return traitsPredicate(obj.GetTraits()) + } +} + +// isExpired returns true if the traits have a non-nil expires_at timestamp in the past. +func isExpired(traits *storage.Traits, now time.Time) bool { + expiresAt := traits.GetExpiresAt() + return expiresAt != nil && now.After(expiresAt.AsTime()) +} + +// removeExpiredDynamicRBACObjects removes roles, permission sets, and access scopes +// that have an expiry timestamp in the past and have IMPERATIVE origin. +// These objects are created dynamically by the internal token API for sensors. +func (g *garbageCollectorImpl) removeExpiredDynamicRBACObjects() { + // Check if dynamic RBAC pruning is enabled. + if !dynamicRBACPruningEnabled.Load() { + return + } + + defer metrics.SetPruningDuration(time.Now(), "DynamicRBACObjects") + + now := time.Now() + expiredFilter := func(traits *storage.Traits) bool { + return isExpired(traits, now) + } + + // First, remove expired roles (must be done before permission sets/access scopes + // because roles reference them and deletion will fail if still referenced). + expiredRoleCount, err := g.roleStore.RemoveFilteredRoles(pruningCtx, withTraitsFilter[*storage.Role](expiredFilter)) + if err != nil { + log.Error("Failed to remove expired roles: ", err) + } + + // Then remove expired permission sets that are no longer referenced. + expiredPSCount, err := g.roleStore.RemoveFilteredPermissionSets(pruningCtx, withTraitsFilter[*storage.PermissionSet](expiredFilter)) + if err != nil { + log.Error("Failed to remove expired permission sets: ", err) + } + + // Finally remove expired access scopes that are no longer referenced. + expiredASCount, err := g.roleStore.RemoveFilteredAccessScopes(pruningCtx, withTraitsFilter[*storage.SimpleAccessScope](expiredFilter)) + if err != nil { + log.Error("Failed to remove expired access scopes: ", err) + } + + if expiredRoleCount+expiredPSCount+expiredASCount > 0 { + log.Infof("[Expired objects pruning] Removed %d roles, %d permission sets, %d access scopes", + expiredRoleCount, expiredPSCount, expiredASCount) + } +} + func (g *garbageCollectorImpl) Stop() { g.stopper.Client().Stop() _ = g.stopper.Client().Stopped().Wait() diff --git a/central/pruning/pruning_test.go b/central/pruning/pruning_test.go index dcb8873f09409..7810806dee076 100644 --- a/central/pruning/pruning_test.go +++ b/central/pruning/pruning_test.go @@ -13,6 +13,7 @@ import ( clusterDatastore "github.com/stackrox/rox/central/cluster/datastore" clusterPostgres "github.com/stackrox/rox/central/cluster/store/cluster/postgres" clusterHealthPostgres "github.com/stackrox/rox/central/cluster/store/clusterhealth/postgres" + clusterInitStoreMocks "github.com/stackrox/rox/central/clusterinit/store/mocks" compliancePrunerMocks "github.com/stackrox/rox/central/complianceoperator/v2/pruner/mocks" configDatastore "github.com/stackrox/rox/central/config/datastore" configDatastoreMocks "github.com/stackrox/rox/central/config/datastore/mocks" @@ -50,10 +51,12 @@ import ( roleBindingMocks "github.com/stackrox/rox/central/rbac/k8srolebinding/datastore/mocks" riskDatastore "github.com/stackrox/rox/central/risk/datastore" riskDatastoreMocks "github.com/stackrox/rox/central/risk/datastore/mocks" + roleDataStore "github.com/stackrox/rox/central/role/datastore" secretMocks "github.com/stackrox/rox/central/secret/datastore/mocks" connectionMocks "github.com/stackrox/rox/central/sensor/service/connection/mocks" serviceAccountDataStore "github.com/stackrox/rox/central/serviceaccount/datastore" serviceAccountMocks "github.com/stackrox/rox/central/serviceaccount/datastore/mocks" + watchedImageDatastore "github.com/stackrox/rox/central/watchedimage/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/alert/convert" @@ -168,7 +171,7 @@ func newImageInstance(id string, daysOld int) *storage.Image { } func newDeployment(imageIDs ...string) *storage.Deployment { - var containers []*storage.Container + containers := make([]*storage.Container, 0, len(imageIDs)) for _, id := range imageIDs { digest := types.NewDigest(id).Digest() containers = append(containers, &storage.Container{ @@ -214,6 +217,7 @@ func newPod(live bool, imageIDs ...string) *storage.Pod { if live { return &storage.Pod{ Id: fixtureconsts.PodUID1, + DeploymentId: fixtureconsts.Deployment1, LiveInstances: instances, TerminatedInstances: instanceLists, } @@ -221,6 +225,7 @@ func newPod(live bool, imageIDs ...string) *storage.Pod { return &storage.Pod{ Id: fixtureconsts.PodUID2, + DeploymentId: fixtureconsts.Deployment1, TerminatedInstances: instanceLists, } } @@ -356,6 +361,7 @@ func (s *PruningTestSuite) generateClusterDataStructures() (configDatastore.Data clusterFlows := networkFlowDatastoreMocks.NewMockClusterDataStore(mockCtrl) flows := networkFlowDatastoreMocks.NewMockFlowDataStore(mockCtrl) clusterCVEs := clusterCVEDS.NewMockDataStore(mockCtrl) + clusterInitStore := clusterInitStoreMocks.NewMockStore(mockCtrl) // A bunch of these get called when a cluster is deleted flowsDataStore.EXPECT().CreateFlowStore(gomock.Any(), gomock.Any()).AnyTimes().Return(networkFlowDatastoreMocks.NewMockFlowDataStore(mockCtrl), nil) @@ -414,7 +420,10 @@ func (s *PruningTestSuite) generateClusterDataStructures() (configDatastore.Data notifierMock, ranking.NewRanker(), networkBaselineMgr, - compliancePruner) + compliancePruner, + clusterInitStore, + nil, + nil) require.NoError(s.T(), err) return mockConfigDatastore, deployments, clusterDataStore @@ -569,16 +578,19 @@ func (s *PruningTestSuite) TestImagePruning() { nodes := s.generateNodeDataStructures() gc := newGarbageCollector(alerts, nodes, images, imagesV2, nil, deployments, pods, - nil, nil, nil, config, nil, nil, + nil, nil, nil, config, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil).(*garbageCollectorImpl) + nil, nil, nil, nil).(*garbageCollectorImpl) + gc.postgres = s.pool // Add images, deployments, and pods into the datastores if c.deployment != nil { require.NoError(t, deployments.UpsertDeployment(ctx, c.deployment)) } if c.pod != nil { - c.pod.DeploymentId = c.deployment.GetId() + if c.deployment != nil { + c.pod.DeploymentId = c.deployment.GetId() + } require.NoError(t, pods.UpsertPod(ctx, c.pod)) } for _, image := range c.images { @@ -594,15 +606,13 @@ func (s *PruningTestSuite) TestImagePruning() { privateConfig, err := config.GetPrivateConfig(ctx) require.NoError(t, err, "failed to get config") - // Garbage collect all of the images gc.collectImages(privateConfig) - // Grab the actual remaining images and make sure they match the images expected to be remaining if features.FlattenImageData.Enabled() { remainingImages, err := imagesV2.SearchRawImages(ctx, search.EmptyQuery()) require.NoError(t, err) - var ids []string + ids := make([]string, 0, len(remainingImages)) for _, i := range remainingImages { ids = append(ids, i.GetDigest()) } @@ -612,7 +622,7 @@ func (s *PruningTestSuite) TestImagePruning() { assert.ElementsMatch(t, c.expectedIDs, ids) - var cleanUpIDs []string + cleanUpIDs := make([]string, 0, len(remainingImages)) for _, image := range remainingImages { cleanUpIDs = append(cleanUpIDs, image.GetId()) } @@ -625,7 +635,7 @@ func (s *PruningTestSuite) TestImagePruning() { remainingImages, err := images.SearchListImages(ctx, search.EmptyQuery()) require.NoError(t, err) - var ids []string + ids := make([]string, 0, len(remainingImages)) for _, i := range remainingImages { ids = append(ids, i.GetId()) } @@ -649,6 +659,181 @@ func (s *PruningTestSuite) TestImagePruning() { } } +func (s *PruningTestSuite) TestImagePruningSameDigestDifferentName() { + if !features.FlattenImageData.Enabled() { + s.T().Skip("only applicable when FlattenImageData is enabled") + } + + ctx := sac.WithAllAccess(context.Background()) + + digest := types.NewDigest("shareddigest").Digest() + differentDigest := types.NewDigest("newdigest").Digest() + + nameA := &storage.ImageName{FullName: "ghcr.io/stackrox/imageA:latest@" + digest} + nameB := &storage.ImageName{FullName: "ghcr.io/stackrox/imageB:latest@" + digest} + oldTimestamp := protoconv.ConvertTimeToTimestamp( + time.Now().Add(-24 * time.Duration(configDatastore.DefaultImageRetention+1) * time.Hour), + ) + + imageA := &storage.ImageV2{ + Id: utils.NewImageV2ID(nameA, digest), + Digest: digest, + Name: nameA, + LastUpdated: oldTimestamp, + } + imageB := &storage.ImageV2{ + Id: utils.NewImageV2ID(nameB, digest), + Digest: digest, + Name: nameB, + LastUpdated: oldTimestamp, + } + + type testCase struct { + name string + images []*storage.ImageV2 + deployments []*storage.Deployment + pods []*storage.Pod + expectedImageIDs []string + } + + cases := []testCase{ + { + name: "active pod from different deployment with same digest - should prune unreferenced image", + images: []*storage.ImageV2{imageA, imageB}, + deployments: []*storage.Deployment{ + { + Id: fixtureconsts.Deployment1, + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{ + Id: digest, + IdV2: imageB.GetId(), + Name: nameB, + }, + }, + }, + }, + }, + pods: []*storage.Pod{ + { + Id: fixtureconsts.PodUID1, + DeploymentId: fixtureconsts.Deployment1, + LiveInstances: []*storage.ContainerInstance{ + {ImageDigest: digest}, + }, + }, + }, + expectedImageIDs: []string{imageB.GetId()}, + }, + { + name: "stuck pod - deployment no longer references digest - should not prune", + images: []*storage.ImageV2{imageA}, + deployments: []*storage.Deployment{ + { + Id: fixtureconsts.Deployment1, + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{ + Id: differentDigest, + IdV2: uuid.NewV5FromNonUUIDs("ghcr.io/stackrox/imageA:v2@"+differentDigest, differentDigest).String(), + Name: &storage.ImageName{FullName: "ghcr.io/stackrox/imageA:v2@" + differentDigest}, + }, + }, + }, + }, + }, + pods: []*storage.Pod{ + { + Id: fixtureconsts.PodUID1, + DeploymentId: fixtureconsts.Deployment1, + LiveInstances: []*storage.ContainerInstance{ + {ImageDigest: digest}, + }, + }, + }, + expectedImageIDs: []string{imageA.GetId()}, + }, + { + name: "stuck pod in one deployment, different deployment has same digest - should not prune", + images: []*storage.ImageV2{imageA, imageB}, + deployments: []*storage.Deployment{ + { + Id: fixtureconsts.Deployment1, + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{ + Id: differentDigest, + IdV2: uuid.NewV5FromNonUUIDs("ghcr.io/stackrox/imageA:v2@"+differentDigest, differentDigest).String(), + Name: &storage.ImageName{FullName: "ghcr.io/stackrox/imageA:v2@" + differentDigest}, + }, + }, + }, + }, + { + Id: fixtureconsts.Deployment2, + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{ + Id: digest, + IdV2: imageB.GetId(), + Name: nameB, + }, + }, + }, + }, + }, + pods: []*storage.Pod{ + { + Id: fixtureconsts.PodUID1, + DeploymentId: fixtureconsts.Deployment1, + LiveInstances: []*storage.ContainerInstance{ + {ImageDigest: digest}, + }, + }, + }, + expectedImageIDs: []string{imageA.GetId(), imageB.GetId()}, + }, + } + + for _, c := range cases { + s.T().Run(c.name, func(t *testing.T) { + defer func() { + _, _ = s.pool.Exec(ctx, "TRUNCATE images_v2, deployments, pods CASCADE") + }() + + _, config, _, imagesV2, deployments, pods := s.generateImageDataStructures(ctx) + gc := newGarbageCollector(nil, nil, nil, imagesV2, nil, deployments, pods, + nil, nil, nil, config, nil, + nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil).(*garbageCollectorImpl) + gc.postgres = s.pool + + for _, dep := range c.deployments { + require.NoError(t, deployments.UpsertDeployment(ctx, dep)) + } + for _, pod := range c.pods { + require.NoError(t, pods.UpsertPod(ctx, pod)) + } + for _, img := range c.images { + require.NoError(t, imagesV2.UpsertImage(ctx, img)) + } + + privateConfig, err := config.GetPrivateConfig(ctx) + require.NoError(t, err) + gc.collectImages(privateConfig) + + remainingImages, err := imagesV2.SearchRawImages(ctx, search.EmptyQuery()) + require.NoError(t, err) + + remainingIDs := make([]string, 0, len(remainingImages)) + for _, img := range remainingImages { + remainingIDs = append(remainingIDs, img.GetId()) + } + assert.ElementsMatch(t, c.expectedImageIDs, remainingIDs) + }) + } +} + func (s *PruningTestSuite) TestClusterPruning() { s.T().Setenv(defaults.ImageFlavorEnvName, defaults.ImageFlavorNameRHACSRelease) @@ -862,9 +1047,9 @@ func (s *PruningTestSuite) TestClusterPruning() { } gc := newGarbageCollector(nil, nil, nil, nil, clusterDS, deploymentsDS, nil, - nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, - nil, nil, nil).(*garbageCollectorImpl) + nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil).(*garbageCollectorImpl) gc.collectClusters(c.config) // Now get all clusters and compare the names to ensure only the expected ones exist @@ -989,9 +1174,9 @@ func (s *PruningTestSuite) TestClusterPruningCentralCheck() { lastClusterPruneTime = time.Now().Add(-24 * time.Hour) gc := newGarbageCollector(nil, nil, nil, nil, clusterDS, deploymentsDS, nil, - nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, - nil, nil, nil).(*garbageCollectorImpl) + nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil).(*garbageCollectorImpl) gc.collectClusters(getCluserRetentionConfig(60, 90, 72)) // Now get all clusters and compare the names to ensure only the expected ones exist @@ -1167,9 +1352,9 @@ func (s *PruningTestSuite) TestAlertPruning() { nodes := s.generateNodeDataStructures() gc := newGarbageCollector(alerts, nodes, images, imagesV2, nil, deployments, nil, - nil, nil, nil, config, nil, nil, - nil, nil, nil, nil, nil, nil, - nil, nil, nil).(*garbageCollectorImpl) + nil, nil, nil, config, nil, + nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil).(*garbageCollectorImpl) // Add alerts into the datastores for _, alert := range c.alerts { @@ -1195,7 +1380,7 @@ func (s *PruningTestSuite) TestAlertPruning() { require.NoError(t, err) log.Infof("Remaining alerts: %v", remainingAlerts) - var ids []string + ids := make([]string, 0, len(remainingAlerts)) for _, i := range remainingAlerts { ids = append(ids, i.GetId()) } @@ -1342,12 +1527,12 @@ func (s *PruningTestSuite) TestRemoveOrphanedProcesses() { // Populate some actual data so the query returns what needs deleted deploymentDS, err := deploymentDatastore.GetTestPostgresDataStore(t, db.DB) s.Nil(err) - for _, deploymentID := range c.deployments.AsSlice() { + for deploymentID := range c.deployments.All() { s.NoError(deploymentDS.UpsertDeployment(s.ctx, &storage.Deployment{Id: deploymentID, ClusterId: fixtureconsts.Cluster1})) } podDS := podDatastore.GetTestPostgresDataStore(t, db.DB) - for _, podID := range c.pods.AsSlice() { + for podID := range c.pods.All() { err := podDS.UpsertPod(s.ctx, &storage.Pod{Id: podID, ClusterId: fixtureconsts.Cluster1}) s.Nil(err) } @@ -1355,7 +1540,7 @@ func (s *PruningTestSuite) TestRemoveOrphanedProcesses() { actualProcessDatastore := processIndicatorDatastore.GetTestPostgresDataStore(t, db.DB) s.NoError(actualProcessDatastore.AddProcessIndicators(s.ctx, c.initialProcesses...)) - processes.EXPECT().PruneProcessIndicators(gomock.Any(), c.expectedDeletions).AnyTimes() + processes.EXPECT().PruneProcessIndicators(gomock.Any(), c.expectedDeletions, gomock.Any()).AnyTimes() gci.removeOrphanedProcesses() }) } @@ -1498,30 +1683,6 @@ func (s *PruningTestSuite) TestRemoveOrphanedPLOPs() { pods: set.NewFrozenStringSet(fixtureconsts.PodUID1), expectedDeletions: []string{}, }, - { - name: "Plop does not have a poduid so it is removed", - initialPlops: []*storage.ProcessListeningOnPortStorage{ - { - Id: plopID1, - Port: 1234, - Protocol: storage.L4Protocol_L4_PROTOCOL_TCP, - CloseTimestamp: nil, - ProcessIndicatorId: fixtureconsts.ProcessIndicatorID1, - Closed: false, - Process: &storage.ProcessIndicatorUniqueKey{ - PodId: fixtureconsts.PodUID1, - ContainerName: "test_container1", - ProcessName: "test_process1", - ProcessArgs: "test_arguments1", - ProcessExecFilePath: "test_path1", - }, - DeploymentId: fixtureconsts.Deployment1, - }, - }, - deployments: set.NewFrozenStringSet(fixtureconsts.Deployment1), - pods: set.NewFrozenStringSet(fixtureconsts.PodUID1), - expectedDeletions: []string{plopID1}, - }, } for _, c := range cases { @@ -1533,17 +1694,16 @@ func (s *PruningTestSuite) TestRemoveOrphanedPLOPs() { postgres: db, plops: plopDS, } - prunedPLOPsWithoutPodUIDs = false // Populate some actual data so the query returns what needs deleted deploymentDS, err := deploymentDatastore.GetTestPostgresDataStore(t, db.DB) s.Nil(err) - for _, deploymentID := range c.deployments.AsSlice() { + for deploymentID := range c.deployments.All() { s.NoError(deploymentDS.UpsertDeployment(s.ctx, &storage.Deployment{Id: deploymentID, ClusterId: fixtureconsts.Cluster1})) } podDS := podDatastore.GetTestPostgresDataStore(t, db.DB) - for _, podID := range c.pods.AsSlice() { + for podID := range c.pods.All() { err := podDS.UpsertPod(s.ctx, &storage.Pod{Id: podID, ClusterId: fixtureconsts.Cluster1}) s.Nil(err) } @@ -1652,7 +1812,7 @@ func (s *PruningTestSuite) TestMarkOrphanedAlerts() { deploymentDS, err := deploymentDatastore.GetTestPostgresDataStore(t, db.DB) assert.NoError(t, err) - for _, depID := range c.deployments.AsSlice() { + for depID := range c.deployments.All() { assert.NoError(t, deploymentDS.UpsertDeployment(pruningCtx, &storage.Deployment{Id: depID})) } for _, la := range c.initialAlerts { @@ -2221,6 +2381,94 @@ func (s *PruningTestSuite) TestRemoveOrphanedPods() { s.Equal(updatedCount, podCount-cluster2PodCount) } +func (s *PruningTestSuite) TestRemoveOrphanedDeployments() { + ctrl := gomock.NewController(s.T()) + + mockRisk := riskDatastoreMocks.NewMockDataStore(ctrl) + mockRisk.EXPECT().RemoveRisk(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + mockBaseline := processBaselineDatastoreMocks.NewMockDataStore(ctrl) + mockBaseline.EXPECT().RemoveProcessBaselinesByDeployment(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + mockFilter := filterMocks.NewMockFilter(ctrl) + mockFilter.EXPECT().Delete(gomock.Any()).AnyTimes() + + flowStore := networkFlowDatastoreMocks.NewMockFlowDataStore(ctrl) + flowStore.EXPECT().RemoveFlowsForDeployment(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + flows := networkFlowDatastoreMocks.NewMockClusterDataStore(ctrl) + flows.EXPECT().GetFlowStore(gomock.Any(), gomock.Any()).AnyTimes().Return(flowStore, nil) + + deployments, err := deploymentDatastore.New(s.pool, nil, nil, mockBaseline, flows, mockRisk, nil, mockFilter, + ranking.NewRanker(), ranking.NewRanker(), ranking.NewRanker(), + platformmatcher.GetTestPlatformMatcherWithDefaultPlatformComponentConfig(ctrl)) + s.Require().NoError(err) + + _, _, clusterDS := s.generateClusterDataStructures() + liveClusterID, err := clusterDS.AddCluster(s.ctx, &storage.Cluster{Name: "testOrphanDeploymentCluster", MainImage: "docker.io/stackrox/rox:latest"}) + s.Require().NoError(err) + + // Deployment on a live (healthy) cluster - must survive the sweep. + liveDeployment := newDeployment("id1") + liveDeployment.Id = fixtureconsts.Deployment2 + liveDeployment.ClusterId = liveClusterID + s.Require().NoError(deployments.UpsertDeployment(s.ctx, liveDeployment)) + + // Clusters still in the DB but unhealthy/disconnected must keep their deployments - + // the sweep keys on the cluster row existing, not on health. + unhealthyClusters := []struct { + name string + deploymentID string + status storage.ClusterHealthStatus_HealthStatusLabel + }{ + {"testUnhealthyDeploymentCluster", fixtureconsts.Deployment3, storage.ClusterHealthStatus_UNHEALTHY}, + {"testDisconnectedDeploymentCluster", fixtureconsts.Deployment4, storage.ClusterHealthStatus_UNAVAILABLE}, + } + var unhealthyDeployments []*storage.Deployment + for _, c := range unhealthyClusters { + clusterID, err := clusterDS.AddCluster(s.ctx, &storage.Cluster{Name: c.name, MainImage: "docker.io/stackrox/rox:latest"}) + s.Require().NoError(err) + s.Require().NoError(clusterDS.UpdateClusterHealth(s.ctx, clusterID, + &storage.ClusterHealthStatus{SensorHealthStatus: c.status})) + + dep := newDeployment("img") + dep.Id = c.deploymentID + dep.ClusterId = clusterID + s.Require().NoError(deployments.UpsertDeployment(s.ctx, dep)) + unhealthyDeployments = append(unhealthyDeployments, dep) + } + + // Deployment whose clusterid has no cluster row - orphaned. + orphanedDeployment := newDeployment("id2") + orphanedDeployment.ClusterId = fixtureconsts.Cluster1 + s.Require().NoError(deployments.UpsertDeployment(s.ctx, orphanedDeployment)) + + gci := &garbageCollectorImpl{ + deployments: deployments, + postgres: s.pool, + } + + gci.removeOrphanedDeployments() + + _, exists, err := deployments.GetDeployment(s.ctx, orphanedDeployment.GetId()) + s.Require().NoError(err) + s.False(exists, "orphaned deployment should be pruned") + + _, exists, err = deployments.GetDeployment(s.ctx, liveDeployment.GetId()) + s.Require().NoError(err) + s.True(exists, "deployment of a live cluster must not be pruned") + + for _, dep := range unhealthyDeployments { + _, exists, err = deployments.GetDeployment(s.ctx, dep.GetId()) + s.Require().NoError(err) + s.True(exists, "deployment of a cluster still in the DB must not be pruned") + } + + // Cascaded deletion of deployments_containers rows is what makes the referenced + // image prunable - assert none remain. + var containerCount int + err = s.pool.QueryRow(s.ctx, "SELECT count(*) FROM deployments_containers WHERE deployments_id = $1", orphanedDeployment.GetId()).Scan(&containerCount) + s.Require().NoError(err) + s.Zero(containerCount) +} + func (s *PruningTestSuite) TestRemoveOrphanedNodes() { _, _, clusterDS := s.generateClusterDataStructures() @@ -2325,8 +2573,380 @@ func (s *PruningTestSuite) TestPruneOrphanedNodeCVEs() { } } +func (s *PruningTestSuite) TestRemoveExpiredDynamicRBACObjects() { + EnableDynamicRBACPruning() + + now := time.Now() + yesterday := now.Add(-24 * time.Hour) + tomorrow := now.Add(24 * time.Hour) + twoDaysAgo := now.Add(-48 * time.Hour) + + // Create classic (non-expiring) access scope and permission set for roles to reference. + classicPS := &storage.PermissionSet{ + Id: uuid.NewV4().String(), + Name: "classic-ps", + ResourceToAccess: map[string]storage.Access{"Cluster": storage.Access_READ_ACCESS}, + Traits: nil, + } + classicAS := &storage.SimpleAccessScope{ + Id: uuid.NewV4().String(), + Name: "classic-as", + Rules: &storage.SimpleAccessScope_Rules{}, + Traits: nil, + } + + // Pre-generate UUIDs for test cases where we need to reference them in expected deletions. + ps1ID := uuid.NewV4().String() + ps2ID := uuid.NewV4().String() + ps3ID := uuid.NewV4().String() + as1ID := uuid.NewV4().String() + as2ID := uuid.NewV4().String() + as3ID := uuid.NewV4().String() + psExpiredID := uuid.NewV4().String() + psNoExpiryID := uuid.NewV4().String() + asExpiredID := uuid.NewV4().String() + asActiveID := uuid.NewV4().String() + + cases := []struct { + name string + roles []*storage.Role + permissionSets []*storage.PermissionSet + accessScopes []*storage.SimpleAccessScope + expectedRoleDeletions set.FrozenStringSet + expectedPSDeletions set.FrozenStringSet + expectedASDeletions set.FrozenStringSet + }{ + { + name: "remove expired roles only", + roles: []*storage.Role{ + { + Name: "role-1", + PermissionSetId: classicPS.GetId(), + AccessScopeId: classicAS.GetId(), + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(yesterday), + }, + }, + { + Name: "role-2", + PermissionSetId: classicPS.GetId(), + AccessScopeId: classicAS.GetId(), + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(tomorrow), + }, + }, + { + Name: "role-3", + PermissionSetId: classicPS.GetId(), + AccessScopeId: classicAS.GetId(), + Traits: &storage.Traits{}, + }, + }, + expectedRoleDeletions: set.NewFrozenStringSet("role-1"), + expectedPSDeletions: set.NewFrozenStringSet(), + expectedASDeletions: set.NewFrozenStringSet(), + }, + { + name: "remove expired permission sets only", + permissionSets: []*storage.PermissionSet{ + { + Id: ps1ID, + Name: "ps-1", + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(twoDaysAgo), + }, + }, + { + Id: ps2ID, + Name: "ps-2", + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(tomorrow), + }, + }, + { + Id: ps3ID, + Name: "ps-3", + Traits: &storage.Traits{}, + }, + }, + expectedRoleDeletions: set.NewFrozenStringSet(), + expectedPSDeletions: set.NewFrozenStringSet(ps1ID), + expectedASDeletions: set.NewFrozenStringSet(), + }, + { + name: "remove expired access scopes only", + accessScopes: []*storage.SimpleAccessScope{ + { + Id: as1ID, + Name: "as-1", + Rules: &storage.SimpleAccessScope_Rules{}, + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(yesterday), + }, + }, + { + Id: as2ID, + Name: "as-2", + Rules: &storage.SimpleAccessScope_Rules{}, + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(tomorrow), + }, + }, + { + Id: as3ID, + Name: "as-3", + Rules: &storage.SimpleAccessScope_Rules{}, + Traits: &storage.Traits{}, + }, + }, + expectedRoleDeletions: set.NewFrozenStringSet(), + expectedPSDeletions: set.NewFrozenStringSet(), + expectedASDeletions: set.NewFrozenStringSet(as1ID), + }, + { + name: "remove expired objects of all types", + roles: []*storage.Role{ + { + Name: "role-expired", + PermissionSetId: classicPS.GetId(), + AccessScopeId: classicAS.GetId(), + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(yesterday), + }, + }, + { + Name: "role-active", + PermissionSetId: classicPS.GetId(), + AccessScopeId: classicAS.GetId(), + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(tomorrow), + }, + }, + }, + permissionSets: []*storage.PermissionSet{ + { + Id: psExpiredID, + Name: "ps-expired", + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(twoDaysAgo), + }, + }, + { + Id: psNoExpiryID, + Name: "ps-no-expiry", + Traits: &storage.Traits{}, + }, + }, + accessScopes: []*storage.SimpleAccessScope{ + { + Id: asExpiredID, + Name: "as-expired", + Rules: &storage.SimpleAccessScope_Rules{}, + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(yesterday), + }, + }, + { + Id: asActiveID, + Name: "as-active", + Rules: &storage.SimpleAccessScope_Rules{}, + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(tomorrow), + }, + }, + }, + expectedRoleDeletions: set.NewFrozenStringSet("role-expired"), + expectedPSDeletions: set.NewFrozenStringSet(psExpiredID), + expectedASDeletions: set.NewFrozenStringSet(asExpiredID), + }, + { + name: "nothing to remove when all unexpired", + roles: []*storage.Role{ + { + Name: "role-1", + PermissionSetId: classicPS.GetId(), + AccessScopeId: classicAS.GetId(), + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(tomorrow), + }, + }, + }, + permissionSets: []*storage.PermissionSet{ + { + Id: uuid.NewV4().String(), + Name: "ps-1", + Traits: &storage.Traits{}, + }, + }, + accessScopes: []*storage.SimpleAccessScope{ + { + Id: uuid.NewV4().String(), + Name: "as-1", + Rules: &storage.SimpleAccessScope_Rules{}, + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(tomorrow), + }, + }, + }, + expectedRoleDeletions: set.NewFrozenStringSet(), + expectedPSDeletions: set.NewFrozenStringSet(), + expectedASDeletions: set.NewFrozenStringSet(), + }, + { + name: "nothing to remove when no objects exist", + expectedRoleDeletions: set.NewFrozenStringSet(), + expectedPSDeletions: set.NewFrozenStringSet(), + expectedASDeletions: set.NewFrozenStringSet(), + }, + } + + for _, c := range cases { + s.T().Run(c.name, func(t *testing.T) { + roleStore := roleDataStore.GetTestPostgresDataStore(t, s.pool) + + // Add classic permission set and access scope for roles to reference. + assert.NoError(t, roleStore.AddPermissionSet(pruningCtx, classicPS)) + assert.NoError(t, roleStore.AddAccessScope(pruningCtx, classicAS)) + t.Cleanup(func() { + _ = roleStore.RemovePermissionSet(pruningCtx, classicPS.GetId()) + _ = roleStore.RemoveAccessScope(pruningCtx, classicAS.GetId()) + }) + + for _, role := range c.roles { + assert.NoError(t, roleStore.AddRole(pruningCtx, role)) + t.Cleanup(func() { + _ = roleStore.RemoveRole(pruningCtx, role.GetName()) + }) + } + + for _, ps := range c.permissionSets { + assert.NoError(t, roleStore.AddPermissionSet(pruningCtx, ps)) + t.Cleanup(func() { + _ = roleStore.RemovePermissionSet(pruningCtx, ps.GetId()) + }) + } + + for _, as := range c.accessScopes { + assert.NoError(t, roleStore.AddAccessScope(pruningCtx, as)) + t.Cleanup(func() { + _ = roleStore.RemoveAccessScope(pruningCtx, as.GetId()) + }) + } + + gc := &garbageCollectorImpl{ + roleStore: roleStore, + } + + gc.removeExpiredDynamicRBACObjects() + + for _, role := range c.roles { + _, ok, err := roleStore.GetRole(pruningCtx, role.GetName()) + assert.NoError(t, err) + assert.Equal(t, !c.expectedRoleDeletions.Contains(role.GetName()), ok) + } + + for _, ps := range c.permissionSets { + _, ok, err := roleStore.GetPermissionSet(pruningCtx, ps.GetId()) + assert.NoError(t, err) + assert.Equal(t, !c.expectedPSDeletions.Contains(ps.GetId()), ok) + } + + for _, as := range c.accessScopes { + _, ok, err := roleStore.GetAccessScope(pruningCtx, as.GetId()) + assert.NoError(t, err) + assert.Equal(t, !c.expectedASDeletions.Contains(as.GetId()), ok) + } + }) + } +} + +func (s *PruningTestSuite) TestRemoveExpiredDynamicRBACObjects_WhenDisabled() { + disableDynamicRBACPruningForTest(s.T()) + + now := time.Now() + yesterday := now.Add(-24 * time.Hour) + + // Create classic (non-expiring) access scope and permission set for roles to reference. + classicPS := &storage.PermissionSet{ + Id: uuid.NewV4().String(), + Name: "classic-ps-disabled", + ResourceToAccess: map[string]storage.Access{"Cluster": storage.Access_READ_ACCESS}, + Traits: nil, + } + classicAS := &storage.SimpleAccessScope{ + Id: uuid.NewV4().String(), + Name: "classic-as-disabled", + Rules: &storage.SimpleAccessScope_Rules{}, + Traits: nil, + } + + roleStore := roleDataStore.GetTestPostgresDataStore(s.T(), s.pool) + + // Add classic permission set and access scope for roles to reference. + s.Require().NoError(roleStore.AddPermissionSet(pruningCtx, classicPS)) + s.Require().NoError(roleStore.AddAccessScope(pruningCtx, classicAS)) + s.T().Cleanup(func() { + _ = roleStore.RemovePermissionSet(pruningCtx, classicPS.GetId()) + _ = roleStore.RemoveAccessScope(pruningCtx, classicAS.GetId()) + }) + + // Create expired RBAC objects. + expiredRole := &storage.Role{ + Name: "expired-role-disabled", + PermissionSetId: classicPS.GetId(), + AccessScopeId: classicAS.GetId(), + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(yesterday), + }, + } + expiredPS := &storage.PermissionSet{ + Id: uuid.NewV4().String(), + Name: "expired-ps-disabled", + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(yesterday), + }, + } + expiredAS := &storage.SimpleAccessScope{ + Id: uuid.NewV4().String(), + Name: "expired-as-disabled", + Rules: &storage.SimpleAccessScope_Rules{}, + Traits: &storage.Traits{ + ExpiresAt: timestamppb.New(yesterday), + }, + } + + s.Require().NoError(roleStore.AddRole(pruningCtx, expiredRole)) + s.Require().NoError(roleStore.AddPermissionSet(pruningCtx, expiredPS)) + s.Require().NoError(roleStore.AddAccessScope(pruningCtx, expiredAS)) + s.T().Cleanup(func() { + _ = roleStore.RemoveRole(pruningCtx, expiredRole.GetName()) + _ = roleStore.RemovePermissionSet(pruningCtx, expiredPS.GetId()) + _ = roleStore.RemoveAccessScope(pruningCtx, expiredAS.GetId()) + }) + + gc := &garbageCollectorImpl{ + roleStore: roleStore, + } + + // Call pruning - it should be a no-op since pruning is disabled. + gc.removeExpiredDynamicRBACObjects() + + // Verify that none of the expired objects were deleted. + _, ok, err := roleStore.GetRole(pruningCtx, expiredRole.GetName()) + s.NoError(err) + s.True(ok, "expired role should still exist when pruning is disabled") + + _, ok, err = roleStore.GetPermissionSet(pruningCtx, expiredPS.GetId()) + s.NoError(err) + s.True(ok, "expired permission set should still exist when pruning is disabled") + + _, ok, err = roleStore.GetAccessScope(pruningCtx, expiredAS.GetId()) + s.NoError(err) + s.True(ok, "expired access scope should still exist when pruning is disabled") +} + func (s *PruningTestSuite) addSomePods(podDS podDatastore.DataStore, clusterID string, numberPods int) { - for i := 0; i < numberPods; i++ { + for range numberPods { pod := &storage.Pod{ Id: uuid.NewV4().String(), ClusterId: clusterID, @@ -2337,7 +2957,7 @@ func (s *PruningTestSuite) addSomePods(podDS podDatastore.DataStore, clusterID s } func (s *PruningTestSuite) addNodes(nodeDS testNodeDatastore.DataStore, clusterID string, numberOfNodes int) { - for i := 0; i < numberOfNodes; i++ { + for range numberOfNodes { pod := &storage.Node{ Id: uuid.NewV4().String(), ClusterId: clusterID, @@ -2362,3 +2982,204 @@ func resourceWithAccess(access storage.Access, resource permissions.ResourceMeta Resource: resource, } } + +func (s *PruningTestSuite) generateImageV1PruningDataStructures() (imageDatastore.DataStore, imageV2Datastore.DataStore, deploymentDatastore.DataStore, watchedImageDatastore.DataStore) { + ctrl := gomock.NewController(s.T()) + + mockRiskDatastore := riskDatastoreMocks.NewMockDataStore(ctrl) + mockRiskDatastore.EXPECT().RemoveRisk(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + mockFilter := filterMocks.NewMockFilter(ctrl) + mockFilter.EXPECT().UpdateByPod(gomock.Any()).AnyTimes() + mockFilter.EXPECT().DeleteByPod(gomock.Any()).AnyTimes() + mockFilter.EXPECT().Delete(gomock.Any()).AnyTimes() + + mockBaselineDataStore := processBaselineDatastoreMocks.NewMockDataStore(ctrl) + mockBaselineDataStore.EXPECT().RemoveProcessBaselinesByDeployment(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + + mockNetworkFlows := networkFlowDatastoreMocks.NewMockClusterDataStore(ctrl) + mockFlowStore := networkFlowDatastoreMocks.NewMockFlowDataStore(ctrl) + mockNetworkFlows.EXPECT().GetFlowStore(gomock.Any(), gomock.Any()).AnyTimes().Return(mockFlowStore, nil) + mockFlowStore.EXPECT().RemoveFlowsForDeployment(gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + + images := imageDatastore.NewWithPostgres( + imagePostgresV2.New(s.pool, true, concurrency.NewKeyFence()), + mockRiskDatastore, + ranking.ImageRanker(), + ranking.ComponentRanker(), + ) + imagesV2 := imageV2Datastore.NewWithPostgres( + imageV2Postgres.New(s.pool, true, concurrency.NewKeyFence()), + mockRiskDatastore, + ranking.ImageRanker(), + ranking.ComponentRanker(), + ) + deployments, err := deploymentDatastore.New(s.pool, nil, nil, mockBaselineDataStore, mockNetworkFlows, mockRiskDatastore, nil, mockFilter, + ranking.NewRanker(), ranking.NewRanker(), ranking.NewRanker(), + platformmatcher.GetTestPlatformMatcherWithDefaultPlatformComponentConfig(ctrl)) + s.Require().NoError(err) + + watchedImages := watchedImageDatastore.GetTestPostgresDataStore(s.T(), s.pool) + + return images, imagesV2, deployments, watchedImages +} + +func newV1ImageWithCVEs(id string, cves int32) *storage.Image { + digest := types.NewDigest(id).Digest() + name := &storage.ImageName{FullName: "test/" + id + ":latest"} + return &storage.Image{ + Id: digest, + LastUpdated: protoconv.ConvertTimeToTimestamp(time.Now()), + Name: name, + Names: []*storage.ImageName{name}, + SetCves: &storage.Image_Cves{Cves: cves}, + SetComponents: &storage.Image_Components{Components: cves}, + } +} + +func (s *PruningTestSuite) TestPruneImageV1Batch() { + if !features.FlattenImageData.Enabled() { + s.T().Skip("V1 image pruning only runs when FlattenImageData is enabled") + } + if !env.PruneV1Images.BooleanSetting() { + s.T().Skip("V1 image pruning is disabled") + } + + ctx := sac.WithAllAccess(context.Background()) + images, imagesV2, depDS, watchedDS := s.generateImageV1PruningDataStructures() + + // Create V1 images covering all pruning scenarios. + image1 := newV1ImageWithCVEs("img-no-cves", 0) + image2 := newV1ImageWithCVEs("img-enriched", 5) + image3 := newV1ImageWithCVEs("img-not-enriched", 3) + image4 := newV1ImageWithCVEs("img-no-twin", 2) + image5 := newV1ImageWithCVEs("img-missing-v2", 4) + image6 := newV1ImageWithCVEs("img-watched-enriched", 6) + image7 := newV1ImageWithCVEs("img-watched-not-enriched", 7) + image8 := newV1ImageWithCVEs("img-watched-no-v2", 8) + + v1Images := []*storage.Image{image1, image2, image3, image4, image5, image6, image7, image8} + for _, img := range v1Images { + s.Require().NoError(images.UpsertImage(ctx, img)) + } + + // Create V2 images: one enriched, one not enriched. + v2IDImg2 := utils.NewImageV2ID(image2.GetName(), image2.GetId()) + v2IDImg3 := utils.NewImageV2ID(image3.GetName(), image3.GetId()) + v2IDImg5 := utils.NewImageV2ID(image5.GetName(), image5.GetId()) + v2IDImg6 := utils.NewImageV2ID(image6.GetName(), image6.GetId()) + v2IDImg7 := utils.NewImageV2ID(image7.GetName(), image7.GetId()) + + s.Require().NoError(imagesV2.UpsertImage(ctx, &storage.ImageV2{ + Id: v2IDImg2, Digest: image2.GetId(), + Name: image2.GetName(), + ScanStats: &storage.ImageV2_ScanStats{CveCount: 5}, + })) + s.Require().NoError(imagesV2.UpsertImage(ctx, &storage.ImageV2{ + Id: v2IDImg3, Digest: image3.GetId(), + Name: image3.GetName(), + ScanStats: &storage.ImageV2_ScanStats{CveCount: 0}, + })) + // Watched image6: V2 twin enriched. + s.Require().NoError(imagesV2.UpsertImage(ctx, &storage.ImageV2{ + Id: v2IDImg6, Digest: image6.GetId(), + Name: image6.GetName(), + ScanStats: &storage.ImageV2_ScanStats{CveCount: 6}, + })) + // Watched image7: V2 twin not enriched. + s.Require().NoError(imagesV2.UpsertImage(ctx, &storage.ImageV2{ + Id: v2IDImg7, Digest: image7.GetId(), + Name: image7.GetName(), + ScanStats: &storage.ImageV2_ScanStats{CveCount: 0}, + })) + + // Watch image6, image7, and image8 (none are deployed). + s.Require().NoError(watchedDS.UpsertWatchedImage(ctx, image6.GetName().GetFullName())) + s.Require().NoError(watchedDS.UpsertWatchedImage(ctx, image7.GetName().GetFullName())) + s.Require().NoError(watchedDS.UpsertWatchedImage(ctx, image8.GetName().GetFullName())) + + // Create deployment referencing image2 (enriched), image3 (not enriched), and image5 (missing from images_v2). + dep := &storage.Deployment{ + Id: fixtureconsts.Deployment1, + Containers: []*storage.Container{ + {Image: &storage.ContainerImage{Id: image2.GetId(), IdV2: v2IDImg2, Name: image2.GetName()}}, + {Image: &storage.ContainerImage{Id: image3.GetId(), IdV2: v2IDImg3, Name: image3.GetName()}}, + {Image: &storage.ContainerImage{Id: image5.GetId(), IdV2: v2IDImg5, Name: image5.GetName()}}, + }, + } + s.Require().NoError(depDS.UpsertDeployment(ctx, dep)) + + gc := &garbageCollectorImpl{ + images: images, + imagesV2: imagesV2, + deployments: depDS, + watchedImages: watchedDS, + postgres: s.pool, + } + + lastPrunedV1ImageID = "" + gc.pruneImageV1Batch() + + remaining, err := images.SearchListImages(ctx, search.EmptyQuery()) + s.Require().NoError(err) + + var remainingIDs []string + for _, img := range remaining { + remainingIDs = append(remainingIDs, img.GetId()) + } + + // image1 (0 CVEs): deleted + // image2 (deployed, V2 twin enriched): deleted + // image3 (deployed, V2 twin not enriched): kept + // image4 (not deployed, not watched): deleted + // image5 (deployed, V2 missing from images_v2): kept + // image6 (not deployed, watched, V2 twin enriched): deleted + // image7 (not deployed, watched, V2 twin not enriched): kept + // image8 (not deployed, watched, V2 missing): kept + s.Assert().ElementsMatch([]string{image3.GetId(), image5.GetId(), image7.GetId(), image8.GetId()}, remainingIDs) + + // Cleanup + _ = images.DeleteImages(ctx, image3.GetId(), image5.GetId(), image7.GetId(), image8.GetId()) + _ = imagesV2.DeleteImages(ctx, v2IDImg2, v2IDImg3, v2IDImg6, v2IDImg7) + _ = depDS.RemoveDeployment(ctx, "", dep.GetId()) +} + +func (s *PruningTestSuite) TestPruneImageV1BatchCursor() { + if !features.FlattenImageData.Enabled() { + s.T().Skip("V1 image pruning only runs when FlattenImageData is enabled") + } + if !env.PruneV1Images.BooleanSetting() { + s.T().Skip("V1 image pruning is disabled") + } + + ctx := sac.WithAllAccess(context.Background()) + images, imagesV2, depDS, watchedDS := s.generateImageV1PruningDataStructures() + + for range 3 { + img := newV1ImageWithCVEs(uuid.NewV4().String(), 0) + s.Require().NoError(images.UpsertImage(ctx, img)) + } + + gc := &garbageCollectorImpl{ + images: images, + imagesV2: imagesV2, + deployments: depDS, + watchedImages: watchedDS, + postgres: s.pool, + } + + // With batch size = 2, first batch processes 2, cursor advances. + s.T().Setenv("ROX_V1_IMAGE_PRUNE_BATCH_SIZE", "2") + + lastPrunedV1ImageID = "" + gc.pruneImageV1Batch() + s.Assert().NotEmpty(lastPrunedV1ImageID, "cursor should advance after full batch") + + // Second batch processes remaining 1, cursor resets. + gc.pruneImageV1Batch() + s.Assert().Empty(lastPrunedV1ImageID, "cursor should reset when batch < batchSize") + + remaining, err := images.SearchListImages(ctx, search.EmptyQuery()) + s.Require().NoError(err) + s.Assert().Empty(remaining, "all 0-CVE images should be pruned") +} diff --git a/central/pruning/singleton.go b/central/pruning/singleton.go index 1f6640d32460d..cf2cb4dd6f4f8 100644 --- a/central/pruning/singleton.go +++ b/central/pruning/singleton.go @@ -8,7 +8,6 @@ import ( nodeCVEDS "github.com/stackrox/rox/central/cve/node/datastore" deploymentDatastore "github.com/stackrox/rox/central/deployment/datastore" imagesDatastore "github.com/stackrox/rox/central/image/datastore" - imageComponentV2Datastore "github.com/stackrox/rox/central/imagecomponent/v2/datastore" imageV2Datastore "github.com/stackrox/rox/central/imagev2/datastore" logimbueStore "github.com/stackrox/rox/central/logimbue/store" networkFlowsDataStore "github.com/stackrox/rox/central/networkgraph/flow/datastore" @@ -21,8 +20,10 @@ import ( k8srolebindingStore "github.com/stackrox/rox/central/rbac/k8srolebinding/datastore" snapshotDataStore "github.com/stackrox/rox/central/reports/snapshot/datastore" riskDataStore "github.com/stackrox/rox/central/risk/datastore" + roleDataStore "github.com/stackrox/rox/central/role/datastore" serviceAccountDataStore "github.com/stackrox/rox/central/serviceaccount/datastore" vulnReqDataStore "github.com/stackrox/rox/central/vulnmgmt/vulnerabilityrequest/datastore" + watchedImageDatastore "github.com/stackrox/rox/central/watchedimage/datastore" "github.com/stackrox/rox/pkg/sync" ) @@ -45,7 +46,6 @@ func Singleton() GarbageCollector { processBaselineDatastore.Singleton(), networkFlowsDataStore.Singleton(), configDatastore.Singleton(), - imageComponentV2Datastore.Singleton(), riskDataStore.Singleton(), vulnReqDataStore.Singleton(), serviceAccountDataStore.Singleton(), @@ -56,6 +56,8 @@ func Singleton() GarbageCollector { plopDataStore.Singleton(), blobDS.Singleton(), nodeCVEDS.Singleton(), + roleDataStore.Singleton(), + watchedImageDatastore.Singleton(), ) }) return gc diff --git a/central/rbac/k8srole/datastore/datastore_impl.go b/central/rbac/k8srole/datastore/datastore_impl.go index a8861375b9daf..ac79f60beca73 100644 --- a/central/rbac/k8srole/datastore/datastore_impl.go +++ b/central/rbac/k8srole/datastore/datastore_impl.go @@ -90,7 +90,3 @@ func (c *K8SRoleSearchResultConverter) BuildLocation(result *searchPkg.Result) s func (c *K8SRoleSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_ROLES } - -func (c *K8SRoleSearchResultConverter) GetScore(result *searchPkg.Result) float64 { - return result.Score -} diff --git a/central/rbac/k8srolebinding/datastore/datastore_impl.go b/central/rbac/k8srolebinding/datastore/datastore_impl.go index 369a5eca145c2..485c062bf721e 100644 --- a/central/rbac/k8srolebinding/datastore/datastore_impl.go +++ b/central/rbac/k8srolebinding/datastore/datastore_impl.go @@ -96,7 +96,3 @@ func (c *K8SRoleBindingSearchResultConverter) BuildLocation(result *searchPkg.Re func (c *K8SRoleBindingSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_ROLEBINDINGS } - -func (c *K8SRoleBindingSearchResultConverter) GetScore(result *searchPkg.Result) float64 { - return result.Score -} diff --git a/central/reports/common/query_builder.go b/central/reports/common/query_builder.go index 5de45059da538..cbf2ada69d1c3 100644 --- a/central/reports/common/query_builder.go +++ b/central/reports/common/query_builder.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/pkg/errors" collectionDataStore "github.com/stackrox/rox/central/resourcecollection/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" @@ -27,17 +28,20 @@ type queryBuilder struct { collection *storage.ResourceCollection collectionQueryResolver collectionDataStore.QueryResolver dataStartTime time.Time + entityScope *storage.EntityScope } // NewVulnReportQueryBuilder builds a query builder to build scope and cve filtering queries for vuln reporting -func NewVulnReportQueryBuilder(collection *storage.ResourceCollection, vulnFilters *storage.VulnerabilityReportFilters, +func NewVulnReportQueryBuilder(collection *storage.ResourceCollection, entityScope *storage.EntityScope, vulnFilters *storage.VulnerabilityReportFilters, collectionQueryRes collectionDataStore.QueryResolver, dataStartTime time.Time) *queryBuilder { return &queryBuilder{ - vulnFilters: vulnFilters, collection: collection, + entityScope: entityScope, + vulnFilters: vulnFilters, collectionQueryResolver: collectionQueryRes, dataStartTime: dataStartTime, } + } // BuildQuery builds scope and cve filtering queries for vuln reporting @@ -46,7 +50,13 @@ func (q *queryBuilder) BuildQuery( clusters []effectiveaccessscope.Cluster, namespaces []effectiveaccessscope.Namespace, ) (*ReportQuery, error) { - deploymentsQuery, err := q.collectionQueryResolver.ResolveCollectionQuery(ctx, q.collection) + deploymentsQuery := search.EmptyQuery() + var err error + if q.collection != nil { + deploymentsQuery, err = q.collectionQueryResolver.ResolveCollectionQuery(ctx, q.collection) + } else if q.entityScope != nil { + deploymentsQuery, err = q.buildEntityScopeQuery() + } if err != nil { return nil, err } @@ -66,7 +76,9 @@ func (q *queryBuilder) BuildQuery( }, nil } -func (q *queryBuilder) buildCVEAttributesQuery() (string, error) { +// buildLegacyFilterQuery() adds severity, fixability filters for collection scoped reports +func (q *queryBuilder) buildLegacyFilterQuery() []string { + vulnReportFilters := q.vulnFilters var conjuncts []string @@ -86,13 +98,26 @@ func (q *queryBuilder) buildCVEAttributesQuery() (string, error) { if len(severities) > 0 { conjuncts = append(conjuncts, search.NewQueryBuilder().AddExactMatches(search.Severity, severities...).Query()) } + return conjuncts +} +func (q *queryBuilder) buildCVEAttributesQuery() (string, error) { + + vulnReportFilters := q.vulnFilters + var conjuncts []string + + if q.collection != nil { + // for collections only add fixability, severity filters for CVE + conjuncts = q.buildLegacyFilterQuery() + } else if q.entityScope != nil { + // for entity scoped reports add all the search filters from query string + conjuncts = append(conjuncts, q.vulnFilters.GetQuery()) + } if filterVulnsByFirstOccurrenceTime(vulnReportFilters) { startTimeStr := fmt.Sprintf(">=%s", q.dataStartTime.Format("01/02/2006 3:04:05 PM MST")) tsQ := search.NewQueryBuilder().AddStrings(search.FirstImageOccurrenceTimestamp, startTimeStr).Query() conjuncts = append(conjuncts, tsQ) } - return strings.Join(conjuncts, "+"), nil } @@ -129,6 +154,132 @@ func (q *queryBuilder) buildAccessScopeQuery( return scopeQuery, nil } +// buildEntityScopeQuery uses entity scope object to build v1 query +func (q *queryBuilder) buildEntityScopeQuery() (*v1.Query, error) { + rules := q.entityScope.GetRules() + if len(rules) == 0 { + return search.EmptyQuery(), nil + } + + var conjuncts []*v1.Query + for _, rule := range rules { + if len(rule.GetValues()) == 0 { + continue + } + + fieldLabel, err := entityScopeRuleToFieldLabel(rule) + if err != nil { + return nil, err + } + isMapField := fieldLabel == search.DeploymentLabel || + fieldLabel == search.NamespaceLabel || + fieldLabel == search.ClusterLabel || + fieldLabel == search.DeploymentAnnotation || + fieldLabel == search.NamespaceAnnotation + + if isMapField { + conjuncts = append(conjuncts, MapFieldQuery(fieldLabel, rule)) + } else { + conjuncts = append(conjuncts, ScalarFieldQuery(fieldLabel, rule)) + } + } + + if len(conjuncts) == 0 { + return search.EmptyQuery(), nil + } + return search.ConjunctionQuery(conjuncts...), nil +} + +// MapFieldQuery builds a disjunction query for a map-type search field +// (labels/annotations) from the values in an entity scope rule. +func MapFieldQuery(fieldLabel search.FieldLabel, rule *storage.EntityScopeRule) *v1.Query { + mapQueries := make([]*v1.Query, 0, len(rule.GetValues())) + for _, rv := range rule.GetValues() { + val := rv.GetValue() + key, value := splitLabelValue(val) + if rv.GetMatchType() == storage.MatchType_REGEX { + rawParts := strings.SplitN(val, "=", 2) + rawValue := "" + if len(rawParts) == 2 { + rawValue = rawParts[1] + } + key = mapPartQueryString(rawParts[0]) + value = mapPartQueryString(rawValue) + } + mapQueries = append(mapQueries, + search.NewQueryBuilder().AddMapQuery(fieldLabel, key, value).ProtoQuery()) + } + return search.DisjunctionQuery(mapQueries...) +} + +// ScalarFieldQuery builds a disjunction query for a scalar-type search field +// (names, IDs) from the values in an entity scope rule. +func ScalarFieldQuery(fieldLabel search.FieldLabel, rule *storage.EntityScopeRule) *v1.Query { + values := rule.GetValues() + ruleQueries := make([]*v1.Query, 0, len(values)) + for _, rv := range values { + val := rv.GetValue() + if rv.GetMatchType() == storage.MatchType_REGEX { + ruleQueries = append(ruleQueries, + search.NewQueryBuilder().AddRegexes(fieldLabel, val).ProtoQuery()) + } else { + ruleQueries = append(ruleQueries, + search.NewQueryBuilder().AddExactMatches(fieldLabel, val).ProtoQuery()) + } + } + return search.DisjunctionQuery(ruleQueries...) +} + +// entityScopeRuleToFieldLabel returns search filter for given entity field pair +func entityScopeRuleToFieldLabel(rule *storage.EntityScopeRule) (search.FieldLabel, error) { + switch rule.GetEntity() { + case storage.EntityType_ENTITY_TYPE_DEPLOYMENT: + switch rule.GetField() { + case storage.EntityField_FIELD_NAME: + return search.DeploymentName, nil + case storage.EntityField_FIELD_LABEL: + return search.DeploymentLabel, nil + case storage.EntityField_FIELD_ANNOTATION: + return search.DeploymentAnnotation, nil + } + case storage.EntityType_ENTITY_TYPE_NAMESPACE: + switch rule.GetField() { + case storage.EntityField_FIELD_NAME: + return search.Namespace, nil + case storage.EntityField_FIELD_LABEL: + return search.NamespaceLabel, nil + case storage.EntityField_FIELD_ANNOTATION: + return search.NamespaceAnnotation, nil + } + case storage.EntityType_ENTITY_TYPE_CLUSTER: + switch rule.GetField() { + case storage.EntityField_FIELD_NAME: + return search.Cluster, nil + case storage.EntityField_FIELD_LABEL: + return search.ClusterLabel, nil + } + } + return "", errors.Errorf("Unsupported entity/field combination %s/%s", rule.GetEntity(), rule.GetField()) +} + func filterVulnsByFirstOccurrenceTime(vulnReportFilters *storage.VulnerabilityReportFilters) bool { return vulnReportFilters.GetSinceLastSentScheduledReport() || vulnReportFilters.GetSinceStartDate() != nil } + +// split map field values like namespace labels(key=val) to key,val pair +func splitLabelValue(labelVal string) (string, string) { + parts := strings.SplitN(labelVal, "=", 2) + if len(parts) == 2 { + return fmt.Sprintf("%q", parts[0]), fmt.Sprintf("%q", parts[1]) + } + return fmt.Sprintf("%q", labelVal), fmt.Sprintf("%q", "") +} + +// mapPartQueryString returns a regex query string (r/...) if the part dos not contain +// a regex prefix; otherwise returns the part. +func mapPartQueryString(part string) string { + if strings.HasPrefix(part, search.RegexPrefix) { + return part + } + return search.RegexQueryString(part) +} diff --git a/central/reports/common/query_builder_test.go b/central/reports/common/query_builder_test.go index cb30397775eae..33124ea2f53be 100644 --- a/central/reports/common/query_builder_test.go +++ b/central/reports/common/query_builder_test.go @@ -197,3 +197,274 @@ func TestBuildAccessScopeQuery(t *testing.T) { func assertByDirectComparison(t testing.TB, expected *v1.Query, actual *v1.Query) { protoassert.Equal(t, expected, actual) } + +func TestBuildClusterOnlyAccessScopeQuery(t *testing.T) { + testCases := map[string]struct { + rules []*storage.SimpleAccessScope_Rules + expectedQ *v1.Query + }{ + "nil rules returns empty query": { + rules: nil, + expectedQ: search.EmptyQuery(), + }, + "cluster included by name": { + rules: []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"remote"}}, + }, + expectedQ: search.NewQueryBuilder().AddExactMatches(search.ClusterID, clusters[0].GetId()).ProtoQuery(), + }, + "namespace-only rule grants cluster visibility": { + rules: []*storage.SimpleAccessScope_Rules{ + { + IncludedNamespaces: []*storage.SimpleAccessScope_Rules_Namespace{ + {ClusterName: "secured", NamespaceName: "ns2"}, + }, + }, + }, + expectedQ: search.NewQueryBuilder().AddExactMatches(search.ClusterID, clusters[1].GetId()).ProtoQuery(), + }, + "exclude all rules returns match none": { + rules: []*storage.SimpleAccessScope_Rules{ + {}, + }, + expectedQ: getMatchNoneQuery(), + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + result, err := BuildClusterOnlyAccessScopeQuery(tc.rules, clusters, namespaces) + assert.NoError(t, err) + protoassert.Equal(t, tc.expectedQ, result) + }) + } +} + +func TestBuildEntityScopeQuery(t *testing.T) { + testCases := []struct { + name string + scope *storage.EntityScope + expected *v1.Query + assertQueries func(t testing.TB, expected *v1.Query, actual *v1.Query) + hasError bool + }{ + { + name: "Empty rules returns empty query (match all)", + scope: &storage.EntityScope{}, + expected: search.EmptyQuery(), + assertQueries: assertByDirectComparison, + }, + { + name: "Namespace rule", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "prod", MatchType: storage.MatchType_EXACT}, + {Value: "staging", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + expected: search.DisjunctionQuery( + search.NewQueryBuilder().AddExactMatches(search.Namespace, "prod").ProtoQuery(), + search.NewQueryBuilder().AddExactMatches(search.Namespace, "staging").ProtoQuery(), + ), + assertQueries: assertByDirectComparison, + }, + { + name: "Single deployment name rule", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "web-server", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + expected: search.NewQueryBuilder().AddExactMatches(search.DeploymentName, "web-server").ProtoQuery(), + assertQueries: assertByDirectComparison, + }, + { + name: "Cluster name rule", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "prod-us", MatchType: storage.MatchType_EXACT}, + {Value: "prod-eu", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + expected: search.DisjunctionQuery( + search.NewQueryBuilder().AddExactMatches(search.Cluster, "prod-us").ProtoQuery(), + search.NewQueryBuilder().AddExactMatches(search.Cluster, "prod-eu").ProtoQuery(), + ), + assertQueries: assertByDirectComparison, + }, + { + name: "Multiple rules are ANDed", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "prod", MatchType: storage.MatchType_EXACT}, + }, + }, + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "backend", MatchType: storage.MatchType_EXACT}, + {Value: "frontend", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + expected: search.ConjunctionQuery( + search.NewQueryBuilder().AddExactMatches(search.Namespace, "prod").ProtoQuery(), + search.DisjunctionQuery( + search.NewQueryBuilder().AddExactMatches(search.DeploymentName, "backend").ProtoQuery(), + search.NewQueryBuilder().AddExactMatches(search.DeploymentName, "frontend").ProtoQuery(), + ), + ), + assertQueries: assertByDirectComparison, + }, + { + name: "Label rule uses map query", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_LABEL, + Values: []*storage.RuleValue{ + {Value: "env=prod", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + expected: search.NewQueryBuilder().AddMapQuery(search.NamespaceLabel, `"env"`, `"prod"`).ProtoQuery(), + assertQueries: assertByDirectComparison, + }, + { + name: "Regex match type adds r/ prefix", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "web-.*", MatchType: storage.MatchType_REGEX}, + }, + }, + }, + }, + expected: search.NewQueryBuilder().AddStrings(search.DeploymentName, "r/web-.*").ProtoQuery(), + assertQueries: assertByDirectComparison, + }, + { + name: "Namespace label regex match type with regex prefix", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_LABEL, + Values: []*storage.RuleValue{ + {Value: "r/env=pr.*", MatchType: storage.MatchType_REGEX}, + }, + }, + }, + }, + // key "env" has regex prefix + expected: search.NewQueryBuilder().AddMapQuery(search.NamespaceLabel, "r/env", "r/pr.*").ProtoQuery(), + assertQueries: assertByDirectComparison, + }, + { + name: "Namespace label regex match type", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_LABEL, + Values: []*storage.RuleValue{ + {Value: "env=pr.*", MatchType: storage.MatchType_REGEX}, + }, + }, + }, + }, + // key "env" has no metacharacters → exact match; value "pr.*" has "." and "*" → regex + expected: search.NewQueryBuilder().AddMapQuery(search.NamespaceLabel, "r/env", "r/pr.*").ProtoQuery(), + assertQueries: assertByDirectComparison, + }, + { + name: "Rule with empty values is skipped", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{}, + }, + }, + }, + expected: search.EmptyQuery(), + assertQueries: assertByDirectComparison, + }, + { + name: "Unsupported entity/field returns error", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_ANNOTATION, + Values: []*storage.RuleValue{ + {Value: "team=infra", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + hasError: true, + }, + { + name: "Deployment annotation rule", + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_ANNOTATION, + Values: []*storage.RuleValue{ + {Value: "owner=team-a", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + expected: search.NewQueryBuilder().AddMapQuery(search.DeploymentAnnotation, `"owner"`, `"team-a"`).ProtoQuery(), + assertQueries: assertByDirectComparison, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + qb := &queryBuilder{ + entityScope: tc.scope, + } + result, err := qb.buildEntityScopeQuery() + if tc.hasError { + assert.Error(t, err) + return + } + assert.NoError(t, err) + tc.assertQueries(t, tc.expected, result) + }) + } +} diff --git a/central/reports/common/report_formatter.go b/central/reports/common/report_formatter.go index 9554a275fce82..3fe21053bd4a8 100644 --- a/central/reports/common/report_formatter.go +++ b/central/reports/common/report_formatter.go @@ -171,7 +171,13 @@ func Format(deployedImagesResults []DeployedImagesResult, watchedImagesResults [ var zipBuf bytes.Buffer zipWriter := zip.NewWriter(&zipBuf) - zipFile, err := zipWriter.Create(makeFileName(configName, time.Now())) + now := time.Now() + header := &zip.FileHeader{ + Name: makeFileName(configName, now), + Method: zip.Deflate, + Modified: now, + } + zipFile, err := zipWriter.CreateHeader(header) if err != nil { return nil, errors.Wrap(err, "unable to create a zip file of the vuln report") } diff --git a/central/reports/common/report_formatter_test.go b/central/reports/common/report_formatter_test.go index 9a454b161a1e6..fe675eec3d3fd 100644 --- a/central/reports/common/report_formatter_test.go +++ b/central/reports/common/report_formatter_test.go @@ -1,11 +1,14 @@ package common import ( + "archive/zip" + "bytes" "strings" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_replaceUnsafeRunes(t *testing.T) { @@ -25,6 +28,18 @@ func Test_replaceUnsafeRunes(t *testing.T) { } } +func TestFormat_ZipEntryHasModifiedTimestamp(t *testing.T) { + before := time.Now().Add(-time.Minute) + result, err := Format(nil, nil, "test", false) + require.NoError(t, err) + + reader, err := zip.NewReader(bytes.NewReader(result.ZippedCsv.Bytes()), int64(result.ZippedCsv.Len())) + require.NoError(t, err) + require.Len(t, reader.File, 1) + assert.False(t, reader.File[0].Modified.IsZero(), "ZIP entry Modified timestamp should not be zero") + assert.True(t, reader.File[0].Modified.After(before), "ZIP entry Modified timestamp should be recent") +} + func Test_makeFileName(t *testing.T) { cases := map[string]string{ "file name": "RHACS_Vulnerability_Report_file_name_31_December_2023.csv", diff --git a/central/reports/common/testutils.go b/central/reports/common/testutils.go deleted file mode 100644 index 4d49a74c2766c..0000000000000 --- a/central/reports/common/testutils.go +++ /dev/null @@ -1,54 +0,0 @@ -package common - -import ( - "testing" - - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/fixtures" -) - -// GetTestReportConfigsV1 returns v1 configs for testing -func GetTestReportConfigsV1(_ *testing.T, notifierID, collectionID string) []*storage.ReportConfiguration { - v1Config1 := fixtures.GetValidReportConfigWithMultipleNotifiersV1() - v1Config1.Id = "" - v1Config1.Name = "Report Config - 1" - v1Config1.GetEmailConfig().NotifierId = notifierID - v1Config1.ScopeId = collectionID - - v1Config2 := fixtures.GetValidReportConfigWithMultipleNotifiersV1() - v1Config2.Id = "" - v1Config2.Name = "Report Config - 2" - v1Config2.GetEmailConfig().NotifierId = notifierID - v1Config2.ScopeId = collectionID - - return []*storage.ReportConfiguration{v1Config1, v1Config2} -} - -// GetTestReportConfigsV2 returns v2 configs for testing -func GetTestReportConfigsV2(_ *testing.T, notifierID, collectionID string) []*storage.ReportConfiguration { - v2Config1 := fixtures.GetValidReportConfigWithMultipleNotifiersV2() - v2Config1.Id = "" - v2Config1.Name = "Report Config - 1" - for _, notifierConf := range v2Config1.GetNotifiers() { - notifierConf.Ref = &storage.NotifierConfiguration_Id{ - Id: notifierID, - } - } - v2Config1.ResourceScope.ScopeReference = &storage.ResourceScope_CollectionId{ - CollectionId: collectionID, - } - - v2Config2 := fixtures.GetValidReportConfigWithMultipleNotifiersV2() - v2Config2.Id = "" - v2Config2.Name = "Report Config - 2" - for _, notifierConf := range v2Config2.GetNotifiers() { - notifierConf.Ref = &storage.NotifierConfiguration_Id{ - Id: notifierID, - } - } - v2Config2.ResourceScope.ScopeReference = &storage.ResourceScope_CollectionId{ - CollectionId: collectionID, - } - - return []*storage.ReportConfiguration{v2Config1, v2Config2} -} diff --git a/central/reports/common/utils.go b/central/reports/common/utils.go index 324747d223e9e..ebac8257908a8 100644 --- a/central/reports/common/utils.go +++ b/central/reports/common/utils.go @@ -39,28 +39,40 @@ func ExtractAccessScopeRules(identity authn.Identity) []*storage.SimpleAccessSco return accessScopeRulesList } -// IsV1ReportConfig returns true if the given config belongs to reporting version 1.0 -func IsV1ReportConfig(config *storage.ReportConfiguration) bool { - return config.GetResourceScope() == nil -} - -// IsV2ReportConfig returns true if the given config belongs to reporting version 2.0 -func IsV2ReportConfig(config *storage.ReportConfiguration) bool { - return config.GetResourceScope() != nil -} - -// WithoutV2ReportConfigs adds a conjunction query to exclude v2 report configs -func WithoutV2ReportConfigs(query *v1.Query) *v1.Query { - return search.ConjunctionQuery( - query, - search.NewQueryBuilder().AddExactMatches(search.CollectionID, "").ProtoQuery()) +// HasValidResourceScope returns true if the report config has a non-empty resource scope +// containing either a collection ID , entity scope etc. Returns false when ResourceScope +// is nil or is an empty message with no scope_reference set. Later can happen in downgrade scenarios. +func HasValidResourceScope(scope *storage.ResourceScope) bool { + if scope == nil || scope.GetScopeReference() == nil { + return false + } + return true } -// WithoutV1ReportConfigs adds a conjunction query to exclude v1 report configs -func WithoutV1ReportConfigs(query *v1.Query) *v1.Query { - return search.ConjunctionQuery( - query, - search.NewQueryBuilder().AddExactMatches(search.EmbeddedCollectionID, "").ProtoQuery()) +// BuildClusterOnlyAccessScopeQuery builds a v1 query that filters by cluster ID +// only. Currently nodes from partially included clusters (where only a subset of +// namespaces are accessible) are visible to users, so the scope is computed with +// both clusters and namespaces to match that behavior. +func BuildClusterOnlyAccessScopeQuery( + accessScopeRules []*storage.SimpleAccessScope_Rules, + clusters []effectiveaccessscope.Cluster, + namespaces []effectiveaccessscope.Namespace, +) (*v1.Query, error) { + if accessScopeRules == nil { + return search.EmptyQuery(), nil + } + scopeTree, err := computeScopeTree(accessScopeRules, clusters, namespaces) + if err != nil { + return nil, err + } + scopeQuery, err := sac.BuildClusterLevelSACQueryFilter(scopeTree) + if err != nil { + return nil, err + } + if scopeQuery == nil { + return search.EmptyQuery(), nil + } + return scopeQuery, nil } // BuildAccessScopeQuery builds v1 query for given access scope rules @@ -72,6 +84,25 @@ func BuildAccessScopeQuery( if accessScopeRules == nil { return search.EmptyQuery(), nil } + scopeTree, err := computeScopeTree(accessScopeRules, clusters, namespaces) + if err != nil { + return nil, err + } + scopeQuery, err := sac.BuildClusterNamespaceLevelSACQueryFilter(scopeTree) + if err != nil { + return nil, err + } + if scopeQuery == nil { + return search.EmptyQuery(), nil + } + return scopeQuery, nil +} + +func computeScopeTree( + accessScopeRules []*storage.SimpleAccessScope_Rules, + clusters []effectiveaccessscope.Cluster, + namespaces []effectiveaccessscope.Namespace, +) (*effectiveaccessscope.ScopeTree, error) { var scopeTree *effectiveaccessscope.ScopeTree for _, rules := range accessScopeRules { sct, err := effectiveaccessscope.ComputeEffectiveAccessScope(rules, clusters, namespaces, v1.ComputeEffectiveAccessScopeRequest_MINIMAL) @@ -84,12 +115,5 @@ func BuildAccessScopeQuery( scopeTree.Merge(sct) } } - scopeQuery, err := sac.BuildClusterNamespaceLevelSACQueryFilter(scopeTree) - if err != nil { - return nil, err - } - if scopeQuery == nil { - return search.EmptyQuery(), nil - } - return scopeQuery, nil + return scopeTree, nil } diff --git a/central/reports/config/datastore/datastore_impl_postgres_test.go b/central/reports/config/datastore/datastore_impl_postgres_test.go index ba98c32b4b550..2e664a3d74316 100644 --- a/central/reports/config/datastore/datastore_impl_postgres_test.go +++ b/central/reports/config/datastore/datastore_impl_postgres_test.go @@ -7,6 +7,7 @@ import ( "fmt" "testing" + notifierDS "github.com/stackrox/rox/central/notifier/datastore" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/postgres/pgtest" @@ -25,14 +26,16 @@ func TestReportConfigurationPostgresDatastore(t *testing.T) { type ReportConfigurationPostgresDatastoreTests struct { suite.Suite - testDB *pgtest.TestPostgres - datastore DataStore - ctx context.Context + testDB *pgtest.TestPostgres + datastore DataStore + notifierDataStore notifierDS.DataStore + ctx context.Context } func (s *ReportConfigurationPostgresDatastoreTests) SetupSuite() { s.testDB = pgtest.ForT(s.T()) s.datastore = GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.notifierDataStore = notifierDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) s.ctx = sac.WithGlobalAccessScopeChecker(context.Background(), sac.AllowFixedScopes( sac.AccessModeScopeKeys(storage.Access_READ_ACCESS, storage.Access_READ_WRITE_ACCESS), @@ -41,10 +44,13 @@ func (s *ReportConfigurationPostgresDatastoreTests) SetupSuite() { func (s *ReportConfigurationPostgresDatastoreTests) TearDownTest() { s.truncateTable(postgresSchema.ReportConfigurationsTableName) + s.truncateTable(postgresSchema.NotifiersTableName) } func (s *ReportConfigurationPostgresDatastoreTests) TestReportsConfigDataStore() { reportConfig := fixtures.GetValidReportConfiguration() + s.storeNotifiers(reportConfig) + // Test add _, err := s.datastore.AddReportConfiguration(s.ctx, reportConfig) s.Require().NoError(err) @@ -92,7 +98,8 @@ func (s *ReportConfigurationPostgresDatastoreTests) TestReportsConfigDataStore() } func (s *ReportConfigurationPostgresDatastoreTests) TestMultipleReportNotifiers() { - reportConfig := fixtures.GetValidReportConfigWithMultipleNotifiersV1() + reportConfig := fixtures.GetValidReportConfigWithMultipleNotifiersV2() + s.storeNotifiers(reportConfig) // Test add _, err := s.datastore.AddReportConfiguration(s.ctx, reportConfig) @@ -106,7 +113,7 @@ func (s *ReportConfigurationPostgresDatastoreTests) TestMultipleReportNotifiers( } func (s *ReportConfigurationPostgresDatastoreTests) TestNoNotifiers() { - reportConfig := fixtures.GetValidReportConfigWithMultipleNotifiersV1() + reportConfig := fixtures.GetValidReportConfigWithMultipleNotifiersV2() reportConfig.Notifiers = nil // Test add @@ -120,6 +127,15 @@ func (s *ReportConfigurationPostgresDatastoreTests) TestNoNotifiers() { protoassert.Equal(s.T(), reportConfig, foundReportConfig) } +func (s *ReportConfigurationPostgresDatastoreTests) storeNotifiers(config *storage.ReportConfiguration) { + allCtx := sac.WithAllAccess(context.Background()) + for i, n := range config.GetNotifiers() { + id, err := s.notifierDataStore.AddNotifier(allCtx, &storage.Notifier{Name: n.GetId()}) + s.Require().NoError(err) + config.Notifiers[i].Ref = &storage.NotifierConfiguration_Id{Id: id} + } +} + func (s *ReportConfigurationPostgresDatastoreTests) truncateTable(name string) { sql := fmt.Sprintf("TRUNCATE %s CASCADE", name) _, err := s.testDB.Exec(s.ctx, sql) diff --git a/central/reports/config/service/config_separation_test.go b/central/reports/config/service/config_separation_test.go deleted file mode 100644 index 2970776384d7c..0000000000000 --- a/central/reports/config/service/config_separation_test.go +++ /dev/null @@ -1,172 +0,0 @@ -//go:build sql_integration - -package service - -import ( - "context" - "fmt" - "testing" - - notifierDS "github.com/stackrox/rox/central/notifier/datastore" - "github.com/stackrox/rox/central/reports/common" - reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" - managerMocks "github.com/stackrox/rox/central/reports/manager/mocks" - collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" - apiV1 "github.com/stackrox/rox/generated/api/v1" - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/postgres/pgtest" - "github.com/stackrox/rox/pkg/protoassert" - "github.com/stackrox/rox/pkg/sac" - "github.com/stretchr/testify/suite" - "go.uber.org/mock/gomock" -) - -func TestServiceLevelConfigSeparation(t *testing.T) { - suite.Run(t, new(ServiceLevelConfigSeparationSuite)) -} - -type ServiceLevelConfigSeparationSuite struct { - suite.Suite - service Service - - testDB *pgtest.TestPostgres - ctx context.Context - reportConfigDatastore reportConfigDS.DataStore - notifierDatastore notifierDS.DataStore - collectionDatastore collectionDS.DataStore - manager *managerMocks.MockManager - mockCtrl *gomock.Controller - - v1Configs []*storage.ReportConfiguration - v2Configs []*storage.ReportConfiguration -} - -func (s *ServiceLevelConfigSeparationSuite) SetupSuite() { - s.testDB = pgtest.ForT(s.T()) - s.reportConfigDatastore = reportConfigDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) - s.notifierDatastore = notifierDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) - - var err error - s.collectionDatastore, _, err = collectionDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) - s.Require().NoError(err) - - s.mockCtrl = gomock.NewController(s.T()) - s.manager = managerMocks.NewMockManager(s.mockCtrl) - s.service = New(s.reportConfigDatastore, s.notifierDatastore, s.collectionDatastore, s.manager) - - s.ctx = sac.WithAllAccess(context.Background()) - - // Add notifier - notifierID, err := s.notifierDatastore.AddNotifier(s.ctx, &storage.Notifier{Name: "notifier"}) - s.Require().NoError(err) - - // Add collection - collectionID, err := s.collectionDatastore.AddCollection(s.ctx, &storage.ResourceCollection{Name: "collection"}) - s.Require().NoError(err) - - // Add report configs - s.v1Configs = common.GetTestReportConfigsV1(s.T(), notifierID, collectionID) - for _, conf := range s.v1Configs { - conf.Id, err = s.reportConfigDatastore.AddReportConfiguration(s.ctx, conf) - s.Require().NoError(err) - } - - s.v2Configs = common.GetTestReportConfigsV2(s.T(), notifierID, collectionID) - for _, conf := range s.v2Configs { - conf.Id, err = s.reportConfigDatastore.AddReportConfiguration(s.ctx, conf) - s.Require().NoError(err) - } -} - -func (s *ServiceLevelConfigSeparationSuite) TearDownSuite() { - s.mockCtrl.Finish() - -} - -func (s *ServiceLevelConfigSeparationSuite) TestGetReportConfigurations() { - // Empty Query - res, err := s.service.GetReportConfigurations(s.ctx, &apiV1.RawQuery{Query: ""}) - s.Require().NoError(err) - protoassert.ElementsMatch(s.T(), s.v1Configs, res.GetReportConfigs()) - - // Non empty query - res, err = s.service.GetReportConfigurations(s.ctx, - &apiV1.RawQuery{Query: fmt.Sprintf("Report Name:%s", s.v1Configs[0].GetName())}) - s.Require().NoError(err) - s.Require().Equal(1, len(res.GetReportConfigs())) - protoassert.Equal(s.T(), s.v1Configs[0], res.GetReportConfigs()[0]) -} - -func (s *ServiceLevelConfigSeparationSuite) TestGetReportConfiguration() { - // returns v1 config - res, err := s.service.GetReportConfiguration(s.ctx, &apiV1.ResourceByID{Id: s.v1Configs[0].GetId()}) - s.Require().NoError(err) - protoassert.Equal(s.T(), s.v1Configs[0], res.GetReportConfig()) - - // error on requesting v2 config - _, err = s.service.GetReportConfiguration(s.ctx, &apiV1.ResourceByID{Id: s.v2Configs[0].GetId()}) - s.Require().Error(err) -} - -func (s *ServiceLevelConfigSeparationSuite) TestCountReportConfigurations() { - // Empty query - res, err := s.service.CountReportConfigurations(s.ctx, &apiV1.RawQuery{Query: ""}) - s.Require().NoError(err) - s.Require().Equal(int32(len(s.v1Configs)), res.GetCount()) - - // Non empty query - res, err = s.service.CountReportConfigurations(s.ctx, - &apiV1.RawQuery{Query: fmt.Sprintf("Report Name:%s", s.v1Configs[0].GetName())}) - s.Require().NoError(err) - s.Require().Equal(int32(1), res.GetCount()) -} - -func (s *ServiceLevelConfigSeparationSuite) TestPostReportConfiguration() { - // Error on v2 config - config := s.v2Configs[0].CloneVT() - config.Id = "" - config.NotifierConfig = s.v1Configs[0].GetNotifierConfig() - _, err := s.service.PostReportConfiguration(s.ctx, &apiV1.PostReportConfigurationRequest{ReportConfig: config}) - s.Require().Error(err) - - // No error on v1 config - s.manager.EXPECT().Upsert(gomock.Any(), gomock.Any()).Return(nil).Times(1) - config = s.v1Configs[0].CloneVT() - config.Id = "" - res, err := s.service.PostReportConfiguration(s.ctx, &apiV1.PostReportConfigurationRequest{ReportConfig: config}) - s.Require().NoError(err) - - err = s.reportConfigDatastore.RemoveReportConfiguration(s.ctx, res.GetReportConfig().GetId()) - s.Require().NoError(err) -} - -func (s *ServiceLevelConfigSeparationSuite) TestUpdateReportConfiguration() { - // Error on v2 config - config := s.v2Configs[0].CloneVT() - config.NotifierConfig = s.v1Configs[0].GetNotifierConfig() - config.GetVulnReportFilters().SinceLastReport = true - _, err := s.service.UpdateReportConfiguration(s.ctx, &apiV1.UpdateReportConfigurationRequest{ReportConfig: config}) - s.Require().Error(err) - - // No error on v1 config - s.manager.EXPECT().Upsert(gomock.Any(), gomock.Any()).Return(nil).Times(1) - s.v1Configs[0].GetVulnReportFilters().SinceLastReport = true - _, err = s.service.UpdateReportConfiguration(s.ctx, &apiV1.UpdateReportConfigurationRequest{ReportConfig: s.v1Configs[0]}) - s.Require().NoError(err) -} - -func (s *ServiceLevelConfigSeparationSuite) TestDeleteReportConfiguration() { - // Error on v2 config ID - _, err := s.service.DeleteReportConfiguration(s.ctx, &apiV1.ResourceByID{Id: s.v2Configs[0].GetId()}) - s.Require().Error(err) - - // No error on v1 config ID - s.manager.EXPECT().Remove(gomock.Any(), gomock.Any()).Return(nil).Times(1) - config := s.v1Configs[0].CloneVT() - config.Id = "" - config.Name = "Delete report config" - config.Id, err = s.reportConfigDatastore.AddReportConfiguration(s.ctx, config) - s.Require().NoError(err) - _, err = s.service.DeleteReportConfiguration(s.ctx, &apiV1.ResourceByID{Id: config.GetId()}) - s.Require().NoError(err) -} diff --git a/central/reports/config/service/service.go b/central/reports/config/service/service.go deleted file mode 100644 index ce2f4065df8c0..0000000000000 --- a/central/reports/config/service/service.go +++ /dev/null @@ -1,31 +0,0 @@ -package service - -import ( - "context" - - notifierDataStore "github.com/stackrox/rox/central/notifier/datastore" - "github.com/stackrox/rox/central/reports/config/datastore" - "github.com/stackrox/rox/central/reports/manager" - collectionDataStore "github.com/stackrox/rox/central/resourcecollection/datastore" - v1 "github.com/stackrox/rox/generated/api/v1" - "github.com/stackrox/rox/pkg/grpc" -) - -// Service provides the interface to the gRPC service for roles. -type Service interface { - grpc.APIService - - AuthFuncOverride(ctx context.Context, fullMethodName string) (context.Context, error) - v1.ReportConfigurationServiceServer -} - -// New returns a new instance of the service. Please use the Singleton instead. -func New(reportConfigStore datastore.DataStore, notifierStore notifierDataStore.DataStore, - collectionDatastore collectionDataStore.DataStore, manager manager.Manager) Service { - return &serviceImpl{ - manager: manager, - reportConfigStore: reportConfigStore, - collectionDatastore: collectionDatastore, - notifierDatastore: notifierStore, - } -} diff --git a/central/reports/config/service/service_impl.go b/central/reports/config/service/service_impl.go deleted file mode 100644 index 080d18807857f..0000000000000 --- a/central/reports/config/service/service_impl.go +++ /dev/null @@ -1,167 +0,0 @@ -package service - -import ( - "context" - - "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" - "github.com/pkg/errors" - notifierDS "github.com/stackrox/rox/central/notifier/datastore" - "github.com/stackrox/rox/central/reports/common" - "github.com/stackrox/rox/central/reports/config/datastore" - "github.com/stackrox/rox/central/reports/manager" - collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" - v1 "github.com/stackrox/rox/generated/api/v1" - "github.com/stackrox/rox/pkg/auth/permissions" - "github.com/stackrox/rox/pkg/errox" - "github.com/stackrox/rox/pkg/grpc/authz" - "github.com/stackrox/rox/pkg/grpc/authz/perrpc" - "github.com/stackrox/rox/pkg/grpc/authz/user" - "github.com/stackrox/rox/pkg/sac/resources" - "github.com/stackrox/rox/pkg/search" - "github.com/stackrox/rox/pkg/search/paginated" - "google.golang.org/grpc" -) - -var ( - // authorizer is used for authorizing report configuration grpc service calls. - authorizer = perrpc.FromMap(map[authz.Authorizer][]string{ - user.With(permissions.View(resources.WorkflowAdministration)): { - v1.ReportConfigurationService_GetReportConfigurations_FullMethodName, - v1.ReportConfigurationService_GetReportConfiguration_FullMethodName, - v1.ReportConfigurationService_CountReportConfigurations_FullMethodName, - }, - user.With(permissions.Modify(resources.WorkflowAdministration), permissions.View(resources.Integration)): { - v1.ReportConfigurationService_PostReportConfiguration_FullMethodName, - v1.ReportConfigurationService_UpdateReportConfiguration_FullMethodName, - }, - user.With(permissions.Modify(resources.WorkflowAdministration)): { - v1.ReportConfigurationService_DeleteReportConfiguration_FullMethodName, - }, - }) -) - -type serviceImpl struct { - v1.UnimplementedReportConfigurationServiceServer - - manager manager.Manager - reportConfigStore datastore.DataStore - collectionDatastore collectionDS.DataStore - notifierDatastore notifierDS.DataStore -} - -func (s *serviceImpl) GetReportConfigurations(ctx context.Context, query *v1.RawQuery) (*v1.GetReportConfigurationsResponse, error) { - // Fill in Query. - parsedQuery, err := search.ParseQuery(query.GetQuery(), search.MatchAllIfEmpty()) - if err != nil { - return nil, errors.Wrap(errox.InvalidArgs, err.Error()) - } - - filteredQ := common.WithoutV2ReportConfigs(parsedQuery) - - // Fill in pagination. - paginated.FillPagination(filteredQ, query.GetPagination(), 1000) - - reportConfigs, err := s.reportConfigStore.GetReportConfigurations(ctx, filteredQ) - if err != nil { - return nil, errors.Wrap(err, "failed to retrieve report configurations") - } - return &v1.GetReportConfigurationsResponse{ReportConfigs: reportConfigs}, nil -} - -func (s *serviceImpl) GetReportConfiguration(ctx context.Context, id *v1.ResourceByID) (*v1.GetReportConfigurationResponse, error) { - reportConfig, exists, err := s.reportConfigStore.GetReportConfiguration(ctx, id.GetId()) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.Wrapf(errox.NotFound, "report configuration with id '%s' does not exist", id) - } - if !common.IsV1ReportConfig(reportConfig) { - return nil, errors.Wrap(errox.InvalidArgs, "report configuration does not belong to reporting version 1.0") - } - return &v1.GetReportConfigurationResponse{ - ReportConfig: reportConfig, - }, nil -} - -func (s *serviceImpl) PostReportConfiguration(ctx context.Context, request *v1.PostReportConfigurationRequest) (*v1.PostReportConfigurationResponse, error) { - if err := s.validateReportConfiguration(ctx, request.GetReportConfig()); err != nil { - return nil, err - } - id, err := s.reportConfigStore.AddReportConfiguration(ctx, request.GetReportConfig()) - if err != nil { - return nil, err - } - - createdReportConfig, _, err := s.reportConfigStore.GetReportConfiguration(ctx, id) - if err := s.manager.Upsert(ctx, createdReportConfig); err != nil { - return nil, err - } - - return &v1.PostReportConfigurationResponse{ - ReportConfig: createdReportConfig, - }, err -} - -func (s *serviceImpl) UpdateReportConfiguration(ctx context.Context, request *v1.UpdateReportConfigurationRequest) (*v1.Empty, error) { - if err := s.validateReportConfiguration(ctx, request.GetReportConfig()); err != nil { - return &v1.Empty{}, err - } - if err := s.manager.Upsert(ctx, request.GetReportConfig()); err != nil { - return nil, err - } - - err := s.reportConfigStore.UpdateReportConfiguration(ctx, request.GetReportConfig()) - if err != nil { - return nil, err - } - return &v1.Empty{}, nil -} - -func (s *serviceImpl) DeleteReportConfiguration(ctx context.Context, id *v1.ResourceByID) (*v1.Empty, error) { - if id.GetId() == "" { - return nil, errors.Wrap(errox.InvalidArgs, "Report configuration id is required for deletion") - } - - config, found, err := s.reportConfigStore.GetReportConfiguration(ctx, id.GetId()) - if err != nil { - return nil, errors.Wrap(err, "Error finding report config") - } - if !found { - return nil, errors.Wrapf(errox.NotFound, "Report config ID '%s' not found", id.GetId()) - } - if !common.IsV1ReportConfig(config) { - return nil, errors.Wrap(errox.InvalidArgs, "report configuration does not belong to reporting version 1.0") - } - - if err := s.reportConfigStore.RemoveReportConfiguration(ctx, id.GetId()); err != nil { - return &v1.Empty{}, err - } - return &v1.Empty{}, s.manager.Remove(ctx, id.GetId()) -} - -func (s *serviceImpl) CountReportConfigurations(ctx context.Context, request *v1.RawQuery) (*v1.CountReportConfigurationsResponse, error) { - parsedQuery, err := search.ParseQuery(request.GetQuery(), search.MatchAllIfEmpty()) - if err != nil { - return nil, errors.Wrap(errox.InvalidArgs, err.Error()) - } - filteredQ := common.WithoutV2ReportConfigs(parsedQuery) - - numReportConfigs, err := s.reportConfigStore.Count(ctx, filteredQ) - if err != nil { - return nil, err - } - return &v1.CountReportConfigurationsResponse{Count: int32(numReportConfigs)}, nil -} - -func (s *serviceImpl) RegisterServiceServer(grpcServer *grpc.Server) { - v1.RegisterReportConfigurationServiceServer(grpcServer, s) -} - -func (s *serviceImpl) RegisterServiceHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { - return v1.RegisterReportConfigurationServiceHandler(ctx, mux, conn) -} - -func (*serviceImpl) AuthFuncOverride(ctx context.Context, fullMethodName string) (context.Context, error) { - return ctx, authorizer.Authorized(ctx, fullMethodName) -} diff --git a/central/reports/config/service/service_impl_postgres_test.go b/central/reports/config/service/service_impl_postgres_test.go deleted file mode 100644 index 09c6f120e3e22..0000000000000 --- a/central/reports/config/service/service_impl_postgres_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package service - -import ( - "context" - "testing" - - notifierMocks "github.com/stackrox/rox/central/notifier/datastore/mocks" - "github.com/stackrox/rox/central/reports/config/datastore/mocks" - managerMocks "github.com/stackrox/rox/central/reports/manager/mocks" - collectionMocks "github.com/stackrox/rox/central/resourcecollection/datastore/mocks" - v1 "github.com/stackrox/rox/generated/api/v1" - "github.com/stackrox/rox/pkg/fixtures" - "github.com/stretchr/testify/suite" - "go.uber.org/mock/gomock" -) - -func TestReportConfigurationServicePostgres(t *testing.T) { - suite.Run(t, new(ReportConfigurationServicePostgresTestSuite)) -} - -type ReportConfigurationServicePostgresTestSuite struct { - suite.Suite - service Service - reportConfigDatastore *mocks.MockDataStore - notifierDatastore *notifierMocks.MockDataStore - collectionDatastore *collectionMocks.MockDataStore - manager *managerMocks.MockManager - mockCtrl *gomock.Controller -} - -func (s *ReportConfigurationServicePostgresTestSuite) SetupTest() { - s.mockCtrl = gomock.NewController(s.T()) - s.reportConfigDatastore = mocks.NewMockDataStore(s.mockCtrl) - s.notifierDatastore = notifierMocks.NewMockDataStore(s.mockCtrl) - s.collectionDatastore = collectionMocks.NewMockDataStore(s.mockCtrl) - s.manager = managerMocks.NewMockManager(s.mockCtrl) - s.service = New(s.reportConfigDatastore, s.notifierDatastore, s.collectionDatastore, s.manager) -} - -func (s *ReportConfigurationServicePostgresTestSuite) TearDownTest() { - s.mockCtrl.Finish() -} - -func (s *ReportConfigurationServicePostgresTestSuite) TestAddValidReportConfiguration() { - ctx := context.Background() - - reportConfig := fixtures.GetValidReportConfiguration() - s.reportConfigDatastore.EXPECT().AddReportConfiguration(ctx, reportConfig).Return(reportConfig.GetId(), nil) - s.reportConfigDatastore.EXPECT().GetReportConfiguration(ctx, reportConfig.GetId()).Return(reportConfig, true, nil) - - s.notifierDatastore.EXPECT().Exists(ctx, gomock.Any()).Return(true, nil).AnyTimes() - s.collectionDatastore.EXPECT().Exists(ctx, gomock.Any()).Return(true, nil).AnyTimes() - - s.manager.EXPECT().Upsert(ctx, reportConfig).Return(nil) - _, err := s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: reportConfig, - }) - s.NoError(err) -} - -func (s *ReportConfigurationServicePostgresTestSuite) TestAddInvalidValidReportConfigurations() { - ctx := context.Background() - - s.notifierDatastore.EXPECT().Exists(ctx, gomock.Any()).Return(true, nil).AnyTimes() - s.collectionDatastore.EXPECT().Exists(ctx, gomock.Any()).Return(true, nil).AnyTimes() - - noNotifierReportConfig := fixtures.GetInvalidReportConfigurationNoNotifier() - _, err := s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: noNotifierReportConfig, - }) - s.Error(err) - - incorrectScheduleReportConfig := fixtures.GetInvalidReportConfigurationIncorrectSchedule() - _, err = s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: incorrectScheduleReportConfig, - }) - s.Error(err) - - missingScheduleReportConfig := fixtures.GetInvalidReportConfigurationMissingSchedule() - _, err = s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: missingScheduleReportConfig, - }) - s.Error(err) - - missingDaysOfWeekReportConfig := fixtures.GetInvalidReportConfigurationMissingDaysOfWeek() - _, err = s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: missingDaysOfWeekReportConfig, - }) - s.Error(err) - - missingDaysOfMonthReportConfig := fixtures.GetInvalidReportConfigurationMissingDaysOfMonth() - _, err = s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: missingDaysOfMonthReportConfig, - }) - s.Error(err) - - dailyScheduleReportConfig := fixtures.GetInvalidReportConfigurationDailySchedule() - _, err = s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: dailyScheduleReportConfig, - }) - s.Error(err) - - incorrectEmailReportConfig := fixtures.GetInvalidReportConfigurationIncorrectEmailV1() - _, err = s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: incorrectEmailReportConfig, - }) - s.Error(err) -} - -func (s *ReportConfigurationServicePostgresTestSuite) TestUpdateInvalidValidReportConfigurations() { - ctx := context.Background() - - s.notifierDatastore.EXPECT().Exists(ctx, gomock.Any()).Return(true, nil).AnyTimes() - s.collectionDatastore.EXPECT().Exists(ctx, gomock.Any()).Return(true, nil).AnyTimes() - - noNotifierReportConfig := fixtures.GetInvalidReportConfigurationNoNotifier() - _, err := s.service.UpdateReportConfiguration(ctx, &v1.UpdateReportConfigurationRequest{ - ReportConfig: noNotifierReportConfig, - }) - s.Error(err) - - incorrectScheduleReportConfig := fixtures.GetInvalidReportConfigurationIncorrectSchedule() - _, err = s.service.UpdateReportConfiguration(ctx, &v1.UpdateReportConfigurationRequest{ - ReportConfig: incorrectScheduleReportConfig, - }) - s.Error(err) - - missingScheduleReportConfig := fixtures.GetInvalidReportConfigurationMissingSchedule() - _, err = s.service.UpdateReportConfiguration(ctx, &v1.UpdateReportConfigurationRequest{ - ReportConfig: missingScheduleReportConfig, - }) - s.Error(err) - - incorrectEmailReportConfig := fixtures.GetInvalidReportConfigurationIncorrectEmailV1() - _, err = s.service.UpdateReportConfiguration(ctx, &v1.UpdateReportConfigurationRequest{ - ReportConfig: incorrectEmailReportConfig, - }) - s.Error(err) -} - -func (s *ReportConfigurationServicePostgresTestSuite) TestNotifierDoesNotExist() { - ctx := context.Background() - - s.notifierDatastore.EXPECT().Exists(ctx, gomock.Any()).Return(false, nil) - s.collectionDatastore.EXPECT().Exists(ctx, gomock.Any()).Return(true, nil).AnyTimes() - - reportConfig := fixtures.GetValidReportConfiguration() - _, err := s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: reportConfig, - }) - s.Error(err) -} - -func (s *ReportConfigurationServicePostgresTestSuite) TestAccessScopeDoesNotExist() { - ctx := context.Background() - - s.notifierDatastore.EXPECT().Exists(ctx, gomock.Any()).Return(true, nil).AnyTimes() - s.collectionDatastore.EXPECT().Exists(ctx, gomock.Any()).Return(false, nil) - - reportConfig := fixtures.GetValidReportConfiguration() - _, err := s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: reportConfig, - }) - s.Error(err) -} - -func (s *ReportConfigurationServicePostgresTestSuite) TestNoMailingAddresses() { - ctx := context.Background() - reportConfig := fixtures.GetValidReportConfiguration() - reportConfig.GetEmailConfig().MailingLists = []string{} - - _, err := s.service.PostReportConfiguration(ctx, &v1.PostReportConfigurationRequest{ - ReportConfig: reportConfig, - }) - s.Error(err) -} diff --git a/central/reports/config/service/singleton.go b/central/reports/config/service/singleton.go deleted file mode 100644 index ad887fce05c25..0000000000000 --- a/central/reports/config/service/singleton.go +++ /dev/null @@ -1,25 +0,0 @@ -package service - -import ( - notifierDataStore "github.com/stackrox/rox/central/notifier/datastore" - reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" - "github.com/stackrox/rox/central/reports/manager" - collectionDataStore "github.com/stackrox/rox/central/resourcecollection/datastore" - "github.com/stackrox/rox/pkg/sync" -) - -var ( - svc Service - once sync.Once -) - -func initialize() { - collectionDS, _ := collectionDataStore.Singleton() - svc = New(reportConfigDS.Singleton(), notifierDataStore.Singleton(), collectionDS, manager.Singleton()) -} - -// Singleton provides the instance of the service to register. -func Singleton() Service { - once.Do(initialize) - return svc -} diff --git a/central/reports/config/service/validate.go b/central/reports/config/service/validate.go deleted file mode 100644 index fda6fa8d9a9af..0000000000000 --- a/central/reports/config/service/validate.go +++ /dev/null @@ -1,116 +0,0 @@ -package service - -import ( - "context" - "net/mail" - - "github.com/pkg/errors" - "github.com/stackrox/rox/central/reports/common" - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/errorhelpers" - "github.com/stackrox/rox/pkg/errox" -) - -// validateReportConfiguration validates the given report configuration object -func (s *serviceImpl) validateReportConfiguration(ctx context.Context, config *storage.ReportConfiguration) error { - if config.GetName() == "" { - return errors.Wrap(errox.InvalidArgs, "Report configuration name is empty") - } - - if err := s.validateSchedule(config); err != nil { - return err - } - if err := s.validateNotifier(ctx, config); err != nil { - return err - } - if err := s.validateResourceScope(ctx, config); err != nil { - return err - } - - return nil -} - -func (s *serviceImpl) validateSchedule(config *storage.ReportConfiguration) error { - schedule := config.GetSchedule() - if schedule == nil { - return errors.Wrap(errox.InvalidArgs, "Report configuration must have a schedule") - } - switch schedule.GetIntervalType() { - case storage.Schedule_UNSET: - case storage.Schedule_DAILY: - return errors.Wrap(errox.InvalidArgs, "Report configuration must have a valid schedule type") - case storage.Schedule_WEEKLY: - if schedule.GetDaysOfWeek() == nil || len(schedule.GetDaysOfWeek().GetDays()) == 0 { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify days of week for the schedule") - } - for _, day := range schedule.GetDaysOfWeek().GetDays() { - if day < 0 || day > 6 { - return errors.Wrap(errox.InvalidArgs, "Invalid schedule: Days of the week can be Sunday (0) - Saturday(6)") - } - } - case storage.Schedule_MONTHLY: - if schedule.GetDaysOfMonth() == nil || len(schedule.GetDaysOfMonth().GetDays()) == 0 { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify days of the month for the schedule") - } - for _, day := range schedule.GetDaysOfMonth().GetDays() { - if day != 1 && day != 15 { - return errors.Wrap(errox.InvalidArgs, "Reports can be sent out only 1st or 15th day of the month") - } - } - } - return nil -} - -func (s *serviceImpl) validateNotifier(ctx context.Context, config *storage.ReportConfiguration) error { - if config.GetEmailConfig() == nil { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify an email notifier configuration") - } - return s.validateEmailConfig(ctx, config.GetEmailConfig()) -} - -func (s *serviceImpl) validateEmailConfig(ctx context.Context, emailConfig *storage.EmailNotifierConfiguration) error { - if emailConfig.GetNotifierId() == "" { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify a valid email notifier") - } - if len(emailConfig.GetMailingLists()) == 0 { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify at least one email recipient to send the report to") - } - - errorList := errorhelpers.NewErrorList("Invalid email addresses in mailing list: ") - for _, addr := range emailConfig.GetMailingLists() { - if _, err := mail.ParseAddress(addr); err != nil { - errorList.AddError(errors.Wrapf(errox.InvalidArgs, "Invalid email recipient address: %s", addr)) - } - } - if !errorList.Empty() { - return errorList.ToError() - } - - exists, err := s.notifierDatastore.Exists(ctx, emailConfig.GetNotifierId()) - if err != nil { - return errors.Errorf("Error looking up attached notifier, Notifier: %s, Error: %s", emailConfig.GetNotifierId(), err) - } - if !exists { - return errors.Wrapf(errox.NotFound, "Notifier %s not found.", emailConfig.GetNotifierId()) - } - return nil -} - -func (s *serviceImpl) validateResourceScope(ctx context.Context, config *storage.ReportConfiguration) error { - if !common.IsV1ReportConfig(config) { - return errors.Wrap(errox.InvalidArgs, "Report configuration belonging to reporting version 1.0 should not set the 'resourceScope' field."+ - "Instead, set the 'scopeId' field to the desired collection ID.") - } - if config.GetScopeId() == "" { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify a valid collection ID in the 'scopeId' field") - } - - exists, err := s.collectionDatastore.Exists(ctx, config.GetScopeId()) - if err != nil { - return errors.Errorf("Error trying to lookup attached collection, Collection: %s, Error: %s", config.GetScopeId(), err) - } - if !exists { - return errors.Wrapf(errox.NotFound, "Collection %s not found.", config.GetScopeId()) - } - return nil -} diff --git a/central/reports/manager/manager.go b/central/reports/manager/manager.go deleted file mode 100644 index 9e4d71ea24dc7..0000000000000 --- a/central/reports/manager/manager.go +++ /dev/null @@ -1,21 +0,0 @@ -package manager - -import ( - "context" - - "github.com/stackrox/rox/generated/storage" -) - -// Manager implements the interface for scheduled reports -// -//go:generate mockgen-wrapper -type Manager interface { - // Upsert adds/updates a report configuration into the scheduler - Upsert(ctx context.Context, rc *storage.ReportConfiguration) error - // Remove removes a report configuration from the scheduler and from future scheduled runs - Remove(ctx context.Context, id string) error - // RunReport queues an on demand report generation request for execution - RunReport(ctx context.Context, rc *storage.ReportConfiguration) error - Start() - Stop() -} diff --git a/central/reports/manager/manager_impl.go b/central/reports/manager/manager_impl.go deleted file mode 100644 index cc1dc49842c58..0000000000000 --- a/central/reports/manager/manager_impl.go +++ /dev/null @@ -1,80 +0,0 @@ -package manager - -import ( - "context" - - "github.com/pkg/errors" - "github.com/stackrox/rox/central/graphql/resolvers/loaders" - "github.com/stackrox/rox/central/reports/common" - "github.com/stackrox/rox/central/reports/scheduler" - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/concurrency" - "github.com/stackrox/rox/pkg/contextutil" - "github.com/stackrox/rox/pkg/errox" - "github.com/stackrox/rox/pkg/sac" - "github.com/stackrox/rox/pkg/sac/resources" -) - -var ( - reportsSAC = sac.ForResource(resources.WorkflowAdministration) -) - -type managerImpl struct { - scheduler scheduler.Scheduler - inProgress concurrency.Flag -} - -func (m *managerImpl) Remove(ctx context.Context, id string) error { - if ok, err := reportsSAC.WriteAllowed(ctx); err != nil { - return err - } else if !ok { - return sac.ErrResourceAccessDenied - } - m.scheduler.RemoveReportSchedule(id) - return nil -} - -func (m *managerImpl) Upsert(ctx context.Context, reportConfig *storage.ReportConfiguration) error { - if ok, err := reportsSAC.WriteAllowed(ctx); err != nil { - return err - } else if !ok { - return sac.ErrResourceAccessDenied - } - if !common.IsV1ReportConfig(reportConfig) { - return errors.Wrap(errox.InvalidArgs, "report configuration does not belong to reporting version 1.0") - } - if err := m.scheduler.UpsertReportSchedule(reportConfig); err != nil { - return err - } - return nil -} - -func (m *managerImpl) RunReport(ctx context.Context, reportConfig *storage.ReportConfiguration) error { - /* - * Multiple on demand reports cannot be executed concurrently. - * An on demand report may be submitted while other scheduled reports are being run and will be - * executed in FIFO order (in case multiple scheduled reports are already queued up). - */ - if !common.IsV1ReportConfig(reportConfig) { - return errors.Wrap(errox.InvalidArgs, "report configuration does not belong to reporting version 1.0") - } - if m.inProgress.TestAndSet(true) { - return errors.New("report generation already in progress, please try again later") - } - defer m.inProgress.Set(false) - - m.scheduler.SubmitReport(&scheduler.ReportRequest{ - ReportConfig: reportConfig, - OnDemand: true, - Ctx: loaders.WithLoaderContext(contextutil.WithValuesFrom(context.Background(), ctx)), - }) - return nil -} - -func (m *managerImpl) Start() { - m.scheduler.Start() -} - -func (m *managerImpl) Stop() { - m.scheduler.Stop() -} diff --git a/central/reports/manager/mocks/manager.go b/central/reports/manager/mocks/manager.go deleted file mode 100644 index 4438dd1f0ec48..0000000000000 --- a/central/reports/manager/mocks/manager.go +++ /dev/null @@ -1,108 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: manager.go -// -// Generated by this command: -// -// mockgen -package mocks -destination mocks/manager.go -source manager.go -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - storage "github.com/stackrox/rox/generated/storage" - gomock "go.uber.org/mock/gomock" -) - -// MockManager is a mock of Manager interface. -type MockManager struct { - ctrl *gomock.Controller - recorder *MockManagerMockRecorder - isgomock struct{} -} - -// MockManagerMockRecorder is the mock recorder for MockManager. -type MockManagerMockRecorder struct { - mock *MockManager -} - -// NewMockManager creates a new mock instance. -func NewMockManager(ctrl *gomock.Controller) *MockManager { - mock := &MockManager{ctrl: ctrl} - mock.recorder = &MockManagerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockManager) EXPECT() *MockManagerMockRecorder { - return m.recorder -} - -// Remove mocks base method. -func (m *MockManager) Remove(ctx context.Context, id string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Remove", ctx, id) - ret0, _ := ret[0].(error) - return ret0 -} - -// Remove indicates an expected call of Remove. -func (mr *MockManagerMockRecorder) Remove(ctx, id any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockManager)(nil).Remove), ctx, id) -} - -// RunReport mocks base method. -func (m *MockManager) RunReport(ctx context.Context, rc *storage.ReportConfiguration) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RunReport", ctx, rc) - ret0, _ := ret[0].(error) - return ret0 -} - -// RunReport indicates an expected call of RunReport. -func (mr *MockManagerMockRecorder) RunReport(ctx, rc any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunReport", reflect.TypeOf((*MockManager)(nil).RunReport), ctx, rc) -} - -// Start mocks base method. -func (m *MockManager) Start() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Start") -} - -// Start indicates an expected call of Start. -func (mr *MockManagerMockRecorder) Start() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockManager)(nil).Start)) -} - -// Stop mocks base method. -func (m *MockManager) Stop() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Stop") -} - -// Stop indicates an expected call of Stop. -func (mr *MockManagerMockRecorder) Stop() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockManager)(nil).Stop)) -} - -// Upsert mocks base method. -func (m *MockManager) Upsert(ctx context.Context, rc *storage.ReportConfiguration) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Upsert", ctx, rc) - ret0, _ := ret[0].(error) - return ret0 -} - -// Upsert indicates an expected call of Upsert. -func (mr *MockManagerMockRecorder) Upsert(ctx, rc any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upsert", reflect.TypeOf((*MockManager)(nil).Upsert), ctx, rc) -} diff --git a/central/reports/manager/singleton.go b/central/reports/manager/singleton.go deleted file mode 100644 index 4e9bf6620383d..0000000000000 --- a/central/reports/manager/singleton.go +++ /dev/null @@ -1,42 +0,0 @@ -package manager - -import ( - clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" - deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" - namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" - notifierDataStore "github.com/stackrox/rox/central/notifier/datastore" - "github.com/stackrox/rox/central/notifier/processor" - reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" - "github.com/stackrox/rox/central/reports/scheduler" - collectionDataStore "github.com/stackrox/rox/central/resourcecollection/datastore" - roleDataStore "github.com/stackrox/rox/central/role/datastore" - "github.com/stackrox/rox/pkg/sync" -) - -var ( - once sync.Once - instance Manager -) - -func initialize() { - collectionDS, collectionQueryRes := collectionDataStore.Singleton() - instance = &managerImpl{ - scheduler: scheduler.New( - reportConfigDS.Singleton(), - notifierDataStore.Singleton(), - clusterDataStore.Singleton(), - namespaceDataStore.Singleton(), - deploymentDataStore.Singleton(), - collectionDS, - roleDataStore.Singleton(), - collectionQueryRes, - processor.Singleton(), - ), - } -} - -// Singleton provides the instance of Manager to use. -func Singleton() Manager { - once.Do(initialize) - return instance -} diff --git a/central/reports/scheduler/schedule.go b/central/reports/scheduler/schedule.go deleted file mode 100644 index 02f6bf9a381b8..0000000000000 --- a/central/reports/scheduler/schedule.go +++ /dev/null @@ -1,448 +0,0 @@ -package scheduler - -import ( - "context" - "encoding/json" - "fmt" - "sort" - "strings" - "text/template" - "time" - - "github.com/graph-gophers/graphql-go" - "github.com/pkg/errors" - clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" - deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" - "github.com/stackrox/rox/central/graphql/resolvers" - "github.com/stackrox/rox/central/graphql/resolvers/loaders" - namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" - notifierDataStore "github.com/stackrox/rox/central/notifier/datastore" - "github.com/stackrox/rox/central/reports/common" - reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" - collectionDataStore "github.com/stackrox/rox/central/resourcecollection/datastore" - roleDataStore "github.com/stackrox/rox/central/role/datastore" - v1 "github.com/stackrox/rox/generated/api/v1" - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/branding" - "github.com/stackrox/rox/pkg/concurrency" - "github.com/stackrox/rox/pkg/grpc/authz/allow" - "github.com/stackrox/rox/pkg/logging" - "github.com/stackrox/rox/pkg/notifier" - "github.com/stackrox/rox/pkg/notifiers" - "github.com/stackrox/rox/pkg/protocompat" - "github.com/stackrox/rox/pkg/protoconv" - "github.com/stackrox/rox/pkg/protoconv/schedule" - "github.com/stackrox/rox/pkg/retry" - "github.com/stackrox/rox/pkg/sac" - "github.com/stackrox/rox/pkg/search" - "github.com/stackrox/rox/pkg/sync" - "github.com/stackrox/rox/pkg/templates" - "github.com/stackrox/rox/pkg/timestamp" - "gopkg.in/robfig/cron.v2" -) - -const ( - deploymentsPaginationLimit = 50 - - reportQueryPostgres = `query getVulnReportData($scopequery: String, - $cvequery: String, $pagination: Pagination) { - deployments: deployments(query: $scopequery, pagination: $pagination) { - clusterName - namespace - name - images { - name { - full_name:fullName - } - imageComponents { - name - imageVulnerabilities(query: $cvequery) { - ...cveFields - } - } - } - } - } - fragment cveFields on ImageVulnerability { - cve - severity - fixedByVersion - isFixable - discoveredAtImage - link - }` - - vulnReportEmailTemplate = ` - {{.BrandedProductName}} has found vulnerabilities associated with the running container images owned by your organization. Please review the attached vulnerability report {{.WhichVulns}} for {{.DateStr}}. - - To address these findings, please review the impacted software packages in the container images running within deployments you are responsible for and update them to a version containing the fix, if one is available.` - - noVulnsFoundEmailTemplate = ` - {{.BrandedProductName}} has found zero vulnerabilities associated with the running container images owned by your organization.` - - paginatedQueryStartOffset = 0 -) - -var ( - log = logging.LoggerForModule() - - scheduledCtx = resolvers.SetAuthorizerOverride(loaders.WithLoaderContext(sac.WithAllAccess(context.Background())), allow.Anonymous()) -) - -// Scheduler maintains the schedules for reports -type Scheduler interface { - UpsertReportSchedule(reportConfig *storage.ReportConfiguration) error - RemoveReportSchedule(reportConfigID string) - SubmitReport(request *ReportRequest) - Start() - Stop() -} - -type scheduler struct { - lock sync.Mutex - cron *cron.Cron - reportConfigToEntryIDs map[string]cron.EntryID - - reportConfigDatastore reportConfigDS.DataStore - notifierDatastore notifierDataStore.DataStore - clusterDatastore clusterDataStore.DataStore - namespaceDatastore namespaceDataStore.DataStore - deploymentDatastore deploymentDataStore.DataStore - roleDatastore roleDataStore.DataStore - collectionDatastore collectionDataStore.DataStore - collectionQueryResolver collectionDataStore.QueryResolver - notificationProcessor notifier.Processor - - reportsToRun chan *ReportRequest - - stopper concurrency.Stopper - - Schema *graphql.Schema -} - -// ReportRequest is a request to the scheduler to run a scheduled or on demand report -type ReportRequest struct { - ReportConfig *storage.ReportConfiguration - OnDemand bool - Ctx context.Context -} - -type reportEmailFormat struct { - BrandedProductName string - WhichVulns string - DateStr string -} - -// New instantiates a new cron scheduler and supports adding and removing report configurations -func New(reportConfigDS reportConfigDS.DataStore, notifierDS notifierDataStore.DataStore, - clusterDS clusterDataStore.DataStore, namespaceDS namespaceDataStore.DataStore, - deploymentDS deploymentDataStore.DataStore, collectionDS collectionDataStore.DataStore, roleDS roleDataStore.DataStore, - collectionQueryRes collectionDataStore.QueryResolver, notificationProcessor notifier.Processor) Scheduler { - cronScheduler := cron.New() - cronScheduler.Start() - - ourSchema, err := graphql.ParseSchema(resolvers.Schema(), resolvers.New()) - if err != nil { - panic(err) - } - - return newSchedulerImpl(reportConfigDS, notifierDS, clusterDS, namespaceDS, deploymentDS, collectionDS, roleDS, - collectionQueryRes, notificationProcessor, cronScheduler, ourSchema) -} - -func newSchedulerImpl(reportConfigDS reportConfigDS.DataStore, notifierDS notifierDataStore.DataStore, - clusterDS clusterDataStore.DataStore, namespaceDS namespaceDataStore.DataStore, - deploymentDS deploymentDataStore.DataStore, collectionDS collectionDataStore.DataStore, roleDS roleDataStore.DataStore, - collectionQueryRes collectionDataStore.QueryResolver, notificationProcessor notifier.Processor, - cronScheduler *cron.Cron, schema *graphql.Schema) *scheduler { - return &scheduler{ - reportConfigToEntryIDs: make(map[string]cron.EntryID), - cron: cronScheduler, - reportConfigDatastore: reportConfigDS, - notifierDatastore: notifierDS, - clusterDatastore: clusterDS, - namespaceDatastore: namespaceDS, - deploymentDatastore: deploymentDS, - collectionDatastore: collectionDS, - roleDatastore: roleDS, - collectionQueryResolver: collectionQueryRes, - notificationProcessor: notificationProcessor, - reportsToRun: make(chan *ReportRequest, 100), - Schema: schema, - - stopper: concurrency.NewStopper(), - } -} - -func (s *scheduler) reportClosure(reportConfig *storage.ReportConfiguration) func() { - return func() { - log.Infof("Submitting report request for '%s' at %v", reportConfig.GetName(), time.Now().Format(time.RFC850)) - s.SubmitReport(&ReportRequest{ - ReportConfig: reportConfig, - OnDemand: false, - Ctx: scheduledCtx, - }) - } -} - -func (s *scheduler) UpsertReportSchedule(reportConfig *storage.ReportConfiguration) error { - s.lock.Lock() - defer s.lock.Unlock() - cronSpec, err := schedule.ConvertToCronTab(reportConfig.GetSchedule()) - if err != nil { - return err - } - entryID, err := s.cron.AddFunc(cronSpec, s.reportClosure(reportConfig)) - if err != nil { - return err - } - - // Remove the old entry if this is an update - if oldEntryID, ok := s.reportConfigToEntryIDs[reportConfig.GetId()]; ok { - s.cron.Remove(oldEntryID) - } - s.reportConfigToEntryIDs[reportConfig.GetId()] = entryID - return nil -} - -func (s *scheduler) RemoveReportSchedule(id string) { - s.lock.Lock() - defer s.lock.Unlock() - - oldEntryID := s.reportConfigToEntryIDs[id] - s.cron.Remove(oldEntryID) - delete(s.reportConfigToEntryIDs, id) -} - -func (s *scheduler) SubmitReport(reportRequest *ReportRequest) { - log.Infof("Submitting report '%s' at %v for execution", reportRequest.ReportConfig.GetName(), time.Now().Format(time.RFC822)) - s.reportsToRun <- reportRequest -} - -func (s *scheduler) runReports() { - defer s.stopper.Flow().ReportStopped() - for { - select { - case <-s.stopper.Flow().StopRequested(): - return - case req := <-s.reportsToRun: - log.Infof("Executing report '%s' at %v", req.ReportConfig.GetName(), time.Now().Format(time.RFC822)) - err := s.sendReportResults(req) - if err != nil { - log.Errorf("error executing report %s: %s", req.ReportConfig.GetName(), err) - } - if err := s.updateLastRunStatus(req, err); err != nil { - log.Errorf("error updating run status for report config '%s': %s", req.ReportConfig.GetName(), err) - } - } - } -} - -func (s *scheduler) updateLastRunStatus(req *ReportRequest, err error) error { - if req.OnDemand { - return nil - } - if err != nil { - // TODO: @khushboo for more accuracy, save timestamp when the vuln data is pulled aka the query is run - req.ReportConfig.LastRunStatus = &storage.ReportLastRunStatus{ - ReportStatus: storage.ReportLastRunStatus_FAILURE, - LastRunTime: protoconv.ConvertMicroTSToProtobufTS(timestamp.Now()), - ErrorMsg: err.Error(), - } - } else { - req.ReportConfig.LastRunStatus = &storage.ReportLastRunStatus{ - ReportStatus: storage.ReportLastRunStatus_SUCCESS, - LastRunTime: protoconv.ConvertMicroTSToProtobufTS(timestamp.Now()), - ErrorMsg: "", - } - req.ReportConfig.LastSuccessfulRunTime = protocompat.TimestampNow() - } - if err = s.UpsertReportSchedule(req.ReportConfig); err != nil { - return err - } - return s.reportConfigDatastore.UpdateReportConfiguration(req.Ctx, req.ReportConfig) -} - -func (s *scheduler) sendReportResults(req *ReportRequest) error { - rc := req.ReportConfig - - reportName := rc.GetName() - notifier := s.notificationProcessor.GetNotifier(req.Ctx, rc.GetEmailConfig().GetNotifierId()) - reportNotifier, ok := notifier.(notifiers.ReportNotifier) - if !ok { - return errors.Errorf("incorrect notifier type in report config '%s'", rc.GetName()) - } - // Get the results of running the report query - reportData, err := s.getReportData(req.Ctx, rc) - if err != nil { - return err - } - // Format results into CSV - watchedImagesReportData := []common.WatchedImagesResult{} - configName := "" - includedNvd := false - zippedCSVResult, err := common.Format(reportData, watchedImagesReportData, configName, includedNvd) - if err != nil { - return errors.Wrap(err, "error formatting the report data") - } - zippedCSVData := zippedCSVResult.ZippedCsv - - // If it is an empty report, do not send an attachment in the final notification email and the email body - // will indicate that no vulns were found - templateStr := vulnReportEmailTemplate - if zippedCSVResult.NumDeployedImageCVEs == 0 && zippedCSVResult.NumWatchedImageCVEs == 0 { - // If it is an empty report, the email body will indicate that no vulns were found - zippedCSVData = nil - templateStr = noVulnsFoundEmailTemplate - } - - messageText, err := formatMessage(rc, templateStr, time.Now()) - if err != nil { - return errors.Wrap(err, "error formatting the report email text") - } - - if err = retry.WithRetry(func() error { - return reportNotifier.ReportNotify(req.Ctx, zippedCSVData, - rc.GetEmailConfig().GetMailingLists(), "", messageText, reportName) - }, - retry.OnlyRetryableErrors(), - retry.Tries(3), - retry.BetweenAttempts(func(previousAttempt int) { - wait := time.Duration(previousAttempt * previousAttempt * 100) - time.Sleep(wait * time.Millisecond) - }), - ); err != nil { - return err - } - log.Infof("Report generation for '%s' completed, email notification sent.", rc.GetName()) - return nil -} - -func formatMessage(rc *storage.ReportConfiguration, emailTemplate string, date time.Time) (string, error) { - data := &reportEmailFormat{ - BrandedProductName: branding.GetProductName(), - WhichVulns: "for all vulnerabilities", - DateStr: date.Format("January 02, 2006"), - } - if rc.GetVulnReportFilters().GetSinceLastReport() && rc.GetLastSuccessfulRunTime() != nil { - data.WhichVulns = fmt.Sprintf("for new vulnerabilities since %s", - timestamp.FromProtobuf(rc.GetLastSuccessfulRunTime()).GoTime().Format("January 02, 2006")) - } - tmpl, err := template.New("emailBody").Parse(emailTemplate) - if err != nil { - return "", err - } - return templates.ExecuteToString(tmpl, data) -} - -func (s *scheduler) getReportData(ctx context.Context, rc *storage.ReportConfiguration) ([]common.DeployedImagesResult, error) { - collection, found, err := s.collectionDatastore.Get(ctx, rc.GetScopeId()) - if err != nil { - return nil, errors.Wrapf(err, "error building report query: unable to get the collection %s", rc.GetScopeId()) - } - if !found { - return nil, errors.Errorf("error building report query: collection with id %s not found", rc.GetScopeId()) - } - rQuery, err := s.buildReportQuery(ctx, rc, collection) - if err != nil { - return nil, err - } - deploymentIds, err := s.getDeploymentIDs(ctx, rQuery.DeploymentsQuery) - if err != nil { - return nil, err - } - result, err := s.runPaginatedDeploymentsQuery(ctx, rQuery.CveFieldsQuery, deploymentIds) - if err != nil { - return nil, err - } - result.Deployments = orderByClusterAndNamespace(result.Deployments) - return []common.DeployedImagesResult{result}, nil -} - -func (s *scheduler) buildReportQuery(ctx context.Context, rc *storage.ReportConfiguration, - collection *storage.ResourceCollection) (*common.ReportQuery, error) { - qb := common.NewVulnReportQueryBuilder(collection, rc.GetVulnReportFilters(), s.collectionQueryResolver, - timestamp.FromProtobuf(rc.GetLastSuccessfulRunTime()).GoTime()) - rQuery, err := qb.BuildQuery(ctx, nil, nil) - if err != nil { - return nil, errors.Wrap(err, "error building report query") - } - return rQuery, nil -} - -// Returns vuln report data from deployments matched by embedded resource collection. -func (s *scheduler) runPaginatedDeploymentsQuery(ctx context.Context, cveQuery string, deploymentIds []string) (common.DeployedImagesResult, error) { - offset := paginatedQueryStartOffset - var resultData common.DeployedImagesResult - for { - if offset >= len(deploymentIds) { - break - } - // deploymentsQuery is a *v1.Query and graphQL resolvers accept a string queries. - // With nested collections, deploymentsQuery could be a disjunction of conjunctions like the example below. - // [(Cluster: c1 AND Namespace: n1 AND Deployment: d1) OR (Cluster: c2 AND Namespace: n2 AND Deployment: d2)] - // Current string query language doesn't support disjunction of multiple conjunctions, so we cannot build a - // string equivalent of deploymentsQuery for graphQL. Because of this, we first fetch deploymentIDs from - // deploymentDatastore using deploymentsQuery and then build string query for graphQL using those deploymentIDs. - scopeQuery := fmt.Sprintf("%s:%s", search.DeploymentID.String(), - strings.Join(deploymentIds[offset:min(offset+deploymentsPaginationLimit, len(deploymentIds))], ",")) - r, err := s.execReportDataQuery(ctx, reportQueryPostgres, scopeQuery, cveQuery, paginatedQueryStartOffset) - if err != nil { - return r, err - } - resultData.Deployments = append(resultData.Deployments, r.Deployments...) - offset += deploymentsPaginationLimit - } - return resultData, nil -} - -func (s *scheduler) execReportDataQuery(ctx context.Context, gqlQuery, scopeQuery, cveQuery string, offset int) (common.DeployedImagesResult, error) { - response := s.Schema.Exec(ctx, - gqlQuery, "getVulnReportData", map[string]interface{}{ - "scopequery": scopeQuery, - "cvequery": cveQuery, - "pagination": map[string]interface{}{ - "offset": offset, - "limit": deploymentsPaginationLimit, - }, - }) - if len(response.Errors) > 0 { - log.Errorf("error running graphql query: %s", response.Errors[0].Message) - return common.DeployedImagesResult{}, response.Errors[0].Err - } - var res common.DeployedImagesResult - if err := json.Unmarshal(response.Data, &res); err != nil { - return common.DeployedImagesResult{}, err - } - return res, nil -} - -func (s *scheduler) getDeploymentIDs(ctx context.Context, deploymentsQuery *v1.Query) ([]string, error) { - results, err := s.deploymentDatastore.Search(ctx, deploymentsQuery) - if err != nil { - return nil, err - } - return search.ResultsToIDs(results), nil -} - -func (s *scheduler) Start() { - go s.runReports() -} - -func (s *scheduler) Stop() { - s.stopper.Client().Stop() - err := s.stopper.Client().Stopped().Wait() - if err != nil { - log.Errorf("Error stopping vulnerability report scheduler : %v", err) - } -} - -func orderByClusterAndNamespace(deployments []*common.Deployment) []*common.Deployment { - sort.SliceStable(deployments, func(i, j int) bool { - if deployments[i].Cluster.GetName() == deployments[j].Cluster.GetName() { - return deployments[i].Namespace < deployments[j].Namespace - } - return deployments[i].Cluster.GetName() < deployments[j].Cluster.GetName() - }) - return deployments -} diff --git a/central/reports/scheduler/schedule_test.go b/central/reports/scheduler/schedule_test.go deleted file mode 100644 index 0f6879598f14a..0000000000000 --- a/central/reports/scheduler/schedule_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package scheduler - -import ( - "testing" - "time" - - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/branding" - "github.com/stackrox/rox/pkg/fixtures" - "github.com/stretchr/testify/suite" -) - -const ( - expectedVulnReportEmailTemplateRhacsBranding = ` - Red Hat Advanced Cluster Security for Kubernetes has found vulnerabilities associated with the running container images owned by your organization. Please review the attached vulnerability report for all vulnerabilities for December 31, 1999. - - To address these findings, please review the impacted software packages in the container images running within deployments you are responsible for and update them to a version containing the fix, if one is available.` - - expectedVulnReportEmailTemplateStackroxBranding = ` - StackRox has found vulnerabilities associated with the running container images owned by your organization. Please review the attached vulnerability report for all vulnerabilities for December 31, 1999. - - To address these findings, please review the impacted software packages in the container images running within deployments you are responsible for and update them to a version containing the fix, if one is available.` - - expectedNoVulnsFoundEmailTemplateRhacsBranding = ` - Red Hat Advanced Cluster Security for Kubernetes has found zero vulnerabilities associated with the running container images owned by your organization.` - - expectedNoVulnsFoundEmailTemplateStackroxBranding = ` - StackRox has found zero vulnerabilities associated with the running container images owned by your organization.` -) - -var _ suite.SetupAllSuite = (*ScheduleTestSuite)(nil) - -func TestSchedule(t *testing.T) { - suite.Run(t, new(ScheduleTestSuite)) -} - -type ScheduleTestSuite struct { - suite.Suite - - time time.Time - rc *storage.ReportConfiguration -} - -func (s *ScheduleTestSuite) SetupSuite() { - s.rc = fixtures.GetValidReportConfiguration() - s.time = time.Date(1999, 12, 31, 23, 59, 59, 999, time.Local) -} - -func (s *ScheduleTestSuite) TestFormatVulnMessage() { - tests := map[string]struct { - vulnReport string - noVulnReport string - }{ - "RHACS_BRANDING": { - vulnReport: expectedVulnReportEmailTemplateRhacsBranding, - noVulnReport: expectedNoVulnsFoundEmailTemplateRhacsBranding, - }, - "STACKROX_BRANDING": { - vulnReport: expectedVulnReportEmailTemplateStackroxBranding, - noVulnReport: expectedNoVulnsFoundEmailTemplateStackroxBranding, - }, - } - for productBranding, tt := range tests { - s.Run(productBranding, func() { - s.T().Setenv(branding.ProductBrandingEnvName, productBranding) - - receivedBrandedVulnFound, err := formatMessage(s.rc, vulnReportEmailTemplate, s.time) - s.NoError(err) - receivedBrandedNoVulnFound, err := formatMessage(s.rc, noVulnsFoundEmailTemplate, s.time) - s.NoError(err) - - s.Equal(tt.vulnReport, receivedBrandedVulnFound) - s.Equal(tt.noVulnReport, receivedBrandedNoVulnFound) - }) - } -} diff --git a/central/reports/scheduler/v2/mocks/scheduler.go b/central/reports/scheduler/v2/mocks/scheduler.go index 08de8c1662566..1b1dbb2e5e93a 100644 --- a/central/reports/scheduler/v2/mocks/scheduler.go +++ b/central/reports/scheduler/v2/mocks/scheduler.go @@ -15,6 +15,7 @@ import ( reportgenerator "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" storage "github.com/stackrox/rox/generated/storage" + postgres "github.com/stackrox/rox/pkg/postgres" gomock "go.uber.org/mock/gomock" ) @@ -72,6 +73,20 @@ func (mr *MockSchedulerMockRecorder) CancelReportRequest(ctx, reportID any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelReportRequest", reflect.TypeOf((*MockScheduler)(nil).CancelReportRequest), ctx, reportID) } +// GetScheduledConfigIDs mocks base method. +func (m *MockScheduler) GetScheduledConfigIDs() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetScheduledConfigIDs") + ret0, _ := ret[0].([]string) + return ret0 +} + +// GetScheduledConfigIDs indicates an expected call of GetScheduledConfigIDs. +func (mr *MockSchedulerMockRecorder) GetScheduledConfigIDs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScheduledConfigIDs", reflect.TypeOf((*MockScheduler)(nil).GetScheduledConfigIDs)) +} + // RemoveReportSchedule mocks base method. func (m *MockScheduler) RemoveReportSchedule(reportConfigID string) { m.ctrl.T.Helper() @@ -85,15 +100,15 @@ func (mr *MockSchedulerMockRecorder) RemoveReportSchedule(reportConfigID any) *g } // Start mocks base method. -func (m *MockScheduler) Start() { +func (m *MockScheduler) Start(db postgres.DB) { m.ctrl.T.Helper() - m.ctrl.Call(m, "Start") + m.ctrl.Call(m, "Start", db) } // Start indicates an expected call of Start. -func (mr *MockSchedulerMockRecorder) Start() *gomock.Call { +func (mr *MockSchedulerMockRecorder) Start(db any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockScheduler)(nil).Start)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockScheduler)(nil).Start), db) } // Stop mocks base method. diff --git a/central/reports/scheduler/v2/reportgenerator/common_helpers.go b/central/reports/scheduler/v2/reportgenerator/common_helpers.go new file mode 100644 index 0000000000000..5ded30d95a420 --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/common_helpers.go @@ -0,0 +1,74 @@ +package reportgenerator + +import ( + "bytes" + "context" + "time" + + "github.com/pkg/errors" + blobDS "github.com/stackrox/rox/central/blob/datastore" + "github.com/stackrox/rox/central/reports/common" + reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/notifiers" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/retry" + "github.com/stackrox/rox/pkg/utils" +) + +// RetryableSendReportResults sends report results via the notifier with retry logic. +func RetryableSendReportResults(ctx context.Context, reportNotifier notifiers.ReportNotifier, mailingList []string, + zippedCSVData *bytes.Buffer, emailSubject, emailBody, baseFilename string) error { + return retry.WithRetry(func() error { + return reportNotifier.ReportNotify(ctx, zippedCSVData, mailingList, emailSubject, emailBody, baseFilename) + }, + retry.OnlyRetryableErrors(), + retry.Tries(3), + retry.BetweenAttempts(func(previousAttempt int) { + wait := time.Duration(previousAttempt * previousAttempt * 100) + time.Sleep(wait * time.Millisecond) + }), + ) +} + +// SaveReportData saves zipped CSV report data to blob storage. +func SaveReportData(ctx context.Context, blobStore blobDS.Datastore, configID, reportID string, data *bytes.Buffer) error { + if data == nil { + return errors.Errorf("No data found for report config %q and id %q", configID, reportID) + } + b := &storage.Blob{ + Name: common.GetReportBlobPath(configID, reportID), + LastUpdated: protocompat.TimestampNow(), + ModifiedTime: protocompat.TimestampNow(), + Length: int64(data.Len()), + } + return blobStore.Upsert(ctx, b, data) +} + +// UpdateReportStatus updates the run state of a report snapshot. +func UpdateReportStatus(ctx context.Context, snapshotStore reportSnapshotDS.DataStore, snapshot *storage.ReportSnapshot, status storage.ReportStatus_RunState) error { + snapshot.ReportStatus.RunState = status + return snapshotStore.UpdateReportSnapshot(ctx, snapshot) +} + +// LogAndUpsertError logs the error and updates the report snapshot status to FAILURE. +// requestCtx is used to check for user cancellation; statusCtx is used for the DB update +// (should be an all-access context since requestCtx may be cancelled). +func LogAndUpsertError(requestCtx, statusCtx context.Context, snapshotStore reportSnapshotDS.DataStore, reportErr error, req *ReportRequest) { + if req.ReportSnapshot == nil || req.ReportSnapshot.GetReportStatus() == nil { + utils.Should(errors.New("Request does not have non-nil report snapshot with a non-nil report status")) + return + } + if errors.Is(context.Cause(requestCtx), ErrUserCancelled) { + log.Infof("Report for config '%s' was cancelled by user", req.ReportSnapshot.GetName()) + req.ReportSnapshot.ReportStatus.ErrorMsg = ErrUserCancelled.Error() + } else if reportErr != nil { + log.Errorf("Error while running report for config '%s': %s", req.ReportSnapshot.GetName(), reportErr) + req.ReportSnapshot.ReportStatus.ErrorMsg = reportErr.Error() + } + req.ReportSnapshot.ReportStatus.CompletedAt = protocompat.TimestampNow() + if err := UpdateReportStatus(statusCtx, snapshotStore, req.ReportSnapshot, storage.ReportStatus_FAILURE); err != nil { + log.Errorf("Error changing report status to FAILURE for report config '%s', report ID '%s': %s", + req.ReportSnapshot.GetName(), req.ReportSnapshot.GetReportId(), err) + } +} diff --git a/central/reports/scheduler/v2/reportgenerator/consts.go b/central/reports/scheduler/v2/reportgenerator/consts.go index 4ae09a1c5c034..ba1007eb7a762 100644 --- a/central/reports/scheduler/v2/reportgenerator/consts.go +++ b/central/reports/scheduler/v2/reportgenerator/consts.go @@ -8,4 +8,15 @@ const ( "Please review the vulnerable software packages/components from the impacted images and update them to a version containing the fix, if one is available.\n" defaultNoVulnsEmailBodyTemplate = "{{.BrandedPrefix}} for Kubernetes has found no workload CVEs in the images matched by the following report configuration parameters.\n" + + // NodeDefaultEmailSubjectTemplate is the default email subject for node CVE reports. + NodeDefaultEmailSubjectTemplate = "{{.BrandedProductNameShort}} Node CVE Report for {{.ReportConfigName}}" + + // NodeDefaultEmailBodyTemplate is the default email body for node CVE reports. + NodeDefaultEmailBodyTemplate = "{{.BrandedPrefix}} for Kubernetes has identified node CVEs in the nodes matched by the following report configuration parameters. " + + "The attached vulnerability report lists those node CVEs and associated details to help with remediation. " + + "Please review the vulnerable software packages/components from the impacted nodes and update them to a version containing the fix, if one is available.\n" + + // NodeDefaultNoVulnsEmailBodyTemplate is the default email body when no node CVEs are found. + NodeDefaultNoVulnsEmailBodyTemplate = "{{.BrandedPrefix}} for Kubernetes has found no node CVEs in the nodes matched by the following report configuration parameters.\n" ) diff --git a/central/reports/scheduler/v2/reportgenerator/csv_gen.go b/central/reports/scheduler/v2/reportgenerator/csv_gen.go index cf87fc0eda7dd..490b53aa6ed9f 100644 --- a/central/reports/scheduler/v2/reportgenerator/csv_gen.go +++ b/central/reports/scheduler/v2/reportgenerator/csv_gen.go @@ -4,6 +4,7 @@ import ( "archive/zip" "bytes" "fmt" + "slices" "strconv" "strings" "time" @@ -11,6 +12,7 @@ import ( "github.com/pkg/errors" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/csv" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/stringutils" ) @@ -21,50 +23,164 @@ var ( "Deployment", "Image", "Component", + "Component Version", "CVE", "Fixable", "CVE Fixed In", "Severity", "CVSS", + "NVDCVSS", + "EPSS Probability Percentage", "Discovered At", + "Image Created Date", "Reference", + "Advisory Name", + "Advisory Link", + "CVE Origin", + } + + // originDisplayNames maps VulnOrigin enum values to human-readable names for the + // CSV report. This must be kept in sync with the equivalent UI map. + originDisplayNames = map[storage.VulnOrigin]string{ + storage.VulnOrigin_VULN_ORIGIN_ALPINE: "Alpine Linux", + storage.VulnOrigin_VULN_ORIGIN_AMAZON: "Amazon Linux", + storage.VulnOrigin_VULN_ORIGIN_DEBIAN: "Debian", + storage.VulnOrigin_VULN_ORIGIN_ORACLE: "Oracle Linux", + storage.VulnOrigin_VULN_ORIGIN_OSV: "OSV.dev", + storage.VulnOrigin_VULN_ORIGIN_PHOTON: "Photon OS", + storage.VulnOrigin_VULN_ORIGIN_RED_HAT: "Red Hat", + storage.VulnOrigin_VULN_ORIGIN_SUSE: "SUSE", + storage.VulnOrigin_VULN_ORIGIN_UBUNTU: "Ubuntu", + storage.VulnOrigin_VULN_ORIGIN_OTHER: "Other", } ) -// GenerateCSV takes in the results of vuln report query, converts to CSV and returns zipped data -func GenerateCSV(cveResponses []*ImageCVEQueryResponse, configName string, reportSnapshot *storage.ReportSnapshot) (*bytes.Buffer, error) { - reportFilters := reportSnapshot.GetVulnReportFilters() - csvHeaderClone := make([]string, len(csvHeader)) - copy(csvHeaderClone, csvHeader) - if reportSnapshot.GetViewBasedVulnReportFilters() != nil { - csvHeaderClone = append(csvHeaderClone, "NVDCVSS") - csvHeaderClone = append(csvHeaderClone, "EPSS Probability Percentage") - csvHeaderClone = append(csvHeaderClone, "Advisory Name") - csvHeaderClone = append(csvHeaderClone, "Advisory Link") +func originDisplayName(origin storage.VulnOrigin) string { + if name, ok := originDisplayNames[origin]; ok { + return name + } + return origin.String() +} + +func formatCSVRow(r *ImageCVEQueryResponse) []string { + var epssScore string + if r.GetEPSSProbability() != nil { + epssScore = strconv.FormatFloat(*r.GetEPSSProbability()*100, 'f', 3, 64) } else { - csvHeaderClone = addOptionalColumnstoHeader(csvHeaderClone, reportFilters) + epssScore = "Not Available" + } + + csvRow := []string{ + r.GetCluster(), + r.GetNamespace(), + r.GetDeployment(), + r.GetImage(), + r.GetComponent(), + r.GetComponentVersion(), + r.GetCVE(), + strconv.FormatBool(r.GetFixable()), + r.GetFixedByVersion(), + strings.ToTitle(stringutils.GetUpTo(r.GetSeverity().String(), "_")), + strconv.FormatFloat(r.GetCVSS(), 'f', 2, 64), + strconv.FormatFloat(r.GetNVDCVSS(), 'f', 2, 64), + epssScore} + + if features.KnownExploitedVulnerabilities.Enabled() { + var cisaKev string + if r.GetCisaKev() != nil { + cisaKev = strconv.FormatBool(*r.GetCisaKev()) + } else { + cisaKev = "Not Available" + } + var knownRansomware string + if r.GetKnownRansomwareCampaign() != nil { + knownRansomware = strconv.FormatBool(*r.GetKnownRansomwareCampaign()) + } else { + knownRansomware = "Not Available" + } + csvRow = append(csvRow, cisaKev, knownRansomware) + } + + csvRow = append(csvRow, + r.GetDiscoveredAtImage(), + r.GetImageCreatedAt(), + r.Link, + r.GetAdvisoryName(), + r.GetAdvisoryLink(), + originDisplayName(r.GetOrigin()), + ) + + return csvRow +} + +func formatCol() []string { + csvHeaderCols := csvHeader + if features.KnownExploitedVulnerabilities.Enabled() { + csvHeaderCols = append(csvHeaderCols[:0:0], csvHeader...) + epssIdx := slices.Index(csvHeaderCols, "EPSS Probability Percentage") + csvHeaderCols = slices.Insert(csvHeaderCols, epssIdx+1, "CISA KEV", "Known Ransomware Campaign") + } + return csvHeaderCols +} + +func csvReportName(configName string) string { + truncatedName := configName + if len(configName) > 80 { + truncatedName = configName[0:80] + "..." } - csvWriter := csv.NewGenericWriter(csvHeaderClone, true) + now := time.Now() + return fmt.Sprintf("RHACS_Vulnerability_Report_%s_%s.csv", truncatedName, now.Format("02_January_2006")) +} + +// GenerateCSV takes in the results of vuln report query, converts to CSV and returns zipped data +func GenerateCSV(cveResponses []*ImageCVEQueryResponse, configName string) (*bytes.Buffer, error) { + csvWriter := csv.NewGenericWriter(formatCol(), true) + for _, r := range cveResponses { + var epssScore string + if r.GetEPSSProbability() != nil { + epssScore = strconv.FormatFloat(*r.GetEPSSProbability()*100, 'f', 3, 64) + } else { + epssScore = "Not Available" + } row := csv.Value{ r.GetCluster(), r.GetNamespace(), r.GetDeployment(), r.GetImage(), r.GetComponent(), + r.GetComponentVersion(), r.GetCVE(), strconv.FormatBool(r.GetFixable()), r.GetFixedByVersion(), strings.ToTitle(stringutils.GetUpTo(r.GetSeverity().String(), "_")), strconv.FormatFloat(r.GetCVSS(), 'f', 2, 64), - r.GetDiscoveredAtImage(), - r.Link, + strconv.FormatFloat(r.GetNVDCVSS(), 'f', 2, 64), + epssScore, } - if reportSnapshot.GetViewBasedVulnReportFilters() != nil { - addOtherColumns(&row, csvWriter, r) - } else { - addOptionalColumnstoRow(reportFilters, &row, csvWriter, r) + if features.KnownExploitedVulnerabilities.Enabled() { + var cisaKev string + if r.GetCisaKev() != nil { + cisaKev = strconv.FormatBool(*r.GetCisaKev()) + } else { + cisaKev = "Not Available" + } + var knownRansomware string + if r.GetKnownRansomwareCampaign() != nil { + knownRansomware = strconv.FormatBool(*r.GetKnownRansomwareCampaign()) + } else { + knownRansomware = "Not Available" + } + row = append(row, cisaKev, knownRansomware) } + row = append(row, + r.GetDiscoveredAtImage(), + r.GetImageCreatedAt(), + r.Link, + r.GetAdvisoryName(), + r.GetAdvisoryLink(), + originDisplayName(r.GetOrigin()), + ) csvWriter.AddValue(row) } @@ -76,13 +192,12 @@ func GenerateCSV(cveResponses []*ImageCVEQueryResponse, configName string, repor var zipBuf bytes.Buffer zipWriter := zip.NewWriter(&zipBuf) - truncatedName := configName - if len(configName) > 80 { - truncatedName = configName[0:80] + "..." + header := &zip.FileHeader{ + Name: csvReportName(configName), + Method: zip.Deflate, + Modified: time.Now(), } - - reportName := fmt.Sprintf("RHACS_Vulnerability_Report_%s_%s.csv", truncatedName, time.Now().Format("02_January_2006")) - zipFile, err := zipWriter.Create(reportName) + zipFile, err := zipWriter.CreateHeader(header) if err != nil { return nil, errors.Wrapf(err, "unable to create the zip file for report config '%s'", configName) } @@ -95,50 +210,5 @@ func GenerateCSV(cveResponses []*ImageCVEQueryResponse, configName string, repor return nil, errors.Wrapf(err, "unable to close the zip file for report config %s", configName) } return &zipBuf, nil -} - -func addOptionalColumnstoHeader(csvHeaderClone []string, optionalColumns *storage.VulnerabilityReportFilters) []string { - if optionalColumns.GetIncludeNvdCvss() { - csvHeaderClone = append(csvHeaderClone, "NVDCVSS") - } - if optionalColumns.GetIncludeEpssProbability() { - csvHeaderClone = append(csvHeaderClone, "EPSS Probability Percentage") - } - if optionalColumns.GetIncludeAdvisory() { - csvHeaderClone = append(csvHeaderClone, "Advisory Name") - csvHeaderClone = append(csvHeaderClone, "Advisory Link") - } - return csvHeaderClone -} - -func addOptionalColumnstoRow(optionalColumns *storage.VulnerabilityReportFilters, row *csv.Value, csvWriter *csv.GenericWriter, resp *ImageCVEQueryResponse) { - if optionalColumns.GetIncludeNvdCvss() { - csvWriter.AppendToValue(row, strconv.FormatFloat(resp.GetNVDCVSS(), 'f', 2, 64)) - } - if optionalColumns.GetIncludeEpssProbability() { - epssScore := resp.GetEPSSProbability() - if epssScore != nil { - csvWriter.AppendToValue(row, strconv.FormatFloat(*resp.GetEPSSProbability()*100, 'f', 3, 64)) - } else { - csvWriter.AppendToValue(row, "Not Available") - } - } - if optionalColumns.GetIncludeAdvisory() { - csvWriter.AppendToValue(row, resp.GetAdvisoryName()) - csvWriter.AppendToValue(row, resp.GetAdvisoryLink()) - - } -} - -func addOtherColumns(row *csv.Value, csvWriter *csv.GenericWriter, resp *ImageCVEQueryResponse) { - csvWriter.AppendToValue(row, strconv.FormatFloat(resp.GetNVDCVSS(), 'f', 2, 64)) - epssScore := resp.GetEPSSProbability() - if epssScore != nil { - csvWriter.AppendToValue(row, strconv.FormatFloat(*resp.GetEPSSProbability()*100, 'f', 3, 64)) - } else { - csvWriter.AppendToValue(row, "Not Available") - } - csvWriter.AppendToValue(row, resp.GetAdvisoryName()) - csvWriter.AppendToValue(row, resp.GetAdvisoryLink()) } diff --git a/central/reports/scheduler/v2/reportgenerator/csv_gen_test.go b/central/reports/scheduler/v2/reportgenerator/csv_gen_test.go new file mode 100644 index 0000000000000..3de7b466a8b13 --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/csv_gen_test.go @@ -0,0 +1,134 @@ +package reportgenerator + +import ( + "archive/zip" + "bytes" + "strings" + "testing" + "time" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateCSV_ZipEntryHasModifiedTimestamp(t *testing.T) { + before := time.Now().Add(-time.Minute) + buf, err := GenerateCSV(nil, "test") + require.NoError(t, err) + + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + require.Len(t, reader.File, 1) + assert.False(t, reader.File[0].Modified.IsZero(), "ZIP entry Modified timestamp should not be zero") + assert.True(t, reader.File[0].Modified.After(before), "ZIP entry Modified timestamp should be recent") +} + +func TestFormatCSVRow(t *testing.T) { + cluster := "test-cluster" + ns := "test-ns" + deploy := "test-deploy" + image := "quay.io/test:latest" + component := "openssl" + compVersion := "1.1.1" + cve := "CVE-2024-1234" + fixable := true + fixedBy := "1.1.2" + severity := storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY + cvss := 9.8 + nvdcvss := 9.5 + epss := 0.95 + discovered := time.Date(2024, 6, 15, 0, 0, 0, 0, time.UTC) + advisoryName := "RHSA-2024:1234" + advisoryLink := "https://access.redhat.com/errata/RHSA-2024:1234" + + r := &ImageCVEQueryResponse{ + Cluster: &cluster, + Namespace: &ns, + Deployment: &deploy, + Image: &image, + Component: &component, + ComponentVersion: &compVersion, + CVE: &cve, + Fixable: &fixable, + FixedByVersion: &fixedBy, + Severity: &severity, + CVSS: &cvss, + NVDCVSS: &nvdcvss, + EPSSProbability: &epss, + DiscoveredAtImage: &discovered, + AdvisoryName: &advisoryName, + AdvisoryLink: &advisoryLink, + Link: "https://nvd.nist.gov/vuln/detail/CVE-2024-1234", + } + + t.Run("KEV disabled", func(t *testing.T) { + t.Setenv(features.KnownExploitedVulnerabilities.EnvVar(), "false") + + row := formatCSVRow(r) + assert.Equal(t, len(formatCol()), len(row), "row should have same number of columns as header") + assert.Equal(t, "test-cluster", row[0]) + assert.Equal(t, "test-ns", row[1]) + assert.Equal(t, "test-deploy", row[2]) + assert.Equal(t, "CVE-2024-1234", row[6]) + assert.Equal(t, "true", row[7]) + assert.Equal(t, "CRITICAL", row[9]) + assert.Equal(t, "95.000", row[12]) + assert.Equal(t, "Not Available", row[14]) // Image Created Date (nil → "Not Available") + assert.Equal(t, "https://nvd.nist.gov/vuln/detail/CVE-2024-1234", row[15]) + }) + + t.Run("KEV enabled", func(t *testing.T) { + t.Setenv(features.KnownExploitedVulnerabilities.EnvVar(), "true") + + // The two KEV columns are inserted after EPSS (index 12), shifting all + // trailing columns right by two. + row := formatCSVRow(r) + assert.Equal(t, len(formatCol()), len(row), "row should have same number of columns as header") + assert.Equal(t, "test-cluster", row[0]) + assert.Equal(t, "95.000", row[12]) + assert.Equal(t, "Not Available", row[13]) // CISA KEV (nil → "Not Available") + assert.Equal(t, "Not Available", row[14]) // Known Ransomware Campaign (nil → "Not Available") + assert.Equal(t, "Not Available", row[16]) // Image Created Date (nil → "Not Available") + assert.Equal(t, "https://nvd.nist.gov/vuln/detail/CVE-2024-1234", row[17]) + }) +} + +func TestFormatCSVRow_NilFields(t *testing.T) { + r := &ImageCVEQueryResponse{} + + t.Run("KEV disabled", func(t *testing.T) { + t.Setenv(features.KnownExploitedVulnerabilities.EnvVar(), "false") + + row := formatCSVRow(r) + assert.Equal(t, len(formatCol()), len(row)) + assert.Equal(t, "", row[0]) + assert.Equal(t, "Not Available", row[12]) // EPSS Probability Percentage + assert.Equal(t, "Not Available", row[13]) // Discovered At + assert.Equal(t, "Not Available", row[14]) // Image Created Date + }) + + t.Run("KEV enabled", func(t *testing.T) { + t.Setenv(features.KnownExploitedVulnerabilities.EnvVar(), "true") + + row := formatCSVRow(r) + assert.Equal(t, len(formatCol()), len(row)) + assert.Equal(t, "", row[0]) + assert.Equal(t, "Not Available", row[12]) // EPSS Probability Percentage + assert.Equal(t, "Not Available", row[13]) // CISA KEV + assert.Equal(t, "Not Available", row[14]) // Known Ransomware Campaign + assert.Equal(t, "Not Available", row[15]) // Discovered At + assert.Equal(t, "Not Available", row[16]) // Image Created Date + }) +} + +func TestCsvReportName(t *testing.T) { + name := csvReportName("My Report") + assert.Contains(t, name, "RHACS_Vulnerability_Report_My Report_") + assert.Contains(t, name, ".csv") + + longName := strings.Repeat("a", 100) + name = csvReportName(longName) + assert.Contains(t, name, "...") +} diff --git a/central/reports/scheduler/v2/reportgenerator/email_formatter.go b/central/reports/scheduler/v2/reportgenerator/email_formatter.go index a50a4e00b11db..87d5d99284906 100644 --- a/central/reports/scheduler/v2/reportgenerator/email_formatter.go +++ b/central/reports/scheduler/v2/reportgenerator/email_formatter.go @@ -39,20 +39,26 @@ var ( } ) -func formatEmailSubject(subjectTemplate string, snapshot *storage.ReportSnapshot) (string, error) { +// FormatEmailSubject formats an email subject line from the given Go template +// and report snapshot. +func FormatEmailSubject(subjectTemplate string, snapshot *storage.ReportSnapshot) (string, error) { configName := snapshot.GetName() if len(configName) > maxConfigNameLenInSubject { configName = fmt.Sprintf("%s...", configName[0:maxConfigNameLenInSubject]) } - collectionName := snapshot.GetCollection().GetName() - if len(collectionName) > maxCollectionNameLenInSubject { - collectionName = fmt.Sprintf("%s...", collectionName[0:maxCollectionNameLenInSubject]) + scopeName := "Custom Scope" + if snapshot.GetCollection() != nil { + scopeName = snapshot.GetCollection().GetName() + } + + if len(scopeName) > maxCollectionNameLenInSubject { + scopeName = fmt.Sprintf("%s...", scopeName[0:maxCollectionNameLenInSubject]) } data := &reportEmailSubjectFormat{ BrandedProductNameShort: branding.GetProductNameShort(), ReportConfigName: configName, - CollectionName: collectionName, + CollectionName: scopeName, } tmpl, err := template.New("emailSubject").Parse(subjectTemplate) if err != nil { @@ -61,7 +67,8 @@ func formatEmailSubject(subjectTemplate string, snapshot *storage.ReportSnapshot return templates.ExecuteToString(tmpl, data) } -func formatEmailBody(emailTemplate string) (string, error) { +// FormatEmailBody formats an email body from the given Go template. +func FormatEmailBody(emailTemplate string) (string, error) { data := &reportEmailBodyFormat{ BrandedPrefix: branding.GetCombinedProductAndShortName(), } @@ -73,7 +80,8 @@ func formatEmailBody(emailTemplate string) (string, error) { return templates.ExecuteToString(tmpl, data) } -func addReportConfigDetails(emailBody, configDetailsHTML string) string { +// AddReportConfigDetails appends report configuration details HTML to the email body. +func AddReportConfigDetails(emailBody, configDetailsHTML string) string { var writer strings.Builder writer.WriteString(emailBody) writer.WriteString("

") @@ -89,7 +97,6 @@ func formatReportConfigDetails(snapshot *storage.ReportSnapshot, numDeployedImag if err != nil { return "", err } - reportFilters := snapshot.GetVulnReportFilters() writer.WriteString("
") @@ -100,27 +107,37 @@ func formatReportConfigDetails(snapshot *storage.ReportSnapshot, numDeployedImag formatSingleDetail(&writer, "Number of CVEs found", fmt.Sprintf("%d in Deployed images", numDeployedImageCVEs), fmt.Sprintf("%d in Watched images", numWatchedImageCVEs)) + // Collection scope: show severity, fixability, collection, image types, CVEs since + reportFilters := snapshot.GetVulnReportFilters() - // Severities - // create a copy because severities will be sorted in descending order (critical, important, moderate, low) - severities := append([]storage.VulnerabilitySeverity{}, reportFilters.GetSeverities()...) - sort.Slice(severities, func(i, j int) bool { - return severities[i] > severities[j] - }) - formatSingleDetail(&writer, "CVE severity", severities...) + if entityScope := snapshot.GetResourceScope().GetEntityScope(); entityScope != nil { + // Entity scope: show filter query and scope rules + if query := reportFilters.GetQuery(); query != "" { + formatSingleDetail(&writer, "Filter", query) + } + scopeParts := formatEntityScope(entityScope) + if len(scopeParts) > 0 { + formatSingleDetail(&writer, "Report scope", scopeParts...) + } + } else { - // Fixability - fixabilities := expandFixability(reportFilters.GetFixability()) - formatSingleDetail(&writer, "CVE status", fixabilities...) + // Severities + severities := append([]storage.VulnerabilitySeverity{}, reportFilters.GetSeverities()...) + sort.Slice(severities, func(i, j int) bool { + return severities[i] > severities[j] + }) + formatSingleDetail(&writer, "CVE severity", severities...) - // Collection - formatSingleDetail(&writer, "Report scope", snapshot.GetCollection()) + // Fixability + fixabilities := expandFixability(reportFilters.GetFixability()) + formatSingleDetail(&writer, "CVE status", fixabilities...) + // Collection + formatSingleDetail(&writer, "Report scope", snapshot.GetCollection()) + } // Image types - // create a copy because image types will be sorted in ascending order (deployed, watched) imageTypes := append([]storage.VulnerabilityReportFilters_ImageType{}, reportFilters.GetImageTypes()...) sliceutils.NaturalSort(imageTypes) - formatSingleDetail(&writer, "Image type", imageTypes...) // CVEs discovered since @@ -131,6 +148,76 @@ func formatReportConfigDetails(snapshot *storage.ReportSnapshot, numDeployedImag return writer.String(), nil } +// FormatNodeReportConfigDetails formats node report configuration details as HTML +// for inclusion in the notification email. +func FormatNodeReportConfigDetails(snapshot *storage.ReportSnapshot, numNodeCVEs int) (string, error) { + filters := snapshot.GetNodeVulnReportFilters() + if filters == nil { + return "", errors.New("report snapshot is missing node vulnerability report filters") + } + + var writer strings.Builder + writer.WriteString("
") + + formatSingleDetail(&writer, "Config name", snapshot.GetName()) + formatSingleDetail(&writer, "Number of CVEs found", fmt.Sprintf("%d", numNodeCVEs)) + + if query := filters.GetQuery(); query != "" { + formatSingleDetail(&writer, "Filter", query) + } + + if entityScope := snapshot.GetResourceScope().GetEntityScope(); entityScope != nil { + scopeParts := formatEntityScope(entityScope) + if len(scopeParts) > 0 { + formatSingleDetail(&writer, "Report scope", scopeParts...) + } + } + + writer.WriteString("
") + return writer.String(), nil +} + +func formatEntityScope(entityScope *storage.EntityScope) []string { + parts := make([]string, 0, len(entityScope.GetRules())) + for _, rule := range entityScope.GetRules() { + entityType := friendlyEntityType(rule.GetEntity()) + field := friendlyEntityField(rule.GetField()) + values := make([]string, 0, len(rule.GetValues())) + for _, v := range rule.GetValues() { + values = append(values, v.GetValue()) + } + parts = append(parts, fmt.Sprintf("%s %s: %s", entityType, field, strings.Join(values, ", "))) + } + return parts +} + +var entityTypeToText = map[storage.EntityType]string{ + storage.EntityType_ENTITY_TYPE_DEPLOYMENT: "Deployment", + storage.EntityType_ENTITY_TYPE_NAMESPACE: "Namespace", + storage.EntityType_ENTITY_TYPE_CLUSTER: "Cluster", +} + +var entityFieldToText = map[storage.EntityField]string{ + storage.EntityField_FIELD_ID: "ID", + storage.EntityField_FIELD_NAME: "Name", + storage.EntityField_FIELD_LABEL: "Label", + storage.EntityField_FIELD_ANNOTATION: "Annotation", +} + +func friendlyEntityType(t storage.EntityType) string { + if text, ok := entityTypeToText[t]; ok { + return text + } + return t.String() +} + +func friendlyEntityField(f storage.EntityField) string { + if text, ok := entityFieldToText[f]; ok { + return text + } + return f.String() +} + func expandFixability(fixability storage.VulnerabilityReportFilters_Fixability) []storage.VulnerabilityReportFilters_Fixability { if fixability == storage.VulnerabilityReportFilters_BOTH { return []storage.VulnerabilityReportFilters_Fixability{ @@ -192,8 +279,11 @@ func validateSnapshot(snapshot *storage.ReportSnapshot) error { if reportFilters == nil { return errors.New("Report snapshot is missing vulnerability report filters") } - if snapshot.GetCollection() == nil { - return errors.New("Report snapshot is missing collection snapshot") + + hasCollection := snapshot.GetCollection() != nil + hasEntityScope := snapshot.GetResourceScope().GetEntityScope() != nil + if !hasCollection && !hasEntityScope { + return errors.New("Report snapshot is missing both collection snapshot and entity scope") } if len(reportFilters.GetImageTypes()) == 0 { return errors.New("Report snapshot is missing image type filters") diff --git a/central/reports/scheduler/v2/reportgenerator/email_formatter_test.go b/central/reports/scheduler/v2/reportgenerator/email_formatter_test.go index 10e93e7bcb056..e1ae9b76e3d22 100644 --- a/central/reports/scheduler/v2/reportgenerator/email_formatter_test.go +++ b/central/reports/scheduler/v2/reportgenerator/email_formatter_test.go @@ -184,5 +184,170 @@ func (s *EmailFormatterTestSuite) configDetailsTestCases() []configDetailsTestCa
`, }, } + cases = append(cases, s.entityScopeTestCases()...) return cases } + +func (s *EmailFormatterTestSuite) TestFormatNodeReportConfigDetails() { + for _, tc := range s.nodeConfigDetailsTestCases() { + s.T().Run(tc.desc, func(t *testing.T) { + configHTML, err := FormatNodeReportConfigDetails(tc.snapshot, 25) + s.Require().NoError(err) + expectedHTML := strings.ReplaceAll(tc.expectedHTML, "\n", "") + expectedHTML = strings.ReplaceAll(expectedHTML, "\t", "") + s.Require().Equal(expectedHTML, configHTML) + }) + } +} + +func (s *EmailFormatterTestSuite) TestFormatNodeReportConfigDetails_NilFilters() { + snap := &storage.ReportSnapshot{ + Name: "Node Report", + } + _, err := FormatNodeReportConfigDetails(snap, 0) + s.Require().Error(err) +} + +func (s *EmailFormatterTestSuite) nodeConfigDetailsTestCases() []configDetailsTestCase { + return []configDetailsTestCase{ + { + desc: "Node report with entity scope and filter query", + snapshot: &storage.ReportSnapshot{ + Name: "Node Team Report", + Filter: &storage.ReportSnapshot_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "CVSS > 7", + }, + }, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "prod-us"}, + {Value: "prod-eu"}, + }, + }, + }, + }, + }, + }, + }, + expectedHTML: `
+
+ + Config name: + Node Team Report +
+
+ + Number of CVEs found: + 25 +
+
+ + Filter: + CVSS > 7 +
+
+ + Report scope: + Cluster Name: prod-us, prod-eu +
+
`, + }, + { + desc: "Node report without filter query or entity scope", + snapshot: &storage.ReportSnapshot{ + Name: "Simple Node Report", + Filter: &storage.ReportSnapshot_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{}, + }, + }, + expectedHTML: `
+
+ + Config name: + Simple Node Report +
+
+ + Number of CVEs found: + 25 +
+
`, + }, + } +} + +func (s *EmailFormatterTestSuite) entityScopeTestCases() []configDetailsTestCase { + return []configDetailsTestCase{ + { + desc: "Entity scope with filter query cvss > 7", + snapshot: func() *storage.ReportSnapshot { + snap := fixtures.GetReportSnapshot() + snap.Collection = nil + snap.ResourceScope = &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "production"}, + {Value: "staging"}, + }, + }, + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "main-cluster"}, + }, + }, + }, + }, + }, + } + snap.GetVulnReportFilters().Query = "CVSS > 7" + return snap + }(), + expectedHTML: `
+
+ + Config name: + App Team 1 Report +
+
+ + Number of CVEs found: + 50 in Deployed images, 30 in Watched images +
+
+ + Filter: + CVSS > 7 +
+
+ + Report scope: + Namespace Name: production, staging, Cluster Name: main-cluster +
+
+ + Image type: + Deployed images, Watched images +
+
+ + CVEs discovered since: + All time +
+
`, + }, + } +} diff --git a/central/reports/scheduler/v2/reportgenerator/mocks/report_gen.go b/central/reports/scheduler/v2/reportgenerator/mocks/report_gen.go index 75f2537d8acc8..0208b9c3abf3e 100644 --- a/central/reports/scheduler/v2/reportgenerator/mocks/report_gen.go +++ b/central/reports/scheduler/v2/reportgenerator/mocks/report_gen.go @@ -10,6 +10,7 @@ package mocks import ( + context "context" reflect "reflect" reportgenerator "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" @@ -41,13 +42,13 @@ func (m *MockReportGenerator) EXPECT() *MockReportGeneratorMockRecorder { } // ProcessReportRequest mocks base method. -func (m *MockReportGenerator) ProcessReportRequest(req *reportgenerator.ReportRequest) { +func (m *MockReportGenerator) ProcessReportRequest(ctx context.Context, req *reportgenerator.ReportRequest) { m.ctrl.T.Helper() - m.ctrl.Call(m, "ProcessReportRequest", req) + m.ctrl.Call(m, "ProcessReportRequest", ctx, req) } // ProcessReportRequest indicates an expected call of ProcessReportRequest. -func (mr *MockReportGeneratorMockRecorder) ProcessReportRequest(req any) *gomock.Call { +func (mr *MockReportGeneratorMockRecorder) ProcessReportRequest(ctx, req any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessReportRequest", reflect.TypeOf((*MockReportGenerator)(nil).ProcessReportRequest), req) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessReportRequest", reflect.TypeOf((*MockReportGenerator)(nil).ProcessReportRequest), ctx, req) } diff --git a/central/reports/scheduler/v2/reportgenerator/node/csv_gen.go b/central/reports/scheduler/v2/reportgenerator/node/csv_gen.go new file mode 100644 index 0000000000000..7117c8b747712 --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/node/csv_gen.go @@ -0,0 +1,77 @@ +package node + +import ( + "archive/zip" + "bytes" + "fmt" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/stackrox/rox/pkg/csv" + "github.com/stackrox/rox/pkg/stringutils" +) + +var csvHeader = []string{ + "Cluster", + "Node", + "Operating System", + "Component", + "Component Version", + "CVE", + "Fixable", + "CVE Fixed In", + "Severity", + "CVSS", + "First System Occurrence", + "Reference", +} + +func generateCSV(cveResponses []*NodeCVEQueryResponse, configName string) (*bytes.Buffer, error) { + csvWriter := csv.NewGenericWriter(csvHeader, true) + + for _, r := range cveResponses { + row := csv.Value{ + r.GetCluster(), + r.GetNode(), + r.GetOperatingSystem(), + r.GetComponent(), + r.GetComponentVersion(), + r.GetCVE(), + strconv.FormatBool(r.GetFixable()), + r.GetFixedByVersion(), + strings.ToTitle(stringutils.GetUpTo(r.GetSeverity().String(), "_")), + strconv.FormatFloat(r.GetCVSS(), 'f', 2, 64), + r.GetFirstOccurrence(), + r.GetLink(), + } + csvWriter.AddValue(row) + } + + var zipBuf bytes.Buffer + zipWriter := zip.NewWriter(&zipBuf) + truncatedName := configName + if runes := []rune(configName); len(runes) > 80 { + truncatedName = string(runes[:80]) + "..." + } + + now := time.Now() + reportName := fmt.Sprintf("RHACS_Node_Vulnerability_Report_%s_%s.csv", truncatedName, now.Format("02_January_2006")) + header := &zip.FileHeader{ + Name: reportName, + Method: zip.Deflate, + Modified: now, + } + zipFile, err := zipWriter.CreateHeader(header) + if err != nil { + return nil, errors.Wrapf(err, "unable to create the zip file for report config '%s'", configName) + } + if err = csvWriter.WriteCSV(zipFile); err != nil { + return nil, errors.Wrapf(err, "unable to write the zip file for report config '%s'", configName) + } + if err = zipWriter.Close(); err != nil { + return nil, errors.Wrapf(err, "unable to close the zip file for report config %s", configName) + } + return &zipBuf, nil +} diff --git a/central/reports/scheduler/v2/reportgenerator/node/csv_gen_test.go b/central/reports/scheduler/v2/reportgenerator/node/csv_gen_test.go new file mode 100644 index 0000000000000..4943131f8ea0a --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/node/csv_gen_test.go @@ -0,0 +1,174 @@ +package node + +import ( + "archive/zip" + "bytes" + "encoding/csv" + "testing" + "time" + + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateCSV_EmptyResponses(t *testing.T) { + buf, err := generateCSV(nil, "test-report") + require.NoError(t, err) + require.NotNil(t, buf) + + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + require.Len(t, reader.File, 1) + assert.Contains(t, reader.File[0].Name, "RHACS_Node_Vulnerability_Report_") + assert.Contains(t, reader.File[0].Name, "test-report") +} + +func TestGenerateCSV_WithResponses(t *testing.T) { + cluster := "prod-us" + node := "node-1" + operatingSystem := "rhcos:4.14" + component := "openssl" + componentVersion := "1.1.1" + cve := "CVE-2021-1234" + fixable := true + fixedBy := "1.1.2" + severity := storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY + cvss := 9.8 + + firstOccurrence := time.Date(2021, 6, 15, 0, 0, 0, 0, time.UTC) + + responses := []*NodeCVEQueryResponse{ + { + Cluster: &cluster, + Node: &node, + OperatingSystem: &operatingSystem, + Component: &component, + ComponentVersion: &componentVersion, + CVE: &cve, + Fixable: &fixable, + FixedByVersion: &fixedBy, + Severity: &severity, + CVSS: &cvss, + FirstOccurrence: &firstOccurrence, + Link: "https://nvd.nist.gov/vuln/detail/CVE-2021-1234", + }, + } + + buf, err := generateCSV(responses, "test-report") + require.NoError(t, err) + + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + require.Len(t, reader.File, 1) + + rc, err := reader.File[0].Open() + require.NoError(t, err) + defer func() { _ = rc.Close() }() + + csvReader := csv.NewReader(rc) + records, err := csvReader.ReadAll() + require.NoError(t, err) + + // Header + 1 data row + require.Len(t, records, 2) + assert.Equal(t, csvHeader, records[0]) + assert.Equal(t, "prod-us", records[1][0]) + assert.Equal(t, "node-1", records[1][1]) + assert.Equal(t, "rhcos:4.14", records[1][2]) + assert.Equal(t, "openssl", records[1][3]) + assert.Equal(t, "1.1.1", records[1][4]) + assert.Equal(t, "CVE-2021-1234", records[1][5]) + assert.Equal(t, "true", records[1][6]) + assert.Equal(t, "1.1.2", records[1][7]) + assert.Equal(t, "CRITICAL", records[1][8]) + assert.Equal(t, "9.80", records[1][9]) + assert.Equal(t, "June 15, 2021", records[1][10]) + assert.Equal(t, "https://nvd.nist.gov/vuln/detail/CVE-2021-1234", records[1][11]) +} + +func TestGenerateCSV_LongConfigNameIsTruncated(t *testing.T) { + longName := "" + for range 100 { + longName += "a" + } + buf, err := generateCSV(nil, longName) + require.NoError(t, err) + + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + require.Len(t, reader.File, 1) + assert.Contains(t, reader.File[0].Name, "...") + assert.LessOrEqual(t, len(reader.File[0].Name), 200) +} + +func TestGenerateCSV_ZipEntryHasModifiedTimestamp(t *testing.T) { + before := time.Now().Add(-time.Minute) + buf, err := generateCSV(nil, "test") + require.NoError(t, err) + + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + require.Len(t, reader.File, 1) + assert.False(t, reader.File[0].Modified.IsZero(), "ZIP entry Modified timestamp should not be zero") + assert.True(t, reader.File[0].Modified.After(before), "ZIP entry Modified timestamp should be recent") +} + +func TestGenerateCSV_NilFieldsProduceDefaults(t *testing.T) { + responses := []*NodeCVEQueryResponse{ + {}, + } + + buf, err := generateCSV(responses, "defaults-test") + require.NoError(t, err) + + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + + rc, err := reader.File[0].Open() + require.NoError(t, err) + defer func() { _ = rc.Close() }() + + csvReader := csv.NewReader(rc) + records, err := csvReader.ReadAll() + require.NoError(t, err) + require.Len(t, records, 2) + + row := records[1] + assert.Equal(t, "", row[0]) // Cluster + assert.Equal(t, "", row[1]) // Node + assert.Equal(t, "Not Available", row[2]) // Operating System + assert.Equal(t, "", row[3]) // Component + assert.Equal(t, "", row[4]) // ComponentVersion + assert.Equal(t, "", row[5]) // CVE + assert.Equal(t, "false", row[6]) // Fixable + assert.Equal(t, "", row[7]) // FixedBy + assert.Equal(t, "UNKNOWN", row[8]) // Severity + assert.Equal(t, "0.00", row[9]) // CVSS + assert.Equal(t, "Not Available", row[10]) // First System Occurrence + assert.Equal(t, "", row[11]) // Link +} + +func TestGenerateCSV_EmptyOperatingSystemProducesDefault(t *testing.T) { + emptyOS := "" + responses := []*NodeCVEQueryResponse{ + {OperatingSystem: &emptyOS}, + } + + buf, err := generateCSV(responses, "empty-os-test") + require.NoError(t, err) + + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + + rc, err := reader.File[0].Open() + require.NoError(t, err) + defer func() { _ = rc.Close() }() + + csvReader := csv.NewReader(rc) + records, err := csvReader.ReadAll() + require.NoError(t, err) + require.Len(t, records, 2) + + assert.Equal(t, "Not Available", records[1][2]) +} diff --git a/central/reports/scheduler/v2/reportgenerator/node/query_builder.go b/central/reports/scheduler/v2/reportgenerator/node/query_builder.go new file mode 100644 index 0000000000000..007d7b4cf7424 --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/node/query_builder.go @@ -0,0 +1,84 @@ +package node + +import ( + "github.com/pkg/errors" + "github.com/stackrox/rox/central/reports/common" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/sac/effectiveaccessscope" + "github.com/stackrox/rox/pkg/search" +) + +type queryBuilder struct { + filters *storage.NodeVulnerabilityReportFilters + entityScope *storage.EntityScope +} + +func newQueryBuilder(entityScope *storage.EntityScope, filters *storage.NodeVulnerabilityReportFilters) *queryBuilder { + return &queryBuilder{ + filters: filters, + entityScope: entityScope, + } +} + +func (q *queryBuilder) buildQuery(clusters []effectiveaccessscope.Cluster, namespaces []effectiveaccessscope.Namespace) (*v1.Query, error) { + var conjuncts []*v1.Query + + if q.entityScope != nil { + scopeQuery, err := q.buildEntityScopeQuery() + if err != nil { + return nil, err + } + conjuncts = append(conjuncts, scopeQuery) + } + + accessScopeQuery, err := common.BuildClusterOnlyAccessScopeQuery(q.filters.GetAccessScopeRules(), clusters, namespaces) + if err != nil { + return nil, err + } + conjuncts = append(conjuncts, accessScopeQuery) + + cveFilterQuery, err := search.ParseQuery(q.filters.GetQuery(), search.MatchAllIfEmpty()) + if err != nil { + return nil, err + } + conjuncts = append(conjuncts, cveFilterQuery) + + return search.ConjunctionQuery(conjuncts...), nil +} + +func (q *queryBuilder) buildEntityScopeQuery() (*v1.Query, error) { + var conjuncts []*v1.Query + for _, rule := range q.entityScope.GetRules() { + if len(rule.GetValues()) == 0 { + continue + } + fieldLabel, err := nodeEntityScopeRuleToFieldLabel(rule) + if err != nil { + return nil, err + } + if fieldLabel == search.ClusterLabel { + conjuncts = append(conjuncts, common.MapFieldQuery(fieldLabel, rule)) + } else { + conjuncts = append(conjuncts, common.ScalarFieldQuery(fieldLabel, rule)) + } + } + + if len(conjuncts) == 0 { + return search.EmptyQuery(), nil + } + return search.ConjunctionQuery(conjuncts...), nil +} + +func nodeEntityScopeRuleToFieldLabel(rule *storage.EntityScopeRule) (search.FieldLabel, error) { + if rule.GetEntity() != storage.EntityType_ENTITY_TYPE_CLUSTER { + return "", errors.Errorf("unsupported entity type %s for node reports; only ENTITY_TYPE_CLUSTER is allowed", rule.GetEntity()) + } + switch rule.GetField() { + case storage.EntityField_FIELD_NAME: + return search.Cluster, nil + case storage.EntityField_FIELD_LABEL: + return search.ClusterLabel, nil + } + return "", errors.Errorf("unsupported field %s for cluster entity scope", rule.GetField()) +} diff --git a/central/reports/scheduler/v2/reportgenerator/node/query_builder_test.go b/central/reports/scheduler/v2/reportgenerator/node/query_builder_test.go new file mode 100644 index 0000000000000..cbd75121835fe --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/node/query_builder_test.go @@ -0,0 +1,277 @@ +package node + +import ( + "testing" + + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/sac/effectiveaccessscope" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/assert" +) + +var testClusters = []effectiveaccessscope.Cluster{ + &storage.Cluster{ + Id: uuid.NewV4().String(), + Name: "prod-us", + }, + &storage.Cluster{ + Id: uuid.NewV4().String(), + Name: "prod-eu", + }, +} + +func TestNodeEntityScopeRuleToFieldLabel(t *testing.T) { + testCases := map[string]struct { + rule *storage.EntityScopeRule + expected search.FieldLabel + hasError bool + }{ + "Cluster name returns search.Cluster": { + rule: &storage.EntityScopeRule{ + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + }, + expected: search.Cluster, + }, + "Cluster label returns search.ClusterLabel": { + rule: &storage.EntityScopeRule{ + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_LABEL, + }, + expected: search.ClusterLabel, + }, + "Namespace entity is not allowed": { + rule: &storage.EntityScopeRule{ + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + }, + hasError: true, + }, + "Deployment entity is not allowed": { + rule: &storage.EntityScopeRule{ + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + }, + hasError: true, + }, + "Cluster annotation is not allowed": { + rule: &storage.EntityScopeRule{ + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_ANNOTATION, + }, + hasError: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + result, err := nodeEntityScopeRuleToFieldLabel(tc.rule) + if tc.hasError { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestBuildEntityScopeQuery(t *testing.T) { + testCases := map[string]struct { + scope *storage.EntityScope + expected *v1.Query + hasError bool + }{ + "Empty rules returns empty query": { + scope: &storage.EntityScope{}, + expected: search.EmptyQuery(), + }, + "Single cluster name rule": { + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "prod-us", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + expected: search.NewQueryBuilder().AddExactMatches(search.Cluster, "prod-us").ProtoQuery(), + }, + "Multiple cluster name values are ORed": { + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "prod-us", MatchType: storage.MatchType_EXACT}, + {Value: "prod-eu", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + expected: search.DisjunctionQuery( + search.NewQueryBuilder().AddExactMatches(search.Cluster, "prod-us").ProtoQuery(), + search.NewQueryBuilder().AddExactMatches(search.Cluster, "prod-eu").ProtoQuery(), + ), + }, + "Cluster label rule uses map query": { + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_LABEL, + Values: []*storage.RuleValue{ + {Value: "env=prod", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + expected: search.NewQueryBuilder().AddMapQuery(search.ClusterLabel, `"env"`, `"prod"`).ProtoQuery(), + }, + "Regex match type": { + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "prod-.*", MatchType: storage.MatchType_REGEX}, + }, + }, + }, + }, + expected: search.NewQueryBuilder().AddRegexes(search.Cluster, "prod-.*").ProtoQuery(), + }, + "Multiple rules are ANDed": { + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "prod-us", MatchType: storage.MatchType_EXACT}, + }, + }, + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_LABEL, + Values: []*storage.RuleValue{ + {Value: "tier=premium", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + expected: search.ConjunctionQuery( + search.NewQueryBuilder().AddExactMatches(search.Cluster, "prod-us").ProtoQuery(), + search.NewQueryBuilder().AddMapQuery(search.ClusterLabel, `"tier"`, `"premium"`).ProtoQuery(), + ), + }, + "Rule with empty values is skipped": { + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{}, + }, + }, + }, + expected: search.EmptyQuery(), + }, + "Namespace rule returns error": { + scope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "default", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + hasError: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + qb := &queryBuilder{entityScope: tc.scope} + result, err := qb.buildEntityScopeQuery() + if tc.hasError { + assert.Error(t, err) + return + } + assert.NoError(t, err) + protoassert.Equal(t, tc.expected, result) + }) + } +} + +func TestBuildQuery(t *testing.T) { + testCases := map[string]struct { + entityScope *storage.EntityScope + filters *storage.NodeVulnerabilityReportFilters + hasError bool + }{ + "View-based (nil entity scope) builds without error": { + entityScope: nil, + filters: &storage.NodeVulnerabilityReportFilters{}, + }, + "Config-based with entity scope builds without error": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "prod-us", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + filters: &storage.NodeVulnerabilityReportFilters{}, + }, + "Config-based with query filter": { + entityScope: &storage.EntityScope{}, + filters: &storage.NodeVulnerabilityReportFilters{ + Query: "CVE:CVE-2021-1234", + }, + }, + "Invalid entity scope returns error": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "default", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + filters: &storage.NodeVulnerabilityReportFilters{}, + hasError: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + qb := newQueryBuilder(tc.entityScope, tc.filters) + result, err := qb.buildQuery(testClusters, nil) + if tc.hasError { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.NotNil(t, result) + }) + } +} diff --git a/central/reports/scheduler/v2/reportgenerator/node/report_gen.go b/central/reports/scheduler/v2/reportgenerator/node/report_gen.go new file mode 100644 index 0000000000000..3e1ead3cb5235 --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/node/report_gen.go @@ -0,0 +1,33 @@ +package node + +import ( + blobDS "github.com/stackrox/rox/central/blob/datastore" + clusterDS "github.com/stackrox/rox/central/cluster/datastore" + nodeCVEDS "github.com/stackrox/rox/central/cve/node/datastore" + namespaceDS "github.com/stackrox/rox/central/namespace/datastore" + reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" + reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + "github.com/stackrox/rox/pkg/notifier" + "github.com/stackrox/rox/pkg/postgres" +) + +// New creates a new NodeReportGenerator instance. +func New( + db postgres.DB, + reportSnapshotStore reportSnapshotDS.DataStore, + notificationProcessor notifier.Processor, + blobDatastore blobDS.Datastore, + clusterDatastore clusterDS.DataStore, + namespaceDatastore namespaceDS.DataStore, + nodeCVEDatastore nodeCVEDS.DataStore, +) reportGen.ReportGenerator { + return &nodeReportGeneratorImpl{ + reportSnapshotStore: reportSnapshotStore, + notificationProcessor: notificationProcessor, + blobStore: blobDatastore, + clusterDatastore: clusterDatastore, + namespaceDatastore: namespaceDatastore, + nodeCVEDatastore: nodeCVEDatastore, + db: db, + } +} diff --git a/central/reports/scheduler/v2/reportgenerator/node/report_gen_impl.go b/central/reports/scheduler/v2/reportgenerator/node/report_gen_impl.go new file mode 100644 index 0000000000000..af815384121a8 --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/node/report_gen_impl.go @@ -0,0 +1,266 @@ +package node + +import ( + "bytes" + "context" + + "github.com/pkg/errors" + clusterDS "github.com/stackrox/rox/central/cluster/datastore" + "github.com/stackrox/rox/central/convert/storagetoeffectiveaccessscope" + nodeCVEDS "github.com/stackrox/rox/central/cve/node/datastore" + namespaceDS "github.com/stackrox/rox/central/namespace/datastore" + reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" + reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/errorhelpers" + "github.com/stackrox/rox/pkg/notifier" + "github.com/stackrox/rox/pkg/notifiers" + "github.com/stackrox/rox/pkg/postgres" + pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/effectiveaccessscope" + "github.com/stackrox/rox/pkg/search" + pgSearch "github.com/stackrox/rox/pkg/search/postgres" + "github.com/stackrox/rox/pkg/set" + + blobDS "github.com/stackrox/rox/central/blob/datastore" +) + +var ( + nodeReportGenCtx = sac.WithAllAccess(context.Background()) + + nodeQueryParts = &reportGen.ReportQueryParts{ + Schema: pkgSchema.NodeCvesSchema, + Selects: []*v1.QuerySelect{ + search.NewQuerySelect(search.Cluster).Proto(), + search.NewQuerySelect(search.Node).Proto(), + search.NewQuerySelect(search.OperatingSystem).Proto(), + search.NewQuerySelect(search.Component).Proto(), + search.NewQuerySelect(search.ComponentVersion).Proto(), + search.NewQuerySelect(search.CVEID).Proto(), + search.NewQuerySelect(search.CVE).Proto(), + search.NewQuerySelect(search.Fixable).Proto(), + search.NewQuerySelect(search.FixedBy).Proto(), + search.NewQuerySelect(search.Severity).Proto(), + search.NewQuerySelect(search.CVSS).Proto(), + search.NewQuerySelect(search.CVECreatedTime).Proto(), + }, + Pagination: search.NewPagination(). + Limit(int32(env.ReportMaxRows.IntegerSetting())). + Offset(0). + AddSortOption(search.NewSortOption(search.Cluster)). + AddSortOption(search.NewSortOption(search.Node)). + Proto(), + } +) + +type nodeReportGeneratorImpl struct { + reportSnapshotStore reportSnapshotDS.DataStore + notificationProcessor notifier.Processor + blobStore blobDS.Datastore + clusterDatastore clusterDS.DataStore + namespaceDatastore namespaceDS.DataStore + nodeCVEDatastore nodeCVEDS.DataStore + db postgres.DB +} + +func (rg *nodeReportGeneratorImpl) ProcessReportRequest(ctx context.Context, req *reportGen.ReportRequest) { + ctx = sac.WithAllAccess(ctx) + + if err := reportGen.ValidateReportRequest(req); err != nil { + reportGen.LogAndUpsertError(ctx, nodeReportGenCtx, rg.reportSnapshotStore, errors.Wrap(err, "Invalid report request"), req) + return + } + + if err := reportGen.UpdateReportStatus(nodeReportGenCtx, rg.reportSnapshotStore, req.ReportSnapshot, storage.ReportStatus_PREPARING); err != nil { + reportGen.LogAndUpsertError(ctx, nodeReportGenCtx, rg.reportSnapshotStore, errors.Wrap(err, "Error changing report status to PREPARING"), req) + return + } + + if err := rg.generateReportAndNotify(ctx, req); err != nil { + reportGen.LogAndUpsertError(ctx, nodeReportGenCtx, rg.reportSnapshotStore, err, req) + return + } + + if req.ReportSnapshot.GetReportStatus().GetReportNotificationMethod() == storage.ReportStatus_EMAIL { + if err := reportGen.UpdateReportStatus(nodeReportGenCtx, rg.reportSnapshotStore, req.ReportSnapshot, storage.ReportStatus_DELIVERED); err != nil { + reportGen.LogAndUpsertError(ctx, nodeReportGenCtx, rg.reportSnapshotStore, errors.Wrap(err, "Error changing report status to DELIVERED"), req) + } + } +} + +func (rg *nodeReportGeneratorImpl) generateReportAndNotify(ctx context.Context, req *reportGen.ReportRequest) error { + if err := ctx.Err(); err != nil { + return err + } + + cveResponses, numResults, err := rg.getReportData(ctx, req.ReportSnapshot) + if err != nil { + return err + } + + if err := ctx.Err(); err != nil { + return err + } + + zippedCSVData, err := generateCSV(cveResponses, req.ReportSnapshot.GetName()) + if err != nil { + return err + } + + req.ReportSnapshot.ReportStatus.CompletedAt = protocompat.TimestampNow() + if err := reportGen.UpdateReportStatus(nodeReportGenCtx, rg.reportSnapshotStore, req.ReportSnapshot, storage.ReportStatus_GENERATED); err != nil { + return errors.Wrap(err, "Error changing report status to GENERATED") + } + if err := ctx.Err(); err != nil { + return err + } + + switch req.ReportSnapshot.GetReportStatus().GetReportNotificationMethod() { + case storage.ReportStatus_DOWNLOAD: + parentDir := req.ReportSnapshot.GetReportConfigurationId() + if req.ReportSnapshot.GetReportStatus().GetReportRequestType() == storage.ReportStatus_VIEW_BASED { + parentDir = "view-based-report" + } + if err := reportGen.SaveReportData(nodeReportGenCtx, rg.blobStore, parentDir, req.ReportSnapshot.GetReportId(), zippedCSVData); err != nil { + return errors.Wrap(err, "error persisting blob") + } + case storage.ReportStatus_EMAIL: + return rg.sendEmailNotification(req, zippedCSVData, numResults) + } + return nil +} + +func (rg *nodeReportGeneratorImpl) sendEmailNotification(req *reportGen.ReportRequest, zippedCSVData *bytes.Buffer, numResults int) error { + defaultEmailSubject, err := reportGen.FormatEmailSubject(reportGen.NodeDefaultEmailSubjectTemplate, req.ReportSnapshot) + if err != nil { + return errors.Wrap(err, "Error generating email subject") + } + + templateStr := reportGen.NodeDefaultEmailBodyTemplate + if numResults == 0 { + zippedCSVData = nil + templateStr = reportGen.NodeDefaultNoVulnsEmailBodyTemplate + } + + defaultEmailBody, err := reportGen.FormatEmailBody(templateStr) + if err != nil { + return errors.Wrap(err, "Error generating email body") + } + + configDetailsHTML, err := reportGen.FormatNodeReportConfigDetails(req.ReportSnapshot, numResults) + if err != nil { + return errors.Wrap(err, "Error adding report config details") + } + + errorList := errorhelpers.NewErrorList("Error sending email notifications: ") + for _, notifierSnap := range req.ReportSnapshot.GetNotifiers() { + nf := rg.notificationProcessor.GetNotifier(nodeReportGenCtx, notifierSnap.GetEmailConfig().GetNotifierId()) + reportNotifier, ok := nf.(notifiers.ReportNotifier) + if !ok { + errorList.AddError(errors.Errorf("incorrect type of notifier '%s'", notifierSnap.GetEmailConfig().GetNotifierId())) + continue + } + customBody := notifierSnap.GetEmailConfig().GetCustomBody() + emailBody := defaultEmailBody + if customBody != "" { + emailBody = customBody + } + customSubject := notifierSnap.GetEmailConfig().GetCustomSubject() + emailSubject := defaultEmailSubject + if customSubject != "" { + emailSubject = customSubject + } + emailBodyWithConfigDetails := reportGen.AddReportConfigDetails(emailBody, configDetailsHTML) + reportName := req.ReportSnapshot.GetName() + if err := reportGen.RetryableSendReportResults(nodeReportGenCtx, reportNotifier, notifierSnap.GetEmailConfig().GetMailingLists(), + zippedCSVData, emailSubject, emailBodyWithConfigDetails, reportName); err != nil { + errorList.AddError(errors.Errorf("Error sending email for notifier '%s': %s", + notifierSnap.GetEmailConfig().GetNotifierId(), err)) + } + } + if !errorList.Empty() { + return errorList.ToError() + } + return nil +} + +func (rg *nodeReportGeneratorImpl) getReportData(ctx context.Context, snap *storage.ReportSnapshot) ([]*NodeCVEQueryResponse, int, error) { + clusters, namespaces, err := rg.getClustersAndNamespacesForSAC(ctx) + if err != nil { + return nil, 0, err + } + + var entityScope *storage.EntityScope + if snap.GetReportStatus().GetReportRequestType() != storage.ReportStatus_VIEW_BASED { + entityScope = snap.GetResourceScope().GetEntityScope() + } + + qb := newQueryBuilder(entityScope, snap.GetNodeVulnReportFilters()) + query, err := qb.buildQuery(clusters, namespaces) + if err != nil { + return nil, 0, errors.Wrap(err, "error building node report query") + } + + return rg.executeQuery(ctx, query) +} + +func (rg *nodeReportGeneratorImpl) executeQuery(ctx context.Context, query *v1.Query) ([]*NodeCVEQueryResponse, int, error) { + query.Pagination = nodeQueryParts.Pagination + query.Selects = nodeQueryParts.Selects + + var cveResponses []*NodeCVEQueryResponse + cveIDs := set.NewStringSet() + err := pgSearch.RunSelectRequestForSchemaFn[NodeCVEQueryResponse](ctx, rg.db, + nodeQueryParts.Schema, query, func(r *NodeCVEQueryResponse) error { + cveResponses = append(cveResponses, r) + if r.GetCVEID() != "" { + cveIDs.Add(r.GetCVEID()) + } + return nil + }) + if err != nil { + return nil, 0, errors.Wrap(err, "Failed to collect node CVE report data") + } + + cveResponses, err = rg.withCVEReferenceLinks(ctx, cveResponses, cveIDs) + if err != nil { + return nil, 0, err + } + + return cveResponses, len(cveResponses), nil +} + +func (rg *nodeReportGeneratorImpl) getClustersAndNamespacesForSAC(ctx context.Context) ([]effectiveaccessscope.Cluster, []effectiveaccessscope.Namespace, error) { + allClusters, err := rg.clusterDatastore.GetClusters(ctx) + if err != nil { + return nil, nil, errors.Wrap(err, "error fetching clusters to build report query") + } + allNamespaces, err := rg.namespaceDatastore.GetAllNamespaces(ctx) + if err != nil { + return nil, nil, errors.Wrap(err, "error fetching namespaces to build report query") + } + return storagetoeffectiveaccessscope.Clusters(allClusters), storagetoeffectiveaccessscope.Namespaces(allNamespaces), nil +} + +func (rg *nodeReportGeneratorImpl) withCVEReferenceLinks(ctx context.Context, responses []*NodeCVEQueryResponse, cveIDs set.StringSet) ([]*NodeCVEQueryResponse, error) { + nodeCVEs, err := rg.nodeCVEDatastore.GetBatch(ctx, cveIDs.AsSlice()) + if err != nil { + return nil, err + } + + cveRefLinks := make(map[string]string, len(nodeCVEs)) + for _, cve := range nodeCVEs { + cveRefLinks[cve.GetId()] = cve.GetCveBaseInfo().GetLink() + } + + for _, res := range responses { + if link, ok := cveRefLinks[res.GetCVEID()]; ok { + res.Link = link + } + } + return responses, nil +} diff --git a/central/reports/scheduler/v2/reportgenerator/node/report_gen_impl_test.go b/central/reports/scheduler/v2/reportgenerator/node/report_gen_impl_test.go new file mode 100644 index 0000000000000..81a0c959e7a1c --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/node/report_gen_impl_test.go @@ -0,0 +1,387 @@ +//go:build sql_integration + +package node + +import ( + "context" + "fmt" + "testing" + "time" + + clusterDSMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" + nodeCVEDS "github.com/stackrox/rox/central/cve/node/datastore" + namespaceDSMocks "github.com/stackrox/rox/central/namespace/datastore/mocks" + nodeDS "github.com/stackrox/rox/central/node/datastore" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/fixtures" + "github.com/stackrox/rox/pkg/postgres/pgtest" + postgresSchema "github.com/stackrox/rox/pkg/postgres/schema" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/set" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/suite" + "go.uber.org/mock/gomock" +) + +type nodeReportData struct { + clusterNames []string + nodeNames []string + componentNames []string + cveNames []string +} + +func TestNodeReportGenerator(t *testing.T) { + suite.Run(t, new(NodeReportGeneratorTestSuite)) +} + +type NodeReportGeneratorTestSuite struct { + suite.Suite + + ctx context.Context + testDB *pgtest.TestPostgres + clusterDatastore *clusterDSMocks.MockDataStore + namespaceDatastore *namespaceDSMocks.MockDataStore + nodeDatastore nodeDS.DataStore + nodeCVEDatastore nodeCVEDS.DataStore + reportGenerator *nodeReportGeneratorImpl + + clusters []*storage.Cluster + namespaces []*storage.NamespaceMetadata +} + +func (s *NodeReportGeneratorTestSuite) SetupSuite() { + s.ctx = sac.WithAllAccess(context.Background()) + mockCtrl := gomock.NewController(s.T()) + s.testDB = pgtest.ForT(s.T()) + + s.clusterDatastore = clusterDSMocks.NewMockDataStore(mockCtrl) + s.namespaceDatastore = namespaceDSMocks.NewMockDataStore(mockCtrl) + + s.nodeDatastore = nodeDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + var err error + s.nodeCVEDatastore, err = nodeCVEDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(err) + + s.clusters = []*storage.Cluster{ + {Id: uuid.NewV4().String(), Name: "c1"}, + {Id: uuid.NewV4().String(), Name: "c2"}, + } + s.namespaces = []*storage.NamespaceMetadata{ + {Id: uuid.NewV4().String(), Name: "ns1", ClusterId: s.clusters[0].GetId(), ClusterName: "c1"}, + {Id: uuid.NewV4().String(), Name: "ns2", ClusterId: s.clusters[1].GetId(), ClusterName: "c2"}, + } + s.clusterDatastore.EXPECT().GetClusters(gomock.Any()). + Return(s.clusters, nil).AnyTimes() + s.namespaceDatastore.EXPECT().GetAllNamespaces(gomock.Any()). + Return(s.namespaces, nil).AnyTimes() + + nodes := s.testNodes() + for _, node := range nodes { + s.Require().NoError(s.nodeDatastore.UpsertNode(s.ctx, node)) + } + + s.reportGenerator = &nodeReportGeneratorImpl{ + clusterDatastore: s.clusterDatastore, + namespaceDatastore: s.namespaceDatastore, + nodeCVEDatastore: s.nodeCVEDatastore, + db: s.testDB.DB, + } +} + +func (s *NodeReportGeneratorTestSuite) TearDownSuite() { + s.truncateTable(postgresSchema.NodeCvesTableName) + s.truncateTable(postgresSchema.NodeComponentsCvesEdgesTableName) + s.truncateTable(postgresSchema.NodeComponentsTableName) + s.truncateTable(postgresSchema.NodeComponentEdgesTableName) + s.truncateTable(postgresSchema.NodesTableName) +} + +func (s *NodeReportGeneratorTestSuite) truncateTable(name string) { + sql := fmt.Sprintf("TRUNCATE %s CASCADE", name) + _, err := s.testDB.Exec(s.ctx, sql) + s.NoError(err) +} + +func (s *NodeReportGeneratorTestSuite) testNodes() []*storage.Node { + t, err := protocompat.ConvertTimeToTimestampOrError(time.Unix(0, 1000)) + s.Require().NoError(err) + + var nodes []*storage.Node + for _, cluster := range s.clusters { + for i := range 2 { + nodeName := fmt.Sprintf("%s_node%d", cluster.GetName(), i) + node := &storage.Node{ + Id: uuid.NewV4().String(), + Name: nodeName, + ClusterId: cluster.GetId(), + ClusterName: cluster.GetName(), + Scan: &storage.NodeScan{ + ScanTime: protocompat.TimestampNow(), + OperatingSystem: "linux", + Components: []*storage.EmbeddedNodeScanComponent{ + { + Name: fmt.Sprintf("%s_comp", nodeName), + Version: "1.0", + Vulnerabilities: []*storage.NodeVulnerability{ + { + CveBaseInfo: &storage.CVEInfo{ + Cve: fmt.Sprintf("CVE-critical-%s_comp", nodeName), + CreatedAt: t, + }, + Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, + Cvss: 9.0, + SetFixedBy: &storage.NodeVulnerability_FixedBy{ + FixedBy: "1.1", + }, + }, + { + CveBaseInfo: &storage.CVEInfo{ + Cve: fmt.Sprintf("CVE-low-%s_comp", nodeName), + CreatedAt: t, + }, + Severity: storage.VulnerabilitySeverity_LOW_VULNERABILITY_SEVERITY, + Cvss: 2.0, + }, + }, + }, + }, + }, + } + nodes = append(nodes, node) + } + } + return nodes +} + +func (s *NodeReportGeneratorTestSuite) TestGetReportData_AllNodes() { + snap := s.testNodeReportSnapshot(nil, "", nil, storage.ReportStatus_ON_DEMAND) + cveResponses, _, err := s.reportGenerator.getReportData(s.ctx, snap) + s.NoError(err) + + collected := collectNodeReportData(cveResponses) + s.ElementsMatch([]string{"c1", "c2"}, collected.clusterNames) + s.ElementsMatch( + []string{"c1_node0", "c1_node1", "c2_node0", "c2_node1"}, + collected.nodeNames, + ) + s.ElementsMatch( + []string{"c1_node0_comp", "c1_node1_comp", "c2_node0_comp", "c2_node1_comp"}, + collected.componentNames, + ) + s.ElementsMatch( + []string{ + "CVE-critical-c1_node0_comp", "CVE-low-c1_node0_comp", + "CVE-critical-c1_node1_comp", "CVE-low-c1_node1_comp", + "CVE-critical-c2_node0_comp", "CVE-low-c2_node0_comp", + "CVE-critical-c2_node1_comp", "CVE-low-c2_node1_comp", + }, + collected.cveNames, + ) +} + +func (s *NodeReportGeneratorTestSuite) TestGetReportData_EntityScope_SingleCluster() { + entityScope := &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + } + scopeRules := []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1"}}, + } + snap := s.testNodeReportSnapshot(entityScope, "", scopeRules, storage.ReportStatus_ON_DEMAND) + cveResponses, _, err := s.reportGenerator.getReportData(s.ctx, snap) + s.NoError(err) + + collected := collectNodeReportData(cveResponses) + s.ElementsMatch([]string{"c1"}, collected.clusterNames) + s.ElementsMatch([]string{"c1_node0", "c1_node1"}, collected.nodeNames) + s.ElementsMatch([]string{"c1_node0_comp", "c1_node1_comp"}, collected.componentNames) + s.ElementsMatch( + []string{ + "CVE-critical-c1_node0_comp", "CVE-low-c1_node0_comp", + "CVE-critical-c1_node1_comp", "CVE-low-c1_node1_comp", + }, + collected.cveNames, + ) +} + +func (s *NodeReportGeneratorTestSuite) TestGetReportData_PartiallyIncludedCluster() { + // User has namespace-only access (no cluster-level rule) for c1. + // Nodes from c1 should still be visible because the cluster is partially included. + scopeRules := []*storage.SimpleAccessScope_Rules{ + { + IncludedNamespaces: []*storage.SimpleAccessScope_Rules_Namespace{ + {ClusterName: "c1", NamespaceName: "ns1"}, + }, + }, + } + snap := s.testNodeReportSnapshot(nil, "", scopeRules, storage.ReportStatus_VIEW_BASED) + cveResponses, _, err := s.reportGenerator.getReportData(s.ctx, snap) + s.NoError(err) + + collected := collectNodeReportData(cveResponses) + s.ElementsMatch([]string{"c1"}, collected.clusterNames) + s.ElementsMatch([]string{"c1_node0", "c1_node1"}, collected.nodeNames) +} + +func (s *NodeReportGeneratorTestSuite) TestGetReportData_EntityScope_WithCVSSFilter() { + entityScope := &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + } + scopeRules := []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1"}}, + } + snap := s.testNodeReportSnapshot(entityScope, "CVSS:>=7.0", scopeRules, storage.ReportStatus_ON_DEMAND) + cveResponses, _, err := s.reportGenerator.getReportData(s.ctx, snap) + s.NoError(err) + + collected := collectNodeReportData(cveResponses) + s.ElementsMatch([]string{"c1"}, collected.clusterNames) + s.ElementsMatch([]string{"c1_node0", "c1_node1"}, collected.nodeNames) + s.ElementsMatch([]string{"c1_node0_comp", "c1_node1_comp"}, collected.componentNames) + s.ElementsMatch( + []string{ + "CVE-critical-c1_node0_comp", + "CVE-critical-c1_node1_comp", + }, + collected.cveNames, + ) +} + +func (s *NodeReportGeneratorTestSuite) TestGetReportData_ViewBased() { + scopeRules := []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1"}}, + } + snap := s.testNodeReportSnapshot(nil, "CVSS:>=7.0", scopeRules, storage.ReportStatus_VIEW_BASED) + cveResponses, _, err := s.reportGenerator.getReportData(s.ctx, snap) + s.NoError(err) + + collected := collectNodeReportData(cveResponses) + s.ElementsMatch([]string{"c1"}, collected.clusterNames) + s.ElementsMatch([]string{"c1_node0", "c1_node1"}, collected.nodeNames) + s.ElementsMatch([]string{"c1_node0_comp", "c1_node1_comp"}, collected.componentNames) + s.ElementsMatch( + []string{ + "CVE-critical-c1_node0_comp", + "CVE-critical-c1_node1_comp", + }, + collected.cveNames, + ) +} + +func (s *NodeReportGeneratorTestSuite) TestGetReportData_ClusterRegexScope() { + entityScope := &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c.*", MatchType: storage.MatchType_REGEX}, + }, + }, + }, + } + snap := s.testNodeReportSnapshot(entityScope, "", nil, storage.ReportStatus_ON_DEMAND) + cveResponses, _, err := s.reportGenerator.getReportData(s.ctx, snap) + s.NoError(err) + + collected := collectNodeReportData(cveResponses) + s.ElementsMatch([]string{"c1", "c2"}, collected.clusterNames) + s.ElementsMatch( + []string{"c1_node0", "c1_node1", "c2_node0", "c2_node1"}, + collected.nodeNames, + ) + s.ElementsMatch( + []string{"c1_node0_comp", "c1_node1_comp", "c2_node0_comp", "c2_node1_comp"}, + collected.componentNames, + ) + s.ElementsMatch( + []string{ + "CVE-critical-c1_node0_comp", "CVE-low-c1_node0_comp", + "CVE-critical-c1_node1_comp", "CVE-low-c1_node1_comp", + "CVE-critical-c2_node0_comp", "CVE-low-c2_node0_comp", + "CVE-critical-c2_node1_comp", "CVE-low-c2_node1_comp", + }, + collected.cveNames, + ) +} + +func (s *NodeReportGeneratorTestSuite) TestGetReportData_CancelledContext() { + snap := s.testNodeReportSnapshot(nil, "", nil, storage.ReportStatus_ON_DEMAND) + ctx, cancel := context.WithCancel(s.ctx) + cancel() + + _, _, err := s.reportGenerator.getReportData(ctx, snap) + s.Error(err) + s.ErrorIs(err, context.Canceled) +} + +func (s *NodeReportGeneratorTestSuite) testNodeReportSnapshot( + entityScope *storage.EntityScope, + query string, + scopeRules []*storage.SimpleAccessScope_Rules, + requestType storage.ReportStatus_RunMethod, +) *storage.ReportSnapshot { + snap := fixtures.GetReportSnapshot() + snap.Filter = &storage.ReportSnapshot_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: query, + AccessScopeRules: scopeRules, + }, + } + snap.ReportStatus = &storage.ReportStatus{ + ReportRequestType: requestType, + } + if entityScope != nil { + snap.ResourceScope = &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: entityScope, + }, + } + } else { + snap.ResourceScope = nil + } + snap.Collection = nil + return snap +} + +func collectNodeReportData(responses []*NodeCVEQueryResponse) *nodeReportData { + clusterNames := set.NewStringSet() + nodeNames := set.NewStringSet() + componentNames := set.NewStringSet() + cveNames := make([]string, 0, len(responses)) + + for _, res := range responses { + if res.GetCluster() != "" { + clusterNames.Add(res.GetCluster()) + } + if res.GetNode() != "" { + nodeNames.Add(res.GetNode()) + } + if res.GetComponent() != "" { + componentNames.Add(res.GetComponent()) + } + cveNames = append(cveNames, res.GetCVE()) + } + return &nodeReportData{ + clusterNames: clusterNames.AsSlice(), + nodeNames: nodeNames.AsSlice(), + componentNames: componentNames.AsSlice(), + cveNames: cveNames, + } +} diff --git a/central/reports/scheduler/v2/reportgenerator/node/singleton.go b/central/reports/scheduler/v2/reportgenerator/node/singleton.go new file mode 100644 index 0000000000000..20788359a00d3 --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/node/singleton.go @@ -0,0 +1,41 @@ +package node + +import ( + blobDS "github.com/stackrox/rox/central/blob/datastore" + clusterDS "github.com/stackrox/rox/central/cluster/datastore" + nodeCVEDS "github.com/stackrox/rox/central/cve/node/datastore" + "github.com/stackrox/rox/central/globaldb" + namespaceDS "github.com/stackrox/rox/central/namespace/datastore" + notifierProcessor "github.com/stackrox/rox/central/notifier/processor" + reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" + reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/sync" +) + +var ( + once sync.Once + rg reportGen.ReportGenerator +) + +func initialize() { + if !features.NodeVulnerabilityReports.Enabled() { + return + } + rg = New( + globaldb.GetPostgres(), + reportSnapshotDS.Singleton(), + notifierProcessor.Singleton(), + blobDS.Singleton(), + clusterDS.Singleton(), + namespaceDS.Singleton(), + nodeCVEDS.Singleton(), + ) +} + +// Singleton returns a singleton instance of the node ReportGenerator. +// Returns nil when the NodeVulnerabilityReports feature flag is disabled. +func Singleton() reportGen.ReportGenerator { + once.Do(initialize) + return rg +} diff --git a/central/reports/scheduler/v2/reportgenerator/node/types.go b/central/reports/scheduler/v2/reportgenerator/node/types.go new file mode 100644 index 0000000000000..7ae1c5418e752 --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/node/types.go @@ -0,0 +1,115 @@ +package node + +import ( + "time" + + "github.com/stackrox/rox/generated/storage" +) + +// NodeCVEQueryResponse contains the fields of a node CVE report query response. +type NodeCVEQueryResponse struct { + Cluster *string `db:"cluster"` + Node *string `db:"node"` + OperatingSystem *string `db:"operating_system"` + Component *string `db:"component"` + ComponentVersion *string `db:"component_version"` + CVEID *string `db:"cve_id"` + CVE *string `db:"cve"` + Fixable *bool `db:"fixable"` + FixedByVersion *string `db:"fixed_by"` + Severity *storage.VulnerabilitySeverity `db:"severity"` + CVSS *float64 `db:"cvss"` + FirstOccurrence *time.Time `db:"cve_created_time"` + + Link string +} + +func (r *NodeCVEQueryResponse) GetCluster() string { + if r.Cluster == nil { + return "" + } + return *r.Cluster +} + +func (r *NodeCVEQueryResponse) GetNode() string { + if r.Node == nil { + return "" + } + return *r.Node +} + +func (r *NodeCVEQueryResponse) GetOperatingSystem() string { + // Nodes scanned without OS metadata come back as an empty string rather than + // NULL, so both cases need the placeholder. + if r.OperatingSystem == nil || *r.OperatingSystem == "" { + return "Not Available" + } + return *r.OperatingSystem +} + +func (r *NodeCVEQueryResponse) GetComponent() string { + if r.Component == nil { + return "" + } + return *r.Component +} + +func (r *NodeCVEQueryResponse) GetComponentVersion() string { + if r.ComponentVersion == nil { + return "" + } + return *r.ComponentVersion +} + +func (r *NodeCVEQueryResponse) GetCVEID() string { + if r.CVEID == nil { + return "" + } + return *r.CVEID +} + +func (r *NodeCVEQueryResponse) GetCVE() string { + if r.CVE == nil { + return "" + } + return *r.CVE +} + +func (r *NodeCVEQueryResponse) GetFixable() bool { + if r.Fixable == nil { + return false + } + return *r.Fixable +} + +func (r *NodeCVEQueryResponse) GetFixedByVersion() string { + if r.FixedByVersion == nil { + return "" + } + return *r.FixedByVersion +} + +func (r *NodeCVEQueryResponse) GetSeverity() storage.VulnerabilitySeverity { + if r.Severity == nil { + return storage.VulnerabilitySeverity_UNKNOWN_VULNERABILITY_SEVERITY + } + return *r.Severity +} + +func (r *NodeCVEQueryResponse) GetCVSS() float64 { + if r.CVSS == nil { + return 0.0 + } + return *r.CVSS +} + +func (r *NodeCVEQueryResponse) GetFirstOccurrence() string { + if r.FirstOccurrence == nil { + return "Not Available" + } + return r.FirstOccurrence.Format("January 02, 2006") +} + +func (r *NodeCVEQueryResponse) GetLink() string { + return r.Link +} diff --git a/central/reports/scheduler/v2/reportgenerator/report_gen.go b/central/reports/scheduler/v2/reportgenerator/report_gen.go index 7e40985387259..92879bcc33aa3 100644 --- a/central/reports/scheduler/v2/reportgenerator/report_gen.go +++ b/central/reports/scheduler/v2/reportgenerator/report_gen.go @@ -1,7 +1,8 @@ package reportgenerator import ( - "github.com/graph-gophers/graphql-go" + "context" + blobDS "github.com/stackrox/rox/central/blob/datastore" clusterDS "github.com/stackrox/rox/central/cluster/datastore" imageCVE2DS "github.com/stackrox/rox/central/cve/image/v2/datastore" @@ -21,7 +22,8 @@ type ReportGenerator interface { // ProcessReportRequest will generate a report and send notification via the requested notification method. // On success, report will be generated and notified, and report snapshot will be stored to the db. // On failure, it will log any errors and store it in the report snapshot. - ProcessReportRequest(req *ReportRequest) + // The context is used for cancellation; cancelling it will abort in-flight database queries. + ProcessReportRequest(ctx context.Context, req *ReportRequest) } // New will create a new instance of the ReportGenerator @@ -36,7 +38,6 @@ func New( clusterDatastore clusterDS.DataStore, namespaceDatastore namespaceDS.DataStore, imageCVE2DataStore imageCVE2DS.DataStore, - schema *graphql.Schema, ) ReportGenerator { return newReportGeneratorImpl( db, @@ -49,7 +50,6 @@ func New( clusterDatastore, namespaceDatastore, imageCVE2DataStore, - schema, ) } @@ -64,7 +64,6 @@ func newReportGeneratorImpl( clusterDatastore clusterDS.DataStore, namespaceDatastore namespaceDS.DataStore, imageCVE2Datastore imageCVE2DS.DataStore, - schema *graphql.Schema, ) *reportGeneratorImpl { return &reportGeneratorImpl{ reportSnapshotStore: reportSnapshotStore, @@ -77,6 +76,5 @@ func newReportGeneratorImpl( imageCVE2Datastore: imageCVE2Datastore, blobStore: blobStore, db: db, - Schema: schema, } } diff --git a/central/reports/scheduler/v2/reportgenerator/report_gen_bench_flat_data_model_test.go b/central/reports/scheduler/v2/reportgenerator/report_gen_bench_flat_data_model_test.go index 892899de56d10..f0a1f356215d0 100644 --- a/central/reports/scheduler/v2/reportgenerator/report_gen_bench_flat_data_model_test.go +++ b/central/reports/scheduler/v2/reportgenerator/report_gen_bench_flat_data_model_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" - "github.com/graph-gophers/graphql-go" clusterDSMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" "github.com/stackrox/rox/central/graphql/resolvers" "github.com/stackrox/rox/central/graphql/resolvers/loaders" + namespaceDS "github.com/stackrox/rox/central/namespace/datastore" namespaceDSMocks "github.com/stackrox/rox/central/namespace/datastore/mocks" collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" collectionPostgres "github.com/stackrox/rox/central/resourcecollection/datastore/store/postgres" @@ -18,6 +18,8 @@ import ( imagesView "github.com/stackrox/rox/central/views/images" watchedImageDS "github.com/stackrox/rox/central/watchedimage/datastore" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/uuid" @@ -78,7 +80,7 @@ func BenchmarkFlatDataModelReportGenerator(b *testing.B) { b.Run("GetReportDataSQF", func(b *testing.B) { for i := 0; i < b.N; i++ { - reportData, err := bts.reportGenerator.getReportDataSQF(reportSnap, collection, time.Time{}) + reportData, err := bts.reportGenerator.getReportDataSQF(bts.ctx, reportSnap, collection, time.Time{}) require.NoError(b, err) require.Equal(b, expectedRowCount, len(reportData.CVEResponses)) } @@ -92,17 +94,29 @@ func (bts *FlatDataModelReportGeneratorBenchmarkTestSuite) setupTestSuite() { bts.testDB = pgtest.ForT(bts.b) // set up tables - imageDataStore := resolvers.CreateTestImageV2Datastore(bts.b, bts.testDB, mockCtrl) bts.watchedImageDatastore = watchedImageDS.GetTestPostgresDataStore(bts.b, bts.testDB.DB) - var schema *graphql.Schema - bts.resolver, schema = resolvers.SetupTestResolver(bts.b, - imagesView.NewImageView(bts.testDB.DB), - imageDataStore, - resolvers.CreateTestImageComponentV2Datastore(bts.b, bts.testDB, mockCtrl), - resolvers.CreateTestImageCVEV2Datastore(bts.b, bts.testDB), - resolvers.CreateTestDeploymentDatastore(bts.b, bts.testDB, mockCtrl, imageDataStore), - deploymentsView.NewDeploymentView(bts.testDB.DB), - ) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + imgV2DataStore := resolvers.CreateTestImageV2Datastore(bts.b, bts.testDB, mockCtrl) + bts.resolver, _ = resolvers.SetupTestResolver(bts.b, + imagesView.NewImageView(bts.testDB.DB), + imgV2DataStore, + resolvers.CreateTestImageComponentV2Datastore(bts.b, bts.testDB, mockCtrl), + resolvers.CreateTestImageCVEV2Datastore(bts.b, bts.testDB), + resolvers.CreateTestDeploymentDatastoreWithImageV2(bts.b, bts.testDB, mockCtrl, imgV2DataStore), + deploymentsView.NewDeploymentView(bts.testDB.DB), + ) + } else { + imageDataStore := resolvers.CreateTestImageDatastore(bts.b, bts.testDB, mockCtrl) + bts.resolver, _ = resolvers.SetupTestResolver(bts.b, + imagesView.NewImageView(bts.testDB.DB), + imageDataStore, + resolvers.CreateTestImageComponentV2Datastore(bts.b, bts.testDB, mockCtrl), + resolvers.CreateTestImageCVEV2Datastore(bts.b, bts.testDB), + resolvers.CreateTestDeploymentDatastore(bts.b, bts.testDB, mockCtrl, imageDataStore), + deploymentsView.NewDeploymentView(bts.testDB.DB), + ) + } collectionStore := collectionPostgres.New(bts.testDB) _, collectionQueryResolver, err := collectionDS.New(collectionStore) require.NoError(bts.b, err) @@ -111,13 +125,215 @@ func (bts *FlatDataModelReportGeneratorBenchmarkTestSuite) setupTestSuite() { bts.reportGenerator = newReportGeneratorImpl(bts.testDB, nil, bts.resolver.DeploymentDataStore, bts.watchedImageDatastore, collectionQueryResolver, nil, nil, bts.clusterDatastore, - bts.namespaceDatastore, bts.resolver.ImageCVEV2DataStore, schema) + bts.namespaceDatastore, bts.resolver.ImageCVEV2DataStore) +} + +func BenchmarkEntityScopeReportGenerator(b *testing.B) { + ctx := loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) + mockCtrl := gomock.NewController(b) + testDB := pgtest.ForT(b) + + watchedImageDatastore := watchedImageDS.GetTestPostgresDataStore(b, testDB.DB) + var resolver *resolvers.Resolver + if features.FlattenImageData.Enabled() { + imgV2DataStore := resolvers.CreateTestImageV2Datastore(b, testDB, mockCtrl) + resolver, _ = resolvers.SetupTestResolver(b, + imagesView.NewImageView(testDB.DB), + imgV2DataStore, + resolvers.CreateTestImageComponentV2Datastore(b, testDB, mockCtrl), + resolvers.CreateTestImageCVEV2Datastore(b, testDB), + resolvers.CreateTestDeploymentDatastoreWithImageV2(b, testDB, mockCtrl, imgV2DataStore), + deploymentsView.NewDeploymentView(testDB.DB), + ) + } else { + imageDataStore := resolvers.CreateTestImageDatastore(b, testDB, mockCtrl) + resolver, _ = resolvers.SetupTestResolver(b, + imagesView.NewImageView(testDB.DB), + imageDataStore, + resolvers.CreateTestImageComponentV2Datastore(b, testDB, mockCtrl), + resolvers.CreateTestImageCVEV2Datastore(b, testDB), + resolvers.CreateTestDeploymentDatastore(b, testDB, mockCtrl, imageDataStore), + deploymentsView.NewDeploymentView(testDB.DB), + ) + } + + clusterDatastore := clusterDSMocks.NewMockDataStore(mockCtrl) + nsDatastore, err := namespaceDS.GetTestPostgresDataStore(b, testDB.DB) + require.NoError(b, err) + + reportGenerator := newReportGeneratorImpl(testDB, nil, resolver.DeploymentDataStore, + watchedImageDatastore, nil, nil, nil, clusterDatastore, + nsDatastore, resolver.ImageCVEV2DataStore) + + clusters := []*storage.Cluster{ + {Id: uuid.NewV4().String(), Name: "c1"}, + {Id: uuid.NewV4().String(), Name: "c2"}, + } + + namespaces := testNamespaces(clusters, 10) + + // Add labels to first 5 namespaces of each cluster (ns1-ns5 get team=backend) + for i, ns := range namespaces { + if i%10 < 5 { + ns.Labels = map[string]string{"team": "backend"} + } else { + ns.Labels = map[string]string{"team": "frontend"} + } + err := nsDatastore.AddNamespace(ctx, ns) + require.NoError(b, err) + } + + deployments, images := testDeploymentsWithImages(namespaces, 100) + if features.FlattenImageData.Enabled() { + for _, img := range images { + require.NoError(b, resolver.ImageV2DataStore.UpsertImage(ctx, imageUtils.ConvertToV2(img))) + } + } else { + for _, img := range images { + require.NoError(b, resolver.ImageDataStore.UpsertImage(ctx, img)) + } + } + for _, dep := range deployments { + require.NoError(b, resolver.DeploymentDataStore.UpsertDeployment(ctx, dep)) + } + + watchedImages := testWatchedImages(500) + if features.FlattenImageData.Enabled() { + for _, img := range watchedImages { + require.NoError(b, resolver.ImageV2DataStore.UpsertImage(ctx, imageUtils.ConvertToV2(img))) + } + } else { + for _, img := range watchedImages { + require.NoError(b, resolver.ImageDataStore.UpsertImage(ctx, img)) + } + } + for _, img := range watchedImages { + require.NoError(b, watchedImageDatastore.UpsertWatchedImage(ctx, img.GetName().GetFullName())) + } + + clusterDatastore.EXPECT().GetClusters(gomock.Any()). + Return(clusters, nil).AnyTimes() + + allImageTypes := []storage.VulnerabilityReportFilters_ImageType{ + storage.VulnerabilityReportFilters_DEPLOYED, + storage.VulnerabilityReportFilters_WATCHED, + } + sinceStartDate := time.Now().AddDate(-1, 0, 0) + + // c1: 10 namespaces × 100 deployments = 1000 deployed images (2 CVEs each) + // c2: 10 namespaces × 100 deployments = 1000 deployed images (2 CVEs each) + // ns1-ns5 per cluster have label team=backend (5 ns × 100 deps = 500 per cluster) + // 500 watched images (2 CVEs each): registry=quay.io, label=app:watch + // Each image has: 1 fixable_critical CVE (CVSS=9.0, EPSS=0.7) + 1 nonFixable_low CVE (CVSS=2.0, EPSS=0.1) + + b.Run("EntityScope_ClusterScope_WithNamespaceLabel", func(b *testing.B) { + entityScope := &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1", MatchType: storage.MatchType_EXACT}, + }, + }, + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_LABEL, + Values: []*storage.RuleValue{ + {Value: "team=backend", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + } + scopeRules := []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1"}}, + } + // CVSS>=7.0 matches fixable_critical only + // Entity scope: cluster=c1 AND namespace label team=backend + // Deployed: c1 ns1-ns5 = 500 images × 1 CVE = 500; Watched: 500 × 1 CVE = 500 + expectedRowCount := 1000 + reportSnap := testEntityScopeReportSnapshot(entityScope, "CVSS:>=7.0", allImageTypes, scopeRules) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + reportData, err := reportGenerator.getReportDataSQF(ctx, reportSnap, nil, sinceStartDate) + require.NoError(b, err) + require.Equal(b, expectedRowCount, len(reportData.CVEResponses)) + } + }) + + b.Run("EntityScope_NamespaceScope_WithEPSSFilter", func(b *testing.B) { + entityScope := &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "ns1", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + } + scopeRules := []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1", "c2"}}, + } + // EPSS>=0.5 matches fixable_critical only (EPSS=0.7) + // Deployed: ns1 across 2 clusters = 200 images × 1 CVE = 200; Watched: 500 × 1 CVE = 500 + expectedRowCount := 700 + reportSnap := testEntityScopeReportSnapshot(entityScope, "EPSS Probability:>=0.5", allImageTypes, scopeRules) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + reportData, err := reportGenerator.getReportDataSQF(ctx, reportSnap, nil, sinceStartDate) + require.NoError(b, err) + require.Equal(b, expectedRowCount, len(reportData.CVEResponses)) + } + }) + + b.Run("EntityScope_DeploymentRegex_WithCVSSFilter", func(b *testing.B) { + entityScope := &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1_.*", MatchType: storage.MatchType_REGEX}, + }, + }, + }, + } + scopeRules := []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1"}}, + } + // CVSS>=7.0 matches fixable_critical only + // Deployed: 1000 c1 deployments matching c1_.* × 1 CVE = 1000; Watched: 500 × 1 CVE = 500 + expectedRowCount := 1500 + reportSnap := testEntityScopeReportSnapshot(entityScope, "CVSS:>=7.0", allImageTypes, scopeRules) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + reportData, err := reportGenerator.getReportDataSQF(ctx, reportSnap, nil, sinceStartDate) + require.NoError(b, err) + require.Equal(b, expectedRowCount, len(reportData.CVEResponses)) + } + }) } func (bts *FlatDataModelReportGeneratorBenchmarkTestSuite) upsertManyImages(images []*storage.Image) { - for _, img := range images { - err := bts.resolver.ImageDataStore.UpsertImage(bts.ctx, img) - require.NoError(bts.b, err) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + for _, img := range images { + err := bts.resolver.ImageV2DataStore.UpsertImage(bts.ctx, imageUtils.ConvertToV2(img)) + require.NoError(bts.b, err) + } + } else { + for _, img := range images { + err := bts.resolver.ImageDataStore.UpsertImage(bts.ctx, img) + require.NoError(bts.b, err) + } } } diff --git a/central/reports/scheduler/v2/reportgenerator/report_gen_impl.go b/central/reports/scheduler/v2/reportgenerator/report_gen_impl.go index 487027f7a4930..ce4e4f4b144b2 100644 --- a/central/reports/scheduler/v2/reportgenerator/report_gen_impl.go +++ b/central/reports/scheduler/v2/reportgenerator/report_gen_impl.go @@ -1,11 +1,14 @@ package reportgenerator import ( + "archive/zip" "bytes" "context" + "encoding/csv" + "io" + "slices" "time" - "github.com/graph-gophers/graphql-go" "github.com/pkg/errors" blobDS "github.com/stackrox/rox/central/blob/datastore" clusterDS "github.com/stackrox/rox/central/cluster/datastore" @@ -23,6 +26,7 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/errorhelpers" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/grpc/authz/allow" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/notifier" @@ -31,13 +35,11 @@ import ( pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" "github.com/stackrox/rox/pkg/postgres/walker" "github.com/stackrox/rox/pkg/protocompat" - "github.com/stackrox/rox/pkg/retry" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/effectiveaccessscope" "github.com/stackrox/rox/pkg/search" pgSearch "github.com/stackrox/rox/pkg/search/postgres" "github.com/stackrox/rox/pkg/set" - "github.com/stackrox/rox/pkg/utils" ) var ( @@ -63,6 +65,7 @@ var ( Offset(int32(0)). AddSortOption(search.NewSortOption(search.ImageName)).Proto(), } + cursorBatchSize = env.PostgresDefaultCursorBatchSize.IntegerSetting() ) type reportGeneratorImpl struct { @@ -76,8 +79,6 @@ type reportGeneratorImpl struct { namespaceDatastore namespaceDS.DataStore imageCVE2Datastore imageCVE2DS.DataStore db postgres.DB - - Schema *graphql.Schema } type ImageCVEInterface interface { @@ -85,11 +86,13 @@ type ImageCVEInterface interface { GetCveBaseInfo() *storage.CVEInfo } -func (rg *reportGeneratorImpl) ProcessReportRequest(req *ReportRequest) { +func (rg *reportGeneratorImpl) ProcessReportRequest(ctx context.Context, req *ReportRequest) { + ctx = resolvers.SetAuthorizerOverride(loaders.WithLoaderContext(sac.WithAllAccess(ctx)), allow.Anonymous()) + // First do some basic validation checks on the request. err := ValidateReportRequest(req) if err != nil { - rg.logAndUpsertError(errors.Wrap(err, "Invalid report request"), req) + rg.logAndUpsertError(ctx, errors.Wrap(err, "Invalid report request"), req) return } @@ -97,14 +100,14 @@ func (rg *reportGeneratorImpl) ProcessReportRequest(req *ReportRequest) { if req.ReportSnapshot.GetVulnReportFilters().GetSinceLastSentScheduledReport() { req.DataStartTime, err = rg.lastSuccessfulScheduledReportTime(req.ReportSnapshot) if err != nil { - rg.logAndUpsertError(errors.Wrap(err, "Error finding last successful scheduled report time"), req) + rg.logAndUpsertError(ctx, errors.Wrap(err, "Error finding last successful scheduled report time"), req) return } } else if req.ReportSnapshot.GetVulnReportFilters().GetSinceStartDate() != nil { sinceStartDate := req.ReportSnapshot.GetVulnReportFilters().GetSinceStartDate() req.DataStartTime, err = protocompat.ConvertTimestampToTimeOrError(sinceStartDate) if err != nil { - rg.logAndUpsertError(errors.Wrap(err, "Error finding last successful scheduled report time"), req) + rg.logAndUpsertError(ctx, errors.Wrap(err, "Error finding last successful scheduled report time"), req) return } } @@ -113,41 +116,53 @@ func (rg *reportGeneratorImpl) ProcessReportRequest(req *ReportRequest) { // Change report status to PREPARING err = rg.updateReportStatus(req.ReportSnapshot, storage.ReportStatus_PREPARING) if err != nil { - rg.logAndUpsertError(errors.Wrap(err, "Error changing report status to PREPARING"), req) + rg.logAndUpsertError(ctx, errors.Wrap(err, "Error changing report status to PREPARING"), req) return } - err = rg.generateReportAndNotify(req) + err = rg.generateReportAndNotify(ctx, req) if err != nil { - rg.logAndUpsertError(err, req) + rg.logAndUpsertError(ctx, err, req) return } if req.ReportSnapshot.GetReportStatus().GetReportNotificationMethod() == storage.ReportStatus_EMAIL { err = rg.updateReportStatus(req.ReportSnapshot, storage.ReportStatus_DELIVERED) if err != nil { - rg.logAndUpsertError(errors.Wrap(err, "Error changing report status to DELIVERED"), req) + rg.logAndUpsertError(ctx, errors.Wrap(err, "Error changing report status to DELIVERED"), req) } } } /* Report generation helper functions */ -func (rg *reportGeneratorImpl) generateReportAndNotify(req *ReportRequest) error { +func (rg *reportGeneratorImpl) generateReportAndNotify(ctx context.Context, req *ReportRequest) error { + if err := ctx.Err(); err != nil { + return err + } + // Get the results of running the report query var err error var reportData *ReportData - if req.ReportSnapshot.GetVulnReportFilters() != nil { - reportData, err = rg.getReportDataSQF(req.ReportSnapshot, req.Collection, req.DataStartTime) - } - if req.ReportSnapshot.GetViewBasedVulnReportFilters() != nil { - reportData, err = rg.getReportDataViewBased(req.ReportSnapshot) + if req.ReportSnapshot.GetReportStatus().GetReportNotificationMethod() == storage.ReportStatus_DOWNLOAD { + if features.VulnerabilityReportStreamingDownload.Enabled() { + log.Info("Streaming report generation") + return rg.generateReportTransaction(ctx, req) + } + return rg.generateReportInMemoryDownload(ctx, req) } + + // EMAIL path: use existing in-memory approach (email attachments have practical size limits) + reportData, err = rg.getReportDataSQF(ctx, req.ReportSnapshot, req.Collection, req.DataStartTime) if err != nil { return err } + if err := ctx.Err(); err != nil { + return err + } + // Format results into CSV - zippedCSVData, err := GenerateCSV(reportData.CVEResponses, req.ReportSnapshot.GetName(), req.ReportSnapshot) + zippedCSVData, err := GenerateCSV(reportData.CVEResponses, req.ReportSnapshot.GetName()) if err != nil { return err } @@ -157,94 +172,321 @@ func (rg *reportGeneratorImpl) generateReportAndNotify(req *ReportRequest) error if err != nil { return errors.Wrap(err, "Error changing report status to GENERATED") } - switch req.ReportSnapshot.GetReportStatus().GetReportNotificationMethod() { - case storage.ReportStatus_DOWNLOAD: - parentDir := req.ReportSnapshot.GetReportConfigurationId() - if req.ReportSnapshot.GetVulnReportFilters() == nil { - parentDir = "view-based-report" + if err := ctx.Err(); err != nil { + return err + } + + defaultEmailSubject, err := FormatEmailSubject(defaultEmailSubjectTemplate, req.ReportSnapshot) + if err != nil { + return errors.Wrap(err, "Error generating email subject") + } + templateStr := defaultEmailBodyTemplate + if reportData.NumDeployedImageResults == 0 && reportData.NumWatchedImageResults == 0 { + zippedCSVData = nil + templateStr = defaultNoVulnsEmailBodyTemplate + } + + defaultEmailBody, err := FormatEmailBody(templateStr) + if err != nil { + return errors.Wrap(err, "Error generating email body") + } + + configDetailsHTML, err := formatReportConfigDetails(req.ReportSnapshot, reportData.NumDeployedImageResults, + reportData.NumWatchedImageResults) + if err != nil { + return errors.Wrap(err, "Error adding report config details") + } + + errorList := errorhelpers.NewErrorList("Error sending email notifications: ") + for _, notifierSnap := range req.ReportSnapshot.GetNotifiers() { + nf := rg.notificationProcessor.GetNotifier(reportGenCtx, notifierSnap.GetEmailConfig().GetNotifierId()) + reportNotifier, ok := nf.(notifiers.ReportNotifier) + if !ok { + errorList.AddError(errors.Errorf("incorrect type of notifier '%s'", notifierSnap.GetEmailConfig().GetNotifierId())) + continue } - if err = rg.saveReportData(parentDir, - req.ReportSnapshot.GetReportId(), zippedCSVData); err != nil { - return errors.Wrap(err, "error persisting blob") + customBody := notifierSnap.GetEmailConfig().GetCustomBody() + emailBody := defaultEmailBody + if customBody != "" { + emailBody = customBody } + customSubject := notifierSnap.GetEmailConfig().GetCustomSubject() + emailSubject := defaultEmailSubject + if customSubject != "" { + emailSubject = customSubject + } + emailBodyWithConfigDetails := AddReportConfigDetails(emailBody, configDetailsHTML) + reportName := req.ReportSnapshot.GetName() + err := rg.retryableSendReportResults(reportNotifier, notifierSnap.GetEmailConfig().GetMailingLists(), + zippedCSVData, emailSubject, emailBodyWithConfigDetails, reportName) + if err != nil { + errorList.AddError(errors.Errorf("Error sending email for notifier '%s': %s", + notifierSnap.GetEmailConfig().GetNotifierId(), err)) + } + } + if !errorList.Empty() { + return errorList.ToError() + } + return nil +} + +// generateReportInMemoryDownload accumulates report data in memory, builds the CSV/ZIP, and stores it in blob storage. +func (rg *reportGeneratorImpl) generateReportInMemoryDownload(ctx context.Context, req *ReportRequest) error { + snap := req.ReportSnapshot + + var reportData *ReportData + var err error + if snap.GetViewBasedVulnReportFilters() != nil { + reportData, err = rg.getReportDataViewBased(ctx, snap) + } else { + reportData, err = rg.getReportDataSQF(ctx, snap, req.Collection, req.DataStartTime) + } + if err != nil { + return err + } + + zippedCSVData, err := GenerateCSV(reportData.CVEResponses, snap.GetName()) + if err != nil { + return err + } - case storage.ReportStatus_EMAIL: - defaultEmailSubject, err := formatEmailSubject(defaultEmailSubjectTemplate, req.ReportSnapshot) + parentDir := snap.GetReportConfigurationId() + if snap.GetVulnReportFilters() == nil { + parentDir = "view-based-report" + } + if err := rg.saveReportData(ctx, parentDir, snap.GetReportId(), zippedCSVData); err != nil { + return errors.Wrap(err, "error saving report to blob store") + } + + snap.ReportStatus.CompletedAt = protocompat.TimestampNow() + if err := rg.updateReportStatus(snap, storage.ReportStatus_GENERATED); err != nil { + return errors.Wrap(err, "Error changing report status to GENERATED") + } + return nil +} + +type querySpec struct { + schema *walker.Schema + query *v1.Query +} + +// buildReportQueries constructs the cursor queries for a report request. +func (rg *reportGeneratorImpl) buildReportQueries(ctx context.Context, req *ReportRequest) ([]querySpec, error) { + var queries []querySpec + snap := req.ReportSnapshot + + if snap.GetVulnReportFilters() != nil { + rQuery, err := rg.buildReportQuery(ctx, snap, req.Collection, req.DataStartTime) if err != nil { - return errors.Wrap(err, "Error generating email subject") + return nil, err } - // If it is an empty report, do not send an attachment in the final notification email and the email body - // will indicate that no vulns were found - templateStr := defaultEmailBodyTemplate - if reportData.NumDeployedImageResults == 0 && reportData.NumWatchedImageResults == 0 { - // If it is an empty report, the email body will indicate that no vulns were found - zippedCSVData = nil - templateStr = defaultNoVulnsEmailBodyTemplate + cveFilterQuery, err := search.ParseQuery(rQuery.CveFieldsQuery, search.MatchAllIfEmpty()) + if err != nil { + return nil, err } - - defaultEmailBody, err := formatEmailBody(templateStr) + if slices.Contains(snap.GetVulnReportFilters().GetImageTypes(), storage.VulnerabilityReportFilters_DEPLOYED) { + q := search.ConjunctionQuery(rQuery.DeploymentsQuery, cveFilterQuery) + q.Pagination = deployedImagesQueryParts.Pagination + q.Selects = deployedImagesQueryParts.Selects + queries = append(queries, querySpec{schema: deployedImagesQueryParts.Schema, query: q}) + } + if slices.Contains(snap.GetVulnReportFilters().GetImageTypes(), storage.VulnerabilityReportFilters_WATCHED) { + watchedImages, err := rg.getWatchedImages(ctx) + if err != nil { + return nil, err + } + if len(watchedImages) != 0 { + q := search.ConjunctionQuery( + search.NewQueryBuilder().AddExactMatches(search.ImageName, watchedImages...).ProtoQuery(), + cveFilterQuery) + q.Pagination = watchedImagesQueryParts.Pagination + q.Selects = watchedImagesQueryParts.Selects + queries = append(queries, querySpec{schema: watchedImagesQueryParts.Schema, query: q}) + } + } + } + if snap.GetViewBasedVulnReportFilters() != nil { + watchedImages, err := rg.getWatchedImages(ctx) if err != nil { - return errors.Wrap(err, "Error generating email body") + return nil, err } - - configDetailsHTML, err := formatReportConfigDetails(req.ReportSnapshot, reportData.NumDeployedImageResults, - reportData.NumWatchedImageResults) + vbQuery, err := rg.buildReportQueryViewBased(ctx, snap, watchedImages) if err != nil { - return errors.Wrap(err, "Error adding report config details") + return nil, err } + vbQuery.DeployedImagesQuery.Pagination = deployedImagesQueryParts.Pagination + vbQuery.DeployedImagesQuery.Selects = deployedImagesQueryParts.Selects + queries = append(queries, querySpec{schema: deployedImagesQueryParts.Schema, query: vbQuery.DeployedImagesQuery}) - errorList := errorhelpers.NewErrorList("Error sending email notifications: ") - for _, notifierSnap := range req.ReportSnapshot.GetNotifiers() { - nf := rg.notificationProcessor.GetNotifier(reportGenCtx, notifierSnap.GetEmailConfig().GetNotifierId()) - reportNotifier, ok := nf.(notifiers.ReportNotifier) - if !ok { - errorList.AddError(errors.Errorf("incorrect type of notifier '%s'", notifierSnap.GetEmailConfig().GetNotifierId())) - continue + if len(watchedImages) != 0 { + vbQuery.WatchedImagesQuery.Pagination = watchedImagesQueryParts.Pagination + vbQuery.WatchedImagesQuery.Selects = watchedImagesQueryParts.Selects + queries = append(queries, querySpec{schema: watchedImagesQueryParts.Schema, query: vbQuery.WatchedImagesQuery}) + } + } + return queries, nil +} + +// generateReportTransaction runs all report operations — cursor reads, CVE +// lookups, CSV/ZIP generation, and blob store write — within a single database +// transaction. The writeFn callback writes directly to a PostgreSQL large +// object, so memory usage stays constant regardless of report size. +// +// Connection usage: 1 connection. All operations (cursor FETCH, CVE lookups, +// large object writes, metadata upsert) share the same transaction on a single +// goroutine, so there are no concurrent-tx-access issues. +func (rg *reportGeneratorImpl) generateReportTransaction(ctx context.Context, req *ReportRequest) error { + snap := req.ReportSnapshot + queries, err := rg.buildReportQueries(ctx, req) + if err != nil { + return err + } + + tx, err := rg.db.Begin(ctx) + if err != nil { + return errors.Wrap(err, "starting report transaction") + } + committed := false + defer func() { + if !committed { + if rbErr := tx.Rollback(context.Background()); rbErr != nil { + log.Errorf("failed to rollback report transaction: %v", rbErr) } - customBody := notifierSnap.GetEmailConfig().GetCustomBody() - emailBody := defaultEmailBody - if customBody != "" { - emailBody = customBody + } + }() + txCtx := postgres.ContextWithTx(ctx, tx) + + parentDir := snap.GetReportConfigurationId() + if snap.GetVulnReportFilters() == nil { + parentDir = "view-based-report" + } + blobPath := common.GetReportBlobPath(parentDir, snap.GetReportId()) + blob := &storage.Blob{ + Name: blobPath, + LastUpdated: protocompat.TimestampNow(), + ModifiedTime: protocompat.TimestampNow(), + Length: -1, + } + + // cveRefLinksCache is a map[string]string that maps CVE ID -> reference URL. + // It caches the "Reference" link for each CVE (the external URL pointing to the CVE advisory, e.g., on NVD). + cveRefLinksCache := make(map[string]string) + rowCount := 0 + + err = rg.blobStore.UpsertWithWriter(txCtx, tx, blob, func(w io.Writer) error { + zipWriter := zip.NewWriter(w) + zipEntry, err := zipWriter.Create(csvReportName(snap.GetName())) + if err != nil { + return errors.Wrap(err, "creating zip entry") + } + csvW := csv.NewWriter(zipEntry) + csvW.UseCRLF = true + if err := csvW.Write(formatCol()); err != nil { + return errors.Wrap(err, "writing CSV header") + } + + for _, qs := range queries { + if err := rg.streamQueryToCSV(txCtx, qs.schema, qs.query, csvW, cveRefLinksCache, &rowCount); err != nil { + return err } - customSubject := notifierSnap.GetEmailConfig().GetCustomSubject() - emailSubject := defaultEmailSubject - if customSubject != "" { - emailSubject = customSubject + } + + csvW.Flush() + if err := csvW.Error(); err != nil { + return errors.Wrap(err, "flushing CSV writer") + } + return zipWriter.Close() + }) + if err != nil { + return errors.Wrap(err, "error streaming report to blob store") + } + + if err := tx.Commit(context.Background()); err != nil { + return errors.Wrap(err, "committing report transaction") + } + committed = true + + snap.ReportStatus.CompletedAt = protocompat.TimestampNow() + if err := rg.updateReportStatus(snap, storage.ReportStatus_GENERATED); err != nil { + return errors.Wrap(err, "Error changing report status to GENERATED") + } + return nil +} + +// streamQueryToCSV runs a cursor-based query and streams each row directly through CSV formatting +// to the provided csv.Writer. CVE reference links are resolved incrementally in batches and cached. +func (rg *reportGeneratorImpl) streamQueryToCSV( + ctx context.Context, + schema *walker.Schema, + query *v1.Query, + csvW *csv.Writer, + cveRefLinksCache map[string]string, + rowCount *int, +) error { + var batch []*ImageCVEQueryResponse + + flushBatch := func() error { + unseenIDs := set.NewStringSet() + for _, r := range batch { + id := r.GetCVEID() + if id != "" { + if _, cached := cveRefLinksCache[id]; !cached { + unseenIDs.Add(id) + } } - emailBodyWithConfigDetails := addReportConfigDetails(emailBody, configDetailsHTML) - reportName := req.ReportSnapshot.GetName() - err := rg.retryableSendReportResults(reportNotifier, notifierSnap.GetEmailConfig().GetMailingLists(), - zippedCSVData, emailSubject, emailBodyWithConfigDetails, reportName) + } + if unseenIDs.Cardinality() > 0 { + cves, err := rg.imageCVE2Datastore.GetBatch(ctx, unseenIDs.AsSlice()) if err != nil { - errorList.AddError(errors.Errorf("Error sending email for notifier '%s': %s", - notifierSnap.GetEmailConfig().GetNotifierId(), err)) + return errors.Wrap(err, "fetching CVE reference links") + } + for _, cve := range cves { + cveRefLinksCache[cve.GetId()] = cve.GetCveBaseInfo().GetLink() } } - if !errorList.Empty() { - return errorList.ToError() + for _, r := range batch { + if link, ok := cveRefLinksCache[r.GetCVEID()]; ok { + r.Link = link + } + if err := csvW.Write(formatCSVRow(r)); err != nil { + return err + } } + batch = batch[:0] + return nil } - return nil -} -func (rg *reportGeneratorImpl) saveReportData(configID, reportID string, data *bytes.Buffer) error { - if data == nil { - return errors.Errorf("No data found for report config %q and id %q", configID, reportID) + err := pgSearch.RunSelectCursorForSchemaFn[ImageCVEQueryResponse]( + ctx, rg.db, schema, query, + func(r *ImageCVEQueryResponse) error { + batch = append(batch, r) + *rowCount++ + if len(batch) >= cursorBatchSize { + return flushBatch() + } + return nil + }) + if err != nil { + return err } - // Store downloadable report in blob storage - b := &storage.Blob{ - Name: common.GetReportBlobPath(configID, reportID), - LastUpdated: protocompat.TimestampNow(), - ModifiedTime: protocompat.TimestampNow(), - Length: int64(data.Len()), + if err := ctx.Err(); err != nil { + return err + } + + if len(batch) > 0 { + return flushBatch() } - return rg.blobStore.Upsert(reportGenCtx, b, data) + return nil +} + +func (rg *reportGeneratorImpl) saveReportData(ctx context.Context, configID, reportID string, data *bytes.Buffer) error { + return SaveReportData(ctx, rg.blobStore, configID, reportID, data) } -func (rg *reportGeneratorImpl) getReportDataSQF(snap *storage.ReportSnapshot, collection *storage.ResourceCollection, +func (rg *reportGeneratorImpl) getReportDataSQF(ctx context.Context, snap *storage.ReportSnapshot, collection *storage.ResourceCollection, dataStartTime time.Time) (*ReportData, error) { - rQuery, err := rg.buildReportQuery(snap, collection, dataStartTime) + rQuery, err := rg.buildReportQuery(ctx, snap, collection, dataStartTime) if err != nil { return nil, err } @@ -256,21 +498,28 @@ func (rg *reportGeneratorImpl) getReportDataSQF(snap *storage.ReportSnapshot, co numDeployedImageResults := 0 var cveResponses []*ImageCVEQueryResponse - if filterOnImageType(snap.GetVulnReportFilters().GetImageTypes(), storage.VulnerabilityReportFilters_DEPLOYED) { + if slices.Contains(snap.GetVulnReportFilters().GetImageTypes(), storage.VulnerabilityReportFilters_DEPLOYED) { query := search.ConjunctionQuery(rQuery.DeploymentsQuery, cveFilterQuery) query.Pagination = deployedImagesQueryParts.Pagination query.Selects = deployedImagesQueryParts.Selects - cveResponses, err = pgSearch.RunSelectRequestForSchema[ImageCVEQueryResponse](reportGenCtx, rg.db, - deployedImagesQueryParts.Schema, query) + err = pgSearch.RunSelectRequestForSchemaFn[ImageCVEQueryResponse](ctx, rg.db, + deployedImagesQueryParts.Schema, query, func(r *ImageCVEQueryResponse) error { + cveResponses = append(cveResponses, r) + numDeployedImageResults++ + return nil + }) if err != nil { return nil, errors.Wrap(err, "Failed to collect report data for deployed images") } - numDeployedImageResults = len(cveResponses) + } + + if err := ctx.Err(); err != nil { + return nil, err } numWatchedImageResults := 0 - if filterOnImageType(snap.GetVulnReportFilters().GetImageTypes(), storage.VulnerabilityReportFilters_WATCHED) { - watchedImages, err := rg.getWatchedImages() + if slices.Contains(snap.GetVulnReportFilters().GetImageTypes(), storage.VulnerabilityReportFilters_WATCHED) { + watchedImages, err := rg.getWatchedImages(ctx) if err != nil { return nil, err } @@ -280,17 +529,19 @@ func (rg *reportGeneratorImpl) getReportDataSQF(snap *storage.ReportSnapshot, co cveFilterQuery) query.Pagination = watchedImagesQueryParts.Pagination query.Selects = watchedImagesQueryParts.Selects - watchedImageCVEResponses, err := pgSearch.RunSelectRequestForSchema[ImageCVEQueryResponse](reportGenCtx, rg.db, - watchedImagesQueryParts.Schema, query) + err := pgSearch.RunSelectRequestForSchemaFn[ImageCVEQueryResponse](ctx, rg.db, + watchedImagesQueryParts.Schema, query, func(r *ImageCVEQueryResponse) error { + cveResponses = append(cveResponses, r) + numWatchedImageResults++ + return nil + }) if err != nil { return nil, errors.Wrap(err, "Failed to collect report data for watched images") } - numWatchedImageResults = len(watchedImageCVEResponses) - cveResponses = append(cveResponses, watchedImageCVEResponses...) } } - cveResponses, err = rg.withCVEReferenceLinks(cveResponses) + cveResponses, err = rg.withCVEReferenceLinks(ctx, cveResponses) if err != nil { return nil, err } @@ -302,42 +553,52 @@ func (rg *reportGeneratorImpl) getReportDataSQF(snap *storage.ReportSnapshot, co }, nil } -func (rg *reportGeneratorImpl) getReportDataViewBased(snap *storage.ReportSnapshot) (*ReportData, error) { - watchedImages, err := rg.getWatchedImages() +func (rg *reportGeneratorImpl) getReportDataViewBased(ctx context.Context, snap *storage.ReportSnapshot) (*ReportData, error) { + watchedImages, err := rg.getWatchedImages(ctx) if err != nil { return nil, err } - query, err := rg.buildReportQueryViewBased(snap, watchedImages) + query, err := rg.buildReportQueryViewBased(ctx, snap, watchedImages) if err != nil { return nil, err } numDeployedImageResults := 0 var cveResponses []*ImageCVEQueryResponse + query.DeployedImagesQuery.Pagination = deployedImagesQueryParts.Pagination query.DeployedImagesQuery.Selects = deployedImagesQueryParts.Selects - cveResponses, err = pgSearch.RunSelectRequestForSchema[ImageCVEQueryResponse](reportGenCtx, rg.db, - deployedImagesQueryParts.Schema, query.DeployedImagesQuery) + err = pgSearch.RunSelectRequestForSchemaFn[ImageCVEQueryResponse](ctx, rg.db, + deployedImagesQueryParts.Schema, query.DeployedImagesQuery, func(r *ImageCVEQueryResponse) error { + cveResponses = append(cveResponses, r) + return nil + }) if err != nil { return nil, errors.Wrap(err, "Failed to collect report data for deployed images") } numDeployedImageResults = len(cveResponses) + if err := ctx.Err(); err != nil { + return nil, err + } + numWatchedImageResults := 0 if len(watchedImages) != 0 { query.WatchedImagesQuery.Pagination = watchedImagesQueryParts.Pagination query.WatchedImagesQuery.Selects = watchedImagesQueryParts.Selects - watchedImageCVEResponses, err := pgSearch.RunSelectRequestForSchema[ImageCVEQueryResponse](reportGenCtx, rg.db, - watchedImagesQueryParts.Schema, query.WatchedImagesQuery) + err := pgSearch.RunSelectRequestForSchemaFn[ImageCVEQueryResponse](ctx, rg.db, + watchedImagesQueryParts.Schema, query.WatchedImagesQuery, func(r *ImageCVEQueryResponse) error { + cveResponses = append(cveResponses, r) + numWatchedImageResults++ + return nil + }) if err != nil { return nil, errors.Wrap(err, "Failed to collect report data for watched images") } - numWatchedImageResults = len(watchedImageCVEResponses) - cveResponses = append(cveResponses, watchedImageCVEResponses...) } - cveResponses, err = rg.withCVEReferenceLinks(cveResponses) + cveResponses, err = rg.withCVEReferenceLinks(ctx, cveResponses) if err != nil { return nil, err } @@ -350,13 +611,13 @@ func (rg *reportGeneratorImpl) getReportDataViewBased(snap *storage.ReportSnapsh } -func (rg *reportGeneratorImpl) getClustersAndNamespacesForSAC() ([]effectiveaccessscope.Cluster, []effectiveaccessscope.Namespace, error) { - allClusters, err := rg.clusterDatastore.GetClusters(reportGenCtx) +func (rg *reportGeneratorImpl) getClustersAndNamespacesForSAC(ctx context.Context) ([]effectiveaccessscope.Cluster, []effectiveaccessscope.Namespace, error) { + allClusters, err := rg.clusterDatastore.GetClusters(ctx) if err != nil { return nil, nil, errors.Wrap(err, "error fetching clusters to build report query") } sacClusters := storagetoeffectiveaccessscope.Clusters(allClusters) - allNamespaces, err := rg.namespaceDatastore.GetAllNamespaces(reportGenCtx) + allNamespaces, err := rg.namespaceDatastore.GetAllNamespaces(ctx) if err != nil { return nil, nil, errors.Wrap(err, "error fetching namespaces to build report query") } @@ -364,9 +625,9 @@ func (rg *reportGeneratorImpl) getClustersAndNamespacesForSAC() ([]effectiveacce return sacClusters, sacNamespaces, nil } -func (rg *reportGeneratorImpl) buildReportQueryViewBased(snap *storage.ReportSnapshot, watchedImages []string) (*common.ReportQueryViewBased, error) { +func (rg *reportGeneratorImpl) buildReportQueryViewBased(ctx context.Context, snap *storage.ReportSnapshot, watchedImages []string) (*common.ReportQueryViewBased, error) { qb := common.NewVulnReportQueryBuilderViewBased(snap.GetViewBasedVulnReportFilters()) - allClusters, allNamespaces, err := rg.getClustersAndNamespacesForSAC() + allClusters, allNamespaces, err := rg.getClustersAndNamespacesForSAC(ctx) if err != nil { return nil, err } @@ -377,15 +638,15 @@ func (rg *reportGeneratorImpl) buildReportQueryViewBased(snap *storage.ReportSna return rQuery, nil } -func (rg *reportGeneratorImpl) buildReportQuery(snap *storage.ReportSnapshot, +func (rg *reportGeneratorImpl) buildReportQuery(ctx context.Context, snap *storage.ReportSnapshot, collection *storage.ResourceCollection, dataStartTime time.Time) (*common.ReportQuery, error) { - qb := common.NewVulnReportQueryBuilder(collection, snap.GetVulnReportFilters(), rg.collectionQueryResolver, + qb := common.NewVulnReportQueryBuilder(collection, snap.GetResourceScope().GetEntityScope(), snap.GetVulnReportFilters(), rg.collectionQueryResolver, dataStartTime) - allClusters, allNamespaces, err := rg.getClustersAndNamespacesForSAC() + allClusters, allNamespaces, err := rg.getClustersAndNamespacesForSAC(ctx) if err != nil { return nil, err } - rQuery, err := qb.BuildQuery(reportGenCtx, allClusters, allNamespaces) + rQuery, err := qb.BuildQuery(ctx, allClusters, allNamespaces) if err != nil { return nil, errors.Wrap(err, "error building report query") } @@ -396,16 +657,8 @@ func (rg *reportGeneratorImpl) buildReportQuery(snap *storage.ReportSnapshot, func (rg *reportGeneratorImpl) retryableSendReportResults(reportNotifier notifiers.ReportNotifier, mailingList []string, zippedCSVData *bytes.Buffer, emailSubject, emailBody, baseFilename string) error { - return retry.WithRetry(func() error { - return reportNotifier.ReportNotify(reportGenCtx, zippedCSVData, mailingList, emailSubject, emailBody, baseFilename) - }, - retry.OnlyRetryableErrors(), - retry.Tries(3), - retry.BetweenAttempts(func(previousAttempt int) { - wait := time.Duration(previousAttempt * previousAttempt * 100) - time.Sleep(wait * time.Millisecond) - }), - ) + return RetryableSendReportResults(reportGenCtx, reportNotifier, mailingList, + zippedCSVData, emailSubject, emailBody, baseFilename) } func (rg *reportGeneratorImpl) lastSuccessfulScheduledReportTime(snap *storage.ReportSnapshot) (time.Time, error) { @@ -434,8 +687,8 @@ func (rg *reportGeneratorImpl) lastSuccessfulScheduledReportTime(snap *storage.R return completedAt, nil } -func (rg *reportGeneratorImpl) getWatchedImages() ([]string, error) { - watched, err := rg.watchedImageDatastore.GetAllWatchedImages(reportGenCtx) +func (rg *reportGeneratorImpl) getWatchedImages(ctx context.Context) ([]string, error) { + watched, err := rg.watchedImageDatastore.GetAllWatchedImages(ctx) if err != nil { return nil, err } @@ -446,7 +699,7 @@ func (rg *reportGeneratorImpl) getWatchedImages() ([]string, error) { return results, nil } -func (rg *reportGeneratorImpl) withCVEReferenceLinks(imageCVEResponses []*ImageCVEQueryResponse) ([]*ImageCVEQueryResponse, error) { +func (rg *reportGeneratorImpl) withCVEReferenceLinks(ctx context.Context, imageCVEResponses []*ImageCVEQueryResponse) ([]*ImageCVEQueryResponse, error) { cveIDs := set.NewStringSet() for _, res := range imageCVEResponses { if res.GetCVEID() != "" { @@ -455,7 +708,7 @@ func (rg *reportGeneratorImpl) withCVEReferenceLinks(imageCVEResponses []*ImageC } var cves []ImageCVEInterface - imageCVEV2, err := rg.imageCVE2Datastore.GetBatch(reportGenCtx, cveIDs.AsSlice()) + imageCVEV2, err := rg.imageCVE2Datastore.GetBatch(ctx, cveIDs.AsSlice()) if err != nil { return nil, err } @@ -463,6 +716,7 @@ func (rg *reportGeneratorImpl) withCVEReferenceLinks(imageCVEResponses []*ImageC cves = append(cves, v2) } + // cveRefLinks is a map[string]string that maps CVE ID -> reference URL cveRefLinks := make(map[string]string) for _, cve := range cves { cveRefLinks[cve.GetId()] = cve.GetCveBaseInfo().GetLink() @@ -477,36 +731,11 @@ func (rg *reportGeneratorImpl) withCVEReferenceLinks(imageCVEResponses []*ImageC } func (rg *reportGeneratorImpl) updateReportStatus(snapshot *storage.ReportSnapshot, status storage.ReportStatus_RunState) error { - snapshot.ReportStatus.RunState = status - return rg.reportSnapshotStore.UpdateReportSnapshot(reportGenCtx, snapshot) + return UpdateReportStatus(reportGenCtx, rg.reportSnapshotStore, snapshot, status) } -func (rg *reportGeneratorImpl) logAndUpsertError(reportErr error, req *ReportRequest) { - if req.ReportSnapshot == nil || req.ReportSnapshot.GetReportStatus() == nil { - utils.Should(errors.New("Request does not have non-nil report snapshot with a non-nil report status")) - return - } - if reportErr != nil { - log.Errorf("Error while running report for config '%s': %s", req.ReportSnapshot.GetName(), reportErr) - req.ReportSnapshot.ReportStatus.ErrorMsg = reportErr.Error() - } - req.ReportSnapshot.ReportStatus.CompletedAt = protocompat.TimestampNow() - err := rg.updateReportStatus(req.ReportSnapshot, storage.ReportStatus_FAILURE) - - if err != nil { - log.Errorf("Error changing report status to FAILURE for report config '%s', report ID '%s': %s", - req.ReportSnapshot.GetName(), req.ReportSnapshot.GetReportId(), err) - } -} - -func filterOnImageType(imageTypes []storage.VulnerabilityReportFilters_ImageType, - target storage.VulnerabilityReportFilters_ImageType) bool { - for _, typ := range imageTypes { - if typ == target { - return true - } - } - return false +func (rg *reportGeneratorImpl) logAndUpsertError(ctx context.Context, reportErr error, req *ReportRequest) { + LogAndUpsertError(ctx, reportGenCtx, rg.reportSnapshotStore, reportErr, req) } func selectSchema() *walker.Schema { @@ -517,6 +746,7 @@ func getSelectsWatchedImages() []*v1.QuerySelect { ret := []*v1.QuerySelect{ search.NewQuerySelect(search.ImageName).Proto(), search.NewQuerySelect(search.Component).Proto(), + search.NewQuerySelect(search.ComponentVersion).Proto(), search.NewQuerySelect(search.CVEID).Proto(), search.NewQuerySelect(search.CVE).Proto(), search.NewQuerySelect(search.Fixable).Proto(), @@ -525,9 +755,13 @@ func getSelectsWatchedImages() []*v1.QuerySelect { search.NewQuerySelect(search.CVSS).Proto(), search.NewQuerySelect(search.NVDCVSS).Proto(), search.NewQuerySelect(search.FirstImageOccurrenceTimestamp).Proto(), + search.NewQuerySelect(search.ImageCreatedTime).Proto(), search.NewQuerySelect(search.EPSSProbablity).Proto(), + search.NewQuerySelect(search.CisaKev).Proto(), + search.NewQuerySelect(search.KnownRansomwareCampaign).Proto(), search.NewQuerySelect(search.AdvisoryName).Proto(), search.NewQuerySelect(search.AdvisoryLink).Proto(), + search.NewQuerySelect(search.CVEOrigin).Proto(), } return ret } @@ -536,6 +770,7 @@ func getSelectsDeployedImages() []*v1.QuerySelect { ret := []*v1.QuerySelect{ search.NewQuerySelect(search.ImageName).Proto(), search.NewQuerySelect(search.Component).Proto(), + search.NewQuerySelect(search.ComponentVersion).Proto(), search.NewQuerySelect(search.CVEID).Proto(), search.NewQuerySelect(search.CVE).Proto(), search.NewQuerySelect(search.Fixable).Proto(), @@ -544,12 +779,16 @@ func getSelectsDeployedImages() []*v1.QuerySelect { search.NewQuerySelect(search.CVSS).Proto(), search.NewQuerySelect(search.NVDCVSS).Proto(), search.NewQuerySelect(search.FirstImageOccurrenceTimestamp).Proto(), + search.NewQuerySelect(search.ImageCreatedTime).Proto(), search.NewQuerySelect(search.Cluster).Proto(), search.NewQuerySelect(search.Namespace).Proto(), search.NewQuerySelect(search.DeploymentName).Proto(), search.NewQuerySelect(search.EPSSProbablity).Proto(), + search.NewQuerySelect(search.CisaKev).Proto(), + search.NewQuerySelect(search.KnownRansomwareCampaign).Proto(), search.NewQuerySelect(search.AdvisoryName).Proto(), search.NewQuerySelect(search.AdvisoryLink).Proto(), + search.NewQuerySelect(search.CVEOrigin).Proto(), } return ret } diff --git a/central/reports/scheduler/v2/reportgenerator/report_gen_impl_flat_data_model_test.go b/central/reports/scheduler/v2/reportgenerator/report_gen_impl_flat_data_model_test.go index df85ecbe32a54..b45f5ff05d5d3 100644 --- a/central/reports/scheduler/v2/reportgenerator/report_gen_impl_flat_data_model_test.go +++ b/central/reports/scheduler/v2/reportgenerator/report_gen_impl_flat_data_model_test.go @@ -12,13 +12,15 @@ import ( clusterDSMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" "github.com/stackrox/rox/central/graphql/resolvers" "github.com/stackrox/rox/central/graphql/resolvers/loaders" - namespaceDSMocks "github.com/stackrox/rox/central/namespace/datastore/mocks" + namespaceDS "github.com/stackrox/rox/central/namespace/datastore" collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" collectionPostgres "github.com/stackrox/rox/central/resourcecollection/datastore/store/postgres" deploymentsView "github.com/stackrox/rox/central/views/deployments" imagesView "github.com/stackrox/rox/central/views/images" watchedImageDS "github.com/stackrox/rox/central/watchedimage/datastore" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres/pgtest" postgresSchema "github.com/stackrox/rox/pkg/postgres/schema" "github.com/stackrox/rox/pkg/sac" @@ -47,16 +49,17 @@ type NewDataModelEnhancedReportingTestSuite struct { testDB *pgtest.TestPostgres watchedImageDatastore watchedImageDS.DataStore clusterDatastore *clusterDSMocks.MockDataStore - namespaceDatastore *namespaceDSMocks.MockDataStore + namespaceDatastore namespaceDS.DataStore reportGenerator *reportGeneratorImpl } -func (s *NewDataModelEnhancedReportingTestSuite) TearDownTest() { +func (s *NewDataModelEnhancedReportingTestSuite) TearDownSuite() { s.truncateTable(postgresSchema.DeploymentsTableName) s.truncateTable(postgresSchema.ImagesTableName) s.truncateTable(postgresSchema.ImageComponentV2TableName) s.truncateTable(postgresSchema.ImageCvesV2TableName) s.truncateTable(postgresSchema.CollectionsTableName) + s.truncateTable(postgresSchema.NamespacesTableName) } func (s *NewDataModelEnhancedReportingTestSuite) SetupSuite() { @@ -65,21 +68,38 @@ func (s *NewDataModelEnhancedReportingTestSuite) SetupSuite() { s.testDB = pgtest.ForT(s.T()) // set up tables - imageDataStore := resolvers.CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl) s.watchedImageDatastore = watchedImageDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) - resolver, schema := resolvers.SetupTestResolver(s.T(), - imagesView.NewImageView(s.testDB.DB), - imageDataStore, - resolvers.CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), - resolvers.CreateTestImageCVEV2Datastore(s.T(), s.testDB), - resolvers.CreateTestDeploymentDatastore(s.T(), s.testDB, mockCtrl, imageDataStore), - deploymentsView.NewDeploymentView(s.testDB.DB), - ) + + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + var resolver *resolvers.Resolver + if features.FlattenImageData.Enabled() { + imgV2DataStore := resolvers.CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl) + resolver, _ = resolvers.SetupTestResolver(s.T(), + imagesView.NewImageView(s.testDB.DB), + imgV2DataStore, + resolvers.CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), + resolvers.CreateTestImageCVEV2Datastore(s.T(), s.testDB), + resolvers.CreateTestDeploymentDatastoreWithImageV2(s.T(), s.testDB, mockCtrl, imgV2DataStore), + deploymentsView.NewDeploymentView(s.testDB.DB), + ) + } else { + imageDataStore := resolvers.CreateTestImageDatastore(s.T(), s.testDB, mockCtrl) + resolver, _ = resolvers.SetupTestResolver(s.T(), + imagesView.NewImageView(s.testDB.DB), + imageDataStore, + resolvers.CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), + resolvers.CreateTestImageCVEV2Datastore(s.T(), s.testDB), + resolvers.CreateTestDeploymentDatastore(s.T(), s.testDB, mockCtrl, imageDataStore), + deploymentsView.NewDeploymentView(s.testDB.DB), + ) + } collectionStore := collectionPostgres.New(s.testDB) _, collectionQueryResolver, err := collectionDS.New(collectionStore) s.NoError(err) s.clusterDatastore = clusterDSMocks.NewMockDataStore(mockCtrl) - s.namespaceDatastore = namespaceDSMocks.NewMockDataStore(mockCtrl) + var nsErr error + s.namespaceDatastore, nsErr = namespaceDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(nsErr) // Add Test Data to DataStores clusters := []*storage.Cluster{ @@ -88,6 +108,18 @@ func (s *NewDataModelEnhancedReportingTestSuite) SetupSuite() { } namespaces := testNamespaces(clusters, 2) + // Add labels to namespaces so entity scope label queries can be tested: + // ns1 gets env=prod, ns2 gets env=dev. + for _, ns := range namespaces { + if ns.GetName() == "ns1" { + ns.Labels = map[string]string{"env": "prod"} + } else { + ns.Labels = map[string]string{"env": "dev"} + } + nsErr := s.namespaceDatastore.AddNamespace(s.ctx, ns) + s.Require().NoError(nsErr) + } + deployments, images := testDeploymentsWithImages(namespaces, 1) // insert deployments in deployment table for _, dep := range deployments { @@ -95,30 +127,43 @@ func (s *NewDataModelEnhancedReportingTestSuite) SetupSuite() { s.NoError(err) } // upsert deployed image in image table - for _, image := range images { - err := resolver.ImageDataStore.UpsertImage(s.ctx, image) - s.NoError(err) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + for _, image := range images { + err := resolver.ImageV2DataStore.UpsertImage(s.ctx, imageUtils.ConvertToV2(image)) + s.NoError(err) + } + } else { + for _, image := range images { + err := resolver.ImageDataStore.UpsertImage(s.ctx, image) + s.NoError(err) + } } // upsert watched images watchedImages := testWatchedImages(2) - for _, image := range watchedImages { - err := resolver.ImageDataStore.UpsertImage(s.ctx, image) - s.NoError(err) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + for _, image := range watchedImages { + err := resolver.ImageV2DataStore.UpsertImage(s.ctx, imageUtils.ConvertToV2(image)) + s.NoError(err) + } + } else { + for _, image := range watchedImages { + err := resolver.ImageDataStore.UpsertImage(s.ctx, image) + s.NoError(err) + } } s.upsertManyWatchedImages(watchedImages) s.clusterDatastore.EXPECT().GetClusters(gomock.Any()). Return(clusters, nil).AnyTimes() - s.namespaceDatastore.EXPECT().GetAllNamespaces(gomock.Any()). - Return(namespaces, nil).AnyTimes() - blobStore := blobDS.NewTestDatastore(s.T(), s.testDB.DB) s.reportGenerator = newReportGeneratorImpl(s.testDB, nil, resolver.DeploymentDataStore, s.watchedImageDatastore, collectionQueryResolver, nil, blobStore, s.clusterDatastore, - s.namespaceDatastore, resolver.ImageCVEV2DataStore, schema) + s.namespaceDatastore, resolver.ImageCVEV2DataStore) } func (s *NewDataModelEnhancedReportingTestSuite) upsertManyWatchedImages(images []*storage.Image) { for _, img := range images { @@ -169,7 +214,7 @@ func (s *NewDataModelEnhancedReportingTestSuite) TestGetReportData() { s.T().Run(tc.name, func(t *testing.T) { reportSnap := testReportSnapshot(tc.collection.GetId(), tc.fixability, tc.severities, tc.imageTypes, tc.scopeRules) // Test get data using SQF - reportData, err := s.reportGenerator.getReportDataSQF(reportSnap, tc.collection, time.Time{}) + reportData, err := s.reportGenerator.getReportDataSQF(s.ctx, reportSnap, tc.collection, time.Time{}) s.NoError(err) collected := collectVulnReportDataSQFNewDataModel(reportData.CVEResponses) s.ElementsMatch(tc.expected.deploymentNames, collected.deploymentNames) @@ -183,12 +228,28 @@ func (s *NewDataModelEnhancedReportingTestSuite) TestGetReportData() { } +func (s *NewDataModelEnhancedReportingTestSuite) TestGetReportDataWithCancelledContext() { + collection := testCollection("col-cancel", "", "", "") + reportSnap := testReportSnapshot(collection.GetId(), + storage.VulnerabilityReportFilters_BOTH, allSeverities(), + []storage.VulnerabilityReportFilters_ImageType{storage.VulnerabilityReportFilters_DEPLOYED}, + nil) + + // Cancel the context before calling getReportDataSQF to prove cancellation propagates to the DB query + ctx, cancel := context.WithCancel(s.ctx) + cancel() + + _, err := s.reportGenerator.getReportDataSQF(ctx, reportSnap, collection, time.Time{}) + s.Error(err, "Expected error when context is cancelled before DB query") + s.ErrorIs(err, context.Canceled) +} + func collectVulnReportDataSQFNewDataModel(cveResponses []*ImageCVEQueryResponse) *vulnReportDataNewDataModel { deploymentNames := set.NewStringSet() imageNames := set.NewStringSet() componentNames := set.NewStringSet() - cveNames := make([]string, 0) - cvss := make([]float64, 0) + cveNames := make([]string, 0, len(cveResponses)) + cvss := make([]float64, 0, len(cveResponses)) for _, res := range cveResponses { if res.GetDeployment() != "" { @@ -207,3 +268,368 @@ func collectVulnReportDataSQFNewDataModel(cveResponses []*ImageCVEQueryResponse) cvss: cvss, } } + +func (s *NewDataModelEnhancedReportingTestSuite) TestGetReportDataWithEntityScopeAndQueryFilters() { + allImageTypes := []storage.VulnerabilityReportFilters_ImageType{ + storage.VulnerabilityReportFilters_DEPLOYED, + storage.VulnerabilityReportFilters_WATCHED, + } + deployedOnly := []storage.VulnerabilityReportFilters_ImageType{ + storage.VulnerabilityReportFilters_DEPLOYED, + } + + testCases := map[string]struct { + entityScope *storage.EntityScope + query string + imageTypes []storage.VulnerabilityReportFilters_ImageType + scopeRules []*storage.SimpleAccessScope_Rules + dataStartTime time.Time + expected *vulnReportDataNewDataModel + }{ + "Cluster scope with CVSS filter narrows by scope and score": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + query: "CVSS:>=7.0", + imageTypes: allImageTypes, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1"}}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0", "c1_ns2_dep0"}, + imageNames: []string{"c1_ns1_dep0_img", "c1_ns2_dep0_img", "w0_img", "w1_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp", "c1_ns2_dep0_img_comp", "w0_img_comp", "w1_img_comp"}, + cveNames: []string{ + "CVE-fixable_critical-c1_ns1_dep0_img_comp", + "CVE-fixable_critical-c1_ns2_dep0_img_comp", + "CVE-fixable_critical-w0_img_comp", + "CVE-fixable_critical-w1_img_comp", + }, + }, + }, + "Namespace scope with EPSS probability filter and c1 access scope": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "ns1", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + query: "EPSS Probability:>=0.5", + imageTypes: allImageTypes, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1"}}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0"}, + imageNames: []string{"c1_ns1_dep0_img", "w0_img", "w1_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp", "w0_img_comp", "w1_img_comp"}, + cveNames: []string{ + "CVE-fixable_critical-c1_ns1_dep0_img_comp", + "CVE-fixable_critical-w0_img_comp", + "CVE-fixable_critical-w1_img_comp", + }, + }, + }, + "Deployment scope with low NVD CVSS filter": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1_ns1_dep0", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + query: "NVD CVSS:<=5.0", + imageTypes: allImageTypes, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedNamespaces: []*storage.SimpleAccessScope_Rules_Namespace{ + {ClusterName: "c1", NamespaceName: "ns1"}, + }}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0"}, + imageNames: []string{"c1_ns1_dep0_img", "w0_img", "w1_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp", "w0_img_comp", "w1_img_comp"}, + cveNames: []string{ + "CVE-nonFixable_low-c1_ns1_dep0_img_comp", + "CVE-nonFixable_low-w0_img_comp", + "CVE-nonFixable_low-w1_img_comp", + }, + }, + }, + "Cluster and namespace AND rules with fixability filter": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1", MatchType: storage.MatchType_EXACT}, + }, + }, + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "ns1", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + query: "Fixable:true", + imageTypes: allImageTypes, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedNamespaces: []*storage.SimpleAccessScope_Rules_Namespace{ + {ClusterName: "c1", NamespaceName: "ns1"}, + }}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0"}, + imageNames: []string{"c1_ns1_dep0_img", "w0_img", "w1_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp", "w0_img_comp", "w1_img_comp"}, + cveNames: []string{ + "CVE-fixable_critical-c1_ns1_dep0_img_comp", + "CVE-fixable_critical-w0_img_comp", + "CVE-fixable_critical-w1_img_comp", + }, + }, + }, + "Deployment scope deployed-only with component source filter": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1_ns1_dep0", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + query: "Component Source:OS", + imageTypes: deployedOnly, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedNamespaces: []*storage.SimpleAccessScope_Rules_Namespace{ + {ClusterName: "c1", NamespaceName: "ns1"}, + }}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0"}, + imageNames: []string{"c1_ns1_dep0_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp"}, + cveNames: []string{ + "CVE-fixable_critical-c1_ns1_dep0_img_comp", + "CVE-nonFixable_low-c1_ns1_dep0_img_comp", + }, + }, + }, + "Deployment regex scope with CVSS filter": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1_.*", MatchType: storage.MatchType_REGEX}, + }, + }, + }, + }, + query: "CVSS:>=7.0", + imageTypes: allImageTypes, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1"}}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0", "c1_ns2_dep0"}, + imageNames: []string{"c1_ns1_dep0_img", "c1_ns2_dep0_img", "w0_img", "w1_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp", "c1_ns2_dep0_img_comp", "w0_img_comp", "w1_img_comp"}, + cveNames: []string{ + "CVE-fixable_critical-c1_ns1_dep0_img_comp", + "CVE-fixable_critical-c1_ns2_dep0_img_comp", + "CVE-fixable_critical-w0_img_comp", + "CVE-fixable_critical-w1_img_comp", + }, + }, + }, + "Multiple deployment values AND with low EPSS filter": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1_ns1_dep0", MatchType: storage.MatchType_EXACT}, + {Value: "c2_ns1_dep0", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + query: "EPSS Probability:<=0.5", + imageTypes: allImageTypes, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedNamespaces: []*storage.SimpleAccessScope_Rules_Namespace{ + {ClusterName: "c1", NamespaceName: "ns1"}, + {ClusterName: "c2", NamespaceName: "ns1"}, + }}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0", "c2_ns1_dep0"}, + imageNames: []string{"c1_ns1_dep0_img", "c2_ns1_dep0_img", "w0_img", "w1_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp", "c2_ns1_dep0_img_comp", "w0_img_comp", "w1_img_comp"}, + cveNames: []string{ + "CVE-nonFixable_low-c1_ns1_dep0_img_comp", + "CVE-nonFixable_low-c2_ns1_dep0_img_comp", + "CVE-nonFixable_low-w0_img_comp", + "CVE-nonFixable_low-w1_img_comp", + }, + }, + }, + "Namespace scope with image registry and CVSS combined filter": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "ns1", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + query: "Image Registry:docker.io+CVSS:>=7.0", + imageTypes: allImageTypes, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1", "c2"}}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0", "c2_ns1_dep0"}, + imageNames: []string{"c1_ns1_dep0_img", "c2_ns1_dep0_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp", "c2_ns1_dep0_img_comp"}, + cveNames: []string{ + "CVE-fixable_critical-c1_ns1_dep0_img_comp", + "CVE-fixable_critical-c2_ns1_dep0_img_comp", + }, + }, + }, + "Deployment scope with image label filter": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1_ns1_dep0", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + query: "Image Label:app=test", + imageTypes: allImageTypes, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedNamespaces: []*storage.SimpleAccessScope_Rules_Namespace{ + {ClusterName: "c1", NamespaceName: "ns1"}, + }}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0"}, + imageNames: []string{"c1_ns1_dep0_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp"}, + cveNames: []string{ + "CVE-fixable_critical-c1_ns1_dep0_img_comp", "CVE-nonFixable_low-c1_ns1_dep0_img_comp", + }, + }, + }, + "Namespace label regex scope with CVSS filter": { + // ns1 has label env=prod; regex "env=pr.*" matches prod but not dev. + // With all-cluster SAC access and CVSS:>=7.0, only fixable_critical CVEs are returned + // for deployments in ns1 across both clusters, plus watched images. + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_LABEL, + Values: []*storage.RuleValue{ + {Value: "env=pr.*", MatchType: storage.MatchType_REGEX}, + }, + }, + }, + }, + query: "CVSS:>=7.0", + imageTypes: allImageTypes, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1", "c2"}}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0", "c2_ns1_dep0"}, + imageNames: []string{"c1_ns1_dep0_img", "c2_ns1_dep0_img", "w0_img", "w1_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp", "c2_ns1_dep0_img_comp", "w0_img_comp", "w1_img_comp"}, + cveNames: []string{ + "CVE-fixable_critical-c1_ns1_dep0_img_comp", + "CVE-fixable_critical-c2_ns1_dep0_img_comp", + "CVE-fixable_critical-w0_img_comp", + "CVE-fixable_critical-w1_img_comp", + }, + }, + }, + "Cluster scope deployed-only with image regex and EPSS combined filter": { + entityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "c1", MatchType: storage.MatchType_EXACT}, + {Value: "c2", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + }, + query: "Image:r/c1_.*+EPSS Probability:>=0.5", + imageTypes: deployedOnly, + scopeRules: []*storage.SimpleAccessScope_Rules{ + {IncludedClusters: []string{"c1", "c2"}}, + }, + expected: &vulnReportDataNewDataModel{ + deploymentNames: []string{"c1_ns1_dep0", "c1_ns2_dep0"}, + imageNames: []string{"c1_ns1_dep0_img", "c1_ns2_dep0_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp", "c1_ns2_dep0_img_comp"}, + cveNames: []string{ + "CVE-fixable_critical-c1_ns1_dep0_img_comp", + "CVE-fixable_critical-c1_ns2_dep0_img_comp", + }, + }, + }, + } + + for name, tc := range testCases { + s.T().Run(name, func(t *testing.T) { + reportSnap := testEntityScopeReportSnapshot(tc.entityScope, tc.query, tc.imageTypes, tc.scopeRules) + reportData, err := s.reportGenerator.getReportDataSQF(s.ctx, reportSnap, nil, time.Time{}) + s.NoError(err) + collected := collectVulnReportDataSQFNewDataModel(reportData.CVEResponses) + s.ElementsMatch(tc.expected.deploymentNames, collected.deploymentNames) + s.ElementsMatch(tc.expected.imageNames, collected.imageNames) + s.ElementsMatch(tc.expected.componentNames, collected.componentNames) + s.ElementsMatch(tc.expected.cveNames, collected.cveNames) + s.Equal(len(tc.expected.cveNames), reportData.NumDeployedImageResults+reportData.NumWatchedImageResults) + s.Equal(len(tc.expected.cveNames), len(collected.cvss)) + }) + } +} diff --git a/central/reports/scheduler/v2/reportgenerator/report_gen_memory_bench_test.go b/central/reports/scheduler/v2/reportgenerator/report_gen_memory_bench_test.go new file mode 100644 index 0000000000000..3ddb8a21990c6 --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/report_gen_memory_bench_test.go @@ -0,0 +1,308 @@ +//go:build sql_integration + +package reportgenerator + +import ( + "context" + "runtime" + "testing" + "time" + + blobDS "github.com/stackrox/rox/central/blob/datastore" + clusterDSMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" + "github.com/stackrox/rox/central/graphql/resolvers" + "github.com/stackrox/rox/central/graphql/resolvers/loaders" + namespaceDS "github.com/stackrox/rox/central/namespace/datastore" + reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" + reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" + collectionPostgres "github.com/stackrox/rox/central/resourcecollection/datastore/store/postgres" + deploymentsView "github.com/stackrox/rox/central/views/deployments" + imagesView "github.com/stackrox/rox/central/views/images" + watchedImageDS "github.com/stackrox/rox/central/watchedimage/datastore" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + imageUtils "github.com/stackrox/rox/pkg/images/utils" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// memoryBenchSuite holds shared test infrastructure for the memory benchmarks. +type memoryBenchSuite struct { + b *testing.B + ctx context.Context + testDB *pgtest.TestPostgres + rg *reportGeneratorImpl + collection *storage.ResourceCollection + snap *storage.ReportSnapshot + rowCount int +} + +func setupMemoryBench(b *testing.B, numNamespacesPerCluster, numDeploymentsPerNamespace, numWatchedImages int) *memoryBenchSuite { + b.Helper() + + ctx := loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) + mockCtrl := gomock.NewController(b) + testDB := pgtest.ForT(b) + + watchedImageDatastore := watchedImageDS.GetTestPostgresDataStore(b, testDB.DB) + + var resolver *resolvers.Resolver + if features.FlattenImageData.Enabled() { + imgV2DataStore := resolvers.CreateTestImageV2Datastore(b, testDB, mockCtrl) + resolver, _ = resolvers.SetupTestResolver(b, + imagesView.NewImageView(testDB.DB), + imgV2DataStore, + resolvers.CreateTestImageComponentV2Datastore(b, testDB, mockCtrl), + resolvers.CreateTestImageCVEV2Datastore(b, testDB), + resolvers.CreateTestDeploymentDatastoreWithImageV2(b, testDB, mockCtrl, imgV2DataStore), + deploymentsView.NewDeploymentView(testDB.DB), + ) + } else { + imageDataStore := resolvers.CreateTestImageDatastore(b, testDB, mockCtrl) + resolver, _ = resolvers.SetupTestResolver(b, + imagesView.NewImageView(testDB.DB), + imageDataStore, + resolvers.CreateTestImageComponentV2Datastore(b, testDB, mockCtrl), + resolvers.CreateTestImageCVEV2Datastore(b, testDB), + resolvers.CreateTestDeploymentDatastore(b, testDB, mockCtrl, imageDataStore), + deploymentsView.NewDeploymentView(testDB.DB), + ) + } + + collectionStore := collectionPostgres.New(testDB) + _, collectionQueryResolver, err := collectionDS.New(collectionStore) + require.NoError(b, err) + + clusterDatastore := clusterDSMocks.NewMockDataStore(mockCtrl) + nsDatastore, err := namespaceDS.GetTestPostgresDataStore(b, testDB.DB) + require.NoError(b, err) + + blobStore := blobDS.NewTestDatastore(b, testDB.DB) + reportSnapshotStore := reportSnapshotDS.GetTestPostgresDataStore(b, testDB.DB) + + rg := newReportGeneratorImpl(testDB, reportSnapshotStore, resolver.DeploymentDataStore, + watchedImageDatastore, collectionQueryResolver, nil, blobStore, clusterDatastore, + nsDatastore, resolver.ImageCVEV2DataStore) + + // pgtest limits all pools to pool_max_conns=1 which is too restrictive for + // report generation. Use a shared pool with enough connections. + multiConnCfg := testDB.DB.Config().Copy() + multiConnCfg.Config.MaxConns = 4 + multiConnPool, err := postgres.New(context.Background(), multiConnCfg) + require.NoError(b, err) + b.Cleanup(func() { multiConnPool.Close() }) + rg.db = multiConnPool + rg.blobStore = blobDS.NewTestDatastore(b, multiConnPool) + + clusters := []*storage.Cluster{ + {Id: uuid.NewV4().String(), Name: "c1"}, + {Id: uuid.NewV4().String(), Name: "c2"}, + } + namespaces := testNamespaces(clusters, numNamespacesPerCluster) + for _, ns := range namespaces { + require.NoError(b, nsDatastore.AddNamespace(ctx, ns)) + } + + deployments, images := testDeploymentsWithImages(namespaces, numDeploymentsPerNamespace) + if features.FlattenImageData.Enabled() { + for _, img := range images { + require.NoError(b, resolver.ImageV2DataStore.UpsertImage(ctx, imageUtils.ConvertToV2(img))) + } + } else { + for _, img := range images { + require.NoError(b, resolver.ImageDataStore.UpsertImage(ctx, img)) + } + } + for _, dep := range deployments { + require.NoError(b, resolver.DeploymentDataStore.UpsertDeployment(ctx, dep)) + } + + watchedImages := testWatchedImages(numWatchedImages) + if features.FlattenImageData.Enabled() { + for _, img := range watchedImages { + require.NoError(b, resolver.ImageV2DataStore.UpsertImage(ctx, imageUtils.ConvertToV2(img))) + } + } else { + for _, img := range watchedImages { + require.NoError(b, resolver.ImageDataStore.UpsertImage(ctx, img)) + } + } + for _, img := range watchedImages { + require.NoError(b, watchedImageDatastore.UpsertWatchedImage(ctx, img.GetName().GetFullName())) + } + + clusterDatastore.EXPECT().GetClusters(gomock.Any()).Return(clusters, nil).AnyTimes() + + collection := testCollection("col_bench", "", "", "") + imageTypes := []storage.VulnerabilityReportFilters_ImageType{ + storage.VulnerabilityReportFilters_DEPLOYED, + storage.VulnerabilityReportFilters_WATCHED, + } + snap := testReportSnapshot(collection.GetId(), storage.VulnerabilityReportFilters_BOTH, allSeverities(), imageTypes, nil) + // updateReportStatus upserts the snapshot into report_snapshots. That table has a FK to + // report_configurations, so the parent config row must exist first. + reportConfigStore := reportConfigDS.GetTestPostgresDataStore(b, testDB.DB) + _, err = reportConfigStore.AddReportConfiguration(ctx, &storage.ReportConfiguration{ + Id: snap.GetReportConfigurationId(), + Name: "bench-report-config", + Type: storage.ReportConfiguration_VULNERABILITY, + }) + require.NoError(b, err) + // updateReportStatus also requires a non-empty ReportId. + snap.ReportId = uuid.NewV4().String() + + // Each image has 2 CVEs, so: + // deployed rows = 2 clusters * numNamespacesPerCluster * numDeploymentsPerNamespace * 2 + // watched rows = numWatchedImages * 2 + expectedRows := 2*numNamespacesPerCluster*numDeploymentsPerNamespace*2 + numWatchedImages*2 + + return &memoryBenchSuite{ + b: b, + ctx: ctx, + testDB: testDB, + rg: rg, + collection: collection, + snap: snap, + rowCount: expectedRows, + } +} + +func forceGC() { + runtime.GC() + runtime.GC() +} + +func heapAllocBytes() uint64 { + var m runtime.MemStats + runtime.ReadMemStats(&m) + return m.HeapAlloc +} + +// BenchmarkMemoryComparison runs InMemory and Transaction paths side-by-side for easy comparison. +// Use: go test -tags sql_integration -bench BenchmarkMemoryComparison -benchtime 3x -v +func BenchmarkMemoryComparison(b *testing.B) { + s := setupMemoryBench(b, 10, 50, 500) + if !features.FlattenImageData.Enabled() { + b.Skip("benchmark requires FlattenImageData (image v2 model) to be enabled") + } + + b.Run("InMemory", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + iterCtx := loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) + forceGC() + before := heapAllocBytes() + + reportData, err := s.rg.getReportDataSQF(iterCtx, s.snap, s.collection, time.Time{}) + require.NoError(b, err) + + zippedCSV, err := GenerateCSV(reportData.CVEResponses, s.snap.GetName()) + require.NoError(b, err) + + after := heapAllocBytes() + b.ReportMetric(float64(after-before), "heap-delta-bytes") + + err = s.rg.saveReportData(iterCtx, s.snap.GetReportConfigurationId(), s.snap.GetReportId(), zippedCSV) + require.NoError(b, err) + } + }) + + s.snap.ReportStatus.ReportNotificationMethod = storage.ReportStatus_DOWNLOAD + b.Run("Transaction", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + iterCtx := loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) + forceGC() + before := heapAllocBytes() + + err := s.rg.generateReportTransaction(iterCtx, &ReportRequest{ + ReportSnapshot: s.snap, + Collection: s.collection, + DataStartTime: time.Time{}, + }) + require.NoError(b, err) + + after := heapAllocBytes() + b.ReportMetric(float64(after-before), "heap-delta-bytes") + } + }) +} + +// BenchmarkMemoryAtScale uses larger data volumes to amplify the difference. +// 2 clusters * 20 namespaces * 100 deployments * 2 CVEs = 8000 deployed rows +// + 1000 watched images * 2 CVEs = 2000 watched rows = 10000 total rows. +func BenchmarkMemoryAtScale(b *testing.B) { + if !features.FlattenImageData.Enabled() { + b.Skip("benchmark requires FlattenImageData (image v2 model) to be enabled") + } + s := setupMemoryBench(b, 20, 100, 1000) + + s.snap.ReportStatus.ReportNotificationMethod = storage.ReportStatus_DOWNLOAD + b.Run("Transaction", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + iterCtx := loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) + forceGC() + before := heapAllocBytes() + + err := s.rg.generateReportTransaction(iterCtx, &ReportRequest{ + ReportSnapshot: s.snap, + Collection: s.collection, + DataStartTime: time.Time{}, + }) + require.NoError(b, err) + + after := heapAllocBytes() + b.ReportMetric(float64(after-before), "heap-delta-bytes") + } + }) +} + +// BenchmarkReportDuration measures time for InMemory vs Transaction report generation. +// The built-in ns/op from testing.B captures this automatically, but this benchmark also reports +// explicit millisecond metrics for easier comparison. +// Use: go test -tags sql_integration -bench BenchmarkReportDuration -benchtime 3x -v +func BenchmarkReportDuration(b *testing.B) { + s := setupMemoryBench(b, 10, 50, 500) + if !features.FlattenImageData.Enabled() { + b.Skip("benchmark requires FlattenImageData (image v2 model) to be enabled") + } + + b.Run("InMemory", func(b *testing.B) { + for i := 0; i < b.N; i++ { + iterCtx := loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) + + start := time.Now() + reportData, err := s.rg.getReportDataSQF(iterCtx, s.snap, s.collection, time.Time{}) + require.NoError(b, err) + + zippedCSV, err := GenerateCSV(reportData.CVEResponses, s.snap.GetName()) + require.NoError(b, err) + + err = s.rg.saveReportData(iterCtx, s.snap.GetReportConfigurationId(), s.snap.GetReportId(), zippedCSV) + require.NoError(b, err) + b.ReportMetric(float64(time.Since(start).Milliseconds()), "ms/report") + } + }) + + s.snap.ReportStatus.ReportNotificationMethod = storage.ReportStatus_DOWNLOAD + b.Run("Transaction", func(b *testing.B) { + for i := 0; i < b.N; i++ { + iterCtx := loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) + + start := time.Now() + err := s.rg.generateReportTransaction(iterCtx, &ReportRequest{ + ReportSnapshot: s.snap, + Collection: s.collection, + DataStartTime: time.Time{}, + }) + require.NoError(b, err) + b.ReportMetric(float64(time.Since(start).Milliseconds()), "ms/report") + } + }) +} diff --git a/central/reports/scheduler/v2/reportgenerator/report_gen_transaction_integration_test.go b/central/reports/scheduler/v2/reportgenerator/report_gen_transaction_integration_test.go new file mode 100644 index 0000000000000..d34af5f597329 --- /dev/null +++ b/central/reports/scheduler/v2/reportgenerator/report_gen_transaction_integration_test.go @@ -0,0 +1,304 @@ +//go:build sql_integration + +package reportgenerator + +import ( + "archive/zip" + "bytes" + "context" + "encoding/csv" + "io" + "slices" + "time" + + blobDS "github.com/stackrox/rox/central/blob/datastore" + "github.com/stackrox/rox/central/reports/common" + reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" + reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/uuid" +) + +// CSV column indices for the fixed prefix that precedes any feature-flag-gated +// columns (see csvHeader in csv_gen.go). Columns after EPSS can shift depending +// on enabled features, so look those up with colOf instead of hardcoding them. +const ( + colCluster = 0 + colNamespace = 1 + colDeployment = 2 + colImage = 3 + colComponent = 4 + colCompVersion = 5 + colCVE = 6 + colFixable = 7 + colFixedBy = 8 + colSeverity = 9 + colCVSS = 10 + colNVDCVSS = 11 + colEPSS = 12 +) + +// colOf returns the index of the named column in the current CSV layout, which +// depends on which feature-flag-gated columns formatCol includes. +func colOf(name string) int { + return slices.Index(formatCol(), name) +} + +// collectColumn extracts a single column from all rows. +func collectColumn(rows [][]string, col int) []string { + out := make([]string, len(rows)) + for i, row := range rows { + out[i] = row[col] + } + return out +} + +// setupTransactionDeps creates a multi-connection pool and the datastores +// required by generateReportTransaction. The returned reportGeneratorImpl +// is a shallow copy of s.reportGenerator with db, blobStore, and +// reportSnapshotStore replaced. +func (s *NewDataModelEnhancedReportingTestSuite) setupTransactionDeps() ( + rg *reportGeneratorImpl, + blobStore blobDS.Datastore, + configStore reportConfigDS.DataStore, +) { + s.T().Helper() + + // generateReportTransaction runs cursor reads, CVE lookups, and blob + // store writes all within a single transaction (1 connection) on a + // single goroutine, so no concurrent tx access occurs. + poolCfg := s.testDB.DB.Config().Copy() + poolCfg.Config.MaxConns = 1 + pool, err := postgres.New(context.Background(), poolCfg) + s.Require().NoError(err) + s.T().Cleanup(func() { pool.Close() }) + + blobStore = blobDS.NewTestDatastore(s.T(), pool) + snapStore := reportSnapshotDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + configStore = reportConfigDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + + rgCopy := *s.reportGenerator + rgCopy.db = pool + rgCopy.blobStore = blobStore + rgCopy.reportSnapshotStore = snapStore + return &rgCopy, blobStore, configStore +} + +// prepareSnapshot creates a report snapshot with a backing report configuration +// (FK constraint) and sets the notification method to DOWNLOAD so the streaming +// path is exercised. +func (s *NewDataModelEnhancedReportingTestSuite) prepareSnapshot( + configStore reportConfigDS.DataStore, + collectionID string, + fixability storage.VulnerabilityReportFilters_Fixability, + severities []storage.VulnerabilitySeverity, + imageTypes []storage.VulnerabilityReportFilters_ImageType, +) *storage.ReportSnapshot { + s.T().Helper() + + snap := testReportSnapshot(collectionID, fixability, severities, imageTypes, nil) + configID := uuid.NewV4().String() + snap.ReportConfigurationId = configID + snap.ReportId = uuid.NewV4().String() + snap.ReportStatus.ReportNotificationMethod = storage.ReportStatus_DOWNLOAD + + _, err := configStore.AddReportConfiguration(s.ctx, &storage.ReportConfiguration{ + Id: configID, + Name: "txn-test-config", + Type: storage.ReportConfiguration_VULNERABILITY, + }) + s.Require().NoError(err) + return snap +} + +// readBlobCSV fetches a zipped CSV from blob store, decompresses it, and returns +// the parsed CSV rows (including the header row). +func (s *NewDataModelEnhancedReportingTestSuite) readBlobCSV( + blobStore blobDS.Datastore, + configID, reportID string, +) [][]string { + s.T().Helper() + + blobPath := common.GetReportBlobPath(configID, reportID) + var buf bytes.Buffer + _, found, err := blobStore.Get(reportGenCtx, blobPath, &buf) + s.Require().NoError(err) + s.Require().True(found, "blob not found at %s", blobPath) + + zipReader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + s.Require().NoError(err) + s.Require().Len(zipReader.File, 1, "expected exactly one entry in the ZIP archive") + + rc, err := zipReader.File[0].Open() + s.Require().NoError(err) + defer func() { _ = rc.Close() }() + + csvBytes, err := io.ReadAll(rc) + s.Require().NoError(err) + + r := csv.NewReader(bytes.NewReader(csvBytes)) + r.FieldsPerRecord = -1 + rows, err := r.ReadAll() + s.Require().NoError(err) + return rows +} + +// --------------------------------------------------------------------------- +// Tests for generateReportTransaction +// --------------------------------------------------------------------------- + +// TestGenerateReportTransaction_AllDeployedImages verifies the full pipeline: +// cursor reads in a single transaction → batched CSV → ZIP → blob store. +// +// Test data: 4 deployments × 2 CVEs = 8 rows, plus header. +func (s *NewDataModelEnhancedReportingTestSuite) TestGenerateReportTransaction_AllDeployedImages() { + rg, blobStore, configStore := s.setupTransactionDeps() + snap := s.prepareSnapshot(configStore, "col1", + storage.VulnerabilityReportFilters_BOTH, allSeverities(), + []storage.VulnerabilityReportFilters_ImageType{storage.VulnerabilityReportFilters_DEPLOYED}) + + err := rg.generateReportTransaction(s.ctx, &ReportRequest{ + ReportSnapshot: snap, + Collection: testCollection("col1", "", "", ""), + DataStartTime: time.Time{}, + }) + s.Require().NoError(err) + + rows := s.readBlobCSV(blobStore, snap.GetReportConfigurationId(), snap.GetReportId()) + s.Require().Len(rows, 9, "1 header + 8 data rows") + s.Equal(formatCol(), rows[0]) + + dataRows := rows[1:] + s.ElementsMatch([]string{ + "CVE-fixable_critical-c1_ns1_dep0_img_comp", "CVE-nonFixable_low-c1_ns1_dep0_img_comp", + "CVE-fixable_critical-c1_ns2_dep0_img_comp", "CVE-nonFixable_low-c1_ns2_dep0_img_comp", + "CVE-fixable_critical-c2_ns1_dep0_img_comp", "CVE-nonFixable_low-c2_ns1_dep0_img_comp", + "CVE-fixable_critical-c2_ns2_dep0_img_comp", "CVE-nonFixable_low-c2_ns2_dep0_img_comp", + }, collectColumn(dataRows, colCVE)) +} + +// TestGenerateReportTransaction_FixableOnlyFilter verifies that the fixable +// filter is applied through the transactional pipeline. +func (s *NewDataModelEnhancedReportingTestSuite) TestGenerateReportTransaction_FixableOnlyFilter() { + rg, blobStore, configStore := s.setupTransactionDeps() + snap := s.prepareSnapshot(configStore, "col1", + storage.VulnerabilityReportFilters_FIXABLE, allSeverities(), + []storage.VulnerabilityReportFilters_ImageType{storage.VulnerabilityReportFilters_DEPLOYED}) + + err := rg.generateReportTransaction(s.ctx, &ReportRequest{ + ReportSnapshot: snap, + Collection: testCollection("col1", "", "", ""), + DataStartTime: time.Time{}, + }) + s.Require().NoError(err) + + rows := s.readBlobCSV(blobStore, snap.GetReportConfigurationId(), snap.GetReportId()) + s.Require().Len(rows, 5, "1 header + 4 fixable rows") + + for _, row := range rows[1:] { + s.Equal("true", row[colFixable]) + s.Equal("CRITICAL", row[colSeverity]) + } +} + +// TestGenerateReportTransaction_DeployedAndWatched verifies that both deployed +// and watched image queries are executed within the same transaction. +// +// Test data: 4 deployed CVEs + 4 watched CVEs (2 watched images × 2 CVEs) = 8 data rows. +func (s *NewDataModelEnhancedReportingTestSuite) TestGenerateReportTransaction_DeployedAndWatched() { + rg, blobStore, configStore := s.setupTransactionDeps() + snap := s.prepareSnapshot(configStore, "col1", + storage.VulnerabilityReportFilters_BOTH, allSeverities(), + []storage.VulnerabilityReportFilters_ImageType{ + storage.VulnerabilityReportFilters_DEPLOYED, + storage.VulnerabilityReportFilters_WATCHED, + }) + + err := rg.generateReportTransaction(s.ctx, &ReportRequest{ + ReportSnapshot: snap, + Collection: testCollection("col1", "", "", ""), + DataStartTime: time.Time{}, + }) + s.Require().NoError(err) + + rows := s.readBlobCSV(blobStore, snap.GetReportConfigurationId(), snap.GetReportId()) + // 8 deployed (4 deps × 2 CVEs) + 4 watched (2 images × 2 CVEs) + 1 header = 13 + s.Require().Len(rows, 13, "1 header + 8 deployed + 4 watched") +} + +// TestGenerateReportTransaction_EmptyResult verifies that a query matching +// no data produces a valid ZIP with a CSV containing only the header row. +func (s *NewDataModelEnhancedReportingTestSuite) TestGenerateReportTransaction_EmptyResult() { + rg, blobStore, configStore := s.setupTransactionDeps() + snap := s.prepareSnapshot(configStore, "col1", + storage.VulnerabilityReportFilters_BOTH, allSeverities(), + []storage.VulnerabilityReportFilters_ImageType{storage.VulnerabilityReportFilters_DEPLOYED}) + + err := rg.generateReportTransaction(s.ctx, &ReportRequest{ + ReportSnapshot: snap, + Collection: testCollection("col1", "nonexistent-cluster", "", ""), + DataStartTime: time.Time{}, + }) + s.Require().NoError(err) + + rows := s.readBlobCSV(blobStore, snap.GetReportConfigurationId(), snap.GetReportId()) + s.Require().Len(rows, 1, "only header row for empty result") + s.Equal(formatCol(), rows[0]) +} + +// TestGenerateReportTransaction_CSVFieldValues spot-checks individual CSV field +// values for a single known row to verify end-to-end correctness. +func (s *NewDataModelEnhancedReportingTestSuite) TestGenerateReportTransaction_CSVFieldValues() { + rg, blobStore, configStore := s.setupTransactionDeps() + snap := s.prepareSnapshot(configStore, "col1", + storage.VulnerabilityReportFilters_FIXABLE, + []storage.VulnerabilitySeverity{storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY}, + []storage.VulnerabilityReportFilters_ImageType{storage.VulnerabilityReportFilters_DEPLOYED}) + + err := rg.generateReportTransaction(s.ctx, &ReportRequest{ + ReportSnapshot: snap, + Collection: testCollection("col1", "c1", "ns1", "c1_ns1_dep0"), + DataStartTime: time.Time{}, + }) + s.Require().NoError(err) + + rows := s.readBlobCSV(blobStore, snap.GetReportConfigurationId(), snap.GetReportId()) + s.Require().Len(rows, 2, "1 header + 1 data row") + + row := rows[1] + s.Equal("c1", row[colCluster]) + s.Equal("ns1", row[colNamespace]) + s.Equal("c1_ns1_dep0", row[colDeployment]) + s.Equal("c1_ns1_dep0_img", row[colImage]) + s.Equal("c1_ns1_dep0_img_comp", row[colComponent]) + s.Equal("1.0", row[colCompVersion]) + s.Equal("CVE-fixable_critical-c1_ns1_dep0_img_comp", row[colCVE]) + s.Equal("true", row[colFixable]) + s.Equal("1.1", row[colFixedBy]) + s.Equal("CRITICAL", row[colSeverity]) + s.Equal("9.00", row[colCVSS]) + s.Equal("10.00", row[colNVDCVSS]) + s.Equal("70.000", row[colEPSS]) + s.NotEqual("Not Available", row[colOf("Discovered At")]) + s.Equal("RHSA-2025-CVE-fixable", row[colOf("Advisory Name")]) + s.Equal("test-rhsa-link", row[colOf("Advisory Link")]) +} + +// TestGenerateReportTransaction_ReportStatusUpdated verifies that the report +// snapshot status is set to GENERATED after a successful run. +func (s *NewDataModelEnhancedReportingTestSuite) TestGenerateReportTransaction_ReportStatusUpdated() { + rg, _, configStore := s.setupTransactionDeps() + snap := s.prepareSnapshot(configStore, "col1", + storage.VulnerabilityReportFilters_BOTH, allSeverities(), + []storage.VulnerabilityReportFilters_ImageType{storage.VulnerabilityReportFilters_DEPLOYED}) + + err := rg.generateReportTransaction(s.ctx, &ReportRequest{ + ReportSnapshot: snap, + Collection: testCollection("col1", "", "", ""), + DataStartTime: time.Time{}, + }) + s.Require().NoError(err) + s.Equal(storage.ReportStatus_GENERATED, snap.GetReportStatus().GetRunState()) + s.NotNil(snap.GetReportStatus().GetCompletedAt()) +} diff --git a/central/reports/scheduler/v2/reportgenerator/report_gen_view_based_test.go b/central/reports/scheduler/v2/reportgenerator/report_gen_view_based_test.go index 9be4cb8ccee02..ffee70a669748 100644 --- a/central/reports/scheduler/v2/reportgenerator/report_gen_view_based_test.go +++ b/central/reports/scheduler/v2/reportgenerator/report_gen_view_based_test.go @@ -16,6 +16,8 @@ import ( imagesView "github.com/stackrox/rox/central/views/images" watchedImageDS "github.com/stackrox/rox/central/watchedimage/datastore" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres/pgtest" postgresSchema "github.com/stackrox/rox/pkg/postgres/schema" "github.com/stackrox/rox/pkg/sac" @@ -59,16 +61,28 @@ func (s *ViewBasedReportingTestSuite) SetupSuite() { func (s *ViewBasedReportingTestSuite) SetupTest() { s.mockCtrl = gomock.NewController(s.T()) - // Create data stores based on feature flag - imageDataStore := resolvers.CreateTestImageV2Datastore(s.T(), s.testDB, s.mockCtrl) - s.resolver, _ = resolvers.SetupTestResolver(s.T(), - imagesView.NewImageView(s.testDB.DB), - imageDataStore, - resolvers.CreateTestImageComponentV2Datastore(s.T(), s.testDB, s.mockCtrl), - resolvers.CreateTestImageCVEV2Datastore(s.T(), s.testDB), - resolvers.CreateTestDeploymentDatastore(s.T(), s.testDB, s.mockCtrl, imageDataStore), - deploymentsView.NewDeploymentView(s.testDB.DB), - ) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + imgV2DataStore := resolvers.CreateTestImageV2Datastore(s.T(), s.testDB, s.mockCtrl) + s.resolver, _ = resolvers.SetupTestResolver(s.T(), + imagesView.NewImageView(s.testDB.DB), + imgV2DataStore, + resolvers.CreateTestImageComponentV2Datastore(s.T(), s.testDB, s.mockCtrl), + resolvers.CreateTestImageCVEV2Datastore(s.T(), s.testDB), + resolvers.CreateTestDeploymentDatastoreWithImageV2(s.T(), s.testDB, s.mockCtrl, imgV2DataStore), + deploymentsView.NewDeploymentView(s.testDB.DB), + ) + } else { + imageDataStore := resolvers.CreateTestImageDatastore(s.T(), s.testDB, s.mockCtrl) + s.resolver, _ = resolvers.SetupTestResolver(s.T(), + imagesView.NewImageView(s.testDB.DB), + imageDataStore, + resolvers.CreateTestImageComponentV2Datastore(s.T(), s.testDB, s.mockCtrl), + resolvers.CreateTestImageCVEV2Datastore(s.T(), s.testDB), + resolvers.CreateTestDeploymentDatastore(s.T(), s.testDB, s.mockCtrl, imageDataStore), + deploymentsView.NewDeploymentView(s.testDB.DB), + ) + } s.watchedImageDatastore = watchedImageDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) s.clusterDatastore = clusterDSMocks.NewMockDataStore(s.mockCtrl) @@ -77,7 +91,7 @@ func (s *ViewBasedReportingTestSuite) SetupTest() { s.reportGenerator = newReportGeneratorImpl(s.testDB, nil, s.resolver.DeploymentDataStore, s.watchedImageDatastore, nil, nil, s.blobStore, s.clusterDatastore, - s.namespaceDatastore, s.resolver.ImageCVEV2DataStore, nil) + s.namespaceDatastore, s.resolver.ImageCVEV2DataStore) } func (s *ViewBasedReportingTestSuite) TearDownTest() { @@ -278,15 +292,13 @@ func (s *ViewBasedReportingTestSuite) TestGetReportDataViewBased() { query: "Image Registry:docker.io", expected: &viewBasedReportData{ deploymentNames: []string{"c1_ns1_dep0", "c1_ns2_dep0", "c2_ns1_dep0", "c2_ns2_dep0"}, - imageNames: []string{"c1_ns1_dep0_img", "c1_ns2_dep0_img", "c2_ns1_dep0_img", "c2_ns2_dep0_img", "w0_img", "w1_img"}, - componentNames: []string{"c1_ns1_dep0_img_comp", "c1_ns2_dep0_img_comp", "c2_ns1_dep0_img_comp", "c2_ns2_dep0_img_comp", "w0_img_comp", "w1_img_comp"}, + imageNames: []string{"c1_ns1_dep0_img", "c1_ns2_dep0_img", "c2_ns1_dep0_img", "c2_ns2_dep0_img"}, + componentNames: []string{"c1_ns1_dep0_img_comp", "c1_ns2_dep0_img_comp", "c2_ns1_dep0_img_comp", "c2_ns2_dep0_img_comp"}, cveNames: []string{ "CVE-fixable_critical-c1_ns1_dep0_img_comp", "CVE-nonFixable_low-c1_ns1_dep0_img_comp", "CVE-fixable_critical-c1_ns2_dep0_img_comp", "CVE-nonFixable_low-c1_ns2_dep0_img_comp", "CVE-fixable_critical-c2_ns1_dep0_img_comp", "CVE-nonFixable_low-c2_ns1_dep0_img_comp", "CVE-fixable_critical-c2_ns2_dep0_img_comp", "CVE-nonFixable_low-c2_ns2_dep0_img_comp", - "CVE-fixable_critical-w0_img_comp", "CVE-nonFixable_low-w0_img_comp", - "CVE-fixable_critical-w1_img_comp", "CVE-nonFixable_low-w1_img_comp", }, }, }, @@ -451,7 +463,7 @@ func (s *ViewBasedReportingTestSuite) TestGetReportDataViewBased() { log.Infof("Running test %s", tc.name) reportSnap := testViewBasedReportSnapshot(tc.query, nil) // Test get data using view-based approach - reportData, err := s.reportGenerator.getReportDataViewBased(reportSnap) + reportData, err := s.reportGenerator.getReportDataViewBased(s.ctx, reportSnap) s.NoError(err) collected := s.collectViewBasedReportData(reportData.CVEResponses) s.ElementsMatch(tc.expected.deploymentNames, collected.deploymentNames) @@ -499,7 +511,7 @@ func (s *ViewBasedReportingTestSuite) TestGetReportDataViewBasedWithInvalidQuery for _, tc := range testCases { s.T().Run(tc.name, func(t *testing.T) { reportSnap := testViewBasedReportSnapshot(tc.query, nil) - reportData, err := s.reportGenerator.getReportDataViewBased(reportSnap) + reportData, err := s.reportGenerator.getReportDataViewBased(s.ctx, reportSnap) if tc.expectErr { s.Error(err) } else { @@ -570,7 +582,7 @@ func (s *ViewBasedReportingTestSuite) TestGetReportDataViewBasedAccessScope() { s.T().Run(tc.name, func(t *testing.T) { reportSnap := testViewBasedReportSnapshot(tc.query, tc.scopeRules) // Test get data using view-based approach - reportData, err := s.reportGenerator.getReportDataViewBased(reportSnap) + reportData, err := s.reportGenerator.getReportDataViewBased(s.ctx, reportSnap) s.NoError(err) collected := s.collectViewBasedReportData(reportData.CVEResponses) s.ElementsMatch(tc.expected.deploymentNames, collected.deploymentNames) @@ -584,6 +596,30 @@ func (s *ViewBasedReportingTestSuite) TestGetReportDataViewBasedAccessScope() { } +func (s *ViewBasedReportingTestSuite) TestGetReportDataViewBasedWithCancelledContext() { + clusters := []*storage.Cluster{ + {Id: uuid.NewV4().String(), Name: "c1"}, + } + namespaces := testNamespaces(clusters, 1) + deployments, images := testDeploymentsWithImages(namespaces, 1) + s.upsertManyImages(images) + s.upsertManyDeployments(deployments) + + s.clusterDatastore.EXPECT().GetClusters(gomock.Any()). + Return(clusters, nil).AnyTimes() + s.namespaceDatastore.EXPECT().GetAllNamespaces(gomock.Any()). + Return(namespaces, nil).AnyTimes() + + reportSnap := testViewBasedReportSnapshot("", nil) + + ctx, cancel := context.WithCancel(s.ctx) + cancel() + + _, err := s.reportGenerator.getReportDataViewBased(ctx, reportSnap) + s.Error(err, "Expected error when context is cancelled before DB query") + s.ErrorIs(err, context.Canceled) +} + // Helper functions func (s *ViewBasedReportingTestSuite) truncateTable(name string) { sql := fmt.Sprintf("TRUNCATE %s CASCADE", name) @@ -592,9 +628,17 @@ func (s *ViewBasedReportingTestSuite) truncateTable(name string) { } func (s *ViewBasedReportingTestSuite) upsertManyImages(images []*storage.Image) { - for _, img := range images { - err := s.resolver.ImageDataStore.UpsertImage(s.ctx, img) - s.NoError(err) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + for _, img := range images { + err := s.resolver.ImageV2DataStore.UpsertImage(s.ctx, imageUtils.ConvertToV2(img)) + s.NoError(err) + } + } else { + for _, img := range images { + err := s.resolver.ImageDataStore.UpsertImage(s.ctx, img) + s.NoError(err) + } } } @@ -616,8 +660,8 @@ func (s *ViewBasedReportingTestSuite) collectViewBasedReportData(cveResponses [] deploymentNames := set.NewStringSet() imageNames := set.NewStringSet() componentNames := set.NewStringSet() - cveNames := make([]string, 0) - cvss := make([]float64, 0) + cveNames := make([]string, 0, len(cveResponses)) + cvss := make([]float64, 0, len(cveResponses)) for _, res := range cveResponses { if res.GetDeployment() != "" { diff --git a/central/reports/scheduler/v2/reportgenerator/singleton.go b/central/reports/scheduler/v2/reportgenerator/singleton.go index 8584149ead221..3bb242c54d343 100644 --- a/central/reports/scheduler/v2/reportgenerator/singleton.go +++ b/central/reports/scheduler/v2/reportgenerator/singleton.go @@ -1,20 +1,17 @@ package reportgenerator import ( - "github.com/graph-gophers/graphql-go" blobDS "github.com/stackrox/rox/central/blob/datastore" clusterDS "github.com/stackrox/rox/central/cluster/datastore" imageCVE2DS "github.com/stackrox/rox/central/cve/image/v2/datastore" deploymentDS "github.com/stackrox/rox/central/deployment/datastore" "github.com/stackrox/rox/central/globaldb" - "github.com/stackrox/rox/central/graphql/resolvers" namespaceDS "github.com/stackrox/rox/central/namespace/datastore" notifierProcessor "github.com/stackrox/rox/central/notifier/processor" reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" watchedImageDS "github.com/stackrox/rox/central/watchedimage/datastore" "github.com/stackrox/rox/pkg/sync" - "github.com/stackrox/rox/pkg/utils" ) var ( @@ -24,8 +21,6 @@ var ( func initialize() { _, collectionQueryRes := collectionDS.Singleton() - schema, err := graphql.ParseSchema(resolvers.Schema(), resolvers.New()) - utils.CrashOnError(err) rg = New(globaldb.GetPostgres(), reportSnapshotDS.Singleton(), deploymentDS.Singleton(), @@ -36,7 +31,6 @@ func initialize() { clusterDS.Singleton(), namespaceDS.Singleton(), imageCVE2DS.Singleton(), - schema, ) } diff --git a/central/reports/scheduler/v2/reportgenerator/testutils.go b/central/reports/scheduler/v2/reportgenerator/testutils.go index d875bac92dfdb..b000149800fc2 100644 --- a/central/reports/scheduler/v2/reportgenerator/testutils.go +++ b/central/reports/scheduler/v2/reportgenerator/testutils.go @@ -16,7 +16,7 @@ import ( func testNamespaces(clusters []*storage.Cluster, namespacesPerCluster int) []*storage.NamespaceMetadata { namespaces := make([]*storage.NamespaceMetadata, 0) for _, cluster := range clusters { - for i := 0; i < namespacesPerCluster; i++ { + for i := range namespacesPerCluster { namespaceName := fmt.Sprintf("ns%d", i+1) namespaces = append(namespaces, &storage.NamespaceMetadata{ Id: uuid.NewV4().String(), @@ -44,9 +44,9 @@ func testDeploymentsWithImages(namespaces []*storage.NamespaceMetadata, numDeplo images := make([]*storage.Image, 0, capacity) for _, namespace := range namespaces { - for i := 0; i < numDeploymentsPerNamespace; i++ { + for i := range numDeploymentsPerNamespace { depName := fmt.Sprintf("%s_%s_dep%d", namespace.GetClusterName(), namespace.GetName(), i) - image := testImage(depName) + image := testImage(depName, map[string]string{"app": "test"}, "docker.io") deployment := testDeployment(depName, namespace, image) deployments = append(deployments, deployment) images = append(images, image) @@ -74,15 +74,15 @@ func testDeployment(deploymentName string, namespace *storage.NamespaceMetadata, func testWatchedImages(numImages int) []*storage.Image { images := make([]*storage.Image, 0, numImages) - for i := 0; i < numImages; i++ { + for i := range numImages { imgNamePrefix := fmt.Sprintf("w%d", i) - image := testImage(imgNamePrefix) + image := testImage(imgNamePrefix, map[string]string{"app": "watch"}, "quay.io") images = append(images, image) } return images } -func testImage(prefix string) *storage.Image { +func testImage(prefix string, labels map[string]string, registry string) *storage.Image { t, err := protocompat.ConvertTimeToTimestampOrError(time.Unix(0, 1000)) utils.CrashOnError(err) nvdCvss := &storage.CVSSScore{ @@ -97,10 +97,15 @@ func testImage(prefix string) *storage.Image { Id: fmt.Sprintf("%s_img", prefix), Name: &storage.ImageName{ FullName: fmt.Sprintf("%s_img", prefix), - Registry: "docker.io", + Registry: registry, Remote: fmt.Sprintf("library/%s_img", prefix), Tag: "latest", }, + Metadata: &storage.ImageMetadata{ + V1: &storage.V1Metadata{ + Labels: labels, + }, + }, SetComponents: &storage.Image_Components{ Components: 1, }, @@ -277,3 +282,24 @@ func testViewBasedReportSnapshot(query string, scopeRules []*storage.SimpleAcces } return snap } + +func testEntityScopeReportSnapshot(entityScope *storage.EntityScope, query string, + imageTypes []storage.VulnerabilityReportFilters_ImageType, + scopeRules []*storage.SimpleAccessScope_Rules) *storage.ReportSnapshot { + snap := fixtures.GetReportSnapshot() + filters := &storage.VulnerabilityReportFilters{ + ImageTypes: imageTypes, + AccessScopeRules: scopeRules, + Query: query, + } + snap.Filter = &storage.ReportSnapshot_VulnReportFilters{ + VulnReportFilters: filters, + } + snap.ResourceScope = &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: entityScope, + }, + } + snap.Collection = nil + return snap +} diff --git a/central/reports/scheduler/v2/reportgenerator/types.go b/central/reports/scheduler/v2/reportgenerator/types.go index ca2b13af5b5c9..2a1c5b2493859 100644 --- a/central/reports/scheduler/v2/reportgenerator/types.go +++ b/central/reports/scheduler/v2/reportgenerator/types.go @@ -3,11 +3,15 @@ package reportgenerator import ( "time" + "github.com/pkg/errors" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/postgres/walker" ) +// ErrUserCancelled is the cause attached to a context when a user cancels a running report. +var ErrUserCancelled = errors.New("report cancelled by user") + // ReportRequest contains information needed to generate and notify a report type ReportRequest struct { ReportSnapshot *storage.ReportSnapshot @@ -27,22 +31,27 @@ type reportEmailSubjectFormat struct { // ImageCVEQueryResponse contains the fields of report query response type ImageCVEQueryResponse struct { - Cluster *string `db:"cluster"` - Namespace *string `db:"namespace"` - Deployment *string `db:"deployment"` - Image *string `db:"image"` - Component *string `db:"component"` - CVEID *string `db:"cve_id"` - CVE *string `db:"cve"` - Fixable *bool `db:"fixable"` - FixedByVersion *string `db:"fixed_by"` - Severity *storage.VulnerabilitySeverity `db:"severity"` - CVSS *float64 `db:"cvss"` - NVDCVSS *float64 `db:"nvd_cvss"` - EPSSProbability *float64 `db:"epss_probability"` - DiscoveredAtImage *time.Time `db:"first_image_occurrence_timestamp"` - AdvisoryName *string `db:"advisory_name"` - AdvisoryLink *string `db:"advisory_link"` + Cluster *string `db:"cluster"` + Namespace *string `db:"namespace"` + Deployment *string `db:"deployment"` + Image *string `db:"image"` + Component *string `db:"component"` + ComponentVersion *string `db:"component_version"` + CVEID *string `db:"cve_id"` + CVE *string `db:"cve"` + Fixable *bool `db:"fixable"` + FixedByVersion *string `db:"fixed_by"` + Severity *storage.VulnerabilitySeverity `db:"severity"` + CVSS *float64 `db:"cvss"` + NVDCVSS *float64 `db:"nvd_cvss"` + EPSSProbability *float64 `db:"epss_probability"` + CisaKev *bool `db:"cisa_kev"` + KnownRansomwareCampaign *bool `db:"known_ransomware_campaign"` + DiscoveredAtImage *time.Time `db:"first_image_occurrence_timestamp"` + ImageCreatedAt *time.Time `db:"image_created_time"` + AdvisoryName *string `db:"advisory_name"` + AdvisoryLink *string `db:"advisory_link"` + Origin *storage.VulnOrigin `db:"cve_origin"` Link string } @@ -82,6 +91,13 @@ func (res *ImageCVEQueryResponse) GetComponent() string { return *res.Component } +func (res *ImageCVEQueryResponse) GetComponentVersion() string { + if res.ComponentVersion == nil { + return "" + } + return *res.ComponentVersion +} + func (res *ImageCVEQueryResponse) GetCVEID() string { if res.CVEID == nil { return "" @@ -124,6 +140,13 @@ func (res *ImageCVEQueryResponse) GetCVSS() float64 { return *res.CVSS } +func (res *ImageCVEQueryResponse) GetOrigin() storage.VulnOrigin { + if res.Origin == nil { + return storage.VulnOrigin_VULN_ORIGIN_OTHER + } + return *res.Origin +} + func (res *ImageCVEQueryResponse) GetNVDCVSS() float64 { if res.NVDCVSS == nil { return 0.0 @@ -135,6 +158,14 @@ func (res *ImageCVEQueryResponse) GetEPSSProbability() *float64 { return res.EPSSProbability } +func (res *ImageCVEQueryResponse) GetCisaKev() *bool { + return res.CisaKev +} + +func (res *ImageCVEQueryResponse) GetKnownRansomwareCampaign() *bool { + return res.KnownRansomwareCampaign +} + func (res *ImageCVEQueryResponse) GetAdvisoryName() string { if res.AdvisoryName == nil { return "Not Available" @@ -156,6 +187,13 @@ func (res *ImageCVEQueryResponse) GetDiscoveredAtImage() string { return res.DiscoveredAtImage.Format("January 02, 2006") } +func (res *ImageCVEQueryResponse) GetImageCreatedAt() string { + if res.ImageCreatedAt == nil { + return "Not Available" + } + return res.ImageCreatedAt.Format("2006-01-02") +} + // ReportQueryParts contains the parts used to build the report query type ReportQueryParts struct { Schema *walker.Schema diff --git a/central/reports/scheduler/v2/reportgenerator/validate.go b/central/reports/scheduler/v2/reportgenerator/validate.go index 0340c100ab978..bec14d35abd0c 100644 --- a/central/reports/scheduler/v2/reportgenerator/validate.go +++ b/central/reports/scheduler/v2/reportgenerator/validate.go @@ -2,6 +2,7 @@ package reportgenerator import ( "github.com/pkg/errors" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/errorhelpers" ) @@ -19,9 +20,14 @@ func ValidateReportRequest(request *ReportRequest) error { } else if request.ReportSnapshot.GetReportStatus() == nil { errorList.AddError(errors.New("Report request does not have a valid report snapshot with report status")) } - // only check collection is non nil if report snapshot is for config based vuln reports - if request.ReportSnapshot.GetVulnReportFilters() != nil && request.Collection == nil { - errorList.AddError(errors.New("Report request does not have a valid non-nil collection.")) + // only check resource scope is non nil if report snapshot is for config based reports + if request.ReportSnapshot.GetVulnReportFilters() != nil && request.ReportSnapshot.GetResourceScope() == nil { + errorList.AddError(errors.New("Report request does not have a valid non-nil resource scope.")) + } + if request.ReportSnapshot.GetNodeVulnReportFilters() != nil && + request.ReportSnapshot.GetReportStatus().GetReportRequestType() != storage.ReportStatus_VIEW_BASED && + request.ReportSnapshot.GetResourceScope() == nil { + errorList.AddError(errors.New("Report request does not have a valid non-nil resource scope.")) } return errorList.ToError() diff --git a/central/reports/scheduler/v2/reportqueue/report_queue.go b/central/reports/scheduler/v2/reportqueue/report_queue.go new file mode 100644 index 0000000000000..e94cebe26ba1e --- /dev/null +++ b/central/reports/scheduler/v2/reportqueue/report_queue.go @@ -0,0 +1,114 @@ +package reportqueue + +import ( + "context" + + reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/queue" + "github.com/stackrox/rox/pkg/set" + "github.com/stackrox/rox/pkg/sync" +) + +// ReportQueue is a thread-safe FIFO queue of report requests. It tracks which +// report configurations are currently running so that at most one report per +// config executes at a time, and stores cancel functions for running reports +// to support user-initiated cancellation. +type ReportQueue struct { + mu sync.Mutex + queue *queue.Queue[*reportGen.ReportRequest] + runningConfigs set.StringSet + jobIDToCancelFunc map[string]context.CancelCauseFunc +} + +// New creates a new empty ReportQueue. +func New() *ReportQueue { + return &ReportQueue{ + queue: queue.NewQueue[*reportGen.ReportRequest](), + runningConfigs: set.NewStringSet(), + jobIDToCancelFunc: make(map[string]context.CancelCauseFunc), + } +} + +// Enqueue adds a report request to the back of the queue. +// The underlying queue is internally synchronized, so no additional locking is needed. +func (q *ReportQueue) Enqueue(req *reportGen.ReportRequest) { + q.queue.Push(req) +} + +// Dequeue removes and returns the first runnable report request from the queue. +// VIEW_BASED reports are always runnable. Config-based reports are runnable only +// if their config ID is not in the running set. Returns nil if no runnable +// request is found. For non-VIEW_BASED reports, the config ID is added to the +// running set upon dequeue. +func (q *ReportQueue) Dequeue() *reportGen.ReportRequest { + q.mu.Lock() + defer q.mu.Unlock() + + req, ok := q.queue.PullWithPred(func(req *reportGen.ReportRequest) bool { + if req.ReportSnapshot.GetReportStatus().GetReportRequestType() == storage.ReportStatus_VIEW_BASED { + return true + } + return !q.runningConfigs.Contains(req.ReportSnapshot.GetReportConfigurationId()) + }) + if !ok { + return nil + } + if req.ReportSnapshot.GetReportStatus().GetReportRequestType() != storage.ReportStatus_VIEW_BASED { + q.runningConfigs.Add(req.ReportSnapshot.GetReportConfigurationId()) + } + return req +} + +// Remove removes a queued (not yet running) request by report snapshot ID. +// Returns the removed request, or nil if not found. +// The underlying queue is internally synchronized, so no additional locking is needed. +func (q *ReportQueue) Remove(reportID string) *reportGen.ReportRequest { + req, _ := q.queue.PullWithPred(func(req *reportGen.ReportRequest) bool { + return req.ReportSnapshot.GetReportId() == reportID + }) + return req +} + +// MarkReportDoneForConfig removes a config ID from the running set, allowing +// another report for the same config to be dequeued. +func (q *ReportQueue) MarkReportDoneForConfig(configID string) { + q.mu.Lock() + defer q.mu.Unlock() + q.runningConfigs.Remove(configID) +} + +// AddCancelFunc stores a cancel function for a running report, keyed by report ID. +func (q *ReportQueue) AddCancelFunc(reportID string, cancel context.CancelCauseFunc) { + q.mu.Lock() + defer q.mu.Unlock() + q.jobIDToCancelFunc[reportID] = cancel +} + +// RemoveCancelFunc removes the cancel function for the given report ID. +func (q *ReportQueue) RemoveCancelFunc(reportID string) { + q.mu.Lock() + defer q.mu.Unlock() + delete(q.jobIDToCancelFunc, reportID) +} + +// TryCancel calls the cancel function for the given report ID with +// ErrUserCancelled as the cause. Returns true if a cancel function was found +// and invoked. +func (q *ReportQueue) TryCancel(reportID string) bool { + cancel := concurrency.WithLock1(&q.mu, func() context.CancelCauseFunc { + return q.jobIDToCancelFunc[reportID] + }) + if cancel == nil { + return false + } + cancel(reportGen.ErrUserCancelled) + return true +} + +// Len returns the number of items in the queue. +// The underlying queue is internally synchronized, so no additional locking is needed. +func (q *ReportQueue) Len() int { + return q.queue.Len() +} diff --git a/central/reports/scheduler/v2/reportqueue/report_queue_test.go b/central/reports/scheduler/v2/reportqueue/report_queue_test.go new file mode 100644 index 0000000000000..73ead9e751a86 --- /dev/null +++ b/central/reports/scheduler/v2/reportqueue/report_queue_test.go @@ -0,0 +1,182 @@ +package reportqueue + +import ( + "context" + "testing" + + reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func makeRequest(reportID, configID string, requestType storage.ReportStatus_RunMethod) *reportGen.ReportRequest { + return &reportGen.ReportRequest{ + ReportSnapshot: &storage.ReportSnapshot{ + ReportId: reportID, + ReportConfigurationId: configID, + ReportStatus: &storage.ReportStatus{ + ReportRequestType: requestType, + }, + }, + } +} + +func TestEnqueueDequeueFIFO(t *testing.T) { + q := New() + + req1 := makeRequest("r1", "c1", storage.ReportStatus_ON_DEMAND) + req2 := makeRequest("r2", "c2", storage.ReportStatus_ON_DEMAND) + req3 := makeRequest("r3", "c3", storage.ReportStatus_SCHEDULED) + + q.Enqueue(req1) + q.Enqueue(req2) + q.Enqueue(req3) + assert.Equal(t, 3, q.Len()) + + got := q.Dequeue() + assert.Equal(t, "r1", got.ReportSnapshot.GetReportId()) + got = q.Dequeue() + assert.Equal(t, "r2", got.ReportSnapshot.GetReportId()) + got = q.Dequeue() + assert.Equal(t, "r3", got.ReportSnapshot.GetReportId()) + + assert.Nil(t, q.Dequeue()) + assert.Equal(t, 0, q.Len()) +} + +func TestDequeueSkipsRunningConfig(t *testing.T) { + q := New() + + req1 := makeRequest("r1", "c1", storage.ReportStatus_ON_DEMAND) + req2 := makeRequest("r2", "c1", storage.ReportStatus_ON_DEMAND) + req3 := makeRequest("r3", "c2", storage.ReportStatus_ON_DEMAND) + + q.Enqueue(req1) + q.Enqueue(req2) + q.Enqueue(req3) + + // Dequeue r1 → c1 is now in runningConfigs + got := q.Dequeue() + assert.Equal(t, "r1", got.ReportSnapshot.GetReportId()) + + // r2 is for c1 (running), should be skipped; r3 (c2) is returned + got = q.Dequeue() + assert.Equal(t, "r3", got.ReportSnapshot.GetReportId()) + + // r2 still blocked + assert.Nil(t, q.Dequeue()) + assert.Equal(t, 1, q.Len()) +} + +func TestDequeueViewBasedAlwaysRunnable(t *testing.T) { + q := New() + + req1 := makeRequest("r1", "c1", storage.ReportStatus_ON_DEMAND) + reqView := makeRequest("r-view", "", storage.ReportStatus_VIEW_BASED) + + q.Enqueue(req1) + q.Enqueue(reqView) + + // Dequeue r1 → c1 in runningConfigs + got := q.Dequeue() + assert.Equal(t, "r1", got.ReportSnapshot.GetReportId()) + + // VIEW_BASED always passes regardless of runningConfigs + got = q.Dequeue() + require.NotNil(t, got) + assert.Equal(t, "r-view", got.ReportSnapshot.GetReportId()) +} + +func TestMarkReportDoneForConfigUnblocks(t *testing.T) { + q := New() + + req1 := makeRequest("r1", "c1", storage.ReportStatus_ON_DEMAND) + req2 := makeRequest("r2", "c1", storage.ReportStatus_SCHEDULED) + + q.Enqueue(req1) + q.Enqueue(req2) + + got := q.Dequeue() + assert.Equal(t, "r1", got.ReportSnapshot.GetReportId()) + + // r2 blocked because c1 is running + assert.Nil(t, q.Dequeue()) + + q.MarkReportDoneForConfig("c1") + + // Now r2 should be dequeued + got = q.Dequeue() + require.NotNil(t, got) + assert.Equal(t, "r2", got.ReportSnapshot.GetReportId()) +} + +func TestRemoveFromQueue(t *testing.T) { + q := New() + + req1 := makeRequest("r1", "c1", storage.ReportStatus_ON_DEMAND) + req2 := makeRequest("r2", "c2", storage.ReportStatus_ON_DEMAND) + + q.Enqueue(req1) + q.Enqueue(req2) + + removed := q.Remove("r1") + require.NotNil(t, removed) + assert.Equal(t, "r1", removed.ReportSnapshot.GetReportId()) + assert.Equal(t, 1, q.Len()) + + // Removing non-existent returns nil + assert.Nil(t, q.Remove("nonexistent")) + assert.Equal(t, 1, q.Len()) +} + +func TestAddCancelFuncAndTryCancel(t *testing.T) { + q := New() + + var capturedCause error + ctx, cancel := context.WithCancelCause(context.Background()) + defer cancel(nil) + + q.AddCancelFunc("r1", cancel) + + cancelled := q.TryCancel("r1") + assert.True(t, cancelled) + + capturedCause = context.Cause(ctx) + assert.ErrorIs(t, capturedCause, reportGen.ErrUserCancelled) +} + +func TestTryCancelReturnsFalseForUnknown(t *testing.T) { + q := New() + assert.False(t, q.TryCancel("nonexistent")) +} + +func TestRemoveCancelFunc(t *testing.T) { + q := New() + + _, cancel := context.WithCancelCause(context.Background()) + + q.AddCancelFunc("r1", cancel) + q.RemoveCancelFunc("r1") + + // After removal, TryCancel should return false + assert.False(t, q.TryCancel("r1")) +} + +func TestViewBasedNotAddedToRunningConfigs(t *testing.T) { + q := New() + + reqView1 := makeRequest("r-view-1", "", storage.ReportStatus_VIEW_BASED) + reqView2 := makeRequest("r-view-2", "", storage.ReportStatus_VIEW_BASED) + + q.Enqueue(reqView1) + q.Enqueue(reqView2) + + got := q.Dequeue() + assert.Equal(t, "r-view-1", got.ReportSnapshot.GetReportId()) + + // Second VIEW_BASED should also dequeue without being blocked + got = q.Dequeue() + require.NotNil(t, got) + assert.Equal(t, "r-view-2", got.ReportSnapshot.GetReportId()) +} diff --git a/central/reports/scheduler/v2/scheduler.go b/central/reports/scheduler/v2/scheduler.go index 1e933155dfeeb..682a264747d6a 100644 --- a/central/reports/scheduler/v2/scheduler.go +++ b/central/reports/scheduler/v2/scheduler.go @@ -5,6 +5,7 @@ import ( reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/postgres" ) // Scheduler maintains the schedules for reports @@ -15,6 +16,8 @@ type Scheduler interface { UpsertReportSchedule(reportConfig *storage.ReportConfiguration) error // RemoveReportSchedule removes the given report configuration from scheduled execution. RemoveReportSchedule(reportConfigID string) + // GetScheduledConfigIDs returns the IDs of all report configurations that have active schedules. + GetScheduledConfigIDs() []string // CanSubmitReportRequest returns true if the given user can submit an on-demand report request for the given report configuration. CanSubmitReportRequest(user *storage.SlimUser, reportConfig *storage.ReportConfiguration) (bool, error) @@ -28,9 +31,10 @@ type Scheduler interface { // If the report is already being prepared or has completed execution, it cannot be cancelled. CancelReportRequest(ctx context.Context, reportID string) (bool, error) - // Start scheduler. A scheduler instance can only be started once. It cannot be re-started once stopped. - // This func will log errors if the scheduler fails to start. - Start() + // Start acquires a PostgreSQL advisory lock and starts the scheduler. + // If the lock is already held by another process, the scheduler is not started. + // A scheduler instance can only be started once and cannot be re-started once stopped. + Start(db postgres.DB) // Stop scheduler Stop() } diff --git a/central/reports/scheduler/v2/scheduler_impl.go b/central/reports/scheduler/v2/scheduler_impl.go index 99683cd88ba13..b418608d39d40 100644 --- a/central/reports/scheduler/v2/scheduler_impl.go +++ b/central/reports/scheduler/v2/scheduler_impl.go @@ -1,31 +1,31 @@ package v2 import ( - "container/list" "context" "sync/atomic" "time" - "github.com/graph-gophers/graphql-go" "github.com/pkg/errors" - "github.com/stackrox/rox/central/graphql/resolvers" notifierDS "github.com/stackrox/rox/central/notifier/datastore" "github.com/stackrox/rox/central/reports/common" reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" + "github.com/stackrox/rox/central/reports/scheduler/v2/reportqueue" reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" "github.com/stackrox/rox/central/reports/validation" collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/dblock" "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/protoconv/schedule" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/search" - "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/sync" "golang.org/x/sync/semaphore" "gopkg.in/robfig/cron.v2" @@ -37,6 +37,14 @@ var ( scheduledCtx = sac.WithAllAccess(context.Background()) ) +// queueGeneratorBinding pairs a report job queue with the generator that processes +// jobs from that queue. E.g., image CVE report job queue <-> image CVE report +// generator, node CVE report job queue <-> node CVE report generator. +type queueGeneratorBinding struct { + queue *reportqueue.ReportQueue + generator reportGen.ReportGenerator +} + type scheduler struct { // Used to map reportConfigs to their cron jobs. This is only used for scheduled reports, On-demand reports are directly added to reportsQueue reportConfigToEntryIDs map[string]cron.EntryID @@ -45,18 +53,19 @@ type scheduler struct { reportSnapshotStore reportSnapshotDS.DataStore collectionDatastore collectionDS.DataStore notifierDatastore notifierDS.DataStore - reportGenerator reportGen.ReportGenerator validator *validation.Validator - reportRequestsQueue *list.List + // Ordered list of queue-generator bindings. The main loop round-robins through + // these when selecting the next report to run. + queues []queueGeneratorBinding + nextQueueIdx int + // Maps report type to its queue for routing submissions. + queueByType map[storage.ReportSnapshot_ReportType]*reportqueue.ReportQueue - // Use to signal the scheduler to find and run a new report if a routine is available - // This signal is triggered when a new request is added to reportsQueue. It is also triggered when a report completes - // execution to inform the scheduler that a routine is free. The signal is reset when there is no report to run. + // Signals the scheduler to find and run a new report. Triggered when a new + // request is enqueued or when a report completes. Reset when no runnable report + // is found across any queue. readyForReports concurrency.Signal - // Stores config IDs for which a report is currently running. Used to make sure only one report per config runs at a time. - runningReportConfigs set.StringSet - Schema *graphql.Schema /* Concurrency and synchronization related fields */ // isStarted will make sure only one scheduling routine runs for an instance of scheduler @@ -64,53 +73,62 @@ type scheduler struct { // isStopped will prevent scheduler from being re-started once it is stopped isStopped atomic.Bool - /* Concurrency and synchronization related fields */ stopper concurrency.Stopper // Use to synchronize access to reportConfigToEntryIDs map cronJobsLock sync.Mutex - // Use to synchronize access to reportsQueue and runningReportConfigs - schedulerLock sync.Mutex // Use to lock any database tables if needed to prevent race conditions dbLock sync.Mutex // NOTE: Lock only one mutex at a time. Do not lock another mutex when one is already held. // If you need to lock another mutex, you must free the locked one first. - cron *cron.Cron - concurrencySema *semaphore.Weighted + cron *cron.Cron + concurrencySema *semaphore.Weighted + advisoryLockRelease func() } // New instantiates a new cron scheduler and supports adding and removing report requests func New(reportConfigDatastore reportConfigDS.DataStore, reportSnapshotStore reportSnapshotDS.DataStore, collectionDatastore collectionDS.DataStore, notifierDatastore notifierDS.DataStore, - reportGenerator reportGen.ReportGenerator, validator *validation.Validator) Scheduler { + imageReportGenerator reportGen.ReportGenerator, nodeReportGenerator reportGen.ReportGenerator, + validator *validation.Validator) Scheduler { cronScheduler := cron.New() cronScheduler.Start() - ourSchema, err := graphql.ParseSchema(resolvers.Schema(), resolvers.New()) - if err != nil { - panic(err) - } return newSchedulerImpl(reportConfigDatastore, reportSnapshotStore, collectionDatastore, notifierDatastore, - reportGenerator, validator, cronScheduler, ourSchema) + imageReportGenerator, nodeReportGenerator, validator, cronScheduler) } func newSchedulerImpl(reportConfigDatastore reportConfigDS.DataStore, reportSnapshotStore reportSnapshotDS.DataStore, collectionDatastore collectionDS.DataStore, notifierDatastore notifierDS.DataStore, - reportGenerator reportGen.ReportGenerator, validator *validation.Validator, cronScheduler *cron.Cron, - schema *graphql.Schema) *scheduler { + imageReportGenerator reportGen.ReportGenerator, nodeReportGenerator reportGen.ReportGenerator, + validator *validation.Validator, cronScheduler *cron.Cron) *scheduler { + + imageQueue := reportqueue.New() + queues := []queueGeneratorBinding{ + {queue: imageQueue, generator: imageReportGenerator}, + } + queueByType := map[storage.ReportSnapshot_ReportType]*reportqueue.ReportQueue{ + storage.ReportSnapshot_VULNERABILITY: imageQueue, + } + + if features.NodeVulnerabilityReports.Enabled() && nodeReportGenerator != nil { + nodeQueue := reportqueue.New() + queues = append(queues, queueGeneratorBinding{queue: nodeQueue, generator: nodeReportGenerator}) + queueByType[storage.ReportSnapshot_NODE_VULNERABILITY] = nodeQueue + } + s := &scheduler{ reportConfigToEntryIDs: make(map[string]cron.EntryID), reportConfigDatastore: reportConfigDatastore, reportSnapshotStore: reportSnapshotStore, collectionDatastore: collectionDatastore, notifierDatastore: notifierDatastore, - reportGenerator: reportGenerator, validator: validator, - reportRequestsQueue: list.New(), + queues: queues, + nextQueueIdx: 0, + queueByType: queueByType, readyForReports: concurrency.NewSignal(), - runningReportConfigs: set.NewStringSet(), - Schema: schema, stopper: concurrency.NewStopper(), cron: cronScheduler, concurrencySema: semaphore.NewWeighted(int64(env.ReportExecutionMaxConcurrency.IntegerSetting())), @@ -120,19 +138,33 @@ func newSchedulerImpl(reportConfigDatastore reportConfigDS.DataStore, reportSnap /* Concurrency and scheduling functions */ -// Start scheduler. A scheduler instance can only be started once. It cannot be re-started once stopped. -// This func will log errors if the scheduler fails to start. -func (s *scheduler) Start() { +// Start acquires a PostgreSQL advisory lock and starts the scheduler. +// If the lock is already held by another process, the scheduler is not started. +// A scheduler instance can only be started once and cannot be re-started once stopped. +func (s *scheduler) Start(db postgres.DB) { + acquired, release, err := dblock.TryAcquireAdvisoryLock(scheduledCtx, db, dblock.ReportSchedulerLockID) + if err != nil { + log.Errorf("Report scheduler: failed to acquire advisory lock: %v", err) + return + } + if !acquired { + log.Info("Report scheduler: advisory lock held by another process, not starting") + return + } if s.isStopped.Load() { + release() log.Error("Scheduler already stopped. It cannot be re-started once stopped.") return } swapped := s.isStarted.CompareAndSwap(false, true) if !swapped { + release() log.Error("Scheduler already running") return } + s.advisoryLockRelease = release s.queuePendingReports() + s.recoverMissedSchedules() s.queueScheduledReports() go s.runReports() } @@ -153,6 +185,9 @@ func (s *scheduler) Stop() { if err != nil { log.Errorf("Error stopping vulnerability report scheduler : %v", err) } + if s.advisoryLockRelease != nil { + s.advisoryLockRelease() + } } func (s *scheduler) runReports() { @@ -162,8 +197,8 @@ func (s *scheduler) runReports() { case <-s.stopper.Flow().StopRequested(): return case <-s.readyForReports.Done(): - reportRequest := s.selectNextRunnableReport() - if reportRequest == nil { + req, q, gen := s.selectNextJobRoundRobin() + if req == nil { s.readyForReports.Reset() continue } @@ -171,43 +206,37 @@ func (s *scheduler) runReports() { log.Errorf("Error acquiring semaphore to run new report: %v", err) continue } - log.Infof("Executing report '%s' at %v", reportRequest.ReportSnapshot.GetName(), time.Now().Format(time.RFC822)) - go s.runSingleReport(reportRequest) + log.Infof("Executing report '%s' at %v", req.ReportSnapshot.GetName(), time.Now().Format(time.RFC822)) + go s.runSingleReport(q, gen, req) } } } -func (s *scheduler) selectNextRunnableReport() *reportGen.ReportRequest { - s.schedulerLock.Lock() - defer s.schedulerLock.Unlock() - - request := findAndRemoveFromQueue(s.reportRequestsQueue, func(req *reportGen.ReportRequest) bool { - if req.ReportSnapshot.GetReportStatus().GetReportRequestType() == storage.ReportStatus_VIEW_BASED { - return true +func (s *scheduler) selectNextJobRoundRobin() (*reportGen.ReportRequest, *reportqueue.ReportQueue, reportGen.ReportGenerator) { + n := len(s.queues) + for i := range n { + idx := (s.nextQueueIdx + i) % n + qb := s.queues[idx] + if req := qb.queue.Dequeue(); req != nil { + s.nextQueueIdx = (idx + 1) % n + return req, qb.queue, qb.generator } - return !s.runningReportConfigs.Contains(req.ReportSnapshot.GetReportConfigurationId()) - }) - if request == nil { - return nil - } - if request.ReportSnapshot.GetVulnReportFilters() != nil { - s.runningReportConfigs.Add(request.ReportSnapshot.GetReportConfigurationId()) } - return request + return nil, nil, nil } -func (s *scheduler) runSingleReport(req *reportGen.ReportRequest) { +func (s *scheduler) runSingleReport(q *reportqueue.ReportQueue, gen reportGen.ReportGenerator, req *reportGen.ReportRequest) { defer s.readyForReports.Signal() defer s.concurrencySema.Release(1) - defer s.removeFromRunningReportConfigs(req.ReportSnapshot.GetReportConfigurationId()) + defer q.MarkReportDoneForConfig(req.ReportSnapshot.GetReportConfigurationId()) - s.reportGenerator.ProcessReportRequest(req) -} + reportID := req.ReportSnapshot.GetReportId() + ctx, cancel := context.WithCancelCause(context.Background()) + q.AddCancelFunc(reportID, cancel) + defer cancel(nil) + defer q.RemoveCancelFunc(reportID) -func (s *scheduler) removeFromRunningReportConfigs(configID string) { - s.schedulerLock.Lock() - defer s.schedulerLock.Unlock() - s.runningReportConfigs.Remove(configID) + gen.ProcessReportRequest(ctx, req) } // UpsertReportSchedule adds/updates the schedule at which reports for the given report config are executed. @@ -245,35 +274,47 @@ func (s *scheduler) RemoveReportSchedule(reportConfigID string) { } } -/* Functions to add/remove report jobs from queue */ +func (s *scheduler) GetScheduledConfigIDs() []string { + s.cronJobsLock.Lock() + defer s.cronJobsLock.Unlock() -// CancelReportRequest cancels a report request that is still waiting in queue. A user can only cancel a report requested by them. -// If the report is already being prepared or has completed execution, it cannot be cancelled. -func (s *scheduler) CancelReportRequest(ctx context.Context, reportID string) (bool, error) { - removed := s.tryRemoveFromRequestQueue(reportID) - if !removed { - return false, nil - } - err := s.reportSnapshotStore.DeleteReportSnapshot(ctx, reportID) - if err != nil { - return false, errors.Wrapf(err, "Error deleting report ID '%s' from storage", reportID) + ids := make([]string, 0, len(s.reportConfigToEntryIDs)) + for id := range s.reportConfigToEntryIDs { + ids = append(ids, id) } - return true, nil + return ids } -// Returns true if the request was successfully removed from the ReportRequests queue -func (s *scheduler) tryRemoveFromRequestQueue(reportID string) bool { - s.schedulerLock.Lock() - defer s.schedulerLock.Unlock() +/* Functions to add/remove report jobs from queue */ - request := findAndRemoveFromQueue(s.reportRequestsQueue, func(req *reportGen.ReportRequest) bool { - return req.ReportSnapshot.GetReportId() == reportID - }) - return request != nil +// CancelReportRequest cancels a report request. If the report is waiting in queue, it is removed +// and its snapshot is updated to FAILURE with a cancellation message. If the report is already +// being prepared, its context is cancelled, which propagates cancellation to in-flight database +// queries and blob store writes. +func (s *scheduler) CancelReportRequest(ctx context.Context, reportID string) (bool, error) { + for _, qb := range s.queues { + req := qb.queue.Remove(reportID) + if req != nil { + req.ReportSnapshot.ReportStatus.ErrorMsg = reportGen.ErrUserCancelled.Error() + req.ReportSnapshot.ReportStatus.CompletedAt = protocompat.TimestampNow() + req.ReportSnapshot.ReportStatus.RunState = storage.ReportStatus_FAILURE + if err := s.reportSnapshotStore.UpdateReportSnapshot(ctx, req.ReportSnapshot); err != nil { + return false, errors.Wrapf(err, "Error updating report snapshot to FAILURE for report ID '%s'", reportID) + } + return true, nil + } + } + for _, qb := range s.queues { + if qb.queue.TryCancel(reportID) { + return true, nil + } + } + return false, nil } func (s *scheduler) CanSubmitReportRequest(user *storage.SlimUser, reportConfig *storage.ReportConfiguration) (bool, error) { - return s.doesUserHavePendingReport(reportConfig.GetId(), user.GetId()) + return s.doesUserHavePendingReport(reportConfig.GetId(), user.GetId(), + storage.ReportSnapshot_ReportType(reportConfig.GetType())) } // SubmitReportRequest submits a report execution request. The report request can be either for an on demand report or a scheduled report. @@ -285,6 +326,11 @@ func (s *scheduler) SubmitReportRequest(ctx context.Context, request *reportGen. return "", err } + q := s.queueForSnapshot(request.ReportSnapshot) + if q == nil { + return "", errors.New("node vulnerability reports are not enabled") + } + request.ReportSnapshot.ReportStatus.RunState = storage.ReportStatus_WAITING request.ReportSnapshot.ReportStatus.QueuedAt = protocompat.TimestampNow() request.ReportSnapshot.ReportId, err = s.validateAndPersistSnapshot(ctx, request.ReportSnapshot, reSubmission) @@ -292,16 +338,16 @@ func (s *scheduler) SubmitReportRequest(ctx context.Context, request *reportGen. return "", err } - s.appendToReportsQueue(request) + if s.isStarted.Load() { + q.Enqueue(request) + s.readyForReports.Signal() + } return request.ReportSnapshot.GetReportId(), nil } -func (s *scheduler) appendToReportsQueue(req *reportGen.ReportRequest) { - s.schedulerLock.Lock() - defer s.schedulerLock.Unlock() - s.reportRequestsQueue.PushBack(req) - s.readyForReports.Signal() +func (s *scheduler) queueForSnapshot(snap *storage.ReportSnapshot) *reportqueue.ReportQueue { + return s.queueByType[snap.GetType()] } func (s *scheduler) reportClosure(reportConfig *storage.ReportConfiguration) func() { @@ -331,6 +377,18 @@ func (s *scheduler) queuePendingReports() { } for _, snap := range pendingReports { + // View-based reports have no associated report configuration, resource scope, or collection. + if snap.GetReportStatus().GetReportRequestType() == storage.ReportStatus_VIEW_BASED { + repRequest := &reportGen.ReportRequest{ + ReportSnapshot: snap, + } + _, err = s.SubmitReportRequest(scheduledCtx, repRequest, true) + if err != nil { + log.Errorf("Error rescheduling pending view-based report job '%s': %s", snap.GetReportId(), err) + } + continue + } + _, found, err := s.reportConfigDatastore.GetReportConfiguration(scheduledCtx, snap.GetReportConfigurationId()) if err != nil { log.Errorf("Error rescheduling pending report job for report config ID '%s': %s", snap.GetReportConfigurationId(), err) @@ -342,19 +400,30 @@ func (s *scheduler) queuePendingReports() { continue } - collection, found, err := s.collectionDatastore.Get(scheduledCtx, snap.GetCollection().GetId()) - if err != nil { - log.Errorf("Error finding collection ID '%s': %s", snap.GetCollection().GetId(), err) + if !common.HasValidResourceScope(snap.GetResourceScope()) { + log.Errorf("Report configuration '%s' has an empty resource scope (no collection ID or entity scope)", snap.GetReportConfigurationId()) continue } - if !found { - log.Errorf("Collection ID '%s' not found", snap.GetCollection().GetId()) - } - _, err = s.SubmitReportRequest(scheduledCtx, &reportGen.ReportRequest{ + repRequest := &reportGen.ReportRequest{ ReportSnapshot: snap, - Collection: collection, - }, true) + } + + if snap.GetCollection() != nil { + collection, found, err := s.collectionDatastore.Get(scheduledCtx, snap.GetCollection().GetId()) + if err != nil { + log.Errorf("Error finding collection ID '%s': %s", snap.GetCollection().GetId(), err) + continue + } + if !found { + log.Errorf("Collection ID '%s' not found", snap.GetCollection().GetId()) + } + + repRequest.Collection = collection + + } + + _, err = s.SubmitReportRequest(scheduledCtx, repRequest, true) if err != nil { log.Errorf("Error rescheduling pending report job for report config ID '%s': %s", snap.GetReportConfigurationId(), err) } @@ -362,16 +431,24 @@ func (s *scheduler) queuePendingReports() { } func (s *scheduler) queueScheduledReports() { + reportTypes := []string{storage.ReportConfiguration_VULNERABILITY.String()} + if features.NodeVulnerabilityReports.Enabled() { + reportTypes = append(reportTypes, storage.ReportConfiguration_NODE_VULNERABILITY.String()) + } query := search.NewQueryBuilder(). - AddExactMatches(search.ReportType, storage.ReportConfiguration_VULNERABILITY.String()). + AddExactMatches(search.ReportType, reportTypes...). ProtoQuery() - filteredQ := common.WithoutV1ReportConfigs(query) - reportConfigs, err := s.reportConfigDatastore.GetReportConfigurations(scheduledCtx, filteredQ) + reportConfigs, err := s.reportConfigDatastore.GetReportConfigurations(scheduledCtx, query) if err != nil { log.Errorf("Error finding scheduled reports: %s", err) return } for _, rc := range reportConfigs { + if !common.HasValidResourceScope(rc.GetResourceScope()) { + log.Errorf("Skipping scheduled report for config '%s' (ID: %s): resource scope is empty", + rc.GetName(), rc.GetId()) + continue + } if rc.GetSchedule() != nil { if err := s.UpsertReportSchedule(rc); err != nil { log.Errorf("Error queuing scheduled report for report configuration with ID %s: %v", rc.GetId(), err) @@ -380,26 +457,120 @@ func (s *scheduler) queueScheduledReports() { } } -/* Utility Functions */ - -// findAndRemoveFromQueue will find the first element that matches the given predicate and returns ReportRequest from that element -// Elements with values that are not of type *reportGen.ReportRequest will be skipped. -// Note: This function does not lock the queue, so any locks to prevent race conditions must be taken by the caller. -func findAndRemoveFromQueue(reportRequestsQueue *list.List, pred func(req *reportGen.ReportRequest) bool) *reportGen.ReportRequest { - var toRemove *list.Element - cur := reportRequestsQueue.Front() - for cur != nil { - req, ok := cur.Value.(*reportGen.ReportRequest) - if ok && pred(req) { - toRemove = cur - break +func (s *scheduler) recoverMissedSchedules() { + if !env.ReportMissedScheduleRecovery.BooleanSetting() { + return + } + + reportTypes := []string{storage.ReportConfiguration_VULNERABILITY.String()} + if features.NodeVulnerabilityReports.Enabled() { + reportTypes = append(reportTypes, storage.ReportConfiguration_NODE_VULNERABILITY.String()) + } + query := search.NewQueryBuilder(). + AddExactMatches(search.ReportType, reportTypes...). + ProtoQuery() + reportConfigs, err := s.reportConfigDatastore.GetReportConfigurations(scheduledCtx, query) + if err != nil { + log.Errorf("Error finding report configs for missed schedule recovery: %s", err) + return + } + + for _, rc := range reportConfigs { + if rc.GetSchedule() == nil { + continue + } + + cronSpec, err := schedule.ConvertToCronTab(rc.GetSchedule()) + if err != nil { + log.Errorf("Error converting schedule to crontab for config '%s': %v", rc.GetId(), err) + continue + } + + cronSchedule, err := cron.Parse(cronSpec) + if err != nil { + log.Errorf("Error parsing cron spec for config '%s': %v", rc.GetId(), err) + continue + } + + // Find the most recent time this schedule should have fired. + // We approximate the previous fire time by stepping back from now. + now := time.Now() + previousFireTime := findPreviousFireTime(cronSchedule, now) + if previousFireTime.IsZero() { + continue + } + + // Query for the most recent report snapshot for this config + snapshotQuery := search.NewQueryBuilder(). + AddExactMatches(search.ReportConfigID, rc.GetId()). + AddExactMatches(search.ReportRequestType, storage.ReportStatus_SCHEDULED.String()). + WithPagination(search.NewPagination(). + AddSortOption(search.NewSortOption(search.ReportQueuedTime).Reversed(true)). + Limit(1)). + ProtoQuery() + snapshots, err := s.reportSnapshotStore.SearchReportSnapshots(scheduledCtx, snapshotQuery) + if err != nil { + log.Errorf("Error querying snapshots for missed schedule recovery, config '%s': %v", rc.GetId(), err) + continue + } + + // If there are no snapshots, the schedule has never run yet. Don't recover it; + // the cron job will fire at the correct time. + if len(snapshots) == 0 { + continue + } + + lastSnapshot := snapshots[0] + // If the most recent snapshot is still pending, queuePendingReports already handles it. + runState := lastSnapshot.GetReportStatus().GetRunState() + if runState == storage.ReportStatus_WAITING || runState == storage.ReportStatus_PREPARING { + continue + } + + shouldRecover := false + lastTime := lastSnapshot.GetReportStatus().GetQueuedAt() + if lastTime != nil { + lastQueuedAt, err := protocompat.ConvertTimestampToTimeOrError(lastTime) + if err != nil { + log.Errorf("Error converting timestamp for config '%s': %v", rc.GetId(), err) + continue + } + if lastQueuedAt.Before(previousFireTime) { + shouldRecover = true + } + } + + if shouldRecover { + log.Infof("Recovering missed scheduled report for config '%s' (name: '%s')", rc.GetId(), rc.GetName()) + reportReq, err := s.validator.ValidateAndGenerateReportRequest(rc.GetId(), storage.ReportStatus_EMAIL, + storage.ReportStatus_SCHEDULED, nil) + if err != nil { + log.Errorf("Error generating report request for missed schedule recovery, config '%s': %v", rc.GetId(), err) + continue + } + _, err = s.SubmitReportRequest(scheduledCtx, reportReq, false) + if err != nil { + log.Errorf("Error submitting missed scheduled report for config '%s': %v", rc.GetId(), err) + } } - cur = cur.Next() } - if toRemove == nil { - return nil +} + +// findPreviousFireTime finds the most recent time before `now` that the cron schedule would have fired. +// It does this by stepping back in time and checking when the next fire time from that point would be. +func findPreviousFireTime(cronSchedule cron.Schedule, now time.Time) time.Time { + // Start from 32 days ago to cover monthly schedules (max interval between fires) + candidate := now.Add(-32 * 24 * time.Hour) + var previousFire time.Time + for { + next := cronSchedule.Next(candidate) + if next.After(now) { + break + } + previousFire = next + candidate = next } - return reportRequestsQueue.Remove(toRemove).(*reportGen.ReportRequest) + return previousFire } // Validate report snapshot and store it to db if validation succeeds. @@ -410,8 +581,11 @@ func (s *scheduler) validateAndPersistSnapshot(ctx context.Context, snapshot *st defer s.dbLock.Unlock() var err error if !reSubmission { - if snapshot.GetVulnReportFilters() != nil && snapshot.GetReportStatus().GetReportRequestType() == storage.ReportStatus_ON_DEMAND { - userHasAnotherReport, err := s.doesUserHavePendingReport(snapshot.GetReportConfigurationId(), snapshot.GetRequester().GetId()) + requestType := snapshot.GetReportStatus().GetReportRequestType() + reportType := snapshot.GetType() + + if requestType == storage.ReportStatus_ON_DEMAND { + userHasAnotherReport, err := s.doesUserHavePendingReport(snapshot.GetReportConfigurationId(), snapshot.GetRequester().GetId(), reportType) if err != nil { return "", err } @@ -421,9 +595,8 @@ func (s *scheduler) validateAndPersistSnapshot(ctx context.Context, snapshot *st } } - // if user has an existing view based report job then dont queue a new one - if snapshot.GetViewBasedVulnReportFilters() != nil { - userHasAnotherReport, err := s.doesUserHaveViewBasedPendingReport(snapshot.GetRequester().GetId()) + if requestType == storage.ReportStatus_VIEW_BASED { + userHasAnotherReport, err := s.doesUserHaveViewBasedPendingReport(snapshot.GetRequester().GetId(), reportType) if err != nil { return "", err } @@ -432,7 +605,14 @@ func (s *scheduler) validateAndPersistSnapshot(ctx context.Context, snapshot *st } } - snapshot.ReportId, err = s.reportSnapshotStore.AddReportSnapshot(ctx, snapshot) + // View-based reports are authorized at the gRPC layer with only view permissions + // (no WorkflowAdministration write required). The snapshot datastore requires + // WorkflowAdministration write, so we elevate to a privileged context here. + persistCtx := ctx + if requestType == storage.ReportStatus_VIEW_BASED { + persistCtx = sac.WithAllAccess(ctx) + } + snapshot.ReportId, err = s.reportSnapshotStore.AddReportSnapshot(persistCtx, snapshot) } else { err = s.reportSnapshotStore.UpdateReportSnapshot(ctx, snapshot) } @@ -443,27 +623,26 @@ func (s *scheduler) validateAndPersistSnapshot(ctx context.Context, snapshot *st return snapshot.GetReportId(), nil } -func (s *scheduler) doesUserHaveViewBasedPendingReport(userID string) (bool, error) { +func (s *scheduler) doesUserHaveViewBasedPendingReport(userID string, reportType storage.ReportSnapshot_ReportType) (bool, error) { query := search.NewQueryBuilder(). AddExactMatches(search.ReportState, storage.ReportStatus_WAITING.String(), storage.ReportStatus_PREPARING.String()). AddExactMatches(search.ReportRequestType, storage.ReportStatus_VIEW_BASED.String()). AddExactMatches(search.UserID, userID). + AddExactMatches(search.ReportType, reportType.String()). ProtoQuery() runningReports, err := s.reportSnapshotStore.Count(scheduledCtx, query) if err != nil { return false, err } - if runningReports > 0 { - return true, nil - } - return false, nil + return runningReports > 0, nil } -func (s *scheduler) doesUserHavePendingReport(configID string, userID string) (bool, error) { +func (s *scheduler) doesUserHavePendingReport(configID string, userID string, reportType storage.ReportSnapshot_ReportType) (bool, error) { query := search.NewQueryBuilder(). AddExactMatches(search.ReportConfigID, configID). AddExactMatches(search.ReportState, storage.ReportStatus_WAITING.String(), storage.ReportStatus_PREPARING.String()). AddExactMatches(search.ReportRequestType, storage.ReportStatus_ON_DEMAND.String()). + AddExactMatches(search.ReportType, reportType.String()). ProtoQuery() runningReports, err := s.reportSnapshotStore.SearchReportSnapshots(scheduledCtx, query) if err != nil { diff --git a/central/reports/scheduler/v2/scheduler_impl_test.go b/central/reports/scheduler/v2/scheduler_impl_test.go new file mode 100644 index 0000000000000..0eb70524f3ee5 --- /dev/null +++ b/central/reports/scheduler/v2/scheduler_impl_test.go @@ -0,0 +1,337 @@ +package v2 + +import ( + "context" + "testing" + "time" + + "github.com/stackrox/rox/central/reports/config/datastore/mocks" + reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" + reportGenMocks "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator/mocks" + snapshotMocks "github.com/stackrox/rox/central/reports/snapshot/datastore/mocks" + collectionMocks "github.com/stackrox/rox/central/resourcecollection/datastore/mocks" + "github.com/stackrox/rox/generated/storage" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" + "gopkg.in/robfig/cron.v2" +) + +func TestFindPreviousFireTime(t *testing.T) { + // Note: robfig/cron.v2 interprets cron specs in the system's local timezone. + // In production, Central runs in UTC. Tests use time.Local to be portable. + loc := time.Now().Location() + + var cases = []struct { + testname string + cronSpec string + now time.Time + expectedTime time.Time + }{ + { + testname: "Daily at 14:30, now is 16:00 same day", + cronSpec: "30 14 * * *", + now: time.Date(2026, 2, 16, 16, 0, 0, 0, loc), + expectedTime: time.Date(2026, 2, 16, 14, 30, 0, 0, loc), + }, + { + testname: "Daily at 14:30, now is 10:00 next day - missed yesterday's run", + cronSpec: "30 14 * * *", + now: time.Date(2026, 2, 17, 10, 0, 0, 0, loc), + expectedTime: time.Date(2026, 2, 16, 14, 30, 0, 0, loc), + }, + { + testname: "Weekly Monday at 09:00, now is Wednesday", + cronSpec: "0 9 * * 1", + now: time.Date(2026, 2, 18, 12, 0, 0, 0, loc), // Wednesday + expectedTime: time.Date(2026, 2, 16, 9, 0, 0, 0, loc), // Previous Monday + }, + { + testname: "Daily at 00:00, now is 23:59 same day", + cronSpec: "0 0 * * *", + now: time.Date(2026, 2, 16, 23, 59, 0, 0, loc), + expectedTime: time.Date(2026, 2, 16, 0, 0, 0, 0, loc), + }, + { + testname: "Daily at 23:59, now is 00:01 next day", + cronSpec: "59 23 * * *", + now: time.Date(2026, 2, 17, 0, 1, 0, 0, loc), + expectedTime: time.Date(2026, 2, 16, 23, 59, 0, 0, loc), + }, + } + + for _, c := range cases { + t.Run(c.testname, func(t *testing.T) { + schedule, err := cron.Parse(c.cronSpec) + assert.NoError(t, err) + + previousFire := findPreviousFireTime(schedule, c.now) + assert.Equal(t, c.expectedTime, previousFire) + }) + } +} + +func TestFindPreviousFireTimeReturnsZeroWhenNoFireInWindow(t *testing.T) { + // Monthly on the 15th at 10:00, now is Jan 16. + // The lookback window is 32 days, starting from Dec 15. + // Dec 15 at 10:00 is the only fire in that window, so it should be found. + // But if the schedule fires only on Feb 29 (leap year), and now is Jan 1, + // there may be no fire in the 32-day window. Use a far-future date to simulate. + loc := time.Now().Location() + schedule, err := cron.Parse("0 10 29 2 *") // Only Feb 29 + assert.NoError(t, err) + + // Now is March 1, 2027 (non-leap year). Feb 29 doesn't exist, so no fire in window. + previousFire := findPreviousFireTime(schedule, time.Date(2027, 3, 1, 0, 0, 0, 0, loc)) + assert.True(t, previousFire.IsZero(), "Expected zero time when no fire exists in lookback window") +} + +func TestQueueScheduledReportsSkipsEmptyResourceScope(t *testing.T) { + ctrl := gomock.NewController(t) + mockReportConfigDS := mocks.NewMockDataStore(ctrl) + + weeklySchedule := &storage.Schedule{ + IntervalType: storage.Schedule_WEEKLY, + Interval: &storage.Schedule_DaysOfWeek_{ + DaysOfWeek: &storage.Schedule_DaysOfWeek{ + Days: []int32{1}, + }, + }, + } + + validCollectionScope := &storage.ReportConfiguration{ + Id: "config-with-collection", + Name: "Valid Collection Config", + Type: storage.ReportConfiguration_VULNERABILITY, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_CollectionId{CollectionId: "collection-1"}, + }, + Schedule: weeklySchedule, + } + validEntityScope := &storage.ReportConfiguration{ + Id: "config-with-entity-scope", + Name: "Valid Entity Scope Config", + Type: storage.ReportConfiguration_VULNERABILITY, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{{Value: "production"}}, + }, + }, + }, + }, + }, + Schedule: weeklySchedule, + } + emptyResourceScope := &storage.ReportConfiguration{ + Id: "config-empty-scope", + Name: "Empty Scope Config (downgrade scenario)", + Type: storage.ReportConfiguration_VULNERABILITY, + ResourceScope: &storage.ResourceScope{}, + Schedule: weeklySchedule, + } + nilResourceScope := &storage.ReportConfiguration{ + Id: "config-nil-scope", + Name: "Nil Scope Config", + Type: storage.ReportConfiguration_VULNERABILITY, + ResourceScope: nil, + Schedule: weeklySchedule, + } + + mockReportConfigDS.EXPECT(). + GetReportConfigurations(gomock.Any(), gomock.Any()). + Return([]*storage.ReportConfiguration{ + validCollectionScope, + validEntityScope, + emptyResourceScope, + nilResourceScope, + }, nil) + + cronScheduler := cron.New() + cronScheduler.Start() + defer cronScheduler.Stop() + + s := newSchedulerImpl(mockReportConfigDS, nil, nil, nil, nil, nil, nil, cronScheduler) + s.queueScheduledReports() + + // Only the two valid configs should have been scheduled + assert.Len(t, s.reportConfigToEntryIDs, 2) + assert.Contains(t, s.reportConfigToEntryIDs, "config-with-collection") + assert.Contains(t, s.reportConfigToEntryIDs, "config-with-entity-scope") + assert.NotContains(t, s.reportConfigToEntryIDs, "config-empty-scope") + assert.NotContains(t, s.reportConfigToEntryIDs, "config-nil-scope") +} + +func TestCancelReportRequestCancelsRunningReport(t *testing.T) { + ctrl := gomock.NewController(t) + mockReportGen := reportGenMocks.NewMockReportGenerator(ctrl) + + cronScheduler := cron.New() + cronScheduler.Start() + defer cronScheduler.Stop() + + s := newSchedulerImpl(nil, nil, nil, nil, mockReportGen, nil, nil, cronScheduler) + imageQueue := s.queueByType[storage.ReportSnapshot_VULNERABILITY] + + started := make(chan struct{}) + done := make(chan struct{}) + var capturedCtx context.Context + + mockReportGen.EXPECT().ProcessReportRequest(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, req *reportGen.ReportRequest) { + capturedCtx = ctx + close(started) + <-done + }, + ) + + req := &reportGen.ReportRequest{ + ReportSnapshot: &storage.ReportSnapshot{ + ReportId: "running-report-id", + ReportConfigurationId: "test-config-id", + Type: storage.ReportSnapshot_VULNERABILITY, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_WAITING, + }, + }, + } + + // Acquire the semaphore since runSingleReport releases it + err := s.concurrencySema.Acquire(context.Background(), 1) + assert.NoError(t, err) + + go s.runSingleReport(imageQueue, mockReportGen, req) + <-started + + // Report is not in queue (it's running), so CancelReportRequest should cancel the running context + cancelled, cancelErr := s.CancelReportRequest(context.Background(), "running-report-id") + assert.NoError(t, cancelErr) + assert.True(t, cancelled) + + assert.Error(t, capturedCtx.Err()) + assert.ErrorIs(t, context.Cause(capturedCtx), reportGen.ErrUserCancelled) + + close(done) +} + +func TestCancelReportRequestReturnsFalseForUnknownReport(t *testing.T) { + cronScheduler := cron.New() + cronScheduler.Start() + defer cronScheduler.Stop() + + s := newSchedulerImpl(nil, nil, nil, nil, nil, nil, nil, cronScheduler) + + cancelled, err := s.CancelReportRequest(context.Background(), "nonexistent-id") + assert.NoError(t, err) + assert.False(t, cancelled) +} + +func TestQueuePendingReports(t *testing.T) { + ctrl := gomock.NewController(t) + mockSnapshotStore := snapshotMocks.NewMockDataStore(ctrl) + mockReportConfigDS := mocks.NewMockDataStore(ctrl) + mockCollectionDS := collectionMocks.NewMockDataStore(ctrl) + + viewBasedSnap := &storage.ReportSnapshot{ + ReportId: "view-based-report-1", + Name: "View Based Report", + Type: storage.ReportSnapshot_VULNERABILITY, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_PREPARING, + ReportRequestType: storage.ReportStatus_VIEW_BASED, + }, + Filter: &storage.ReportSnapshot_ViewBasedVulnReportFilters{ + ViewBasedVulnReportFilters: &storage.ViewBasedVulnerabilityReportFilters{ + Query: "CVE Type:IMAGE_CVE", + }, + }, + Requester: &storage.SlimUser{Id: "user-1", Name: "user-1"}, + } + + configBasedSnap := &storage.ReportSnapshot{ + ReportId: "config-based-report-1", + ReportConfigurationId: "config-1", + Name: "Config Based Report", + Type: storage.ReportSnapshot_VULNERABILITY, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_WAITING, + ReportRequestType: storage.ReportStatus_ON_DEMAND, + }, + Filter: &storage.ReportSnapshot_VulnReportFilters{ + VulnReportFilters: &storage.VulnerabilityReportFilters{}, + }, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_CollectionId{CollectionId: "collection-1"}, + }, + Collection: &storage.CollectionSnapshot{Id: "collection-1", Name: "collection-1"}, + Requester: &storage.SlimUser{Id: "user-2", Name: "user-2"}, + } + + mockSnapshotStore.EXPECT(). + SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return([]*storage.ReportSnapshot{viewBasedSnap, configBasedSnap}, nil) + + // View-based report should NOT trigger a config lookup. + // Config-based report should trigger a config lookup. + mockReportConfigDS.EXPECT(). + GetReportConfiguration(gomock.Any(), "config-1"). + Return(&storage.ReportConfiguration{Id: "config-1"}, true, nil) + + mockCollectionDS.EXPECT(). + Get(gomock.Any(), "collection-1"). + Return(&storage.ResourceCollection{Id: "collection-1"}, true, nil) + + // Both should be updated via resubmission. + mockSnapshotStore.EXPECT().UpdateReportSnapshot(gomock.Any(), gomock.Any()).Return(nil).Times(2) + + cronScheduler := cron.New() + cronScheduler.Start() + defer cronScheduler.Stop() + + s := newSchedulerImpl(mockReportConfigDS, mockSnapshotStore, mockCollectionDS, nil, nil, nil, nil, cronScheduler) + s.isStarted.Store(true) + s.queuePendingReports() + + assert.Equal(t, 2, s.queues[0].queue.Len()) +} + +func TestCancelReportRequestUpdatesWaitingReportToFailure(t *testing.T) { + ctrl := gomock.NewController(t) + mockSnapshotStore := snapshotMocks.NewMockDataStore(ctrl) + + cronScheduler := cron.New() + cronScheduler.Start() + defer cronScheduler.Stop() + + s := newSchedulerImpl(nil, mockSnapshotStore, nil, nil, nil, nil, nil, cronScheduler) + imageQueue := s.queueByType[storage.ReportSnapshot_VULNERABILITY] + + req := &reportGen.ReportRequest{ + ReportSnapshot: &storage.ReportSnapshot{ + ReportId: "waiting-report-id", + ReportConfigurationId: "test-config-id", + Name: "test-report", + Type: storage.ReportSnapshot_VULNERABILITY, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_WAITING, + }, + }, + } + imageQueue.Enqueue(req) + + mockSnapshotStore.EXPECT().UpdateReportSnapshot(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, snap *storage.ReportSnapshot) error { + assert.Equal(t, storage.ReportStatus_FAILURE, snap.GetReportStatus().GetRunState()) + assert.Equal(t, reportGen.ErrUserCancelled.Error(), snap.GetReportStatus().GetErrorMsg()) + assert.NotNil(t, snap.GetReportStatus().GetCompletedAt()) + return nil + }) + + cancelled, err := s.CancelReportRequest(context.Background(), "waiting-report-id") + assert.NoError(t, err) + assert.True(t, cancelled) + assert.Equal(t, 0, imageQueue.Len()) +} diff --git a/central/reports/scheduler/v2/singleton.go b/central/reports/scheduler/v2/singleton.go index 8d1788568e943..bafe399d2806e 100644 --- a/central/reports/scheduler/v2/singleton.go +++ b/central/reports/scheduler/v2/singleton.go @@ -4,6 +4,7 @@ import ( notifierDS "github.com/stackrox/rox/central/notifier/datastore" reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" + nodeReportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator/node" reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" "github.com/stackrox/rox/central/reports/validation" collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" @@ -24,6 +25,7 @@ func initialize() { collectionDatastore, notifierDS.Singleton(), reportGen.Singleton(), + nodeReportGen.Singleton(), validation.Singleton(), ) } diff --git a/central/reports/scheduler/vuln_report_collections_test.go b/central/reports/scheduler/vuln_report_collections_test.go deleted file mode 100644 index 4f91906ec90b9..0000000000000 --- a/central/reports/scheduler/vuln_report_collections_test.go +++ /dev/null @@ -1,380 +0,0 @@ -//go:build sql_integration - -package scheduler - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/graph-gophers/graphql-go" - "github.com/stackrox/rox/central/graphql/resolvers" - "github.com/stackrox/rox/central/graphql/resolvers/loaders" - "github.com/stackrox/rox/central/reports/common" - collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" - collectionPostgres "github.com/stackrox/rox/central/resourcecollection/datastore/store/postgres" - deploymentsView "github.com/stackrox/rox/central/views/deployments" - "github.com/stackrox/rox/central/views/imagecomponentflat" - "github.com/stackrox/rox/central/views/imagecve" - "github.com/stackrox/rox/central/views/imagecveflat" - imagesView "github.com/stackrox/rox/central/views/images" - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/fixtures" - "github.com/stackrox/rox/pkg/grpc/authz/allow" - types2 "github.com/stackrox/rox/pkg/images/types" - "github.com/stackrox/rox/pkg/postgres/pgtest" - postgresSchema "github.com/stackrox/rox/pkg/postgres/schema" - "github.com/stackrox/rox/pkg/protocompat" - "github.com/stackrox/rox/pkg/sac" - pkgSearch "github.com/stackrox/rox/pkg/search" - "github.com/stackrox/rox/pkg/utils" - "github.com/stackrox/rox/pkg/uuid" - "github.com/stretchr/testify/suite" - "go.uber.org/mock/gomock" -) - -func TestReportingWithCollections(t *testing.T) { - suite.Run(t, new(ReportingWithCollectionsTestSuite)) -} - -type ReportingWithCollectionsTestSuite struct { - suite.Suite - - ctx context.Context - testDB *pgtest.TestPostgres - reportScheduler *scheduler - resolver *resolvers.Resolver - schema *graphql.Schema - - collectionDatastore collectionDS.DataStore - collectionQueryResolver collectionDS.QueryResolver -} - -type vulnReportData struct { - deploymentNames []string - imageNames []string - componentNames []string - cveNames []string -} - -func (s *ReportingWithCollectionsTestSuite) SetupSuite() { - - s.ctx = loaders.WithLoaderContext(sac.WithAllAccess(context.Background())) - mockCtrl := gomock.NewController(s.T()) - s.testDB = resolvers.SetupTestPostgresConn(s.T()) - - imgDataStore := resolvers.CreateTestImageV2Datastore(s.T(), s.testDB, mockCtrl) - s.resolver, s.schema = resolvers.SetupTestResolver(s.T(), - imgDataStore, - imagesView.NewImageView(s.testDB.DB), - resolvers.CreateTestImageComponentV2Datastore(s.T(), s.testDB, mockCtrl), - resolvers.CreateTestImageCVEV2Datastore(s.T(), s.testDB), - resolvers.CreateTestDeploymentDatastore(s.T(), s.testDB, mockCtrl, imgDataStore), - imagecve.NewCVEView(s.testDB.DB), - imagecveflat.NewCVEFlatView(s.testDB.DB), - imagecomponentflat.NewComponentFlatView(s.testDB.DB), - deploymentsView.NewDeploymentView(s.testDB.DB), - ) - - var err error - collectionStore := collectionPostgres.New(s.testDB) - s.collectionDatastore, s.collectionQueryResolver, err = collectionDS.New(collectionStore) - s.NoError(err) - - s.reportScheduler = newSchedulerImpl(nil, nil, nil, nil, - s.resolver.DeploymentDataStore, s.collectionDatastore, nil, s.collectionQueryResolver, - nil, nil, s.schema) -} - -func (s *ReportingWithCollectionsTestSuite) TearDownTest() { - s.truncateTable(postgresSchema.DeploymentsTableName) - s.truncateTable(postgresSchema.ImagesTableName) - s.truncateTable(postgresSchema.CollectionsTableName) - s.truncateTable(postgresSchema.ImageComponentV2TableName) -} - -func (s *ReportingWithCollectionsTestSuite) TestGetReportData() { - ctx := resolvers.SetAuthorizerOverride(s.ctx, allow.Anonymous()) - clusters := []string{"c1", "c2"} - namespaces := []string{"ns1", "ns2"} - deployments, images := testDeploymentsWithImages(clusters, namespaces, 1) - s.upsertManyImages(images) - s.upsertManyDeployments(deployments) - - testCases := []struct { - name string - collection *storage.ResourceCollection - fixability storage.VulnerabilityReportFilters_Fixability - severities []storage.VulnerabilitySeverity - expected *vulnReportData - }{ - { - name: "Include all deployments; CVEs with both fixabilities and all severities", - collection: testCollection("col1", "", "", ""), - fixability: storage.VulnerabilityReportFilters_BOTH, - severities: []storage.VulnerabilitySeverity{ - storage.VulnerabilitySeverity_LOW_VULNERABILITY_SEVERITY, - storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, - storage.VulnerabilitySeverity_MODERATE_VULNERABILITY_SEVERITY, - storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, - }, - expected: &vulnReportData{ - deploymentNames: []string{"c1_ns1_dep0", "c1_ns2_dep0", "c2_ns1_dep0", "c2_ns2_dep0"}, - imageNames: []string{"c1_ns1_dep0_img", "c1_ns2_dep0_img", "c2_ns1_dep0_img", "c2_ns2_dep0_img"}, - componentNames: []string{"c1_ns1_dep0_img_comp", "c1_ns2_dep0_img_comp", "c2_ns1_dep0_img_comp", "c2_ns2_dep0_img_comp"}, - cveNames: []string{ - "CVE-fixable_critical-c1_ns1_dep0_img_comp", "CVE-nonFixable_low-c1_ns1_dep0_img_comp", - "CVE-fixable_critical-c1_ns2_dep0_img_comp", "CVE-nonFixable_low-c1_ns2_dep0_img_comp", - "CVE-fixable_critical-c2_ns1_dep0_img_comp", "CVE-nonFixable_low-c2_ns1_dep0_img_comp", - "CVE-fixable_critical-c2_ns2_dep0_img_comp", "CVE-nonFixable_low-c2_ns2_dep0_img_comp", - }, - }, - }, - { - name: "Include all deployments; Fixable CVEs with CRITICAL severity", - collection: testCollection("col2", "", "", ""), - fixability: storage.VulnerabilityReportFilters_FIXABLE, - severities: []storage.VulnerabilitySeverity{ - storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, - }, - expected: &vulnReportData{ - deploymentNames: []string{"c1_ns1_dep0", "c1_ns2_dep0", "c2_ns1_dep0", "c2_ns2_dep0"}, - imageNames: []string{"c1_ns1_dep0_img", "c1_ns2_dep0_img", "c2_ns1_dep0_img", "c2_ns2_dep0_img"}, - componentNames: []string{"c1_ns1_dep0_img_comp", "c1_ns2_dep0_img_comp", "c2_ns1_dep0_img_comp", "c2_ns2_dep0_img_comp"}, - cveNames: []string{ - "CVE-fixable_critical-c1_ns1_dep0_img_comp", - "CVE-fixable_critical-c1_ns2_dep0_img_comp", - "CVE-fixable_critical-c2_ns1_dep0_img_comp", - "CVE-fixable_critical-c2_ns2_dep0_img_comp", - }, - }, - }, - { - name: "Include deployments from cluster c1 and namespace ns1; CVEs with both fixabilities and all severities", - collection: testCollection("col3", "c1", "ns1", ""), - fixability: storage.VulnerabilityReportFilters_BOTH, - severities: []storage.VulnerabilitySeverity{ - storage.VulnerabilitySeverity_LOW_VULNERABILITY_SEVERITY, - storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, - storage.VulnerabilitySeverity_MODERATE_VULNERABILITY_SEVERITY, - storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, - }, - expected: &vulnReportData{ - deploymentNames: []string{"c1_ns1_dep0"}, - imageNames: []string{"c1_ns1_dep0_img"}, - componentNames: []string{"c1_ns1_dep0_img_comp"}, - cveNames: []string{ - "CVE-fixable_critical-c1_ns1_dep0_img_comp", "CVE-nonFixable_low-c1_ns1_dep0_img_comp", - }, - }, - }, - } - - for _, c := range testCases { - _, err := s.collectionDatastore.AddCollection(s.ctx, c.collection) - s.NoError(err) - - reportConfig := testReportConfig(c.collection.GetId(), c.fixability, c.severities) - results, err := s.reportScheduler.getReportData(ctx, reportConfig) - s.NoError(err) - reportData := extractVulnReportData(results) - s.ElementsMatch(c.expected.deploymentNames, reportData.deploymentNames) - s.ElementsMatch(c.expected.imageNames, reportData.imageNames) - s.ElementsMatch(c.expected.componentNames, reportData.componentNames) - s.ElementsMatch(c.expected.cveNames, reportData.cveNames) - } -} - -func (s *ReportingWithCollectionsTestSuite) truncateTable(name string) { - sql := fmt.Sprintf("TRUNCATE %s CASCADE", name) - _, err := s.testDB.Exec(s.ctx, sql) - s.NoError(err) -} - -func (s *ReportingWithCollectionsTestSuite) upsertManyImages(images []*storage.Image) { - for _, img := range images { - err := s.resolver.ImageDataStore.UpsertImage(s.ctx, img) - s.NoError(err) - } -} - -func (s *ReportingWithCollectionsTestSuite) upsertManyDeployments(deployments []*storage.Deployment) { - for _, dep := range deployments { - err := s.resolver.DeploymentDataStore.UpsertDeployment(s.ctx, dep) - s.NoError(err) - } -} - -func testDeploymentsWithImages(clusters, namespaces []string, numDeploymentsPerNamespace int) ([]*storage.Deployment, []*storage.Image) { - capacity := len(clusters) * len(namespaces) * numDeploymentsPerNamespace - deployments := make([]*storage.Deployment, 0, capacity) - images := make([]*storage.Image, 0, capacity) - for _, cluster := range clusters { - for _, namespace := range namespaces { - for i := 0; i < numDeploymentsPerNamespace; i++ { - depName := fmt.Sprintf("%s_%s_dep%d", cluster, namespace, i) - image := testImage(depName) - deployment := testDeployment(depName, cluster, namespace, image) - deployments = append(deployments, deployment) - images = append(images, image) - } - } - } - return deployments, images -} - -func testDeployment(deploymentName, cluster, namespace string, image *storage.Image) *storage.Deployment { - return &storage.Deployment{ - Name: deploymentName, - Id: uuid.NewV4().String(), - ClusterName: cluster, - ClusterId: uuid.NewV4().String(), - Namespace: namespace, - NamespaceId: uuid.NewV4().String(), - Containers: []*storage.Container{ - { - Name: fmt.Sprintf("%s_container", deploymentName), - Image: types2.ToContainerImage(image), - }, - }, - } -} - -func testImage(deployment string) *storage.Image { - t, err := protocompat.ConvertTimeToTimestampOrError(time.Unix(0, 1000)) - utils.CrashOnError(err) - return &storage.Image{ - Id: fmt.Sprintf("%s_img", deployment), - Name: &storage.ImageName{FullName: fmt.Sprintf("%s_img", deployment)}, - SetComponents: &storage.Image_Components{ - Components: 1, - }, - SetCves: &storage.Image_Cves{ - Cves: 2, - }, - Scan: &storage.ImageScan{ - ScanTime: t, - Components: []*storage.EmbeddedImageScanComponent{ - { - Name: fmt.Sprintf("%s_img_comp", deployment), - Version: "1.0", - Vulns: []*storage.EmbeddedVulnerability{ - { - Cve: fmt.Sprintf("CVE-fixable_critical-%s_img_comp", deployment), - SetFixedBy: &storage.EmbeddedVulnerability_FixedBy{ - FixedBy: "1.1", - }, - Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, - Link: "link", - }, - { - Cve: fmt.Sprintf("CVE-nonFixable_low-%s_img_comp", deployment), - Severity: storage.VulnerabilitySeverity_LOW_VULNERABILITY_SEVERITY, - Link: "link", - }, - }, - }, - }, - }, - } -} - -func testCollection(collectionName, cluster, namespace, deployment string) *storage.ResourceCollection { - collection := &storage.ResourceCollection{ - Name: collectionName, - ResourceSelectors: []*storage.ResourceSelector{ - { - Rules: []*storage.SelectorRule{}, - }, - }, - } - if cluster != "" { - collection.ResourceSelectors[0].Rules = append(collection.ResourceSelectors[0].Rules, &storage.SelectorRule{ - FieldName: pkgSearch.Cluster.String(), - Operator: storage.BooleanOperator_OR, - Values: []*storage.RuleValue{ - { - Value: cluster, - MatchType: storage.MatchType_EXACT, - }, - }, - }) - } - if namespace != "" { - collection.ResourceSelectors[0].Rules = append(collection.ResourceSelectors[0].Rules, &storage.SelectorRule{ - FieldName: pkgSearch.Namespace.String(), - Operator: storage.BooleanOperator_OR, - Values: []*storage.RuleValue{ - { - Value: namespace, - MatchType: storage.MatchType_EXACT, - }, - }, - }) - } - var deploymentVal string - var matchType storage.MatchType - if deployment != "" { - deploymentVal = deployment - matchType = storage.MatchType_EXACT - } else { - deploymentVal = ".*" - matchType = storage.MatchType_REGEX - } - collection.ResourceSelectors[0].Rules = append(collection.ResourceSelectors[0].Rules, &storage.SelectorRule{ - FieldName: pkgSearch.DeploymentName.String(), - Operator: storage.BooleanOperator_OR, - Values: []*storage.RuleValue{ - { - Value: deploymentVal, - MatchType: matchType, - }, - }, - }) - - return collection -} - -func testReportConfig(collectionID string, fixability storage.VulnerabilityReportFilters_Fixability, - severities []storage.VulnerabilitySeverity) *storage.ReportConfiguration { - config := fixtures.GetValidReportConfiguration() - config.Filter = &storage.ReportConfiguration_VulnReportFilters{ - VulnReportFilters: &storage.VulnerabilityReportFilters{ - Fixability: fixability, - SinceLastReport: false, - Severities: severities, - }, - } - config.ScopeId = collectionID - return config -} - -func extractVulnReportData(results []common.DeployedImagesResult) *vulnReportData { - deploymentNames := make([]string, 0) - imageNames := make([]string, 0) - componentNames := make([]string, 0) - cveNames := make([]string, 0) - - for _, res := range results { - for _, dep := range res.Deployments { - deploymentNames = append(deploymentNames, dep.DeploymentName) - for _, img := range dep.Images { - imageNames = append(imageNames, img.Name.GetFullName()) - for _, comp := range img.ImageComponents { - componentNames = append(componentNames, comp.Name) - for _, cve := range comp.ImageVulnerabilities { - cveNames = append(cveNames, cve.Cve) - } - } - } - } - } - - return &vulnReportData{ - deploymentNames: deploymentNames, - imageNames: imageNames, - componentNames: componentNames, - cveNames: cveNames, - } -} diff --git a/central/reports/service/service.go b/central/reports/service/service.go deleted file mode 100644 index 4bfe00ed0fe0a..0000000000000 --- a/central/reports/service/service.go +++ /dev/null @@ -1,28 +0,0 @@ -package service - -import ( - "context" - - notifierDataStore "github.com/stackrox/rox/central/notifier/datastore" - "github.com/stackrox/rox/central/reports/config/datastore" - "github.com/stackrox/rox/central/reports/manager" - v1 "github.com/stackrox/rox/generated/api/v1" - "github.com/stackrox/rox/pkg/grpc" -) - -// Service provides the interface to the gRPC service for roles. -type Service interface { - grpc.APIService - - AuthFuncOverride(ctx context.Context, fullMethodName string) (context.Context, error) - v1.ReportServiceServer -} - -// New returns a new instance of the service. Please use the Singleton instead. -func New(reportConfigStore datastore.DataStore, notifierStore notifierDataStore.DataStore, manager manager.Manager) Service { - return &serviceImpl{ - manager: manager, - reportConfigStore: reportConfigStore, - notifierStore: notifierStore, - } -} diff --git a/central/reports/service/service_impl.go b/central/reports/service/service_impl.go deleted file mode 100644 index 8f3cd2dbe2dbc..0000000000000 --- a/central/reports/service/service_impl.go +++ /dev/null @@ -1,72 +0,0 @@ -package service - -import ( - "context" - - "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" - "github.com/pkg/errors" - notifierDataStore "github.com/stackrox/rox/central/notifier/datastore" - "github.com/stackrox/rox/central/reports/common" - reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" - "github.com/stackrox/rox/central/reports/manager" - v1 "github.com/stackrox/rox/generated/api/v1" - "github.com/stackrox/rox/pkg/auth/permissions" - "github.com/stackrox/rox/pkg/errox" - "github.com/stackrox/rox/pkg/grpc/authz" - "github.com/stackrox/rox/pkg/grpc/authz/perrpc" - "github.com/stackrox/rox/pkg/grpc/authz/user" - "github.com/stackrox/rox/pkg/logging" - "github.com/stackrox/rox/pkg/sac/resources" - "google.golang.org/grpc" -) - -var ( - authorizer = perrpc.FromMap(map[authz.Authorizer][]string{ - user.With(permissions.View(resources.WorkflowAdministration), permissions.View(resources.Integration), - permissions.View(resources.Image)): { - v1.ReportService_RunReport_FullMethodName, - }, - }) -) - -var ( - log = logging.LoggerForModule() -) - -type serviceImpl struct { - v1.UnimplementedReportServiceServer - - manager manager.Manager - reportConfigStore reportConfigDS.DataStore - notifierStore notifierDataStore.DataStore -} - -func (s *serviceImpl) RegisterServiceServer(grpcServer *grpc.Server) { - v1.RegisterReportServiceServer(grpcServer, s) -} - -func (s *serviceImpl) RegisterServiceHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { - return v1.RegisterReportServiceHandler(ctx, mux, conn) -} - -func (*serviceImpl) AuthFuncOverride(ctx context.Context, fullMethodName string) (context.Context, error) { - return ctx, authorizer.Authorized(ctx, fullMethodName) -} - -func (s *serviceImpl) RunReport(ctx context.Context, id *v1.ResourceByID) (*v1.Empty, error) { - rc, found, err := s.reportConfigStore.GetReportConfiguration(ctx, id.GetId()) - if err != nil { - return &v1.Empty{}, errors.Wrapf(err, "error finding report configuration %s", id) - } - if !found { - return &v1.Empty{}, errors.Wrapf(errox.NotFound, "unable to find report configuration %s", id) - } - if !common.IsV1ReportConfig(rc) { - return &v1.Empty{}, errors.Wrap(errox.InvalidArgs, "report configuration does not belong to reporting version 1.0") - } - - if err := s.manager.RunReport(ctx, rc); err != nil { - return &v1.Empty{}, err - } - return &v1.Empty{}, nil -} diff --git a/central/reports/service/service_impl_test.go b/central/reports/service/service_impl_test.go deleted file mode 100644 index a18da09319509..0000000000000 --- a/central/reports/service/service_impl_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package service - -import ( - "context" - "testing" - - reportConfigDSMocks "github.com/stackrox/rox/central/reports/config/datastore/mocks" - managerMocks "github.com/stackrox/rox/central/reports/manager/mocks" - apiV1 "github.com/stackrox/rox/generated/api/v1" - "github.com/stackrox/rox/pkg/fixtures" - "github.com/stretchr/testify/assert" - "go.uber.org/mock/gomock" -) - -func TestConfigSeparation(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - reportConfigStore := reportConfigDSMocks.NewMockDataStore(mockCtrl) - manager := managerMocks.NewMockManager(mockCtrl) - service := New(reportConfigStore, nil, manager) - ctx := context.Background() - - // Error on v2 config - configV2 := fixtures.GetValidReportConfigWithMultipleNotifiersV2() - reportConfigStore.EXPECT().GetReportConfiguration(gomock.Any(), configV2.GetId()).Return(configV2, true, nil).Times(1) - _, err := service.RunReport(ctx, &apiV1.ResourceByID{Id: configV2.GetId()}) - assert.Error(t, err) - - // No error on v1 config - configV1 := fixtures.GetValidReportConfigWithMultipleNotifiersV1() - reportConfigStore.EXPECT().GetReportConfiguration(gomock.Any(), configV1.GetId()).Return(configV1, true, nil).Times(1) - manager.EXPECT().RunReport(gomock.Any(), gomock.Any()).Return(nil).Times(1) - _, err = service.RunReport(ctx, &apiV1.ResourceByID{Id: configV1.GetId()}) - assert.NoError(t, err) -} diff --git a/central/reports/service/singleton.go b/central/reports/service/singleton.go deleted file mode 100644 index 9ce15e754b3e4..0000000000000 --- a/central/reports/service/singleton.go +++ /dev/null @@ -1,57 +0,0 @@ -package service - -import ( - "context" - - notifierDataStore "github.com/stackrox/rox/central/notifier/datastore" - "github.com/stackrox/rox/central/reports/common" - "github.com/stackrox/rox/central/reports/config/datastore" - "github.com/stackrox/rox/central/reports/manager" - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/sac" - "github.com/stackrox/rox/pkg/sac/resources" - "github.com/stackrox/rox/pkg/search" - "github.com/stackrox/rox/pkg/sync" -) - -var ( - once sync.Once - - as Service -) - -func initialize() { - mgr := initializeManager() - as = New(datastore.Singleton(), notifierDataStore.Singleton(), mgr) -} - -func initializeManager() manager.Manager { - ctx := sac.WithGlobalAccessScopeChecker(context.Background(), - sac.AllowFixedScopes( - sac.AccessModeScopeKeys(storage.Access_READ_ACCESS, storage.Access_READ_WRITE_ACCESS), - sac.ResourceScopeKeys(resources.WorkflowAdministration))) - - query := search.NewQueryBuilder().AddExactMatches(search.ReportType, storage.ReportConfiguration_VULNERABILITY.String()).ProtoQuery() - filteredQ := common.WithoutV2ReportConfigs(query) - - reportConfigs, err := datastore.Singleton().GetReportConfigurations(ctx, filteredQ) - mgr := manager.Singleton() - if err != nil { - log.Errorf("Error finding scheduled reports: %s", err) - return mgr - } - - for _, rc := range reportConfigs { - if err := mgr.Upsert(ctx, rc); err != nil { - log.Errorf("error upserting report config: %v", err) - } - } - mgr.Start() - return mgr -} - -// Singleton provides the instance of the Service interface to register. -func Singleton() Service { - once.Do(initialize) - return as -} diff --git a/central/reports/service/v2/config_separation_test.go b/central/reports/service/v2/config_separation_test.go deleted file mode 100644 index 4eb2304547ef9..0000000000000 --- a/central/reports/service/v2/config_separation_test.go +++ /dev/null @@ -1,175 +0,0 @@ -//go:build sql_integration - -package v2 - -import ( - "context" - "fmt" - "testing" - - notifierDS "github.com/stackrox/rox/central/notifier/datastore" - "github.com/stackrox/rox/central/reports/common" - reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" - schedulerMocks "github.com/stackrox/rox/central/reports/scheduler/v2/mocks" - snapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" - "github.com/stackrox/rox/central/reports/validation" - collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" - apiV2 "github.com/stackrox/rox/generated/api/v2" - "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/postgres/pgtest" - "github.com/stackrox/rox/pkg/protoassert" - "github.com/stackrox/rox/pkg/sac" - "github.com/stretchr/testify/suite" - "go.uber.org/mock/gomock" -) - -func TestServiceLevelConfigSeparationV2(t *testing.T) { - suite.Run(t, new(ServiceLevelConfigSeparationSuiteV2)) -} - -type ServiceLevelConfigSeparationSuiteV2 struct { - suite.Suite - service *serviceImpl - - testDB *pgtest.TestPostgres - ctx context.Context - reportConfigDatastore reportConfigDS.DataStore - notifierDatastore notifierDS.DataStore - collectionDatastore collectionDS.DataStore - scheduler *schedulerMocks.MockScheduler - snapshotDS snapshotDS.DataStore - - mockCtrl *gomock.Controller - - v1Configs []*storage.ReportConfiguration - v2Configs []*storage.ReportConfiguration -} - -func (s *ServiceLevelConfigSeparationSuiteV2) SetupSuite() { - s.testDB = pgtest.ForT(s.T()) - s.reportConfigDatastore = reportConfigDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) - s.notifierDatastore = notifierDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) - s.snapshotDS = snapshotDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) - - var err error - s.collectionDatastore, _, err = collectionDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) - s.Require().NoError(err) - - s.mockCtrl = gomock.NewController(s.T()) - s.scheduler = schedulerMocks.NewMockScheduler(s.mockCtrl) - - validator := validation.New(s.reportConfigDatastore, nil, s.collectionDatastore, s.notifierDatastore) - s.service = &serviceImpl{ - reportConfigStore: s.reportConfigDatastore, - collectionDatastore: s.collectionDatastore, - notifierDatastore: s.notifierDatastore, - scheduler: s.scheduler, - snapshotDatastore: s.snapshotDS, - validator: validator, - } - - s.ctx = sac.WithAllAccess(context.Background()) - - // Add notifier - notifierID, err := s.notifierDatastore.AddNotifier(s.ctx, &storage.Notifier{Name: "notifier"}) - s.Require().NoError(err) - - // Add collection - collectionID, err := s.collectionDatastore.AddCollection(s.ctx, &storage.ResourceCollection{Name: "collection"}) - s.Require().NoError(err) - - // Add report configs - s.v1Configs = common.GetTestReportConfigsV1(s.T(), notifierID, collectionID) - for _, conf := range s.v1Configs { - conf.Id, err = s.reportConfigDatastore.AddReportConfiguration(s.ctx, conf) - s.Require().NoError(err) - } - - s.v2Configs = common.GetTestReportConfigsV2(s.T(), notifierID, collectionID) - for _, conf := range s.v2Configs { - conf.Id, err = s.reportConfigDatastore.AddReportConfiguration(s.ctx, conf) - s.Require().NoError(err) - } -} - -func (s *ServiceLevelConfigSeparationSuiteV2) TearDownSuite() { - s.mockCtrl.Finish() - -} - -func (s *ServiceLevelConfigSeparationSuiteV2) TestListReportConfigurations() { - apiV2Configs := s.convertConfigs(s.v2Configs) - - // Empty Query - res, err := s.service.ListReportConfigurations(s.ctx, &apiV2.RawQuery{Query: ""}) - s.Require().NoError(err) - protoassert.ElementsMatch(s.T(), apiV2Configs, res.GetReportConfigs()) - - // Non empty query - res, err = s.service.ListReportConfigurations(s.ctx, - &apiV2.RawQuery{Query: fmt.Sprintf("Report Name:%s", s.v2Configs[0].GetName())}) - s.Require().NoError(err) - s.Require().Equal(1, len(res.GetReportConfigs())) - protoassert.Equal(s.T(), apiV2Configs[0], res.GetReportConfigs()[0]) -} - -func (s *ServiceLevelConfigSeparationSuiteV2) TestGetReportConfiguration() { - apiV2Configs := s.convertConfigs(s.v2Configs) - - // returns v2 config - res, err := s.service.GetReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: s.v2Configs[0].GetId()}) - s.Require().NoError(err) - protoassert.Equal(s.T(), apiV2Configs[0], res) - - // error on requesting v1 config - _, err = s.service.GetReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: s.v1Configs[0].GetId()}) - s.Require().Error(err) -} - -func (s *ServiceLevelConfigSeparationSuiteV2) TestCountReportConfigurations() { - // Empty query - res, err := s.service.CountReportConfigurations(s.ctx, &apiV2.RawQuery{Query: ""}) - s.Require().NoError(err) - s.Require().Equal(int32(len(s.v2Configs)), res.GetCount()) - - // Non empty query - res, err = s.service.CountReportConfigurations(s.ctx, - &apiV2.RawQuery{Query: fmt.Sprintf("Report Name:%s", s.v2Configs[0].GetName())}) - s.Require().NoError(err) - s.Require().Equal(int32(1), res.GetCount()) -} - -func (s *ServiceLevelConfigSeparationSuiteV2) TestDeleteReportConfiguration() { - // Error on v1 config ID - _, err := s.service.DeleteReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: s.v1Configs[0].GetId()}) - s.Require().Error(err) - - // No error on v2 config ID - s.scheduler.EXPECT().RemoveReportSchedule(gomock.Any()).Return().Times(1) - config := s.v2Configs[0].CloneVT() - config.Id = "" - config.Name = "Delete report config" - config.Id, err = s.reportConfigDatastore.AddReportConfiguration(s.ctx, config) - s.Require().NoError(err) - _, err = s.service.DeleteReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: config.GetId()}) - s.Require().NoError(err) -} - -func (s *ServiceLevelConfigSeparationSuiteV2) TestRunReport() { - // Error on v1 config - _, err := s.service.RunReport(s.ctx, &apiV2.RunReportRequest{ - ReportConfigId: s.v1Configs[0].GetId(), - ReportNotificationMethod: apiV2.NotificationMethod_EMAIL, - }) - s.Require().Error(err) -} - -func (s *ServiceLevelConfigSeparationSuiteV2) convertConfigs(configs []*storage.ReportConfiguration) []*apiV2.ReportConfiguration { - apiV2Configs := make([]*apiV2.ReportConfiguration, 0, len(configs)) - for _, conf := range configs { - c, err := s.service.convertProtoReportConfigurationToV2(conf) - s.Require().NoError(err) - apiV2Configs = append(apiV2Configs, c) - } - return apiV2Configs -} diff --git a/central/reports/service/v2/convert.go b/central/reports/service/v2/convert.go index b4c90bace68b7..9c25a59fe7c3d 100644 --- a/central/reports/service/v2/convert.go +++ b/central/reports/service/v2/convert.go @@ -4,9 +4,11 @@ import ( "context" "github.com/pkg/errors" + notifierDS "github.com/stackrox/rox/central/notifier/datastore" "github.com/stackrox/rox/central/reports/common" apiV2 "github.com/stackrox/rox/generated/api/v2" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/set" @@ -17,11 +19,12 @@ var ( apiV2.ReportSchedule_UNSET: storage.Schedule_UNSET, apiV2.ReportSchedule_WEEKLY: storage.Schedule_WEEKLY, apiV2.ReportSchedule_MONTHLY: storage.Schedule_MONTHLY, + apiV2.ReportSchedule_DAILY: storage.Schedule_DAILY, } storageIntervalTypeToV2 = map[storage.Schedule_IntervalType]apiV2.ReportSchedule_IntervalType{ storage.Schedule_UNSET: apiV2.ReportSchedule_UNSET, - storage.Schedule_DAILY: apiV2.ReportSchedule_UNSET, + storage.Schedule_DAILY: apiV2.ReportSchedule_DAILY, storage.Schedule_WEEKLY: apiV2.ReportSchedule_WEEKLY, storage.Schedule_MONTHLY: apiV2.ReportSchedule_MONTHLY, } @@ -54,7 +57,7 @@ func (s *serviceImpl) convertV2ReportConfigurationToProto(config *apiV2.ReportCo Name: config.GetName(), Description: config.GetDescription(), Type: storage.ReportConfiguration_ReportType(config.GetType()), - Schedule: s.convertV2ScheduleToProto(config.GetSchedule()), + Schedule: ConvertV2ScheduleToProto(config.GetSchedule()), ResourceScope: s.convertV2ResourceScopeToProto(config.GetResourceScope()), Creator: creator, Version: 2, @@ -67,7 +70,7 @@ func (s *serviceImpl) convertV2ReportConfigurationToProto(config *apiV2.ReportCo } for _, notifier := range config.GetNotifiers() { - ret.Notifiers = append(ret.Notifiers, s.convertV2NotifierConfigToProto(notifier)) + ret.Notifiers = append(ret.Notifiers, ConvertV2NotifierConfigToProto(notifier)) } return ret @@ -87,6 +90,9 @@ func (s *serviceImpl) convertV2VulnReportFiltersToProto(filters *apiV2.Vulnerabi IncludeAdvisory: filters.GetIncludeAdvisory(), } + if features.VulnerabilityReportsEnhancedFiltering.Enabled() { + ret.Query = filters.GetQuery() + } for _, severity := range filters.GetSeverities() { ret.Severities = append(ret.Severities, storage.VulnerabilitySeverity(severity)) } @@ -121,13 +127,81 @@ func (s *serviceImpl) convertV2ResourceScopeToProto(scope *apiV2.ResourceScope) } ret := &storage.ResourceScope{} - if scope.GetCollectionScope() != nil { - ret.ScopeReference = &storage.ResourceScope_CollectionId{CollectionId: scope.GetCollectionScope().GetCollectionId()} + switch ref := scope.GetScopeReference().(type) { + case *apiV2.ResourceScope_CollectionScope: + ret.ScopeReference = &storage.ResourceScope_CollectionId{CollectionId: ref.CollectionScope.GetCollectionId()} + case *apiV2.ResourceScope_EntityScope: + if features.VulnerabilityReportsEnhancedFiltering.Enabled() { + ret.ScopeReference = &storage.ResourceScope_EntityScope{EntityScope: convertV2EntityScopeToStorage(ref.EntityScope)} + } } return ret } -func (s *serviceImpl) convertV2NotifierConfigToProto(notifier *apiV2.NotifierConfiguration) *storage.NotifierConfiguration { +// convertV2EntityScopeToStorage converts an apiV2.EntityScope into a storage.EntityScope. +func convertV2EntityScopeToStorage(es *apiV2.EntityScope) *storage.EntityScope { + if es == nil { + return nil + } + rules := make([]*storage.EntityScopeRule, 0, len(es.GetRules())) + for _, rule := range es.GetRules() { + sr := &storage.EntityScopeRule{ + Entity: v2EntityTypeToStorage(rule.GetEntity()), + Field: v2EntityFieldToStorage(rule.GetField()), + } + for _, rv := range rule.GetValues() { + sr.Values = append(sr.Values, &storage.RuleValue{ + Value: rv.GetValue(), + MatchType: v2MatchTypeToStorage(rv.GetMatchType()), + }) + } + rules = append(rules, sr) + } + return &storage.EntityScope{Rules: rules} +} + +// v2EntityTypeToStorage converts an API ScopeEntity to the corresponding storage EntityType. +func v2EntityTypeToStorage(e apiV2.ScopeEntity) storage.EntityType { + switch e { + case apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT: + return storage.EntityType_ENTITY_TYPE_DEPLOYMENT + case apiV2.ScopeEntity_SCOPE_ENTITY_NAMESPACE: + return storage.EntityType_ENTITY_TYPE_NAMESPACE + case apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER: + return storage.EntityType_ENTITY_TYPE_CLUSTER + default: + return storage.EntityType_ENTITY_TYPE_UNSET + } +} + +// v2EntityFieldToStorage converts an API ScopeField to the corresponding storage EntityField. +func v2EntityFieldToStorage(f apiV2.ScopeField) storage.EntityField { + switch f { + case apiV2.ScopeField_FIELD_ID: + return storage.EntityField_FIELD_ID + case apiV2.ScopeField_FIELD_NAME: + return storage.EntityField_FIELD_NAME + case apiV2.ScopeField_FIELD_LABEL: + return storage.EntityField_FIELD_LABEL + case apiV2.ScopeField_FIELD_ANNOTATION: + return storage.EntityField_FIELD_ANNOTATION + default: + return storage.EntityField_FIELD_UNSET + } +} + +// v2MatchTypeToStorage converts an API MatchType to the corresponding storage MatchType. +func v2MatchTypeToStorage(m apiV2.MatchType) storage.MatchType { + switch m { + case apiV2.MatchType_REGEX: + return storage.MatchType_REGEX + default: + return storage.MatchType_EXACT + } +} + +// ConvertV2NotifierConfigToProto converts apiV2.NotifierConfiguration to storage.NotifierConfiguration. +func ConvertV2NotifierConfigToProto(notifier *apiV2.NotifierConfiguration) *storage.NotifierConfiguration { if notifier == nil { return nil } @@ -150,8 +224,8 @@ func (s *serviceImpl) convertV2NotifierConfigToProto(notifier *apiV2.NotifierCon return ret } -// convertV2ScheduleToProto converts v2.ReportSchedule to storage.Schedule. Does not validate v2.ReportSchedule -func (s *serviceImpl) convertV2ScheduleToProto(schedule *apiV2.ReportSchedule) *storage.Schedule { +// ConvertV2ScheduleToProto converts v2.ReportSchedule to storage.Schedule. Does not validate v2.ReportSchedule. +func ConvertV2ScheduleToProto(schedule *apiV2.ReportSchedule) *storage.Schedule { if schedule == nil { return nil } @@ -194,22 +268,32 @@ func (s *serviceImpl) convertProtoReportConfigurationToV2(config *storage.Report Name: config.GetName(), Description: config.GetDescription(), Type: apiV2.ReportConfiguration_ReportType(config.GetType()), - Schedule: s.convertProtoScheduleToV2(config.GetSchedule()), + Schedule: ConvertProtoScheduleToV2(config.GetSchedule()), ResourceScope: resourceScope, } if config.GetVulnReportFilters() != nil { + vulnReportFilter := s.convertProtoVulnReportFiltersToV2(config.GetVulnReportFilters()) + + if config.GetResourceScope().GetEntityScope() != nil { + // fixability field value does not matter for enhanced filters. Hence, set the value to BOTH to disable the field. + vulnReportFilter.Fixability = apiV2.VulnerabilityReportFilters_BOTH + // severities field not apply when enhanced filters are enabled. Hence, set the value to empty list. + vulnReportFilter.Severities = []apiV2.VulnerabilityReportFilters_VulnerabilitySeverity{} + } ret.Filter = &apiV2.ReportConfiguration_VulnReportFilters{ - VulnReportFilters: s.convertProtoVulnReportFiltersToV2(config.GetVulnReportFilters()), + VulnReportFilters: vulnReportFilter, } } for _, notifier := range config.GetNotifiers() { - converted, err := s.convertProtoNotifierConfigToV2(notifier) + converted, err := ConvertProtoNotifierConfigToV2(s.notifierDatastore, notifier) if err != nil { return nil, err } - ret.Notifiers = append(ret.Notifiers, converted) + if converted != nil { + ret.Notifiers = append(ret.Notifiers, converted) + } } return ret, nil @@ -223,9 +307,12 @@ func (s *serviceImpl) convertProtoVulnReportFiltersToV2(filters *storage.Vulnera ret := &apiV2.VulnerabilityReportFilters{ Fixability: apiV2.VulnerabilityReportFilters_Fixability(filters.GetFixability()), - IncludeNvdCvss: filters.GetIncludeNvdCvss(), - IncludeEpssProbability: filters.GetIncludeEpssProbability(), - IncludeAdvisory: filters.GetIncludeAdvisory(), + IncludeNvdCvss: true, + IncludeEpssProbability: true, + IncludeAdvisory: true, + } + if features.VulnerabilityReportsEnhancedFiltering.Enabled() { + ret.Query = filters.GetQuery() } for _, severity := range filters.GetSeverities() { @@ -262,29 +349,93 @@ func (s *serviceImpl) convertProtoResourceScopeToV2(scope *storage.ResourceScope } ret := &apiV2.ResourceScope{} - if scope.GetScopeReference() != nil { - var collectionName string - collection, found, err := s.collectionDatastore.Get(allAccessCtx, scope.GetCollectionId()) + switch ref := scope.GetScopeReference().(type) { + case *storage.ResourceScope_CollectionId: + collection, found, err := s.collectionDatastore.Get(allAccessCtx, ref.CollectionId) if err != nil { return nil, err } if !found { - return nil, errors.Errorf("Collection with ID %s no longer exists", scope.GetCollectionId()) + return nil, errors.Errorf("Collection with ID %s no longer exists", ref.CollectionId) } - collectionName = collection.GetName() - ret.ScopeReference = &apiV2.ResourceScope_CollectionScope{ CollectionScope: &apiV2.CollectionReference{ - CollectionId: scope.GetCollectionId(), - CollectionName: collectionName, + CollectionId: ref.CollectionId, + CollectionName: collection.GetName(), }, } + case *storage.ResourceScope_EntityScope: + if features.VulnerabilityReportsEnhancedFiltering.Enabled() { + ret.ScopeReference = &apiV2.ResourceScope_EntityScope{EntityScope: convertStorageEntityScopeToV2(ref.EntityScope)} + } } return ret, nil } -// convertProtoNotifierConfigToV2 converts storage.NotifierConfiguration to apiV2.NotifierConfiguration -func (s *serviceImpl) convertProtoNotifierConfigToV2(notifierConfig *storage.NotifierConfiguration) (*apiV2.NotifierConfiguration, error) { +// convertStorageEntityScopeToV2 converts a storage.EntityScope into an apiV2.EntityScope. +func convertStorageEntityScopeToV2(es *storage.EntityScope) *apiV2.EntityScope { + if es == nil { + return nil + } + rules := make([]*apiV2.EntityScopeRule, 0, len(es.GetRules())) + for _, rule := range es.GetRules() { + ar := &apiV2.EntityScopeRule{ + Entity: storageEntityTypeToV2(rule.GetEntity()), + Field: storageEntityFieldToV2(rule.GetField()), + } + for _, rv := range rule.GetValues() { + ar.Values = append(ar.Values, &apiV2.RuleValue{ + Value: rv.GetValue(), + MatchType: storageMatchTypeToV2(rv.GetMatchType()), + }) + } + rules = append(rules, ar) + } + return &apiV2.EntityScope{Rules: rules} +} + +// storageEntityTypeToV2 converts a storage EntityType to the corresponding API ScopeEntity. +func storageEntityTypeToV2(e storage.EntityType) apiV2.ScopeEntity { + switch e { + case storage.EntityType_ENTITY_TYPE_DEPLOYMENT: + return apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT + case storage.EntityType_ENTITY_TYPE_NAMESPACE: + return apiV2.ScopeEntity_SCOPE_ENTITY_NAMESPACE + case storage.EntityType_ENTITY_TYPE_CLUSTER: + return apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER + default: + return apiV2.ScopeEntity_SCOPE_ENTITY_UNSET + } +} + +// storageEntityFieldToV2 converts a storage EntityField to the corresponding API ScopeField. +func storageEntityFieldToV2(f storage.EntityField) apiV2.ScopeField { + switch f { + case storage.EntityField_FIELD_ID: + return apiV2.ScopeField_FIELD_ID + case storage.EntityField_FIELD_NAME: + return apiV2.ScopeField_FIELD_NAME + case storage.EntityField_FIELD_LABEL: + return apiV2.ScopeField_FIELD_LABEL + case storage.EntityField_FIELD_ANNOTATION: + return apiV2.ScopeField_FIELD_ANNOTATION + default: + return apiV2.ScopeField_FIELD_UNSET + } +} + +// storageMatchTypeToV2 converts a storage MatchType to the corresponding API MatchType. +func storageMatchTypeToV2(m storage.MatchType) apiV2.MatchType { + switch m { + case storage.MatchType_REGEX: + return apiV2.MatchType_REGEX + default: + return apiV2.MatchType_EXACT + } +} + +// ConvertProtoNotifierConfigToV2 converts storage.NotifierConfiguration to apiV2.NotifierConfiguration. +func ConvertProtoNotifierConfigToV2(ds notifierDS.DataStore, notifierConfig *storage.NotifierConfiguration) (*apiV2.NotifierConfiguration, error) { if notifierConfig == nil { return nil, nil } @@ -293,7 +444,7 @@ func (s *serviceImpl) convertProtoNotifierConfigToV2(notifierConfig *storage.Not return nil, nil } - notifier, found, err := s.notifierDatastore.GetNotifier(allAccessCtx, notifierConfig.GetId()) + notifier, found, err := ds.GetNotifier(allAccessCtx, notifierConfig.GetId()) if err != nil { return nil, err } @@ -314,8 +465,8 @@ func (s *serviceImpl) convertProtoNotifierConfigToV2(notifierConfig *storage.Not }, nil } -// convertProtoScheduleToV2 converts storage.Schedule to v2.ReportSchedule. Does not validate storage.Schedule -func (s *serviceImpl) convertProtoScheduleToV2(schedule *storage.Schedule) *apiV2.ReportSchedule { +// ConvertProtoScheduleToV2 converts storage.Schedule to v2.ReportSchedule +func ConvertProtoScheduleToV2(schedule *storage.Schedule) *apiV2.ReportSchedule { if schedule == nil { return nil } @@ -363,13 +514,15 @@ func (s *serviceImpl) convertProtoReportCollectiontoV2(collection *storage.Colle } } -// convertProtoNotifierSnapshotToV2 converts notifiersnapshot proto to v2 -func (s *serviceImpl) convertProtoNotifierSnapshotToV2(notifierSnapshot *storage.NotifierSnapshot) *apiV2.NotifierConfiguration { +// ConvertProtoNotifierSnapshotToV2 converts storage.NotifierSnapshot to apiV2.NotifierConfiguration. +func ConvertProtoNotifierSnapshotToV2(notifierSnapshot *storage.NotifierSnapshot) *apiV2.NotifierConfiguration { if notifierSnapshot == nil { return nil } if notifierSnapshot.GetEmailConfig() == nil { - return &apiV2.NotifierConfiguration{} + return &apiV2.NotifierConfiguration{ + NotifierName: notifierSnapshot.GetNotifierName(), + } } return &apiV2.NotifierConfiguration{ @@ -432,25 +585,38 @@ func (s *serviceImpl) convertProtoReportSnapshotstoV2(snapshots []*storage.Repor } v2snaps := make([]*apiV2.ReportSnapshot, 0, len(snapshots)) for _, snapshot := range snapshots { + var resourceScope *apiV2.ResourceScope + if features.VulnerabilityReportsEnhancedFiltering.Enabled() { + resourceScope, err = s.convertProtoResourceScopeToV2(snapshot.GetResourceScope()) + if err != nil { + return nil, err + } + } + parentDir := snapshot.GetReportConfigurationId() + if snapshot.GetViewBasedVulnReportFilters() != nil { + parentDir = "view-based-report" + } snapshotv2 := &apiV2.ReportSnapshot{ ReportStatus: s.convertPrototoV2Reportstatus(snapshot.GetReportStatus()), ReportConfigId: snapshot.GetReportConfigurationId(), ReportJobId: snapshot.GetReportId(), Name: snapshot.GetName(), Description: snapshot.GetDescription(), + Type: apiV2.ReportSnapshot_ReportType(snapshot.GetType()), CollectionSnapshot: s.convertProtoReportCollectiontoV2(snapshot.GetCollection()), User: &apiV2.SlimUser{ Id: snapshot.GetRequester().GetId(), Name: snapshot.GetRequester().GetName(), }, - Schedule: s.convertProtoScheduleToV2(snapshot.GetSchedule()), + Schedule: ConvertProtoScheduleToV2(snapshot.GetSchedule()), + ResourceScope: resourceScope, + IsDownloadAvailable: blobNames.Contains(common.GetReportBlobPath(parentDir, snapshot.GetReportId())), Filter: &apiV2.ReportSnapshot_VulnReportFilters{ VulnReportFilters: s.convertProtoVulnReportFiltersToV2(snapshot.GetVulnReportFilters()), }, - IsDownloadAvailable: blobNames.Contains(common.GetReportBlobPath(snapshot.GetReportConfigurationId(), snapshot.GetReportId())), } for _, notifier := range snapshot.GetNotifiers() { - converted := s.convertProtoNotifierSnapshotToV2(notifier) + converted := ConvertProtoNotifierSnapshotToV2(notifier) if converted != nil { snapshotv2.Notifiers = append(snapshotv2.Notifiers, converted) } @@ -465,7 +631,7 @@ func (s *serviceImpl) getExistingBlobNames(snapshots []*storage.ReportSnapshot) blobNames := make([]string, 0) for _, snap := range snapshots { status := snap.GetReportStatus() - if status.GetReportNotificationMethod() == storage.ReportStatus_DOWNLOAD { + if status != nil && status.GetReportNotificationMethod() == storage.ReportStatus_DOWNLOAD { if status.GetRunState() == storage.ReportStatus_GENERATED || status.GetRunState() == storage.ReportStatus_DELIVERED { parentDir := snap.GetReportConfigurationId() diff --git a/central/reports/service/v2/convert_test.go b/central/reports/service/v2/convert_test.go index f180686c145c7..d362061f463ab 100644 --- a/central/reports/service/v2/convert_test.go +++ b/central/reports/service/v2/convert_test.go @@ -7,6 +7,7 @@ import ( collectionDSMocks "github.com/stackrox/rox/central/resourcecollection/datastore/mocks" apiV2 "github.com/stackrox/rox/generated/api/v2" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/protoassert" "github.com/stretchr/testify/assert" @@ -356,6 +357,89 @@ func (s *typeConversionTestSuite) TestConvertProtoReportConfigurationToV2() { } } +func (s *typeConversionTestSuite) TestConvertEntityScope() { + s.T().Setenv(features.VulnerabilityReportsEnhancedFiltering.EnvVar(), "true") + apiScope := &apiV2.EntityScope{ + Rules: []*apiV2.EntityScopeRule{ + { + Entity: apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, + Field: apiV2.ScopeField_FIELD_NAME, + Values: []*apiV2.RuleValue{ + {Value: "frontend", MatchType: apiV2.MatchType_EXACT}, + {Value: "back.*", MatchType: apiV2.MatchType_REGEX}, + }, + }, + { + Entity: apiV2.ScopeEntity_SCOPE_ENTITY_NAMESPACE, + Field: apiV2.ScopeField_FIELD_LABEL, + Values: []*apiV2.RuleValue{ + {Value: "env=prod", MatchType: apiV2.MatchType_EXACT}, + }, + }, + }, + } + storageScope := &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT, + Field: storage.EntityField_FIELD_NAME, + Values: []*storage.RuleValue{ + {Value: "frontend", MatchType: storage.MatchType_EXACT}, + {Value: "back.*", MatchType: storage.MatchType_REGEX}, + }, + }, + { + Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE, + Field: storage.EntityField_FIELD_LABEL, + Values: []*storage.RuleValue{ + {Value: "env=prod", MatchType: storage.MatchType_EXACT}, + }, + }, + }, + } + + s.T().Run("API to storage entity scope", func(t *testing.T) { + got := convertV2EntityScopeToStorage(apiScope) + protoassert.Equal(t, storageScope, got) + }) + + s.T().Run("Storage to API entity scope", func(t *testing.T) { + got := convertStorageEntityScopeToV2(storageScope) + protoassert.Equal(t, apiScope, got) + }) + + s.T().Run("convertProtoResourceScopeToV2 with entity scope does not call collection datastore", func(t *testing.T) { + // collection datastore must NOT be called — no EXPECT set up + scope := &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{EntityScope: storageScope}, + } + got, err := s.service.convertProtoResourceScopeToV2(scope) + assert.NoError(t, err) + protoassert.Equal(t, &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{EntityScope: apiScope}, + }, got) + }) + + s.T().Run("convertV2ResourceScopeToProto with entity scope", func(t *testing.T) { + scope := &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{EntityScope: apiScope}, + } + got := s.service.convertV2ResourceScopeToProto(scope) + protoassert.Equal(t, &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{EntityScope: storageScope}, + }, got) + }) + + s.T().Run("Query field round-trips through vuln filters", func(t *testing.T) { + apiFilters := &apiV2.VulnerabilityReportFilters{Query: "CVE:CVE-2024-1234"} + storageFilters := s.service.convertV2VulnReportFiltersToProto(apiFilters, nil) + assert.Equal(t, "CVE:CVE-2024-1234", storageFilters.GetQuery()) + + backToAPI := s.service.convertProtoVulnReportFiltersToV2(storageFilters) + assert.Equal(t, "CVE:CVE-2024-1234", backToAPI.GetQuery()) + }) +} + func (s *typeConversionTestSuite) TestConvertProtoScheduleToV2() { var cases = []struct { testname string @@ -395,12 +479,77 @@ func (s *typeConversionTestSuite) TestConvertProtoScheduleToV2() { for _, c := range cases { s.T().Run(c.testname, func(t *testing.T) { - converted := s.service.convertProtoScheduleToV2(c.schedule) + converted := ConvertProtoScheduleToV2(c.schedule) protoassert.Equal(t, c.result, converted) }) } } +func (s *typeConversionTestSuite) TestConvertSchedulePreservesTime() { + var cases = []struct { + testname string + hour int32 + minute int32 + }{ + { + testname: "Midnight", + hour: 0, + minute: 0, + }, + { + testname: "Afternoon with non-zero minute", + hour: 14, + minute: 30, + }, + { + testname: "End of day", + hour: 23, + minute: 59, + }, + } + + for _, c := range cases { + s.T().Run(c.testname+" v2 to storage", func(t *testing.T) { + v2Schedule := &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_DAILY, + Hour: c.hour, + Minute: c.minute, + } + converted := ConvertV2ScheduleToProto(v2Schedule) + assert.Equal(t, c.hour, converted.GetHour()) + assert.Equal(t, c.minute, converted.GetMinute()) + assert.Equal(t, storage.Schedule_DAILY, converted.GetIntervalType()) + }) + + s.T().Run(c.testname+" storage to v2", func(t *testing.T) { + storageSchedule := &storage.Schedule{ + IntervalType: storage.Schedule_DAILY, + Hour: c.hour, + Minute: c.minute, + } + converted := ConvertProtoScheduleToV2(storageSchedule) + assert.Equal(t, c.hour, converted.GetHour()) + assert.Equal(t, c.minute, converted.GetMinute()) + assert.Equal(t, apiV2.ReportSchedule_DAILY, converted.GetIntervalType()) + }) + + s.T().Run(c.testname+" weekly roundtrip", func(t *testing.T) { + v2Schedule := &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_WEEKLY, + Hour: c.hour, + Minute: c.minute, + Interval: &apiV2.ReportSchedule_DaysOfWeek_{ + DaysOfWeek: &apiV2.ReportSchedule_DaysOfWeek{Days: []int32{1}}, + }, + } + storage := ConvertV2ScheduleToProto(v2Schedule) + roundTripped := ConvertProtoScheduleToV2(storage) + assert.Equal(t, c.hour, roundTripped.GetHour()) + assert.Equal(t, c.minute, roundTripped.GetMinute()) + }) + } +} + func (s *typeConversionTestSuite) TestConvertV2ScheduleToProto() { var cases = []struct { testname string @@ -422,11 +571,16 @@ func (s *typeConversionTestSuite) TestConvertV2ScheduleToProto() { schedule: newScheduleV2(34, 12, []int32{}, []int32{1}), result: newSchedule(34, 12, []int32{}, false, []int32{1}), }, + { + testname: "Report Schedule with Daily interval", + schedule: newScheduleV2(12, 12, []int32{}, []int32{}), + result: newSchedule(12, 12, []int32{}, false, []int32{}), + }, } for _, c := range cases { s.T().Run(c.testname, func(t *testing.T) { - converted := s.service.convertV2ScheduleToProto(c.schedule) + converted := ConvertV2ScheduleToProto(c.schedule) protoassert.Equal(t, c.result, converted) }) } @@ -482,7 +636,7 @@ func newScheduleV2(minute int32, hour int32, weekdays []int32, daysOfMonth []int return &sched } if len(weekdays) == 0 { - sched.IntervalType = apiV2.ReportSchedule_UNSET + sched.IntervalType = apiV2.ReportSchedule_DAILY } else { sched.IntervalType = apiV2.ReportSchedule_WEEKLY sched.Interval = &apiV2.ReportSchedule_DaysOfWeek_{ @@ -493,3 +647,120 @@ func newScheduleV2(minute int32, hour int32, weekdays []int32, daysOfMonth []int } return &sched } + +func TestEnumConversions(t *testing.T) { + t.Run("v2EntityTypeToStorage", func(t *testing.T) { + tests := map[string]struct { + input apiV2.ScopeEntity + expected storage.EntityType + }{ + "deployment": {apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, storage.EntityType_ENTITY_TYPE_DEPLOYMENT}, + "namespace": {apiV2.ScopeEntity_SCOPE_ENTITY_NAMESPACE, storage.EntityType_ENTITY_TYPE_NAMESPACE}, + "cluster": {apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, storage.EntityType_ENTITY_TYPE_CLUSTER}, + "unset": {apiV2.ScopeEntity_SCOPE_ENTITY_UNSET, storage.EntityType_ENTITY_TYPE_UNSET}, + "unknown": {apiV2.ScopeEntity(999), storage.EntityType_ENTITY_TYPE_UNSET}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + result := v2EntityTypeToStorage(tc.input) + assert.Equal(t, tc.expected, result) + }) + } + }) + + t.Run("v2EntityFieldToStorage", func(t *testing.T) { + tests := map[string]struct { + input apiV2.ScopeField + expected storage.EntityField + }{ + "id": {apiV2.ScopeField_FIELD_ID, storage.EntityField_FIELD_ID}, + "name": {apiV2.ScopeField_FIELD_NAME, storage.EntityField_FIELD_NAME}, + "label": {apiV2.ScopeField_FIELD_LABEL, storage.EntityField_FIELD_LABEL}, + "annotation": {apiV2.ScopeField_FIELD_ANNOTATION, storage.EntityField_FIELD_ANNOTATION}, + "unset": {apiV2.ScopeField_FIELD_UNSET, storage.EntityField_FIELD_UNSET}, + "unknown": {apiV2.ScopeField(999), storage.EntityField_FIELD_UNSET}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + result := v2EntityFieldToStorage(tc.input) + assert.Equal(t, tc.expected, result) + }) + } + }) + + t.Run("storageEntityTypeToV2", func(t *testing.T) { + tests := map[string]struct { + input storage.EntityType + expected apiV2.ScopeEntity + }{ + "deployment": {storage.EntityType_ENTITY_TYPE_DEPLOYMENT, apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT}, + "namespace": {storage.EntityType_ENTITY_TYPE_NAMESPACE, apiV2.ScopeEntity_SCOPE_ENTITY_NAMESPACE}, + "cluster": {storage.EntityType_ENTITY_TYPE_CLUSTER, apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER}, + "unset": {storage.EntityType_ENTITY_TYPE_UNSET, apiV2.ScopeEntity_SCOPE_ENTITY_UNSET}, + "unknown": {storage.EntityType(999), apiV2.ScopeEntity_SCOPE_ENTITY_UNSET}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + result := storageEntityTypeToV2(tc.input) + assert.Equal(t, tc.expected, result) + }) + } + }) + + t.Run("storageEntityFieldToV2", func(t *testing.T) { + tests := map[string]struct { + input storage.EntityField + expected apiV2.ScopeField + }{ + "id": {storage.EntityField_FIELD_ID, apiV2.ScopeField_FIELD_ID}, + "name": {storage.EntityField_FIELD_NAME, apiV2.ScopeField_FIELD_NAME}, + "label": {storage.EntityField_FIELD_LABEL, apiV2.ScopeField_FIELD_LABEL}, + "annotation": {storage.EntityField_FIELD_ANNOTATION, apiV2.ScopeField_FIELD_ANNOTATION}, + "unset": {storage.EntityField_FIELD_UNSET, apiV2.ScopeField_FIELD_UNSET}, + "unknown": {storage.EntityField(999), apiV2.ScopeField_FIELD_UNSET}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + result := storageEntityFieldToV2(tc.input) + assert.Equal(t, tc.expected, result) + }) + } + }) +} + +func TestConvertProtoNotifierSnapshotToV2(t *testing.T) { + t.Run("nil snapshot", func(t *testing.T) { + result := ConvertProtoNotifierSnapshotToV2(nil) + assert.Nil(t, result) + }) + + t.Run("snapshot without email config", func(t *testing.T) { + snapshot := &storage.NotifierSnapshot{ + NotifierName: "test-notifier", + } + result := ConvertProtoNotifierSnapshotToV2(snapshot) + assert.NotNil(t, result) + assert.Equal(t, "test-notifier", result.GetNotifierName()) + }) + + t.Run("snapshot with email config", func(t *testing.T) { + snapshot := &storage.NotifierSnapshot{ + NotifierName: "test-notifier", + NotifierConfig: &storage.NotifierSnapshot_EmailConfig{ + EmailConfig: &storage.EmailNotifierConfiguration{ + NotifierId: "notifier-1", + MailingLists: []string{"user@example.com"}, + CustomSubject: "Test Subject", + CustomBody: "Test Body", + }, + }, + } + result := ConvertProtoNotifierSnapshotToV2(snapshot) + assert.NotNil(t, result) + assert.Equal(t, "test-notifier", result.GetNotifierName()) + assert.Equal(t, "notifier-1", result.GetEmailConfig().GetNotifierId()) + assert.Equal(t, []string{"user@example.com"}, result.GetEmailConfig().GetMailingLists()) + assert.Equal(t, "Test Subject", result.GetEmailConfig().GetCustomSubject()) + assert.Equal(t, "Test Body", result.GetEmailConfig().GetCustomBody()) + }) +} diff --git a/central/reports/service/v2/download.go b/central/reports/service/v2/download.go index 78644c66c95d6..8d974ec6e1cfc 100644 --- a/central/reports/service/v2/download.go +++ b/central/reports/service/v2/download.go @@ -34,17 +34,27 @@ func parseJobID(r *http.Request) (id string, err error) { return } -// NewDownloadHandler is an HTTP handler for downloading reports +// NewDownloadHandler is an HTTP handler for downloading image vulnerability reports. func NewDownloadHandler() http.HandlerFunc { + return newTypedDownloadHandler(storage.ReportSnapshot_VULNERABILITY) +} + +// NewNodeDownloadHandler is an HTTP handler for downloading node vulnerability reports. +func NewNodeDownloadHandler() http.HandlerFunc { + return newTypedDownloadHandler(storage.ReportSnapshot_NODE_VULNERABILITY) +} + +func newTypedDownloadHandler(expectedType storage.ReportSnapshot_ReportType) http.HandlerFunc { snapshotStore := snapshotDataStore.Singleton() blobStore := blobDS.Singleton() - handler := &downloadHandler{snapshotStore: snapshotStore, blobStore: blobStore} + handler := &downloadHandler{snapshotStore: snapshotStore, blobStore: blobStore, expectedType: expectedType} return handler.handle } type downloadHandler struct { snapshotStore snapshotDataStore.DataStore blobStore blobDS.Datastore + expectedType storage.ReportSnapshot_ReportType } func (h *downloadHandler) handle(w http.ResponseWriter, r *http.Request) { @@ -77,6 +87,12 @@ func (h *downloadHandler) handle(w http.ResponseWriter, r *http.Request) { return } + if rep.GetType() != h.expectedType { + httputil.WriteGRPCStyleError(w, codes.InvalidArgument, + errors.Errorf("report job %q is not a %s report; use the matching download endpoint", id, h.expectedType)) + return + } + if slimUser.GetId() != rep.GetRequester().GetId() { httputil.WriteGRPCStyleError(w, codes.PermissionDenied, errors.Errorf("Report cannot be downloaded by a user %q who did not request the report.", slimUser.GetId())) @@ -110,7 +126,8 @@ func (h *downloadHandler) handle(w http.ResponseWriter, r *http.Request) { ) parentDir := rep.GetReportConfigurationId() - if rep.GetViewBasedVulnReportFilters() != nil { + // View-based report blobs are stored under "view-based-report" dir, not under a dir named after reportConfigID. + if status.GetReportRequestType() == storage.ReportStatus_VIEW_BASED { parentDir = "view-based-report" } _, exists, err := h.blobStore.Get(ctx, common.GetReportBlobPath(parentDir, id), buf) diff --git a/central/reports/service/v2/download_test.go b/central/reports/service/v2/download_test.go index 3e41d17c36fb0..005e6c4710644 100644 --- a/central/reports/service/v2/download_test.go +++ b/central/reports/service/v2/download_test.go @@ -44,6 +44,7 @@ func (s *handlerTestSuite) SetupSuite() { s.handler = &downloadHandler{ snapshotStore: s.reportSnapshotDataStore, blobStore: s.blobStore, + expectedType: storage.ReportSnapshot_VULNERABILITY, } } @@ -116,6 +117,18 @@ func (s *handlerTestSuite) TestDownloadReport() { }, statusCode: http.StatusBadRequest, }, + { + desc: "Node vulnerability snapshot is rejected", + id: reportSnapshot.GetReportId(), + ctx: userContext, + mockGen: func() { + snap := reportSnapshot.CloneVT() + snap.Type = storage.ReportSnapshot_NODE_VULNERABILITY + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportSnapshot.GetReportId()). + Return(snap, true, nil).Times(1) + }, + statusCode: http.StatusBadRequest, + }, { desc: "Report is not ready yet", id: reportSnapshot.GetReportId(), @@ -201,6 +214,23 @@ func (s *handlerTestSuite) TestDownloadReport() { } } +func (s *handlerTestSuite) TestDownloadNodeReport_RejectsImageType() { + orig := s.handler.expectedType + s.handler.expectedType = storage.ReportSnapshot_NODE_VULNERABILITY + defer func() { s.handler.expectedType = orig }() + + reportSnapshot := fixtures.GetReportSnapshot() + reportSnapshot.ReportId = uuid.NewV4().String() + reportSnapshot.ReportStatus.RunState = storage.ReportStatus_GENERATED + reportSnapshot.ReportStatus.ReportNotificationMethod = storage.ReportStatus_DOWNLOAD + userContext := s.getContextForUser(reportSnapshot.GetRequester()) + + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportSnapshot.GetReportId()). + Return(reportSnapshot, true, nil).Times(1) + + s.downloadAndVerify(userContext, reportSnapshot.GetReportId(), http.StatusBadRequest, nil) +} + func (s *handlerTestSuite) downloadAndVerify(ctx context.Context, id string, code int, expectData []byte) { req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("https://example.com/api/reports/jobs/download?id=%s", id), nil) req = req.WithContext(ctx) diff --git a/central/reports/service/v2/node/convert.go b/central/reports/service/v2/node/convert.go new file mode 100644 index 0000000000000..00e8671837ee7 --- /dev/null +++ b/central/reports/service/v2/node/convert.go @@ -0,0 +1,373 @@ +package node + +import ( + "context" + + "github.com/pkg/errors" + "github.com/stackrox/rox/central/reports/common" + v2 "github.com/stackrox/rox/central/reports/service/v2" + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/set" +) + +var ( + storageRunStateToV2 = map[storage.ReportStatus_RunState]apiV2.ReportStatus_RunState{ + storage.ReportStatus_WAITING: apiV2.ReportStatus_WAITING, + storage.ReportStatus_PREPARING: apiV2.ReportStatus_PREPARING, + storage.ReportStatus_GENERATED: apiV2.ReportStatus_GENERATED, + storage.ReportStatus_DELIVERED: apiV2.ReportStatus_DELIVERED, + storage.ReportStatus_FAILURE: apiV2.ReportStatus_FAILURE, + } + + // Use this context only to populate notifier names and IsDownloadAvailable fields in converted responses + allAccessCtx = sac.WithAllAccess(context.Background()) +) + +// convertV2ReportConfigurationToProto converts v2.ReportConfiguration to storage.ReportConfiguration for node reports +func (s *serviceImpl) convertV2ReportConfigurationToProto(config *apiV2.ReportConfiguration, creator *storage.SlimUser, + accessScopeRules []*storage.SimpleAccessScope_Rules) (*storage.ReportConfiguration, error) { + if config == nil { + return nil, nil + } + + resourceScope, err := s.convertV2ResourceScopeToProto(config.GetResourceScope()) + if err != nil { + return nil, err + } + + ret := &storage.ReportConfiguration{ + Id: config.GetId(), + Name: config.GetName(), + Description: config.GetDescription(), + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + Schedule: v2.ConvertV2ScheduleToProto(config.GetSchedule()), + ResourceScope: resourceScope, + Creator: creator, + Version: 2, + } + + if config.GetNodeVulnReportFilters() != nil { + ret.Filter = &storage.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: convertV2NodeReportFiltersToProto(config.GetNodeVulnReportFilters(), accessScopeRules), + } + } + + for _, notifier := range config.GetNotifiers() { + ret.Notifiers = append(ret.Notifiers, v2.ConvertV2NotifierConfigToProto(notifier)) + } + + return ret, nil +} + +func convertV2NodeReportFiltersToProto(filters *apiV2.NodeVulnerabilityReportFilters, + accessScopeRules []*storage.SimpleAccessScope_Rules) *storage.NodeVulnerabilityReportFilters { + if filters == nil { + return nil + } + + ret := &storage.NodeVulnerabilityReportFilters{ + AccessScopeRules: accessScopeRules, + Query: filters.GetQuery(), + } + + switch filters.GetCvesSince().(type) { + case *apiV2.NodeVulnerabilityReportFilters_AllVuln: + ret.CvesSince = &storage.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: filters.GetAllVuln(), + } + } + + return ret +} + +func (s *serviceImpl) convertV2ResourceScopeToProto(scope *apiV2.ResourceScope) (*storage.ResourceScope, error) { + if scope == nil { + return nil, nil + } + + ret := &storage.ResourceScope{} + switch ref := scope.GetScopeReference().(type) { + case *apiV2.ResourceScope_EntityScope: + ret.ScopeReference = &storage.ResourceScope_EntityScope{ + EntityScope: convertV2EntityScopeToStorage(ref.EntityScope), + } + case nil: + return nil, errors.New("resource scope reference is required") + default: + return nil, errors.Errorf("unsupported resource scope type: %T", ref) + } + return ret, nil +} + +func convertV2EntityScopeToStorage(es *apiV2.EntityScope) *storage.EntityScope { + if es == nil { + return nil + } + rules := make([]*storage.EntityScopeRule, 0, len(es.GetRules())) + for _, rule := range es.GetRules() { + sr := &storage.EntityScopeRule{ + Entity: v2EntityTypeToStorage(rule.GetEntity()), + Field: v2EntityFieldToStorage(rule.GetField()), + } + for _, rv := range rule.GetValues() { + sr.Values = append(sr.Values, &storage.RuleValue{ + Value: rv.GetValue(), + MatchType: v2MatchTypeToStorage(rv.GetMatchType()), + }) + } + rules = append(rules, sr) + } + return &storage.EntityScope{Rules: rules} +} + +func v2EntityTypeToStorage(e apiV2.ScopeEntity) storage.EntityType { + switch e { + case apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER: + return storage.EntityType_ENTITY_TYPE_CLUSTER + default: + return storage.EntityType_ENTITY_TYPE_UNSET + } +} + +func v2EntityFieldToStorage(f apiV2.ScopeField) storage.EntityField { + switch f { + case apiV2.ScopeField_FIELD_ID: + return storage.EntityField_FIELD_ID + case apiV2.ScopeField_FIELD_NAME: + return storage.EntityField_FIELD_NAME + case apiV2.ScopeField_FIELD_LABEL: + return storage.EntityField_FIELD_LABEL + default: + return storage.EntityField_FIELD_UNSET + } +} + +func v2MatchTypeToStorage(m apiV2.MatchType) storage.MatchType { + switch m { + case apiV2.MatchType_REGEX: + return storage.MatchType_REGEX + default: + return storage.MatchType_EXACT + } +} + +// Proto to V2 conversions + +func (s *serviceImpl) convertProtoReportConfigurationToV2(config *storage.ReportConfiguration) (*apiV2.ReportConfiguration, error) { + if config == nil { + return nil, nil + } + + resourceScope := convertProtoResourceScopeToV2(config.GetResourceScope()) + + ret := &apiV2.ReportConfiguration{ + Id: config.GetId(), + Name: config.GetName(), + Description: config.GetDescription(), + Type: apiV2.ReportConfiguration_NODE_VULNERABILITY, + Schedule: v2.ConvertProtoScheduleToV2(config.GetSchedule()), + ResourceScope: resourceScope, + } + + if config.GetNodeVulnReportFilters() != nil { + ret.Filter = &apiV2.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: convertProtoNodeReportFiltersToV2(config.GetNodeVulnReportFilters()), + } + } + + for _, notifier := range config.GetNotifiers() { + converted, err := v2.ConvertProtoNotifierConfigToV2(s.notifierDatastore, notifier) + if err != nil { + return nil, err + } + if converted != nil { + ret.Notifiers = append(ret.Notifiers, converted) + } + } + + return ret, nil +} + +func convertProtoNodeReportFiltersToV2(filters *storage.NodeVulnerabilityReportFilters) *apiV2.NodeVulnerabilityReportFilters { + if filters == nil { + return nil + } + + ret := &apiV2.NodeVulnerabilityReportFilters{ + Query: filters.GetQuery(), + } + + switch filters.GetCvesSince().(type) { + case *storage.NodeVulnerabilityReportFilters_AllVuln: + ret.CvesSince = &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: filters.GetAllVuln(), + } + } + + return ret +} + +func convertProtoResourceScopeToV2(scope *storage.ResourceScope) *apiV2.ResourceScope { + if scope == nil { + return nil + } + + ret := &apiV2.ResourceScope{} + if ref, ok := scope.GetScopeReference().(*storage.ResourceScope_EntityScope); ok { + ret.ScopeReference = &apiV2.ResourceScope_EntityScope{EntityScope: convertStorageEntityScopeToV2(ref.EntityScope)} + } + return ret +} + +func convertStorageEntityScopeToV2(es *storage.EntityScope) *apiV2.EntityScope { + if es == nil { + return nil + } + rules := make([]*apiV2.EntityScopeRule, 0, len(es.GetRules())) + for _, rule := range es.GetRules() { + sr := &apiV2.EntityScopeRule{ + Entity: storageEntityTypeToV2(rule.GetEntity()), + Field: storageEntityFieldToV2(rule.GetField()), + } + for _, rv := range rule.GetValues() { + sr.Values = append(sr.Values, &apiV2.RuleValue{ + Value: rv.GetValue(), + MatchType: storageMatchTypeToV2(rv.GetMatchType()), + }) + } + rules = append(rules, sr) + } + return &apiV2.EntityScope{Rules: rules} +} + +func storageEntityTypeToV2(e storage.EntityType) apiV2.ScopeEntity { + switch e { + case storage.EntityType_ENTITY_TYPE_CLUSTER: + return apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER + default: + return apiV2.ScopeEntity_SCOPE_ENTITY_UNSET + } +} + +func storageEntityFieldToV2(f storage.EntityField) apiV2.ScopeField { + switch f { + case storage.EntityField_FIELD_ID: + return apiV2.ScopeField_FIELD_ID + case storage.EntityField_FIELD_NAME: + return apiV2.ScopeField_FIELD_NAME + case storage.EntityField_FIELD_LABEL: + return apiV2.ScopeField_FIELD_LABEL + default: + return apiV2.ScopeField_FIELD_UNSET + } +} + +func storageMatchTypeToV2(m storage.MatchType) apiV2.MatchType { + switch m { + case storage.MatchType_REGEX: + return apiV2.MatchType_REGEX + default: + return apiV2.MatchType_EXACT + } +} + +// reportBlobParentDir returns the blob directory for a snapshot. View-based +// report blobs are stored under "view-based-report", not under a report config ID. +func reportBlobParentDir(snapshot *storage.ReportSnapshot) string { + if snapshot.GetReportStatus().GetReportRequestType() == storage.ReportStatus_VIEW_BASED { + return "view-based-report" + } + return snapshot.GetReportConfigurationId() +} + +func isDownloadAvailable(blobNames set.FrozenStringSet, snapshot *storage.ReportSnapshot) bool { + return blobNames.Contains(common.GetReportBlobPath(reportBlobParentDir(snapshot), snapshot.GetReportId())) +} + +func (s *serviceImpl) convertProtoReportSnapshotToV2(snapshot *storage.ReportSnapshot, blobNames set.FrozenStringSet) (*apiV2.ReportSnapshot, error) { + if snapshot == nil { + return nil, nil + } + + resourceScope := convertProtoResourceScopeToV2(snapshot.GetResourceScope()) + + ret := &apiV2.ReportSnapshot{ + ReportStatus: convertPrototoV2Reportstatus(snapshot.GetReportStatus()), + ReportConfigId: snapshot.GetReportConfigurationId(), + ReportJobId: snapshot.GetReportId(), + Name: snapshot.GetName(), + Description: snapshot.GetDescription(), + AreaOfConcern: snapshot.GetAreaOfConcern(), + Type: apiV2.ReportSnapshot_NODE_VULNERABILITY, + User: &apiV2.SlimUser{ + Id: snapshot.GetRequester().GetId(), + Name: snapshot.GetRequester().GetName(), + }, + Schedule: v2.ConvertProtoScheduleToV2(snapshot.GetSchedule()), + ResourceScope: resourceScope, + IsDownloadAvailable: isDownloadAvailable(blobNames, snapshot), + } + + if snapshot.GetNodeVulnReportFilters() != nil { + ret.Filter = &apiV2.ReportSnapshot_NodeVulnReportFilters{ + NodeVulnReportFilters: convertProtoNodeReportFiltersToV2(snapshot.GetNodeVulnReportFilters()), + } + } + + for _, notifier := range snapshot.GetNotifiers() { + converted := v2.ConvertProtoNotifierSnapshotToV2(notifier) + if converted != nil { + ret.Notifiers = append(ret.Notifiers, converted) + } + } + + return ret, nil +} + +func convertPrototoV2Reportstatus(status *storage.ReportStatus) *apiV2.ReportStatus { + if status == nil { + return nil + } + + ret := &apiV2.ReportStatus{ + ReportNotificationMethod: apiV2.NotificationMethod(status.GetReportNotificationMethod()), + ReportRequestType: apiV2.ReportStatus_ReportMethod(status.GetReportRequestType()), + RunState: storageRunStateToV2[status.GetRunState()], + ErrorMsg: status.GetErrorMsg(), + CompletedAt: status.GetCompletedAt(), + } + + return ret +} + +func (s *serviceImpl) getExistingBlobNames(snapshots []*storage.ReportSnapshot) (set.FrozenStringSet, error) { + if len(snapshots) == 0 { + return set.NewFrozenStringSet(), nil + } + + blobNames := make([]string, 0, len(snapshots)) + for _, snapshot := range snapshots { + status := snapshot.GetReportStatus() + if status.GetReportNotificationMethod() == storage.ReportStatus_DOWNLOAD { + if status.GetRunState() == storage.ReportStatus_GENERATED || + status.GetRunState() == storage.ReportStatus_DELIVERED { + blobNames = append(blobNames, common.GetReportBlobPath(reportBlobParentDir(snapshot), snapshot.GetReportId())) + } + } + } + + if len(blobNames) == 0 { + return set.NewFrozenStringSet(), nil + } + + query := search.NewQueryBuilder().AddExactMatches(search.BlobName, blobNames...).ProtoQuery() + results, err := s.blobStore.Search(allAccessCtx, query) + if err != nil { + return set.NewFrozenStringSet(), errors.Wrap(err, "failed to search for report blobs") + } + + return search.ResultsToIDSet(results).Freeze(), nil +} diff --git a/central/reports/service/v2/node/convert_test.go b/central/reports/service/v2/node/convert_test.go new file mode 100644 index 0000000000000..e5401b88c7875 --- /dev/null +++ b/central/reports/service/v2/node/convert_test.go @@ -0,0 +1,859 @@ +package node + +import ( + "testing" + + blobDSMocks "github.com/stackrox/rox/central/blob/datastore/mocks" + notifierDSMocks "github.com/stackrox/rox/central/notifier/datastore/mocks" + "github.com/stackrox/rox/central/reports/common" + reportConfigDSMocks "github.com/stackrox/rox/central/reports/config/datastore/mocks" + schedulerMocks "github.com/stackrox/rox/central/reports/scheduler/v2/mocks" + v2 "github.com/stackrox/rox/central/reports/service/v2" + reportSnapshotDSMocks "github.com/stackrox/rox/central/reports/snapshot/datastore/mocks" + "github.com/stackrox/rox/central/reports/validation" + collectionDSMocks "github.com/stackrox/rox/central/resourcecollection/datastore/mocks" + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/set" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "go.uber.org/mock/gomock" +) + +func TestConversions(t *testing.T) { + suite.Run(t, new(ConversionTestSuite)) +} + +type ConversionTestSuite struct { + suite.Suite + mockCtrl *gomock.Controller + service *serviceImpl +} + +func (s *ConversionTestSuite) SetupTest() { + s.mockCtrl = gomock.NewController(s.T()) + reportConfigDataStore := reportConfigDSMocks.NewMockDataStore(s.mockCtrl) + reportSnapshotDataStore := reportSnapshotDSMocks.NewMockDataStore(s.mockCtrl) + collectionDataStore := collectionDSMocks.NewMockDataStore(s.mockCtrl) + notifierDataStore := notifierDSMocks.NewMockDataStore(s.mockCtrl) + blobStore := blobDSMocks.NewMockDatastore(s.mockCtrl) + scheduler := schedulerMocks.NewMockScheduler(s.mockCtrl) + validator := validation.New(reportConfigDataStore, reportSnapshotDataStore, collectionDataStore, notifierDataStore) + + s.service = &serviceImpl{ + reportConfigStore: reportConfigDataStore, + snapshotDatastore: reportSnapshotDataStore, + notifierDatastore: notifierDataStore, + scheduler: scheduler, + blobStore: blobStore, + validator: validator, + } +} + +func (s *ConversionTestSuite) TearDownTest() { + s.mockCtrl.Finish() +} + +func (s *ConversionTestSuite) TestScheduleConversions() { + testCases := []struct { + name string + v2Schedule *apiV2.ReportSchedule + }{ + { + name: "Daily schedule", + v2Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_DAILY, + Hour: 14, + Minute: 30, + }, + }, + { + name: "Weekly schedule", + v2Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_WEEKLY, + Hour: 9, + Minute: 0, + Interval: &apiV2.ReportSchedule_DaysOfWeek_{ + DaysOfWeek: &apiV2.ReportSchedule_DaysOfWeek{ + Days: []int32{1, 3, 5}, + }, + }, + }, + }, + { + name: "Monthly schedule", + v2Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_MONTHLY, + Hour: 10, + Minute: 15, + Interval: &apiV2.ReportSchedule_DaysOfMonth_{ + DaysOfMonth: &apiV2.ReportSchedule_DaysOfMonth{ + Days: []int32{1, 15}, + }, + }, + }, + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + protoSchedule := v2.ConvertV2ScheduleToProto(tc.v2Schedule) + assert.NotNil(s.T(), protoSchedule) + assert.Equal(s.T(), tc.v2Schedule.GetHour(), protoSchedule.GetHour()) + assert.Equal(s.T(), tc.v2Schedule.GetMinute(), protoSchedule.GetMinute()) + + v2Schedule := v2.ConvertProtoScheduleToV2(protoSchedule) + assert.NotNil(s.T(), v2Schedule) + assert.Equal(s.T(), tc.v2Schedule.GetIntervalType(), v2Schedule.GetIntervalType()) + assert.Equal(s.T(), tc.v2Schedule.GetHour(), v2Schedule.GetHour()) + assert.Equal(s.T(), tc.v2Schedule.GetMinute(), v2Schedule.GetMinute()) + + if tc.v2Schedule.GetDaysOfWeek() != nil { + assert.Equal(s.T(), tc.v2Schedule.GetDaysOfWeek().GetDays(), v2Schedule.GetDaysOfWeek().GetDays()) + } + if tc.v2Schedule.GetDaysOfMonth() != nil { + assert.Equal(s.T(), tc.v2Schedule.GetDaysOfMonth().GetDays(), v2Schedule.GetDaysOfMonth().GetDays()) + } + }) + } +} + +func (s *ConversionTestSuite) TestNotifierConversions() { + v2Notifier := &apiV2.NotifierConfiguration{ + NotifierConfig: &apiV2.NotifierConfiguration_EmailConfig{ + EmailConfig: &apiV2.EmailNotifierConfiguration{ + NotifierId: "notifier-123", + MailingLists: []string{"test@example.com", "team@example.com"}, + CustomSubject: "Custom Subject", + CustomBody: "Custom Body", + }, + }, + } + + protoNotifier := v2.ConvertV2NotifierConfigToProto(v2Notifier) + assert.NotNil(s.T(), protoNotifier) + assert.Equal(s.T(), "notifier-123", protoNotifier.GetId()) + assert.Equal(s.T(), "Custom Subject", protoNotifier.GetEmailConfig().GetCustomSubject()) + + // Mock GetNotifier for the conversion back + notifierDataStore := s.service.notifierDatastore.(*notifierDSMocks.MockDataStore) + notifierDataStore.EXPECT().GetNotifier(gomock.Any(), "notifier-123").Return(&storage.Notifier{ + Id: "notifier-123", + Name: "Test Notifier", + }, true, nil).Times(1) + + v2NotifierBack, err := v2.ConvertProtoNotifierConfigToV2(s.service.notifierDatastore, protoNotifier) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), v2NotifierBack) + assert.Equal(s.T(), "Test Notifier", v2NotifierBack.GetNotifierName()) + assert.Equal(s.T(), v2Notifier.GetEmailConfig().GetNotifierId(), v2NotifierBack.GetEmailConfig().GetNotifierId()) + assert.Equal(s.T(), v2Notifier.GetEmailConfig().GetMailingLists(), v2NotifierBack.GetEmailConfig().GetMailingLists()) +} + +func (s *ConversionTestSuite) TestNotifierSnapshotConversion() { + protoSnapshot := &storage.NotifierSnapshot{ + NotifierName: "Email Notifier", + NotifierConfig: &storage.NotifierSnapshot_EmailConfig{ + EmailConfig: &storage.EmailNotifierConfiguration{ + NotifierId: "notifier-456", + MailingLists: []string{"alerts@example.com"}, + CustomSubject: "Snapshot Subject", + CustomBody: "Snapshot Body", + }, + }, + } + + v2Config := v2.ConvertProtoNotifierSnapshotToV2(protoSnapshot) + assert.NotNil(s.T(), v2Config) + assert.Equal(s.T(), "Email Notifier", v2Config.GetNotifierName()) + assert.Equal(s.T(), "notifier-456", v2Config.GetEmailConfig().GetNotifierId()) + assert.Equal(s.T(), "Snapshot Subject", v2Config.GetEmailConfig().GetCustomSubject()) + assert.Equal(s.T(), []string{"alerts@example.com"}, v2Config.GetEmailConfig().GetMailingLists()) +} + +func (s *ConversionTestSuite) TestEntityScopeConversions() { + testCases := []struct { + name string + entityType apiV2.ScopeEntity + field apiV2.ScopeField + matchType apiV2.MatchType + }{ + { + name: "Cluster ID exact match", + entityType: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + field: apiV2.ScopeField_FIELD_ID, + matchType: apiV2.MatchType_EXACT, + }, + { + name: "Cluster name regex match", + entityType: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + field: apiV2.ScopeField_FIELD_NAME, + matchType: apiV2.MatchType_REGEX, + }, + { + name: "Cluster label exact match", + entityType: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + field: apiV2.ScopeField_FIELD_LABEL, + matchType: apiV2.MatchType_EXACT, + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + storageType := v2EntityTypeToStorage(tc.entityType) + v2Type := storageEntityTypeToV2(storageType) + assert.Equal(s.T(), tc.entityType, v2Type) + + storageField := v2EntityFieldToStorage(tc.field) + v2Field := storageEntityFieldToV2(storageField) + assert.Equal(s.T(), tc.field, v2Field) + + storageMatch := v2MatchTypeToStorage(tc.matchType) + v2Match := storageMatchTypeToV2(storageMatch) + assert.Equal(s.T(), tc.matchType, v2Match) + }) + } +} + +func (s *ConversionTestSuite) TestReportStatusConversion() { + testCases := map[string]struct { + requestType storage.ReportStatus_RunMethod + want apiV2.ReportStatus_ReportMethod + }{ + "on-demand": { + requestType: storage.ReportStatus_ON_DEMAND, + want: apiV2.ReportStatus_ON_DEMAND, + }, + "scheduled": { + requestType: storage.ReportStatus_SCHEDULED, + want: apiV2.ReportStatus_SCHEDULED, + }, + "view-based": { + requestType: storage.ReportStatus_VIEW_BASED, + want: apiV2.ReportStatus_ReportMethod(storage.ReportStatus_VIEW_BASED), + }, + } + + for name, tc := range testCases { + s.Run(name, func() { + protoStatus := &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + ReportRequestType: tc.requestType, + ErrorMsg: "test error", + } + + v2Status := convertPrototoV2Reportstatus(protoStatus) + assert.NotNil(s.T(), v2Status) + assert.Equal(s.T(), apiV2.ReportStatus_GENERATED, v2Status.GetRunState()) + assert.Equal(s.T(), apiV2.NotificationMethod_DOWNLOAD, v2Status.GetReportNotificationMethod()) + assert.Equal(s.T(), "test error", v2Status.GetErrorMsg()) + assert.Equal(s.T(), tc.want, v2Status.GetReportRequestType()) + }) + } +} + +func (s *ConversionTestSuite) TestNodeFiltersConversion() { + v2Filters := &apiV2.NodeVulnerabilityReportFilters{ + Query: "Cluster:prod", + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + } + + accessScopeRules := []*storage.SimpleAccessScope_Rules{ + { + IncludedClusters: []string{"cluster-1"}, + }, + } + + protoFilters := convertV2NodeReportFiltersToProto(v2Filters, accessScopeRules) + assert.NotNil(s.T(), protoFilters) + assert.Equal(s.T(), "Cluster:prod", protoFilters.GetQuery()) + assert.True(s.T(), protoFilters.GetAllVuln()) + protoassert.SlicesEqual(s.T(), accessScopeRules, protoFilters.GetAccessScopeRules()) + + v2FiltersBack := convertProtoNodeReportFiltersToV2(protoFilters) + assert.NotNil(s.T(), v2FiltersBack) + assert.Equal(s.T(), v2Filters.GetQuery(), v2FiltersBack.GetQuery()) + assert.True(s.T(), v2FiltersBack.GetAllVuln()) +} + +func (s *ConversionTestSuite) TestReportConfigurationRoundTrip() { + creator := &storage.SlimUser{ + Id: "user-123", + Name: "Test User", + } + + accessScopeRules := []*storage.SimpleAccessScope_Rules{ + { + IncludedClusters: []string{"cluster-1", "cluster-2"}, + }, + } + + v2Config := &apiV2.ReportConfiguration{ + Id: "config-123", + Name: "Node Report", + Description: "Test node vulnerability report", + Type: apiV2.ReportConfiguration_NODE_VULNERABILITY, + ResourceScope: &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{ + EntityScope: &apiV2.EntityScope{ + Rules: []*apiV2.EntityScopeRule{ + { + Entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + Field: apiV2.ScopeField_FIELD_ID, + Values: []*apiV2.RuleValue{ + { + Value: "cluster-1", + MatchType: apiV2.MatchType_EXACT, + }, + }, + }, + }, + }, + }, + }, + Filter: &apiV2.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + Query: "Cluster:prod", + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_WEEKLY, + Hour: 9, + Minute: 0, + Interval: &apiV2.ReportSchedule_DaysOfWeek_{ + DaysOfWeek: &apiV2.ReportSchedule_DaysOfWeek{ + Days: []int32{1, 3, 5}, + }, + }, + }, + Notifiers: []*apiV2.NotifierConfiguration{ + { + NotifierConfig: &apiV2.NotifierConfiguration_EmailConfig{ + EmailConfig: &apiV2.EmailNotifierConfiguration{ + NotifierId: "email-1", + MailingLists: []string{"team@example.com"}, + }, + }, + }, + }, + } + + protoConfig, err := s.service.convertV2ReportConfigurationToProto(v2Config, creator, accessScopeRules) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), protoConfig) + assert.Equal(s.T(), v2Config.GetId(), protoConfig.GetId()) + assert.Equal(s.T(), v2Config.GetName(), protoConfig.GetName()) + assert.Equal(s.T(), storage.ReportConfiguration_NODE_VULNERABILITY, protoConfig.GetType()) + protoassert.Equal(s.T(), creator, protoConfig.GetCreator()) + + // Mock GetNotifier for the conversion back + notifierDataStore := s.service.notifierDatastore.(*notifierDSMocks.MockDataStore) + notifierDataStore.EXPECT().GetNotifier(gomock.Any(), "email-1").Return(&storage.Notifier{ + Id: "email-1", + Name: "Email Notifier", + }, true, nil).Times(1) + + v2ConfigBack, err := s.service.convertProtoReportConfigurationToV2(protoConfig) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), v2ConfigBack) + assert.Equal(s.T(), v2Config.GetId(), v2ConfigBack.GetId()) + assert.Equal(s.T(), v2Config.GetName(), v2ConfigBack.GetName()) + assert.Equal(s.T(), apiV2.ReportConfiguration_NODE_VULNERABILITY, v2ConfigBack.GetType()) + assert.Len(s.T(), v2ConfigBack.GetNotifiers(), 1) + assert.Equal(s.T(), "Email Notifier", v2ConfigBack.GetNotifiers()[0].GetNotifierName()) +} + +func (s *ConversionTestSuite) TestNilConversions() { + assert.Nil(s.T(), v2.ConvertV2ScheduleToProto(nil)) + assert.Nil(s.T(), v2.ConvertProtoScheduleToV2(nil)) + assert.Nil(s.T(), v2.ConvertV2NotifierConfigToProto(nil)) + + result, err := v2.ConvertProtoNotifierConfigToV2(s.service.notifierDatastore, nil) + assert.NoError(s.T(), err) + assert.Nil(s.T(), result) + + assert.Nil(s.T(), v2.ConvertProtoNotifierSnapshotToV2(nil)) + assert.Nil(s.T(), convertPrototoV2Reportstatus(nil)) + assert.Nil(s.T(), convertV2NodeReportFiltersToProto(nil, nil)) + assert.Nil(s.T(), convertProtoNodeReportFiltersToV2(nil)) +} + +func (s *ConversionTestSuite) TestEntityTypeEdgeCases() { + assert.Equal(s.T(), storage.EntityType_ENTITY_TYPE_UNSET, v2EntityTypeToStorage(apiV2.ScopeEntity_SCOPE_ENTITY_UNSET)) + assert.Equal(s.T(), apiV2.ScopeEntity_SCOPE_ENTITY_UNSET, storageEntityTypeToV2(storage.EntityType_ENTITY_TYPE_UNSET)) + + assert.Equal(s.T(), storage.EntityField_FIELD_UNSET, v2EntityFieldToStorage(apiV2.ScopeField_FIELD_UNSET)) + assert.Equal(s.T(), apiV2.ScopeField_FIELD_UNSET, storageEntityFieldToV2(storage.EntityField_FIELD_UNSET)) + + assert.Equal(s.T(), storage.MatchType_EXACT, v2MatchTypeToStorage(apiV2.MatchType_EXACT)) + assert.Equal(s.T(), apiV2.MatchType_EXACT, storageMatchTypeToV2(storage.MatchType_EXACT)) +} + +func (s *ConversionTestSuite) TestConvertProtoReportSnapshotToV2() { + configID := "test-config" + reportID := "test-report" + + snapshot := &storage.ReportSnapshot{ + ReportConfigurationId: configID, + ReportId: reportID, + Name: "Test Report", + Description: "Test description", + AreaOfConcern: "User Workloads", + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + Requester: &storage.SlimUser{ + Id: "user-1", + Name: "Test User", + }, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{}, + }, + }, + Filter: &storage.ReportSnapshot_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "test", + }, + }, + } + + blobStore := s.service.blobStore.(*blobDSMocks.MockDatastore) + blobStore.EXPECT().Search(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + + result, err := s.service.convertProtoReportSnapshotToV2(snapshot, set.NewFrozenStringSet()) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), result) + assert.Equal(s.T(), reportID, result.GetReportJobId()) + assert.Equal(s.T(), configID, result.GetReportConfigId()) + assert.Equal(s.T(), "Test Report", result.GetName()) + assert.Equal(s.T(), "Test description", result.GetDescription()) + assert.Equal(s.T(), "User Workloads", result.GetAreaOfConcern()) + assert.False(s.T(), result.GetIsDownloadAvailable()) +} + +func (s *ConversionTestSuite) TestReportSnapshotDownloadAvailable() { + configID := "test-config" + reportID := "test-report" + + testCases := map[string]struct { + snapshot *storage.ReportSnapshot + blobNames set.FrozenStringSet + available bool + }{ + "config-based with matching blob": { + snapshot: &storage.ReportSnapshot{ + ReportConfigurationId: configID, + ReportId: reportID, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + }, + blobNames: set.NewFrozenStringSet(common.GetReportBlobPath(configID, reportID)), + available: true, + }, + "view-based with matching blob": { + snapshot: &storage.ReportSnapshot{ + ReportId: reportID, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + ReportRequestType: storage.ReportStatus_VIEW_BASED, + }, + }, + blobNames: set.NewFrozenStringSet(common.GetReportBlobPath("view-based-report", reportID)), + available: true, + }, + "view-based without blob": { + snapshot: &storage.ReportSnapshot{ + ReportId: reportID, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + ReportRequestType: storage.ReportStatus_VIEW_BASED, + }, + }, + blobNames: set.NewFrozenStringSet(), + available: false, + }, + "view-based does not match config-id blob path": { + snapshot: &storage.ReportSnapshot{ + ReportConfigurationId: configID, + ReportId: reportID, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + ReportRequestType: storage.ReportStatus_VIEW_BASED, + }, + }, + blobNames: set.NewFrozenStringSet(common.GetReportBlobPath(configID, reportID)), + available: false, + }, + } + + for name, tc := range testCases { + s.Run(name, func() { + result, err := s.service.convertProtoReportSnapshotToV2(tc.snapshot, tc.blobNames) + assert.NoError(s.T(), err) + assert.Equal(s.T(), tc.available, result.GetIsDownloadAvailable()) + }) + } +} + +func (s *ConversionTestSuite) TestRunStateConversions() { + assert.Equal(s.T(), apiV2.ReportStatus_WAITING, storageRunStateToV2[storage.ReportStatus_WAITING]) + assert.Equal(s.T(), apiV2.ReportStatus_PREPARING, storageRunStateToV2[storage.ReportStatus_PREPARING]) + assert.Equal(s.T(), apiV2.ReportStatus_GENERATED, storageRunStateToV2[storage.ReportStatus_GENERATED]) + assert.Equal(s.T(), apiV2.ReportStatus_DELIVERED, storageRunStateToV2[storage.ReportStatus_DELIVERED]) + assert.Equal(s.T(), apiV2.ReportStatus_FAILURE, storageRunStateToV2[storage.ReportStatus_FAILURE]) +} + +func (s *ConversionTestSuite) TestResourceScopeConversions() { + v2Scope := &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{ + EntityScope: &apiV2.EntityScope{ + Rules: []*apiV2.EntityScopeRule{ + { + Entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + Field: apiV2.ScopeField_FIELD_NAME, + Values: []*apiV2.RuleValue{ + { + Value: "prod-cluster", + MatchType: apiV2.MatchType_REGEX, + }, + }, + }, + }, + }, + }, + } + + protoScope, err := s.service.convertV2ResourceScopeToProto(v2Scope) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), protoScope) + + v2ScopeBack := convertProtoResourceScopeToV2(protoScope) + assert.NotNil(s.T(), v2ScopeBack) + assert.NotNil(s.T(), v2ScopeBack.GetEntityScope()) + assert.Len(s.T(), v2ScopeBack.GetEntityScope().GetRules(), 1) +} + +func (s *ConversionTestSuite) TestEmptyResourceScope() { + result, err := s.service.convertV2ResourceScopeToProto(nil) + assert.NoError(s.T(), err) + assert.Nil(s.T(), result) + + v2Result := convertProtoResourceScopeToV2(nil) + assert.Nil(s.T(), v2Result) +} + +func (s *ConversionTestSuite) TestReportConfigurationWithoutNotifiers() { + creator := &storage.SlimUser{ + Id: "user-123", + Name: "Test User", + } + + v2Config := &apiV2.ReportConfiguration{ + Id: "config-no-notifiers", + Name: "Node Report No Notifiers", + Description: "Test without notifiers", + Type: apiV2.ReportConfiguration_NODE_VULNERABILITY, + ResourceScope: &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{ + EntityScope: &apiV2.EntityScope{ + Rules: []*apiV2.EntityScopeRule{ + { + Entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + Field: apiV2.ScopeField_FIELD_ID, + Values: []*apiV2.RuleValue{ + { + Value: "cluster-1", + MatchType: apiV2.MatchType_EXACT, + }, + }, + }, + }, + }, + }, + }, + Filter: &apiV2.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + Query: "Cluster:prod", + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + Notifiers: nil, + } + + protoConfig, err := s.service.convertV2ReportConfigurationToProto(v2Config, creator, nil) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), protoConfig) + assert.Empty(s.T(), protoConfig.GetNotifiers()) +} + +func (s *ConversionTestSuite) TestProtoReportConfigurationToV2() { + protoConfig := &storage.ReportConfiguration{ + Id: "config-proto", + Name: "Proto Config", + Description: "Test proto to v2", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_ID, + Values: []*storage.RuleValue{ + { + Value: "cluster-1", + MatchType: storage.MatchType_EXACT, + }, + }, + }, + }, + }, + }, + }, + Filter: &storage.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "test query", + CvesSince: &storage.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + Notifiers: []*storage.NotifierConfiguration{ + { + Ref: &storage.NotifierConfiguration_Id{ + Id: "notifier-1", + }, + NotifierConfig: &storage.NotifierConfiguration_EmailConfig{ + EmailConfig: &storage.EmailNotifierConfiguration{ + NotifierId: "notifier-1", + MailingLists: []string{"team@example.com"}, + }, + }, + }, + }, + } + + // Mock GetNotifier for the conversion + notifierDataStore := s.service.notifierDatastore.(*notifierDSMocks.MockDataStore) + notifierDataStore.EXPECT().GetNotifier(gomock.Any(), "notifier-1").Return(&storage.Notifier{ + Id: "notifier-1", + Name: "Notifier 1", + }, true, nil).Times(1) + + v2Config, err := s.service.convertProtoReportConfigurationToV2(protoConfig) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), v2Config) + assert.Equal(s.T(), "config-proto", v2Config.GetId()) + assert.Equal(s.T(), "Proto Config", v2Config.GetName()) + assert.Equal(s.T(), apiV2.ReportConfiguration_NODE_VULNERABILITY, v2Config.GetType()) + assert.Len(s.T(), v2Config.GetNotifiers(), 1) + assert.Equal(s.T(), "Notifier 1", v2Config.GetNotifiers()[0].GetNotifierName()) +} + +func (s *ConversionTestSuite) TestProtoReportConfigurationToV2_SkipsNotifierWithoutEmail() { + protoConfig := &storage.ReportConfiguration{ + Id: "config-proto", + Name: "Proto Config", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + Notifiers: []*storage.NotifierConfiguration{ + { + Ref: &storage.NotifierConfiguration_Id{Id: "notifier-1"}, + }, + }, + } + + v2Config, err := s.service.convertProtoReportConfigurationToV2(protoConfig) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), v2Config) + assert.Empty(s.T(), v2Config.GetNotifiers()) +} + +func (s *ConversionTestSuite) TestGetExistingBlobNames_NotDownloadMethod() { + snapshot := &storage.ReportSnapshot{ + ReportConfigurationId: "config-1", + ReportId: "report-1", + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_EMAIL, + }, + } + + result, err := s.service.getExistingBlobNames([]*storage.ReportSnapshot{snapshot}) + assert.NoError(s.T(), err) + assert.Equal(s.T(), 0, result.Cardinality()) +} + +func (s *ConversionTestSuite) TestGetExistingBlobNames_WrongRunState() { + testCases := []struct { + name string + runState storage.ReportStatus_RunState + }{ + {"Waiting", storage.ReportStatus_WAITING}, + {"Preparing", storage.ReportStatus_PREPARING}, + {"Failure", storage.ReportStatus_FAILURE}, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + snapshot := &storage.ReportSnapshot{ + ReportConfigurationId: "config-1", + ReportId: "report-1", + ReportStatus: &storage.ReportStatus{ + RunState: tc.runState, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + } + + result, err := s.service.getExistingBlobNames([]*storage.ReportSnapshot{snapshot}) + assert.NoError(s.T(), err) + assert.Equal(s.T(), 0, result.Cardinality()) + }) + } +} + +func (s *ConversionTestSuite) TestGetExistingBlobNames_BlobSearchError() { + snapshot := &storage.ReportSnapshot{ + ReportConfigurationId: "config-1", + ReportId: "report-1", + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + } + + blobStore := s.service.blobStore.(*blobDSMocks.MockDatastore) + blobStore.EXPECT().Search(gomock.Any(), gomock.Any()).Return(nil, assert.AnError).Times(1) + + result, err := s.service.getExistingBlobNames([]*storage.ReportSnapshot{snapshot}) + assert.Error(s.T(), err) + assert.Equal(s.T(), 0, result.Cardinality()) +} + +func (s *ConversionTestSuite) TestGetExistingBlobNames_NoBlobFound() { + snapshot := &storage.ReportSnapshot{ + ReportConfigurationId: "config-1", + ReportId: "report-1", + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + } + + blobStore := s.service.blobStore.(*blobDSMocks.MockDatastore) + blobStore.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{}, nil).Times(1) + + result, err := s.service.getExistingBlobNames([]*storage.ReportSnapshot{snapshot}) + assert.NoError(s.T(), err) + assert.Equal(s.T(), 0, result.Cardinality()) +} + +func (s *ConversionTestSuite) TestGetExistingBlobNames_BlobFound() { + testCases := []struct { + name string + runState storage.ReportStatus_RunState + }{ + {"Generated state", storage.ReportStatus_GENERATED}, + {"Delivered state", storage.ReportStatus_DELIVERED}, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + snapshot := &storage.ReportSnapshot{ + ReportConfigurationId: "config-1", + ReportId: "report-1", + ReportStatus: &storage.ReportStatus{ + RunState: tc.runState, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + } + + blobStore := s.service.blobStore.(*blobDSMocks.MockDatastore) + blobStore.EXPECT().Search(allAccessCtx, gomock.Any()).Return([]search.Result{ + {ID: common.GetReportBlobPath("config-1", "report-1")}, + }, nil).Times(1) + + result, err := s.service.getExistingBlobNames([]*storage.ReportSnapshot{snapshot}) + assert.NoError(s.T(), err) + assert.True(s.T(), result.Contains(common.GetReportBlobPath("config-1", "report-1"))) + }) + } +} + +func (s *ConversionTestSuite) TestGetExistingBlobNames_ViewBased() { + snapshot := &storage.ReportSnapshot{ + ReportId: "report-1", + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + ReportRequestType: storage.ReportStatus_VIEW_BASED, + }, + } + + blobStore := s.service.blobStore.(*blobDSMocks.MockDatastore) + blobStore.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{ + {ID: common.GetReportBlobPath("view-based-report", "report-1")}, + }, nil).Times(1) + + result, err := s.service.getExistingBlobNames([]*storage.ReportSnapshot{snapshot}) + assert.NoError(s.T(), err) + assert.True(s.T(), result.Contains(common.GetReportBlobPath("view-based-report", "report-1"))) +} + +func (s *ConversionTestSuite) TestReportBlobParentDir() { + testCases := map[string]struct { + snapshot *storage.ReportSnapshot + want string + }{ + "config-based on-demand": { + snapshot: &storage.ReportSnapshot{ + ReportConfigurationId: "config-1", + ReportStatus: &storage.ReportStatus{ + ReportRequestType: storage.ReportStatus_ON_DEMAND, + }, + }, + want: "config-1", + }, + "config-based scheduled": { + snapshot: &storage.ReportSnapshot{ + ReportConfigurationId: "config-1", + ReportStatus: &storage.ReportStatus{ + ReportRequestType: storage.ReportStatus_SCHEDULED, + }, + }, + want: "config-1", + }, + "view-based uses shared blob directory": { + snapshot: &storage.ReportSnapshot{ + ReportConfigurationId: "config-1", + ReportStatus: &storage.ReportStatus{ + ReportRequestType: storage.ReportStatus_VIEW_BASED, + }, + }, + want: "view-based-report", + }, + } + + for name, tc := range testCases { + s.Run(name, func() { + assert.Equal(s.T(), tc.want, reportBlobParentDir(tc.snapshot)) + }) + } +} diff --git a/central/reports/service/v2/node/service.go b/central/reports/service/v2/node/service.go new file mode 100644 index 0000000000000..ddffe2c64c06c --- /dev/null +++ b/central/reports/service/v2/node/service.go @@ -0,0 +1,38 @@ +package node + +import ( + "context" + + blobDS "github.com/stackrox/rox/central/blob/datastore" + notifierDS "github.com/stackrox/rox/central/notifier/datastore" + reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" + schedulerV2 "github.com/stackrox/rox/central/reports/scheduler/v2" + snapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + "github.com/stackrox/rox/central/reports/validation" + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/pkg/grpc" + "github.com/stackrox/rox/pkg/postgres" +) + +// Service provides the interface to the gRPC service for node vulnerability reports. +type Service interface { + grpc.APIService + + AuthFuncOverride(ctx context.Context, fullMethodName string) (context.Context, error) + apiV2.NodeReportServiceServer +} + +// New returns a new instance of the node report service. +func New(reportConfigStore reportConfigDS.DataStore, snapshotDatastore snapshotDS.DataStore, + notifierDatastore notifierDS.DataStore, scheduler schedulerV2.Scheduler, blobStore blobDS.Datastore, + validator *validation.Validator, db postgres.DB) Service { + return &serviceImpl{ + reportConfigStore: reportConfigStore, + snapshotDatastore: snapshotDatastore, + notifierDatastore: notifierDatastore, + scheduler: scheduler, + blobStore: blobStore, + validator: validator, + db: db, + } +} diff --git a/central/reports/service/v2/node/service_impl.go b/central/reports/service/v2/node/service_impl.go new file mode 100644 index 0000000000000..57e12d15e4b77 --- /dev/null +++ b/central/reports/service/v2/node/service_impl.go @@ -0,0 +1,617 @@ +package node + +import ( + "context" + + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/pkg/errors" + blobDS "github.com/stackrox/rox/central/blob/datastore" + notifierDS "github.com/stackrox/rox/central/notifier/datastore" + "github.com/stackrox/rox/central/reports/common" + reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" + schedulerV2 "github.com/stackrox/rox/central/reports/scheduler/v2" + reportsv2 "github.com/stackrox/rox/central/reports/service/v2" + snapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + "github.com/stackrox/rox/central/reports/validation" + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/auth/permissions" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/grpc/authn" + "github.com/stackrox/rox/pkg/grpc/authz" + "github.com/stackrox/rox/pkg/grpc/authz/perrpc" + "github.com/stackrox/rox/pkg/grpc/authz/user" + "github.com/stackrox/rox/pkg/postgres" + pgNotify "github.com/stackrox/rox/pkg/postgres/notify" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/paginated" + "github.com/stackrox/rox/pkg/stringutils" + "google.golang.org/grpc" +) + +const ( + maxPaginationLimit = 1000 +) + +var ( + workflowSAC = sac.ForResource(resources.WorkflowAdministration) + + authorizer = perrpc.FromMap(map[authz.Authorizer][]string{ + user.With(permissions.View(resources.WorkflowAdministration), permissions.View(resources.Node), permissions.View(resources.Cluster)): { + apiV2.NodeReportService_ListNodeReportConfigurations_FullMethodName, + apiV2.NodeReportService_GetNodeReportConfiguration_FullMethodName, + apiV2.NodeReportService_CountNodeReportConfigurations_FullMethodName, + apiV2.NodeReportService_GetNodeReportHistory_FullMethodName, + apiV2.NodeReportService_GetMyNodeReportHistory_FullMethodName, + }, + user.With(permissions.Modify(resources.WorkflowAdministration), permissions.View(resources.Integration), permissions.View(resources.Node), permissions.View(resources.Cluster)): { + apiV2.NodeReportService_PostNodeReportConfiguration_FullMethodName, + apiV2.NodeReportService_UpdateNodeReportConfiguration_FullMethodName, + }, + user.With(permissions.Modify(resources.WorkflowAdministration), permissions.View(resources.Node), permissions.View(resources.Cluster)): { + apiV2.NodeReportService_DeleteNodeReportConfiguration_FullMethodName, + apiV2.NodeReportService_RunNodeReport_FullMethodName, + }, + // Cancel/Delete job match the image service: View is enough for the requester's own job. + // Modify(WorkflowAdministration) is checked in-handler when cancelling another user's job. + // Design table listed Modify(WorkflowAdministration) on those RPCs; image-parity won in review. + // GetNodeReportStatus uses the same View(Node)+View(Cluster) set so view-based users can poll jobs. + user.With(permissions.View(resources.Node), permissions.View(resources.Cluster)): { + apiV2.NodeReportService_GetNodeReportStatus_FullMethodName, + apiV2.NodeReportService_GetViewBasedNodeReportHistory_FullMethodName, + apiV2.NodeReportService_GetViewBasedMyNodeReportHistory_FullMethodName, + apiV2.NodeReportService_PostViewBasedNodeReport_FullMethodName, + apiV2.NodeReportService_DeleteNodeReport_FullMethodName, + apiV2.NodeReportService_CancelNodeReport_FullMethodName, + }, + }) +) + +type serviceImpl struct { + apiV2.UnimplementedNodeReportServiceServer + reportConfigStore reportConfigDS.DataStore + snapshotDatastore snapshotDS.DataStore + notifierDatastore notifierDS.DataStore + scheduler schedulerV2.Scheduler + blobStore blobDS.Datastore + validator *validation.Validator + db postgres.DB +} + +func (s *serviceImpl) RegisterServiceServer(grpcServer *grpc.Server) { + apiV2.RegisterNodeReportServiceServer(grpcServer, s) +} + +func (s *serviceImpl) RegisterServiceHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { + return apiV2.RegisterNodeReportServiceHandler(ctx, mux, conn) +} + +func (s *serviceImpl) AuthFuncOverride(ctx context.Context, fullMethodName string) (context.Context, error) { + return ctx, authorizer.Authorized(ctx, fullMethodName) +} + +func (s *serviceImpl) PostNodeReportConfiguration(ctx context.Context, request *apiV2.ReportConfiguration) (*apiV2.ReportConfiguration, error) { + creatorID, err := identityFromContext(ctx) + if err != nil { + return nil, err + } + + if request.GetType() != apiV2.ReportConfiguration_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.CausedBy("report type must be NODE_VULNERABILITY") + } + + if err := s.validator.ValidateReportConfiguration(request); err != nil { + return nil, errors.Wrap(err, "validating report configuration") + } + + creator := &storage.SlimUser{ + Id: creatorID.UID(), + Name: stringutils.FirstNonEmpty(creatorID.FullName(), creatorID.FriendlyName()), + } + + protoReportConfig, err := s.convertV2ReportConfigurationToProto(request, creator, common.ExtractAccessScopeRules(creatorID)) + if err != nil { + return nil, errors.Wrap(err, "converting report configuration") + } + + id, err := s.reportConfigStore.AddReportConfiguration(ctx, protoReportConfig) + if err != nil { + return nil, err + } + + createdReportConfig, exists, err := s.reportConfigStore.GetReportConfiguration(ctx, id) + if err != nil { + return nil, err + } + if !exists { + return nil, errox.NotFound.CausedByf("report configuration with id '%s' was created but could not be read back", id) + } + + if env.CentralWorkerEnabled.BooleanSetting() { + reportsv2.NotifyWithRetry(ctx, s.db, pgNotify.ReportConfigChanged, id) + } else if err := s.scheduler.UpsertReportSchedule(createdReportConfig); err != nil { + return nil, err + } + + resp, err := s.convertProtoReportConfigurationToV2(createdReportConfig) + if err != nil { + return nil, errors.Wrap(err, "report config created, but encountered error generating the response") + } + return resp, nil +} + +func (s *serviceImpl) UpdateNodeReportConfiguration(ctx context.Context, request *apiV2.ReportConfiguration) (*apiV2.Empty, error) { + if request.GetId() == "" { + return nil, errox.InvalidArgs.CausedBy("report configuration id is required") + } + + if request.GetType() != apiV2.ReportConfiguration_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.CausedBy("report type must be NODE_VULNERABILITY") + } + + if err := s.validator.ValidateReportConfiguration(request); err != nil { + return nil, errors.Wrap(err, "validating report configuration") + } + + currentConfig, exists, err := s.reportConfigStore.GetReportConfiguration(ctx, request.GetId()) + if err != nil { + return nil, err + } + if !exists { + return nil, errox.NotFound.CausedByf("report configuration with id '%s' does not exist", request.GetId()) + } + + if currentConfig.GetType() != storage.ReportConfiguration_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.CausedByf("report configuration '%s' is not a node vulnerability report", request.GetId()) + } + + query := search.NewQueryBuilder().AddExactMatches(search.ReportConfigID, request.GetId()).AddExactMatches(search.ReportState, storage.ReportStatus_WAITING.String(), storage.ReportStatus_PREPARING.String()).ProtoQuery() + reportSnapshots, err := s.snapshotDatastore.SearchReportSnapshots(ctx, query) + if err != nil { + return nil, err + } + slimUser, err := userFromContext(ctx) + if err != nil { + return nil, err + } + for _, reportSnapshot := range reportSnapshots { + if slimUser.GetId() == reportSnapshot.GetRequester().GetId() { + return nil, errox.InvalidArgs.CausedBy("user has a report job running for this configuration") + } + } + + var accessScopeRules []*storage.SimpleAccessScope_Rules + if filters := currentConfig.GetNodeVulnReportFilters(); filters != nil { + accessScopeRules = filters.GetAccessScopeRules() + } + updatedConfig, err := s.convertV2ReportConfigurationToProto(request, currentConfig.GetCreator(), accessScopeRules) + if err != nil { + return nil, errors.Wrap(err, "converting report configuration") + } + + err = s.reportConfigStore.UpdateReportConfiguration(ctx, updatedConfig) + if err != nil { + return nil, err + } + + if env.CentralWorkerEnabled.BooleanSetting() { + reportsv2.NotifyWithRetry(ctx, s.db, pgNotify.ReportConfigChanged, updatedConfig.GetId()) + } else if err := s.scheduler.UpsertReportSchedule(updatedConfig); err != nil { + return nil, err + } + return &apiV2.Empty{}, nil +} + +func (s *serviceImpl) ListNodeReportConfigurations(ctx context.Context, query *apiV2.RawQuery) (*apiV2.ListReportConfigurationsResponse, error) { + parsedQuery, err := search.ParseQuery(query.GetQuery(), search.MatchAllIfEmpty()) + if err != nil { + return nil, errox.InvalidArgs.CausedBy(err.Error()) + } + + // Filter for NODE_VULNERABILITY type + parsedQuery = search.ConjunctionQuery( + parsedQuery, + search.NewQueryBuilder().AddExactMatches(search.ReportType, storage.ReportConfiguration_NODE_VULNERABILITY.String()).ProtoQuery(), + ) + + paginated.FillPaginationV2(parsedQuery, query.GetPagination(), maxPaginationLimit) + + reportConfigs, err := s.reportConfigStore.GetReportConfigurations(ctx, parsedQuery) + if err != nil { + return nil, errors.Wrap(err, "failed to retrieve report configurations") + } + v2Configs := make([]*apiV2.ReportConfiguration, 0, len(reportConfigs)) + + for _, config := range reportConfigs { + converted, err := s.convertProtoReportConfigurationToV2(config) + if err != nil { + return nil, errors.Wrapf(err, "error converting storage report configuration with id %s to response", config.GetId()) + } + v2Configs = append(v2Configs, converted) + } + return &apiV2.ListReportConfigurationsResponse{ReportConfigs: v2Configs}, nil +} + +func (s *serviceImpl) GetNodeReportConfiguration(ctx context.Context, req *apiV2.ResourceByID) (*apiV2.ReportConfiguration, error) { + if req.GetId() == "" { + return nil, errox.InvalidArgs.CausedBy("report configuration id is required") + } + config, exists, err := s.reportConfigStore.GetReportConfiguration(ctx, req.GetId()) + if err != nil { + return nil, err + } + if !exists { + return nil, errox.NotFound.CausedByf("report configuration with id '%s' does not exist", req.GetId()) + } + + if config.GetType() != storage.ReportConfiguration_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.CausedByf("report configuration '%s' is not a node vulnerability report", req.GetId()) + } + + if !common.HasValidResourceScope(config.GetResourceScope()) { + return nil, errox.InvalidArgs.CausedByf("report configuration '%s' has an empty resource scope (no entity scope)", req.GetId()) + } + + converted, err := s.convertProtoReportConfigurationToV2(config) + if err != nil { + return nil, errors.Wrapf(err, "error converting storage report configuration with id %s to response", config.GetId()) + } + return converted, nil +} + +func (s *serviceImpl) CountNodeReportConfigurations(ctx context.Context, request *apiV2.RawQuery) (*apiV2.CountReportConfigurationsResponse, error) { + parsedQuery, err := search.ParseQuery(request.GetQuery(), search.MatchAllIfEmpty()) + if err != nil { + return nil, errox.InvalidArgs.CausedBy(err.Error()) + } + + // Filter for NODE_VULNERABILITY type + parsedQuery = search.ConjunctionQuery( + parsedQuery, + search.NewQueryBuilder().AddExactMatches(search.ReportType, storage.ReportConfiguration_NODE_VULNERABILITY.String()).ProtoQuery(), + ) + + numReportConfigs, err := s.reportConfigStore.Count(ctx, parsedQuery) + if err != nil { + return nil, err + } + return &apiV2.CountReportConfigurationsResponse{Count: int32(numReportConfigs)}, nil +} + +func (s *serviceImpl) DeleteNodeReportConfiguration(ctx context.Context, id *apiV2.ResourceByID) (*apiV2.Empty, error) { + if id.GetId() == "" { + return nil, errox.InvalidArgs.CausedBy("report configuration id is required for deletion") + } + config, found, err := s.reportConfigStore.GetReportConfiguration(ctx, id.GetId()) + if err != nil { + return nil, errors.Wrap(err, "error finding report config") + } + if !found { + return nil, errox.NotFound.CausedByf("report config ID '%s' not found", id.GetId()) + } + + if config.GetType() != storage.ReportConfiguration_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.CausedByf("report configuration '%s' is not a node vulnerability report", id.GetId()) + } + + query := search.NewQueryBuilder().AddExactMatches(search.ReportConfigID, id.GetId()).AddExactMatches(search.ReportState, storage.ReportStatus_WAITING.String(), storage.ReportStatus_PREPARING.String()).ProtoQuery() + reportSnapshots, err := s.snapshotDatastore.SearchReportSnapshots(ctx, query) + if err != nil { + return nil, errors.Wrap(err, "failed to search for active report snapshots") + } + if len(reportSnapshots) > 0 { + return nil, errox.InvalidArgs.CausedByf("report config ID '%s' has job in preparing or waiting state", id.GetId()) + } + + if err := s.reportConfigStore.RemoveReportConfiguration(ctx, id.GetId()); err != nil { + return nil, err + } + + if env.CentralWorkerEnabled.BooleanSetting() { + reportsv2.NotifyWithRetry(ctx, s.db, pgNotify.ReportConfigChanged, id.GetId()) + } else { + s.scheduler.RemoveReportSchedule(id.GetId()) + } + return &apiV2.Empty{}, nil +} + +func (s *serviceImpl) RunNodeReport(ctx context.Context, req *apiV2.RunReportRequest) (*apiV2.RunReportResponse, error) { + if req.GetReportConfigId() == "" { + return nil, errox.InvalidArgs.CausedBy("report configuration id is required") + } + + config, exists, err := s.reportConfigStore.GetReportConfiguration(ctx, req.GetReportConfigId()) + if err != nil { + return nil, err + } + if !exists { + return nil, errox.NotFound.CausedByf("report configuration with id '%s' does not exist", req.GetReportConfigId()) + } + + if config.GetType() != storage.ReportConfiguration_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.CausedByf("report configuration '%s' is not a node vulnerability report", req.GetReportConfigId()) + } + + requesterID, err := identityFromContext(ctx) + if err != nil { + return nil, err + } + + reportRequest, err := s.validator.ValidateAndGenerateReportRequest( + req.GetReportConfigId(), + storage.ReportStatus_NotificationMethod(req.GetReportNotificationMethod()), + storage.ReportStatus_ON_DEMAND, + requesterID, + ) + if err != nil { + return nil, errors.Wrap(err, "error validating report request") + } + + if env.CentralWorkerEnabled.BooleanSetting() { + reportID, err := reportsv2.PersistSnapshotAndNotify(ctx, s.validator, s.db, reportRequest, pgNotify.ReportRequestSubmitted) + if err != nil { + return nil, err + } + return &apiV2.RunReportResponse{ReportConfigId: req.GetReportConfigId(), ReportId: reportID}, nil + } + + reportID, err := s.scheduler.SubmitReportRequest(ctx, reportRequest, false) + if err != nil { + return nil, errox.ServerError.CausedByf("scheduler error: %s", err) + } + + return &apiV2.RunReportResponse{ReportConfigId: req.GetReportConfigId(), ReportId: reportID}, nil +} + +// getReportHistory is a helper that fetches report snapshots with user filtering +func (s *serviceImpl) getReportHistory(ctx context.Context, queryBuilder *search.QueryBuilder, pagination *apiV2.Pagination, query string, userID string) (*apiV2.ReportHistoryResponse, error) { + parsedQuery, err := search.ParseQuery(query, search.MatchAllIfEmpty()) + if err != nil { + return nil, errox.InvalidArgs.CausedBy(err.Error()) + } + + baseQuery := queryBuilder.ProtoQuery() + if userID != "" { + if err := reportsv2.VerifyNoUserSearchLabels(parsedQuery); err != nil { + return nil, errox.InvalidArgs.CausedBy(err.Error()) + } + userIDQuery := search.NewQueryBuilder().AddExactMatches(search.UserID, userID).ProtoQuery() + baseQuery = search.ConjunctionQuery(baseQuery, userIDQuery) + } + + conjunctionQuery := search.ConjunctionQuery(baseQuery, parsedQuery) + paginated.FillPaginationV2(conjunctionQuery, pagination, maxPaginationLimit) + + results, err := s.snapshotDatastore.SearchReportSnapshots(ctx, conjunctionQuery) + if err != nil { + return nil, err + } + + blobNames, err := s.getExistingBlobNames(results) + if err != nil { + return nil, errors.Wrap(err, "failed to check blob availability") + } + + v2Snapshots := make([]*apiV2.ReportSnapshot, 0, len(results)) + for _, snapshot := range results { + converted, err := s.convertProtoReportSnapshotToV2(snapshot, blobNames) + if err != nil { + return nil, errors.Wrapf(err, "error converting snapshot with id %s", snapshot.GetReportId()) + } + v2Snapshots = append(v2Snapshots, converted) + } + + return &apiV2.ReportHistoryResponse{ReportSnapshots: v2Snapshots}, nil +} + +func (s *serviceImpl) GetNodeReportHistory(ctx context.Context, req *apiV2.GetReportHistoryRequest) (*apiV2.ReportHistoryResponse, error) { + if req == nil || req.GetId() == "" { + return nil, errox.InvalidArgs.CausedBy("empty request or id") + } + + queryBuilder := search.NewQueryBuilder(). + AddExactMatches(search.ReportConfigID, req.GetId()). + AddExactMatches(search.ReportType, storage.ReportSnapshot_NODE_VULNERABILITY.String()) + return s.getReportHistory(ctx, queryBuilder, req.GetReportParamQuery().GetPagination(), req.GetReportParamQuery().GetQuery(), "") +} + +func (s *serviceImpl) GetMyNodeReportHistory(ctx context.Context, req *apiV2.GetReportHistoryRequest) (*apiV2.ReportHistoryResponse, error) { + if req == nil || req.GetId() == "" { + return nil, errox.InvalidArgs.CausedBy("empty request or id") + } + + slimUser, err := userFromContext(ctx) + if err != nil { + return nil, err + } + + queryBuilder := search.NewQueryBuilder(). + AddExactMatches(search.ReportConfigID, req.GetId()). + AddExactMatches(search.ReportType, storage.ReportSnapshot_NODE_VULNERABILITY.String()) + return s.getReportHistory(ctx, queryBuilder, req.GetReportParamQuery().GetPagination(), req.GetReportParamQuery().GetQuery(), slimUser.GetId()) +} + +func (s *serviceImpl) GetNodeReportStatus(ctx context.Context, req *apiV2.ResourceByID) (*apiV2.ReportStatusResponse, error) { + if req == nil || req.GetId() == "" { + return nil, errox.InvalidArgs.CausedBy("empty request or id") + } + rep, found, err := s.snapshotDatastore.Get(ctx, req.GetId()) + if err != nil { + return nil, err + } + if !found { + return nil, errox.NotFound.CausedByf("report snapshot not found for job id %s", req.GetId()) + } + if rep.GetType() != storage.ReportSnapshot_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.CausedByf("report snapshot '%s' is not a node vulnerability report", req.GetId()) + } + status := convertPrototoV2Reportstatus(rep.GetReportStatus()) + return &apiV2.ReportStatusResponse{Status: status}, nil +} + +func (s *serviceImpl) CancelNodeReport(ctx context.Context, req *apiV2.ResourceByID) (*apiV2.Empty, error) { + if req == nil || req.GetId() == "" { + return nil, errox.InvalidArgs.CausedBy("empty request or id") + } + snapshot, found, err := s.snapshotDatastore.Get(ctx, req.GetId()) + if err != nil { + return nil, err + } + if !found { + return nil, errox.NotFound.CausedByf("report snapshot not found for id %s", req.GetId()) + } + + if snapshot.GetType() != storage.ReportSnapshot_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.CausedByf("report snapshot '%s' is not a node vulnerability report", req.GetId()) + } + + slimUser, err := userFromContext(ctx) + if err != nil { + return nil, err + } + if slimUser.GetId() != snapshot.GetRequester().GetId() { + if err := sac.VerifyAuthzOK(workflowSAC.WriteAllowed(ctx)); err != nil { + return nil, errors.Wrap(err, "user can only cancel a job created by the user unless they have Modify(WorkflowAdministration) permission") + } + } + + reportStatus := snapshot.GetReportStatus() + if reportStatus.GetRunState() != storage.ReportStatus_WAITING && reportStatus.GetRunState() != storage.ReportStatus_PREPARING { + return nil, errox.InvalidArgs.CausedBy("cannot cancel a job that is not in WAITING or PREPARING state") + } + + if env.CentralWorkerEnabled.BooleanSetting() { + reportsv2.NotifyWithRetry(ctx, s.db, pgNotify.ReportRequestCancelled, req.GetId()) + return &apiV2.Empty{}, nil + } + + cancelled, err := s.scheduler.CancelReportRequest(ctx, req.GetId()) + if err != nil { + return nil, err + } + if !cancelled { + return nil, errors.Errorf("failed to cancel report job %s", req.GetId()) + } + + return &apiV2.Empty{}, nil +} + +func (s *serviceImpl) DeleteNodeReport(ctx context.Context, req *apiV2.DeleteReportRequest) (*apiV2.Empty, error) { + if req == nil || req.GetId() == "" { + return nil, errox.InvalidArgs.CausedBy("empty request or id") + } + snapshot, found, err := s.snapshotDatastore.Get(ctx, req.GetId()) + if err != nil { + return nil, err + } + if !found { + return nil, errox.NotFound.CausedByf("report snapshot not found for id %s", req.GetId()) + } + + if snapshot.GetType() != storage.ReportSnapshot_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.CausedByf("report snapshot '%s' is not a node vulnerability report", req.GetId()) + } + + slimUser, err := userFromContext(ctx) + if err != nil { + return nil, err + } + if slimUser.GetId() != snapshot.GetRequester().GetId() { + return nil, errox.NotAuthorized.CausedBy("report cannot be deleted by a user who did not request the report") + } + + status := snapshot.GetReportStatus() + if status.GetReportNotificationMethod() != storage.ReportStatus_DOWNLOAD { + return nil, errox.InvalidArgs.CausedByf("report job id %q did not generate a downloadable report and hence no report to delete", req.GetId()) + } + switch status.GetRunState() { + case storage.ReportStatus_FAILURE: + return nil, errox.InvalidArgs.CausedByf("report job %q has failed and no downloadable report to delete", req.GetId()) + case storage.ReportStatus_PREPARING, storage.ReportStatus_WAITING: + return nil, errox.InvalidArgs.CausedByf("report job %q is still running. Please cancel it or wait for its completion", req.GetId()) + } + + blobPath := common.GetReportBlobPath(reportBlobParentDir(snapshot), snapshot.GetReportId()) + blobCtx := sac.WithGlobalAccessScopeChecker(ctx, + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_WRITE_ACCESS), + sac.ResourceScopeKeys(resources.Administration), + ), + ) + if err := s.blobStore.Delete(blobCtx, blobPath); err != nil { + return nil, errox.InvariantViolation.CausedByf("failed to delete downloadable report %q", req.GetId()) + } + + return &apiV2.Empty{}, nil +} + +func (s *serviceImpl) PostViewBasedNodeReport(ctx context.Context, req *apiV2.ReportRequestViewBased) (*apiV2.RunReportResponseViewBased, error) { + if req.GetType() != apiV2.ReportRequestViewBased_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.CausedBy("report type must be NODE_VULNERABILITY") + } + + requesterID, err := identityFromContext(ctx) + if err != nil { + return nil, err + } + + reportRequest, err := s.validator.ValidateAndGenerateViewBasedReportRequest(req, requesterID) + if err != nil { + return nil, errors.Wrap(err, "error validating view-based report request") + } + + if env.CentralWorkerEnabled.BooleanSetting() { + reportID, err := reportsv2.PersistSnapshotAndNotify(ctx, s.validator, s.db, reportRequest, pgNotify.ReportRequestSubmitted) + if err != nil { + return nil, err + } + return &apiV2.RunReportResponseViewBased{ReportID: reportID, RequestName: reportRequest.ReportSnapshot.GetName()}, nil + } + + reportID, err := s.scheduler.SubmitReportRequest(ctx, reportRequest, false) + if err != nil { + return nil, errox.ServerError.CausedByf("scheduler error: %s", err) + } + + return &apiV2.RunReportResponseViewBased{ReportID: reportID, RequestName: reportRequest.ReportSnapshot.GetName()}, nil +} + +func (s *serviceImpl) GetViewBasedNodeReportHistory(ctx context.Context, req *apiV2.GetViewBasedReportHistoryRequest) (*apiV2.ReportHistoryResponse, error) { + queryBuilder := search.NewQueryBuilder(). + AddExactMatches(search.ReportType, storage.ReportSnapshot_NODE_VULNERABILITY.String()). + AddExactMatches(search.ReportRequestType, storage.ReportStatus_VIEW_BASED.String()) + return s.getReportHistory(reportsv2.SnapshotReadContext(ctx), queryBuilder, req.GetReportParamQuery().GetPagination(), req.GetReportParamQuery().GetQuery(), "") +} + +func (s *serviceImpl) GetViewBasedMyNodeReportHistory(ctx context.Context, req *apiV2.GetViewBasedReportHistoryRequest) (*apiV2.ReportHistoryResponse, error) { + slimUser, err := userFromContext(ctx) + if err != nil { + return nil, err + } + + queryBuilder := search.NewQueryBuilder(). + AddExactMatches(search.ReportType, storage.ReportSnapshot_NODE_VULNERABILITY.String()). + AddExactMatches(search.ReportRequestType, storage.ReportStatus_VIEW_BASED.String()) + return s.getReportHistory(reportsv2.SnapshotReadContext(ctx), queryBuilder, req.GetReportParamQuery().GetPagination(), req.GetReportParamQuery().GetQuery(), slimUser.GetId()) +} + +func identityFromContext(ctx context.Context) (authn.Identity, error) { + id := authn.IdentityFromContextOrNil(ctx) + if id == nil { + return nil, errox.NoCredentials.New("could not determine user identity from provided context") + } + return id, nil +} + +func userFromContext(ctx context.Context) (*storage.SlimUser, error) { + slimUser := authn.UserFromContext(ctx) + if slimUser == nil { + return nil, errox.NoCredentials.New("could not determine user identity from provided context") + } + return slimUser, nil +} + +var _ Service = (*serviceImpl)(nil) diff --git a/central/reports/service/v2/node/service_impl_test.go b/central/reports/service/v2/node/service_impl_test.go new file mode 100644 index 0000000000000..8de5b2e97c062 --- /dev/null +++ b/central/reports/service/v2/node/service_impl_test.go @@ -0,0 +1,1396 @@ +package node + +import ( + "context" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + blobDSMocks "github.com/stackrox/rox/central/blob/datastore/mocks" + notifierDSMocks "github.com/stackrox/rox/central/notifier/datastore/mocks" + "github.com/stackrox/rox/central/reports/common" + reportConfigDSMocks "github.com/stackrox/rox/central/reports/config/datastore/mocks" + schedulerMocks "github.com/stackrox/rox/central/reports/scheduler/v2/mocks" + reportSnapshotDSMocks "github.com/stackrox/rox/central/reports/snapshot/datastore/mocks" + "github.com/stackrox/rox/central/reports/validation" + collectionDSMocks "github.com/stackrox/rox/central/resourcecollection/datastore/mocks" + v1 "github.com/stackrox/rox/generated/api/v1" + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/auth/permissions" + permissionsMocks "github.com/stackrox/rox/pkg/auth/permissions/mocks" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/grpc/authn" + mockIdentity "github.com/stackrox/rox/pkg/grpc/authn/mocks" + "github.com/stackrox/rox/pkg/grpc/testutils" + "github.com/stackrox/rox/pkg/notifiers" + postgresMocks "github.com/stackrox/rox/pkg/postgres/mocks" + pgNotify "github.com/stackrox/rox/pkg/postgres/notify" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "go.uber.org/mock/gomock" +) + +func TestNodeReportService(t *testing.T) { + suite.Run(t, new(NodeReportServiceTestSuite)) +} + +type NodeReportServiceTestSuite struct { + suite.Suite + mockCtrl *gomock.Controller + + ctx context.Context + reportConfigDataStore *reportConfigDSMocks.MockDataStore + reportSnapshotDataStore *reportSnapshotDSMocks.MockDataStore + collectionDataStore *collectionDSMocks.MockDataStore + notifierDataStore *notifierDSMocks.MockDataStore + blobStore *blobDSMocks.MockDatastore + scheduler *schedulerMocks.MockScheduler + db *postgresMocks.MockDB + service Service +} + +func (s *NodeReportServiceTestSuite) SetupTest() { + s.mockCtrl = gomock.NewController(s.T()) + s.ctx = sac.WithAllAccess(context.Background()) + s.reportConfigDataStore = reportConfigDSMocks.NewMockDataStore(s.mockCtrl) + s.reportSnapshotDataStore = reportSnapshotDSMocks.NewMockDataStore(s.mockCtrl) + s.collectionDataStore = collectionDSMocks.NewMockDataStore(s.mockCtrl) + s.notifierDataStore = notifierDSMocks.NewMockDataStore(s.mockCtrl) + s.blobStore = blobDSMocks.NewMockDatastore(s.mockCtrl) + s.scheduler = schedulerMocks.NewMockScheduler(s.mockCtrl) + s.db = postgresMocks.NewMockDB(s.mockCtrl) + validator := validation.New(s.reportConfigDataStore, s.reportSnapshotDataStore, s.collectionDataStore, s.notifierDataStore) + s.service = New(s.reportConfigDataStore, s.reportSnapshotDataStore, s.notifierDataStore, s.scheduler, s.blobStore, validator, s.db) +} + +func (s *NodeReportServiceTestSuite) TearDownSuite() { + s.mockCtrl.Finish() +} + +func (s *NodeReportServiceTestSuite) TearDownTest() { + s.mockCtrl.Finish() +} + +func (s *NodeReportServiceTestSuite) getContextForUser(user *storage.SlimUser) context.Context { + mockID := mockIdentity.NewMockIdentity(s.mockCtrl) + mockID.EXPECT().UID().Return(user.GetId()).AnyTimes() + mockID.EXPECT().FullName().Return(user.GetName()).AnyTimes() + mockID.EXPECT().FriendlyName().Return(user.GetName()).AnyTimes() + mockRole := permissionsMocks.NewMockResolvedRole(s.mockCtrl) + mockRole.EXPECT().GetAccessScope().Return(&storage.SimpleAccessScope{ + Rules: &storage.SimpleAccessScope_Rules{ + IncludedClusters: []string{"cluster-1"}, + }, + }).AnyTimes() + mockID.EXPECT().Roles().Return([]permissions.ResolvedRole{mockRole}).AnyTimes() + return authn.ContextWithIdentity(s.ctx, mockID, s.T()) +} + +func (s *NodeReportServiceTestSuite) getValidNodeReportConfig() *apiV2.ReportConfiguration { + return &apiV2.ReportConfiguration{ + Id: uuid.NewV4().String(), + Name: "test node report", + Description: "test description", + Type: apiV2.ReportConfiguration_NODE_VULNERABILITY, + ResourceScope: &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{ + EntityScope: &apiV2.EntityScope{ + Rules: []*apiV2.EntityScopeRule{ + { + Entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + Field: apiV2.ScopeField_FIELD_NAME, + Values: []*apiV2.RuleValue{ + { + Value: "cluster-1", + MatchType: apiV2.MatchType_EXACT, + }, + }, + }, + }, + }, + }, + }, + Filter: &apiV2.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + Notifiers: []*apiV2.NotifierConfiguration{ + { + NotifierConfig: &apiV2.NotifierConfiguration_EmailConfig{ + EmailConfig: &apiV2.EmailNotifierConfiguration{ + NotifierId: "email-notifier-id", + MailingLists: []string{"test@example.com"}, + }, + }, + }, + }, + } +} + +func (s *NodeReportServiceTestSuite) TestPostNodeReportConfiguration() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + + accessScope := &storage.SimpleAccessScope{ + Rules: &storage.SimpleAccessScope_Rules{ + IncludedClusters: []string{"cluster-1"}, + }, + } + + requestConfig := s.getValidNodeReportConfig() + mockID := mockIdentity.NewMockIdentity(s.mockCtrl) + ctx := authn.ContextWithIdentity(s.ctx, mockID, s.T()) + + mockID.EXPECT().UID().Return(creator.GetId()).AnyTimes() + mockID.EXPECT().FullName().Return(creator.GetName()).AnyTimes() + mockID.EXPECT().FriendlyName().Return(creator.GetName()).AnyTimes() + + mockRole := permissionsMocks.NewMockResolvedRole(s.mockCtrl) + mockRole.EXPECT().GetAccessScope().Return(accessScope).Times(1) + mockID.EXPECT().Roles().Return([]permissions.ResolvedRole{mockRole}).Times(1) + + s.notifierDataStore.EXPECT().GetScrubbedNotifier(gomock.Any(), "email-notifier-id"). + Return(&storage.Notifier{Type: notifiers.EmailType}, true, nil).Times(1) + + s.reportConfigDataStore.EXPECT().AddReportConfiguration(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, cfg *storage.ReportConfiguration) (string, error) { + s.Equal(storage.ReportConfiguration_NODE_VULNERABILITY, cfg.GetType()) + protoassert.Equal(s.T(), creator, cfg.GetCreator()) + s.NotNil(cfg.GetNodeVulnReportFilters()) + return cfg.GetId(), nil + }).Times(1) + + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), requestConfig.GetId()). + DoAndReturn(func(_ context.Context, _ string) (*storage.ReportConfiguration, bool, error) { + return &storage.ReportConfiguration{ + Id: requestConfig.GetId(), + Name: requestConfig.GetName(), + Description: requestConfig.GetDescription(), + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + Creator: creator, + Filter: &storage.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: &storage.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + }, true, nil + }).Times(1) + + s.scheduler.EXPECT().UpsertReportSchedule(gomock.Any()).Return(nil).Times(1) + + result, err := s.service.PostNodeReportConfiguration(ctx, requestConfig) + s.NoError(err) + s.Equal(requestConfig.GetId(), result.GetId()) + s.Equal(apiV2.ReportConfiguration_NODE_VULNERABILITY, result.GetType()) +} + +func (s *NodeReportServiceTestSuite) TestPostNodeReportConfiguration_GetAfterCreateMissing() { + creator := &storage.SlimUser{Id: "uid", Name: "name"} + requestConfig := s.getValidNodeReportConfig() + ctx := s.getContextForUser(creator) + + s.notifierDataStore.EXPECT().GetScrubbedNotifier(gomock.Any(), "email-notifier-id"). + Return(&storage.Notifier{Type: notifiers.EmailType}, true, nil).Times(1) + s.reportConfigDataStore.EXPECT().AddReportConfiguration(gomock.Any(), gomock.Any()). + Return(requestConfig.GetId(), nil).Times(1) + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), requestConfig.GetId()). + Return(nil, false, nil).Times(1) + + result, err := s.service.PostNodeReportConfiguration(ctx, requestConfig) + s.Error(err) + s.Nil(result) +} + +func (s *NodeReportServiceTestSuite) TestPostNodeReportConfiguration_ValidationError() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + invalidConfig := s.getValidNodeReportConfig() + invalidConfig.Filter = &apiV2.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: nil, + }, + } + + s.notifierDataStore.EXPECT().GetScrubbedNotifier(gomock.Any(), "email-notifier-id"). + Return(&storage.Notifier{Type: notifiers.EmailType}, true, nil).Times(1) + + _, err := s.service.PostNodeReportConfiguration(ctx, invalidConfig) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestGetNodeReportConfiguration() { + protoReportConfig := &storage.ReportConfiguration{ + Id: "test-id", + Name: "test node report", + Description: "test description", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_ID, + Values: []*storage.RuleValue{ + { + Value: "cluster-1", + MatchType: storage.MatchType_EXACT, + }, + }, + }, + }, + }, + }, + }, + Filter: &storage.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: &storage.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + } + + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "test-id"). + Return(protoReportConfig, true, nil).Times(1) + + result, err := s.service.GetNodeReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: "test-id"}) + s.NoError(err) + s.Equal("test-id", result.GetId()) + s.Equal(apiV2.ReportConfiguration_NODE_VULNERABILITY, result.GetType()) +} + +func (s *NodeReportServiceTestSuite) TestGetNodeReportConfiguration_NotFound() { + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "nonexistent"). + Return(nil, false, nil).Times(1) + + _, err := s.service.GetNodeReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: "nonexistent"}) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestDeleteNodeReportConfiguration() { + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "test-id"). + Return(&storage.ReportConfiguration{ + Id: "test-id", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + }, true, nil).Times(1) + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return([]*storage.ReportSnapshot{}, nil).Times(1) + s.reportConfigDataStore.EXPECT().RemoveReportConfiguration(gomock.Any(), "test-id"). + Return(nil).Times(1) + s.scheduler.EXPECT().RemoveReportSchedule("test-id").Times(1) + + _, err := s.service.DeleteNodeReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: "test-id"}) + s.NoError(err) +} + +func (s *NodeReportServiceTestSuite) TestDeleteNodeReportConfiguration_HasRunningJob() { + runningSnapshot := &storage.ReportSnapshot{ + ReportId: "test-report-id", + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_PREPARING, + }, + } + + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "test-id"). + Return(&storage.ReportConfiguration{ + Id: "test-id", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + }, true, nil).Times(1) + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return([]*storage.ReportSnapshot{runningSnapshot}, nil).Times(1) + + resp, err := s.service.DeleteNodeReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: "test-id"}) + s.Error(err) + s.Nil(resp) +} + +func (s *NodeReportServiceTestSuite) TestRunNodeReport() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + configID := "test-config-id" + protoReportConfig := &storage.ReportConfiguration{ + Id: configID, + Name: "test node report", + Description: "test description", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + Creator: creator, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_ID, + Values: []*storage.RuleValue{ + { + Value: "cluster-1", + MatchType: storage.MatchType_EXACT, + }, + }, + }, + }, + }, + }, + }, + Filter: &storage.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: &storage.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + Notifiers: []*storage.NotifierConfiguration{ + { + NotifierConfig: &storage.NotifierConfiguration_EmailConfig{ + EmailConfig: &storage.EmailNotifierConfiguration{ + NotifierId: "email-notifier-id", + MailingLists: []string{"test@example.com"}, + }, + }, + }, + }, + } + + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), configID). + Return(protoReportConfig, true, nil).AnyTimes() + + s.notifierDataStore.EXPECT().GetManyNotifiers(gomock.Any(), gomock.Any()). + Return([]*storage.Notifier{ + { + Id: "email-notifier-id", + Name: "Email Notifier", + Type: "email", + }, + }, nil).AnyTimes() + + s.scheduler.EXPECT().SubmitReportRequest(gomock.Any(), gomock.Any(), false). + DoAndReturn(func(_ context.Context, reportReq interface{}, _ bool) (string, error) { + return "on-demand-report-id", nil + }).Times(1) + + result, err := s.service.RunNodeReport(ctx, &apiV2.RunReportRequest{ + ReportConfigId: configID, + }) + s.NoError(err) + s.Equal(configID, result.GetReportConfigId()) + s.Equal("on-demand-report-id", result.GetReportId()) +} + +func (s *NodeReportServiceTestSuite) TestCancelNodeReport() { + reportID := "test-report-id" + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + snapshot := &storage.ReportSnapshot{ + ReportId: reportID, + ReportConfigurationId: "test-config-id", + Requester: creator, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_PREPARING, + }, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + } + + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportID). + Return(snapshot, true, nil).Times(1) + + s.scheduler.EXPECT().CancelReportRequest(gomock.Any(), reportID). + Return(false, nil).Times(1) + + _, err := s.service.CancelNodeReport(ctx, &apiV2.ResourceByID{Id: reportID}) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestDeleteNodeReport() { + reportID := "test-report-id" + configID := "test-config-id" + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + snapshot := &storage.ReportSnapshot{ + ReportId: reportID, + ReportConfigurationId: configID, + Requester: creator, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + } + + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportID). + Return(snapshot, true, nil).Times(1) + + blobPath := common.GetReportBlobPath(configID, reportID) + s.blobStore.EXPECT().Delete(gomock.Any(), blobPath).Return(nil).Times(1) + + _, err := s.service.DeleteNodeReport(ctx, &apiV2.DeleteReportRequest{ + Id: reportID, + }) + s.NoError(err) +} + +func (s *NodeReportServiceTestSuite) TestPostViewBasedNodeReport() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + + req := &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_NODE_VULNERABILITY, + Filter: &apiV2.ReportRequestViewBased_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + } + + mockRole := permissionsMocks.NewMockResolvedRole(s.mockCtrl) + mockID := mockIdentity.NewMockIdentity(s.mockCtrl) + mockID.EXPECT().UID().Return(creator.GetId()).AnyTimes() + mockID.EXPECT().FullName().Return(creator.GetName()).AnyTimes() + mockID.EXPECT().FriendlyName().Return(creator.GetName()).AnyTimes() + mockID.EXPECT().Roles().Return([]permissions.ResolvedRole{mockRole}).AnyTimes() + mockRole.EXPECT().GetAccessScope().Return(&storage.SimpleAccessScope{ + Rules: &storage.SimpleAccessScope_Rules{ + IncludedClusters: []string{"cluster-1"}, + }, + }).AnyTimes() + + ctxWithIdentity := authn.ContextWithIdentity(s.ctx, mockID, s.T()) + + s.scheduler.EXPECT().SubmitReportRequest(gomock.Any(), gomock.Any(), false). + DoAndReturn(func(_ context.Context, reportReq interface{}, _ bool) (string, error) { + return "view-based-report-id", nil + }).Times(1) + + result, err := s.service.PostViewBasedNodeReport(ctxWithIdentity, req) + s.NoError(err) + s.NotEmpty(result.GetReportID()) + s.Equal("view-based-report-id", result.GetReportID()) +} + +func (s *NodeReportServiceTestSuite) TestListNodeReportConfigurations() { + protoReportConfigs := []*storage.ReportConfiguration{ + { + Id: "config-1", + Name: "Node Report 1", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + }, + { + Id: "config-2", + Name: "Node Report 2", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + }, + } + + s.reportConfigDataStore.EXPECT().GetReportConfigurations(gomock.Any(), gomock.Any()). + Return(protoReportConfigs, nil).Times(1) + + result, err := s.service.ListNodeReportConfigurations(s.ctx, &apiV2.RawQuery{Query: ""}) + s.NoError(err) + s.Len(result.GetReportConfigs(), 2) + assert.Equal(s.T(), apiV2.ReportConfiguration_NODE_VULNERABILITY, result.GetReportConfigs()[0].GetType()) +} + +func (s *NodeReportServiceTestSuite) TestCountNodeReportConfigurations() { + s.reportConfigDataStore.EXPECT().Count(gomock.Any(), gomock.Any()). + Return(5, nil).Times(1) + + result, err := s.service.CountNodeReportConfigurations(s.ctx, &apiV2.RawQuery{Query: ""}) + s.NoError(err) + s.Equal(int32(5), result.GetCount()) +} + +func (s *NodeReportServiceTestSuite) TestGetMyNodeReportHistory() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + configID := "test-config-id" + // Query-time filtering: only return creator's snapshot + snapshots := []*storage.ReportSnapshot{ + { + ReportId: "report-1", + ReportConfigurationId: configID, + Name: "Report 1", + Requester: creator, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + }, + } + + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return(snapshots, nil).Times(1) + + s.blobStore.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{}, nil).AnyTimes() + + result, err := s.service.GetMyNodeReportHistory(ctx, &apiV2.GetReportHistoryRequest{ + Id: configID, + ReportParamQuery: &apiV2.RawQuery{ + Query: "", + }, + }) + s.NoError(err) + s.Len(result.GetReportSnapshots(), 1) + s.Equal(creator.GetId(), result.GetReportSnapshots()[0].GetUser().GetId()) +} + +func (s *NodeReportServiceTestSuite) TestUpdateNodeReportConfiguration() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + + accessScope := &storage.SimpleAccessScope{ + Rules: &storage.SimpleAccessScope_Rules{ + IncludedClusters: []string{"cluster-1"}, + }, + } + + updateConfig := s.getValidNodeReportConfig() + updateConfig.Name = "Updated Node Report" + + mockID := mockIdentity.NewMockIdentity(s.mockCtrl) + ctx := authn.ContextWithIdentity(s.ctx, mockID, s.T()) + + mockID.EXPECT().UID().Return(creator.GetId()).AnyTimes() + mockID.EXPECT().FullName().Return(creator.GetName()).AnyTimes() + mockID.EXPECT().FriendlyName().Return(creator.GetName()).AnyTimes() + + mockRole := permissionsMocks.NewMockResolvedRole(s.mockCtrl) + mockRole.EXPECT().GetAccessScope().Return(accessScope).AnyTimes() + mockID.EXPECT().Roles().Return([]permissions.ResolvedRole{mockRole}).AnyTimes() + + s.notifierDataStore.EXPECT().GetScrubbedNotifier(gomock.Any(), "email-notifier-id"). + Return(&storage.Notifier{Type: notifiers.EmailType}, true, nil).Times(1) + + existingConfig := &storage.ReportConfiguration{ + Id: updateConfig.GetId(), + Name: "Old Name", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + Creator: creator, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_ID, + Values: []*storage.RuleValue{ + { + Value: "cluster-1", + MatchType: storage.MatchType_EXACT, + }, + }, + }, + }, + }, + }, + }, + Filter: &storage.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: &storage.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + } + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), updateConfig.GetId()). + Return(existingConfig, true, nil).Times(1) + + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return([]*storage.ReportSnapshot{}, nil).Times(1) + + s.reportConfigDataStore.EXPECT().UpdateReportConfiguration(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, cfg *storage.ReportConfiguration) error { + s.Equal("Updated Node Report", cfg.GetName()) + return nil + }).Times(1) + + s.scheduler.EXPECT().UpsertReportSchedule(gomock.Any()).Return(nil).Times(1) + + result, err := s.service.UpdateNodeReportConfiguration(ctx, updateConfig) + s.NoError(err) + s.NotNil(result) +} + +func (s *NodeReportServiceTestSuite) TestGetNodeReportHistory() { + configID := "test-config-id" + snapshots := []*storage.ReportSnapshot{ + { + ReportId: "report-1", + ReportConfigurationId: configID, + Name: "Report 1", + Requester: &storage.SlimUser{Id: "uid", Name: "name"}, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{}, + }, + }, + }, + } + + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return(snapshots, nil).Times(1) + + s.blobStore.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{}, nil).AnyTimes() + + result, err := s.service.GetNodeReportHistory(s.ctx, &apiV2.GetReportHistoryRequest{ + Id: configID, + ReportParamQuery: &apiV2.RawQuery{ + Query: "", + }, + }) + s.NoError(err) + s.Len(result.GetReportSnapshots(), 1) + s.Equal("report-1", result.GetReportSnapshots()[0].GetReportJobId()) +} + +func (s *NodeReportServiceTestSuite) TestGetNodeReportStatus() { + reportID := "test-report-id" + snapshot := &storage.ReportSnapshot{ + ReportId: reportID, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + } + + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportID). + Return(snapshot, true, nil).Times(1) + + result, err := s.service.GetNodeReportStatus(s.ctx, &apiV2.ResourceByID{Id: reportID}) + s.NoError(err) + s.NotNil(result.GetStatus()) + s.Equal(apiV2.ReportStatus_GENERATED, result.GetStatus().GetRunState()) +} + +func (s *NodeReportServiceTestSuite) TestGetViewBasedNodeReportHistory() { + snapshots := []*storage.ReportSnapshot{ + { + ReportId: "view-report-1", + ReportConfigurationId: "", + Name: "View Report 1", + Requester: &storage.SlimUser{Id: "uid", Name: "name"}, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{}, + }, + }, + }, + } + + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return(snapshots, nil).Times(1) + + s.blobStore.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{}, nil).AnyTimes() + + result, err := s.service.GetViewBasedNodeReportHistory(s.ctx, &apiV2.GetViewBasedReportHistoryRequest{ + ReportParamQuery: &apiV2.RawQuery{ + Query: "", + }, + }) + s.NoError(err) + s.Len(result.GetReportSnapshots(), 1) +} + +func (s *NodeReportServiceTestSuite) TestGetViewBasedMyNodeReportHistory() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + // Query-time filtering: only return creator's snapshot + snapshots := []*storage.ReportSnapshot{ + { + ReportId: "view-report-1", + ReportConfigurationId: "", + Name: "View Report 1", + Requester: creator, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{}, + }, + }, + }, + } + + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return(snapshots, nil).Times(1) + + s.blobStore.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{}, nil).AnyTimes() + + result, err := s.service.GetViewBasedMyNodeReportHistory(ctx, &apiV2.GetViewBasedReportHistoryRequest{ + ReportParamQuery: &apiV2.RawQuery{ + Query: "", + }, + }) + s.NoError(err) + s.Len(result.GetReportSnapshots(), 1) + s.Equal(creator.GetId(), result.GetReportSnapshots()[0].GetUser().GetId()) +} + +func (s *NodeReportServiceTestSuite) TestGetNodeReportConfiguration_InvalidType() { + protoReportConfig := &storage.ReportConfiguration{ + Id: "test-id", + Type: storage.ReportConfiguration_VULNERABILITY, + } + + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "test-id"). + Return(protoReportConfig, true, nil).Times(1) + + _, err := s.service.GetNodeReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: "test-id"}) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestDeleteNodeReportConfiguration_WrongType() { + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "test-id"). + Return(&storage.ReportConfiguration{ + Id: "test-id", + Type: storage.ReportConfiguration_VULNERABILITY, + }, true, nil).Times(1) + + _, err := s.service.DeleteNodeReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: "test-id"}) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestRunNodeReport_WrongType() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "config-id"). + Return(&storage.ReportConfiguration{ + Id: "config-id", + Type: storage.ReportConfiguration_VULNERABILITY, + }, true, nil).Times(1) + + _, err := s.service.RunNodeReport(ctx, &apiV2.RunReportRequest{ + ReportConfigId: "config-id", + }) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestUpdateNodeReportConfiguration_WrongType() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + wrongConfig := s.getValidNodeReportConfig() + wrongConfig.Type = apiV2.ReportConfiguration_VULNERABILITY + + _, err := s.service.UpdateNodeReportConfiguration(ctx, wrongConfig) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestPostNodeReportConfiguration_WrongType() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + wrongConfig := s.getValidNodeReportConfig() + wrongConfig.Type = apiV2.ReportConfiguration_VULNERABILITY + + _, err := s.service.PostNodeReportConfiguration(ctx, wrongConfig) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestPostNodeReportConfiguration_RequiresIdentity() { + _, err := s.service.PostNodeReportConfiguration(s.ctx, s.getValidNodeReportConfig()) + s.Error(err) + s.ErrorIs(err, errox.NoCredentials) +} + +func (s *NodeReportServiceTestSuite) TestCancelNodeReport_NotFound() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), "nonexistent"). + Return(nil, false, nil).Times(1) + + _, err := s.service.CancelNodeReport(ctx, &apiV2.ResourceByID{Id: "nonexistent"}) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestGetNodeReportStatus_NotFound() { + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), "nonexistent"). + Return(nil, false, nil).Times(1) + + _, err := s.service.GetNodeReportStatus(s.ctx, &apiV2.ResourceByID{Id: "nonexistent"}) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestCancelNodeReport_WrongState() { + reportID := "test-report-id" + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + snapshot := &storage.ReportSnapshot{ + ReportId: reportID, + ReportConfigurationId: "test-config-id", + Requester: creator, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_DELIVERED, + }, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + } + + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportID). + Return(snapshot, true, nil).Times(1) + + _, err := s.service.CancelNodeReport(ctx, &apiV2.ResourceByID{Id: reportID}) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestCancelNodeReport_WrongType() { + reportID := "test-report-id" + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + snapshot := &storage.ReportSnapshot{ + ReportId: reportID, + ReportConfigurationId: "test-config-id", + Requester: creator, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_PREPARING, + }, + Type: storage.ReportSnapshot_VULNERABILITY, + } + + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportID). + Return(snapshot, true, nil).Times(1) + + _, err := s.service.CancelNodeReport(ctx, &apiV2.ResourceByID{Id: reportID}) + s.Error(err) + s.Contains(err.Error(), "not a node vulnerability report") +} + +func (s *NodeReportServiceTestSuite) TestPostViewBasedNodeReport_WrongType() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + req := &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_VULNERABILITY, + } + + _, err := s.service.PostViewBasedNodeReport(ctx, req) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestDeleteNodeReport_NotFound() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), "nonexistent"). + Return(nil, false, nil).Times(1) + + _, err := s.service.DeleteNodeReport(ctx, &apiV2.DeleteReportRequest{ + Id: "nonexistent", + }) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestDeleteNodeReport_WrongType() { + reportID := "test-report-id" + configID := "test-config-id" + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + snapshot := &storage.ReportSnapshot{ + ReportId: reportID, + ReportConfigurationId: configID, + Requester: creator, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_GENERATED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + Type: storage.ReportSnapshot_VULNERABILITY, + } + + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportID). + Return(snapshot, true, nil).Times(1) + + _, err := s.service.DeleteNodeReport(ctx, &apiV2.DeleteReportRequest{ + Id: reportID, + }) + s.Error(err) + s.Contains(err.Error(), "not a node vulnerability report") +} + +func (s *NodeReportServiceTestSuite) TestUpdateNodeReportConfiguration_NotFound() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + updateConfig := s.getValidNodeReportConfig() + + s.notifierDataStore.EXPECT().GetScrubbedNotifier(gomock.Any(), "email-notifier-id"). + Return(&storage.Notifier{Type: notifiers.EmailType}, true, nil).Times(1) + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), updateConfig.GetId()). + Return(nil, false, nil).Times(1) + + _, err := s.service.UpdateNodeReportConfiguration(ctx, updateConfig) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestRunNodeReport_ConfigNotFound() { + creator := &storage.SlimUser{ + Id: "uid", + Name: "name", + } + ctx := s.getContextForUser(creator) + + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "nonexistent"). + Return(nil, false, nil).Times(1) + + _, err := s.service.RunNodeReport(ctx, &apiV2.RunReportRequest{ + ReportConfigId: "nonexistent", + }) + s.Error(err) +} + +func (s *NodeReportServiceTestSuite) TestRunNodeReport_NoIdentity() { + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "test-config-id"). + Return(&storage.ReportConfiguration{ + Id: "test-config-id", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + }, true, nil).Times(1) + + _, err := s.service.RunNodeReport(s.ctx, &apiV2.RunReportRequest{ + ReportConfigId: "test-config-id", + }) + s.Error(err) + s.Contains(err.Error(), "user identity") +} + +func (s *NodeReportServiceTestSuite) TestGetNodeReportStatus_WrongType() { + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), "report-id"). + Return(&storage.ReportSnapshot{ + ReportId: "report-id", + Type: storage.ReportSnapshot_VULNERABILITY, + }, true, nil).Times(1) + + _, err := s.service.GetNodeReportStatus(s.ctx, &apiV2.ResourceByID{Id: "report-id"}) + s.Error(err) + s.Contains(err.Error(), "not a node vulnerability report") +} + +func (s *NodeReportServiceTestSuite) TestGetMyNodeReportHistory_UsesUserID() { + creator := &storage.SlimUser{Id: "uid", Name: "name"} + ctx := s.getContextForUser(creator) + snapshots := []*storage.ReportSnapshot{ + { + ReportId: "report-1", + ReportConfigurationId: "test-config-id", + Requester: creator, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + }, + } + + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, q *v1.Query) ([]*storage.ReportSnapshot, error) { + var foundUserID bool + search.ApplyFnToAllBaseQueries(q, func(bq *v1.BaseQuery) { + mfQ, ok := bq.GetQuery().(*v1.BaseQuery_MatchFieldQuery) + if !ok { + return + } + s.NotEqual("Requester User ID", mfQ.MatchFieldQuery.GetField()) + if mfQ.MatchFieldQuery.GetField() == search.UserID.String() { + foundUserID = true + s.Contains(mfQ.MatchFieldQuery.GetValue(), "uid") + } + }) + s.True(foundUserID) + return snapshots, nil + }).Times(1) + s.blobStore.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{}, nil).AnyTimes() + + result, err := s.service.GetMyNodeReportHistory(ctx, &apiV2.GetReportHistoryRequest{ + Id: "test-config-id", + ReportParamQuery: &apiV2.RawQuery{ + Query: "", + }, + }) + s.NoError(err) + s.Len(result.GetReportSnapshots(), 1) +} + +func (s *NodeReportServiceTestSuite) TestGetViewBasedNodeReportHistory_ElevatesWorkflowAdministrationRead() { + limitedCtx := sac.WithGlobalAccessScopeChecker(context.Background(), + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS), + sac.ResourceScopeKeys(resources.Node, resources.Cluster), + ), + ) + snapshots := []*storage.ReportSnapshot{ + { + ReportId: "view-report-1", + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + ReportStatus: &storage.ReportStatus{ + ReportRequestType: storage.ReportStatus_VIEW_BASED, + }, + }, + } + + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, _ *v1.Query) ([]*storage.ReportSnapshot, error) { + allowed, err := sac.ForResource(resources.WorkflowAdministration).ReadAllowed(ctx) + s.NoError(err) + s.True(allowed) + return snapshots, nil + }).Times(1) + s.blobStore.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{}, nil).AnyTimes() + + result, err := s.service.GetViewBasedNodeReportHistory(limitedCtx, &apiV2.GetViewBasedReportHistoryRequest{ + ReportParamQuery: &apiV2.RawQuery{Query: ""}, + }) + s.NoError(err) + s.Len(result.GetReportSnapshots(), 1) +} + +func (s *NodeReportServiceTestSuite) TestAuthz() { + svc := serviceImpl{snapshotDatastore: s.reportSnapshotDataStore} + testutils.AssertAuthzWorks(s.T(), &svc) +} + +func (s *NodeReportServiceTestSuite) TestAuthzPermissions() { + svc := &serviceImpl{} + + ctxWithPerms := func(perms map[string]storage.Access) context.Context { + mockID := mockIdentity.NewMockIdentity(s.mockCtrl) + mockID.EXPECT().Permissions().Return(perms).AnyTimes() + return authn.ContextWithIdentity(context.Background(), mockID, s.T()) + } + + viewNodeCluster := map[string]storage.Access{ + "Node": storage.Access_READ_ACCESS, + "Cluster": storage.Access_READ_ACCESS, + } + viewWorkflow := map[string]storage.Access{ + "WorkflowAdministration": storage.Access_READ_ACCESS, + "Node": storage.Access_READ_ACCESS, + "Cluster": storage.Access_READ_ACCESS, + } + modifyWorkflow := map[string]storage.Access{ + "WorkflowAdministration": storage.Access_READ_WRITE_ACCESS, + "Node": storage.Access_READ_ACCESS, + "Cluster": storage.Access_READ_ACCESS, + } + nodeOnly := map[string]storage.Access{ + "Node": storage.Access_READ_ACCESS, + } + + testCases := map[string]struct { + method string + allowedPerms map[string]storage.Access + deniedPerms map[string]storage.Access + }{ + "GetNodeReportStatus allowed with View Node+Cluster, denied without Cluster": { + method: apiV2.NodeReportService_GetNodeReportStatus_FullMethodName, + allowedPerms: viewNodeCluster, + deniedPerms: nodeOnly, + }, + "CancelNodeReport allowed with View Node+Cluster, denied without Cluster": { + method: apiV2.NodeReportService_CancelNodeReport_FullMethodName, + allowedPerms: viewNodeCluster, + deniedPerms: nodeOnly, + }, + "DeleteNodeReport allowed with View Node+Cluster, denied without Cluster": { + method: apiV2.NodeReportService_DeleteNodeReport_FullMethodName, + allowedPerms: viewNodeCluster, + deniedPerms: nodeOnly, + }, + "PostViewBasedNodeReport allowed with View Node+Cluster, denied without Cluster": { + method: apiV2.NodeReportService_PostViewBasedNodeReport_FullMethodName, + allowedPerms: viewNodeCluster, + deniedPerms: nodeOnly, + }, + "ListNodeReportConfigurations requires View WorkflowAdministration": { + method: apiV2.NodeReportService_ListNodeReportConfigurations_FullMethodName, + allowedPerms: viewWorkflow, + deniedPerms: viewNodeCluster, + }, + "RunNodeReport requires Modify WorkflowAdministration": { + method: apiV2.NodeReportService_RunNodeReport_FullMethodName, + allowedPerms: modifyWorkflow, + deniedPerms: viewNodeCluster, + }, + } + + for name, tc := range testCases { + s.T().Run(name, func(t *testing.T) { + _, err := svc.AuthFuncOverride(ctxWithPerms(tc.allowedPerms), tc.method) + assert.NoError(t, err, "should be allowed with sufficient permissions") + + _, err = svc.AuthFuncOverride(ctxWithPerms(tc.deniedPerms), tc.method) + assert.Error(t, err, "should be denied with insufficient permissions") + }) + } +} + +func (s *NodeReportServiceTestSuite) expectNotify(ctx context.Context, channel, payload string) { + s.db.EXPECT().Exec(ctx, "SELECT pg_notify($1, $2)", channel, payload). + Return(pgconn.NewCommandTag("SELECT 1"), nil).Times(1) +} + +func (s *NodeReportServiceTestSuite) TestPostNodeReportConfigurationWithCentralWorker() { + s.T().Setenv(env.CentralWorkerEnabled.EnvVar(), "true") + + creator := &storage.SlimUser{Id: "uid", Name: "name"} + accessScope := &storage.SimpleAccessScope{ + Rules: &storage.SimpleAccessScope_Rules{IncludedClusters: []string{"cluster-1"}}, + } + requestConfig := s.getValidNodeReportConfig() + mockID := mockIdentity.NewMockIdentity(s.mockCtrl) + ctx := authn.ContextWithIdentity(s.ctx, mockID, s.T()) + mockID.EXPECT().UID().Return(creator.GetId()).AnyTimes() + mockID.EXPECT().FullName().Return(creator.GetName()).AnyTimes() + mockID.EXPECT().FriendlyName().Return(creator.GetName()).AnyTimes() + mockRole := permissionsMocks.NewMockResolvedRole(s.mockCtrl) + mockRole.EXPECT().GetAccessScope().Return(accessScope).Times(1) + mockID.EXPECT().Roles().Return([]permissions.ResolvedRole{mockRole}).Times(1) + + s.notifierDataStore.EXPECT().GetScrubbedNotifier(gomock.Any(), "email-notifier-id"). + Return(&storage.Notifier{Type: notifiers.EmailType}, true, nil).Times(1) + s.reportConfigDataStore.EXPECT().AddReportConfiguration(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, cfg *storage.ReportConfiguration) (string, error) { + return cfg.GetId(), nil + }).Times(1) + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), requestConfig.GetId()). + Return(&storage.ReportConfiguration{ + Id: requestConfig.GetId(), + Name: requestConfig.GetName(), + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + Creator: creator, + Filter: &storage.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: &storage.NodeVulnerabilityReportFilters_AllVuln{AllVuln: true}, + }, + }, + }, true, nil).Times(1) + s.expectNotify(ctx, pgNotify.ReportConfigChanged, requestConfig.GetId()) + + result, err := s.service.PostNodeReportConfiguration(ctx, requestConfig) + s.NoError(err) + s.Equal(requestConfig.GetId(), result.GetId()) +} + +func (s *NodeReportServiceTestSuite) TestUpdateNodeReportConfigurationWithCentralWorker() { + s.T().Setenv(env.CentralWorkerEnabled.EnvVar(), "true") + + creator := &storage.SlimUser{Id: "uid", Name: "name"} + updateConfig := s.getValidNodeReportConfig() + ctx := s.getContextForUser(creator) + + s.notifierDataStore.EXPECT().GetScrubbedNotifier(gomock.Any(), "email-notifier-id"). + Return(&storage.Notifier{Type: notifiers.EmailType}, true, nil).Times(1) + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), updateConfig.GetId()). + Return(&storage.ReportConfiguration{ + Id: updateConfig.GetId(), + Name: "Old Name", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + Creator: creator, + Filter: &storage.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: &storage.NodeVulnerabilityReportFilters_AllVuln{AllVuln: true}, + }, + }, + }, true, nil).Times(1) + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return([]*storage.ReportSnapshot{}, nil).Times(1) + s.reportConfigDataStore.EXPECT().UpdateReportConfiguration(gomock.Any(), gomock.Any()).Return(nil).Times(1) + s.expectNotify(ctx, pgNotify.ReportConfigChanged, updateConfig.GetId()) + + _, err := s.service.UpdateNodeReportConfiguration(ctx, updateConfig) + s.NoError(err) +} + +func (s *NodeReportServiceTestSuite) TestDeleteNodeReportConfigurationWithCentralWorker() { + s.T().Setenv(env.CentralWorkerEnabled.EnvVar(), "true") + + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "test-id"). + Return(&storage.ReportConfiguration{ + Id: "test-id", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + }, true, nil).Times(1) + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return([]*storage.ReportSnapshot{}, nil).Times(1) + s.reportConfigDataStore.EXPECT().RemoveReportConfiguration(gomock.Any(), "test-id"). + Return(nil).Times(1) + s.expectNotify(s.ctx, pgNotify.ReportConfigChanged, "test-id") + + _, err := s.service.DeleteNodeReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: "test-id"}) + s.NoError(err) +} + +func (s *NodeReportServiceTestSuite) TestRunNodeReportWithCentralWorker() { + s.T().Setenv(env.CentralWorkerEnabled.EnvVar(), "true") + + creator := &storage.SlimUser{Id: "uid", Name: "name"} + ctx := s.getContextForUser(creator) + configID := "test-config-id" + protoReportConfig := &storage.ReportConfiguration{ + Id: configID, + Name: "test node report", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + Creator: creator, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_ID, + Values: []*storage.RuleValue{{Value: "cluster-1", MatchType: storage.MatchType_EXACT}}, + }, + }, + }, + }, + }, + Filter: &storage.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: &storage.NodeVulnerabilityReportFilters_AllVuln{AllVuln: true}, + }, + }, + } + + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), configID). + Return(protoReportConfig, true, nil).AnyTimes() + s.notifierDataStore.EXPECT().GetManyNotifiers(gomock.Any(), gomock.Any()). + Return(nil, nil).AnyTimes() + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + Return([]*storage.ReportSnapshot{}, nil).Times(1) + s.reportSnapshotDataStore.EXPECT().AddReportSnapshot(gomock.Any(), gomock.Any()). + Return("on-demand-report-id", nil).Times(1) + s.db.EXPECT().Exec(gomock.Any(), "SELECT pg_notify($1, $2)", pgNotify.ReportRequestSubmitted, "on-demand-report-id"). + Return(pgconn.NewCommandTag("SELECT 1"), nil).Times(1) + + result, err := s.service.RunNodeReport(ctx, &apiV2.RunReportRequest{ + ReportConfigId: configID, + ReportNotificationMethod: apiV2.NotificationMethod_DOWNLOAD, + }) + s.NoError(err) + s.Equal("on-demand-report-id", result.GetReportId()) +} + +func (s *NodeReportServiceTestSuite) TestPostViewBasedNodeReportWithCentralWorker() { + s.T().Setenv(env.CentralWorkerEnabled.EnvVar(), "true") + + creator := &storage.SlimUser{Id: "uid", Name: "name"} + ctx := s.getContextForUser(creator) + req := &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_NODE_VULNERABILITY, + Filter: &apiV2.ReportRequestViewBased_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + Query: "Cluster:cluster-1", + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{AllVuln: true}, + }, + }, + } + + s.reportSnapshotDataStore.EXPECT().Count(gomock.Any(), gomock.Any()). + Return(0, nil).Times(1) + s.reportSnapshotDataStore.EXPECT().AddReportSnapshot(gomock.Any(), gomock.Any()). + Return("view-based-report-id", nil).Times(1) + s.db.EXPECT().Exec(gomock.Any(), "SELECT pg_notify($1, $2)", pgNotify.ReportRequestSubmitted, "view-based-report-id"). + Return(pgconn.NewCommandTag("SELECT 1"), nil).Times(1) + + result, err := s.service.PostViewBasedNodeReport(ctx, req) + s.NoError(err) + s.Equal("view-based-report-id", result.GetReportID()) +} + +func (s *NodeReportServiceTestSuite) TestCancelNodeReportWithCentralWorker() { + s.T().Setenv(env.CentralWorkerEnabled.EnvVar(), "true") + + reportID := "test-report-id" + creator := &storage.SlimUser{Id: "uid", Name: "name"} + ctx := s.getContextForUser(creator) + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportID). + Return(&storage.ReportSnapshot{ + ReportId: reportID, + Requester: creator, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_PREPARING, + }, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + }, true, nil).Times(1) + s.expectNotify(ctx, pgNotify.ReportRequestCancelled, reportID) + + _, err := s.service.CancelNodeReport(ctx, &apiV2.ResourceByID{Id: reportID}) + s.NoError(err) +} diff --git a/central/reports/service/v2/node/singleton.go b/central/reports/service/v2/node/singleton.go new file mode 100644 index 0000000000000..a3d0abe2cbafd --- /dev/null +++ b/central/reports/service/v2/node/singleton.go @@ -0,0 +1,33 @@ +package node + +import ( + blobDS "github.com/stackrox/rox/central/blob/datastore" + "github.com/stackrox/rox/central/globaldb" + notifierDS "github.com/stackrox/rox/central/notifier/datastore" + reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" + schedulerV2 "github.com/stackrox/rox/central/reports/scheduler/v2" + snapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + "github.com/stackrox/rox/central/reports/validation" + "github.com/stackrox/rox/pkg/sync" +) + +var ( + once sync.Once + svc Service +) + +// Singleton returns the singleton instance of the node report service. +func Singleton() Service { + once.Do(func() { + svc = New( + reportConfigDS.Singleton(), + snapshotDS.Singleton(), + notifierDS.Singleton(), + schedulerV2.Singleton(), + blobDS.Singleton(), + validation.Singleton(), + globaldb.GetPostgres(), + ) + }) + return svc +} diff --git a/central/reports/service/v2/service.go b/central/reports/service/v2/service.go index 6ef015e4d6adf..2e9625112eb65 100644 --- a/central/reports/service/v2/service.go +++ b/central/reports/service/v2/service.go @@ -12,6 +12,7 @@ import ( collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" apiV2 "github.com/stackrox/rox/generated/api/v2" "github.com/stackrox/rox/pkg/grpc" + "github.com/stackrox/rox/pkg/postgres" ) // Service provides the interface to the gRPC service for reports. @@ -25,7 +26,8 @@ type Service interface { // New returns a new instance of the service. func New(reportConfigStore reportConfigDS.DataStore, snapshotDatastore snapshotDS.DataStore, collectionDatastore collectionDS.DataStore, notifierDatastore notifierDS.DataStore, - scheduler schedulerV2.Scheduler, blobStore blobDS.Datastore, validator *validation.Validator) Service { + scheduler schedulerV2.Scheduler, blobStore blobDS.Datastore, validator *validation.Validator, + db postgres.DB) Service { return &serviceImpl{ reportConfigStore: reportConfigStore, snapshotDatastore: snapshotDatastore, @@ -34,5 +36,6 @@ func New(reportConfigStore reportConfigDS.DataStore, snapshotDatastore snapshotD scheduler: scheduler, blobStore: blobStore, validator: validator, + db: db, } } diff --git a/central/reports/service/v2/service_impl.go b/central/reports/service/v2/service_impl.go index efeb9fe5d73b2..d4e3def92eb8f 100644 --- a/central/reports/service/v2/service_impl.go +++ b/central/reports/service/v2/service_impl.go @@ -10,6 +10,7 @@ import ( "github.com/stackrox/rox/central/reports/common" reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" schedulerV2 "github.com/stackrox/rox/central/reports/scheduler/v2" + reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" snapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" "github.com/stackrox/rox/central/reports/validation" collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" @@ -17,12 +18,16 @@ import ( apiV2 "github.com/stackrox/rox/generated/api/v2" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/auth/permissions" + "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/grpc/authn" "github.com/stackrox/rox/pkg/grpc/authz" "github.com/stackrox/rox/pkg/grpc/authz/perrpc" "github.com/stackrox/rox/pkg/grpc/authz/user" + "github.com/stackrox/rox/pkg/postgres" + pgNotify "github.com/stackrox/rox/pkg/postgres/notify" + "github.com/stackrox/rox/pkg/retry" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" @@ -41,6 +46,7 @@ var ( authorizer = perrpc.FromMap(map[authz.Authorizer][]string{ // V2 API authorization + // TO DO ROX-35954: add view deployment permission to report config APIs user.With(permissions.View(resources.WorkflowAdministration), permissions.View(resources.Image)): { apiV2.ReportService_ListReportConfigurations_FullMethodName, apiV2.ReportService_GetReportConfiguration_FullMethodName, @@ -52,19 +58,23 @@ var ( }, user.With(permissions.Modify(resources.WorkflowAdministration), permissions.View(resources.Image)): { apiV2.ReportService_DeleteReportConfiguration_FullMethodName, + apiV2.ReportService_RunReport_FullMethodName, }, user.With(permissions.View(resources.WorkflowAdministration), permissions.View(resources.Image)): { apiV2.ReportService_GetReportStatus_FullMethodName, apiV2.ReportService_GetReportHistory_FullMethodName, apiV2.ReportService_GetMyReportHistory_FullMethodName, + }, + user.With(permissions.View(resources.Image), permissions.View(resources.Deployment)): { apiV2.ReportService_GetViewBasedReportHistory_FullMethodName, apiV2.ReportService_GetViewBasedMyReportHistory_FullMethodName, + apiV2.ReportService_PostViewBasedReport_FullMethodName, }, - user.With(permissions.Modify(resources.WorkflowAdministration), permissions.View(resources.Image)): { - apiV2.ReportService_RunReport_FullMethodName, - apiV2.ReportService_CancelReport_FullMethodName, + // view permissions are enough if user is deleting a job created by the user + // TO DO ROX-35954: add view deployment permission + user.With(permissions.View(resources.Image)): { apiV2.ReportService_DeleteReport_FullMethodName, - apiV2.ReportService_PostViewBasedReport_FullMethodName, + apiV2.ReportService_CancelReport_FullMethodName, }, }) ) @@ -78,6 +88,7 @@ type serviceImpl struct { scheduler schedulerV2.Scheduler blobStore blobDS.Datastore validator *validation.Validator + db postgres.DB } func (s *serviceImpl) RegisterServiceServer(grpcServer *grpc.Server) { @@ -98,6 +109,10 @@ func (s *serviceImpl) PostReportConfiguration(ctx context.Context, request *apiV return nil, errors.New("Could not determine user identity from provided context") } + if request.GetType() == apiV2.ReportConfiguration_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.New("node vulnerability reports must be created via the node report service") + } + if err := s.validator.ValidateReportConfiguration(request); err != nil { return nil, errors.Wrap(err, "Validating report configuration") } @@ -119,8 +134,9 @@ func (s *serviceImpl) PostReportConfiguration(ctx context.Context, request *apiV return nil, err } - err = s.scheduler.UpsertReportSchedule(createdReportConfig) - if err != nil { + if env.CentralWorkerEnabled.BooleanSetting() { + notifyWithRetry(ctx, s.db, pgNotify.ReportConfigChanged, id) + } else if err := s.scheduler.UpsertReportSchedule(createdReportConfig); err != nil { return nil, err } @@ -146,6 +162,9 @@ func (s *serviceImpl) UpdateReportConfiguration(ctx context.Context, request *ap if !exists { return nil, errors.Wrapf(errox.NotFound, "report configuration with id '%s' does not exist", request.GetId()) } + if err := rejectNodeReportConfiguration(currentConfig); err != nil { + return nil, err + } query := search.NewQueryBuilder().AddExactMatches(search.ReportConfigID, request.GetId()).AddExactMatches(search.ReportState, storage.ReportStatus_WAITING.String(), storage.ReportStatus_PREPARING.String()).ProtoQuery() reportSnapshots, err := s.snapshotDatastore.SearchReportSnapshots(ctx, query) @@ -159,16 +178,17 @@ func (s *serviceImpl) UpdateReportConfiguration(ctx context.Context, request *ap } } - updatedConfig := s.convertV2ReportConfigurationToProto(request, currentConfig.GetCreator(), - currentConfig.GetVulnReportFilters().GetAccessScopeRules()) + accessScopeRules := currentConfig.GetVulnReportFilters().GetAccessScopeRules() + updatedConfig := s.convertV2ReportConfigurationToProto(request, currentConfig.GetCreator(), accessScopeRules) err = s.reportConfigStore.UpdateReportConfiguration(ctx, updatedConfig) if err != nil { return nil, err } - err = s.scheduler.UpsertReportSchedule(updatedConfig) - if err != nil { + if env.CentralWorkerEnabled.BooleanSetting() { + notifyWithRetry(ctx, s.db, pgNotify.ReportConfigChanged, updatedConfig.GetId()) + } else if err := s.scheduler.UpsertReportSchedule(updatedConfig); err != nil { return nil, err } return &apiV2.Empty{}, nil @@ -180,12 +200,14 @@ func (s *serviceImpl) ListReportConfigurations(ctx context.Context, query *apiV2 if err != nil { return nil, errors.Wrap(errox.InvalidArgs, err.Error()) } - filteredQ := common.WithoutV1ReportConfigs(parsedQuery) - + parsedQuery = search.ConjunctionQuery( + parsedQuery, + search.NewQueryBuilder().AddExactMatches(search.ReportType, storage.ReportConfiguration_VULNERABILITY.String()).ProtoQuery(), + ) // Fill in pagination. - paginated.FillPaginationV2(filteredQ, query.GetPagination(), maxPaginationLimit) + paginated.FillPaginationV2(parsedQuery, query.GetPagination(), maxPaginationLimit) - reportConfigs, err := s.reportConfigStore.GetReportConfigurations(ctx, filteredQ) + reportConfigs, err := s.reportConfigStore.GetReportConfigurations(ctx, parsedQuery) if err != nil { return nil, errors.Wrap(err, "failed to retrieve report configurations") } @@ -212,8 +234,13 @@ func (s *serviceImpl) GetReportConfiguration(ctx context.Context, req *apiV2.Res if !exists { return nil, errors.Wrapf(errox.NotFound, "report configuration with id '%s' does not exist", req.GetId()) } - if !common.IsV2ReportConfig(config) { - return nil, errors.Wrap(errox.InvalidArgs, "report configuration does not belong to reporting version 2.0") + if err := rejectNodeReportConfiguration(config); err != nil { + return nil, err + } + // Remove report configs with empty scope. This can happen after downgrade to a version that has less scoping methods and doesn't support the new scoping method. + if !common.HasValidResourceScope(config.GetResourceScope()) { + return nil, errors.Wrapf(errox.InvalidArgs, + "Report configuration '%s' has an empty resource scope (no collection ID or entity scope)", req.GetId()) } converted, err := s.convertProtoReportConfigurationToV2(config) @@ -228,9 +255,11 @@ func (s *serviceImpl) CountReportConfigurations(ctx context.Context, request *ap if err != nil { return nil, errors.Wrap(errox.InvalidArgs, err.Error()) } - filteredQ := common.WithoutV1ReportConfigs(parsedQuery) - - numReportConfigs, err := s.reportConfigStore.Count(ctx, filteredQ) + parsedQuery = search.ConjunctionQuery( + parsedQuery, + search.NewQueryBuilder().AddExactMatches(search.ReportType, storage.ReportConfiguration_VULNERABILITY.String()).ProtoQuery(), + ) + numReportConfigs, err := s.reportConfigStore.Count(ctx, parsedQuery) if err != nil { return nil, err } @@ -248,8 +277,8 @@ func (s *serviceImpl) DeleteReportConfiguration(ctx context.Context, id *apiV2.R if !found { return nil, errors.Wrapf(errox.NotFound, "Report config ID '%s' not found", id.GetId()) } - if !common.IsV2ReportConfig(config) { - return nil, errors.Wrap(errox.InvalidArgs, "report configuration does not belong to reporting version 2.0") + if err := rejectNodeReportConfiguration(config); err != nil { + return nil, err } query := search.NewQueryBuilder().AddExactMatches(search.ReportConfigID, id.GetId()).AddExactMatches(search.ReportState, storage.ReportStatus_WAITING.String(), storage.ReportStatus_PREPARING.String()).ProtoQuery() reportSnapshots, _ := s.snapshotDatastore.SearchReportSnapshots(ctx, query) @@ -261,7 +290,11 @@ func (s *serviceImpl) DeleteReportConfiguration(ctx context.Context, id *apiV2.R return nil, err } - s.scheduler.RemoveReportSchedule(id.GetId()) + if env.CentralWorkerEnabled.BooleanSetting() { + notifyWithRetry(ctx, s.db, pgNotify.ReportConfigChanged, id.GetId()) + } else { + s.scheduler.RemoveReportSchedule(id.GetId()) + } return &apiV2.Empty{}, nil } @@ -276,6 +309,9 @@ func (s *serviceImpl) GetReportStatus(ctx context.Context, req *apiV2.ResourceBy if !found { return nil, errors.Wrapf(errox.NotFound, "Report snapshot not found for job id %s", req.GetId()) } + if err := rejectNodeReportSnapshot(rep); err != nil { + return nil, err + } status := s.convertPrototoV2Reportstatus(rep.GetReportStatus()) return &apiV2.ReportStatusResponse{Status: status}, err } @@ -290,7 +326,10 @@ func (s *serviceImpl) GetReportHistory(ctx context.Context, req *apiV2.GetReport } conjunctionQuery := search.ConjunctionQuery( - search.NewQueryBuilder().AddExactMatches(search.ReportConfigID, req.GetId()).ProtoQuery(), + search.NewQueryBuilder(). + AddExactMatches(search.ReportConfigID, req.GetId()). + AddExactMatches(search.ReportType, storage.ReportSnapshot_VULNERABILITY.String()). + ProtoQuery(), parsedQuery, ) // Fill in pagination. @@ -332,7 +371,9 @@ func (s *serviceImpl) GetMyReportHistory(ctx context.Context, req *apiV2.GetRepo conjunctionQuery := search.ConjunctionQuery( search.NewQueryBuilder(). AddExactMatches(search.ReportConfigID, req.GetId()). - AddExactMatches(search.UserID, slimUser.GetId()).ProtoQuery(), + AddExactMatches(search.UserID, slimUser.GetId()). + AddExactMatches(search.ReportType, storage.ReportSnapshot_VULNERABILITY.String()). + ProtoQuery(), parsedQuery, ) @@ -377,6 +418,20 @@ func (s *serviceImpl) RunReport(ctx context.Context, req *apiV2.RunReportRequest if err != nil { return nil, err } + if reportReq.ReportSnapshot.GetType() == storage.ReportSnapshot_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.Newf("report configuration '%s' is a node vulnerability report; use the node report service", req.GetReportConfigId()) + } + + if env.CentralWorkerEnabled.BooleanSetting() { + reportID, err := s.persistSnapshotAndNotify(ctx, reportReq, pgNotify.ReportRequestSubmitted) + if err != nil { + return nil, err + } + return &apiV2.RunReportResponse{ + ReportConfigId: req.GetReportConfigId(), + ReportId: reportID, + }, nil + } reportID, err := s.scheduler.SubmitReportRequest(ctx, reportReq, false) if err != nil { @@ -390,9 +445,6 @@ func (s *serviceImpl) RunReport(ctx context.Context, req *apiV2.RunReportRequest } func (s *serviceImpl) CancelReport(ctx context.Context, req *apiV2.ResourceByID) (*apiV2.Empty, error) { - if err := sac.VerifyAuthzOK(workflowSAC.WriteAllowed(ctx)); err != nil { - return nil, err - } if req.GetId() == "" { return nil, errors.Wrap(errox.InvalidArgs, "Report job ID is empty") } @@ -406,6 +458,11 @@ func (s *serviceImpl) CancelReport(ctx context.Context, req *apiV2.ResourceByID) return nil, err } + if env.CentralWorkerEnabled.BooleanSetting() { + notifyWithRetry(ctx, s.db, pgNotify.ReportRequestCancelled, req.GetId()) + return &apiV2.Empty{}, nil + } + cancelled, err := s.scheduler.CancelReportRequest(ctx, req.GetId()) if err != nil { return nil, err @@ -436,6 +493,9 @@ func (s *serviceImpl) DeleteReport(ctx context.Context, req *apiV2.DeleteReportR if !found { return nil, errors.Wrapf(errox.NotFound, "Error finding report snapshot with job ID '%q'.", req.GetId()) } + if err := rejectNodeReportSnapshot(rep); err != nil { + return nil, err + } if slimUser.GetId() != rep.GetRequester().GetId() { return nil, errors.Wrap(errox.NotAuthorized, "Report cannot be deleted by a user who did not request the report.") @@ -472,14 +532,12 @@ func (s *serviceImpl) PostViewBasedReport(ctx context.Context, req *apiV2.Report return nil, errors.Wrap(errox.NotImplemented, "View-based vulnerability reports are not enabled. Please enable the ROX_VULNERABILITY_VIEW_BASED_REPORTS feature flag.") } - // Authorisation: must have write access on workflow administration. - if err := sac.VerifyAuthzOK(workflowSAC.WriteAllowed(ctx)); err != nil { - return nil, err - } - if req == nil { return nil, errors.Wrap(errox.InvalidArgs, "Empty Request Body") } + if req.GetType() == apiV2.ReportRequestViewBased_NODE_VULNERABILITY { + return nil, errox.InvalidArgs.New("node vulnerability reports must be created via the node report service") + } requesterID := authn.IdentityFromContextOrNil(ctx) if requesterID == nil { @@ -492,7 +550,14 @@ func (s *serviceImpl) PostViewBasedReport(ctx context.Context, req *apiV2.Report return nil, err } - // Submit to scheduler. view-based reports are always on-demand, not re-submissions. + if env.CentralWorkerEnabled.BooleanSetting() { + reportID, err := s.persistSnapshotAndNotify(ctx, reportReq, pgNotify.ReportRequestSubmitted) + if err != nil { + return nil, err + } + return &apiV2.RunReportResponseViewBased{ReportID: reportID, RequestName: reportReq.ReportSnapshot.GetName()}, nil + } + reportID, err := s.scheduler.SubmitReportRequest(ctx, reportReq, false) if err != nil { return nil, errors.Wrapf(errox.ServerError, "Scheduler error:%s", err) @@ -515,13 +580,18 @@ func (s *serviceImpl) GetViewBasedReportHistory(ctx context.Context, req *apiV2. conjunctionQuery := search.ConjunctionQuery( search.NewQueryBuilder().AddExactMatches( search.ReportRequestType, - storage.ReportStatus_VIEW_BASED.String()).ProtoQuery(), + storage.ReportStatus_VIEW_BASED.String()). + AddExactMatches(search.ReportType, storage.ReportSnapshot_VULNERABILITY.String()). + ProtoQuery(), parsedQuery, ) // Fill in pagination. paginated.FillPaginationV2(conjunctionQuery, req.GetReportParamQuery().GetPagination(), maxPaginationLimit) - results, err := s.snapshotDatastore.SearchReportSnapshots(ctx, conjunctionQuery) + // View-based history endpoints are authorized with only Image+Deployment view. + // The snapshot datastore requires WorkflowAdministration read, so elevate the + // context to that single read scope instead of granting unrestricted access. + results, err := s.snapshotDatastore.SearchReportSnapshots(SnapshotReadContext(ctx), conjunctionQuery) if err != nil { return nil, err } @@ -560,6 +630,7 @@ func (s *serviceImpl) GetViewBasedMyReportHistory(ctx context.Context, req *apiV search.NewQueryBuilder(). AddExactMatches(search.UserID, slimUser.GetId()). AddExactMatches(search.ReportRequestType, storage.ReportStatus_VIEW_BASED.String()). + AddExactMatches(search.ReportType, storage.ReportSnapshot_VULNERABILITY.String()). ProtoQuery(), parsedQuery, ) @@ -567,7 +638,11 @@ func (s *serviceImpl) GetViewBasedMyReportHistory(ctx context.Context, req *apiV // Fill in pagination. paginated.FillPaginationV2(conjunctionQuery, req.GetReportParamQuery().GetPagination(), maxPaginationLimit) - results, err := s.snapshotDatastore.SearchReportSnapshots(ctx, conjunctionQuery) + // View-based history endpoints are authorized with only Image+Deployment view. + // The snapshot datastore requires WorkflowAdministration read, so elevate the + // context to that single read scope. Results are already scoped to the requesting + // user via the UserID query filter above. + results, err := s.snapshotDatastore.SearchReportSnapshots(SnapshotReadContext(ctx), conjunctionQuery) if err != nil { return nil, err } @@ -582,6 +657,11 @@ func (s *serviceImpl) GetViewBasedMyReportHistory(ctx context.Context, req *apiV } func verifyNoUserSearchLabels(q *v1.Query) error { + return VerifyNoUserSearchLabels(q) +} + +// VerifyNoUserSearchLabels rejects queries that filter on user identity fields. +func VerifyNoUserSearchLabels(q *v1.Query) error { unexpectedLabels := set.NewStringSet(search.UserID.String(), search.UserName.String()) var err error search.ApplyFnToAllBaseQueries(q, func(bq *v1.BaseQuery) { @@ -593,3 +673,62 @@ func verifyNoUserSearchLabels(q *v1.Query) error { }) return err } + +func notifyWithRetry(ctx context.Context, db postgres.DB, channel, payload string) { + NotifyWithRetry(ctx, db, channel, payload) +} + +// NotifyWithRetry sends a pg_notify on the given channel, retrying a few times on failure. +// Failures are logged and not returned so that the originating RPC can still succeed. +func NotifyWithRetry(ctx context.Context, db postgres.DB, channel, payload string) { + if db == nil { + return + } + err := retry.WithRetry(func() error { + return pgNotify.Notify(ctx, db, channel, payload) + }, retry.Tries(3), retry.BetweenAttempts(func(previousAttempt int) { + log.Errorf("pg_notify %s failed (attempt %d), retrying", channel, previousAttempt+1) + })) + if err != nil { + log.Errorf("pg_notify %s failed after retries: %v", channel, err) + } +} + +func (s *serviceImpl) persistSnapshotAndNotify(ctx context.Context, reportReq *reportGen.ReportRequest, channel string) (string, error) { + return PersistSnapshotAndNotify(ctx, s.validator, s.db, reportReq, channel) +} + +func rejectNodeReportSnapshot(snapshot *storage.ReportSnapshot) error { + if snapshot.GetType() == storage.ReportSnapshot_NODE_VULNERABILITY { + return errox.InvalidArgs.Newf("report job '%s' is a node vulnerability report; use the node report service", snapshot.GetReportId()) + } + return nil +} + +// SnapshotReadContext elevates the request context with WorkflowAdministration +// read so view-based history RPCs can query the snapshot datastore. +func SnapshotReadContext(ctx context.Context) context.Context { + return sac.WithGlobalAccessScopeChecker(ctx, + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS), + sac.ResourceScopeKeys(resources.WorkflowAdministration), + ), + ) +} + +// PersistSnapshotAndNotify persists a WAITING snapshot and notifies the worker. +func PersistSnapshotAndNotify(ctx context.Context, validator *validation.Validator, db postgres.DB, reportReq *reportGen.ReportRequest, channel string) (string, error) { + reportID, err := validator.PersistReportSnapshot(ctx, reportReq.ReportSnapshot) + if err != nil { + return "", err + } + NotifyWithRetry(ctx, db, channel, reportID) + return reportID, nil +} + +func rejectNodeReportConfiguration(config *storage.ReportConfiguration) error { + if config.GetType() == storage.ReportConfiguration_NODE_VULNERABILITY { + return errox.InvalidArgs.Newf("report configuration '%s' is a node vulnerability report; use the node report service", config.GetId()) + } + return nil +} diff --git a/central/reports/service/v2/service_impl_test.go b/central/reports/service/v2/service_impl_test.go index c454b4c48d685..5f574ba81d094 100644 --- a/central/reports/service/v2/service_impl_test.go +++ b/central/reports/service/v2/service_impl_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/jackc/pgx/v5/pgconn" "github.com/pkg/errors" blobDSMocks "github.com/stackrox/rox/central/blob/datastore/mocks" notifierDSMocks "github.com/stackrox/rox/central/notifier/datastore/mocks" @@ -25,6 +26,9 @@ import ( "github.com/stackrox/rox/pkg/grpc/authn" mockIdentity "github.com/stackrox/rox/pkg/grpc/authn/mocks" "github.com/stackrox/rox/pkg/grpc/testutils" + "github.com/stackrox/rox/pkg/notifiers" + postgresMocks "github.com/stackrox/rox/pkg/postgres/mocks" + pgNotify "github.com/stackrox/rox/pkg/postgres/notify" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/search" @@ -34,10 +38,6 @@ import ( "go.uber.org/mock/gomock" ) -var ( - withoutV1ConfigsQuery = search.NewQueryBuilder().AddExactMatches(search.EmbeddedCollectionID, "").ProtoQuery() -) - type upsertTestCase struct { desc string setMocksAndGenReportConfig func() *apiV2.ReportConfiguration @@ -60,6 +60,7 @@ type ReportServiceTestSuite struct { notifierDataStore *notifierDSMocks.MockDataStore blobStore *blobDSMocks.MockDatastore scheduler *schedulerMocks.MockScheduler + db *postgresMocks.MockDB service Service } @@ -72,8 +73,9 @@ func (s *ReportServiceTestSuite) SetupTest() { s.notifierDataStore = notifierDSMocks.NewMockDataStore(s.mockCtrl) s.blobStore = blobDSMocks.NewMockDatastore(s.mockCtrl) s.scheduler = schedulerMocks.NewMockScheduler(s.mockCtrl) + s.db = postgresMocks.NewMockDB(s.mockCtrl) validator := validation.New(s.reportConfigDataStore, s.reportSnapshotDataStore, s.collectionDataStore, s.notifierDataStore) - s.service = New(s.reportConfigDataStore, s.reportSnapshotDataStore, s.collectionDataStore, s.notifierDataStore, s.scheduler, s.blobStore, validator) + s.service = New(s.reportConfigDataStore, s.reportSnapshotDataStore, s.collectionDataStore, s.notifierDataStore, s.scheduler, s.blobStore, validator, s.db) } func (s *ReportServiceTestSuite) TearDownSuite() { @@ -138,6 +140,142 @@ func (s *ReportServiceTestSuite) TestCreateReportConfiguration() { s.Error(err) } +func (s *ReportServiceTestSuite) TestPostReportConfiguration_RejectsNodeType() { + creator := &storage.SlimUser{Id: "uid", Name: "name"} + ctx := s.getContextForUser(creator) + _, err := s.service.PostReportConfiguration(ctx, &apiV2.ReportConfiguration{ + Name: "node report", + Type: apiV2.ReportConfiguration_NODE_VULNERABILITY, + }) + s.Error(err) + s.Contains(err.Error(), "node report service") +} + +func (s *ReportServiceTestSuite) TestUpdateReportConfiguration_RejectsNodeType() { + protoReportConfig := &storage.ReportConfiguration{ + Id: "node-config", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + } + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "node-config"). + Return(protoReportConfig, true, nil).Times(1) + s.collectionDataStore.EXPECT().Exists(gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() + s.notifierDataStore.EXPECT().GetScrubbedNotifier(gomock.Any(), gomock.Any()). + Return(&storage.Notifier{Type: notifiers.EmailType}, true, nil).AnyTimes() + + requestConfig := fixtures.GetValidV2ReportConfigWithMultipleNotifiers() + requestConfig.Id = "node-config" + _, err := s.service.UpdateReportConfiguration(s.ctx, requestConfig) + s.Error(err) + s.Contains(err.Error(), "node report service") +} + +func (s *ReportServiceTestSuite) TestGetReportConfiguration_RejectsNodeType() { + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "node-config"). + Return(&storage.ReportConfiguration{ + Id: "node-config", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + }, true, nil).Times(1) + + _, err := s.service.GetReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: "node-config"}) + s.Error(err) + s.Contains(err.Error(), "node report service") +} + +func (s *ReportServiceTestSuite) TestDeleteReportConfiguration_RejectsNodeType() { + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "node-config"). + Return(&storage.ReportConfiguration{ + Id: "node-config", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + }, true, nil).Times(1) + + _, err := s.service.DeleteReportConfiguration(s.ctx, &apiV2.ResourceByID{Id: "node-config"}) + s.Error(err) + s.Contains(err.Error(), "node report service") +} + +func (s *ReportServiceTestSuite) TestRunReport_RejectsNodeType() { + creator := &storage.SlimUser{Id: "uid", Name: "name"} + ctx := s.getContextForUser(creator) + nodeConfig := &storage.ReportConfiguration{ + Id: "node-config", + Name: "node report", + Type: storage.ReportConfiguration_NODE_VULNERABILITY, + ResourceScope: &storage.ResourceScope{ + ScopeReference: &storage.ResourceScope_EntityScope{ + EntityScope: &storage.EntityScope{ + Rules: []*storage.EntityScopeRule{ + { + Entity: storage.EntityType_ENTITY_TYPE_CLUSTER, + Field: storage.EntityField_FIELD_ID, + Values: []*storage.RuleValue{{Value: "cluster-1", MatchType: storage.MatchType_EXACT}}, + }, + }, + }, + }, + }, + } + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), "node-config"). + Return(nodeConfig, true, nil).Times(1) + s.notifierDataStore.EXPECT().GetManyNotifiers(gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) + + _, err := s.service.RunReport(ctx, &apiV2.RunReportRequest{ + ReportConfigId: "node-config", + ReportNotificationMethod: apiV2.NotificationMethod_DOWNLOAD, + }) + s.Error(err) + s.Contains(err.Error(), "node report service") +} + +func (s *ReportServiceTestSuite) TestPostViewBasedReport_RejectsNodeType() { + s.T().Setenv(features.VulnerabilityViewBasedReports.EnvVar(), "true") + creator := &storage.SlimUser{Id: "uid", Name: "name"} + ctx := s.getContextForUser(creator) + _, err := s.service.PostViewBasedReport(ctx, &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_NODE_VULNERABILITY, + }) + s.Error(err) + s.Contains(err.Error(), "node report service") +} + +func imageReportTypeQuery(base *v1.Query) *v1.Query { + return search.ConjunctionQuery( + base, + search.NewQueryBuilder().AddExactMatches(search.ReportType, storage.ReportConfiguration_VULNERABILITY.String()).ProtoQuery(), + ) +} + +func (s *ReportServiceTestSuite) TestCreateReportConfigurationWithCentralWorker() { + s.T().Setenv(env.CentralWorkerEnabled.EnvVar(), "true") + + requestConfig := fixtures.GetValidV2ReportConfigWithMultipleNotifiers() + requestConfig.Notifiers = nil + requestConfig.Schedule = nil + s.mockCollectionStoreCalls(requestConfig, true, false, false) + + creator := &storage.SlimUser{Id: "uid", Name: "name"} + identity := mockIdentity.NewMockIdentity(s.mockCtrl) + identity.EXPECT().UID().Return(creator.GetId()).AnyTimes() + identity.EXPECT().FullName().Return(creator.GetName()).AnyTimes() + identity.EXPECT().FriendlyName().Return(creator.GetName()).AnyTimes() + role := permissionsMocks.NewMockResolvedRole(s.mockCtrl) + role.EXPECT().GetAccessScope().Return(&storage.SimpleAccessScope{}).Times(1) + identity.EXPECT().Roles().Return([]permissions.ResolvedRole{role}).Times(1) + ctx := authn.ContextWithIdentity(s.ctx, identity, s.T()) + + storedConfig := fixtures.GetValidReportConfigWithMultipleNotifiersV2() + storedConfig.Notifiers = nil + storedConfig.Schedule = nil + storedConfig.Creator = creator + storedConfig.GetVulnReportFilters().AccessScopeRules = []*storage.SimpleAccessScope_Rules{{}} + s.reportConfigDataStore.EXPECT().AddReportConfiguration(ctx, storedConfig).Return(storedConfig.GetId(), nil).Times(1) + s.reportConfigDataStore.EXPECT().GetReportConfiguration(ctx, storedConfig.GetId()).Return(storedConfig, true, nil).Times(1) + s.db.EXPECT().Exec(ctx, "SELECT pg_notify($1, $2)", pgNotify.ReportConfigChanged, storedConfig.GetId()). + Return(pgconn.NewCommandTag("SELECT 1"), nil).Times(1) + + _, err := s.service.PostReportConfiguration(ctx, requestConfig) + s.NoError(err) +} + func (s *ReportServiceTestSuite) TestUpdateReportConfigurationError() { requester := &storage.SlimUser{ @@ -232,6 +370,30 @@ func (s *ReportServiceTestSuite) TestUpdateReportConfiguration() { } } +func (s *ReportServiceTestSuite) TestUpdateReportConfigurationWithCentralWorker() { + s.T().Setenv(env.CentralWorkerEnabled.EnvVar(), "true") + + creator := &storage.SlimUser{Id: "uid", Name: "name"} + ctx := s.getContextForUser(creator) + requestConfig := fixtures.GetValidV2ReportConfigWithMultipleNotifiers() + requestConfig.Notifiers = nil + requestConfig.Schedule = nil + s.mockCollectionStoreCalls(requestConfig, true, false, true) + + storedConfig := fixtures.GetValidReportConfigWithMultipleNotifiersV2() + storedConfig.Notifiers = nil + storedConfig.Schedule = nil + storedConfig.Creator = creator + s.reportConfigDataStore.EXPECT().GetReportConfiguration(ctx, storedConfig.GetId()).Return(storedConfig, true, nil).Times(1) + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(ctx, gomock.Any()).Return([]*storage.ReportSnapshot{}, nil).Times(1) + s.reportConfigDataStore.EXPECT().UpdateReportConfiguration(ctx, storedConfig).Return(nil).Times(1) + s.db.EXPECT().Exec(ctx, "SELECT pg_notify($1, $2)", pgNotify.ReportConfigChanged, storedConfig.GetId()). + Return(pgconn.NewCommandTag("SELECT 1"), nil).Times(1) + + _, err := s.service.UpdateReportConfiguration(ctx, requestConfig) + s.NoError(err) +} + func (s *ReportServiceTestSuite) TestListReportConfigurations() { allAccessContext := sac.WithAllAccess(context.Background()) testCases := []struct { @@ -243,9 +405,7 @@ func (s *ReportServiceTestSuite) TestListReportConfigurations() { desc: "Empty query", query: &apiV2.RawQuery{Query: ""}, expectedQ: func() *v1.Query { - query := search.ConjunctionQuery( - search.EmptyQuery(), - withoutV1ConfigsQuery) + query := imageReportTypeQuery(search.EmptyQuery()) query.Pagination = &v1.QueryPagination{Limit: maxPaginationLimit} return query }(), @@ -254,9 +414,7 @@ func (s *ReportServiceTestSuite) TestListReportConfigurations() { desc: "Query with search field", query: &apiV2.RawQuery{Query: "Report Name:name"}, expectedQ: func() *v1.Query { - query := search.ConjunctionQuery( - search.NewQueryBuilder().AddStrings(search.ReportName, "name").ProtoQuery(), - withoutV1ConfigsQuery) + query := imageReportTypeQuery(search.NewQueryBuilder().AddStrings(search.ReportName, "name").ProtoQuery()) query.Pagination = &v1.QueryPagination{Limit: maxPaginationLimit} return query }(), @@ -268,9 +426,7 @@ func (s *ReportServiceTestSuite) TestListReportConfigurations() { Pagination: &apiV2.Pagination{Limit: 25}, }, expectedQ: func() *v1.Query { - query := search.ConjunctionQuery( - search.EmptyQuery(), - withoutV1ConfigsQuery) + query := imageReportTypeQuery(search.EmptyQuery()) query.Pagination = &v1.QueryPagination{Limit: 25} return query }(), @@ -362,18 +518,14 @@ func (s *ReportServiceTestSuite) TestCountReportConfigurations() { expectedQ *v1.Query }{ { - desc: "Empty query", - query: &apiV2.RawQuery{Query: ""}, - expectedQ: search.ConjunctionQuery( - search.NewQueryBuilder().ProtoQuery(), - withoutV1ConfigsQuery), + desc: "Empty query", + query: &apiV2.RawQuery{Query: ""}, + expectedQ: imageReportTypeQuery(search.NewQueryBuilder().ProtoQuery()), }, { - desc: "Query with search field", - query: &apiV2.RawQuery{Query: "Report Name:name"}, - expectedQ: search.ConjunctionQuery( - search.NewQueryBuilder().AddStrings(search.ReportName, "name").ProtoQuery(), - withoutV1ConfigsQuery), + desc: "Query with search field", + query: &apiV2.RawQuery{Query: "Report Name:name"}, + expectedQ: imageReportTypeQuery(search.NewQueryBuilder().AddStrings(search.ReportName, "name").ProtoQuery()), }, } @@ -422,6 +574,22 @@ func (s *ReportServiceTestSuite) TestDeleteReportConfiguration() { } } +func (s *ReportServiceTestSuite) TestDeleteReportConfigurationWithCentralWorker() { + s.T().Setenv(env.CentralWorkerEnabled.EnvVar(), "true") + + const configID = "config-id" + ctx := sac.WithAllAccess(context.Background()) + s.reportConfigDataStore.EXPECT().GetReportConfiguration(gomock.Any(), configID). + Return(fixtures.GetValidReportConfigWithMultipleNotifiersV2(), true, nil).Times(1) + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()).Return([]*storage.ReportSnapshot{}, nil).Times(1) + s.reportConfigDataStore.EXPECT().RemoveReportConfiguration(ctx, configID).Return(nil).Times(1) + s.db.EXPECT().Exec(ctx, "SELECT pg_notify($1, $2)", pgNotify.ReportConfigChanged, configID). + Return(pgconn.NewCommandTag("SELECT 1"), nil).Times(1) + + _, err := s.service.DeleteReportConfiguration(ctx, &apiV2.ResourceByID{Id: configID}) + s.NoError(err) +} + func (s *ReportServiceTestSuite) upsertReportConfigTestCases(isUpdate bool) []upsertTestCase { cases := []upsertTestCase{ { @@ -684,11 +852,11 @@ func (s *ReportServiceTestSuite) upsertReportConfigTestCases(isUpdate bool) []up func (s *ReportServiceTestSuite) mockNotifierStoreCalls(reqNotifier *apiV2.NotifierConfiguration, notifierIDExits, isValidationError, isUpdate bool) { if notifierIDExits { - s.notifierDataStore.EXPECT().Exists(gomock.Any(), reqNotifier.GetEmailConfig().GetNotifierId()). - Return(true, nil).Times(1) + s.notifierDataStore.EXPECT().GetScrubbedNotifier(gomock.Any(), reqNotifier.GetEmailConfig().GetNotifierId()). + Return(&storage.Notifier{Type: notifiers.EmailType}, true, nil).Times(1) } else { - s.notifierDataStore.EXPECT().Exists(gomock.Any(), reqNotifier.GetEmailConfig().GetNotifierId()). - Return(false, nil).Times(1) + s.notifierDataStore.EXPECT().GetScrubbedNotifier(gomock.Any(), reqNotifier.GetEmailConfig().GetNotifierId()). + Return(nil, false, nil).Times(1) } if !isValidationError && !isUpdate { @@ -746,6 +914,53 @@ func (s *ReportServiceTestSuite) TestGetReportStatus() { assert.Equal(s.T(), repStatusResponse.GetStatus().GetErrorMsg(), status.GetErrorMsg()) } +func (s *ReportServiceTestSuite) TestGetReportStatus_RejectsNodeType() { + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), "node-job").Return(&storage.ReportSnapshot{ + ReportId: "node-job", + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + ReportStatus: &storage.ReportStatus{}, + }, true, nil) + + _, err := s.service.GetReportStatus(s.ctx, &apiV2.ResourceByID{Id: "node-job"}) + s.Error(err) + s.Contains(err.Error(), "node report service") +} + +func (s *ReportServiceTestSuite) TestGetReportHistory_FiltersImageReportType() { + var captured *v1.Query + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, q *v1.Query) ([]*storage.ReportSnapshot, error) { + captured = q + return nil, nil + }) + + _, err := s.service.GetReportHistory(s.ctx, &apiV2.GetReportHistoryRequest{ + Id: "test_report_config", + ReportParamQuery: &apiV2.RawQuery{Query: ""}, + }) + s.NoError(err) + s.True(queryHasFieldValue(captured, search.ReportType, storage.ReportSnapshot_VULNERABILITY.String()), + "history query must constrain ReportType to VULNERABILITY, got %s", captured) +} + +func (s *ReportServiceTestSuite) TestGetMyReportHistory_FiltersImageReportType() { + user := &storage.SlimUser{Id: "user-a", Name: "user-a"} + var captured *v1.Query + s.reportSnapshotDataStore.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, q *v1.Query) ([]*storage.ReportSnapshot, error) { + captured = q + return nil, nil + }) + + _, err := s.service.GetMyReportHistory(s.getContextForUser(user), &apiV2.GetReportHistoryRequest{ + Id: "test_report_config", + ReportParamQuery: &apiV2.RawQuery{Query: ""}, + }) + s.NoError(err) + s.True(queryHasFieldValue(captured, search.ReportType, storage.ReportSnapshot_VULNERABILITY.String()), + "my-history query must constrain ReportType to VULNERABILITY, got %s", captured) +} + func (s *ReportServiceTestSuite) TestGetReportHistory() { reportSnapshot := &storage.ReportSnapshot{ ReportId: "test_report", @@ -859,6 +1074,111 @@ func (s *ReportServiceTestSuite) TestAuthz() { testutils.AssertAuthzWorks(s.T(), &svc) } +// TestAuthzPermissions verifies that each reporting API method enforces the correct permissions. +// View-based report methods require only View Image + View Deployment (matching the CVE Results +// page) while collection-based report management requires WorkflowAdministration. +func (s *ReportServiceTestSuite) TestAuthzPermissions() { + svc := &serviceImpl{} + + ctxWithPerms := func(perms map[string]storage.Access) context.Context { + mockID := mockIdentity.NewMockIdentity(s.mockCtrl) + mockID.EXPECT().Permissions().Return(perms).AnyTimes() + return authn.ContextWithIdentity(context.Background(), mockID, s.T()) + } + + // viewImageDeployment is the minimum permission set for view-based report APIs — + // the same permissions required to view CVE results in the UI. + viewImageDeployment := map[string]storage.Access{ + "Image": storage.Access_READ_ACCESS, + "Deployment": storage.Access_READ_ACCESS, + } + // modifyWorkflow is required for collection-based report lifecycle operations. + modifyWorkflow := map[string]storage.Access{ + "WorkflowAdministration": storage.Access_READ_WRITE_ACCESS, + "Image": storage.Access_READ_ACCESS, + "Deployment": storage.Access_READ_ACCESS, + } + // viewWorkflow is required for read-only collection-based report APIs. + viewWorkflow := map[string]storage.Access{ + "WorkflowAdministration": storage.Access_READ_ACCESS, + "Image": storage.Access_READ_ACCESS, + "Deployment": storage.Access_READ_ACCESS, + } + // viewImageOnly is insufficient for view-based report APIs since Deployment access is missing. + viewImageOnly := map[string]storage.Access{ + "Image": storage.Access_READ_ACCESS, + } + // deploymentOnly has no Image access, so it is denied by any report API that requires View Image. + deploymentOnly := map[string]storage.Access{ + "Deployment": storage.Access_READ_ACCESS, + } + + testCases := map[string]struct { + method string + allowedPerms map[string]storage.Access + deniedPerms map[string]storage.Access + }{ + // View-based report APIs require View Image + View Deployment; no WorkflowAdministration needed. + "PostViewBasedReport allowed with View Image+Deployment, denied without Deployment": { + method: apiV2.ReportService_PostViewBasedReport_FullMethodName, + allowedPerms: viewImageDeployment, + deniedPerms: viewImageOnly, + }, + "GetViewBasedReportHistory allowed with View Image+Deployment, denied without Deployment": { + method: apiV2.ReportService_GetViewBasedReportHistory_FullMethodName, + allowedPerms: viewImageDeployment, + deniedPerms: viewImageOnly, + }, + "GetViewBasedMyReportHistory allowed with View Image+Deployment, denied without Deployment": { + method: apiV2.ReportService_GetViewBasedMyReportHistory_FullMethodName, + allowedPerms: viewImageDeployment, + deniedPerms: viewImageOnly, + }, + // CancelReport and DeleteReport only require View Image (per-user job operations). + "CancelReport allowed with View Image, denied without Image": { + method: apiV2.ReportService_CancelReport_FullMethodName, + allowedPerms: viewImageOnly, + deniedPerms: deploymentOnly, + }, + // Collection-based report APIs require WorkflowAdministration; View Image+Deployment alone is insufficient. + "RunReport requires Modify WorkflowAdministration, denied with View Image+Deployment only": { + method: apiV2.ReportService_RunReport_FullMethodName, + allowedPerms: modifyWorkflow, + deniedPerms: viewImageDeployment, + }, + "DeleteReport allowed with View Image, denied without Image": { + method: apiV2.ReportService_DeleteReport_FullMethodName, + allowedPerms: viewImageOnly, + deniedPerms: deploymentOnly, + }, + "ListReportConfigurations requires View WorkflowAdministration, denied without it": { + method: apiV2.ReportService_ListReportConfigurations_FullMethodName, + allowedPerms: viewWorkflow, + deniedPerms: viewImageOnly, + }, + "GetReportStatus requires View WorkflowAdministration, denied without it": { + method: apiV2.ReportService_GetReportStatus_FullMethodName, + allowedPerms: viewWorkflow, + deniedPerms: viewImageDeployment, + }, + "GetReportHistory requires View WorkflowAdministration, denied without it": { + method: apiV2.ReportService_GetReportHistory_FullMethodName, + allowedPerms: viewWorkflow, + deniedPerms: viewImageDeployment, + }, + } + + for name, tc := range testCases { + s.T().Run(name, func(t *testing.T) { + _, err := svc.AuthFuncOverride(ctxWithPerms(tc.allowedPerms), tc.method) + assert.NoError(t, err, "should be allowed with sufficient permissions") + + _, err = svc.AuthFuncOverride(ctxWithPerms(tc.deniedPerms), tc.method) + assert.Error(t, err, "should be denied with insufficient permissions") + }) + } +} + func (s *ReportServiceTestSuite) TestRunReport() { reportConfig := fixtures.GetValidReportConfigWithMultipleNotifiersV2() notifierIDs := make([]string, 0, len(reportConfig.GetNotifiers())) @@ -1132,6 +1452,24 @@ func (s *ReportServiceTestSuite) TestCancelReport() { snap.ReportStatus.RunState = storage.ReportStatus_PREPARING s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportSnapshot.GetReportId()). Return(snap, true, nil).Times(1) + s.scheduler.EXPECT().CancelReportRequest(gomock.Any(), reportSnapshot.GetReportId()). + Return(true, nil).Times(1) + }, + isError: false, + }, + { + desc: "Report in PREPARING state but no longer in running map", + req: &apiV2.ResourceByID{ + Id: reportSnapshot.GetReportId(), + }, + ctx: userContext, + mockGen: func() { + snap := reportSnapshot.CloneVT() + snap.ReportStatus.RunState = storage.ReportStatus_PREPARING + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), reportSnapshot.GetReportId()). + Return(snap, true, nil).Times(1) + s.scheduler.EXPECT().CancelReportRequest(gomock.Any(), reportSnapshot.GetReportId()). + Return(false, nil).Times(1) }, isError: true, }, @@ -1325,6 +1663,20 @@ func (s *ReportServiceTestSuite) TestDeleteReport() { }, isError: false, }, + { + desc: "Node vulnerability snapshot is rejected", + req: &apiV2.DeleteReportRequest{ + Id: reportSnapshot.GetReportId(), + }, + ctx: userContext, + mockGen: func() { + snap := reportSnapshot.CloneVT() + snap.Type = storage.ReportSnapshot_NODE_VULNERABILITY + s.reportSnapshotDataStore.EXPECT().Get(gomock.Any(), snap.GetReportId()). + Return(snap, true, nil).Times(1) + }, + isError: true, + }, } for _, tc := range testCases { s.T().Run(tc.desc, func(t *testing.T) { @@ -1695,3 +2047,17 @@ func (s *ReportServiceTestSuite) getContextForUser(user *storage.SlimUser) conte return authn.ContextWithIdentity(s.ctx, mockID, s.T()) } + +func queryHasFieldValue(q *v1.Query, field search.FieldLabel, want string) bool { + found := false + search.ApplyFnToAllBaseQueries(q, func(bq *v1.BaseQuery) { + mfQ, ok := bq.GetQuery().(*v1.BaseQuery_MatchFieldQuery) + if !ok { + return + } + if mfQ.MatchFieldQuery.GetField() == field.String() && strings.Contains(mfQ.MatchFieldQuery.GetValue(), want) { + found = true + } + }) + return found +} diff --git a/central/reports/service/v2/singleton.go b/central/reports/service/v2/singleton.go index f8b262110c896..45e900480243b 100644 --- a/central/reports/service/v2/singleton.go +++ b/central/reports/service/v2/singleton.go @@ -2,12 +2,14 @@ package v2 import ( blobDS "github.com/stackrox/rox/central/blob/datastore" + "github.com/stackrox/rox/central/globaldb" notifierDS "github.com/stackrox/rox/central/notifier/datastore" reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" schedulerV2 "github.com/stackrox/rox/central/reports/scheduler/v2" snapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" "github.com/stackrox/rox/central/reports/validation" collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" + "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/sync" ) @@ -18,12 +20,14 @@ var ( func initialize() { scheduler := schedulerV2.Singleton() - // Start() also queues previously pending reports and scheduled reports, so running it in a separate routine to prevent - // blocking main routine - go scheduler.Start() + if !env.CentralWorkerEnabled.BooleanSetting() { + go scheduler.Start(globaldb.GetPostgres()) + } else { + log.Info("Report scheduling is managed by central-worker, skipping start in Central") + } collectionDatastore, _ := collectionDS.Singleton() svc = New(reportConfigDS.Singleton(), snapshotDS.Singleton(), collectionDatastore, notifierDS.Singleton(), scheduler, - blobDS.Singleton(), validation.Singleton()) + blobDS.Singleton(), validation.Singleton(), globaldb.GetPostgres()) } // Singleton provides the instance of the service to register. diff --git a/central/reports/snapshot/datastore/datastore_impl.go b/central/reports/snapshot/datastore/datastore_impl.go index e5fcf3ee11604..fe8cb89fb8c5b 100644 --- a/central/reports/snapshot/datastore/datastore_impl.go +++ b/central/reports/snapshot/datastore/datastore_impl.go @@ -188,7 +188,3 @@ func (c *ReportSnapshotSearchResultConverter) BuildLocation(result *pkgSearch.Re func (c *ReportSnapshotSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_REPORT_SNAPSHOT } - -func (c *ReportSnapshotSearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/reports/snapshot/datastore/store/postgres/store.go b/central/reports/snapshot/datastore/store/postgres/store.go index d019cc69b89cd..12fc029ac5500 100644 --- a/central/reports/snapshot/datastore/store/postgres/store.go +++ b/central/reports/snapshot/datastore/store/postgres/store.go @@ -104,6 +104,7 @@ func insertIntoReportSnapshots(batch *pgx.Batch, obj *storage.ReportSnapshot) er pgutils.NilOrUUID(obj.GetReportId()), pgutils.NilOrString(obj.GetReportConfigurationId()), obj.GetName(), + obj.GetType(), obj.GetReportStatus().GetRunState(), protocompat.NilOrTime(obj.GetReportStatus().GetQueuedAt()), protocompat.NilOrTime(obj.GetReportStatus().GetCompletedAt()), @@ -115,7 +116,7 @@ func insertIntoReportSnapshots(batch *pgx.Batch, obj *storage.ReportSnapshot) er serialized, } - finalStr := "INSERT INTO report_snapshots (ReportId, ReportConfigurationId, Name, ReportStatus_RunState, ReportStatus_QueuedAt, ReportStatus_CompletedAt, ReportStatus_ReportRequestType, ReportStatus_ReportNotificationMethod, Requester_Id, Requester_Name, AreaOfConcern, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT(ReportId) DO UPDATE SET ReportId = EXCLUDED.ReportId, ReportConfigurationId = EXCLUDED.ReportConfigurationId, Name = EXCLUDED.Name, ReportStatus_RunState = EXCLUDED.ReportStatus_RunState, ReportStatus_QueuedAt = EXCLUDED.ReportStatus_QueuedAt, ReportStatus_CompletedAt = EXCLUDED.ReportStatus_CompletedAt, ReportStatus_ReportRequestType = EXCLUDED.ReportStatus_ReportRequestType, ReportStatus_ReportNotificationMethod = EXCLUDED.ReportStatus_ReportNotificationMethod, Requester_Id = EXCLUDED.Requester_Id, Requester_Name = EXCLUDED.Requester_Name, AreaOfConcern = EXCLUDED.AreaOfConcern, serialized = EXCLUDED.serialized" + finalStr := "INSERT INTO report_snapshots (ReportId, ReportConfigurationId, Name, Type, ReportStatus_RunState, ReportStatus_QueuedAt, ReportStatus_CompletedAt, ReportStatus_ReportRequestType, ReportStatus_ReportNotificationMethod, Requester_Id, Requester_Name, AreaOfConcern, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT(ReportId) DO UPDATE SET ReportId = EXCLUDED.ReportId, ReportConfigurationId = EXCLUDED.ReportConfigurationId, Name = EXCLUDED.Name, Type = EXCLUDED.Type, ReportStatus_RunState = EXCLUDED.ReportStatus_RunState, ReportStatus_QueuedAt = EXCLUDED.ReportStatus_QueuedAt, ReportStatus_CompletedAt = EXCLUDED.ReportStatus_CompletedAt, ReportStatus_ReportRequestType = EXCLUDED.ReportStatus_ReportRequestType, ReportStatus_ReportNotificationMethod = EXCLUDED.ReportStatus_ReportNotificationMethod, Requester_Id = EXCLUDED.Requester_Id, Requester_Name = EXCLUDED.Requester_Name, AreaOfConcern = EXCLUDED.AreaOfConcern, serialized = EXCLUDED.serialized" batch.Queue(finalStr, values...) return nil @@ -125,6 +126,7 @@ var copyColsReportSnapshots = []string{ "reportid", "reportconfigurationid", "name", + "type", "reportstatus_runstate", "reportstatus_queuedat", "reportstatus_completedat", @@ -170,6 +172,7 @@ func copyFromReportSnapshots(ctx context.Context, s pgSearch.Deleter, tx *postgr pgutils.NilOrUUID(obj.GetReportId()), pgutils.NilOrString(obj.GetReportConfigurationId()), obj.GetName(), + obj.GetType(), obj.GetReportStatus().GetRunState(), protocompat.NilOrTime(obj.GetReportStatus().GetQueuedAt()), protocompat.NilOrTime(obj.GetReportStatus().GetCompletedAt()), diff --git a/central/reports/validation/entity_scope_boundary_test.go b/central/reports/validation/entity_scope_boundary_test.go new file mode 100644 index 0000000000000..26ddadc18da91 --- /dev/null +++ b/central/reports/validation/entity_scope_boundary_test.go @@ -0,0 +1,63 @@ +package validation + +import ( + "testing" + + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/pkg/errox" + "github.com/stretchr/testify/require" +) + +func TestValidateEntityScopeRejectsUnsupportedRulesAndValues(t *testing.T) { + tests := map[string]struct { + entity apiV2.ScopeEntity + field apiV2.ScopeField + value *apiV2.RuleValue + }{ + "unsupported cluster ID": { + entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + field: apiV2.ScopeField_FIELD_ID, + value: &apiV2.RuleValue{Value: "cluster-id"}, + }, + "unknown entity": { + entity: apiV2.ScopeEntity(99), + field: apiV2.ScopeField_FIELD_NAME, + value: &apiV2.RuleValue{Value: "prod"}, + }, + "unknown field": { + entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + field: apiV2.ScopeField(99), + value: &apiV2.RuleValue{Value: "prod"}, + }, + "unknown match type": { + entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + field: apiV2.ScopeField_FIELD_NAME, + value: &apiV2.RuleValue{Value: "prod", MatchType: apiV2.MatchType(99)}, + }, + "nil value": { + entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + field: apiV2.ScopeField_FIELD_NAME, + }, + "empty scalar value": { + entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + field: apiV2.ScopeField_FIELD_NAME, + value: &apiV2.RuleValue{}, + }, + "invalid regex key": { + entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + field: apiV2.ScopeField_FIELD_LABEL, + value: &apiV2.RuleValue{Value: "[=prod", MatchType: apiV2.MatchType_REGEX}, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + err := validateEntityScope(&apiV2.EntityScope{Rules: []*apiV2.EntityScopeRule{{ + Entity: tc.entity, + Field: tc.field, + Values: []*apiV2.RuleValue{tc.value}, + }}}) + require.ErrorIs(t, err, errox.InvalidArgs) + }) + } +} diff --git a/central/reports/validation/filter_enum_boundary_test.go b/central/reports/validation/filter_enum_boundary_test.go new file mode 100644 index 0000000000000..b1e707cbbc4f6 --- /dev/null +++ b/central/reports/validation/filter_enum_boundary_test.go @@ -0,0 +1,55 @@ +package validation + +import ( + "testing" + + reportConfigDSMocks "github.com/stackrox/rox/central/reports/config/datastore/mocks" + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/errox" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestValidateAndGenerateReportRequestRejectsInvalidNotificationMethodBeforeLookup(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + validator := &Validator{reportConfigDatastore: reportConfigDSMocks.NewMockDataStore(ctrl)} + + _, err := validator.ValidateAndGenerateReportRequest( + "config", + storage.ReportStatus_NotificationMethod(99), + storage.ReportStatus_ON_DEMAND, + nil, + ) + require.ErrorIs(t, err, errox.InvalidArgs) +} + +func TestValidateImageFiltersRejectsInvalidEnumsAndDates(t *testing.T) { + tests := map[string]*apiV2.VulnerabilityReportFilters{ + "image type": { + ImageTypes: []apiV2.VulnerabilityReportFilters_ImageType{apiV2.VulnerabilityReportFilters_ImageType(99)}, + CvesSince: &apiV2.VulnerabilityReportFilters_AllVuln{AllVuln: true}, + }, + "fixability": { + ImageTypes: []apiV2.VulnerabilityReportFilters_ImageType{apiV2.VulnerabilityReportFilters_DEPLOYED}, + Fixability: apiV2.VulnerabilityReportFilters_Fixability(99), + CvesSince: &apiV2.VulnerabilityReportFilters_AllVuln{AllVuln: true}, + }, + "severity": { + ImageTypes: []apiV2.VulnerabilityReportFilters_ImageType{apiV2.VulnerabilityReportFilters_DEPLOYED}, + Severities: []apiV2.VulnerabilityReportFilters_VulnerabilitySeverity{apiV2.VulnerabilityReportFilters_VulnerabilitySeverity(99)}, + CvesSince: &apiV2.VulnerabilityReportFilters_AllVuln{AllVuln: true}, + }, + "nil start date": { + ImageTypes: []apiV2.VulnerabilityReportFilters_ImageType{apiV2.VulnerabilityReportFilters_DEPLOYED}, + CvesSince: &apiV2.VulnerabilityReportFilters_SinceStartDate{}, + }, + } + + for name, filters := range tests { + t.Run(name, func(t *testing.T) { + require.ErrorIs(t, (&Validator{}).validateImageFilters(filters), errox.InvalidArgs) + }) + } +} diff --git a/central/reports/validation/notifier_type_boundary_test.go b/central/reports/validation/notifier_type_boundary_test.go new file mode 100644 index 0000000000000..56825c94b2cd3 --- /dev/null +++ b/central/reports/validation/notifier_type_boundary_test.go @@ -0,0 +1,24 @@ +package validation + +import ( + "testing" + + notifierDSMocks "github.com/stackrox/rox/central/notifier/datastore/mocks" + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/errox" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestValidateEmailConfigRejectsNonEmailNotifier(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + notifierDS := notifierDSMocks.NewMockDataStore(ctrl) + notifierDS.EXPECT().GetScrubbedNotifier(gomock.Any(), "notifier").Return(&storage.Notifier{Type: "slack"}, true, nil) + + err := (&Validator{notifierDatastore: notifierDS}).validateEmailConfig(&apiV2.EmailNotifierConfiguration{ + NotifierId: "notifier", MailingLists: []string{"reader@example.com"}, + }) + require.ErrorIs(t, err, errox.InvalidArgs) +} diff --git a/central/reports/validation/schedule_boundary_test.go b/central/reports/validation/schedule_boundary_test.go new file mode 100644 index 0000000000000..459d8a7ca4729 --- /dev/null +++ b/central/reports/validation/schedule_boundary_test.go @@ -0,0 +1,28 @@ +package validation + +import ( + "testing" + + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/pkg/errox" + "github.com/stretchr/testify/require" +) + +func TestValidateScheduleRejectsInvalidBoundsAndIntervals(t *testing.T) { + tests := map[string]*apiV2.ReportSchedule{ + "negative hour": {IntervalType: apiV2.ReportSchedule_DAILY, Hour: -1}, + "hour 24": {IntervalType: apiV2.ReportSchedule_DAILY, Hour: 24}, + "negative minute": {IntervalType: apiV2.ReportSchedule_DAILY, Minute: -1}, + "minute 60": {IntervalType: apiV2.ReportSchedule_DAILY, Minute: 60}, + "unknown interval": { + IntervalType: apiV2.ReportSchedule_IntervalType(99), + }, + } + + for name, schedule := range tests { + t.Run(name, func(t *testing.T) { + err := (&Validator{}).validateSchedule(&apiV2.ReportConfiguration{Schedule: schedule}) + require.ErrorIs(t, err, errox.InvalidArgs) + }) + } +} diff --git a/central/reports/validation/validate.go b/central/reports/validation/validate.go index 3214cb96a423e..4c00f9431d603 100644 --- a/central/reports/validation/validate.go +++ b/central/reports/validation/validate.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/mail" + "regexp" "strings" "time" @@ -20,10 +21,16 @@ import ( "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/errorhelpers" "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/grpc/authn" + "github.com/stackrox/rox/pkg/notifiers" + "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/stringutils" "github.com/stackrox/rox/pkg/uuid" + k8sValidation "k8s.io/apimachinery/pkg/api/validate/content" ) // Use this context only to @@ -53,7 +60,7 @@ func New(reportConfigDatastore reportConfigDS.DataStore, reportSnapshotDatastore // ValidateReportConfiguration validates the given report configuration object func (v *Validator) ValidateReportConfiguration(config *apiV2.ReportConfiguration) error { if config.GetName() == "" { - return errors.Wrap(errox.InvalidArgs, "Report configuration name is empty") + return errox.InvalidArgs.New("Report configuration name is empty") } if err := v.validateSchedule(config); err != nil { @@ -77,25 +84,35 @@ func (v *Validator) validateSchedule(config *apiV2.ReportConfiguration) error { if schedule == nil { return nil } + if schedule.GetHour() < 0 || schedule.GetHour() > 23 { + return errox.InvalidArgs.New("schedule hour must be within 0-23") + } + if schedule.GetMinute() < 0 || schedule.GetMinute() > 59 { + return errox.InvalidArgs.New("schedule minute must be within 0-59") + } switch schedule.GetIntervalType() { - case apiV2.ReportSchedule_UNSET: - return errors.Wrap(errox.InvalidArgs, "Report configuration schedule must be one of WEEKLY or MONTHLY") + default: + return errox.InvalidArgs.New("report configuration schedule must be one of DAILY, WEEKLY, or MONTHLY") + case apiV2.ReportSchedule_DAILY: + if schedule.GetDaysOfWeek() != nil || schedule.GetDaysOfMonth() != nil { + return errox.InvalidArgs.New("daily schedule must not specify days of week or days of month") + } case apiV2.ReportSchedule_WEEKLY: if schedule.GetDaysOfWeek() == nil || len(schedule.GetDaysOfWeek().GetDays()) == 0 { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify days of week for weekly schedule") + return errox.InvalidArgs.New("report configuration must specify days of week for weekly schedule") } for _, day := range schedule.GetDaysOfWeek().GetDays() { if day < 0 || day > 6 { - return errors.Wrap(errox.InvalidArgs, "Invalid schedule: Days of the week can be Sunday (0) - Saturday(6)") + return errox.InvalidArgs.New("invalid schedule: days of the week can be Sunday (0) - Saturday(6)") } } case apiV2.ReportSchedule_MONTHLY: if schedule.GetDaysOfMonth() == nil || len(schedule.GetDaysOfMonth().GetDays()) == 0 { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify days of the month for monthly schedule") + return errox.InvalidArgs.New("report configuration must specify days of the month for monthly schedule") } for _, day := range schedule.GetDaysOfMonth().GetDays() { if day != 1 && day != 15 { - return errors.Wrap(errox.InvalidArgs, "Reports can be sent out only 1st or 15th day of the month") + return errox.InvalidArgs.New("reports can be sent out only 1st or 15th day of the month") } } } @@ -106,13 +123,13 @@ func (v *Validator) validateNotifiers(config *apiV2.ReportConfiguration) error { notifiers := config.GetNotifiers() if len(notifiers) == 0 { if config.GetSchedule() != nil { - return errors.Wrap(errox.InvalidArgs, "Report configurations with a schedule must specify a notifier.") + return errox.InvalidArgs.New("report configurations with a schedule must specify a notifier") } return nil } for _, notifier := range notifiers { if notifier.GetEmailConfig() == nil { - return errors.Wrap(errox.InvalidArgs, "Notifier must specify an email notifier configuration") + return errox.InvalidArgs.New("notifier must specify an email notifier configuration") } if err := v.validateEmailConfig(notifier.GetEmailConfig()); err != nil { return err @@ -123,24 +140,24 @@ func (v *Validator) validateNotifiers(config *apiV2.ReportConfiguration) error { func (v *Validator) validateEmailConfig(emailConfig *apiV2.EmailNotifierConfiguration) error { if emailConfig.GetNotifierId() == "" { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify a valid email notifier") + return errox.InvalidArgs.New("report configuration must specify a valid email notifier") } if len(emailConfig.GetMailingLists()) == 0 { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify at least one email recipient to send the report to") + return errox.InvalidArgs.New("report configuration must specify at least one email recipient to send the report to") } subjectMaxLen := env.ReportCustomEmailSubjectMaxLen.IntegerSetting() if len(emailConfig.GetCustomSubject()) > subjectMaxLen { - return errors.Wrapf(errox.InvalidArgs, "Custom email subject must be fewer than %d characters", subjectMaxLen) + return errox.InvalidArgs.Newf("custom email subject must be fewer than %d characters", subjectMaxLen) } bodyMaxLen := env.ReportCustomEmailBodyMaxLen.IntegerSetting() if len(emailConfig.GetCustomBody()) > bodyMaxLen { - return errors.Wrapf(errox.InvalidArgs, "Custom email body must be fewer than than %d characters", bodyMaxLen) + return errox.InvalidArgs.Newf("custom email body must be fewer than %d characters", bodyMaxLen) } errorList := errorhelpers.NewErrorList("Invalid email addresses in mailing list: ") for _, addr := range emailConfig.GetMailingLists() { if _, err := mail.ParseAddress(addr); err != nil { - errorList.AddError(errors.Wrapf(errox.InvalidArgs, "Invalid email recipient address: %s", addr)) + errorList.AddError(errox.InvalidArgs.Newf("invalid email recipient address: %s", addr)) } } if !errorList.Empty() { @@ -148,26 +165,72 @@ func (v *Validator) validateEmailConfig(emailConfig *apiV2.EmailNotifierConfigur } // Use allAccessCtx since report creator/updater might not have permissions for integrationSAC - exists, err := v.notifierDatastore.Exists(allAccessCtx, emailConfig.GetNotifierId()) + notifier, exists, err := v.notifierDatastore.GetScrubbedNotifier(allAccessCtx, emailConfig.GetNotifierId()) if err != nil { return errors.Errorf("Error looking up attached notifier, Notifier ID: %s, Error: %s", emailConfig.GetNotifierId(), err) } if !exists { return errors.Wrapf(errox.NotFound, "Notifier with ID %s not found.", emailConfig.GetNotifierId()) } + if notifier.GetType() != notifiers.EmailType && notifier.GetType() != notifiers.ACSCSEmailType { + return errox.InvalidArgs.New("report configuration requires an email notifier") + } return nil } func (v *Validator) validateResourceScope(config *apiV2.ReportConfiguration) error { - if config.GetResourceScope() == nil || config.GetResourceScope().GetCollectionScope() == nil { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify a valid resource scope") + scope := config.GetResourceScope() + if scope == nil { + return errox.InvalidArgs.New("report configuration must specify a valid resource scope") } - collectionID := config.GetResourceScope().GetCollectionScope().GetCollectionId() - if collectionID == "" { - return errors.Wrap(errox.InvalidArgs, "Report configuration must specify a valid collection ID") + // Nodes are cluster-level resources without namespace hierarchy + if config.GetType() == apiV2.ReportConfiguration_NODE_VULNERABILITY { + return v.validateNodeResourceScope(scope) } + switch ref := scope.GetScopeReference().(type) { + case *apiV2.ResourceScope_CollectionScope: + return v.validateCollectionScope(ref.CollectionScope) + case *apiV2.ResourceScope_EntityScope: + if !features.VulnerabilityReportsEnhancedFiltering.Enabled() { + return errox.InvalidArgs.New("report configuration must specify a valid collection as resource scope") + } + return validateEntityScope(ref.EntityScope) + default: + return errox.InvalidArgs.New("report configuration must specify a valid resource scope") + } +} + +func (v *Validator) validateNodeResourceScope(scope *apiV2.ResourceScope) error { + entityScope, ok := scope.GetScopeReference().(*apiV2.ResourceScope_EntityScope) + if !ok { + return errox.InvalidArgs.New("node vulnerability reports must use entity scope (cluster-based scoping only)") + } + + if entityScope.EntityScope == nil { + return errox.InvalidArgs.New("entity scope cannot be nil") + } + + if err := validateEntityScope(entityScope.EntityScope); err != nil { + return err + } + + // Nodes exist at cluster scope only - unlike deployments, they have no namespace association + for _, rule := range entityScope.EntityScope.GetRules() { + if rule.GetEntity() != apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER { + return errox.InvalidArgs.Newf("node vulnerability reports only support cluster-level scoping, got: %s", rule.GetEntity()) + } + } + + return nil +} + +func (v *Validator) validateCollectionScope(collectionRef *apiV2.CollectionReference) error { + if collectionRef == nil || collectionRef.GetCollectionId() == "" { + return errox.InvalidArgs.New("report configuration must specify a valid collection ID") + } + collectionID := collectionRef.GetCollectionId() // Use allAccessCtx since report creator/updater might not have permissions for workflowAdministrationSAC exists, err := v.collectionDatastore.Exists(allAccessCtx, collectionID) if err != nil { @@ -176,28 +239,166 @@ func (v *Validator) validateResourceScope(config *apiV2.ReportConfiguration) err if !exists { return errors.Wrapf(errox.NotFound, "Collection %s not found.", collectionID) } + return nil +} +// validateEntityScope validates the provided EntityScope and its rules. +// It returns an error if: +// 1. the scope is nil; +// 2. any rule has an unset entity or field; +// 3. a rule uses the unsupported (cluster, annotation) combination; +// 4. a duplicate (entity, field) pair appears +// 5. a rule has no values, or a label rule contains values that are not in `key=value` format. +func validateEntityScope(es *apiV2.EntityScope) error { + if es == nil { + return errox.InvalidArgs.New("report configuration must specify a valid resource scope: either a collection scope with a valid collection ID or a non-nil entity scope") + } + type entityFieldKey struct { + entity apiV2.ScopeEntity + field apiV2.ScopeField + } + seen := set.NewSet[entityFieldKey]() + for _, rule := range es.GetRules() { + switch rule.GetEntity() { + case apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, apiV2.ScopeEntity_SCOPE_ENTITY_NAMESPACE, apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT: + default: + return errox.InvalidArgs.Newf("unexpected entity scope rule: %s", rule.GetEntity()) + } + if rule.GetField() == apiV2.ScopeField_FIELD_UNSET { + return errox.InvalidArgs.Newf("unexpected entity in scope rule for %s with an unset field", rule.GetEntity()) + } + switch rule.GetField() { + case apiV2.ScopeField_FIELD_NAME, apiV2.ScopeField_FIELD_LABEL, apiV2.ScopeField_FIELD_ANNOTATION: + default: + return errox.InvalidArgs.Newf("unsupported field %s for entity scope", rule.GetField()) + } + // Cluster annotation is not indexed and therefore unsupported. + if rule.GetEntity() == apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER && rule.GetField() == apiV2.ScopeField_FIELD_ANNOTATION { + return errox.InvalidArgs.New("annotation field is not supported for cluster entity scope rules") + } + key := entityFieldKey{entity: rule.GetEntity(), field: rule.GetField()} + if !seen.Add(key) { + return errox.InvalidArgs.Newf( + "one rule per (entity, field) pair in entity scope rules is expected; duplicate (entity, field) pair found: entity=%v field=%v", rule.GetEntity(), rule.GetField()) + } + if len(rule.GetValues()) == 0 { + return errox.InvalidArgs.Newf( + "provide at least one matching value for entity=%v field=%v rule", rule.GetEntity(), rule.GetField()) + } + isMapField := rule.GetField() == apiV2.ScopeField_FIELD_LABEL || rule.GetField() == apiV2.ScopeField_FIELD_ANNOTATION + for _, rv := range rule.GetValues() { + valOfValue := rv.GetValue() + if valOfValue == "" { + return errox.InvalidArgs.New("entity scope rule values must not be empty") + } + if rv.GetMatchType() != apiV2.MatchType_EXACT && rv.GetMatchType() != apiV2.MatchType_REGEX { + return errox.InvalidArgs.Newf("unsupported match type %s", rv.GetMatchType()) + } + if isMapField { + mapKey, mapValue, found := strings.Cut(valOfValue, "=") + if !found { + return errox.InvalidArgs.Newf("%v values must be in 'key=value' format", rule.GetField()) + } + // Check the key for a Kubernetes qualified name. + if rv.GetMatchType() == apiV2.MatchType_EXACT { + if errs := k8sValidation.IsLabelKey(mapKey); len(errs) > 0 { + return errox.InvalidArgs.Newf("invalid %v key %q: %s", rule.GetField(), mapKey, strings.Join(errs, "; ")) + } + } else if _, err := regexp.Compile(mapKey); err != nil { + return errox.InvalidArgs.CausedByf("invalid regex key %q: %v", mapKey, err) + } + valOfValue = mapValue + } + if rv.GetMatchType() == apiV2.MatchType_REGEX { + if _, err := regexp.Compile(valOfValue); err != nil { + return errox.InvalidArgs.CausedByf("invalid regex %q: %v", valOfValue, err) + } + } + } + } return nil } func (v *Validator) validateReportFilters(config *apiV2.ReportConfiguration) error { - filters := config.GetVulnReportFilters() + switch config.GetType() { + case apiV2.ReportConfiguration_VULNERABILITY: + return v.validateImageFilters(config.GetVulnReportFilters()) + case apiV2.ReportConfiguration_NODE_VULNERABILITY: + return v.validateNodeFilters(config.GetNodeVulnReportFilters()) + default: + return errox.InvalidArgs.New("unsupported report type") + } +} + +func (v *Validator) validateImageFilters(filters *apiV2.VulnerabilityReportFilters) error { if filters == nil { - return errors.Wrap(errox.InvalidArgs, "Report configuration must include Vulnerability report filters") + return errox.InvalidArgs.New("report configuration must include vulnerability report filters") } if len(filters.GetImageTypes()) == 0 { - return errors.Wrap(errox.InvalidArgs, "Vulnerability report filters should specify which image types to scan for CVEs. "+ - "The valid options are 'DEPLOYED' and 'WATCHED'.") + return errox.InvalidArgs.New("vulnerability report filters should specify which image types to scan for CVEs; " + + "the valid options are 'DEPLOYED' and 'WATCHED'") + } + for _, imageType := range filters.GetImageTypes() { + if _, ok := apiV2.VulnerabilityReportFilters_ImageType_name[int32(imageType)]; !ok { + return errox.InvalidArgs.Newf("unsupported image type %d", imageType) + } + } + if _, ok := apiV2.VulnerabilityReportFilters_Fixability_name[int32(filters.GetFixability())]; !ok { + return errox.InvalidArgs.Newf("unsupported fixability %d", filters.GetFixability()) + } + for _, severity := range filters.GetSeverities() { + if _, ok := apiV2.VulnerabilityReportFilters_VulnerabilitySeverity_name[int32(severity)]; !ok { + return errox.InvalidArgs.Newf("unsupported severity %d", severity) + } + } + if since, ok := filters.GetCvesSince().(*apiV2.VulnerabilityReportFilters_SinceStartDate); ok { + if since.SinceStartDate == nil || since.SinceStartDate.CheckValid() != nil { + return errox.InvalidArgs.New("invalid CVE start date") + } } if filters.GetCvesSince() == nil { - return errors.Wrap(errox.InvalidArgs, "Vulnerability report filters must specify how far back in time to look for CVEs. "+ - "The valid options are 'sinceLastSentScheduledReport', 'allVuln', and 'startDate'") + return errox.InvalidArgs.New("vulnerability report filters must specify how far back in time to look for CVEs; " + + "the valid options are 'sinceLastSentScheduledReport', 'allVuln', and 'startDate'") + } + if features.VulnerabilityReportsEnhancedFiltering.Enabled() { + if q := filters.GetQuery(); q != "" { + if _, err := search.ParseQuery(q); err != nil { + return errox.InvalidArgs.CausedByf("invalid query in vulnerability report filters: %v", err) + } + } } return nil } +func (v *Validator) validateNodeFilters(filters *apiV2.NodeVulnerabilityReportFilters) error { + if filters == nil { + return errox.InvalidArgs.New("node vulnerability report filters cannot be nil") + } + + // TODO: Add support for since_last_sent_scheduled_report and since_start_date filters + // once FirstNodeOccurrence timestamp is available in the data model + if filters.GetCvesSince() == nil { + return errox.InvalidArgs.New("node vulnerability report filters must specify CVE time filter") + } + + switch filters.GetCvesSince().(type) { + case *apiV2.NodeVulnerabilityReportFilters_AllVuln: + // Valid - this is the supported type + default: + return errox.InvalidArgs.New("only all vulnerabilities filter is currently supported for node reports (since_last_sent and since_start_date require FirstNodeOccurrence timestamp)") + } + + if q := filters.GetQuery(); q != "" { + if _, err := search.ParseQuery(q); err != nil { + return errox.InvalidArgs.CausedByf("invalid query in node vulnerability report filters: %v", err) + } + } + + return nil +} + // ValidateAndGenerateReportRequest validates the report configuration for which report is requested and generates a report request func (v *Validator) ValidateAndGenerateReportRequest( configID string, @@ -205,6 +406,9 @@ func (v *Validator) ValidateAndGenerateReportRequest( requestType storage.ReportStatus_RunMethod, requesterID authn.Identity, ) (*reportGen.ReportRequest, error) { + if notificationMethod != storage.ReportStatus_EMAIL && notificationMethod != storage.ReportStatus_DOWNLOAD { + return nil, errox.InvalidArgs.Newf("unsupported notification method %d", notificationMethod) + } config, found, err := v.reportConfigDatastore.GetReportConfiguration(allAccessCtx, configID) if err != nil { return nil, errors.Wrapf(err, "Error finding report configuration %s", configID) @@ -212,21 +416,29 @@ func (v *Validator) ValidateAndGenerateReportRequest( if !found { return nil, errors.Wrapf(errox.NotFound, "Report configuration id not found %s", configID) } - if !common.IsV2ReportConfig(config) { - return nil, errors.Wrap(errox.InvalidArgs, "report configuration does not belong to reporting version 2.0") + // Verify ResourceScope is non-nil + if !common.HasValidResourceScope(config.GetResourceScope()) { + return nil, errox.InvalidArgs.Newf( + "report configuration '%s' has an empty resource scope (no collection ID or entity scope)", + configID) } if notificationMethod == storage.ReportStatus_EMAIL && len(config.GetNotifiers()) == 0 { - return nil, errors.Wrap(errox.InvalidArgs, - "Email request sent for a report configuration that does not have any email notifiers configured") + return nil, errox.InvalidArgs.New( + "email request sent for a report configuration that does not have any email notifiers configured") } - collection, found, err := v.collectionDatastore.Get(allAccessCtx, config.GetResourceScope().GetCollectionId()) - if err != nil { - return nil, errors.Wrapf(err, "Error finding collection ID '%s'", config.GetResourceScope().GetCollectionId()) - } - if !found { - return nil, errors.Wrapf(errox.NotFound, "Collection ID '%s' not found", config.GetResourceScope().GetCollectionId()) + var collection *storage.ResourceCollection + if collectionID := config.GetResourceScope().GetCollectionId(); collectionID != "" { + var found bool + var err error + collection, found, err = v.collectionDatastore.Get(allAccessCtx, collectionID) + if err != nil { + return nil, errors.Wrapf(err, "Error finding collection ID '%s'", collectionID) + } + if !found { + return nil, errors.Wrapf(errox.NotFound, "Collection ID '%s' not found", collectionID) + } } notifierIDs := make([]string, 0, len(config.GetNotifiers())) @@ -256,20 +468,105 @@ func (v *Validator) ValidateCancelReportRequest(reportID string, requester *stor if !found { return errors.Wrapf(errox.NotFound, "Report snapshot with job ID '%s' does not exist", reportID) } + if snapshot.GetType() == storage.ReportSnapshot_NODE_VULNERABILITY { + return errox.InvalidArgs.Newf("report job '%s' is a node vulnerability report; use the node report service", reportID) + } switch snapshot.GetReportStatus().GetRunState() { - case storage.ReportStatus_DELIVERED, storage.ReportStatus_GENERATED, storage.ReportStatus_FAILURE: + case storage.ReportStatus_WAITING, storage.ReportStatus_PREPARING: + // valid states for cancellation — fall through + default: return errors.Wrapf(errox.InvalidArgs, "Cannot cancel. Report job ID '%s' has already completed execution.", reportID) - case storage.ReportStatus_PREPARING: - return errors.Wrapf(errox.InvalidArgs, "Cannot cancel. Report job ID '%s' is currently being prepared.", reportID) } - if requester.GetId() != snapshot.GetRequester().GetId() { return errors.Wrap(errox.NotAuthorized, "Report job cannot be cancelled by a user who did not request the report.") } return nil } +// PersistReportSnapshot validates that the user does not already have a pending +// report for the same config, sets the snapshot to WAITING, and persists it to +// the database. Returns the report ID. +func (v *Validator) PersistReportSnapshot(ctx context.Context, snapshot *storage.ReportSnapshot) (string, error) { + reportType := snapshot.GetType() + requestType := snapshot.GetReportStatus().GetReportRequestType() + + if requestType == storage.ReportStatus_ON_DEMAND { + hasPending, err := v.doesUserHavePendingReport(snapshot.GetReportConfigurationId(), snapshot.GetRequester().GetId(), reportType) + if err != nil { + return "", err + } + if hasPending { + return "", errors.Wrapf(errox.AlreadyExists, "User already has a report running for config ID '%s'", + snapshot.GetReportConfigurationId()) + } + } + + if requestType == storage.ReportStatus_VIEW_BASED { + hasPending, err := v.doesUserHaveViewBasedPendingReport(snapshot.GetRequester().GetId(), reportType) + if err != nil { + return "", err + } + if hasPending { + return "", errors.New("User already has a view based report queued") + } + // View-based reports are authorized at the gRPC layer with only view permissions. + // The snapshot datastore requires WorkflowAdministration write. + ctx = sac.WithAllAccess(ctx) + } + + snapshot.ReportStatus.RunState = storage.ReportStatus_WAITING + snapshot.ReportStatus.QueuedAt = protocompat.TimestampNow() + reportID, err := v.snapshotDatastore.AddReportSnapshot(ctx, snapshot) + if err != nil { + return "", err + } + return reportID, nil +} + +func (v *Validator) doesUserHavePendingReport(configID string, userID string, reportType storage.ReportSnapshot_ReportType) (bool, error) { + query := search.NewQueryBuilder(). + AddExactMatches(search.ReportConfigID, configID). + AddExactMatches(search.ReportState, storage.ReportStatus_WAITING.String(), storage.ReportStatus_PREPARING.String()). + AddExactMatches(search.ReportRequestType, storage.ReportStatus_ON_DEMAND.String()). + AddExactMatches(search.ReportType, reportType.String()). + ProtoQuery() + snapshots, err := v.snapshotDatastore.SearchReportSnapshots(allAccessCtx, query) + if err != nil { + return false, err + } + for _, snap := range snapshots { + if snap.GetRequester().GetId() == userID { + return true, nil + } + } + return false, nil +} + +func (v *Validator) doesUserHaveViewBasedPendingReport(userID string, reportType storage.ReportSnapshot_ReportType) (bool, error) { + query := search.NewQueryBuilder(). + AddExactMatches(search.ReportState, storage.ReportStatus_WAITING.String(), storage.ReportStatus_PREPARING.String()). + AddExactMatches(search.ReportRequestType, storage.ReportStatus_VIEW_BASED.String()). + AddExactMatches(search.UserID, userID). + AddExactMatches(search.ReportType, reportType.String()). + ProtoQuery() + count, err := v.snapshotDatastore.Count(allAccessCtx, query) + if err != nil { + return false, err + } + return count > 0, nil +} + +func collectionSnapshot(collection *storage.ResourceCollection) *storage.CollectionSnapshot { + if collection == nil { + return nil + } + return &storage.CollectionSnapshot{ + Id: collection.GetId(), + Name: collection.GetName(), + } +} + func generateReportSnapshot( config *storage.ReportConfiguration, collection *storage.ResourceCollection, @@ -278,16 +575,33 @@ func generateReportSnapshot( requestType storage.ReportStatus_RunMethod, requesterID authn.Identity, ) *storage.ReportSnapshot { + var requester *storage.SlimUser + switch requestType { + case storage.ReportStatus_ON_DEMAND: + requester = &storage.SlimUser{ + Id: requesterID.UID(), + Name: stringutils.FirstNonEmpty(requesterID.FullName(), requesterID.FriendlyName()), + } + case storage.ReportStatus_SCHEDULED: + requester = config.GetCreator() + } + + var snapshotType storage.ReportSnapshot_ReportType + switch config.GetType() { + case storage.ReportConfiguration_VULNERABILITY: + snapshotType = storage.ReportSnapshot_VULNERABILITY + case storage.ReportConfiguration_NODE_VULNERABILITY: + snapshotType = storage.ReportSnapshot_NODE_VULNERABILITY + } + snapshot := &storage.ReportSnapshot{ ReportConfigurationId: config.GetId(), Name: config.GetName(), Description: config.GetDescription(), - Type: storage.ReportSnapshot_VULNERABILITY, - Collection: &storage.CollectionSnapshot{ - Id: config.GetResourceScope().GetCollectionId(), - Name: collection.GetName(), - }, - Schedule: config.GetSchedule(), + Type: snapshotType, + Collection: collectionSnapshot(collection), + Schedule: config.GetSchedule(), + Requester: requester, ReportStatus: &storage.ReportStatus{ RunState: storage.ReportStatus_WAITING, ReportRequestType: requestType, @@ -295,21 +609,33 @@ func generateReportSnapshot( }, } - reportFilters := config.GetVulnReportFilters().CloneVT() - var requester *storage.SlimUser - switch requestType { - case storage.ReportStatus_ON_DEMAND: - reportFilters.AccessScopeRules = common.ExtractAccessScopeRules(requesterID) - requester = &storage.SlimUser{ - Id: requesterID.UID(), - Name: stringutils.FirstNonEmpty(requesterID.FullName(), requesterID.FriendlyName()), - } - case storage.ReportStatus_SCHEDULED: - requester = config.GetCreator() + if features.VulnerabilityReportsEnhancedFiltering.Enabled() || config.GetType() == storage.ReportConfiguration_NODE_VULNERABILITY { + snapshot.ResourceScope = config.GetResourceScope().CloneVT() } - snapshot.Requester = requester - snapshot.Filter = &storage.ReportSnapshot_VulnReportFilters{ - VulnReportFilters: reportFilters, + + switch config.GetType() { + case storage.ReportConfiguration_VULNERABILITY: + reportFilters := config.GetVulnReportFilters() + if reportFilters != nil { + reportFilters = reportFilters.CloneVT() + if requestType == storage.ReportStatus_ON_DEMAND { + reportFilters.AccessScopeRules = common.ExtractAccessScopeRules(requesterID) + } + snapshot.Filter = &storage.ReportSnapshot_VulnReportFilters{ + VulnReportFilters: reportFilters, + } + } + case storage.ReportConfiguration_NODE_VULNERABILITY: + nodeFilters := config.GetNodeVulnReportFilters() + if nodeFilters != nil { + nodeFilters = nodeFilters.CloneVT() + if requestType == storage.ReportStatus_ON_DEMAND { + nodeFilters.AccessScopeRules = common.ExtractAccessScopeRules(requesterID) + } + snapshot.Filter = &storage.ReportSnapshot_NodeVulnReportFilters{ + NodeVulnReportFilters: nodeFilters, + } + } } notifierSnaps := make([]*storage.NotifierSnapshot, 0, len(config.GetNotifiers())) @@ -336,7 +662,7 @@ func generateViewBasedRequestName(user *storage.SlimUser) string { now := time.Now() date := now.Format("Jan02") year := now.Format("2006") - shortUUID := strings.Split(uuid.NewV4().String(), "-")[0] + shortUUID, _, _ := strings.Cut(uuid.NewV4().String(), "-") return fmt.Sprintf("%s-%s-%s-%s", shortName, strings.ToLower(date), year, shortUUID) } @@ -347,7 +673,7 @@ func getShortName(user *storage.SlimUser) string { name := strings.ToUpper(user.GetName()) parts := strings.Split(name, " ") - for i := 0; i < len(parts); i++ { + for i := range parts { parts[i] = strings.TrimSpace(parts[i]) } @@ -365,44 +691,97 @@ func (v *Validator) ValidateAndGenerateViewBasedReportRequest( requesterID authn.Identity, ) (*reportGen.ReportRequest, error) { if req == nil { - return nil, errors.Wrap(errox.InvalidArgs, "Empty request") + return nil, errox.InvalidArgs.New("empty request") } - // Currently only vulnerability view-based reports are supported. - if req.GetType() != apiV2.ReportRequestViewBased_VULNERABILITY { - return nil, errors.Wrap(errox.InvalidArgs, "unsupported report type") - } - - // Validate filters. - vbFilters := req.GetViewBasedVulnReportFilters() - if vbFilters == nil { - return nil, errors.Wrap(errox.InvalidArgs, "view-based vulnerability report filters must be provided") - } - - // Convert API filters to storage filters. - storageFilters := &storage.ViewBasedVulnerabilityReportFilters{ - Query: vbFilters.GetQuery(), - AccessScopeRules: common.ExtractAccessScopeRules(requesterID), - } requester := &storage.SlimUser{ Id: requesterID.UID(), Name: stringutils.FirstNonEmpty(requesterID.FullName(), requesterID.FriendlyName()), } - // Build report snapshot. - snapshot := &storage.ReportSnapshot{ - Name: generateViewBasedRequestName(requester), - Type: storage.ReportSnapshot_VULNERABILITY, - AreaOfConcern: req.GetAreaOfConcern(), - ReportStatus: &storage.ReportStatus{ - RunState: storage.ReportStatus_WAITING, - ReportRequestType: storage.ReportStatus_VIEW_BASED, - ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, - }, - Filter: &storage.ReportSnapshot_ViewBasedVulnReportFilters{ - ViewBasedVulnReportFilters: storageFilters, - }, - Requester: requester, + var snapshot *storage.ReportSnapshot + + switch req.GetType() { + case apiV2.ReportRequestViewBased_VULNERABILITY: + // Validate filters. + vbFilters := req.GetViewBasedVulnReportFilters() + if vbFilters == nil { + return nil, errox.InvalidArgs.New("view-based vulnerability report filters must be provided") + } + + // Convert API filters to storage filters. + storageFilters := &storage.ViewBasedVulnerabilityReportFilters{ + Query: vbFilters.GetQuery(), + AccessScopeRules: common.ExtractAccessScopeRules(requesterID), + } + + // Build report snapshot. + snapshot = &storage.ReportSnapshot{ + Name: generateViewBasedRequestName(requester), + Type: storage.ReportSnapshot_VULNERABILITY, + AreaOfConcern: req.GetAreaOfConcern(), + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_WAITING, + ReportRequestType: storage.ReportStatus_VIEW_BASED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + Filter: &storage.ReportSnapshot_ViewBasedVulnReportFilters{ + ViewBasedVulnReportFilters: storageFilters, + }, + Requester: requester, + } + + case apiV2.ReportRequestViewBased_NODE_VULNERABILITY: + // Validate filters. + nodeFilters := req.GetNodeVulnReportFilters() + if nodeFilters == nil { + return nil, errox.InvalidArgs.New("node vulnerability report filters must be provided") + } + + // Node vulnerability view based reports should default to allVuln in CVEs since time filter if not provided. + if nodeFilters.GetCvesSince() == nil { + nodeFilters.CvesSince = &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + } + } + + if err := v.validateNodeFilters(nodeFilters); err != nil { + return nil, err + } + + // Convert API filters to storage filters. + storageFilters := &storage.NodeVulnerabilityReportFilters{ + Query: nodeFilters.GetQuery(), + AccessScopeRules: common.ExtractAccessScopeRules(requesterID), + } + + switch nodeFilters.GetCvesSince().(type) { + case *apiV2.NodeVulnerabilityReportFilters_AllVuln: + storageFilters.CvesSince = &storage.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: nodeFilters.GetAllVuln(), + } + default: + return nil, errox.InvalidArgs.New("unsupported CVE time filter for node vulnerability reports") + } + + // Build report snapshot. + snapshot = &storage.ReportSnapshot{ + Name: generateViewBasedRequestName(requester), + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + AreaOfConcern: req.GetAreaOfConcern(), + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_WAITING, + ReportRequestType: storage.ReportStatus_VIEW_BASED, + ReportNotificationMethod: storage.ReportStatus_DOWNLOAD, + }, + Filter: &storage.ReportSnapshot_NodeVulnReportFilters{ + NodeVulnReportFilters: storageFilters, + }, + Requester: requester, + } + + default: + return nil, errox.InvalidArgs.New("unsupported report type") } return &reportGen.ReportRequest{ diff --git a/central/reports/validation/validate_comprehensive_test.go b/central/reports/validation/validate_comprehensive_test.go new file mode 100644 index 0000000000000..809b8c9a66be6 --- /dev/null +++ b/central/reports/validation/validate_comprehensive_test.go @@ -0,0 +1,861 @@ +package validation + +import ( + "context" + "testing" + + notifierDSMocks "github.com/stackrox/rox/central/notifier/datastore/mocks" + reportConfigDSMocks "github.com/stackrox/rox/central/reports/config/datastore/mocks" + snapshotDSMocks "github.com/stackrox/rox/central/reports/snapshot/datastore/mocks" + collectionDSMocks "github.com/stackrox/rox/central/resourcecollection/datastore/mocks" + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + authnMocks "github.com/stackrox/rox/pkg/grpc/authn/mocks" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestValidateReportConfiguration(t *testing.T) { + t.Setenv(features.VulnerabilityReportsEnhancedFiltering.EnvVar(), "true") + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + notifierDS := notifierDSMocks.NewMockDataStore(ctrl) + collectionDS := collectionDSMocks.NewMockDataStore(ctrl) + reportConfigDS := reportConfigDSMocks.NewMockDataStore(ctrl) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + + validator := New(reportConfigDS, snapshotDS, collectionDS, notifierDS) + + tests := map[string]struct { + config *apiV2.ReportConfiguration + setupMocks func() + expectError bool + errContains string + }{ + "empty name fails": { + config: &apiV2.ReportConfiguration{ + Name: "", + }, + setupMocks: func() {}, + expectError: true, + errContains: "name is empty", + }, + "scheduled report without notifiers fails": { + config: &apiV2.ReportConfiguration{ + Name: "Test Report", + Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_DAILY, + Hour: 9, + Minute: 0, + }, + }, + setupMocks: func() {}, + expectError: true, + errContains: "must specify a notifier", + }, + "missing resource scope fails": { + config: &apiV2.ReportConfiguration{ + Name: "Test Report", + }, + setupMocks: func() {}, + expectError: true, + errContains: "must specify a valid resource scope", + }, + "valid node report with entity scope": { + config: &apiV2.ReportConfiguration{ + Name: "Node Report", + Type: apiV2.ReportConfiguration_NODE_VULNERABILITY, + ResourceScope: &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{ + EntityScope: &apiV2.EntityScope{ + Rules: []*apiV2.EntityScopeRule{ + { + Entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + Field: apiV2.ScopeField_FIELD_NAME, + Values: []*apiV2.RuleValue{ + {Value: "prod", MatchType: apiV2.MatchType_EXACT}, + }, + }, + }, + }, + }, + }, + Filter: &apiV2.ReportConfiguration_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + }, + setupMocks: func() {}, + expectError: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + tc.setupMocks() + err := validator.ValidateReportConfiguration(tc.config) + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateNotifiers(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + notifierDS := notifierDSMocks.NewMockDataStore(ctrl) + collectionDS := collectionDSMocks.NewMockDataStore(ctrl) + reportConfigDS := reportConfigDSMocks.NewMockDataStore(ctrl) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + + validator := New(reportConfigDS, snapshotDS, collectionDS, notifierDS) + + tests := map[string]struct { + config *apiV2.ReportConfiguration + setupMocks func() + expectError bool + errContains string + }{ + "no notifiers without schedule is valid": { + config: &apiV2.ReportConfiguration{ + Notifiers: nil, + Schedule: nil, + }, + setupMocks: func() {}, + expectError: false, + }, + "notifier without email config fails": { + config: &apiV2.ReportConfiguration{ + Notifiers: []*apiV2.NotifierConfiguration{ + {}, + }, + }, + setupMocks: func() {}, + expectError: true, + errContains: "must specify an email notifier configuration", + }, + "valid notifier": { + config: &apiV2.ReportConfiguration{ + Notifiers: []*apiV2.NotifierConfiguration{ + { + NotifierConfig: &apiV2.NotifierConfiguration_EmailConfig{ + EmailConfig: &apiV2.EmailNotifierConfiguration{ + NotifierId: "notifier-1", + MailingLists: []string{"user@example.com"}, + }, + }, + }, + }, + }, + setupMocks: func() { + notifierDS.EXPECT().GetScrubbedNotifier(gomock.Any(), "notifier-1").Return(&storage.Notifier{Type: "email"}, true, nil) + }, + expectError: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + tc.setupMocks() + err := validator.validateNotifiers(tc.config) + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateEmailConfig(t *testing.T) { + tests := map[string]struct { + emailConfig *apiV2.EmailNotifierConfiguration + setupMocks func(*notifierDSMocks.MockDataStore) + expectError bool + errContains string + }{ + "empty notifier ID fails": { + emailConfig: &apiV2.EmailNotifierConfiguration{ + NotifierId: "", + MailingLists: []string{"user@example.com"}, + }, + setupMocks: func(notifierDS *notifierDSMocks.MockDataStore) {}, + expectError: true, + errContains: "must specify a valid email notifier", + }, + "empty mailing list fails": { + emailConfig: &apiV2.EmailNotifierConfiguration{ + NotifierId: "notifier-1", + MailingLists: []string{}, + }, + setupMocks: func(notifierDS *notifierDSMocks.MockDataStore) {}, + expectError: true, + errContains: "must specify at least one email recipient", + }, + "invalid email address fails": { + emailConfig: &apiV2.EmailNotifierConfiguration{ + NotifierId: "notifier-1", + MailingLists: []string{"invalid-email"}, + }, + setupMocks: func(notifierDS *notifierDSMocks.MockDataStore) {}, + expectError: true, + errContains: "invalid email recipient address", + }, + "notifier not found fails": { + emailConfig: &apiV2.EmailNotifierConfiguration{ + NotifierId: "missing-notifier", + MailingLists: []string{"user@example.com"}, + }, + setupMocks: func(notifierDS *notifierDSMocks.MockDataStore) { + notifierDS.EXPECT().GetScrubbedNotifier(gomock.Any(), "missing-notifier").Return(nil, false, nil) + }, + expectError: true, + errContains: "not found", + }, + "valid email config": { + emailConfig: &apiV2.EmailNotifierConfiguration{ + NotifierId: "notifier-1", + MailingLists: []string{"user@example.com", "admin@example.com"}, + CustomSubject: "Test Report", + CustomBody: "Report body", + }, + setupMocks: func(notifierDS *notifierDSMocks.MockDataStore) { + notifierDS.EXPECT().GetScrubbedNotifier(gomock.Any(), "notifier-1").Return(&storage.Notifier{Type: "email"}, true, nil) + }, + expectError: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + // Create fresh mocks for each subtest to avoid mock conflicts + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + notifierDS := notifierDSMocks.NewMockDataStore(ctrl) + collectionDS := collectionDSMocks.NewMockDataStore(ctrl) + reportConfigDS := reportConfigDSMocks.NewMockDataStore(ctrl) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + + validator := New(reportConfigDS, snapshotDS, collectionDS, notifierDS) + + tc.setupMocks(notifierDS) + err := validator.validateEmailConfig(tc.emailConfig) + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateNodeResourceScope(t *testing.T) { + t.Setenv(features.VulnerabilityReportsEnhancedFiltering.EnvVar(), "true") + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + notifierDS := notifierDSMocks.NewMockDataStore(ctrl) + collectionDS := collectionDSMocks.NewMockDataStore(ctrl) + reportConfigDS := reportConfigDSMocks.NewMockDataStore(ctrl) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + + validator := New(reportConfigDS, snapshotDS, collectionDS, notifierDS) + + tests := map[string]struct { + scope *apiV2.ResourceScope + expectError bool + errContains string + }{ + "collection scope not allowed for nodes": { + scope: &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_CollectionScope{ + CollectionScope: &apiV2.CollectionReference{ + CollectionId: "collection-1", + }, + }, + }, + expectError: true, + errContains: "must use entity scope", + }, + "nil entity scope fails": { + scope: &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{ + EntityScope: nil, + }, + }, + expectError: true, + errContains: "cannot be nil", + }, + "namespace scope not allowed for nodes": { + scope: &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{ + EntityScope: &apiV2.EntityScope{ + Rules: []*apiV2.EntityScopeRule{ + { + Entity: apiV2.ScopeEntity_SCOPE_ENTITY_NAMESPACE, + Field: apiV2.ScopeField_FIELD_NAME, + Values: []*apiV2.RuleValue{ + {Value: "default", MatchType: apiV2.MatchType_EXACT}, + }, + }, + }, + }, + }, + }, + expectError: true, + errContains: "only support cluster-level scoping", + }, + "valid cluster scope": { + scope: &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{ + EntityScope: &apiV2.EntityScope{ + Rules: []*apiV2.EntityScopeRule{ + { + Entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + Field: apiV2.ScopeField_FIELD_NAME, + Values: []*apiV2.RuleValue{ + {Value: "prod", MatchType: apiV2.MatchType_EXACT}, + }, + }, + }, + }, + }, + }, + expectError: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + err := validator.validateNodeResourceScope(tc.scope) + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateCollectionScope(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + notifierDS := notifierDSMocks.NewMockDataStore(ctrl) + collectionDS := collectionDSMocks.NewMockDataStore(ctrl) + reportConfigDS := reportConfigDSMocks.NewMockDataStore(ctrl) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + + validator := New(reportConfigDS, snapshotDS, collectionDS, notifierDS) + + tests := map[string]struct { + collectionRef *apiV2.CollectionReference + setupMocks func() + expectError bool + errContains string + }{ + "nil collection reference fails": { + collectionRef: nil, + setupMocks: func() {}, + expectError: true, + errContains: "must specify a valid collection ID", + }, + "empty collection ID fails": { + collectionRef: &apiV2.CollectionReference{ + CollectionId: "", + }, + setupMocks: func() {}, + expectError: true, + errContains: "must specify a valid collection ID", + }, + "collection not found fails": { + collectionRef: &apiV2.CollectionReference{ + CollectionId: "missing-collection", + }, + setupMocks: func() { + collectionDS.EXPECT().Exists(gomock.Any(), "missing-collection").Return(false, nil) + }, + expectError: true, + errContains: "not found", + }, + "valid collection": { + collectionRef: &apiV2.CollectionReference{ + CollectionId: "collection-1", + }, + setupMocks: func() { + collectionDS.EXPECT().Exists(gomock.Any(), "collection-1").Return(true, nil) + }, + expectError: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + tc.setupMocks() + err := validator.validateCollectionScope(tc.collectionRef) + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateNodeFilters(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + notifierDS := notifierDSMocks.NewMockDataStore(ctrl) + collectionDS := collectionDSMocks.NewMockDataStore(ctrl) + reportConfigDS := reportConfigDSMocks.NewMockDataStore(ctrl) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + + validator := New(reportConfigDS, snapshotDS, collectionDS, notifierDS) + + tests := map[string]struct { + filters *apiV2.NodeVulnerabilityReportFilters + expectError bool + errContains string + }{ + "nil filters fail": { + filters: nil, + expectError: true, + errContains: "cannot be nil", + }, + "missing CVE time filter fails": { + filters: &apiV2.NodeVulnerabilityReportFilters{ + CvesSince: nil, + }, + expectError: true, + errContains: "must specify CVE time filter", + }, + "valid all vulnerabilities filter": { + filters: &apiV2.NodeVulnerabilityReportFilters{ + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + expectError: false, + }, + "valid filter with query": { + filters: &apiV2.NodeVulnerabilityReportFilters{ + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + Query: "Cluster:prod", + }, + expectError: false, + }, + "invalid query fails": { + filters: &apiV2.NodeVulnerabilityReportFilters{ + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + Query: "invalid query syntax [", + }, + expectError: true, + errContains: "invalid query", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + err := validator.validateNodeFilters(tc.filters) + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateResourceScope(t *testing.T) { + t.Setenv(features.VulnerabilityReportsEnhancedFiltering.EnvVar(), "true") + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + notifierDS := notifierDSMocks.NewMockDataStore(ctrl) + collectionDS := collectionDSMocks.NewMockDataStore(ctrl) + reportConfigDS := reportConfigDSMocks.NewMockDataStore(ctrl) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + + validator := New(reportConfigDS, snapshotDS, collectionDS, notifierDS) + + tests := map[string]struct { + config *apiV2.ReportConfiguration + setupMocks func() + expectError bool + errContains string + }{ + "nil scope fails": { + config: &apiV2.ReportConfiguration{ + Type: apiV2.ReportConfiguration_VULNERABILITY, + ResourceScope: nil, + }, + setupMocks: func() {}, + expectError: true, + errContains: "must specify a valid resource scope", + }, + "valid collection scope for vulnerability report": { + config: &apiV2.ReportConfiguration{ + Type: apiV2.ReportConfiguration_VULNERABILITY, + ResourceScope: &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_CollectionScope{ + CollectionScope: &apiV2.CollectionReference{ + CollectionId: "collection-1", + }, + }, + }, + }, + setupMocks: func() { + collectionDS.EXPECT().Exists(gomock.Any(), "collection-1").Return(true, nil) + }, + expectError: false, + }, + "valid entity scope for vulnerability report": { + config: &apiV2.ReportConfiguration{ + Type: apiV2.ReportConfiguration_VULNERABILITY, + ResourceScope: &apiV2.ResourceScope{ + ScopeReference: &apiV2.ResourceScope_EntityScope{ + EntityScope: &apiV2.EntityScope{ + Rules: []*apiV2.EntityScopeRule{ + { + Entity: apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, + Field: apiV2.ScopeField_FIELD_NAME, + Values: []*apiV2.RuleValue{ + {Value: "prod", MatchType: apiV2.MatchType_EXACT}, + }, + }, + }, + }, + }, + }, + }, + setupMocks: func() {}, + expectError: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + tc.setupMocks() + err := validator.validateResourceScope(tc.config) + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateCancelReportRequest(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + notifierDS := notifierDSMocks.NewMockDataStore(ctrl) + collectionDS := collectionDSMocks.NewMockDataStore(ctrl) + reportConfigDS := reportConfigDSMocks.NewMockDataStore(ctrl) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + + validator := New(reportConfigDS, snapshotDS, collectionDS, notifierDS) + + tests := map[string]struct { + reportID string + setupMocks func() + expectError bool + errContains string + }{ + "snapshot not found": { + reportID: "missing-report", + setupMocks: func() { + snapshotDS.EXPECT().Get(gomock.Any(), "missing-report").Return(nil, false, nil) + }, + expectError: true, + errContains: "not found", + }, + "node vulnerability snapshot is rejected": { + reportID: "node-job", + setupMocks: func() { + snapshotDS.EXPECT().Get(gomock.Any(), "node-job").Return(&storage.ReportSnapshot{ + ReportId: "node-job", + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + ReportStatus: &storage.ReportStatus{ + RunState: storage.ReportStatus_WAITING, + }, + Requester: &storage.SlimUser{Id: "user-1"}, + }, true, nil) + }, + expectError: true, + errContains: "node report service", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + tc.setupMocks() + requester := &storage.SlimUser{ + Id: "user-1", + Name: "Test User", + } + err := validator.ValidateCancelReportRequest(tc.reportID, requester) + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestPersistReportSnapshot_NodeOnDemandPending(t *testing.T) { + ctrl := gomock.NewController(t) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + validator := New(reportConfigDSMocks.NewMockDataStore(ctrl), snapshotDS, collectionDSMocks.NewMockDataStore(ctrl), notifierDSMocks.NewMockDataStore(ctrl)) + + pending := &storage.ReportSnapshot{ + ReportConfigurationId: "cfg", + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + Requester: &storage.SlimUser{Id: "u1"}, + } + snapshotDS.EXPECT().SearchReportSnapshots(gomock.Any(), gomock.Any()).Return([]*storage.ReportSnapshot{pending}, nil) + + _, err := validator.PersistReportSnapshot(t.Context(), nodeOnDemandSnapshot("cfg", "u1")) + require.Error(t, err) + assert.Contains(t, err.Error(), "already has a report running") +} + +func TestPersistReportSnapshot_NodeViewBasedPending(t *testing.T) { + ctrl := gomock.NewController(t) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + validator := New(reportConfigDSMocks.NewMockDataStore(ctrl), snapshotDS, collectionDSMocks.NewMockDataStore(ctrl), notifierDSMocks.NewMockDataStore(ctrl)) + + snapshotDS.EXPECT().Count(gomock.Any(), gomock.Any()).Return(1, nil) + + _, err := validator.PersistReportSnapshot(t.Context(), nodeViewBasedSnapshot("u1")) + require.Error(t, err) + assert.Contains(t, err.Error(), "view based report queued") +} + +func TestPersistReportSnapshot_NodeViewBasedElevatesSAC(t *testing.T) { + ctrl := gomock.NewController(t) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + validator := New(reportConfigDSMocks.NewMockDataStore(ctrl), snapshotDS, collectionDSMocks.NewMockDataStore(ctrl), notifierDSMocks.NewMockDataStore(ctrl)) + + snapshotDS.EXPECT().Count(gomock.Any(), gomock.Any()).Return(0, nil) + snapshotDS.EXPECT().AddReportSnapshot(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, _ *storage.ReportSnapshot) (string, error) { + ok, err := sac.ForResource(resources.WorkflowAdministration).WriteAllowed(ctx) + require.NoError(t, err) + assert.True(t, ok) + return "report-id", nil + }) + + id, err := validator.PersistReportSnapshot(sac.WithNoAccess(t.Context()), nodeViewBasedSnapshot("u1")) + require.NoError(t, err) + assert.Equal(t, "report-id", id) +} + +func nodeOnDemandSnapshot(configID, userID string) *storage.ReportSnapshot { + return &storage.ReportSnapshot{ + ReportConfigurationId: configID, + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + Requester: &storage.SlimUser{Id: userID}, + Filter: &storage.ReportSnapshot_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "Cluster:c1", + }, + }, + ReportStatus: &storage.ReportStatus{ + ReportRequestType: storage.ReportStatus_ON_DEMAND, + }, + } +} + +func nodeViewBasedSnapshot(userID string) *storage.ReportSnapshot { + return &storage.ReportSnapshot{ + Type: storage.ReportSnapshot_NODE_VULNERABILITY, + Requester: &storage.SlimUser{Id: userID}, + Filter: &storage.ReportSnapshot_NodeVulnReportFilters{ + NodeVulnReportFilters: &storage.NodeVulnerabilityReportFilters{ + Query: "Cluster:c1", + }, + }, + ReportStatus: &storage.ReportStatus{ + ReportRequestType: storage.ReportStatus_VIEW_BASED, + }, + } +} + +func TestValidateAndGenerateViewBasedReportRequest(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + notifierDS := notifierDSMocks.NewMockDataStore(ctrl) + collectionDS := collectionDSMocks.NewMockDataStore(ctrl) + reportConfigDS := reportConfigDSMocks.NewMockDataStore(ctrl) + snapshotDS := snapshotDSMocks.NewMockDataStore(ctrl) + + validator := New(reportConfigDS, snapshotDS, collectionDS, notifierDS) + + identity := authnMocks.NewMockIdentity(ctrl) + identity.EXPECT().UID().Return("user-1").AnyTimes() + identity.EXPECT().FullName().Return("John Doe").AnyTimes() + identity.EXPECT().FriendlyName().Return("jdoe").AnyTimes() + identity.EXPECT().Roles().Return(nil).AnyTimes() + + tests := map[string]struct { + req *apiV2.ReportRequestViewBased + expectError bool + errContains string + // verify runs on the generated snapshot when no error is expected. + verify func(t *testing.T, snapshot *storage.ReportSnapshot) + }{ + "nil request fails": { + req: nil, + expectError: true, + errContains: "empty request", + }, + "unsupported report type fails": { + req: &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_ReportType(-1), + }, + expectError: true, + errContains: "unsupported report type", + }, + "vulnerability report without filters fails": { + req: &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_VULNERABILITY, + }, + expectError: true, + errContains: "view-based vulnerability report filters must be provided", + }, + "vulnerability report with filters succeeds": { + req: &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_VULNERABILITY, + Filter: &apiV2.ReportRequestViewBased_ViewBasedVulnReportFilters{ + ViewBasedVulnReportFilters: &apiV2.ViewBasedVulnerabilityReportFilters{ + Query: "Cluster:prod", + }, + }, + AreaOfConcern: "User Workloads", + }, + verify: func(t *testing.T, snapshot *storage.ReportSnapshot) { + assert.Equal(t, storage.ReportSnapshot_VULNERABILITY, snapshot.GetType()) + assert.Equal(t, "User Workloads", snapshot.GetAreaOfConcern()) + assert.Equal(t, "Cluster:prod", snapshot.GetViewBasedVulnReportFilters().GetQuery()) + }, + }, + "node report without filters fails": { + req: &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_NODE_VULNERABILITY, + }, + expectError: true, + errContains: "node vulnerability report filters must be provided", + }, + "node report without CVE time filter defaults to all vulnerabilities": { + req: &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_NODE_VULNERABILITY, + Filter: &apiV2.ReportRequestViewBased_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + Query: "Cluster:prod", + }, + }, + }, + verify: func(t *testing.T, snapshot *storage.ReportSnapshot) { + filters := snapshot.GetNodeVulnReportFilters() + assert.Equal(t, "Cluster:prod", filters.GetQuery()) + assert.True(t, filters.GetAllVuln()) + }, + }, + "node report with explicit all vulnerabilities filter is preserved": { + req: &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_NODE_VULNERABILITY, + Filter: &apiV2.ReportRequestViewBased_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: true, + }, + }, + }, + }, + verify: func(t *testing.T, snapshot *storage.ReportSnapshot) { + assert.True(t, snapshot.GetNodeVulnReportFilters().GetAllVuln()) + }, + }, + "node report with all vulnerabilities set to false is not overridden": { + req: &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_NODE_VULNERABILITY, + Filter: &apiV2.ReportRequestViewBased_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + CvesSince: &apiV2.NodeVulnerabilityReportFilters_AllVuln{ + AllVuln: false, + }, + }, + }, + }, + verify: func(t *testing.T, snapshot *storage.ReportSnapshot) { + assert.False(t, snapshot.GetNodeVulnReportFilters().GetAllVuln()) + }, + }, + "node report with invalid query fails": { + req: &apiV2.ReportRequestViewBased{ + Type: apiV2.ReportRequestViewBased_NODE_VULNERABILITY, + Filter: &apiV2.ReportRequestViewBased_NodeVulnReportFilters{ + NodeVulnReportFilters: &apiV2.NodeVulnerabilityReportFilters{ + Query: "invalid query syntax [", + }, + }, + }, + expectError: true, + errContains: "invalid query", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + result, err := validator.ValidateAndGenerateViewBasedReportRequest(tc.req, identity) + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + assert.Nil(t, result) + return + } + require.NoError(t, err) + require.NotNil(t, result) + + snapshot := result.ReportSnapshot + require.NotNil(t, snapshot) + assert.Equal(t, storage.ReportStatus_WAITING, snapshot.GetReportStatus().GetRunState()) + assert.Equal(t, storage.ReportStatus_VIEW_BASED, snapshot.GetReportStatus().GetReportRequestType()) + assert.Equal(t, storage.ReportStatus_DOWNLOAD, snapshot.GetReportStatus().GetReportNotificationMethod()) + assert.Equal(t, "user-1", snapshot.GetRequester().GetId()) + assert.Equal(t, "John Doe", snapshot.GetRequester().GetName()) + assert.NotEmpty(t, snapshot.GetName()) + tc.verify(t, snapshot) + }) + } +} diff --git a/central/reports/validation/validate_test.go b/central/reports/validation/validate_test.go new file mode 100644 index 0000000000000..da51f83371d26 --- /dev/null +++ b/central/reports/validation/validate_test.go @@ -0,0 +1,335 @@ +package validation + +import ( + "testing" + + apiV2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/pkg/features" + "github.com/stretchr/testify/assert" +) + +func makeEntityScope(rules ...*apiV2.EntityScopeRule) *apiV2.EntityScope { + return &apiV2.EntityScope{Rules: rules} +} + +func makeRule(entity apiV2.ScopeEntity, field apiV2.ScopeField, values ...*apiV2.RuleValue) *apiV2.EntityScopeRule { + return &apiV2.EntityScopeRule{Entity: entity, Field: field, Values: values} +} + +func exactValue(v string) *apiV2.RuleValue { + return &apiV2.RuleValue{Value: v, MatchType: apiV2.MatchType_EXACT} +} + +func regexValue(v string) *apiV2.RuleValue { + return &apiV2.RuleValue{Value: v, MatchType: apiV2.MatchType_REGEX} +} + +func TestValidateEntityScope(t *testing.T) { + t.Setenv(features.VulnerabilityReportsEnhancedFiltering.EnvVar(), "true") + cases := map[string]struct { + scope *apiV2.EntityScope + expectError bool + errContains string + }{ + "nil entity scope": { + scope: nil, + expectError: true, + errContains: "either a collection scope with a valid collection ID or a non-nil entity scope", + }, + "empty rules": { + scope: makeEntityScope(), + expectError: false, + }, + "valid deployment name rule": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_NAME, exactValue("frontend")), + ), + expectError: false, + }, + "valid namespace label rule": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_NAMESPACE, apiV2.ScopeField_FIELD_LABEL, exactValue("env=prod")), + ), + expectError: false, + }, + "label key with empty value is valid": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_NAMESPACE, apiV2.ScopeField_FIELD_LABEL, exactValue("required=")), + ), + expectError: false, + }, + "valid multiple rules different entities": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, apiV2.ScopeField_FIELD_NAME, exactValue("prod")), + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_NAMESPACE, apiV2.ScopeField_FIELD_NAME, exactValue("payments")), + ), + expectError: false, + }, + "unset entity": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_UNSET, apiV2.ScopeField_FIELD_NAME, exactValue("x")), + ), + expectError: true, + errContains: "unexpected entity scope rule:", + }, + "unset field": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_UNSET, exactValue("x")), + ), + expectError: true, + errContains: "with an unset field", + }, + "duplicate entity+field pair": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_NAME, exactValue("a")), + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_NAME, exactValue("b")), + ), + expectError: true, + errContains: "duplicate", + }, + "no values in rule": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_NAME), + ), + expectError: true, + errContains: "provide at least one matching value", + }, + "invalid regex is rejected": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_NAME, regexValue("[invalid")), + ), + expectError: true, + errContains: "invalid regex", + }, + "valid regex is accepted": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_NAME, regexValue("web-.*")), + ), + expectError: false, + }, + "label with regex match type is valid": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_LABEL, regexValue("env=prod")), + ), + expectError: false, + }, + "label value missing equals sign": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_LABEL, exactValue("noequalssign")), + ), + expectError: true, + errContains: "key=value", + }, + "label with invalid k8s key": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_LABEL, exactValue("invalid key!=value")), + ), + expectError: true, + errContains: "invalid", + }, + "cluster annotation unsupported": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_CLUSTER, apiV2.ScopeField_FIELD_ANNOTATION, exactValue("k=v")), + ), + expectError: true, + errContains: "annotation", + }, + "annotation value missing equals sign": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_ANNOTATION, exactValue("noequalssign")), + ), + expectError: true, + errContains: "key=value", + }, + "annotation value with equals sign is valid": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_ANNOTATION, exactValue("k=v")), + ), + expectError: false, + }, + "deployment regex on name is valid": { + scope: makeEntityScope( + makeRule(apiV2.ScopeEntity_SCOPE_ENTITY_DEPLOYMENT, apiV2.ScopeField_FIELD_NAME, regexValue("front.*")), + ), + expectError: false, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + err := validateEntityScope(tc.scope) + if tc.expectError { + assert.Error(t, err) + if tc.errContains != "" { + assert.Contains(t, err.Error(), tc.errContains) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateSchedule(t *testing.T) { + v := &Validator{} + + var cases = []struct { + testname string + config *apiV2.ReportConfiguration + expectError bool + errContains string + }{ + { + testname: "Nil schedule is valid", + config: &apiV2.ReportConfiguration{ + Schedule: nil, + }, + expectError: false, + }, + { + testname: "UNSET interval type is invalid", + config: &apiV2.ReportConfiguration{ + Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_UNSET, + }, + }, + expectError: true, + errContains: "DAILY, WEEKLY, or MONTHLY", + }, + { + testname: "Valid daily schedule", + config: &apiV2.ReportConfiguration{ + Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_DAILY, + Hour: 14, + Minute: 30, + }, + }, + expectError: false, + }, + { + testname: "Daily schedule with days_of_week is invalid", + config: &apiV2.ReportConfiguration{ + Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_DAILY, + Hour: 14, + Minute: 30, + Interval: &apiV2.ReportSchedule_DaysOfWeek_{ + DaysOfWeek: &apiV2.ReportSchedule_DaysOfWeek{Days: []int32{1}}, + }, + }, + }, + expectError: true, + errContains: "daily schedule must not specify", + }, + { + testname: "Daily schedule with days_of_month is invalid", + config: &apiV2.ReportConfiguration{ + Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_DAILY, + Hour: 14, + Minute: 30, + Interval: &apiV2.ReportSchedule_DaysOfMonth_{ + DaysOfMonth: &apiV2.ReportSchedule_DaysOfMonth{Days: []int32{1}}, + }, + }, + }, + expectError: true, + errContains: "daily schedule must not specify", + }, + { + testname: "Valid weekly schedule", + config: &apiV2.ReportConfiguration{ + Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_WEEKLY, + Hour: 10, + Minute: 0, + Interval: &apiV2.ReportSchedule_DaysOfWeek_{ + DaysOfWeek: &apiV2.ReportSchedule_DaysOfWeek{Days: []int32{1, 3}}, + }, + }, + }, + expectError: false, + }, + { + testname: "Weekly schedule without days is invalid", + config: &apiV2.ReportConfiguration{ + Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_WEEKLY, + Hour: 10, + Minute: 0, + }, + }, + expectError: true, + errContains: "days of week", + }, + { + testname: "Valid monthly schedule", + config: &apiV2.ReportConfiguration{ + Schedule: &apiV2.ReportSchedule{ + IntervalType: apiV2.ReportSchedule_MONTHLY, + Hour: 8, + Minute: 45, + Interval: &apiV2.ReportSchedule_DaysOfMonth_{ + DaysOfMonth: &apiV2.ReportSchedule_DaysOfMonth{Days: []int32{1}}, + }, + }, + }, + expectError: false, + }, + } + + for _, c := range cases { + t.Run(c.testname, func(t *testing.T) { + err := v.validateSchedule(c.config) + if c.expectError { + assert.Error(t, err) + if c.errContains != "" { + assert.Contains(t, err.Error(), c.errContains) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestValidateReportFiltersQuery(t *testing.T) { + t.Setenv(features.VulnerabilityReportsEnhancedFiltering.EnvVar(), "true") + v := &Validator{} + baseFilters := func(query string) *apiV2.ReportConfiguration { + return &apiV2.ReportConfiguration{ + Type: apiV2.ReportConfiguration_VULNERABILITY, + Filter: &apiV2.ReportConfiguration_VulnReportFilters{ + VulnReportFilters: &apiV2.VulnerabilityReportFilters{ + ImageTypes: []apiV2.VulnerabilityReportFilters_ImageType{apiV2.VulnerabilityReportFilters_DEPLOYED}, + CvesSince: &apiV2.VulnerabilityReportFilters_AllVuln{AllVuln: true}, + Query: query, + }, + }, + } + } + + cases := map[string]struct { + query string + expectError bool + }{ + "empty query is valid": {query: "", expectError: false}, + "valid CVE query": {query: "CVE:CVE-2024-1234", expectError: false}, + "valid severity query": {query: "Severity:CRITICAL", expectError: false}, + "valid compound query": {query: "Severity:CRITICAL+CVE:CVE-2024-1234", expectError: false}, + "bare value without field label is invalid": {query: "nofieldvalue", expectError: true}, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + err := v.validateReportFilters(baseFilters(tc.query)) + if tc.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid query") + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/central/reprocessor/reprocessor.go b/central/reprocessor/reprocessor.go index 344b316e8b849..c2f7e1ffbba7b 100644 --- a/central/reprocessor/reprocessor.go +++ b/central/reprocessor/reprocessor.go @@ -164,6 +164,13 @@ type imageReprocessingFunc func(ctx context.Context, enrichCtx imageEnricher.Enr type imageReprocessingFuncV2 func(ctx context.Context, enrichCtx imageEnricher.EnrichmentContext, image *storage.ImageV2) (imageEnricher.EnrichmentResult, error) +// imageRef identifies an image for reprocessing. +type imageRef struct { + id string // V2 image ID (UUID) + digest string // SHA256 digest + name *storage.ImageName // registry/remote:tag +} + type loopImpl struct { enrichAndDetectTickerDuration time.Duration enrichAndDetectTicker *time.Ticker @@ -326,45 +333,45 @@ func (l *loopImpl) runReprocessingForObjects(entityType string, getIDsFunc func( // TODO(ROX-30117): Remove this function after ImageV2 model is fully rolled out func (l *loopImpl) reprocessImage(id string, fetchOpt imageEnricher.FetchOption, - reprocessingFunc imageReprocessingFunc) (*storage.Image, bool) { - image, exists, err := l.images.GetImage(allAccessCtx, id) + reprocessingFunc imageReprocessingFunc) (image *storage.Image, successfullyProcessed bool, imageUpdated bool) { + img, exists, err := l.images.GetImage(allAccessCtx, id) if err != nil { log.Errorw("Error fetching image from database", logging.ImageID(id), logging.Err(err)) - return nil, false + return nil, false, false } - if !exists || image.GetNotPullable() || image.GetIsClusterLocal() { - return nil, false + if !exists || img.GetNotPullable() || img.GetIsClusterLocal() { + return nil, false, false } result, err := reprocessingFunc(emptyCtx, imageEnricher.EnrichmentContext{ FetchOpt: fetchOpt, - }, image) + }, img) if err != nil { - log.Errorw("Error enriching image", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId()), logging.Err(err)) - return nil, false + log.Errorw("Error enriching image", logging.ImageName(img.GetName().GetFullName()), logging.ImageID(img.GetId()), logging.Err(err)) + return nil, false, false } if result.ImageUpdated { - if err := l.risk.CalculateRiskAndUpsertImage(image); err != nil { + if err := l.risk.CalculateRiskAndUpsertImage(img); err != nil { log.Errorw("Error upserting image into datastore", - logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId()), logging.Err(err)) - return nil, false + logging.ImageName(img.GetName().GetFullName()), logging.ImageID(img.GetId()), logging.Err(err)) + return nil, false, false } // We need to fetch the image again to make sure all fields are populated. // GetImage will internally call a Merge function which will use the CVEEdges table to enrich fields like // FirstImageOccurrence and FirstSystemOccurrence. newImage, exists, err := l.images.GetImage(allAccessCtx, id) if err != nil { - log.Errorw("Error fetching image from database", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId()), logging.Err(err)) - return nil, false + log.Errorw("Error fetching image from database", logging.ImageName(img.GetName().GetFullName()), logging.ImageID(img.GetId()), logging.Err(err)) + return nil, false, false } if !exists { - log.Errorw("The image was not found after enrichement", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId())) - return nil, false + log.Errorw("The image was not found after enrichment", logging.ImageName(img.GetName().GetFullName()), logging.ImageID(img.GetId())) + return nil, false, false } - return newImage, true + return newImage, true, true } - return image, true + return img, true, false } // TODO(ROX-30117): Remove this function after ImageV2 model is fully rolled out @@ -387,7 +394,7 @@ func (l *loopImpl) reprocessImagesAndResyncDeployments(fetchOpt imageEnricher.Fe sema := semaphore.NewWeighted(imageReprocessorSemaphoreSize) wg := concurrency.NewWaitGroup(0) nReprocessed := atomic.NewInt32(0) - skipClusterIDs := maputil.NewSyncMap[string, struct{}]() + state := newReprocessingState(fetchOpt == imageEnricher.UseCachesIfPossible) for _, result := range results { wg.Add(1) if err := sema.Acquire(concurrency.AsContext(&l.stopSig), 1); err != nil { @@ -400,7 +407,7 @@ func (l *loopImpl) reprocessImagesAndResyncDeployments(fetchOpt imageEnricher.Fe defer sema.Release(1) defer wg.Add(-1) - image, successfullyProcessed := l.reprocessImage(id, fetchOpt, imgReprocessingFunc) + image, successfullyProcessed, imageUpdated := l.reprocessImage(id, fetchOpt, imgReprocessingFunc) if !successfullyProcessed { return } @@ -409,21 +416,21 @@ func (l *loopImpl) reprocessImagesAndResyncDeployments(fetchOpt imageEnricher.Fe utils.FilterSuppressedCVEsNoClone(image) utils.StripCVEDescriptionsNoClone(image) - // Send the updated image to relevant clusters. + imageKey := ¢ral.ImageKey{ + ImageId: image.GetId(), + ImageFullName: image.GetName().GetFullName(), + } + for clusterID := range clusterIDs { conn := l.connManager.GetConnection(clusterID) if conn == nil { continue } - msg := ¢ral.MsgToSensor{ - Msg: ¢ral.MsgToSensor_UpdatedImage{ - UpdatedImage: image, - }, - } - - // If were prior errors, do not attempt to send a message to this cluster. - if skipClusterIDs.Contains(clusterID) { + if state.skipClusterIDs.Contains(clusterID) { + msg := ¢ral.MsgToSensor{ + Msg: ¢ral.MsgToSensor_UpdatedImage{UpdatedImage: image}, + } metrics.IncrementMsgToSensorNotSentCounter(clusterID, msg, metrics.NotSentSkip) log.Debugw("Not sending updated image to cluster due to prior errors", logging.ImageID(image.GetId()), @@ -433,14 +440,12 @@ func (l *loopImpl) reprocessImagesAndResyncDeployments(fetchOpt imageEnricher.Fe continue } - err := l.injectMessage(concurrency.AsContext(&l.stopSig), conn, msg) - if err != nil { - skipClusterIDs.Store(clusterID, struct{}{}) - log.Errorw("Error sending updated image to cluster, skipping cluster until next reprocessing cycle", + if err := l.sendReprocessingMessages(conn, image, imageKey, imageUpdated, state); err != nil { + log.Errorw("Error sending reprocessing messages to cluster, skipping cluster until next reprocessing cycle", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId()), logging.Err(err), - // Not using logging.ClusterID() to avoid "duplicate resource ID field found" panic - logging.String("dst_cluster", clusterID), + // Not using logging.ClusterID() to avoid "duplicate resource ID field found" panic. + logging.String("dst_cluster", conn.ClusterID()), ) } } @@ -454,46 +459,46 @@ func (l *loopImpl) reprocessImagesAndResyncDeployments(fetchOpt imageEnricher.Fe } log.Infof("Successfully reprocessed %d/%d images", nReprocessed.Load(), len(results)) + l.sendRefreshImageCacheTTL(state) + log.Info("Resyncing deployments now that images have been reprocessed...") - l.sendReprocessDeployments(skipClusterIDs) + l.sendReprocessDeployments(state) } -func (l *loopImpl) reprocessImageV2(id string, digest string, fetchOpt imageEnricher.FetchOption, - reprocessingFunc imageReprocessingFuncV2) (*storage.ImageV2, bool) { - image, exists, err := l.imagesV2.GetImage(allAccessCtx, id) +func (l *loopImpl) reprocessImageV2(ref imageRef, fetchOpt imageEnricher.FetchOption, + reprocessingFunc imageReprocessingFuncV2) (img *storage.ImageV2, successfullyProcessed bool, imageUpdated bool) { + image, exists, err := l.imagesV2.GetImage(allAccessCtx, ref.id) if err != nil { - log.Errorw("Error fetching image from database", logging.ImageID(id), logging.Err(err)) - return nil, false + log.Errorw("Error fetching image from database", logging.ImageID(ref.id), logging.Err(err)) + return nil, false, false } migrateToV2 := false if !exists { - // The image was not found in ImageV2 store, but it might be in the legacy ImageV1 store var legacyImage *storage.Image - legacyImage, exists, err = l.images.GetImageMetadata(allAccessCtx, digest) + legacyImage, exists, err = l.images.GetImageMetadata(allAccessCtx, ref.digest) if err != nil { - log.Errorw("Error fetching legacy image from database", logging.ImageID(id), logging.Err(err)) - return nil, false + log.Errorw("Error fetching legacy image from database", logging.ImageID(ref.id), logging.Err(err)) + return nil, false, false } if !exists { - return nil, false + return nil, false, false } - image = utils.ConvertToV2(legacyImage) + image = utils.ConvertToV2WithNameOverride(legacyImage, ref.name) migrateToV2 = true } if image == nil { - return nil, false + return nil, false, false } if image.GetNotPullable() || image.GetIsClusterLocal() { - // Skip reprocessing. Sensor will handle cluster-local images. But we still need to migrate the image to V2. if migrateToV2 { if err := l.imagesV2.UpsertImage(allAccessCtx, image); err != nil { log.Errorw("Error migrating image to imageV2 store", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId()), logging.Err(err)) - return nil, false + return nil, false, false } } - return nil, false + return nil, false, false } result, err := reprocessingFunc(emptyCtx, imageEnricher.EnrichmentContext{ @@ -502,29 +507,29 @@ func (l *loopImpl) reprocessImageV2(id string, digest string, fetchOpt imageEnri if err != nil { log.Errorw("Error enriching image", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId()), logging.Err(err)) - return nil, false + return nil, false, false } if result.ImageUpdated { if err := l.risk.CalculateRiskAndUpsertImageV2(image); err != nil { log.Errorw("Error upserting image into datastore", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId()), logging.Err(err)) - return nil, false + return nil, false, false } // We need to fetch the image again to make sure all fields are populated. // GetImage will internally call a Merge function which will use the CVEEdges table to enrich fields like // FirstImageOccurrence and FirstSystemOccurrence. - newImage, exists, err := l.imagesV2.GetImage(allAccessCtx, id) + newImage, exists, err := l.imagesV2.GetImage(allAccessCtx, ref.id) if err != nil { log.Errorw("Error fetching image from database", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId()), logging.Err(err)) - return nil, false + return nil, false, false } if !exists { - log.Errorw("The image was not found after enrichement", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId())) - return nil, false + log.Errorw("The image was not found after enrichment", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId())) + return nil, false, false } - return newImage, true + return newImage, true, true } - return image, true + return image, true, false } func (l *loopImpl) reprocessImagesV2AndResyncDeployments(fetchOpt imageEnricher.FetchOption, @@ -546,7 +551,7 @@ func (l *loopImpl) reprocessImagesV2AndResyncDeployments(fetchOpt imageEnricher. sema := semaphore.NewWeighted(imageReprocessorSemaphoreSize) wg := concurrency.NewWaitGroup(0) nReprocessed := atomic.NewInt32(0) - skipClusterIDs := maputil.NewSyncMap[string, struct{}]() + state := newReprocessingState(fetchOpt == imageEnricher.UseCachesIfPossible) for _, result := range results { wg.Add(1) if err := sema.Acquire(concurrency.AsContext(&l.stopSig), 1); err != nil { @@ -554,11 +559,11 @@ func (l *loopImpl) reprocessImagesV2AndResyncDeployments(fetchOpt imageEnricher. return } clusterIDSet := set.NewStringSet(result.GetClusterIDs()...) - go func(id string, digest string, clusterIDs set.StringSet) { + go func(ref imageRef, clusterIDs set.StringSet) { defer sema.Release(1) defer wg.Add(-1) - image, successfullyProcessed := l.reprocessImageV2(id, digest, fetchOpt, imgReprocessingFunc) + image, successfullyProcessed, imageUpdated := l.reprocessImageV2(ref, fetchOpt, imgReprocessingFunc) if !successfullyProcessed { return } @@ -584,21 +589,23 @@ func (l *loopImpl) reprocessImagesV2AndResyncDeployments(fetchOpt imageEnricher. } } convertedImage := utils.ConvertToV1(image, allNames...) - // Send the updated image to relevant clusters. + + imageKey := ¢ral.ImageKey{ + ImageId: image.GetDigest(), + ImageIdV2: image.GetId(), + ImageFullName: image.GetName().GetFullName(), + } + for clusterID := range clusterIDs { conn := l.connManager.GetConnection(clusterID) if conn == nil { continue } - msg := ¢ral.MsgToSensor{ - Msg: ¢ral.MsgToSensor_UpdatedImage{ - UpdatedImage: convertedImage, - }, - } - - // If were prior errors, do not attempt to send a message to this cluster. - if skipClusterIDs.Contains(clusterID) { + if state.skipClusterIDs.Contains(clusterID) { + msg := ¢ral.MsgToSensor{ + Msg: ¢ral.MsgToSensor_UpdatedImage{UpdatedImage: convertedImage}, + } metrics.IncrementMsgToSensorNotSentCounter(clusterID, msg, metrics.NotSentSkip) log.Debugw("Not sending updated image to cluster due to prior errors", logging.ImageID(image.GetId()), @@ -608,18 +615,16 @@ func (l *loopImpl) reprocessImagesV2AndResyncDeployments(fetchOpt imageEnricher. continue } - err := l.injectMessage(concurrency.AsContext(&l.stopSig), conn, msg) - if err != nil { - skipClusterIDs.Store(clusterID, struct{}{}) - log.Errorw("Error sending updated image to cluster, skipping cluster until next reprocessing cycle", + if err := l.sendReprocessingMessages(conn, convertedImage, imageKey, imageUpdated, state); err != nil { + log.Errorw("Error sending reprocessing messages to cluster, skipping cluster until next reprocessing cycle", logging.ImageName(image.GetName().GetFullName()), logging.ImageID(image.GetId()), logging.Err(err), - // Not using logging.ClusterID() to avoid "duplicate resource ID field found" panic - logging.String("dst_cluster", clusterID), + // Not using logging.ClusterID() to avoid "duplicate resource ID field found" panic. + logging.String("dst_cluster", conn.ClusterID()), ) } } - }(result.GetImageID(), result.GetImageDigest(), clusterIDSet) + }(imageRef{id: result.GetImageID(), digest: result.GetImageDigest(), name: result.GetImageName()}, clusterIDSet) } select { case <-wg.Done(): @@ -629,19 +634,113 @@ func (l *loopImpl) reprocessImagesV2AndResyncDeployments(fetchOpt imageEnricher. } log.Infof("Successfully reprocessed %d/%d images", nReprocessed.Load(), len(results)) + l.sendRefreshImageCacheTTL(state) + log.Info("Resyncing deployments now that images have been reprocessed...") - l.sendReprocessDeployments(skipClusterIDs) + l.sendReprocessDeployments(state) +} + +// reprocessingState holds per-cycle concurrent state shared across image goroutines. +type reprocessingState struct { + skipClusterIDs maputil.SyncMap[string, struct{}] + refreshTTLKeys maputil.SyncMap[string, []*central.ImageKey] + // When true, Sensor uses targeted invalidation instead of full AC cache flush. + skipCacheFlush bool +} + +func newReprocessingState(skipCacheFlush bool) *reprocessingState { + return &reprocessingState{ + skipClusterIDs: maputil.NewSyncMap[string, struct{}](), + refreshTTLKeys: maputil.NewSyncMap[string, []*central.ImageKey](), + skipCacheFlush: skipCacheFlush, + } } -// sendReprocessDeployments sends a reprocess deployments message to every connected -// secured cluster. -func (l *loopImpl) sendReprocessDeployments(skipClusterIDs maputil.SyncMap[string, struct{}]) { +// sendReprocessingMessages handles per-image messaging to a single cluster. +// +// Short-circuit path (skipCacheFlush=true) with capable sensors: +// - imageUpdated=true → sends UpdatedImage (Sensor handles AC invalidation) +// - imageUpdated=false → accumulates the key for batched RefreshImageCacheTTL +// +// Periodic path (skipCacheFlush=false) or sensors without the capability: +// - always sends UpdatedImage (FlushCache handles AC invalidation) +func (l *loopImpl) sendReprocessingMessages( + conn connection.SensorConnection, + image *storage.Image, + imageKey *central.ImageKey, + imageUpdated bool, + state *reprocessingState, +) error { + ctx := concurrency.AsContext(&l.stopSig) + clusterID := conn.ClusterID() + + if state.skipCacheFlush && conn.HasCapability(centralsensor.TargetedImageCacheInvalidation) && !imageUpdated { + state.refreshTTLKeys.Access(func(m *map[string][]*central.ImageKey) { + (*m)[clusterID] = append((*m)[clusterID], imageKey) + }) + return nil + } + + updateMsg := ¢ral.MsgToSensor{ + Msg: ¢ral.MsgToSensor_UpdatedImage{UpdatedImage: image}, + } + if err := l.injectMessage(ctx, conn, updateMsg); err != nil { + state.skipClusterIDs.Store(clusterID, struct{}{}) + return err + } + return nil +} + +// sendRefreshImageCacheTTL sends a single batched RefreshImageCacheTTL message +// per cluster for all unchanged images accumulated during reprocessing. +func (l *loopImpl) sendRefreshImageCacheTTL(state *reprocessingState) { + if l.stopSig.IsDone() { + return + } + ctx := concurrency.AsContext(&l.stopSig) + state.refreshTTLKeys.RAccess(func(m map[string][]*central.ImageKey) { + for clusterID, keys := range m { + if state.skipClusterIDs.Contains(clusterID) { + log.Debugw("Not sending refresh image cache TTL to cluster due to prior errors", + logging.String("dst_cluster", clusterID), + logging.Int("key_count", len(keys)), + ) + continue + } + conn := l.connManager.GetConnection(clusterID) + if conn == nil { + continue + } + msg := ¢ral.MsgToSensor{ + Msg: ¢ral.MsgToSensor_RefreshImageCacheTtl{ + RefreshImageCacheTtl: ¢ral.RefreshImageCacheTTL{ + ImageKeys: keys, + }, + }, + } + if err := l.injectMessage(ctx, conn, msg); err != nil { + state.skipClusterIDs.Store(clusterID, struct{}{}) + log.Errorw("Error sending refresh image cache TTL to cluster", + logging.String("dst_cluster", clusterID), + logging.Err(err), + ) + continue + } + log.Infof("Sent refresh image cache TTL to cluster %s with %d keys", clusterID, len(keys)) + } + }) +} + +// sendReprocessDeployments sends a ReprocessDeployments message to every connected cluster. +func (l *loopImpl) sendReprocessDeployments(state *reprocessingState) { // Once the images have been rescanned, then reprocess the deployments. // This should not take a particularly long period of time. if !l.stopSig.IsDone() { msg := ¢ral.MsgToSensor{ Msg: ¢ral.MsgToSensor_ReprocessDeployments{ - ReprocessDeployments: ¢ral.ReprocessDeployments{}, + ReprocessDeployments: ¢ral.ReprocessDeployments{ + SkipCacheFlush: state.skipCacheFlush, + }, }, } ctx := concurrency.AsContext(&l.stopSig) @@ -656,7 +755,7 @@ func (l *loopImpl) sendReprocessDeployments(skipClusterIDs maputil.SyncMap[strin firstMessage := true for i, conn := range conns { clusterID := conn.ClusterID() - if skipClusterIDs.Contains(clusterID) { + if state.skipClusterIDs.Contains(clusterID) { metrics.IncrementMsgToSensorNotSentCounter(clusterID, msg, metrics.NotSentSkip) log.Errorw("Not sending reprocess deployments to cluster due to prior errors", logging.ClusterID(clusterID), diff --git a/central/reprocessor/reprocessor_test.go b/central/reprocessor/reprocessor_test.go index d95e80c37a9db..8ed6d65bd8694 100644 --- a/central/reprocessor/reprocessor_test.go +++ b/central/reprocessor/reprocessor_test.go @@ -26,93 +26,84 @@ import ( ) func TestImagesWithSignaturesQuery(t *testing.T) { - testutils.MustUpdateFeature(t, features.FlattenImageData, false) - testCtx := sac.WithAllAccess(context.Background()) testingDB := pgtest.ForT(t) pool := testingDB.DB defer pool.Close() - imageDS := imageDatastore.NewWithPostgres(imagePostgresV2.New(pool, false, concurrency.NewKeyFence()), nil, ranking.ImageRanker(), ranking.ComponentRanker()) - - imgWithSignature := fixtures.GetImage() - imgWithoutSignature := fixtures.GetImageWithUniqueComponents(10) - oneHourAgo := time.Now().Add(-1 * time.Hour) oneMinuteAgo := time.Now().Add(-1 * time.Minute) + const imgWithSignatureID = "sha256:image-with-signature" const imgWithoutSignatureID = "sha256:image-without-signature" - imgWithSignature.Id = imgWithSignatureID - imgWithSignature.Signature = &storage.ImageSignature{ - Fetched: protocompat.ConvertTimeToTimestampOrNil(&oneHourAgo), - } - imgWithoutSignature.Id = imgWithoutSignatureID - - require.NoError(t, imageDS.UpsertImage(testCtx, imgWithSignature)) - require.NoError(t, imageDS.UpsertImage(testCtx, imgWithoutSignature)) - - results, err := imageDS.Search(testCtx, imagesWithSignaturesQuery) - assert.NoError(t, err) - - require.Len(t, results, 1) - assert.Equal(t, results[0].ID, imgWithSignatureID) + t.Run("V1", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.FlattenImageData, false) - imgWithSignature.Signature = &storage.ImageSignature{ - Fetched: protocompat.ConvertTimeToTimestampOrNil(&oneMinuteAgo), - } - require.NoError(t, imageDS.UpsertImage(testCtx, imgWithSignature)) + imageDS := imageDatastore.NewWithPostgres(imagePostgresV2.New(pool, false, concurrency.NewKeyFence()), nil, ranking.ImageRanker(), ranking.ComponentRanker()) - results, err = imageDS.Search(testCtx, imagesWithSignaturesQuery) - assert.NoError(t, err) + imgWithSignature := fixtures.GetImage() + imgWithoutSignature := fixtures.GetImageWithUniqueComponents(10) - require.Len(t, results, 1) - assert.Equal(t, results[0].ID, imgWithSignatureID) -} + imgWithSignature.Id = imgWithSignatureID + imgWithSignature.Signature = &storage.ImageSignature{ + Fetched: protocompat.ConvertTimeToTimestampOrNil(&oneHourAgo), + } + imgWithoutSignature.Id = imgWithoutSignatureID -func TestImagesWithSignaturesQueryV2(t *testing.T) { - testutils.MustUpdateFeature(t, features.FlattenImageData, true) + require.NoError(t, imageDS.UpsertImage(testCtx, imgWithSignature)) + require.NoError(t, imageDS.UpsertImage(testCtx, imgWithoutSignature)) - testCtx := sac.WithAllAccess(context.Background()) + results, err := imageDS.Search(testCtx, imagesWithSignaturesQuery) + assert.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, results[0].ID, imgWithSignatureID) - testingDB := pgtest.ForT(t) - pool := testingDB.DB - defer pool.Close() + imgWithSignature.Signature = &storage.ImageSignature{ + Fetched: protocompat.ConvertTimeToTimestampOrNil(&oneMinuteAgo), + } + require.NoError(t, imageDS.UpsertImage(testCtx, imgWithSignature)) - imageDS := imageV2Datastore.NewWithPostgres(imageV2PG.New(pool, false, concurrency.NewKeyFence()), nil, ranking.ImageRanker(), ranking.ComponentRanker()) + results, err = imageDS.Search(testCtx, imagesWithSignaturesQuery) + assert.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, results[0].ID, imgWithSignatureID) + }) - imgWithSignature := fixtures.GetImageV2() - imgWithoutSignature := fixtures.GetImageV2WithUniqueComponents(10) + t.Run("V2", func(t *testing.T) { + testutils.MustUpdateFeature(t, features.FlattenImageData, true) - oneHourAgo := time.Now().Add(-1 * time.Hour) - oneMinuteAgo := time.Now().Add(-1 * time.Minute) - imgWithSignatureID := utils.NewImageV2ID(imgWithSignature.GetName(), "sha256:image-with-signature") - imgWithoutSignatureID := utils.NewImageV2ID(imgWithoutSignature.GetName(), "sha256:image-without-signature") + imageDS := imageV2Datastore.NewWithPostgres(imageV2PG.New(pool, false, concurrency.NewKeyFence()), nil, ranking.ImageRanker(), ranking.ComponentRanker()) - imgWithSignature.Id = imgWithSignatureID - imgWithSignature.Signature = &storage.ImageSignature{ - Fetched: protocompat.ConvertTimeToTimestampOrNil(&oneHourAgo), - } - imgWithoutSignature.Id = imgWithoutSignatureID + imgWithSignature := fixtures.GetImageV2() + imgWithoutSignature := fixtures.GetImageV2WithUniqueComponents(10) - require.NoError(t, imageDS.UpsertImage(testCtx, imgWithSignature)) - require.NoError(t, imageDS.UpsertImage(testCtx, imgWithoutSignature)) + imgWithSignatureID := utils.NewImageV2ID(imgWithSignature.GetName(), imgWithSignatureID) + imgWithoutSignatureID := utils.NewImageV2ID(imgWithoutSignature.GetName(), imgWithoutSignatureID) - results, err := imageDS.Search(testCtx, imagesWithSignaturesQuery) - assert.NoError(t, err) + imgWithSignature.Id = imgWithSignatureID + imgWithSignature.Signature = &storage.ImageSignature{ + Fetched: protocompat.ConvertTimeToTimestampOrNil(&oneHourAgo), + } + imgWithoutSignature.Id = imgWithoutSignatureID - require.Len(t, results, 1) - assert.Equal(t, results[0].ID, imgWithSignatureID) + require.NoError(t, imageDS.UpsertImage(testCtx, imgWithSignature)) + require.NoError(t, imageDS.UpsertImage(testCtx, imgWithoutSignature)) - imgWithSignature.Signature = &storage.ImageSignature{ - Fetched: protocompat.ConvertTimeToTimestampOrNil(&oneMinuteAgo), - } - require.NoError(t, imageDS.UpsertImage(testCtx, imgWithSignature)) + results, err := imageDS.Search(testCtx, imagesWithSignaturesQuery) + assert.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, results[0].ID, imgWithSignatureID) - results, err = imageDS.Search(testCtx, imagesWithSignaturesQuery) - assert.NoError(t, err) + imgWithSignature.Signature = &storage.ImageSignature{ + Fetched: protocompat.ConvertTimeToTimestampOrNil(&oneMinuteAgo), + } + require.NoError(t, imageDS.UpsertImage(testCtx, imgWithSignature)) - require.Len(t, results, 1) - assert.Equal(t, results[0].ID, imgWithSignatureID) + results, err = imageDS.Search(testCtx, imagesWithSignaturesQuery) + assert.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, results[0].ID, imgWithSignatureID) + }) } diff --git a/central/reprocessor/reprocessor_unit_test.go b/central/reprocessor/reprocessor_unit_test.go index fc0214eba7c00..a52c38e30af72 100644 --- a/central/reprocessor/reprocessor_unit_test.go +++ b/central/reprocessor/reprocessor_unit_test.go @@ -17,6 +17,7 @@ import ( connectionMocks "github.com/stackrox/rox/central/sensor/service/connection/mocks" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/features" imageEnricher "github.com/stackrox/rox/pkg/images/enricher" @@ -34,6 +35,61 @@ import ( "google.golang.org/protobuf/proto" ) +var ( + isInvalidateImageCache = gomock.Cond(func(msg *central.MsgToSensor) bool { + return event.GetEventTypeWithoutPrefix(msg.GetMsg()) == "InvalidateImageCache" + }) + isRefreshImageCacheTTL = gomock.Cond(func(msg *central.MsgToSensor) bool { + return event.GetEventTypeWithoutPrefix(msg.GetMsg()) == "RefreshImageCacheTtl" + }) + isUpdatedImage = gomock.Cond(func(msg *central.MsgToSensor) bool { + return event.GetEventTypeWithoutPrefix(msg.GetMsg()) == "UpdatedImage" + }) + isReprocessDeployments = gomock.Cond(func(msg *central.MsgToSensor) bool { + return event.GetEventTypeWithoutPrefix(msg.GetMsg()) == "ReprocessDeployments" + }) + isReprocessDeploymentsSkipFlush = gomock.Cond(func(msg *central.MsgToSensor) bool { + return event.GetEventTypeWithoutPrefix(msg.GetMsg()) == "ReprocessDeployments" && + msg.GetReprocessDeployments().GetSkipCacheFlush() + }) + isReprocessDeploymentsDoFlush = gomock.Cond(func(msg *central.MsgToSensor) bool { + return event.GetEventTypeWithoutPrefix(msg.GetMsg()) == "ReprocessDeployments" && + !msg.GetReprocessDeployments().GetSkipCacheFlush() + }) + + v1ReprocessUpdate = func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.Image) (imageEnricher.EnrichmentResult, error) { + return imageEnricher.EnrichmentResult{ImageUpdated: true}, nil + } + v1ReprocessNoUpdate = func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.Image) (imageEnricher.EnrichmentResult, error) { + return imageEnricher.EnrichmentResult{ImageUpdated: false}, nil + } + v1ReprocessError = func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.Image) (imageEnricher.EnrichmentResult, error) { + return imageEnricher.EnrichmentResult{}, errors.New("some error") + } + + v2ReprocessUpdate = func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.ImageV2) (imageEnricher.EnrichmentResult, error) { + return imageEnricher.EnrichmentResult{ImageUpdated: true}, nil + } + v2ReprocessNoUpdate = func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.ImageV2) (imageEnricher.EnrichmentResult, error) { + return imageEnricher.EnrichmentResult{ImageUpdated: false}, nil + } + v2ReprocessError = func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.ImageV2) (imageEnricher.EnrichmentResult, error) { + return imageEnricher.EnrichmentResult{}, errors.New("some error") + } +) + +// newConnWithCapability creates a mock connection and manager for a single +// cluster "a" that advertises TargetedImageCacheInvalidation. +func newConnWithCapability(ctrl *gomock.Controller) (*connectionMocks.MockSensorConnection, *connectionMocks.MockManager) { + conn := connectionMocks.NewMockSensorConnection(ctrl) + mgr := connectionMocks.NewMockManager(ctrl) + conn.EXPECT().ClusterID().AnyTimes().Return("a") + conn.EXPECT().HasCapability(centralsensor.TargetedImageCacheInvalidation).AnyTimes().Return(true) + mgr.EXPECT().GetConnection("a").AnyTimes().Return(conn) + mgr.EXPECT().GetActiveConnections().Return([]connection.SensorConnection{conn}) + return conn, mgr +} + func Test_loopImpl_reprocessNode(t *testing.T) { type args struct { id string @@ -214,59 +270,56 @@ func TestReprocessImage(t *testing.T) { } return testLoop, imageDS, riskManager } - reprocessFuncError := func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.Image) (imageEnricher.EnrichmentResult, error) { - return imageEnricher.EnrichmentResult{}, errors.New("some error") - } - reprocessFuncUpdate := func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.Image) (imageEnricher.EnrichmentResult, error) { - return imageEnricher.EnrichmentResult{ImageUpdated: true}, nil - } - reprocessFuncNoUpdate := func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.Image) (imageEnricher.EnrichmentResult, error) { - return imageEnricher.EnrichmentResult{ImageUpdated: false}, nil - } imageID := "id" t.Run("error retrieving the image", func(tt *testing.T) { testLoop, imageDS, _ := newTestLoop(tt) imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(nil, false, errors.New("some error")) - image, reprocessed := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("image does not exist", func(tt *testing.T) { testLoop, imageDS, _ := newTestLoop(tt) imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(nil, false, nil) - image, reprocessed := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("image is not pullable", func(tt *testing.T) { testLoop, imageDS, _ := newTestLoop(tt) imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(&storage.Image{NotPullable: true}, true, nil) - image, reprocessed := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("image is cluster local", func(tt *testing.T) { testLoop, imageDS, _ := newTestLoop(tt) imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(&storage.Image{IsClusterLocal: true}, true, nil) - image, reprocessed := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("reprocessingFunc error", func(tt *testing.T) { testLoop, imageDS, _ := newTestLoop(tt) imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(&storage.Image{}, true, nil) - image, reprocessed := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, reprocessFuncError) + image, reprocessed, updated := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, v1ReprocessError) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("reprocessingFunc update calculate risk and upsert error", func(tt *testing.T) { testLoop, imageDS, riskManager := newTestLoop(tt) image := &storage.Image{} imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(image, true, nil) riskManager.EXPECT().CalculateRiskAndUpsertImage(gomock.Eq(image)).Times(1).Return(errors.New("some error")) - retImage, reprocessed := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, reprocessFuncUpdate) + retImage, reprocessed, updated := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, v1ReprocessUpdate) assert.Nil(tt, retImage) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("re-fetch error", func(tt *testing.T) { testLoop, imageDS, riskManager := newTestLoop(tt) @@ -276,9 +329,10 @@ func TestReprocessImage(t *testing.T) { riskManager.EXPECT().CalculateRiskAndUpsertImage(gomock.Eq(image)).Times(1).Return(nil), imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(nil, false, errors.New("some error")), ) - retImage, reprocessed := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, reprocessFuncUpdate) + retImage, reprocessed, updated := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, v1ReprocessUpdate) assert.Nil(tt, retImage) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("re-fetch not found", func(tt *testing.T) { testLoop, imageDS, riskManager := newTestLoop(tt) @@ -288,9 +342,10 @@ func TestReprocessImage(t *testing.T) { riskManager.EXPECT().CalculateRiskAndUpsertImage(gomock.Eq(image)).Times(1).Return(nil), imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(nil, false, nil), ) - retImage, reprocessed := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, reprocessFuncUpdate) + retImage, reprocessed, updated := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, v1ReprocessUpdate) assert.Nil(tt, retImage) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("re-fetch image", func(tt *testing.T) { testLoop, imageDS, riskManager := newTestLoop(tt) @@ -301,20 +356,22 @@ func TestReprocessImage(t *testing.T) { riskManager.EXPECT().CalculateRiskAndUpsertImage(gomock.Eq(initialImage)).Times(1).Return(nil), imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(secondImage, true, nil), ) - retImage, reprocessed := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, reprocessFuncUpdate) + retImage, reprocessed, updated := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, v1ReprocessUpdate) assert.NotNil(tt, retImage) assert.False(tt, proto.Equal(initialImage, retImage)) assert.True(tt, proto.Equal(secondImage, retImage)) assert.True(tt, reprocessed) + assert.True(tt, updated) }) t.Run("reprocessingFunc no scan update", func(tt *testing.T) { testLoop, imageDS, _ := newTestLoop(tt) image := &storage.Image{} imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(image, true, nil) - retImage, reprocessed := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, reprocessFuncNoUpdate) + retImage, reprocessed, updated := testLoop.reprocessImage(imageID, imageEnricher.UseCachesIfPossible, v1ReprocessNoUpdate) assert.NotNil(tt, retImage) assert.True(tt, proto.Equal(image, retImage)) assert.True(tt, reprocessed) + assert.False(tt, updated) }) } @@ -332,23 +389,22 @@ func TestReprocessImageV2(t *testing.T) { } return testLoop, imageDS, legacyImageDS, riskManager } - reprocessFuncError := func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.ImageV2) (imageEnricher.EnrichmentResult, error) { - return imageEnricher.EnrichmentResult{}, errors.New("some error") - } - reprocessFuncUpdate := func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.ImageV2) (imageEnricher.EnrichmentResult, error) { - return imageEnricher.EnrichmentResult{ImageUpdated: true}, nil - } - reprocessFuncNoUpdate := func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.ImageV2) (imageEnricher.EnrichmentResult, error) { - return imageEnricher.EnrichmentResult{ImageUpdated: false}, nil - } imageID := "id" imageDigest := "sha256:test" + imageName := &storage.ImageName{ + Registry: "docker.io", + Remote: "library/nginx", + Tag: "latest", + FullName: "docker.io/library/nginx:latest", + } + ref := imageRef{id: imageID, digest: imageDigest, name: imageName} t.Run("error retrieving the image", func(tt *testing.T) { testLoop, imageDS, _, _ := newTestLoop(tt) imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(nil, false, errors.New("some error")) - image, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("image does not exist in V2 or legacy store", func(tt *testing.T) { testLoop, imageDS, legacyImageDS, _ := newTestLoop(tt) @@ -356,9 +412,10 @@ func TestReprocessImageV2(t *testing.T) { imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(nil, false, nil), legacyImageDS.EXPECT().GetImageMetadata(gomock.Any(), gomock.Eq(imageDigest)).Times(1).Return(nil, false, nil), ) - image, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("image does not exist in V2 and error fetching from legacy store", func(tt *testing.T) { testLoop, imageDS, legacyImageDS, _ := newTestLoop(tt) @@ -366,23 +423,26 @@ func TestReprocessImageV2(t *testing.T) { imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(nil, false, nil), legacyImageDS.EXPECT().GetImageMetadata(gomock.Any(), gomock.Eq(imageDigest)).Times(1).Return(nil, false, errors.New("some error")), ) - image, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("image is not pullable", func(tt *testing.T) { testLoop, imageDS, _, _ := newTestLoop(tt) imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(&storage.ImageV2{NotPullable: true}, true, nil) - image, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("image is cluster local", func(tt *testing.T) { testLoop, imageDS, _, _ := newTestLoop(tt) imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(&storage.ImageV2{IsClusterLocal: true}, true, nil) - image, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("legacy image not pullable triggers migration", func(tt *testing.T) { testLoop, imageDS, legacyImageDS, _ := newTestLoop(tt) @@ -392,9 +452,10 @@ func TestReprocessImageV2(t *testing.T) { legacyImageDS.EXPECT().GetImageMetadata(gomock.Any(), gomock.Eq(imageDigest)).Times(1).Return(legacyImage, true, nil), imageDS.EXPECT().UpsertImage(gomock.Any(), gomock.Any()).Times(1).Return(nil), ) - image, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("legacy image cluster local triggers migration", func(tt *testing.T) { testLoop, imageDS, legacyImageDS, _ := newTestLoop(tt) @@ -404,9 +465,10 @@ func TestReprocessImageV2(t *testing.T) { legacyImageDS.EXPECT().GetImageMetadata(gomock.Any(), gomock.Eq(imageDigest)).Times(1).Return(legacyImage, true, nil), imageDS.EXPECT().UpsertImage(gomock.Any(), gomock.Any()).Times(1).Return(nil), ) - image, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("legacy image migration upsert error", func(tt *testing.T) { testLoop, imageDS, legacyImageDS, _ := newTestLoop(tt) @@ -416,25 +478,28 @@ func TestReprocessImageV2(t *testing.T) { legacyImageDS.EXPECT().GetImageMetadata(gomock.Any(), gomock.Eq(imageDigest)).Times(1).Return(legacyImage, true, nil), imageDS.EXPECT().UpsertImage(gomock.Any(), gomock.Any()).Times(1).Return(errors.New("upsert error")), ) - image, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, nil) + image, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, nil) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("reprocessingFunc error", func(tt *testing.T) { testLoop, imageDS, _, _ := newTestLoop(tt) imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(&storage.ImageV2{}, true, nil) - image, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, reprocessFuncError) + image, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, v2ReprocessError) assert.Nil(tt, image) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("reprocessingFunc update calculate risk and upsert error", func(tt *testing.T) { testLoop, imageDS, _, riskManager := newTestLoop(tt) image := &storage.ImageV2{} imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(image, true, nil) riskManager.EXPECT().CalculateRiskAndUpsertImageV2(gomock.Eq(image)).Times(1).Return(errors.New("some error")) - retImage, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, reprocessFuncUpdate) + retImage, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, v2ReprocessUpdate) assert.Nil(tt, retImage) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("re-fetch error", func(tt *testing.T) { testLoop, imageDS, _, riskManager := newTestLoop(tt) @@ -444,9 +509,10 @@ func TestReprocessImageV2(t *testing.T) { riskManager.EXPECT().CalculateRiskAndUpsertImageV2(gomock.Eq(image)).Times(1).Return(nil), imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(nil, false, errors.New("some error")), ) - retImage, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, reprocessFuncUpdate) + retImage, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, v2ReprocessUpdate) assert.Nil(tt, retImage) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("re-fetch not found", func(tt *testing.T) { testLoop, imageDS, _, riskManager := newTestLoop(tt) @@ -456,9 +522,10 @@ func TestReprocessImageV2(t *testing.T) { riskManager.EXPECT().CalculateRiskAndUpsertImageV2(gomock.Eq(image)).Times(1).Return(nil), imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(nil, false, nil), ) - retImage, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, reprocessFuncUpdate) + retImage, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, v2ReprocessUpdate) assert.Nil(tt, retImage) assert.False(tt, reprocessed) + assert.False(tt, updated) }) t.Run("re-fetch image", func(tt *testing.T) { testLoop, imageDS, _, riskManager := newTestLoop(tt) @@ -469,20 +536,22 @@ func TestReprocessImageV2(t *testing.T) { riskManager.EXPECT().CalculateRiskAndUpsertImageV2(gomock.Eq(initialImage)).Times(1).Return(nil), imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(secondImage, true, nil), ) - retImage, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, reprocessFuncUpdate) + retImage, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, v2ReprocessUpdate) assert.NotNil(tt, retImage) assert.False(tt, proto.Equal(initialImage, retImage)) assert.True(tt, proto.Equal(secondImage, retImage)) assert.True(tt, reprocessed) + assert.True(tt, updated) }) t.Run("reprocessingFunc no scan update", func(tt *testing.T) { testLoop, imageDS, _, _ := newTestLoop(tt) image := &storage.ImageV2{} imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(image, true, nil) - retImage, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, reprocessFuncNoUpdate) + retImage, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, v2ReprocessNoUpdate) assert.NotNil(tt, retImage) assert.True(tt, proto.Equal(image, retImage)) assert.True(tt, reprocessed) + assert.False(tt, updated) }) t.Run("legacy image found and successfully reprocessed", func(tt *testing.T) { testLoop, imageDS, legacyImageDS, riskManager := newTestLoop(tt) @@ -494,16 +563,17 @@ func TestReprocessImageV2(t *testing.T) { riskManager.EXPECT().CalculateRiskAndUpsertImageV2(gomock.Any()).Times(1).Return(nil), imageDS.EXPECT().GetImage(gomock.Any(), gomock.Eq(imageID)).Times(1).Return(secondImage, true, nil), ) - retImage, reprocessed := testLoop.reprocessImageV2(imageID, imageDigest, imageEnricher.UseCachesIfPossible, reprocessFuncUpdate) + retImage, reprocessed, updated := testLoop.reprocessImageV2(ref, imageEnricher.UseCachesIfPossible, v2ReprocessUpdate) assert.NotNil(tt, retImage) assert.True(tt, proto.Equal(secondImage, retImage)) assert.True(tt, reprocessed) + assert.True(tt, updated) }) } func TestReprocessImagesAndResyncDeployments_SkipBrokenSensor(t *testing.T) { testutils.MustUpdateFeature(t, features.FlattenImageData, false) - imgs := []*storage.Image{} + imgs := make([]*storage.Image, 0, (imageReprocessorSemaphoreSize+1)*2) for _, cluster := range []string{"a", "b"} { // two clusters // Create at least one more image than max semaphore size to ensure skip logic is executed. for i := range imageReprocessorSemaphoreSize + 1 { @@ -511,7 +581,7 @@ func TestReprocessImagesAndResyncDeployments_SkipBrokenSensor(t *testing.T) { } } - results := []search.Result{} + results := make([]search.Result, 0, len(imgs)) for _, img := range imgs { results = append(results, search.Result{ ID: img.GetId(), @@ -554,49 +624,40 @@ func TestReprocessImagesAndResyncDeployments_SkipBrokenSensor(t *testing.T) { return testLoop, connA, connB } - reprocessFuncUpdate := func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.Image) (imageEnricher.EnrichmentResult, error) { - return imageEnricher.EnrichmentResult{ImageUpdated: true}, nil - } - updatedImageTypeCond := gomock.Cond(func(msg *central.MsgToSensor) bool { - return event.GetEventTypeWithoutPrefix(msg.GetMsg()) == "UpdatedImage" - }) - reprocessDeploymentsTypeCond := gomock.Cond(func(msg *central.MsgToSensor) bool { - return event.GetEventTypeWithoutPrefix(msg.GetMsg()) == "ReprocessDeployments" - }) - t.Run("send all messages when clusters are healthy", func(t *testing.T) { testLoop, connA, connB := newReprocessorLoop(t) - // Expect every updated image to be sent. - connA.EXPECT().InjectMessage(gomock.Any(), updatedImageTypeCond).Times(len(imgs) / 2).Return(nil) - connB.EXPECT().InjectMessage(gomock.Any(), updatedImageTypeCond).Times(len(imgs) / 2).Return(nil) + connA.EXPECT().HasCapability(centralsensor.TargetedImageCacheInvalidation).AnyTimes().Return(false) + connB.EXPECT().HasCapability(centralsensor.TargetedImageCacheInvalidation).AnyTimes().Return(false) + + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(len(imgs) / 2).Return(nil) + connB.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(len(imgs) / 2).Return(nil) - // Expect each cluster to be sent a reprocess deployments message. - connA.EXPECT().InjectMessage(gomock.Any(), reprocessDeploymentsTypeCond).Times(1).Return(nil) - connB.EXPECT().InjectMessage(gomock.Any(), reprocessDeploymentsTypeCond).Times(1).Return(nil) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeployments).Times(1).Return(nil) + connB.EXPECT().InjectMessage(gomock.Any(), isReprocessDeployments).Times(1).Return(nil) - testLoop.reprocessImagesAndResyncDeployments(0, reprocessFuncUpdate, nil) + testLoop.reprocessImagesAndResyncDeployments(0, v1ReprocessUpdate, nil) }) t.Run("skip some messages when are broken clusters", func(t *testing.T) { testLoop, connA, connB := newReprocessorLoop(t) - // Cluster "a" is healthy, expect all applicable images to be sent. - connA.EXPECT().InjectMessage(gomock.Any(), updatedImageTypeCond).Times(len(imgs) / 2).Return(nil) - // Cluster "b" is not healthy, expect at MOST imageReprocessorSemaphoreSize attempted messages. - connB.EXPECT().InjectMessage(gomock.Any(), updatedImageTypeCond).MaxTimes(int(imageReprocessorSemaphoreSize)).Return(errors.New("broken")) + connA.EXPECT().HasCapability(centralsensor.TargetedImageCacheInvalidation).AnyTimes().Return(false) + connB.EXPECT().HasCapability(centralsensor.TargetedImageCacheInvalidation).AnyTimes().Return(false) - connA.EXPECT().InjectMessage(gomock.Any(), reprocessDeploymentsTypeCond).Times(1).Return(nil) - // No reprocess deployments message is sent due to previous failures. - connB.EXPECT().InjectMessage(gomock.Any(), reprocessDeploymentsTypeCond).Times(0).Return(nil) + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(len(imgs) / 2).Return(nil) + connB.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).MaxTimes(int(imageReprocessorSemaphoreSize)).Return(errors.New("broken")) - testLoop.reprocessImagesAndResyncDeployments(0, reprocessFuncUpdate, nil) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeployments).Times(1).Return(nil) + connB.EXPECT().InjectMessage(gomock.Any(), isReprocessDeployments).Times(0).Return(nil) + + testLoop.reprocessImagesAndResyncDeployments(0, v1ReprocessUpdate, nil) }) } func TestReprocessImagesV2AndResyncDeployments_SkipBrokenSensor(t *testing.T) { testutils.MustUpdateFeature(t, features.FlattenImageData, true) - imgs := []*storage.ImageV2{} + imgs := make([]*storage.ImageV2, 0, (imageReprocessorSemaphoreSize+1)*2) for _, cluster := range []string{"a", "b"} { // two clusters // Create at least one more image than max semaphore size to ensure skip logic is executed. for i := range imageReprocessorSemaphoreSize + 1 { @@ -604,10 +665,11 @@ func TestReprocessImagesV2AndResyncDeployments_SkipBrokenSensor(t *testing.T) { } } - containerImageViews := []*views.ContainerImageView{} + containerImageViews := make([]*views.ContainerImageView, 0, len(imgs)) for _, img := range imgs { containerImageViews = append(containerImageViews, &views.ContainerImageView{ - ImageIDV2: img.GetId(), + ImageIDV2: img.GetId(), + ImageNameFullName: fmt.Sprintf("docker.io/library/%s:latest", img.GetId()), // Last character of image ID is the cluster. ClusterIDs: []string{img.GetId()[len(img.GetId())-1:]}, }) @@ -651,43 +713,34 @@ func TestReprocessImagesV2AndResyncDeployments_SkipBrokenSensor(t *testing.T) { return testLoop, connA, connB } - reprocessFuncUpdate := func(_ context.Context, _ imageEnricher.EnrichmentContext, _ *storage.ImageV2) (imageEnricher.EnrichmentResult, error) { - return imageEnricher.EnrichmentResult{ImageUpdated: true}, nil - } - updatedImageTypeCond := gomock.Cond(func(msg *central.MsgToSensor) bool { - return event.GetEventTypeWithoutPrefix(msg.GetMsg()) == "UpdatedImage" - }) - reprocessDeploymentsTypeCond := gomock.Cond(func(msg *central.MsgToSensor) bool { - return event.GetEventTypeWithoutPrefix(msg.GetMsg()) == "ReprocessDeployments" - }) - t.Run("send all messages when clusters are healthy", func(t *testing.T) { testLoop, connA, connB := newReprocessorLoop(t) - // Expect every updated image to be sent. - connA.EXPECT().InjectMessage(gomock.Any(), updatedImageTypeCond).Times(len(imgs) / 2).Return(nil) - connB.EXPECT().InjectMessage(gomock.Any(), updatedImageTypeCond).Times(len(imgs) / 2).Return(nil) + connA.EXPECT().HasCapability(centralsensor.TargetedImageCacheInvalidation).AnyTimes().Return(false) + connB.EXPECT().HasCapability(centralsensor.TargetedImageCacheInvalidation).AnyTimes().Return(false) + + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(len(imgs) / 2).Return(nil) + connB.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(len(imgs) / 2).Return(nil) - // Expect each cluster to be sent a reprocess deployments message. - connA.EXPECT().InjectMessage(gomock.Any(), reprocessDeploymentsTypeCond).Times(1).Return(nil) - connB.EXPECT().InjectMessage(gomock.Any(), reprocessDeploymentsTypeCond).Times(1).Return(nil) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeployments).Times(1).Return(nil) + connB.EXPECT().InjectMessage(gomock.Any(), isReprocessDeployments).Times(1).Return(nil) - testLoop.reprocessImagesV2AndResyncDeployments(0, reprocessFuncUpdate, search.EmptyQuery()) + testLoop.reprocessImagesV2AndResyncDeployments(0, v2ReprocessUpdate, search.EmptyQuery()) }) t.Run("skip some messages when are broken clusters", func(t *testing.T) { testLoop, connA, connB := newReprocessorLoop(t) - // Cluster "a" is healthy, expect all applicable images to be sent. - connA.EXPECT().InjectMessage(gomock.Any(), updatedImageTypeCond).Times(len(imgs) / 2).Return(nil) - // Cluster "b" is not healthy, expect at MOST imageReprocessorSemaphoreSize attempted messages. - connB.EXPECT().InjectMessage(gomock.Any(), updatedImageTypeCond).MaxTimes(int(imageReprocessorSemaphoreSize)).Return(errors.New("broken")) + connA.EXPECT().HasCapability(centralsensor.TargetedImageCacheInvalidation).AnyTimes().Return(false) + connB.EXPECT().HasCapability(centralsensor.TargetedImageCacheInvalidation).AnyTimes().Return(false) - connA.EXPECT().InjectMessage(gomock.Any(), reprocessDeploymentsTypeCond).Times(1).Return(nil) - // No reprocess deployments message is sent due to previous failures. - connB.EXPECT().InjectMessage(gomock.Any(), reprocessDeploymentsTypeCond).Times(0).Return(nil) + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(len(imgs) / 2).Return(nil) + connB.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).MaxTimes(int(imageReprocessorSemaphoreSize)).Return(errors.New("broken")) - testLoop.reprocessImagesV2AndResyncDeployments(0, reprocessFuncUpdate, search.EmptyQuery()) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeployments).Times(1).Return(nil) + connB.EXPECT().InjectMessage(gomock.Any(), isReprocessDeployments).Times(0).Return(nil) + + testLoop.reprocessImagesV2AndResyncDeployments(0, v2ReprocessUpdate, search.EmptyQuery()) }) } @@ -736,3 +789,214 @@ func TestInjectMessage(t *testing.T) { require.NoError(t, err) }) } + +func TestReprocessImagesAndResyncDeployments_WithCapability(t *testing.T) { + testutils.MustUpdateFeature(t, features.FlattenImageData, false) + + imgUpdated := &storage.Image{Id: "img-updated-a"} + imgUnchanged := &storage.Image{Id: "img-unchanged-a"} + + resultsUpdated := search.Result{ + ID: imgUpdated.GetId(), + Matches: map[string][]string{imageClusterIDFieldPath: {"a"}}, + } + resultsUnchanged := search.Result{ + ID: imgUnchanged.GetId(), + Matches: map[string][]string{imageClusterIDFieldPath: {"a"}}, + } + + t.Run("updated image sends UpdatedImage only", func(t *testing.T) { + ctrl := gomock.NewController(t) + connA, connManager := newConnWithCapability(ctrl) + imageDS := mockImageDataStore.NewMockDataStore(ctrl) + riskManager := riskManagerMocks.NewMockManager(ctrl) + + imageDS.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{resultsUpdated}, nil) + imageDS.EXPECT().GetImage(gomock.Any(), imgUpdated.GetId()).AnyTimes().Return(imgUpdated, true, nil) + riskManager.EXPECT().CalculateRiskAndUpsertImage(gomock.Any()).AnyTimes().Return(nil) + + connA.EXPECT().InjectMessage(gomock.Any(), isInvalidateImageCache).Times(0) + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(1).Return(nil) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeploymentsSkipFlush).Times(1).Return(nil) + + testLoop := &loopImpl{ + images: imageDS, + risk: riskManager, + connManager: connManager, + stopSig: concurrency.NewSignal(), + } + testLoop.reprocessImagesAndResyncDeployments(imageEnricher.UseCachesIfPossible, v1ReprocessUpdate, nil) + }) + + t.Run("unchanged image sends RefreshImageCacheTTL only", func(t *testing.T) { + ctrl := gomock.NewController(t) + connA, connManager := newConnWithCapability(ctrl) + imageDS := mockImageDataStore.NewMockDataStore(ctrl) + + imageDS.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{resultsUnchanged}, nil) + imageDS.EXPECT().GetImage(gomock.Any(), imgUnchanged.GetId()).AnyTimes().Return(imgUnchanged, true, nil) + + connA.EXPECT().InjectMessage(gomock.Any(), isRefreshImageCacheTTL).Times(1).Return(nil) + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(0) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeploymentsSkipFlush).Times(1).Return(nil) + + testLoop := &loopImpl{ + images: imageDS, + connManager: connManager, + stopSig: concurrency.NewSignal(), + } + testLoop.reprocessImagesAndResyncDeployments(imageEnricher.UseCachesIfPossible, v1ReprocessNoUpdate, nil) + }) + + t.Run("periodic path skips InvalidateImageCache and flushes", func(t *testing.T) { + ctrl := gomock.NewController(t) + connA, connManager := newConnWithCapability(ctrl) + imageDS := mockImageDataStore.NewMockDataStore(ctrl) + riskManager := riskManagerMocks.NewMockManager(ctrl) + + imageDS.EXPECT().Search(gomock.Any(), gomock.Any()).Return([]search.Result{resultsUpdated}, nil) + imageDS.EXPECT().GetImage(gomock.Any(), imgUpdated.GetId()).AnyTimes().Return(imgUpdated, true, nil) + riskManager.EXPECT().CalculateRiskAndUpsertImage(gomock.Any()).AnyTimes().Return(nil) + + connA.EXPECT().InjectMessage(gomock.Any(), isInvalidateImageCache).Times(0) + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(1).Return(nil) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeploymentsDoFlush).Times(1).Return(nil) + + testLoop := &loopImpl{ + images: imageDS, + risk: riskManager, + connManager: connManager, + stopSig: concurrency.NewSignal(), + } + testLoop.reprocessImagesAndResyncDeployments(imageEnricher.ForceRefetchCachedValuesOnly, v1ReprocessUpdate, nil) + }) + + t.Run("broken cluster with capability skips after first error", func(t *testing.T) { + ctrl := gomock.NewController(t) + connA, connManager := newConnWithCapability(ctrl) + imageDS := mockImageDataStore.NewMockDataStore(ctrl) + riskManager := riskManagerMocks.NewMockManager(ctrl) + + imgs := make([]*storage.Image, 0, imageReprocessorSemaphoreSize+1) + results := make([]search.Result, 0, imageReprocessorSemaphoreSize+1) + for i := range imageReprocessorSemaphoreSize + 1 { + img := &storage.Image{Id: fmt.Sprintf("img%d-a", i)} + imgs = append(imgs, img) + results = append(results, search.Result{ + ID: img.GetId(), + Matches: map[string][]string{imageClusterIDFieldPath: {"a"}}, + }) + } + + imageDS.EXPECT().Search(gomock.Any(), gomock.Any()).Return(results, nil) + for _, img := range imgs { + imageDS.EXPECT().GetImage(gomock.Any(), img.GetId()).AnyTimes().Return(img, true, nil) + } + riskManager.EXPECT().CalculateRiskAndUpsertImage(gomock.Any()).AnyTimes().Return(nil) + + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).MaxTimes(int(imageReprocessorSemaphoreSize)).Return(errors.New("broken")) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeployments).Times(0) + + testLoop := &loopImpl{ + images: imageDS, + risk: riskManager, + connManager: connManager, + stopSig: concurrency.NewSignal(), + } + testLoop.reprocessImagesAndResyncDeployments(0, v1ReprocessUpdate, nil) + }) +} + +func TestReprocessImagesV2AndResyncDeployments_WithCapability(t *testing.T) { + testutils.MustUpdateFeature(t, features.FlattenImageData, true) + + imgUpdated := &storage.ImageV2{Id: "img-updated-a"} + imgUnchanged := &storage.ImageV2{Id: "img-unchanged-a"} + + viewUpdated := &views.ContainerImageView{ + ImageIDV2: imgUpdated.GetId(), + ClusterIDs: []string{"a"}, + } + viewUnchanged := &views.ContainerImageView{ + ImageIDV2: imgUnchanged.GetId(), + ClusterIDs: []string{"a"}, + } + + t.Run("updated image sends UpdatedImage only", func(t *testing.T) { + ctrl := gomock.NewController(t) + connA, connManager := newConnWithCapability(ctrl) + connManager.EXPECT().AllSensorsHaveCapability(gomock.Any()).AnyTimes().Return(false) + deploymentDS := mockDeploymentDataStore.NewMockDataStore(ctrl) + imageDS := mockImageV2DataStore.NewMockDataStore(ctrl) + riskManager := riskManagerMocks.NewMockManager(ctrl) + + deploymentDS.EXPECT().GetContainerImageViews(gomock.Any(), gomock.Any()).Return([]*views.ContainerImageView{viewUpdated}, nil) + imageDS.EXPECT().GetImage(gomock.Any(), imgUpdated.GetId()).AnyTimes().Return(imgUpdated, true, nil) + imageDS.EXPECT().GetImageNames(gomock.Any(), gomock.Any()).AnyTimes().Return([]*storage.ImageName{imgUpdated.GetName()}, nil) + riskManager.EXPECT().CalculateRiskAndUpsertImageV2(gomock.Any()).AnyTimes().Return(nil) + + connA.EXPECT().InjectMessage(gomock.Any(), isInvalidateImageCache).Times(0) + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(1).Return(nil) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeploymentsSkipFlush).Times(1).Return(nil) + + testLoop := &loopImpl{ + deployments: deploymentDS, + imagesV2: imageDS, + risk: riskManager, + connManager: connManager, + stopSig: concurrency.NewSignal(), + } + testLoop.reprocessImagesV2AndResyncDeployments(imageEnricher.UseCachesIfPossible, v2ReprocessUpdate, search.EmptyQuery()) + }) + + t.Run("unchanged image sends RefreshImageCacheTTL only", func(t *testing.T) { + ctrl := gomock.NewController(t) + connA, connManager := newConnWithCapability(ctrl) + connManager.EXPECT().AllSensorsHaveCapability(gomock.Any()).AnyTimes().Return(false) + deploymentDS := mockDeploymentDataStore.NewMockDataStore(ctrl) + imageDS := mockImageV2DataStore.NewMockDataStore(ctrl) + + deploymentDS.EXPECT().GetContainerImageViews(gomock.Any(), gomock.Any()).Return([]*views.ContainerImageView{viewUnchanged}, nil) + imageDS.EXPECT().GetImage(gomock.Any(), imgUnchanged.GetId()).AnyTimes().Return(imgUnchanged, true, nil) + imageDS.EXPECT().GetImageNames(gomock.Any(), gomock.Any()).AnyTimes().Return([]*storage.ImageName{imgUnchanged.GetName()}, nil) + + connA.EXPECT().InjectMessage(gomock.Any(), isRefreshImageCacheTTL).Times(1).Return(nil) + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(0) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeploymentsSkipFlush).Times(1).Return(nil) + + testLoop := &loopImpl{ + deployments: deploymentDS, + imagesV2: imageDS, + connManager: connManager, + stopSig: concurrency.NewSignal(), + } + testLoop.reprocessImagesV2AndResyncDeployments(imageEnricher.UseCachesIfPossible, v2ReprocessNoUpdate, search.EmptyQuery()) + }) + + t.Run("periodic path skips InvalidateImageCache and flushes", func(t *testing.T) { + ctrl := gomock.NewController(t) + connA, connManager := newConnWithCapability(ctrl) + connManager.EXPECT().AllSensorsHaveCapability(gomock.Any()).AnyTimes().Return(false) + deploymentDS := mockDeploymentDataStore.NewMockDataStore(ctrl) + imageDS := mockImageV2DataStore.NewMockDataStore(ctrl) + riskManager := riskManagerMocks.NewMockManager(ctrl) + + deploymentDS.EXPECT().GetContainerImageViews(gomock.Any(), gomock.Any()).Return([]*views.ContainerImageView{viewUpdated}, nil) + imageDS.EXPECT().GetImage(gomock.Any(), imgUpdated.GetId()).AnyTimes().Return(imgUpdated, true, nil) + imageDS.EXPECT().GetImageNames(gomock.Any(), gomock.Any()).AnyTimes().Return([]*storage.ImageName{imgUpdated.GetName()}, nil) + riskManager.EXPECT().CalculateRiskAndUpsertImageV2(gomock.Any()).AnyTimes().Return(nil) + + connA.EXPECT().InjectMessage(gomock.Any(), isInvalidateImageCache).Times(0) + connA.EXPECT().InjectMessage(gomock.Any(), isUpdatedImage).Times(1).Return(nil) + connA.EXPECT().InjectMessage(gomock.Any(), isReprocessDeploymentsDoFlush).Times(1).Return(nil) + + testLoop := &loopImpl{ + deployments: deploymentDS, + imagesV2: imageDS, + risk: riskManager, + connManager: connManager, + stopSig: concurrency.NewSignal(), + } + testLoop.reprocessImagesV2AndResyncDeployments(imageEnricher.ForceRefetchCachedValuesOnly, v2ReprocessUpdate, search.EmptyQuery()) + }) +} diff --git a/central/resourcecollection/datastore/datastore.go b/central/resourcecollection/datastore/datastore.go index 62957394d09b9..ae69ac2f0339d 100644 --- a/central/resourcecollection/datastore/datastore.go +++ b/central/resourcecollection/datastore/datastore.go @@ -60,7 +60,7 @@ var ( // GetSupportedFieldLabels returns a list of the supported search.FieldLabel values for resolving deployments for a collection func GetSupportedFieldLabels() []pkgSearch.FieldLabel { - var ret []pkgSearch.FieldLabel + ret := make([]pkgSearch.FieldLabel, 0, len(supportedFieldNames)) for _, label := range supportedFieldNames { ret = append(ret, label.fieldLabel) } diff --git a/central/resourcecollection/datastore/datastore_bench_postgres_test.go b/central/resourcecollection/datastore/datastore_bench_postgres_test.go index 1770ce40eec46..7a5ea8d4bef72 100644 --- a/central/resourcecollection/datastore/datastore_bench_postgres_test.go +++ b/central/resourcecollection/datastore/datastore_bench_postgres_test.go @@ -27,7 +27,7 @@ func BenchmarkCollections(b *testing.B) { ids := make([]string, 0, numSeedObjects) collections := make([]*storage.ResourceCollection, 0, numSeedObjects) - for i := 0; i < numSeedObjects; i++ { + for i := range numSeedObjects { name := fmt.Sprintf("%d", i) collections = append(collections, getTestCollection(name, nil)) id, err := datastore.AddCollection(ctx, collections[i]) diff --git a/central/resourcecollection/datastore/datastore_impl.go b/central/resourcecollection/datastore/datastore_impl.go index 912fc95856ab4..30cb87cd1f876 100644 --- a/central/resourcecollection/datastore/datastore_impl.go +++ b/central/resourcecollection/datastore/datastore_impl.go @@ -661,7 +661,3 @@ func (c *ResourceCollectionSearchResultConverter) BuildLocation(result *pkgSearc func (c *ResourceCollectionSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_COLLECTIONS } - -func (c *ResourceCollectionSearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/resourcecollection/datastore/datastore_impl_postgres_test.go b/central/resourcecollection/datastore/datastore_impl_postgres_test.go index 07414627b1aa3..506e1d121b2ad 100644 --- a/central/resourcecollection/datastore/datastore_impl_postgres_test.go +++ b/central/resourcecollection/datastore/datastore_impl_postgres_test.go @@ -5,6 +5,7 @@ package datastore import ( "context" "fmt" + "slices" "testing" "time" @@ -113,8 +114,8 @@ func (s *CollectionPostgresDataStoreTestSuite) TestGraphInit() { protoassert.ElementsMatch(s.T(), objs, batch) // clean up data - for i := len(objIDs) - 1; i >= 0; i-- { - assert.NoError(s.T(), s.datastore.DeleteCollection(ctx, objIDs[i])) + for _, objID := range slices.Backward(objIDs) { + assert.NoError(s.T(), s.datastore.DeleteCollection(ctx, objID)) } assert.NoError(s.T(), resetLocalGraph(s.datastore.(*datastoreImpl))) }) @@ -542,9 +543,10 @@ func (s *CollectionPostgresDataStoreTestSuite) TestVerifyCollectionConstraints() func (s *CollectionPostgresDataStoreTestSuite) TestCollectionToQueries() { - var supportedLabelRules []*storage.SelectorRule + supportedFieldLabels := GetSupportedFieldLabels() + supportedLabelRules := make([]*storage.SelectorRule, 0, len(supportedFieldLabels)) - for _, label := range GetSupportedFieldLabels() { + for _, label := range supportedFieldLabels { supportedLabelRules = append(supportedLabelRules, &storage.SelectorRule{ FieldName: label.String(), Operator: storage.BooleanOperator_OR, diff --git a/central/risk/getters/alert.go b/central/risk/getters/alert.go index 7f47b83915bef..770ec3d8efe60 100644 --- a/central/risk/getters/alert.go +++ b/central/risk/getters/alert.go @@ -3,6 +3,7 @@ package getters import ( "context" + alertviews "github.com/stackrox/rox/central/alert/views" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" ) @@ -10,4 +11,5 @@ import ( // AlertSearcher provides the required access to alerts for risk scoring. type AlertSearcher interface { SearchListAlerts(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*storage.ListAlert, error) + SearchAlertPolicyNamesAndSeverities(ctx context.Context, q *v1.Query, excludeResolved bool) ([]*alertviews.PolicyNameAndSeverity, error) } diff --git a/central/risk/getters/alert_mock.go b/central/risk/getters/alert_mock.go index 4af4f46599f2e..af60509c983f0 100644 --- a/central/risk/getters/alert_mock.go +++ b/central/risk/getters/alert_mock.go @@ -4,6 +4,7 @@ import ( "context" "strings" + alertviews "github.com/stackrox/rox/central/alert/views" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/search" @@ -31,3 +32,24 @@ func (m MockAlertsSearcher) SearchListAlerts(_ context.Context, q *v1.Query, _ b } return } + +// SearchAlertPolicyNamesAndSeverities implements the AlertSearcher interface +func (m MockAlertsSearcher) SearchAlertPolicyNamesAndSeverities(_ context.Context, q *v1.Query, _ bool) (results []*alertviews.PolicyNameAndSeverity, err error) { + state := storage.ViolationState_ACTIVE.String() + search.ApplyFnToAllBaseQueries(q, func(bq *v1.BaseQuery) { + mfQ, ok := bq.GetQuery().(*v1.BaseQuery_MatchFieldQuery) + if ok && mfQ.MatchFieldQuery.GetField() == search.ViolationState.String() { + state = mfQ.MatchFieldQuery.GetValue() + } + }) + + for _, a := range m.Alerts { + if a.GetState().String() == strings.Trim(state, "\"") { + results = append(results, &alertviews.PolicyNameAndSeverity{ + PolicyName: a.GetPolicy().GetName(), + Severity: int(a.GetPolicy().GetSeverity()), + }) + } + } + return +} diff --git a/central/risk/manager/manager.go b/central/risk/manager/manager.go index 9d071d452f64a..052cda4ea5169 100644 --- a/central/risk/manager/manager.go +++ b/central/risk/manager/manager.go @@ -356,7 +356,12 @@ func (e *managerImpl) reprocessImageComponentRisk(imageComponent *storage.Embedd } imageComponent.RiskScore = risk.GetScore() - // skip direct upsert here since it is handled during image upsert + // Update the in-memory ranker so that downstream updateComponentRisk + // in the imagev2 datastore reads the correct score before DB write. + e.imageComponentRanker.Add( + scancomponent.ComponentIDV2(imageComponent, imageID, componentIndex), + risk.GetScore(), + ) } // reprocessNodeComponentRisk will reprocess risk of node components and save the results. @@ -370,7 +375,12 @@ func (e *managerImpl) reprocessNodeComponentRisk(nodeComponent *storage.Embedded } nodeComponent.RiskScore = risk.GetScore() - // skip direct upsert here since it is handled during node upsert + // Update the in-memory ranker so that downstream updateComponentRisk + // in the node datastore reads the correct score before DB write. + e.nodeComponentRanker.Add( + scancomponent.ComponentID(nodeComponent.GetName(), nodeComponent.GetVersion(), os), + risk.GetScore(), + ) } func (e *managerImpl) updateNamespaceRisk(nsID string, oldDeploymentScore float32, newDeploymentScore float32) { diff --git a/central/risk/manager/manager_test.go b/central/risk/manager/manager_test.go index ca2f373bfe91f..e446292908034 100644 --- a/central/risk/manager/manager_test.go +++ b/central/risk/manager/manager_test.go @@ -1,6 +1,7 @@ package manager import ( + "context" "fmt" "testing" @@ -18,6 +19,7 @@ import ( "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/images/integration" integrationSetMocks "github.com/stackrox/rox/pkg/images/integration/mocks" + "github.com/stackrox/rox/pkg/scancomponent" scannerSetMocks "github.com/stackrox/rox/pkg/scanners/mocks" "github.com/stackrox/rox/pkg/scanners/types" scannerTypesMocks "github.com/stackrox/rox/pkg/scanners/types/mocks" @@ -386,3 +388,92 @@ func TestReprocessDeploymentRiskUsesCorrectImageID(t *testing.T) { }) } } + +// stubImageComponentScorer returns a fixed risk score for any component. +type stubImageComponentScorer struct { + score float32 +} + +func (s *stubImageComponentScorer) Score(_ context.Context, sc scancomponent.ScanComponent, _ string, imageComponent *storage.EmbeddedImageScanComponent, imageID string, index int) *storage.Risk { + componentID := scancomponent.ComponentIDV2(imageComponent, imageID, index) + return &storage.Risk{ + Score: s.score, + Results: []*storage.Risk_Result{ + {Name: "stub", Score: s.score}, + }, + Subject: &storage.RiskSubject{ + Id: componentID, + Type: storage.RiskSubjectType_IMAGE_COMPONENT, + }, + } +} + +// stubNodeComponentScorer returns a fixed risk score for any component. +type stubNodeComponentScorer struct { + score float32 +} + +func (s *stubNodeComponentScorer) Score(_ context.Context, _ scancomponent.ScanComponent, _ string) *storage.Risk { + return &storage.Risk{ + Score: s.score, + Results: []*storage.Risk_Result{ + {Name: "stub", Score: s.score}, + }, + Subject: &storage.RiskSubject{ + Type: storage.RiskSubjectType_NODE_COMPONENT, + }, + } +} + +func TestReprocessImageComponentRiskUpdatesRanker(t *testing.T) { + ranker := ranking.NewRanker() + mgr := &managerImpl{ + imageComponentScorer: &stubImageComponentScorer{score: 3.5}, + imageComponentRanker: ranker, + } + + component := &storage.EmbeddedImageScanComponent{ + Name: "openssl", + Version: "1.1.1", + Vulns: []*storage.EmbeddedVulnerability{ + {Cve: "CVE-2021-1234", Cvss: 7.5}, + }, + } + imageID := "sha256:abc123" + componentIndex := 0 + + mgr.reprocessImageComponentRisk(component, "linux", imageID, componentIndex) + + assert.InDelta(t, 3.5, component.GetRiskScore(), 0.001, "component RiskScore should be set") + + expectedID := scancomponent.ComponentIDV2(component, imageID, componentIndex) + rankerScore := ranker.GetScoreForID(expectedID) + assert.InDelta(t, 3.5, rankerScore, 0.001, + "ranker must be updated with the calculated score so downstream updateComponentRisk does not clobber it") +} + +func TestReprocessNodeComponentRiskUpdatesRanker(t *testing.T) { + ranker := ranking.NewRanker() + mgr := &managerImpl{ + nodeComponentScorer: &stubNodeComponentScorer{score: 2.8}, + nodeComponentRanker: ranker, + } + + component := &storage.EmbeddedNodeScanComponent{ + Name: "kernel", + Version: "5.4.0", + Vulns: []*storage.EmbeddedVulnerability{ + {Cve: "CVE-2022-5678", Cvss: 6.0}, + }, + } + os := "linux" + + mgr.reprocessNodeComponentRisk(component, os) + + assert.InDelta(t, 2.8, component.GetRiskScore(), 0.001, "component RiskScore should be set") + + expectedID := scancomponent.ComponentID(component.GetName(), component.GetVersion(), os) + rankerScore := ranker.GetScoreForID(expectedID) + assert.InDelta(t, 2.8, rankerScore, 0.001, + "ranker must be updated with the calculated score so downstream updateComponentRisk does not clobber it") +} diff --git a/central/risk/multipliers/deployment/process_baseline_violations.go b/central/risk/multipliers/deployment/process_baseline_violations.go index 2ad12013de856..06907a8a2d170 100644 --- a/central/risk/multipliers/deployment/process_baseline_violations.go +++ b/central/risk/multipliers/deployment/process_baseline_violations.go @@ -2,6 +2,7 @@ package deployment import ( "context" + "strconv" "strings" "github.com/stackrox/rox/central/processbaseline/evaluator" @@ -9,11 +10,12 @@ import ( "github.com/stackrox/rox/central/risk/multipliers" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/env" - "github.com/stackrox/rox/pkg/stringutils" ) const ( - processBaselineHeading = `Suspicious Process Executions` + // ProcessBaselineHeading is the risk result name for process baseline violations. + // Exported for use in risk data sanitization. + ProcessBaselineHeading = `Suspicious Process Executions` processBaselineSaturation = 10 processBaselineValue = 4 @@ -56,13 +58,19 @@ func NewProcessBaselines(evaluator evaluator.Evaluator) Multiplier { } } -func formatProcess(process *views.ProcessIndicatorRiskView) string { +// FormatProcess formats a process indicator into a human-readable message for risk factors. +// Exported for use in tests to ensure sanitization regex stays in sync with format changes. +func FormatProcess(process *views.ProcessIndicatorRiskView) string { sb := strings.Builder{} - stringutils.WriteStringf(&sb, "Detected execution of suspicious process %q", process.SignalName) + sb.Grow(128) + sb.WriteString("Detected execution of suspicious process ") + sb.WriteString(strconv.Quote(process.SignalName)) if len(process.SignalArgs) > 0 { - stringutils.WriteStringf(&sb, " with args %q", process.SignalArgs) + sb.WriteString(" with args ") + sb.WriteString(strconv.Quote(process.SignalArgs)) } - stringutils.WriteStrings(&sb, " in container ", process.ContainerName) + sb.WriteString(" in container ") + sb.WriteString(process.ContainerName) return sb.String() } @@ -78,13 +86,13 @@ func (p *processBaselineMultiplier) Score(_ context.Context, deployment *storage scorer := newScorer() riskResult := &storage.Risk_Result{ - Name: processBaselineHeading, + Name: ProcessBaselineHeading, } for _, process := range violatingProcesses { scorer.addProcess() riskResult.Factors = append(riskResult.Factors, &storage.Risk_Result_Factor{ - Message: formatProcess(process), + Message: FormatProcess(process), }) } diff --git a/central/risk/multipliers/deployment/process_baseline_violations_test.go b/central/risk/multipliers/deployment/process_baseline_violations_test.go index 1d47bdec6db7d..f5cede11106e1 100644 --- a/central/risk/multipliers/deployment/process_baseline_violations_test.go +++ b/central/risk/multipliers/deployment/process_baseline_violations_test.go @@ -44,7 +44,7 @@ func TestProcessBaselines(t *testing.T) { }, }, expected: &storage.Risk_Result{ - Name: processBaselineHeading, + Name: ProcessBaselineHeading, Score: 1.6, Factors: []*storage.Risk_Result_Factor{ {Message: "Detected execution of suspicious process \"apt-get\" with args \"install nmap\" in container containerName"}, @@ -66,7 +66,7 @@ func TestProcessBaselines(t *testing.T) { }, }, expected: &storage.Risk_Result{ - Name: processBaselineHeading, + Name: ProcessBaselineHeading, Score: 2.14, Factors: []*storage.Risk_Result_Factor{ {Message: "Detected execution of suspicious process \"apt-get\" with args \"install nmap\" in container containerName"}, diff --git a/central/risk/multipliers/deployment/violations.go b/central/risk/multipliers/deployment/violations.go index f3b3e04a006d5..a4ce9404ab681 100644 --- a/central/risk/multipliers/deployment/violations.go +++ b/central/risk/multipliers/deployment/violations.go @@ -46,21 +46,19 @@ func NewViolations(searcher getters.AlertSearcher) *ViolationsMultiplier { func (v *ViolationsMultiplier) Score(ctx context.Context, deployment *storage.Deployment, _ map[string][]*storage.Risk_Result) *storage.Risk_Result { qb := search.NewQueryBuilder().AddExactMatches(search.DeploymentID, deployment.GetId()).AddExactMatches(search.ViolationState, storage.ViolationState_ACTIVE.String()) - alerts, err := v.searcher.SearchListAlerts(ctx, qb.ProtoQuery(), true) + results, err := v.searcher.SearchAlertPolicyNamesAndSeverities(ctx, qb.ProtoQuery(), true) if err != nil { log.Errorf("Couldn't get risk violations for %s: %s", deployment.GetId(), err) return nil } var severitySum float32 - var count int var factors []policyFactor - for _, alert := range alerts { - count++ - severitySum += severityImpact(alert.GetPolicy().GetSeverity()) + for _, result := range results { + severitySum += severityImpact(result.GetSeverity()) factors = append(factors, policyFactor{ - name: alert.GetPolicy().GetName(), - severity: alert.GetPolicy().GetSeverity(), + name: result.GetPolicyName(), + severity: result.GetSeverity(), }) } diff --git a/central/risk/multipliers/image/component_count_test.go b/central/risk/multipliers/image/component_count_test.go index 1de28bf0b3a77..aa0c6b396b26a 100644 --- a/central/risk/multipliers/image/component_count_test.go +++ b/central/risk/multipliers/image/component_count_test.go @@ -18,7 +18,7 @@ func TestComponentCountScore(t *testing.T) { // We need 14 components added for the count to be 15 image := multipliers.GetMockImages()[0] components := image.GetScan().GetComponents() - for i := 0; i < 14; i++ { + for i := range 14 { components = append(components, &storage.EmbeddedImageScanComponent{ Name: strconv.Itoa(i), Version: "1.0", @@ -44,7 +44,7 @@ func TestComponentCountScoreV2(t *testing.T) { // The image already has one unique component. So we need 14 components added for the count to be 15. image := multipliers.GetMockImagesV2()[0] components := image.GetScan().GetComponents() - for i := 0; i < 14; i++ { + for i := range 14 { components = append(components, &storage.EmbeddedImageScanComponent{ Name: strconv.Itoa(i), Version: "1.0", diff --git a/central/risk/multipliers/image/vulnerabilities.go b/central/risk/multipliers/image/vulnerabilities.go index f99dd7a3abb90..b37950fd5bfee 100644 --- a/central/risk/multipliers/image/vulnerabilities.go +++ b/central/risk/multipliers/image/vulnerabilities.go @@ -8,6 +8,7 @@ import ( "github.com/stackrox/rox/central/risk/scorer/vulns" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/scancomponent" + "github.com/stackrox/rox/pkg/set" ) const ( @@ -21,7 +22,7 @@ const ( // vulnerabilitiesMultiplier is a scorer for the vulnerabilities in an image type vulnerabilitiesMultiplier struct{} -// NewVulnerabilities provides a multiplier that scores the data based on the CVSS scores and number of CVEs +// NewVulnerabilities provides a multiplier that scores known-severity vulnerability findings across image components. func NewVulnerabilities() Multiplier { return &vulnerabilitiesMultiplier{} } @@ -43,8 +44,8 @@ func (c *vulnerabilitiesMultiplier) Score(_ context.Context, image *storage.Imag Name: VulnerabilitiesHeading, Factors: []*storage.Risk_Result_Factor{ { - Message: fmt.Sprintf("Image %q contains %d CVEs with severities ranging between %s and %s", - image.GetName().GetFullName(), num, min.Severity, max.Severity), + Message: fmt.Sprintf("Image %q contains %d distinct CVEs with severities ranging between %s and %s", + image.GetName().GetFullName(), countDistinctCVEs(imgComponents), min.Severity, max.Severity), }, }, Score: score, @@ -68,10 +69,23 @@ func (c *vulnerabilitiesMultiplier) ScoreV2(_ context.Context, image *storage.Im Name: VulnerabilitiesHeading, Factors: []*storage.Risk_Result_Factor{ { - Message: fmt.Sprintf("Image %q contains %d CVEs with severities ranging between %s and %s", - image.GetName().GetFullName(), num, min.Severity, max.Severity), + Message: fmt.Sprintf("Image %q contains %d distinct CVEs with severities ranging between %s and %s", + image.GetName().GetFullName(), countDistinctCVEs(imgComponents), min.Severity, max.Severity), }, }, Score: score, } } + +// countDistinctCVEs includes unknown-severity findings, which do not affect the risk score. +func countDistinctCVEs(components []*storage.EmbeddedImageScanComponent) int { + cves := set.NewStringSet() + for _, component := range components { + for _, vuln := range component.GetVulns() { + if cve := vuln.GetCve(); cve != "" { + cves.Add(cve) + } + } + } + return cves.Cardinality() +} diff --git a/central/risk/multipliers/image/vulnerabilities_test.go b/central/risk/multipliers/image/vulnerabilities_test.go index 959ac3af49969..5a7460316dda3 100644 --- a/central/risk/multipliers/image/vulnerabilities_test.go +++ b/central/risk/multipliers/image/vulnerabilities_test.go @@ -18,6 +18,7 @@ func TestVulnerabilitiesScore(t *testing.T) { images := multipliers.GetMockImages() result := mult.Score(ctx, images[0]) assert.Equal(t, float32(1.59025), result.GetScore()) + assert.Equal(t, "Image \"docker.io/library/nginx:1.10\" contains 2 distinct CVEs with severities ranging between Low and Critical", result.GetFactors()[0].GetMessage()) // Changing CVSS score should not affect result. images[0].GetScan().GetComponents()[0].GetVulns()[0].Cvss = 0 @@ -27,7 +28,14 @@ func TestVulnerabilitiesScore(t *testing.T) { result = mult.Score(ctx, images[0]) assert.Equal(t, float32(1.59025), result.GetScore()) - // Set both severity to unknown and then there should be a nil RiskResult + // Unknown-severity CVEs are displayed, but do not contribute to the score. + images[0].GetScan().GetComponents()[0].Vulns = append(images[0].GetScan().GetComponents()[0].GetVulns(), + &storage.EmbeddedVulnerability{Cve: "CVE-2019-0003", Severity: storage.VulnerabilitySeverity_UNKNOWN_VULNERABILITY_SEVERITY}) + result = mult.Score(ctx, images[0]) + assert.Equal(t, float32(1.59025), result.GetScore()) + assert.Equal(t, "Image \"docker.io/library/nginx:1.10\" contains 3 distinct CVEs with severities ranging between Low and Critical", result.GetFactors()[0].GetMessage()) + + // Only unknown-severity vulnerabilities remain, so there is no risk result. images[0].GetScan().GetComponents()[0].GetVulns()[0].Severity = storage.VulnerabilitySeverity_UNKNOWN_VULNERABILITY_SEVERITY images[0].GetScan().GetComponents()[0].GetVulns()[1].Severity = storage.VulnerabilitySeverity_UNKNOWN_VULNERABILITY_SEVERITY images[0].GetScan().GetComponents()[1].GetVulns()[0].Severity = storage.VulnerabilitySeverity_UNKNOWN_VULNERABILITY_SEVERITY @@ -44,6 +52,7 @@ func TestVulnerabilitiesScoreV2(t *testing.T) { images := multipliers.GetMockImagesV2() result := mult.ScoreV2(ctx, images[0]) assert.Equal(t, float32(1.59025), result.GetScore()) + assert.Equal(t, "Image \"docker.io/library/nginx:1.10\" contains 2 distinct CVEs with severities ranging between Low and Critical", result.GetFactors()[0].GetMessage()) // Changing CVSS score should not affect result. images[0].GetScan().GetComponents()[0].GetVulns()[0].Cvss = 0 @@ -53,7 +62,14 @@ func TestVulnerabilitiesScoreV2(t *testing.T) { result = mult.ScoreV2(ctx, images[0]) assert.Equal(t, float32(1.59025), result.GetScore()) - // Set both severity to unknown and then there should be a nil RiskResult + // Unknown-severity CVEs are displayed, but do not contribute to the score. + images[0].GetScan().GetComponents()[0].Vulns = append(images[0].GetScan().GetComponents()[0].GetVulns(), + &storage.EmbeddedVulnerability{Cve: "CVE-2019-0003", Severity: storage.VulnerabilitySeverity_UNKNOWN_VULNERABILITY_SEVERITY}) + result = mult.ScoreV2(ctx, images[0]) + assert.Equal(t, float32(1.59025), result.GetScore()) + assert.Equal(t, "Image \"docker.io/library/nginx:1.10\" contains 3 distinct CVEs with severities ranging between Low and Critical", result.GetFactors()[0].GetMessage()) + + // Only unknown-severity vulnerabilities remain, so there is no risk result. images[0].GetScan().GetComponents()[0].GetVulns()[0].Severity = storage.VulnerabilitySeverity_UNKNOWN_VULNERABILITY_SEVERITY images[0].GetScan().GetComponents()[0].GetVulns()[1].Severity = storage.VulnerabilitySeverity_UNKNOWN_VULNERABILITY_SEVERITY images[0].GetScan().GetComponents()[1].GetVulns()[0].Severity = storage.VulnerabilitySeverity_UNKNOWN_VULNERABILITY_SEVERITY diff --git a/central/risk/scorer/deployment/scorer_test.go b/central/risk/scorer/deployment/scorer_test.go index 59a5838e41802..faf919428da8e 100644 --- a/central/risk/scorer/deployment/scorer_test.go +++ b/central/risk/scorer/deployment/scorer_test.go @@ -62,7 +62,7 @@ func TestScore(t *testing.T) { { Name: imageMultiplier.VulnerabilitiesHeading, Factors: []*storage.Risk_Result_Factor{ - {Message: "Image \"docker.io/library/nginx:1.10\" contains 3 CVEs with severities ranging between Moderate and Critical"}, + {Message: "Image \"docker.io/library/nginx:1.10\" contains 3 distinct CVEs with severities ranging between Moderate and Critical"}, }, Score: 1.5535, }, diff --git a/central/risk/scorer/image/scorer_test.go b/central/risk/scorer/image/scorer_test.go index c89e474ce1f2c..902014711f510 100644 --- a/central/risk/scorer/image/scorer_test.go +++ b/central/risk/scorer/image/scorer_test.go @@ -28,7 +28,7 @@ func TestScore(t *testing.T) { { Name: imageMultiplier.VulnerabilitiesHeading, Factors: []*storage.Risk_Result_Factor{ - {Message: "Image \"docker.io/library/nginx:1.10\" contains 3 CVEs with severities ranging between Moderate and Critical"}, + {Message: "Image \"docker.io/library/nginx:1.10\" contains 3 distinct CVEs with severities ranging between Moderate and Critical"}, }, Score: 1.5535, }, @@ -67,7 +67,7 @@ func TestScoreV2(t *testing.T) { { Name: imageMultiplier.VulnerabilitiesHeading, Factors: []*storage.Risk_Result_Factor{ - {Message: "Image \"docker.io/library/nginx:1.10\" contains 3 CVEs with severities ranging between Moderate and Critical"}, + {Message: "Image \"docker.io/library/nginx:1.10\" contains 3 distinct CVEs with severities ranging between Moderate and Critical"}, }, Score: 1.5535, }, diff --git a/central/role/datastore/datastore.go b/central/role/datastore/datastore.go index 5ea32e1d500d3..d17aea8b02b20 100644 --- a/central/role/datastore/datastore.go +++ b/central/role/datastore/datastore.go @@ -20,6 +20,7 @@ type DataStore interface { AddRole(ctx context.Context, role *storage.Role) error UpdateRole(ctx context.Context, role *storage.Role) error RemoveRole(ctx context.Context, name string) error + RemoveFilteredRoles(ctx context.Context, filter func(*storage.Role) bool) (int, error) GetPermissionSet(ctx context.Context, id string) (*storage.PermissionSet, bool, error) GetAllPermissionSets(ctx context.Context) ([]*storage.PermissionSet, error) @@ -29,6 +30,7 @@ type DataStore interface { UpdatePermissionSet(ctx context.Context, permissionSet *storage.PermissionSet) error UpsertPermissionSet(ctx context.Context, permissionSet *storage.PermissionSet) error RemovePermissionSet(ctx context.Context, id string) error + RemoveFilteredPermissionSets(ctx context.Context, filter func(*storage.PermissionSet) bool) (int, error) GetAccessScope(ctx context.Context, id string) (*storage.SimpleAccessScope, bool, error) GetAllAccessScopes(ctx context.Context) ([]*storage.SimpleAccessScope, error) @@ -39,6 +41,7 @@ type DataStore interface { UpdateAccessScope(ctx context.Context, scope *storage.SimpleAccessScope) error UpsertAccessScope(ctx context.Context, scope *storage.SimpleAccessScope) error RemoveAccessScope(ctx context.Context, id string) error + RemoveFilteredAccessScopes(ctx context.Context, filter func(*storage.SimpleAccessScope) bool) (int, error) GetAllResolvedRoles(ctx context.Context) ([]permissions.ResolvedRole, error) GetAndResolveRole(ctx context.Context, name string) (permissions.ResolvedRole, error) diff --git a/central/role/datastore/datastore_impl.go b/central/role/datastore/datastore_impl.go index 9c9361fa238df..4ac95edc33db2 100644 --- a/central/role/datastore/datastore_impl.go +++ b/central/role/datastore/datastore_impl.go @@ -16,6 +16,7 @@ import ( "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/sync" ) @@ -62,8 +63,7 @@ func (ds *dataStoreImpl) UpsertRole(ctx context.Context, newRole *storage.Role) return err } - // Constraints ok, write the object. We expect the underlying store to - // verify there is no role with the same name. + // Constraints ok, upsert the object (create if new, update if exists). if err := ds.roleStorage.Upsert(ctx, newRole); err != nil { return err } @@ -95,8 +95,7 @@ func (ds *dataStoreImpl) UpsertPermissionSet(ctx context.Context, newPS *storage return err } - // Constraints ok, write the object. We expect the underlying store to - // verify there is no permission set with the same name. + // Constraints ok, upsert the object (create if new, update if exists). if err := ds.permissionSetStorage.Upsert(ctx, newPS); err != nil { return err } @@ -128,8 +127,7 @@ func (ds *dataStoreImpl) UpsertAccessScope(ctx context.Context, newScope *storag return err } - // Constraints ok, write the object. We expect the underlying store to - // verify there is no access scope with the same name. + // Constraints ok, upsert the object (create if new, update if exists). if err := ds.accessScopeStorage.Upsert(ctx, newScope); err != nil { return err } @@ -278,7 +276,14 @@ func (ds *dataStoreImpl) RemoveRole(ctx context.Context, name string) error { return err } - if err := ds.verifyRoleForDeletion(ctx, name); err != nil { + role, found, err := ds.roleStorage.Get(ctx, name) + if err != nil { + return err + } + if !found { + return errors.Wrapf(errox.NotFound, "name = %q", name) + } + if err := ds.verifyRoleForDeletion(ctx, role); err != nil { return err } @@ -856,15 +861,7 @@ func (ds *dataStoreImpl) verifyRoleNameExists(ctx context.Context, name string) // It will: // - verify that the role is not a default role // - verify that the role exists -func (ds *dataStoreImpl) verifyRoleForDeletion(ctx context.Context, name string) error { - role, found, err := ds.roleStorage.Get(ctx, name) - - if err != nil { - return err - } - if !found { - return errors.Wrapf(errox.NotFound, "name = %q", name) - } +func (ds *dataStoreImpl) verifyRoleForDeletion(ctx context.Context, role *storage.Role) error { if err := verifyRoleOrigin(ctx, role); err != nil { return err } @@ -937,3 +934,147 @@ func getGroupIDs(groups []*storage.Group) []string { } return groupIDs } + +//////////////////////////////////////////////////////////////////////////////// +// Removal of filtered objects // +// // + +func (ds *dataStoreImpl) RemoveFilteredRoles(ctx context.Context, filter func(*storage.Role) bool) (int, error) { + if err := sac.VerifyAuthzOK(roleSAC.WriteAllowed(ctx)); err != nil { + return 0, err + } + + ds.lock.Lock() + defer ds.lock.Unlock() + + rolesToDelete := make(map[string]*storage.Role) + walkFn := func() error { + return ds.roleStorage.Walk(ctx, func(role *storage.Role) error { + if filter(role) { + rolesToDelete[role.GetName()] = role + } + return nil + }) + } + if err := pgutils.RetryIfPostgres(ctx, walkFn); err != nil { + return 0, err + } + + for name, role := range rolesToDelete { + if err := ds.verifyRoleForDeletion(ctx, role); err != nil { + log.Debugf("Skipping role %q: %v", name, err) + delete(rolesToDelete, name) + } + } + + deletedCount := 0 + for name := range rolesToDelete { + if err := ds.roleStorage.Delete(ctx, name); err != nil { + log.Errorf("Failed to delete filtered role %q: %v", name, err) + } else { + deletedCount++ + } + } + + return deletedCount, nil +} + +func (ds *dataStoreImpl) RemoveFilteredPermissionSets(ctx context.Context, filter func(*storage.PermissionSet) bool) (int, error) { + if err := sac.VerifyAuthzOK(roleSAC.WriteAllowed(ctx)); err != nil { + return 0, err + } + + ds.lock.Lock() + defer ds.lock.Unlock() + + // First, identify permission sets that match the filter. + candidateSet := set.NewStringSet() + walkPSFn := func() error { + return ds.permissionSetStorage.Walk(ctx, func(permissionSet *storage.PermissionSet) error { + if filter(permissionSet) { + candidateSet.Add(permissionSet.GetId()) + } + return nil + }) + } + if err := pgutils.RetryIfPostgres(ctx, walkPSFn); err != nil { + return 0, err + } + + if candidateSet.Cardinality() == 0 { + return 0, nil + } + + // Walk through roles to find which candidate permission sets are referenced. + walkRolesFn := func() error { + return ds.roleStorage.Walk(ctx, func(role *storage.Role) error { + candidateSet.Remove(role.GetPermissionSetId()) + return nil + }) + } + if err := pgutils.RetryIfPostgres(ctx, walkRolesFn); err != nil { + return 0, err + } + + // Delete permission sets that are not referenced. + deletedCount := 0 + for id := range candidateSet { + if err := ds.permissionSetStorage.Delete(ctx, id); err != nil { + log.Errorf("Failed to delete filtered permission set %q: %v", id, err) + } else { + deletedCount++ + } + } + + return deletedCount, nil +} + +func (ds *dataStoreImpl) RemoveFilteredAccessScopes(ctx context.Context, filter func(*storage.SimpleAccessScope) bool) (int, error) { + if err := sac.VerifyAuthzOK(roleSAC.WriteAllowed(ctx)); err != nil { + return 0, err + } + + ds.lock.Lock() + defer ds.lock.Unlock() + + // First, identify access scopes that match the filter. + candidateSet := set.NewStringSet() + walkASFn := func() error { + return ds.accessScopeStorage.Walk(ctx, func(accessScope *storage.SimpleAccessScope) error { + if filter(accessScope) { + candidateSet.Add(accessScope.GetId()) + } + return nil + }) + } + if err := pgutils.RetryIfPostgres(ctx, walkASFn); err != nil { + return 0, err + } + + if candidateSet.Cardinality() == 0 { + return 0, nil + } + + // Walk through roles to find which candidate access scopes are referenced. + walkRolesFn := func() error { + return ds.roleStorage.Walk(ctx, func(role *storage.Role) error { + candidateSet.Remove(role.GetAccessScopeId()) + return nil + }) + } + if err := pgutils.RetryIfPostgres(ctx, walkRolesFn); err != nil { + return 0, err + } + + // Delete access scopes that are not referenced. + deletedCount := 0 + for id := range candidateSet { + if err := ds.accessScopeStorage.Delete(ctx, id); err != nil { + log.Errorf("Failed to delete filtered access scope %q: %v", id, err) + } else { + deletedCount++ + } + } + + return deletedCount, nil +} diff --git a/central/role/datastore/defaults.go b/central/role/datastore/defaults.go index f5ee36fc230dd..23801c6ce3dcc 100644 --- a/central/role/datastore/defaults.go +++ b/central/role/datastore/defaults.go @@ -143,7 +143,7 @@ var defaultPermissionSets = map[string]permSetAttributes{ func getDefaultRoles() []*storage.Role { roles := make([]*storage.Role, 0, len(defaultPermissionSets)) - for _, roleName := range accesscontrol.DefaultRoleNames.AsSlice() { + for roleName := range accesscontrol.DefaultRoleNames { attributes, found := defaultPermissionSets[roleName] if !found { utils.Should(errors.Errorf("Default role %s does not have permission set defined", roleName)) diff --git a/central/role/datastore/mocks/datastore.go b/central/role/datastore/mocks/datastore.go index d0896c976e5f5..e20b4cc6cb3f9 100644 --- a/central/role/datastore/mocks/datastore.go +++ b/central/role/datastore/mocks/datastore.go @@ -342,6 +342,51 @@ func (mr *MockDataStoreMockRecorder) RemoveAccessScope(ctx, id any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveAccessScope", reflect.TypeOf((*MockDataStore)(nil).RemoveAccessScope), ctx, id) } +// RemoveFilteredAccessScopes mocks base method. +func (m *MockDataStore) RemoveFilteredAccessScopes(ctx context.Context, filter func(*storage.SimpleAccessScope) bool) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveFilteredAccessScopes", ctx, filter) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveFilteredAccessScopes indicates an expected call of RemoveFilteredAccessScopes. +func (mr *MockDataStoreMockRecorder) RemoveFilteredAccessScopes(ctx, filter any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveFilteredAccessScopes", reflect.TypeOf((*MockDataStore)(nil).RemoveFilteredAccessScopes), ctx, filter) +} + +// RemoveFilteredPermissionSets mocks base method. +func (m *MockDataStore) RemoveFilteredPermissionSets(ctx context.Context, filter func(*storage.PermissionSet) bool) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveFilteredPermissionSets", ctx, filter) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveFilteredPermissionSets indicates an expected call of RemoveFilteredPermissionSets. +func (mr *MockDataStoreMockRecorder) RemoveFilteredPermissionSets(ctx, filter any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveFilteredPermissionSets", reflect.TypeOf((*MockDataStore)(nil).RemoveFilteredPermissionSets), ctx, filter) +} + +// RemoveFilteredRoles mocks base method. +func (m *MockDataStore) RemoveFilteredRoles(ctx context.Context, filter func(*storage.Role) bool) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveFilteredRoles", ctx, filter) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveFilteredRoles indicates an expected call of RemoveFilteredRoles. +func (mr *MockDataStoreMockRecorder) RemoveFilteredRoles(ctx, filter any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveFilteredRoles", reflect.TypeOf((*MockDataStore)(nil).RemoveFilteredRoles), ctx, filter) +} + // RemovePermissionSet mocks base method. func (m *MockDataStore) RemovePermissionSet(ctx context.Context, id string) error { m.ctrl.T.Helper() diff --git a/central/role/service/service_impl.go b/central/role/service/service_impl.go index 2e01f43ecfb70..abfe7437bdfc8 100644 --- a/central/role/service/service_impl.go +++ b/central/role/service/service_impl.go @@ -14,13 +14,13 @@ import ( "github.com/stackrox/rox/central/role/sachelper" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/accessscope" "github.com/stackrox/rox/pkg/auth/permissions" "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/grpc/authn" "github.com/stackrox/rox/pkg/grpc/authz" "github.com/stackrox/rox/pkg/grpc/authz/perrpc" "github.com/stackrox/rox/pkg/grpc/authz/user" + "github.com/stackrox/rox/pkg/sac/accessscope" "github.com/stackrox/rox/pkg/sac/effectiveaccessscope" "github.com/stackrox/rox/pkg/sac/resources" "google.golang.org/grpc" @@ -82,7 +82,10 @@ func (*serviceImpl) AuthFuncOverride(ctx context.Context, fullMethodName string) } func (s *serviceImpl) GetRoles(ctx context.Context, _ *v1.Empty) (*v1.GetRolesResponse, error) { - roles, err := s.roleDataStore.GetAllRoles(ctx) + roles, err := s.roleDataStore.GetRolesFiltered(ctx, func(role *storage.Role) bool { + // filter out dynamic roles created to back Rox tokens issued for internal purposes. + return role.GetTraits().GetOrigin() != storage.Traits_EPHEMERAL + }) if err != nil { return nil, errors.Wrap(err, "failed to retrieve roles") } @@ -117,6 +120,9 @@ func (s *serviceImpl) CreateRole(ctx context.Context, roleRequest *v1.CreateRole if role.GetName() != "" && role.GetName() != roleRequest.GetName() { return nil, errox.InvalidArgs.CausedBy("different role names in path and body") } + if role.GetTraits().GetOrigin() == storage.Traits_EPHEMERAL { + return nil, errox.InvalidArgs.CausedBy("dynamic roles can only be created by internal services") + } role.Name = roleRequest.GetName() err := s.roleDataStore.AddRole(ctx, role) @@ -127,6 +133,9 @@ func (s *serviceImpl) CreateRole(ctx context.Context, roleRequest *v1.CreateRole } func (s *serviceImpl) UpdateRole(ctx context.Context, role *storage.Role) (*v1.Empty, error) { + if role.GetTraits().GetOrigin() == storage.Traits_EPHEMERAL { + return nil, errox.InvalidArgs.CausedBy("dynamic roles cannot be modified by API") + } err := s.roleDataStore.UpdateRole(ctx, role) if err != nil { return nil, err @@ -183,7 +192,10 @@ func (s *serviceImpl) GetPermissionSet(ctx context.Context, id *v1.ResourceByID) } func (s *serviceImpl) ListPermissionSets(ctx context.Context, _ *v1.Empty) (*v1.ListPermissionSetsResponse, error) { - permissionSets, err := s.roleDataStore.GetAllPermissionSets(ctx) + permissionSets, err := s.roleDataStore.GetPermissionSetsFiltered(ctx, func(permissionSet *storage.PermissionSet) bool { + // filter out dynamic permission sets created to back Rox tokens issued for internal purposes. + return permissionSet.GetTraits().GetOrigin() != storage.Traits_EPHEMERAL + }) if err != nil { return nil, errors.Wrap(err, "failed to retrieve permission sets") } @@ -202,6 +214,10 @@ func (s *serviceImpl) PostPermissionSet(ctx context.Context, permissionSet *stor } permissionSet.Id = rolePkg.GeneratePermissionSetID() + if permissionSet.GetTraits().GetOrigin() == storage.Traits_EPHEMERAL { + return nil, errox.InvalidArgs.CausedBy("dynamic permission sets can only be created by internal services") + } + // Store the augmented permission set; report back on error. Note the // permission set is referenced by its name because that's what the caller // knows. @@ -215,6 +231,9 @@ func (s *serviceImpl) PostPermissionSet(ctx context.Context, permissionSet *stor } func (s *serviceImpl) PutPermissionSet(ctx context.Context, permissionSet *storage.PermissionSet) (*v1.Empty, error) { + if permissionSet.GetTraits().GetOrigin() == storage.Traits_EPHEMERAL { + return nil, errox.InvalidArgs.CausedBy("dynamic permission sets cannot be modified by API") + } err := s.roleDataStore.UpdatePermissionSet(ctx, permissionSet) if err != nil { return nil, errors.Wrapf(err, "failed to update permission set %s", permissionSet.GetId()) @@ -249,7 +268,10 @@ func (s *serviceImpl) GetSimpleAccessScope(ctx context.Context, id *v1.ResourceB } func (s *serviceImpl) ListSimpleAccessScopes(ctx context.Context, _ *v1.Empty) (*v1.ListSimpleAccessScopesResponse, error) { - scopes, err := s.roleDataStore.GetAllAccessScopes(ctx) + scopes, err := s.roleDataStore.GetAccessScopesFiltered(ctx, func(scope *storage.SimpleAccessScope) bool { + // filter out dynamic access scopes created to back Rox tokens issued for internal purposes. + return scope.GetTraits().GetOrigin() != storage.Traits_EPHEMERAL + }) if err != nil { return nil, errors.Wrap(err, "failed to retrieve access scopes") } @@ -267,6 +289,9 @@ func (s *serviceImpl) PostSimpleAccessScope(ctx context.Context, scope *storage. return nil, errox.InvalidArgs.CausedBy("setting id field is not allowed") } scope.Id = rolePkg.GenerateAccessScopeID() + if scope.GetTraits().GetOrigin() == storage.Traits_EPHEMERAL { + return nil, errox.InvalidArgs.CausedBy("dynamic access scopes can only be created by internal services") + } // Store the augmented access scope; report back on error. Note the access // scope is referenced by its name because that's what the caller knows. @@ -280,6 +305,9 @@ func (s *serviceImpl) PostSimpleAccessScope(ctx context.Context, scope *storage. } func (s *serviceImpl) PutSimpleAccessScope(ctx context.Context, scope *storage.SimpleAccessScope) (*v1.Empty, error) { + if scope.GetTraits().GetOrigin() == storage.Traits_EPHEMERAL { + return nil, errox.InvalidArgs.CausedBy("dynamic access scopes cannot be modified by API") + } err := s.roleDataStore.UpdateAccessScope(ctx, scope) if err != nil { return nil, errors.Wrapf(err, "failed to update access scope %s", scope.GetId()) diff --git a/central/role/service/service_impl_accessscope_postgres_test.go b/central/role/service/service_impl_accessscope_postgres_test.go new file mode 100644 index 0000000000000..1ebcbcfdd317e --- /dev/null +++ b/central/role/service/service_impl_accessscope_postgres_test.go @@ -0,0 +1,110 @@ +//go:build sql_integration + +package service + +import ( + "testing" + + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/sac" + "github.com/stretchr/testify/suite" +) + +func TestServiceImplWithDB_AccessScopes(t *testing.T) { + suite.Run(t, new(serviceImplAccessScopeTestSuite)) +} + +type serviceImplAccessScopeTestSuite struct { + suite.Suite + + tester *serviceImplTester +} + +func (s *serviceImplAccessScopeTestSuite) SetupSuite() { + s.tester = &serviceImplTester{} + s.tester.Setup(s.T()) +} + +func (s *serviceImplAccessScopeTestSuite) SetupTest() { + s.Require().NotNil(s.tester) + s.tester.SetupTest(s.T()) +} + +func (s *serviceImplAccessScopeTestSuite) TearDownTest() { + s.Require().NotNil(s.tester) + s.tester.TearDownTest(s.T()) +} + +func (s *serviceImplAccessScopeTestSuite) TestListAccessScopes() { + t := s.T() + ctx := sac.WithAllAccess(t.Context()) + + accessScopeName1 := "TestListAccessScopes_noTraits" + accessScopeName2 := "TestListAccessScopes_imperativeOriginTraits" + accessScopeName3 := "TestListAccessScopes_declarativeOriginTraits" + accessScopeName4 := "TestListAccessScopes_orphanedDeclarativeOriginTraits" + accessScopeName5 := "TestListAccessScopes_ephemeralOriginTraits" + scope1 := s.tester.createAccessScope(t, accessScopeName1, nilTraits) + scope2 := s.tester.createAccessScope(t, accessScopeName2, imperativeOriginTraits) + scope3 := s.tester.createAccessScope(t, accessScopeName3, declarativeOriginTraits) + scope4 := s.tester.createAccessScope(t, accessScopeName4, orphanedDeclarativeOriginTraits) + scope5 := s.tester.createAccessScope(t, accessScopeName5, ephemeralOriginTraits) + + scopes, err := s.tester.service.ListSimpleAccessScopes(ctx, &v1.Empty{}) + s.NoError(err) + s.Len(scopes.GetAccessScopes(), 4) + + protoassert.SliceContains(s.T(), scopes.GetAccessScopes(), scope1) + protoassert.SliceContains(s.T(), scopes.GetAccessScopes(), scope2) + protoassert.SliceContains(s.T(), scopes.GetAccessScopes(), scope3) + protoassert.SliceContains(s.T(), scopes.GetAccessScopes(), scope4) + // Roles with ephemeral origin are filtered out. + protoassert.SliceNotContains(s.T(), scopes.GetAccessScopes(), scope5) +} + +func (s *serviceImplAccessScopeTestSuite) TestPostAccessScope() { + s.Run("Scope without specified origin can be created by API", func() { + inputScope := &storage.SimpleAccessScope{ + Name: "Test basic access scope", + Rules: &storage.SimpleAccessScope_Rules{}, + } + ctx := sac.WithAllAccess(s.T().Context()) + scope, err := s.tester.service.PostSimpleAccessScope(ctx, inputScope) + s.NoError(err) + inputScope.Id = scope.GetId() + protoassert.Equal(s.T(), inputScope, scope) + }) + s.Run("ephemeral scopes cannot be created by API", func() { + inputScope := &storage.SimpleAccessScope{ + Traits: ephemeralOriginTraits, + } + ctx := sac.WithAllAccess(s.T().Context()) + scope, err := s.tester.service.PostSimpleAccessScope(ctx, inputScope) + s.ErrorIs(err, errox.InvalidArgs) + s.Nil(scope) + }) +} + +func (s *serviceImplAccessScopeTestSuite) TestPutAccessScope() { + s.Run("Scope without specified origin can be updated by API", func() { + scopeName := "Access scope without origin" + inputScope := s.tester.createAccessScope(s.T(), scopeName, nilTraits) + updatedScope := inputScope.CloneVT() + updatedScope.Description = "Updated description" + ctx := sac.WithAllAccess(s.T().Context()) + _, err := s.tester.service.PutSimpleAccessScope(ctx, updatedScope) + s.NoError(err) + }) + s.Run("Ephemeral scopes cannot be created by API", func() { + scopeName := "Ephemeral access scope" + inputScope := s.tester.createAccessScope(s.T(), scopeName, ephemeralOriginTraits) + updatedScope := inputScope.CloneVT() + updatedScope.Description = "Updated description" + ctx := sac.WithAllAccess(s.T().Context()) + _, err := s.tester.service.PutSimpleAccessScope(ctx, updatedScope) + s.ErrorIs(err, errox.InvalidArgs) + }) +} diff --git a/central/role/service/service_impl_other_postgres_test.go b/central/role/service/service_impl_other_postgres_test.go new file mode 100644 index 0000000000000..d159cf587a64d --- /dev/null +++ b/central/role/service/service_impl_other_postgres_test.go @@ -0,0 +1,487 @@ +//go:build sql_integration + +package service + +import ( + "context" + "testing" + + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/auth/permissions" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stretchr/testify/suite" +) + +const ( + clusterPermission = "Cluster" + compliancePermission = "Compliance" + deploymentPermission = "Deployment" + deploymentExtensionPermission = "DeploymentExtension" + integrationPermission = "Integration" + namespacePermission = "Namespace" + networkGraphPermission = "NetworkGraph" + nodePermission = "Node" + rolePermission = "Role" +) + +func TestServiceImplWithDB_Other(t *testing.T) { + suite.Run(t, new(serviceImplOtherTestSuite)) +} + +type serviceImplOtherTestSuite struct { + suite.Suite + + tester *serviceImplTester +} + +func (s *serviceImplOtherTestSuite) SetupSuite() { + s.tester = &serviceImplTester{} + s.tester.Setup(s.T()) +} + +func (s *serviceImplOtherTestSuite) SetupTest() { + s.Require().NotNil(s.tester) + s.tester.SetupTest(s.T()) +} + +func (s *serviceImplOtherTestSuite) TearDownTest() { + s.Require().NotNil(s.tester) + s.tester.TearDownTest(s.T()) +} + +func (s *serviceImplOtherTestSuite) TestGetClustersForPermissions() { + deepPurpleClusterID := s.tester.clusterNameToIDMap[clusterDeepPurple.GetName()] + queenClusterID := s.tester.clusterNameToIDMap[clusterQueen.GetName()] + testResourceScope1 := getTestResourceScopeSingleNamespace( + queenClusterID, + namespaceInnuendoInClusterQueen.GetName()) + pinkFloydClusterID := s.tester.clusterNameToIDMap[clusterPinkFloyd.GetName()] + testResourceScope2 := getTestResourceScopeSingleNamespace( + pinkFloydClusterID, + namespaceTheWallInClusterPinkFloyd.GetName()) + + testScopeMap := sac.TestScopeMap{ + storage.Access_READ_ACCESS: map[permissions.Resource]*sac.TestResourceScope{ + resources.Integration.GetResource(): { + Included: true, + }, + resources.Node.GetResource(): testResourceScope1, + resources.Deployment.GetResource(): testResourceScope1, + resources.NetworkGraph.GetResource(): testResourceScope2, + }, + } + + extendedAccessTestScopeMap := sac.TestScopeMap{ + storage.Access_READ_ACCESS: map[permissions.Resource]*sac.TestResourceScope{ + resources.Compliance.GetResource(): { + Included: true, + }, + resources.Integration.GetResource(): { + Included: true, + }, + resources.Node.GetResource(): testResourceScope1, + resources.Deployment.GetResource(): testResourceScope1, + resources.NetworkGraph.GetResource(): testResourceScope2, + }, + } + + deepPurpleClusterResponse := &v1.ScopeObject{ + Id: deepPurpleClusterID, + Name: clusterDeepPurple.GetName(), + } + + queenClusterResponse := &v1.ScopeObject{ + Id: queenClusterID, + Name: clusterQueen.GetName(), + } + + pinkFloydClusterResponse := &v1.ScopeObject{ + Id: pinkFloydClusterID, + Name: clusterPinkFloyd.GetName(), + } + + testCtx := sac.WithGlobalAccessScopeChecker(context.Background(), + sac.TestScopeCheckerCoreFromFullScopeMap(s.T(), testScopeMap)) + + extendedAccessTestCtx := sac.WithGlobalAccessScopeChecker(context.Background(), + sac.TestScopeCheckerCoreFromFullScopeMap(s.T(), extendedAccessTestScopeMap)) + + testCases := []struct { + name string + context context.Context + testedPermissions []string + expectedClusters []*v1.ScopeObject + }{ + { + name: "Global permission (Not Granted) gets no cluster data.", + context: testCtx, + testedPermissions: []string{rolePermission}, + expectedClusters: []*v1.ScopeObject{}, + }, + { + name: "Global permission (Granted) gets no cluster data.", + context: testCtx, + testedPermissions: []string{integrationPermission}, + expectedClusters: []*v1.ScopeObject{}, + }, + { + name: "Not granted Cluster scoped permission gets no cluster data.", + context: testCtx, + testedPermissions: []string{clusterPermission}, + expectedClusters: []*v1.ScopeObject{}, + }, + { + name: "Granted Cluster scoped permission gets only cluster data for clusters in permission scope.", + context: testCtx, + testedPermissions: []string{nodePermission}, + expectedClusters: []*v1.ScopeObject{queenClusterResponse}, + }, + { + name: "Not granted Namespace scoped permission gets no cluster data.", + context: testCtx, + testedPermissions: []string{namespacePermission}, + expectedClusters: []*v1.ScopeObject{}, + }, + { + name: "Granted Namespace scoped permission gets only cluster data for clusters in permission scope.", + context: testCtx, + testedPermissions: []string{deploymentPermission}, + expectedClusters: []*v1.ScopeObject{queenClusterResponse}, + }, + { + name: "Multiple not granted Namespace scoped permissions get no cluster data.", + context: testCtx, + testedPermissions: []string{namespacePermission, deploymentExtensionPermission}, + expectedClusters: []*v1.ScopeObject{}, + }, + { + name: "Multiple Namespace scoped permissions get only cluster data for clusters in granted permission scopes.", + context: testCtx, + testedPermissions: []string{namespacePermission, deploymentPermission}, + expectedClusters: []*v1.ScopeObject{queenClusterResponse}, + }, + { + name: "empty permission list get cluster data for all cluster data in scope of granted cluster and namespace permissions.", + context: testCtx, + testedPermissions: []string{}, + expectedClusters: []*v1.ScopeObject{queenClusterResponse, pinkFloydClusterResponse}, + }, + { + name: "Extended Access - Global permission (Not Granted) gets no cluster data.", + context: extendedAccessTestCtx, + testedPermissions: []string{rolePermission}, + expectedClusters: []*v1.ScopeObject{}, + }, + { + name: "Extended Access - Global permission (Granted) gets no cluster data.", + context: extendedAccessTestCtx, + testedPermissions: []string{integrationPermission}, + expectedClusters: []*v1.ScopeObject{}, + }, + { + name: "Extended Access - Not granted Cluster scoped permission gets no cluster data.", + context: extendedAccessTestCtx, + testedPermissions: []string{clusterPermission}, + expectedClusters: []*v1.ScopeObject{}, + }, + { + name: "Extended Access - Granted Cluster scoped permission at resource level gets all cluster data", + context: extendedAccessTestCtx, + testedPermissions: []string{compliancePermission}, + expectedClusters: []*v1.ScopeObject{ + deepPurpleClusterResponse, + queenClusterResponse, + pinkFloydClusterResponse, + }, + }, + { + name: "Extended Access - Granted Cluster scoped permission gets only cluster data for clusters in permission scope.", + context: extendedAccessTestCtx, + testedPermissions: []string{nodePermission}, + expectedClusters: []*v1.ScopeObject{queenClusterResponse}, + }, + { + name: "Extended Access - Not granted Namespace scoped permission gets no cluster data.", + context: extendedAccessTestCtx, + testedPermissions: []string{namespacePermission}, + expectedClusters: []*v1.ScopeObject{}, + }, + { + name: "Extended Access - Granted Namespace scoped permission gets only cluster data for clusters in permission scope.", + context: extendedAccessTestCtx, + testedPermissions: []string{deploymentPermission}, + expectedClusters: []*v1.ScopeObject{queenClusterResponse}, + }, + { + name: "Extended Access - Multiple not granted Namespace scoped permissions get no cluster data.", + context: extendedAccessTestCtx, + testedPermissions: []string{namespacePermission, deploymentExtensionPermission}, + expectedClusters: []*v1.ScopeObject{}, + }, + { + name: "Extended Access - Multiple Namespace scoped permissions get only cluster data for clusters in granted permission scopes.", + context: extendedAccessTestCtx, + testedPermissions: []string{namespacePermission, deploymentPermission}, + expectedClusters: []*v1.ScopeObject{queenClusterResponse}, + }, + { + name: "Extended Access - empty permission list get cluster data for all cluster data in scope of granted cluster and namespace permissions.", + context: extendedAccessTestCtx, + testedPermissions: []string{}, + expectedClusters: []*v1.ScopeObject{ + deepPurpleClusterResponse, + queenClusterResponse, + pinkFloydClusterResponse, + }, + }, + } + + for _, c := range testCases { + s.Run(c.name, func() { + clusterResponse, err := s.tester.service.GetClustersForPermissions(c.context, &v1.GetClustersForPermissionsRequest{ + Pagination: nil, + Permissions: c.testedPermissions, + }) + s.NoError(err) + protoassert.ElementsMatch(s.T(), clusterResponse.GetClusters(), c.expectedClusters) + }) + } +} + +func (s *serviceImplOtherTestSuite) TestGetClustersForPermissionsPagination() { + queenClusterID := s.tester.clusterNameToIDMap[clusterQueen.GetName()] + testResourceScope1 := getTestResourceScopeSingleNamespace( + queenClusterID, + namespaceInnuendoInClusterQueen.GetName()) + pinkFloydClusterID := s.tester.clusterNameToIDMap[clusterPinkFloyd.GetName()] + testResourceScope2 := getTestResourceScopeSingleNamespace( + pinkFloydClusterID, + namespaceTheWallInClusterPinkFloyd.GetName()) + testScopeMap := sac.TestScopeMap{ + storage.Access_READ_ACCESS: map[permissions.Resource]*sac.TestResourceScope{ + resources.Integration.GetResource(): { + Included: true, + }, + resources.Node.GetResource(): testResourceScope1, + resources.Deployment.GetResource(): testResourceScope1, + resources.NetworkGraph.GetResource(): testResourceScope2, + }, + } + + queenClusterResponse := &v1.ScopeObject{ + Id: queenClusterID, + Name: clusterQueen.GetName(), + } + + pinkFloydClusterResponse := &v1.ScopeObject{ + Id: pinkFloydClusterID, + Name: clusterPinkFloyd.GetName(), + } + + testCtx := sac.WithGlobalAccessScopeChecker(context.Background(), + sac.TestScopeCheckerCoreFromFullScopeMap(s.T(), testScopeMap)) + + testCases := []struct { + name string + pagination *v1.Pagination + expectedClusters []*v1.ScopeObject + }{ + { + name: "No offset and a limit restricts to a list of appropriate size", + pagination: &v1.Pagination{ + Limit: 1, + Offset: 0, + SortOption: &v1.SortOption{ + Field: "Cluster", + Reversed: false, + }, + }, + expectedClusters: []*v1.ScopeObject{pinkFloydClusterResponse}, + }, + { + name: "Offset and no limit restricts to a list of appropriate size starting with expected value", + pagination: &v1.Pagination{ + Limit: 0, + Offset: 1, + SortOption: &v1.SortOption{ + Field: "Cluster", + Reversed: false, + }, + }, + expectedClusters: []*v1.ScopeObject{queenClusterResponse}, + }, + { + name: "Sort options without offset nor limit return the expected results", + pagination: &v1.Pagination{ + Limit: 0, + Offset: 0, + SortOption: &v1.SortOption{ + Field: "Cluster", + Reversed: false, + }, + }, + expectedClusters: []*v1.ScopeObject{pinkFloydClusterResponse, queenClusterResponse}, + }, + { + name: "Reversed sort without offset nor limit return the expected results", + pagination: &v1.Pagination{ + Limit: 0, + Offset: 0, + SortOption: &v1.SortOption{ + Field: "Cluster", + Reversed: true, + }, + }, + expectedClusters: []*v1.ScopeObject{queenClusterResponse, pinkFloydClusterResponse}, + }, + } + + for _, c := range testCases { + s.Run(c.name, func() { + clusterResponse, err := s.tester.service.GetClustersForPermissions(testCtx, &v1.GetClustersForPermissionsRequest{ + Pagination: c.pagination, + Permissions: []string{}, + }) + s.NoError(err) + protoassert.SlicesEqual(s.T(), clusterResponse.GetClusters(), c.expectedClusters) + }) + } +} + +func (s *serviceImplOtherTestSuite) TestGetNamespacesForClusterAndPermissions() { + queenClusterID := s.tester.clusterNameToIDMap[clusterQueen.GetName()] + testResourceScope1 := getTestResourceScopeSingleNamespace( + queenClusterID, + namespaceInnuendoInClusterQueen.GetName()) + pinkFloydClusterID := s.tester.clusterNameToIDMap[clusterPinkFloyd.GetName()] + testResourceScope2 := getTestResourceScopeSingleNamespace( + pinkFloydClusterID, + namespaceTheWallInClusterPinkFloyd.GetName()) + testResourceScope3 := getTestResourceScopeSingleNamespace( + queenClusterID, + namespaceQueenInClusterQueen.GetName()) + testScopeMap := sac.TestScopeMap{ + storage.Access_READ_ACCESS: map[permissions.Resource]*sac.TestResourceScope{ + resources.Integration.GetResource(): { + Included: true, + }, + resources.Node.GetResource(): testResourceScope1, + resources.Deployment.GetResource(): testResourceScope1, + resources.NetworkGraph.GetResource(): testResourceScope2, + resources.Image.GetResource(): testResourceScope3, + }, + } + + queenQueenNamespaceResponse := &v1.ScopeObject{ + Id: getNamespaceID(namespaceQueenInClusterQueen.GetName()), + Name: namespaceQueenInClusterQueen.GetName(), + } + + queenInnuendoNamespaceResponse := &v1.ScopeObject{ + Id: getNamespaceID(namespaceInnuendoInClusterQueen.GetName()), + Name: namespaceInnuendoInClusterQueen.GetName(), + } + + pinkFloydTheWallNamespaceResponse := &v1.ScopeObject{ + Id: getNamespaceID(namespaceTheWallInClusterPinkFloyd.GetName()), + Name: namespaceTheWallInClusterPinkFloyd.GetName(), + } + + testCases := []struct { + name string + testedClusterID string + testedPermissions []string + expectedNamespaces []*v1.ScopeObject + }{ + { + name: "Global permission (Not Granted) gets no namespace data.", + testedClusterID: s.tester.clusterNameToIDMap[clusterQueen.GetName()], + testedPermissions: []string{rolePermission}, + expectedNamespaces: []*v1.ScopeObject{}, + }, + { + name: "Global permission (Granted) gets no namespace data.", + testedClusterID: s.tester.clusterNameToIDMap[clusterQueen.GetName()], + testedPermissions: []string{integrationPermission}, + expectedNamespaces: []*v1.ScopeObject{}, + }, + { + name: "Not granted Cluster scoped permission gets no namespace data.", + testedClusterID: s.tester.clusterNameToIDMap[clusterQueen.GetName()], + testedPermissions: []string{clusterPermission}, + expectedNamespaces: []*v1.ScopeObject{}, + }, + { + name: "Granted Cluster scoped permission gets no namespace data.", + testedClusterID: s.tester.clusterNameToIDMap[clusterQueen.GetName()], + testedPermissions: []string{nodePermission}, + expectedNamespaces: []*v1.ScopeObject{}, + }, + { + name: "Not granted Namespace scoped permission gets no namespace data.", + testedClusterID: s.tester.clusterNameToIDMap[clusterQueen.GetName()], + testedPermissions: []string{namespacePermission}, + expectedNamespaces: []*v1.ScopeObject{}, + }, + { + name: "Granted Namespace scoped permission gets only namespace data for namespaces in cluster and permission scope.", + testedClusterID: s.tester.clusterNameToIDMap[clusterQueen.GetName()], + testedPermissions: []string{deploymentPermission}, + expectedNamespaces: []*v1.ScopeObject{queenInnuendoNamespaceResponse}, + }, + { + name: "Granted Namespace scoped permission gets only namespace data for namespaces in cluster and permission scope (other permission).", + testedClusterID: s.tester.clusterNameToIDMap[clusterPinkFloyd.GetName()], + testedPermissions: []string{networkGraphPermission}, + expectedNamespaces: []*v1.ScopeObject{pinkFloydTheWallNamespaceResponse}, + }, + { + name: "Multiple not granted Namespace scoped permissions get no namespace data.", + testedClusterID: s.tester.clusterNameToIDMap[clusterQueen.GetName()], + testedPermissions: []string{namespacePermission, deploymentExtensionPermission}, + expectedNamespaces: []*v1.ScopeObject{}, + }, + { + name: "Multiple Namespace scoped permissions get only namespace data for namespaces in granted permission scopes.", + testedClusterID: s.tester.clusterNameToIDMap[clusterQueen.GetName()], + testedPermissions: []string{namespacePermission, deploymentPermission}, + expectedNamespaces: []*v1.ScopeObject{queenInnuendoNamespaceResponse}, + }, + { + name: "empty permission list get namespace data for all namespaces in scope of target cluster and granted namespace permissions.", + testedClusterID: s.tester.clusterNameToIDMap[clusterQueen.GetName()], + testedPermissions: []string{}, + expectedNamespaces: []*v1.ScopeObject{queenQueenNamespaceResponse, queenInnuendoNamespaceResponse}, + }, + } + + testCtx := sac.WithGlobalAccessScopeChecker(context.Background(), + sac.TestScopeCheckerCoreFromFullScopeMap(s.T(), testScopeMap)) + + for _, c := range testCases { + s.Run(c.name, func() { + request := &v1.GetNamespaceForClusterAndPermissionsRequest{ + ClusterId: c.testedClusterID, + Permissions: c.testedPermissions, + } + namespaceResponse, err := s.tester.service.GetNamespacesForClusterAndPermissions(testCtx, request) + s.NoError(err) + protoassert.ElementsMatch(s.T(), namespaceResponse.GetNamespaces(), c.expectedNamespaces) + }) + } +} + +func getTestResourceScopeSingleNamespace(clusterID string, namespace string) *sac.TestResourceScope { + return &sac.TestResourceScope{ + Clusters: map[string]*sac.TestClusterScope{ + clusterID: { + Namespaces: []string{namespace}, + Included: false, + }, + }, + Included: false, + } +} diff --git a/central/role/service/service_impl_permissionset_postgres_test.go b/central/role/service/service_impl_permissionset_postgres_test.go new file mode 100644 index 0000000000000..449f38569b1af --- /dev/null +++ b/central/role/service/service_impl_permissionset_postgres_test.go @@ -0,0 +1,109 @@ +//go:build sql_integration + +package service + +import ( + "testing" + + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/sac" + "github.com/stretchr/testify/suite" +) + +func TestServiceImplWithDB_PermissionSet(t *testing.T) { + suite.Run(t, new(serviceImplPermissionSetTestSuite)) +} + +type serviceImplPermissionSetTestSuite struct { + suite.Suite + + tester *serviceImplTester +} + +func (s *serviceImplPermissionSetTestSuite) SetupSuite() { + s.tester = &serviceImplTester{} + s.tester.Setup(s.T()) +} + +func (s *serviceImplPermissionSetTestSuite) SetupTest() { + s.Require().NotNil(s.tester) + s.tester.SetupTest(s.T()) +} + +func (s *serviceImplPermissionSetTestSuite) TearDownTest() { + s.Require().NotNil(s.tester) + s.tester.TearDownTest(s.T()) +} + +func (s *serviceImplPermissionSetTestSuite) TestListPermissionSets() { + t := s.T() + ctx := sac.WithAllAccess(t.Context()) + + permissionSetName1 := "TestListPermissionSets_noTraits" + permissionSetName2 := "TestListPermissionSets_imperativeOriginTraits" + permissionSetName3 := "TestListPermissionSets_declarativeOriginTraits" + permissionSetName4 := "TestListPermissionSets_orphanedDeclarativeOriginTraits" + permissionSetName5 := "TestListPermissionSets_ephemeralOriginTraits" + permissionSet1 := s.tester.createPermissionSet(t, permissionSetName1, nilTraits) + permissionSet2 := s.tester.createPermissionSet(t, permissionSetName2, imperativeOriginTraits) + permissionSet3 := s.tester.createPermissionSet(t, permissionSetName3, declarativeOriginTraits) + permissionSet4 := s.tester.createPermissionSet(t, permissionSetName4, orphanedDeclarativeOriginTraits) + permissionSet5 := s.tester.createPermissionSet(t, permissionSetName5, ephemeralOriginTraits) + + permissionSets, err := s.tester.service.ListPermissionSets(ctx, &v1.Empty{}) + s.NoError(err) + s.Len(permissionSets.GetPermissionSets(), 4) + + protoassert.SliceContains(s.T(), permissionSets.GetPermissionSets(), permissionSet1) + protoassert.SliceContains(s.T(), permissionSets.GetPermissionSets(), permissionSet2) + protoassert.SliceContains(s.T(), permissionSets.GetPermissionSets(), permissionSet3) + protoassert.SliceContains(s.T(), permissionSets.GetPermissionSets(), permissionSet4) + // Roles with ephemeral origin are filtered out. + protoassert.SliceNotContains(s.T(), permissionSets.GetPermissionSets(), permissionSet5) +} + +func (s *serviceImplPermissionSetTestSuite) TestPostPermissionSet() { + s.Run("Permission set without specified origin can be created by API", func() { + inputPermissionSet := &storage.PermissionSet{ + Name: "Test basic permission set", + } + ctx := sac.WithAllAccess(s.T().Context()) + permissionSet, err := s.tester.service.PostPermissionSet(ctx, inputPermissionSet) + s.NoError(err) + inputPermissionSet.Id = permissionSet.GetId() + protoassert.Equal(s.T(), inputPermissionSet, permissionSet) + }) + s.Run("Ephemeral scopes cannot be created by API", func() { + inputScope := &storage.SimpleAccessScope{ + Traits: ephemeralOriginTraits, + } + ctx := sac.WithAllAccess(s.T().Context()) + scope, err := s.tester.service.PostSimpleAccessScope(ctx, inputScope) + s.ErrorIs(err, errox.InvalidArgs) + s.Nil(scope) + }) +} + +func (s *serviceImplPermissionSetTestSuite) TestPutPermissionSet() { + s.Run("Permission set without specified origin can be updated by API", func() { + permissionSetName := "Permission set without origin" + inputPermissionSet := s.tester.createPermissionSet(s.T(), permissionSetName, nilTraits) + updatedPermissionSet := inputPermissionSet.CloneVT() + updatedPermissionSet.Description = "Updated description" + ctx := sac.WithAllAccess(s.T().Context()) + _, err := s.tester.service.PutPermissionSet(ctx, updatedPermissionSet) + s.NoError(err) + }) + s.Run("Ephemeral scopes cannot be created by API", func() { + permissionSetName := "Ephemeral permission set" + inputPermissionSet := s.tester.createPermissionSet(s.T(), permissionSetName, ephemeralOriginTraits) + updatedPermissionSet := inputPermissionSet.CloneVT() + updatedPermissionSet.Description = "Updated description" + ctx := sac.WithAllAccess(s.T().Context()) + _, err := s.tester.service.PutPermissionSet(ctx, updatedPermissionSet) + s.ErrorIs(err, errox.InvalidArgs) + }) +} diff --git a/central/role/service/service_impl_role_postgres_test.go b/central/role/service/service_impl_role_postgres_test.go new file mode 100644 index 0000000000000..5de1d22edb0d1 --- /dev/null +++ b/central/role/service/service_impl_role_postgres_test.go @@ -0,0 +1,226 @@ +//go:build sql_integration + +package service + +import ( + "fmt" + "testing" + + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/defaults/accesscontrol" + "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/sac" + "github.com/stretchr/testify/suite" +) + +func TestServiceImplWithDB_Roles(t *testing.T) { + suite.Run(t, new(serviceImplRoleTestSuite)) +} + +type serviceImplRoleTestSuite struct { + suite.Suite + + tester *serviceImplTester +} + +func (s *serviceImplRoleTestSuite) SetupSuite() { + s.tester = &serviceImplTester{} + s.tester.Setup(s.T()) +} + +func (s *serviceImplRoleTestSuite) SetupTest() { + s.Require().NotNil(s.tester) + s.tester.SetupTest(s.T()) +} + +func (s *serviceImplRoleTestSuite) TearDownTest() { + s.Require().NotNil(s.tester) + s.tester.TearDownTest(s.T()) +} + +func (s *serviceImplRoleTestSuite) TestCreateRoleValidAccessScopeID() { + t := s.T() + ctx := sac.WithAllAccess(t.Context()) + roleName := "TestCreateRoleValidAccessScopeID" + + ps := s.tester.createPermissionSet(t, roleName, nilTraits) + scope := s.tester.createAccessScope(t, roleName, nilTraits) + + role := getValidRole(roleName) + role.PermissionSetId = ps.GetId() + role.AccessScopeId = scope.GetId() + createRoleRequest := &v1.CreateRoleRequest{ + Name: roleName, + Role: role, + } + _, err := s.tester.service.CreateRole(ctx, createRoleRequest) + s.NoError(err) + s.tester.storedRoleNames = append(s.tester.storedRoleNames, role.GetName()) +} + +func (s *serviceImplRoleTestSuite) TestCreateRoleEmptyAccessScopeID() { + t := s.T() + ctx := sac.WithAllAccess(t.Context()) + roleName := "TestCreateRoleEmptyAccessScopeID" + + ps := s.tester.createPermissionSet(t, roleName, nilTraits) + + role := getValidRole(roleName) + role.PermissionSetId = ps.GetId() + role.AccessScopeId = "" + createRoleRequest := &v1.CreateRoleRequest{ + Name: roleName, + Role: role, + } + _, err := s.tester.service.CreateRole(ctx, createRoleRequest) + s.ErrorContains(err, "role access_scope_id field must be set") +} + +func (s *serviceImplRoleTestSuite) TestUpdateExistingRoleValidAccessScopeID() { + t := s.T() + ctx := sac.WithAllAccess(t.Context()) + roleName := "TestUpdateExistingRoleValidAccessScopeID" + role := s.tester.createRole(t, roleName, nilTraits) + newScope := s.tester.createAccessScope(t, "new scope", nilTraits) + role.AccessScopeId = newScope.GetId() + _, err := s.tester.service.UpdateRole(ctx, role) + s.NoError(err) +} + +func (s *serviceImplRoleTestSuite) TestUpdateExistingRoleEmptyAccessScopeID() { + t := s.T() + ctx := sac.WithAllAccess(t.Context()) + roleName := "TestUpdateExistingRoleEmptyAccessScopeID" + role := s.tester.createRole(t, roleName, nilTraits) + role.AccessScopeId = "" + _, err := s.tester.service.UpdateRole(ctx, role) + s.ErrorContains(err, "role access_scope_id field must be set") +} + +func (s *serviceImplRoleTestSuite) TestUpdateMissingRoleValidAccessScopeID() { + t := s.T() + ctx := sac.WithAllAccess(t.Context()) + roleName := "TestUpdateMissingRoleValidAccessScopeID" + ps := s.tester.createPermissionSet(t, roleName, nilTraits) + scope := s.tester.createAccessScope(t, roleName, nilTraits) + role := getValidRole(roleName) + role.PermissionSetId = ps.GetId() + role.AccessScopeId = scope.GetId() + _, err := s.tester.service.UpdateRole(ctx, role) + s.ErrorIs(err, errox.NotFound) +} + +func (s *serviceImplRoleTestSuite) TestUpdateMissingRoleEmptyAccessScopeID() { + t := s.T() + ctx := sac.WithAllAccess(t.Context()) + roleName := "TestUpdateMissingRoleEmptyAccessScopeID" + ps := s.tester.createPermissionSet(t, roleName, nilTraits) + role := getValidRole(roleName) + role.PermissionSetId = ps.GetId() + role.AccessScopeId = "" + _, err := s.tester.service.UpdateRole(ctx, role) + s.ErrorContains(err, "role access_scope_id field must be set") +} + +func (s *serviceImplRoleTestSuite) TestGetRoles() { + t := s.T() + ctx := sac.WithAllAccess(t.Context()) + + roleName1 := "TestGetRoles_noTraits" + roleName2 := "TestGetRoles_imperativeOriginTraits" + roleName3 := "TestGetRoles_declarativeOriginTraits" + roleName4 := "TestGetRoles_orphanedDeclarativeOriginTraits" + roleName5 := "TestGetRoles_ephemeralOriginTraits" + role1 := s.tester.createRole(t, roleName1, nilTraits) + role2 := s.tester.createRole(t, roleName2, imperativeOriginTraits) + role3 := s.tester.createRole(t, roleName3, declarativeOriginTraits) + role4 := s.tester.createRole(t, roleName4, orphanedDeclarativeOriginTraits) + role5 := s.tester.createRole(t, roleName5, ephemeralOriginTraits) + + roles, err := s.tester.service.GetRoles(ctx, &v1.Empty{}) + s.NoError(err) + s.Len(roles.GetRoles(), 4) + + protoassert.SliceContains(s.T(), roles.GetRoles(), role1) + protoassert.SliceContains(s.T(), roles.GetRoles(), role2) + protoassert.SliceContains(s.T(), roles.GetRoles(), role3) + protoassert.SliceContains(s.T(), roles.GetRoles(), role4) + // Roles with ephemeral origin are filtered out. + protoassert.SliceNotContains(s.T(), roles.GetRoles(), role5) +} + +func (s *serviceImplRoleTestSuite) TestCreateRole() { + s.Run("Role without specified origin can be created by API", func() { + roleName := "Basic test role" + ps := s.tester.createPermissionSet(s.T(), roleName, nilTraits) + as := s.tester.createAccessScope(s.T(), roleName, nilTraits) + roleCreationRequest := &v1.CreateRoleRequest{ + Name: roleName, + Role: &storage.Role{ + Name: roleName, + PermissionSetId: ps.GetId(), + AccessScopeId: as.GetId(), + Traits: nilTraits, + }, + } + ctx := sac.WithAllAccess(s.T().Context()) + _, err := s.tester.service.CreateRole(ctx, roleCreationRequest) + s.NoError(err) + s.tester.storedRoleNames = append(s.tester.storedRoleNames, roleName) + roleLookupRequest := &v1.ResourceByID{Id: roleName} + role, fetchErr := s.tester.service.GetRole(ctx, roleLookupRequest) + s.NoError(fetchErr) + protoassert.Equal(s.T(), roleCreationRequest.GetRole(), role) + }) + s.Run("Ephemeral roles cannot be created by API", func() { + roleName := "Ephemeral test role" + roleCreationRequest := &v1.CreateRoleRequest{ + Name: roleName, + Role: &storage.Role{ + Name: roleName, + PermissionSetId: accesscontrol.DefaultPermissionSetIDs[accesscontrol.Admin], + AccessScopeId: accesscontrol.DefaultAccessScopeIDs[accesscontrol.UnrestrictedAccessScope], + Traits: ephemeralOriginTraits, + }, + } + ctx := sac.WithAllAccess(s.T().Context()) + _, err := s.tester.service.CreateRole(ctx, roleCreationRequest) + s.ErrorIs(err, errox.InvalidArgs) + }) +} + +func (s *serviceImplRoleTestSuite) TestUpdateRole() { + s.Run("Roles without specified origin can be updated by API", func() { + roleName := "Test Update of basic role" + role := s.tester.createRole(s.T(), roleName, nilTraits) + updatedRole := role.CloneVT() + updatedRole.Description = "Updated description" + ctx := sac.WithAllAccess(s.T().Context()) + _, err := s.tester.service.UpdateRole(ctx, role) + s.NoError(err) + }) + s.Run("Ephemeral roles cannot be updated by API", func() { + roleName := "Test update of ephemeral role" + role := s.tester.createRole(s.T(), roleName, ephemeralOriginTraits) + updatedRole := role.CloneVT() + updatedRole.Description = "Updated description" + ctx := sac.WithAllAccess(s.T().Context()) + updated, err := s.tester.service.UpdateRole(ctx, role) + s.ErrorIs(err, errox.InvalidArgs) + s.Nil(updated) + }) +} + +func getValidRole(name string) *storage.Role { + permissionSetID := accesscontrol.DefaultPermissionSetIDs[accesscontrol.Admin] + scopeID := accesscontrol.DefaultAccessScopeIDs[accesscontrol.UnrestrictedAccessScope] + return &storage.Role{ + Name: name, + Description: fmt.Sprintf("Test role for %s", name), + PermissionSetId: permissionSetID, + AccessScopeId: scopeID, + Traits: nil, + } +} diff --git a/central/role/service/service_impl_test.go b/central/role/service/service_impl_test.go index ab6f9ffc7f16a..9ab7605724f0c 100644 --- a/central/role/service/service_impl_test.go +++ b/central/role/service/service_impl_test.go @@ -4,31 +4,20 @@ package service import ( "context" - "fmt" "testing" - "time" - clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" - namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" - roleDatastore "github.com/stackrox/rox/central/role/datastore" - "github.com/stackrox/rox/central/role/sachelper" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/auth/authproviders" - "github.com/stackrox/rox/pkg/auth/permissions" - "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/defaults/accesscontrol" "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/grpc/authn/basic" "github.com/stackrox/rox/pkg/labels" - "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protoassert" - "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/effectiveaccessscope" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/uuid" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -515,723 +504,6 @@ func TestEffectiveAccessScopeForSimpleAccessScope(t *testing.T) { } } -func TestServiceImplWithDB(t *testing.T) { - suite.Run(t, new(serviceImplTestSuite)) -} - -type serviceImplTestSuite struct { - suite.Suite - - postgres *pgtest.TestPostgres - service *serviceImpl - - storedClusterIDs []string - storedNamespaceIDs []string - clusterNameToIDMap map[string]string - - storedPermissionSetIDs []string - storedAccessScopeIDs []string - storedRoleNames []string -} - -func (s *serviceImplTestSuite) SetupSuite() { - var err error - s.postgres = pgtest.ForT(s.T()) - s.Require().NotNil(s.postgres) - roleStore := roleDatastore.GetTestPostgresDataStore(s.T(), s.postgres.DB) - clusterStore, err := clusterDataStore.GetTestPostgresDataStore(s.T(), s.postgres.DB) - s.Require().NoError(err) - namespaceStore, err := namespaceDataStore.GetTestPostgresDataStore(s.T(), s.postgres.DB) - s.Require().NoError(err) - - s.service = &serviceImpl{ - roleDataStore: roleStore, - clusterDataStore: clusterStore, - namespaceDataStore: namespaceStore, - clusterSACHelper: sachelper.NewClusterSacHelper(clusterStore), - namespaceSACHelper: sachelper.NewClusterNamespaceSacHelper(clusterStore, namespaceStore), - } -} - -func (s *serviceImplTestSuite) SetupTest() { - s.storedAccessScopeIDs = make([]string, 0) - s.storedPermissionSetIDs = make([]string, 0) - s.storedRoleNames = make([]string, 0) - - s.storedClusterIDs = make([]string, 0) - s.storedNamespaceIDs = make([]string, 0) - s.clusterNameToIDMap = make(map[string]string, 0) - - writeCtx := sac.WithAllAccess(context.Background()) - - for _, cluster := range storageClusters { - clusterToAdd := cluster.CloneVT() - clusterToAdd.Id = "" - clusterToAdd.MainImage = "quay.io/rhacs-eng/main:latest" - id, err := s.service.clusterDataStore.AddCluster(writeCtx, clusterToAdd) - s.Require().NoError(err) - s.clusterNameToIDMap[clusterToAdd.GetName()] = id - s.storedClusterIDs = append(s.storedClusterIDs, id) - } - - for _, namespace := range storageNamespaces { - ns := namespace.CloneVT() - ns.Id = getNamespaceID(ns.GetName()) - ns.ClusterId = s.clusterNameToIDMap[ns.GetClusterName()] - s.Require().NoError(s.service.namespaceDataStore.AddNamespace(writeCtx, ns)) - s.storedNamespaceIDs = append(s.storedNamespaceIDs, ns.GetId()) - } -} - -func (s *serviceImplTestSuite) TearDownTest() { - writeCtx := sac.WithAllAccess(context.Background()) - for _, clusterID := range s.storedClusterIDs { - doneSignal := concurrency.NewSignal() - s.Require().NoError(s.service.clusterDataStore.RemoveCluster(writeCtx, clusterID, &doneSignal)) - require.Eventually(s.T(), - func() bool { return doneSignal.IsDone() }, - 5*time.Second, - 10*time.Millisecond, - ) - } - s.storedClusterIDs = s.storedClusterIDs[:0] - for _, namespaceID := range s.storedNamespaceIDs { - s.Require().NoError(s.service.namespaceDataStore.RemoveNamespace(writeCtx, namespaceID)) - } - for _, roleName := range s.storedRoleNames { - s.deleteRole(roleName) - } - for _, permissionSetID := range s.storedPermissionSetIDs { - s.deletePermissionSet(permissionSetID) - } - for _, accessScopeID := range s.storedAccessScopeIDs { - s.deleteAccessScope(accessScopeID) - } -} - -const ( - namespaceUUIDNamespace = "namespace" - - clusterPermission = "Cluster" - compliancePermission = "Compliance" - deploymentPermission = "Deployment" - deploymentExtensionPermission = "DeploymentExtension" - integrationPermission = "Integration" - namespacePermission = "Namespace" - networkGraphPermission = "NetworkGraph" - nodePermission = "Node" - rolePermission = "Role" -) - -func getTestResourceScopeSingleNamespace(clusterID string, namespace string) *sac.TestResourceScope { - return &sac.TestResourceScope{ - Clusters: map[string]*sac.TestClusterScope{ - clusterID: { - Namespaces: []string{namespace}, - Included: false, - }, - }, - Included: false, - } -} - -func getNamespaceID(namespaceName string) string { - return uuid.NewV5FromNonUUIDs(namespaceUUIDNamespace, namespaceName).String() -} - -func (s *serviceImplTestSuite) TestGetClustersForPermissions() { - deepPurpleClusterID := s.clusterNameToIDMap[clusterDeepPurple.GetName()] - queenClusterID := s.clusterNameToIDMap[clusterQueen.GetName()] - testResourceScope1 := getTestResourceScopeSingleNamespace( - queenClusterID, - namespaceInnuendoInClusterQueen.GetName()) - pinkFloydClusterID := s.clusterNameToIDMap[clusterPinkFloyd.GetName()] - testResourceScope2 := getTestResourceScopeSingleNamespace( - pinkFloydClusterID, - namespaceTheWallInClusterPinkFloyd.GetName()) - - testScopeMap := sac.TestScopeMap{ - storage.Access_READ_ACCESS: map[permissions.Resource]*sac.TestResourceScope{ - resources.Integration.GetResource(): { - Included: true, - }, - resources.Node.GetResource(): testResourceScope1, - resources.Deployment.GetResource(): testResourceScope1, - resources.NetworkGraph.GetResource(): testResourceScope2, - }, - } - - extendedAccessTestScopeMap := sac.TestScopeMap{ - storage.Access_READ_ACCESS: map[permissions.Resource]*sac.TestResourceScope{ - resources.Compliance.GetResource(): { - Included: true, - }, - resources.Integration.GetResource(): { - Included: true, - }, - resources.Node.GetResource(): testResourceScope1, - resources.Deployment.GetResource(): testResourceScope1, - resources.NetworkGraph.GetResource(): testResourceScope2, - }, - } - - deepPurpleClusterResponse := &v1.ScopeObject{ - Id: deepPurpleClusterID, - Name: clusterDeepPurple.GetName(), - } - - queenClusterResponse := &v1.ScopeObject{ - Id: queenClusterID, - Name: clusterQueen.GetName(), - } - - pinkFloydClusterResponse := &v1.ScopeObject{ - Id: pinkFloydClusterID, - Name: clusterPinkFloyd.GetName(), - } - - testCtx := sac.WithGlobalAccessScopeChecker(context.Background(), - sac.TestScopeCheckerCoreFromFullScopeMap(s.T(), testScopeMap)) - - extendedAccessTestCtx := sac.WithGlobalAccessScopeChecker(context.Background(), - sac.TestScopeCheckerCoreFromFullScopeMap(s.T(), extendedAccessTestScopeMap)) - - testCases := []struct { - name string - context context.Context - testedPermissions []string - expectedClusters []*v1.ScopeObject - }{ - { - name: "Global permission (Not Granted) gets no cluster data.", - context: testCtx, - testedPermissions: []string{rolePermission}, - expectedClusters: []*v1.ScopeObject{}, - }, - { - name: "Global permission (Granted) gets no cluster data.", - context: testCtx, - testedPermissions: []string{integrationPermission}, - expectedClusters: []*v1.ScopeObject{}, - }, - { - name: "Not granted Cluster scoped permission gets no cluster data.", - context: testCtx, - testedPermissions: []string{clusterPermission}, - expectedClusters: []*v1.ScopeObject{}, - }, - { - name: "Granted Cluster scoped permission gets only cluster data for clusters in permission scope.", - context: testCtx, - testedPermissions: []string{nodePermission}, - expectedClusters: []*v1.ScopeObject{queenClusterResponse}, - }, - { - name: "Not granted Namespace scoped permission gets no cluster data.", - context: testCtx, - testedPermissions: []string{namespacePermission}, - expectedClusters: []*v1.ScopeObject{}, - }, - { - name: "Granted Namespace scoped permission gets only cluster data for clusters in permission scope.", - context: testCtx, - testedPermissions: []string{deploymentPermission}, - expectedClusters: []*v1.ScopeObject{queenClusterResponse}, - }, - { - name: "Multiple not granted Namespace scoped permissions get no cluster data.", - context: testCtx, - testedPermissions: []string{namespacePermission, deploymentExtensionPermission}, - expectedClusters: []*v1.ScopeObject{}, - }, - { - name: "Multiple Namespace scoped permissions get only cluster data for clusters in granted permission scopes.", - context: testCtx, - testedPermissions: []string{namespacePermission, deploymentPermission}, - expectedClusters: []*v1.ScopeObject{queenClusterResponse}, - }, - { - name: "empty permission list get cluster data for all cluster data in scope of granted cluster and namespace permissions.", - context: testCtx, - testedPermissions: []string{}, - expectedClusters: []*v1.ScopeObject{queenClusterResponse, pinkFloydClusterResponse}, - }, - { - name: "Extended Access - Global permission (Not Granted) gets no cluster data.", - context: extendedAccessTestCtx, - testedPermissions: []string{rolePermission}, - expectedClusters: []*v1.ScopeObject{}, - }, - { - name: "Extended Access - Global permission (Granted) gets no cluster data.", - context: extendedAccessTestCtx, - testedPermissions: []string{integrationPermission}, - expectedClusters: []*v1.ScopeObject{}, - }, - { - name: "Extended Access - Not granted Cluster scoped permission gets no cluster data.", - context: extendedAccessTestCtx, - testedPermissions: []string{clusterPermission}, - expectedClusters: []*v1.ScopeObject{}, - }, - { - name: "Extended Access - Granted Cluster scoped permission at resource level gets all cluster data", - context: extendedAccessTestCtx, - testedPermissions: []string{compliancePermission}, - expectedClusters: []*v1.ScopeObject{ - deepPurpleClusterResponse, - queenClusterResponse, - pinkFloydClusterResponse, - }, - }, - { - name: "Extended Access - Granted Cluster scoped permission gets only cluster data for clusters in permission scope.", - context: extendedAccessTestCtx, - testedPermissions: []string{nodePermission}, - expectedClusters: []*v1.ScopeObject{queenClusterResponse}, - }, - { - name: "Extended Access - Not granted Namespace scoped permission gets no cluster data.", - context: extendedAccessTestCtx, - testedPermissions: []string{namespacePermission}, - expectedClusters: []*v1.ScopeObject{}, - }, - { - name: "Extended Access - Granted Namespace scoped permission gets only cluster data for clusters in permission scope.", - context: extendedAccessTestCtx, - testedPermissions: []string{deploymentPermission}, - expectedClusters: []*v1.ScopeObject{queenClusterResponse}, - }, - { - name: "Extended Access - Multiple not granted Namespace scoped permissions get no cluster data.", - context: extendedAccessTestCtx, - testedPermissions: []string{namespacePermission, deploymentExtensionPermission}, - expectedClusters: []*v1.ScopeObject{}, - }, - { - name: "Extended Access - Multiple Namespace scoped permissions get only cluster data for clusters in granted permission scopes.", - context: extendedAccessTestCtx, - testedPermissions: []string{namespacePermission, deploymentPermission}, - expectedClusters: []*v1.ScopeObject{queenClusterResponse}, - }, - { - name: "Extended Access - empty permission list get cluster data for all cluster data in scope of granted cluster and namespace permissions.", - context: extendedAccessTestCtx, - testedPermissions: []string{}, - expectedClusters: []*v1.ScopeObject{ - deepPurpleClusterResponse, - queenClusterResponse, - pinkFloydClusterResponse, - }, - }, - } - - for _, c := range testCases { - s.Run(c.name, func() { - clusterResponse, err := s.service.GetClustersForPermissions(c.context, &v1.GetClustersForPermissionsRequest{ - Pagination: nil, - Permissions: c.testedPermissions, - }) - s.NoError(err) - protoassert.ElementsMatch(s.T(), clusterResponse.GetClusters(), c.expectedClusters) - }) - } -} - -func (s *serviceImplTestSuite) TestGetClustersForPermissionsPagination() { - queenClusterID := s.clusterNameToIDMap[clusterQueen.GetName()] - testResourceScope1 := getTestResourceScopeSingleNamespace( - queenClusterID, - namespaceInnuendoInClusterQueen.GetName()) - pinkFloydClusterID := s.clusterNameToIDMap[clusterPinkFloyd.GetName()] - testResourceScope2 := getTestResourceScopeSingleNamespace( - pinkFloydClusterID, - namespaceTheWallInClusterPinkFloyd.GetName()) - testScopeMap := sac.TestScopeMap{ - storage.Access_READ_ACCESS: map[permissions.Resource]*sac.TestResourceScope{ - resources.Integration.GetResource(): { - Included: true, - }, - resources.Node.GetResource(): testResourceScope1, - resources.Deployment.GetResource(): testResourceScope1, - resources.NetworkGraph.GetResource(): testResourceScope2, - }, - } - - queenClusterResponse := &v1.ScopeObject{ - Id: queenClusterID, - Name: clusterQueen.GetName(), - } - - pinkFloydClusterResponse := &v1.ScopeObject{ - Id: pinkFloydClusterID, - Name: clusterPinkFloyd.GetName(), - } - - testCtx := sac.WithGlobalAccessScopeChecker(context.Background(), - sac.TestScopeCheckerCoreFromFullScopeMap(s.T(), testScopeMap)) - - testCases := []struct { - name string - pagination *v1.Pagination - expectedClusters []*v1.ScopeObject - }{ - { - name: "No offset and a limit restricts to a list of appropriate size", - pagination: &v1.Pagination{ - Limit: 1, - Offset: 0, - SortOption: &v1.SortOption{ - Field: "Cluster", - Reversed: false, - }, - }, - expectedClusters: []*v1.ScopeObject{pinkFloydClusterResponse}, - }, - { - name: "Offset and no limit restricts to a list of appropriate size starting with expected value", - pagination: &v1.Pagination{ - Limit: 0, - Offset: 1, - SortOption: &v1.SortOption{ - Field: "Cluster", - Reversed: false, - }, - }, - expectedClusters: []*v1.ScopeObject{queenClusterResponse}, - }, - { - name: "Sort options without offset nor limit return the expected results", - pagination: &v1.Pagination{ - Limit: 0, - Offset: 0, - SortOption: &v1.SortOption{ - Field: "Cluster", - Reversed: false, - }, - }, - expectedClusters: []*v1.ScopeObject{pinkFloydClusterResponse, queenClusterResponse}, - }, - { - name: "Reversed sort without offset nor limit return the expected results", - pagination: &v1.Pagination{ - Limit: 0, - Offset: 0, - SortOption: &v1.SortOption{ - Field: "Cluster", - Reversed: true, - }, - }, - expectedClusters: []*v1.ScopeObject{queenClusterResponse, pinkFloydClusterResponse}, - }, - } - - for _, c := range testCases { - s.Run(c.name, func() { - clusterResponse, err := s.service.GetClustersForPermissions(testCtx, &v1.GetClustersForPermissionsRequest{ - Pagination: c.pagination, - Permissions: []string{}, - }) - s.NoError(err) - protoassert.SlicesEqual(s.T(), clusterResponse.GetClusters(), c.expectedClusters) - }) - } -} - -func (s *serviceImplTestSuite) TestGetNamespacesForClusterAndPermissions() { - queenClusterID := s.clusterNameToIDMap[clusterQueen.GetName()] - testResourceScope1 := getTestResourceScopeSingleNamespace( - queenClusterID, - namespaceInnuendoInClusterQueen.GetName()) - pinkFloydClusterID := s.clusterNameToIDMap[clusterPinkFloyd.GetName()] - testResourceScope2 := getTestResourceScopeSingleNamespace( - pinkFloydClusterID, - namespaceTheWallInClusterPinkFloyd.GetName()) - testResourceScope3 := getTestResourceScopeSingleNamespace( - queenClusterID, - namespaceQueenInClusterQueen.GetName()) - testScopeMap := sac.TestScopeMap{ - storage.Access_READ_ACCESS: map[permissions.Resource]*sac.TestResourceScope{ - resources.Integration.GetResource(): { - Included: true, - }, - resources.Node.GetResource(): testResourceScope1, - resources.Deployment.GetResource(): testResourceScope1, - resources.NetworkGraph.GetResource(): testResourceScope2, - resources.Image.GetResource(): testResourceScope3, - }, - } - - queenQueenNamespaceResponse := &v1.ScopeObject{ - Id: getNamespaceID(namespaceQueenInClusterQueen.GetName()), - Name: namespaceQueenInClusterQueen.GetName(), - } - - queenInnuendoNamespaceResponse := &v1.ScopeObject{ - Id: getNamespaceID(namespaceInnuendoInClusterQueen.GetName()), - Name: namespaceInnuendoInClusterQueen.GetName(), - } - - pinkFloydTheWallNamespaceResponse := &v1.ScopeObject{ - Id: getNamespaceID(namespaceTheWallInClusterPinkFloyd.GetName()), - Name: namespaceTheWallInClusterPinkFloyd.GetName(), - } - - testCases := []struct { - name string - testedClusterID string - testedPermissions []string - expectedNamespaces []*v1.ScopeObject - }{ - { - name: "Global permission (Not Granted) gets no namespace data.", - testedClusterID: s.clusterNameToIDMap[clusterQueen.GetName()], - testedPermissions: []string{rolePermission}, - expectedNamespaces: []*v1.ScopeObject{}, - }, - { - name: "Global permission (Granted) gets no namespace data.", - testedClusterID: s.clusterNameToIDMap[clusterQueen.GetName()], - testedPermissions: []string{integrationPermission}, - expectedNamespaces: []*v1.ScopeObject{}, - }, - { - name: "Not granted Cluster scoped permission gets no namespace data.", - testedClusterID: s.clusterNameToIDMap[clusterQueen.GetName()], - testedPermissions: []string{clusterPermission}, - expectedNamespaces: []*v1.ScopeObject{}, - }, - { - name: "Granted Cluster scoped permission gets no namespace data.", - testedClusterID: s.clusterNameToIDMap[clusterQueen.GetName()], - testedPermissions: []string{nodePermission}, - expectedNamespaces: []*v1.ScopeObject{}, - }, - { - name: "Not granted Namespace scoped permission gets no namespace data.", - testedClusterID: s.clusterNameToIDMap[clusterQueen.GetName()], - testedPermissions: []string{namespacePermission}, - expectedNamespaces: []*v1.ScopeObject{}, - }, - { - name: "Granted Namespace scoped permission gets only namespace data for namespaces in cluster and permission scope.", - testedClusterID: s.clusterNameToIDMap[clusterQueen.GetName()], - testedPermissions: []string{deploymentPermission}, - expectedNamespaces: []*v1.ScopeObject{queenInnuendoNamespaceResponse}, - }, - { - name: "Granted Namespace scoped permission gets only namespace data for namespaces in cluster and permission scope (other permission).", - testedClusterID: s.clusterNameToIDMap[clusterPinkFloyd.GetName()], - testedPermissions: []string{networkGraphPermission}, - expectedNamespaces: []*v1.ScopeObject{pinkFloydTheWallNamespaceResponse}, - }, - { - name: "Multiple not granted Namespace scoped permissions get no namespace data.", - testedClusterID: s.clusterNameToIDMap[clusterQueen.GetName()], - testedPermissions: []string{namespacePermission, deploymentExtensionPermission}, - expectedNamespaces: []*v1.ScopeObject{}, - }, - { - name: "Multiple Namespace scoped permissions get only namespace data for namespaces in granted permission scopes.", - testedClusterID: s.clusterNameToIDMap[clusterQueen.GetName()], - testedPermissions: []string{namespacePermission, deploymentPermission}, - expectedNamespaces: []*v1.ScopeObject{queenInnuendoNamespaceResponse}, - }, - { - name: "empty permission list get namespace data for all namespaces in scope of target cluster and granted namespace permissions.", - testedClusterID: s.clusterNameToIDMap[clusterQueen.GetName()], - testedPermissions: []string{}, - expectedNamespaces: []*v1.ScopeObject{queenQueenNamespaceResponse, queenInnuendoNamespaceResponse}, - }, - } - - testCtx := sac.WithGlobalAccessScopeChecker(context.Background(), - sac.TestScopeCheckerCoreFromFullScopeMap(s.T(), testScopeMap)) - - for _, c := range testCases { - s.Run(c.name, func() { - request := &v1.GetNamespaceForClusterAndPermissionsRequest{ - ClusterId: c.testedClusterID, - Permissions: c.testedPermissions, - } - namespaceResponse, err := s.service.GetNamespacesForClusterAndPermissions(testCtx, request) - s.NoError(err) - protoassert.ElementsMatch(s.T(), namespaceResponse.GetNamespaces(), c.expectedNamespaces) - }) - } -} - -func getValidRole(name string) *storage.Role { - permissionSetID := accesscontrol.DefaultPermissionSetIDs[accesscontrol.Admin] - scopeID := accesscontrol.DefaultAccessScopeIDs[accesscontrol.UnrestrictedAccessScope] - return &storage.Role{ - Name: name, - Description: fmt.Sprintf("Test role for %s", name), - PermissionSetId: permissionSetID, - AccessScopeId: scopeID, - Traits: nil, - } -} - -func (s *serviceImplTestSuite) TestCreateRoleValidAccessScopeID() { - ctx := sac.WithAllAccess(context.Background()) - roleName := "TestCreateRoleValidAccessScopeID" - - ps := s.createPermissionSet(roleName) - scope := s.createAccessScope(roleName) - - role := getValidRole(roleName) - role.PermissionSetId = ps.GetId() - role.AccessScopeId = scope.GetId() - createRoleRequest := &v1.CreateRoleRequest{ - Name: roleName, - Role: role, - } - _, err := s.service.CreateRole(ctx, createRoleRequest) - s.NoError(err) - s.storedRoleNames = append(s.storedRoleNames, role.GetName()) -} - -func (s *serviceImplTestSuite) TestCreateRoleEmptyAccessScopeID() { - ctx := sac.WithAllAccess(context.Background()) - roleName := "TestCreateRoleEmptyAccessScopeID" - - ps := s.createPermissionSet(roleName) - - role := getValidRole(roleName) - role.PermissionSetId = ps.GetId() - role.AccessScopeId = "" - createRoleRequest := &v1.CreateRoleRequest{ - Name: roleName, - Role: role, - } - _, err := s.service.CreateRole(ctx, createRoleRequest) - s.ErrorContains(err, "role access_scope_id field must be set") -} - -func (s *serviceImplTestSuite) TestUpdateExistingRoleValidAccessScopeID() { - ctx := sac.WithAllAccess(context.Background()) - role := s.createRole("TestUpdateExistingRoleValidAccessScopeID") - newScope := s.createAccessScope("new scope") - role.AccessScopeId = newScope.GetId() - _, err := s.service.UpdateRole(ctx, role) - s.NoError(err) -} - -func (s *serviceImplTestSuite) TestUpdateExistingRoleEmptyAccessScopeID() { - ctx := sac.WithAllAccess(context.Background()) - roleName := "TestUpdateExistingRoleEmptyAccessScopeID" - role := s.createRole(roleName) - role.AccessScopeId = "" - _, err := s.service.UpdateRole(ctx, role) - s.ErrorContains(err, "role access_scope_id field must be set") -} - -func (s *serviceImplTestSuite) TestUpdateMissingRoleValidAccessScopeID() { - ctx := sac.WithAllAccess(context.Background()) - roleName := "TestUpdateMissingRoleValidAccessScopeID" - ps := s.createPermissionSet(roleName) - scope := s.createAccessScope(roleName) - role := getValidRole(roleName) - role.PermissionSetId = ps.GetId() - role.AccessScopeId = scope.GetId() - _, err := s.service.UpdateRole(ctx, role) - s.ErrorIs(err, errox.NotFound) -} - -func (s *serviceImplTestSuite) TestUpdateMissingRoleEmptyAccessScopeID() { - ctx := sac.WithAllAccess(context.Background()) - roleName := "TestUpdateMissingRoleEmptyAccessScopeID" - ps := s.createPermissionSet(roleName) - role := getValidRole(roleName) - role.PermissionSetId = ps.GetId() - role.AccessScopeId = "" - _, err := s.service.UpdateRole(ctx, role) - s.ErrorContains(err, "role access_scope_id field must be set") -} - -func (s *serviceImplTestSuite) createAccessScope(name string) *storage.SimpleAccessScope { - ctx := sac.WithAllAccess(context.Background()) - scope := &storage.SimpleAccessScope{ - Name: name, - Description: fmt.Sprintf("Test access scope for %s", name), - Rules: &storage.SimpleAccessScope_Rules{ - IncludedClusters: []string{"test"}, - }, - Traits: nil, - } - postedScope, postErr := s.service.PostSimpleAccessScope(ctx, scope) - s.Require().NoError(postErr) - s.storedAccessScopeIDs = append(s.storedAccessScopeIDs, postedScope.GetId()) - return postedScope -} - -func (s *serviceImplTestSuite) deleteAccessScope(id string) { - ctx := sac.WithAllAccess(context.Background()) - request := &v1.ResourceByID{ - Id: id, - } - _, deleteErr := s.service.DeleteSimpleAccessScope(ctx, request) - s.Require().NoError(deleteErr) -} - -func (s *serviceImplTestSuite) createPermissionSet(name string) *storage.PermissionSet { - ctx := sac.WithAllAccess(context.Background()) - permissionSet := &storage.PermissionSet{ - Name: name, - Description: fmt.Sprintf("Test permission set for %s", name), - ResourceToAccess: nil, - Traits: nil, - } - ps, postErr := s.service.PostPermissionSet(ctx, permissionSet) - s.Require().NoError(postErr) - s.storedPermissionSetIDs = append(s.storedPermissionSetIDs, ps.GetId()) - return ps -} - -func (s *serviceImplTestSuite) deletePermissionSet(id string) { - ctx := sac.WithAllAccess(context.Background()) - request := &v1.ResourceByID{ - Id: id, - } - _, deleteErr := s.service.DeletePermissionSet(ctx, request) - s.Require().NoError(deleteErr) -} - -func (s *serviceImplTestSuite) createRole(roleName string) *storage.Role { - ctx := sac.WithAllAccess(context.Background()) - - ps := s.createPermissionSet(roleName) - scope := s.createAccessScope(roleName) - - createRoleRequest := &v1.CreateRoleRequest{ - Name: roleName, - Role: getValidRole(roleName), - } - createRoleRequest.Role.PermissionSetId = ps.GetId() - createRoleRequest.Role.AccessScopeId = scope.GetId() - - _, createErr := s.service.CreateRole(ctx, createRoleRequest) - s.Require().NoError(createErr) - s.storedRoleNames = append(s.storedRoleNames, roleName) - - readRoleRequest := &v1.ResourceByID{ - Id: roleName, - } - role, readErr := s.service.GetRole(ctx, readRoleRequest) - s.Require().NoError(readErr) - return role -} - -func (s *serviceImplTestSuite) deleteRole(name string) { - ctx := sac.WithAllAccess(context.Background()) - request := &v1.ResourceByID{ - Id: name, - } - _, deleteErr := s.service.DeleteRole(ctx, request) - s.Require().NoError(deleteErr) -} - func TestGetMyPermissions(t *testing.T) { suite.Run(t, new(roleServiceGetMyPermissionsTestSuite)) } diff --git a/central/role/service/test_helpers_test.go b/central/role/service/test_helpers_test.go new file mode 100644 index 0000000000000..65db4100ea652 --- /dev/null +++ b/central/role/service/test_helpers_test.go @@ -0,0 +1,240 @@ +//go:build sql_integration + +package service + +import ( + "fmt" + "testing" + "time" + + clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" + namespaceDataStore "github.com/stackrox/rox/central/namespace/datastore" + rolePkg "github.com/stackrox/rox/central/role" + roleDataStore "github.com/stackrox/rox/central/role/datastore" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/declarativeconfig" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/require" +) + +const ( + namespaceUUIDNamespace = "namespace" +) + +var ( + nilTraits *storage.Traits = nil + + imperativeOriginTraits = &storage.Traits{Origin: storage.Traits_IMPERATIVE} + + declarativeOriginTraits = &storage.Traits{Origin: storage.Traits_DECLARATIVE} + + orphanedDeclarativeOriginTraits = &storage.Traits{Origin: storage.Traits_DECLARATIVE_ORPHANED} + + ephemeralOriginTraits = &storage.Traits{Origin: storage.Traits_EPHEMERAL} +) + +type serviceImplTester struct { + postgres *pgtest.TestPostgres + service Service + + roleStore roleDataStore.DataStore + clusterStore clusterDataStore.DataStore + namespaceStore namespaceDataStore.DataStore + + storedClusterIDs []string + storedNamespaceIDs []string + clusterNameToIDMap map[string]string + + storedPermissionSetIDs []string + storedAccessScopeIDs []string + storedRoleNames []string +} + +func (s *serviceImplTester) Setup(t *testing.T) { + if s == nil { + return + } + + var err error + s.postgres = pgtest.ForT(t) + require.NotNil(t, s.postgres) + s.roleStore = roleDataStore.GetTestPostgresDataStore(t, s.postgres.DB) + clusterStore, err := clusterDataStore.GetTestPostgresDataStore(t, s.postgres.DB) + require.NoError(t, err) + s.clusterStore = clusterStore + namespaceStore, err := namespaceDataStore.GetTestPostgresDataStore(t, s.postgres.DB) + require.NoError(t, err) + s.namespaceStore = namespaceStore + + s.service = New(s.roleStore, s.clusterStore, s.namespaceStore) +} + +func (s *serviceImplTester) SetupTest(t *testing.T) { + if s == nil { + return + } + + s.storedAccessScopeIDs = make([]string, 0) + s.storedPermissionSetIDs = make([]string, 0) + s.storedRoleNames = make([]string, 0) + + s.storedClusterIDs = make([]string, 0) + s.storedNamespaceIDs = make([]string, 0) + s.clusterNameToIDMap = make(map[string]string, 0) + + writeCtx := sac.WithAllAccess(t.Context()) + + for _, cluster := range storageClusters { + clusterToAdd := cluster.CloneVT() + clusterToAdd.Id = "" + clusterToAdd.MainImage = "quay.io/rhacs-eng/main:latest" + id, err := s.clusterStore.AddCluster(writeCtx, clusterToAdd) + require.NoError(t, err) + s.clusterNameToIDMap[clusterToAdd.GetName()] = id + s.storedClusterIDs = append(s.storedClusterIDs, id) + } + + for _, namespace := range storageNamespaces { + ns := namespace.CloneVT() + ns.Id = getNamespaceID(ns.GetName()) + ns.ClusterId = s.clusterNameToIDMap[ns.GetClusterName()] + require.NoError(t, s.namespaceStore.AddNamespace(writeCtx, ns)) + s.storedNamespaceIDs = append(s.storedNamespaceIDs, ns.GetId()) + } +} + +func (s *serviceImplTester) TearDownTest(t *testing.T) { + if s == nil { + return + } + + writeCtx := sac.WithAllAccess(t.Context()) + for _, clusterID := range s.storedClusterIDs { + doneSignal := concurrency.NewSignal() + require.NoError(t, s.clusterStore.RemoveCluster(writeCtx, clusterID, &doneSignal)) + require.Eventually(t, + func() bool { return doneSignal.IsDone() }, + 5*time.Second, + 10*time.Millisecond, + ) + } + s.storedClusterIDs = s.storedClusterIDs[:0] + for _, namespaceID := range s.storedNamespaceIDs { + require.NoError(t, s.namespaceStore.RemoveNamespace(writeCtx, namespaceID)) + } + for _, roleName := range s.storedRoleNames { + s.deleteRole(t, roleName) + } + for _, permissionSetID := range s.storedPermissionSetIDs { + s.deletePermissionSet(t, permissionSetID) + } + for _, accessScopeID := range s.storedAccessScopeIDs { + s.deleteAccessScope(t, accessScopeID) + } +} + +func (s *serviceImplTester) createRole(t *testing.T, roleName string, traits *storage.Traits) *storage.Role { + ctx := sac.WithAllAccess(t.Context()) + ctx = declarativeconfig.WithModifyDeclarativeOrImperative(ctx) + + ps := s.createPermissionSet(t, roleName, traits) + scope := s.createAccessScope(t, roleName, traits) + + role := &storage.Role{ + Name: roleName, + Description: fmt.Sprintf("Test role for %s", roleName), + PermissionSetId: ps.GetId(), + AccessScopeId: scope.GetId(), + Traits: traits, + } + + err := s.roleStore.AddRole(ctx, role) + require.NoError(t, err) + s.storedRoleNames = append(s.storedRoleNames, roleName) + + return role +} + +func (s *serviceImplTester) deleteRole(t *testing.T, roleName string) { + if s == nil { + return + } + + ctx := sac.WithAllAccess(t.Context()) + ctx = declarativeconfig.WithModifyDeclarativeOrImperative(ctx) + request := &v1.ResourceByID{ + Id: roleName, + } + _, deleteErr := s.service.DeleteRole(ctx, request) + require.NoError(t, deleteErr) +} + +func (s *serviceImplTester) createPermissionSet(t *testing.T, name string, traits *storage.Traits) *storage.PermissionSet { + ctx := sac.WithAllAccess(t.Context()) + ctx = declarativeconfig.WithModifyDeclarativeOrImperative(ctx) + permissionSet := &storage.PermissionSet{ + Name: name, + Description: fmt.Sprintf("Test permission set for %s", name), + ResourceToAccess: nil, + Traits: traits, + } + permissionSet.Id = rolePkg.GeneratePermissionSetID() + err := s.roleStore.UpsertPermissionSet(ctx, permissionSet) + require.NoError(t, err) + s.storedPermissionSetIDs = append(s.storedPermissionSetIDs, permissionSet.GetId()) + return permissionSet +} + +func (s *serviceImplTester) deletePermissionSet(t *testing.T, permissionSetID string) { + if s == nil { + return + } + + ctx := sac.WithAllAccess(t.Context()) + ctx = declarativeconfig.WithModifyDeclarativeOrImperative(ctx) + request := &v1.ResourceByID{ + Id: permissionSetID, + } + _, deleteErr := s.service.DeletePermissionSet(ctx, request) + require.NoError(t, deleteErr) +} + +func (s *serviceImplTester) createAccessScope(t *testing.T, name string, traits *storage.Traits) *storage.SimpleAccessScope { + ctx := sac.WithAllAccess(t.Context()) + ctx = declarativeconfig.WithModifyDeclarativeOrImperative(ctx) + scope := &storage.SimpleAccessScope{ + Name: name, + Description: fmt.Sprintf("Test access scope for %s", name), + Rules: &storage.SimpleAccessScope_Rules{ + IncludedClusters: []string{"test"}, + }, + Traits: traits, + } + scope.Id = rolePkg.GenerateAccessScopeID() + err := s.roleStore.UpsertAccessScope(ctx, scope) + require.NoError(t, err) + s.storedAccessScopeIDs = append(s.storedAccessScopeIDs, scope.GetId()) + return scope +} + +func (s *serviceImplTester) deleteAccessScope(t *testing.T, accessScopeID string) { + if s == nil { + return + } + + ctx := sac.WithAllAccess(t.Context()) + ctx = declarativeconfig.WithModifyDeclarativeOrImperative(ctx) + request := &v1.ResourceByID{ + Id: accessScopeID, + } + _, deleteErr := s.service.DeleteSimpleAccessScope(ctx, request) + require.NoError(t, deleteErr) +} + +func getNamespaceID(namespaceName string) string { + return uuid.NewV5FromNonUUIDs(namespaceUUIDNamespace, namespaceName).String() +} diff --git a/central/role/validate.go b/central/role/validate.go index 07f72c2d2c76f..a55364615ebaf 100644 --- a/central/role/validate.go +++ b/central/role/validate.go @@ -4,8 +4,8 @@ import ( "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/accessscope" "github.com/stackrox/rox/pkg/auth/permissions" + "github.com/stackrox/rox/pkg/sac/accessscope" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/uuid" ) diff --git a/central/role/validate_test.go b/central/role/validate_test.go index 36f1c90dbd3cd..411a7e74f7aa2 100644 --- a/central/role/validate_test.go +++ b/central/role/validate_test.go @@ -274,8 +274,7 @@ func TestValidateSimpleAccessScope(t *testing.T) { for _, tc := range testCasesBad { t.Run(tc.name, func(t *testing.T) { err := ValidateSimpleAccessScope(tc.scope) - var target *multierror.Error - if errors.As(err, &target) { + if target, converted := errors.AsType[*multierror.Error](err); converted { assert.Equal(t, tc.expectedNumberOfErrors, target.Len()) } else { assert.Zero(t, tc.expectedNumberOfErrors) diff --git a/central/sac/authorizer/builtin_scoped_authorizer_test.go b/central/sac/authorizer/builtin_scoped_authorizer_test.go index c5e82792eb36f..2f4e6a2b102f2 100644 --- a/central/sac/authorizer/builtin_scoped_authorizer_test.go +++ b/central/sac/authorizer/builtin_scoped_authorizer_test.go @@ -834,3 +834,78 @@ func countAllowedResults(xs []bool) int { } return result } + +func TestNoAddedResources(t *testing.T) { + knownResources := []permissions.ResourceMetadata{ + resources.Access, + resources.Administration, + resources.Alert, + resources.CVE, + resources.Cluster, + resources.Compliance, + resources.Deployment, + resources.DeploymentExtension, + resources.Detection, + resources.Image, + resources.ImageAdministration, + resources.Integration, + resources.K8sRole, + resources.K8sRoleBinding, + resources.K8sSubject, + resources.Namespace, + resources.NetworkGraph, + resources.NetworkPolicy, + resources.Node, + resources.Secret, + resources.ServiceAccount, + resources.VirtualMachine, + resources.VulnerabilityManagementApprovals, + resources.VulnerabilityManagementRequests, + resources.WatchedImage, + resources.WorkflowAdministration, + } + knownDisabledResources := ([]permissions.ResourceMetadata)(nil) + knownInternalResources := []permissions.ResourceMetadata{ + resources.ComplianceOperator, + resources.Hash, + resources.InitBundleMeta, + resources.InstallationInfo, + resources.NetworkEntity, + resources.Notifications, + resources.Version, + resources.VulnerabilityRequest, + } + + const failureMessage = "" + + "Please contact the Sensor & Ecosystems team to validate " + + "your addition of resource. For more information about " + + "the resource creation process, check " + + "https://github.com/stackrox/stackrox/tree/master/pkg/sac/resources/README.md ." + + for _, metadata := range resources.ListAllMetadata() { + assert.Contains( + t, + knownResources, + metadata, + failureMessage+" "+metadata.String(), + ) + } + for _, metadata := range resources.ListAllDisabledMetadata() { + assert.Contains( + t, + knownDisabledResources, + metadata, + failureMessage, + metadata.String(), + ) + } + for _, metadata := range resources.ListAllInternalMetadata() { + assert.Contains( + t, + knownInternalResources, + metadata, + failureMessage, + metadata.String(), + ) + } +} diff --git a/central/scanner/handler.go b/central/scanner/handler.go index f08694843bea5..9728df49a6a08 100644 --- a/central/scanner/handler.go +++ b/central/scanner/handler.go @@ -93,13 +93,10 @@ func generateFilesForScannerV1(params *apiparams.Scanner, clusterType storage.Cl ClusterType: clusterType, K8sConfig: &renderer.K8sConfig{ CommonConfig: renderer.CommonConfig{ - ScannerImage: stringutils.OrDefault(params.ScannerImage, flavor.ScannerImage()), - ScannerDBImage: stringutils.OrDefault(params.ScannerDBImage, flavor.ScannerDBImage()), ScannerV4Image: stringutils.OrDefault(params.ScannerV4Image, flavor.ScannerV4Image()), ScannerV4DBImage: stringutils.OrDefault(params.ScannerV4DBImage, flavor.ScannerV4DBImage()), }, - OfflineMode: params.OfflineMode, - IstioVersion: params.IstioVersion, + OfflineMode: params.OfflineMode, }, SecretsByteMap: map[string][]byte{ "ca.pem": centralCA, diff --git a/central/scrape/scrape_impl.go b/central/scrape/scrape_impl.go index 9bb69efe1e803..313b9873a8572 100644 --- a/central/scrape/scrape_impl.go +++ b/central/scrape/scrape_impl.go @@ -3,6 +3,7 @@ package scrape import ( "errors" "fmt" + "maps" "time" "github.com/stackrox/rox/generated/internalapi/central" @@ -57,9 +58,7 @@ func (s *scrapeImpl) GetResults() map[string]*compliance.ComplianceReturn { defer s.lock.RUnlock() ret := make(map[string]*compliance.ComplianceReturn, len(s.results)) - for id, cr := range s.results { - ret[id] = cr - } + maps.Copy(ret, s.results) return ret } diff --git a/central/search/category.go b/central/search/category.go index c7728996d8b08..418f0eb32a2ce 100644 --- a/central/search/category.go +++ b/central/search/category.go @@ -8,6 +8,13 @@ import ( // GetGlobalSearchCategories returns a set of search categories func GetGlobalSearchCategories() set.Set[v1.SearchCategory] { // globalSearchCategories is exposed for e2e options test + // + // IMAGES_V2 is intentionally omitted. With FlattenImageData enabled, IMAGES is + // already routed to the imagesV2 store (see central/search/service), so adding + // IMAGES_V2 here would run the same image search twice and return every image + // twice (both labeled IMAGES). IMAGES remains the public category used by the UI + // and API clients; IMAGES_V2 is the internal storage/schema category and is still + // searchable when requested explicitly. globalSearchCategories := set.NewSet( v1.SearchCategory_ALERTS, v1.SearchCategory_CLUSTERS, diff --git a/central/search/service/service_impl.go b/central/search/service/service_impl.go index 34f4dc8a971af..1bde76f53169f 100644 --- a/central/search/service/service_impl.go +++ b/central/search/service/service_impl.go @@ -3,7 +3,6 @@ package service import ( "context" "fmt" - "sort" "strconv" "strings" @@ -66,7 +65,6 @@ var ( type autocompleteResult struct { value string - score float64 } // SearchFunc represents a function that goes from a query to a proto search result. @@ -170,7 +168,7 @@ func handleMatch(fieldPath, value string) string { return value } -func handleMapResults(matches map[string][]string, score float64) []autocompleteResult { +func handleMapResults(matches map[string][]string) []autocompleteResult { var keys []string var values []string for k, match := range matches { @@ -182,7 +180,7 @@ func handleMapResults(matches map[string][]string, score float64) []autocomplete } results := make([]autocompleteResult, 0, len(keys)) for i := 0; i < len(keys); i++ { - results = append(results, autocompleteResult{value: fmt.Sprintf("%s=%s", keys[i], values[i]), score: score}) + results = append(results, autocompleteResult{value: fmt.Sprintf("%s=%s", keys[i], values[i])}) } return results } @@ -267,20 +265,19 @@ func RunAutoComplete(ctx context.Context, queryString string, categories []v1.Se // // This implies that the object is a map because it has multiple values if isMapMatch(matches) { - autocompleteResults = append(autocompleteResults, handleMapResults(matches, r.Score)...) + autocompleteResults = append(autocompleteResults, handleMapResults(matches)...) continue } for fieldPath, match := range matches { for _, v := range match { value := handleMatch(fieldPath, v) - autocompleteResults = append(autocompleteResults, autocompleteResult{value: value, score: r.Score}) + autocompleteResults = append(autocompleteResults, autocompleteResult{value: value}) } } } } - sort.Slice(autocompleteResults, func(i, j int) bool { return autocompleteResults[i].score > autocompleteResults[j].score }) resultSet := set.NewStringSet() var stringResults []string @@ -382,8 +379,6 @@ func GlobalSearch(ctx context.Context, query string, categories []v1.SearchCateg counts = append(counts, &v1.SearchResponse_Count{Category: category, Count: int64(len(resultsFromCategory))}) results = append(results, resultsFromCategory...) } - // Sort from highest score to lowest - sort.SliceStable(results, func(i, j int) bool { return results[i].GetScore() > results[j].GetScore() }) return } diff --git a/central/search/service/service_impl_test.go b/central/search/service/service_impl_test.go index 25d1ba2d88d43..025e495188e3c 100644 --- a/central/search/service/service_impl_test.go +++ b/central/search/service/service_impl_test.go @@ -52,6 +52,15 @@ func TestSearchCategoryToOptionsMultiMap(t *testing.T) { } } +// TestPoliciesCategoryAutocompleteFieldPath verifies that the POLICIES multimap resolves "Category" +// to the PolicyCategoryName field path so autocomplete works via the policy_categories join. +func TestPoliciesCategoryAutocompleteFieldPath(t *testing.T) { + policiesMultimap := categoryToOptionsMultimap[v1.SearchCategory_POLICIES] + fields := policiesMultimap.GetAll(search.Category.String()) + assert.Len(t, fields, 1, `POLICIES multimap must have exactly one field for the "Category" key`) + assert.Equal(t, "policycategory.name", fields[0].GetFieldPath()) +} + func TestGetSearchFuncs_FlattenImageDataRoutesImageSearch(t *testing.T) { ctx := context.Background() q := &v1.Query{} @@ -249,6 +258,7 @@ func TestSearchFuncs(t *testing.T) { WithAlertStore(alertMocks.NewMockDataStore(mockCtrl)). WithDeploymentStore(deploymentMocks.NewMockDataStore(mockCtrl)). WithImageStore(imageMocks.NewMockDataStore(mockCtrl)). + WithImageV2Store(imageV2DatastoreMocks.NewMockDataStore(mockCtrl)). WithPolicyStore(policyMocks.NewMockDataStore(mockCtrl)). WithSecretStore(secretMocks.NewMockDataStore(mockCtrl)). WithServiceAccountStore(serviceAccountMocks.NewMockDataStore(mockCtrl)). @@ -347,6 +357,7 @@ func (s *SearchOperationsTestSuite) TestAutocomplete() { WithAlertStore(alertMocks.NewMockDataStore(s.mockCtrl)). WithDeploymentStore(deploymentDS). WithImageStore(imageMocks.NewMockDataStore(s.mockCtrl)). + WithImageV2Store(imageV2DatastoreMocks.NewMockDataStore(s.mockCtrl)). WithPolicyStore(policyMocks.NewMockDataStore(s.mockCtrl)). WithSecretStore(secretMocks.NewMockDataStore(s.mockCtrl)). WithServiceAccountStore(serviceAccountMocks.NewMockDataStore(s.mockCtrl)). @@ -451,6 +462,7 @@ func (s *SearchOperationsTestSuite) TestAutocompleteForEnums() { WithAlertStore(alertMocks.NewMockDataStore(s.mockCtrl)). WithDeploymentStore(deploymentMocks.NewMockDataStore(s.mockCtrl)). WithImageStore(imageMocks.NewMockDataStore(s.mockCtrl)). + WithImageV2Store(imageV2DatastoreMocks.NewMockDataStore(s.mockCtrl)). WithPolicyStore(ds). WithSecretStore(secretMocks.NewMockDataStore(s.mockCtrl)). WithServiceAccountStore(serviceAccountMocks.NewMockDataStore(s.mockCtrl)). @@ -502,6 +514,7 @@ func (s *SearchOperationsTestSuite) TestAutocompleteAuthz() { WithAlertStore(alertsDS). WithDeploymentStore(deploymentDS). WithImageStore(imageMocks.NewMockDataStore(s.mockCtrl)). + WithImageV2Store(imageV2DatastoreMocks.NewMockDataStore(s.mockCtrl)). WithPolicyStore(policyMocks.NewMockDataStore(s.mockCtrl)). WithSecretStore(secretMocks.NewMockDataStore(s.mockCtrl)). WithServiceAccountStore(serviceAccountMocks.NewMockDataStore(s.mockCtrl)). @@ -573,6 +586,7 @@ func (s *SearchOperationsTestSuite) TestSearchAuthz() { WithAlertStore(alertsDS). WithDeploymentStore(deploymentDS). WithImageStore(imageMocks.NewMockDataStore(s.mockCtrl)). + WithImageV2Store(imageV2DatastoreMocks.NewMockDataStore(s.mockCtrl)). WithPolicyStore(policyMocks.NewMockDataStore(s.mockCtrl)). WithSecretStore(secretMocks.NewMockDataStore(s.mockCtrl)). WithServiceAccountStore(serviceAccountMocks.NewMockDataStore(s.mockCtrl)). diff --git a/central/secret/datastore/datastore_impl.go b/central/secret/datastore/datastore_impl.go index 5c404752284f4..11d4af7a6bb38 100644 --- a/central/secret/datastore/datastore_impl.go +++ b/central/secret/datastore/datastore_impl.go @@ -9,18 +9,24 @@ import ( v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/postgres" + pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" pkgSearch "github.com/stackrox/rox/pkg/search" - "github.com/stackrox/rox/pkg/secret/convert" + "github.com/stackrox/rox/pkg/search/paginated" + pgSearch "github.com/stackrox/rox/pkg/search/postgres" ) type datastoreImpl struct { storage store.Store + // TODO(ROX-31142): No way to call RunSelectRequestForSchemaFn via + // the store so we have to allow for access to the DB here. + db postgres.DB } func newPostgres(db postgres.DB) DataStore { dbStore := pgStore.New(db) return &datastoreImpl{ storage: dbStore, + db: db, } } @@ -62,12 +68,38 @@ func (d *datastoreImpl) SearchSecrets(ctx context.Context, q *v1.Query) ([]*v1.S } func (d *datastoreImpl) SearchListSecrets(ctx context.Context, request *v1.Query) ([]*storage.ListSecret, error) { - results, err := d.Search(ctx, request) + query := request.CloneVT() + if query == nil { + query = pkgSearch.EmptyQuery() + } + + defaultSort := &v1.QuerySortOption{ + Field: pkgSearch.CreatedTime.String(), + Reversed: false, + } + query = paginated.FillDefaultSortOption(query, defaultSort) + + query.Selects = []*v1.QuerySelect{ + pkgSearch.NewQuerySelect(pkgSearch.SecretID).Proto(), + pkgSearch.NewQuerySelect(pkgSearch.SecretName).Proto(), + pkgSearch.NewQuerySelect(pkgSearch.ClusterID).Proto(), + pkgSearch.NewQuerySelect(pkgSearch.Cluster).Proto(), + pkgSearch.NewQuerySelect(pkgSearch.Namespace).Proto(), + pkgSearch.NewQuerySelect(pkgSearch.CreatedTime).Proto(), + pkgSearch.NewQuerySelect(pkgSearch.SecretType).Proto(), + } + + var listSecrets []*storage.ListSecret + err := pgSearch.RunSelectRequestForSchemaFn(ctx, d.db, pkgSchema.SecretsSchema, query, + func(r *listSecretResponse) error { + listSecrets = append(listSecrets, r.toListSecret()) + return nil + }) if err != nil { return nil, err } - secrets, _, err := d.resultsToListSecrets(ctx, results) - return secrets, err + + return listSecrets, nil } func (d *datastoreImpl) SearchRawSecrets(ctx context.Context, request *v1.Query) ([]*storage.Secret, error) { @@ -100,21 +132,6 @@ func (d *datastoreImpl) Count(ctx context.Context, q *v1.Query) (int, error) { return d.storage.Count(ctx, q) } -// ToSecrets returns the secrets from the db for the given search results. -func (d *datastoreImpl) resultsToListSecrets(ctx context.Context, results []pkgSearch.Result) ([]*storage.ListSecret, []int, error) { - ids := pkgSearch.ResultsToIDs(results) - - secrets, missingIndices, err := d.storage.GetMany(ctx, ids) - if err != nil { - return nil, nil, err - } - listSecrets := make([]*storage.ListSecret, 0, len(secrets)) - for _, s := range secrets { - listSecrets = append(listSecrets, convert.SecretToSecretList(s)) - } - return listSecrets, missingIndices, nil -} - type SecretSearchResultConverter struct{} func (c *SecretSearchResultConverter) BuildName(result *pkgSearch.Result) string { @@ -129,7 +146,3 @@ func (c *SecretSearchResultConverter) BuildLocation(result *pkgSearch.Result) st func (c *SecretSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_SECRETS } - -func (c *SecretSearchResultConverter) GetScore(result *pkgSearch.Result) float64 { - return result.Score -} diff --git a/central/secret/datastore/datastore_test.go b/central/secret/datastore/datastore_test.go index 393c8ddc19fd3..313887e810a03 100644 --- a/central/secret/datastore/datastore_test.go +++ b/central/secret/datastore/datastore_test.go @@ -5,6 +5,7 @@ package datastore import ( "context" "testing" + "time" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" @@ -12,6 +13,7 @@ import ( "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" @@ -45,6 +47,11 @@ func (suite *SecretDataStoreTestSuite) SetupSuite() { sac.ResourceScopeKeys(resources.Secret))) } +func (suite *SecretDataStoreTestSuite) TearDownTest() { + _, err := suite.pool.Exec(suite.ctx, "TRUNCATE TABLE secrets CASCADE") + suite.Require().NoError(err) +} + func (suite *SecretDataStoreTestSuite) TearDownSuite() { suite.pool.Close() } @@ -78,6 +85,169 @@ func (suite *SecretDataStoreTestSuite) assertSearchResults(q *v1.Query, s *stora } } +func (suite *SecretDataStoreTestSuite) TestSearchListSecrets() { + secret1 := fixtures.GetSecret() + secret1.Id = uuid.NewV4().String() + secret1.Name = "test-secret-1" + secret1.Namespace = "default" + secret1.Files = []*storage.SecretDataFile{ + {Type: storage.SecretType_PUBLIC_CERTIFICATE}, + {Type: storage.SecretType_RSA_PRIVATE_KEY}, + } + + secret2 := fixtures.GetSecret() + secret2.Id = uuid.NewV4().String() + secret2.Name = "test-secret-2" + secret2.Namespace = "kube-system" + secret2.Files = []*storage.SecretDataFile{ + {Type: storage.SecretType_IMAGE_PULL_SECRET}, + } + + suite.NoError(suite.datastore.UpsertSecret(suite.ctx, secret1)) + suite.NoError(suite.datastore.UpsertSecret(suite.ctx, secret2)) + + // Test retrieval with empty query + results, err := suite.datastore.SearchListSecrets(suite.ctx, search.EmptyQuery()) + suite.NoError(err) + suite.Equal(len(results), 2) + + // Find our test secrets + var found1, found2 *storage.ListSecret + for _, r := range results { + if r.GetId() == secret1.GetId() { + found1 = r + } else if r.GetId() == secret2.GetId() { + found2 = r + } + } + + suite.NotNil(found1) + suite.Equal(secret1.GetName(), found1.GetName()) + suite.Equal(secret1.GetNamespace(), found1.GetNamespace()) + suite.ElementsMatch( + []storage.SecretType{storage.SecretType_PUBLIC_CERTIFICATE, storage.SecretType_RSA_PRIVATE_KEY}, + found1.GetTypes(), + ) + + suite.NotNil(found2) + suite.Equal([]storage.SecretType{storage.SecretType_IMAGE_PULL_SECRET}, found2.GetTypes()) +} + +func (suite *SecretDataStoreTestSuite) TestSearchListSecrets_NoFiles() { + // Test secret with no files (should return UNDETERMINED type) + secret := fixtures.GetSecret() + secret.Id = uuid.NewV4().String() + secret.Name = "empty-secret" + secret.Files = nil // No files + + suite.NoError(suite.datastore.UpsertSecret(suite.ctx, secret)) + + query := search.NewQueryBuilder().AddStrings(search.SecretID, secret.GetId()).ProtoQuery() + results, err := suite.datastore.SearchListSecrets(suite.ctx, query) + + suite.NoError(err) + suite.Len(results, 1) + suite.Equal([]storage.SecretType{storage.SecretType_UNDETERMINED}, results[0].GetTypes()) +} + +func (suite *SecretDataStoreTestSuite) TestSearchListSecrets_WithFilter() { + // Test that search filters work correctly with single-pass query + secret1 := fixtures.GetSecret() + secret1.Id = uuid.NewV4().String() + secret1.Name = "filter-test-1" + secret1.Namespace = "default" + + secret2 := fixtures.GetSecret() + secret2.Id = uuid.NewV4().String() + secret2.Name = "filter-test-2" + secret2.Namespace = "kube-system" + + suite.NoError(suite.datastore.UpsertSecret(suite.ctx, secret1)) + suite.NoError(suite.datastore.UpsertSecret(suite.ctx, secret2)) + + // Filter by namespace + query := search.NewQueryBuilder().AddExactMatches(search.Namespace, "kube-system").ProtoQuery() + results, err := suite.datastore.SearchListSecrets(suite.ctx, query) + + suite.NoError(err) + // Find our test secret in results + var found *storage.ListSecret + for _, r := range results { + if r.GetId() == secret2.GetId() { + found = r + break + } + } + suite.NotNil(found) + suite.Equal("kube-system", found.GetNamespace()) +} + +func (suite *SecretDataStoreTestSuite) TestSearchListSecrets_DuplicateTypes() { + // Test that framework correctly deduplicates types via jsonb_agg + secret := fixtures.GetSecret() + secret.Id = uuid.NewV4().String() + secret.Name = "duplicate-types-secret" + secret.Files = []*storage.SecretDataFile{ + {Type: storage.SecretType_PUBLIC_CERTIFICATE}, + {Type: storage.SecretType_PUBLIC_CERTIFICATE}, // Duplicate + {Type: storage.SecretType_RSA_PRIVATE_KEY}, + } + + suite.NoError(suite.datastore.UpsertSecret(suite.ctx, secret)) + + query := search.NewQueryBuilder().AddStrings(search.SecretID, secret.GetId()).ProtoQuery() + results, err := suite.datastore.SearchListSecrets(suite.ctx, query) + + suite.NoError(err) + suite.Len(results, 1) + // Should have only 2 unique types + suite.Len(results[0].GetTypes(), 2) + suite.ElementsMatch( + []storage.SecretType{storage.SecretType_PUBLIC_CERTIFICATE, storage.SecretType_RSA_PRIVATE_KEY}, + results[0].GetTypes(), + ) +} + +func (suite *SecretDataStoreTestSuite) TestSearchListSecrets_DefaultSortOrder() { + now := time.Now() + older := now.Add(-1 * time.Hour) + newer := now.Add(1 * time.Hour) + + secret1 := fixtures.GetSecret() + secret1.Id = uuid.NewV4().String() + secret1.Name = "newer-secret" + secret1.CreatedAt = protocompat.ConvertTimeToTimestampOrNil(&newer) + + secret2 := fixtures.GetSecret() + secret2.Id = uuid.NewV4().String() + secret2.Name = "older-secret" + secret2.CreatedAt = protocompat.ConvertTimeToTimestampOrNil(&older) + + // Insert newer first to ensure ordering comes from sort, not insertion order + suite.NoError(suite.datastore.UpsertSecret(suite.ctx, secret1)) + suite.NoError(suite.datastore.UpsertSecret(suite.ctx, secret2)) + + // Default sort is CreatedTime ascending + results, err := suite.datastore.SearchListSecrets(suite.ctx, search.EmptyQuery()) + suite.NoError(err) + suite.Require().Len(results, 2) + suite.Equal(secret2.GetId(), results[0].GetId(), "older secret should be first in ascending order") + suite.Equal(secret1.GetId(), results[1].GetId(), "newer secret should be second in ascending order") + + // Caller-provided descending sort should override default + q := search.EmptyQuery() + q.Pagination = &v1.QueryPagination{ + SortOptions: []*v1.QuerySortOption{ + {Field: search.CreatedTime.String(), Reversed: true}, + }, + } + results, err = suite.datastore.SearchListSecrets(suite.ctx, q) + suite.NoError(err) + suite.Require().Len(results, 2) + suite.Equal(secret1.GetId(), results[0].GetId(), "newer secret should be first in descending order") + suite.Equal(secret2.GetId(), results[1].GetId(), "older secret should be second in descending order") +} + func (suite *SecretDataStoreTestSuite) TestSecretsDataStore() { secret := fixtures.GetSecret() err := suite.datastore.UpsertSecret(suite.ctx, secret) @@ -187,14 +357,4 @@ func (suite *SecretDataStoreTestSuite) TestSearchSecrets() { } }) } - - // Clean up - suite.NoError(suite.datastore.RemoveSecret(suite.ctx, secret1.GetId())) - suite.NoError(suite.datastore.RemoveSecret(suite.ctx, secret2.GetId())) - suite.NoError(suite.datastore.RemoveSecret(suite.ctx, secret3.GetId())) - - // Verify cleanup - results, err := suite.datastore.SearchSecrets(suite.ctx, search.EmptyQuery()) - suite.NoError(err) - suite.Empty(results) } diff --git a/central/secret/datastore/list_secret_response.go b/central/secret/datastore/list_secret_response.go new file mode 100644 index 0000000000000..b3a555a432984 --- /dev/null +++ b/central/secret/datastore/list_secret_response.go @@ -0,0 +1,37 @@ +package datastore + +import ( + "time" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/protocompat" +) + +// listSecretResponse is a helper struct for scanning ListSecret query results. +type listSecretResponse struct { + ID string `db:"secret_id"` + Name string `db:"secret"` + ClusterID string `db:"cluster_id"` + ClusterName string `db:"cluster"` + Namespace string `db:"namespace"` + CreatedAt *time.Time `db:"created_time"` + Types []storage.SecretType `db:"secret_type"` +} + +// toListSecret converts the database response to a storage.ListSecret protobuf. +func (r *listSecretResponse) toListSecret() *storage.ListSecret { + types := r.Types + if len(types) == 0 { + types = []storage.SecretType{storage.SecretType_UNDETERMINED} + } + + return &storage.ListSecret{ + Id: r.ID, + Name: r.Name, + ClusterId: r.ClusterID, + ClusterName: r.ClusterName, + Namespace: r.Namespace, + CreatedAt: protocompat.ConvertTimeToTimestampOrNil(r.CreatedAt), + Types: types, + } +} diff --git a/central/secret/service/service_impl.go b/central/secret/service/service_impl.go index e60e9773fc140..03bfb93b5e77b 100644 --- a/central/secret/service/service_impl.go +++ b/central/secret/service/service_impl.go @@ -30,6 +30,7 @@ var ( v1.SecretService_GetSecret_FullMethodName, v1.SecretService_CountSecrets_FullMethodName, v1.SecretService_ListSecrets_FullMethodName, + v1.SecretService_ListSecretsExtended_FullMethodName, }, }) ) @@ -57,16 +58,15 @@ func (s *serviceImpl) AuthFuncOverride(ctx context.Context, fullMethodName strin return ctx, authorizer.Authorized(ctx, fullMethodName) } -// GetSecret returns the secret for the id. -func (s *serviceImpl) GetSecret(ctx context.Context, request *v1.ResourceByID) (*storage.Secret, error) { - secret, exists, err := s.secrets.GetSecret(ctx, request.GetId()) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.Wrapf(errox.NotFound, "secret with id '%s' does not exist", request.GetId()) - } +// common interface for secrets and list secrets based on identifying fields. +type secretIdentity interface { + GetClusterId() string + GetNamespace() string + GetName() string +} +// getDeploymentRelationships returns the deployment relationships for a secret. +func (s *serviceImpl) getDeploymentRelationships(ctx context.Context, secret secretIdentity) ([]*storage.SecretDeploymentRelationship, error) { psr := search.NewQueryBuilder(). AddExactMatches(search.ClusterID, secret.GetClusterId()). AddExactMatches(search.Namespace, secret.GetNamespace()). @@ -77,7 +77,6 @@ func (s *serviceImpl) GetSecret(ctx context.Context, request *v1.ResourceByID) ( if err != nil { return nil, err } - var deployments []*storage.SecretDeploymentRelationship for _, r := range deploymentResults { deployments = append(deployments, &storage.SecretDeploymentRelationship{ @@ -85,6 +84,24 @@ func (s *serviceImpl) GetSecret(ctx context.Context, request *v1.ResourceByID) ( Name: r.GetName(), }) } + return deployments, nil +} + +// GetSecret returns the secret for the id. +func (s *serviceImpl) GetSecret(ctx context.Context, request *v1.ResourceByID) (*storage.Secret, error) { + secret, exists, err := s.secrets.GetSecret(ctx, request.GetId()) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.Wrapf(errox.NotFound, "secret with id '%s' does not exist", request.GetId()) + } + + deployments, err := s.getDeploymentRelationships(ctx, secret) + if err != nil { + return nil, err + } + secret.Relationship = &storage.SecretRelationship{ DeploymentRelationships: deployments, } @@ -106,20 +123,56 @@ func (s *serviceImpl) CountSecrets(ctx context.Context, request *v1.RawQuery) (* return &v1.CountSecretsResponse{Count: int32(numSecrets)}, nil } -// ListSecrets returns all secrets that match the query. -func (s *serviceImpl) ListSecrets(ctx context.Context, request *v1.RawQuery) (*v1.ListSecretsResponse, error) { +func (s *serviceImpl) listSecrets(ctx context.Context, request *v1.ListSecretsExtendedRequest) ([]*storage.ListSecret, error) { + rawQuery := request.GetQuery() // Fill in query. - parsedQuery, err := search.ParseQuery(request.GetQuery(), search.MatchAllIfEmpty()) + parsedQuery, err := search.ParseQuery(rawQuery.GetQuery(), search.MatchAllIfEmpty()) if err != nil { return nil, errors.Wrap(errox.InvalidArgs, err.Error()) } // Fill in pagination. - paginated.FillPagination(parsedQuery, request.GetPagination(), maxSecretsReturned) + paginated.FillPagination(parsedQuery, rawQuery.GetPagination(), maxSecretsReturned) secrets, err := s.secrets.SearchListSecrets(ctx, parsedQuery) if err != nil { return nil, err } + + // This N+1 query pattern is only used when the caller explicitly requests the relationships, so + // is acceptable in the opt-in case. + if request.GetIncludeRelationships() { + for _, secret := range secrets { + deployments, err := s.getDeploymentRelationships(ctx, secret) + if err != nil { + return nil, err + } + secret.Relationship = &storage.SecretRelationship{ + DeploymentRelationships: deployments, + } + } + } + + return secrets, nil +} + +// ListSecrets returns all secrets that match the query. +func (s *serviceImpl) ListSecrets(ctx context.Context, request *v1.RawQuery) (*v1.ListSecretsResponse, error) { + secrets, err := s.listSecrets(ctx, &v1.ListSecretsExtendedRequest{ + Query: request, + IncludeRelationships: false, + }) + if err != nil { + return nil, err + } + return &v1.ListSecretsResponse{Secrets: secrets}, nil +} + +// ListSecretsExtended returns all secrets that match the query and optionally includes their workload relationships. +func (s *serviceImpl) ListSecretsExtended(ctx context.Context, request *v1.ListSecretsExtendedRequest) (*v1.ListSecretsResponse, error) { + secrets, err := s.listSecrets(ctx, request) + if err != nil { + return nil, err + } return &v1.ListSecretsResponse{Secrets: secrets}, nil } diff --git a/central/secret/service/service_impl_test.go b/central/secret/service/service_impl_test.go index 90cbae49bc2f5..271b1f652556d 100644 --- a/central/secret/service/service_impl_test.go +++ b/central/secret/service/service_impl_test.go @@ -180,3 +180,78 @@ func (suite *SecretServiceTestSuite) TestSearchSecretFailure() { _, actualErr := suite.service.ListSecrets((context.Context)(nil), &v1.RawQuery{}) suite.True(strings.Contains(actualErr.Error(), expectedError.Error())) } + +// Test that when we search secrets without including relationships, the deployment relationships are not queried. +func (suite *SecretServiceTestSuite) TestSearchSecretsExtendedWithoutIncludeRelationships() { + expectedSecrets := []*storage.ListSecret{ + {Id: "id1"}, + } + + emptyWithPag := search.EmptyQuery() + emptyWithPag.Pagination = &v1.QueryPagination{ + Limit: maxSecretsReturned, + } + + suite.mockSecretStore.EXPECT().SearchListSecrets(gomock.Any(), emptyWithPag).Return([]*storage.ListSecret{ + {Id: "id1"}, + }, nil) + suite.mockDeploymentStore.EXPECT().SearchDeployments(gomock.Any(), gomock.Any()).Times(0) + + response, err := suite.service.ListSecretsExtended((context.Context)(nil), &v1.ListSecretsExtendedRequest{IncludeRelationships: false}) + protoassert.SlicesEqual(suite.T(), expectedSecrets, response.GetSecrets()) + suite.NoError(err) +} + +// Test that when we list secrets with including relationships, the deployment relationships are queried. +func (suite *SecretServiceTestSuite) TestSearchSecretsExtendedWithIncludeRelationships() { + emptyWithPag := search.EmptyQuery() + emptyWithPag.Pagination = &v1.QueryPagination{ + Limit: maxSecretsReturned, + } + secret := &storage.ListSecret{ + Id: "id1", + Name: "secretname", + ClusterId: "cluster", + Namespace: "namespace", + } + + suite.mockSecretStore.EXPECT().SearchListSecrets(gomock.Any(), emptyWithPag).Return([]*storage.ListSecret{secret}, nil) + psr := search.NewQueryBuilder(). + AddExactMatches(search.ClusterID, secret.GetClusterId()). + AddExactMatches(search.Namespace, secret.GetNamespace()). + AddExactMatches(search.SecretName, secret.GetName()). + ProtoQuery() + suite.mockDeploymentStore.EXPECT().SearchDeployments(gomock.Any(), psr).Return([]*v1.SearchResult{ + {Id: "d1", Name: "deployment1"}, + }, nil) + + response, err := suite.service.ListSecretsExtended((context.Context)(nil), &v1.ListSecretsExtendedRequest{IncludeRelationships: true}) + + expectedSecrets := []*storage.ListSecret{ + { + Id: "id1", + Name: "secretname", + ClusterId: "cluster", + Namespace: "namespace", + Relationship: &storage.SecretRelationship{DeploymentRelationships: []*storage.SecretDeploymentRelationship{ + {Id: "d1", Name: "deployment1"}, + }}, + }, + } + protoassert.SlicesEqual(suite.T(), expectedSecrets, response.GetSecrets()) + suite.NoError(err) +} + +func (suite *SecretServiceTestSuite) TestSearchSecretsExtendedWithIncludeRelationshipsFailure() { + emptyWithPag := search.EmptyQuery() + emptyWithPag.Pagination = &v1.QueryPagination{ + Limit: maxSecretsReturned, + } + suite.mockSecretStore.EXPECT().SearchListSecrets(gomock.Any(), emptyWithPag).Return([]*storage.ListSecret{ + {Id: "id1"}, + }, nil) + suite.mockDeploymentStore.EXPECT().SearchDeployments(gomock.Any(), gomock.Any()).Return(([]*v1.SearchResult)(nil), errors.New("failure")) + + _, err := suite.service.ListSecretsExtended((context.Context)(nil), &v1.ListSecretsExtendedRequest{IncludeRelationships: true}) + suite.Error(err) +} diff --git a/central/securedclustercertgen/certificates.go b/central/securedclustercertgen/certificates.go index 6207806b61010..0ad193df79a4e 100644 --- a/central/securedclustercertgen/certificates.go +++ b/central/securedclustercertgen/certificates.go @@ -21,6 +21,8 @@ var ( // secretDataMap represents data stored as part of a secret. type secretDataMap = map[string][]byte +// TODO(ROX-36705): Remove Scanner V2 certs once 4.11 goes out of support. +// Older Secured Clusters may still run Scanner V2 and need these certificates. var scannerV2ServiceTypes = set.NewFrozenSet[storage.ServiceType](storage.ServiceType_SCANNER_SERVICE, storage.ServiceType_SCANNER_DB_SERVICE) var scannerV4ServiceTypes = set.NewFrozenSet[storage.ServiceType](storage.ServiceType_SCANNER_V4_INDEXER_SERVICE, storage.ServiceType_SCANNER_V4_DB_SERVICE) var localScannerServiceTypes = scannerV2ServiceTypes.Union(scannerV4ServiceTypes) @@ -129,7 +131,7 @@ func (c *certIssuerImpl) issueCertificates(namespace string, clusterID string) ( var caPem []byte serviceCerts := make([]*storage.TypedServiceCertificate, 0, c.serviceTypes.Cardinality()) - for _, serviceType := range c.serviceTypes.AsSlice() { + for serviceType := range c.serviceTypes.All() { ca, cert, err := c.certificateFor(serviceType, namespace, clusterID) if err != nil { certIssueError = multierror.Append(certIssueError, err) diff --git a/central/securedclustercertgen/certificates_test.go b/central/securedclustercertgen/certificates_test.go index b9fafb6e5c467..01c8014bb1522 100644 --- a/central/securedclustercertgen/certificates_test.go +++ b/central/securedclustercertgen/certificates_test.go @@ -194,7 +194,7 @@ func (s *securedClusterCertGenSuite) TestServiceIssueLocalScannerCerts() { serviceTypes = localScannerServiceTypes } serviceTypeNames := make([]string, 0, serviceTypes.Cardinality()) - for _, serviceType := range serviceTypes.AsSlice() { + for serviceType := range serviceTypes.All() { serviceTypeNames = append(serviceTypeNames, serviceType.String()) } return set.NewFrozenSet(serviceTypeNames...) diff --git a/central/sensor/networkentities/controller_impl.go b/central/sensor/networkentities/controller_impl.go index 8c0473d37b06c..cf9f39b78e8e8 100644 --- a/central/sensor/networkentities/controller_impl.go +++ b/central/sensor/networkentities/controller_impl.go @@ -17,7 +17,7 @@ type controller struct { clusterID string netEntityMgr common.NetworkEntityManager graphEvaluator graph.Evaluator - requestSeqID int64 + requestSeqID atomic.Int64 stopSig concurrency.ReadOnlyErrorSignal injector common.MessageInjector @@ -46,7 +46,7 @@ func (c *controller) SyncNow(ctx context.Context) error { } // Stop early if the request message is outdated. - if msg.GetPushNetworkEntitiesRequest().GetSeqID() != atomic.LoadInt64(&c.requestSeqID) { + if msg.GetPushNetworkEntitiesRequest().GetSeqID() != c.requestSeqID.Load() { return nil } @@ -63,7 +63,7 @@ func (c *controller) getPushNetworkEntitiesRequestMsg(ctx context.Context) (*cen c.lock.Lock() defer c.lock.Unlock() - requestSeqID := atomic.AddInt64(&c.requestSeqID, 1) + requestSeqID := c.requestSeqID.Add(1) netEntities, err := c.netEntityMgr.GetAllEntitiesForCluster(ctx, c.clusterID) if err != nil { diff --git a/central/sensor/networkpolicies/controller_impl.go b/central/sensor/networkpolicies/controller_impl.go index fdf052d365cec..41e952c23988b 100644 --- a/central/sensor/networkpolicies/controller_impl.go +++ b/central/sensor/networkpolicies/controller_impl.go @@ -20,7 +20,7 @@ type controller struct { returnChans map[int64]chan *central.NetworkPoliciesResponse_Payload returnChansMutex sync.Mutex - currSeqID int64 + currSeqID atomic.Int64 injector common.MessageInjector } @@ -34,7 +34,7 @@ func newController(injector common.MessageInjector, stopSig concurrency.ReadOnly } func (c *controller) ApplyNetworkPolicies(ctx context.Context, mod *storage.NetworkPolicyModification) (*storage.NetworkPolicyModification, error) { - seqID := atomic.AddInt64(&c.currSeqID, 1) + seqID := c.currSeqID.Add(1) applyID := uuid.NewV4().String() diff --git a/central/sensor/service/common/message_injector.go b/central/sensor/service/common/message_injector.go index 387a1539ab757..3e2cdab17fd61 100644 --- a/central/sensor/service/common/message_injector.go +++ b/central/sensor/service/common/message_injector.go @@ -2,6 +2,7 @@ package common import ( "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/concurrency" ) @@ -9,4 +10,5 @@ import ( type MessageInjector interface { InjectMessage(ctx concurrency.Waitable, msg *central.MsgToSensor) error InjectMessageIntoQueue(msg *central.MsgFromSensor) + HasCapability(capability centralsensor.SensorCapability) bool } diff --git a/central/sensor/service/common/sensor_ack.go b/central/sensor/service/common/sensor_ack.go new file mode 100644 index 0000000000000..bd57afeca1806 --- /dev/null +++ b/central/sensor/service/common/sensor_ack.go @@ -0,0 +1,69 @@ +package common + +import ( + "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/pkg/centralsensor" + "github.com/stackrox/rox/pkg/concurrency" +) + +const vmIndexACKResourceIDSeparator = ":" + +// SendSensorACK sends a SensorACK only when sensor capability support is explicitly advertised. +func SendSensorACK(ctx concurrency.Waitable, action central.SensorACK_Action, messageType central.SensorACK_MessageType, resourceID, reason string, injector MessageInjector) { + if injector == nil { + return + } + + if !injector.HasCapability(centralsensor.SensorACKSupport) { + return + } + + if err := injector.InjectMessage(ctx, ¢ral.MsgToSensor{ + Msg: ¢ral.MsgToSensor_SensorAck{ + SensorAck: ¢ral.SensorACK{ + Action: action, + MessageType: messageType, + ResourceId: resourceID, + Reason: reason, + }, + }, + }); err != nil { + log.Warnf("Failed injecting SensorACK (%v) for %v (resource_id=%s): %v", action, messageType, resourceID, err) + } +} + +// VMIndexACKResourceID builds the correlation key for VM index ACK/NACK. +// +// The format is always "vmID:vsockCID" with the separator present even when +// one component is empty (e.g. ":100" or "vm-1:"). This makes it unambiguous +// which part is which when debugging logs or parsing the resource ID. +// +// Limitation: this pair cannot distinguish multiple in-flight reports from the +// same VM while it keeps the same CID; a stale ACK may still match the latest +// VMID:CID entry. +func VMIndexACKResourceID(vmID, vsockCID string) string { + if vmID == "" && vsockCID == "" { + return "" + } + return vmID + vmIndexACKResourceIDSeparator + vsockCID +} + +// SendLegacyNodeInventoryACK sends the legacy NodeInventoryACK message supported since version 4.1. +func SendLegacyNodeInventoryACK(ctx concurrency.Waitable, clusterID, nodeName string, action central.NodeInventoryACK_Action, messageType central.NodeInventoryACK_MessageType, injector MessageInjector) { + if injector == nil { + return + } + + if err := injector.InjectMessage(ctx, ¢ral.MsgToSensor{ + Msg: ¢ral.MsgToSensor_NodeInventoryAck{ + NodeInventoryAck: ¢ral.NodeInventoryACK{ + ClusterId: clusterID, + NodeName: nodeName, + Action: action, + MessageType: messageType, + }, + }, + }); err != nil { + log.Warnf("Failed injecting legacy NodeInventoryACK (%v) for cluster=%s node=%s: %v", messageType, clusterID, nodeName, err) + } +} diff --git a/central/sensor/service/common/sensor_ack_test.go b/central/sensor/service/common/sensor_ack_test.go new file mode 100644 index 0000000000000..a6cdffa361a7f --- /dev/null +++ b/central/sensor/service/common/sensor_ack_test.go @@ -0,0 +1,103 @@ +package common + +import ( + "testing" + + "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/pkg/centralsensor" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stretchr/testify/assert" +) + +func TestSendSensorACK_NACK(t *testing.T) { + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + + SendSensorACK(t.Context(), central.SensorACK_NACK, central.SensorACK_VM_INDEX_REPORT, "vm-nack", centralsensor.SensorACKReasonRateLimited, injector) + + assert.Len(t, injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + assert.NotNil(t, ack) + assert.Equal(t, central.SensorACK_NACK, ack.GetAction()) + assert.Equal(t, central.SensorACK_VM_INDEX_REPORT, ack.GetMessageType()) + assert.Equal(t, "vm-nack", ack.GetResourceId()) + assert.Equal(t, centralsensor.SensorACKReasonRateLimited, ack.GetReason()) +} + +func TestSendSensorACK_NilInjector(t *testing.T) { + assert.NotPanics(t, func() { + SendSensorACK(t.Context(), central.SensorACK_ACK, central.SensorACK_VM_INDEX_REPORT, "vm-1", "", nil) + }) +} + +func TestSendSensorACK_InjectorWithoutCapabilitySupport(t *testing.T) { + injector := &mockInjector{} + + SendSensorACK(t.Context(), central.SensorACK_ACK, central.SensorACK_VM_INDEX_REPORT, "vm-1", "", injector) + + assert.Empty(t, injector.messages, "should not send when SensorACKSupport capability is not advertised") +} + +func TestVMIndexACKResourceID(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + vmID string + vsockCID string + expected string + }{ + { + name: "returns pair when both vm id and cid are present", + vmID: "vm-1", + vsockCID: "100", + expected: "vm-1:100", + }, + { + name: "keeps separator when cid is missing", + vmID: "vm-1", + vsockCID: "", + expected: "vm-1:", + }, + { + name: "keeps separator when vm id is missing", + vmID: "", + vsockCID: "100", + expected: ":100", + }, + { + name: "returns empty string when both vm id and cid are missing", + vmID: "", + vsockCID: "", + expected: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + actual := VMIndexACKResourceID(tc.vmID, tc.vsockCID) + assert.Equalf(t, tc.expected, actual, "expected resource id %q, but got %q", tc.expected, actual) + }) + } +} + +type mockInjector struct { + messages []*central.MsgToSensor + injectErr error + capabilities map[centralsensor.SensorCapability]bool +} + +func (m *mockInjector) InjectMessage(_ concurrency.Waitable, msg *central.MsgToSensor) error { + m.messages = append(m.messages, msg) + return m.injectErr +} + +func (m *mockInjector) InjectMessageIntoQueue(_ *central.MsgFromSensor) {} + +func (m *mockInjector) HasCapability(cap centralsensor.SensorCapability) bool { + return m.capabilities[cap] +} diff --git a/central/sensor/service/connection/README.md b/central/sensor/service/connection/README.md new file mode 100644 index 0000000000000..248c74f642ceb --- /dev/null +++ b/central/sensor/service/connection/README.md @@ -0,0 +1,156 @@ +# Sensor-to-Central Message Flow + +![Data flow diagram](sensor-central-data-flow.excalidraw.svg) + +## Overview + +Central receives events from N Sensor clusters concurrently. Each cluster gets a fully +isolated set of queues and workers, so a slow or misbehaving cluster cannot stall others. +Within a cluster, messages are routed through two levels of queuing before reaching a +pipeline fragment that performs validation, enrichment, and datastore writes. + +## Architecture + +### Layer 1 — gRPC stream (one per cluster) + +[`central/sensor/service/service_impl.go`](../service_impl.go) `Communicate()` accepts the +bidirectional gRPC stream opened by Sensor. +[`connection/manager_impl.go`](manager_impl.go) `HandleConnection()` wraps it in a +`sensorConnection` struct that owns all state for that cluster. + +`runRecv()` in [`connection_impl.go`](connection_impl.go) is the read loop: it pulls +`MsgFromSensor` off the gRPC stream and forwards each message into the first queue. + +### Layer 2 — Per-message-type DedupingQueue + +`multiplexedPush()` in [`connection_impl.go`](connection_impl.go) routes every incoming +message into a `DedupingQueue` keyed by `MsgFromSensor` type. There are ~16 distinct +message types (Event, NetworkFlowUpdate, ClusterHealthInfo, ComplianceResponse, …). Queues +are allocated lazily on first use; a dedicated goroutine drains each one via +`handleMessages()`. + +The per-type fan-out is primarily about **category isolation**: a surge of one message type +(e.g. `Event` during initial cluster sync) cannot stall processing of another (e.g. +`NetworkFlowUpdate`). These queues use `DedupingQueue`, so messages carrying a `DedupeKey` +are collapsed (see [Deduplication](#deduplication) for which messages set this key and when). + +**Gotcha:** only `Event` messages continue into further queuing (Layers 3–4). All other +types (NetworkFlow, Compliance, Telemetry, …) are handled inline in `handleMessage()` +without further queuing. + +### Layer 3 — Per-resource-type workerQueue (Events only) + +[`sensorevents.go`](sensorevents.go) `addMultiplexed()` receives a decoded `SensorEvent` +and routes it to a `workerQueue` keyed by resource type (Deployment, Pod, Node, …). Up to +~30 workerQueues per `sensorConnection`, allocated lazily. + +Before pushing, the content-hash deduper (`deduper.ShouldProcess()`) decides whether this +event carries new information or can be dropped as a duplicate (see +[Content-hash deduplication](#content-hash-deduplication-layer-3)). + +### Layer 4 — Sharded DedupingQueues inside each workerQueue + +Each `workerQueue` contains **17** internal `DedupingQueue` slots and 17 goroutines — one +per slot. Slot 0 handles messages with no `HashKey`; slots 1–16 are sharded by +`FNV32(HashKey) % 16 + 1`. Because all messages for the same entity (same ID) always land +in the same slot, processing is serialized per entity without a global lock. + +This design solves two problems at once: different entities are processed **in parallel** +across slots, while updates to the **same** entity are **serialized** within one slot +(preventing lost-update races in the pipeline). As with Layer 2, these are `DedupingQueue`s, +so messages with a matching `DedupeKey` are collapsed before they reach the pipeline. + +**Gotcha (queue count):** the total number of goroutines and queues grows multiplicatively: + +``` +N clusters × M resource types seen × 17 workers +``` + +For 10 clusters each sending 15 resource types that is 2 550 goroutines just for workers, +plus Layer-2 queues and their goroutines on top. + +**Gotcha (unbounded queues):** the Layer-4 `DedupingQueue` slots have no capacity cap. +Items flow through Layers 2 and 3 quickly (dispatch only), but sit in Layer 4 until a +worker goroutine runs the pipeline fragment — which involves DB writes and can be slow. If +events arrive faster than the pipeline can process them, the queues grow without limit and +will eventually cause an OOM. Replace-in-place dedup provides partial relief for messages +that carry a `DedupeKey`, but messages with an empty key are always appended. + +### Layer 5 — Pipeline fragments + +[`central/sensor/service/pipeline/all/pipeline.go`](../pipeline/all/pipeline.go) `Run()` +finds the fragment matching the resource type and runs it. Each fragment handles +validation, enrichment, and datastore writes. For deployments this includes SAC checks, +risk scoring, network baseline updates, and policy re-evaluation. + +## Deduplication + +Two independent deduplication mechanisms operate on the message flow. Both are optional and +apply to different subsets of messages. + +### DedupingQueue replace-in-place semantics + +[`pkg/dedupingqueue/deduping_queue.go`](../../../../pkg/dedupingqueue/deduping_queue.go) +is used at both Layer 2 and Layer 4. When a message is pushed with a non-empty `DedupeKey` +that matches an already-queued item, the queue **replaces the old item in-place** (the new +item takes the old item's queue position). This means only the latest state of an entity is +ever processed — intermediate updates are silently dropped. + +When `DedupeKey` is empty (the zero value), the queue behaves as a plain FIFO — every push +appends without deduplication. + +#### Which messages carry a DedupeKey + +For the vast majority of traffic — regular Sensor Events arriving from the gRPC stream — no +`DedupeKey` is set when messages enter Layer 2, so those queues behave as plain FIFOs. +The `DedupeKey` is assigned later, in `handleMessage()`, via `shallDedupe()`, which means it +only takes effect at Layer 4. The rules are: + +- **No dedup** (key = empty): `CREATE` actions; and for non-`REMOVE` actions: + `NodeInventory`, `IndexReport`, `VirtualMachine`, `VirtualMachineIndexReport`. +- **Dedup on entity ID**: `UPDATE` and `REMOVE` actions for all other resource types + (Pod, Deployment, Namespace, etc.). + +`shallDedupe()` in [`connection_impl.go`](connection_impl.go) controls this logic. + +Two special cases carry a `DedupeKey` **before** reaching Layer 2, so deduplication occurs +at both layers: + +- **`AuditLogStatusInfo`** — Sensor sets `DedupeKey = clusterID` before sending + (`sensor/common/compliance/auditlog_manager_impl.go`). Multiple audit-log state + snapshots queued before the goroutine drains them collapse to the latest, avoiding + redundant syncs. +- **`ReprocessDeployment`** — Central's reprocessor (`central/reprocessor/reprocessor.go`) + injects synthetic Event messages with `DedupeKey = UUIDv5(riskDedupeNamespace, + deploymentID)`. Because the reprocessor also sets `HashKey` and the message type is + `ReprocessDeployment`, `handleMessage()` short-circuits before `shallDedupe()`, + preserving the key through both layers. This collapses redundant risk-reprocessing + requests for the same deployment. + +All other non-Event message types (`NetworkFlowUpdate`, `ComplianceResponse`, +`TelemetryDataResponse`, …) carry no `DedupeKey` and are therefore never deduplicated. + +### Content-hash deduplication (Layer 3) + +Before an Event enters a `workerQueue`, `deduper.ShouldProcess()` +([`central/hash/manager/deduper.go`](../../../hash/manager/deduper.go)) checks the +content hash of the event. The hash is normally **computed by Sensor** and sent as +`SensorHash`; Central only computes it as a backward-compatibility fallback when +`SensorHashOneof` is nil. If the same entity was already processed with an identical hash, +the message is dropped. + +This deduplication state is periodically flushed to the `hashes` postgres table and +reloaded on Central restart, so it can survive restarts. However, persistence is +**conditional**: when the `ROX_HASH_FLUSH_INTERVAL` environment variable is set to `0`, +Central truncates the `hashes` table on startup and does not flush — effectively disabling +cross-restart deduplication. + +## Key Code Locations + +- [central/sensor/service/service_impl.go](../service_impl.go) — gRPC entry point `Communicate():88` +- [central/sensor/service/connection/connection_impl.go](connection_impl.go) — `sensorConnection` struct; `multiplexedPush():171`; `handleMessage():342`; `shallDedupe()` +- [central/sensor/service/connection/sensorevents.go](sensorevents.go) — Layer-3 queue dispatch `addMultiplexed():80` +- [central/sensor/service/connection/worker_queue.go](worker_queue.go) — 17-slot sharded queue; FNV hashing `indexFromKey():51` +- [central/hash/manager/deduper.go](../../../hash/manager/deduper.go) — content-hash deduplication `ShouldProcess()` +- [pkg/dedupingqueue/deduping_queue.go](../../../../pkg/dedupingqueue/deduping_queue.go) — replace-in-place queue semantics +- [central/sensor/service/pipeline/all/pipeline.go](../pipeline/all/pipeline.go) — pipeline fragment dispatch `Run():64` diff --git a/central/sensor/service/connection/connection_impl.go b/central/sensor/service/connection/connection_impl.go index 3b57b49686008..b59fdec096b88 100644 --- a/central/sensor/service/connection/connection_impl.go +++ b/central/sensor/service/connection/connection_impl.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "math" + "slices" "github.com/pkg/errors" compScanSetting "github.com/stackrox/rox/central/complianceoperator/v2/scanconfigurations/datastore" + "github.com/stackrox/rox/central/convert/internaltov2storage" delegatedRegistryConfigConvert "github.com/stackrox/rox/central/delegatedregistryconfig/convert" "github.com/stackrox/rox/central/delegatedregistryconfig/util/imageintegration" hashManager "github.com/stackrox/rox/central/hash/manager" @@ -23,6 +25,8 @@ import ( v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/administration/events" + adminResources "github.com/stackrox/rox/pkg/administration/events/resources" "github.com/stackrox/rox/pkg/booleanpolicy/policyversion" "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/concurrency" @@ -32,6 +36,7 @@ import ( "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/postgres/pgutils" "github.com/stackrox/rox/pkg/protoconv/schedule" + "github.com/stackrox/rox/pkg/rate" "github.com/stackrox/rox/pkg/reflectutils" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/safe" @@ -81,6 +86,13 @@ type sensorConnection struct { capabilities set.Set[centralsensor.SensorCapability] hashDeduper hashManager.Deduper + + rl rateLimiter + adminEventsStream events.Stream +} + +type rateLimiter interface { + TryConsume(clientID string, msg *central.MsgFromSensor) (allowed bool, reason string) } func newConnection(ctx context.Context, @@ -97,6 +109,8 @@ func newConnection(ctx context.Context, hashMgr hashManager.Manager, complianceOperatorMgr common.ComplianceOperatorManager, initSyncMgr *initSyncManager, + rl rateLimiter, + adminEventsStream events.Stream, ) *sensorConnection { conn := &sensorConnection{ @@ -120,6 +134,8 @@ func newConnection(ctx context.Context, sensorHello: sensorHello, capabilities: set.NewSet(sliceutils. FromStringSlice[centralsensor.SensorCapability](sensorHello.GetCapabilities()...)...), + rl: rl, + adminEventsStream: adminEventsStream, } // Need a reference to conn for injector @@ -161,6 +177,23 @@ func (c *sensorConnection) multiplexedPush(ctx context.Context, msg *central.Msg return } + allowed, reason := c.rl.TryConsume(c.clusterID, msg) + if !allowed { + logging.GetRateLimitedLogger().WarnL( + "vm_index_reports_rate_limiter", + "Request is rate-limited for cluster %s and event type %s. Reason: %s", + c.clusterID, + event.GetEventTypeWithoutPrefix(msg.GetEvent().GetResource()), + reason, + ) + c.emitRateLimitedAdminEvent(c.clusterID, reason) + if vmReport := msg.GetEvent().GetVirtualMachineIndexReport(); vmReport != nil { + resourceID := common.VMIndexACKResourceID(vmReport.GetId(), vmReport.GetIndex().GetVsockCid()) + common.SendSensorACK(ctx, central.SensorACK_NACK, central.SensorACK_VM_INDEX_REPORT, resourceID, centralsensor.SensorACKReasonRateLimited, c) + } + return + } + typ := reflectutils.Type(msg.GetMsg()) queue := queues[typ] if queue == nil { @@ -181,6 +214,31 @@ func (c *sensorConnection) multiplexedPush(ctx context.Context, msg *central.Msg queue.Push(msg) } +func (c *sensorConnection) emitRateLimitedAdminEvent(clusterID, reason string) { + if c.adminEventsStream == nil { + return + } + // The texts are tuned for the rate.ReasonRateLimitExceeded, so skip adding log entry if the reason is different. + if reason != rate.ReasonRateLimitExceeded { + return + } + + c.adminEventsStream.Produce(&events.AdministrationEvent{ + Type: storage.AdministrationEventType_ADMINISTRATION_EVENT_TYPE_GENERIC, + Level: storage.AdministrationEventLevel_ADMINISTRATION_EVENT_LEVEL_WARNING, + Domain: events.DefaultDomain, + Message: fmt.Sprintf("VM index reports from cluster %s are being rate limited: %s", clusterID, reason), + ResourceType: adminResources.Cluster, + ResourceID: clusterID, + Hint: fmt.Sprintf("VM index reports are being rate limited to avoid overwhelming the system. "+ + "Consider either: (1) scaling up the Scanner V4 deployments and increasing values of %s or %s, "+ + "or (2) reducing the index-report frequency in roxagents running in the Virtual Machines.", + env.VMIndexReportRateLimit.EnvVar(), + env.VMIndexReportBucketCapacity.EnvVar(), + ), + }) +} + func getSensorMessageTypeString(msg *central.MsgFromSensor) string { messageType := reflectutils.Type(msg.GetMsg()) var eventType string @@ -487,6 +545,13 @@ func (c *sensorConnection) getPolicySyncMsgFromPolicies(policies []*storage.Poli // Otherwise, sensor supports the same version or newer as central } + // Remove exclude_by_kind exclusions before syncing to a Sensor that + // lacks WorkloadKindExclusionCap. Sensor forwards this same list to + // the admission controller, so AC evaluates the same copy. + if !c.HasCapability(centralsensor.WorkloadKindExclusionCap) { + policies = stripExcludeByKindExclusions(policies) + } + return ¢ral.MsgToSensor{ Msg: ¢ral.MsgToSensor_PolicySync{ PolicySync: ¢ral.PolicySync{ @@ -496,6 +561,32 @@ func (c *sensorConnection) getPolicySyncMsgFromPolicies(policies []*storage.Poli }, nil } +// stripExcludeByKindExclusions returns a copy of policies with every exclusion +// that sets exclude_by_kind removed. Stored policies are not mutated. An old +// Sensor compiles an exclude_by_kind item as an unconstrained exclusion (nil +// name and nil scope match every workload). Sensor does not evaluate image +// exclusions, so an image+kind item with only the kind matcher removed is that +// same unconstrained exclusion and is removed entirely. An exclusion with no +// kind matcher is kept, including an image combined with a deployment name or +// scope. +func stripExcludeByKindExclusions(policies []*storage.Policy) []*storage.Policy { + out := make([]*storage.Policy, 0, len(policies)) + for _, p := range policies { + if !slices.ContainsFunc(p.GetExclusions(), exclusionSetsExcludeByKind) { + out = append(out, p) + continue + } + cloned := p.CloneVT() + cloned.Exclusions = slices.DeleteFunc(cloned.GetExclusions(), exclusionSetsExcludeByKind) + out = append(out, cloned) + } + return out +} + +func exclusionSetsExcludeByKind(e *storage.Exclusion) bool { + return e.GetExcludeByKind() != nil +} + func (c *sensorConnection) getNetworkBaselineSyncMsg(ctx context.Context) (*central.MsgToSensor, error) { var networkBaselines []*storage.NetworkBaseline walkFn := func() error { @@ -590,6 +681,7 @@ func (c *sensorConnection) getScanConfigurationMsg(ctx context.Context) (*centra for _, profile := range scanConfig.GetProfiles() { profiles = append(profiles, profile.GetProfileName()) } + profileRefs := internaltov2storage.ScanConfigRefsToCentral(scanConfig.GetProfileRefs()) cron, err := schedule.ConvertToCronTab(scanConfig.GetSchedule()) if err != nil { return nil, err @@ -600,6 +692,7 @@ func (c *sensorConnection) getScanConfigurationMsg(ctx context.Context) (*centra ScanSettings: ¢ral.ApplyComplianceScanConfigRequest_BaseScanSettings{ ScanName: scanConfig.GetScanConfigName(), Profiles: profiles, + ProfileRefs: profileRefs, StrictNodeScan: scanConfig.GetStrictNodeScan(), AutoApplyRemediations: scanConfig.GetAutoApplyRemediations(), AutoUpdateRemediations: scanConfig.GetAutoUpdateRemediations(), diff --git a/central/sensor/service/connection/connection_test.go b/central/sensor/service/connection/connection_test.go index 13fe4bcafe4f9..eaf8523107feb 100644 --- a/central/sensor/service/connection/connection_test.go +++ b/central/sensor/service/connection/connection_test.go @@ -4,12 +4,14 @@ import ( "context" "crypto/x509" "encoding/pem" + "maps" "slices" "testing" "time" "github.com/pkg/errors" scanConfigMocks "github.com/stackrox/rox/central/complianceoperator/v2/scanconfigurations/datastore/mocks" + "github.com/stackrox/rox/central/convert/internaltov2storage" "github.com/stackrox/rox/central/hash/manager/mocks" clusterMgrMock "github.com/stackrox/rox/central/sensor/service/common/mocks" pipelineMock "github.com/stackrox/rox/central/sensor/service/pipeline/mocks" @@ -61,6 +63,26 @@ var ( }, }, }, + { + ScanConfigName: "TestConfigWithRefs", + Profiles: []*storage.ComplianceOperatorScanConfigurationV2_ProfileName{ + {ProfileName: "ocp4-cis"}, + {ProfileName: "ocp4-cis-tailored"}, + }, + ProfileRefs: []*storage.ComplianceOperatorScanConfigurationV2_ProfileReference{ + {Name: "ocp4-cis", Kind: storage.ComplianceOperatorProfileV2_PROFILE}, + {Name: "ocp4-cis-tailored", Kind: storage.ComplianceOperatorProfileV2_TAILORED_PROFILE}, + }, + Schedule: &storage.Schedule{ + IntervalType: storage.Schedule_DAILY, + Hour: 1, + Minute: 2, Interval: &storage.Schedule_DaysOfWeek_{ + DaysOfWeek: &storage.Schedule_DaysOfWeek{ + Days: []int32{1}, + }, + }, + }, + }, } ) @@ -144,10 +166,18 @@ func (s *testSuite) TestSendsScanConfigurationMsgOnRun() { return slice.GetScanConfigName() == sc.GetUpdateScan().GetScanSettings().GetScanName() }) s.Require().NotEqual(-1, idx) - s.Assert().Equal(scanConfigs[idx].GetScanConfigName(), sc.GetUpdateScan().GetScanSettings().GetScanName()) - cron, err := schedule.ConvertToCronTab(scanConfigs[idx].GetSchedule()) + stored := scanConfigs[idx] + settings := sc.GetUpdateScan().GetScanSettings() + s.Assert().Equal(stored.GetScanConfigName(), settings.GetScanName()) + cron, err := schedule.ConvertToCronTab(stored.GetSchedule()) s.Require().NoError(err) s.Assert().Equal(cron, sc.GetUpdateScan().GetCron()) + // profile_refs must be forwarded with correct kinds + s.Require().Len(settings.GetProfileRefs(), len(stored.GetProfileRefs())) + for i, ref := range settings.GetProfileRefs() { + s.Assert().Equal(stored.GetProfileRefs()[i].GetName(), ref.GetName()) + s.Assert().Equal(internaltov2storage.StorageToCentralProfileKind(stored.GetProfileRefs()[i].GetKind()), ref.GetKind()) + } } } } @@ -300,9 +330,7 @@ func (s *testSuite) TestSendDeduperStateIfSensorReconciliation() { } deduperStateSent := make(map[string]uint64) for _, state := range deduperStates { - for k, v := range state.GetResourceHashes() { - deduperStateSent[k] = v - } + maps.Copy(deduperStateSent, state.GetResourceHashes()) s.Equal(tc.expectNumberOfDeduperStates, int(state.GetTotal())) s.True(currentSet.Contains(int(state.GetCurrent()))) currentSet.Remove(int(state.GetCurrent())) @@ -337,6 +365,102 @@ func (s *testSuite) TestGetPolicySyncMsgFromPoliciesDoesntDowngradeInvalidVersio s.Equal(policyversion.CurrentVersion().String(), policySync.GetPolicies()[0].GetPolicyVersion()) } +func (s *testSuite) TestGetPolicySyncMsgFromPoliciesStripsExcludeByKind() { + kindExclusion := &storage.Exclusion{ + Name: "skip-cronjobs", + Matcher: &storage.Exclusion_ExcludeByKind_{ + ExcludeByKind: &storage.Exclusion_ExcludeByKind{ + Kinds: []storage.Exclusion_WorkloadKind{storage.Exclusion_CRON_JOB}, + }, + }, + } + nameExclusion := &storage.Exclusion{ + Name: "skip-web", + Matcher: &storage.Exclusion_Deployment_{ + Deployment: &storage.Exclusion_Deployment{Name: "web"}, + }, + } + imageExclusion := &storage.Exclusion{ + Name: "skip-image", + Image: &storage.Exclusion_Image{Name: "stackrox.io"}, + } + imageAndKindExclusion := &storage.Exclusion{ + Name: "skip-image-and-jobs", + Image: &storage.Exclusion_Image{Name: "example.io"}, + Matcher: &storage.Exclusion_ExcludeByKind_{ + ExcludeByKind: &storage.Exclusion_ExcludeByKind{ + Kinds: []storage.Exclusion_WorkloadKind{storage.Exclusion_JOB}, + }, + }, + } + imageAndDeploymentExclusion := &storage.Exclusion{ + Name: "skip-image-and-web", + Image: &storage.Exclusion_Image{Name: "nginx"}, + Matcher: &storage.Exclusion_Deployment_{ + Deployment: &storage.Exclusion_Deployment{Name: "web-image"}, + }, + } + policy := &storage.Policy{ + Name: "test-policy", + Exclusions: []*storage.Exclusion{ + kindExclusion, + nameExclusion, + imageExclusion, + imageAndKindExclusion, + imageAndDeploymentExclusion, + }, + } + + s.Run("without capability strips kind items and keeps others", func() { + original := policy.CloneVT() + msg, err := (&sensorConnection{}).getPolicySyncMsgFromPolicies([]*storage.Policy{policy}) + s.Require().NoError(err) + + synced := msg.GetPolicySync().GetPolicies() + s.Require().Len(synced, 1) + got := synced[0].GetExclusions() + s.Require().Len(got, 3) + s.Equal("skip-web", got[0].GetName()) + s.Equal("web", got[0].GetDeployment().GetName()) + s.Equal("skip-image", got[1].GetName()) + s.Equal("stackrox.io", got[1].GetImage().GetName()) + s.Equal("skip-image-and-web", got[2].GetName()) + s.Equal("nginx", got[2].GetImage().GetName()) + s.Equal("web-image", got[2].GetDeployment().GetName()) + protoassert.Equal(s.T(), original, policy) + }) + + s.Run("with capability keeps kind items", func() { + conn := &sensorConnection{ + capabilities: set.NewSet(centralsensor.WorkloadKindExclusionCap), + } + msg, err := conn.getPolicySyncMsgFromPolicies([]*storage.Policy{policy}) + s.Require().NoError(err) + + got := msg.GetPolicySync().GetPolicies()[0].GetExclusions() + s.Require().Len(got, 5) + s.NotNil(got[0].GetExcludeByKind()) + s.NotNil(got[3].GetExcludeByKind()) + s.Equal(storage.Exclusion_JOB, got[3].GetExcludeByKind().GetKinds()[0]) + s.Equal("example.io", got[3].GetImage().GetName()) + s.Equal("web-image", got[4].GetDeployment().GetName()) + }) + + s.Run("later capable sensor still sees original kind items", func() { + // PreparePoliciesAndBroadcast reuses the same slice across connections. + input := []*storage.Policy{policy} + _, err := (&sensorConnection{}).getPolicySyncMsgFromPolicies(input) + s.Require().NoError(err) + + capable := &sensorConnection{ + capabilities: set.NewSet(centralsensor.WorkloadKindExclusionCap), + } + msg, err := capable.getPolicySyncMsgFromPolicies(input) + s.Require().NoError(err) + s.NotNil(msg.GetPolicySync().GetPolicies()[0].GetExclusions()[0].GetExcludeByKind()) + }) +} + func (s *testSuite) TestSendsAuditLogSyncMessageIfEnabledOnRun() { ctx := context.Background() clusterID := "this-cluster" diff --git a/central/sensor/service/connection/duplicate_synced_regression_test.go b/central/sensor/service/connection/duplicate_synced_regression_test.go new file mode 100644 index 0000000000000..6874c0b90b972 --- /dev/null +++ b/central/sensor/service/connection/duplicate_synced_regression_test.go @@ -0,0 +1,63 @@ +package connection + +import ( + "testing" + + hashManager "github.com/stackrox/rox/central/hash/manager" + pipelineMock "github.com/stackrox/rox/central/sensor/service/pipeline/mocks" + "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" +) + +func TestDuplicateSyncedEventDoesNotPanic(t *testing.T) { + const clusterID = "duplicate-synced-cluster" + + ctrl := gomock.NewController(t) + mockPipeline := pipelineMock.NewMockClusterPipeline(ctrl) + var reconcileCalls int + mockPipeline.EXPECT().Reconcile(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(func(_ any, _ any) error { + reconcileCalls++ + return nil + }) + + stopSig := concurrency.NewErrorSignal() + deduper := hashManager.NewDeduper(nil, clusterID) + deduper.StartSync() + + handler := newSensorEventHandler( + &storage.Cluster{Id: clusterID, Name: "duplicate-synced-cluster"}, + "regression-sensor-version", + mockPipeline, + nil, + &stopSig, + deduper, + NewInitSyncManager(), + ) + + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Id: "duplicate-synced-regression", + Resource: ¢ral.SensorEvent_Synced{ + Synced: ¢ral.SensorEvent_ResourcesSynced{ + UnchangedIds: []string{ + "Deployment:deployment-1", + "Pod:pod-1", + }, + }, + }, + }, + }, + } + + handler.addMultiplexed(t.Context(), msg) + + assert.NotPanics(t, func() { + handler.addMultiplexed(t.Context(), msg.CloneVT()) + }) + assert.Equal(t, 1, reconcileCalls) + assert.True(t, handler.reconciliationMap.IsClosed()) +} diff --git a/central/sensor/service/connection/initsync_manager_test.go b/central/sensor/service/connection/initsync_manager_test.go index 07fb84535d487..f5b0d32ea9b86 100644 --- a/central/sensor/service/connection/initsync_manager_test.go +++ b/central/sensor/service/connection/initsync_manager_test.go @@ -37,7 +37,7 @@ func TestInitSyncManager(t *testing.T) { t.Setenv(env.CentralMaxInitSyncSensors.EnvVar(), "3") m := NewInitSyncManager() - for i := 0; i < 3; i++ { + for i := range 3 { assert.True(t, m.Add(fmt.Sprintf("test-%d", i))) } assert.False(t, m.Add("test-a"), "Unable to add after limit is reached") diff --git a/central/sensor/service/connection/manager_impl.go b/central/sensor/service/connection/manager_impl.go index 6d3e7426f758a..61762872fc408 100644 --- a/central/sensor/service/connection/manager_impl.go +++ b/central/sensor/service/connection/manager_impl.go @@ -11,13 +11,17 @@ import ( "github.com/stackrox/rox/central/sensor/service/pipeline" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/administration/events" + adminEventStream "github.com/stackrox/rox/pkg/administration/events/stream" "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/clusterhealth" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/errorhelpers" "github.com/stackrox/rox/pkg/errox" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/protoconv" + "github.com/stackrox/rox/pkg/rate" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/sync" @@ -73,6 +77,8 @@ type manager struct { complianceOperatorMgr common.ComplianceOperatorManager initSyncMgr *initSyncManager autoTriggerUpgrades *concurrency.Flag + rateLimiter *rate.Limiter + adminEventsStream events.Stream } // NewManager returns a new connection manager @@ -81,9 +87,27 @@ func NewManager(mgr hashManager.Manager) Manager { connectionsByClusterID: make(map[string]connectionAndUpgradeController), manager: mgr, initSyncMgr: NewInitSyncManager(), + rateLimiter: newVMIndexReportRateLimiter(), + adminEventsStream: adminEventStream.Singleton(), } } +func newVMIndexReportRateLimiter() *rate.Limiter { + rl, err := rate.NewLimiter( + "vm_index_reports", + env.VMIndexReportRateLimit.FloatSetting(), + env.VMIndexReportBucketCapacity.IntegerSetting()). + ForWorkload(func(msg *central.MsgFromSensor) bool { + return msg.GetEvent().GetVirtualMachineIndexReport() != nil + }) + + if err != nil { + utils.Should(errors.Wrap(err, "Creating rate-limiter for VM index reports")) + } + + return rl +} + func (m *manager) initializeUpgradeControllers() error { clusters, err := m.clusters.GetClusters(managerCtx) if err != nil { @@ -252,6 +276,8 @@ func (m *manager) CloseConnection(clusterID string) { if err := m.manager.Delete(ctx, clusterID); err != nil { log.Errorf("deleting cluster id %q from hash manager: %v", clusterID, err) } + + m.rateLimiter.OnClientDisconnect(clusterID) } func (m *manager) HandleConnection(ctx context.Context, sensorHello *central.SensorHello, cluster *storage.Cluster, eventPipeline pipeline.ClusterPipeline, server central.SensorService_CommunicateServer) error { @@ -278,7 +304,12 @@ func (m *manager) HandleConnection(ctx context.Context, sensorHello *central.Sen m.manager, m.complianceOperatorMgr, m.initSyncMgr, + m.rateLimiter, + m.adminEventsStream, ) + + defer m.rateLimiter.OnClientDisconnect(clusterID) + ctx = WithConnection(ctx, conn) oldConnection, err := m.replaceConnection(ctx, cluster, conn) diff --git a/central/sensor/service/connection/sensor-central-data-flow.excalidraw.svg b/central/sensor/service/connection/sensor-central-data-flow.excalidraw.svg new file mode 100644 index 0000000000000..0bb309cddfd13 --- /dev/null +++ b/central/sensor/service/connection/sensor-central-data-flow.excalidraw.svg @@ -0,0 +1,4 @@ + + +eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO19aVfq2rbt9/MrbOt+ea+1be6si/1NUbGu63dus1GLXCIghYi3nf/+xkCFQFxuklx1MDAxMFx1MDAxNdnk7LbOWlx1MDAwNmJmMmafvc9R/e+/Vlb+dPrN0p+/V/6UXlx1MDAwYrlatdjK9f78hT9/KbXa1UZcdTAwMWROscG/241uqzD45H2n02z//d//PfqGU2g8vX+rVCs9leqdNnzu/8G/V1b+d/AnnKlcdTAwMTbxu5erJ2v7nG+fZ/ZNM3+9rzLna9XBV1x1MDAwN1x1MDAxZvq8mVap0MnVK7XS6NQr/JxcdTAwMGLuaFxyf2o5/HlcdTAwMWZ+vsq0dohWzHIrJKdsdLpXLXbu4SOUXG7qfJ4lcLDhR+5L1cp9XHUwMDA3PmONcqjSVlx1MDAxOT34XGJcdTAwMWR+5P1e/l4hw5+0O63GYynTqDVaeMP/RUv4v9Ht5nOFx0qr0a1cdTAwMTdHnynpsiyXR58pV2u1s05/cGV4vvAs/0xcXP/q4+7ZxM+Dvlx1MDAwNb+wcl8vtfH5j+6+0cxcdTAwMTWqXHUwMDFkfFCUjEaAd9fcKVx1MDAwZV7V/4zuqZV7Ku3gu6p3a7Xhj6v1Ylx038CfXHUwMDFjXHUwMDE5+2314sdv+3zPo5fIP37yn9G9l0p4YS41t0wwMTwxsjVKzeRPXHUwMDBmXHUwMDFi9YHdMVwiXHUwMDA1N1x1MDAwNl7Q6K7aXHUwMDFiYHCdwVXLuVq7NHr8eGubk8boNsgxe+uUXjvDcbnMtZBcdTAwMTGbd+S6snp+s56/bmSL/e554c/wc//5+Nvo8XWbxdz7/VCtmVx1MDAxNFZbMFc+PF+r1lx1MDAxZiefba1RePRcdTAwMTlCu1x1MDAwZuMurqFh/mGEqVXC4b9zKv5cdTAwMTbsb8pcdTAwMWPJ1a3LXobTs9xq1DulevFcdTAwMGUv8GfscufVp1K7k3tqXHUwMDA2XpPq2/fhfby1ienr/zw803fscX7MXFxcdTAwMTM0c9mUmauMw1x1MDAxNU5HRVx0l1Qq78R1fSuVmdpp5ertZq5cdTAwMDXG89tn66X/dFx1MDAxZPv4x7yUknKlXHUwMDE4Zz7zUtCgaak4oUTBm00yK79k4lxmrFx1MDAwZlx1MDAwNp+B39LK1Vxcb1x1MDAxMCbGWfXtXHUwMDFkTMZ+upV7qtbwkcuxq6zVqlx1MDAxNVx1MDAxY/2fWqnsslx1MDAwNHhcdTAwMDKdKqx9w9OdRnN0tlx1MDAwMNfLVeul1k6UXHUwMDE1r9GqVqr1XFztPOimc91O47TUfr/tTqtbcj+S0vbnXHUwMDE0oI5rXHUwMDEyzCN0mJ3S3YPgR5nV4mbmuf7Syp7urEZf+SWH30A1NZNcdTAwMDBcdTAwMDJG61CqXHUwMDE4JdZcci6f6MGAMkjDyFx1MDAxOLRcZoGDwlmi0oaPxVjo+cxcdTAwMGK9oZxcYmO0XHUwMDFmoFx1MDAwNOJcdLwqZph0TdAvX+WfyNNcdTAwMTYv9lx1MDAxZatH5nCtfZpcdTAwMTO9t0Letcr/5X/Z9y+vXz62zzbrq/lM6a2ePym2brqF6/Hf8vn7c61Wo1x1MDAxN4c9cGGB71D3jP/N7MH/OUdgXHUwMDBm8CBcdTAwMWPFmeaMW1xu9FGMQ4CQwiFGWOVcdTAwMGZcdTAwMDFcdTAwMDaIv1x1MDAwMP7ArdLMSPpcclx1MDAwNGIxXHUwMDEwQETnXHUwMDBl1Fx1MDAxYVivXHLw+jhzXHUwMDFkcJtwoahkSSZ78LxhXHUwMDE21lx0I5PMm0/ycFaqt1x1MDAxYq1cdTAwMTWaXHUwMDA2eyjAXHUwMDFkl1oh/OGpWiy6V7xcdFxuMWXpnKRcdTAwMTDeW19cdTAwMTRcdTAwMGVxffKaedvQh9dPJ6tcdTAwMTenLV2ud2o2ModQgjlMXHUwMDEw5qNCkEQg901IXCKko7TUhi5hxFx1MDAwZkbkzESCamWEtlIr91x1MDAxY5qKLsLCXHUwMDFik0aNlosvp1x1MDAxMk22cXJw2t3sXHUwMDFm6053/yp72+NX11GpxFpho69y+budNcVcdTAwMGLiYjNX3uifpEUlJNFAolx1MDAxN4VK+D/nXGJUQjNcdTAwMDJUgjJcdTAwMDFcdTAwMDYlKKN0XHUwMDFjXGJcdTAwMDToXGbCXGafmOtDKqFcdTAwMWODW1x1MDAxMZzAXHLijF9Sifd7moZcdTAwMDEqXHUwMDA2lZCcM2nd0zbKZJdEXG5cIlx1MDAxOEmkXHUwMDFiwqiEMMQkouBcdTAwMTNUgs1cdTAwMDOVmLKCXHUwMDA2UFx0tnhUonpfXju1tVx1MDAxYmY2mkd9vnV0/2oqkalcdTAwMDQlwFxijFx1MDAxY2xHMlx1MDAwZpdgjpbEajV2blx1MDAxMblEMde+L30vkOjZyVx1MDAwNOhEaqhcdTAwMTFcdTAwMWVPQ7hUXHUwMDAxhWMliMzvI1x1MDAxMyeX9dZ9w1x1MDAxNFdzrcPdvbvOa+u4RKOSiVxyazR96N7kePeK0uOXRmv19CklMiEoXHUwMDA341x1MDAxNu6p/5vJhP9zjkAmKGXMsVxmSFx1MDAwMyHMMqvsJJtgXHUwMDBlPCtipP/GhHRcdTAwMDRuTDB4oGJsXHUwMDFicskmQkHARGdcdTAwMTNKWYbbPvEmO+FGK5CJiSZ7XHUwMDE4mVBIw1MgXHUwMDEzh/NAJqasoVx1MDAwMWTicPHIRH3tcb17Ue7tllx1MDAxZSp3O+x2/TV73PNcdTAwMDJcYlx1MDAwZWpcdTAwMWNALIPlnnI2tty/Q1x1MDAwN9dcdTAwMGVRlnN/6NDEYcrKXHUwMDAwXHUwMDEy4Wg2Tj7mXHUwMDE2O4qNTuebXHUwMDE5hI3KIFggg2BgZJaxmFxmgktl4UhcdTAwMWZUNOVCu6dOVFBpNqqTXHUwMDA0ZfS3lZG1XGb+Mfz7//zl+2lcdTAwMWaTxGN1aI2jXHUwMDBieFxiRi3X7mRcdTAwMWFPT1W0hmO8K1x1MDAwZpx3cq3OOrzFar0yeVx1MDAwZWZ1wJnBt9aQ3tyXclx1MDAxZZuA71x1MDAwNZ77MZhxvZxavzIwqcFLXHUwMDBigZ/9m8Jz8dZS2uvfXFyeVLdcdTAwMWL6JJ+NrGUkXHUwMDEzXHUwMDBlWjRlZrCrMTLq91x1MDAwMFxyZlx1MDAxY2qktpRQX0FcdTAwMDO/XHUwMDE3XHUwMDA0jTWAWFx1MDAwMu17xIGGgGSYdoA7XHUwMDAzhVaEMcpGMyBcdTAwMWRgKpfLtphPXHUwMDE3mL5f2mSjXHUwMDAyU7C0UVJcdTAwMThBYnpcXEHWwKvVavTiv1xc2VxcUr1Xv1q9aGyK1cP1l1bBiJeHWFxuRGnOzcJsZ/o/j1xiXG5EMlx1MDAxYj55efjkZVxuPlx1MDAwMFx1MDAxYURIqlx1MDAxNPmW4KrFkCDb0SWI1Vxm1mdcdTAwMTVPgnBJXHS+05TJXHUwMDAy51x1MDAxNoxEJZk341xuJNOo12E9gTteSVOSzFx1MDAxMGg1ZVx1MDAxNfRXI6HDWFx1MDAxNHlcdTAwMTK+XHUwMDE1NLFccjRcdTAwMDYvlFx1MDAxYeaAXHUwMDFh5oxcdTAwMWLDOFFcdTAwMTP4wlx1MDAxNepcdTAwMTcwLMBcdTAwMTimyWhHaFx1MDAxOHbNlaOFXHUwMDAxXmgolUz5kFx1MDAwM0aBf8CliVx1MDAwNZRcdTAwMTJcZlxmdMlcdTAwMGX8XHUwMDAwJyCSM45uMUbCM4bFXCJcdTAwMGVcdTAwMTJJQSRcdTAwMTecJfKiXHUwMDA2I5GRSlx1MDAxMpZcYolSlS2rgSaKh9c401Qxrnf3kThcdTAwMTFlY1x1MDAwNT791ChcdTAwMGXssNHKV91cdTAwMDRcdTAwMGJs+3X0q8dcdTAwMWWKo2CEhFx1MDAxMW6k1ED41F9hZ71ibWRNY3IraFxiT+zuwrz1K3eH5Pm4/XS8/rR2Xpp1XGKSWIBcdTAwMWFccm+CXG5uxt9cdTAwMTRxhCXo0jXwXCKlosyGXGYhli6cXHUwMDA0xlIt3+j9XHUwMDEyYllcXKudXHUwMDFmnF1cdTAwMGLTypzeNOu7NzXNi17oXHUwMDBmkIbwkFx1MDAxZG2kUJZIXHUwMDFmdklB8ltAfYOB+ZJz7cV/K7ybUSNZSKUjXGJcdTAwMTdUc/lLZOH3XHUwMDEzzZ1cdTAwMTSiZ2BpXHUwMDA2sVx1MDAxNC80T8Jsolayb1SFrFxcqOxedM71iyXXmUwmK15PVVx1MDAxY1WoXHUwMDAwwIVdmHhZ/+dcdTAwMTFBXHUwMDE1XG7Mk1x1MDAwYpm3wNqmzFumhCOXgS7xp+tunEBcdTAwMTfMuOHaxFxudFHoaUw5yEVozrSwSebNNFWYStTLXGaqcMpcdTAwMDJcdTAwMThVXHUwMDE1LmBcdTAwMDRMeLRhmCrUljqGK82JIEyriWh8rrijmDBAJTnFjScvusAnXHUwMDE4M5Rrqoyh7kjtXHUwMDExujBM+CHEcGIoo4qnjTZcdTAwMGJBXHUwMDBldlNcdTAwMTCFwsJCQayMR1x1MDAwZoRGMWnT3p+aXHUwMDE3VVx1MDAxOGiheHht0yM1vkBcdTAwMTSGh+4lVVQ4j1x1MDAxNVx1MDAwMTnIMX3mr7CT1DPK+dCEozx9PODVgIhcdTAwMDc5XHUwMDAx9sSpXHUwMDExJZApITf+j1GC4Vx1MDAwZn9q/qXljpB6PGrxnUxy6lx1MDAxMMa1NspcdTAwMTBgM6Pdu2hcdTAwMWGQXHUwMDEw+DpRwEaXXHUwMDFhMFxi5vdm14Cg51x1MDAxOKbQxcvFZJh+aHWi9KzhzcXSgGtPeveEN5p9slNVje1K7qJ4vvdcclx0XHUwMDE0odf9yljKXHUwMDE5Vrx5hFx1MDAxOf/3XHUwMDE3QbNya4JcdTAwMTCGqSlcYsOkdYCLXHUwMDEybpdRlNFRZT+GVEXssMSdXVx1MDAxMcmHXHTcklqTaFx1MDAwYulLZsw0tZpKuudcZmp1ylwiXHUwMDFkVa0uYOpn5rJQ7lrF38yhvO49ZW+UvFx1MDAxN5Hpi1x1MDAxNsSBtVxmKIhvmIRcIlx1MDAwZafAcYVcdTAwMTG+6Z9cdTAwMDBAXHUwMDEzWaNDaNEghdV4WYolcVx1MDAxOULM0ezEhTNcdTAwMDX6g8TcvbaGS4XVJ76PucizfvX4rVfU/Wzz5ny3y25f8uJcdTAwMWKqSIRet7T3cHKr8sV2NkduatvXV4eV3kFcbtc9vt542JL1yutRs93eejvNtVv72Vx1MDAxNK57ojKl8tWOqVx1MDAxNJvHXHUwMDBm2fZhpbp2Xk2JaVx0So1cdTAwMTQjO/rlTMvf3lwiMC1DtEOM0cJYXHUwMDEwyO48qI/9O8dcdTAwMDDP0lx1MDAwMbnwljmKYdKKXHUwMDE2apDH9lxydGshsPA4Ot3SXHUwMDAy9DZI4lhsXHUwMDBiU+Q1sSTlOFxyjmaSLHH8k221uvXTUuHl//zfNNjVrEkrU4jEJMXyufdFoVTha06YXHUwMDAzQFx1MDAwMutcdTAwMDFLXHUwMDAzPiX9/ItcXFx1MDAxOcdYYpSdSGtzR51Okq0henBcIlx1MDAxZE/27Fx1MDAxMkeGOHJcdTAwMTKVU1x1MDAwNW/5U1wiJSdWxYpItVx1MDAwMtORiEnZ9/he4yrRqpzqjr+PTeLhtsbRJTxcXCO1Xf7wWj8ryfbItVGKW4kxborT0Vx1MDAwNorfWVx1MDAxNbZbXHUwMDFlaZs/XHUwMDFjYVx1MDAxM1x1MDAwZVx1MDAwMW5ccuiIpVx1MDAxNMDFsFFcdTAwMDTj+1lcdTAwMDGrJbBcdTAwMTJjYFx1MDAwMFRcdTAwMWKrQsbwj9nxz1/cZYvZq+PVbOHwrr55Ur/f7USvliT4YNdcdTAwMWWELfeRzFpidlwiQFx1MDAwMjPWXHUwMDFm4jl3XGZcdTAwMWZkXHUwMDA1XHJqKfvxRFxyy1x1MDAwNFXiK1xuMObL+VLpl1x1MDAwM/3pzOJcdTAwMTlrXHUwMDFmsbHyalx1MDAxMXDeXHUwMDEwhbXcklx1MDAxNVJcdTAwMTneWyzpXFzPr5+z68xLfr1ZIK36y71aK99ElYyP3dfe9eHry/bD3u7jJVdcdTAwMTeqYyd+S7JN/6uLvYunyrY4O+vuZLd2XHUwMDFle5nNm1x1MDAxNK57fUsyZ1dcdTAwMWQq1/b2b1x1MDAxZW5uz/onpctcdTAwMTSuO4PU9yxnXulMLF2cdCt/e4sgnTGwTiil0S02qFx1MDAxNjdcdTAwMWX7Qlx0dSycXHUwMDE0drKgw1x1MDAxMFx1MDAxNilcdTAwMTlUs4ZlimnLXFy5XHUwMDBlS/VcdTAwMWNcboZn0dUzI4pqpWi8uDp4XHUwMDFm8DWqSMreXG5u4FUnq+H6qZ83X+DXrGyUit0mMK+TbqnrXtp+TElP4Vx1MDAxN5NKOnRcdTAwMTSLoqk7m28nlyftl/t27vT2/ohXL8+LMjLn0kQ5SjHBXHUwMDE4NcSPczFguHbJub5cdTAwMGVmzmd3WMBCKbS18ZKsKNGMYVx04290WJxcdTAwMTZcdTAwMGYuN2+Kr4Wt+/37m96VbD6a529wXHUwMDAwTGVcdTAwMWJKXGKhXHUwMDE3JyTC/zlHYFx1MDAxYlpYh1n0PH44J8fgwFhcdTAwMDewwNhArkGFcSSWq1x1MDAwNOZGiVx1MDAxND5cXEOSlEFgIbjGRZzAXGKhjJLu1Ncos1x1MDAxZNgjyCx3q41UuMZcZlx1MDAxM+eTa1x1MDAxY5Y6W7VG72Jw5X/X5450TFlgJ0nH2HBcdTAwMTadfFx1MDAxY/TLLbtuVuWV1TfN4nW/drBbjkw+KNBmR9BA9iEwZ8gwSa1/n56FY1x1MDAxZj+Q5Fx1MDAxZDmcP5B+WFx1MDAxMDFcdTAwMDJcdTAwMDR7rK19ZbnmhiXb2k9GPlxur2190d9vNmzuaWet93B7YFx1MDAwZraXUVx1MDAwMul37PJ9zlx1MDAxMchcdTAwMDcwUmy8XHUwMDA1s1oyXCKJO1x1MDAwNfCDfXBtQ0plU2tcdTAwMWQrgHxoXHJ/WOmDXHUwMDA2S/bhh1x1MDAwMVfR2YcxXHUwMDA0INnGJFx1MDAxZsxcbitx9yld8pHCRlx1MDAwN1x1MDAxZF+hV5ql1r/rXHUwMDA37cpWq/H0XHUwMDEx2jgw2DngIVPW2klcdTAwMWXiM7KVsIEtXG4lyb10ds/3bePkqvp2fnq08XpZ2Tzwwo9PZUzgXHUwMDFiVGKxXHUwMDA3YUD78PHKupRqR0hcdTAwMDMjYsZQYCwjdPnEXHUwMDFmKVx1MDAwMX5cdTAwMTixXHUwMDFhg1x1MDAxNDg1PurnXHUwMDE3wM9cdTAwMGbUx7yOykGC41x1MDAwYrBoKcPUsXiiSMH3gFx1MDAxZc7PXHUwMDA2bKpcdTAwMDFcdTAwMDaBXHUwMDA2OfFtXHUwMDBmQ1lcdTAwMTbHnKU45t314fHe3dPpyfP64V22/XyiV1+6MbzgzCGGcqVcdTAwMDXWkZ7QRCh5pIC3yVx1MDAxOFx1MDAwYqj2/6OiqGzKpFxc+OV86CYqXHUwMDFlhbVcdTAwMGbSTJuAjsOBeFx1MDAwNMyWYaeAb6z4n1dcdTAwMDePN4XGeuU6e9h/3u6frp4+bvy0I/yrXHUwMDFj1runzef2hiDV+sVzt70rmtv2fj+F63b2Lo6v869cdTAwMTf3b+3CjiquZ0q3XHUwMDBmaTjui8WnxuPl9Wnr7KJ2mbk6OCWbV/lo1/Vgukd1XHUwMDFhXHRr38J0VPC341xiqlMo6Vx1MDAxMEZhtsKyjVUzR5NvXHUwMDAwuIpcdTAwMDDv49jJSvpcdTAwMTc0pUo7Vlx1MDAxMbiGXHLMOV562P1w9tZcdTAwMWZnfXe9tVx1MDAwNF7OaDzhyZjQRFKasu5cdTAwMTToXHUwMDA1STZ1PnXncaO40mu0XHUwMDFlS6252eeeQlsm9WXgXGJcdTAwMTZFSFx1MDAxZb+tPXdBQ1x1MDAxNjeet2rqKdNbv+757GNcdTAwMDU61rVDKMajY0VcdTAwMGI/XHUwMDFhp5VgZknjvlx1MDAwZV5ys9M4ySg3cT3rXGbrtHCVUFYmo3FZ0tnsrauDLjneb1x1MDAxZr6VXHUwMDBm61x1MDAwN1dcdTAwMDc/XHUwMDFkd3jBry/XqubopSHzjd5Fdm+78HqYwnVfVq2s3qx2ykdXN3uXrcZL1rS2U7huuXR6ftg8O66fXHUwMDFmXHUwMDE0j/Y3+rXzaqWWXHUwMDE23bLoVF+YXGJcdTAwMDN/e4tAt7QwXHUwMDBlU1xuyFx1MDAxNtHAqsh45Vx1MDAwNVx1MDAwNmephj+D2ZakXHUwMDBl59ZahXXsMCrJi4vLXX4/OMzHYFvUMKs0j1x1MDAxOc9oKS5YabfWNlx1MDAwMkvwzlQq8PJgXHUwMDA3XHUwMDFmw2mp2Wh1/l2fM+I1hWhMXHUwMDEyr7HBLDZcdTAwMDUr3bcvc9Xsereynz3avFeWNNhVZFxuNlxiL1x1MDAwMD1cdTAwMTfCwYDKXHUwMDBiSifbYS0sXHUwMDA3+4H4gsLMJFxmpJaSNm4/Pa6sXHUwMDEw3H5nMeHNveKLPOk81+8r7bvM6ct6/7bX/OmtNHncfljPPD/svFx1MDAxZF+fiMPs5uvx4V1cbtd9O+ucnW3n9OPdvr14vTy/XHUwMDEytdJaXG7X3Xu6fCtcdTAwMTdeuqerjaOmXrvafNq6SavwlSRcXHKzMIFcdTAwMTb+9lx1MDAxNoGDUWKJg601XHUwMDE4MjDh7lx1MDAxM/NOwrSDpZypYFx1MDAxM22BhyRMK1x1MDAwN8s0MvnZSXRJwt7vaVx1MDAxYVx1MDAxY1x1MDAxNuOQMEmBKceOtSDaaMKS1aFcdNvzklryXHUwMDE5Yy1cXGTlPdLiPVx1MDAxMuE9T2Ne4iymkFx1MDAwZW+cxcSoVoJcdTAwMDe1MLysly13Kt2X41K1X9i9u34s7Fx1MDAxNu+jxlhISrhcdTAwMDJCpulkjFx1MDAwNdPSXHUwMDAxmUxcdTAwMTlcdTAwMWNSXHQ2mrc/XHUwMDE5YvFcdTAwMDU07PtDLEpRaVhIK1x1MDAxZqtcZjWExOJhjFx1MDAxMmPhvSbjYWEhXHUwMDE2glx1MDAwYtdcdTAwMTKzXGaxcH9r4UMswtlyWCFcdTAwMTlcdTAwMDWakCnQdMKvUVx1MDAwNZWIT5YqOVnZ81x1MDAxM4G4lU5AXHUwMDE5XHUwMDE5IEVUiPFQ9fnFoO+XguWZMVxiXHUwMDAze4Vk8aIqQNxzTsB6U0Ygy1x1MDAxOaGJtqVSRSCPQeLhMsU0MSio/kp4Xu5KsvorIN5cdKg1gj2subRcdTAwMTOV4lx1MDAxNdGYeGBcdTAwMDUnRFx0zyBjXHUwMDE2kFx053xcdFx1MDAwN6Bg6dNcdTAwMTJcdTAwMTRcdTAwMTblUlx1MDAwMJn/a2J4XHUwMDE0WDpCkVx1MDAxMtKosPZn/5j6MeGOqND6YFx1MDAwNmA9sP9cdTAwMTCVzNGYI+sp2OzOXFxcdTAwMTRgTsr4XCI7TLHJ6oRzi+zfL2orM1x1MDAwMztwKKYx0TyW1DWMXHUwMDBig41cIlNHdkvZz4fv+tkkXHUwMDFlLmucK3Cv1tvVYikqOMK6XHUwMDA1yzlcdTAwMTdY8Vx1MDAxZp75ODhcdTAwMGXatnOYzFxcXHUwMDEwYVRYca1I+Fx1MDAxZe5iSYjvzDBtNdO4XHUwMDA1hlx1MDAwMWR/hZ01S3yfXHUwMDE211x1MDAxOYrv2Fx1MDAwMIdTa6hcdTAwMDWEN1x1MDAxM6lhSNtcdTAwMTlWI4V5XHUwMDBiM0a6XHUwMDEykkfJYciSXHUwMDE0Y1ZTTsC8/GDeOExjdFx1MDAxN3zKMumK4F+C/Vx1MDAxMOzvZ1x1MDAwNnt4fUbYmFx0o1RcZkjg14Ty/Xz3p2DzxMNjmHOF+7FIvZJaXHUwMDAxoFx1MDAwYqxw6SpcdTAwMTP3flbBcsAph1x1MDAxN1xy6G8k91xmMybsh4c0Jlx1MDAxY8Jk+yfvLs8/XHUwMDEw2Vc3tjf05Vopo17l8e164/agtXboRXbvjjDlRjvUcKNAJIGNi/Hwa8VcdTAwMTRMXGZM0aKcauLqtDp00lvn3YnlR9xcdTAwMTlzPEHZc+uN+oFt4erMYK5cciZrm5hgrlxypVx1MDAwML9pd1x1MDAxNVVcdTAwMTRj8Eev+scq+3ptXHUwMDEyj1WXOaaJ4Mut4XdcdTAwMTN638pcdC1KXHUwMDE52mRUiLA9XHUwMDA0jYtcdTAwMTeXanKL91x1MDAxM4i0ciyjvjAklVx1MDAwM4u6XFzuXHUwMDFm+KPQw+zOKSUlaC0qYlx1MDAxNkXSXHUwMDEycSPtXHUwMDA2XHUwMDA2M9QuTVx1MDAxNYZWPSaJh8tcdTAwMTi/g0Z+SW1uarGtXHUwMDEy0fhcdTAwMDalNlx1MDAxM243x2AjeWGxVFxivIzZ24h+XHRcdTAwMTVWRGNJLs1cclx1MDAwMS00Slx1MDAxNvA7u+wn+ve0XHUwMDAygGHIbrAundKWcYLqaVx1MDAxNDUwXHUwMDAwMOGgx88y+E9cdTAwMThGiLfbXHUwMDFmdShRXHUwMDE4OmasVSBcXFxcXHUwMDBlqFx1MDAxMcZjcTyLPSpcZlx1MDAwMXZD5egzS6hcdTAwMWZC/WNcblCPjYCs5jH3ii0xhlOVNuNcdTAwMWPUv1x1MDAxYlnMj0F9sIni4TXOX1x1MDAwYv2T8ntmcFx1MDAwZq/vl/QuLVx1MDAxN4JTqYjgkomxXHUwMDA1mDhcdTAwMDJelVRGcFx1MDAwYnRcdTAwMTKIZpj/8lx1MDAxZlx1MDAwM+/hJdbC4N1q7OiMXHUwMDFkU/yaXHUwMDAzwa/WWGU7kLhb4XjSjYawroTDl/H9wYBem1x1MDAxOdBhXHUwMDFlXGKYXHUwMDA3Mlx1MDAxNp5LmDRcdTAwMWGbTaTN3JlcdTAwMTXk511/PiaJh8tcdTAwMTh/LX4rYlB+XGIgYkpTPbZMXHKiXCIwX0dqwTkzzGedionu4VXTXHUwMDEyjkHASlxuz0WhuFx1MDAxMnoy6lx1MDAwZsBcdTAwMWbOc8WxhKGEUSzRXHUwMDFkwHZ1x1xcbJfsY77AyVU729g9rm550T2oKJKxXHUwMDBlo1x1MDAwMN7ErzWQXHUwMDAwXG5cdTAwMGa3XHUwMDA0yyyl441xh1x1MDAwNN5cYkdcdTAwMTksXHUwMDEyXHUwMDA21Fx1MDAxMVx1MDAwZp/AYVxmXHUwMDEyXHUwMDAx7vRFUVx1MDAxZbac+/VQL6JifXBCPTFMMFx1MDAxYrdPhtEghFWyrt7Dm4uVy1XcqGa3WHmtUs4+N87pSea5enM7XHUwMDBmZXuwXHUwMDExOU3msJxHWPB/zlx1MDAxZVjwKdtjpUM+0GA8SWmAXGJKOlx1MDAxMi2N2ok92VFcblx1MDAxM3dcYlwiXHUwMDAxMYNcbubyW1KYXHUwMDE2g/JcdTAwMTl/XHUwMDFj8O+MXHUwMDAzT5gybmNV0NCDKr9umEhHrFx1MDAwZuZOXCKx7qrbM1GiXHUwMDFlru16oT+WtTRlffWr3lx1MDAxMzyMRclTXG5cdTAwMDfi8HgjrMKoXHTIdFx1MDAwYkJDjnulOeWOxiBcdTAwMDUgf9ZSb7SRkFx1MDAwZceCx9ixllhtfcDFcodjwFx1MDAxMrY7ZMSF6EvK4YKazOzyXHUwMDEy1Fx1MDAwNjC6eFx1MDAwMESNpFxu41RS3i6EK1x1MDAxYZYsizJVfVx1MDAxOWifeExapke+fIHU/JJIXHUwMDFkLrixRFOKvSVA2Y+rNFjyKcG6XHUwMDAyXHUwMDE2OKnRM+8jhiNwwiEsg43w/yZgfffhQt5cXOcvL9r982r5+fS+t7+7XHUwMDE3XUtcbutQmIagzX12XHUwMDBihVx1MDAwMsuwQChJsJaET7xvwpCgqiC/S0t+v1x1MDAxYqiWjYrrwVKSUcI48a9cYumJRlx1MDAxYXU9I1x1MDAwMlZt4cLgL9eS91lSfF7vP5xcdTAwMTfyzbt6/5BcdTAwMTNyVv+G0rJcdTAwMWW0nlxcjTCzwLCFKVx1MDAwMev/nKNoSSlcdTAwMWSDXHUwMDFkUqWR2HF+nOzByuFQq4BcdTAwMGZcdTAwMDRqSaVcdTAwMWQsSWa5XHUwMDE5NE3jxotcYkst6Vx1MDAwZlx1MDAwNFx1MDAwMf2H/LQkXHUwMDE3lGDNTN9cdTAwMWH/IdVcdTAwMTi1XHUwMDAwXHUwMDFjdmVqpsLkZpg7IVqSzYOUnLK8TpeSrlEsipJcZofhUCVJMfZEXHUwMDAw0+ZcdTAwMDBccsylQ96VJLY8ooBcdTAwMWNCXHUwMDBm4kK9WtJiXHUwMDFkWcVcdTAwMDF4JJNcdTAwMWGUjFx1MDAwZt3gysGKo1x1MDAxZpXIhP2C2JNcdTAwMDUgXHUwMDFks/sqJXa1pP6+ykDOIbFlJtM0XHUwMDEx5ZiGQD+furJcdTAwMWFooHh4TdOjYH6NnMTOpkZxzFx1MDAwZSXaTFx1MDAwNnXALLZcdTAwMTaGq4jRQFx1MDAxNGZOXlx0h+GkY6BcdTAwMDJew+B9Kan5RNDhxFnxz1x0O1x1MDAxY37lqfFSOqi+P732VbVz/zFcdTAwMTn/TI88XHUwMDE33fLz00U2+3D33HmrZM9a+bddXHUwMDFhXZYy6XBBqH9cdTAwMTCL0MaxRlx1MDAwMlxytUG9d7V2/Fx1MDAwMliWUjR8VXiKuihcdTAwMDRcdTAwMTeoXHUwMDE0hFhD4lx1MDAxNajE0nmAJyyRi2N4b7F06DW7Kzzlj3fu+1xc1Hezra3OcyFy9+1l61x1MDAxNFeU0GK1TvG3iyi6mSO0SKqYheVf6XFqKzlxgPHCkvlep9tPN1PHgnA2mlx1MDAwMpGh+ltKeS+EbK77g5afalx1MDAxNoz4e189P1x1MDAxYnlfXHUwMDAxl1x1MDAxNEmfrlLiKlx1MDAwM5eS83VcdTAwMWVcdTAwMDTzlIU/gu918Vx1MDAwNLPmLHf4unFY2Lyurbezr+3X61x1MDAxZJ94L29CMCCJXHUwMDAzxkex8La3XHKntMTBXHUwMDEy81JcdTAwMTI+aMPpXHUwMDAxlWU+8EpyXFxpRiVDwelcdTAwMTlcdTAwMDTeXHUwMDFhQL//Ll0gXHUwMDFiotxySlnaXHUwMDExXHUwMDFmoNaEYT9fpG2ZXHUwMDEw/HNcdMHHO4XMyT4rbN+96lx1MDAwYvpSOtvsMeFFoqA+TlxuR1x1MDAxM5RcdTAwMTQsMLmAXGJKXHUwMDAympVcdTAwMTjjoJ9qXHUwMDE4duqTN/a79NlcdTAwMGZwnVZUTFxudlx1MDAxNWKQtnS31YxcdTAwMDBJuGGrXHUwMDAxPVx1MDAxMkWA+CuTaVx1MDAwMu1yM39AXHUwMDBl+3uZq87reTWTub59PVmdh+ZFXHUwMDAw6Li7tSiCx/85R1x1MDAxMDxcdTAwMWFcdTAwMTONhFwiXGZzj32CToVcdTAwMDO3Q8V4fdhlxOnMXHUwMDEw0I5cdTAwMTFwarnkmijfXGLzYMkjXHUwMDE5odjPIGXNoylROs3WRfNcdTAwMTh6OmWBXHLvX7T4Qah78vRMbFdebnq1fX6ao4+HXHUwMDA3bs0zjX9cdTAwMDB15Lgrj1x1MDAwNVx1MDAwZrz8Q8JZLa32b1wiiaFKWr3HMVxyQpV8IOd38Y/v31x1MDAxZu7MTD8w1sxyLv02YXiwJOJcdTAwMWMwxIhvXGZU2uq1VO74+altz8ubT5nL7Crrm29oyjiVf1jNqUlW7X9cdTAwMWVcdTAwMTHB/zlH4Vx1MDAxZlxcOYRhPzzGqU+gXHUwMDEycYxcdTAwMTaEcX/+sYxSSopcdTAwMDHd6PxcdTAwMDMkn+FYscVvtktcdTAwMWU424nQklx1MDAwYldpkpRcYlxiTpyZuvaEXHUwMDEykLmIV5qywMbhXHUwMDFmXHUwMDBiXHUwMDE4uXRQLF7Uri5fZLHXPKvZ/T47PKpEp1x1MDAxZoQ5lFxmUlx1MDAwMfzoh9ZcdTAwMGVGftD31NqlUzotwHmZmXRcdTAwMDBvxFx1MDAwNDpfXHUwMDE4XG7p4CotXHUwMDA2fiRqlZGMc9yfvGw8POxztp/J9bcq16dHR6zzi1x1MDAxYkxP5TJcdTAwMTLXbrUwXFzG//1F4TLEOloqY0Ce+DiPXHUwMDE5JmlQo2Qgm1n6jpNhSy9cdTAwMGWZgdfPXHUwMDE5891NXHTcTKFcZqtcdTAwMWZcdTAwMDKhXHUwMDE5sZ2UPMhcdTAwMTab2nxcdTAwMTmZmVx1MDAwYl/ylOU61mbK4pGZdk5cXGZo5pHzyt5++eDhfPPuYCOKV9lS6XDFLGGa+3iVjXVcdTAwMTCWKcV+XCJSeCtcdTAwMDAuvcoryVx1MDAxMac/s1dZ4Fx1MDAwNi3o3ZgtXHUwMDAzpNSwMqSfxMtcdTAwMTmXiVwiWZZO5Vx1MDAwNXEqXHUwMDFmXGJ++7y6etnUZ3Rn5+7kTdremlx1MDAxN4iCOtNTorBtvKZcdTAwMDFeZen4+JhGu7rSke5cZlS/jJBfJbC+n1x1MDAwNeVcdJlZYjEqXHUwMDAxXHUwMDBl4tacXHUwMDA2wsso1eRcdTAwMWL9yneb590yPzlcdTAwMTJ35KbV3njtt3bWI1x1MDAwN/5+ZUN2bFx1MDAwNKNIXCIknUd24v+cI2ghIFx1MDAxZdwhwlqmfKtcdTAwMTlpTLExUo7zjziO5WX1Sn9cdTAwMTSg/ijgXHUwMDFiSGuwa4WIXHUwMDE3RVwiJcd6u8nC/L9k4nxKoXn0JE9ZVSfFz+L7jp87jdZ+O3+YPcpuXHUwMDFlqGLn+f62eFx1MDAxY51mXHUwMDEwJVx1MDAxZMNVUNlEaVx1MDAxZGOpXG5cZl6zXHUwMDBlXHUwMDAzLHqPXFxbXHUwMDA2ryVcdTAwMDNcdTAwMTg2M83g2nCiaLx655JzwzlcdTAwMTUpsox3i3w765ydbef0492+vXi9PL9cdTAwMTK10lo0Olx1MDAxMEAzpkbFXHUwMDE12vnW3t5rVcvbu7fS6jm963fjsVxmpXmyIOB5xFx1MDAwNP/nXHUwMDExhWVcdTAwMTCtXHUwMDFjrTFtmPvVuWDcMVx1MDAxZq7hhO7jtFlGp5Wrt5u5XHUwMDE25l+mXHRcdTAwMDTf7tDJXHUwMDEzXHUwMDFlY9dcdTAwMTU0XHUwMDAyyD9cdTAwMTGvZJnE71x1MDAxMJ52M1xcY7RSiiWZPP5MYy5cXMZT1tVworGATuKb24OnTKN7q0Sv3yrtXHUwMDFkvFx1MDAxNW5zOlx1MDAwNs/g1Fx1MDAwMSpcdTAwMTBQgFx1MDAxZnOXsVO74EFeYqtcdTAwMWOLtfk/qcbIyEdMQ32BN2dcdTAwMTFcYkZcbkWZObFGXHUwMDAyzYi1kVx1MDAwMauqZYIla8iajFx03J6UT/N3T/r+Vl1cdTAwMTbEgz1pb99eRN3HkMfth/XM88PO2/H1iTjMbr5cdTAwMWVcdTAwMWbepeArnoFcdTAwMTBNZy5KaM5cdTAwMTfGV+z//lwiMVx1MDAxN2GRmzBGP+pcdTAwMWaMM1x1MDAxNzJccmBA6VxiXHLAgk3iJLe+cbBk6S72XHUwMDA1XHUwMDE4XHUwMDE5g7lg815cdTAwMDE0M1byXHUwMDFmkElcdTAwMDVcdTAwMTCU+lx1MDAxZUnyyfPJXFzoSrvUWWmU4YpcdTAwMTPO4va/681Sa+WsVG83WpsvcDf/rlx1MDAwZux3XHUwMDBlyM2UxXyS3LhcdTAwMDY57kBur0xcZnFlfISLQn5e5Pb1S8nWT1x1MDAwZliZXHUwMDFk37+sXHUwMDBive5cdTAwMDNLPr2LmULJpDjG3IIoXHUwMDFhXHUwMDE58GeqMuFGc7V0Kn9YXqq4pKNcdTAwMTKfQK8y+mG0XHUwMDEy1jcyP9iDQ8yg96ROXHUwMDE0JVx1MDAxN1JVRFx1MDAxOViZXHUwMDEy4dXSrbwgbuXwKjvTXG7WW6zkLbGBXHUwMDE1MSPzfS8ziFx1MDAxYsFEcWtcdTAwMTihLn32XHRF3GLqolx1MDAwMNO2RGhK/VxcyphcdM2oJMpIYGFSj1x1MDAwMG1uXHUwMDA12Vx1MDAwZmz0rM1cZktcdTAwMTZEXHUwMDE1U0bEXCJRTFwii5Kpg5KxQpGf76i+XHUwMDFhaKB4eE0zTYT63lwig1ZwhFikXGaWuVx1MDAxYVW9n4Z3oVx1MDAwNNdYg1x1MDAxMmSQsZ5xxiwyXHUwMDE4Xrom4Vx1MDAxOJhEMWYwZVG7XHUwMDFi4fqe9S4mi1pjMFx1MDAwNPfDk/dDm9ZL4UjBXHUwMDE5zlJ4rq5cZqPPRiUgmjlX2lgrXFxOkE/gV8rBTWPGXHUwMDE5sbh97LOrb4VDmVTESiCxxrh+xdzi/k/o5PWZgV9cdTAwMWJcdC/R8Fh0XHUwMDE0qFx1MDAxYShcdTAwMTDJXHUwMDEyNUf7kpJcdTAwMGapXHUwMDAyf6B94uGxzO+A/eO3tedu9e28uPG8VVNPmd76dW97VsikymC7d1xccLHvxHjbT1x1MDAwMtyNYsKnkJJcdTAwMWHDrJlcdTAwMTX2w1P2ky5dhGKoXHUwMDAxZZpcbsLtxFx1MDAxMCbOqrD6uP9cdTAwMTjcXHUwMDBmTzRcbsd9hj0vtVJWS2wxPYH7xpFgT+azrLhcdTAwMDf3seu95Fx1MDAxNub6wNtIfTticuFoUMfwXHUwMDAxM9hY/Vx1MDAwNU2qfoDxz96jXG57S1x1MDAxOEG1by+T4Fx1MDAxMlx1MDAwMVx1MDAwNjuQXHUwMDAxl0q7tcEgaTiRyzdV6Fx1MDAwZjbRwVmPcf5e8Mdmh1x1MDAxOL8plFx1MDAwNYU+1oyLXGbixFx1MDAwN1HekmNt7rBuwpGwPzxdOinlZ1x1MDAxYaRcIlx1MDAwM2WCXHUwMDFhdMRg/M4y+c/piDz8ylxmdcXDs1xcp4lcdTAwMDOQkVx1MDAxYZttXHUwMDBieP6TvSe4XHUwMDAztFx0bM4whpmWI/RcdTAwMTmFXHUwMDE2O1pbXHUwMDA211FEW+5cdTAwMWJaTFx1MDAxNXe4UnSQhym1dIWlLleJ0SqxmcIqoTXlXHUwMDAxpa2C1lxi0H5cYp2cpOxde09j/vlOhqvBJoqH1zh/7yqBZV+wgyRcdTAwMTBoY8xE61x1MDAwNkO00Fx1MDAxMt42M5LCMesqXHUwMDExnoeadFxiglhsumix7VwiU/yvkLPShKmcxVolQsC/Ut2t1rM5Ub47uur1u52H/Vx1MDAxZOFTK8gn41VcdFx1MDAwN0RcdTAwMTjR2FpI6ImUXHUwMDEySZVDqTVcdTAwMDaVXHUwMDE4iH2vR0BKx1xuXHUwMDEw3Vx1MDAxYYO6uLtt8lx1MDAxMPl/Q8DEXHUwMDBmOCa3UkB6XHUwMDAyoIWbXHUwMDAxsaBcdTAwMWW0XHUwMDAwbqjalH1cdTAwMDBGccvmoIhyoEVOfHt2XFxf+iTfrSdCXHUwMDAyYFx1MDAxOP2kYJGOpVx1MDAxY9Ccg1x1MDAxOTE96ZRcdTAwMDSBp+F9XG5cdTAwMDNcdTAwMWNFevcomHRcZqxcdTAwMDbKaFx1MDAxNHrKXG6fXGZcdTAwMTSrXHUwMDFjuDBcdTAwMDNDxWYq8lx1MDAxN/gkf2JvendmSEL/glx1MDAwMe5cdTAwMTmvyVxykULDq09cdTAwMTYkOi397edcdTAwMTFpNdhC8Zi0ze+gnqX79mWuml3vVvazR5v3ypJcdTAwMDa7mnlnVzAglkAujeW4hTuxQ6FA1CtcdTAwMTCSXG77c1pcdTAwMWLm0YvGPUPTXHUwMDAwXHUwMDEzjoFg4zbJJOghXHUwMDFjXHUwMDA1nSCfk6eXbsmpsc3h0C+xcJyyXHUwMDA2XHUwMDBiOClcIrx+SSCdzGI7PVx1MDAwNYu6t+2lpJirZFx00Fx1MDAxOWYs81x1MDAwYkhhXHUwMDE41atgXHUwMDEwzFx1MDAwMCRQs9x48MP+vZmxXHUwMDFm3lx1MDAxMWVaxNx4wK6XXHUwMDE4ypCyY9IoyYWcg7aXQVx1MDAwNoqH1zR/L/ZcdTAwMWLAe6OEwNRcdTAwMWXubXppNExmXHRaXHUwMDA1tya8e/AxoT88MSvhXHUwMDEwMD+JKmWM1Vx1MDAxYdewv8LOXHUwMDEyb4PSfyDyh2fLTCX9jDIhqKbEjqeCIeWHnzKcMtKS0Vx1MDAwZdBox1x1MDAxOWvfXGJlLbxcdTAwMTKwKWp9XHUwMDAyUqhcdTAwMDIlXGLnleGMXHUwMDEz6+qIvMT9XHUwMDEx7u/PjPtYqIZJIK8xSb/ihGDYYNrAj/lcdTAwMWM/75ZcXFxyNlE8PMb5i5FfMobSnGilXHUwMDE5o1x1MDAxZehcdTAwMTdaU2U1OpbM7H7J8LSVpKxcdTAwMWZ4qNVcdTAwMDQraVx1MDAxMbzXiU2jyfN2yfvxO5eMnveP997WzlQlw2/OXHS9Wrvyor83UVx1MDAwZuR+SLcszqWjQGIp9d5cdTAwMTPLJyqFXHUwMDE5x1pAXHUwMDFjwTGWkfrVjmYpbzsvUJGBI3/I9+2Tg4xcdTAwMWWIacySilpwhc3P0yb1ckBcYpJg+2eu3tl9rlUsXHUwMDE1V/L9lftcXPve9SpHiXiu2k1RXHUwMDEy8WqlssskPGl4nYarXeJ4XHUwMDBl3tiNTybcXHUwMDA13umiJNRdXHUwMDFlXHUwMDFk35nnOj8rX29cdTAwMTa7XHUwMDBm1e7FTe80XG58XHUwMDAwPDuCa0swTsRcdTAwMDJPnIBcdTAwMGbhSE1cdTAwMTR8alx0XHUwMDFmXHUwMDFmppUqfFx1MDAxY0eHXHUwMDBmXHUwMDEwTcRcIr2PXHUwMDA1XHUwMDFmVmjKsWpi2uihuUrWj3uJXHUwMDFlc4dcdTAwMWV8+2I7f17JXZ6eVlj+ZstcdTAwMWWY7Z0o6DHYc5RSgmFcdTAwMWE/+Fx1MDAxMMpRhlx1MDAxMMmX8PFpW6nCx0l0+Fx1MDAxMNJIg07uOOghqKBMsbRlpdRaykRcdTAwMTluS/D4SfD414eS/pNrNs868E6H2vLPS7XUW/fPfIdcdTAwMDO/P4BcdTAwMWWcZ6XBnsh//vWf/1x1MDAwZqX7rXUifQ==CentralSensor 1Sensor 2Sensor NSensorConnection Sensor NSensorConnection Sensor 2SensorConnection Sensor 1runRecv()Event DedupingQueueNetFlowUpdateDedupingQueue1 DedupingQueue perMsgFromSensor typePod workerQueueVMIndexReportworkerQueue1 workerQueue perSensorEvent typePodDedupingQueue 17PodDedupingQueue 2PodDedupingQueue 1VMIndexReportDedupingQueue 17VMIndexReportDedupingQueue 2VMIndexReportDedupingQueue 1DedupingQueue 17DedupingQueue 21 set of 17DedupingQueuesper SensorEventtypeSharded by hashSharded by hashSharded by hash \ No newline at end of file diff --git a/central/sensor/service/connection/sensorevents.go b/central/sensor/service/connection/sensorevents.go index 2df8b5aae82db..236163f92c4dc 100644 --- a/central/sensor/service/connection/sensorevents.go +++ b/central/sensor/service/connection/sensorevents.go @@ -89,6 +89,10 @@ func (s *sensorEventHandler) addMultiplexed(ctx context.Context, msg *central.Ms // The occurrence of a "Synced" event from the sensor marks the conclusion // of the initial synchronization process. case *central.SensorEvent_Synced: + if s.reconciliationMap.IsClosed() { + log.Warnf("Ignoring duplicate reconciliation event for cluster %s (%s)", s.cluster.GetName(), s.cluster.GetId()) + return + } // Call the reconcile functions log.Info("Receiving reconciliation event") diff --git a/central/sensor/service/connection/synced_event_regression_test.go b/central/sensor/service/connection/synced_event_regression_test.go new file mode 100644 index 0000000000000..a1541d5e1b914 --- /dev/null +++ b/central/sensor/service/connection/synced_event_regression_test.go @@ -0,0 +1,72 @@ +package connection + +import ( + "testing" + + hashManager "github.com/stackrox/rox/central/hash/manager" + pipelineMock "github.com/stackrox/rox/central/sensor/service/pipeline/mocks" + "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/set" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" +) + +// Malformed unchanged_ids should be ignored rather than crashing the Synced +// event reconciliation path. Valid entries still need to reconcile, and sync +// bookkeeping still needs to complete. +func TestSyncedEventMalformedUnchangedIDsDoesNotPanic(t *testing.T) { + const clusterID = "regression-cluster" + + ctrl := gomock.NewController(t) + mockPipeline := pipelineMock.NewMockClusterPipeline(ctrl) + var reconcileCalls int + mockPipeline.EXPECT().Reconcile(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn(func(_ any, _ any) error { + reconcileCalls++ + return nil + }) + + stopSig := concurrency.NewErrorSignal() + initSyncMgr := &initSyncManager{ + maxSensors: 1, + sensors: set.NewStringSet(clusterID), + } + deduper := hashManager.NewDeduper(nil, clusterID) + deduper.StartSync() + + handler := newSensorEventHandler( + &storage.Cluster{Id: clusterID, Name: "regression-cluster"}, + "regression-sensor-version", + mockPipeline, + nil, + &stopSig, + deduper, + initSyncMgr, + ) + + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Id: "synced-regression", + Resource: ¢ral.SensorEvent_Synced{ + Synced: ¢ral.SensorEvent_ResourcesSynced{ + UnchangedIds: []string{ + "Deployment:deployment-2", + "definitely-not-a-deduper-key", + "AlertResults:alert-1", + }, + }, + }, + }, + }, + } + + assert.NotPanics(t, func() { + handler.addMultiplexed(t.Context(), msg) + }) + assert.Equal(t, 1, reconcileCalls) + assert.True(t, handler.reconciliationMap.IsClosed()) + assert.False(t, initSyncMgr.sensors.Contains(clusterID)) + assert.Len(t, handler.workerQueues, 0) +} diff --git a/central/sensor/service/connection/upgradecontroller/register_connection_test.go b/central/sensor/service/connection/upgradecontroller/register_connection_test.go index be02b8f750800..00ab06834625d 100644 --- a/central/sensor/service/connection/upgradecontroller/register_connection_test.go +++ b/central/sensor/service/connection/upgradecontroller/register_connection_test.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/version/testutils" "github.com/stretchr/testify/assert" @@ -86,6 +87,10 @@ func (f fakeConnection) InjectMessageIntoQueue(_ *central.MsgFromSensor) { panic("not implemented") } +func (f fakeConnection) HasCapability(_ centralsensor.SensorCapability) bool { + return false +} + func (f fakeConnection) CheckAutoUpgradeSupport() error { if f.autoUpgradeSupported { return nil diff --git a/central/sensor/service/connection/upgradecontroller/stateutils/all_transitions.go b/central/sensor/service/connection/upgradecontroller/stateutils/all_transitions.go index d47442242a6b8..9eac3b6b4e583 100644 --- a/central/sensor/service/connection/upgradecontroller/stateutils/all_transitions.go +++ b/central/sensor/service/connection/upgradecontroller/stateutils/all_transitions.go @@ -2,18 +2,9 @@ package stateutils import ( "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/sensorupgrader" ) -func statePtr(state storage.UpgradeProgress_UpgradeState) *storage.UpgradeProgress_UpgradeState { - return &state -} - -func upgradeTypePtr(typ storage.ClusterUpgradeStatus_UpgradeProcessStatus_UpgradeProcessType) *storage.ClusterUpgradeStatus_UpgradeProcessStatus_UpgradeProcessType { - return &typ -} - var ( // These define all the valid transitions we handle. // Note that the first match wins, so order these transitions with that in mind. @@ -32,7 +23,7 @@ var ( // So we MUST handle all non-terminal states through the below transitions. { - workflowMatch: pointers.String(""), + workflowMatch: new(""), currentStateMatch: anyStateFrom( storage.UpgradeProgress_UPGRADE_INITIALIZING, // This should basically never happen, but being defensive can't hurt. @@ -49,89 +40,89 @@ var ( ), workflowToExecute: sensorupgrader.RollForwardWorkflow, - nextState: statePtr(storage.UpgradeProgress_UPGRADER_LAUNCHED), + nextState: new(storage.UpgradeProgress_UPGRADER_LAUNCHED), }, { // Upgrader restarted in the middle of rolling back. Tell it to keep rolling back. - workflowMatch: pointers.String(""), + workflowMatch: new(""), currentStateMatch: anyStateFrom(storage.UpgradeProgress_UPGRADE_ERROR_ROLLING_BACK), workflowToExecute: sensorupgrader.RollBackWorkflow, - nextState: statePtr(storage.UpgradeProgress_UPGRADE_ERROR_ROLLING_BACK), + nextState: new(storage.UpgradeProgress_UPGRADE_ERROR_ROLLING_BACK), }, // The following are roll-forward transitions. // Note that we don't check the starting state here (we know it's not terminal since that was checked above, // and the end state only depends on the upgrader action). { - workflowMatch: pointers.String(sensorupgrader.RollForwardWorkflow), + workflowMatch: new(sensorupgrader.RollForwardWorkflow), stageMatch: anyStageFrom(rollForwardStagesBeforePreFlight...), - errOccurredMatch: pointers.Bool(false), + errOccurredMatch: new(false), workflowToExecute: sensorupgrader.RollForwardWorkflow, - nextState: statePtr(storage.UpgradeProgress_UPGRADER_LAUNCHED), + nextState: new(storage.UpgradeProgress_UPGRADER_LAUNCHED), }, // An error occurred before we could even do pre-flight checks! // Mark it as a fatal error, and tell the upgrader to clean up. { - workflowMatch: pointers.String(sensorupgrader.RollForwardWorkflow), + workflowMatch: new(sensorupgrader.RollForwardWorkflow), stageMatch: anyStageFrom(rollForwardStagesBeforePreFlight...), - errOccurredMatch: pointers.Bool(true), + errOccurredMatch: new(true), workflowToExecute: sensorupgrader.CleanupWorkflow, - nextState: statePtr(storage.UpgradeProgress_UPGRADE_INITIALIZATION_ERROR), + nextState: new(storage.UpgradeProgress_UPGRADE_INITIALIZATION_ERROR), updateDetail: true, }, // Yay, passed pre-flight checks! { - workflowMatch: pointers.String(sensorupgrader.RollForwardWorkflow), + workflowMatch: new(sensorupgrader.RollForwardWorkflow), stageMatch: anyStageFrom(sensorupgrader.PreflightStage), - errOccurredMatch: pointers.Bool(false), + errOccurredMatch: new(false), workflowToExecute: sensorupgrader.RollForwardWorkflow, - nextState: statePtr(storage.UpgradeProgress_PRE_FLIGHT_CHECKS_COMPLETE), + nextState: new(storage.UpgradeProgress_PRE_FLIGHT_CHECKS_COMPLETE), }, // Oh no, pre-flight checks failed! { - workflowMatch: pointers.String(sensorupgrader.RollForwardWorkflow), + workflowMatch: new(sensorupgrader.RollForwardWorkflow), stageMatch: anyStageFrom(sensorupgrader.PreflightStage), - errOccurredMatch: pointers.Bool(true), + errOccurredMatch: new(true), workflowToExecute: sensorupgrader.CleanupWorkflow, - nextState: statePtr(storage.UpgradeProgress_PRE_FLIGHT_CHECKS_FAILED), + nextState: new(storage.UpgradeProgress_PRE_FLIGHT_CHECKS_FAILED), updateDetail: true, }, // Ooh yeah, upgrade done from the PoV of the upgrader! { - workflowMatch: pointers.String(sensorupgrader.RollForwardWorkflow), + workflowMatch: new(sensorupgrader.RollForwardWorkflow), stageMatch: anyStageFrom(sensorupgrader.ExecuteStage), - errOccurredMatch: pointers.Bool(false), - upgradeTypeMatch: upgradeTypePtr(storage.ClusterUpgradeStatus_UpgradeProcessStatus_UPGRADE), + errOccurredMatch: new(false), + upgradeTypeMatch: new(storage.ClusterUpgradeStatus_UpgradeProcessStatus_UPGRADE), // For upgrades, tell the upgrader to stay in the roll-forward workflow, and keep polling until // we ask it to clean up (after we hear from the sensor). workflowToExecute: sensorupgrader.RollForwardWorkflow, - nextState: statePtr(storage.UpgradeProgress_UPGRADE_OPERATIONS_DONE), + nextState: new(storage.UpgradeProgress_UPGRADE_OPERATIONS_DONE), }, // Ooh yeah, upgrade done from the PoV of the upgrader! { - workflowMatch: pointers.String(sensorupgrader.RollForwardWorkflow), + workflowMatch: new(sensorupgrader.RollForwardWorkflow), stageMatch: anyStageFrom(sensorupgrader.ExecuteStage), - errOccurredMatch: pointers.Bool(false), - upgradeTypeMatch: upgradeTypePtr(storage.ClusterUpgradeStatus_UpgradeProcessStatus_CERT_ROTATION), + errOccurredMatch: new(false), + upgradeTypeMatch: new(storage.ClusterUpgradeStatus_UpgradeProcessStatus_CERT_ROTATION), // For cert rotation, when the upgrader says it's done, we mark the upgrade complete. workflowToExecute: sensorupgrader.CleanupWorkflow, - nextState: statePtr(storage.UpgradeProgress_UPGRADE_COMPLETE), + nextState: new(storage.UpgradeProgress_UPGRADE_COMPLETE), }, // Oh no, upgrade operations failed. :( Tell the upgrader to roll back. { - workflowMatch: pointers.String(sensorupgrader.RollForwardWorkflow), + workflowMatch: new(sensorupgrader.RollForwardWorkflow), stageMatch: anyStageFrom(sensorupgrader.ExecuteStage), - errOccurredMatch: pointers.Bool(true), + errOccurredMatch: new(true), workflowToExecute: sensorupgrader.RollBackWorkflow, - nextState: statePtr(storage.UpgradeProgress_UPGRADE_ERROR_ROLLING_BACK), + nextState: new(storage.UpgradeProgress_UPGRADE_ERROR_ROLLING_BACK), updateDetail: true, }, @@ -139,42 +130,42 @@ var ( // Rollback still in progress. { - workflowMatch: pointers.String(sensorupgrader.RollBackWorkflow), + workflowMatch: new(sensorupgrader.RollBackWorkflow), stageMatch: anyStageFrom(sensorupgrader.SnapshotForRollbackStage, sensorupgrader.GenerateRollbackPlanStage, sensorupgrader.PreflightNoFailStage), - errOccurredMatch: pointers.Bool(false), + errOccurredMatch: new(false), workflowToExecute: sensorupgrader.RollBackWorkflow, - nextState: statePtr(storage.UpgradeProgress_UPGRADE_ERROR_ROLLING_BACK), + nextState: new(storage.UpgradeProgress_UPGRADE_ERROR_ROLLING_BACK), }, // Rollback done, now clean up. { - workflowMatch: pointers.String(sensorupgrader.RollBackWorkflow), + workflowMatch: new(sensorupgrader.RollBackWorkflow), stageMatch: anyStageFrom(sensorupgrader.ExecuteStage), - errOccurredMatch: pointers.Bool(false), - upgradeTypeMatch: upgradeTypePtr(storage.ClusterUpgradeStatus_UpgradeProcessStatus_UPGRADE), + errOccurredMatch: new(false), + upgradeTypeMatch: new(storage.ClusterUpgradeStatus_UpgradeProcessStatus_UPGRADE), workflowToExecute: sensorupgrader.CleanupWorkflow, // On upgrades, don't mark as rolled back until the sensor checks in. - nextState: statePtr(storage.UpgradeProgress_UPGRADE_ERROR_ROLLING_BACK), + nextState: new(storage.UpgradeProgress_UPGRADE_ERROR_ROLLING_BACK), }, { - workflowMatch: pointers.String(sensorupgrader.RollBackWorkflow), + workflowMatch: new(sensorupgrader.RollBackWorkflow), stageMatch: anyStageFrom(sensorupgrader.ExecuteStage), - errOccurredMatch: pointers.Bool(false), - upgradeTypeMatch: upgradeTypePtr(storage.ClusterUpgradeStatus_UpgradeProcessStatus_CERT_ROTATION), + errOccurredMatch: new(false), + upgradeTypeMatch: new(storage.ClusterUpgradeStatus_UpgradeProcessStatus_CERT_ROTATION), workflowToExecute: sensorupgrader.CleanupWorkflow, // On cert rotation, we mark as rolled back when the upgrader says it has rolled back. - nextState: statePtr(storage.UpgradeProgress_UPGRADE_ERROR_ROLLED_BACK), + nextState: new(storage.UpgradeProgress_UPGRADE_ERROR_ROLLED_BACK), }, // Any error when rolling back => rollback failed. Not much we can do at this point. :( { - workflowMatch: pointers.String(sensorupgrader.RollBackWorkflow), - errOccurredMatch: pointers.Bool(true), + workflowMatch: new(sensorupgrader.RollBackWorkflow), + errOccurredMatch: new(true), // Upgrader might as well clean up. workflowToExecute: sensorupgrader.CleanupWorkflow, - nextState: statePtr(storage.UpgradeProgress_UPGRADE_ERROR_ROLLBACK_FAILED), + nextState: new(storage.UpgradeProgress_UPGRADE_ERROR_ROLLBACK_FAILED), updateDetail: true, }, @@ -184,7 +175,7 @@ var ( // expect to clean up eventually. { currentStateMatch: anyStateFrom(storage.UpgradeProgress_UPGRADE_ERROR_ROLLING_BACK), - workflowMatch: pointers.String(sensorupgrader.CleanupWorkflow), + workflowMatch: new(sensorupgrader.CleanupWorkflow), workflowToExecute: sensorupgrader.CleanupWorkflow, noStateChange: true, }, diff --git a/central/sensor/service/connection/upgradecontroller/stateutils/state_machine_test.go b/central/sensor/service/connection/upgradecontroller/stateutils/state_machine_test.go index 0d6e0b30e4b74..9c70aeb560cdc 100644 --- a/central/sensor/service/connection/upgradecontroller/stateutils/state_machine_test.go +++ b/central/sensor/service/connection/upgradecontroller/stateutils/state_machine_test.go @@ -64,15 +64,15 @@ type mockUpgrader struct { done concurrency.Flag - numRunsDone int32 + numRunsDone atomic.Int32 } func (m *mockUpgrader) incrNumRunsDone() { - atomic.AddInt32(&m.numRunsDone, 1) + m.numRunsDone.Add(1) } func (m *mockUpgrader) getNumRunsDone() int { - return int(atomic.LoadInt32(&m.numRunsDone)) + return int(m.numRunsDone.Load()) } // A simplified version of the logic we expect the upgrader to perform. diff --git a/central/sensor/service/connection/upgradecontroller/upgrade_controller_test.go b/central/sensor/service/connection/upgradecontroller/upgrade_controller_test.go index f77903e231d7a..040a06daf8d76 100644 --- a/central/sensor/service/connection/upgradecontroller/upgrade_controller_test.go +++ b/central/sensor/service/connection/upgradecontroller/upgrade_controller_test.go @@ -9,6 +9,7 @@ import ( "github.com/stackrox/rox/central/sensor/service/connection/upgradecontroller/stateutils" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/sensorupgrader" @@ -95,6 +96,10 @@ func (r *recordingConn) InjectMessage(_ concurrency.Waitable, msg *central.MsgTo func (r *recordingConn) InjectMessageIntoQueue(_ *central.MsgFromSensor) {} +func (*recordingConn) HasCapability(_ centralsensor.SensorCapability) bool { + return false +} + func (r *recordingConn) getSentTriggers() []*central.SensorUpgradeTrigger { r.lock.Lock() defer r.lock.Unlock() diff --git a/central/sensor/service/connection/worker_queue.go b/central/sensor/service/connection/worker_queue.go index bbfd2ab92b6fd..66d0868c63a6f 100644 --- a/central/sensor/service/connection/worker_queue.go +++ b/central/sensor/service/connection/worker_queue.go @@ -34,7 +34,7 @@ type workerQueue struct { func newWorkerQueue(poolSize int, typ string, injector common.MessageInjector) *workerQueue { totalSize := poolSize + 1 queues := make([]*dedupingqueue.DedupingQueue[string], totalSize) - for i := 0; i < totalSize; i++ { + for i := range totalSize { queues[i] = dedupingqueue.NewDedupingQueue[string]( dedupingqueue.WithQueueName[string](typ), dedupingqueue.WithOperationMetricsFunc[string](metrics.IncrementSensorEventQueueCounter)) diff --git a/central/sensor/service/pipeline/clustermetrics/pipeline.go b/central/sensor/service/pipeline/clustermetrics/pipeline.go index 2e7f9e258140d..6051a4fc326ad 100644 --- a/central/sensor/service/pipeline/clustermetrics/pipeline.go +++ b/central/sensor/service/pipeline/clustermetrics/pipeline.go @@ -79,7 +79,7 @@ func (p *pipelineImpl) Run( ctx context.Context, clusterID string, msg *central.MsgFromSensor, - _ common.MessageInjector, + injector common.MessageInjector, ) error { clusterMetrics := msg.GetClusterMetrics() p.metricsStore.Set(clusterID, clusterMetrics) @@ -93,7 +93,7 @@ func (p *pipelineImpl) Run( logging.GetRateLimitedLogger().Warn( "Error while trying to update secured units usage: ", err.Error()) } - go clusterTelemetry.UpdateSecuredClusterIdentity(ctx, clusterID, clusterMetrics) + go clusterTelemetry.UpdateSecuredClusterIdentity(ctx, clusterID, clusterMetrics, hasVMTelemetryCap(injector)) return nil } @@ -101,3 +101,9 @@ func (p *pipelineImpl) OnFinish(clusterID string) { p.metricsStore.Set(clusterID, ¢ral.ClusterMetrics{}) p.telemetryMetrics.DeleteClusterMetrics(clusterID) } + +// hasVMTelemetryCap is true when the Sensor advertised VirtualMachineTelemetryCap. +// A nil injector is treated as no capability. +func hasVMTelemetryCap(injector common.MessageInjector) bool { + return injector != nil && injector.HasCapability(centralsensor.VirtualMachineTelemetryCap) +} diff --git a/central/sensor/service/pipeline/clustermetrics/pipeline_test.go b/central/sensor/service/pipeline/clustermetrics/pipeline_test.go index d6c8e2e62a941..2e7708398dcaa 100644 --- a/central/sensor/service/pipeline/clustermetrics/pipeline_test.go +++ b/central/sensor/service/pipeline/clustermetrics/pipeline_test.go @@ -6,9 +6,12 @@ import ( usageMocks "github.com/stackrox/rox/central/administration/usage/datastore/securedunits/mocks" telemetryMocks "github.com/stackrox/rox/central/metrics/telemetry/mocks" + "github.com/stackrox/rox/central/sensor/service/common" metricsMocks "github.com/stackrox/rox/central/sensor/service/pipeline/clustermetrics/mocks" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/centralsensor" + "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/fixtures" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" @@ -70,3 +73,58 @@ func (suite *PipelineTestSuite) TestClusterMetricsResetOnPipelineFinish() { suite.pipeline.OnFinish(clusterID) } + +func (suite *PipelineTestSuite) TestClusterMetricsMessageFromSensor_QueriesVMTelemetryCap() { + tests := map[string]struct { + caps map[centralsensor.SensorCapability]bool + }{ + "should query VirtualMachineTelemetryCap when Sensor advertises it": { + caps: map[centralsensor.SensorCapability]bool{ + centralsensor.VirtualMachineTelemetryCap: true, + }, + }, + "should query VirtualMachineTelemetryCap when Sensor lacks it": { + caps: map[centralsensor.SensorCapability]bool{}, + }, + } + for name, tc := range tests { + suite.Run(name, func() { + clusterID := fixtures.GetDeployment().GetClusterId() + expectedMetrics := ¢ral.ClusterMetrics{NodeCount: 1, CpuCapacity: 10} + + suite.metricsStore.EXPECT().Set(clusterID, expectedMetrics) + suite.telemetryMetrics.EXPECT().SetClusterMetrics(clusterID, expectedMetrics) + suite.usageStore.EXPECT().UpdateUsage(gomock.Any(), clusterID, &storage.SecuredUnits{ + NumNodes: expectedMetrics.GetNodeCount(), + NumCpuUnits: expectedMetrics.GetCpuCapacity(), + }).Return(nil) + + injector := &fakeInjector{caps: tc.caps} + err := suite.pipeline.Run(suite.T().Context(), clusterID, ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_ClusterMetrics{ + ClusterMetrics: expectedMetrics, + }, + }, injector) + suite.NoError(err) + suite.Equal([]centralsensor.SensorCapability{centralsensor.VirtualMachineTelemetryCap}, injector.queried) + }) + } +} + +var _ common.MessageInjector = (*fakeInjector)(nil) + +type fakeInjector struct { + caps map[centralsensor.SensorCapability]bool + queried []centralsensor.SensorCapability +} + +func (f *fakeInjector) HasCapability(cap centralsensor.SensorCapability) bool { + f.queried = append(f.queried, cap) + return f.caps[cap] +} + +func (f *fakeInjector) InjectMessage(concurrency.Waitable, *central.MsgToSensor) error { + return nil +} + +func (f *fakeInjector) InjectMessageIntoQueue(*central.MsgFromSensor) {} diff --git a/central/sensor/service/pipeline/networkflowupdate/flow_store_updater_impl_test.go b/central/sensor/service/pipeline/networkflowupdate/flow_store_updater_impl_test.go index 0c6453e59f3da..50b36b7167905 100644 --- a/central/sensor/service/pipeline/networkflowupdate/flow_store_updater_impl_test.go +++ b/central/sensor/service/pipeline/networkflowupdate/flow_store_updater_impl_test.go @@ -3,6 +3,7 @@ package networkflowupdate import ( "context" "math" + "slices" "testing" "time" @@ -456,12 +457,7 @@ func (suite *FlowStoreUpdaterTestSuite) TestUpdateWithExternalIPs() { // not discoveredEntity3 since the flow was filtered } - for i := range expectedEntities { - if updatedEntity.GetInfo().EqualVT(expectedEntities[i]) { - return true - } - } - return false + return slices.ContainsFunc(expectedEntities, updatedEntity.GetInfo().EqualVT) }), true).Times(2).Return(nil) // Run test. diff --git a/central/sensor/service/pipeline/nodeindex/pipeline.go b/central/sensor/service/pipeline/nodeindex/pipeline.go index 17b83ccf1807a..dc032fe4cbf36 100644 --- a/central/sensor/service/pipeline/nodeindex/pipeline.go +++ b/central/sensor/service/pipeline/nodeindex/pipeline.go @@ -2,9 +2,11 @@ package nodeindex import ( "context" + "time" "github.com/pkg/errors" clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" + "github.com/stackrox/rox/central/cluster/lifecycle" "github.com/stackrox/rox/central/enrichment" countMetrics "github.com/stackrox/rox/central/metrics" nodeDatastore "github.com/stackrox/rox/central/node/datastore" @@ -19,6 +21,7 @@ import ( "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/metrics" "github.com/stackrox/rox/pkg/nodes/enricher" + "github.com/stackrox/rox/pkg/protocompat" ) var ( @@ -115,38 +118,73 @@ func (p *pipelineImpl) Run(ctx context.Context, _ string, msg *central.MsgFromSe } log.Infof("Scanned index report and found %d components for node %s", len(node.GetScan().GetComponents()), nodeDatastore.NodeString(node)) + incomingScanTime := node.GetScan().GetScanTime() // Update the whole node in the database with the new and previous information. - err = p.riskManager.CalculateRiskAndUpsertNode(node) + err = p.upsertNodeIfClusterActive(ctx, node) if err != nil { return errors.Wrapf(err, "failed calculating risk and upserting node %s", nodeDatastore.NodeString(node)) } + // The store may have rejected this scan as not newer than what's already persisted, in which case it + // rewrites node.Scan in place to the previously stored one before returning - see isUpdated() in + // central/node/datastore/store/postgres/store.go. Comparing scan times here, on the same object we just + // tried to upsert, tells us definitively whether the fresh scan we just processed actually landed. + persistedScanTime := node.GetScan().GetScanTime() + switch cmp := protocompat.CompareTimestamps(incomingScanTime, persistedScanTime); { + case cmp > 0: + // The store kept a scan_time older than the one we just processed. isUpdated() only rejects scans + // that are not strictly newer than the stored one, so a fresh scan should have been accepted here; + // a persisted scan older than the processed one means the fresh scan was silently dropped, which the + // freshness guard should never do. + log.Warnf("Persisted scan_time for node %s is older than the freshly processed scan_time: "+ + "processed=%s persisted=%s. The store kept an older scan than the one just processed, which the "+ + "freshness guard should have accepted - possible scan regression.", + nodeDatastore.NodeString(node), + protocompat.ConvertTimestampToString(incomingScanTime, time.RFC3339Nano), + protocompat.ConvertTimestampToString(persistedScanTime, time.RFC3339Nano)) + case cmp < 0: + // The store kept a scan_time newer than the one we just processed - the expected shape of an + // isUpdated() rejection, where node.Scan was rewritten back to the newer stored scan. + log.Warnf("Persisted scan_time for node %s does not match the freshly processed scan_time: "+ + "processed=%s persisted=%s. The store rejected this scan as not newer than the stored one.", + nodeDatastore.NodeString(node), + protocompat.ConvertTimestampToString(incomingScanTime, time.RFC3339Nano), + protocompat.ConvertTimestampToString(persistedScanTime, time.RFC3339Nano)) + } + sendComplianceAck(ctx, node, injector) return nil } -func sendComplianceAck(ctx context.Context, node *storage.Node, injector common.MessageInjector) { - if injector == nil { - return +func (p *pipelineImpl) upsertNodeIfClusterActive(ctx context.Context, node *storage.Node) error { + active, release := lifecycle.Singleton().Enter(node.GetClusterId()) + if !active { + return nil } - reply := replyCompliance(node.GetClusterId(), node.GetName(), central.NodeInventoryACK_ACK) - if err := injector.InjectMessage(ctx, reply); err != nil { - log.Warnf("Failed sending node-indexing-ACK to Sensor for %s: %v", nodeDatastore.NodeString(node), err) - } else { - log.Debugf("Sent node-indexing-ACK for %s", nodeDatastore.NodeString(node)) + defer release() + + _, exists, err := p.clusterStore.GetClusterName(ctx, node.GetClusterId()) + if err != nil || !exists { + return err } + return p.riskManager.CalculateRiskAndUpsertNode(node) } -func replyCompliance(clusterID, nodeName string, t central.NodeInventoryACK_Action) *central.MsgToSensor { - return ¢ral.MsgToSensor{ - Msg: ¢ral.MsgToSensor_NodeInventoryAck{ - NodeInventoryAck: ¢ral.NodeInventoryACK{ - ClusterId: clusterID, - NodeName: nodeName, - Action: t, - MessageType: central.NodeInventoryACK_NodeIndexer, - }, - }, +func sendComplianceAck(ctx context.Context, node *storage.Node, injector common.MessageInjector) { + if injector == nil { + return } + common.SendSensorACK(ctx, central.SensorACK_ACK, central.SensorACK_NODE_INDEX_REPORT, node.GetName(), "", injector) + + common.SendLegacyNodeInventoryACK( + ctx, + node.GetClusterId(), + node.GetName(), + central.NodeInventoryACK_ACK, + central.NodeInventoryACK_NodeIndexer, + injector, + ) + + log.Debugf("Sent node-indexing ACKs for node %s in cluster %s", node.GetName(), node.GetClusterId()) } diff --git a/central/sensor/service/pipeline/nodeindex/pipeline_test.go b/central/sensor/service/pipeline/nodeindex/pipeline_test.go index 7a1a78a34f8b3..7d98420ce86bc 100644 --- a/central/sensor/service/pipeline/nodeindex/pipeline_test.go +++ b/central/sensor/service/pipeline/nodeindex/pipeline_test.go @@ -2,19 +2,129 @@ package nodeindex import ( "testing" + "time" clusterDatastoreMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" + "github.com/stackrox/rox/central/cluster/lifecycle" nodeDatastoreMocks "github.com/stackrox/rox/central/node/datastore/mocks" riskManagerMocks "github.com/stackrox/rox/central/risk/manager/mocks" "github.com/stackrox/rox/generated/internalapi/central" v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/centralsensor" + "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/logging" nodesEnricherMocks "github.com/stackrox/rox/pkg/nodes/enricher/mocks" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/sync" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) +const ( + scanRejectedWarnSnippet = "rejected this scan as not newer than the stored one" + scanRegressionWarnSnippet = "possible scan regression" +) + +func swapObservedLogger(t *testing.T) *observer.ObservedLogs { + core, logs := observer.New(zap.WarnLevel) + orig := log + log = &logging.LoggerImpl{InnerLogger: zap.New(core).Sugar()} + t.Cleanup(func() { log = orig }) + return logs +} + +// TestPipelineWarnsWhenPersistedScanTimeRegresses validates the post-upsert diagnostic: after the upsert, +// the pipeline compares the scan_time it processed against what the store actually kept (the store rewrites +// node.Scan in place when it rejects a scan as not-newer) and warns on a mismatch. +func TestPipelineWarnsWhenPersistedScanTimeRegresses(t *testing.T) { + t.Setenv(features.NodeIndexEnabled.EnvVar(), "true") + t.Setenv(features.ScannerV4.EnvVar(), "true") + + processed := time.Now() + + cases := map[string]struct { + // persistedScanTime is what the store leaves on node.Scan after the upsert returns. + persistedScanTime time.Time + // expectSnippet is the warning we expect to see; empty means no warning at all. + expectSnippet string + }{ + // A genuine isUpdated() rejection keeps a scan strictly newer than the one we just processed. + "warns (rejection) when the store kept a newer scan": { + persistedScanTime: processed.Add(time.Hour), + expectSnippet: scanRejectedWarnSnippet, + }, + // The store kept an older scan than the fresh one - a scan the freshness guard should have accepted. + "warns (regression) when the store kept an older scan": { + persistedScanTime: processed.Add(-time.Hour), + expectSnippet: scanRegressionWarnSnippet, + }, + "silent when the fresh scan was accepted": { + persistedScanTime: processed, + expectSnippet: "", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + logs := swapObservedLogger(t) + node := &storage.Node{Id: "1", Name: "node-name", ClusterId: "cluster-id"} + + ctrl := gomock.NewController(t) + clusterStore := clusterDatastoreMocks.NewMockDataStore(ctrl) + nodeDatastore := nodeDatastoreMocks.NewMockDataStore(ctrl) + riskManager := riskManagerMocks.NewMockManager(ctrl) + enricher := nodesEnricherMocks.NewMockNodeEnricher(ctrl) + + gomock.InOrder( + nodeDatastore.EXPECT().GetNode(gomock.Any(), gomock.Eq(node.GetId())).Times(1).Return(node, true, nil), + // Enrichment stamps a fresh scan_time, as the real enricher does with TimestampNow(). + enricher.EXPECT().EnrichNodeWithVulnerabilities(gomock.Any(), nil, gomock.Any()).Times(1). + DoAndReturn(func(n *storage.Node, _ *storage.NodeInventory, _ *v4.IndexReport) error { + n.Scan = &storage.NodeScan{ScanTime: protocompat.ConvertTimeToTimestampOrNil(&processed)} + return nil + }), + // Upsert rewrites node.Scan in place to whatever was persisted (mimics isUpdated()). + riskManager.EXPECT().CalculateRiskAndUpsertNode(gomock.Any()).Times(1). + DoAndReturn(func(n *storage.Node) error { + n.Scan.ScanTime = protocompat.ConvertTimeToTimestampOrNil(&tc.persistedScanTime) + return nil + }), + ) + clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Eq(node.GetClusterId())).Return("", true, nil) + + p := &pipelineImpl{ + clusterStore: clusterStore, + nodeDatastore: nodeDatastore, + riskManager: riskManager, + enricher: enricher, + } + + err := p.Run(t.Context(), node.GetClusterId(), createMsg(mockIndexReport), nil) + require.NoError(t, err) + + rejections := logs.FilterMessageSnippet(scanRejectedWarnSnippet).All() + regressions := logs.FilterMessageSnippet(scanRegressionWarnSnippet).All() + switch tc.expectSnippet { + case scanRejectedWarnSnippet: + assert.Len(t, rejections, 1) + assert.Empty(t, regressions) + case scanRegressionWarnSnippet: + assert.Len(t, regressions, 1) + assert.Empty(t, rejections) + default: + assert.Empty(t, rejections) + assert.Empty(t, regressions) + } + }) + } +} + func TestPipelineWithEmptyIndex(t *testing.T) { t.Setenv(features.NodeIndexEnabled.EnvVar(), "true") t.Setenv(features.ScannerV4.EnvVar(), "true") @@ -51,6 +161,7 @@ func TestPipelineEnrichesAndUpserts(t *testing.T) { } ctrl := gomock.NewController(t) clusterStore := clusterDatastoreMocks.NewMockDataStore(ctrl) + clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).AnyTimes().Return("", true, nil) nodeDatastore := nodeDatastoreMocks.NewMockDataStore(ctrl) nodeDatastore.EXPECT().GetNode(gomock.Any(), gomock.Eq("1")).Times(2).Return(&node, true, nil) riskManager := riskManagerMocks.NewMockManager(ctrl) @@ -77,6 +188,132 @@ func TestPipelineEnrichesAndUpserts(t *testing.T) { } } +func TestPipelineSkipsUpsertAfterClusterDeletionBegins(t *testing.T) { + t.Setenv(features.NodeIndexEnabled.EnvVar(), "true") + t.Setenv(features.ScannerV4.EnvVar(), "true") + + ctrl := gomock.NewController(t) + node := &storage.Node{Id: "1", Name: "node-name", ClusterId: "cluster-id"} + nodeDatastore := nodeDatastoreMocks.NewMockDataStore(ctrl) + nodeDatastore.EXPECT().GetNode(gomock.Any(), gomock.Eq(node.GetId())).Return(node, true, nil) + enricher := nodesEnricherMocks.NewMockNodeEnricher(ctrl) + enricher.EXPECT().EnrichNodeWithVulnerabilities(gomock.Any(), nil, gomock.Any()).Return(nil) + + releaseDeletion := lifecycle.Singleton().BeginDeletion(node.GetClusterId()) + defer releaseDeletion() + + p := &pipelineImpl{ + nodeDatastore: nodeDatastore, + enricher: enricher, + } + require.NoError(t, p.Run(t.Context(), node.GetClusterId(), createMsg(mockIndexReport), nil)) +} + +func TestPipelineSendsSensorAndLegacyACKs(t *testing.T) { + t.Setenv(features.NodeIndexEnabled.EnvVar(), "true") + t.Setenv(features.ScannerV4.EnvVar(), "true") + + ctrl := gomock.NewController(t) + clusterStore := clusterDatastoreMocks.NewMockDataStore(ctrl) + clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).AnyTimes().Return("", true, nil) + nodeDatastore := nodeDatastoreMocks.NewMockDataStore(ctrl) + riskManager := riskManagerMocks.NewMockManager(ctrl) + enricher := nodesEnricherMocks.NewMockNodeEnricher(ctrl) + + node := storage.Node{ + Id: "1", + Name: "node-name", + ClusterId: "cluster-id", + } + msg := createMsg(mockIndexReport) + + gomock.InOrder( + nodeDatastore.EXPECT().GetNode(gomock.Any(), gomock.Eq(node.GetId())).Times(1).Return(&node, true, nil), + enricher.EXPECT().EnrichNodeWithVulnerabilities(gomock.Any(), nil, gomock.Any()).Times(1).Return(nil), + riskManager.EXPECT().CalculateRiskAndUpsertNode(gomock.Any()).Times(1).Return(nil), + ) + + injector := &recordingInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + p := &pipelineImpl{ + clusterStore: clusterStore, + nodeDatastore: nodeDatastore, + riskManager: riskManager, + enricher: enricher, + } + + err := p.Run(t.Context(), node.GetClusterId(), msg, injector) + assert.NoError(t, err) + + protoassert.SlicesEqual(t, []*central.SensorACK{ + { + Action: central.SensorACK_ACK, + MessageType: central.SensorACK_NODE_INDEX_REPORT, + ResourceId: node.GetName(), + }, + }, injector.getSentSensorACKs()) + + protoassert.SlicesEqual(t, []*central.NodeInventoryACK{ + { + ClusterId: node.GetClusterId(), + NodeName: node.GetName(), + Action: central.NodeInventoryACK_ACK, + MessageType: central.NodeInventoryACK_NodeIndexer, + }, + }, injector.getSentACKs()) +} + +func TestPipelineSkipsSensorACKWhenCapabilityMissing(t *testing.T) { + t.Setenv(features.NodeIndexEnabled.EnvVar(), "true") + t.Setenv(features.ScannerV4.EnvVar(), "true") + + ctrl := gomock.NewController(t) + clusterStore := clusterDatastoreMocks.NewMockDataStore(ctrl) + clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).AnyTimes().Return("", true, nil) + nodeDatastore := nodeDatastoreMocks.NewMockDataStore(ctrl) + riskManager := riskManagerMocks.NewMockManager(ctrl) + enricher := nodesEnricherMocks.NewMockNodeEnricher(ctrl) + + node := storage.Node{ + Id: "1", + Name: "node-name", + ClusterId: "cluster-id", + } + msg := createMsg(mockIndexReport) + + gomock.InOrder( + nodeDatastore.EXPECT().GetNode(gomock.Any(), gomock.Eq(node.GetId())).Times(1).Return(&node, true, nil), + enricher.EXPECT().EnrichNodeWithVulnerabilities(gomock.Any(), nil, gomock.Any()).Times(1).Return(nil), + riskManager.EXPECT().CalculateRiskAndUpsertNode(gomock.Any()).Times(1).Return(nil), + ) + + injector := &recordingInjector{ + capabilities: map[centralsensor.SensorCapability]bool{}, + } + p := &pipelineImpl{ + clusterStore: clusterStore, + nodeDatastore: nodeDatastore, + riskManager: riskManager, + enricher: enricher, + } + + err := p.Run(t.Context(), node.GetClusterId(), msg, injector) + assert.NoError(t, err) + assert.Empty(t, injector.getSentSensorACKs(), "sensor ACK should be skipped without SensorACKSupport") + + protoassert.SlicesEqual(t, []*central.NodeInventoryACK{ + { + ClusterId: node.GetClusterId(), + NodeName: node.GetName(), + Action: central.NodeInventoryACK_ACK, + MessageType: central.NodeInventoryACK_NodeIndexer, + }, + }, injector.getSentACKs()) +} + func createMsg(ir *v4.IndexReport) *central.MsgFromSensor { return ¢ral.MsgFromSensor{ Msg: ¢ral.MsgFromSensor_Event{ @@ -90,6 +327,55 @@ func createMsg(ir *v4.IndexReport) *central.MsgFromSensor { } } +type recordingInjector struct { + lock sync.Mutex + legacy []*central.NodeInventoryACK + sensor []*central.SensorACK + capabilities map[centralsensor.SensorCapability]bool +} + +func (r *recordingInjector) HasCapability(cap centralsensor.SensorCapability) bool { + return r.capabilities[cap] +} + +func (r *recordingInjector) InjectMessage(_ concurrency.Waitable, msg *central.MsgToSensor) error { + r.lock.Lock() + defer r.lock.Unlock() + if ack := msg.GetNodeInventoryAck(); ack != nil { + r.legacy = append(r.legacy, ack.CloneVT()) + } + if ack := msg.GetSensorAck(); ack != nil { + r.sensor = append(r.sensor, ack.CloneVT()) + } + return nil +} + +func (r *recordingInjector) InjectMessageIntoQueue(_ *central.MsgFromSensor) {} + +func (r *recordingInjector) getSentACKs() []*central.NodeInventoryACK { + r.lock.Lock() + defer r.lock.Unlock() + out := make([]*central.NodeInventoryACK, 0, len(r.legacy)) + for _, ack := range r.legacy { + if ack != nil { + out = append(out, ack) + } + } + return out +} + +func (r *recordingInjector) getSentSensorACKs() []*central.SensorACK { + r.lock.Lock() + defer r.lock.Unlock() + out := make([]*central.SensorACK, 0, len(r.sensor)) + for _, ack := range r.sensor { + if ack != nil { + out = append(out, ack) + } + } + return out +} + var ( mockIndexReport = &v4.IndexReport{ HashId: "", diff --git a/central/sensor/service/pipeline/nodeindex/three_pipelines_test.go b/central/sensor/service/pipeline/nodeindex/three_pipelines_test.go index f57672c774c86..069ed968a6fbf 100644 --- a/central/sensor/service/pipeline/nodeindex/three_pipelines_test.go +++ b/central/sensor/service/pipeline/nodeindex/three_pipelines_test.go @@ -28,6 +28,7 @@ import ( "github.com/stackrox/rox/pkg/scancomponent" "github.com/stackrox/rox/pkg/scanners" "github.com/stackrox/rox/pkg/scanners/types" + "github.com/stackrox/rox/pkg/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -140,13 +141,12 @@ func Test_ThreePipelines_Run(t *testing.T) { }, }, setUpMocksAndEnv: func(t *testing.T, m *usedMocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, false) t.Setenv(features.ScannerV4.EnvVar(), "true") t.Setenv(features.NodeIndexEnabled.EnvVar(), "true") gomock.InOrder( - // node arrives + // node arrives: non-RHCOS with LegacyScanner off upserts UNSUPPORTED and skips Clairify m.clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Eq(clusterID)).Times(1).Return(clusterID, true, nil), - m.cveDatastore.EXPECT().EnrichNodeWithSuppressedCVEs(gomock.Any()).Times(1).Return(), - m.riskStorage.EXPECT().UpsertRisk(gomock.Any(), gomock.Any()).MinTimes(1).Return(nil), m.nodeDatastore.EXPECT().UpsertNode(gomock.Any(), gomock.Any()).Times(1).Return(nil), // node index arrives m.nodeDatastore.EXPECT().GetNode(gomock.Any(), gomock.Eq(nodeID)).Times(1).Return(nodeWithScore, true, nil), @@ -175,6 +175,7 @@ func Test_ThreePipelines_Run(t *testing.T) { }, }, setUpMocksAndEnv: func(t *testing.T, m *usedMocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) t.Setenv(features.ScannerV4.EnvVar(), "true") t.Setenv(features.NodeIndexEnabled.EnvVar(), "true") gomock.InOrder( @@ -211,6 +212,7 @@ func Test_ThreePipelines_Run(t *testing.T) { }, }, setUpMocksAndEnv: func(t *testing.T, m *usedMocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) t.Setenv(features.ScannerV4.EnvVar(), "true") t.Setenv(features.NodeIndexEnabled.EnvVar(), "true") gomock.InOrder( @@ -244,6 +246,7 @@ func Test_ThreePipelines_Run(t *testing.T) { }, }, setUpMocksAndEnv: func(t *testing.T, m *usedMocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) t.Setenv(features.ScannerV4.EnvVar(), "false") t.Setenv(features.NodeIndexEnabled.EnvVar(), "false") gomock.InOrder( @@ -269,6 +272,7 @@ func Test_ThreePipelines_Run(t *testing.T) { }, }, setUpMocksAndEnv: func(t *testing.T, m *usedMocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) t.Setenv(features.ScannerV4.EnvVar(), "true") t.Setenv(features.NodeIndexEnabled.EnvVar(), "true") gomock.InOrder( @@ -356,6 +360,7 @@ func Test_ThreePipelines_Run(t *testing.T) { if tt.setUpMocksAndEnv != nil { tt.setUpMocksAndEnv(t, tt.mocks) } + tt.mocks.clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).AnyTimes().Return("", true, nil) pNode := nodes.NewPipeline(tt.mocks.clusterStore, tt.mocks.nodeDatastore, tt.enricher, tt.riskManager) pNodeInv := nodeinventory.NewPipeline(tt.mocks.clusterStore, tt.mocks.nodeDatastore, tt.enricher, tt.riskManager) pNodeIdx := newPipeline(tt.mocks.clusterStore, tt.mocks.nodeDatastore, tt.enricher, tt.riskManager) diff --git a/central/sensor/service/pipeline/nodeinventory/pipeline.go b/central/sensor/service/pipeline/nodeinventory/pipeline.go index 90b0ef3b02157..f9cfff63913f0 100644 --- a/central/sensor/service/pipeline/nodeinventory/pipeline.go +++ b/central/sensor/service/pipeline/nodeinventory/pipeline.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" + "github.com/stackrox/rox/central/cluster/lifecycle" "github.com/stackrox/rox/central/enrichment" countMetrics "github.com/stackrox/rox/central/metrics" nodeDatastore "github.com/stackrox/rox/central/node/datastore" @@ -79,6 +80,12 @@ func (p *pipelineImpl) Run(ctx context.Context, _ string, msg *central.MsgFromSe log.Warn("Removal of node inventory is unsupported action") return nil } + if !features.LegacyScanner.Enabled() { + replyCompliance(ctx, "", ninv.GetNodeName(), central.NodeInventoryACK_ACK, injector) + log.Debug("Discarding v2 NodeInventory because the legacy scanner is disabled") + return nil + } + ninv = ninv.CloneVT() // Read the node from the database, if not found we fail. @@ -94,7 +101,7 @@ func (p *pipelineImpl) Run(ctx context.Context, _ string, msg *central.MsgFromSe if shouldDiscardMsg(node) { // To prevent resending the inventory, still acknowledge receipt of it - sendComplianceAck(ctx, node, ninv, injector) + replyCompliance(ctx, node.GetClusterId(), ninv.GetNodeName(), central.NodeInventoryACK_ACK, injector) log.Debug("Discarding v2 NodeScan in favor of v4 NodeScan") return nil } @@ -109,16 +116,30 @@ func (p *pipelineImpl) Run(ctx context.Context, _ string, msg *central.MsgFromSe len(node.GetScan().GetComponents())) // Update the whole node in the database with the new and previous information. - err = p.riskManager.CalculateRiskAndUpsertNode(node) + err = p.calculateRiskAndUpsertNodeIfClusterActive(ctx, node) if err != nil { log.Error(err) return err } - sendComplianceAck(ctx, node, ninv, injector) + replyCompliance(ctx, node.GetClusterId(), ninv.GetNodeName(), central.NodeInventoryACK_ACK, injector) return nil } +func (p *pipelineImpl) calculateRiskAndUpsertNodeIfClusterActive(ctx context.Context, node *storage.Node) error { + active, release := lifecycle.Singleton().Enter(node.GetClusterId()) + if !active { + return nil + } + defer release() + + _, exists, err := p.clusterStore.GetClusterName(ctx, node.GetClusterId()) + if err != nil || !exists { + return err + } + return p.riskManager.CalculateRiskAndUpsertNode(node) +} + // shouldDiscardMsg returns true if the pipeline should discard an incoming message or false if it should keep processing it. func shouldDiscardMsg(node *storage.Node) bool { // In a mixed environment, there might be v2-only clusters, so there is no Scanner v4 scan available for that cluster. @@ -138,29 +159,31 @@ func shouldDiscardMsg(node *storage.Node) bool { return false } -func sendComplianceAck(ctx context.Context, node *storage.Node, ninv *storage.NodeInventory, injector common.MessageInjector) { +// replyCompliance uses injector to send a SensorACK and NodeInventoryACK to Compliance. +func replyCompliance(ctx context.Context, clusterID, nodeName string, t central.NodeInventoryACK_Action, injector common.MessageInjector) { if injector == nil { return } - reply := replyCompliance(node.GetClusterId(), ninv.GetNodeName(), central.NodeInventoryACK_ACK) - if err := injector.InjectMessage(ctx, reply); err != nil { - log.Warnf("Failed sending node-inventory-ACK to Sensor for %s: %v", nodeDatastore.NodeString(node), err) - } else { - log.Debugf("Sent node-inventory-ACK for %s", nodeDatastore.NodeString(node)) - } -} -func replyCompliance(clusterID, nodeName string, t central.NodeInventoryACK_Action) *central.MsgToSensor { - return ¢ral.MsgToSensor{ - Msg: ¢ral.MsgToSensor_NodeInventoryAck{ - NodeInventoryAck: ¢ral.NodeInventoryACK{ - ClusterId: clusterID, - NodeName: nodeName, - Action: t, - MessageType: central.NodeInventoryACK_NodeInventory, - }, - }, - } + common.SendSensorACK(ctx, convertLegacyActionToSensor(t), central.SensorACK_NODE_INVENTORY, nodeName, "", injector) + + common.SendLegacyNodeInventoryACK( + ctx, + clusterID, + nodeName, + t, + central.NodeInventoryACK_NodeInventory, + injector, + ) + + log.Debugf("Sent node-inventory ACKs for node %s in cluster %s", nodeName, clusterID) } func (p *pipelineImpl) OnFinish(_ string) {} + +func convertLegacyActionToSensor(action central.NodeInventoryACK_Action) central.SensorACK_Action { + if action == central.NodeInventoryACK_ACK { + return central.SensorACK_ACK + } + return central.SensorACK_NACK +} diff --git a/central/sensor/service/pipeline/nodeinventory/pipeline_test.go b/central/sensor/service/pipeline/nodeinventory/pipeline_test.go index eb6777d7740e2..a6e70ea76acfc 100644 --- a/central/sensor/service/pipeline/nodeinventory/pipeline_test.go +++ b/central/sensor/service/pipeline/nodeinventory/pipeline_test.go @@ -11,10 +11,13 @@ import ( "github.com/stackrox/rox/central/sensor/service/common" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" nodesEnricherMocks "github.com/stackrox/rox/pkg/nodes/enricher/mocks" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/sync" + "github.com/stackrox/rox/pkg/testutils" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" ) @@ -67,9 +70,20 @@ func Test_pipelineImpl_Run(t *testing.T) { }, wantInjectorContain: []*central.NodeInventoryACK{}, }, + { + name: "when LegacyScanner is disabled then ACK and discard", + setUp: func(t *testing.T, a *args, m *mocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, false) + a.msg = createMsg("node1") + a.msg.GetEvent().GetNodeInventory().NodeName = "test-node" + a.injector = &recordingInjector{} + }, + wantInjectorContain: []*central.NodeInventoryACK{{Action: central.NodeInventoryACK_ACK, NodeName: "test-node"}}, + }, { name: "when event action is CREATE_RESOURCE then do not ignore event", setUp: func(t *testing.T, a *args, m *mocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) node := storage.Node{ Id: "test node id", } @@ -90,6 +104,7 @@ func Test_pipelineImpl_Run(t *testing.T) { {Action: central.NodeInventoryACK_ACK}, }, setUp: func(t *testing.T, a *args, m *mocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) node := storage.Node{ Id: "test node id", } @@ -105,6 +120,7 @@ func Test_pipelineImpl_Run(t *testing.T) { { name: "when injector is nil then handle normally and don't panic", setUp: func(t *testing.T, a *args, m *mocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) node := storage.Node{ Id: "test node id", } @@ -121,6 +137,7 @@ func Test_pipelineImpl_Run(t *testing.T) { name: "when event has inventory for unknown node then no ACK should be sent", wantInjectorContain: []*central.NodeInventoryACK{}, setUp: func(t *testing.T, a *args, m *mocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) a.msg = createMsg("node1") a.injector = &recordingInjector{} gomock.InOrder( @@ -133,6 +150,7 @@ func Test_pipelineImpl_Run(t *testing.T) { wantInjectorContain: []*central.NodeInventoryACK{}, wantErr: "fetching error from DB", setUp: func(t *testing.T, a *args, m *mocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) a.msg = createMsg("node1") a.injector = &recordingInjector{} gomock.InOrder( @@ -153,6 +171,7 @@ func Test_pipelineImpl_Run(t *testing.T) { if tt.setUp != nil { tt.setUp(t, &tt.args, &tt.mocks) } + tt.mocks.clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).AnyTimes().Return("", true, nil) p := &pipelineImpl{ clusterStore: tt.mocks.clusterStore, nodeDatastore: tt.mocks.nodeDatastore, @@ -167,24 +186,162 @@ func Test_pipelineImpl_Run(t *testing.T) { if len(tt.wantInjectorContain) == 0 { assert.Len(t, inj.getSentACKs(), 0) } else { - protoassert.SlicesEqual(t, tt.wantInjectorContain, inj.getSentACKs()) + protoassert.SlicesEqual(t, tt.wantInjectorContain, inj.getSentACKs(), "sent ACKs: %v", inj.getSentACKs()) } } }) } } +func Test_pipelineImpl_Run_SendsSensorAndLegacyACKs(t *testing.T) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) + ctrl := gomock.NewController(t) + clusterStore := clusterDatastoreMocks.NewMockDataStore(ctrl) + clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).AnyTimes().Return("", true, nil) + nodeDatastore := nodeDatastoreMocks.NewMockDataStore(ctrl) + riskManager := riskManagerMocks.NewMockManager(ctrl) + enricher := nodesEnricherMocks.NewMockNodeEnricher(ctrl) + + clusterID := "cluster-1" + nodeName := "node-name" + node := storage.Node{ + Id: "node-id", + ClusterId: clusterID, + } + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Action: central.ResourceAction_CREATE_RESOURCE, + Resource: ¢ral.SensorEvent_NodeInventory{ + NodeInventory: &storage.NodeInventory{ + NodeId: node.GetId(), + NodeName: nodeName, + }, + }, + }, + }, + } + injector := &recordingInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + + gomock.InOrder( + nodeDatastore.EXPECT().GetNode(gomock.Any(), gomock.Eq(node.GetId())).Times(1).Return(&node, true, nil), + enricher.EXPECT().EnrichNodeWithVulnerabilities(gomock.Any(), gomock.Any(), nil).Times(1).Return(nil), + riskManager.EXPECT().CalculateRiskAndUpsertNode(gomock.Any()).Times(1).Return(nil), + ) + + p := &pipelineImpl{ + clusterStore: clusterStore, + nodeDatastore: nodeDatastore, + enricher: enricher, + riskManager: riskManager, + } + + err := p.Run(context.Background(), clusterID, msg, injector) + assert.NoError(t, err) + + protoassert.SlicesEqual(t, []*central.NodeInventoryACK{ + { + ClusterId: clusterID, + NodeName: nodeName, + Action: central.NodeInventoryACK_ACK, + MessageType: central.NodeInventoryACK_NodeInventory, + }, + }, injector.getSentACKs(), "legacy ACKs") + + protoassert.SlicesEqual(t, []*central.SensorACK{ + { + Action: central.SensorACK_ACK, + MessageType: central.SensorACK_NODE_INVENTORY, + ResourceId: nodeName, + }, + }, injector.getSentSensorACKs(), "sensor ACKs") +} + +func Test_pipelineImpl_Run_SkipsSensorACKWhenCapabilityMissing(t *testing.T) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) + ctrl := gomock.NewController(t) + clusterStore := clusterDatastoreMocks.NewMockDataStore(ctrl) + clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).AnyTimes().Return("", true, nil) + nodeDatastore := nodeDatastoreMocks.NewMockDataStore(ctrl) + riskManager := riskManagerMocks.NewMockManager(ctrl) + enricher := nodesEnricherMocks.NewMockNodeEnricher(ctrl) + + clusterID := "cluster-1" + nodeName := "node-name" + node := storage.Node{ + Id: "node-id", + ClusterId: clusterID, + } + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Action: central.ResourceAction_CREATE_RESOURCE, + Resource: ¢ral.SensorEvent_NodeInventory{ + NodeInventory: &storage.NodeInventory{ + NodeId: node.GetId(), + NodeName: nodeName, + }, + }, + }, + }, + } + injector := &recordingInjector{ + capabilities: map[centralsensor.SensorCapability]bool{}, + } + + gomock.InOrder( + nodeDatastore.EXPECT().GetNode(gomock.Any(), gomock.Eq(node.GetId())).Times(1).Return(&node, true, nil), + enricher.EXPECT().EnrichNodeWithVulnerabilities(gomock.Any(), gomock.Any(), nil).Times(1).Return(nil), + riskManager.EXPECT().CalculateRiskAndUpsertNode(gomock.Any()).Times(1).Return(nil), + ) + + p := &pipelineImpl{ + clusterStore: clusterStore, + nodeDatastore: nodeDatastore, + enricher: enricher, + riskManager: riskManager, + } + + err := p.Run(context.Background(), clusterID, msg, injector) + assert.NoError(t, err) + assert.Empty(t, injector.getSentSensorACKs(), "sensor ACK should be skipped without SensorACKSupport") + + protoassert.SlicesEqual(t, []*central.NodeInventoryACK{ + { + ClusterId: clusterID, + NodeName: nodeName, + Action: central.NodeInventoryACK_ACK, + MessageType: central.NodeInventoryACK_NodeInventory, + }, + }, injector.getSentACKs(), "legacy ACKs") +} + var _ common.MessageInjector = (*recordingInjector)(nil) type recordingInjector struct { - lock sync.Mutex - messages []*central.NodeInventoryACK + lock sync.Mutex + messages []*central.NodeInventoryACK + sensor []*central.SensorACK + capabilities map[centralsensor.SensorCapability]bool +} + +func (r *recordingInjector) HasCapability(cap centralsensor.SensorCapability) bool { + return r.capabilities[cap] } func (r *recordingInjector) InjectMessage(_ concurrency.Waitable, msg *central.MsgToSensor) error { r.lock.Lock() defer r.lock.Unlock() - r.messages = append(r.messages, msg.GetNodeInventoryAck().CloneVT()) + if ack := msg.GetNodeInventoryAck(); ack != nil { + r.messages = append(r.messages, ack.CloneVT()) + } + if ack := msg.GetSensorAck(); ack != nil { + r.sensor = append(r.sensor, ack.CloneVT()) + } return nil } @@ -194,6 +351,22 @@ func (r *recordingInjector) getSentACKs() []*central.NodeInventoryACK { r.lock.Lock() defer r.lock.Unlock() copied := make([]*central.NodeInventoryACK, 0, len(r.messages)) - copied = append(copied, r.messages...) + for _, m := range r.messages { + if m != nil { + copied = append(copied, m) + } + } + return copied +} + +func (r *recordingInjector) getSentSensorACKs() []*central.SensorACK { + r.lock.Lock() + defer r.lock.Unlock() + copied := make([]*central.SensorACK, 0, len(r.sensor)) + for _, m := range r.sensor { + if m != nil { + copied = append(copied, m) + } + } return copied } diff --git a/central/sensor/service/pipeline/nodeinventory/two_pipelines_test.go b/central/sensor/service/pipeline/nodeinventory/two_pipelines_test.go index 566793f840382..9f8b72efd5fa8 100644 --- a/central/sensor/service/pipeline/nodeinventory/two_pipelines_test.go +++ b/central/sensor/service/pipeline/nodeinventory/two_pipelines_test.go @@ -21,11 +21,13 @@ import ( "github.com/stackrox/rox/generated/internalapi/central" v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/metrics" nodeEnricher "github.com/stackrox/rox/pkg/nodes/enricher" "github.com/stackrox/rox/pkg/scancomponent" "github.com/stackrox/rox/pkg/scanners" "github.com/stackrox/rox/pkg/scanners/types" + "github.com/stackrox/rox/pkg/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -105,6 +107,7 @@ func Test_TwoPipelines_Run(t *testing.T) { }, }, setUpMocks: func(t *testing.T, m *usedMocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) m.nodeDatastore.EXPECT().GetNode(gomock.Any(), gomock.Eq(nodeID)).MinTimes(1).Return(nil, false, nil) }, wantNodeExists: false, @@ -122,6 +125,7 @@ func Test_TwoPipelines_Run(t *testing.T) { }, }, setUpMocks: func(t *testing.T, m *usedMocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) gomock.InOrder( // node arrives m.clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Eq(clusterID)).Times(1).Return(clusterID, true, nil), @@ -154,6 +158,7 @@ func Test_TwoPipelines_Run(t *testing.T) { }, }, setUpMocks: func(t *testing.T, m *usedMocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) gomock.InOrder( // node inventory arrives m.nodeDatastore.EXPECT().GetNode(gomock.Any(), gomock.Eq(nodeID)).Return(nil, false, nil), @@ -225,6 +230,7 @@ func Test_TwoPipelines_Run(t *testing.T) { if tt.setUpMocks != nil { tt.setUpMocks(t, tt.mocks) } + tt.mocks.clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).AnyTimes().Return("", true, nil) pNode := nodes.NewPipeline(tt.mocks.clusterStore, tt.mocks.nodeDatastore, tt.enricher, tt.riskManager) pNodeInv := NewPipeline(tt.mocks.clusterStore, tt.mocks.nodeDatastore, tt.enricher, tt.riskManager) diff --git a/central/sensor/service/pipeline/nodes/pipeline.go b/central/sensor/service/pipeline/nodes/pipeline.go index dd70cfa02da55..9bf5a7e053ab4 100644 --- a/central/sensor/service/pipeline/nodes/pipeline.go +++ b/central/sensor/service/pipeline/nodes/pipeline.go @@ -5,6 +5,7 @@ import ( "github.com/pkg/errors" clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" + "github.com/stackrox/rox/central/cluster/lifecycle" "github.com/stackrox/rox/central/enrichment" countMetrics "github.com/stackrox/rox/central/metrics" nodeDatastore "github.com/stackrox/rox/central/node/datastore" @@ -13,10 +14,13 @@ import ( "github.com/stackrox/rox/central/sensor/service/pipeline" "github.com/stackrox/rox/central/sensor/service/pipeline/reconciliation" "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/centralsensor" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/metrics" "github.com/stackrox/rox/pkg/nodes/enricher" + "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/stringutils" ) @@ -100,14 +104,28 @@ func (p *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.M // ROX-22002: Remove invalid null characters in annotations stringutils.SanitizeMapValues(node.GetAnnotations()) - if enricher.SupportsNodeScanning(node) { // If supports node scanning, this pipeline should only update the node's // metadata. We call upsert without scan. Upsert will read scan information from // the database before writing. This is safe because NodeInventory and Node // pipelines never run concurrently by the Sensor Event worker queues. node.Scan = nil - if err := p.nodeDatastore.UpsertNode(ctx, node); err != nil { + if err := p.upsertNodeIfClusterActive(ctx, clusterID, node); err != nil { + err = errors.Wrapf(err, "upserting node %s", nodeDatastore.NodeString(node)) + log.Error(err) + return err + } + return nil + } + + // If the node does NOT support v4 Node Scanning, AND the legacy scanner is not enabled, + // then we set the scan to unsupported and skip enrichment. + if !features.LegacyScanner.Enabled() { + node.Scan = &storage.NodeScan{ + ScanTime: protocompat.TimestampNow(), + Notes: []storage.NodeScan_Note{storage.NodeScan_UNSUPPORTED}, + } + if err := p.upsertNodeIfClusterActive(ctx, clusterID, node); err != nil { err = errors.Wrapf(err, "upserting node %s", nodeDatastore.NodeString(node)) log.Error(err) return err @@ -121,7 +139,7 @@ func (p *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.M "risk information will not be updated): %v", nodeDatastore.NodeString(node), err) } - if err := p.riskManager.CalculateRiskAndUpsertNode(node); err != nil { + if err := p.calculateRiskAndUpsertNodeIfClusterActive(ctx, clusterID, node); err != nil { log.Error(err) return err } @@ -129,4 +147,30 @@ func (p *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.M return nil } +func (p *pipelineImpl) upsertNodeIfClusterActive(ctx context.Context, clusterID string, node *storage.Node) error { + return p.withActiveCluster(ctx, clusterID, func() error { + return p.nodeDatastore.UpsertNode(ctx, node) + }) +} + +func (p *pipelineImpl) calculateRiskAndUpsertNodeIfClusterActive(ctx context.Context, clusterID string, node *storage.Node) error { + return p.withActiveCluster(ctx, clusterID, func() error { + return p.riskManager.CalculateRiskAndUpsertNode(node) + }) +} + +func (p *pipelineImpl) withActiveCluster(ctx context.Context, clusterID string, write func() error) error { + active, release := lifecycle.Singleton().Enter(clusterID) + if !active { + return nil + } + defer release() + + _, exists, err := p.clusterStore.GetClusterName(ctx, clusterID) + if err != nil || !exists { + return err + } + return write() +} + func (p *pipelineImpl) OnFinish(_ string) {} diff --git a/central/sensor/service/pipeline/nodes/pipeline_test.go b/central/sensor/service/pipeline/nodes/pipeline_test.go index d81067fdc05e1..c673578f9288b 100644 --- a/central/sensor/service/pipeline/nodes/pipeline_test.go +++ b/central/sensor/service/pipeline/nodes/pipeline_test.go @@ -10,8 +10,10 @@ import ( "github.com/stackrox/rox/central/sensor/service/common" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" nodesEnricherMocks "github.com/stackrox/rox/pkg/nodes/enricher/mocks" "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/testutils" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" ) @@ -75,6 +77,7 @@ func Test_pipelineImpl_Run(t *testing.T) { { name: "when node is not full host scanned then enrich and upsert with risk", setUp: func(t *testing.T, a *args, m *mocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, true) a.msg = createMsg("Something that is not RHCOS") a.clusterID = "test cluster id" gomock.InOrder( @@ -96,6 +99,27 @@ func Test_pipelineImpl_Run(t *testing.T) { ) }, }, + { + name: "when LegacyScanner is disabled and node is non-RHCOS then set UNSUPPORTED note", + setUp: func(t *testing.T, a *args, m *mocks) { + testutils.MustUpdateFeature(t, features.LegacyScanner, false) + a.msg = createMsg("Ubuntu 22.04.3 LTS") + a.clusterID = "test cluster id" + gomock.InOrder( + m.clusterStore.EXPECT(). + GetClusterName(gomock.Any(), gomock.Eq(a.clusterID)). + Times(1). + Return("test cluster name", true, nil), + m.nodeDatastore.EXPECT(). + UpsertNode(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, node *storage.Node) error { + assert.Contains(t, node.GetScan().GetNotes(), storage.NodeScan_UNSUPPORTED) + assert.NotNil(t, node.GetScan().GetScanTime()) + return nil + }).Times(1), + ) + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -109,6 +133,7 @@ func Test_pipelineImpl_Run(t *testing.T) { if tt.setUp != nil { tt.setUp(t, &tt.args, &tt.mocks) } + tt.mocks.clusterStore.EXPECT().GetClusterName(gomock.Any(), gomock.Any()).AnyTimes().Return("test cluster name", true, nil) p := &pipelineImpl{ clusterStore: tt.mocks.clusterStore, nodeDatastore: tt.mocks.nodeDatastore, diff --git a/central/sensor/service/pipeline/reprocessing/pipeline.go b/central/sensor/service/pipeline/reprocessing/pipeline.go index c7e274ade9b2c..fce2986ddd93f 100644 --- a/central/sensor/service/pipeline/reprocessing/pipeline.go +++ b/central/sensor/service/pipeline/reprocessing/pipeline.go @@ -3,6 +3,8 @@ package reprocessing import ( "context" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/stackrox/rox/central/deployment/datastore" "github.com/stackrox/rox/central/deployment/utils" "github.com/stackrox/rox/central/detection/lifecycle" @@ -14,15 +16,43 @@ import ( "github.com/stackrox/rox/central/sensor/service/pipeline/reconciliation" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/pkg/centralsensor" + "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/metrics" "github.com/stackrox/rox/pkg/search" + "golang.org/x/sync/semaphore" ) var ( + log = logging.LoggerForModule() + _ pipeline.Fragment = (*pipelineImpl)(nil) + + riskSemaphoreQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: "central", + Name: "deployment_risk_semaphore_queue_size", + Help: "Number of deployment risk reprocessing operations waiting for a semaphore slot.", + }) + riskSemaphoreHoldingSize = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: "central", + Name: "deployment_risk_semaphore_holding_size", + Help: "Number of deployment risk reprocessing operations currently holding a semaphore slot.", + }) + riskSemaphoreTimeouts = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: "central", + Name: "deployment_risk_semaphore_timeouts_total", + Help: "Total number of deployment risk reprocessing operations that timed out waiting for a semaphore slot.", + }) ) +func init() { + prometheus.MustRegister(riskSemaphoreQueueSize, riskSemaphoreHoldingSize, riskSemaphoreTimeouts) +} + // GetPipeline returns an instantiation of this particular pipeline func GetPipeline() pipeline.Fragment { return NewPipeline(datastore.Singleton(), lifecycle.SingletonManager(), riskManager.Singleton(), reprocessor.Singleton()) @@ -30,11 +60,13 @@ func GetPipeline() pipeline.Fragment { // NewPipeline returns a new instance of Pipeline. func NewPipeline(deployments datastore.DataStore, manager lifecycle.Manager, riskManager riskManager.Manager, riskReprocessor reprocessor.Loop) pipeline.Fragment { + maxConcurrency := int64(env.DeploymentRiskMaxConcurrency.IntegerSetting()) return &pipelineImpl{ riskManager: riskManager, riskReprocessor: riskReprocessor, manager: manager, deployments: deployments, + riskSemaphore: semaphore.NewWeighted(maxConcurrency), } } @@ -43,6 +75,7 @@ type pipelineImpl struct { riskManager riskManager.Manager riskReprocessor reprocessor.Loop manager lifecycle.Manager + riskSemaphore *semaphore.Weighted } func (s *pipelineImpl) Capabilities() []centralsensor.CentralCapability { @@ -68,6 +101,16 @@ func (s *pipelineImpl) Match(msg *central.MsgFromSensor) bool { func (s *pipelineImpl) Run(ctx context.Context, _ string, msg *central.MsgFromSensor, _ common.MessageInjector) error { defer countMetrics.IncrementResourceProcessedCounter(pipeline.ActionToOperation(msg.GetEvent().GetAction()), metrics.DeploymentReprocess) + // Throttle concurrent risk reprocessing to prevent DB connection pool exhaustion. + // + // A timeout is applied to prevent indefinite blocking if risk operations are stuck. + // On timeout, the operation is dropped -- it will be retried on the next reprocessing + // cycle. + if err := s.acquireRiskSemaphore(ctx); err != nil { + return err + } + defer s.releaseRiskSemaphore() + reprocessMsg := msg.GetEvent().GetReprocessDeployment() deployment, exists, err := s.deployments.GetDeployment(ctx, reprocessMsg.GetDeploymentId()) @@ -84,4 +127,44 @@ func (s *pipelineImpl) Run(ctx context.Context, _ string, msg *central.MsgFromSe return nil } +// acquireRiskSemaphore acquires the risk reprocessing semaphore with an optional timeout. +// This follows the same pattern as the image scan semaphore in central/image/service. +func (s *pipelineImpl) acquireRiskSemaphore(ctx context.Context) error { + waitTime := env.DeploymentRiskSemaphoreWaitTime.DurationSetting() + + acquireCtx := ctx + if waitTime > 0 { + var cancel context.CancelFunc + acquireCtx, cancel = context.WithTimeout(ctx, waitTime) + defer cancel() + } + + riskSemaphoreQueueSize.Inc() + defer riskSemaphoreQueueSize.Dec() + + if err := s.riskSemaphore.Acquire(acquireCtx, 1); err != nil { + if ctx.Err() != nil { + // Parent context was cancelled (sensor disconnected). This is expected. + log.Debugf("Unable to acquire context to reprocess deployment risk: %v", err) + } else if errors.Is(err, context.DeadlineExceeded) { + // Semaphore wait timed out... + riskSemaphoreTimeouts.Inc() + log.Warnf("Timed out waiting to reprocess deployment risk (waited %v, queue is full): %v", + waitTime, err) + } else { + // unexpected error + log.Errorf("Unexpected error acquiring risk semaphore: %v", err) + } + return err + } + + riskSemaphoreHoldingSize.Inc() + return nil +} + +func (s *pipelineImpl) releaseRiskSemaphore() { + s.riskSemaphore.Release(1) + riskSemaphoreHoldingSize.Dec() +} + func (s *pipelineImpl) OnFinish(_ string) {} diff --git a/central/sensor/service/pipeline/reprocessing/pipeline_test.go b/central/sensor/service/pipeline/reprocessing/pipeline_test.go new file mode 100644 index 0000000000000..4db245b00e2c8 --- /dev/null +++ b/central/sensor/service/pipeline/reprocessing/pipeline_test.go @@ -0,0 +1,178 @@ +package reprocessing + +import ( + "context" + "sync/atomic" + "testing" + "time" + + deploymentMocks "github.com/stackrox/rox/central/deployment/datastore/mocks" + lifecycleMocks "github.com/stackrox/rox/central/detection/lifecycle/mocks" + reprocessorMocks "github.com/stackrox/rox/central/reprocessor/mocks" + riskManagerMocks "github.com/stackrox/rox/central/risk/manager/mocks" + "github.com/stackrox/rox/generated/internalapi/central" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "golang.org/x/sync/semaphore" +) + +func TestRunCallsReprocessDeploymentRisk(t *testing.T) { + ctrl := gomock.NewController(t) + deployments := deploymentMocks.NewMockDataStore(ctrl) + manager := lifecycleMocks.NewMockManager(ctrl) + riskMgr := riskManagerMocks.NewMockManager(ctrl) + reprocessor := reprocessorMocks.NewMockLoop(ctrl) + + p := &pipelineImpl{ + deployments: deployments, + riskManager: riskMgr, + riskReprocessor: reprocessor, + manager: manager, + riskSemaphore: semaphore.NewWeighted(15), + } + + depID := uuid.NewV4().String() + dep := &storage.Deployment{Id: depID, Name: "test"} + deployments.EXPECT().GetDeployment(gomock.Any(), depID).Return(dep, true, nil) + riskMgr.EXPECT().ReprocessDeploymentRisk(dep) + + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Resource: ¢ral.SensorEvent_ReprocessDeployment{ + ReprocessDeployment: ¢ral.ReprocessDeploymentRisk{ + DeploymentId: depID, + }, + }, + }, + }, + } + + err := p.Run(context.Background(), "cluster-1", msg, nil) + assert.NoError(t, err) +} + +func TestRunThrottlesConcurrency(t *testing.T) { + ctrl := gomock.NewController(t) + deployments := deploymentMocks.NewMockDataStore(ctrl) + manager := lifecycleMocks.NewMockManager(ctrl) + riskMgr := riskManagerMocks.NewMockManager(ctrl) + reprocessor := reprocessorMocks.NewMockLoop(ctrl) + + const maxConcurrency = 3 + p := &pipelineImpl{ + deployments: deployments, + riskManager: riskMgr, + riskReprocessor: reprocessor, + manager: manager, + riskSemaphore: semaphore.NewWeighted(maxConcurrency), + } + + // Track peak concurrency inside ReprocessDeploymentRisk. + var currentConcurrency atomic.Int32 + var peakConcurrency atomic.Int32 + var completed atomic.Int32 + + const numDeployments = 20 + + // Set up mocks: each ReprocessDeploymentRisk call sleeps briefly to simulate DB work. + for range numDeployments { + depID := uuid.NewV4().String() + dep := &storage.Deployment{Id: depID, Name: "test"} + deployments.EXPECT().GetDeployment(gomock.Any(), depID).Return(dep, true, nil) + riskMgr.EXPECT().ReprocessDeploymentRisk(dep).Do(func(_ *storage.Deployment) { + cur := currentConcurrency.Add(1) + // Update peak + for { + peak := peakConcurrency.Load() + if cur <= peak { + break + } + if peakConcurrency.CompareAndSwap(peak, cur) { + break + } + } + time.Sleep(50 * time.Millisecond) + currentConcurrency.Add(-1) + completed.Add(1) + }) + + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Resource: ¢ral.SensorEvent_ReprocessDeployment{ + ReprocessDeployment: ¢ral.ReprocessDeploymentRisk{ + DeploymentId: depID, + }, + }, + }, + }, + } + + go func() { + err := p.Run(context.Background(), "cluster-1", msg, nil) + assert.NoError(t, err) + }() + } + + // Wait for all goroutines to finish. + require.Eventually(t, func() bool { + return completed.Load() == numDeployments + }, 10*time.Second, 10*time.Millisecond) + + // The peak concurrency should not exceed our limit. + assert.LessOrEqual(t, peakConcurrency.Load(), int32(maxConcurrency), + "peak concurrency (%d) should not exceed the semaphore limit (%d)", + peakConcurrency.Load(), maxConcurrency) + t.Logf("Peak concurrency: %d (limit: %d)", peakConcurrency.Load(), maxConcurrency) +} + +func TestRunRespectsContextCancellation(t *testing.T) { + ctrl := gomock.NewController(t) + deployments := deploymentMocks.NewMockDataStore(ctrl) + manager := lifecycleMocks.NewMockManager(ctrl) + riskMgr := riskManagerMocks.NewMockManager(ctrl) + reprocessor := reprocessorMocks.NewMockLoop(ctrl) + + // Semaphore with capacity 1, pre-acquired so the next Acquire blocks. + sem := semaphore.NewWeighted(1) + require.NoError(t, sem.Acquire(context.Background(), 1)) + + p := &pipelineImpl{ + deployments: deployments, + riskManager: riskMgr, + riskReprocessor: reprocessor, + manager: manager, + riskSemaphore: sem, + } + + // Note: no GetDeployment expectation because the semaphore is acquired first. + // With a cancelled context and a full semaphore, Acquire returns immediately + // with an error before any DB call is made. + + depID := uuid.NewV4().String() + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Resource: ¢ral.SensorEvent_ReprocessDeployment{ + ReprocessDeployment: ¢ral.ReprocessDeploymentRisk{ + DeploymentId: depID, + }, + }, + }, + }, + } + + // Cancel context so the semaphore Acquire fails. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := p.Run(ctx, "cluster-1", msg, nil) + assert.Error(t, err, "Run should return an error when context is cancelled") + + // Release the pre-acquired semaphore to clean up. + sem.Release(1) +} diff --git a/central/sensor/service/pipeline/virtualmachineindex/pipeline.go b/central/sensor/service/pipeline/virtualmachineindex/pipeline.go index dea0d7f86ba57..e11ebef4f189d 100644 --- a/central/sensor/service/pipeline/virtualmachineindex/pipeline.go +++ b/central/sensor/service/pipeline/virtualmachineindex/pipeline.go @@ -2,86 +2,74 @@ package virtualmachineindex import ( "context" - "strconv" + "strings" "github.com/pkg/errors" + "github.com/stackrox/rox/central/convert/v1tov2storage" countMetrics "github.com/stackrox/rox/central/metrics" "github.com/stackrox/rox/central/sensor/service/common" - "github.com/stackrox/rox/central/sensor/service/connection" "github.com/stackrox/rox/central/sensor/service/pipeline" "github.com/stackrox/rox/central/sensor/service/pipeline/reconciliation" - vmDatastore "github.com/stackrox/rox/central/virtualmachine/datastore" + virtualMachineDataStore "github.com/stackrox/rox/central/virtualmachine/datastore" + virtualMachineV2DataStore "github.com/stackrox/rox/central/virtualmachine/v2/datastore" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/centralsensor" - "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/metrics" - "github.com/stackrox/rox/pkg/rate" + pkgVM "github.com/stackrox/rox/pkg/virtualmachine" vmEnricher "github.com/stackrox/rox/pkg/virtualmachine/enricher" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -const ( - // rateLimiterWorkload is the workload name used for rate limiting VM index reports. - rateLimiterWorkload = "vm_index_report" -) - -// rateLimiter defines the interface for rate limiting operations used by this pipeline. -// This interface is satisfied by *rate.Limiter and allows for easier testing. -type rateLimiter interface { - TryConsume(clientID string) (allowed bool, reason string) - OnClientDisconnect(clientID string) -} - var ( log = logging.LoggerForModule() _ pipeline.Fragment = (*pipelineImpl)(nil) ) +const ( + matcherNotReadyReasonText = "the matcher is not initialized" +) + // GetPipeline returns an instantiation of this particular pipeline func GetPipeline() pipeline.Fragment { - rateLimit, err := strconv.ParseFloat(env.VMIndexReportRateLimit.Setting(), 64) - if err != nil { - log.Warnf("Invalid %s value: %v. Using fallback value of 0.3", env.VMIndexReportRateLimit.EnvVar(), err) - rateLimit = 0.3 // Keep in sync with the default value in env.VMIndexReportRateLimit. - } - bucketCapacity := env.VMIndexReportBucketCapacity.IntegerSetting() - rateLimiter, err := rate.NewLimiter(rateLimiterWorkload, rateLimit, bucketCapacity) - if err != nil { - log.Errorf("Failed to create rate limiter for %s: %v", rateLimiterWorkload, err) - } return newPipeline( - vmDatastore.Singleton(), + virtualMachineDataStore.Singleton(), vmEnricher.Singleton(), - rateLimiter, + virtualMachineV2DataStore.Singleton(), ) } // newPipeline returns a new instance of Pipeline. -func newPipeline(vms vmDatastore.DataStore, enricher vmEnricher.VirtualMachineEnricher, rl rateLimiter) pipeline.Fragment { +func newPipeline( + virtualMachineStore virtualMachineDataStore.DataStore, + enricher vmEnricher.VirtualMachineEnricher, + virtualMachineV2Store virtualMachineV2DataStore.DataStore, +) pipeline.Fragment { return &pipelineImpl{ - vmDatastore: vms, - enricher: enricher, - rateLimiter: rl, + virtualMachineStore: virtualMachineStore, + enricher: enricher, + virtualMachineV2Store: virtualMachineV2Store, } } type pipelineImpl struct { - vmDatastore vmDatastore.DataStore - enricher vmEnricher.VirtualMachineEnricher - rateLimiter rateLimiter + virtualMachineStore virtualMachineDataStore.DataStore + enricher vmEnricher.VirtualMachineEnricher + virtualMachineV2Store virtualMachineV2DataStore.DataStore } -func (p *pipelineImpl) OnFinish(clusterID string) { - // Notify rate limiter that this client (Sensor) has disconnected so it can rebalance the limiters. - if p.rateLimiter != nil { - p.rateLimiter.OnClientDisconnect(clusterID) - } -} +func (p *pipelineImpl) OnFinish(string) {} +// Capabilities omits VirtualMachinesSupported when ROX_VIRTUAL_MACHINES is +// off so Sensor treats this Central like a pre-VM build. func (p *pipelineImpl) Capabilities() []centralsensor.CentralCapability { + if !features.VirtualMachines.Enabled() { + return nil + } return []centralsensor.CentralCapability{centralsensor.VirtualMachinesSupported} } @@ -93,14 +81,51 @@ func (p *pipelineImpl) Match(msg *central.MsgFromSensor) bool { return msg.GetEvent().GetVirtualMachineIndexReport() != nil } +// sendVMIndexACK is a convenience wrapper around common.SendSensorACK for VM index reports. +func sendVMIndexACK(ctx context.Context, resourceID, reason string, injector common.MessageInjector) { + common.SendSensorACK(ctx, central.SensorACK_ACK, central.SensorACK_VM_INDEX_REPORT, resourceID, reason, injector) +} + +// sendVMIndexNACK is a convenience wrapper around common.SendSensorACK for VM index reports. +func sendVMIndexNACK(ctx context.Context, resourceID, reason string, injector common.MessageInjector) { + common.SendSensorACK(ctx, central.SensorACK_NACK, central.SensorACK_VM_INDEX_REPORT, resourceID, reason, injector) +} + +func reasonForEnrichmentFailure(err error) string { + if isMatcherNotReadyError(err) { + return centralsensor.SensorACKReasonMatcherNotReady + } + return centralsensor.SensorACKReasonEnrichmentFailed +} + +func isMatcherNotReadyError(err error) bool { + if err == nil { + return false + } + + grpcStatus, ok := status.FromError(err) + if !ok || grpcStatus.Code() != codes.FailedPrecondition { + return false + } + + return strings.Contains(grpcStatus.Message(), matcherNotReadyReasonText) +} + func (p *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.MsgFromSensor, injector common.MessageInjector) error { defer countMetrics.IncrementResourceProcessedCounter(pipeline.ActionToOperation(msg.GetEvent().GetAction()), metrics.VirtualMachineIndex) + event := msg.GetEvent() + index := event.GetVirtualMachineIndexReport() + resourceID := "" + if index != nil { + resourceID = common.VMIndexACKResourceID(index.GetId(), index.GetIndex().GetVsockCid()) + } + if !features.VirtualMachines.Enabled() { + // ACK to prevent the sender from retrying when the feature is disabled on Central. + sendVMIndexACK(ctx, resourceID, centralsensor.SensorACKReasonFeatureDisabled, injector) return nil } - event := msg.GetEvent() - index := event.GetVirtualMachineIndexReport() if index == nil { return errors.Errorf("unexpected resource type %T for virtual machine index report", event.GetResource()) } @@ -110,91 +135,102 @@ func (p *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.M event.GetAction().String(), central.ResourceAction_SYNC_RESOURCE.String(), ) + sendVMIndexNACK(ctx, resourceID, centralsensor.SensorACKReasonUnsupportedAction, injector) return nil } log.Debugf("Received virtual machine index report: %s", index.GetId()) if clusterID == "" { + sendVMIndexNACK(ctx, resourceID, centralsensor.SensorACKReasonMissingClusterID, injector) return errors.New("missing cluster ID in pipeline context") } - // Extract connection for capability checks; cluster ID is taken from the pipeline argument. - conn := connection.FromContext(ctx) - - // Rate limit check. Drop message if rate limiter is misconfigured (defensive behavior against misconfiguration) - // or rate limit exceeded. Afterwards, send NACK to Sensor if Sensor supports it. - if p.rateLimiter == nil { - logging.GetRateLimitedLogger().ErrorL( - "vm_index_report_nil_rate_limiter", - "No rate limiter found for workload %q. Dropping VM index report from cluster %s", - rateLimiterWorkload, - clusterID, - ) - if conn != nil && conn.HasCapability(centralsensor.SensorACKSupport) { - sendVMIndexReportResponse(ctx, clusterID, index.GetId(), central.SensorACK_NACK, "rate limiter not configured", injector) - } - return nil // Don't return error - would cause pipeline retry - } - - allowed, reason := p.rateLimiter.TryConsume(clusterID) - if !allowed { - log.Infof("Dropping VM index report %s from cluster %s: %s", index.GetId(), clusterID, reason) - if conn != nil && conn.HasCapability(centralsensor.SensorACKSupport) { - sendVMIndexReportResponse(ctx, clusterID, index.GetId(), central.SensorACK_NACK, reason, injector) - } - return nil // Don't return error - would cause pipeline retry - } - // Get or create VM vm := &storage.VirtualMachine{Id: index.GetId()} // Extract Scanner V4 index report from VM index report event indexV4 := index.GetIndex().GetIndexV4() if indexV4 == nil { + sendVMIndexNACK(ctx, resourceID, centralsensor.SensorACKReasonMissingScanData, injector) return errors.Errorf("VM index report %s missing Scanner V4 index data", index.GetId()) } // Enrich VM with vulnerabilities err := p.enricher.EnrichVirtualMachineWithVulnerabilities(vm, indexV4) if err != nil { + sendVMIndexNACK(ctx, resourceID, reasonForEnrichmentFailure(err), injector) return errors.Wrapf(err, "failed to enrich VM %s with vulnerabilities", index.GetId()) } - // Store enriched VM - if err := p.vmDatastore.UpdateVirtualMachineScan(ctx, vm.GetId(), vm.GetScan()); err != nil { - return errors.Wrapf(err, "failed to upsert VM %s to datastore", index.GetId()) + if guestOS := p.lookupGuestOS(ctx, index.GetId()); guestOS != "" && guestOS != pkgVM.UnknownGuestOS && vm.GetScan() != nil { + vm.Scan.OperatingSystem = guestOS + } + + // Store enriched VM via v1 or v2 path. + if features.VirtualMachinesEnhancedDataModel.Enabled() { + if err := p.storeV2Scan(ctx, clusterID, vm); err != nil { + sendVMIndexNACK(ctx, resourceID, centralsensor.SensorACKReasonStorageFailed, injector) + return err + } + } else { + if err := p.storeV1Scan(ctx, vm); err != nil { + sendVMIndexNACK(ctx, resourceID, centralsensor.SensorACKReasonStorageFailed, injector) + return err + } } log.Debugf("Successfully enriched and stored VM %s with %d components", vm.GetId(), len(vm.GetScan().GetComponents())) - // Send ACK to Sensor if Sensor supports it - if conn != nil && conn.HasCapability(centralsensor.SensorACKSupport) { - sendVMIndexReportResponse(ctx, clusterID, index.GetId(), central.SensorACK_ACK, "", injector) + sendVMIndexACK(ctx, resourceID, "", injector) + return nil +} + +func (p *pipelineImpl) storeV1Scan(ctx context.Context, vm *storage.VirtualMachine) error { + if err := p.virtualMachineStore.UpdateVirtualMachineScan(ctx, vm.GetId(), vm.GetScan()); err != nil { + return errors.Wrapf(err, "failed to upsert VM %s to datastore", vm.GetId()) } return nil } -// sendVMIndexReportResponse sends an ACK or NACK for a VM index report. -func sendVMIndexReportResponse(ctx context.Context, clusterID, vmID string, action central.SensorACK_Action, reason string, injector common.MessageInjector) { - if injector == nil { - log.Debugf("Cannot send %s to Sensor for cluster %s - no injector", action.String(), clusterID) - return - } - msg := ¢ral.MsgToSensor{ - Msg: ¢ral.MsgToSensor_SensorAck{ - SensorAck: ¢ral.SensorACK{ - Action: action, - MessageType: central.SensorACK_VM_INDEX_REPORT, - ResourceId: vmID, - Reason: reason, - }, - }, - } - if err := injector.InjectMessage(ctx, msg); err != nil { - log.Warnf("Failed sending VM index report %s for VM %s in cluster %s: %v", action.String(), vmID, clusterID, err) - } else { - log.Debugf("Sent VM index report %s for VM %s in cluster %s (reason=%q)", action.String(), vmID, clusterID, reason) +func (p *pipelineImpl) storeV2Scan(ctx context.Context, clusterID string, vm *storage.VirtualMachine) error { + if err := p.virtualMachineV2Store.EnsureVirtualMachineExists(ctx, vm.GetId(), clusterID); err != nil { + return errors.Wrapf(err, "failed to ensure VM %s exists in v2 datastore", vm.GetId()) + } + + parts := v1tov2storage.ScanPartsFromV1Scan(vm.GetId(), vm.GetScan()) + if parts == nil { + return nil + } + + if err := p.virtualMachineV2Store.UpsertScan(ctx, vm.GetId(), *parts); err != nil { + return errors.Wrapf(err, "failed to upsert v2 scan for VM %s", vm.GetId()) + } + return nil +} + +// lookupGuestOS returns facts["guestOS"] so scan identity stays on the KubeVirt +// family name rather than the versioned display string in guest_os. +func (p *pipelineImpl) lookupGuestOS(ctx context.Context, vmID string) string { + if features.VirtualMachinesEnhancedDataModel.Enabled() { + vm, found, err := p.virtualMachineV2Store.GetVirtualMachine(ctx, vmID) + if err != nil { + log.Warnf("Failed to look up guest OS for VM %s: %v", vmID, err) + return "" + } + if !found { + return "" + } + return vm.GetFacts()[pkgVM.GuestOSKey] + } + vm, found, err := p.virtualMachineStore.GetVirtualMachine(ctx, vmID) + if err != nil { + log.Warnf("Failed to look up guest OS for VM %s: %v", vmID, err) + return "" + } + if !found { + return "" } + return vm.GetFacts()[pkgVM.GuestOSKey] } diff --git a/central/sensor/service/pipeline/virtualmachineindex/pipeline_test.go b/central/sensor/service/pipeline/virtualmachineindex/pipeline_test.go index f1c6c4edd0399..e5bfdbca75e8c 100644 --- a/central/sensor/service/pipeline/virtualmachineindex/pipeline_test.go +++ b/central/sensor/service/pipeline/virtualmachineindex/pipeline_test.go @@ -2,14 +2,14 @@ package virtualmachineindex import ( "context" + "strconv" "testing" "github.com/pkg/errors" - "github.com/stackrox/rox/central/sensor/service/common" - "github.com/stackrox/rox/central/sensor/service/connection" - connMocks "github.com/stackrox/rox/central/sensor/service/connection/mocks" "github.com/stackrox/rox/central/sensor/service/pipeline/reconciliation" - vmDatastoreMocks "github.com/stackrox/rox/central/virtualmachine/datastore/mocks" + virtualMachineDSMocks "github.com/stackrox/rox/central/virtualmachine/datastore/mocks" + virtualMachineV2DSMocks "github.com/stackrox/rox/central/virtualmachine/v2/datastore/mocks" + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" "github.com/stackrox/rox/generated/internalapi/central" v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4" v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" @@ -17,13 +17,13 @@ import ( "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/features" - "github.com/stackrox/rox/pkg/rate" - "github.com/stackrox/rox/pkg/sync" + pkgVM "github.com/stackrox/rox/pkg/virtualmachine" vmEnricherMocks "github.com/stackrox/rox/pkg/virtualmachine/enricher/mocks" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -32,13 +32,6 @@ const ( var ctx = context.Background() -// mustNewLimiter creates a rate limiter or fails the test. -func mustNewLimiter(t require.TestingT, workloadName string, globalRate float64, bucketCapacity int) *rate.Limiter { - limiter, err := rate.NewLimiter(workloadName, globalRate, bucketCapacity) - require.NoError(t, err) - return limiter -} - func TestPipeline(t *testing.T) { suite.Run(t, new(PipelineTestSuite)) } @@ -46,23 +39,20 @@ func TestPipeline(t *testing.T) { type PipelineTestSuite struct { suite.Suite - vmDatastore *vmDatastoreMocks.MockDataStore - enricher *vmEnricherMocks.MockVirtualMachineEnricher - pipeline *pipelineImpl + virtualMachineStore *virtualMachineDSMocks.MockDataStore + enricher *vmEnricherMocks.MockVirtualMachineEnricher + pipeline *pipelineImpl mockCtrl *gomock.Controller } func (suite *PipelineTestSuite) SetupTest() { suite.mockCtrl = gomock.NewController(suite.T()) - suite.vmDatastore = vmDatastoreMocks.NewMockDataStore(suite.mockCtrl) + suite.virtualMachineStore = virtualMachineDSMocks.NewMockDataStore(suite.mockCtrl) suite.enricher = vmEnricherMocks.NewMockVirtualMachineEnricher(suite.mockCtrl) - // Use unlimited rate limiter for tests (rate=0) - rateLimiter := mustNewLimiter(suite.T(), "test", 0, 50) suite.pipeline = &pipelineImpl{ - vmDatastore: suite.vmDatastore, - enricher: suite.enricher, - rateLimiter: rateLimiter, + virtualMachineStore: suite.virtualMachineStore, + enricher: suite.enricher, } } @@ -126,6 +116,7 @@ func (suite *PipelineTestSuite) TestMatch_VirtualMachineMessage() { func (suite *PipelineTestSuite) TestRun_NilVirtualMachine() { suite.T().Setenv(features.VirtualMachines.EnvVar(), "true") + suite.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") msg := ¢ral.MsgFromSensor{ Msg: ¢ral.MsgFromSensor_Event{ Event: ¢ral.SensorEvent{ @@ -145,18 +136,21 @@ func (suite *PipelineTestSuite) TestRun_NilVirtualMachine() { func (suite *PipelineTestSuite) TestRun_UpdateScanError() { suite.T().Setenv(features.VirtualMachines.EnvVar(), "true") + suite.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") vmID := "vm-1" msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) - // Expect enricher to be called successfully - suite.enricher.EXPECT(). - EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). - Return(nil) - - expectedError := errors.New("datastore error") - suite.vmDatastore.EXPECT(). - UpdateVirtualMachineScan(ctx, vmID, gomock.Any()). - Return(expectedError) + gomock.InOrder( + suite.enricher.EXPECT(). + EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). + Return(nil), + suite.virtualMachineStore.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(nil, false, nil), + suite.virtualMachineStore.EXPECT(). + UpdateVirtualMachineScan(ctx, vmID, gomock.Any()). + Return(errors.New("datastore error")), + ) err := suite.pipeline.Run(ctx, testClusterID, msg, nil) suite.Error(err) @@ -165,8 +159,25 @@ func (suite *PipelineTestSuite) TestRun_UpdateScanError() { } func (suite *PipelineTestSuite) TestCapabilities() { - capabilities := suite.pipeline.Capabilities() - suite.Contains(capabilities, centralsensor.CentralCapability(centralsensor.VirtualMachinesSupported)) + cases := map[string]struct { + enabled bool + want []centralsensor.CentralCapability + }{ + "flag on advertises VirtualMachinesSupported": { + enabled: true, + want: []centralsensor.CentralCapability{centralsensor.VirtualMachinesSupported}, + }, + "flag off advertises nothing": { + enabled: false, + want: nil, + }, + } + for name, tc := range cases { + suite.Run(name, func() { + suite.T().Setenv(features.VirtualMachines.EnvVar(), strconv.FormatBool(tc.enabled)) + suite.Equal(tc.want, suite.pipeline.Capabilities()) + }) + } } func (suite *PipelineTestSuite) TestOnFinish() { @@ -191,21 +202,21 @@ func (suite *PipelineTestSuite) TestGetPipeline() { } func (suite *PipelineTestSuite) TestNewPipeline() { - mockDatastore := vmDatastoreMocks.NewMockDataStore(suite.mockCtrl) + mockDatastore := virtualMachineDSMocks.NewMockDataStore(suite.mockCtrl) mockEnricher := vmEnricherMocks.NewMockVirtualMachineEnricher(suite.mockCtrl) - rateLimiter := mustNewLimiter(suite.T(), "test", 0, 50) - pipeline := newPipeline(mockDatastore, mockEnricher, rateLimiter) + pipeline := newPipeline(mockDatastore, mockEnricher, nil) suite.NotNil(pipeline) impl, ok := pipeline.(*pipelineImpl) suite.True(ok, "Should return pipelineImpl instance") - suite.Equal(mockDatastore, impl.vmDatastore) + suite.Equal(mockDatastore, impl.virtualMachineStore) suite.Equal(mockEnricher, impl.enricher) - suite.Equal(rateLimiter, impl.rateLimiter) + suite.Nil(impl.virtualMachineV2Store) } // Test table-driven approach for different actions func TestPipelineRun_DifferentActions(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") tests := []struct { name string action central.ResourceAction @@ -246,25 +257,24 @@ func TestPipelineRun_DifferentActions(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - vmDatastore := vmDatastoreMocks.NewMockDataStore(ctrl) + virtualMachineStore := virtualMachineDSMocks.NewMockDataStore(ctrl) enricher := vmEnricherMocks.NewMockVirtualMachineEnricher(ctrl) - rateLimiter := mustNewLimiter(t, "test", 0, 50) pipeline := &pipelineImpl{ - vmDatastore: vmDatastore, - enricher: enricher, - rateLimiter: rateLimiter, + virtualMachineStore: virtualMachineStore, + enricher: enricher, } vmID := "vm-1" msg := createVMIndexMessage(vmID, tt.action) if tt.expectUpdate { - // Expect enricher to be called for action enricher.EXPECT(). EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). Return(nil) - - vmDatastore.EXPECT(). + virtualMachineStore.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(nil, false, nil) + virtualMachineStore.EXPECT(). UpdateVirtualMachineScan(ctx, vmID, gomock.Any()). Do(func(ctx context.Context, virtualMachineID string, _ *storage.VirtualMachineScan) { assert.Equal(t, vmID, virtualMachineID) @@ -289,14 +299,13 @@ func TestPipelineRun_DifferentActions(t *testing.T) { // Test edge cases with malformed messages func TestPipelineEdgeCases(t *testing.T) { t.Setenv(features.VirtualMachines.EnvVar(), "true") + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") ctrl := gomock.NewController(t) defer ctrl.Finish() - vmDatastore := vmDatastoreMocks.NewMockDataStore(ctrl) - rateLimiter := mustNewLimiter(t, "test", 0, 50) + virtualMachineStore := virtualMachineDSMocks.NewMockDataStore(ctrl) pipeline := &pipelineImpl{ - vmDatastore: vmDatastore, - rateLimiter: rateLimiter, + virtualMachineStore: virtualMachineStore, } t.Run("nil message", func(t *testing.T) { @@ -346,314 +355,652 @@ func TestPipelineEdgeCases(t *testing.T) { }) } +// mockInjector records InjectMessage calls. +type mockInjector struct { + messages []*central.MsgToSensor + injectErr error + capabilities map[centralsensor.SensorCapability]bool +} + +func (m *mockInjector) InjectMessage(_ concurrency.Waitable, msg *central.MsgToSensor) error { + m.messages = append(m.messages, msg) + return m.injectErr +} + +func (m *mockInjector) InjectMessageIntoQueue(_ *central.MsgFromSensor) {} + +func (m *mockInjector) HasCapability(cap centralsensor.SensorCapability) bool { + return m.capabilities[cap] +} + +func (suite *PipelineTestSuite) TestRun_SendsACKOnSuccess() { + suite.T().Setenv(features.VirtualMachines.EnvVar(), "true") + suite.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + vmID := "vm-ack-test" + msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) + + suite.enricher.EXPECT(). + EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). + Return(nil) + suite.virtualMachineStore.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(nil, false, nil) + suite.virtualMachineStore.EXPECT(). + UpdateVirtualMachineScan(ctx, vmID, gomock.Any()). + Return(nil) + + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + + err := suite.pipeline.Run(ctx, testClusterID, msg, injector) + suite.NoError(err) + + suite.Require().Len(injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + suite.Require().NotNil(ack) + suite.Equal(central.SensorACK_ACK, ack.GetAction()) + suite.Equal(central.SensorACK_VM_INDEX_REPORT, ack.GetMessageType()) + suite.Equal(vmID+":", ack.GetResourceId()) + suite.Empty(ack.GetReason()) +} + +func (suite *PipelineTestSuite) TestRun_SendsACKWithVMIDAndVsockCIDResourceID() { + suite.T().Setenv(features.VirtualMachines.EnvVar(), "true") + suite.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + vmID := "vm-ack-vsock-correlation" + vsockCID := "1337" + msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) + msg.GetEvent().GetVirtualMachineIndexReport().GetIndex().VsockCid = vsockCID + + suite.enricher.EXPECT(). + EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). + Return(nil) + suite.virtualMachineStore.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(nil, false, nil) + suite.virtualMachineStore.EXPECT(). + UpdateVirtualMachineScan(ctx, vmID, gomock.Any()). + Return(nil) + + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + + err := suite.pipeline.Run(ctx, testClusterID, msg, injector) + suite.NoError(err) + + suite.Require().Len(injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + suite.Require().NotNil(ack) + suite.Equal(central.SensorACK_ACK, ack.GetAction()) + suite.Equal(central.SensorACK_VM_INDEX_REPORT, ack.GetMessageType()) + suite.Equal(vmID+":"+vsockCID, ack.GetResourceId(), "expected ACK resource_id to match VMID:CID pair for VMID:CID resource ID correlation") + suite.Empty(ack.GetReason()) +} + +func (suite *PipelineTestSuite) TestRun_NoACKWhenCapabilityMissing() { + suite.T().Setenv(features.VirtualMachines.EnvVar(), "true") + suite.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + vmID := "vm-no-cap" + msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) + + suite.enricher.EXPECT(). + EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). + Return(nil) + suite.virtualMachineStore.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(nil, false, nil) + suite.virtualMachineStore.EXPECT(). + UpdateVirtualMachineScan(ctx, vmID, gomock.Any()). + Return(nil) + + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{}, + } + + err := suite.pipeline.Run(ctx, testClusterID, msg, injector) + suite.NoError(err) + suite.Empty(injector.messages, "should not send ACK when SensorACKSupport is missing") +} + +func (suite *PipelineTestSuite) TestRun_NACKOnDBError() { + suite.T().Setenv(features.VirtualMachines.EnvVar(), "true") + suite.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + vmID := "vm-error" + msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) + + suite.enricher.EXPECT(). + EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). + Return(nil) + suite.virtualMachineStore.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(nil, false, nil) + suite.virtualMachineStore.EXPECT(). + UpdateVirtualMachineScan(ctx, vmID, gomock.Any()). + Return(errors.New("db error")) + + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + + err := suite.pipeline.Run(ctx, testClusterID, msg, injector) + suite.Error(err) + + suite.Require().Len(injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + suite.Require().NotNil(ack) + suite.Equal(central.SensorACK_NACK, ack.GetAction()) + suite.Equal(central.SensorACK_VM_INDEX_REPORT, ack.GetMessageType()) + suite.Equal(vmID+":", ack.GetResourceId()) + suite.Equal(centralsensor.SensorACKReasonStorageFailed, ack.GetReason()) +} + +func (suite *PipelineTestSuite) TestRun_NACKOnEnrichmentError() { + suite.T().Setenv(features.VirtualMachines.EnvVar(), "true") + suite.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + vmID := "vm-enrich-fail" + msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) + + suite.enricher.EXPECT(). + EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). + Return(errors.New("scanner unavailable")) + + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + + err := suite.pipeline.Run(ctx, testClusterID, msg, injector) + suite.Error(err) + + suite.Require().Len(injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + suite.Require().NotNil(ack) + suite.Equal(central.SensorACK_NACK, ack.GetAction()) + suite.Equal(central.SensorACK_VM_INDEX_REPORT, ack.GetMessageType()) + suite.Equal(vmID+":", ack.GetResourceId()) + suite.Equal(centralsensor.SensorACKReasonEnrichmentFailed, ack.GetReason()) +} + +func (suite *PipelineTestSuite) TestRun_NACKOnMatcherNotInitializedError() { + suite.T().Setenv(features.VirtualMachines.EnvVar(), "true") + suite.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + vmID := "vm-matcher-not-initialized" + msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) + + suite.enricher.EXPECT(). + EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). + Return(errors.Wrap( + status.Error(codes.FailedPrecondition, "the matcher is not initialized: initial load for the vulnerability store is in progress"), + "getting scan for VM", + )) + + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + + err := suite.pipeline.Run(ctx, testClusterID, msg, injector) + suite.Error(err) + + suite.Require().Len(injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + suite.Require().NotNil(ack) + suite.Equal(central.SensorACK_NACK, ack.GetAction()) + suite.Equal(central.SensorACK_VM_INDEX_REPORT, ack.GetMessageType()) + suite.Equal(vmID+":", ack.GetResourceId()) + suite.Equal(centralsensor.SensorACKReasonMatcherNotReady, ack.GetReason()) +} + +func (suite *PipelineTestSuite) TestRun_NACKOnMissingClusterID() { + suite.T().Setenv(features.VirtualMachines.EnvVar(), "true") + suite.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + vmID := "vm-no-cluster" + msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) + + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + + err := suite.pipeline.Run(ctx, "", msg, injector) + suite.ErrorContains(err, "missing cluster ID") + + suite.Require().Len(injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + suite.Require().NotNil(ack) + suite.Equal(central.SensorACK_NACK, ack.GetAction()) + suite.Equal(central.SensorACK_VM_INDEX_REPORT, ack.GetMessageType()) + suite.Equal(vmID+":", ack.GetResourceId()) + suite.Equal(centralsensor.SensorACKReasonMissingClusterID, ack.GetReason()) +} + +func (suite *PipelineTestSuite) TestRun_NACKOnMissingScannerIndexPayload() { + suite.T().Setenv(features.VirtualMachines.EnvVar(), "true") + suite.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + tests := []struct { + name string + index *v1.IndexReport + }{ + { + name: "nil Index", + index: nil, + }, + { + name: "Index without Scanner V4 payload", + index: &v1.IndexReport{}, + }, + } + + for _, tt := range tests { + suite.Run(tt.name, func() { + vmID := "vm-missing-payload-" + tt.name + msg := ¢ral.MsgFromSensor{ + Msg: ¢ral.MsgFromSensor_Event{ + Event: ¢ral.SensorEvent{ + Id: vmID, + Action: central.ResourceAction_SYNC_RESOURCE, + Resource: ¢ral.SensorEvent_VirtualMachineIndexReport{ + VirtualMachineIndexReport: &v1.IndexReportEvent{ + Id: vmID, + Index: tt.index, + }, + }, + }, + }, + } + + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + + err := suite.pipeline.Run(ctx, testClusterID, msg, injector) + suite.ErrorContains(err, "missing Scanner V4 index data") + + suite.Require().Len(injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + suite.Require().NotNil(ack) + suite.Equal(central.SensorACK_NACK, ack.GetAction()) + suite.Equal(central.SensorACK_VM_INDEX_REPORT, ack.GetMessageType()) + suite.Equal(vmID+":", ack.GetResourceId()) + suite.Equal(centralsensor.SensorACKReasonMissingScanData, ack.GetReason()) + }) + } +} + func TestPipelineRun_DisabledFeature(t *testing.T) { t.Setenv(features.VirtualMachines.EnvVar(), "false") + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") ctrl := gomock.NewController(t) defer ctrl.Finish() - vmDatastore := vmDatastoreMocks.NewMockDataStore(ctrl) + virtualMachineStore := virtualMachineDSMocks.NewMockDataStore(ctrl) enricher := vmEnricherMocks.NewMockVirtualMachineEnricher(ctrl) - rateLimiter := mustNewLimiter(t, "test", 0, 50) pipeline := &pipelineImpl{ - vmDatastore: vmDatastore, - enricher: enricher, - rateLimiter: rateLimiter, + virtualMachineStore: virtualMachineStore, + enricher: enricher, } vmID := "vm-1" msg := createVMIndexMessage(vmID, central.ResourceAction_CREATE_RESOURCE) - err := pipeline.Run(ctx, testClusterID, msg, nil) + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, + } + + err := pipeline.Run(ctx, testClusterID, msg, injector) assert.NoError(t, err) + assert.Len(t, injector.messages, 1, "should ACK to prevent retries when feature is disabled") + ack := injector.messages[0].GetSensorAck() + assert.NotNil(t, ack) + assert.Equal(t, central.SensorACK_ACK, ack.GetAction()) + assert.Equal(t, central.SensorACK_VM_INDEX_REPORT, ack.GetMessageType()) + assert.Equal(t, vmID+":", ack.GetResourceId()) + assert.Equal(t, centralsensor.SensorACKReasonFeatureDisabled, ack.GetReason()) } -// TestPipelineRun_RateLimitDisabled tests that rate limiting is disabled when configured with 0 -func TestPipelineRun_RateLimitDisabled(t *testing.T) { +func TestPipelineRunV2_StoresScanViaV2Datastore(t *testing.T) { t.Setenv(features.VirtualMachines.EnvVar(), "true") + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") ctrl := gomock.NewController(t) defer ctrl.Finish() - vmDatastore := vmDatastoreMocks.NewMockDataStore(ctrl) enricher := vmEnricherMocks.NewMockVirtualMachineEnricher(ctrl) - rateLimiter := mustNewLimiter(t, "test", 0, 50) // Disabled + virtualMachineV2Store := virtualMachineV2DSMocks.NewMockDataStore(ctrl) pipeline := &pipelineImpl{ - vmDatastore: vmDatastore, - enricher: enricher, - rateLimiter: rateLimiter, + enricher: enricher, + virtualMachineV2Store: virtualMachineV2Store, } - vmID := "vm-1" + vmID := "vm-v2-scan" msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) - // Should process all 100 requests without rate limiting - for i := range 100 { - enricher.EXPECT(). - EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). - Return(nil) - vmDatastore.EXPECT(). - UpdateVirtualMachineScan(ctx, vmID, gomock.Any()). - Return(nil) + enricher.EXPECT(). + EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). + DoAndReturn(func(vm *storage.VirtualMachine, _ interface{}) error { + vm.Scan = &storage.VirtualMachineScan{ + Components: []*storage.EmbeddedVirtualMachineScanComponent{ + {Name: "test-pkg", Version: "1.0"}, + }, + } + return nil + }) + virtualMachineV2Store.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(nil, false, nil) + virtualMachineV2Store.EXPECT(). + EnsureVirtualMachineExists(gomock.Any(), vmID, testClusterID). + Return(nil) + virtualMachineV2Store.EXPECT(). + UpsertScan(gomock.Any(), vmID, gomock.Any()). + DoAndReturn(func(_ context.Context, id string, parts common.VMScanParts) error { + assert.Equal(t, vmID, id) + assert.NotNil(t, parts.Scan) + assert.Equal(t, vmID, parts.Scan.GetVmV2Id()) + return nil + }) - err := pipeline.Run(ctx, testClusterID, msg, nil) - assert.NoError(t, err, "request %d should succeed with rate limiting disabled", i) + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, } + + err := pipeline.Run(ctx, testClusterID, msg, injector) + assert.NoError(t, err) + + assert.Len(t, injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + assert.NotNil(t, ack) + assert.Equal(t, central.SensorACK_ACK, ack.GetAction()) } -// TestPipelineRun_RateLimitEnabled tests that rate limiting rejects requests when enabled. -// This test verifies that: -// 1. First N requests (within burst) succeed and perform enrichment/datastore writes -// 2. Rate-limited request does NOT perform enrichment or datastore writes -// 3. A NACK is sent for rate-limited requests when ACK support is enabled -func TestPipelineRun_RateLimitEnabled(t *testing.T) { +func TestPipelineRunV2_NACKOnEnsureError(t *testing.T) { t.Setenv(features.VirtualMachines.EnvVar(), "true") + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") ctrl := gomock.NewController(t) defer ctrl.Finish() - vmDatastore := vmDatastoreMocks.NewMockDataStore(ctrl) enricher := vmEnricherMocks.NewMockVirtualMachineEnricher(ctrl) - rateLimiter := mustNewLimiter(t, "test", 5, 5) // 5 req/s, bucket capacity=5 - - // Recording injector to capture sent messages - injector := &recordingInjector{} - - // Mock connection with SensorACKSupport capability - mockConn := connMocks.NewMockSensorConnection(ctrl) - mockConn.EXPECT().HasCapability(centralsensor.SensorACKSupport).Return(true).AnyTimes() + virtualMachineV2Store := virtualMachineV2DSMocks.NewMockDataStore(ctrl) pipeline := &pipelineImpl{ - vmDatastore: vmDatastore, - enricher: enricher, - rateLimiter: rateLimiter, + enricher: enricher, + virtualMachineV2Store: virtualMachineV2Store, } - vmID := "vm-1" + vmID := "vm-v2-ensure-fail" msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) - // Build a context with the mocked connection that has SensorACKSupport - ctxWithConn := connection.WithConnection(context.Background(), mockConn) - - // Expect enrichment and datastore writes ONLY for the first 5 (non-rate-limited) requests. - // The 6th request should be rate-limited and these methods should NOT be called. enricher.EXPECT(). EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). - Return(nil). - Times(5) - - vmDatastore.EXPECT(). - UpdateVirtualMachineScan(gomock.Any(), vmID, gomock.Any()). - Return(nil). - Times(5) - - // Send 6 requests - the first 5 should be processed successfully, - // the 6th should be rate-limited. - for i := range 6 { - err := pipeline.Run(ctxWithConn, testClusterID, msg, injector) - assert.NoError(t, err, "Run should not return an error even when rate-limited (request %d)", i+1) + Return(nil) + virtualMachineV2Store.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(nil, false, nil) + virtualMachineV2Store.EXPECT(). + EnsureVirtualMachineExists(gomock.Any(), vmID, testClusterID). + Return(errors.New("ensure failed")) + + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, } - // Verify ACKs were sent for successful requests and NACK for rate-limited request - acks := injector.getSentACKs() - require.Len(t, acks, 6, "expected 6 ACK/NACK messages (5 ACKs + 1 NACK)") - - // First 5 should be ACKs - for i := range 5 { - assert.Equal(t, central.SensorACK_ACK, acks[i].GetAction(), "request %d should be ACKed", i+1) - assert.Equal(t, central.SensorACK_VM_INDEX_REPORT, acks[i].GetMessageType()) - } + err := pipeline.Run(ctx, testClusterID, msg, injector) + assert.Error(t, err) + assert.Contains(t, err.Error(), "ensure failed") - // 6th should be NACK - assert.Equal(t, central.SensorACK_NACK, acks[5].GetAction(), "request 6 should be NACKed (rate limited)") - assert.Equal(t, central.SensorACK_VM_INDEX_REPORT, acks[5].GetMessageType()) - assert.Contains(t, acks[5].GetReason(), "rate limit exceeded") + assert.Len(t, injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + assert.NotNil(t, ack) + assert.Equal(t, central.SensorACK_NACK, ack.GetAction()) + assert.Equal(t, centralsensor.SensorACKReasonStorageFailed, ack.GetReason()) } -// TestPipelineRun_RateLimitEnabled_NoACKSupport tests that when the connection -// does not support SensorACKSupport, rate limiting still applies but no ACK/NACK -// messages are sent. -func TestPipelineRun_RateLimitEnabled_NoACKSupport(t *testing.T) { +func TestPipelineRunV2_NilScanNoUpsert(t *testing.T) { t.Setenv(features.VirtualMachines.EnvVar(), "true") + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") ctrl := gomock.NewController(t) defer ctrl.Finish() - vmDatastore := vmDatastoreMocks.NewMockDataStore(ctrl) enricher := vmEnricherMocks.NewMockVirtualMachineEnricher(ctrl) - rateLimiter := mustNewLimiter(t, "test-no-ack", 5, 5) // 5 req/s, bucket capacity=5 - - // Recording injector to capture any ACK/NACK attempts - should remain empty. - injector := &recordingInjector{} - - // Mock connection WITHOUT SensorACKSupport capability - mockConn := connMocks.NewMockSensorConnection(ctrl) - mockConn.EXPECT().HasCapability(centralsensor.SensorACKSupport).Return(false).AnyTimes() + virtualMachineV2Store := virtualMachineV2DSMocks.NewMockDataStore(ctrl) pipeline := &pipelineImpl{ - vmDatastore: vmDatastore, - enricher: enricher, - rateLimiter: rateLimiter, + enricher: enricher, + virtualMachineV2Store: virtualMachineV2Store, } - vmID := "vm-1" + vmID := "vm-v2-nil-scan" msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) - // Build a context with the mocked connection that does NOT have SensorACKSupport - ctxWithConn := connection.WithConnection(context.Background(), mockConn) - - // Expect enrichment and datastore writes ONLY for the first 5 (non-rate-limited) requests. enricher.EXPECT(). EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). - Return(nil). - Times(5) - - vmDatastore.EXPECT(). - UpdateVirtualMachineScan(gomock.Any(), vmID, gomock.Any()). - Return(nil). - Times(5) - - // Send 6 requests - the first 5 should be processed successfully, - // the 6th should be rate-limited. - for i := range 6 { - err := pipeline.Run(ctxWithConn, testClusterID, msg, injector) - assert.NoError(t, err, "Run should not return an error even when rate-limited (request %d)", i+1) + Return(nil) + virtualMachineV2Store.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(nil, false, nil) + virtualMachineV2Store.EXPECT(). + EnsureVirtualMachineExists(gomock.Any(), vmID, testClusterID). + Return(nil) + + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, } - // Verify NO ACK/NACK messages were sent (SensorACKSupport is not available) - acks := injector.getSentACKs() - assert.Empty(t, acks, "no ACK/NACKs should be sent when SensorACKSupport is not available") + err := pipeline.Run(ctx, testClusterID, msg, injector) + assert.NoError(t, err) + + assert.Len(t, injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + assert.NotNil(t, ack) + assert.Equal(t, central.SensorACK_ACK, ack.GetAction()) } -// TestPipelineRun_NilRateLimiter_WithACKSupport tests behavior when the rateLimiter is nil and ACKs are supported. -// This covers the nil-limiter branch and verifies that: -// 1. No enrichment/datastore calls occur -// 2. A NACK with MessageType=VM_INDEX_REPORT is sent -func TestPipelineRun_NilRateLimiter_WithACKSupport(t *testing.T) { +func TestPipelineRunV2_NACKOnUpsertScanError(t *testing.T) { t.Setenv(features.VirtualMachines.EnvVar(), "true") + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") ctrl := gomock.NewController(t) defer ctrl.Finish() - // Mocks for datastore and enricher - no expectations should be set on these, - // because the pipeline must short-circuit before doing any work. - vmDatastore := vmDatastoreMocks.NewMockDataStore(ctrl) enricher := vmEnricherMocks.NewMockVirtualMachineEnricher(ctrl) - - // Recording injector to capture sent messages - injector := &recordingInjector{} - - // Mock connection with SensorACKSupport capability - mockConn := connMocks.NewMockSensorConnection(ctrl) - mockConn.EXPECT().HasCapability(centralsensor.SensorACKSupport).Return(true).AnyTimes() + virtualMachineV2Store := virtualMachineV2DSMocks.NewMockDataStore(ctrl) pipeline := &pipelineImpl{ - vmDatastore: vmDatastore, - enricher: enricher, - rateLimiter: nil, // nil rate limiter to cover the nil-limiter branch + enricher: enricher, + virtualMachineV2Store: virtualMachineV2Store, } - vmID := "vm-1" + vmID := "vm-v2-upsert-fail" msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) - // Build a context with the mocked connection that has SensorACKSupport - ctxWithConn := connection.WithConnection(context.Background(), mockConn) - - // Run the pipeline - it should short-circuit due to nil rateLimiter, - // emit a NACK, and not call any datastore/enricher methods. - err := pipeline.Run(ctxWithConn, testClusterID, msg, injector) - assert.NoError(t, err, "pipeline Run should not error when rateLimiter is nil") - - // Verify exactly one NACK was sent - acks := injector.getSentACKs() - require.Len(t, acks, 1, "expected exactly one ACK/NACK to be sent") - - ack := acks[0] - assert.Equal(t, central.SensorACK_NACK, ack.GetAction(), "expected NACK action") - assert.Equal(t, central.SensorACK_VM_INDEX_REPORT, ack.GetMessageType(), "expected VM_INDEX_REPORT message type") - assert.Equal(t, vmID, ack.GetResourceId(), "expected resource ID to match VM ID") - assert.Equal(t, "rate limiter not configured", ack.GetReason(), "expected reason to indicate nil rate limiter") -} - -// recordingInjector is a test double that records all SensorACK messages sent via InjectMessage. -var _ common.MessageInjector = (*recordingInjector)(nil) - -type recordingInjector struct { - lock sync.Mutex - messages []*central.SensorACK -} + enricher.EXPECT(). + EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). + DoAndReturn(func(vm *storage.VirtualMachine, _ interface{}) error { + vm.Scan = &storage.VirtualMachineScan{ + Components: []*storage.EmbeddedVirtualMachineScanComponent{ + {Name: "test-pkg", Version: "1.0"}, + }, + } + return nil + }) + virtualMachineV2Store.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(nil, false, nil) + virtualMachineV2Store.EXPECT(). + EnsureVirtualMachineExists(gomock.Any(), vmID, testClusterID). + Return(nil) + virtualMachineV2Store.EXPECT(). + UpsertScan(gomock.Any(), vmID, gomock.Any()). + Return(errors.New("upsert scan failed")) -func (r *recordingInjector) InjectMessage(_ concurrency.Waitable, msg *central.MsgToSensor) error { - r.lock.Lock() - defer r.lock.Unlock() - if ack := msg.GetSensorAck(); ack != nil { - r.messages = append(r.messages, ack.CloneVT()) + injector := &mockInjector{ + capabilities: map[centralsensor.SensorCapability]bool{ + centralsensor.SensorACKSupport: true, + }, } - return nil -} -func (r *recordingInjector) InjectMessageIntoQueue(_ *central.MsgFromSensor) {} + err := pipeline.Run(ctx, testClusterID, msg, injector) + assert.Error(t, err) + assert.Contains(t, err.Error(), "upsert scan failed") -func (r *recordingInjector) getSentACKs() []*central.SensorACK { - r.lock.Lock() - defer r.lock.Unlock() - copied := make([]*central.SensorACK, 0, len(r.messages)) - copied = append(copied, r.messages...) - return copied + assert.Len(t, injector.messages, 1) + ack := injector.messages[0].GetSensorAck() + assert.NotNil(t, ack) + assert.Equal(t, central.SensorACK_NACK, ack.GetAction()) + assert.Equal(t, centralsensor.SensorACKReasonStorageFailed, ack.GetReason()) } -// TestOnFinishPropagatesClusterDisconnect verifies that OnFinish propagates the cluster ID -// to the rate limiter's OnClientDisconnect method. -func TestOnFinishPropagatesClusterDisconnect(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - vmDatastore := vmDatastoreMocks.NewMockDataStore(ctrl) - enricher := vmEnricherMocks.NewMockVirtualMachineEnricher(ctrl) - - // Use a fake limiter so we can observe calls to OnClientDisconnect. - fakeLimiter := &fakeRateLimiter{} - - p := &pipelineImpl{ - vmDatastore: vmDatastore, - enricher: enricher, - rateLimiter: fakeLimiter, +func TestLookupGuestOS(t *testing.T) { + tests := map[string]struct { + v2Enabled bool + v1VM *storage.VirtualMachine + v1Found bool + v1Err error + v2VM *storage.VirtualMachineV2 + v2Found bool + v2Err error + wantOS string + }{ + "v1 guest OS populates scan": { + v1VM: &storage.VirtualMachine{ + Facts: map[string]string{pkgVM.GuestOSKey: "Red Hat Enterprise Linux 9"}, + }, + v1Found: true, + wantOS: "Red Hat Enterprise Linux 9", + }, + "v2 guest OS populates scan": { + v2Enabled: true, + v2VM: &storage.VirtualMachineV2{ + GuestOs: "Red Hat Enterprise Linux 9", + Facts: map[string]string{pkgVM.GuestOSKey: "Red Hat Enterprise Linux 9"}, + }, + v2Found: true, + wantOS: "Red Hat Enterprise Linux 9", + }, + "v2 stamps scan from informer fact not display column": { + v2Enabled: true, + v2VM: &storage.VirtualMachineV2{ + GuestOs: "Red Hat Enterprise Linux 9.2", + Facts: map[string]string{pkgVM.GuestOSKey: "Red Hat Enterprise Linux"}, + }, + v2Found: true, + wantOS: "Red Hat Enterprise Linux", + }, + "v1 VM not found leaves scan OS empty": { + v1Found: false, + wantOS: "", + }, + "v2 VM not found leaves scan OS empty": { + v2Enabled: true, + v2Found: false, + wantOS: "", + }, + "v1 unknown guest OS leaves scan OS empty": { + v1VM: &storage.VirtualMachine{ + Facts: map[string]string{pkgVM.GuestOSKey: pkgVM.UnknownGuestOS}, + }, + v1Found: true, + wantOS: "", + }, + "v2 unknown guest OS leaves scan OS empty": { + v2Enabled: true, + v2VM: &storage.VirtualMachineV2{ + GuestOs: pkgVM.UnknownGuestOS, + Facts: map[string]string{pkgVM.GuestOSKey: pkgVM.UnknownGuestOS}, + }, + v2Found: true, + wantOS: "", + }, + "v1 store error leaves scan OS empty": { + v1Err: errors.New("db error"), + wantOS: "", + }, + "v2 store error leaves scan OS empty": { + v2Enabled: true, + v2Err: errors.New("db error"), + wantOS: "", + }, } - const clusterID = "cluster-1" - - p.OnFinish(clusterID) - - assert.Equal(t, clusterID, fakeLimiter.lastDisconnectedClientID, "OnFinish should propagate cluster disconnect to the rate limiter") -} - -// TestOnFinishWithNilRateLimiter verifies that OnFinish doesn't panic when rateLimiter is nil. -func TestOnFinishWithNilRateLimiter(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + t.Setenv(features.VirtualMachines.EnvVar(), "true") + if tt.v2Enabled { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + } else { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + } - vmDatastore := vmDatastoreMocks.NewMockDataStore(ctrl) - enricher := vmEnricherMocks.NewMockVirtualMachineEnricher(ctrl) + ctrl := gomock.NewController(t) - p := &pipelineImpl{ - vmDatastore: vmDatastore, - enricher: enricher, - rateLimiter: nil, - } + vmID := "vm-guest-os" + msg := createVMIndexMessage(vmID, central.ResourceAction_SYNC_RESOURCE) - // Should not panic - assert.NotPanics(t, func() { - p.OnFinish("cluster-1") - }) -} + v1Store := virtualMachineDSMocks.NewMockDataStore(ctrl) + v2Store := virtualMachineV2DSMocks.NewMockDataStore(ctrl) + enricher := vmEnricherMocks.NewMockVirtualMachineEnricher(ctrl) -// fakeRateLimiter is a test double that records the last client ID passed to OnClientDisconnect. -// It satisfies the interface used by pipelineImpl.rateLimiter. -type fakeRateLimiter struct { - lastDisconnectedClientID string -} + enricher.EXPECT(). + EnrichVirtualMachineWithVulnerabilities(gomock.Any(), gomock.Any()). + DoAndReturn(func(vm *storage.VirtualMachine, _ interface{}) error { + vm.Scan = &storage.VirtualMachineScan{} + return nil + }) + + if tt.v2Enabled { + v2Store.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(tt.v2VM, tt.v2Found, tt.v2Err) + v2Store.EXPECT(). + EnsureVirtualMachineExists(gomock.Any(), vmID, testClusterID). + Return(nil) + v2Store.EXPECT(). + UpsertScan(gomock.Any(), vmID, gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, parts common.VMScanParts) error { + assert.Equal(t, tt.wantOS, parts.Scan.GetScanOs()) + return nil + }) + } else { + v1Store.EXPECT(). + GetVirtualMachine(gomock.Any(), vmID). + Return(tt.v1VM, tt.v1Found, tt.v1Err) + v1Store.EXPECT(). + UpdateVirtualMachineScan(gomock.Any(), vmID, gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, scan *storage.VirtualMachineScan) error { + assert.Equal(t, tt.wantOS, scan.GetOperatingSystem()) + return nil + }) + } -func (f *fakeRateLimiter) TryConsume(_ string) (bool, string) { - return true, "" -} + p := &pipelineImpl{ + virtualMachineStore: v1Store, + virtualMachineV2Store: v2Store, + enricher: enricher, + } -func (f *fakeRateLimiter) OnClientDisconnect(clientID string) { - f.lastDisconnectedClientID = clientID + err := p.Run(ctx, testClusterID, msg, nil) + assert.NoError(t, err) + }) + } } diff --git a/central/sensor/service/pipeline/virtualmachines/pipeline.go b/central/sensor/service/pipeline/virtualmachines/pipeline.go index 83686a8ebadf2..3418898057a11 100644 --- a/central/sensor/service/pipeline/virtualmachines/pipeline.go +++ b/central/sensor/service/pipeline/virtualmachines/pipeline.go @@ -3,6 +3,7 @@ package virtualmachines import ( "context" "fmt" + "math" "github.com/pkg/errors" clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" @@ -12,9 +13,12 @@ import ( "github.com/stackrox/rox/central/sensor/service/pipeline" "github.com/stackrox/rox/central/sensor/service/pipeline/reconciliation" virtualMachineDataStore "github.com/stackrox/rox/central/virtualmachine/datastore" + virtualMachineV2DataStore "github.com/stackrox/rox/central/virtualmachine/v2/datastore" + v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/internalapi/central" virtualMachineV1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" "github.com/stackrox/rox/pkg/centralsensor" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/metrics" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/set" @@ -25,27 +29,35 @@ var ( ) func GetPipeline() pipeline.Fragment { - return newPipeline(clusterDataStore.Singleton(), virtualMachineDataStore.Singleton()) + return newPipeline(clusterDataStore.Singleton(), virtualMachineDataStore.Singleton(), virtualMachineV2DataStore.Singleton()) } func newPipeline( clusterStore clusterDataStore.DataStore, virtualMachineStore virtualMachineDataStore.DataStore, + virtualMachineV2Store virtualMachineV2DataStore.DataStore, ) pipeline.Fragment { return &pipelineImpl{ - clusterStore: clusterStore, - virtualMachineStore: virtualMachineStore, + clusterStore: clusterStore, + virtualMachineStore: virtualMachineStore, + virtualMachineV2Store: virtualMachineV2Store, } } type pipelineImpl struct { - clusterStore clusterDataStore.DataStore - virtualMachineStore virtualMachineDataStore.DataStore + clusterStore clusterDataStore.DataStore + virtualMachineStore virtualMachineDataStore.DataStore + virtualMachineV2Store virtualMachineV2DataStore.DataStore } func (p *pipelineImpl) OnFinish(_ string) {} +// Capabilities omits VirtualMachinesSupported when ROX_VIRTUAL_MACHINES is +// off so Sensor treats this Central like a pre-VM build. func (p *pipelineImpl) Capabilities() []centralsensor.CentralCapability { + if !features.VirtualMachines.Enabled() { + return nil + } return []centralsensor.CentralCapability{centralsensor.VirtualMachinesSupported} } @@ -54,15 +66,26 @@ func (p *pipelineImpl) Match(msg *central.MsgFromSensor) bool { } func (p *pipelineImpl) Reconcile(ctx context.Context, clusterID string, storeMap *reconciliation.StoreMap) error { - virtualMachines, err := p.virtualMachineStore.SearchRawVirtualMachines(ctx, search.EmptyQuery()) + if !features.VirtualMachines.Enabled() { + // An empty Sensor snapshot would otherwise delete every VM row for the cluster. + return nil + } + if features.VirtualMachinesEnhancedDataModel.Enabled() { + return p.reconcileV2(ctx, clusterID, storeMap) + } + return p.reconcileV1(ctx, clusterID, storeMap) +} + +func (p *pipelineImpl) reconcileV1(ctx context.Context, clusterID string, storeMap *reconciliation.StoreMap) error { + query := search.NewQueryBuilder().AddExactMatches(search.ClusterID, clusterID).ProtoQuery() + query.Pagination = &v1.QueryPagination{Limit: math.MaxInt32} + virtualMachines, err := p.virtualMachineStore.SearchRawVirtualMachines(ctx, query) if err != nil { return errors.Wrap(err, "retrieving virtual machines for reconciliation") } clusterVMIDs := set.NewStringSet() for _, vm := range virtualMachines { - if vm.GetClusterId() == clusterID { - clusterVMIDs.Add(vm.GetId()) - } + clusterVMIDs.Add(vm.GetId()) } store := storeMap.Get((*central.SensorEvent_VirtualMachine)(nil)) @@ -71,26 +94,63 @@ func (p *pipelineImpl) Reconcile(ctx context.Context, clusterID string, storeMap }) } +func (p *pipelineImpl) reconcileV2(ctx context.Context, clusterID string, storeMap *reconciliation.StoreMap) error { + query := search.NewQueryBuilder().AddExactMatches(search.ClusterID, clusterID).ProtoQuery() + query.Pagination = &v1.QueryPagination{Limit: math.MaxInt32} + results, err := p.virtualMachineV2Store.Search(ctx, query) + if err != nil { + return errors.Wrap(err, "retrieving v2 virtual machines for reconciliation") + } + + store := storeMap.Get((*central.SensorEvent_VirtualMachine)(nil)) + return reconciliation.Perform(store, search.ResultsToIDSet(results), "virtualmachines", func(id string) error { + return p.virtualMachineV2Store.DeleteVirtualMachines(ctx, id) + }) +} + func (p *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.MsgFromSensor, _ common.MessageInjector) error { defer countMetrics.IncrementResourceProcessedCounter(pipeline.ActionToOperation(msg.GetEvent().GetAction()), metrics.VirtualMachine) + if !features.VirtualMachines.Enabled() { + // Returning nil avoids Sensor retries; the rows stay untouched. + return nil + } + event := msg.GetEvent() virtualMachine := event.GetVirtualMachine() if virtualMachine == nil { return errors.Errorf("unexpected resource type %T for virtual machine", event.GetResource()) } - switch event.GetAction() { + if features.VirtualMachinesEnhancedDataModel.Enabled() { + return p.runV2(ctx, clusterID, event.GetAction(), virtualMachine) + } + return p.runV1(ctx, clusterID, event.GetAction(), virtualMachine) +} + +func (p *pipelineImpl) runV1(ctx context.Context, clusterID string, action central.ResourceAction, vm *virtualMachineV1.VirtualMachine) error { + switch action { + case central.ResourceAction_REMOVE_RESOURCE: + return p.virtualMachineStore.DeleteVirtualMachines(ctx, vm.GetId()) + case central.ResourceAction_CREATE_RESOURCE, central.ResourceAction_UPDATE_RESOURCE, central.ResourceAction_SYNC_RESOURCE: + return p.runUpsertPipelineV1(ctx, clusterID, vm) + default: + return fmt.Errorf("event action '%s' for virtual machine does not exist", action) + } +} + +func (p *pipelineImpl) runV2(ctx context.Context, clusterID string, action central.ResourceAction, vm *virtualMachineV1.VirtualMachine) error { + switch action { case central.ResourceAction_REMOVE_RESOURCE: - return p.virtualMachineStore.DeleteVirtualMachines(ctx, virtualMachine.GetId()) + return p.virtualMachineV2Store.DeleteVirtualMachines(ctx, vm.GetId()) case central.ResourceAction_CREATE_RESOURCE, central.ResourceAction_UPDATE_RESOURCE, central.ResourceAction_SYNC_RESOURCE: - return p.runUpsertPipeline(ctx, clusterID, virtualMachine) + return p.runUpsertPipelineV2(ctx, clusterID, vm) default: - return fmt.Errorf("event action '%s' for virtual machine does not exist", event.GetAction()) + return fmt.Errorf("event action '%s' for virtual machine does not exist", action) } } -func (p *pipelineImpl) runUpsertPipeline( +func (p *pipelineImpl) runUpsertPipelineV1( ctx context.Context, clusterID string, vm *virtualMachineV1.VirtualMachine, @@ -104,3 +164,18 @@ func (p *pipelineImpl) runUpsertPipeline( return p.virtualMachineStore.UpsertVirtualMachine(ctx, virtualMachineToStore) } + +func (p *pipelineImpl) runUpsertPipelineV2( + ctx context.Context, + clusterID string, + vm *virtualMachineV1.VirtualMachine, +) error { + vmToStore := internaltostorage.VirtualMachineV2(vm) + + clusterName, ok, err := p.clusterStore.GetClusterName(ctx, clusterID) + if err == nil && ok { + vmToStore.ClusterName = clusterName + } + + return p.virtualMachineV2Store.UpsertVirtualMachine(ctx, vmToStore) +} diff --git a/central/sensor/service/pipeline/virtualmachines/pipeline_test.go b/central/sensor/service/pipeline/virtualmachines/pipeline_test.go index 6bbef00e2a3c9..98edf66f1dba2 100644 --- a/central/sensor/service/pipeline/virtualmachines/pipeline_test.go +++ b/central/sensor/service/pipeline/virtualmachines/pipeline_test.go @@ -1,19 +1,23 @@ package virtualmachines import ( + "strconv" "testing" clusterDSMocks "github.com/stackrox/rox/central/cluster/datastore/mocks" "github.com/stackrox/rox/central/convert/internaltostorage" "github.com/stackrox/rox/central/sensor/service/pipeline/reconciliation" virtualMachineDSMocks "github.com/stackrox/rox/central/virtualmachine/datastore/mocks" + virtualMachineV2DSMocks "github.com/stackrox/rox/central/virtualmachine/v2/datastore/mocks" "github.com/stackrox/rox/generated/internalapi/central" virtualMachineV1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" "github.com/stackrox/rox/pkg/protomock" + "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/uuid" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" @@ -21,11 +25,25 @@ import ( func TestCapabilities(t *testing.T) { pipeline := &pipelineImpl{} - assert.ElementsMatch( - t, - []centralsensor.CentralCapability{centralsensor.VirtualMachinesSupported}, - pipeline.Capabilities(), - ) + cases := map[string]struct { + enabled bool + want []centralsensor.CentralCapability + }{ + "flag on advertises VirtualMachinesSupported": { + enabled: true, + want: []centralsensor.CentralCapability{centralsensor.VirtualMachinesSupported}, + }, + "flag off advertises nothing": { + enabled: false, + want: nil, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Setenv(features.VirtualMachines.EnvVar(), strconv.FormatBool(tc.enabled)) + assert.Equal(t, tc.want, pipeline.Capabilities()) + }) + } } func TestMatch(t *testing.T) { @@ -86,6 +104,7 @@ func TestMatch(t *testing.T) { } func TestPipelineRun(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") testClusterID := fixtureconsts.Cluster1 type mocks struct { clusters *clusterDSMocks.MockDataStore @@ -164,7 +183,7 @@ func TestPipelineRun(t *testing.T) { clusters: clusterDSMocks.NewMockDataStore(mockCtrl), virtualMachines: virtualMachineDSMocks.NewMockDataStore(mockCtrl), } - pipeline := newPipeline(testMocks.clusters, testMocks.virtualMachines) + pipeline := newPipeline(testMocks.clusters, testMocks.virtualMachines, nil) if tt.setupMocks != nil { tt.setupMocks(testMocks) } @@ -179,8 +198,8 @@ func TestPipelineRun(t *testing.T) { } func TestPipelineReconcile(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") testClusterID := fixtureconsts.Cluster1 - otherClusterID := fixtureconsts.Cluster2 tests := []struct { name string setupStoreMap func(*reconciliation.StoreMap) @@ -208,21 +227,18 @@ func TestPipelineReconcile(t *testing.T) { m.Add((*central.SensorEvent_VirtualMachine)(nil), "existing-vm") }, setupMock: func(m *virtualMachineDSMocks.MockDataStore) { + // Query is now cluster-scoped, so only VMs from testClusterID are returned. m.EXPECT().SearchRawVirtualMachines(gomock.Any(), gomock.Any()). Return([]*storage.VirtualMachine{ { Id: "existing-vm", ClusterId: testClusterID, }, - { - Id: "existing-vm-in-other-cluster", - ClusterId: otherClusterID, - }, }, nil) }, }, { - name: "reconciliation does not remove virtual machines from other clusters", + name: "reconciliation removes stale virtual machines", setupStoreMap: func(m *reconciliation.StoreMap) { m.Add((*central.SensorEvent_VirtualMachine)(nil), "existing-vm") }, @@ -273,7 +289,242 @@ func TestPipelineReconcile(t *testing.T) { tt.setupStoreMap(storeMap) } - pipeline := newPipeline(nil, mockVMStore) + pipeline := newPipeline(nil, mockVMStore, nil) + err := pipeline.Reconcile(it.Context(), testClusterID, storeMap) + if !tt.expectsError { + assert.NoError(it, err) + } else { + assert.Error(it, err) + } + }) + } +} + +func TestPipelineRun_FeatureDisabledDoesNotWrite(t *testing.T) { + t.Setenv(features.VirtualMachines.EnvVar(), "false") + testClusterID := fixtureconsts.Cluster1 + upsertTestVM := &virtualMachineV1.VirtualMachine{ + Id: uuid.NewTestUUID(1).String(), + Namespace: "test-namespace", + Name: "test-virtual-machine", + ClusterId: testClusterID, + } + msgs := map[string]*central.MsgFromSensor{ + "create": getVirtualMachineAdditionMessage(upsertTestVM), + "remove": getVirtualMachineRemovalMessage("removed_vm_id"), + } + models := map[string]bool{ + "v1": false, + "v2": true, + } + for model, enhanced := range models { + t.Run(model, func(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), strconv.FormatBool(enhanced)) + for name, msg := range msgs { + t.Run(name, func(t *testing.T) { + mockCtrl := gomock.NewController(t) + pipeline := newPipeline( + clusterDSMocks.NewMockDataStore(mockCtrl), + virtualMachineDSMocks.NewMockDataStore(mockCtrl), + virtualMachineV2DSMocks.NewMockDataStore(mockCtrl), + ) + assert.NoError(t, pipeline.Run(t.Context(), testClusterID, msg, nil)) + }) + } + }) + } +} + +func TestPipelineReconcile_FeatureDisabledDoesNotDelete(t *testing.T) { + t.Setenv(features.VirtualMachines.EnvVar(), "false") + testClusterID := fixtureconsts.Cluster1 + cases := map[string]bool{ + "v1": false, + "v2": true, + } + for name, enhanced := range cases { + t.Run(name, func(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), strconv.FormatBool(enhanced)) + mockCtrl := gomock.NewController(t) + pipeline := newPipeline( + nil, + virtualMachineDSMocks.NewMockDataStore(mockCtrl), + virtualMachineV2DSMocks.NewMockDataStore(mockCtrl), + ) + assert.NoError(t, pipeline.Reconcile(t.Context(), testClusterID, reconciliation.NewStoreMap())) + }) + } +} + +func TestPipelineRunV2(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + testClusterID := fixtureconsts.Cluster1 + type mocks struct { + clusters *clusterDSMocks.MockDataStore + virtualMachinesV2 *virtualMachineV2DSMocks.MockDataStore + } + var upsertTestVM = &virtualMachineV1.VirtualMachine{ + Id: uuid.NewTestUUID(1).String(), + Namespace: "test-namespace", + Name: "test-virtual-machine", + ClusterId: testClusterID, + VsockCid: 0, + State: virtualMachineV1.VirtualMachine_STOPPED, + } + tests := []struct { + name string + setupMocks func(*mocks) + message *central.MsgFromSensor + expectsError bool + expectedErrorMsg string + }{ + { + name: "nil input", + expectsError: true, + expectedErrorMsg: "unexpected resource type for virtual machine", + }, + { + name: "V2 removal expects call to v2 datastore delete", + setupMocks: func(testMocks *mocks) { + testMocks.virtualMachinesV2.EXPECT(). + DeleteVirtualMachines(gomock.Any(), "removed_vm_id"). + Return(nil) + }, + message: getVirtualMachineRemovalMessage("removed_vm_id"), + }, + { + name: "V2 addition upserts with cluster name", + setupMocks: func(testMock *mocks) { + storedVM := internaltostorage.VirtualMachineV2(upsertTestVM) + storedVM.ClusterName = "test-cluster" + testMock.clusters.EXPECT(). + GetClusterName(gomock.Any(), testClusterID). + Return("test-cluster", true, nil) + testMock.virtualMachinesV2.EXPECT(). + UpsertVirtualMachine(gomock.Any(), protomock.GoMockMatcherEqualMessage(storedVM)). + Return(nil) + }, + message: getVirtualMachineAdditionMessage(upsertTestVM), + }, + { + name: "V2 addition upserts despite cluster name lookup failure", + setupMocks: func(testMock *mocks) { + storedVM := internaltostorage.VirtualMachineV2(upsertTestVM) + storedVM.ClusterName = "" + testMock.clusters.EXPECT(). + GetClusterName(gomock.Any(), testClusterID). + Return("", false, nil) + testMock.virtualMachinesV2.EXPECT(). + UpsertVirtualMachine(gomock.Any(), protomock.GoMockMatcherEqualMessage(storedVM)). + Return(nil) + }, + message: getVirtualMachineAdditionMessage(upsertTestVM), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(it *testing.T) { + mockCtrl := gomock.NewController(it) + defer mockCtrl.Finish() + testMocks := &mocks{ + clusters: clusterDSMocks.NewMockDataStore(mockCtrl), + virtualMachinesV2: virtualMachineV2DSMocks.NewMockDataStore(mockCtrl), + } + pipeline := newPipeline(testMocks.clusters, nil, testMocks.virtualMachinesV2) + if tt.setupMocks != nil { + tt.setupMocks(testMocks) + } + err := pipeline.Run(it.Context(), testClusterID, tt.message, nil) + if tt.expectsError { + assert.ErrorContains(it, err, tt.expectedErrorMsg) + } else { + assert.NoError(it, err) + } + }) + } +} + +func TestPipelineReconcileV2(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + testClusterID := fixtureconsts.Cluster1 + tests := []struct { + name string + setupStoreMap func(*reconciliation.StoreMap) + setupMock func(*virtualMachineV2DSMocks.MockDataStore) + expectsError bool + }{ + { + name: "v2 reconciliation has nothing to remove", + setupStoreMap: func(m *reconciliation.StoreMap) { + m.Add((*central.SensorEvent_VirtualMachine)(nil), "existing-vm") + }, + setupMock: func(m *virtualMachineV2DSMocks.MockDataStore) { + m.EXPECT().Search(gomock.Any(), gomock.Any()). + Return([]search.Result{ + {ID: "existing-vm"}, + }, nil) + }, + }, + { + name: "v2 reconciliation does not remove virtual machines from other clusters", + setupStoreMap: func(m *reconciliation.StoreMap) { + m.Add((*central.SensorEvent_VirtualMachine)(nil), "existing-vm") + }, + setupMock: func(m *virtualMachineV2DSMocks.MockDataStore) { + // Query is now cluster-scoped, so only VMs from testClusterID are returned. + m.EXPECT().Search(gomock.Any(), gomock.Any()). + Return([]search.Result{ + {ID: "existing-vm"}, + }, nil) + }, + }, + { + name: "v2 reconciliation removes stale virtual machines", + setupStoreMap: func(m *reconciliation.StoreMap) { + m.Add((*central.SensorEvent_VirtualMachine)(nil), "existing-vm") + }, + setupMock: func(m *virtualMachineV2DSMocks.MockDataStore) { + m.EXPECT(). + Search(gomock.Any(), gomock.Any()). + Return([]search.Result{ + {ID: "existing-vm"}, + {ID: "vm-to-remove-from-cluster"}, + }, nil) + m.EXPECT(). + DeleteVirtualMachines(gomock.Any(), "vm-to-remove-from-cluster"). + Return(nil) + }, + }, + { + name: "v2 reconciliation fails on virtual machine lookup error", + setupStoreMap: func(m *reconciliation.StoreMap) { + m.Add((*central.SensorEvent_VirtualMachine)(nil), "existing-vm") + }, + setupMock: func(m *virtualMachineV2DSMocks.MockDataStore) { + m.EXPECT(). + Search(gomock.Any(), gomock.Any()). + Return(nil, errox.InvalidArgs) + }, + expectsError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(it *testing.T) { + mockCtrl := gomock.NewController(it) + defer mockCtrl.Finish() + + mockVMV2Store := virtualMachineV2DSMocks.NewMockDataStore(mockCtrl) + if tt.setupMock != nil { + tt.setupMock(mockVMV2Store) + } + + storeMap := reconciliation.NewStoreMap() + if tt.setupStoreMap != nil { + tt.setupStoreMap(storeMap) + } + + pipeline := newPipeline(nil, nil, mockVMV2Store) err := pipeline.Reconcile(it.Context(), testClusterID, storeMap) if !tt.expectsError { assert.NoError(it, err) diff --git a/central/sensor/service/service_impl.go b/central/sensor/service/service_impl.go index 25ae9fcdbb7bc..c1b21e2e5dcef 100644 --- a/central/sensor/service/service_impl.go +++ b/central/sensor/service/service_impl.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/go-multierror" "github.com/pkg/errors" clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" + clusterInitStore "github.com/stackrox/rox/central/clusterinit/store" installationStore "github.com/stackrox/rox/central/installation/store" "github.com/stackrox/rox/central/metrics/telemetry" "github.com/stackrox/rox/central/securedclustercertgen" @@ -15,6 +16,7 @@ import ( "github.com/stackrox/rox/central/sensor/service/pipeline" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/centralproxy" "github.com/stackrox/rox/pkg/centralsensor" "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/errox" @@ -45,19 +47,27 @@ var ( type serviceImpl struct { central.UnimplementedSensorServiceServer - manager connection.Manager - pf pipeline.Factory - clusters clusterDataStore.DataStore - installation installationStore.Store + manager connection.Manager + pf pipeline.Factory + clusters clusterDataStore.DataStore + installation installationStore.Store + clusterInitStore clusterInitStore.Store } // New creates a new Service using the given manager. -func New(manager connection.Manager, pf pipeline.Factory, clusters clusterDataStore.DataStore, installation installationStore.Store) Service { +func New( + manager connection.Manager, + pf pipeline.Factory, + clusters clusterDataStore.DataStore, + installation installationStore.Store, + clusterInitStore clusterInitStore.Store, +) Service { return &serviceImpl{ - manager: manager, - pf: pf, - clusters: clusters, - installation: installation, + manager: manager, + pf: pf, + clusters: clusters, + installation: installation, + clusterInitStore: clusterInitStore, } } @@ -128,6 +138,7 @@ func (s *serviceImpl) Communicate(server central.SensorService_CommunicateServer capabilities = append(capabilities, centralsensor.SendDeduperStateOnReconnect) if features.ComplianceEnhancements.Enabled() { capabilities = append(capabilities, centralsensor.ComplianceV2Integrations) + capabilities = append(capabilities, centralsensor.ComplianceV2TailoredProfiles) } if features.ComplianceRemediationV2.Enabled() { capabilities = append(capabilities, centralsensor.ComplianceV2Remediations) @@ -141,16 +152,24 @@ func (s *serviceImpl) Communicate(server central.SensorService_CommunicateServer if features.FlattenImageData.Enabled() { capabilities = append(capabilities, centralsensor.FlattenImageData) } + if features.InitContainerSupport.Enabled() { + capabilities = append(capabilities, centralsensor.InitContainerSupport) + } + if features.OCPConsoleIntegration.Enabled() { + capabilities = append(capabilities, centralsensor.InternalTokenAPISupported.String()) + capabilities = append(capabilities, centralsensor.CentralProxyPathFiltering.String()) + } preferences := s.manager.GetConnectionPreference(clusterID) // Let's be polite and respond with a greeting from our side. centralHello := ¢ral.CentralHello{ - ClusterId: clusterID, - ManagedCentral: env.ManagedCentral.BooleanSetting(), - CentralId: installInfo.GetId(), - Capabilities: capabilities, - SendDeduperState: preferences.SendDeduperState, + ClusterId: clusterID, + ManagedCentral: env.ManagedCentral.BooleanSetting(), + CentralId: installInfo.GetId(), + Capabilities: capabilities, + SendDeduperState: preferences.SendDeduperState, + AllowedProxyPaths: centralproxy.AllowedProxyPaths.AsSlice(), } if err := safe.RunE(func() error { @@ -166,7 +185,7 @@ func (s *serviceImpl) Communicate(server central.SensorService_CommunicateServer } return nil }); err != nil { - log.Errorf("Could not include certificate bundle in sensor hello message: %s", err) + log.Errorf("Could not include certificate bundle in sensor hello message: %s.", err) } if err := server.Send(¢ral.MsgToSensor{Msg: ¢ral.MsgToSensor_Hello{Hello: centralHello}}); err != nil { @@ -176,20 +195,45 @@ func (s *serviceImpl) Communicate(server central.SensorService_CommunicateServer if svcType == storage.ServiceType_REGISTRANT_SERVICE { // Terminate connection which uses a CRS certificate at this point. + log.Infof("Terminating initial CRS flow from cluster %s (%s).", cluster.GetName(), cluster.GetId()) return nil } + // At this point sensor is connecting with non-CRS service certificates. This could mean either init bundle certificates + // or freshly issued per-cluster service certificates. + + if cluster.GetHealthStatus().GetLastContact() == nil && cluster.GetInitBundleId() != "" { + // The call to MarkClusterRegistrationComplete also updates the revocation state of the CRS used for this + // cluster, if needed (no-op in init bundle case). + log.Infof("First connection from cluster %s (%s) using a sensor service certificate.", cluster.GetName(), cluster.GetId()) + if err := s.clusterInitStore.MarkClusterRegistrationComplete(clusterDSSAC, cluster.GetInitBundleId(), cluster.GetName()); err != nil { + return errors.Wrapf(err, "updating completed-registrations counter for cluster registration secret %q", cluster.GetInitBundleId()) + } + } + if clusterInitArtifactId := cluster.GetInitBundleId(); clusterInitArtifactId != "" && svc.GetInitBundleId() == "" { + // Sensor has connected with a non-init certificate. + // At this point we can clear the cluster's InitBundleId, irregardless of which init artifact type has been used. + // For CRS, this is the point after which we don't need it anymore for any sort of bookkepping. + // For init bundles, this was previously done in `LookupOrCreateClusterFromConfig()`, now it is being done here, + // slightly later in the flow. + cluster.InitBundleId = "" + if err := s.clusters.UpdateCluster(clusterDSSAC, cluster); err != nil { + return errors.Wrapf(err, "clearing init artifact ID of cluster %s", cluster.GetName()) + } + log.Infof("Cleared init artifact ID (%s) of newly created cluster %s.", clusterInitArtifactId, cluster.GetName()) + } + if expiryStatus, err := getCertExpiryStatus(identity); err != nil { notBefore, notAfter := identity.ValidityPeriod() - log.Warnf("Failed to convert expiry status of sensor cert (NotBefore: %v, Expiry: %v) from cluster %s to proto: %v", + log.Warnf("Failed to convert expiry status of sensor cert (NotBefore: %v, Expiry: %v) from cluster %s to proto: %v.", notBefore, notAfter, cluster.GetId(), err) } else if expiryStatus != nil { if err := s.clusters.UpdateClusterCertExpiryStatus(clusterDSSAC, cluster.GetId(), expiryStatus); err != nil { - log.Warnf("Failed to update cluster expiry status for cluster %s: %v", cluster.GetId(), err) + log.Warnf("Failed to update cluster expiry status for cluster %s: %v.", cluster.GetId(), err) } } - log.Infof("Cluster %s (%s) has successfully connected to Central", cluster.GetName(), cluster.GetId()) + log.Infof("Cluster %s (%s) has successfully connected to Central.", cluster.GetName(), cluster.GetId()) return s.manager.HandleConnection(server.Context(), sensorHello, cluster, eventPipeline, server) } diff --git a/central/sensor/service/service_impl_test.go b/central/sensor/service/service_impl_test.go index 7f1c5baf7feaf..2d5c3ad4d14d5 100644 --- a/central/sensor/service/service_impl_test.go +++ b/central/sensor/service/service_impl_test.go @@ -1,15 +1,22 @@ +//go:build sql_integration + package service import ( "context" + "encoding/json" + "fmt" "io" + "os" "testing" "time" - clusterMock "github.com/stackrox/rox/central/cluster/datastore/mocks" + clusterDataStore "github.com/stackrox/rox/central/cluster/datastore" + clusterInitStore "github.com/stackrox/rox/central/clusterinit/store" installationMock "github.com/stackrox/rox/central/installation/store/mocks" "github.com/stackrox/rox/central/sensor/service/connection" connectionMock "github.com/stackrox/rox/central/sensor/service/connection/mocks" + "github.com/stackrox/rox/central/sensor/service/pipeline" pipelineMock "github.com/stackrox/rox/central/sensor/service/pipeline/mocks" "github.com/stackrox/rox/generated/internalapi/central" "github.com/stackrox/rox/generated/storage" @@ -18,12 +25,15 @@ import ( authnMock "github.com/stackrox/rox/pkg/grpc/authn/mocks" "github.com/stackrox/rox/pkg/grpc/authn/service" "github.com/stackrox/rox/pkg/mtls" - "github.com/stackrox/rox/pkg/mtls/testutils" + mtlsTestutils "github.com/stackrox/rox/pkg/mtls/testutils" + "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/sac" + pkgSearch "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/utils" "github.com/stackrox/rox/pkg/uuid" + "github.com/stackrox/rox/pkg/version/testutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "go.uber.org/mock/gomock" @@ -118,78 +128,256 @@ func TestIsCARotationSupported(t *testing.T) { } } -// CRS Test Suite. - -type crsTestSuite struct { +// Sensor Service Test Suite (primarily, but not exclusively, for CRS handshake). +type sensorServiceTestSuite struct { suite.Suite - mockCtrl *gomock.Controller - context context.Context + mockCtrl *gomock.Controller + internalContext context.Context + db *pgtest.TestPostgres + clusterInitStore clusterInitStore.Store + clusterDataStore clusterDataStore.DataStore } -func (s *crsTestSuite) SetupTest() { +func (s *sensorServiceTestSuite) SetupSuite() { + imageFlavor := "rhacs" + utils.Must(os.Setenv("ROX_IMAGE_FLAVOR", imageFlavor)) + testutils.SetExampleVersion(s.T()) + s.internalContext = sac.WithAllAccess(context.Background()) + s.db = pgtest.ForT(s.T()) + var err error + s.clusterDataStore, err = clusterDataStore.GetTestPostgresDataStore(s.T(), s.db.DB) + s.Require().NoError(err, "failed to create cluster data store") + s.clusterInitStore = clusterDataStore.IntrospectClusterInitStore(s.T(), s.clusterDataStore) + s.mockCtrl = gomock.NewController(s.T()) - s.context = sac.WithAllAccess(context.Background()) - utils.Should(testutils.LoadTestMTLSCerts(s.T())) + utils.Should(mtlsTestutils.LoadTestMTLSCerts(s.T())) +} + +func (s *sensorServiceTestSuite) SetupTest() { + log.Infof("Running test: %s", s.T().Name()) } -func TestCrs(t *testing.T) { - suite.Run(t, new(crsTestSuite)) +func TestClusterRegistration(t *testing.T) { + suite.Run(t, new(sensorServiceTestSuite)) } -func (s *crsTestSuite) TestCrsCentralReturnsAllServiceCertificates() { - sensorService, mockServer := newSensorService(s.context, s.mockCtrl) +func newClusterNames() (string, string, string) { + baseUuid := uuid.NewV4().String() + base := baseUuid[:9] + return base + "a", base + "b", base + "c" + +} + +func (s *sensorServiceTestSuite) TestCrsCentralReturnsAllServiceCertificates() { + _, crsMeta := newCrsMeta(0) + sensorService := newSensorService(s, crsMeta) + clusterName, _, _ := newClusterNames() + mockServer := newMockServerForCrsHandshake(s, crsMeta, sensorDeploymentIdentificationA, clusterName) - // First-time CRS cluster registration. - mockServer.prepareNewHandshake(sensorHello) err := sensorService.Communicate(mockServer) s.NoError(err) - s.Len(mockServer.clustersRegistered, 1, "expected exactly one registered cluster") centralHello := retrieveCentralHello(s, mockServer) - assertCertificateBundleComplete(s, centralHello.GetCertBundle()) -} + _ = centralHello -func assertCertificateBundleComplete(s *crsTestSuite, certBundle map[string]string) { - s.Len(certBundle, 15, "expected 15 entries (1 CA cert, 7 service certs, 7 service keys) in bundle") + assertCertificateBundleComplete(s, centralHello.GetCertBundle()) } -func (s *crsTestSuite) TestCrsFlowCanBeRepeated() { - sensorService, mockServer := newSensorService(s.context, s.mockCtrl) +func (s *sensorServiceTestSuite) TestCrsFlowCanBeRepeated() { + _, crsMeta := newCrsMeta(0) + sensorService := newSensorService(s, crsMeta) + clusterName, _, _ := newClusterNames() // First-time CRS cluster registration. - mockServer.prepareNewHandshake(sensorHello) + mockServer := newMockServerForCrsHandshake(s, crsMeta, sensorDeploymentIdentificationA, clusterName) err := sensorService.Communicate(mockServer) s.NoError(err) - s.Len(mockServer.clustersRegistered, 1, "expected exactly one registered cluster") // Initiating the CRS flow a second time should still work. - mockServer.prepareNewHandshake(sensorHello) + mockServer = newMockServerForCrsHandshake(s, crsMeta, sensorDeploymentIdentificationA, clusterName) err = sensorService.Communicate(mockServer) s.NoError(err) - s.Len(mockServer.clustersRegistered, 1, "expected exactly one registered cluster") - // Verify that we again got all certificates we need. - centralHello := retrieveCentralHello(s, mockServer) - assertCertificateBundleComplete(s, centralHello.GetCertBundle()) } -func (s *crsTestSuite) TestCrsFlowFailsAfterLastContact() { - sensorService, mockServer := newSensorService(s.context, s.mockCtrl) +func (s *sensorServiceTestSuite) TestCrsFlowFailsAfterRegistrationComplete() { + _, crsMeta := newCrsMeta(0) + sensorService := newSensorService(s, crsMeta) + clusterName, _, _ := newClusterNames() // CRS cluster registration. - mockServer.prepareNewHandshake(sensorHello) + mockServer := newMockServerForCrsHandshake(s, crsMeta, sensorDeploymentIdentificationA, clusterName) err := sensorService.Communicate(mockServer) s.NoError(err) - // Simulate a connection with service certificates has occurred by updating - // the LastContact field of a cluster. - mockServer.clustersRegistered[0].HealthStatus = &storage.ClusterHealthStatus{ - LastContact: timestamppb.Now(), - } + // Regular connect. + mockServer = newMockServerForRegularConnect(s, sensorDeploymentIdentificationA, clusterName) + err = sensorService.Communicate(mockServer) + s.NoError(err) // Initiating the CRS should fail now. - mockServer.prepareNewHandshake(sensorHello) + mockServer = newMockServerForCrsHandshake(s, crsMeta, sensorDeploymentIdentificationA, clusterName) err = sensorService.Communicate(mockServer) + s.Error(err, "CRS flow succeeded even after regular connect.") s.ErrorContains(err, "forbidden to use a Cluster Registration Certificate for already-existing cluster") - s.Error(err, "CRS flow succeeded even after LastContact field was updated.") +} + +func lookupCrs(s *sensorServiceTestSuite, crsId string) *storage.InitBundleMeta { + crsMeta, err := s.clusterInitStore.Get(s.internalContext, crsId) + s.Require().NoError(err) + return crsMeta +} + +func (s *sensorServiceTestSuite) TestClusterRegistrationWithOneShotCrs() { + crsMetaId, crsMeta := newCrsMeta(1) + sensorService := newSensorService(s, crsMeta) + clusterNameA, clusterNameB, _ := newClusterNames() + + mockServer := newMockServerForCrsHandshake(s, crsMeta, sensorDeploymentIdentificationA, clusterNameA) + + // CRS cluster registration. + err := sensorService.Communicate(mockServer) + s.NoError(err) + + // Verify that cluster registrations is initiated. + crsMeta = lookupCrs(s, crsMetaId) + registrationsInitiated := crsMeta.GetRegistrationsInitiated() + s.Len(registrationsInitiated, 1, "Unexpected number of initiated registrations for CRS") + assert.Containsf(s.T(), registrationsInitiated, clusterNameA, "registrationsInitiated (%v) does not contain %q", registrationsInitiated, clusterNameA) + + // Attempt another cluster registration. + mockServer = newMockServerForCrsHandshake(s, crsMeta, sensorDeploymentIdentificationA, clusterNameB) + err = sensorService.Communicate(mockServer) + s.Error(err) + + // Verify that no other cluster registration is initiated. + crsMeta = lookupCrs(s, crsMetaId) + registrationsInitiated = crsMeta.GetRegistrationsInitiated() + s.Len(registrationsInitiated, 1, "Unexpected number of initiated registrations for CRS after second registration attempt") + + // Execute a regular connect. + mockServer = newMockServerForRegularConnect(s, sensorDeploymentIdentificationA, clusterNameA) + err = sensorService.Communicate(mockServer) + s.NoError(err) + + // Verify that cluster registrations is completed. + crsMeta = lookupCrs(s, crsMetaId) + registrationsInitiated = crsMeta.GetRegistrationsInitiated() + registrationsCompleted := crsMeta.GetRegistrationsCompleted() + s.Len(registrationsInitiated, 0, "Unexpected number of initiated registrations for CRS") + s.Len(registrationsCompleted, 1, "Unexpected number of initiated registrations for CRS") + assert.Containsf(s.T(), registrationsCompleted, clusterNameA, "registrationsCompleted (%v) does not contain %q", registrationsCompleted, clusterNameA) + + // Verify CRS is revoked. + crsMeta = lookupCrs(s, crsMetaId) + s.True(crsMeta.GetIsRevoked(), "CRS is not revoked after one-shot use") +} + +func toPrettyJson(s *sensorServiceTestSuite, v any) string { + bytes, err := json.MarshalIndent(v, "|", " ") + s.Require().NoErrorf(err, "JSON marshalling of value %+v failed", v) + return string(bytes) +} + +func (s *sensorServiceTestSuite) TestClusterRegistrationWithTwoLimitCrs() { + crsMetaId, crsMeta := newCrsMeta(2) + sensorService := newSensorService(s, crsMeta) + clusterNameA, clusterNameB, clusterNameC := newClusterNames() + mockServer := newMockServerForCrsHandshake(s, crsMeta, sensorDeploymentIdentificationA, clusterNameA) + + // CRS cluster registration. + err := sensorService.Communicate(mockServer) + s.NoError(err) + + // Verify that cluster registrations is initiated. + crsMeta = lookupCrs(s, crsMetaId) + registrationsInitiated := crsMeta.GetRegistrationsInitiated() + s.Len(registrationsInitiated, 1, "Unexpected number of initiated registrations for CRS:\n%s\n", toPrettyJson(s, crsMeta)) + assert.Containsf(s.T(), registrationsInitiated, clusterNameA, "registrationsInitiated (%v) does not contain %q", registrationsInitiated, clusterNameA) + + // Attempt another cluster registration. + mockServer = newMockServerForCrsHandshake(s, crsMeta, sensorDeploymentIdentificationB, clusterNameB) + err = sensorService.Communicate(mockServer) + s.NoError(err) + + // Verify that cluster registrations is initiated. + crsMeta = lookupCrs(s, crsMetaId) + registrationsInitiated = crsMeta.GetRegistrationsInitiated() + s.Len(registrationsInitiated, 2, "Unexpected number of initiated registrations for CRS") + assert.Containsf(s.T(), registrationsInitiated, clusterNameB, "registrationsInitiated (%v) does not contain %q", registrationsInitiated, clusterNameB) + + // Execute regular connects. + mockServer = newMockServerForRegularConnect(s, sensorDeploymentIdentificationA, clusterNameA) + err = sensorService.Communicate(mockServer) + s.NoError(err) + mockServer = newMockServerForRegularConnect(s, sensorDeploymentIdentificationB, clusterNameB) + err = sensorService.Communicate(mockServer) + s.NoError(err) + + // Verify that cluster registrations is completed. + crsMeta = lookupCrs(s, crsMetaId) + registrationsInitiated = crsMeta.GetRegistrationsInitiated() + registrationsCompleted := crsMeta.GetRegistrationsCompleted() + s.Len(registrationsInitiated, 0, "Unexpected number of initiated registrations for CRS") + s.Len(registrationsCompleted, 2, "Unexpected number of initiated registrations for CRS") + assert.Containsf(s.T(), registrationsCompleted, clusterNameA, "registrationsCompleted (%v) does not contain %q", registrationsCompleted, clusterNameA) + assert.Containsf(s.T(), registrationsCompleted, clusterNameB, "registrationsCompleted (%v) does not contain %q", registrationsCompleted, clusterNameB) + + // Attempt another cluster registration. + mockServer = newMockServerForCrsHandshake(s, crsMeta, sensorDeploymentIdentificationA, clusterNameC) + err = sensorService.Communicate(mockServer) + s.Error(err) + + // Verify that no other cluster registration has been recorded. + crsMeta = lookupCrs(s, crsMetaId) + registrationsInitiated = crsMeta.GetRegistrationsInitiated() + registrationsCompleted = crsMeta.GetRegistrationsCompleted() + s.Len(registrationsInitiated, 0, "Unexpected number of initiated registrations for CRS after third registration attempt") + s.Len(registrationsCompleted, 2, "Unexpected number of completed registrations for CRS after third registration attempt") + + // Verify CRS is revoked. + crsMeta = lookupCrs(s, crsMetaId) + s.True(crsMeta.GetIsRevoked(), "CRS is not revoked after registering two clusters") +} + +func (s *sensorServiceTestSuite) lookupClusterByName(name string) *storage.Cluster { + query := pkgSearch.NewQueryBuilder().AddExactMatches(pkgSearch.Cluster, name).ProtoQuery() + results, err := s.clusterDataStore.Search(s.internalContext, query) + s.NoError(err) + s.Lenf(results, 1, "unexpected number of search results when looking for cluster by name %s", name) + + resultIds := pkgSearch.ResultsToIDs(results) + s.Len(resultIds, 1) + + clusterId := resultIds[0] + cluster, ok, err := s.clusterDataStore.GetCluster(s.internalContext, clusterId) + s.NoErrorf(err, "failed to retrieve cluster %s (%s)", name) + s.True(ok) + + return cluster +} + +func (s *sensorServiceTestSuite) TestClusterRegistrationWithInitBundle() { + initBundleId, initBundleMeta := newInitBundleMeta() + sensorService := newSensorService(s, initBundleMeta) + clusterNameA, _, _ := newClusterNames() + + mockServer := newMockServerForInitBundleHandshake(s, initBundleMeta, sensorDeploymentIdentificationA, clusterNameA) + + // Init Bundle cluster registration. + err := sensorService.Communicate(mockServer) + s.NoError(err) + + // Verify that init bundle is still associated with cluster. + cluster := s.lookupClusterByName(clusterNameA) + s.NotEmptyf(cluster.GetInitBundleId(), "cluster %s lost association to init bundle %s", clusterNameA, initBundleId) + + // Simulate regular connection with non-init certificate. + mockServer = newMockServerForRegularConnect(s, sensorDeploymentIdentificationA, clusterNameA) + err = sensorService.Communicate(mockServer) + s.NoError(err) + + // Verify that init bundle is not associated with cluster anymore. + cluster = s.lookupClusterByName(clusterNameA) + s.Emptyf(cluster.GetInitBundleId(), "cluster %s still association with init bundle %s", clusterNameA, initBundleId) } // Implementation of a simple mock server to be used in the CRS test suite. @@ -201,24 +389,66 @@ type mockServer struct { clustersRegistered []*storage.Cluster } -func newMockServer(ctx context.Context, ctrl *gomock.Controller) *mockServer { - mockIdentity := authnMock.NewMockIdentity(ctrl) - mockIdentity.EXPECT().Service().AnyTimes().Return(®istrantIdentity) +func newMockServer(s *sensorServiceTestSuite, identity *storage.ServiceIdentity) *mockServer { + notBefore := time.Now().Add(-time.Minute) + notAfter := time.Now().Add(time.Hour) + + mockIdentity := authnMock.NewMockIdentity(s.mockCtrl) + mockIdentity.EXPECT().Service().AnyTimes().Return(identity) + mockIdentity.EXPECT().ValidityPeriod().AnyTimes().Return(notBefore, notAfter) md := metadata.Pairs(centralsensor.SensorHelloMetadataKey, "true") - ctx = authn.ContextWithIdentity(ctx, mockIdentity, nil) + ctx := authn.ContextWithIdentity(s.internalContext, mockIdentity, nil) ctx = metadata.NewIncomingContext(ctx, md) return &mockServer{ context: ctx, } } -func (s *mockServer) prepareNewHandshake(hello *central.SensorHello) { - s.msgsFromSensor = []*central.MsgFromSensor{ +func prepareHelloHandshake(m *mockServer, sensorDeploymentId *storage.SensorDeploymentIdentification, clusterName string) { + hello := ¢ral.SensorHello{ + SensorVersion: "1.0", + DeploymentIdentification: sensorDeploymentId, + HelmManagedConfigInit: ¢ral.HelmManagedConfigInit{ + ClusterName: clusterName, + ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, + }, + } + m.msgsFromSensor = []*central.MsgFromSensor{ { Msg: ¢ral.MsgFromSensor_Hello{Hello: hello}, }, } - s.msgsToSensor = nil + m.msgsToSensor = nil +} + +func newMockServerForCrsHandshake(s *sensorServiceTestSuite, crsMeta *storage.InitBundleMeta, sensorDeploymentId *storage.SensorDeploymentIdentification, clusterName string) *mockServer { + identity := &storage.ServiceIdentity{ + Type: storage.ServiceType_REGISTRANT_SERVICE, + InitBundleId: crsMeta.GetId(), + } + + m := newMockServer(s, identity) + prepareHelloHandshake(m, sensorDeploymentId, clusterName) + return m +} + +func newMockServerForInitBundleHandshake(s *sensorServiceTestSuite, crsMeta *storage.InitBundleMeta, sensorDeploymentId *storage.SensorDeploymentIdentification, clusterName string) *mockServer { + identity := &storage.ServiceIdentity{ + Type: storage.ServiceType_SENSOR_SERVICE, + InitBundleId: crsMeta.GetId(), + } + + m := newMockServer(s, identity) + prepareHelloHandshake(m, sensorDeploymentId, clusterName) + return m +} + +func newMockServerForRegularConnect(s *sensorServiceTestSuite, sensorDeploymentId *storage.SensorDeploymentIdentification, clusterName string) *mockServer { + m := newMockServer(s, &storage.ServiceIdentity{ + Type: storage.ServiceType_SENSOR_SERVICE, + }) + prepareHelloHandshake(m, sensorDeploymentId, clusterName) + return m } func (s *mockServer) Context() context.Context { @@ -268,7 +498,7 @@ func newCluster(clusterName string, hello *central.SensorHello) *storage.Cluster } } -func retrieveCentralHello(s *crsTestSuite, server *mockServer) *central.CentralHello { +func retrieveCentralHello(s *sensorServiceTestSuite, server *mockServer) *central.CentralHello { s.NotEmpty(server.msgsToSensor, "no central response message") centralMsg := server.msgsToSensor[0] centralHello := centralMsg.GetHello() @@ -276,60 +506,93 @@ func retrieveCentralHello(s *crsTestSuite, server *mockServer) *central.CentralH return centralHello } -var registrantIdentity = storage.ServiceIdentity{ - Type: storage.ServiceType_REGISTRANT_SERVICE, -} - -var installInfo *storage.InstallationInfo = &storage.InstallationInfo{ - Id: "some-central-id", -} - -var sensorDeploymentIdentification = &storage.SensorDeploymentIdentification{ +var sensorDeploymentIdentificationA = &storage.SensorDeploymentIdentification{ SystemNamespaceId: uuid.NewV4().String(), DefaultNamespaceId: uuid.NewV4().String(), - AppNamespace: "my-stackrox-namespace", + AppNamespace: "my-stackrox-namespace-a", AppNamespaceId: uuid.NewV4().String(), AppServiceaccountId: uuid.NewV4().String(), K8SNodeName: "my-node", } -var sensorHello = ¢ral.SensorHello{ - SensorVersion: "1.0", - DeploymentIdentification: sensorDeploymentIdentification, - HelmManagedConfigInit: ¢ral.HelmManagedConfigInit{ - ClusterName: "my-new-cluster", - ManagedBy: storage.ManagerType_MANAGER_TYPE_HELM_CHART, - }, +var sensorDeploymentIdentificationB = &storage.SensorDeploymentIdentification{ + SystemNamespaceId: uuid.NewV4().String(), + DefaultNamespaceId: uuid.NewV4().String(), + AppNamespace: "my-stackrox-namespace-b", + AppNamespaceId: uuid.NewV4().String(), + AppServiceaccountId: uuid.NewV4().String(), + K8SNodeName: "my-node", } -func newSensorService(ctx context.Context, ctrl *gomock.Controller) (Service, *mockServer) { - mockServer := newMockServer(ctx, ctrl) - - mockInstallation := installationMock.NewMockStore(ctrl) +func newSensorService(s *sensorServiceTestSuite, crsMeta *storage.InitBundleMeta) Service { + mockInstallation := installationMock.NewMockStore(s.mockCtrl) + installInfo := &storage.InstallationInfo{ + Id: "some-central-id", + } mockInstallation.EXPECT().Get(gomock.Any()).AnyTimes().Return(installInfo, true, nil) - mockConnetionManager := connectionMock.NewMockManager(ctrl) - mockConnetionManager.EXPECT().GetConnectionPreference(gomock.Any()).AnyTimes().Return(connection.Preferences{}) + mockConnectionManager := connectionMock.NewMockManager(s.mockCtrl) + mockConnectionManager.EXPECT(). + GetConnectionPreference(gomock.Any()). + AnyTimes().Return(connection.Preferences{}) + mockConnectionManager.EXPECT(). + HandleConnection(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + AnyTimes(). + DoAndReturn(mockHandleConnection(s.clusterDataStore)) - pipelineFactory := pipelineMock.NewMockFactory(ctrl) - pipeline := pipelineMock.NewMockClusterPipeline(ctrl) + pipelineFactory := pipelineMock.NewMockFactory(s.mockCtrl) + pipeline := pipelineMock.NewMockClusterPipeline(s.mockCtrl) pipeline.EXPECT().Capabilities().AnyTimes().Return([]centralsensor.CentralCapability{}) pipelineFactory.EXPECT().PipelineForCluster(gomock.Any(), gomock.Any()).AnyTimes().Return(pipeline, nil) - clustersDataStore := clusterMock.NewMockDataStore(ctrl) - clustersDataStore.EXPECT().LookupOrCreateClusterFromConfig( - gomock.Any(), // ctx - gomock.Any(), // clusterID - gomock.Any(), // bundleID - gomock.Any(), // sensorHello - ). - AnyTimes(). - DoAndReturn( - func(_ctx context.Context, clusterId, _bundleId string, hello *central.SensorHello) (*storage.Cluster, error) { - return mockServer.LookupOrCreateClusterFromConfig(clusterId, hello) - }, - ) + clusterInitStore := clusterDataStore.IntrospectClusterInitStore(s.T(), s.clusterDataStore) + + if crsMeta != nil { + err := clusterInitStore.Add(s.internalContext, crsMeta) + s.NoError(err, "failed adding dummy CRS to cluster init store") + s.T().Logf("Added dummy CRS with ID %q and name %q", crsMeta.GetId(), crsMeta.GetName()) + } - sensorService := New(mockConnetionManager, pipelineFactory, clustersDataStore, mockInstallation) - return sensorService, mockServer + sensorService := New(mockConnectionManager, pipelineFactory, s.clusterDataStore, mockInstallation, clusterInitStore) + return sensorService +} + +func newCrsMeta(maxRegistrations uint64) (string, *storage.InitBundleMeta) { + id := uuid.NewV4().String() + meta := &storage.InitBundleMeta{ + Id: id, + Name: fmt.Sprintf("name-%s", id), + CreatedAt: timestamppb.New(time.Now()), + ExpiresAt: timestamppb.New(time.Now().Add(10 * time.Minute)), + Version: storage.InitBundleMeta_CRS, + MaxRegistrations: maxRegistrations, + } + return meta.GetId(), meta +} + +func newInitBundleMeta() (string, *storage.InitBundleMeta) { + id := uuid.NewV4().String() + meta := &storage.InitBundleMeta{ + Id: id, + Name: fmt.Sprintf("init-bundle-%s", id), + CreatedAt: timestamppb.New(time.Now()), + ExpiresAt: timestamppb.New(time.Now().Add(10 * time.Minute)), + Version: storage.InitBundleMeta_INIT_BUNDLE, + } + return meta.GetId(), meta +} + +func assertCertificateBundleComplete(s *sensorServiceTestSuite, certBundle map[string]string) { + s.Len(certBundle, 15, "expected 15 entries (1 CA cert, 7 service certs, 7 service keys) in bundle") +} + +type HandleConnectionFunc = func(ctx context.Context, _ *central.SensorHello, cluster *storage.Cluster, _ pipeline.ClusterPipeline, _ central.SensorService_CommunicateServer) error + +func mockHandleConnection(clusterDataStore clusterDataStore.DataStore) HandleConnectionFunc { + return func(ctx context.Context, _ *central.SensorHello, cluster *storage.Cluster, _ pipeline.ClusterPipeline, _ central.SensorService_CommunicateServer) error { + cluster.HealthStatus = &storage.ClusterHealthStatus{ + LastContact: timestamppb.New(time.Now()), + } + return clusterDataStore.UpdateCluster(ctx, cluster) + } } diff --git a/central/serviceaccount/datastore/datastore_impl_test.go b/central/serviceaccount/datastore/datastore_impl_test.go index 02f34e185e209..23b99081e6959 100644 --- a/central/serviceaccount/datastore/datastore_impl_test.go +++ b/central/serviceaccount/datastore/datastore_impl_test.go @@ -116,7 +116,7 @@ func (suite *ServiceAccountDataStoreTestSuite) TestSearchServiceAccounts_NameAnd results, err := suite.datastore.SearchServiceAccounts(suite.ctx, nil) suite.Require().NoError(err) ids := make(map[string]struct{}) - var actualNames []string + actualNames := make([]string, 0, len(results)) for _, r := range results { if r.GetId() == sa1.GetId() || r.GetId() == sa2.GetId() { ids[r.GetId()] = struct{}{} diff --git a/central/signatureintegration/datastore/key_bundle.go b/central/signatureintegration/datastore/key_bundle.go new file mode 100644 index 0000000000000..1a1d22e7417e8 --- /dev/null +++ b/central/signatureintegration/datastore/key_bundle.go @@ -0,0 +1,67 @@ +package datastore + +import ( + "context" + "strings" + + "github.com/stackrox/rox/central/signatureintegration/store" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/filewatcher" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/signatures" +) + +var redHatKeyBundlePath = signatures.RedHatKeyBundlePath() + +func keyBundleHandler(siStore store.SignatureIntegrationStore) filewatcher.Handler { + return func(data []byte) error { + bundle, err := signatures.ParseKeyBundle(data) + if err != nil { + log.Warnf("Invalid key bundle file: %v", err) + watcherFileErrorTotal.Inc() + return nil + } + + si, err := bundle.ToSignatureIntegration() + if err != nil { + log.Warnf("Failed to create Red Hat signature integration from key bundle: %v", err) + watcherFileErrorTotal.Inc() + return nil + } + ctx := sac.WithGlobalAccessScopeChecker(context.Background(), sac.AllowAllAccessScopeChecker()) + if err := siStore.Upsert(ctx, si); err != nil { + log.Errorf("Failed to upsert Red Hat signature integration from key bundle: %v", err) + watcherUpsertTotal.WithLabelValues("error").Inc() + return err + } + + watcherUpsertTotal.WithLabelValues("success").Inc() + cosignKeys := si.GetCosign().GetPublicKeys() + watcherKeyCount.Set(float64(len(cosignKeys))) + watcherLastSuccessTimestamp.SetToCurrentTime() + + keyNames := make([]string, 0, len(cosignKeys)) + for _, k := range cosignKeys { + keyNames = append(keyNames, k.GetName()) + } + log.Infof("Updated Red Hat signature integration with %d key(s) from bundle: [%s]", + len(cosignKeys), strings.Join(keyNames, ", ")) + return nil + } +} + +func startKeyBundleWatcher(siStore store.SignatureIntegrationStore) { + interval := env.RedHatSigningKeyWatchInterval.DurationSetting() + if interval == 0 { + log.Info("Red Hat signing key bundle watcher is disabled (ROX_REDHAT_SIGNING_KEY_WATCH_INTERVAL=0)") + return + } + + w := filewatcher.New(redHatKeyBundlePath, interval, keyBundleHandler(siStore), + filewatcher.WithOnError(func(_ error) { + watcherFileErrorTotal.Inc() + }), + ) + w.Start() + bundleWatcher = w +} diff --git a/central/signatureintegration/datastore/key_bundle_test.go b/central/signatureintegration/datastore/key_bundle_test.go new file mode 100644 index 0000000000000..293a40615e884 --- /dev/null +++ b/central/signatureintegration/datastore/key_bundle_test.go @@ -0,0 +1,141 @@ +package datastore + +import ( + "errors" + "fmt" + "testing" + + storeMocks "github.com/stackrox/rox/central/signatureintegration/store/mocks" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/signatures" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +const ( + testPublicKeyPEM = `-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE16IoQbiiB5exTRLTkl2rn5FuyXys +4TbDn4+GhQD1JmLZnAiA0cXktX+gFdxu/0JM9pcjjaqT7pdXztbBs78cXg== +-----END PUBLIC KEY----- +` + testPublicKeyPEM2 = `-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEQq1X/6XxCA4s0++8Tvl8k+Z0G/GN +LKpdYJEldXnyRE4ppY5d7vnRZHvdZQMSE3KoRSMvVnzZtc9LTKLB3DlS/w== +-----END PUBLIC KEY----- +` +) + +func redHatIntegrationMatcher() gomock.Matcher { + return gomock.Cond(func(x any) bool { + si, ok := x.(*storage.SignatureIntegration) + return ok && si.GetId() == signatures.DefaultRedHatIntegrationID + }) +} + +func validBundleJSON() []byte { + return fmt.Appendf(nil, `{"schemaVersion": "1.0", "cosignKeys": [{"name": "test-key-1", "publicKey": %q}]}`, testPublicKeyPEM) +} + +func validBundleJSON2Keys() []byte { + return fmt.Appendf(nil, `{"schemaVersion": "1.0", "cosignKeys": [{"name": "test-key-1", "publicKey": %q}, {"name": "test-key-2", "publicKey": %q}]}`, + testPublicKeyPEM, testPublicKeyPEM2) +} + +func TestStartKeyBundleWatcherDisabled(t *testing.T) { + t.Setenv("ROX_REDHAT_SIGNING_KEY_WATCH_INTERVAL", "0") + + ctrl := gomock.NewController(t) + mockStore := storeMocks.NewMockSignatureIntegrationStore(ctrl) + + old := bundleWatcher + defer func() { bundleWatcher = old }() + bundleWatcher = nil + + startKeyBundleWatcher(mockStore) + assert.Nil(t, bundleWatcher) +} + +func TestHandlerValidBundle(t *testing.T) { + ctrl := gomock.NewController(t) + mockStore := storeMocks.NewMockSignatureIntegrationStore(ctrl) + mockStore.EXPECT().Upsert(gomock.Any(), redHatIntegrationMatcher()).Return(nil).Times(1) + + handler := keyBundleHandler(mockStore) + err := handler(validBundleJSON()) + assert.NoError(t, err) +} + +func TestHandlerTwoKeys(t *testing.T) { + ctrl := gomock.NewController(t) + mockStore := storeMocks.NewMockSignatureIntegrationStore(ctrl) + mockStore.EXPECT().Upsert(gomock.Any(), redHatIntegrationMatcher()).Return(nil).Times(1) + + handler := keyBundleHandler(mockStore) + err := handler(validBundleJSON2Keys()) + assert.NoError(t, err) +} + +func TestHandlerInvalidBundleReturnsNil(t *testing.T) { + ctrl := gomock.NewController(t) + mockStore := storeMocks.NewMockSignatureIntegrationStore(ctrl) + + handler := keyBundleHandler(mockStore) + err := handler([]byte(`{"cosignKeys": []}`)) + assert.NoError(t, err, "parse errors must return nil to suppress retry") +} + +func TestHandlerMalformedJSONReturnsNil(t *testing.T) { + ctrl := gomock.NewController(t) + mockStore := storeMocks.NewMockSignatureIntegrationStore(ctrl) + + handler := keyBundleHandler(mockStore) + err := handler([]byte(`{not json`)) + assert.NoError(t, err, "parse errors must return nil to suppress retry") +} + +func TestHandlerNoSupportedKeysReturnsNil(t *testing.T) { + ctrl := gomock.NewController(t) + mockStore := storeMocks.NewMockSignatureIntegrationStore(ctrl) + + handler := keyBundleHandler(mockStore) + err := handler([]byte(`{"schemaVersion": "1.0", "pgpKeys": [{"name": "k", "armoredKey": "opaque"}]}`)) + assert.NoError(t, err, "no-supported-keys errors must return nil to suppress retry") +} + +func TestHandlerUpsertErrorReturnsError(t *testing.T) { + ctrl := gomock.NewController(t) + mockStore := storeMocks.NewMockSignatureIntegrationStore(ctrl) + mockStore.EXPECT(). + Upsert(gomock.Any(), redHatIntegrationMatcher()). + Return(errors.New("transient DB error")). + Times(1) + + handler := keyBundleHandler(mockStore) + err := handler(validBundleJSON()) + require.Error(t, err, "upsert errors must be returned to enable retry") + assert.Contains(t, err.Error(), "transient DB error") +} + +func TestHandlerUpsertRetryOnFailure(t *testing.T) { + ctrl := gomock.NewController(t) + mockStore := storeMocks.NewMockSignatureIntegrationStore(ctrl) + + firstCall := mockStore.EXPECT(). + Upsert(gomock.Any(), redHatIntegrationMatcher()). + Return(errors.New("transient DB error")). + Times(1) + mockStore.EXPECT(). + Upsert(gomock.Any(), redHatIntegrationMatcher()). + Return(nil). + Times(1). + After(firstCall) + + handler := keyBundleHandler(mockStore) + + err := handler(validBundleJSON()) + assert.Error(t, err, "first call should fail") + + err = handler(validBundleJSON()) + assert.NoError(t, err, "second call should succeed") +} diff --git a/central/signatureintegration/datastore/metrics.go b/central/signatureintegration/datastore/metrics.go new file mode 100644 index 0000000000000..26630b07eedb7 --- /dev/null +++ b/central/signatureintegration/datastore/metrics.go @@ -0,0 +1,70 @@ +package datastore + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/stackrox/rox/pkg/metrics" +) + +var ( + watcherUpsertTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "key_bundle_watcher_upserts_total", + Help: "Total number of key bundle upsert attempts by the watcher, labeled by result", + }, []string{"result"}) + + watcherFileErrorTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "key_bundle_watcher_file_errors_total", + Help: "Total number of file-level errors in the watcher (stat, read, oversize, parse failures)", + }) + + watcherKeyCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "key_bundle_watcher_keys_count", + Help: "Number of keys in the most recently applied Red Hat signing key bundle", + }) + + watcherLastSuccessTimestamp = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "key_bundle_watcher_last_success_timestamp", + Help: "Unix timestamp of the last successful key bundle upsert by the watcher", + }) + + updaterDownloadsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "key_bundle_updater_downloads_total", + Help: "Total number of key bundle download attempts by the updater, labeled by result", + }, []string{"result"}) + + updaterLastSuccessTimestamp = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "key_bundle_updater_last_success_timestamp", + Help: "Unix timestamp of the last successful key bundle download by the updater", + }) + + updaterDownloadDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "key_bundle_updater_download_duration_seconds", + Help: "Duration of key bundle download attempts by the updater", + Buckets: prometheus.DefBuckets, + }) +) + +func init() { + metrics.EmplaceCollector( + watcherUpsertTotal, + watcherFileErrorTotal, + watcherKeyCount, + watcherLastSuccessTimestamp, + updaterDownloadsTotal, + updaterLastSuccessTimestamp, + updaterDownloadDuration, + ) +} diff --git a/central/signatureintegration/datastore/singleton.go b/central/signatureintegration/datastore/singleton.go index a652ab9826389..df754ee696b46 100644 --- a/central/signatureintegration/datastore/singleton.go +++ b/central/signatureintegration/datastore/singleton.go @@ -2,40 +2,111 @@ package datastore import ( "context" + "time" "github.com/pkg/errors" "github.com/stackrox/rox/central/globaldb" policyDataStore "github.com/stackrox/rox/central/policy/datastore" "github.com/stackrox/rox/central/signatureintegration/store" pgStore "github.com/stackrox/rox/central/signatureintegration/store/postgres" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/filedownloader" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/signatures" "github.com/stackrox/rox/pkg/sync" + "github.com/stackrox/rox/pkg/urlfmt" "github.com/stackrox/rox/pkg/utils" ) +// Stoppable represents a background process that can be stopped. +type Stoppable interface { + Stop() +} + var ( once sync.Once instance DataStore + + bundleUpdater Stoppable + bundleWatcher Stoppable ) -func upsertDefaultRedHatSignatureIntegration(siStore store.SignatureIntegrationStore) { +// KeyBundleUpdater returns the key bundle updater for shutdown registration. +// Returns nil if the updater was not started (e.g. no URL configured). +// Must only be called after Singleton(). +func KeyBundleUpdater() Stoppable { + return bundleUpdater +} + +// KeyBundleWatcher returns the key bundle watcher for shutdown registration. +// Must only be called after Singleton(). +func KeyBundleWatcher() Stoppable { + return bundleWatcher +} + +func seedRedHatDefaultSignatureIntegration(siStore store.SignatureIntegrationStore) { ctx := sac.WithGlobalAccessScopeChecker(context.Background(), sac.AllowAllAccessScopeChecker()) - log.Debugf("Upserting default Red Hat signature integration %q (%s)", - signatures.DefaultRedHatSignatureIntegration.GetName(), - signatures.DefaultRedHatSignatureIntegration.GetId(), + id := signatures.DefaultRedHatIntegrationID + _, exists, err := siStore.Get(ctx, id) + if err != nil { + utils.Should(errors.Wrap(err, "checking for default Red Hat signature integration")) + return + } + if exists { + log.Debugf("Default Red Hat signature integration %q already exists, skipping seed", id) + return + } + + log.Infof("Seeding default Red Hat signature integration %q", id) + err = siStore.Upsert(ctx, signatures.DefaultRedHatSignatureIntegration) + utils.Should(errors.Wrap(err, "seeding default Red Hat signature integration")) +} + +func startKeyBundleUpdater() { + if env.OfflineModeEnv.BooleanSetting() { + log.Infof("Offline mode detected: The Red Hat signing key bundle will not be downloaded automatically. Manual updates are possible by mounting the bundle to %q", redHatKeyBundlePath) + return + } + + rawURL := env.RedHatSigningKeyBundleURL.Setting() + if rawURL == "" { + log.Info("ROX_REDHAT_SIGNING_KEY_BUNDLE_URL is empty, key bundle updater will not start") + return + } + bundleURL := urlfmt.FormatURL(rawURL, urlfmt.HTTPS, urlfmt.HonorInputSlash) + + interval := env.RedHatSigningKeyUpdateInterval.DurationSetting() + + u := filedownloader.New(bundleURL, redHatKeyBundlePath, interval, + filedownloader.WithOnComplete(func(err error, duration time.Duration) { + updaterDownloadDuration.Observe(duration.Seconds()) + if err != nil { + log.Warnf("Failed to download Red Hat signing key bundle from %q: %v", bundleURL, err) + updaterDownloadsTotal.WithLabelValues("error").Inc() + } else { + updaterDownloadsTotal.WithLabelValues("success").Inc() + updaterLastSuccessTimestamp.SetToCurrentTime() + } + }), ) - err := siStore.Upsert(ctx, signatures.DefaultRedHatSignatureIntegration) - utils.Should(errors.Wrap(err, "upserting default Red Hat signature integration")) + // Passing false for waitForInitial means Start does not block startup + // on the first download; WithOnComplete will report its outcome. + if err := u.Start(context.Background(), false); err != nil { + log.Errorf("Red Hat signing key bundle updater will not start: %v", err) + return + } + bundleUpdater = u } // Singleton returns the sole instance of the DataStore service. func Singleton() DataStore { once.Do(func() { storage := pgStore.New(globaldb.GetPostgres()) - upsertDefaultRedHatSignatureIntegration(storage) + seedRedHatDefaultSignatureIntegration(storage) // must run before watcher; bundle file takes precedence on first tick instance = New(storage, policyDataStore.Singleton()) + startKeyBundleWatcher(storage) + startKeyBundleUpdater() }) return instance } diff --git a/central/signatureintegration/datastore/singleton_test.go b/central/signatureintegration/datastore/singleton_test.go new file mode 100644 index 0000000000000..9cde281672c82 --- /dev/null +++ b/central/signatureintegration/datastore/singleton_test.go @@ -0,0 +1,73 @@ +package datastore + +import ( + "testing" + + storeMocks "github.com/stackrox/rox/central/signatureintegration/store/mocks" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/signatures" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" +) + +func TestSeedFirstInstall(t *testing.T) { + ctrl := gomock.NewController(t) + mockStore := storeMocks.NewMockSignatureIntegrationStore(ctrl) + + id := signatures.DefaultRedHatIntegrationID + mockStore.EXPECT().Get(gomock.Any(), id).Return(nil, false, nil).Times(1) + mockStore.EXPECT().Upsert(gomock.Any(), signatures.DefaultRedHatSignatureIntegration).Return(nil).Times(1) + + seedRedHatDefaultSignatureIntegration(mockStore) +} + +func TestSeedSubsequentStartup(t *testing.T) { + ctrl := gomock.NewController(t) + mockStore := storeMocks.NewMockSignatureIntegrationStore(ctrl) + + id := signatures.DefaultRedHatIntegrationID + mockStore.EXPECT().Get(gomock.Any(), id).Return(signatures.DefaultRedHatSignatureIntegration, true, nil).Times(1) + + seedRedHatDefaultSignatureIntegration(mockStore) +} + +func TestStartKeyBundleUpdaterOfflineMode(t *testing.T) { + t.Setenv(env.OfflineModeEnv.EnvVar(), "true") + t.Setenv(env.RedHatSigningKeyBundleURL.EnvVar(), "https://example.com/keys.json") + + old := bundleUpdater + defer func() { bundleUpdater = old }() + bundleUpdater = nil + + startKeyBundleUpdater() + assert.Nil(t, bundleUpdater, "updater should not start in offline mode") +} + +func TestStartKeyBundleUpdaterOnlineWithURL(t *testing.T) { + t.Setenv(env.OfflineModeEnv.EnvVar(), "false") + t.Setenv(env.RedHatSigningKeyBundleURL.EnvVar(), "https://example.com/keys.json") + + old := bundleUpdater + defer func() { + if bundleUpdater != nil { + bundleUpdater.Stop() + } + bundleUpdater = old + }() + bundleUpdater = nil + + startKeyBundleUpdater() + assert.NotNil(t, bundleUpdater, "updater should start in online mode with URL configured") +} + +func TestStartKeyBundleUpdaterOnlineWithoutURL(t *testing.T) { + t.Setenv(env.OfflineModeEnv.EnvVar(), "false") + t.Setenv(env.RedHatSigningKeyBundleURL.EnvVar(), "") + + old := bundleUpdater + defer func() { bundleUpdater = old }() + bundleUpdater = nil + + startKeyBundleUpdater() + assert.Nil(t, bundleUpdater, "updater should not start without URL") +} diff --git a/central/splunk/testdata/file_access_violation.json b/central/splunk/testdata/file_access_violation.json new file mode 100644 index 0000000000000..8122fad739cb4 --- /dev/null +++ b/central/splunk/testdata/file_access_violation.json @@ -0,0 +1,56 @@ +{ + "violations": [ + { + "violationInfo": { + "violationId": "26c0771b-4274-5aaf-8f18-60b023458d20", + "violationMessage": "'/etc/passwd' opened (writable)", + "violationType": "FILE_ACCESS", + "violationTime": "2024-06-14T20:00:00Z" + }, + "alertInfo": { + "alertId": "fa1eacce-0000-4000-a000-000000000001", + "lifecycleStage": "RUNTIME" + }, + "processInfo": { + "processViolationId": "fa1eacce-0000-4000-f000-000000000006", + "processSignalId": "fa1eacce-0000-4000-f000-000000000007", + "processCreationTime": "2024-06-14T20:00:00Z", + "processName": "vi", + "processArgs": "/etc/passwd", + "execFilePath": "/usr/bin/vi", + "pid": 42, + "processUid": 0, + "processGid": 0 + }, + "deploymentInfo": { + "deploymentId": "fa1eacce-0000-4000-c000-000000000003", + "deploymentName": "nginx", + "deploymentType": "Deployment", + "deploymentNamespace": "default", + "clusterId": "fa1eacce-0000-4000-d000-000000000004" + }, + "policyInfo": { + "policyId": "fa1eacce-0000-4000-b000-000000000002", + "policyName": "File Access: /etc/passwd", + "policyDescription": "Detect modifications to /etc/passwd", + "policyLifecycleStages": [ + "RUNTIME" + ], + "policySeverity": "HIGH_SEVERITY", + "policyVersion": "1.1" + }, + "fileAccessInfo": { + "effectivePath": "/etc/passwd", + "actualPath": "/rootfs/etc/passwd", + "operation": "OPEN", + "fileUid": 0, + "fileGid": 0, + "fileMode": 420, + "fileUsername": "root", + "fileGroup": "root", + "hostname": "node-1" + } + } + ], + "newCheckpoint": "2024-06-26T22:00:00Z" +} diff --git a/central/splunk/testdata/generic_violation.json b/central/splunk/testdata/generic_violation.json new file mode 100644 index 0000000000000..02a9b165e5a37 --- /dev/null +++ b/central/splunk/testdata/generic_violation.json @@ -0,0 +1,41 @@ +{ + "newCheckpoint": "2024-06-26T22:00:00Z", + "violations": [ + { + "alertInfo": { + "alertId": "aeaaaaaa-bbbb-4011-0000-111111111111" + }, + "deploymentInfo": {}, + "violationInfo": { + "containerName": "nginx", + "podId": "nginx", + "violationId": "f073c9f5-c766-5dc4-b7cb-aa8ac8f2d445", + "violationMessage": "This is a kube event violation", + "violationMessageAttributes": [ + { + "key": "pod", + "value": "nginx" + }, + { + "key": "container", + "value": "nginx" + } + ], + "violationTime": "2024-06-14T20:00:00Z", + "violationType": "GENERIC" + } + }, + { + "alertInfo": { + "alertId": "aeaaaaaa-bbbb-4011-0000-111111111111" + }, + "deploymentInfo": {}, + "violationInfo": { + "violationId": "aeaaaaaa-bbbb-4011-0000-111111111111", + "violationMessage": "Deployment is affected by 'CVE-2017-15670'", + "violationTime": "2024-06-14T20:00:00Z", + "violationType": "GENERIC" + } + } + ] +} diff --git a/central/splunk/violations.go b/central/splunk/violations.go index c5e75d4a62080..224a461cd465a 100644 --- a/central/splunk/violations.go +++ b/central/splunk/violations.go @@ -37,6 +37,13 @@ const ( // detected but not yet seen in the database. Ten seconds were chosen based on our understanding how quickly // violations are persisted. eventualConsistencyMargin = 10 * time.Second + + // maxProcessDetectionDelay is the maximum expected delay between when a process runs (signal time) and when + // the resulting alert is stored in Central's database. The full pipeline is: sensor detection -> transmission + // to central -> enrichment -> database storage. In CI environments this delay has been observed at 50+ seconds. + // This constant bounds the fallback recovery window for process violations whose signal time has already been + // passed by the checkpoint but whose alert was recently stored. + maxProcessDetectionDelay = 2 * time.Minute ) var ( @@ -210,8 +217,11 @@ func extractViolations(alert *storage.Alert, fromTime time.Time, toTime time.Tim seenViolations = true violationTime := getProcessViolationTime(alert, procIndicator) - if violationTime == nil || violationTime.Compare(fromTime) <= 0 || - violationTime.Compare(toTime) > 0 { + if violationTime == nil { + continue + } + if !isTimeInWindow(violationTime, fromTime, toTime) && + !isProcessViolationRecoverableViaAlertTime(violationTime, alert, fromTime, toTime) { continue } @@ -258,6 +268,14 @@ func extractViolations(alert *storage.Alert, fromTime time.Time, toTime time.Tim NetworkFlowInfo: v.GetNetworkFlowInfo().CloneVT(), } + if v.GetType() == storage.Alert_Violation_FILE_ACCESS { + fileAccess := v.GetFileAccess() + violation.FileAccessInfo = extractFileAccessInfo(fileAccess) + if proc := fileAccess.GetProcess(); proc != nil { + violation.ProcessInfo = extractProcessInfo(alert.GetId(), proc) + } + } + addEntityInfoToSplunkViolation(alert, &violation, deploymentInfo) result = append(result, &violation) @@ -299,6 +317,25 @@ func generateViolationID(alertID string, v *storage.Alert_Violation) (string, er return uuid.NewV5(alertUUID, hex.Dump(data)).String(), nil } +func isTimeInWindow(t *time.Time, fromTime, toTime time.Time) bool { + return t != nil && t.Compare(fromTime) > 0 && t.Compare(toTime) <= 0 +} + +// isProcessViolationRecoverableViaAlertTime returns true when a process violation's signal time has +// fallen behind the checkpoint window but the alert was recently stored (alert.Time is within the +// window). This handles the race where the pipeline delay between process execution and alert +// storage exceeds the checkpoint advancement rate. +func isProcessViolationRecoverableViaAlertTime(signalTime *time.Time, alert *storage.Alert, fromTime, toTime time.Time) bool { + alertTime := protocompat.ConvertTimestampToTimeOrNil(alert.GetTime()) + if alertTime == nil { + return false + } + if !isTimeInWindow(alertTime, fromTime, toTime) { + return false + } + return signalTime.After(alertTime.Add(-maxProcessDetectionDelay)) +} + func getProcessViolationTime(fromAlert *storage.Alert, fromProcIndicator *storage.ProcessIndicator) *time.Time { timestamp := fromProcIndicator.GetSignal().GetTime() if timestamp == nil { @@ -333,6 +370,10 @@ func extractProcessViolationInfo(fromAlert *storage.Alert, fromProcViolation *st func getNonProcessViolationTime(fromAlert *storage.Alert, fromViolation *storage.Alert_Violation) *time.Time { timestamp := fromViolation.GetTime() + if timestamp == nil { + // File access violations store time on the FileAccess message + timestamp = fromViolation.GetFileAccess().GetTimestamp() + } if timestamp == nil { // Use alert timestamp as a fallback in case violation timestamp wasn't provided. timestamp = fromAlert.GetTime() @@ -366,6 +407,8 @@ func extractNonProcessViolationInfo(fromAlert *storage.Alert, fromViolation *sto typ = integrations.SplunkViolation_ViolationInfo_K8S_EVENT case storage.Alert_Violation_NETWORK_FLOW: typ = integrations.SplunkViolation_ViolationInfo_NETWORK_FLOW + case storage.Alert_Violation_FILE_ACCESS: + typ = integrations.SplunkViolation_ViolationInfo_FILE_ACCESS } violationTime := getNonProcessViolationTime(fromAlert, fromViolation) @@ -456,6 +499,35 @@ func extractProcessInfo(alertID string, from *storage.ProcessIndicator) *integra return splunkProcessInfo } +func extractFileAccessInfo(from *storage.FileAccess) *integrations.SplunkViolation_FileAccessInfo { + if from == nil { + return nil + } + info := &integrations.SplunkViolation_FileAccessInfo{ + Operation: from.GetOperation().String(), + Hostname: from.GetHostname(), + } + if f := from.GetFile(); f != nil { + info.EffectivePath = f.GetEffectivePath() + info.ActualPath = f.GetActualPath() + if m := f.GetMeta(); m != nil { + info.FileUid = protocompat.ProtoUInt32Value(m.GetUid()) + info.FileGid = protocompat.ProtoUInt32Value(m.GetGid()) + info.FileMode = protocompat.ProtoUInt32Value(m.GetMode()) + info.FileUsername = m.GetUsername() + info.FileGroup = m.GetGroup() + info.AclType = m.GetAclType() + info.AclEntries = m.GetAclEntries() + info.XattrName = m.GetXattrName() + } + } + if moved := from.GetMoved(); moved != nil { + info.MovedEffectivePath = moved.GetEffectivePath() + info.MovedActualPath = moved.GetActualPath() + } + return info +} + func extractAlertInfo(from *storage.Alert, violationInfo *integrations.SplunkViolation_ViolationInfo) *integrations.SplunkViolation_AlertInfo { alertInfo := &integrations.SplunkViolation_AlertInfo{ AlertId: from.GetId(), diff --git a/central/splunk/violations_query_test.go b/central/splunk/violations_query_test.go index 98f8ab0c8c301..aaf49adb70c98 100644 --- a/central/splunk/violations_query_test.go +++ b/central/splunk/violations_query_test.go @@ -5,7 +5,7 @@ package splunk import ( "context" "fmt" - "sort" + "slices" "testing" "time" @@ -25,8 +25,8 @@ import ( // if the definition is another file type violationsTestSuite struct { suite.Suite - deployAlert, processAlert, k8sAlert, networkAlert, resourceAlert *storage.Alert - allowCtx context.Context + deployAlert, processAlert, k8sAlert, networkAlert, resourceAlert, fileAccessAlert *storage.Alert + allowCtx context.Context } func mustParseTime(timeStr string) time.Time { @@ -137,9 +137,7 @@ func (s *violationsTestSuite) TestQueryAlertsAreSortedByAlertID() { a.Id = uuid.NewV4().String() sortedIDs = append(sortedIDs, a.GetId()) } - sort.Slice(sortedIDs, func(i, j int) bool { - return sortedIDs[i] < sortedIDs[j] - }) + slices.Sort(sortedIDs) var resultIDs []string s.withAlerts(alerts, func(alertsDS datastore.DataStore) { diff --git a/central/splunk/violations_serialization_test.go b/central/splunk/violations_serialization_test.go index 857e429470f2b..d64df2af8a1fd 100644 --- a/central/splunk/violations_serialization_test.go +++ b/central/splunk/violations_serialization_test.go @@ -5,6 +5,7 @@ package splunk import ( "bytes" "context" + _ "embed" "io" "net/http" "net/http/httptest" @@ -12,6 +13,7 @@ import ( "time" alertDataStore "github.com/stackrox/rox/central/alert/datastore" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/fixtures" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protocompat" @@ -22,6 +24,12 @@ import ( var ( violationTime = time.Date(2024, time.June, 14, 20, 00, 00, 0, time.UTC) violationTimestamp = protocompat.ConvertTimeToTimestampOrNil(&violationTime) + + //go:embed testdata/generic_violation.json + expectedGenericViolation string + + //go:embed testdata/file_access_violation.json + expectedFileAccessViolation string ) func TestViolationSerialization(t *testing.T) { @@ -41,21 +49,16 @@ func (s *violationSerializationTestSuite) SetupTest() { s.alertStore = alertStore } -func (s *violationSerializationTestSuite) TestViolationSerialization() { +func (s *violationSerializationTestSuite) queryViolations(alert *storage.Alert) string { ctx := sac.WithAllAccess(context.Background()) - // Inject alert - alert := fixtures.GetSerializationTestAlert() - alert.Time = violationTimestamp err := s.alertStore.UpsertAlert(ctx, alert) s.Require().NoError(err) - // Serve with the target function handler := newViolationsHandler(s.alertStore, defaultPaginationSettings) server := httptest.NewServer(wrapHandler(ctx, handler)) defer server.Close() - // Query server client := server.Client() client.Timeout = 5 * time.Second requestBody := bytes.NewBufferString("") @@ -65,55 +68,85 @@ func (s *violationSerializationTestSuite) TestViolationSerialization() { resp, err := client.Do(req) s.Require().NoError(err) - // Validate response - respBody := resp.Body - defer func() { s.NoError(respBody.Close()) }() + defer func() { s.NoError(resp.Body.Close()) }() + + respBodyData, err := io.ReadAll(resp.Body) + s.Require().NoError(err) + return string(respBodyData) +} + +func (s *violationSerializationTestSuite) TestViolationSerialization() { + alert := fixtures.GetSerializationTestAlert() + alert.Time = violationTimestamp + + s.JSONEq(expectedGenericViolation, s.queryViolations(alert)) +} - expectedViolationResponse := `{ - "newCheckpoint": "2024-06-26T22:00:00Z", - "violations": [ - { - "alertInfo": { - "alertId": "aeaaaaaa-bbbb-4011-0000-111111111111" +func (s *violationSerializationTestSuite) TestFileAccessViolationSerialization() { + alert := &storage.Alert{ + Id: "fa1eacce-0000-4000-a000-000000000001", + Policy: &storage.Policy{ + Id: "fa1eacce-0000-4000-b000-000000000002", + Name: "File Access: /etc/passwd", + Description: "Detect modifications to /etc/passwd", + Severity: storage.Severity_HIGH_SEVERITY, + LifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_RUNTIME}, + PolicyVersion: "1.1", + }, + LifecycleStage: storage.LifecycleStage_RUNTIME, + Entity: &storage.Alert_Deployment_{ + Deployment: &storage.Alert_Deployment{ + Id: "fa1eacce-0000-4000-c000-000000000003", + Name: "nginx", + Type: "Deployment", + Namespace: "default", + ClusterId: "fa1eacce-0000-4000-d000-000000000004", }, - "deploymentInfo": {}, - "violationInfo": { - "containerName": "nginx", - "podId": "nginx", - "violationId": "f073c9f5-c766-5dc4-b7cb-aa8ac8f2d445", - "violationMessage": "This is a kube event violation", - "violationMessageAttributes": [ - { - "key": "pod", - "value": "nginx" - }, - { - "key": "container", - "value": "nginx" - } - ], - "violationTime": "2024-06-14T20:00:00Z", - "violationType": "GENERIC" - } }, - { - "alertInfo": { - "alertId": "aeaaaaaa-bbbb-4011-0000-111111111111" + Violations: []*storage.Alert_Violation{ + { + Message: "'/etc/passwd' opened (writable)", + MessageAttributes: &storage.Alert_Violation_FileAccess{ + FileAccess: &storage.FileAccess{ + File: &storage.FileAccess_File{ + EffectivePath: "/etc/passwd", + ActualPath: "/rootfs/etc/passwd", + Meta: &storage.FileAccess_FileMetadata{ + Uid: 0, + Gid: 0, + Mode: 0644, + Username: "root", + Group: "root", + }, + }, + Operation: storage.FileAccess_OPEN, + Timestamp: violationTimestamp, + Process: &storage.ProcessIndicator{ + Id: "fa1eacce-0000-4000-f000-000000000006", + DeploymentId: "fa1eacce-0000-4000-c000-000000000003", + ContainerName: "nginx", + PodId: "nginx-pod", + Signal: &storage.ProcessSignal{ + Id: "fa1eacce-0000-4000-f000-000000000007", + Name: "vi", + Args: "/etc/passwd", + ExecFilePath: "/usr/bin/vi", + Pid: 42, + Uid: 0, + Gid: 0, + Time: violationTimestamp, + }, + }, + Hostname: "node-1", + }, + }, + Type: storage.Alert_Violation_FILE_ACCESS, }, - "deploymentInfo": {}, - "violationInfo": { - "violationId": "aeaaaaaa-bbbb-4011-0000-111111111111", - "violationMessage": "Deployment is affected by 'CVE-2017-15670'", - "violationTime": "2024-06-14T20:00:00Z", - "violationType": "GENERIC" - } - } - ] -}` - - respBodyData, err := io.ReadAll(respBody) - s.NoError(err) - s.JSONEq(expectedViolationResponse, string(respBodyData)) + }, + Time: violationTimestamp, + } + + s.JSONEq(expectedFileAccessViolation, s.queryViolations(alert)) } func wrapHandler(ctx context.Context, handler http.HandlerFunc) http.HandlerFunc { diff --git a/central/splunk/violations_test.go b/central/splunk/violations_test.go index 3b69e17a6ecff..6a6397caddd3a 100644 --- a/central/splunk/violations_test.go +++ b/central/splunk/violations_test.go @@ -53,19 +53,17 @@ var ( }, LifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_BUILD, storage.LifecycleStage_DEPLOY}, Exclusions: []*storage.Exclusion{{ - Name: "Don't alert on kube-system namespace", - Deployment: &storage.Exclusion_Deployment{ + Name: "Don't alert on kube-system namespace", Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ Scope: &storage.Scope{ Namespace: "kube-system", }, - }, + }}, }, { - Name: "Don't alert on istio-system namespace", - Deployment: &storage.Exclusion_Deployment{ + Name: "Don't alert on istio-system namespace", Matcher: &storage.Exclusion_Deployment_{Deployment: &storage.Exclusion_Deployment{ Scope: &storage.Scope{ Namespace: "istio-system", }, - }, + }}, }}, Severity: storage.Severity_LOW_SEVERITY, SORTName: "90-Day Image Age", @@ -613,6 +611,111 @@ var ( Time: makeTimestamp("2021-07-15T17:36:35.115310605Z"), FirstOccurred: makeTimestamp("2021-07-15T17:26:35.115310605Z"), } + + fileAccessAlert = storage.Alert{ + Id: "fa1eacce-5500-4000-a000-111111111111", + Policy: &storage.Policy{ + Id: "fa1eacce-5500-4000-b000-222222222222", + Name: "File Access: /etc/passwd modified", + Description: "Alert on modifications to /etc/passwd", + Rationale: "Modifications to /etc/passwd may indicate credential tampering", + Categories: []string{"Security Best Practices"}, + LifecycleStages: []storage.LifecycleStage{storage.LifecycleStage_RUNTIME}, + Severity: storage.Severity_HIGH_SEVERITY, + PolicyVersion: "1.1", + }, + LifecycleStage: storage.LifecycleStage_RUNTIME, + Entity: &storage.Alert_Deployment_{ + Deployment: &storage.Alert_Deployment{ + Id: "fa1eacce-5500-4000-c000-333333333333", + Name: "nginx", + Type: "Deployment", + Namespace: "default", + NamespaceId: "fa1eacce-5500-4000-d000-444444444444", + ClusterId: "fa1eacce-5500-4000-e000-555555555555", + ClusterName: "remote", + }, + }, + Violations: []*storage.Alert_Violation{ + { + Message: "'/etc/passwd' opened (writable)", + MessageAttributes: &storage.Alert_Violation_FileAccess{ + FileAccess: &storage.FileAccess{ + File: &storage.FileAccess_File{ + EffectivePath: "/etc/passwd", + ActualPath: "/rootfs/etc/passwd", + Meta: &storage.FileAccess_FileMetadata{ + Uid: 0, + Gid: 0, + Mode: 0644, + Username: "root", + Group: "root", + }, + }, + Operation: storage.FileAccess_OPEN, + Timestamp: makeTimestamp("2021-08-10T14:30:00.123456789Z"), + Process: &storage.ProcessIndicator{ + Id: "fa1eacce-5500-4000-f000-666666666666", + DeploymentId: "fa1eacce-5500-4000-c000-333333333333", + Namespace: "default", + ClusterId: "fa1eacce-5500-4000-e000-555555555555", + ContainerName: "nginx", + PodId: "nginx-abc123", + Signal: &storage.ProcessSignal{ + Name: "vi", + Args: "/etc/passwd", + ExecFilePath: "/usr/bin/vi", + Pid: 1234, + Uid: 0, + Gid: 0, + Time: makeTimestamp("2021-08-10T14:29:59.000000000Z"), + }, + }, + Hostname: "worker-node-01", + }, + }, + Type: storage.Alert_Violation_FILE_ACCESS, + }, + { + Message: "'/var/log/app.log' renamed", + MessageAttributes: &storage.Alert_Violation_FileAccess{ + FileAccess: &storage.FileAccess{ + File: &storage.FileAccess_File{ + EffectivePath: "/var/log/app.log", + ActualPath: "/rootfs/var/log/app.log", + }, + Operation: storage.FileAccess_RENAME, + Moved: &storage.FileAccess_File{ + EffectivePath: "/var/log/app.log.bak", + ActualPath: "/rootfs/var/log/app.log.bak", + }, + Timestamp: makeTimestamp("2021-08-10T14:31:00.987654321Z"), + Process: &storage.ProcessIndicator{ + Id: "fa1eacce-5500-4000-f000-777777777777", + DeploymentId: "fa1eacce-5500-4000-c000-333333333333", + Namespace: "default", + ClusterId: "fa1eacce-5500-4000-e000-555555555555", + ContainerName: "nginx", + PodId: "nginx-abc123", + Signal: &storage.ProcessSignal{ + Name: "mv", + Args: "/var/log/app.log /var/log/app.log.bak", + ExecFilePath: "/usr/bin/mv", + Pid: 1235, + Uid: 0, + Gid: 0, + Time: makeTimestamp("2021-08-10T14:31:00.000000000Z"), + }, + }, + Hostname: "worker-node-01", + }, + }, + Type: storage.Alert_Violation_FILE_ACCESS, + }, + }, + Time: makeTimestamp("2021-08-10T14:31:00.987654321Z"), + FirstOccurred: makeTimestamp("2021-08-10T14:30:00.123456789Z"), + } ) func TestViolations(t *testing.T) { @@ -625,6 +728,7 @@ func (s *violationsTestSuite) SetupTest() { s.k8sAlert = k8sAlert.CloneVT() s.networkAlert = networkAlert.CloneVT() s.resourceAlert = resourceAlert.CloneVT() + s.fileAccessAlert = fileAccessAlert.CloneVT() s.allowCtx = sac.WithAllAccess(context.Background()) } @@ -777,6 +881,53 @@ func (s *violationsTestSuite) TestResourceAlert() { } +func (s *violationsTestSuite) TestFileAccessAlert() { + vs := s.getViolations(s.prepare().setAlerts(s.fileAccessAlert).runRequestAndGetBody()) + s.Len(vs, 2) + + for _, v := range vs { + s.Equal("FILE_ACCESS", s.extr(v, ".violationInfo.violationType")) + s.checkViolationInfo(v) + s.checkAlertInfo(v, ".lifecycleStage") + s.assertPresent(v, ".deploymentInfo", ".deploymentId", ".deploymentName", ".deploymentType", + ".deploymentNamespace", ".clusterId", ".clusterName") + s.checkPolicy(v) + + // File access violations should have process info from FileAccess.process + s.assertPresent(v, ".processInfo", ".processName", ".execFilePath", ".pid") + s.Equal(float64(0), s.extr(v, ".processInfo.processUid")) + s.Equal(float64(0), s.extr(v, ".processInfo.processGid")) + + // File access info should be present + s.assertPresent(v, ".fileAccessInfo", ".effectivePath", ".actualPath", ".operation", ".hostname") + s.Equal("worker-node-01", s.extr(v, ".fileAccessInfo.hostname")) + } + + // First violation: OPEN on /etc/passwd + openViolation := vs[0] + s.Equal("2021-08-10T14:30:00.123456789Z", s.extr(openViolation, ".violationInfo.violationTime")) + s.Equal("OPEN", s.extr(openViolation, ".fileAccessInfo.operation")) + s.Equal("/etc/passwd", s.extr(openViolation, ".fileAccessInfo.effectivePath")) + s.Equal("/rootfs/etc/passwd", s.extr(openViolation, ".fileAccessInfo.actualPath")) + s.Equal("vi", s.extr(openViolation, ".processInfo.processName")) + // File metadata + s.Equal(float64(0), s.extr(openViolation, ".fileAccessInfo.fileUid")) + s.Equal(float64(0), s.extr(openViolation, ".fileAccessInfo.fileGid")) + s.Equal(float64(0644), s.extr(openViolation, ".fileAccessInfo.fileMode")) + s.Equal("root", s.extr(openViolation, ".fileAccessInfo.fileUsername")) + s.Equal("root", s.extr(openViolation, ".fileAccessInfo.fileGroup")) + + // Second violation: RENAME + renameViolation := vs[1] + s.Equal("2021-08-10T14:31:00.987654321Z", s.extr(renameViolation, ".violationInfo.violationTime")) + s.Equal("RENAME", s.extr(renameViolation, ".fileAccessInfo.operation")) + s.Equal("/var/log/app.log", s.extr(renameViolation, ".fileAccessInfo.effectivePath")) + s.Equal("mv", s.extr(renameViolation, ".processInfo.processName")) + // Rename destination paths + s.Equal("/var/log/app.log.bak", s.extr(renameViolation, ".fileAccessInfo.movedEffectivePath")) + s.Equal("/rootfs/var/log/app.log.bak", s.extr(renameViolation, ".fileAccessInfo.movedActualPath")) +} + func (s *violationsTestSuite) TestViolationIdsAreDistinct() { vs := s.getViolations(s.prepare().setAlerts(s.processAlert, s.k8sAlert, s.deployAlert, s.networkAlert).runRequestAndGetBody()) @@ -1190,6 +1341,57 @@ func (s *violationsTestSuite) TestCheckpointTimestampFiltering() { } } +func (s *violationsTestSuite) makeDelayedProcessAlert(signalTime, alertTime string) *storage.Alert { + alert := s.processAlert.CloneVT() + alert.ProcessViolation.Processes = alert.GetProcessViolation().GetProcesses()[:1] + alert.ProcessViolation.Processes[0].Signal.Time = makeTimestamp(signalTime) + alert.Time = makeTimestamp(alertTime) + return alert +} + +func (s *violationsTestSuite) TestProcessViolationDelayedStorage() { + // Signal time is 50 seconds before alert time, simulating the pipeline delay + // from sensor -> central -> enrichment -> storage. + alert := s.makeDelayedProcessAlert("2021-02-01T17:15:00Z", "2021-02-01T17:15:50Z") + + // Checkpoint window: 17:15:30 to 17:16:00 + // Signal time (17:15:00) is before fromTime (17:15:30) — would normally be excluded. + // Alert time (17:15:50) is within [fromTime, toTime] — fallback should include it. + vs := s.getViolations(s.prepare(). + setCheckpoint("2021-02-01T17:15:30Z__2021-02-01T17:16:00Z"). + setAlerts(alert).runRequestAndGetBody()) + s.Len(vs, 1) + // The reported violation time should still be the signal time, not alert time. + s.Equal("2021-02-01T17:15:00Z", s.extr(vs[0], ".violationInfo.violationTime")) +} + +func (s *violationsTestSuite) TestProcessViolationTooOldSignalNotRecovered() { + // Signal time is more than maxProcessDetectionDelay (2 min) before alert time. + alert := s.makeDelayedProcessAlert("2021-02-01T17:12:00Z", "2021-02-01T17:15:50Z") + + // Checkpoint window includes alert time but signal time is >2 min before alert time. + vs := s.getViolations(s.prepare(). + setCheckpoint("2021-02-01T17:15:30Z__2021-02-01T17:16:00Z"). + setAlerts(alert).runRequestAndGetBody()) + s.Len(vs, 0) +} + +func (s *violationsTestSuite) TestProcessViolationDelayedStorageNoDuplicates() { + alert := s.makeDelayedProcessAlert("2021-02-01T17:15:00Z", "2021-02-01T17:15:50Z") + + // First window: should include the violation via fallback. + vs1 := s.getViolations(s.prepare(). + setCheckpoint("2021-02-01T17:15:30Z__2021-02-01T17:16:00Z"). + setAlerts(alert).runRequestAndGetBody()) + s.Len(vs1, 1) + + // Second window: checkpoint advances, should NOT include again. + vs2 := s.getViolations(s.prepare(). + setCheckpoint("2021-02-01T17:16:00Z__2021-02-01T17:16:30Z"). + setAlerts(alert).runRequestAndGetBody()) + s.Len(vs2, 0) +} + func (s *violationsTestSuite) TestCheckpointFromAlertIDFiltering() { smallerID, biggerID := s.processAlert.GetId(), s.k8sAlert.GetId() if smallerID > biggerID { diff --git a/central/splunk/vm.go b/central/splunk/vm.go index e333558e598cd..5c7cc8d9d13e1 100644 --- a/central/splunk/vm.go +++ b/central/splunk/vm.go @@ -6,6 +6,9 @@ import ( "github.com/stackrox/rox/central/deployment/datastore" imageDatastore "github.com/stackrox/rox/central/image/datastore" + imageV2Datastore "github.com/stackrox/rox/central/imagev2/datastore" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/httputil" "github.com/stackrox/rox/pkg/jsonutil" "github.com/stackrox/rox/pkg/protoconv" @@ -44,8 +47,16 @@ type splunkCVEEvent struct { Source string `json:"source"` } +// imageFields is satisfied by both *storage.Image and *storage.ImageV2, +// providing access to the shared fields needed for Splunk events. +type imageFields interface { + GetScan() *storage.ImageScan + GetMetadata() *storage.ImageMetadata + GetName() *storage.ImageName +} + // NewVulnMgmtHandler returns an http.HandlerFunc implementation that returns all the required events for the Splunk TA -func NewVulnMgmtHandler(deployments datastore.DataStore, images imageDatastore.DataStore) http.HandlerFunc { +func NewVulnMgmtHandler(deployments datastore.DataStore, images imageDatastore.DataStore, imagesV2 imageV2Datastore.DataStore) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { arrayWriter := jsonutil.NewJSONArrayWriter(w) if err := arrayWriter.Init(); err != nil { @@ -59,6 +70,8 @@ func NewVulnMgmtHandler(deployments datastore.DataStore, images imageDatastore.D return } + v2Enabled := features.FlattenImageData.Enabled() + imageSet := set.NewStringSet() for _, id := range ids { deployment, exists, err := deployments.GetDeployment(r.Context(), id) @@ -70,10 +83,11 @@ func NewVulnMgmtHandler(deployments datastore.DataStore, images imageDatastore.D continue } for _, c := range deployment.GetContainers() { - if c.GetImage().GetId() == "" { + imgID := containerImageID(c.GetImage(), v2Enabled) + if imgID == "" { continue } - imageSet.Add(c.GetImage().GetId()) + imageSet.Add(imgID) err := arrayWriter.WriteObject(&splunkDeploymentEvent{ Type: "deployment", @@ -90,52 +104,86 @@ func NewVulnMgmtHandler(deployments datastore.DataStore, images imageDatastore.D } } } - for id := range imageSet { - image, exists, err := images.GetImage(r.Context(), id) - if err != nil { - httputil.WriteError(w, err) - return - } - if !exists { - continue + + for imgID := range imageSet { + var img imageFields + var digest string + if v2Enabled { + v2, exists, err := imagesV2.GetImage(r.Context(), imgID) + if err != nil { + httputil.WriteError(w, err) + return + } + if !exists { + continue + } + img = v2 + digest = v2.GetDigest() + } else { + v1, exists, err := images.GetImage(r.Context(), imgID) + if err != nil { + httputil.WriteError(w, err) + return + } + if !exists { + continue + } + img = v1 + digest = imgID } - err = arrayWriter.WriteObject(&splunkImageEvent{ - Type: "image", - ImageDigest: id, - OS: image.GetScan().GetOperatingSystem(), - CreatedTime: protoconv.ConvertTimestampToTimeOrNow(image.GetMetadata().GetV1().GetCreated()), - ImageRegistry: image.GetName().GetRegistry(), - ImageRepo: image.GetName().GetRemote(), - ImageTag: image.GetName().GetTag(), - }) - if err != nil { + if err := writeImageEvents(arrayWriter, digest, img); err != nil { httputil.WriteError(w, err) return } - - for _, c := range image.GetScan().GetComponents() { - for _, v := range c.GetVulns() { - err = arrayWriter.WriteObject(&splunkCVEEvent{ - Type: "cve", - ImageDigest: id, - Component: c.GetName(), - Version: c.GetVersion(), - CVE: v.GetCve(), - CVSS: v.GetCvss(), - FixedBy: v.GetFixedBy(), - FirstSeen: protoconv.ConvertTimestampToTimeOrNow(v.GetFirstImageOccurrence()), - Source: c.GetSource().String(), - }) - if err != nil { - httputil.WriteError(w, err) - return - } - } - } } + if err := arrayWriter.Finish(); err != nil { httputil.WriteError(w, err) } } } + +func writeImageEvents(w *jsonutil.JSONArrayWriter, digest string, img imageFields) error { + err := w.WriteObject(&splunkImageEvent{ + Type: "image", + ImageDigest: digest, + OS: img.GetScan().GetOperatingSystem(), + CreatedTime: protoconv.ConvertTimestampToTimeOrNow(img.GetMetadata().GetV1().GetCreated()), + ImageRegistry: img.GetName().GetRegistry(), + ImageRepo: img.GetName().GetRemote(), + ImageTag: img.GetName().GetTag(), + }) + if err != nil { + return err + } + + for _, c := range img.GetScan().GetComponents() { + for _, v := range c.GetVulns() { + err = w.WriteObject(&splunkCVEEvent{ + Type: "cve", + ImageDigest: digest, + Component: c.GetName(), + Version: c.GetVersion(), + CVE: v.GetCve(), + CVSS: v.GetCvss(), + FixedBy: v.GetFixedBy(), + FirstSeen: protoconv.ConvertTimestampToTimeOrNow(v.GetFirstImageOccurrence()), + Source: c.GetSource().String(), + }) + if err != nil { + return err + } + } + } + return nil +} + +// containerImageID returns the appropriate image identifier for datastore lookups. +// V1 uses the SHA digest (Id), V2 uses the UUID (IdV2). +func containerImageID(ci *storage.ContainerImage, v2Enabled bool) string { + if v2Enabled { + return ci.GetIdV2() + } + return ci.GetId() +} diff --git a/central/splunk/vm_test.go b/central/splunk/vm_test.go new file mode 100644 index 0000000000000..f55a3832d5d75 --- /dev/null +++ b/central/splunk/vm_test.go @@ -0,0 +1,222 @@ +package splunk + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + deploymentMocks "github.com/stackrox/rox/central/deployment/datastore/mocks" + imageMocks "github.com/stackrox/rox/central/image/datastore/mocks" + imageV2Mocks "github.com/stackrox/rox/central/imagev2/datastore/mocks" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestVulnMgmtHandler_V1(t *testing.T) { + t.Setenv(features.FlattenImageData.EnvVar(), "false") + + const ( + deploymentID = "dep-1" + imageDigest = "sha256:abc123" + imageUUID = "uuid-v2-001" + ) + + ctrl := gomock.NewController(t) + deployments := deploymentMocks.NewMockDataStore(ctrl) + images := imageMocks.NewMockDataStore(ctrl) + imagesV2 := imageV2Mocks.NewMockDataStore(ctrl) + + deployments.EXPECT().GetDeploymentIDs(gomock.Any()).Return([]string{deploymentID}, nil) + deployments.EXPECT().GetDeployment(gomock.Any(), deploymentID).Return(&storage.Deployment{ + Id: deploymentID, + Name: "my-deploy", + ClusterName: "cluster-1", + Namespace: "default", + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{Id: imageDigest, IdV2: imageUUID}, + }, + }, + }, true, nil) + + // V1 path: lookup by digest, NOT by UUID. + images.EXPECT().GetImage(gomock.Any(), imageDigest).Return(&storage.Image{ + Id: imageDigest, + Name: &storage.ImageName{ + Registry: "docker.io", + Remote: "library/nginx", + Tag: "latest", + }, + Metadata: &storage.ImageMetadata{ + V1: &storage.V1Metadata{Created: timestamppb.Now()}, + }, + Scan: &storage.ImageScan{ + OperatingSystem: "linux", + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "openssl", + Version: "1.1.1", + Vulns: []*storage.EmbeddedVulnerability{ + {Cve: "CVE-2021-0001", Cvss: 7.5, SetFixedBy: &storage.EmbeddedVulnerability_FixedBy{FixedBy: "1.1.2"}}, + }, + }, + }, + }, + }, true, nil) + + handler := NewVulnMgmtHandler(deployments, images, imagesV2) + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/", nil)) + + assert.Equal(t, http.StatusOK, rr.Code) + + var events []json.RawMessage + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &events)) + require.Len(t, events, 3) // deployment + image + cve + + // Verify deployment event uses SHA digest. + var depEvent splunkDeploymentEvent + require.NoError(t, json.Unmarshal(events[0], &depEvent)) + assert.Equal(t, "deployment", depEvent.Type) + assert.Equal(t, imageDigest, depEvent.ImageDigest) + + // Verify image event uses SHA digest. + var imgEvent splunkImageEvent + require.NoError(t, json.Unmarshal(events[1], &imgEvent)) + assert.Equal(t, "image", imgEvent.Type) + assert.Equal(t, imageDigest, imgEvent.ImageDigest) + + // Verify CVE event uses SHA digest. + var cveEvent splunkCVEEvent + require.NoError(t, json.Unmarshal(events[2], &cveEvent)) + assert.Equal(t, "cve", cveEvent.Type) + assert.Equal(t, imageDigest, cveEvent.ImageDigest) + assert.Equal(t, "CVE-2021-0001", cveEvent.CVE) +} + +func TestVulnMgmtHandler_V2(t *testing.T) { + t.Setenv(features.FlattenImageData.EnvVar(), "true") + + const ( + deploymentID = "dep-1" + imageDigest = "sha256:abc123" + imageUUID = "uuid-v2-001" + ) + + ctrl := gomock.NewController(t) + deployments := deploymentMocks.NewMockDataStore(ctrl) + images := imageMocks.NewMockDataStore(ctrl) + imagesV2 := imageV2Mocks.NewMockDataStore(ctrl) + + deployments.EXPECT().GetDeploymentIDs(gomock.Any()).Return([]string{deploymentID}, nil) + deployments.EXPECT().GetDeployment(gomock.Any(), deploymentID).Return(&storage.Deployment{ + Id: deploymentID, + Name: "my-deploy", + ClusterName: "cluster-1", + Namespace: "default", + Containers: []*storage.Container{ + { + Image: &storage.ContainerImage{Id: imageDigest, IdV2: imageUUID}, + }, + }, + }, true, nil) + + // V2 path: lookup by UUID, NOT by digest. + imagesV2.EXPECT().GetImage(gomock.Any(), imageUUID).Return(&storage.ImageV2{ + Id: imageUUID, + Digest: imageDigest, + Name: &storage.ImageName{ + Registry: "docker.io", + Remote: "library/nginx", + Tag: "latest", + }, + Metadata: &storage.ImageMetadata{ + V1: &storage.V1Metadata{Created: timestamppb.Now()}, + }, + Scan: &storage.ImageScan{ + OperatingSystem: "linux", + Components: []*storage.EmbeddedImageScanComponent{ + { + Name: "openssl", + Version: "1.1.1", + Vulns: []*storage.EmbeddedVulnerability{ + {Cve: "CVE-2021-0001", Cvss: 7.5, SetFixedBy: &storage.EmbeddedVulnerability_FixedBy{FixedBy: "1.1.2"}}, + }, + }, + }, + }, + }, true, nil) + + handler := NewVulnMgmtHandler(deployments, images, imagesV2) + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/", nil)) + + assert.Equal(t, http.StatusOK, rr.Code) + + var events []json.RawMessage + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &events)) + require.Len(t, events, 3) // deployment + image + cve + + // Verify deployment event uses SHA digest, NOT UUID. + var depEvent splunkDeploymentEvent + require.NoError(t, json.Unmarshal(events[0], &depEvent)) + assert.Equal(t, "deployment", depEvent.Type) + assert.Equal(t, imageDigest, depEvent.ImageDigest) + + // Verify image event uses SHA digest, NOT UUID. + var imgEvent splunkImageEvent + require.NoError(t, json.Unmarshal(events[1], &imgEvent)) + assert.Equal(t, "image", imgEvent.Type) + assert.Equal(t, imageDigest, imgEvent.ImageDigest) + + // Verify CVE event uses SHA digest, NOT UUID. + var cveEvent splunkCVEEvent + require.NoError(t, json.Unmarshal(events[2], &cveEvent)) + assert.Equal(t, "cve", cveEvent.Type) + assert.Equal(t, imageDigest, cveEvent.ImageDigest) + assert.Equal(t, "CVE-2021-0001", cveEvent.CVE) +} + +func TestVulnMgmtHandler_V2SkipsEmptyUUID(t *testing.T) { + t.Setenv(features.FlattenImageData.EnvVar(), "true") + + const ( + deploymentID = "dep-1" + imageDigest = "sha256:abc123" + ) + + ctrl := gomock.NewController(t) + deployments := deploymentMocks.NewMockDataStore(ctrl) + images := imageMocks.NewMockDataStore(ctrl) + imagesV2 := imageV2Mocks.NewMockDataStore(ctrl) + + deployments.EXPECT().GetDeploymentIDs(gomock.Any()).Return([]string{deploymentID}, nil) + deployments.EXPECT().GetDeployment(gomock.Any(), deploymentID).Return(&storage.Deployment{ + Id: deploymentID, + Name: "my-deploy", + ClusterName: "cluster-1", + Namespace: "default", + Containers: []*storage.Container{ + { + // IdV2 is empty — this container should be skipped entirely. + Image: &storage.ContainerImage{Id: imageDigest}, + }, + }, + }, true, nil) + + handler := NewVulnMgmtHandler(deployments, images, imagesV2) + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/", nil)) + + assert.Equal(t, http.StatusOK, rr.Code) + + var events []json.RawMessage + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &events)) + // No deployment or image events because the only container has empty IdV2. + assert.Empty(t, events) +} diff --git a/central/telemetry/centralclient/client_test.go b/central/telemetry/centralclient/client_test.go index 61d180b649f04..22bf8f0289f24 100644 --- a/central/telemetry/centralclient/client_test.go +++ b/central/telemetry/centralclient/client_test.go @@ -114,11 +114,9 @@ func Test_centralClient_flow(t *testing.T) { }) var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { c.RegisterCentralClient("basic") - }() + }) c.GrantConsent() diff --git a/central/telemetry/centralclient/interceptors.go b/central/telemetry/centralclient/interceptors.go index ad07161a321fd..bdd34a0bfb68e 100644 --- a/central/telemetry/centralclient/interceptors.go +++ b/central/telemetry/centralclient/interceptors.go @@ -8,20 +8,14 @@ import ( "github.com/stackrox/rox/pkg/telemetry/phonehome" ) -const ( - // The header is set by the RHACS ServiceNow integration. - // See https://github.com/stackrox/service-now/blob/9d1df943f5f0b3052df97c6272814e2303f17685/52616ff6938a1a50c52a72856aba10fd/update/sys_script_include_2b362bbe938a1a50c52a72856aba10b3.xml#L80. - snowIntegrationHeader = "Rh-ServiceNow-Integration" - - userAgentHeaderKey = "User-Agent" -) +const userAgentHeaderKey = "User-Agent" var ( ignoredPaths = glob.Pattern("{/v1/ping,/v1.PingService/Ping,/v1/metadata,/static/*}") permanentTelemetryCampaign = phonehome.APICallCampaign{ { - Headers: map[string]glob.Pattern{ + Headers: phonehome.GlobMap{ userAgentHeaderKey: "*roxctl*", clientconn.RoxctlCommandHeader: phonehome.NoHeaderOrAnyValue, clientconn.RoxctlCommandIndexHeader: phonehome.NoHeaderOrAnyValue, @@ -30,12 +24,12 @@ var ( }, { Path: glob.Pattern("/v1/clusters").Ptr(), - Headers: map[string]glob.Pattern{ + Headers: phonehome.GlobMap{ // ServiceNow default User-Agent includes "ServiceNow", but // customers are free to change it. // See https://support.servicenow.com/kb?id=kb_article_view&sysparm_article=KB1511513. - userAgentHeaderKey: "*ServiceNow*", - snowIntegrationHeader: phonehome.NoHeaderOrAnyValue, + userAgentHeaderKey: "*ServiceNow*", + "Rh-*": phonehome.NoHeaderOrAnyValue, }, }, // Capture requests from GitHub action user agents. @@ -46,7 +40,7 @@ var ( // Capture SBOM generation requests. Corresponding handler in central/image/service/http_handler.go. Path: glob.Pattern("/api/v1/images/sbom").Ptr(), Method: glob.Pattern("POST").Ptr(), - Headers: map[string]glob.Pattern{userAgentHeaderKey: phonehome.NoHeaderOrAnyValue}, + Headers: phonehome.GlobMap{userAgentHeaderKey: phonehome.NoHeaderOrAnyValue}, }, // Capture Jenkins Plugin requests phonehome.HeaderPattern(userAgentHeaderKey, "*stackrox-container-image-scanner*"), @@ -55,13 +49,16 @@ var ( } ) -// apiPathsCampaign constructs an API paths campaign from the apiWhiteList -// environment variable. +// apiPathsCampaign constructs an APICallCampaignCriterion from the apiWhiteList +// setting when that setting is non-empty. +// The criterion matches requests whose path fits the provided glob pattern and +// requires only the presence (or any value) of a User-Agent header. +// Returns nil when the apiWhiteList setting is empty. func apiPathsCampaign() *phonehome.APICallCampaignCriterion { if pattern := apiWhiteList.Setting(); pattern != "" { return &phonehome.APICallCampaignCriterion{ Path: glob.Pattern("{" + pattern + "}").Ptr(), - Headers: map[string]glob.Pattern{ + Headers: phonehome.GlobMap{ userAgentHeaderKey: phonehome.NoHeaderOrAnyValue, }, } @@ -69,11 +66,13 @@ func apiPathsCampaign() *phonehome.APICallCampaignCriterion { return nil } -// userAgentsCampaign constructs an User-Agent campaign from the userAgentsList -// environment variable. +// userAgentsCampaign constructs an APICallCampaignCriterion that matches the +// "User-Agent" header against the glob pattern defined by the userAgentsList +// setting. +// If the setting is empty, it returns nil. func userAgentsCampaign() *phonehome.APICallCampaignCriterion { if pattern := userAgentsList.Setting(); pattern != "" { - return phonehome.HeaderPattern(userAgentHeaderKey, "{"+pattern+"}") + return phonehome.HeaderPattern(userAgentHeaderKey, glob.Pattern("{"+pattern+"}")) } return nil } @@ -94,22 +93,10 @@ func (c *CentralClient) apiCallInterceptor() phonehome.Interceptor { c.campaignMux.RLock() defer c.campaignMux.RUnlock() return !ignoredPaths.Match(rp.Path) && c.telemetryCampaign.CountFulfilled(rp, - func(cc *phonehome.APICallCampaignCriterion) { - addCustomHeaders(rp, cc, props) + func(_ *phonehome.APICallCampaignCriterion, h phonehome.Headers) { + for k, values := range h { + props[k] = strings.Join(values, "; ") + } }) > 0 } } - -// addCustomHeaders adds additional properties to the event if the telemetry -// campaign criterion contains a header pattern condition. -func addCustomHeaders(rp *phonehome.RequestParams, cc *phonehome.APICallCampaignCriterion, props map[string]any) { - if rp.Headers == nil || cc == nil { - return - } - for header := range cc.Headers { - values := rp.Headers(header) - if len(values) != 0 { - props[header] = strings.Join(values, "; ") - } - } -} diff --git a/central/telemetry/centralclient/interceptors_test.go b/central/telemetry/centralclient/interceptors_test.go index 3332e688eab05..8c1b61057a276 100644 --- a/central/telemetry/centralclient/interceptors_test.go +++ b/central/telemetry/centralclient/interceptors_test.go @@ -1,22 +1,19 @@ package centralclient import ( - "net/http" "testing" - "github.com/stackrox/rox/pkg/clientconn" "github.com/stackrox/rox/pkg/telemetry/phonehome" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func withUserAgent(_ *testing.T, headers map[string][]string, ua string) func(string) []string { - return func(key string) []string { - if http.CanonicalHeaderKey(key) == userAgentHeaderKey { - return []string{ua} - } - return headers[key] - } +// The header is set by the RHACS ServiceNow integration. +// See https://github.com/stackrox/service-now/blob/9d1df943f5f0b3052df97c6272814e2303f17685/52616ff6938a1a50c52a72856aba10fd/update/sys_script_include_2b362bbe938a1a50c52a72856aba10b3.xml#L80. +const snowIntegrationHeader = "Rh-Servicenow-Integration" + +func withUserAgent(ua string) phonehome.Headers { + return phonehome.Headers{userAgentHeaderKey: {ua}} } func Test_apiCall(t *testing.T) { @@ -28,7 +25,7 @@ func Test_apiCall(t *testing.T) { }{ "roxctl": { rp: &phonehome.RequestParams{ - Headers: withUserAgent(t, nil, "Some roxctl client"), + Headers: withUserAgent("Some roxctl client"), Method: "GET", Path: "/v1/endpoint", Code: 200, @@ -38,7 +35,7 @@ func Test_apiCall(t *testing.T) { }, "not roxctl": { rp: &phonehome.RequestParams{ - Headers: withUserAgent(t, nil, "Some other client"), + Headers: withUserAgent("Some other client"), Method: "GET", Path: "/v1/endpoint", Code: 200, @@ -48,7 +45,7 @@ func Test_apiCall(t *testing.T) { }, "don't catch user-agent": { rp: &phonehome.RequestParams{ - Headers: withUserAgent(t, nil, "Some client"), + Headers: withUserAgent("Some client"), Method: "GET", Path: "/v1/test-endpoint", Code: 200, @@ -58,7 +55,7 @@ func Test_apiCall(t *testing.T) { }, "roxctl ignored path": { rp: &phonehome.RequestParams{ - Headers: withUserAgent(t, nil, "Some roxctl client"), + Headers: withUserAgent("Some roxctl client"), Method: "GET", Path: "/v1/ping", Code: 200, @@ -68,7 +65,7 @@ func Test_apiCall(t *testing.T) { }, "ServiceNow clusters": { rp: &phonehome.RequestParams{ - Headers: withUserAgent(t, nil, "Some ServiceNow client"), + Headers: withUserAgent("Some ServiceNow client"), Method: "GET", Path: "/v1/clusters", Code: 200, @@ -78,7 +75,7 @@ func Test_apiCall(t *testing.T) { }, "ServiceNow deployments": { rp: &phonehome.RequestParams{ - Headers: withUserAgent(t, nil, "Some ServiceNow client"), + Headers: withUserAgent("Some ServiceNow client"), Method: "GET", Path: "/v1/deployments", Code: 200, @@ -88,9 +85,10 @@ func Test_apiCall(t *testing.T) { }, "ServiceNow from integration": { rp: &phonehome.RequestParams{ - Headers: withUserAgent(t, map[string][]string{ + Headers: phonehome.Headers{ snowIntegrationHeader: {"v1.0.3"}, - }, "RHACS Integration ServiceNow client"), + "User-Agent": {"RHACS Integration ServiceNow client"}, + }, Method: "GET", Path: "/v1/clusters", Code: 200, @@ -103,7 +101,7 @@ func Test_apiCall(t *testing.T) { }, "central-login GitHub action": { rp: &phonehome.RequestParams{ - Headers: withUserAgent(t, nil, "central-login-GHA"), + Headers: withUserAgent("central-login-GHA"), Method: "POST", Path: "/v1/auth/m2m/exchange", }, @@ -114,7 +112,7 @@ func Test_apiCall(t *testing.T) { }, "roxctl-installer GitHub action": { rp: &phonehome.RequestParams{ - Headers: withUserAgent(t, nil, "roxctl-installer-GHA"), + Headers: withUserAgent("roxctl-installer-GHA"), Method: "GET", Path: "/api/cli/download/roxctl-linux-amd64", }, @@ -125,7 +123,7 @@ func Test_apiCall(t *testing.T) { }, "SBOM generation": { rp: &phonehome.RequestParams{ - Headers: withUserAgent(t, nil, "Some SBOM client"), + Headers: withUserAgent("Some SBOM client"), Method: "POST", Path: "/api/v1/images/sbom", Code: 200, @@ -149,87 +147,3 @@ func Test_apiCall(t *testing.T) { }) } } - -func Test_addCustomHeaders(t *testing.T) { - tc := permanentTelemetryCampaign - require.NoError(t, tc.Compile()) - t.Run(snowIntegrationHeader, func(t *testing.T) { - rp := &phonehome.RequestParams{ - Method: "GET", - Path: "/v1/clusters", - Code: 200, - Headers: func(h string) []string { - return map[string][]string{ - userAgentHeaderKey: {"RHACS Integration ServiceNow client"}, - snowIntegrationHeader: {"v1.0.3", "beta"}, - }[h] - }, - } - props := map[string]any{} - addCustomHeaders(rp, tc[1], props) - assert.Equal(t, map[string]any{ - userAgentHeaderKey: "RHACS Integration ServiceNow client", - snowIntegrationHeader: "v1.0.3; beta", - }, props) - }) - t.Run("3rd-party Integration", func(t *testing.T) { - rp := &phonehome.RequestParams{ - Method: "GET", - Path: "/v1/clusters", - Code: 200, - Headers: func(h string) []string { - return map[string][]string{ - userAgentHeaderKey: {"ServiceNow"}, - "3rd-party-integration": {"v1.0.3", "beta"}, - }[h] - }, - } - props := map[string]any{} - addCustomHeaders(rp, tc[1], props) - assert.Equal(t, map[string]any{ - userAgentHeaderKey: "ServiceNow", - }, props) - }) - t.Run("roxctl", func(t *testing.T) { - rp := &phonehome.RequestParams{ - Method: "GET", - Path: "/v1/clusters", - Code: 200, - Headers: func(h string) []string { - return map[string][]string{ - userAgentHeaderKey: {"roxctl"}, - clientconn.RoxctlCommandHeader: {"central"}, - clientconn.RoxctlCommandIndexHeader: {"1"}, - clientconn.ExecutionEnvironment: {"github"}, - }[h] - }, - } - props := map[string]any{} - addCustomHeaders(rp, tc[0], props) - assert.Equal(t, map[string]any{ - userAgentHeaderKey: "roxctl", - clientconn.RoxctlCommandHeader: "central", - clientconn.RoxctlCommandIndexHeader: "1", - clientconn.ExecutionEnvironment: "github", - }, props) - }) - t.Run("add header from the single criterion", func(t *testing.T) { - tc = append(tc, phonehome.HeaderPattern("Custom-Header", "")) - require.NoError(t, tc.Compile()) - rp := &phonehome.RequestParams{ - Method: "GET", - Path: "/v1/config", - Code: 200, - Headers: func(h string) []string { - return map[string][]string{ - userAgentHeaderKey: {"roxctl"}, - }[h] - }, - } - props := map[string]any{} - addCustomHeaders(rp, tc[0], props) - assert.Equal(t, map[string]any{ - userAgentHeaderKey: "roxctl", - }, props) - }) -} diff --git a/central/testutils/export.go b/central/testutils/export.go index 28f4a91130ece..7f91508231468 100644 --- a/central/testutils/export.go +++ b/central/testutils/export.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "math/rand" "os" "slices" @@ -14,10 +15,12 @@ import ( deploymentDataStore "github.com/stackrox/rox/central/deployment/datastore" imageDataStore "github.com/stackrox/rox/central/image/datastore" imageV2DataStore "github.com/stackrox/rox/central/imagev2/datastore" + imageMapperDatastore "github.com/stackrox/rox/central/imagev2/datastore/mapper/datastore" podDatastore "github.com/stackrox/rox/central/pod/datastore" deploymentsView "github.com/stackrox/rox/central/views/deployments" imagesView "github.com/stackrox/rox/central/views/images" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/jsonutil" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/postgres/pgtest" @@ -70,7 +73,11 @@ func (h *ExportServicePostgresTestHelper) SetupTest(tb testing.TB) error { } h.Deployments = deploymentStore h.DeploymentView = deploymentsView.NewDeploymentView(h.pool) - h.Images = imageDataStore.GetTestPostgresDataStore(tb, h.pool) + if features.FlattenImageData.Enabled() { + h.Images = imageMapperDatastore.GetTestPostgresDataStore(tb, h.pool) + } else { + h.Images = imageDataStore.GetTestPostgresDataStore(tb, h.pool) + } h.ImagesV2 = imageV2DataStore.GetTestPostgresDataStore(tb, h.pool) h.ImageView = imagesView.NewImageView(h.pool) h.Pods = podDatastore.GetTestPostgresDataStore(tb, h.pool) @@ -157,7 +164,7 @@ func (h *ExportServicePostgresTestHelper) InjectImages( copies := count / len(baseImages) extras := count % len(baseImages) upsertCtx := sac.WithAllAccess(context.Background()) - for i := 0; i < len(baseImages); i++ { + for i := range baseImages { copyCount := copies if i < extras { copyCount++ @@ -189,7 +196,7 @@ func (h *ExportServicePostgresTestHelper) InjectDeployments( ) ([]*storage.Deployment, error) { upsertCtx := sac.WithAllAccess(context.Background()) deployments := make([]*storage.Deployment, 0, count) - for i := 0; i < count; i++ { + for i := range count { deployment := &storage.Deployment{} err := testutils.FullInit(deployment, testutils.UniqueInitializer(), testutils.JSONFieldsFilter) if err != nil { @@ -197,7 +204,7 @@ func (h *ExportServicePostgresTestHelper) InjectDeployments( } nbContainers := (i % 3) + 1 containers := make([]*storage.Container, 0, nbContainers) - for j := 0; j < nbContainers; j++ { + for range nbContainers { container := &storage.Container{} err := testutils.FullInit(container, testutils.UniqueInitializer(), testutils.JSONFieldsFilter) if err != nil { @@ -342,9 +349,7 @@ func (h *ExportServicePostgresTestHelper) InjectDataAndRunBenchmark( b.Error(err) } imageIDs = append(imageIDs, addedImageIDs...) - for imageID, imageName := range addedImageNamesByID { - imageNamesByIDs[imageID] = imageName - } + maps.Copy(imageNamesByIDs, addedImageNamesByID) } log.Info("Injecting ", delta, " deployments") deployments, err := h.InjectDeployments(b, delta, imageIDs, imageNamesByIDs) diff --git a/central/tlsconfig/manager.go b/central/tlsconfig/manager.go index 97e7d4307a251..097ef513f97e1 100644 --- a/central/tlsconfig/manager.go +++ b/central/tlsconfig/manager.go @@ -18,6 +18,8 @@ const ( DefaultTLSCertSource ServerCertSource = iota // ServiceCertSource is the source for the StackRox internal service TLS certificate. ServiceCertSource + // OpenShiftTLSCertSource is the OpenShift service-serving certificate for the central-ocp Service. + OpenShiftTLSCertSource ) // ClientCASource is an enum type that determines the source for obtaining TLS client certificate authorities. diff --git a/central/tlsconfig/manager_impl.go b/central/tlsconfig/manager_impl.go index f83cbf2d01207..b486f51236c34 100644 --- a/central/tlsconfig/manager_impl.go +++ b/central/tlsconfig/manager_impl.go @@ -26,11 +26,14 @@ type providerData struct { type managerImpl struct { mutex sync.RWMutex + namespace string + internalTrustRoots []*x509.Certificate userTrustRoots []*x509.Certificate - defaultCerts []tls.Certificate - internalCerts []tls.Certificate + defaultCerts []tls.Certificate + internalCerts []tls.Certificate + openShiftCerts []tls.Certificate providerIDToProviderData map[string]providerData certFingerprintToProvider map[string]providerData @@ -57,11 +60,14 @@ func newManager(namespace string) (*managerImpl, error) { mgr := &managerImpl{ providerIDToProviderData: make(map[string]providerData), + namespace: namespace, internalTrustRoots: trustRoots, internalCerts: internalCerts, } - certwatch.WatchCertDir(DefaultCertPath, MaybeGetDefaultTLSCertificateFromDirectory, mgr.UpdateDefaultTLSCertificate) + certwatch.WatchCertDir("default TLS", DefaultCertPath, MaybeGetDefaultTLSCertificateFromDirectory, mgr.UpdateDefaultTLSCertificate) + certwatch.WatchCertDir("internal service", mtls.CertsPrefix, LoadInternalCertificateFromDirectory, mgr.UpdateInternalCertificate, certwatch.WithVerify(false)) + mgr.initOpenShiftTLS() return mgr, nil } @@ -79,6 +85,23 @@ func (m *managerImpl) UpdateDefaultTLSCertificate(defaultCert *tls.Certificate) m.updateConfigurersNoLock() } +func (m *managerImpl) UpdateInternalCertificate(cert *tls.Certificate) { + if cert == nil { + return + } + + internalCerts, err := buildInternalCerts(cert, m.namespace) + if err != nil { + log.Errorf("Failed to build internal certificates (keeping previous certs): %v", err) + return + } + + m.mutex.Lock() + defer m.mutex.Unlock() + m.internalCerts = internalCerts + m.updateConfigurersNoLock() +} + func (m *managerImpl) GetProviderForFingerprint(fingerprint string) authproviders.Provider { m.mutex.RLock() defer m.mutex.RUnlock() @@ -162,6 +185,8 @@ func (m *managerImpl) TLSConfigurer(opts Options) (verifier.TLSConfigurer, error configurer.AddServerCertSource(&m.defaultCerts) case ServiceCertSource: configurer.AddServerCertSource(&m.internalCerts) + case OpenShiftTLSCertSource: + configurer.AddServerCertSource(&m.openShiftCerts) default: return nil, errors.Errorf("invalid server cert source %v", serverCert) } diff --git a/central/tlsconfig/manager_impl_test.go b/central/tlsconfig/manager_impl_test.go index 599835a51c5d0..3153fda08dd1d 100644 --- a/central/tlsconfig/manager_impl_test.go +++ b/central/tlsconfig/manager_impl_test.go @@ -137,6 +137,82 @@ func (s *managerTestSuite) testConnectionWithManager(mgr *managerImpl, acceptedS s.ErrorIs(err, net.ErrClosed) } +func (s *managerTestSuite) TestUpdateInternalCertificateReplacesExisting() { + mgr, err := newManager(namespaces.StackRox) + s.Require().NoError(err) + + s.Require().NotEmpty(mgr.internalCerts) + originalSerial := mgr.internalCerts[0].Leaf.SerialNumber + + ca, err := certgen.GenerateCA() + s.Require().NoError(err) + issuedCert, err := ca.IssueCertForSubject(mtls.CentralSubject) + s.Require().NoError(err) + newCert, err := tls.X509KeyPair(issuedCert.CertPEM, issuedCert.KeyPEM) + s.Require().NoError(err) + + mgr.UpdateInternalCertificate(&newCert) + + s.Len(mgr.internalCerts, 1) + s.NotEqual(originalSerial, mgr.internalCerts[0].Leaf.SerialNumber) +} + +func (s *managerTestSuite) TestUpdateInternalCertificateNilIsNoop() { + mgr, err := newManager(namespaces.StackRox) + s.Require().NoError(err) + + s.Require().NotEmpty(mgr.internalCerts) + certCountBefore := len(mgr.internalCerts) + + mgr.UpdateInternalCertificate(nil) + + s.Len(mgr.internalCerts, certCountBefore) +} + +func (s *managerTestSuite) TestUpdateInternalCertificateIssuesEphemeralForAltNamespace() { + mgr, err := newManager("alt-ns") + s.Require().NoError(err) + + // Cert without alt-ns DNS names triggers ephemeral cert issuance + newCert := testutils.IssueSelfSignedCert(s.T(), "central.stackrox", "central.stackrox.svc") + mgr.UpdateInternalCertificate(&newCert) + + s.Require().Len(mgr.internalCerts, 2, "expected reloaded cert + ephemeral cert for alt namespace") + + // First cert is the one we passed in (no alt-ns DNS names) + s.Require().NotNil(mgr.internalCerts[0].Leaf) + s.Error(mgr.internalCerts[0].Leaf.VerifyHostname("central.alt-ns.svc")) + + // Second cert is the ephemeral one covering the alt namespace + s.Require().NotNil(mgr.internalCerts[1].Leaf) + s.NoError(mgr.internalCerts[1].Leaf.VerifyHostname("central.alt-ns.svc")) +} + +func (s *managerTestSuite) TestLoadInternalCertificateFromDirectory() { + ca, err := mtls.CAForSigning() + s.Require().NoError(err) + + centralCert, err := ca.IssueCertForSubject(mtls.CentralSubject) + s.Require().NoError(err) + + dir := s.T().TempDir() + s.Require().NoError(os.WriteFile(filepath.Join(dir, mtls.ServiceCertFileName), centralCert.CertPEM, 0644)) + s.Require().NoError(os.WriteFile(filepath.Join(dir, mtls.ServiceKeyFileName), centralCert.KeyPEM, 0600)) + + cert, err := LoadInternalCertificateFromDirectory(dir) + s.Require().NoError(err) + s.Require().NotNil(cert) + s.Require().NotNil(cert.Leaf) + s.Contains(cert.Leaf.DNSNames, "central.stackrox.svc") +} + +func (s *managerTestSuite) TestLoadInternalCertificateFromDirectoryMissingFiles() { + dir := s.T().TempDir() + cert, err := LoadInternalCertificateFromDirectory(dir) + s.Error(err) + s.Nil(cert) +} + func getCertPool(certs []*x509.Certificate) *x509.CertPool { pool := x509.NewCertPool() for _, cert := range certs { diff --git a/central/tlsconfig/openshift_tls.go b/central/tlsconfig/openshift_tls.go new file mode 100644 index 0000000000000..8e35aa9bfe869 --- /dev/null +++ b/central/tlsconfig/openshift_tls.go @@ -0,0 +1,48 @@ +package tlsconfig + +import ( + "crypto/tls" + + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/fileutils" + "github.com/stackrox/rox/pkg/mtls/certwatch" +) + +// OpenShiftTLSConfigured reports whether OpenShift service-serving TLS for the +// central-ocp Service should be enabled: Central must run on OpenShift and the +// certificate mount directory must be present. +func OpenShiftTLSConfigured() bool { + if !env.Openshift.BooleanSetting() { + return false + } + exists, err := fileutils.Exists(env.OpenShiftTLSCertDir.Setting()) + if err != nil { + log.Warnf("Cannot check OpenShift TLS certificate directory %s: %v", env.OpenShiftTLSCertDir.Setting(), err) + } + return exists && err == nil +} + +func (m *managerImpl) initOpenShiftTLS() { + if !OpenShiftTLSConfigured() { + return + } + certwatch.WatchCertDir( + "OpenShift service-serving TLS", + env.OpenShiftTLSCertDir.Setting(), + MaybeLoadOpenShiftTLSCertificateFromDirectory, + m.UpdateOpenShiftTLSCertificate, + certwatch.WithVerify(false), + ) +} + +func (m *managerImpl) UpdateOpenShiftTLSCertificate(cert *tls.Certificate) { + m.mutex.Lock() + defer m.mutex.Unlock() + + if cert == nil { + m.openShiftCerts = nil + } else { + m.openShiftCerts = []tls.Certificate{*cert} + } + m.updateConfigurersNoLock() +} diff --git a/central/tlsconfig/tlsconfig.go b/central/tlsconfig/tlsconfig.go index ba10466c648f1..111cf3f817529 100644 --- a/central/tlsconfig/tlsconfig.go +++ b/central/tlsconfig/tlsconfig.go @@ -14,6 +14,7 @@ import ( "github.com/stackrox/rox/pkg/fileutils" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/mtls" + "github.com/stackrox/rox/pkg/mtls/verifier" "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/x509utils" ) @@ -144,33 +145,46 @@ func MaybeGetDefaultTLSCertificateFromDefaultDirectory() (*tls.Certificate, erro // MaybeGetDefaultTLSCertificateFromDirectory loads the default TLS certificate from the given directory. func MaybeGetDefaultTLSCertificateFromDirectory(dir string) (*tls.Certificate, error) { + return maybeLoadTLSCertificateFromDirectory("default TLS", dir) +} + +// MaybeLoadOpenShiftTLSCertificateFromDirectory loads the OpenShift service-serving +// TLS certificate from the given directory. +func MaybeLoadOpenShiftTLSCertificateFromDirectory(dir string) (*tls.Certificate, error) { + return maybeLoadTLSCertificateFromDirectory("OpenShift service-serving TLS", dir) +} + +// maybeLoadTLSCertificateFromDirectory loads TLS certificate and key files from +// the given directory. The label is used in log and error messages to identify +// which certificate is being loaded. +func maybeLoadTLSCertificateFromDirectory(label, dir string) (*tls.Certificate, error) { certFile := filepath.Join(dir, TLSCertFileName) keyFile := filepath.Join(dir, TLSKeyFileName) if exists, err := fileutils.Exists(certFile); err != nil || !exists { if err != nil { - log.Warnw("Error checking if default TLS certificate file exists", logging.Err(err)) + log.Warnw(fmt.Sprintf("Error checking if %s certificate file exists", label), logging.Err(err)) return nil, err } - log.Debugf("Default TLS certificate file %q does not exist. Skipping", certFile) + log.Debugf("%s certificate file %q does not exist. Skipping", label, certFile) return nil, nil } if exists, err := fileutils.Exists(keyFile); err != nil || !exists { if err != nil { - log.Warnw("Error checking if default TLS key file exists", logging.Err(err)) + log.Warnw(fmt.Sprintf("Error checking if %s key file exists", label), logging.Err(err)) return nil, err } - log.Debugf("Default TLS key file %q does not exist. Skipping", keyFile) + log.Debugf("%s key file %q does not exist. Skipping", label, keyFile) return nil, nil } cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { if strings.Contains(err.Error(), "private key does not match public key") { - return nil, errors.Wrap(err, "loading default certificate; if the certificate file contains a certificate chain, ensure that the certificate chain is in the correct order (the first certificate should be the leaf certificate, any following certificates should form the certificate chain)") + return nil, errors.Wrap(err, fmt.Sprintf("loading %s certificate; if the certificate file contains a certificate chain, ensure that the certificate chain is in the correct order (the first certificate should be the leaf certificate, any following certificates should form the certificate chain)", label)) } - return nil, errors.Wrap(err, "loading default certificate failed") + return nil, errors.Wrap(err, fmt.Sprintf("loading %s certificate failed", label)) } cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) @@ -193,6 +207,29 @@ func loadInternalCertificateFromFiles() (*tls.Certificate, error) { return &cert, nil } +// LoadInternalCertificateFromDirectory loads the internal service leaf certificate +// (cert.pem + key.pem) from the given directory and verifies it against the +// internal CA trust roots. +func LoadInternalCertificateFromDirectory(dir string) (*tls.Certificate, error) { + certFile := filepath.Join(dir, mtls.ServiceCertFileName) + keyFile := filepath.Join(dir, mtls.ServiceKeyFileName) + + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, errors.Wrapf(err, "loading internal certificate from %q", dir) + } + + trustPool, err := verifier.TrustedCertPool() + if err != nil { + return nil, errors.Wrap(err, "building trust pool for internal certificate verification") + } + if _, err := cert.Leaf.Verify(x509.VerifyOptions{Roots: trustPool}); err != nil { + return nil, errors.Wrap(err, "verifying internal certificate against trusted CAs") + } + + return &cert, nil +} + func issueInternalCertificate(namespace string) (*tls.Certificate, error) { issuedCert, err := mtls.IssueNewCert(mtls.CentralSubject, mtls.WithNamespace(namespace)) if err != nil { @@ -213,34 +250,39 @@ func issueInternalCertificate(namespace string) (*tls.Certificate, error) { } func getInternalCertificates(namespace string) ([]tls.Certificate, error) { - var internalCerts []tls.Certificate - // First try to load the internal certificate from files. If the files don't exist, issue - // ourselves a cert. - if certFromFiles, err := loadInternalCertificateFromFiles(); err != nil { + certFromFiles, err := loadInternalCertificateFromFiles() + if err != nil { return nil, err - } else if certFromFiles != nil { - internalCerts = append(internalCerts, *certFromFiles) + } + if certFromFiles != nil { + return buildInternalCerts(certFromFiles, namespace) } - if len(internalCerts) > 0 { - serviceCert, err := x509.ParseCertificate(internalCerts[0].Certificate[0]) - if err != nil { - return nil, errors.Wrap(err, "loaded internal certificate is invalid") - } - if validForAllDNSNames(serviceCert, mtls.CentralSubject.AllHostnamesForNamespace(namespace)...) { - return internalCerts, nil // cert loaded from secret is sufficient - } + // No cert files on disk — issue an ephemeral cert from scratch. + ephemeralCert, err := issueInternalCertificate(namespace) + if err != nil { + return nil, errors.Wrapf(err, "issuing ephemeral internal certificate for namespace %s", namespace) + } + return []tls.Certificate{*ephemeralCert}, nil +} + +// buildInternalCerts returns a cert slice containing the given cert. If the cert +// is not valid for all DNS names in the given namespace, an additional ephemeral +// cert with the correct SANs is issued and appended. +func buildInternalCerts(cert *tls.Certificate, namespace string) ([]tls.Certificate, error) { + if cert.Leaf != nil && validForAllDNSNames(cert.Leaf, mtls.CentralSubject.AllHostnamesForNamespace(namespace)...) { + return []tls.Certificate{*cert}, nil } - log.Warnw("Internal TLS certificates are not valid for all cluster-internal DNS names due to deployment in "+ - "alternative namespace, issuing ephemeral certificate with adequate DNS names", - logging.String("namespace", namespace), logging.Strings("internalDNSNames", mtls.CentralSubject.AllHostnamesForNamespace(namespace))) - newInternalCert, err := issueInternalCertificate(namespace) + log.Warnw("Internal TLS certificate is not valid for all cluster-internal DNS names, "+ + "issuing ephemeral certificate with adequate DNS names", + logging.String("namespace", namespace), + logging.Strings("internalDNSNames", mtls.CentralSubject.AllHostnamesForNamespace(namespace))) + ephemeralCert, err := issueInternalCertificate(namespace) if err != nil { - return internalCerts, err + return []tls.Certificate{*cert}, errors.Wrap(err, "issuing ephemeral certificate for alternative namespace") } - internalCerts = append(internalCerts, *newInternalCert) - return internalCerts, nil + return []tls.Certificate{*cert, *ephemeralCert}, nil } func validForAllDNSNames(cert *x509.Certificate, dnsNames ...string) bool { diff --git a/central/tlsconfig/tlsconfig_test.go b/central/tlsconfig/tlsconfig_test.go index 1daa08e09e9ce..01bdb0551ff31 100644 --- a/central/tlsconfig/tlsconfig_test.go +++ b/central/tlsconfig/tlsconfig_test.go @@ -198,7 +198,7 @@ func (s *tlsConfigTestSuite) TestMaybeGetDefaultTLSCertificate_ShouldReturnError func createCertChainWithLengthOf(length int) ([]*x509.Certificate, []*ecdsa.PrivateKey, error) { var certs []*x509.Certificate var privateKeys []*ecdsa.PrivateKey - for i := 0; i < length; i++ { + for i := range length { var parentCert = &x509.Certificate{} var parentKey = &ecdsa.PrivateKey{} if i > 0 { diff --git a/central/version/ensure.go b/central/version/ensure.go index d478254f561e1..3ab439167afe6 100644 --- a/central/version/ensure.go +++ b/central/version/ensure.go @@ -5,11 +5,8 @@ import ( "github.com/pkg/errors" vStore "github.com/stackrox/rox/central/version/store" - "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/migrations" - "github.com/stackrox/rox/pkg/protocompat" - versionUtil "github.com/stackrox/rox/pkg/version" ) var ( @@ -28,30 +25,22 @@ func Ensure(versionStore vStore.Store) error { return errors.Wrap(err, "failed to read version from DB") } - // No version in the DB. This means that we're starting from scratch, with a blank DB, so we can just - // write the current version in and move on. if version == nil { - err = versionStore.UpdateVersion( - &storage.Version{ - SeqNum: int32(migrations.CurrentDBVersionSeqNum()), - Version: versionUtil.GetMainVersion(), - MinSeqNum: int32(migrations.MinimumSupportedDBVersionSeqNum()), - LastPersisted: protocompat.TimestampNow(), - }, - ) - if err != nil { - return errors.Wrap(err, "failed to write version to the DB") - } - log.Info("No version found in the DB. Assuming that this is a fresh install...") - return nil - } - - if int(version.GetSeqNum()) != migrations.CurrentDBVersionSeqNum() { - return fmt.Errorf("invalid DB version found: %s", protocompat.MarshalTextString(version)) + return errors.New("no DB version found") } - // TYPICAL CASE: DB is of the same version. This happens if Central does a regular restart, and was not patched. + actualSeqNum := int(version.GetSeqNum()) + expectedSeqNum := migrations.CurrentDBVersionSeqNum() - log.Info("Version found in the DB was current. We're good to go!") - return nil + switch { + case actualSeqNum < expectedSeqNum: + return fmt.Errorf("DB version %d below expected version %d", version.GetSeqNum(), migrations.CurrentDBVersionSeqNum()) + case actualSeqNum > expectedSeqNum: + // This case allows old version centrals to restart for rollbacks and during RollingUpdates + log.Info("DB version higher than expected version, central can start since it's backward compatible") + return nil + default: + log.Info("Version found in the DB was current. We're good to go!") + return nil + } } diff --git a/central/version/ensure_test.go b/central/version/ensure_test.go index a310fd04eac16..8e55c5497a56d 100644 --- a/central/version/ensure_test.go +++ b/central/version/ensure_test.go @@ -39,10 +39,7 @@ func (suite *EnsurerTestSuite) TearDownTest() { } func (suite *EnsurerTestSuite) TestWithEmptyDB() { - suite.NoError(Ensure(store.NewPostgres(suite.pool))) - version, err := suite.versionStore.GetVersion() - suite.NoError(err) - suite.Equal(migrations.CurrentDBVersionSeqNum(), int(version.GetSeqNum())) + suite.Error(Ensure(store.NewPostgres(suite.pool))) } func (suite *EnsurerTestSuite) TestWithCurrentVersion() { @@ -58,3 +55,8 @@ func (suite *EnsurerTestSuite) TestWithIncorrectVersion() { suite.NoError(suite.versionStore.UpdateVersion(&storage.Version{SeqNum: int32(migrations.CurrentDBVersionSeqNum()) - 2})) suite.Error(Ensure(store.NewPostgres(suite.pool))) } + +func (suite *EnsurerTestSuite) TestWithSmallerVersion() { + suite.NoError(suite.versionStore.UpdateVersion(&storage.Version{SeqNum: int32(migrations.CurrentDBVersionSeqNum()) + 3})) + suite.NoError(Ensure(store.NewPostgres(suite.pool))) +} diff --git a/central/version/utils/utils.go b/central/version/utils/utils.go index a1be106d0b566..85386b5766833 100644 --- a/central/version/utils/utils.go +++ b/central/version/utils/utils.go @@ -1,15 +1,11 @@ package utils import ( - "github.com/pkg/errors" vStore "github.com/stackrox/rox/central/version/store" - "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/migrations" "github.com/stackrox/rox/pkg/postgres" - "github.com/stackrox/rox/pkg/protoconv" "github.com/stackrox/rox/pkg/timestamp" "github.com/stackrox/rox/pkg/utils" - "github.com/stackrox/rox/pkg/version" ) // ReadVersionPostgres - reads the version from the postgres database. @@ -29,26 +25,3 @@ func ReadVersionPostgres(pool postgres.DB) (*migrations.MigrationVersion, error) MinimumSeqNum: int(ver.GetMinSeqNum()), }, nil } - -// SetCurrentVersionPostgres - sets the current version in the postgres database -func SetCurrentVersionPostgres(pool postgres.DB) { - if curr, err := ReadVersionPostgres(pool); err != nil || - curr.MainVersion != version.GetMainVersion() || - curr.SeqNum != migrations.CurrentDBVersionSeqNum() || - curr.MinimumSeqNum != migrations.MinimumSupportedDBVersionSeqNum() { - newVersion := &storage.Version{ - SeqNum: int32(migrations.CurrentDBVersionSeqNum()), - Version: version.GetMainVersion(), - MinSeqNum: int32(migrations.MinimumSupportedDBVersionSeqNum()), - LastPersisted: protoconv.ConvertMicroTSToProtobufTS(timestamp.Now()), - } - setVersionPostgres(pool, newVersion) - } -} - -func setVersionPostgres(pool postgres.DB, updatedVersion *storage.Version) { - store := vStore.NewPostgres(pool) - - err := store.UpdateVersion(updatedVersion) - utils.CrashOnError(errors.Wrapf(err, "failed to write migration version to %s", pool.Config().ConnConfig.Database)) -} diff --git a/central/views/common/queries.go b/central/views/common/queries.go index 0950c558f8a9c..4a74f684c38be 100644 --- a/central/views/common/queries.go +++ b/central/views/common/queries.go @@ -9,15 +9,6 @@ import ( "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" ) -// WithCountQuery returns a query to count the number of distinct values of the given field -func WithCountQuery(q *v1.Query, field search.FieldLabel) *v1.Query { - cloned := q.CloneVT() - cloned.Selects = []*v1.QuerySelect{ - search.NewQuerySelect(field).AggrFunc(aggregatefunc.Count).Distinct().Proto(), - } - return cloned -} - func WithCountBySeverityAndFixabilityQuery(q *v1.Query, countOn search.FieldLabel) *v1.Query { cloned := q.CloneVT() cloned.Selects = append(cloned.Selects, diff --git a/central/views/deployments/view_impl.go b/central/views/deployments/view_impl.go index 53b789c48fb8d..27dc8c41fed5f 100644 --- a/central/views/deployments/view_impl.go +++ b/central/views/deployments/view_impl.go @@ -9,8 +9,8 @@ import ( "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/walker" - "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/paginated" pgSearch "github.com/stackrox/rox/pkg/search/postgres" ) @@ -28,28 +28,21 @@ func (v *deploymentViewImpl) Get(ctx context.Context, query *v1.Query) ([]Deploy return nil, err } - var err error // Update the sort options to use aggregations if necessary as we are grouping by CVEs query = common.UpdateSortAggs(query) - query, err = common.WithSACFilter(ctx, resources.Deployment, query) - if err != nil { - return nil, err - } query = withSelectQuery(query) queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() - var results []*deploymentResponse - results, err = pgSearch.RunSelectRequestForSchema[deploymentResponse](queryCtx, v.db, v.schema, query) + ret := make([]DeploymentCore, 0, paginated.GetLimit(query.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[deploymentResponse](queryCtx, v.db, v.schema, query, func(r *deploymentResponse) error { + ret = append(ret, r) + return nil + }) if err != nil { return nil, err } - - ret := make([]DeploymentCore, 0, len(results)) - for _, r := range results { - ret = append(ret, r) - } return ret, nil } diff --git a/central/views/deployments/view_test.go b/central/views/deployments/view_test.go index 2a6cdb213d0da..14b6d38bdd813 100644 --- a/central/views/deployments/view_test.go +++ b/central/views/deployments/view_test.go @@ -10,11 +10,14 @@ import ( deploymentDS "github.com/stackrox/rox/central/deployment/datastore" imageDS "github.com/stackrox/rox/central/image/datastore" imagePostgresV2 "github.com/stackrox/rox/central/image/datastore/store/v2/postgres" + imageV2DS "github.com/stackrox/rox/central/imagev2/datastore" + imageV2Postgres "github.com/stackrox/rox/central/imagev2/datastore/store/postgres" "github.com/stackrox/rox/central/ranking" mockRisks "github.com/stackrox/rox/central/risk/datastore/mocks" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" imageSamples "github.com/stackrox/rox/pkg/fixtures/image" "github.com/stackrox/rox/pkg/postgres/pgtest" @@ -40,11 +43,17 @@ type testCase struct { ignoreOrder bool } +type testImage interface { + GetId() string + GetName() *storage.ImageName + GetScan() *storage.ImageScan +} + type lessFunc func(records []*deploymentResponse) func(i, j int) bool type filterImpl struct { matchDeployment func(dep *storage.Deployment) bool - matchImage func(image *storage.Image) bool + matchImage func(image testImage) bool matchVuln func(vuln *storage.EmbeddedVulnerability) bool } @@ -53,7 +62,7 @@ func matchAllFilter() *filterImpl { matchDeployment: func(dep *storage.Deployment) bool { return true }, - matchImage: func(_ *storage.Image) bool { + matchImage: func(_ testImage) bool { return true }, matchVuln: func(_ *storage.EmbeddedVulnerability) bool { @@ -67,7 +76,7 @@ func (f *filterImpl) withDeploymentFilter(fn func(dep *storage.Deployment) bool) return f } -func (f *filterImpl) withImageFilter(fn func(image *storage.Image) bool) *filterImpl { +func (f *filterImpl) withImageFilter(fn func(image testImage) bool) *filterImpl { f.matchImage = fn return f } @@ -89,7 +98,7 @@ type DeploymentViewTestSuite struct { testDeployments []*storage.Deployment testDeploymentsMap map[string]*storage.Deployment - testImages []*storage.Image + testImages []testImage scopeToDeploymentIDs map[string]set.StringSet } @@ -100,58 +109,108 @@ func (s *DeploymentViewTestSuite) SetupSuite() { mockRisk := mockRisks.NewMockDataStore(mockCtrl) - // Initialize the datastore. - imageStore := imageDS.NewWithPostgres( - imagePostgresV2.New(s.testDB.DB, false, concurrency.NewKeyFence()), - mockRisk, - ranking.ImageRanker(), - ranking.ComponentRanker(), - ) - deploymentStore, err := deploymentDS.NewTestDataStore( - s.T(), - s.testDB, - &deploymentDS.DeploymentTestStoreParams{ - ImagesDataStore: imageStore, - RisksDataStore: mockRisk, - ClusterRanker: ranking.ClusterRanker(), - NamespaceRanker: ranking.NamespaceRanker(), - DeploymentRanker: ranking.DeploymentRanker(), - }, - ) - s.Require().NoError(err) + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + var deploymentStore deploymentDS.DataStore + if features.FlattenImageData.Enabled() { + imageV2Store := imageV2DS.NewWithPostgres( + imageV2Postgres.New(s.testDB.DB, false, concurrency.NewKeyFence()), + mockRisk, + ranking.ImageRanker(), + ranking.ComponentRanker(), + ) + var err error + deploymentStore, err = deploymentDS.NewTestDataStore( + s.T(), + s.testDB, + &deploymentDS.DeploymentTestStoreParams{ + ImagesV2DataStore: imageV2Store, + RisksDataStore: mockRisk, + ClusterRanker: ranking.ClusterRanker(), + NamespaceRanker: ranking.NamespaceRanker(), + DeploymentRanker: ranking.DeploymentRanker(), + }, + ) + s.Require().NoError(err) - // Upsert test images. - images, err := imageSamples.GetTestImages(s.T()) - s.Require().NoError(err) - // set cvss metrics list with one nvd cvss score - for _, image := range images { - s.Require().NoError(imageStore.UpsertImage(ctx, image)) - } + imagesV2, imagesV2Err := imageSamples.GetTestImagesV2(s.T()) + s.Require().NoError(imagesV2Err) + for _, imgV2 := range imagesV2 { + s.Require().NoError(imageV2Store.UpsertImage(ctx, imgV2)) + } - // Ensure that the image is stored and constructed as expected. - for idx, image := range images { - actual, found, err := imageStore.GetImage(ctx, image.GetId()) + // Verify stored V2 images and use them for expected results. + s.testImages = make([]testImage, len(imagesV2)) + for idx, imgV2 := range imagesV2 { + actual, found, getErr := imageV2Store.GetImage(ctx, imgV2.GetId()) + s.Require().NoError(getErr) + s.Require().True(found) + s.testImages[idx] = actual + } + } else { + imageStore := imageDS.NewWithPostgres( + imagePostgresV2.New(s.testDB.DB, false, concurrency.NewKeyFence()), + mockRisk, + ranking.ImageRanker(), + ranking.ComponentRanker(), + ) + var err error + deploymentStore, err = deploymentDS.NewTestDataStore( + s.T(), + s.testDB, + &deploymentDS.DeploymentTestStoreParams{ + ImagesDataStore: imageStore, + RisksDataStore: mockRisk, + ClusterRanker: ranking.ClusterRanker(), + NamespaceRanker: ranking.NamespaceRanker(), + DeploymentRanker: ranking.DeploymentRanker(), + }, + ) s.Require().NoError(err) - s.Require().True(found) - cloned := actual.CloneVT() - // Adjust dynamic fields and ensure images in ACS are as expected. - standardizeImages(image, cloned) + images, imagesErr := imageSamples.GetTestImages(s.T()) + s.Require().NoError(imagesErr) + + for _, image := range images { + s.Require().NoError(imageStore.UpsertImage(ctx, image)) + } + + // Ensure that the image is stored and constructed as expected. + s.testImages = make([]testImage, len(images)) + for idx, image := range images { + actual, found, getErr := imageStore.GetImage(ctx, image.GetId()) + s.Require().NoError(getErr) + s.Require().True(found) + + cloned := actual.CloneVT() + // Adjust dynamic fields and ensure images in ACS are as expected. + standardizeImages(image, cloned) - // Now that we confirmed that images match, use stored image to establish the expected test results. - // This makes dynamic fields matching (e.g. created at) straightforward. - images[idx] = actual + // Now that we confirmed that images match, use stored image to establish the expected test results. + // This makes dynamic fields matching (e.g. created at) straightforward. + s.testImages[idx] = actual + } } - s.testImages = images s.deploymentView = NewDeploymentView(s.testDB.DB) - s.Require().Len(images, 5) - deployments := []*storage.Deployment{ - fixtures.GetDeploymentWithImage(testconsts.Cluster1, testconsts.NamespaceA, images[0]), - fixtures.GetDeploymentWithImage(testconsts.Cluster1, testconsts.NamespaceA, images[1]), - fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, images[2]), - fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, images[3]), - fixtures.GetDeploymentWithImage(testconsts.Cluster3, testconsts.NamespaceC, images[4]), + s.Require().Len(s.testImages, 5) + var deployments []*storage.Deployment + // TODO(ROX-30117): Remove conditional when FlattenImageData feature flag is removed. + if features.FlattenImageData.Enabled() { + deployments = []*storage.Deployment{ + fixtures.GetDeploymentWithImageV2(testconsts.Cluster1, testconsts.NamespaceA, s.testImages[0].(*storage.ImageV2)), + fixtures.GetDeploymentWithImageV2(testconsts.Cluster1, testconsts.NamespaceA, s.testImages[1].(*storage.ImageV2)), + fixtures.GetDeploymentWithImageV2(testconsts.Cluster2, testconsts.NamespaceB, s.testImages[2].(*storage.ImageV2)), + fixtures.GetDeploymentWithImageV2(testconsts.Cluster2, testconsts.NamespaceB, s.testImages[3].(*storage.ImageV2)), + fixtures.GetDeploymentWithImageV2(testconsts.Cluster3, testconsts.NamespaceC, s.testImages[4].(*storage.ImageV2)), + } + } else { + deployments = []*storage.Deployment{ + fixtures.GetDeploymentWithImage(testconsts.Cluster1, testconsts.NamespaceA, s.testImages[0].(*storage.Image)), + fixtures.GetDeploymentWithImage(testconsts.Cluster1, testconsts.NamespaceA, s.testImages[1].(*storage.Image)), + fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, s.testImages[2].(*storage.Image)), + fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, s.testImages[3].(*storage.Image)), + fixtures.GetDeploymentWithImage(testconsts.Cluster3, testconsts.NamespaceC, s.testImages[4].(*storage.Image)), + } } s.testDeploymentsMap = make(map[string]*storage.Deployment) @@ -199,7 +258,7 @@ func (s *DeploymentViewTestSuite) TestGet() { desc: "filtered query", query: search.NewQueryBuilder().AddExactMatches(search.ImageName, "quay.io/appcontainers/wordpress:latest").ProtoQuery(), ignoreOrder: true, - matchFilter: matchAllFilter().withImageFilter(func(image *storage.Image) bool { + matchFilter: matchAllFilter().withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:latest" }), }, @@ -430,7 +489,7 @@ func (s *DeploymentViewTestSuite) TestGet() { } func (s *DeploymentViewTestSuite) compileExpected(filter *filterImpl, less lessFunc, hasSortBySeverityCounts bool) []DeploymentCore { - imageMap := make(map[string]*storage.Image) + imageMap := make(map[string]testImage) for _, img := range s.testImages { imageMap[img.GetId()] = img } @@ -466,10 +525,14 @@ func (s *DeploymentViewTestSuite) compileExpected(filter *filterImpl, less lessF return ret } -func compileExpectedVulns(containers []*storage.Container, imageMap map[string]*storage.Image, filter *filterImpl) []*storage.EmbeddedVulnerability { +func compileExpectedVulns(containers []*storage.Container, imageMap map[string]testImage, filter *filterImpl) []*storage.EmbeddedVulnerability { results := make([]*storage.EmbeddedVulnerability, 0) for _, container := range containers { - image := imageMap[container.GetImage().GetId()] + imageID := container.GetImage().GetId() + if imageID == "" { + imageID = container.GetImage().GetIdV2() + } + image := imageMap[imageID] if image == nil { continue } diff --git a/central/views/imagecomponentflat/view_impl.go b/central/views/imagecomponentflat/view_impl.go index 3c87cc47b9894..9bdd351c6cca1 100644 --- a/central/views/imagecomponentflat/view_impl.go +++ b/central/views/imagecomponentflat/view_impl.go @@ -4,14 +4,15 @@ import ( "context" "sort" + imagev2common "github.com/stackrox/rox/central/imagev2/common" "github.com/stackrox/rox/central/views/common" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/pkg/contextutil" "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/walker" - "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/paginated" pgSearch "github.com/stackrox/rox/pkg/search/postgres" "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" ) @@ -30,12 +31,6 @@ func (v *imageComponentFlatViewImpl) Count(ctx context.Context, q *v1.Query) (in return 0, err } - var err error - q, err = common.WithSACFilter(ctx, resources.Image, q) - if err != nil { - return 0, err - } - queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() @@ -55,33 +50,27 @@ func (v *imageComponentFlatViewImpl) Get(ctx context.Context, q *v1.Query) ([]Co if err := common.ValidateQuery(q); err != nil { return nil, err } + q = imagev2common.WithRowsFromImageV2Only(q) - var err error // Avoid changing the passed query cloned := q.CloneVT() // Update the sort options to use aggregations if necessary as we are grouping by CVEs cloned = common.UpdateSortAggs(cloned) - cloned, err = common.WithSACFilter(ctx, resources.Image, cloned) - if err != nil { - return nil, err - } queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() - var results []*imageComponentFlatResponse - results, err = pgSearch.RunSelectRequestForSchema[imageComponentFlatResponse](queryCtx, v.db, v.schema, withSelectComponentCoreResponseQuery(cloned)) - if err != nil { - return nil, err - } - - ret := make([]ComponentFlat, 0, len(results)) - for _, r := range results { + ret := make([]ComponentFlat, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[imageComponentFlatResponse](queryCtx, v.db, v.schema, withSelectComponentCoreResponseQuery(cloned), func(r *imageComponentFlatResponse) error { // For each record, sort the IDs so that result looks consistent. sort.SliceStable(r.ComponentIDs, func(i, j int) bool { return r.ComponentIDs[i] < r.ComponentIDs[j] }) ret = append(ret, r) + return nil + }) + if err != nil { + return nil, err } return ret, nil } diff --git a/central/views/imagecomponentflat/view_test.go b/central/views/imagecomponentflat/view_test.go index d044e7f968763..dd37e50655f89 100644 --- a/central/views/imagecomponentflat/view_test.go +++ b/central/views/imagecomponentflat/view_test.go @@ -5,10 +5,12 @@ package imagecomponentflat import ( "context" "fmt" + "slices" "sort" "testing" deploymentDS "github.com/stackrox/rox/central/deployment/datastore" + imageDS "github.com/stackrox/rox/central/image/datastore" imageV2DS "github.com/stackrox/rox/central/imagev2/datastore" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" @@ -97,6 +99,12 @@ func (s *ImageComponentFlatViewTestSuite) SetupSuite() { ctx := sac.WithAllAccess(context.Background()) s.testDB = pgtest.ForT(s.T()) + // Insert a V1 image to verify that V1 rows are excluded from all view queries. + s.T().Setenv(features.FlattenImageData.EnvVar(), "false") + v1Store := imageDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(v1Store.UpsertImage(ctx, fixtures.GetImageSherlockHolmes1())) + s.T().Setenv(features.FlattenImageData.EnvVar(), "true") + // Initialize the ImageV2 datastore imageStore := imageV2DS.GetTestPostgresDataStore(s.T(), s.testDB.DB) deploymentStore, err := deploymentDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) @@ -658,7 +666,7 @@ func (s *ImageComponentFlatViewTestSuite) compileExpected(images []*storage.Imag expected := make([]*imageComponentFlatResponse, 0, len(componentMap)) for _, entry := range componentMap { // Sort component IDs for consistent results (matches the view implementation) - sort.Strings(entry.ComponentIDs) + slices.Sort(entry.ComponentIDs) expected = append(expected, entry) } diff --git a/central/views/imagecve/db_response.go b/central/views/imagecve/db_response.go index 8b7fb8c226de7..5993c7c066e2f 100644 --- a/central/views/imagecve/db_response.go +++ b/central/views/imagecve/db_response.go @@ -80,10 +80,6 @@ func (c *imageCVECoreResponse) GetPublishDate() *time.Time { return c.Published } -type imageCVECoreCount struct { - CVECount int `db:"cve_count"` -} - type imageResponse struct { ImageID string `db:"image_sha"` } diff --git a/central/views/imagecve/view_impl.go b/central/views/imagecve/view_impl.go index 24f9873173437..f452aa5c9c4a6 100644 --- a/central/views/imagecve/view_impl.go +++ b/central/views/imagecve/view_impl.go @@ -4,7 +4,7 @@ import ( "context" "sort" - "github.com/pkg/errors" + imagev2common "github.com/stackrox/rox/central/imagev2/common" "github.com/stackrox/rox/central/views" "github.com/stackrox/rox/central/views/common" v1 "github.com/stackrox/rox/generated/api/v1" @@ -13,11 +13,10 @@ import ( "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/walker" - "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/paginated" pgSearch "github.com/stackrox/rox/pkg/search/postgres" "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" - "github.com/stackrox/rox/pkg/utils" ) var ( @@ -33,75 +32,46 @@ func (v *imageCVECoreViewImpl) Count(ctx context.Context, q *v1.Query) (int, err if err := common.ValidateQuery(q); err != nil { return 0, err } - - var err error - q, err = common.WithSACFilter(ctx, resources.Image, q) - if err != nil { - return 0, err - } + q = imagev2common.WithRowsFromImageV2Only(q) queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() - var results []*imageCVECoreCount - results, err = pgSearch.RunSelectRequestForSchema[imageCVECoreCount](queryCtx, v.db, v.schema, common.WithCountQuery(q, search.CVE)) - if err != nil { - return 0, err - } - if len(results) == 0 { - return 0, nil - } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", q.String()) - utils.Should(err) - return 0, err - } - return results[0].CVECount, nil + return pgSearch.RunDistinctCountForSchema(queryCtx, v.db, v.schema, q, search.CVE) } func (v *imageCVECoreViewImpl) CountBySeverity(ctx context.Context, q *v1.Query) (common.ResourceCountByCVESeverity, error) { if err := common.ValidateQuery(q); err != nil { return nil, err } - - var err error - q, err = common.WithSACFilter(ctx, resources.Image, q) - if err != nil { - return nil, err - } + q = imagev2common.WithRowsFromImageV2Only(q) queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() - var results []*common.ResourceCountByImageCVESeverity - results, err = pgSearch.RunSelectRequestForSchema[common.ResourceCountByImageCVESeverity](queryCtx, v.db, v.schema, common.WithCountBySeverityAndFixabilityQuery(q, search.CVE)) + result, err := pgSearch.RunSelectOneForSchema[common.ResourceCountByImageCVESeverity](queryCtx, v.db, v.schema, common.WithCountBySeverityAndFixabilityQuery(q, search.CVE)) if err != nil { return nil, err } - if len(results) == 0 { + if result == nil { return &common.ResourceCountByImageCVESeverity{}, nil } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", q.String()) - utils.Should(err) - return &common.ResourceCountByImageCVESeverity{}, err - } return &common.ResourceCountByImageCVESeverity{ - CriticalSeverityCount: results[0].CriticalSeverityCount, - FixableCriticalSeverityCount: results[0].FixableCriticalSeverityCount, + CriticalSeverityCount: result.CriticalSeverityCount, + FixableCriticalSeverityCount: result.FixableCriticalSeverityCount, - ImportantSeverityCount: results[0].ImportantSeverityCount, - FixableImportantSeverityCount: results[0].FixableImportantSeverityCount, + ImportantSeverityCount: result.ImportantSeverityCount, + FixableImportantSeverityCount: result.FixableImportantSeverityCount, - ModerateSeverityCount: results[0].ModerateSeverityCount, - FixableModerateSeverityCount: results[0].FixableModerateSeverityCount, + ModerateSeverityCount: result.ModerateSeverityCount, + FixableModerateSeverityCount: result.FixableModerateSeverityCount, - LowSeverityCount: results[0].LowSeverityCount, - FixableLowSeverityCount: results[0].FixableLowSeverityCount, + LowSeverityCount: result.LowSeverityCount, + FixableLowSeverityCount: result.FixableLowSeverityCount, - UnknownSeverityCount: results[0].UnknownSeverityCount, - FixableUnknownSeverityCount: results[0].FixableUnknownSeverityCount, + UnknownSeverityCount: result.UnknownSeverityCount, + FixableUnknownSeverityCount: result.FixableUnknownSeverityCount, }, nil } @@ -109,18 +79,15 @@ func (v *imageCVECoreViewImpl) Get(ctx context.Context, q *v1.Query, options vie if err := common.ValidateQuery(q); err != nil { return nil, err } + q = imagev2common.WithRowsFromImageV2Only(q) - var err error // Avoid changing the passed query cloned := q.CloneVT() // Update the sort options to use aggregations if necessary as we are grouping by CVEs cloned = common.UpdateSortAggs(cloned) - cloned, err = common.WithSACFilter(ctx, resources.Image, cloned) - if err != nil { - return nil, err - } var cveIDsToFilter []string + var err error if cloned.GetPagination().GetLimit() > 0 || cloned.GetPagination().GetOffset() > 0 { cveIDsToFilter, err = v.getFilteredCVEs(ctx, cloned) if err != nil { @@ -136,30 +103,22 @@ func (v *imageCVECoreViewImpl) Get(ctx context.Context, q *v1.Query, options vie queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() - var results []*imageCVECoreResponse - results, err = pgSearch.RunSelectRequestForSchema[imageCVECoreResponse](queryCtx, v.db, v.schema, withSelectCVECoreResponseQuery(cloned, cveIDsToFilter, options)) - if err != nil { - return nil, err - } - - ret := make([]CveCore, 0, len(results)) - for _, r := range results { + ret := make([]CveCore, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + err = pgSearch.RunSelectRequestForSchemaFn[imageCVECoreResponse](queryCtx, v.db, v.schema, withSelectCVECoreResponseQuery(cloned, cveIDsToFilter, options), func(r *imageCVECoreResponse) error { // For each record, sort the IDs so that result looks consistent. sort.SliceStable(r.CVEIDs, func(i, j int) bool { return r.CVEIDs[i] < r.CVEIDs[j] }) ret = append(ret, r) + return nil + }) + if err != nil { + return nil, err } return ret, nil } func (v *imageCVECoreViewImpl) GetDeploymentIDs(ctx context.Context, q *v1.Query) ([]string, error) { - var err error - q, err = common.WithSACFilter(ctx, resources.Deployment, q) - if err != nil { - return nil, err - } - q.Selects = []*v1.QuerySelect{ search.NewQuerySelect(search.DeploymentID).Distinct().Proto(), } @@ -167,26 +126,21 @@ func (v *imageCVECoreViewImpl) GetDeploymentIDs(ctx context.Context, q *v1.Query queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() - var results []*deploymentResponse - results, err = pgSearch.RunSelectRequestForSchema[deploymentResponse](queryCtx, v.db, v.schema, q) - if err != nil || len(results) == 0 { + ret := make([]string, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[deploymentResponse](queryCtx, v.db, v.schema, q, func(r *deploymentResponse) error { + ret = append(ret, r.DeploymentID) + return nil + }) + if err != nil { return nil, err } - - ret := make([]string, 0, len(results)) - for _, r := range results { - ret = append(ret, r.DeploymentID) + if len(ret) == 0 { + return nil, nil } return ret, nil } func (v *imageCVECoreViewImpl) GetImageIDs(ctx context.Context, q *v1.Query) ([]string, error) { - var err error - q, err = common.WithSACFilter(ctx, resources.Image, q) - if err != nil { - return nil, err - } - searchField := search.ImageSHA if features.FlattenImageData.Enabled() { searchField = search.ImageID @@ -198,28 +152,24 @@ func (v *imageCVECoreViewImpl) GetImageIDs(ctx context.Context, q *v1.Query) ([] queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() + ret := make([]string, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + var err error if features.FlattenImageData.Enabled() { - var results []*imageV2Response - results, err = pgSearch.RunSelectRequestForSchema[imageV2Response](queryCtx, v.db, v.schema, q) - if err != nil || len(results) == 0 { - return nil, err - } - - ret := make([]string, 0, len(results)) - for _, r := range results { + err = pgSearch.RunSelectRequestForSchemaFn[imageV2Response](queryCtx, v.db, v.schema, q, func(r *imageV2Response) error { ret = append(ret, r.ImageID) - } - return ret, nil + return nil + }) + } else { + err = pgSearch.RunSelectRequestForSchemaFn[imageResponse](queryCtx, v.db, v.schema, q, func(r *imageResponse) error { + ret = append(ret, r.ImageID) + return nil + }) } - var results []*imageResponse - results, err = pgSearch.RunSelectRequestForSchema[imageResponse](queryCtx, v.db, v.schema, q) - if err != nil || len(results) == 0 { + if err != nil { return nil, err } - - ret := make([]string, 0, len(results)) - for _, r := range results { - ret = append(ret, r.ImageID) + if len(ret) == 0 { + return nil, nil } return ret, nil } @@ -303,15 +253,13 @@ func (v *imageCVECoreViewImpl) getFilteredCVEs(ctx context.Context, q *v1.Query) // TODO(@charmik) : Update the SQL query generator to not include 'ORDER BY' and 'GROUP BY' fields in the select clause (before where). // SQL syntax does not need those fields in the select clause. The below query for example would work fine // "SELECT JSONB_AGG(DISTINCT(image_cves.Id)) AS cve_id FROM image_cves GROUP BY image_cves.CveBaseInfo_Cve ORDER BY MAX(image_cves.Cvss) DESC LIMIT 20;" - var identifiersList []*imageCVECoreResponse - identifiersList, err := pgSearch.RunSelectRequestForSchema[imageCVECoreResponse](queryCtx, v.db, v.schema, withSelectCVEIdentifiersQuery(q)) + err := pgSearch.RunSelectRequestForSchemaFn[imageCVECoreResponse](queryCtx, v.db, v.schema, withSelectCVEIdentifiersQuery(q), func(r *imageCVECoreResponse) error { + cveIDsToFilter = append(cveIDsToFilter, r.CVEIDs...) + return nil + }) if err != nil { return nil, err } - for _, idList := range identifiersList { - cveIDsToFilter = append(cveIDsToFilter, idList.CVEIDs...) - } - return cveIDsToFilter, nil } diff --git a/central/views/imagecve/view_test.go b/central/views/imagecve/view_test.go index e76b475aad2f1..242d64f29703f 100644 --- a/central/views/imagecve/view_test.go +++ b/central/views/imagecve/view_test.go @@ -5,6 +5,7 @@ package imagecve import ( "context" "fmt" + "slices" "sort" "strings" "testing" @@ -14,14 +15,16 @@ import ( deploymentDS "github.com/stackrox/rox/central/deployment/datastore" imageDS "github.com/stackrox/rox/central/image/datastore" imageComponentV2DS "github.com/stackrox/rox/central/imagecomponent/v2/datastore" + imageV2DS "github.com/stackrox/rox/central/imagev2/datastore" "github.com/stackrox/rox/central/views" "github.com/stackrox/rox/central/views/common" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/cve" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" imageSamples "github.com/stackrox/rox/pkg/fixtures/image" - "github.com/stackrox/rox/pkg/pointers" + imageUtils "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protoassert" "github.com/stackrox/rox/pkg/protocompat" @@ -38,6 +41,13 @@ import ( "github.com/stretchr/testify/suite" ) +// testImage is satisfied by both *storage.Image and *storage.ImageV2. +type testImage interface { + GetId() string + GetName() *storage.ImageName + GetScan() *storage.ImageScan +} + type testCase struct { desc string ctx context.Context @@ -57,16 +67,16 @@ type imageIDsPaginationTestCase struct { } type lessFunc func(records []*imageCVECoreResponse) func(i, j int) bool -type lessFuncForImages func(records []*storage.Image) func(i, j int) bool +type lessFuncForImages func(records []testImage) func(i, j int) bool type filterImpl struct { - matchImage func(image *storage.Image) bool + matchImage func(image testImage) bool matchVuln func(vuln *storage.EmbeddedVulnerability) bool } func matchAllFilter() *filterImpl { return &filterImpl{ - matchImage: func(_ *storage.Image) bool { + matchImage: func(_ testImage) bool { return true }, matchVuln: func(_ *storage.EmbeddedVulnerability) bool { @@ -77,7 +87,7 @@ func matchAllFilter() *filterImpl { func matchNoneFilter() *filterImpl { return &filterImpl{ - matchImage: func(_ *storage.Image) bool { + matchImage: func(_ testImage) bool { return false }, matchVuln: func(_ *storage.EmbeddedVulnerability) bool { @@ -86,7 +96,7 @@ func matchNoneFilter() *filterImpl { } } -func (f *filterImpl) withImageFilter(fn func(image *storage.Image) bool) *filterImpl { +func (f *filterImpl) withImageFilter(fn func(image testImage) bool) *filterImpl { f.matchImage = fn return f } @@ -107,7 +117,7 @@ type ImageCVEViewTestSuite struct { cveView CveView suiteCtx context.Context - testImages []*storage.Image + testImages []testImage testImagesToDeployments map[string][]*storage.Deployment componentDatastore imageComponentV2DS.DataStore @@ -119,66 +129,124 @@ func (s *ImageCVEViewTestSuite) SetupSuite() { ctx := s.suiteCtx s.testDB = pgtest.ForT(s.T()) - // Initialize the datastore. - imageStore := imageDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + // Initialize the datastores. deploymentStore, err := deploymentDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) s.Require().NoError(err) s.componentDatastore = imageComponentV2DS.GetTestPostgresDataStore(s.T(), s.testDB) s.cveDatastore = imageCVEV2DS.GetTestPostgresDataStore(s.T(), s.testDB) - // Upsert test images. - images, err := imageSamples.GetTestImages(s.T()) - s.Require().NoError(err) - // set cvss metrics list with one nvd cvss score - for _, image := range images { - for _, components := range image.GetScan().GetComponents() { - for _, vuln := range components.GetVulns() { - cvssScore := &storage.CVSSScore{ + // setCVSSMetrics sets NVD CVSS metrics on all vulns in an image scan. + setCVSSMetrics := func(scan *storage.ImageScan) { + for _, component := range scan.GetComponents() { + for _, vuln := range component.GetVulns() { + vuln.CvssMetrics = []*storage.CVSSScore{{ Source: storage.Source_SOURCE_NVD, CvssScore: &storage.CVSSScore_Cvssv3{ - Cvssv3: &storage.CVSSV3{ - Score: 10, - }, + Cvssv3: &storage.CVSSV3{Score: 10}, }, - } - vuln.CvssMetrics = []*storage.CVSSScore{cvssScore} + }} vuln.NvdCvss = 10 } } - s.Require().NoError(imageStore.UpsertImage(ctx, image)) } - // Ensure that the image is stored and constructed as expected. - for idx, image := range images { - actual, found, err := imageStore.GetImage(ctx, image.GetId()) + // Upsert images using the appropriate datastore based on feature flag. + var deployments []*storage.Deployment + if features.FlattenImageData.Enabled() { + // Insert a V1 image to verify that V1 rows are excluded from all view queries. + s.T().Setenv(features.FlattenImageData.EnvVar(), "false") + v1Store := imageDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(v1Store.UpsertImage(ctx, fixtures.GetImageSherlockHolmes1())) + s.T().Setenv(features.FlattenImageData.EnvVar(), "true") + + imageV2Store := imageV2DS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + imagesV2, err := imageSamples.GetTestImagesV2(s.T()) s.Require().NoError(err) - s.Require().True(found) + for _, imgV2 := range imagesV2 { + setCVSSMetrics(imgV2.GetScan()) + s.Require().NoError(imageV2Store.UpsertImage(ctx, imgV2)) + } + // Verify stored V2 images and use them for expected results. + for idx, imgV2 := range imagesV2 { + actual, found, err := imageV2Store.GetImage(ctx, imgV2.GetId()) + s.Require().NoError(err) + s.Require().True(found) + imagesV2[idx] = actual + } + s.testImages = make([]testImage, len(imagesV2)) + for i, img := range imagesV2 { + s.testImages[i] = img + } + deployments = []*storage.Deployment{ + fixtures.GetDeploymentWithImageV2(testconsts.Cluster1, testconsts.NamespaceA, imagesV2[1]), + fixtures.GetDeploymentWithImageV2(testconsts.Cluster2, testconsts.NamespaceB, imagesV2[1]), + fixtures.GetDeploymentWithImageV2(testconsts.Cluster2, testconsts.NamespaceB, imagesV2[2]), + } + } else { + imageStore := imageDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + images, err := imageSamples.GetTestImages(s.T()) + s.Require().NoError(err) + for _, image := range images { + setCVSSMetrics(image.GetScan()) + s.Require().NoError(imageStore.UpsertImage(ctx, image)) + } + // Ensure that the image is stored and constructed as expected. + for idx, image := range images { + actual, found, err := imageStore.GetImage(ctx, image.GetId()) + s.Require().NoError(err) + s.Require().True(found) - cloned := actual.CloneVT() - // Adjust dynamic fields and ensure images in ACS are as expected. - standardizeImages(image, cloned) - protoassert.Equal(s.T(), image, cloned) + cloned := actual.CloneVT() + standardizeImages(image, cloned) + protoassert.Equal(s.T(), image, cloned) - // Now that we confirmed that images match, use stored image to establish the expected test results. - // This makes dynamic fields matching (e.g. created at) straightforward. - images[idx] = actual + images[idx] = actual + } + s.testImages = make([]testImage, len(images)) + for i, img := range images { + s.testImages[i] = img + } + deployments = []*storage.Deployment{ + fixtures.GetDeploymentWithImage(testconsts.Cluster1, testconsts.NamespaceA, images[1]), + fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, images[1]), + fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, images[2]), + } } - s.testImages = images + s.cveView = NewCVEView(s.testDB.DB) - s.Require().Len(images, 5) - deployments := []*storage.Deployment{ - fixtures.GetDeploymentWithImage(testconsts.Cluster1, testconsts.NamespaceA, images[1]), - fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, images[1]), - fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, images[2]), - } + s.Require().Len(s.testImages, 5) for _, d := range deployments { s.Require().NoError(deploymentStore.UpsertDeployment(ctx, d)) } s.testImagesToDeployments = make(map[string][]*storage.Deployment) - s.testImagesToDeployments[images[1].GetId()] = []*storage.Deployment{deployments[0], deployments[1]} - s.testImagesToDeployments[images[2].GetId()] = []*storage.Deployment{deployments[2]} + s.testImagesToDeployments[s.testImages[1].GetId()] = []*storage.Deployment{deployments[0], deployments[1]} + s.testImagesToDeployments[s.testImages[2].GetId()] = []*storage.Deployment{deployments[2]} +} + +// imageScopeCategory returns the search category for image scoping. +func (s *ImageCVEViewTestSuite) imageScopeCategory() v1.SearchCategory { + if features.FlattenImageData.Enabled() { + return v1.SearchCategory_IMAGES_V2 + } + return v1.SearchCategory_IMAGES +} + +// imageSearchField returns the search field label for image ID. +func imageSearchField() search.FieldLabel { + if features.FlattenImageData.Enabled() { + return search.ImageID + } + return search.ImageSHA +} + +func incrAffectedImageCount(val *imageCVECoreResponse) { + if features.FlattenImageData.Enabled() { + val.AffectedImageCountV2++ + } else { + val.AffectedImageCount++ + } } func (s *ImageCVEViewTestSuite) TestGetImageCVECore() { @@ -230,7 +298,7 @@ func (s *ImageCVEViewTestSuite) TestGetImageCVECoreSAC() { // Wrap image filter with sac filter. matchFilter := *tc.matchFilter baseImageMatchFilter := matchFilter.matchImage - matchFilter.withImageFilter(func(image *storage.Image) bool { + matchFilter.withImageFilter(func(image testImage) bool { if sacTC[image.GetId()] { return baseImageMatchFilter(image) } @@ -317,7 +385,7 @@ func (s *ImageCVEViewTestSuite) TestGetImageIDsSAC() { // Wrap image filter with sac filter. matchFilter := *tc.matchFilter baseImageMatchFilter := matchFilter.matchImage - matchFilter.withImageFilter(func(image *storage.Image) bool { + matchFilter.withImageFilter(func(image testImage) bool { if sacTC[image.GetId()] { return baseImageMatchFilter(image) } @@ -411,7 +479,7 @@ func (s *ImageCVEViewTestSuite) TestCountImageCVECoreSAC() { // Wrap image filter with sac filter. matchFilter := *tc.matchFilter baseImageMatchFilter := matchFilter.matchImage - matchFilter.withImageFilter(func(image *storage.Image) bool { + matchFilter.withImageFilter(func(image testImage) bool { if sacTC[image.GetId()] { return baseImageMatchFilter(image) } @@ -445,7 +513,19 @@ func (s *ImageCVEViewTestSuite) TestCountBySeverity() { } } +func (s *ImageCVEViewTestSuite) findImageByName(fullName string) testImage { + for _, img := range s.testImages { + if img.GetName().GetFullName() == fullName { + return img + } + } + return nil +} + func (s *ImageCVEViewTestSuite) testCases() []testCase { + wordpressDebian := s.findImageByName("quay.io/appcontainers/wordpress:debian") + s.Require().NotNil(wordpressDebian) + return []testCase{ { desc: "search all", @@ -466,7 +546,7 @@ func (s *ImageCVEViewTestSuite) testCases() []testCase { ctx: context.Background(), q: search.NewQueryBuilder(). AddExactMatches(search.ImageName, "quay.io/appcontainers/wordpress:latest").ProtoQuery(), - matchFilter: matchAllFilter().withImageFilter(func(image *storage.Image) bool { + matchFilter: matchAllFilter().withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:latest" }), }, @@ -478,7 +558,7 @@ func (s *ImageCVEViewTestSuite) testCases() []testCase { AddExactMatches(search.ImageName, "quay.io/appcontainers/wordpress:debian"). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:debian" }). withVulnFilter(func(vuln *storage.EmbeddedVulnerability) bool { @@ -554,7 +634,7 @@ func (s *ImageCVEViewTestSuite) testCases() []testCase { AddExactMatches(search.ImageName, "quay.io/appcontainers/wordpress:debian"). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:debian" }). withVulnFilter(func(vuln *storage.EmbeddedVulnerability) bool { @@ -565,7 +645,7 @@ func (s *ImageCVEViewTestSuite) testCases() []testCase { desc: "search one operating system", ctx: context.Background(), q: search.NewQueryBuilder().AddExactMatches(search.OperatingSystem, "debian:8").ProtoQuery(), - matchFilter: matchAllFilter().withImageFilter(func(image *storage.Image) bool { + matchFilter: matchAllFilter().withImageFilter(func(image testImage) bool { return image.GetScan().GetOperatingSystem() == "debian:8" }), }, @@ -604,15 +684,15 @@ func (s *ImageCVEViewTestSuite) testCases() []testCase { { desc: "search one cve w/ image scope", ctx: scoped.Context(context.Background(), scoped.Scope{ - IDs: []string{"sha256:6ef31316f4f9e0c31a8f4e602ba287a210d66934f91b1616f1c9b957201d025c"}, - Level: v1.SearchCategory_IMAGES, + IDs: []string{wordpressDebian.GetId()}, + Level: s.imageScopeCategory(), }), q: search.NewQueryBuilder(). AddExactMatches(search.CVE, "CVE-2022-1552"). AddExactMatches(search.ImageName, "quay.io/appcontainers/wordpress:debian"). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:debian" }). withVulnFilter(func(vuln *storage.EmbeddedVulnerability) bool { @@ -622,8 +702,8 @@ func (s *ImageCVEViewTestSuite) testCases() []testCase { { desc: "search critical severity w/ cve & image scope", ctx: scoped.Context(context.Background(), scoped.Scope{ - IDs: []string{"sha256:6ef31316f4f9e0c31a8f4e602ba287a210d66934f91b1616f1c9b957201d025c"}, - Level: v1.SearchCategory_IMAGES, + IDs: []string{wordpressDebian.GetId()}, + Level: s.imageScopeCategory(), Parent: &scoped.Scope{ IDs: []string{cve.ID("CVE-2022-1552", "debian:8")}, Level: v1.SearchCategory_IMAGE_VULNERABILITIES_V2, @@ -633,7 +713,7 @@ func (s *ImageCVEViewTestSuite) testCases() []testCase { AddExactMatches(search.Severity, storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY.String()). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:debian" && image.GetScan().GetOperatingSystem() == "debian:8" }). @@ -666,7 +746,7 @@ func (s *ImageCVEViewTestSuite) testCases() []testCase { AddStrings(search.PlatformComponent, "false", "-"). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { deps, ok := s.testImagesToDeployments[image.GetId()] if !ok { // include inactive image @@ -691,7 +771,7 @@ func (s *ImageCVEViewTestSuite) testCases() []testCase { AddStrings(search.PlatformComponent, "true", "-"). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { deps, ok := s.testImagesToDeployments[image.GetId()] if !ok { // include inactive image @@ -717,15 +797,15 @@ func (s *ImageCVEViewTestSuite) paginationTestCases() []testCase { desc: "w/ affected image sort", q: search.NewQueryBuilder().WithPagination( search.NewPagination().AddSortOption( - search.NewSortOption(search.ImageSHA).AggregateBy(aggregatefunc.Count, true).Reversed(true), + search.NewSortOption(imageSearchField()).AggregateBy(aggregatefunc.Count, true).Reversed(true), ).AddSortOption(search.NewSortOption(search.CVE)), ).ProtoQuery(), less: func(records []*imageCVECoreResponse) func(i, j int) bool { return func(i, j int) bool { - if records[i].AffectedImageCount == records[j].AffectedImageCount { + if records[i].GetAffectedImageCount() == records[j].GetAffectedImageCount() { return records[i].CVE < records[j].CVE } - return records[i].AffectedImageCount > records[j].AffectedImageCount + return records[i].GetAffectedImageCount() > records[j].GetAffectedImageCount() } }, }, @@ -769,15 +849,16 @@ func (s *ImageCVEViewTestSuite) paginationTestCases() []testCase { } func (s *ImageCVEViewTestSuite) paginationTestCasesForImageIDs() []imageIDsPaginationTestCase { + imgIDField := imageSearchField() return []imageIDsPaginationTestCase{ { desc: "sort by image name", q: search.NewQueryBuilder().WithPagination( search.NewPagination(). AddSortOption(search.NewSortOption(search.ImageName)). - AddSortOption(search.NewSortOption(search.ImageSHA)), + AddSortOption(search.NewSortOption(imgIDField)), ).ProtoQuery(), - less: func(records []*storage.Image) func(i int, j int) bool { + less: func(records []testImage) func(i int, j int) bool { return func(i, j int) bool { if records[i].GetName().GetFullName() == records[j].GetName().GetFullName() { return strings.Compare(records[i].GetId(), records[j].GetId()) < 0 @@ -791,9 +872,9 @@ func (s *ImageCVEViewTestSuite) paginationTestCasesForImageIDs() []imageIDsPagin q: search.NewQueryBuilder().WithPagination( search.NewPagination(). AddSortOption(search.NewSortOption(search.ImageOS)). - AddSortOption(search.NewSortOption(search.ImageSHA)), + AddSortOption(search.NewSortOption(imgIDField)), ).ProtoQuery(), - less: func(records []*storage.Image) func(i int, j int) bool { + less: func(records []testImage) func(i int, j int) bool { return func(i, j int) bool { if records[i].GetScan().GetOperatingSystem() == records[j].GetScan().GetOperatingSystem() { return strings.Compare(records[i].GetId(), records[j].GetId()) < 0 @@ -807,9 +888,9 @@ func (s *ImageCVEViewTestSuite) paginationTestCasesForImageIDs() []imageIDsPagin q: search.NewQueryBuilder().WithPagination( search.NewPagination(). AddSortOption(search.NewSortOption(search.ImageScanTime)). - AddSortOption(search.NewSortOption(search.ImageSHA)), + AddSortOption(search.NewSortOption(imgIDField)), ).ProtoQuery(), - less: func(records []*storage.Image) func(i int, j int) bool { + less: func(records []testImage) func(i int, j int) bool { return func(i, j int) bool { if protocompat.CompareTimestamps(records[i].GetScan().GetScanTime(), records[j].GetScan().GetScanTime()) == 0 { return strings.Compare(records[i].GetId(), records[j].GetId()) < 0 @@ -920,7 +1001,7 @@ func applyPaginationProps(baseTc *testCase, paginationTc testCase) { baseTc.less = paginationTc.less } -func (s *ImageCVEViewTestSuite) compileExpected(images []*storage.Image, filter *filterImpl, options views.ReadOptions, less lessFunc) []CveCore { +func (s *ImageCVEViewTestSuite) compileExpected(images []testImage, filter *filterImpl, options views.ReadOptions, less lessFunc) []CveCore { cveMap := make(map[string]*imageCVECoreResponse) for _, image := range images { @@ -930,7 +1011,7 @@ func (s *ImageCVEViewTestSuite) compileExpected(images []*storage.Image, filter var seenForImage set.Set[string] // Instead of using embedded objects, grab components and CVEs from the datastores to get IDs - components, err := s.componentDatastore.SearchRawImageComponents(s.suiteCtx, search.NewQueryBuilder().AddExactMatches(search.ImageSHA, image.GetId()).ProtoQuery()) + components, err := s.componentDatastore.SearchRawImageComponents(s.suiteCtx, search.NewQueryBuilder().AddExactMatches(imageSearchField(), image.GetId()).ProtoQuery()) s.Require().NoError(err) for _, component := range components { dbVulns, err := s.cveDatastore.SearchRawImageCVEs(s.suiteCtx, search.NewQueryBuilder().AddExactMatches(search.ComponentID, component.GetId()).ProtoQuery()) @@ -962,16 +1043,16 @@ func (s *ImageCVEViewTestSuite) compileExpected(images []*storage.Image, filter if val == nil { val = &imageCVECoreResponse{ CVE: vuln.GetCveBaseInfo().GetCve(), - TopCVSS: pointers.Float32(vuln.GetCvss()), + TopCVSS: new(vuln.GetCvss()), FirstDiscoveredInSystem: &vulnTime, Published: &vulnPublishDate, } for _, metric := range vuln.GetCveBaseInfo().GetCvssMetrics() { if metric.GetSource() == storage.Source_SOURCE_NVD { if metric.GetCvssv2() != nil { - val.TopNVDCVSS = pointers.Float32(metric.GetCvssv2().GetScore()) + val.TopNVDCVSS = new(metric.GetCvssv2().GetScore()) } else { - val.TopNVDCVSS = pointers.Float32(metric.GetCvssv3().GetScore()) + val.TopNVDCVSS = new(metric.GetCvssv3().GetScore()) } } } @@ -980,18 +1061,12 @@ func (s *ImageCVEViewTestSuite) compileExpected(images []*storage.Image, filter // Add CVE ID (V2 format) from database object id := vuln.GetId() - var found bool - for _, seenID := range val.GetCVEIDs() { - if seenID == id { - found = true - break - } - } - if !found { + + if !slices.Contains(val.GetCVEIDs(), id) { val.CVEIDs = append(val.CVEIDs, id) } - val.TopCVSS = pointers.Float32(max(val.GetTopCVSS(), vuln.GetCvss())) + val.TopCVSS = new(max(val.GetTopCVSS(), vuln.GetCvss())) if val.GetFirstDiscoveredInSystem().After(vulnTime) { val.FirstDiscoveredInSystem = &vulnTime @@ -1003,7 +1078,7 @@ func (s *ImageCVEViewTestSuite) compileExpected(images []*storage.Image, filter if !seenForImage.Add(val.CVE) { continue } - val.AffectedImageCount++ + incrAffectedImageCount(val) switch vuln.GetSeverity() { case storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY: @@ -1063,6 +1138,7 @@ func (s *ImageCVEViewTestSuite) compileExpected(images []*storage.Image, filter if options.SkipGetAffectedImages { for _, entry := range cveMap { entry.AffectedImageCount = 0 + entry.AffectedImageCountV2 = 0 } } if options.SkipGetFirstDiscoveredInSystem { @@ -1081,22 +1157,15 @@ func (s *ImageCVEViewTestSuite) compileExpected(images []*storage.Image, filter return ret } -func compileExpectedAffectedImageIDs(images []*storage.Image, filter *filterImpl, less lessFuncForImages) []string { - var affectedImages []*storage.Image +func compileExpectedAffectedImageIDs(images []testImage, filter *filterImpl, less lessFuncForImages) []string { + var affectedImages []testImage for _, image := range images { if !filter.matchImage(image) { continue } for _, component := range image.GetScan().GetComponents() { - var vulnFilterPassed bool - for _, vuln := range component.GetVulns() { - if filter.matchVuln(vuln) { - vulnFilterPassed = true - break - } - } - if vulnFilterPassed { + if slices.ContainsFunc(component.GetVulns(), filter.matchVuln) { affectedImages = append(affectedImages, image) break } @@ -1116,7 +1185,7 @@ func compileExpectedAffectedImageIDs(images []*storage.Image, filter *filterImpl return ret } -func compileExpectedCountBySeverity(images []*storage.Image, filter *filterImpl) *common.ResourceCountByImageCVESeverity { +func compileExpectedCountBySeverity(images []testImage, filter *filterImpl) *common.ResourceCountByImageCVESeverity { sevToCVEsMap := make(map[storage.VulnerabilitySeverity]set.Set[string]) sevToFixableCVEsMap := make(map[storage.VulnerabilitySeverity]set.Set[string]) @@ -1194,17 +1263,79 @@ func standardizeImages(images ...*storage.Image) { } } +// TestImageCVECountFiltering verifies that "Image CVE Count > 0" doesn't drop any +// CVEs from ImageCVEView results when the underlying data is healthy (see ROX-36389 +// for background on this field, including a case where it isn't). +func (s *ImageCVEViewTestSuite) TestImageCVECountFiltering() { + if !features.FlattenImageData.Enabled() { + s.T().Skip("Image CVE Count only maps to images_v2.ScanStats.CveCount when FlattenImageData is enabled") + } + + ctx := s.suiteCtx + + imageV2Store := imageV2DS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + zeroCveImage := &storage.ImageV2{ + Id: imageUtils.NewImageV2ID(&storage.ImageName{Registry: "reg-zero", FullName: "reg-zero"}, "sha-zero"), + Digest: "sha-zero", + Name: &storage.ImageName{Registry: "reg-zero", FullName: "reg-zero"}, + Scan: &storage.ImageScan{ + OperatingSystem: "zero-os", + Components: []*storage.EmbeddedImageScanComponent{}, + }, + } + s.Require().NoError(imageV2Store.UpsertImage(ctx, zeroCveImage)) + actualZeroCve, found, err := imageV2Store.GetImage(ctx, zeroCveImage.GetId()) + s.Require().NoError(err) + s.Require().True(found) + s.Require().Equal(int32(0), actualZeroCve.GetScanStats().GetCveCount()) + + noFilterQuery := search.EmptyQuery() + filteredQuery := search.NewQueryBuilder().AddStrings(search.ImageCVECount, ">0").ProtoQuery() + + noFilterCount, err := s.cveView.Count(ctx, noFilterQuery) + s.Require().NoError(err) + filteredCount, err := s.cveView.Count(ctx, filteredQuery) + s.Require().NoError(err) + + s.T().Logf("ImageCVEView.Count() with no filter: %d; with 'Image CVE Count > 0': %d", noFilterCount, filteredCount) + + s.Equal(noFilterCount, filteredCount, + "expected 'Image CVE Count > 0' to be a no-op on the CVE count") + + noFilterResults, err := s.cveView.Get(ctx, noFilterQuery, views.ReadOptions{}) + s.Require().NoError(err) + filteredResults, err := s.cveView.Get(ctx, filteredQuery, views.ReadOptions{}) + s.Require().NoError(err) + + getCVESet := func(results []CveCore) []string { + ids := make([]string, 0, len(results)) + for _, r := range results { + ids = append(ids, r.GetCVE()) + } + slices.Sort(ids) + return ids + } + + s.Equal(getCVESet(noFilterResults), getCVESet(filteredResults), + "expected the exact same set of CVEs to be returned with and without the 'Image CVE Count > 0' filter") +} + func TestImageCVEUnknownSeverity(t *testing.T) { ctx := sac.WithAllAccess(context.Background()) testDB := pgtest.ForT(t) - // Initialize the datastore. - imageStore := imageDS.GetTestPostgresDataStore(t, testDB.DB) - - // Upsert test images. + // Upsert test images using the appropriate datastore. images := testImages() - for _, image := range images { - assert.NoError(t, imageStore.UpsertImage(ctx, image)) + if features.FlattenImageData.Enabled() { + imageV2Store := imageV2DS.GetTestPostgresDataStore(t, testDB.DB) + for _, image := range images { + assert.NoError(t, imageV2Store.UpsertImage(ctx, imageUtils.ConvertToV2(image))) + } + } else { + imageStore := imageDS.GetTestPostgresDataStore(t, testDB.DB) + for _, image := range images { + assert.NoError(t, imageStore.UpsertImage(ctx, image)) + } } cveView := NewCVEView(testDB.DB) diff --git a/central/views/imagecveflat/db_response.go b/central/views/imagecveflat/db_response.go index 92f59bcf368df..21ac32e0889c9 100644 --- a/central/views/imagecveflat/db_response.go +++ b/central/views/imagecveflat/db_response.go @@ -32,6 +32,10 @@ func (c *imageCVEFlatResponse) GetCVEIDs() []string { } func (c *imageCVEFlatResponse) GetSeverity() *storage.VulnerabilitySeverity { + if c.Severity == nil { + unknown := storage.VulnerabilitySeverity_UNKNOWN_VULNERABILITY_SEVERITY + return &unknown + } return c.Severity } @@ -75,8 +79,10 @@ func (c *imageCVEFlatResponse) GetFirstImageOccurrence() *time.Time { return c.FirstImageOccurrence } -func (c *imageCVEFlatResponse) GetState() *storage.VulnerabilityState { return c.State } - -type imageCVEFlatCount struct { - CVECount int `db:"cve_count"` +func (c *imageCVEFlatResponse) GetState() *storage.VulnerabilityState { + if c.State == nil { + observed := storage.VulnerabilityState_OBSERVED + return &observed + } + return c.State } diff --git a/central/views/imagecveflat/view_impl.go b/central/views/imagecveflat/view_impl.go index f61e866f94072..081c2765748f7 100644 --- a/central/views/imagecveflat/view_impl.go +++ b/central/views/imagecveflat/view_impl.go @@ -4,7 +4,7 @@ import ( "context" "sort" - "github.com/pkg/errors" + imagev2common "github.com/stackrox/rox/central/imagev2/common" "github.com/stackrox/rox/central/views" "github.com/stackrox/rox/central/views/common" v1 "github.com/stackrox/rox/generated/api/v1" @@ -14,11 +14,9 @@ import ( "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/walker" - "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" pgSearch "github.com/stackrox/rox/pkg/search/postgres" "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" - "github.com/stackrox/rox/pkg/utils" ) var ( @@ -35,49 +33,28 @@ func (v *imageCVEFlatViewImpl) Count(ctx context.Context, q *v1.Query) (int, err if err := common.ValidateQuery(q); err != nil { return 0, err } - - var err error - q, err = common.WithSACFilter(ctx, resources.Image, q) - if err != nil { - return 0, err - } + q = imagev2common.WithRowsFromImageV2Only(q) queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() - var results []*imageCVEFlatCount - results, err = pgSearch.RunSelectRequestForSchema[imageCVEFlatCount](queryCtx, v.db, v.schema, common.WithCountQuery(q, search.CVE)) - if err != nil { - return 0, err - } - if len(results) == 0 { - return 0, nil - } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", q.String()) - utils.Should(err) - return 0, err - } - return results[0].CVECount, nil + return pgSearch.RunDistinctCountForSchema(queryCtx, v.db, v.schema, q, search.CVE) } func (v *imageCVEFlatViewImpl) Get(ctx context.Context, q *v1.Query, options views.ReadOptions) ([]CveFlat, error) { if err := common.ValidateQuery(q); err != nil { return nil, err } - var err error + q = imagev2common.WithRowsFromImageV2Only(q) + // Avoid changing the passed query cloned := q.CloneVT() // Update the sort options to use aggregations if necessary as we are grouping by CVEs cloned = common.UpdateSortAggs(cloned) - cloned, err = common.WithSACFilter(ctx, resources.Image, cloned) - if err != nil { - log.Error(err) - return nil, err - } // Performance improvements to narrow aggregations performed var cveIDsToFilter []string + var err error if cloned.GetPagination().GetLimit() > 0 || cloned.GetPagination().GetOffset() > 0 { cveIDsToFilter, err = v.getFilteredCVEs(ctx, cloned) if err != nil { @@ -95,20 +72,18 @@ func (v *imageCVEFlatViewImpl) Get(ctx context.Context, q *v1.Query, options vie queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() - var results []*imageCVEFlatResponse - results, err = pgSearch.RunSelectRequestForSchema[imageCVEFlatResponse](queryCtx, v.db, v.schema, withSelectCVEFlatResponseQuery(cloned, cveIDsToFilter, options)) - if err != nil { - log.Error(err) - return nil, err - } - - ret := make([]CveFlat, 0, len(results)) - for _, r := range results { + var ret []CveFlat + err = pgSearch.RunSelectRequestForSchemaFn[imageCVEFlatResponse](queryCtx, v.db, v.schema, withSelectCVEFlatResponseQuery(cloned, cveIDsToFilter, options), func(r *imageCVEFlatResponse) error { // For each record, sort the IDs so that result looks consistent. sort.SliceStable(r.CVEIDs, func(i, j int) bool { return r.CVEIDs[i] < r.CVEIDs[j] }) ret = append(ret, r) + return nil + }) + if err != nil { + log.Error(err) + return nil, err } return ret, nil } @@ -179,16 +154,14 @@ func (v *imageCVEFlatViewImpl) getFilteredCVEs(ctx context.Context, q *v1.Query) // TODO(@charmik) : Update the SQL query generator to not include 'ORDER BY' and 'GROUP BY' fields in the select clause (before where). // SQL syntax does not need those fields in the select clause. The below query for example would work fine // "SELECT JSONB_AGG(DISTINCT(image_cves.Id)) AS cve_id FROM image_cves GROUP BY image_cves.CveBaseInfo_Cve ORDER BY MAX(image_cves.Cvss) DESC LIMIT 20;" - var identifiersList []*imageCVEFlatResponse - identifiersList, err := pgSearch.RunSelectRequestForSchema[imageCVEFlatResponse](queryCtx, v.db, v.schema, withSelectCVEIdentifiersQuery(q)) + err := pgSearch.RunSelectRequestForSchemaFn[imageCVEFlatResponse](queryCtx, v.db, v.schema, withSelectCVEIdentifiersQuery(q), func(r *imageCVEFlatResponse) error { + cveIDsToFilter = append(cveIDsToFilter, r.CVEIDs...) + return nil + }) if err != nil { log.Error(err) return nil, err } - for _, idList := range identifiersList { - cveIDsToFilter = append(cveIDsToFilter, idList.CVEIDs...) - } - return cveIDsToFilter, nil } diff --git a/central/views/imagecveflat/view_test.go b/central/views/imagecveflat/view_test.go index 9bb239e94698d..5324fd97bb0cd 100644 --- a/central/views/imagecveflat/view_test.go +++ b/central/views/imagecveflat/view_test.go @@ -5,6 +5,7 @@ package imagecveflat import ( "context" "fmt" + "slices" "sort" "testing" "time" @@ -12,18 +13,15 @@ import ( imageCVEV2DS "github.com/stackrox/rox/central/cve/image/v2/datastore" deploymentDS "github.com/stackrox/rox/central/deployment/datastore" imageDS "github.com/stackrox/rox/central/image/datastore" - imagePostgresV2 "github.com/stackrox/rox/central/image/datastore/store/v2/postgres" imageComponentV2DS "github.com/stackrox/rox/central/imagecomponent/v2/datastore" - "github.com/stackrox/rox/central/ranking" - mockRisks "github.com/stackrox/rox/central/risk/datastore/mocks" + imageV2DS "github.com/stackrox/rox/central/imagev2/datastore" "github.com/stackrox/rox/central/views" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" - "github.com/stackrox/rox/pkg/concurrency" "github.com/stackrox/rox/pkg/cve" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/fixtures" imageSamples "github.com/stackrox/rox/pkg/fixtures/image" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/postgres/pgtest" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/sac" @@ -37,7 +35,6 @@ import ( "github.com/stackrox/rox/pkg/set" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "go.uber.org/mock/gomock" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -55,14 +52,21 @@ type testCase struct { type lessFunc func(records []*imageCVEFlatResponse) func(i, j int) bool +// testImage is satisfied by both *storage.Image and *storage.ImageV2. +type testImage interface { + GetId() string + GetName() *storage.ImageName + GetScan() *storage.ImageScan +} + type filterImpl struct { - matchImage func(image *storage.Image) bool + matchImage func(image testImage) bool matchVuln func(cve *storage.ImageCVEV2) bool } func matchAllFilter() *filterImpl { return &filterImpl{ - matchImage: func(_ *storage.Image) bool { + matchImage: func(_ testImage) bool { return true }, matchVuln: func(_ *storage.ImageCVEV2) bool { @@ -73,7 +77,7 @@ func matchAllFilter() *filterImpl { func matchNoneFilter() *filterImpl { return &filterImpl{ - matchImage: func(_ *storage.Image) bool { + matchImage: func(_ testImage) bool { return false }, matchVuln: func(_ *storage.ImageCVEV2) bool { @@ -82,7 +86,7 @@ func matchNoneFilter() *filterImpl { } } -func (f *filterImpl) withImageFilter(fn func(image *storage.Image) bool) *filterImpl { +func (f *filterImpl) withImageFilter(fn func(image testImage) bool) *filterImpl { f.matchImage = fn return f } @@ -103,7 +107,7 @@ type ImageCVEFlatViewTestSuite struct { cveView CveFlatView suiteCtx context.Context - testImages []*storage.Image + testImages []testImage testImagesToDeployments map[string][]*storage.Deployment componentDatastore imageComponentV2DS.DataStore @@ -111,87 +115,128 @@ type ImageCVEFlatViewTestSuite struct { } func (s *ImageCVEFlatViewTestSuite) SetupSuite() { - mockCtrl := gomock.NewController(s.T()) s.suiteCtx = sac.WithAllAccess(context.Background()) + ctx := s.suiteCtx s.testDB = pgtest.ForT(s.T()) - mockRisk := mockRisks.NewMockDataStore(mockCtrl) - - // Initialize the datastore. - imageStore := imageDS.NewWithPostgres( - imagePostgresV2.New(s.testDB.DB, false, concurrency.NewKeyFence()), - mockRisk, - ranking.ImageRanker(), - ranking.ComponentRanker(), - ) - deploymentStore, err := deploymentDS.NewTestDataStore( - s.T(), - s.testDB, - &deploymentDS.DeploymentTestStoreParams{ - ImagesDataStore: imageStore, - RisksDataStore: mockRisk, - ClusterRanker: ranking.ClusterRanker(), - NamespaceRanker: ranking.NamespaceRanker(), - DeploymentRanker: ranking.DeploymentRanker(), - }, - ) + // Initialize the datastores. + deploymentStore, err := deploymentDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(err) s.componentDatastore = imageComponentV2DS.GetTestPostgresDataStore(s.T(), s.testDB) s.cveDatastore = imageCVEV2DS.GetTestPostgresDataStore(s.T(), s.testDB) - s.Require().NoError(err) - - // Upsert test images. - images, err := imageSamples.GetTestImages(s.T()) - s.Require().NoError(err) - // set cvss metrics list with one nvd cvss score - for _, image := range images { - for _, components := range image.GetScan().GetComponents() { - for _, vuln := range components.GetVulns() { - cvssScore := &storage.CVSSScore{ + // setCVSSMetrics sets NVD CVSS metrics on all vulns in an image scan. + setCVSSMetrics := func(scan *storage.ImageScan) { + for _, component := range scan.GetComponents() { + for _, vuln := range component.GetVulns() { + vuln.CvssMetrics = []*storage.CVSSScore{{ Source: storage.Source_SOURCE_NVD, CvssScore: &storage.CVSSScore_Cvssv3{ - Cvssv3: &storage.CVSSV3{ - Score: 10, - }, + Cvssv3: &storage.CVSSV3{Score: 10}, }, - } - vuln.CvssMetrics = []*storage.CVSSScore{cvssScore} + }} vuln.NvdCvss = 10 } } - s.Require().NoError(imageStore.UpsertImage(s.suiteCtx, image)) } - // Ensure that the image is stored and constructed as expected. - for idx, image := range images { - actual, found, err := imageStore.GetImage(s.suiteCtx, image.GetId()) + // Upsert images using the appropriate datastore based on feature flag. + var deployments []*storage.Deployment + if features.FlattenImageData.Enabled() { + // Insert a V1 image to verify that V1 rows are excluded from all view queries. + s.T().Setenv(features.FlattenImageData.EnvVar(), "false") + v1Store := imageDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + s.Require().NoError(v1Store.UpsertImage(ctx, fixtures.GetImageSherlockHolmes1())) + s.T().Setenv(features.FlattenImageData.EnvVar(), "true") + + imageV2Store := imageV2DS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + imagesV2, err := imageSamples.GetTestImagesV2(s.T()) + s.Require().NoError(err) + for _, imgV2 := range imagesV2 { + setCVSSMetrics(imgV2.GetScan()) + s.Require().NoError(imageV2Store.UpsertImage(ctx, imgV2)) + } + // Verify stored V2 images and use them for expected results. + for idx, imgV2 := range imagesV2 { + actual, found, err := imageV2Store.GetImage(ctx, imgV2.GetId()) + s.Require().NoError(err) + s.Require().True(found) + imagesV2[idx] = actual + } + s.testImages = make([]testImage, len(imagesV2)) + for i, img := range imagesV2 { + s.testImages[i] = img + } + deployments = []*storage.Deployment{ + fixtures.GetDeploymentWithImageV2(testconsts.Cluster1, testconsts.NamespaceA, imagesV2[1]), + fixtures.GetDeploymentWithImageV2(testconsts.Cluster2, testconsts.NamespaceB, imagesV2[1]), + fixtures.GetDeploymentWithImageV2(testconsts.Cluster2, testconsts.NamespaceB, imagesV2[2]), + } + } else { + imageStore := imageDS.GetTestPostgresDataStore(s.T(), s.testDB.DB) + images, err := imageSamples.GetTestImages(s.T()) s.Require().NoError(err) - s.Require().True(found) + for _, image := range images { + setCVSSMetrics(image.GetScan()) + s.Require().NoError(imageStore.UpsertImage(ctx, image)) + } + // Ensure that the image is stored and constructed as expected. + for idx, image := range images { + actual, found, err := imageStore.GetImage(ctx, image.GetId()) + s.Require().NoError(err) + s.Require().True(found) - cloned := actual.CloneVT() - // Adjust dynamic fields and ensure images in ACS are as expected. - standardizeImages(image, cloned) + cloned := actual.CloneVT() + standardizeImages(image, cloned) - // Now that we confirmed that images match, use stored image to establish the expected test results. - // This makes dynamic fields matching (e.g. created at) straightforward. - images[idx] = actual + images[idx] = actual + } + s.testImages = make([]testImage, len(images)) + for i, img := range images { + s.testImages[i] = img + } + deployments = []*storage.Deployment{ + fixtures.GetDeploymentWithImage(testconsts.Cluster1, testconsts.NamespaceA, images[1]), + fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, images[1]), + fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, images[2]), + } } - s.testImages = images + s.cveView = NewCVEFlatView(s.testDB.DB) - s.Require().Len(images, 5) - deployments := []*storage.Deployment{ - fixtures.GetDeploymentWithImage(testconsts.Cluster1, testconsts.NamespaceA, images[1]), - fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, images[1]), - fixtures.GetDeploymentWithImage(testconsts.Cluster2, testconsts.NamespaceB, images[2]), - } + s.Require().Len(s.testImages, 5) for _, d := range deployments { - s.Require().NoError(deploymentStore.UpsertDeployment(s.suiteCtx, d)) + s.Require().NoError(deploymentStore.UpsertDeployment(ctx, d)) } s.testImagesToDeployments = make(map[string][]*storage.Deployment) - s.testImagesToDeployments[images[1].GetId()] = []*storage.Deployment{deployments[0], deployments[1]} - s.testImagesToDeployments[images[2].GetId()] = []*storage.Deployment{deployments[2]} + s.testImagesToDeployments[s.testImages[1].GetId()] = []*storage.Deployment{deployments[0], deployments[1]} + s.testImagesToDeployments[s.testImages[2].GetId()] = []*storage.Deployment{deployments[2]} +} + +// imageScopeCategory returns the search category for image scoping. +func (s *ImageCVEFlatViewTestSuite) imageScopeCategory() v1.SearchCategory { + if features.FlattenImageData.Enabled() { + return v1.SearchCategory_IMAGES_V2 + } + return v1.SearchCategory_IMAGES +} + +// imageSearchField returns the search field label for image ID. +func imageSearchField() search.FieldLabel { + if features.FlattenImageData.Enabled() { + return search.ImageID + } + return search.ImageSHA +} + +func (s *ImageCVEFlatViewTestSuite) findImageByName(fullName string) testImage { + for _, img := range s.testImages { + if img.GetName().GetFullName() == fullName { + return img + } + } + return nil } func (s *ImageCVEFlatViewTestSuite) TestGetImageCVEFlat() { @@ -232,7 +277,7 @@ func (s *ImageCVEFlatViewTestSuite) TestGetImageCVEFlatSAC() { // Wrap image filter with sac filter. matchFilter := *tc.matchFilter baseImageMatchFilter := matchFilter.matchImage - matchFilter.withImageFilter(func(image *storage.Image) bool { + matchFilter.withImageFilter(func(image testImage) bool { if sacTC[image.GetId()] { return baseImageMatchFilter(image) } @@ -336,7 +381,7 @@ func (s *ImageCVEFlatViewTestSuite) TestCountImageCVEFlatSAC() { // Wrap image filter with sac filter. matchFilter := *tc.matchFilter baseImageMatchFilter := matchFilter.matchImage - matchFilter.withImageFilter(func(image *storage.Image) bool { + matchFilter.withImageFilter(func(image testImage) bool { if sacTC[image.GetId()] { return baseImageMatchFilter(image) } @@ -351,6 +396,11 @@ func (s *ImageCVEFlatViewTestSuite) TestCountImageCVEFlatSAC() { } func (s *ImageCVEFlatViewTestSuite) testCases() []testCase { + wordpressDebian := s.findImageByName("quay.io/appcontainers/wordpress:debian") + s.Require().NotNil(wordpressDebian) + wordpressLatest := s.findImageByName("quay.io/appcontainers/wordpress:latest") + s.Require().NotNil(wordpressLatest) + return []testCase{ { desc: "search all", @@ -371,7 +421,7 @@ func (s *ImageCVEFlatViewTestSuite) testCases() []testCase { ctx: context.Background(), q: search.NewQueryBuilder(). AddExactMatches(search.ImageName, "quay.io/appcontainers/wordpress:latest").ProtoQuery(), - matchFilter: matchAllFilter().withImageFilter(func(image *storage.Image) bool { + matchFilter: matchAllFilter().withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:latest" }), }, @@ -383,7 +433,7 @@ func (s *ImageCVEFlatViewTestSuite) testCases() []testCase { AddExactMatches(search.ImageName, "quay.io/appcontainers/wordpress:debian"). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:debian" }). withVulnFilter(func(vuln *storage.ImageCVEV2) bool { @@ -459,7 +509,7 @@ func (s *ImageCVEFlatViewTestSuite) testCases() []testCase { AddExactMatches(search.ImageName, "quay.io/appcontainers/wordpress:debian"). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:debian" }). withVulnFilter(func(vuln *storage.ImageCVEV2) bool { @@ -501,15 +551,15 @@ func (s *ImageCVEFlatViewTestSuite) testCases() []testCase { { desc: "search one cve w/ image scope", ctx: scoped.Context(context.Background(), scoped.Scope{ - IDs: []string{"sha256:6ef31316f4f9e0c31a8f4e602ba287a210d66934f91b1616f1c9b957201d025c"}, - Level: v1.SearchCategory_IMAGES, + IDs: []string{wordpressDebian.GetId()}, + Level: s.imageScopeCategory(), }), q: search.NewQueryBuilder(). AddExactMatches(search.CVE, "CVE-2022-1552"). AddExactMatches(search.ImageName, "quay.io/appcontainers/wordpress:debian"). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:debian" }). withVulnFilter(func(vuln *storage.ImageCVEV2) bool { @@ -519,8 +569,8 @@ func (s *ImageCVEFlatViewTestSuite) testCases() []testCase { { desc: "search critical severity w/ cve & image scope", ctx: scoped.Context(context.Background(), scoped.Scope{ - IDs: []string{"sha256:6ef31316f4f9e0c31a8f4e602ba287a210d66934f91b1616f1c9b957201d025c"}, - Level: v1.SearchCategory_IMAGES, + IDs: []string{wordpressDebian.GetId()}, + Level: s.imageScopeCategory(), Parent: &scoped.Scope{ IDs: []string{getTestCVEID(getTestCVE(), getTestComponentID(&storage.EmbeddedImageScanComponent{ @@ -529,7 +579,7 @@ func (s *ImageCVEFlatViewTestSuite) testCases() []testCase { Source: storage.SourceType_OS, Location: "", Architecture: "", - }, "sha256:05dd8ed5c76ad3c9f06481770828cf17b8c89f1e406c91d548426dd70fe94560", 0), 0)}, + }, wordpressLatest.GetId(), 0), 0)}, Level: v1.SearchCategory_IMAGE_VULNERABILITIES_V2, }, }), @@ -537,7 +587,7 @@ func (s *ImageCVEFlatViewTestSuite) testCases() []testCase { AddExactMatches(search.Severity, storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY.String()). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { return image.GetName().GetFullName() == "quay.io/appcontainers/wordpress:debian" && image.GetScan().GetOperatingSystem() == "debian:8" }). @@ -554,7 +604,7 @@ func (s *ImageCVEFlatViewTestSuite) testCases() []testCase { AddStrings(search.PlatformComponent, "false", "-"). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { deps, ok := s.testImagesToDeployments[image.GetId()] if !ok { // include inactive image @@ -579,7 +629,7 @@ func (s *ImageCVEFlatViewTestSuite) testCases() []testCase { AddStrings(search.PlatformComponent, "true", "-"). ProtoQuery(), matchFilter: matchAllFilter(). - withImageFilter(func(image *storage.Image) bool { + withImageFilter(func(image testImage) bool { deps, ok := s.testImagesToDeployments[image.GetId()] if !ok { // include inactive image @@ -605,15 +655,15 @@ func (s *ImageCVEFlatViewTestSuite) paginationTestCases() []testCase { desc: "w/ affected image sort", q: search.NewQueryBuilder().WithPagination( search.NewPagination().AddSortOption( - search.NewSortOption(search.ImageSHA).AggregateBy(aggregatefunc.Count, true).Reversed(true), + search.NewSortOption(imageSearchField()).AggregateBy(aggregatefunc.Count, true).Reversed(true), ).AddSortOption(search.NewSortOption(search.CVE)), ).ProtoQuery(), less: func(records []*imageCVEFlatResponse) func(i, j int) bool { return func(i, j int) bool { - if records[i].AffectedImageCount == records[j].AffectedImageCount { + if records[i].GetAffectedImageCount() == records[j].GetAffectedImageCount() { return records[i].CVE < records[j].CVE } - return records[i].AffectedImageCount > records[j].AffectedImageCount + return records[i].GetAffectedImageCount() > records[j].GetAffectedImageCount() } }, }, @@ -775,7 +825,7 @@ func applyPaginationProps(baseTc *testCase, paginationTc testCase) { baseTc.less = paginationTc.less } -func (s *ImageCVEFlatViewTestSuite) compileExpected(images []*storage.Image, filter *filterImpl, options views.ReadOptions, less lessFunc) []CveFlat { +func (s *ImageCVEFlatViewTestSuite) compileExpected(images []testImage, filter *filterImpl, options views.ReadOptions, less lessFunc) []CveFlat { cveMap := make(map[string]*imageCVEFlatResponse) for _, image := range images { @@ -784,7 +834,7 @@ func (s *ImageCVEFlatViewTestSuite) compileExpected(images []*storage.Image, fil } var seenForImage set.Set[string] - components, err := s.componentDatastore.SearchRawImageComponents(s.suiteCtx, search.NewQueryBuilder().AddExactMatches(search.ImageSHA, image.GetId()).ProtoQuery()) + components, err := s.componentDatastore.SearchRawImageComponents(s.suiteCtx, search.NewQueryBuilder().AddExactMatches(imageSearchField(), image.GetId()).ProtoQuery()) s.Require().NoError(err) // Instead of rebuilding these from what we return in the image, grab them from the component and cve store for _, component := range components { @@ -813,21 +863,21 @@ func (s *ImageCVEFlatViewTestSuite) compileExpected(images []*storage.Image, fil if val == nil { val = &imageCVEFlatResponse{ CVE: vuln.GetCveBaseInfo().GetCve(), - TopCVSS: pointers.Float32(vuln.GetCvss()), + TopCVSS: new(vuln.GetCvss()), FirstDiscoveredInSystem: &vulnTime, Published: &vulnPublishDate, Severity: vuln.GetSeverity().Enum(), FirstImageOccurrence: &vulnImageOccurrence, - State: pointers.Pointer(vuln.GetState()), - ImpactScore: pointers.Float32(impactScore), - EpssProbability: pointers.Float32(vuln.GetCveBaseInfo().GetEpss().GetEpssProbability()), + State: new(vuln.GetState()), + ImpactScore: new(impactScore), + EpssProbability: new(vuln.GetCveBaseInfo().GetEpss().GetEpssProbability()), } for _, metric := range vuln.GetCveBaseInfo().GetCvssMetrics() { if metric.GetSource() == storage.Source_SOURCE_NVD { if metric.GetCvssv2() != nil { - val.TopNVDCVSS = pointers.Float32(metric.GetCvssv2().GetScore()) + val.TopNVDCVSS = new(metric.GetCvssv2().GetScore()) } else { - val.TopNVDCVSS = pointers.Float32(metric.GetCvssv3().GetScore()) + val.TopNVDCVSS = new(metric.GetCvssv3().GetScore()) } } } @@ -835,23 +885,16 @@ func (s *ImageCVEFlatViewTestSuite) compileExpected(images []*storage.Image, fil } id := vuln.GetId() - var found bool - for _, seenID := range val.GetCVEIDs() { - if seenID == id { - found = true - break - } - } - if !found { + if !slices.Contains(val.GetCVEIDs(), id) { val.CVEIDs = append(val.CVEIDs, id) } - val.TopCVSS = pointers.Float32(max(val.GetTopCVSS(), vuln.GetCvss())) - val.ImpactScore = pointers.Float32(max(*val.ImpactScore, impactScore)) - val.EpssProbability = pointers.Float32(max(*val.EpssProbability, vuln.GetCveBaseInfo().GetEpss().GetEpssProbability())) + val.TopCVSS = new(max(val.GetTopCVSS(), vuln.GetCvss())) + val.ImpactScore = new(max(*val.ImpactScore, impactScore)) + val.EpssProbability = new(max(*val.EpssProbability, vuln.GetCveBaseInfo().GetEpss().GetEpssProbability())) if vuln.GetSeverity().Number() > val.GetSeverity().Number() { - val.Severity = pointers.Pointer(vuln.GetSeverity()) + val.Severity = new(vuln.GetSeverity()) } if val.GetFirstDiscoveredInSystem().After(vulnTime) { @@ -867,7 +910,11 @@ func (s *ImageCVEFlatViewTestSuite) compileExpected(images []*storage.Image, fil if !seenForImage.Add(val.CVE) { continue } - val.AffectedImageCount++ + if features.FlattenImageData.Enabled() { + val.AffectedImageCountV2++ + } else { + val.AffectedImageCount++ + } } } } @@ -885,6 +932,7 @@ func (s *ImageCVEFlatViewTestSuite) compileExpected(images []*storage.Image, fil if options.SkipGetAffectedImages { for _, entry := range cveMap { entry.AffectedImageCount = 0 + entry.AffectedImageCountV2 = 0 } } if options.SkipGetFirstDiscoveredInSystem { diff --git a/central/views/images/view_impl.go b/central/views/images/view_impl.go index f223db006efbb..e5499d6359f0a 100644 --- a/central/views/images/view_impl.go +++ b/central/views/images/view_impl.go @@ -10,8 +10,8 @@ import ( "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/walker" - "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/paginated" pgSearch "github.com/stackrox/rox/pkg/search/postgres" ) @@ -29,26 +29,19 @@ func (v *imageCoreViewImpl) Get(ctx context.Context, query *v1.Query) ([]ImageCo return nil, err } - var err error - query, err = common.WithSACFilter(ctx, resources.Image, query) - if err != nil { - return nil, err - } query = withSelectQuery(query) queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) defer cancel() - var results []*imageResponse - results, err = pgSearch.RunSelectRequestForSchema[imageResponse](queryCtx, v.db, v.schema, query) + ret := make([]ImageCore, 0, paginated.GetLimit(query.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[imageResponse](queryCtx, v.db, v.schema, query, func(r *imageResponse) error { + ret = append(ret, r) + return nil + }) if err != nil { return nil, err } - - ret := make([]ImageCore, 0, len(results)) - for _, r := range results { - ret = append(ret, r) - } return ret, nil } diff --git a/central/views/images/view_test.go b/central/views/images/view_test.go index b4a8410acbdeb..741970fec5c1e 100644 --- a/central/views/images/view_test.go +++ b/central/views/images/view_test.go @@ -85,6 +85,8 @@ type ImageViewTestSuite struct { testImagesMap map[string]*storage.Image testImagesV2Map map[string]*storage.ImageV2 scopeToImageIDs map[string]set.StringSet + // zeroCveImageID is used by TestImageCVECountFiltering. + zeroCveImageID string } func (s *ImageViewTestSuite) SetupSuite() { @@ -114,6 +116,15 @@ func (s *ImageViewTestSuite) SetupSuite() { s.testImagesV2Map[actual.GetId()] = image } + // For TestImageCVECountFiltering. + zeroCveImageV2 := utils.ConvertToV2(zeroCveTestImage()) + s.Require().NoError(imageStore.UpsertImage(ctx, zeroCveImageV2)) + actualZeroCve, found, err := imageStore.GetImage(ctx, zeroCveImageV2.GetId()) + s.Require().NoError(err) + s.Require().True(found) + s.zeroCveImageID = actualZeroCve.GetId() + s.testImagesV2Map[s.zeroCveImageID] = zeroCveImageV2 + s.imagesView = NewImageView(s.testDB.DB) deployments := []*storage.Deployment{ @@ -167,6 +178,15 @@ func (s *ImageViewTestSuite) SetupSuite() { s.testImagesMap[actual.GetId()] = actual } + // For TestImageCVECountFiltering. + zeroCveImage := zeroCveTestImage() + s.Require().NoError(imageStore.UpsertImage(ctx, zeroCveImage)) + actualZeroCve, found, err := imageStore.GetImage(ctx, zeroCveImage.GetId()) + s.Require().NoError(err) + s.Require().True(found) + s.zeroCveImageID = actualZeroCve.GetId() + s.testImagesMap[s.zeroCveImageID] = actualZeroCve + s.imagesView = NewImageView(s.testDB.DB) deployments := []*storage.Deployment{ @@ -486,6 +506,24 @@ func (s *ImageViewTestSuite) TestGetImagesCore() { } } +// TestImageCVECountFiltering verifies that "Image CVE Count > 0" excludes zero-CVE +// images from ImageView results (see ROX-36389 for background on this field). +func (s *ImageViewTestSuite) TestImageCVECountFiltering() { + ctx := sac.WithAllAccess(context.Background()) + + query := search.NewQueryBuilder().AddStrings(search.ImageCVECount, ">0").ProtoQuery() + cores, err := s.imagesView.Get(ctx, query) + s.Require().NoError(err) + + ids := make([]string, 0, len(cores)) + for _, c := range cores { + ids = append(ids, c.GetImageID()) + } + + s.NotContains(ids, s.zeroCveImageID, "expected the 0-CVE image to be excluded by 'Image CVE Count > 0'") + s.NotEmpty(ids, "expected images with CVEs to still be returned") +} + func (s *ImageViewTestSuite) compileExpected(filter *filterImpl, less lessFunc, hasVulnFilters, hasSortBySeverityCounts bool) []ImageCore { expected := make([]*imageResponse, 0) if features.FlattenImageData.Enabled() { @@ -814,3 +852,20 @@ func testImages() []*storage.Image { } return []*storage.Image{img1, img2, img3, img4} } + +func zeroCveTestImage() *storage.Image { + return &storage.Image{ + Id: "sha5", + Name: &storage.ImageName{ + Registry: "reg5", + Remote: "img5", + Tag: "tag5", + FullName: "reg5/img5:tag5", + }, + RiskScore: 1, + Scan: &storage.ImageScan{ + OperatingSystem: "os5", + Components: []*storage.EmbeddedImageScanComponent{}, + }, + } +} diff --git a/central/views/nodecve/db_response.go b/central/views/nodecve/db_response.go index e59df0ddd12ff..cb0972aa9edc6 100644 --- a/central/views/nodecve/db_response.go +++ b/central/views/nodecve/db_response.go @@ -77,10 +77,6 @@ func (c *nodeCVECoreResponse) GetOperatingSystemCount() int { return c.OperatingSystemCount } -type nodeCVECoreCount struct { - CVECount int `db:"cve_count"` -} - type countByNodeCVESeverity struct { CriticalSeverityCount int `db:"critical_severity_count"` FixableCriticalSeverityCount int `db:"fixable_critical_severity_count"` diff --git a/central/views/nodecve/view_impl.go b/central/views/nodecve/view_impl.go index e2b3ca109b489..259b06496399b 100644 --- a/central/views/nodecve/view_impl.go +++ b/central/views/nodecve/view_impl.go @@ -4,16 +4,14 @@ import ( "context" "sort" - "github.com/pkg/errors" "github.com/stackrox/rox/central/views/common" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/walker" - "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/paginated" pgSearch "github.com/stackrox/rox/pkg/search/postgres" "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" - "github.com/stackrox/rox/pkg/utils" ) type nodeCVECoreViewImpl struct { @@ -26,46 +24,15 @@ func (n *nodeCVECoreViewImpl) Count(ctx context.Context, q *v1.Query) (int, erro return 0, err } - var err error - q, err = common.WithSACFilter(ctx, resources.Node, q) - if err != nil { - return 0, err - } - - var results []*nodeCVECoreCount - results, err = pgSearch.RunSelectRequestForSchema[nodeCVECoreCount](ctx, n.db, n.schema, common.WithCountQuery(q, search.CVE)) - if err != nil { - return 0, err - } - if len(results) == 0 { - return 0, nil - } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", q.String()) - utils.Should(err) - return 0, err - } - return results[0].CVECount, nil + return pgSearch.RunDistinctCountForSchema(ctx, n.db, n.schema, q, search.CVE) } func (n *nodeCVECoreViewImpl) Get(ctx context.Context, q *v1.Query) ([]CveCore, error) { if err := common.ValidateQuery(q); err != nil { return nil, err } - var err error - q, err = common.WithSACFilter(ctx, resources.Node, q) - if err != nil { - return nil, err - } - - var results []*nodeCVECoreResponse - results, err = pgSearch.RunSelectRequestForSchema[nodeCVECoreResponse](ctx, n.db, n.schema, withSelectQuery(q)) - if err != nil { - return nil, err - } - - ret := make([]CveCore, 0, len(results)) - for _, r := range results { + ret := make([]CveCore, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[nodeCVECoreResponse](ctx, n.db, n.schema, withSelectQuery(q), func(r *nodeCVECoreResponse) error { // For each record, sort the IDs so that result looks consistent. sort.SliceStable(r.CVEIDs, func(i, j int) bool { return r.CVEIDs[i] < r.CVEIDs[j] @@ -74,6 +41,10 @@ func (n *nodeCVECoreViewImpl) Get(ctx context.Context, q *v1.Query) ([]CveCore, return r.NodeIDs[i] < r.NodeIDs[j] }) ret = append(ret, r) + return nil + }) + if err != nil { + return nil, err } return ret, nil } @@ -83,49 +54,31 @@ func (n *nodeCVECoreViewImpl) CountBySeverity(ctx context.Context, q *v1.Query) return nil, err } - var err error - q, err = common.WithSACFilter(ctx, resources.Node, q) + result, err := pgSearch.RunSelectOneForSchema[countByNodeCVESeverity](ctx, n.db, n.schema, common.WithCountBySeverityAndFixabilityQuery(q, search.CVE)) if err != nil { return nil, err } - - var results []*countByNodeCVESeverity - results, err = pgSearch.RunSelectRequestForSchema[countByNodeCVESeverity](ctx, n.db, n.schema, common.WithCountBySeverityAndFixabilityQuery(q, search.CVE)) - if err != nil { - return nil, err - } - if len(results) == 0 { + if result == nil { return common.NewEmptyResourceCountByCVESeverity(), nil } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", q.String()) - utils.Should(err) - return common.NewEmptyResourceCountByCVESeverity(), err - } - - return results[0], nil + return result, nil } func (n *nodeCVECoreViewImpl) GetNodeIDs(ctx context.Context, q *v1.Query) ([]string, error) { - var err error - q, err = common.WithSACFilter(ctx, resources.Node, q) - if err != nil { - return nil, err - } - q.Selects = []*v1.QuerySelect{ search.NewQuerySelect(search.NodeID).Distinct().Proto(), } - var results []*nodeResponse - results, err = pgSearch.RunSelectRequestForSchema[nodeResponse](ctx, n.db, n.schema, q) - if err != nil || len(results) == 0 { + ret := make([]string, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[nodeResponse](ctx, n.db, n.schema, q, func(r *nodeResponse) error { + ret = append(ret, r.GetNodeID()) + return nil + }) + if err != nil { return nil, err } - - ret := make([]string, 0, len(results)) - for _, r := range results { - ret = append(ret, r.GetNodeID()) + if len(ret) == 0 { + return nil, nil } return ret, nil } diff --git a/central/views/nodecve/view_test.go b/central/views/nodecve/view_test.go index 2df49935e3130..49550320b780f 100644 --- a/central/views/nodecve/view_test.go +++ b/central/views/nodecve/view_test.go @@ -815,7 +815,7 @@ func (s *NodeCVEViewTestSuite) compileExpectedCVECores(filter *filterImpl) []Cve } } } - var expected []CveCore + expected := make([]CveCore, 0, len(cveMap)) for _, withStats := range cveMap { core := withStats.response core.CVEIDs = set.NewStringSet(core.CVEIDs...).AsSlice() @@ -904,10 +904,7 @@ func (s *NodeCVEViewTestSuite) compileExpectedCVECoresWithPagination(filter *fil if limit == 0 { return expected } - end := offset + limit - if end > len(expected) { - end = len(expected) - } + end := min(offset+limit, len(expected)) return expected[offset:end] } diff --git a/central/views/platformcve/db_response.go b/central/views/platformcve/db_response.go index 8f4ebe7d53a57..436b08cbba25e 100644 --- a/central/views/platformcve/db_response.go +++ b/central/views/platformcve/db_response.go @@ -65,10 +65,6 @@ func (c *platformCVECoreResponse) GetFirstDiscoveredTime() *time.Time { return c.FirstDiscoveredTime } -type platformCVECoreCount struct { - CVECount int `db:"cve_id_count"` -} - type clusterCountByPlatformType struct { GenericClusterCount int KubernetesClusterCount int diff --git a/central/views/platformcve/view_impl.go b/central/views/platformcve/view_impl.go index f323d04f55fcd..2fc1e57448c67 100644 --- a/central/views/platformcve/view_impl.go +++ b/central/views/platformcve/view_impl.go @@ -3,17 +3,15 @@ package platformcve import ( "context" - "github.com/pkg/errors" "github.com/stackrox/rox/central/views/common" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/postgres/walker" - "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/paginated" pgSearch "github.com/stackrox/rox/pkg/search/postgres" "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" - "github.com/stackrox/rox/pkg/utils" ) type platformCVECoreViewImpl struct { @@ -26,26 +24,7 @@ func (v *platformCVECoreViewImpl) Count(ctx context.Context, q *v1.Query) (int, return 0, err } - var err error - q, err = common.WithSACFilter(ctx, resources.Cluster, q) - if err != nil { - return 0, err - } - - var results []*platformCVECoreCount - results, err = pgSearch.RunSelectRequestForSchema[platformCVECoreCount](ctx, v.db, v.schema, common.WithCountQuery(q, search.CVEID)) - if err != nil { - return 0, err - } - if len(results) == 0 { - return 0, nil - } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", q.String()) - utils.Should(err) - return 0, err - } - return results[0].CVECount, nil + return pgSearch.RunDistinctCountForSchema(ctx, v.db, v.schema, q, search.CVEID) } func (v *platformCVECoreViewImpl) Get(ctx context.Context, q *v1.Query) ([]CveCore, error) { @@ -53,45 +32,32 @@ func (v *platformCVECoreViewImpl) Get(ctx context.Context, q *v1.Query) ([]CveCo return nil, err } - var err error - q, err = common.WithSACFilter(ctx, resources.Cluster, q) - if err != nil { - return nil, err - } - - var results []*platformCVECoreResponse - results, err = pgSearch.RunSelectRequestForSchema[platformCVECoreResponse](ctx, v.db, v.schema, withSelectQuery(q)) + ret := make([]CveCore, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[platformCVECoreResponse](ctx, v.db, v.schema, withSelectQuery(q), func(r *platformCVECoreResponse) error { + ret = append(ret, r) + return nil + }) if err != nil { return nil, err } - - ret := make([]CveCore, 0, len(results)) - for _, r := range results { - ret = append(ret, r) - } return ret, nil } func (v *platformCVECoreViewImpl) GetClusterIDs(ctx context.Context, q *v1.Query) ([]string, error) { - var err error - q, err = common.WithSACFilter(ctx, resources.Cluster, q) - if err != nil { - return nil, err - } - q.Selects = []*v1.QuerySelect{ search.NewQuerySelect(search.ClusterID).Distinct().Proto(), } - var results []*clusterResponse - results, err = pgSearch.RunSelectRequestForSchema[clusterResponse](ctx, v.db, v.schema, q) - if err != nil || len(results) == 0 { + ret := make([]string, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[clusterResponse](ctx, v.db, v.schema, q, func(r *clusterResponse) error { + ret = append(ret, r.ClusterID) + return nil + }) + if err != nil { return nil, err } - - ret := make([]string, 0, len(results)) - for _, r := range results { - ret = append(ret, r.ClusterID) + if len(ret) == 0 { + return nil, nil } return ret, nil } @@ -101,27 +67,15 @@ func (v *platformCVECoreViewImpl) CVECountByType(ctx context.Context, q *v1.Quer return nil, err } - var err error - q, err = common.WithSACFilter(ctx, resources.Cluster, q) + result, err := pgSearch.RunSelectOneForSchema[cveCountByTypeResponse](ctx, v.db, v.schema, withCVECountByTypeQuery(q)) if err != nil { return nil, err } - - var results []*cveCountByTypeResponse - results, err = pgSearch.RunSelectRequestForSchema[cveCountByTypeResponse](ctx, v.db, v.schema, withCVECountByTypeQuery(q)) - if err != nil { - return nil, err - } - if len(results) == 0 { - return NewEmptyCVECountByType(), nil - } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", q.String()) - utils.Should(err) + if result == nil { return NewEmptyCVECountByType(), nil } - return results[0], nil + return result, nil } func (v *platformCVECoreViewImpl) CVECountByFixability(ctx context.Context, q *v1.Query) (common.ResourceCountByFixability, error) { @@ -129,27 +83,15 @@ func (v *platformCVECoreViewImpl) CVECountByFixability(ctx context.Context, q *v return nil, err } - var err error - q, err = common.WithSACFilter(ctx, resources.Cluster, q) + result, err := pgSearch.RunSelectOneForSchema[cveCountByFixabilityResponse](ctx, v.db, v.schema, withCVECountByFixabilityQuery(q)) if err != nil { return nil, err } - - var results []*cveCountByFixabilityResponse - results, err = pgSearch.RunSelectRequestForSchema[cveCountByFixabilityResponse](ctx, v.db, v.schema, withCVECountByFixabilityQuery(q)) - if err != nil { - return nil, err - } - if len(results) == 0 { - return NewEmptyCVECountByFixability(), nil - } - if len(results) > 1 { - err = errors.Errorf("Retrieved multiple rows when only one row is expected for count query %q", q.String()) - utils.Should(err) + if result == nil { return NewEmptyCVECountByFixability(), nil } - return results[0], nil + return result, nil } func withSelectQuery(q *v1.Query) *v1.Query { diff --git a/central/views/platformcve/view_test.go b/central/views/platformcve/view_test.go index bd4ae72626293..2edf42d076896 100644 --- a/central/views/platformcve/view_test.go +++ b/central/views/platformcve/view_test.go @@ -793,10 +793,7 @@ func (s *PlatformCVEViewTestSuite) compileExpectedCVECoresWithPagination(filter if limit == 0 { return expected } - end := offset + limit - if end > len(expected) { - end = len(expected) - } + end := min(offset+limit, len(expected)) return expected[offset:end] } diff --git a/central/views/vmcve/db_response.go b/central/views/vmcve/db_response.go new file mode 100644 index 0000000000000..60570311ab830 --- /dev/null +++ b/central/views/vmcve/db_response.go @@ -0,0 +1,211 @@ +package vmcve + +import ( + "time" + + "github.com/stackrox/rox/central/views/common" +) + +type vmCVECoreResponse struct { + CVE string `db:"cve"` + VMsWithCriticalSeverity int `db:"critical_severity_count"` + FixableVMsWithCriticalSev int `db:"fixable_critical_severity_count"` + VMsWithImportantSeverity int `db:"important_severity_count"` + FixableVMsWithImportantSev int `db:"fixable_important_severity_count"` + VMsWithModerateSeverity int `db:"moderate_severity_count"` + FixableVMsWithModerateSev int `db:"fixable_moderate_severity_count"` + VMsWithLowSeverity int `db:"low_severity_count"` + FixableVMsWithLowSev int `db:"fixable_low_severity_count"` + VMsWithUnknownSeverity int `db:"unknown_severity_count"` + FixableVMsWithUnknownSev int `db:"fixable_unknown_severity_count"` + TopCVSS *float32 `db:"cvss_max"` + AffectedVMCount int `db:"virtual_machine_id_count"` + FirstDiscoveredInSystem *time.Time `db:"cve_created_time_min"` + Published *time.Time `db:"cve_published_on_min"` + EPSSProbabilityMax *float32 `db:"epss_probability_max"` +} + +func (c *vmCVECoreResponse) GetCVE() string { + return c.CVE +} + +func (c *vmCVECoreResponse) GetVMsBySeverity() common.ResourceCountByCVESeverity { + return &resourceCountByVMCVESeverity{ + CriticalSeverityCount: c.VMsWithCriticalSeverity, + FixableCriticalSeverityCount: c.FixableVMsWithCriticalSev, + ImportantSeverityCount: c.VMsWithImportantSeverity, + FixableImportantSeverityCount: c.FixableVMsWithImportantSev, + ModerateSeverityCount: c.VMsWithModerateSeverity, + FixableModerateSeverityCount: c.FixableVMsWithModerateSev, + LowSeverityCount: c.VMsWithLowSeverity, + FixableLowSeverityCount: c.FixableVMsWithLowSev, + UnknownSeverityCount: c.VMsWithUnknownSeverity, + FixableUnknownSeverityCount: c.FixableVMsWithUnknownSev, + } +} + +func (c *vmCVECoreResponse) GetTopCVSS() float32 { + if c.TopCVSS == nil { + return 0.0 + } + return *c.TopCVSS +} + +func (c *vmCVECoreResponse) GetAffectedVMCount() int { + return c.AffectedVMCount +} + +func (c *vmCVECoreResponse) GetFirstDiscoveredInSystem() *time.Time { + return c.FirstDiscoveredInSystem +} + +func (c *vmCVECoreResponse) GetPublishDate() *time.Time { + return c.Published +} + +func (c *vmCVECoreResponse) GetEPSSProbability() float32 { + if c.EPSSProbabilityMax == nil { + return 0.0 + } + return *c.EPSSProbabilityMax +} + +type vmIDResponse struct { + VMID string `db:"virtual_machine_id"` +} + +// resourceCountByVMCVESeverity contains the counts of VMs by CVE severity. +type resourceCountByVMCVESeverity struct { + CriticalSeverityCount int `db:"critical_severity_count"` + FixableCriticalSeverityCount int `db:"fixable_critical_severity_count"` + ImportantSeverityCount int `db:"important_severity_count"` + FixableImportantSeverityCount int `db:"fixable_important_severity_count"` + ModerateSeverityCount int `db:"moderate_severity_count"` + FixableModerateSeverityCount int `db:"fixable_moderate_severity_count"` + LowSeverityCount int `db:"low_severity_count"` + FixableLowSeverityCount int `db:"fixable_low_severity_count"` + UnknownSeverityCount int `db:"unknown_severity_count"` + FixableUnknownSeverityCount int `db:"fixable_unknown_severity_count"` +} + +func (r *resourceCountByVMCVESeverity) GetCriticalSeverityCount() common.ResourceCountByFixability { + return &resourceCountByFixability{total: r.CriticalSeverityCount, fixable: r.FixableCriticalSeverityCount} +} + +func (r *resourceCountByVMCVESeverity) GetImportantSeverityCount() common.ResourceCountByFixability { + return &resourceCountByFixability{total: r.ImportantSeverityCount, fixable: r.FixableImportantSeverityCount} +} + +func (r *resourceCountByVMCVESeverity) GetModerateSeverityCount() common.ResourceCountByFixability { + return &resourceCountByFixability{total: r.ModerateSeverityCount, fixable: r.FixableModerateSeverityCount} +} + +func (r *resourceCountByVMCVESeverity) GetLowSeverityCount() common.ResourceCountByFixability { + return &resourceCountByFixability{total: r.LowSeverityCount, fixable: r.FixableLowSeverityCount} +} + +func (r *resourceCountByVMCVESeverity) GetUnknownSeverityCount() common.ResourceCountByFixability { + return &resourceCountByFixability{total: r.UnknownSeverityCount, fixable: r.FixableUnknownSeverityCount} +} + +type resourceCountByFixability struct { + total int + fixable int +} + +func (r *resourceCountByFixability) GetTotal() int { + return r.total +} + +func (r *resourceCountByFixability) GetFixable() int { + return r.fixable +} + +type cveForVMResponse struct { + CVE string `db:"cve"` + MaxSeverity int32 `db:"severity_max"` + FixableCount int `db:"fixable_count"` + MaxCVSS float32 `db:"cvss_max"` + MaxNVDCVSS float32 `db:"nvd_cvss_max"` + EPSSProbabilityMax *float32 `db:"epss_probability_max"` + AffectedComponentCount int `db:"component_id_count"` + Published *time.Time `db:"cve_published_on_min"` +} + +func (r *cveForVMResponse) GetCVE() string { return r.CVE } +func (r *cveForVMResponse) GetMaxSeverity() int32 { return r.MaxSeverity } +func (r *cveForVMResponse) GetIsFixable() bool { return r.FixableCount > 0 } +func (r *cveForVMResponse) GetMaxCVSS() float32 { return r.MaxCVSS } +func (r *cveForVMResponse) GetMaxNVDCVSS() float32 { return r.MaxNVDCVSS } +func (r *cveForVMResponse) GetAffectedComponentCount() int { return r.AffectedComponentCount } +func (r *cveForVMResponse) GetPublishDate() *time.Time { return r.Published } +func (r *cveForVMResponse) GetEPSSProbability() float32 { + if r.EPSSProbabilityMax == nil { + return 0 + } + return *r.EPSSProbabilityMax +} + +type cveComponentResponse struct { + ComponentName string `db:"component"` + ComponentVersion string `db:"component_version"` + ComponentSource int32 `db:"component_source"` + FixedBy string `db:"fixed_by"` + AdvisoryName string `db:"advisory_name"` + AdvisoryLink string `db:"advisory_link"` +} + +func (c *cveComponentResponse) GetComponentName() string { return c.ComponentName } +func (c *cveComponentResponse) GetComponentVersion() string { return c.ComponentVersion } +func (c *cveComponentResponse) GetComponentSource() int32 { return c.ComponentSource } +func (c *cveComponentResponse) GetFixedBy() string { return c.FixedBy } +func (c *cveComponentResponse) GetAdvisoryName() string { return c.AdvisoryName } +func (c *cveComponentResponse) GetAdvisoryLink() string { return c.AdvisoryLink } + +type vmSeverityCountsResponse struct { + VMID string `db:"virtual_machine_id"` + CriticalSeverityCount int `db:"critical_severity_count"` + FixableCriticalSeverityCount int `db:"fixable_critical_severity_count"` + ImportantSeverityCount int `db:"important_severity_count"` + FixableImportantSeverityCount int `db:"fixable_important_severity_count"` + ModerateSeverityCount int `db:"moderate_severity_count"` + FixableModerateSeverityCount int `db:"fixable_moderate_severity_count"` + LowSeverityCount int `db:"low_severity_count"` + FixableLowSeverityCount int `db:"fixable_low_severity_count"` + UnknownSeverityCount int `db:"unknown_severity_count"` + FixableUnknownSeverityCount int `db:"fixable_unknown_severity_count"` +} + +func (r *vmSeverityCountsResponse) GetVMID() string { return r.VMID } +func (r *vmSeverityCountsResponse) GetSeverityCounts() common.ResourceCountByCVESeverity { + return &resourceCountByVMCVESeverity{ + CriticalSeverityCount: r.CriticalSeverityCount, + FixableCriticalSeverityCount: r.FixableCriticalSeverityCount, + ImportantSeverityCount: r.ImportantSeverityCount, + FixableImportantSeverityCount: r.FixableImportantSeverityCount, + ModerateSeverityCount: r.ModerateSeverityCount, + FixableModerateSeverityCount: r.FixableModerateSeverityCount, + LowSeverityCount: r.LowSeverityCount, + FixableLowSeverityCount: r.FixableLowSeverityCount, + UnknownSeverityCount: r.UnknownSeverityCount, + FixableUnknownSeverityCount: r.FixableUnknownSeverityCount, + } +} + +type affectedVMResponse struct { + VMID string `db:"virtual_machine_id"` + VMName string `db:"virtual_machine_name"` + MaxSeverity int32 `db:"severity_max"` + FixableCount int `db:"fixable_count"` + MaxCVSS float32 `db:"cvss_max"` + GuestOS string `db:"guest_os"` + AffectedComponentCount int `db:"component_id_count"` +} + +func (r *affectedVMResponse) GetVMID() string { return r.VMID } +func (r *affectedVMResponse) GetVMName() string { return r.VMName } +func (r *affectedVMResponse) GetMaxSeverity() int32 { return r.MaxSeverity } +func (r *affectedVMResponse) GetIsFixable() bool { return r.FixableCount > 0 } +func (r *affectedVMResponse) GetMaxCVSS() float32 { return r.MaxCVSS } +func (r *affectedVMResponse) GetGuestOS() string { return r.GuestOS } +func (r *affectedVMResponse) GetAffectedComponentCount() int { return r.AffectedComponentCount } diff --git a/central/views/vmcve/mocks/types.go b/central/views/vmcve/mocks/types.go new file mode 100644 index 0000000000000..f370a8f81b349 --- /dev/null +++ b/central/views/vmcve/mocks/types.go @@ -0,0 +1,721 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: types.go +// +// Generated by this command: +// +// mockgen -package mocks -destination mocks/types.go -source types.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + time "time" + + common "github.com/stackrox/rox/central/views/common" + vmcve "github.com/stackrox/rox/central/views/vmcve" + v1 "github.com/stackrox/rox/generated/api/v1" + search "github.com/stackrox/rox/pkg/search" + gomock "go.uber.org/mock/gomock" +) + +// MockCveCore is a mock of CveCore interface. +type MockCveCore struct { + ctrl *gomock.Controller + recorder *MockCveCoreMockRecorder + isgomock struct{} +} + +// MockCveCoreMockRecorder is the mock recorder for MockCveCore. +type MockCveCoreMockRecorder struct { + mock *MockCveCore +} + +// NewMockCveCore creates a new mock instance. +func NewMockCveCore(ctrl *gomock.Controller) *MockCveCore { + mock := &MockCveCore{ctrl: ctrl} + mock.recorder = &MockCveCoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCveCore) EXPECT() *MockCveCoreMockRecorder { + return m.recorder +} + +// GetAffectedVMCount mocks base method. +func (m *MockCveCore) GetAffectedVMCount() int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAffectedVMCount") + ret0, _ := ret[0].(int) + return ret0 +} + +// GetAffectedVMCount indicates an expected call of GetAffectedVMCount. +func (mr *MockCveCoreMockRecorder) GetAffectedVMCount() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAffectedVMCount", reflect.TypeOf((*MockCveCore)(nil).GetAffectedVMCount)) +} + +// GetCVE mocks base method. +func (m *MockCveCore) GetCVE() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCVE") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetCVE indicates an expected call of GetCVE. +func (mr *MockCveCoreMockRecorder) GetCVE() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCVE", reflect.TypeOf((*MockCveCore)(nil).GetCVE)) +} + +// GetEPSSProbability mocks base method. +func (m *MockCveCore) GetEPSSProbability() float32 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetEPSSProbability") + ret0, _ := ret[0].(float32) + return ret0 +} + +// GetEPSSProbability indicates an expected call of GetEPSSProbability. +func (mr *MockCveCoreMockRecorder) GetEPSSProbability() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEPSSProbability", reflect.TypeOf((*MockCveCore)(nil).GetEPSSProbability)) +} + +// GetFirstDiscoveredInSystem mocks base method. +func (m *MockCveCore) GetFirstDiscoveredInSystem() *time.Time { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFirstDiscoveredInSystem") + ret0, _ := ret[0].(*time.Time) + return ret0 +} + +// GetFirstDiscoveredInSystem indicates an expected call of GetFirstDiscoveredInSystem. +func (mr *MockCveCoreMockRecorder) GetFirstDiscoveredInSystem() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFirstDiscoveredInSystem", reflect.TypeOf((*MockCveCore)(nil).GetFirstDiscoveredInSystem)) +} + +// GetPublishDate mocks base method. +func (m *MockCveCore) GetPublishDate() *time.Time { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPublishDate") + ret0, _ := ret[0].(*time.Time) + return ret0 +} + +// GetPublishDate indicates an expected call of GetPublishDate. +func (mr *MockCveCoreMockRecorder) GetPublishDate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPublishDate", reflect.TypeOf((*MockCveCore)(nil).GetPublishDate)) +} + +// GetTopCVSS mocks base method. +func (m *MockCveCore) GetTopCVSS() float32 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTopCVSS") + ret0, _ := ret[0].(float32) + return ret0 +} + +// GetTopCVSS indicates an expected call of GetTopCVSS. +func (mr *MockCveCoreMockRecorder) GetTopCVSS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopCVSS", reflect.TypeOf((*MockCveCore)(nil).GetTopCVSS)) +} + +// GetVMsBySeverity mocks base method. +func (m *MockCveCore) GetVMsBySeverity() common.ResourceCountByCVESeverity { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVMsBySeverity") + ret0, _ := ret[0].(common.ResourceCountByCVESeverity) + return ret0 +} + +// GetVMsBySeverity indicates an expected call of GetVMsBySeverity. +func (mr *MockCveCoreMockRecorder) GetVMsBySeverity() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVMsBySeverity", reflect.TypeOf((*MockCveCore)(nil).GetVMsBySeverity)) +} + +// MockCVEComponentCore is a mock of CVEComponentCore interface. +type MockCVEComponentCore struct { + ctrl *gomock.Controller + recorder *MockCVEComponentCoreMockRecorder + isgomock struct{} +} + +// MockCVEComponentCoreMockRecorder is the mock recorder for MockCVEComponentCore. +type MockCVEComponentCoreMockRecorder struct { + mock *MockCVEComponentCore +} + +// NewMockCVEComponentCore creates a new mock instance. +func NewMockCVEComponentCore(ctrl *gomock.Controller) *MockCVEComponentCore { + mock := &MockCVEComponentCore{ctrl: ctrl} + mock.recorder = &MockCVEComponentCoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCVEComponentCore) EXPECT() *MockCVEComponentCoreMockRecorder { + return m.recorder +} + +// GetAdvisoryLink mocks base method. +func (m *MockCVEComponentCore) GetAdvisoryLink() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAdvisoryLink") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetAdvisoryLink indicates an expected call of GetAdvisoryLink. +func (mr *MockCVEComponentCoreMockRecorder) GetAdvisoryLink() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAdvisoryLink", reflect.TypeOf((*MockCVEComponentCore)(nil).GetAdvisoryLink)) +} + +// GetAdvisoryName mocks base method. +func (m *MockCVEComponentCore) GetAdvisoryName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAdvisoryName") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetAdvisoryName indicates an expected call of GetAdvisoryName. +func (mr *MockCVEComponentCoreMockRecorder) GetAdvisoryName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAdvisoryName", reflect.TypeOf((*MockCVEComponentCore)(nil).GetAdvisoryName)) +} + +// GetComponentName mocks base method. +func (m *MockCVEComponentCore) GetComponentName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetComponentName") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetComponentName indicates an expected call of GetComponentName. +func (mr *MockCVEComponentCoreMockRecorder) GetComponentName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetComponentName", reflect.TypeOf((*MockCVEComponentCore)(nil).GetComponentName)) +} + +// GetComponentSource mocks base method. +func (m *MockCVEComponentCore) GetComponentSource() int32 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetComponentSource") + ret0, _ := ret[0].(int32) + return ret0 +} + +// GetComponentSource indicates an expected call of GetComponentSource. +func (mr *MockCVEComponentCoreMockRecorder) GetComponentSource() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetComponentSource", reflect.TypeOf((*MockCVEComponentCore)(nil).GetComponentSource)) +} + +// GetComponentVersion mocks base method. +func (m *MockCVEComponentCore) GetComponentVersion() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetComponentVersion") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetComponentVersion indicates an expected call of GetComponentVersion. +func (mr *MockCVEComponentCoreMockRecorder) GetComponentVersion() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetComponentVersion", reflect.TypeOf((*MockCVEComponentCore)(nil).GetComponentVersion)) +} + +// GetFixedBy mocks base method. +func (m *MockCVEComponentCore) GetFixedBy() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFixedBy") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetFixedBy indicates an expected call of GetFixedBy. +func (mr *MockCVEComponentCoreMockRecorder) GetFixedBy() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFixedBy", reflect.TypeOf((*MockCVEComponentCore)(nil).GetFixedBy)) +} + +// MockCveView is a mock of CveView interface. +type MockCveView struct { + ctrl *gomock.Controller + recorder *MockCveViewMockRecorder + isgomock struct{} +} + +// MockCveViewMockRecorder is the mock recorder for MockCveView. +type MockCveViewMockRecorder struct { + mock *MockCveView +} + +// NewMockCveView creates a new mock instance. +func NewMockCveView(ctrl *gomock.Controller) *MockCveView { + mock := &MockCveView{ctrl: ctrl} + mock.recorder = &MockCveViewMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCveView) EXPECT() *MockCveViewMockRecorder { + return m.recorder +} + +// Count mocks base method. +func (m *MockCveView) Count(ctx context.Context, q *v1.Query) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Count", ctx, q) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Count indicates an expected call of Count. +func (mr *MockCveViewMockRecorder) Count(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockCveView)(nil).Count), ctx, q) +} + +// CountAffectedVMs mocks base method. +func (m *MockCveView) CountAffectedVMs(ctx context.Context, q *v1.Query) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountAffectedVMs", ctx, q) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountAffectedVMs indicates an expected call of CountAffectedVMs. +func (mr *MockCveViewMockRecorder) CountAffectedVMs(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountAffectedVMs", reflect.TypeOf((*MockCveView)(nil).CountAffectedVMs), ctx, q) +} + +// CountBySeverity mocks base method. +func (m *MockCveView) CountBySeverity(ctx context.Context, q *v1.Query, countOn search.FieldLabel) (common.ResourceCountByCVESeverity, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountBySeverity", ctx, q, countOn) + ret0, _ := ret[0].(common.ResourceCountByCVESeverity) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountBySeverity indicates an expected call of CountBySeverity. +func (mr *MockCveViewMockRecorder) CountBySeverity(ctx, q, countOn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountBySeverity", reflect.TypeOf((*MockCveView)(nil).CountBySeverity), ctx, q, countOn) +} + +// CountBySeverityPerVM mocks base method. +func (m *MockCveView) CountBySeverityPerVM(ctx context.Context, q *v1.Query) ([]vmcve.VMSeverityCounts, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountBySeverityPerVM", ctx, q) + ret0, _ := ret[0].([]vmcve.VMSeverityCounts) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountBySeverityPerVM indicates an expected call of CountBySeverityPerVM. +func (mr *MockCveViewMockRecorder) CountBySeverityPerVM(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountBySeverityPerVM", reflect.TypeOf((*MockCveView)(nil).CountBySeverityPerVM), ctx, q) +} + +// Get mocks base method. +func (m *MockCveView) Get(ctx context.Context, q *v1.Query) ([]vmcve.CveCore, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, q) + ret0, _ := ret[0].([]vmcve.CveCore) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockCveViewMockRecorder) Get(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockCveView)(nil).Get), ctx, q) +} + +// GetAffectedVMs mocks base method. +func (m *MockCveView) GetAffectedVMs(ctx context.Context, q *v1.Query) ([]vmcve.AffectedVMCore, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAffectedVMs", ctx, q) + ret0, _ := ret[0].([]vmcve.AffectedVMCore) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAffectedVMs indicates an expected call of GetAffectedVMs. +func (mr *MockCveViewMockRecorder) GetAffectedVMs(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAffectedVMs", reflect.TypeOf((*MockCveView)(nil).GetAffectedVMs), ctx, q) +} + +// GetCVEComponents mocks base method. +func (m *MockCveView) GetCVEComponents(ctx context.Context, q *v1.Query) ([]vmcve.CVEComponentCore, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCVEComponents", ctx, q) + ret0, _ := ret[0].([]vmcve.CVEComponentCore) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCVEComponents indicates an expected call of GetCVEComponents. +func (mr *MockCveViewMockRecorder) GetCVEComponents(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCVEComponents", reflect.TypeOf((*MockCveView)(nil).GetCVEComponents), ctx, q) +} + +// GetCVEsForVM mocks base method. +func (m *MockCveView) GetCVEsForVM(ctx context.Context, q *v1.Query) ([]vmcve.CVEForVMCore, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCVEsForVM", ctx, q) + ret0, _ := ret[0].([]vmcve.CVEForVMCore) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCVEsForVM indicates an expected call of GetCVEsForVM. +func (mr *MockCveViewMockRecorder) GetCVEsForVM(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCVEsForVM", reflect.TypeOf((*MockCveView)(nil).GetCVEsForVM), ctx, q) +} + +// GetVMIDs mocks base method. +func (m *MockCveView) GetVMIDs(ctx context.Context, q *v1.Query) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVMIDs", ctx, q) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVMIDs indicates an expected call of GetVMIDs. +func (mr *MockCveViewMockRecorder) GetVMIDs(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVMIDs", reflect.TypeOf((*MockCveView)(nil).GetVMIDs), ctx, q) +} + +// MockCVEForVMCore is a mock of CVEForVMCore interface. +type MockCVEForVMCore struct { + ctrl *gomock.Controller + recorder *MockCVEForVMCoreMockRecorder + isgomock struct{} +} + +// MockCVEForVMCoreMockRecorder is the mock recorder for MockCVEForVMCore. +type MockCVEForVMCoreMockRecorder struct { + mock *MockCVEForVMCore +} + +// NewMockCVEForVMCore creates a new mock instance. +func NewMockCVEForVMCore(ctrl *gomock.Controller) *MockCVEForVMCore { + mock := &MockCVEForVMCore{ctrl: ctrl} + mock.recorder = &MockCVEForVMCoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCVEForVMCore) EXPECT() *MockCVEForVMCoreMockRecorder { + return m.recorder +} + +// GetAffectedComponentCount mocks base method. +func (m *MockCVEForVMCore) GetAffectedComponentCount() int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAffectedComponentCount") + ret0, _ := ret[0].(int) + return ret0 +} + +// GetAffectedComponentCount indicates an expected call of GetAffectedComponentCount. +func (mr *MockCVEForVMCoreMockRecorder) GetAffectedComponentCount() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAffectedComponentCount", reflect.TypeOf((*MockCVEForVMCore)(nil).GetAffectedComponentCount)) +} + +// GetCVE mocks base method. +func (m *MockCVEForVMCore) GetCVE() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCVE") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetCVE indicates an expected call of GetCVE. +func (mr *MockCVEForVMCoreMockRecorder) GetCVE() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCVE", reflect.TypeOf((*MockCVEForVMCore)(nil).GetCVE)) +} + +// GetEPSSProbability mocks base method. +func (m *MockCVEForVMCore) GetEPSSProbability() float32 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetEPSSProbability") + ret0, _ := ret[0].(float32) + return ret0 +} + +// GetEPSSProbability indicates an expected call of GetEPSSProbability. +func (mr *MockCVEForVMCoreMockRecorder) GetEPSSProbability() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEPSSProbability", reflect.TypeOf((*MockCVEForVMCore)(nil).GetEPSSProbability)) +} + +// GetIsFixable mocks base method. +func (m *MockCVEForVMCore) GetIsFixable() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIsFixable") + ret0, _ := ret[0].(bool) + return ret0 +} + +// GetIsFixable indicates an expected call of GetIsFixable. +func (mr *MockCVEForVMCoreMockRecorder) GetIsFixable() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIsFixable", reflect.TypeOf((*MockCVEForVMCore)(nil).GetIsFixable)) +} + +// GetMaxCVSS mocks base method. +func (m *MockCVEForVMCore) GetMaxCVSS() float32 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMaxCVSS") + ret0, _ := ret[0].(float32) + return ret0 +} + +// GetMaxCVSS indicates an expected call of GetMaxCVSS. +func (mr *MockCVEForVMCoreMockRecorder) GetMaxCVSS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaxCVSS", reflect.TypeOf((*MockCVEForVMCore)(nil).GetMaxCVSS)) +} + +// GetMaxNVDCVSS mocks base method. +func (m *MockCVEForVMCore) GetMaxNVDCVSS() float32 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMaxNVDCVSS") + ret0, _ := ret[0].(float32) + return ret0 +} + +// GetMaxNVDCVSS indicates an expected call of GetMaxNVDCVSS. +func (mr *MockCVEForVMCoreMockRecorder) GetMaxNVDCVSS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaxNVDCVSS", reflect.TypeOf((*MockCVEForVMCore)(nil).GetMaxNVDCVSS)) +} + +// GetMaxSeverity mocks base method. +func (m *MockCVEForVMCore) GetMaxSeverity() int32 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMaxSeverity") + ret0, _ := ret[0].(int32) + return ret0 +} + +// GetMaxSeverity indicates an expected call of GetMaxSeverity. +func (mr *MockCVEForVMCoreMockRecorder) GetMaxSeverity() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaxSeverity", reflect.TypeOf((*MockCVEForVMCore)(nil).GetMaxSeverity)) +} + +// GetPublishDate mocks base method. +func (m *MockCVEForVMCore) GetPublishDate() *time.Time { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPublishDate") + ret0, _ := ret[0].(*time.Time) + return ret0 +} + +// GetPublishDate indicates an expected call of GetPublishDate. +func (mr *MockCVEForVMCoreMockRecorder) GetPublishDate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPublishDate", reflect.TypeOf((*MockCVEForVMCore)(nil).GetPublishDate)) +} + +// MockVMSeverityCounts is a mock of VMSeverityCounts interface. +type MockVMSeverityCounts struct { + ctrl *gomock.Controller + recorder *MockVMSeverityCountsMockRecorder + isgomock struct{} +} + +// MockVMSeverityCountsMockRecorder is the mock recorder for MockVMSeverityCounts. +type MockVMSeverityCountsMockRecorder struct { + mock *MockVMSeverityCounts +} + +// NewMockVMSeverityCounts creates a new mock instance. +func NewMockVMSeverityCounts(ctrl *gomock.Controller) *MockVMSeverityCounts { + mock := &MockVMSeverityCounts{ctrl: ctrl} + mock.recorder = &MockVMSeverityCountsMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVMSeverityCounts) EXPECT() *MockVMSeverityCountsMockRecorder { + return m.recorder +} + +// GetSeverityCounts mocks base method. +func (m *MockVMSeverityCounts) GetSeverityCounts() common.ResourceCountByCVESeverity { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSeverityCounts") + ret0, _ := ret[0].(common.ResourceCountByCVESeverity) + return ret0 +} + +// GetSeverityCounts indicates an expected call of GetSeverityCounts. +func (mr *MockVMSeverityCountsMockRecorder) GetSeverityCounts() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSeverityCounts", reflect.TypeOf((*MockVMSeverityCounts)(nil).GetSeverityCounts)) +} + +// GetVMID mocks base method. +func (m *MockVMSeverityCounts) GetVMID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVMID") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetVMID indicates an expected call of GetVMID. +func (mr *MockVMSeverityCountsMockRecorder) GetVMID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVMID", reflect.TypeOf((*MockVMSeverityCounts)(nil).GetVMID)) +} + +// MockAffectedVMCore is a mock of AffectedVMCore interface. +type MockAffectedVMCore struct { + ctrl *gomock.Controller + recorder *MockAffectedVMCoreMockRecorder + isgomock struct{} +} + +// MockAffectedVMCoreMockRecorder is the mock recorder for MockAffectedVMCore. +type MockAffectedVMCoreMockRecorder struct { + mock *MockAffectedVMCore +} + +// NewMockAffectedVMCore creates a new mock instance. +func NewMockAffectedVMCore(ctrl *gomock.Controller) *MockAffectedVMCore { + mock := &MockAffectedVMCore{ctrl: ctrl} + mock.recorder = &MockAffectedVMCoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAffectedVMCore) EXPECT() *MockAffectedVMCoreMockRecorder { + return m.recorder +} + +// GetAffectedComponentCount mocks base method. +func (m *MockAffectedVMCore) GetAffectedComponentCount() int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAffectedComponentCount") + ret0, _ := ret[0].(int) + return ret0 +} + +// GetAffectedComponentCount indicates an expected call of GetAffectedComponentCount. +func (mr *MockAffectedVMCoreMockRecorder) GetAffectedComponentCount() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAffectedComponentCount", reflect.TypeOf((*MockAffectedVMCore)(nil).GetAffectedComponentCount)) +} + +// GetGuestOS mocks base method. +func (m *MockAffectedVMCore) GetGuestOS() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetGuestOS") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetGuestOS indicates an expected call of GetGuestOS. +func (mr *MockAffectedVMCoreMockRecorder) GetGuestOS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGuestOS", reflect.TypeOf((*MockAffectedVMCore)(nil).GetGuestOS)) +} + +// GetIsFixable mocks base method. +func (m *MockAffectedVMCore) GetIsFixable() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIsFixable") + ret0, _ := ret[0].(bool) + return ret0 +} + +// GetIsFixable indicates an expected call of GetIsFixable. +func (mr *MockAffectedVMCoreMockRecorder) GetIsFixable() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIsFixable", reflect.TypeOf((*MockAffectedVMCore)(nil).GetIsFixable)) +} + +// GetMaxCVSS mocks base method. +func (m *MockAffectedVMCore) GetMaxCVSS() float32 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMaxCVSS") + ret0, _ := ret[0].(float32) + return ret0 +} + +// GetMaxCVSS indicates an expected call of GetMaxCVSS. +func (mr *MockAffectedVMCoreMockRecorder) GetMaxCVSS() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaxCVSS", reflect.TypeOf((*MockAffectedVMCore)(nil).GetMaxCVSS)) +} + +// GetMaxSeverity mocks base method. +func (m *MockAffectedVMCore) GetMaxSeverity() int32 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMaxSeverity") + ret0, _ := ret[0].(int32) + return ret0 +} + +// GetMaxSeverity indicates an expected call of GetMaxSeverity. +func (mr *MockAffectedVMCoreMockRecorder) GetMaxSeverity() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaxSeverity", reflect.TypeOf((*MockAffectedVMCore)(nil).GetMaxSeverity)) +} + +// GetVMID mocks base method. +func (m *MockAffectedVMCore) GetVMID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVMID") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetVMID indicates an expected call of GetVMID. +func (mr *MockAffectedVMCoreMockRecorder) GetVMID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVMID", reflect.TypeOf((*MockAffectedVMCore)(nil).GetVMID)) +} + +// GetVMName mocks base method. +func (m *MockAffectedVMCore) GetVMName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVMName") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetVMName indicates an expected call of GetVMName. +func (mr *MockAffectedVMCoreMockRecorder) GetVMName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVMName", reflect.TypeOf((*MockAffectedVMCore)(nil).GetVMName)) +} diff --git a/central/views/vmcve/singleton.go b/central/views/vmcve/singleton.go new file mode 100644 index 0000000000000..6b6bccb86a56b --- /dev/null +++ b/central/views/vmcve/singleton.go @@ -0,0 +1,35 @@ +package vmcve + +import ( + "github.com/stackrox/rox/central/globaldb" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/schema" + "github.com/stackrox/rox/pkg/sync" +) + +var ( + once sync.Once + + vmCVEView CveView +) + +// NewCVEView returns the interface CveView +// that provides searching VM CVEs stored in the database. +func NewCVEView(db postgres.DB) CveView { + return &vmCVECoreViewImpl{ + db: db, + schema: schema.VirtualMachineCvev2Schema, + } +} + +// Singleton provides the interface to search VM CVEs stored in the database. +func Singleton() CveView { + if !features.VirtualMachinesEnhancedDataModel.Enabled() { + return nil + } + once.Do(func() { + vmCVEView = NewCVEView(globaldb.GetPostgres()) + }) + return vmCVEView +} diff --git a/central/views/vmcve/types.go b/central/views/vmcve/types.go new file mode 100644 index 0000000000000..a789ffed3d4bb --- /dev/null +++ b/central/views/vmcve/types.go @@ -0,0 +1,81 @@ +package vmcve + +import ( + "context" + "time" + + "github.com/stackrox/rox/central/views/common" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/pkg/search" +) + +// CveCore is an interface to get VM CVE properties. +// +//go:generate mockgen-wrapper +type CveCore interface { + GetCVE() string + GetVMsBySeverity() common.ResourceCountByCVESeverity + GetTopCVSS() float32 + GetAffectedVMCount() int + GetFirstDiscoveredInSystem() *time.Time + GetPublishDate() *time.Time + GetEPSSProbability() float32 +} + +// CVEComponentCore provides component details for a specific CVE. +// +//go:generate mockgen-wrapper +type CVEComponentCore interface { + GetComponentName() string + GetComponentVersion() string + GetComponentSource() int32 + GetFixedBy() string + GetAdvisoryName() string + GetAdvisoryLink() string +} + +// CveView interface is like a SQL view that provides functionality to fetch VM CVE data +// irrespective of the data model. One CVE can have multiple database entries if that CVE +// impacts multiple VMs or components. However, the core information is the same. +// +//go:generate mockgen-wrapper +type CveView interface { + Count(ctx context.Context, q *v1.Query) (int, error) + CountBySeverity(ctx context.Context, q *v1.Query, countOn search.FieldLabel) (common.ResourceCountByCVESeverity, error) + Get(ctx context.Context, q *v1.Query) ([]CveCore, error) + GetVMIDs(ctx context.Context, q *v1.Query) ([]string, error) + GetCVEComponents(ctx context.Context, q *v1.Query) ([]CVEComponentCore, error) + CountBySeverityPerVM(ctx context.Context, q *v1.Query) ([]VMSeverityCounts, error) + GetAffectedVMs(ctx context.Context, q *v1.Query) ([]AffectedVMCore, error) + CountAffectedVMs(ctx context.Context, q *v1.Query) (int, error) + GetCVEsForVM(ctx context.Context, q *v1.Query) ([]CVEForVMCore, error) +} + +// CVEForVMCore provides per-CVE aggregation scoped to a single VM. +type CVEForVMCore interface { + GetCVE() string + GetMaxSeverity() int32 + GetIsFixable() bool + GetMaxCVSS() float32 + GetMaxNVDCVSS() float32 + GetEPSSProbability() float32 + GetAffectedComponentCount() int + GetPublishDate() *time.Time +} + +// VMSeverityCounts provides per-VM severity counts. +type VMSeverityCounts interface { + GetVMID() string + GetSeverityCounts() common.ResourceCountByCVESeverity +} + +// AffectedVMCore provides per-VM aggregation for a specific CVE. +type AffectedVMCore interface { + GetVMID() string + GetVMName() string + GetMaxSeverity() int32 + GetIsFixable() bool + GetMaxCVSS() float32 + GetGuestOS() string + GetAffectedComponentCount() int +} diff --git a/central/views/vmcve/view_impl.go b/central/views/vmcve/view_impl.go new file mode 100644 index 0000000000000..b159fcafcd330 --- /dev/null +++ b/central/views/vmcve/view_impl.go @@ -0,0 +1,256 @@ +package vmcve + +import ( + "context" + + "github.com/stackrox/rox/central/views/common" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/pkg/contextutil" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/walker" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/paginated" + pgSearch "github.com/stackrox/rox/pkg/search/postgres" + "github.com/stackrox/rox/pkg/search/postgres/aggregatefunc" +) + +var ( + queryTimeout = env.PostgresVMStatementTimeout.DurationSetting() +) + +type vmCVECoreViewImpl struct { + schema *walker.Schema + db postgres.DB +} + +func (v *vmCVECoreViewImpl) Count(ctx context.Context, q *v1.Query) (int, error) { + if err := common.ValidateQuery(q); err != nil { + return 0, err + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + return pgSearch.RunDistinctCountForSchema(queryCtx, v.db, v.schema, q, search.CVE) +} + +func (v *vmCVECoreViewImpl) CountBySeverity(ctx context.Context, q *v1.Query, countOn search.FieldLabel) (common.ResourceCountByCVESeverity, error) { + if err := common.ValidateQuery(q); err != nil { + return nil, err + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + result, err := pgSearch.RunSelectOneForSchema[resourceCountByVMCVESeverity](queryCtx, v.db, v.schema, common.WithCountBySeverityAndFixabilityQuery(q, countOn)) + if err != nil { + return nil, err + } + if result == nil { + return &resourceCountByVMCVESeverity{}, nil + } + return result, nil +} + +func (v *vmCVECoreViewImpl) Get(ctx context.Context, q *v1.Query) ([]CveCore, error) { + if err := common.ValidateQuery(q); err != nil { + return nil, err + } + + cloned := q.CloneVT() + cloned = common.UpdateSortAggs(cloned) + cloned.Selects = []*v1.QuerySelect{ + search.NewQuerySelect(search.CVE).Proto(), + } + cloned.Selects = append(cloned.Selects, + common.WithCountBySeverityAndFixabilityQuery(q, search.VirtualMachineID).GetSelects()..., + ) + cloned.Selects = append(cloned.Selects, + search.NewQuerySelect(search.CVSS).AggrFunc(aggregatefunc.Max).Proto(), + search.NewQuerySelect(search.VirtualMachineID).AggrFunc(aggregatefunc.Count).Distinct().Proto(), + search.NewQuerySelect(search.CVECreatedTime).AggrFunc(aggregatefunc.Min).Proto(), + search.NewQuerySelect(search.CVEPublishedOn).AggrFunc(aggregatefunc.Min).Proto(), + search.NewQuerySelect(search.EPSSProbablity).AggrFunc(aggregatefunc.Max).Proto(), + ) + cloned.GroupBy = &v1.QueryGroupBy{ + Fields: []string{search.CVE.String()}, + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + ret := make([]CveCore, 0, paginated.GetLimit(q.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[vmCVECoreResponse](queryCtx, v.db, v.schema, cloned, func(r *vmCVECoreResponse) error { + ret = append(ret, r) + return nil + }) + if err != nil { + return nil, err + } + return ret, nil +} + +func (v *vmCVECoreViewImpl) GetVMIDs(ctx context.Context, q *v1.Query) ([]string, error) { + cloned := q.CloneVT() + cloned.Selects = []*v1.QuerySelect{ + search.NewQuerySelect(search.VirtualMachineID).Distinct().Proto(), + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + ret := make([]string, 0, paginated.GetLimit(cloned.GetPagination().GetLimit(), 100)) + err := pgSearch.RunSelectRequestForSchemaFn[vmIDResponse](queryCtx, v.db, v.schema, cloned, func(r *vmIDResponse) error { + ret = append(ret, r.VMID) + return nil + }) + if err != nil { + return nil, err + } + if len(ret) == 0 { + return nil, nil + } + return ret, nil +} + +func (v *vmCVECoreViewImpl) CountBySeverityPerVM(ctx context.Context, q *v1.Query) ([]VMSeverityCounts, error) { + if err := common.ValidateQuery(q); err != nil { + return nil, err + } + + cloned := common.WithCountBySeverityAndFixabilityQuery(q, search.CVE) + cloned.Selects = append([]*v1.QuerySelect{ + search.NewQuerySelect(search.VirtualMachineID).Proto(), + }, cloned.GetSelects()...) + cloned.GroupBy = &v1.QueryGroupBy{ + Fields: []string{search.VirtualMachineID.String()}, + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + var ret []VMSeverityCounts + err := pgSearch.RunSelectRequestForSchemaFn[vmSeverityCountsResponse](queryCtx, v.db, v.schema, cloned, func(r *vmSeverityCountsResponse) error { + ret = append(ret, r) + return nil + }) + if err != nil { + return nil, err + } + return ret, nil +} + +func (v *vmCVECoreViewImpl) CountAffectedVMs(ctx context.Context, q *v1.Query) (int, error) { + if err := common.ValidateQuery(q); err != nil { + return 0, err + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + return pgSearch.RunDistinctCountForSchema(queryCtx, v.db, v.schema, q, search.VirtualMachineID) +} + +func (v *vmCVECoreViewImpl) GetAffectedVMs(ctx context.Context, q *v1.Query) ([]AffectedVMCore, error) { + if err := common.ValidateQuery(q); err != nil { + return nil, err + } + + cloned := q.CloneVT() + cloned.Selects = []*v1.QuerySelect{ + search.NewQuerySelect(search.VirtualMachineID).Proto(), + search.NewQuerySelect(search.VirtualMachineName).Proto(), + search.NewQuerySelect(search.Severity).AggrFunc(aggregatefunc.Max).Proto(), + search.NewQuerySelect(search.Fixable).AggrFunc(aggregatefunc.Count). + Filter("fixable_count", + search.NewQueryBuilder().AddBools(search.Fixable, true).ProtoQuery(), + ).Proto(), + search.NewQuerySelect(search.CVSS).AggrFunc(aggregatefunc.Max).Proto(), + search.NewQuerySelect(search.GuestOS).Proto(), + search.NewQuerySelect(search.ComponentID).AggrFunc(aggregatefunc.Count).Distinct().Proto(), + } + cloned.GroupBy = &v1.QueryGroupBy{ + Fields: []string{ + search.VirtualMachineID.String(), + search.VirtualMachineName.String(), + search.GuestOS.String(), + }, + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + var ret []AffectedVMCore + err := pgSearch.RunSelectRequestForSchemaFn[affectedVMResponse](queryCtx, v.db, v.schema, cloned, func(r *affectedVMResponse) error { + ret = append(ret, r) + return nil + }) + if err != nil { + return nil, err + } + return ret, nil +} + +func (v *vmCVECoreViewImpl) GetCVEsForVM(ctx context.Context, q *v1.Query) ([]CVEForVMCore, error) { + if err := common.ValidateQuery(q); err != nil { + return nil, err + } + + cloned := q.CloneVT() + cloned = common.UpdateSortAggs(cloned) + cloned.Selects = []*v1.QuerySelect{ + search.NewQuerySelect(search.CVE).Proto(), + search.NewQuerySelect(search.Severity).AggrFunc(aggregatefunc.Max).Proto(), + search.NewQuerySelect(search.Fixable).AggrFunc(aggregatefunc.Count). + Filter("fixable_count", + search.NewQueryBuilder().AddBools(search.Fixable, true).ProtoQuery(), + ).Proto(), + search.NewQuerySelect(search.CVSS).AggrFunc(aggregatefunc.Max).Proto(), + search.NewQuerySelect(search.NVDCVSS).AggrFunc(aggregatefunc.Max).Proto(), + search.NewQuerySelect(search.EPSSProbablity).AggrFunc(aggregatefunc.Max).Proto(), + search.NewQuerySelect(search.ComponentID).AggrFunc(aggregatefunc.Count).Distinct().Proto(), + search.NewQuerySelect(search.CVEPublishedOn).AggrFunc(aggregatefunc.Min).Proto(), + } + cloned.GroupBy = &v1.QueryGroupBy{ + Fields: []string{search.CVE.String()}, + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + var ret []CVEForVMCore + err := pgSearch.RunSelectRequestForSchemaFn[cveForVMResponse](queryCtx, v.db, v.schema, cloned, func(r *cveForVMResponse) error { + ret = append(ret, r) + return nil + }) + if err != nil { + return nil, err + } + return ret, nil +} + +func (v *vmCVECoreViewImpl) GetCVEComponents(ctx context.Context, q *v1.Query) ([]CVEComponentCore, error) { + cloned := q.CloneVT() + cloned.Selects = []*v1.QuerySelect{ + search.NewQuerySelect(search.Component).Proto(), + search.NewQuerySelect(search.ComponentVersion).Proto(), + search.NewQuerySelect(search.ComponentSource).Proto(), + search.NewQuerySelect(search.FixedBy).Proto(), + search.NewQuerySelect(search.AdvisoryName).Proto(), + search.NewQuerySelect(search.AdvisoryLink).Proto(), + } + + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, queryTimeout) + defer cancel() + + var ret []CVEComponentCore + err := pgSearch.RunSelectRequestForSchemaFn[cveComponentResponse](queryCtx, v.db, v.schema, cloned, func(r *cveComponentResponse) error { + ret = append(ret, r) + return nil + }) + if err != nil { + return nil, err + } + return ret, nil +} diff --git a/central/virtualmachine/component/v2/datastore/datastore.go b/central/virtualmachine/component/v2/datastore/datastore.go new file mode 100644 index 0000000000000..66d6d81d820d2 --- /dev/null +++ b/central/virtualmachine/component/v2/datastore/datastore.go @@ -0,0 +1,39 @@ +package datastore + +import ( + "context" + "testing" + + pgStore "github.com/stackrox/rox/central/virtualmachine/component/v2/datastore/store/postgres" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/postgres" + searchPkg "github.com/stackrox/rox/pkg/search" +) + +// DataStore is an intermediary to VirtualMachineComponentV2 storage. +// +//go:generate mockgen-wrapper +type DataStore interface { + Search(ctx context.Context, q *v1.Query) ([]searchPkg.Result, error) + SearchRawVMComponents(ctx context.Context, q *v1.Query) ([]*storage.VirtualMachineComponentV2, error) + + Exists(ctx context.Context, id string) (bool, error) + Get(ctx context.Context, id string) (*storage.VirtualMachineComponentV2, bool, error) + Count(ctx context.Context, q *v1.Query) (int, error) + GetBatch(ctx context.Context, ids []string) ([]*storage.VirtualMachineComponentV2, error) +} + +// New returns a new instance of a DataStore. +func New(storage pgStore.Store) DataStore { + ds := &datastoreImpl{ + storage: storage, + } + return ds +} + +// GetTestPostgresDataStore provides a datastore connected to postgres for testing purposes. +func GetTestPostgresDataStore(_ testing.TB, pool postgres.DB) DataStore { + dbstore := pgStore.New(pool) + return New(dbstore) +} diff --git a/central/virtualmachine/component/v2/datastore/datastore_impl.go b/central/virtualmachine/component/v2/datastore/datastore_impl.go new file mode 100644 index 0000000000000..68e7f9fa18eb5 --- /dev/null +++ b/central/virtualmachine/component/v2/datastore/datastore_impl.go @@ -0,0 +1,66 @@ +package datastore + +import ( + "context" + "time" + + "github.com/stackrox/rox/central/metrics" + pgStore "github.com/stackrox/rox/central/virtualmachine/component/v2/datastore/store/postgres" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + pkgSearch "github.com/stackrox/rox/pkg/search" +) + +type datastoreImpl struct { + storage pgStore.Store +} + +func (ds *datastoreImpl) Search(ctx context.Context, q *v1.Query) ([]pkgSearch.Result, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineComponentV2", "Search") + return ds.storage.Search(ctx, q) +} + +func (ds *datastoreImpl) Count(ctx context.Context, q *v1.Query) (int, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineComponentV2", "Count") + return ds.storage.Count(ctx, q) +} + +func (ds *datastoreImpl) SearchRawVMComponents(ctx context.Context, q *v1.Query) ([]*storage.VirtualMachineComponentV2, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineComponentV2", "SearchRawVMComponents") + var components []*storage.VirtualMachineComponentV2 + err := ds.storage.GetByQueryFn(ctx, q, func(component *storage.VirtualMachineComponentV2) error { + components = append(components, component) + return nil + }) + if err != nil { + return nil, err + } + return components, nil +} + +func (ds *datastoreImpl) Get(ctx context.Context, id string) (*storage.VirtualMachineComponentV2, bool, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineComponentV2", "Get") + component, found, err := ds.storage.Get(ctx, id) + if err != nil || !found { + return nil, false, err + } + return component, true, nil +} + +func (ds *datastoreImpl) Exists(ctx context.Context, id string) (bool, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineComponentV2", "Exists") + found, err := ds.storage.Exists(ctx, id) + if err != nil || !found { + return false, err + } + return true, nil +} + +func (ds *datastoreImpl) GetBatch(ctx context.Context, ids []string) ([]*storage.VirtualMachineComponentV2, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineComponentV2", "GetBatch") + components, _, err := ds.storage.GetMany(ctx, ids) + if err != nil { + return nil, err + } + return components, nil +} diff --git a/central/virtualmachine/component/v2/datastore/datastore_impl_test.go b/central/virtualmachine/component/v2/datastore/datastore_impl_test.go new file mode 100644 index 0000000000000..9ed1fb064c5a2 --- /dev/null +++ b/central/virtualmachine/component/v2/datastore/datastore_impl_test.go @@ -0,0 +1,344 @@ +//go:build sql_integration + +package datastore + +import ( + "context" + "fmt" + "slices" + "testing" + + componentStore "github.com/stackrox/rox/central/virtualmachine/component/v2/datastore/store/postgres" + scanStore "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore/store/postgres" + vmV2Store "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store" + vmV2Postgres "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/postgres" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/sac/testconsts" + "github.com/stackrox/rox/pkg/sac/testutils" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/suite" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestVMComponentV2DataStore(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + if !features.VirtualMachinesEnhancedDataModel.Enabled() { + t.Skip("VM enhanced data model is not enabled") + } + suite.Run(t, new(vmComponentV2DataStoreTestSuite)) +} + +type vmComponentV2DataStoreTestSuite struct { + suite.Suite + + db *pgtest.TestPostgres + datastore DataStore + vmStore vmV2Store.Store + scanPG scanStore.Store + + sacCtx context.Context + noSacCtx context.Context +} + +func (s *vmComponentV2DataStoreTestSuite) SetupTest() { + s.db = pgtest.ForT(s.T()) + store := componentStore.New(s.db) + s.datastore = New(store) + s.vmStore = vmV2Postgres.New(s.db, concurrency.NewKeyFence()) + s.scanPG = scanStore.New(s.db) + + s.sacCtx = sac.WithGlobalAccessScopeChecker(context.Background(), + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS, storage.Access_READ_WRITE_ACCESS), + sac.ResourceScopeKeys(resources.VirtualMachine))) + s.noSacCtx = sac.WithGlobalAccessScopeChecker(context.Background(), + sac.DenyAllAccessScopeChecker()) +} + +func (s *vmComponentV2DataStoreTestSuite) TearDownTest() { + s.db.Close() +} + +func (s *vmComponentV2DataStoreTestSuite) createTestVM(clusterID, namespace string, index int) *storage.VirtualMachineV2 { + vmID := uuid.NewV5FromNonUUIDs(clusterID, fmt.Sprintf("%s-%d", namespace, index)).String() + return &storage.VirtualMachineV2{ + Id: vmID, + Name: fmt.Sprintf("vm-%d", index), + Namespace: namespace, + ClusterId: clusterID, + ClusterName: fmt.Sprintf("cluster-%s", clusterID), + } +} + +func (s *vmComponentV2DataStoreTestSuite) createTestScan(vmID string) *storage.VirtualMachineScanV2 { + scanID := uuid.NewV5FromNonUUIDs(vmID, "scan").String() + return &storage.VirtualMachineScanV2{ + Id: scanID, + VmV2Id: vmID, + ScanOs: "rhel:9", + ScanTime: timestamppb.Now(), + TopCvss: 7.5, + } +} + +func (s *vmComponentV2DataStoreTestSuite) createTestComponent(scanID string, index int) *storage.VirtualMachineComponentV2 { + componentID := uuid.NewV5FromNonUUIDs(scanID, fmt.Sprintf("component-%d", index)).String() + return &storage.VirtualMachineComponentV2{ + Id: componentID, + VmScanId: scanID, + Name: fmt.Sprintf("pkg-%d", index), + Version: fmt.Sprintf("%d.0.0", index), + Source: storage.SourceType_OS, + OperatingSystem: "rhel:9", + } +} + +// insertVMScanAndComponent creates a VM, scan, and component, returning the component. +func (s *vmComponentV2DataStoreTestSuite) insertVMScanAndComponent(clusterID, namespace string, vmIndex, componentIndex int) *storage.VirtualMachineComponentV2 { + ctx := sac.WithAllAccess(context.Background()) + vm := s.createTestVM(clusterID, namespace, vmIndex) + s.Require().NoError(s.vmStore.UpsertVM(ctx, vm)) + scan := s.createTestScan(vm.GetId()) + s.Require().NoError(s.scanPG.Upsert(ctx, scan)) + component := s.createTestComponent(scan.GetId(), componentIndex) + compPG := componentStore.New(s.db) + s.Require().NoError(compPG.Upsert(ctx, component)) + return component +} + +// TestGet verifies basic get functionality. +func (s *vmComponentV2DataStoreTestSuite) TestGet() { + comp := s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + + retrieved, found, err := s.datastore.Get(s.sacCtx, comp.GetId()) + s.NoError(err) + s.True(found) + s.Equal(comp.GetId(), retrieved.GetId()) + s.Equal(comp.GetName(), retrieved.GetName()) + s.Equal(comp.GetVersion(), retrieved.GetVersion()) + + // Non-existent component + _, found, err = s.datastore.Get(s.sacCtx, uuid.NewDummy().String()) + s.NoError(err) + s.False(found) +} + +// TestExists verifies existence checks. +func (s *vmComponentV2DataStoreTestSuite) TestExists() { + comp := s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + + exists, err := s.datastore.Exists(s.sacCtx, comp.GetId()) + s.NoError(err) + s.True(exists) + + exists, err = s.datastore.Exists(s.sacCtx, uuid.NewDummy().String()) + s.NoError(err) + s.False(exists) +} + +// TestCount verifies count functionality. +func (s *vmComponentV2DataStoreTestSuite) TestCount() { + count, err := s.datastore.Count(s.sacCtx, nil) + s.NoError(err) + s.Equal(0, count) + + s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceA, 2, 1) + + count, err = s.datastore.Count(s.sacCtx, nil) + s.NoError(err) + s.Equal(2, count) +} + +// TestGetBatch verifies batch retrieval. +func (s *vmComponentV2DataStoreTestSuite) TestGetBatch() { + comp1 := s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + comp2 := s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceA, 2, 1) + + components, err := s.datastore.GetBatch(s.sacCtx, []string{comp1.GetId(), comp2.GetId()}) + s.NoError(err) + s.Len(components, 2) + + fetchedIDs := make([]string, len(components)) + for i, c := range components { + fetchedIDs[i] = c.GetId() + } + s.ElementsMatch([]string{comp1.GetId(), comp2.GetId()}, fetchedIDs) +} + +// TestSearch verifies search functionality. +func (s *vmComponentV2DataStoreTestSuite) TestSearch() { + comp1 := s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + comp2 := s.insertVMScanAndComponent(testconsts.Cluster2, testconsts.NamespaceB, 1, 1) + + results, err := s.datastore.Search(s.sacCtx, nil) + s.NoError(err) + s.Len(results, 2) + + fetchedIDs := make([]string, len(results)) + for i, r := range results { + fetchedIDs[i] = r.ID + } + s.ElementsMatch([]string{comp1.GetId(), comp2.GetId()}, fetchedIDs) +} + +// TestSearchRawVMComponents verifies raw search. +func (s *vmComponentV2DataStoreTestSuite) TestSearchRawVMComponents() { + s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + s.insertVMScanAndComponent(testconsts.Cluster2, testconsts.NamespaceB, 1, 1) + + components, err := s.datastore.SearchRawVMComponents(s.sacCtx, search.EmptyQuery()) + s.NoError(err) + s.Len(components, 2) + + for _, c := range components { + s.NotEmpty(c.GetId()) + s.NotEmpty(c.GetName()) + s.NotEmpty(c.GetVersion()) + } +} + +// TestSACDeniedAccess verifies that SAC-denied contexts cannot access components. +func (s *vmComponentV2DataStoreTestSuite) TestSACDeniedAccess() { + comp := s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + + _, found, err := s.datastore.Get(s.noSacCtx, comp.GetId()) + s.NoError(err) + s.False(found) + + exists, err := s.datastore.Exists(s.noSacCtx, comp.GetId()) + s.NoError(err) + s.False(exists) + + count, err := s.datastore.Count(s.noSacCtx, nil) + s.NoError(err) + s.Equal(0, count) + + results, err := s.datastore.Search(s.noSacCtx, nil) + s.NoError(err) + s.Empty(results) + + components, err := s.datastore.GetBatch(s.noSacCtx, []string{comp.GetId()}) + s.NoError(err) + s.Empty(components) + + rawComponents, err := s.datastore.SearchRawVMComponents(s.noSacCtx, search.EmptyQuery()) + s.NoError(err) + s.Empty(rawComponents) +} + +// TestSACNamespaceScopedAccess verifies namespace-scoped SAC filtering. +func (s *vmComponentV2DataStoreTestSuite) TestSACNamespaceScopedAccess() { + comp1 := s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + comp2 := s.insertVMScanAndComponent(testconsts.Cluster1, testconsts.NamespaceB, 1, 1) + comp3 := s.insertVMScanAndComponent(testconsts.Cluster2, testconsts.NamespaceA, 1, 1) + comp4 := s.insertVMScanAndComponent(testconsts.Cluster2, testconsts.NamespaceB, 1, 1) + + contexts := testutils.GetNamespaceScopedTestContexts(context.Background(), s.T(), resources.VirtualMachine) + + for name, tc := range map[string]struct { + scopeKey string + expectedIDs []string + }{ + "Unrestricted read access returns all components": { + scopeKey: testutils.UnrestrictedReadCtx, + expectedIDs: []string{comp1.GetId(), comp2.GetId(), comp3.GetId(), comp4.GetId()}, + }, + "Unrestricted read-write access returns all components": { + scopeKey: testutils.UnrestrictedReadWriteCtx, + expectedIDs: []string{comp1.GetId(), comp2.GetId(), comp3.GetId(), comp4.GetId()}, + }, + "Cluster1 read-write access returns only cluster1 components": { + scopeKey: testutils.Cluster1ReadWriteCtx, + expectedIDs: []string{comp1.GetId(), comp2.GetId()}, + }, + "Cluster2 read-write access returns only cluster2 components": { + scopeKey: testutils.Cluster2ReadWriteCtx, + expectedIDs: []string{comp3.GetId(), comp4.GetId()}, + }, + "Cluster3 read-write access returns no components": { + scopeKey: testutils.Cluster3ReadWriteCtx, + expectedIDs: nil, + }, + "Cluster1 NamespaceA access returns only cluster1 namespaceA components": { + scopeKey: testutils.Cluster1NamespaceAReadWriteCtx, + expectedIDs: []string{comp1.GetId()}, + }, + "Cluster1 NamespaceB access returns only cluster1 namespaceB components": { + scopeKey: testutils.Cluster1NamespaceBReadWriteCtx, + expectedIDs: []string{comp2.GetId()}, + }, + "Cluster1 NamespacesAB access returns cluster1 namespaces A and B components": { + scopeKey: testutils.Cluster1NamespacesABReadWriteCtx, + expectedIDs: []string{comp1.GetId(), comp2.GetId()}, + }, + "Cluster2 NamespaceB access returns only cluster2 namespaceB components": { + scopeKey: testutils.Cluster2NamespaceBReadWriteCtx, + expectedIDs: []string{comp4.GetId()}, + }, + "Mixed cluster and namespace access returns components from allowed scopes": { + scopeKey: testutils.MixedClusterAndNamespaceReadCtx, + expectedIDs: []string{comp1.GetId(), comp3.GetId(), comp4.GetId()}, + }, + } { + s.Run(name, func() { + ctx := contexts[tc.scopeKey] + + // Test Count + count, err := s.datastore.Count(ctx, nil) + s.NoError(err) + s.Equal(len(tc.expectedIDs), count) + + // Test Search + results, err := s.datastore.Search(ctx, nil) + s.NoError(err) + fetchedIDs := make([]string, len(results)) + for i, r := range results { + fetchedIDs[i] = r.ID + } + s.ElementsMatch(tc.expectedIDs, fetchedIDs) + + // Test Exists for a component in cluster1 namespaceA + exists, err := s.datastore.Exists(ctx, comp1.GetId()) + s.NoError(err) + expectedExists := slices.Contains(tc.expectedIDs, comp1.GetId()) + s.Equal(expectedExists, exists) + + // Test Get for a component in cluster1 namespaceA + _, found, err := s.datastore.Get(ctx, comp1.GetId()) + s.NoError(err) + s.Equal(expectedExists, found) + + // Test GetBatch + allIDs := []string{comp1.GetId(), comp2.GetId(), comp3.GetId(), comp4.GetId()} + components, err := s.datastore.GetBatch(ctx, allIDs) + s.NoError(err) + batchIDs := make([]string, len(components)) + for i, c := range components { + batchIDs[i] = c.GetId() + } + s.ElementsMatch(tc.expectedIDs, batchIDs) + + // Test SearchRawVMComponents + rawComponents, err := s.datastore.SearchRawVMComponents(ctx, search.EmptyQuery()) + s.NoError(err) + rawIDs := make([]string, len(rawComponents)) + for i, c := range rawComponents { + rawIDs[i] = c.GetId() + } + s.ElementsMatch(tc.expectedIDs, rawIDs) + }) + } +} + +// TestSingletonGatedByFeatureFlag verifies the singleton is nil when the feature flag is disabled. +func (s *vmComponentV2DataStoreTestSuite) TestSingletonGatedByFeatureFlag() { + s.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + s.Nil(Singleton()) +} diff --git a/central/virtualmachine/component/v2/datastore/mocks/datastore.go b/central/virtualmachine/component/v2/datastore/mocks/datastore.go new file mode 100644 index 0000000000000..bf5a7092690cc --- /dev/null +++ b/central/virtualmachine/component/v2/datastore/mocks/datastore.go @@ -0,0 +1,135 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: datastore.go +// +// Generated by this command: +// +// mockgen -package mocks -destination mocks/datastore.go -source datastore.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + v1 "github.com/stackrox/rox/generated/api/v1" + storage "github.com/stackrox/rox/generated/storage" + search "github.com/stackrox/rox/pkg/search" + gomock "go.uber.org/mock/gomock" +) + +// MockDataStore is a mock of DataStore interface. +type MockDataStore struct { + ctrl *gomock.Controller + recorder *MockDataStoreMockRecorder + isgomock struct{} +} + +// MockDataStoreMockRecorder is the mock recorder for MockDataStore. +type MockDataStoreMockRecorder struct { + mock *MockDataStore +} + +// NewMockDataStore creates a new mock instance. +func NewMockDataStore(ctrl *gomock.Controller) *MockDataStore { + mock := &MockDataStore{ctrl: ctrl} + mock.recorder = &MockDataStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataStore) EXPECT() *MockDataStoreMockRecorder { + return m.recorder +} + +// Count mocks base method. +func (m *MockDataStore) Count(ctx context.Context, q *v1.Query) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Count", ctx, q) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Count indicates an expected call of Count. +func (mr *MockDataStoreMockRecorder) Count(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockDataStore)(nil).Count), ctx, q) +} + +// Exists mocks base method. +func (m *MockDataStore) Exists(ctx context.Context, id string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, id) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockDataStoreMockRecorder) Exists(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockDataStore)(nil).Exists), ctx, id) +} + +// Get mocks base method. +func (m *MockDataStore) Get(ctx context.Context, id string) (*storage.VirtualMachineComponentV2, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, id) + ret0, _ := ret[0].(*storage.VirtualMachineComponentV2) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Get indicates an expected call of Get. +func (mr *MockDataStoreMockRecorder) Get(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDataStore)(nil).Get), ctx, id) +} + +// GetBatch mocks base method. +func (m *MockDataStore) GetBatch(ctx context.Context, ids []string) ([]*storage.VirtualMachineComponentV2, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBatch", ctx, ids) + ret0, _ := ret[0].([]*storage.VirtualMachineComponentV2) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBatch indicates an expected call of GetBatch. +func (mr *MockDataStoreMockRecorder) GetBatch(ctx, ids any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBatch", reflect.TypeOf((*MockDataStore)(nil).GetBatch), ctx, ids) +} + +// Search mocks base method. +func (m *MockDataStore) Search(ctx context.Context, q *v1.Query) ([]search.Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Search", ctx, q) + ret0, _ := ret[0].([]search.Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Search indicates an expected call of Search. +func (mr *MockDataStoreMockRecorder) Search(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockDataStore)(nil).Search), ctx, q) +} + +// SearchRawVMComponents mocks base method. +func (m *MockDataStore) SearchRawVMComponents(ctx context.Context, q *v1.Query) ([]*storage.VirtualMachineComponentV2, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchRawVMComponents", ctx, q) + ret0, _ := ret[0].([]*storage.VirtualMachineComponentV2) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchRawVMComponents indicates an expected call of SearchRawVMComponents. +func (mr *MockDataStoreMockRecorder) SearchRawVMComponents(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchRawVMComponents", reflect.TypeOf((*MockDataStore)(nil).SearchRawVMComponents), ctx, q) +} diff --git a/central/virtualmachine/component/v2/datastore/singleton.go b/central/virtualmachine/component/v2/datastore/singleton.go new file mode 100644 index 0000000000000..62f2beacd43e2 --- /dev/null +++ b/central/virtualmachine/component/v2/datastore/singleton.go @@ -0,0 +1,28 @@ +package datastore + +import ( + "github.com/stackrox/rox/central/globaldb" + pgStore "github.com/stackrox/rox/central/virtualmachine/component/v2/datastore/store/postgres" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/sync" +) + +var ( + once sync.Once + + ds DataStore +) + +func initialize() { + storage := pgStore.New(globaldb.GetPostgres()) + ds = New(storage) +} + +// Singleton returns a singleton instance of the VM component v2 datastore. +func Singleton() DataStore { + if !features.VirtualMachinesEnhancedDataModel.Enabled() { + return nil + } + once.Do(initialize) + return ds +} diff --git a/central/virtualmachine/component/v2/datastore/store/postgres/gen.go b/central/virtualmachine/component/v2/datastore/store/postgres/gen.go new file mode 100644 index 0000000000000..b1c368e1b3d52 --- /dev/null +++ b/central/virtualmachine/component/v2/datastore/store/postgres/gen.go @@ -0,0 +1,3 @@ +package postgres + +//go:generate pg-table-bindings-wrapper --type=storage.VirtualMachineComponentV2 --search-category VIRTUAL_MACHINE_COMPONENTS_V2 --search-scope VIRTUAL_MACHINE_VULNERABILITIES_V2,VIRTUAL_MACHINE_COMPONENTS_V2,VIRTUAL_MACHINE_SCANS_V2,VIRTUAL_MACHINES_V2,NAMESPACES,CLUSTERS --references=storage.VirtualMachineScanV2 diff --git a/central/virtualmachine/component/v2/datastore/store/postgres/store.go b/central/virtualmachine/component/v2/datastore/store/postgres/store.go new file mode 100644 index 0000000000000..22b3d9c47b718 --- /dev/null +++ b/central/virtualmachine/component/v2/datastore/store/postgres/store.go @@ -0,0 +1,179 @@ +// Code generated by pg-bindings generator. DO NOT EDIT. + +package postgres + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5" + "github.com/stackrox/rox/central/metrics" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/logging" + ops "github.com/stackrox/rox/pkg/metrics" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/pgutils" + pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" + pgSearch "github.com/stackrox/rox/pkg/search/postgres" +) + +const ( + baseTable = "virtual_machine_component_v2" + storeName = "VirtualMachineComponentV2" +) + +var ( + log = logging.LoggerForModule() + schema = pkgSchema.VirtualMachineComponentV2Schema + targetResource = resources.VirtualMachine +) + +type ( + storeType = storage.VirtualMachineComponentV2 + callback = func(obj *storeType) error +) + +// Store is the interface to interact with the storage for storage.VirtualMachineComponentV2 +type Store interface { + Upsert(ctx context.Context, obj *storeType) error + UpsertMany(ctx context.Context, objs []*storeType) error + Delete(ctx context.Context, id string) error + DeleteByQuery(ctx context.Context, q *v1.Query) error + DeleteByQueryWithIDs(ctx context.Context, q *v1.Query) ([]string, error) + DeleteMany(ctx context.Context, identifiers []string) error + PruneMany(ctx context.Context, identifiers []string) error + + Count(ctx context.Context, q *v1.Query) (int, error) + Exists(ctx context.Context, id string) (bool, error) + Search(ctx context.Context, q *v1.Query) ([]search.Result, error) + + Get(ctx context.Context, id string) (*storeType, bool, error) + // Deprecated: use GetByQueryFn instead + GetByQuery(ctx context.Context, query *v1.Query) ([]*storeType, error) + GetByQueryFn(ctx context.Context, query *v1.Query, fn callback) error + GetMany(ctx context.Context, identifiers []string) ([]*storeType, []int, error) + GetIDs(ctx context.Context) ([]string, error) + + Walk(ctx context.Context, fn callback) error + WalkByQuery(ctx context.Context, query *v1.Query, fn callback) error +} + +// New returns a new Store instance using the provided sql instance. +func New(db postgres.DB) Store { + return pgSearch.NewGloballyScopedGenericStore[storeType, *storeType]( + db, + schema, + pkGetter, + insertIntoVirtualMachineComponentV2, + copyFromVirtualMachineComponentV2, + metricsSetAcquireDBConnDuration, + metricsSetPostgresOperationDurationTime, + targetResource, + nil, + nil, + ) +} + +// region Helper functions + +func pkGetter(obj *storeType) string { + return obj.GetId() +} + +func metricsSetPostgresOperationDurationTime(start time.Time, op ops.Op) { + metrics.SetPostgresOperationDurationTime(start, op, storeName) +} + +func metricsSetAcquireDBConnDuration(start time.Time, op ops.Op) { + metrics.SetAcquireDBConnDuration(start, op, storeName) +} + +func insertIntoVirtualMachineComponentV2(batch *pgx.Batch, obj *storage.VirtualMachineComponentV2) error { + + serialized, marshalErr := obj.MarshalVT() + if marshalErr != nil { + return marshalErr + } + + values := []interface{}{ + // parent primary keys start + pgutils.NilOrUUID(obj.GetId()), + pgutils.NilOrUUID(obj.GetVmScanId()), + obj.GetName(), + obj.GetVersion(), + obj.GetSource(), + obj.GetOperatingSystem(), + obj.GetTopCvss(), + serialized, + } + + finalStr := "INSERT INTO virtual_machine_component_v2 (Id, VmScanId, Name, Version, Source, OperatingSystem, TopCvss, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, VmScanId = EXCLUDED.VmScanId, Name = EXCLUDED.Name, Version = EXCLUDED.Version, Source = EXCLUDED.Source, OperatingSystem = EXCLUDED.OperatingSystem, TopCvss = EXCLUDED.TopCvss, serialized = EXCLUDED.serialized" + batch.Queue(finalStr, values...) + + return nil +} + +var copyColsVirtualMachineComponentV2 = []string{ + "id", + "vmscanid", + "name", + "version", + "source", + "operatingsystem", + "topcvss", + "serialized", +} + +func copyFromVirtualMachineComponentV2(ctx context.Context, s pgSearch.Deleter, tx *postgres.Tx, objs ...*storage.VirtualMachineComponentV2) error { + if len(objs) == 0 { + return nil + } + + { + // CopyFrom does not upsert, so delete existing rows first to achieve upsert behavior. + // Parent deletion cascades to children, so only the top-level parent needs deletion. + deletes := make([]string, 0, len(objs)) + for _, obj := range objs { + deletes = append(deletes, obj.GetId()) + } + if err := s.DeleteMany(ctx, deletes); err != nil { + return err + } + } + + idx := 0 + inputRows := pgx.CopyFromFunc(func() ([]any, error) { + if idx >= len(objs) { + return nil, nil + } + obj := objs[idx] + idx++ + + serialized, marshalErr := obj.MarshalVT() + if marshalErr != nil { + return nil, marshalErr + } + + return []interface{}{ + pgutils.NilOrUUID(obj.GetId()), + pgutils.NilOrUUID(obj.GetVmScanId()), + obj.GetName(), + obj.GetVersion(), + obj.GetSource(), + obj.GetOperatingSystem(), + obj.GetTopCvss(), + serialized, + }, nil + }) + + if _, err := tx.CopyFrom(ctx, pgx.Identifier{"virtual_machine_component_v2"}, copyColsVirtualMachineComponentV2, inputRows); err != nil { + return err + } + + return nil +} + +// endregion Helper functions diff --git a/central/virtualmachine/component/v2/datastore/store/postgres/store_test.go b/central/virtualmachine/component/v2/datastore/store/postgres/store_test.go new file mode 100644 index 0000000000000..fac1fb8d1afb0 --- /dev/null +++ b/central/virtualmachine/component/v2/datastore/store/postgres/store_test.go @@ -0,0 +1,55 @@ +// Code generated by pg-bindings generator. DO NOT EDIT. + +//go:build sql_integration + +package postgres + +import ( + "context" + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/testutils" + "github.com/stretchr/testify/suite" +) + +type VirtualMachineComponentV2StoreSuite struct { + suite.Suite + store Store + testDB *pgtest.TestPostgres +} + +func TestVirtualMachineComponentV2Store(t *testing.T) { + suite.Run(t, new(VirtualMachineComponentV2StoreSuite)) +} + +func (s *VirtualMachineComponentV2StoreSuite) SetupSuite() { + + s.testDB = pgtest.ForT(s.T()) + s.store = New(s.testDB.DB) +} + +func (s *VirtualMachineComponentV2StoreSuite) SetupTest() { + ctx := sac.WithAllAccess(context.Background()) + tag, err := s.testDB.Exec(ctx, "TRUNCATE virtual_machine_component_v2 CASCADE") + s.T().Log("virtual_machine_component_v2", tag) + s.store = New(s.testDB.DB) + s.NoError(err) +} + +func (s *VirtualMachineComponentV2StoreSuite) TestStore() { + ctx := sac.WithAllAccess(context.Background()) + + store := s.store + + virtualMachineComponentV2 := &storage.VirtualMachineComponentV2{} + s.NoError(testutils.FullInit(virtualMachineComponentV2, testutils.SimpleInitializer(), testutils.JSONFieldsFilter)) + + foundVirtualMachineComponentV2, exists, err := store.Get(ctx, virtualMachineComponentV2.GetId()) + s.NoError(err) + s.False(exists) + s.Nil(foundVirtualMachineComponentV2) + +} diff --git a/central/virtualmachine/cve/v2/datastore/datastore.go b/central/virtualmachine/cve/v2/datastore/datastore.go new file mode 100644 index 0000000000000..e383e42bf40ee --- /dev/null +++ b/central/virtualmachine/cve/v2/datastore/datastore.go @@ -0,0 +1,39 @@ +package datastore + +import ( + "context" + "testing" + + pgStore "github.com/stackrox/rox/central/virtualmachine/cve/v2/datastore/store/postgres" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/postgres" + searchPkg "github.com/stackrox/rox/pkg/search" +) + +// DataStore is an intermediary to VirtualMachineCVEV2 storage. +// +//go:generate mockgen-wrapper +type DataStore interface { + Search(ctx context.Context, q *v1.Query) ([]searchPkg.Result, error) + SearchRawVMCVEs(ctx context.Context, q *v1.Query) ([]*storage.VirtualMachineCVEV2, error) + + Exists(ctx context.Context, id string) (bool, error) + Get(ctx context.Context, id string) (*storage.VirtualMachineCVEV2, bool, error) + Count(ctx context.Context, q *v1.Query) (int, error) + GetBatch(ctx context.Context, ids []string) ([]*storage.VirtualMachineCVEV2, error) +} + +// New returns a new instance of a DataStore. +func New(storage pgStore.Store) DataStore { + ds := &datastoreImpl{ + storage: storage, + } + return ds +} + +// GetTestPostgresDataStore provides a datastore connected to postgres for testing purposes. +func GetTestPostgresDataStore(_ testing.TB, pool postgres.DB) DataStore { + dbstore := pgStore.New(pool) + return New(dbstore) +} diff --git a/central/virtualmachine/cve/v2/datastore/datastore_impl.go b/central/virtualmachine/cve/v2/datastore/datastore_impl.go new file mode 100644 index 0000000000000..8bb72bc9cf5e5 --- /dev/null +++ b/central/virtualmachine/cve/v2/datastore/datastore_impl.go @@ -0,0 +1,66 @@ +package datastore + +import ( + "context" + "time" + + "github.com/stackrox/rox/central/metrics" + pgStore "github.com/stackrox/rox/central/virtualmachine/cve/v2/datastore/store/postgres" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + pkgSearch "github.com/stackrox/rox/pkg/search" +) + +type datastoreImpl struct { + storage pgStore.Store +} + +func (ds *datastoreImpl) Search(ctx context.Context, q *v1.Query) ([]pkgSearch.Result, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineCVEV2", "Search") + return ds.storage.Search(ctx, q) +} + +func (ds *datastoreImpl) Count(ctx context.Context, q *v1.Query) (int, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineCVEV2", "Count") + return ds.storage.Count(ctx, q) +} + +func (ds *datastoreImpl) SearchRawVMCVEs(ctx context.Context, q *v1.Query) ([]*storage.VirtualMachineCVEV2, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineCVEV2", "SearchRawVMCVEs") + var cves []*storage.VirtualMachineCVEV2 + err := ds.storage.GetByQueryFn(ctx, q, func(cve *storage.VirtualMachineCVEV2) error { + cves = append(cves, cve) + return nil + }) + if err != nil { + return nil, err + } + return cves, nil +} + +func (ds *datastoreImpl) Get(ctx context.Context, id string) (*storage.VirtualMachineCVEV2, bool, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineCVEV2", "Get") + cve, found, err := ds.storage.Get(ctx, id) + if err != nil || !found { + return nil, false, err + } + return cve, true, nil +} + +func (ds *datastoreImpl) Exists(ctx context.Context, id string) (bool, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineCVEV2", "Exists") + found, err := ds.storage.Exists(ctx, id) + if err != nil || !found { + return false, err + } + return true, nil +} + +func (ds *datastoreImpl) GetBatch(ctx context.Context, ids []string) ([]*storage.VirtualMachineCVEV2, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineCVEV2", "GetBatch") + cves, _, err := ds.storage.GetMany(ctx, ids) + if err != nil { + return nil, err + } + return cves, nil +} diff --git a/central/virtualmachine/cve/v2/datastore/datastore_impl_test.go b/central/virtualmachine/cve/v2/datastore/datastore_impl_test.go new file mode 100644 index 0000000000000..d776e13234281 --- /dev/null +++ b/central/virtualmachine/cve/v2/datastore/datastore_impl_test.go @@ -0,0 +1,368 @@ +//go:build sql_integration + +package datastore + +import ( + "context" + "fmt" + "slices" + "testing" + + componentStore "github.com/stackrox/rox/central/virtualmachine/component/v2/datastore/store/postgres" + cveStore "github.com/stackrox/rox/central/virtualmachine/cve/v2/datastore/store/postgres" + scanStore "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore/store/postgres" + vmV2Store "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store" + vmV2Postgres "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/postgres" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/sac/testconsts" + "github.com/stackrox/rox/pkg/sac/testutils" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/suite" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestVMCVEV2DataStore(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + if !features.VirtualMachinesEnhancedDataModel.Enabled() { + t.Skip("VM enhanced data model is not enabled") + } + suite.Run(t, new(vmCVEV2DataStoreTestSuite)) +} + +type vmCVEV2DataStoreTestSuite struct { + suite.Suite + + db *pgtest.TestPostgres + datastore DataStore + vmStore vmV2Store.Store + scanPG scanStore.Store + componentPG componentStore.Store + cvePG cveStore.Store + + sacCtx context.Context + noSacCtx context.Context +} + +func (s *vmCVEV2DataStoreTestSuite) SetupTest() { + s.db = pgtest.ForT(s.T()) + store := cveStore.New(s.db) + s.datastore = New(store) + s.vmStore = vmV2Postgres.New(s.db, concurrency.NewKeyFence()) + s.scanPG = scanStore.New(s.db) + s.componentPG = componentStore.New(s.db) + s.cvePG = cveStore.New(s.db) + + s.sacCtx = sac.WithGlobalAccessScopeChecker(context.Background(), + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS, storage.Access_READ_WRITE_ACCESS), + sac.ResourceScopeKeys(resources.VirtualMachine))) + s.noSacCtx = sac.WithGlobalAccessScopeChecker(context.Background(), + sac.DenyAllAccessScopeChecker()) +} + +func (s *vmCVEV2DataStoreTestSuite) TearDownTest() { + s.db.Close() +} + +func (s *vmCVEV2DataStoreTestSuite) createTestVM(clusterID, namespace string, index int) *storage.VirtualMachineV2 { + vmID := uuid.NewV5FromNonUUIDs(clusterID, fmt.Sprintf("%s-%d", namespace, index)).String() + return &storage.VirtualMachineV2{ + Id: vmID, + Name: fmt.Sprintf("vm-%d", index), + Namespace: namespace, + ClusterId: clusterID, + ClusterName: fmt.Sprintf("cluster-%s", clusterID), + } +} + +func (s *vmCVEV2DataStoreTestSuite) createTestScan(vmID string) *storage.VirtualMachineScanV2 { + scanID := uuid.NewV5FromNonUUIDs(vmID, "scan").String() + return &storage.VirtualMachineScanV2{ + Id: scanID, + VmV2Id: vmID, + ScanOs: "rhel:9", + ScanTime: timestamppb.Now(), + TopCvss: 9.8, + } +} + +func (s *vmCVEV2DataStoreTestSuite) createTestComponent(scanID string) *storage.VirtualMachineComponentV2 { + componentID := uuid.NewV5FromNonUUIDs(scanID, "component").String() + return &storage.VirtualMachineComponentV2{ + Id: componentID, + VmScanId: scanID, + Name: "openssl", + Version: "1.1.1", + Source: storage.SourceType_OS, + OperatingSystem: "rhel:9", + } +} + +func (s *vmCVEV2DataStoreTestSuite) createTestCVE(vmID, componentID string, index int) *storage.VirtualMachineCVEV2 { + cveID := uuid.NewV5FromNonUUIDs(componentID, fmt.Sprintf("cve-%d", index)).String() + return &storage.VirtualMachineCVEV2{ + Id: cveID, + VmV2Id: vmID, + VmComponentId: componentID, + CveBaseInfo: &storage.CVEInfo{ + Cve: fmt.Sprintf("CVE-2024-%04d", index), + PublishedOn: timestamppb.Now(), + CreatedAt: timestamppb.Now(), + }, + PreferredCvss: 7.5, + Severity: storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, + IsFixable: true, + HasFixedBy: &storage.VirtualMachineCVEV2_FixedBy{FixedBy: "1.1.2"}, + } +} + +// insertFullChainAndCVE creates a VM -> scan -> component -> CVE chain and returns the CVE. +func (s *vmCVEV2DataStoreTestSuite) insertFullChainAndCVE(clusterID, namespace string, vmIndex, cveIndex int) *storage.VirtualMachineCVEV2 { + ctx := sac.WithAllAccess(context.Background()) + vm := s.createTestVM(clusterID, namespace, vmIndex) + s.Require().NoError(s.vmStore.UpsertVM(ctx, vm)) + scan := s.createTestScan(vm.GetId()) + s.Require().NoError(s.scanPG.Upsert(ctx, scan)) + component := s.createTestComponent(scan.GetId()) + s.Require().NoError(s.componentPG.Upsert(ctx, component)) + cve := s.createTestCVE(vm.GetId(), component.GetId(), cveIndex) + s.Require().NoError(s.cvePG.Upsert(ctx, cve)) + return cve +} + +// TestGet verifies basic get functionality. +func (s *vmCVEV2DataStoreTestSuite) TestGet() { + cve := s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + + retrieved, found, err := s.datastore.Get(s.sacCtx, cve.GetId()) + s.NoError(err) + s.True(found) + s.Equal(cve.GetId(), retrieved.GetId()) + s.Equal(cve.GetCveBaseInfo().GetCve(), retrieved.GetCveBaseInfo().GetCve()) + s.Equal(cve.GetPreferredCvss(), retrieved.GetPreferredCvss()) + + // Non-existent CVE + _, found, err = s.datastore.Get(s.sacCtx, uuid.NewDummy().String()) + s.NoError(err) + s.False(found) +} + +// TestExists verifies existence checks. +func (s *vmCVEV2DataStoreTestSuite) TestExists() { + cve := s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + + exists, err := s.datastore.Exists(s.sacCtx, cve.GetId()) + s.NoError(err) + s.True(exists) + + exists, err = s.datastore.Exists(s.sacCtx, uuid.NewDummy().String()) + s.NoError(err) + s.False(exists) +} + +// TestCount verifies count functionality. +func (s *vmCVEV2DataStoreTestSuite) TestCount() { + count, err := s.datastore.Count(s.sacCtx, nil) + s.NoError(err) + s.Equal(0, count) + + s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceA, 2, 1) + + count, err = s.datastore.Count(s.sacCtx, nil) + s.NoError(err) + s.Equal(2, count) +} + +// TestGetBatch verifies batch retrieval. +func (s *vmCVEV2DataStoreTestSuite) TestGetBatch() { + cve1 := s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + cve2 := s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceA, 2, 1) + + cves, err := s.datastore.GetBatch(s.sacCtx, []string{cve1.GetId(), cve2.GetId()}) + s.NoError(err) + s.Len(cves, 2) + + fetchedIDs := make([]string, len(cves)) + for i, c := range cves { + fetchedIDs[i] = c.GetId() + } + s.ElementsMatch([]string{cve1.GetId(), cve2.GetId()}, fetchedIDs) +} + +// TestSearch verifies search functionality. +func (s *vmCVEV2DataStoreTestSuite) TestSearch() { + cve1 := s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + cve2 := s.insertFullChainAndCVE(testconsts.Cluster2, testconsts.NamespaceB, 1, 1) + + results, err := s.datastore.Search(s.sacCtx, nil) + s.NoError(err) + s.Len(results, 2) + + fetchedIDs := make([]string, len(results)) + for i, r := range results { + fetchedIDs[i] = r.ID + } + s.ElementsMatch([]string{cve1.GetId(), cve2.GetId()}, fetchedIDs) +} + +// TestSearchRawVMCVEs verifies raw search. +func (s *vmCVEV2DataStoreTestSuite) TestSearchRawVMCVEs() { + s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + s.insertFullChainAndCVE(testconsts.Cluster2, testconsts.NamespaceB, 1, 1) + + cves, err := s.datastore.SearchRawVMCVEs(s.sacCtx, search.EmptyQuery()) + s.NoError(err) + s.Len(cves, 2) + + for _, c := range cves { + s.NotEmpty(c.GetId()) + s.NotEmpty(c.GetCveBaseInfo().GetCve()) + s.True(c.GetPreferredCvss() > 0) + } +} + +// TestSACDeniedAccess verifies that SAC-denied contexts cannot access CVEs. +func (s *vmCVEV2DataStoreTestSuite) TestSACDeniedAccess() { + cve := s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + + _, found, err := s.datastore.Get(s.noSacCtx, cve.GetId()) + s.NoError(err) + s.False(found) + + exists, err := s.datastore.Exists(s.noSacCtx, cve.GetId()) + s.NoError(err) + s.False(exists) + + count, err := s.datastore.Count(s.noSacCtx, nil) + s.NoError(err) + s.Equal(0, count) + + results, err := s.datastore.Search(s.noSacCtx, nil) + s.NoError(err) + s.Empty(results) + + cves, err := s.datastore.GetBatch(s.noSacCtx, []string{cve.GetId()}) + s.NoError(err) + s.Empty(cves) + + rawCVEs, err := s.datastore.SearchRawVMCVEs(s.noSacCtx, search.EmptyQuery()) + s.NoError(err) + s.Empty(rawCVEs) +} + +// TestSACNamespaceScopedAccess verifies namespace-scoped SAC filtering. +func (s *vmCVEV2DataStoreTestSuite) TestSACNamespaceScopedAccess() { + cve1 := s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + cve2 := s.insertFullChainAndCVE(testconsts.Cluster1, testconsts.NamespaceB, 1, 1) + cve3 := s.insertFullChainAndCVE(testconsts.Cluster2, testconsts.NamespaceA, 1, 1) + cve4 := s.insertFullChainAndCVE(testconsts.Cluster2, testconsts.NamespaceB, 1, 1) + + contexts := testutils.GetNamespaceScopedTestContexts(context.Background(), s.T(), resources.VirtualMachine) + + for name, tc := range map[string]struct { + scopeKey string + expectedIDs []string + }{ + "Unrestricted read access returns all CVEs": { + scopeKey: testutils.UnrestrictedReadCtx, + expectedIDs: []string{cve1.GetId(), cve2.GetId(), cve3.GetId(), cve4.GetId()}, + }, + "Unrestricted read-write access returns all CVEs": { + scopeKey: testutils.UnrestrictedReadWriteCtx, + expectedIDs: []string{cve1.GetId(), cve2.GetId(), cve3.GetId(), cve4.GetId()}, + }, + "Cluster1 read-write access returns only cluster1 CVEs": { + scopeKey: testutils.Cluster1ReadWriteCtx, + expectedIDs: []string{cve1.GetId(), cve2.GetId()}, + }, + "Cluster2 read-write access returns only cluster2 CVEs": { + scopeKey: testutils.Cluster2ReadWriteCtx, + expectedIDs: []string{cve3.GetId(), cve4.GetId()}, + }, + "Cluster3 read-write access returns no CVEs": { + scopeKey: testutils.Cluster3ReadWriteCtx, + expectedIDs: nil, + }, + "Cluster1 NamespaceA access returns only cluster1 namespaceA CVEs": { + scopeKey: testutils.Cluster1NamespaceAReadWriteCtx, + expectedIDs: []string{cve1.GetId()}, + }, + "Cluster1 NamespaceB access returns only cluster1 namespaceB CVEs": { + scopeKey: testutils.Cluster1NamespaceBReadWriteCtx, + expectedIDs: []string{cve2.GetId()}, + }, + "Cluster1 NamespacesAB access returns cluster1 namespaces A and B CVEs": { + scopeKey: testutils.Cluster1NamespacesABReadWriteCtx, + expectedIDs: []string{cve1.GetId(), cve2.GetId()}, + }, + "Cluster2 NamespaceB access returns only cluster2 namespaceB CVEs": { + scopeKey: testutils.Cluster2NamespaceBReadWriteCtx, + expectedIDs: []string{cve4.GetId()}, + }, + "Mixed cluster and namespace access returns CVEs from allowed scopes": { + scopeKey: testutils.MixedClusterAndNamespaceReadCtx, + expectedIDs: []string{cve1.GetId(), cve3.GetId(), cve4.GetId()}, + }, + } { + s.Run(name, func() { + ctx := contexts[tc.scopeKey] + + // Test Count + count, err := s.datastore.Count(ctx, nil) + s.NoError(err) + s.Equal(len(tc.expectedIDs), count) + + // Test Search + results, err := s.datastore.Search(ctx, nil) + s.NoError(err) + fetchedIDs := make([]string, len(results)) + for i, r := range results { + fetchedIDs[i] = r.ID + } + s.ElementsMatch(tc.expectedIDs, fetchedIDs) + + // Test Exists for a CVE in cluster1 namespaceA + exists, err := s.datastore.Exists(ctx, cve1.GetId()) + s.NoError(err) + expectedExists := slices.Contains(tc.expectedIDs, cve1.GetId()) + s.Equal(expectedExists, exists) + + // Test Get for a CVE in cluster1 namespaceA + _, found, err := s.datastore.Get(ctx, cve1.GetId()) + s.NoError(err) + s.Equal(expectedExists, found) + + // Test GetBatch + allIDs := []string{cve1.GetId(), cve2.GetId(), cve3.GetId(), cve4.GetId()} + cves, err := s.datastore.GetBatch(ctx, allIDs) + s.NoError(err) + batchIDs := make([]string, len(cves)) + for i, c := range cves { + batchIDs[i] = c.GetId() + } + s.ElementsMatch(tc.expectedIDs, batchIDs) + + // Test SearchRawVMCVEs + rawCVEs, err := s.datastore.SearchRawVMCVEs(ctx, search.EmptyQuery()) + s.NoError(err) + rawIDs := make([]string, len(rawCVEs)) + for i, c := range rawCVEs { + rawIDs[i] = c.GetId() + } + s.ElementsMatch(tc.expectedIDs, rawIDs) + }) + } +} + +// TestSingletonGatedByFeatureFlag verifies the singleton is nil when the feature flag is disabled. +func (s *vmCVEV2DataStoreTestSuite) TestSingletonGatedByFeatureFlag() { + s.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + s.Nil(Singleton()) +} diff --git a/central/virtualmachine/cve/v2/datastore/mocks/datastore.go b/central/virtualmachine/cve/v2/datastore/mocks/datastore.go new file mode 100644 index 0000000000000..1bd1050f42e4d --- /dev/null +++ b/central/virtualmachine/cve/v2/datastore/mocks/datastore.go @@ -0,0 +1,135 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: datastore.go +// +// Generated by this command: +// +// mockgen -package mocks -destination mocks/datastore.go -source datastore.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + v1 "github.com/stackrox/rox/generated/api/v1" + storage "github.com/stackrox/rox/generated/storage" + search "github.com/stackrox/rox/pkg/search" + gomock "go.uber.org/mock/gomock" +) + +// MockDataStore is a mock of DataStore interface. +type MockDataStore struct { + ctrl *gomock.Controller + recorder *MockDataStoreMockRecorder + isgomock struct{} +} + +// MockDataStoreMockRecorder is the mock recorder for MockDataStore. +type MockDataStoreMockRecorder struct { + mock *MockDataStore +} + +// NewMockDataStore creates a new mock instance. +func NewMockDataStore(ctrl *gomock.Controller) *MockDataStore { + mock := &MockDataStore{ctrl: ctrl} + mock.recorder = &MockDataStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataStore) EXPECT() *MockDataStoreMockRecorder { + return m.recorder +} + +// Count mocks base method. +func (m *MockDataStore) Count(ctx context.Context, q *v1.Query) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Count", ctx, q) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Count indicates an expected call of Count. +func (mr *MockDataStoreMockRecorder) Count(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockDataStore)(nil).Count), ctx, q) +} + +// Exists mocks base method. +func (m *MockDataStore) Exists(ctx context.Context, id string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, id) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockDataStoreMockRecorder) Exists(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockDataStore)(nil).Exists), ctx, id) +} + +// Get mocks base method. +func (m *MockDataStore) Get(ctx context.Context, id string) (*storage.VirtualMachineCVEV2, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, id) + ret0, _ := ret[0].(*storage.VirtualMachineCVEV2) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Get indicates an expected call of Get. +func (mr *MockDataStoreMockRecorder) Get(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDataStore)(nil).Get), ctx, id) +} + +// GetBatch mocks base method. +func (m *MockDataStore) GetBatch(ctx context.Context, ids []string) ([]*storage.VirtualMachineCVEV2, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBatch", ctx, ids) + ret0, _ := ret[0].([]*storage.VirtualMachineCVEV2) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBatch indicates an expected call of GetBatch. +func (mr *MockDataStoreMockRecorder) GetBatch(ctx, ids any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBatch", reflect.TypeOf((*MockDataStore)(nil).GetBatch), ctx, ids) +} + +// Search mocks base method. +func (m *MockDataStore) Search(ctx context.Context, q *v1.Query) ([]search.Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Search", ctx, q) + ret0, _ := ret[0].([]search.Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Search indicates an expected call of Search. +func (mr *MockDataStoreMockRecorder) Search(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockDataStore)(nil).Search), ctx, q) +} + +// SearchRawVMCVEs mocks base method. +func (m *MockDataStore) SearchRawVMCVEs(ctx context.Context, q *v1.Query) ([]*storage.VirtualMachineCVEV2, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchRawVMCVEs", ctx, q) + ret0, _ := ret[0].([]*storage.VirtualMachineCVEV2) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchRawVMCVEs indicates an expected call of SearchRawVMCVEs. +func (mr *MockDataStoreMockRecorder) SearchRawVMCVEs(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchRawVMCVEs", reflect.TypeOf((*MockDataStore)(nil).SearchRawVMCVEs), ctx, q) +} diff --git a/central/virtualmachine/cve/v2/datastore/singleton.go b/central/virtualmachine/cve/v2/datastore/singleton.go new file mode 100644 index 0000000000000..86384ad677093 --- /dev/null +++ b/central/virtualmachine/cve/v2/datastore/singleton.go @@ -0,0 +1,28 @@ +package datastore + +import ( + "github.com/stackrox/rox/central/globaldb" + pgStore "github.com/stackrox/rox/central/virtualmachine/cve/v2/datastore/store/postgres" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/sync" +) + +var ( + once sync.Once + + ds DataStore +) + +func initialize() { + storage := pgStore.New(globaldb.GetPostgres()) + ds = New(storage) +} + +// Singleton returns a singleton instance of the VM CVE v2 datastore. +func Singleton() DataStore { + if !features.VirtualMachinesEnhancedDataModel.Enabled() { + return nil + } + once.Do(initialize) + return ds +} diff --git a/central/virtualmachine/cve/v2/datastore/store/postgres/gen.go b/central/virtualmachine/cve/v2/datastore/store/postgres/gen.go new file mode 100644 index 0000000000000..0f4082a26b06a --- /dev/null +++ b/central/virtualmachine/cve/v2/datastore/store/postgres/gen.go @@ -0,0 +1,3 @@ +package postgres + +//go:generate pg-table-bindings-wrapper --type=storage.VirtualMachineCVEV2 --search-category VIRTUAL_MACHINE_VULNERABILITIES_V2 --search-scope VIRTUAL_MACHINE_VULNERABILITIES_V2,VIRTUAL_MACHINE_COMPONENTS_V2,VIRTUAL_MACHINE_SCANS_V2,VIRTUAL_MACHINES_V2,NAMESPACES,CLUSTERS --references=storage.VirtualMachineV2,storage.VirtualMachineComponentV2 diff --git a/central/virtualmachine/cve/v2/datastore/store/postgres/store.go b/central/virtualmachine/cve/v2/datastore/store/postgres/store.go new file mode 100644 index 0000000000000..9a817e4e199b1 --- /dev/null +++ b/central/virtualmachine/cve/v2/datastore/store/postgres/store.go @@ -0,0 +1,213 @@ +// Code generated by pg-bindings generator. DO NOT EDIT. + +package postgres + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5" + "github.com/stackrox/rox/central/metrics" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/logging" + ops "github.com/stackrox/rox/pkg/metrics" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/pgutils" + pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" + pgSearch "github.com/stackrox/rox/pkg/search/postgres" +) + +const ( + baseTable = "virtual_machine_cvev2" + storeName = "VirtualMachineCVEV2" +) + +var ( + log = logging.LoggerForModule() + schema = pkgSchema.VirtualMachineCvev2Schema + targetResource = resources.VirtualMachine +) + +type ( + storeType = storage.VirtualMachineCVEV2 + callback = func(obj *storeType) error +) + +// Store is the interface to interact with the storage for storage.VirtualMachineCVEV2 +type Store interface { + Upsert(ctx context.Context, obj *storeType) error + UpsertMany(ctx context.Context, objs []*storeType) error + Delete(ctx context.Context, id string) error + DeleteByQuery(ctx context.Context, q *v1.Query) error + DeleteByQueryWithIDs(ctx context.Context, q *v1.Query) ([]string, error) + DeleteMany(ctx context.Context, identifiers []string) error + PruneMany(ctx context.Context, identifiers []string) error + + Count(ctx context.Context, q *v1.Query) (int, error) + Exists(ctx context.Context, id string) (bool, error) + Search(ctx context.Context, q *v1.Query) ([]search.Result, error) + + Get(ctx context.Context, id string) (*storeType, bool, error) + // Deprecated: use GetByQueryFn instead + GetByQuery(ctx context.Context, query *v1.Query) ([]*storeType, error) + GetByQueryFn(ctx context.Context, query *v1.Query, fn callback) error + GetMany(ctx context.Context, identifiers []string) ([]*storeType, []int, error) + GetIDs(ctx context.Context) ([]string, error) + + Walk(ctx context.Context, fn callback) error + WalkByQuery(ctx context.Context, query *v1.Query, fn callback) error +} + +// New returns a new Store instance using the provided sql instance. +func New(db postgres.DB) Store { + return pgSearch.NewGloballyScopedGenericStore[storeType, *storeType]( + db, + schema, + pkGetter, + insertIntoVirtualMachineCvev2, + copyFromVirtualMachineCvev2, + metricsSetAcquireDBConnDuration, + metricsSetPostgresOperationDurationTime, + targetResource, + nil, + nil, + ) +} + +// region Helper functions + +func pkGetter(obj *storeType) string { + return obj.GetId() +} + +func metricsSetPostgresOperationDurationTime(start time.Time, op ops.Op) { + metrics.SetPostgresOperationDurationTime(start, op, storeName) +} + +func metricsSetAcquireDBConnDuration(start time.Time, op ops.Op) { + metrics.SetAcquireDBConnDuration(start, op, storeName) +} + +func insertIntoVirtualMachineCvev2(batch *pgx.Batch, obj *storage.VirtualMachineCVEV2) error { + + serialized, marshalErr := obj.MarshalVT() + if marshalErr != nil { + return marshalErr + } + + values := []interface{}{ + // parent primary keys start + pgutils.NilOrUUID(obj.GetId()), + pgutils.NilOrUUID(obj.GetVmV2Id()), + pgutils.NilOrUUID(obj.GetVmComponentId()), + obj.GetCveBaseInfo().GetCve(), + protocompat.NilOrTime(obj.GetCveBaseInfo().GetPublishedOn()), + protocompat.NilOrTime(obj.GetCveBaseInfo().GetCreatedAt()), + obj.GetCveBaseInfo().GetEpss().GetEpssProbability(), + obj.GetCveBaseInfo().GetCisaKev(), + obj.GetCveBaseInfo().GetKnownRansomwareCampaign(), + obj.GetPreferredCvss(), + obj.GetSeverity(), + obj.GetImpactScore(), + obj.GetNvdcvss(), + obj.GetIsFixable(), + obj.GetFixedBy(), + obj.GetEpssProbability(), + obj.GetAdvisory().GetName(), + obj.GetAdvisory().GetLink(), + serialized, + } + + finalStr := "INSERT INTO virtual_machine_cvev2 (Id, VmV2Id, VmComponentId, CveBaseInfo_Cve, CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability, CveBaseInfo_CisaKev, CveBaseInfo_KnownRansomwareCampaign, PreferredCvss, Severity, ImpactScore, Nvdcvss, IsFixable, FixedBy, EpssProbability, Advisory_Name, Advisory_Link, serialized) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, VmV2Id = EXCLUDED.VmV2Id, VmComponentId = EXCLUDED.VmComponentId, CveBaseInfo_Cve = EXCLUDED.CveBaseInfo_Cve, CveBaseInfo_PublishedOn = EXCLUDED.CveBaseInfo_PublishedOn, CveBaseInfo_CreatedAt = EXCLUDED.CveBaseInfo_CreatedAt, CveBaseInfo_Epss_EpssProbability = EXCLUDED.CveBaseInfo_Epss_EpssProbability, CveBaseInfo_CisaKev = EXCLUDED.CveBaseInfo_CisaKev, CveBaseInfo_KnownRansomwareCampaign = EXCLUDED.CveBaseInfo_KnownRansomwareCampaign, PreferredCvss = EXCLUDED.PreferredCvss, Severity = EXCLUDED.Severity, ImpactScore = EXCLUDED.ImpactScore, Nvdcvss = EXCLUDED.Nvdcvss, IsFixable = EXCLUDED.IsFixable, FixedBy = EXCLUDED.FixedBy, EpssProbability = EXCLUDED.EpssProbability, Advisory_Name = EXCLUDED.Advisory_Name, Advisory_Link = EXCLUDED.Advisory_Link, serialized = EXCLUDED.serialized" + batch.Queue(finalStr, values...) + + return nil +} + +var copyColsVirtualMachineCvev2 = []string{ + "id", + "vmv2id", + "vmcomponentid", + "cvebaseinfo_cve", + "cvebaseinfo_publishedon", + "cvebaseinfo_createdat", + "cvebaseinfo_epss_epssprobability", + "cvebaseinfo_cisakev", + "cvebaseinfo_knownransomwarecampaign", + "preferredcvss", + "severity", + "impactscore", + "nvdcvss", + "isfixable", + "fixedby", + "epssprobability", + "advisory_name", + "advisory_link", + "serialized", +} + +func copyFromVirtualMachineCvev2(ctx context.Context, s pgSearch.Deleter, tx *postgres.Tx, objs ...*storage.VirtualMachineCVEV2) error { + if len(objs) == 0 { + return nil + } + + { + // CopyFrom does not upsert, so delete existing rows first to achieve upsert behavior. + // Parent deletion cascades to children, so only the top-level parent needs deletion. + deletes := make([]string, 0, len(objs)) + for _, obj := range objs { + deletes = append(deletes, obj.GetId()) + } + if err := s.DeleteMany(ctx, deletes); err != nil { + return err + } + } + + idx := 0 + inputRows := pgx.CopyFromFunc(func() ([]any, error) { + if idx >= len(objs) { + return nil, nil + } + obj := objs[idx] + idx++ + + serialized, marshalErr := obj.MarshalVT() + if marshalErr != nil { + return nil, marshalErr + } + + return []interface{}{ + pgutils.NilOrUUID(obj.GetId()), + pgutils.NilOrUUID(obj.GetVmV2Id()), + pgutils.NilOrUUID(obj.GetVmComponentId()), + obj.GetCveBaseInfo().GetCve(), + protocompat.NilOrTime(obj.GetCveBaseInfo().GetPublishedOn()), + protocompat.NilOrTime(obj.GetCveBaseInfo().GetCreatedAt()), + obj.GetCveBaseInfo().GetEpss().GetEpssProbability(), + obj.GetCveBaseInfo().GetCisaKev(), + obj.GetCveBaseInfo().GetKnownRansomwareCampaign(), + obj.GetPreferredCvss(), + obj.GetSeverity(), + obj.GetImpactScore(), + obj.GetNvdcvss(), + obj.GetIsFixable(), + obj.GetFixedBy(), + obj.GetEpssProbability(), + obj.GetAdvisory().GetName(), + obj.GetAdvisory().GetLink(), + serialized, + }, nil + }) + + if _, err := tx.CopyFrom(ctx, pgx.Identifier{"virtual_machine_cvev2"}, copyColsVirtualMachineCvev2, inputRows); err != nil { + return err + } + + return nil +} + +// endregion Helper functions diff --git a/central/virtualmachine/cve/v2/datastore/store/postgres/store_test.go b/central/virtualmachine/cve/v2/datastore/store/postgres/store_test.go new file mode 100644 index 0000000000000..3c8a0fc3f6daa --- /dev/null +++ b/central/virtualmachine/cve/v2/datastore/store/postgres/store_test.go @@ -0,0 +1,55 @@ +// Code generated by pg-bindings generator. DO NOT EDIT. + +//go:build sql_integration + +package postgres + +import ( + "context" + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/testutils" + "github.com/stretchr/testify/suite" +) + +type VirtualMachineCvev2StoreSuite struct { + suite.Suite + store Store + testDB *pgtest.TestPostgres +} + +func TestVirtualMachineCvev2Store(t *testing.T) { + suite.Run(t, new(VirtualMachineCvev2StoreSuite)) +} + +func (s *VirtualMachineCvev2StoreSuite) SetupSuite() { + + s.testDB = pgtest.ForT(s.T()) + s.store = New(s.testDB.DB) +} + +func (s *VirtualMachineCvev2StoreSuite) SetupTest() { + ctx := sac.WithAllAccess(context.Background()) + tag, err := s.testDB.Exec(ctx, "TRUNCATE virtual_machine_cvev2 CASCADE") + s.T().Log("virtual_machine_cvev2", tag) + s.store = New(s.testDB.DB) + s.NoError(err) +} + +func (s *VirtualMachineCvev2StoreSuite) TestStore() { + ctx := sac.WithAllAccess(context.Background()) + + store := s.store + + virtualMachineCVEV2 := &storage.VirtualMachineCVEV2{} + s.NoError(testutils.FullInit(virtualMachineCVEV2, testutils.SimpleInitializer(), testutils.JSONFieldsFilter)) + + foundVirtualMachineCVEV2, exists, err := store.Get(ctx, virtualMachineCVEV2.GetId()) + s.NoError(err) + s.False(exists) + s.Nil(foundVirtualMachineCVEV2) + +} diff --git a/central/virtualmachine/datastore/datastore_impl_test.go b/central/virtualmachine/datastore/datastore_impl_test.go index 79f0544391533..935af6ffaa226 100644 --- a/central/virtualmachine/datastore/datastore_impl_test.go +++ b/central/virtualmachine/datastore/datastore_impl_test.go @@ -590,7 +590,7 @@ func (s *VirtualMachineDataStoreTestSuite) TestSearchRawVirtualMachinesSort() { } func (s *VirtualMachineDataStoreTestSuite) TestSearchRawVirtualMachinesSliceHandling() { - for i := 0; i < 50; i++ { + for i := range 50 { for _, testCluster := range []string{testconsts.Cluster1, testconsts.Cluster2} { for _, testNamespace := range []string{testconsts.NamespaceA, testconsts.NamespaceB, testconsts.NamespaceC} { err := s.injectNamespacedVirtualMachine(testCluster, testNamespace, i) @@ -796,7 +796,7 @@ func BenchmarkGetVirtualMachine(b *testing.B) { sac.ResourceScopeKeys(resources.VirtualMachine))) // Pre-populate - for i := 0; i < 1000; i++ { + for i := range 1000 { vm := &storage.VirtualMachine{ Id: uuid.NewTestUUID(i).String(), Name: fmt.Sprintf("test-vm-%d", i), diff --git a/central/virtualmachine/datastore/datastore_test_constructors.go b/central/virtualmachine/datastore/datastore_test_constructors.go new file mode 100644 index 0000000000000..903e13a949d3f --- /dev/null +++ b/central/virtualmachine/datastore/datastore_test_constructors.go @@ -0,0 +1,13 @@ +package datastore + +import ( + "testing" + + pgStore "github.com/stackrox/rox/central/virtualmachine/datastore/internal/store/postgres" + "github.com/stackrox/rox/pkg/postgres" +) + +// GetTestPostgresDataStore provides a datastore connected to postgres for testing purposes. +func GetTestPostgresDataStore(_ testing.TB, pool postgres.DB) DataStore { + return newDatastoreImpl(pgStore.New(pool)) +} diff --git a/central/virtualmachine/datastore/telemetry.go b/central/virtualmachine/datastore/telemetry.go index 04bc665791488..777124c8f6e53 100644 --- a/central/virtualmachine/datastore/telemetry.go +++ b/central/virtualmachine/datastore/telemetry.go @@ -4,14 +4,17 @@ import ( "context" "time" + v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/set" "github.com/stackrox/rox/pkg/telemetry/phonehome" + "github.com/stackrox/rox/pkg/timestamp" ) const ( @@ -35,8 +38,8 @@ var ( // 2. Total number of virtual machines // 3. Number of VMs with active agents (received IndexReport within last 24 hours) // -// When the ROX_VIRTUAL_MACHINES feature flag is disabled, this function returns -// an empty map without performing any database queries, ensuring no performance impact. +// When ROX_VIRTUAL_MACHINES is off, or the enhanced data model is on (GatherV2 +// owns these metric names), Gather returns an empty map and does not query storage. func Gather(ds DataStore) phonehome.GatherFunc { return gatherWithTime(ds, time.Now) } @@ -44,8 +47,7 @@ func Gather(ds DataStore) phonehome.GatherFunc { // gatherWithTime allows injecting a time function for deterministic testing. func gatherWithTime(ds DataStore, nowFunc func() time.Time) phonehome.GatherFunc { return func(ctx context.Context) (map[string]any, error) { - // Early return if virtual machines feature is disabled - zero performance impact - if !features.VirtualMachines.Enabled() { + if !features.VirtualMachines.Enabled() || features.VirtualMachinesEnhancedDataModel.Enabled() { return map[string]any{}, nil } @@ -101,3 +103,71 @@ func gatherWithTime(ds DataStore, nowFunc func() time.Time) phonehome.GatherFunc return props, nil } } + +// vmV2Walker is the subset of the VM V2 datastore this gatherer needs. +type vmV2Walker interface { + Walk(ctx context.Context, fn func(vm *storage.VirtualMachineV2) error) error +} + +// scanV2Counter is the subset of the scan V2 datastore this gatherer needs. +type scanV2Counter interface { + Count(ctx context.Context, q *v1.Query) (int, error) +} + +// GatherV2 returns a gatherer that queries the V2 VM and scan datastores. +func GatherV2(vmV2DS vmV2Walker, scanV2DS scanV2Counter) phonehome.GatherFunc { + return gatherV2WithTime(vmV2DS, scanV2DS, time.Now) +} + +func gatherV2WithTime(vmV2DS vmV2Walker, scanV2DS scanV2Counter, nowFunc func() time.Time) phonehome.GatherFunc { + return func(ctx context.Context) (map[string]any, error) { + if !features.VirtualMachines.Enabled() || !features.VirtualMachinesEnhancedDataModel.Enabled() { + return map[string]any{}, nil + } + + ctx = sac.WithGlobalAccessScopeChecker(ctx, + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS), + sac.ResourceScopeKeys(resources.VirtualMachine), + ), + ) + + clusterIDsWithRunningVMs := set.NewStringSet() + totalVMs := 0 + + err := vmV2DS.Walk(ctx, func(vm *storage.VirtualMachineV2) error { + totalVMs++ + if vm.GetState() == storage.VirtualMachineV2_RUNNING { + clusterID := vm.GetClusterId() + if clusterID == "" { + log.Debugf("Virtual machine V2 %s has empty cluster_id", vm.GetId()) + } else { + clusterIDsWithRunningVMs.Add(clusterID) + } + } + return nil + }) + if err != nil { + return nil, err + } + + now := nowFunc() + cutoff := now.Add(-activeVMAgentMaxAgeLimitTelemetry) + q := search.NewQueryBuilder(). + AddTimeRangeField(search.VirtualMachineScanTime, cutoff, timestamp.InfiniteFuture.GoTime()). + ProtoQuery() + + vmsWithActiveAgents, err := scanV2DS.Count(ctx, q) + if err != nil { + return nil, err + } + + props := map[string]any{ + metricClustersWithVMs: clusterIDsWithRunningVMs.Cardinality(), + metricTotalVMs: totalVMs, + metricVMsWithActiveAgents: vmsWithActiveAgents, + } + log.Debugf("Virtual machines V2 telemetry update: %v", props) + return props, nil + } +} diff --git a/central/virtualmachine/datastore/telemetry_test.go b/central/virtualmachine/datastore/telemetry_test.go index f730df2af01e7..0d8f01b6861a2 100644 --- a/central/virtualmachine/datastore/telemetry_test.go +++ b/central/virtualmachine/datastore/telemetry_test.go @@ -2,6 +2,9 @@ package datastore import ( "context" + "errors" + "strconv" + "strings" "testing" "time" @@ -10,6 +13,8 @@ import ( "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/protocompat" "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/timestamp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -20,9 +25,10 @@ var ( arbitraryNowFunc = func() time.Time { return arbitraryNow } ) +// TestVirtualMachineTelemetry covers the v1 gatherer, which only runs when the enhanced data model is off. func TestVirtualMachineTelemetry(t *testing.T) { - // Ensure feature flag is enabled for these tests t.Setenv(features.VirtualMachines.EnvVar(), "true") + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") tests := map[string]struct { vms []*storage.VirtualMachine @@ -181,29 +187,42 @@ func TestVirtualMachineTelemetry(t *testing.T) { } } -func TestVirtualMachineTelemetryWithFeatureFlagDisabled(t *testing.T) { - // Disable the feature flag using t.Setenv for automatic cleanup - t.Setenv(features.VirtualMachines.EnvVar(), "false") - - ctx := sac.WithAllAccess(context.Background()) - - // Create a mock datastore with VMs (which should NOT be queried) - ds := &mockDataStore{ - vms: []*storage.VirtualMachine{ - {Id: "vm1", ClusterId: "cluster1", Name: "test-vm", State: storage.VirtualMachine_RUNNING, Scan: createScanWithAge(1 * time.Hour)}, - {Id: "vm2", ClusterId: "cluster2", Name: "test-vm2", State: storage.VirtualMachine_RUNNING, Scan: createScanWithAge(1 * time.Hour)}, +func TestVirtualMachineTelemetrySkipGather(t *testing.T) { + tests := map[string]struct { + virtualMachinesEnabled bool + enhancedDataModel bool + }{ + "should return empty map when virtual machines flag is disabled": { + virtualMachinesEnabled: false, + enhancedDataModel: false, + }, + "should return empty map when enhanced data model is enabled": { + virtualMachinesEnabled: true, + enhancedDataModel: true, }, } - gatherer := gatherWithTime(ds, arbitraryNowFunc) - props, err := gatherer(ctx) + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Setenv(features.VirtualMachines.EnvVar(), strconv.FormatBool(tc.virtualMachinesEnabled)) + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), strconv.FormatBool(tc.enhancedDataModel)) - require.NoError(t, err) - require.NotNil(t, props) + ctx := sac.WithAllAccess(context.Background()) + ds := &mockDataStore{ + vms: []*storage.VirtualMachine{ + {Id: "vm1", ClusterId: "cluster1", Name: "test-vm", State: storage.VirtualMachine_RUNNING, Scan: createScanWithAge(1 * time.Hour)}, + }, + } - // When feature flag is disabled, should return empty map - // No database query should have been performed - assert.Empty(t, props, "Should return empty map when feature flag is disabled") + gatherer := gatherWithTime(ds, arbitraryNowFunc) + props, err := gatherer(ctx) + + require.NoError(t, err) + require.NotNil(t, props) + assert.Empty(t, props) + assert.False(t, ds.walked) + }) + } } // createScanWithAge creates a scan with a timestamp at the specified duration before arbitraryNow. @@ -216,8 +235,9 @@ func createScanWithAge(age time.Duration) *storage.VirtualMachineScan { } type mockDataStore struct { - vms []*storage.VirtualMachine - err error + vms []*storage.VirtualMachine + err error + walked bool } func (m *mockDataStore) CountVirtualMachines(ctx context.Context, query *v1.Query) (int, error) { @@ -252,6 +272,7 @@ func (m *mockDataStore) SearchRawVirtualMachines(ctx context.Context, query *v1. } func (m *mockDataStore) Walk(ctx context.Context, fn func(vm *storage.VirtualMachine) error) error { + m.walked = true if m.err != nil { return m.err } @@ -262,3 +283,190 @@ func (m *mockDataStore) Walk(ctx context.Context, fn func(vm *storage.VirtualMac } return nil } + +// --- V2 mock types --- + +type mockVMV2DataStore struct { + vms []*storage.VirtualMachineV2 + err error +} + +func (m *mockVMV2DataStore) Walk(_ context.Context, fn func(vm *storage.VirtualMachineV2) error) error { + if m.err != nil { + return m.err + } + for _, vm := range m.vms { + if err := fn(vm); err != nil { + return err + } + } + return nil +} + +type mockScanV2DataStore struct { + count int + err error + lastQuery *v1.Query +} + +func (m *mockScanV2DataStore) Count(_ context.Context, q *v1.Query) (int, error) { + m.lastQuery = q + return m.count, m.err +} + +func TestVirtualMachineTelemetryV2(t *testing.T) { + t.Setenv(features.VirtualMachines.EnvVar(), "true") + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + + tests := map[string]struct { + vms []*storage.VirtualMachineV2 + scanCount int + expectedClustersWithRunningVMs int + expectedTotalVMs int + expectedVMsWithActiveAgents int + }{ + "should return zero for all metrics when no V2 VMs exist": { + vms: nil, + scanCount: 0, + expectedClustersWithRunningVMs: 0, + expectedTotalVMs: 0, + expectedVMsWithActiveAgents: 0, + }, + "should count single cluster with one running V2 VM and one recent scan": { + vms: []*storage.VirtualMachineV2{ + {Id: "vm1", ClusterId: "cluster1", Name: "test-vm", State: storage.VirtualMachineV2_RUNNING}, + }, + scanCount: 1, + expectedClustersWithRunningVMs: 1, + expectedTotalVMs: 1, + expectedVMsWithActiveAgents: 1, + }, + "should count total V2 VMs including stopped ones": { + vms: []*storage.VirtualMachineV2{ + {Id: "vm1", ClusterId: "cluster1", Name: "vm-1", State: storage.VirtualMachineV2_RUNNING}, + {Id: "vm2", ClusterId: "cluster1", Name: "vm-2", State: storage.VirtualMachineV2_STOPPED}, + {Id: "vm3", ClusterId: "cluster2", Name: "vm-3", State: storage.VirtualMachineV2_UNKNOWN}, + }, + scanCount: 0, + expectedClustersWithRunningVMs: 1, + expectedTotalVMs: 3, + expectedVMsWithActiveAgents: 0, + }, + "should count multiple distinct clusters with running V2 VMs": { + vms: []*storage.VirtualMachineV2{ + {Id: "vm1", ClusterId: "cluster1", Name: "vm-1", State: storage.VirtualMachineV2_RUNNING}, + {Id: "vm2", ClusterId: "cluster2", Name: "vm-2", State: storage.VirtualMachineV2_RUNNING}, + {Id: "vm3", ClusterId: "cluster3", Name: "vm-3", State: storage.VirtualMachineV2_RUNNING}, + }, + scanCount: 2, + expectedClustersWithRunningVMs: 3, + expectedTotalVMs: 3, + expectedVMsWithActiveAgents: 2, + }, + "should exclude V2 VMs with empty cluster id from cluster count": { + vms: []*storage.VirtualMachineV2{ + {Id: "vm1", ClusterId: "cluster1", Name: "vm-1", State: storage.VirtualMachineV2_RUNNING}, + {Id: "vm2", ClusterId: "", Name: "vm-orphan", State: storage.VirtualMachineV2_RUNNING}, + }, + scanCount: 1, + expectedClustersWithRunningVMs: 1, + expectedTotalVMs: 2, + expectedVMsWithActiveAgents: 1, + }, + "should handle V2 VM with no scan row yet (never scanned)": { + vms: []*storage.VirtualMachineV2{ + {Id: "vm1", ClusterId: "cluster1", Name: "vm-1", State: storage.VirtualMachineV2_RUNNING}, + {Id: "vm2", ClusterId: "cluster1", Name: "vm-2", State: storage.VirtualMachineV2_RUNNING}, + }, + scanCount: 0, + expectedClustersWithRunningVMs: 1, + expectedTotalVMs: 2, + expectedVMsWithActiveAgents: 0, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + ctx := sac.WithAllAccess(context.Background()) + + vmV2DS := &mockVMV2DataStore{vms: tc.vms} + scanV2DS := &mockScanV2DataStore{count: tc.scanCount} + + gatherer := gatherV2WithTime(vmV2DS, scanV2DS, arbitraryNowFunc) + props, err := gatherer(ctx) + + require.NoError(t, err) + require.NotNil(t, props) + assert.Equal(t, tc.expectedClustersWithRunningVMs, props[metricClustersWithVMs]) + assert.Equal(t, tc.expectedTotalVMs, props[metricTotalVMs]) + assert.Equal(t, tc.expectedVMsWithActiveAgents, props[metricVMsWithActiveAgents]) + assertActiveAgentScanQuery(t, scanV2DS.lastQuery) + }) + } +} + +func TestVirtualMachineTelemetryV2Errors(t *testing.T) { + t.Setenv(features.VirtualMachines.EnvVar(), "true") + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + + walkErr := errors.New("walk failed") + countErr := errors.New("count failed") + + tests := map[string]struct { + walkErr error + countErr error + wantErr error + }{ + "should return Walk error and nil props": { + walkErr: walkErr, + wantErr: walkErr, + }, + "should return Count error and nil props": { + countErr: countErr, + wantErr: countErr, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + ctx := sac.WithAllAccess(context.Background()) + vmV2DS := &mockVMV2DataStore{ + vms: []*storage.VirtualMachineV2{ + {Id: "vm1", ClusterId: "cluster1", Name: "vm-1", State: storage.VirtualMachineV2_RUNNING}, + }, + err: tc.walkErr, + } + scanV2DS := &mockScanV2DataStore{count: 1, err: tc.countErr} + + props, err := gatherV2WithTime(vmV2DS, scanV2DS, arbitraryNowFunc)(ctx) + + require.ErrorIs(t, err, tc.wantErr) + assert.Nil(t, props) + if tc.walkErr != nil { + assert.Nil(t, scanV2DS.lastQuery) + return + } + assertActiveAgentScanQuery(t, scanV2DS.lastQuery) + }) + } +} + +func assertActiveAgentScanQuery(t *testing.T, q *v1.Query) { + t.Helper() + require.NotNil(t, q) + mfq := q.GetBaseQuery().GetMatchFieldQuery() + require.NotNil(t, mfq) + assert.Equal(t, search.VirtualMachineScanTime.String(), mfq.GetField()) + + value, ok := strings.CutPrefix(mfq.GetValue(), search.TimeRangePrefix) + require.True(t, ok) + fromStr, toStr, ok := strings.Cut(value, "-") + require.True(t, ok) + from, err := strconv.ParseInt(fromStr, 10, 64) + require.NoError(t, err) + to, err := strconv.ParseInt(toStr, 10, 64) + require.NoError(t, err) + + assert.Equal(t, arbitraryNow.Add(-activeVMAgentMaxAgeLimitTelemetry).UnixMilli(), from) + assert.Equal(t, timestamp.InfiniteFuture.GoTime().UnixMilli(), to) +} diff --git a/central/virtualmachine/scan/v2/datastore/datastore.go b/central/virtualmachine/scan/v2/datastore/datastore.go new file mode 100644 index 0000000000000..5a5f40ede4703 --- /dev/null +++ b/central/virtualmachine/scan/v2/datastore/datastore.go @@ -0,0 +1,39 @@ +package datastore + +import ( + "context" + "testing" + + pgStore "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore/store/postgres" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/postgres" + searchPkg "github.com/stackrox/rox/pkg/search" +) + +// DataStore is an intermediary to VirtualMachineScanV2 storage. +// +//go:generate mockgen-wrapper +type DataStore interface { + Search(ctx context.Context, q *v1.Query) ([]searchPkg.Result, error) + SearchRawVMScans(ctx context.Context, q *v1.Query) ([]*storage.VirtualMachineScanV2, error) + + Exists(ctx context.Context, id string) (bool, error) + Get(ctx context.Context, id string) (*storage.VirtualMachineScanV2, bool, error) + Count(ctx context.Context, q *v1.Query) (int, error) + GetBatch(ctx context.Context, ids []string) ([]*storage.VirtualMachineScanV2, error) +} + +// New returns a new instance of a DataStore. +func New(storage pgStore.Store) DataStore { + ds := &datastoreImpl{ + storage: storage, + } + return ds +} + +// GetTestPostgresDataStore provides a datastore connected to postgres for testing purposes. +func GetTestPostgresDataStore(_ testing.TB, pool postgres.DB) DataStore { + dbstore := pgStore.New(pool) + return New(dbstore) +} diff --git a/central/virtualmachine/scan/v2/datastore/datastore_impl.go b/central/virtualmachine/scan/v2/datastore/datastore_impl.go new file mode 100644 index 0000000000000..89a99c4335a30 --- /dev/null +++ b/central/virtualmachine/scan/v2/datastore/datastore_impl.go @@ -0,0 +1,66 @@ +package datastore + +import ( + "context" + "time" + + "github.com/stackrox/rox/central/metrics" + pgStore "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore/store/postgres" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + pkgSearch "github.com/stackrox/rox/pkg/search" +) + +type datastoreImpl struct { + storage pgStore.Store +} + +func (ds *datastoreImpl) Search(ctx context.Context, q *v1.Query) ([]pkgSearch.Result, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineScanV2", "Search") + return ds.storage.Search(ctx, q) +} + +func (ds *datastoreImpl) Count(ctx context.Context, q *v1.Query) (int, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineScanV2", "Count") + return ds.storage.Count(ctx, q) +} + +func (ds *datastoreImpl) SearchRawVMScans(ctx context.Context, q *v1.Query) ([]*storage.VirtualMachineScanV2, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineScanV2", "SearchRawVMScans") + var scans []*storage.VirtualMachineScanV2 + err := ds.storage.GetByQueryFn(ctx, q, func(scan *storage.VirtualMachineScanV2) error { + scans = append(scans, scan) + return nil + }) + if err != nil { + return nil, err + } + return scans, nil +} + +func (ds *datastoreImpl) Get(ctx context.Context, id string) (*storage.VirtualMachineScanV2, bool, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineScanV2", "Get") + scan, found, err := ds.storage.Get(ctx, id) + if err != nil || !found { + return nil, false, err + } + return scan, true, nil +} + +func (ds *datastoreImpl) Exists(ctx context.Context, id string) (bool, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineScanV2", "Exists") + found, err := ds.storage.Exists(ctx, id) + if err != nil || !found { + return false, err + } + return true, nil +} + +func (ds *datastoreImpl) GetBatch(ctx context.Context, ids []string) ([]*storage.VirtualMachineScanV2, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineScanV2", "GetBatch") + scans, _, err := ds.storage.GetMany(ctx, ids) + if err != nil { + return nil, err + } + return scans, nil +} diff --git a/central/virtualmachine/scan/v2/datastore/datastore_impl_test.go b/central/virtualmachine/scan/v2/datastore/datastore_impl_test.go new file mode 100644 index 0000000000000..1eebe0167c5b5 --- /dev/null +++ b/central/virtualmachine/scan/v2/datastore/datastore_impl_test.go @@ -0,0 +1,338 @@ +//go:build sql_integration + +package datastore + +import ( + "context" + "fmt" + "slices" + "testing" + + scanStore "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore/store/postgres" + vmV2Store "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store" + vmV2Postgres "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/postgres" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/sac/testconsts" + "github.com/stackrox/rox/pkg/sac/testutils" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/suite" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestVMScanV2DataStore(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + if !features.VirtualMachinesEnhancedDataModel.Enabled() { + t.Skip("VM enhanced data model is not enabled") + } + suite.Run(t, new(vmScanV2DataStoreTestSuite)) +} + +type vmScanV2DataStoreTestSuite struct { + suite.Suite + + db *pgtest.TestPostgres + datastore DataStore + vmStore vmV2Store.Store + + sacCtx context.Context + noSacCtx context.Context +} + +func (s *vmScanV2DataStoreTestSuite) SetupTest() { + s.db = pgtest.ForT(s.T()) + store := scanStore.New(s.db) + s.datastore = New(store) + s.vmStore = vmV2Postgres.New(s.db, concurrency.NewKeyFence()) + + s.sacCtx = sac.WithGlobalAccessScopeChecker(context.Background(), + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS, storage.Access_READ_WRITE_ACCESS), + sac.ResourceScopeKeys(resources.VirtualMachine))) + s.noSacCtx = sac.WithGlobalAccessScopeChecker(context.Background(), + sac.DenyAllAccessScopeChecker()) +} + +func (s *vmScanV2DataStoreTestSuite) TearDownTest() { + s.db.Close() +} + +func (s *vmScanV2DataStoreTestSuite) createTestVM(clusterID, namespace string, index int) *storage.VirtualMachineV2 { + vmID := uuid.NewV5FromNonUUIDs(clusterID, fmt.Sprintf("%s-%d", namespace, index)).String() + return &storage.VirtualMachineV2{ + Id: vmID, + Name: fmt.Sprintf("vm-%d", index), + Namespace: namespace, + ClusterId: clusterID, + ClusterName: fmt.Sprintf("cluster-%s", clusterID), + } +} + +func (s *vmScanV2DataStoreTestSuite) createTestScan(vmID string, index int) *storage.VirtualMachineScanV2 { + scanID := uuid.NewV5FromNonUUIDs(vmID, fmt.Sprintf("scan-%d", index)).String() + return &storage.VirtualMachineScanV2{ + Id: scanID, + VmV2Id: vmID, + ScanOs: "rhel:9", + ScanTime: timestamppb.Now(), + TopCvss: 7.5, + } +} + +func (s *vmScanV2DataStoreTestSuite) insertVMAndScan(clusterID, namespace string, vmIndex, scanIndex int) *storage.VirtualMachineScanV2 { + ctx := sac.WithAllAccess(context.Background()) + vm := s.createTestVM(clusterID, namespace, vmIndex) + s.Require().NoError(s.vmStore.UpsertVM(ctx, vm)) + scan := s.createTestScan(vm.GetId(), scanIndex) + scanPgStore := scanStore.New(s.db) + s.Require().NoError(scanPgStore.Upsert(ctx, scan)) + return scan +} + +// TestGet verifies basic get functionality. +func (s *vmScanV2DataStoreTestSuite) TestGet() { + scan := s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + + retrievedScan, found, err := s.datastore.Get(s.sacCtx, scan.GetId()) + s.NoError(err) + s.True(found) + s.Equal(scan.GetId(), retrievedScan.GetId()) + s.Equal(scan.GetScanOs(), retrievedScan.GetScanOs()) + + // Non-existent scan + _, found, err = s.datastore.Get(s.sacCtx, uuid.NewDummy().String()) + s.NoError(err) + s.False(found) +} + +// TestExists verifies existence checks. +func (s *vmScanV2DataStoreTestSuite) TestExists() { + scan := s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + + exists, err := s.datastore.Exists(s.sacCtx, scan.GetId()) + s.NoError(err) + s.True(exists) + + exists, err = s.datastore.Exists(s.sacCtx, uuid.NewDummy().String()) + s.NoError(err) + s.False(exists) +} + +// TestCount verifies count functionality. +func (s *vmScanV2DataStoreTestSuite) TestCount() { + count, err := s.datastore.Count(s.sacCtx, nil) + s.NoError(err) + s.Equal(0, count) + + s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceA, 2, 1) + + count, err = s.datastore.Count(s.sacCtx, nil) + s.NoError(err) + s.Equal(2, count) +} + +// TestGetBatch verifies batch retrieval. +func (s *vmScanV2DataStoreTestSuite) TestGetBatch() { + scan1 := s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + scan2 := s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceA, 2, 1) + + scans, err := s.datastore.GetBatch(s.sacCtx, []string{scan1.GetId(), scan2.GetId()}) + s.NoError(err) + s.Len(scans, 2) + + fetchedIDs := make([]string, len(scans)) + for i, sc := range scans { + fetchedIDs[i] = sc.GetId() + } + s.ElementsMatch([]string{scan1.GetId(), scan2.GetId()}, fetchedIDs) +} + +// TestSearch verifies search functionality. +func (s *vmScanV2DataStoreTestSuite) TestSearch() { + scan1 := s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + scan2 := s.insertVMAndScan(testconsts.Cluster2, testconsts.NamespaceB, 1, 1) + + results, err := s.datastore.Search(s.sacCtx, nil) + s.NoError(err) + s.Len(results, 2) + + fetchedIDs := make([]string, len(results)) + for i, r := range results { + fetchedIDs[i] = r.ID + } + s.ElementsMatch([]string{scan1.GetId(), scan2.GetId()}, fetchedIDs) +} + +// TestSearchRawVMScans verifies raw search. +func (s *vmScanV2DataStoreTestSuite) TestSearchRawVMScans() { + scan1 := s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + s.insertVMAndScan(testconsts.Cluster2, testconsts.NamespaceB, 1, 1) + + scans, err := s.datastore.SearchRawVMScans(s.sacCtx, search.EmptyQuery()) + s.NoError(err) + s.Len(scans, 2) + + // Verify scan data is populated + for _, sc := range scans { + s.NotEmpty(sc.GetId()) + s.NotEmpty(sc.GetScanOs()) + } + + // Search by VM scan OS + q := search.NewQueryBuilder().AddExactMatches(search.VirtualMachineScanOS, scan1.GetScanOs()).ProtoQuery() + scans, err = s.datastore.SearchRawVMScans(s.sacCtx, q) + s.NoError(err) + s.Len(scans, 2) +} + +// TestSACDeniedAccess verifies that SAC-denied contexts cannot access scans. +func (s *vmScanV2DataStoreTestSuite) TestSACDeniedAccess() { + scan := s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + + // Get should silently return not found + _, found, err := s.datastore.Get(s.noSacCtx, scan.GetId()) + s.NoError(err) + s.False(found) + + // Exists should return false + exists, err := s.datastore.Exists(s.noSacCtx, scan.GetId()) + s.NoError(err) + s.False(exists) + + // Count should return 0 + count, err := s.datastore.Count(s.noSacCtx, nil) + s.NoError(err) + s.Equal(0, count) + + // Search should return empty + results, err := s.datastore.Search(s.noSacCtx, nil) + s.NoError(err) + s.Empty(results) + + // GetBatch should return empty + scans, err := s.datastore.GetBatch(s.noSacCtx, []string{scan.GetId()}) + s.NoError(err) + s.Empty(scans) + + // SearchRaw should return empty + rawScans, err := s.datastore.SearchRawVMScans(s.noSacCtx, search.EmptyQuery()) + s.NoError(err) + s.Empty(rawScans) +} + +// TestSACNamespaceScopedAccess verifies namespace-scoped SAC filtering. +func (s *vmScanV2DataStoreTestSuite) TestSACNamespaceScopedAccess() { + // Insert scans across different clusters and namespaces + scan1 := s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceA, 1, 1) + scan2 := s.insertVMAndScan(testconsts.Cluster1, testconsts.NamespaceB, 1, 1) + scan3 := s.insertVMAndScan(testconsts.Cluster2, testconsts.NamespaceA, 1, 1) + scan4 := s.insertVMAndScan(testconsts.Cluster2, testconsts.NamespaceB, 1, 1) + + contexts := testutils.GetNamespaceScopedTestContexts(context.Background(), s.T(), resources.VirtualMachine) + + for name, tc := range map[string]struct { + scopeKey string + expectedIDs []string + }{ + "Unrestricted read access returns all scans": { + scopeKey: testutils.UnrestrictedReadCtx, + expectedIDs: []string{scan1.GetId(), scan2.GetId(), scan3.GetId(), scan4.GetId()}, + }, + "Unrestricted read-write access returns all scans": { + scopeKey: testutils.UnrestrictedReadWriteCtx, + expectedIDs: []string{scan1.GetId(), scan2.GetId(), scan3.GetId(), scan4.GetId()}, + }, + "Cluster1 read-write access returns only cluster1 scans": { + scopeKey: testutils.Cluster1ReadWriteCtx, + expectedIDs: []string{scan1.GetId(), scan2.GetId()}, + }, + "Cluster2 read-write access returns only cluster2 scans": { + scopeKey: testutils.Cluster2ReadWriteCtx, + expectedIDs: []string{scan3.GetId(), scan4.GetId()}, + }, + "Cluster3 read-write access returns no scans": { + scopeKey: testutils.Cluster3ReadWriteCtx, + expectedIDs: nil, + }, + "Cluster1 NamespaceA access returns only cluster1 namespaceA scans": { + scopeKey: testutils.Cluster1NamespaceAReadWriteCtx, + expectedIDs: []string{scan1.GetId()}, + }, + "Cluster1 NamespaceB access returns only cluster1 namespaceB scans": { + scopeKey: testutils.Cluster1NamespaceBReadWriteCtx, + expectedIDs: []string{scan2.GetId()}, + }, + "Cluster1 NamespacesAB access returns cluster1 namespaces A and B scans": { + scopeKey: testutils.Cluster1NamespacesABReadWriteCtx, + expectedIDs: []string{scan1.GetId(), scan2.GetId()}, + }, + "Cluster2 NamespaceB access returns only cluster2 namespaceB scans": { + scopeKey: testutils.Cluster2NamespaceBReadWriteCtx, + expectedIDs: []string{scan4.GetId()}, + }, + "Mixed cluster and namespace access returns scans from allowed scopes": { + scopeKey: testutils.MixedClusterAndNamespaceReadCtx, + expectedIDs: []string{scan1.GetId(), scan3.GetId(), scan4.GetId()}, + }, + } { + s.Run(name, func() { + ctx := contexts[tc.scopeKey] + + // Test Count + count, err := s.datastore.Count(ctx, nil) + s.NoError(err) + s.Equal(len(tc.expectedIDs), count) + + // Test Search + results, err := s.datastore.Search(ctx, nil) + s.NoError(err) + fetchedIDs := make([]string, len(results)) + for i, r := range results { + fetchedIDs[i] = r.ID + } + s.ElementsMatch(tc.expectedIDs, fetchedIDs) + + // Test Exists for a scan in cluster1 namespaceA + exists, err := s.datastore.Exists(ctx, scan1.GetId()) + s.NoError(err) + expectedExists := slices.Contains(tc.expectedIDs, scan1.GetId()) + s.Equal(expectedExists, exists) + + // Test Get for a scan in cluster1 namespaceA + _, found, err := s.datastore.Get(ctx, scan1.GetId()) + s.NoError(err) + s.Equal(expectedExists, found) + + // Test GetBatch + allIDs := []string{scan1.GetId(), scan2.GetId(), scan3.GetId(), scan4.GetId()} + scans, err := s.datastore.GetBatch(ctx, allIDs) + s.NoError(err) + batchIDs := make([]string, len(scans)) + for i, sc := range scans { + batchIDs[i] = sc.GetId() + } + s.ElementsMatch(tc.expectedIDs, batchIDs) + + // Test SearchRawVMScans + rawScans, err := s.datastore.SearchRawVMScans(ctx, search.EmptyQuery()) + s.NoError(err) + rawIDs := make([]string, len(rawScans)) + for i, sc := range rawScans { + rawIDs[i] = sc.GetId() + } + s.ElementsMatch(tc.expectedIDs, rawIDs) + }) + } +} + +// TestSingletonGatedByFeatureFlag verifies the singleton is nil when the feature flag is disabled. +func (s *vmScanV2DataStoreTestSuite) TestSingletonGatedByFeatureFlag() { + s.T().Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "false") + s.Nil(Singleton()) +} diff --git a/central/virtualmachine/scan/v2/datastore/mocks/datastore.go b/central/virtualmachine/scan/v2/datastore/mocks/datastore.go new file mode 100644 index 0000000000000..770574b88ba23 --- /dev/null +++ b/central/virtualmachine/scan/v2/datastore/mocks/datastore.go @@ -0,0 +1,135 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: datastore.go +// +// Generated by this command: +// +// mockgen -package mocks -destination mocks/datastore.go -source datastore.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + v1 "github.com/stackrox/rox/generated/api/v1" + storage "github.com/stackrox/rox/generated/storage" + search "github.com/stackrox/rox/pkg/search" + gomock "go.uber.org/mock/gomock" +) + +// MockDataStore is a mock of DataStore interface. +type MockDataStore struct { + ctrl *gomock.Controller + recorder *MockDataStoreMockRecorder + isgomock struct{} +} + +// MockDataStoreMockRecorder is the mock recorder for MockDataStore. +type MockDataStoreMockRecorder struct { + mock *MockDataStore +} + +// NewMockDataStore creates a new mock instance. +func NewMockDataStore(ctrl *gomock.Controller) *MockDataStore { + mock := &MockDataStore{ctrl: ctrl} + mock.recorder = &MockDataStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataStore) EXPECT() *MockDataStoreMockRecorder { + return m.recorder +} + +// Count mocks base method. +func (m *MockDataStore) Count(ctx context.Context, q *v1.Query) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Count", ctx, q) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Count indicates an expected call of Count. +func (mr *MockDataStoreMockRecorder) Count(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockDataStore)(nil).Count), ctx, q) +} + +// Exists mocks base method. +func (m *MockDataStore) Exists(ctx context.Context, id string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", ctx, id) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exists indicates an expected call of Exists. +func (mr *MockDataStoreMockRecorder) Exists(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockDataStore)(nil).Exists), ctx, id) +} + +// Get mocks base method. +func (m *MockDataStore) Get(ctx context.Context, id string) (*storage.VirtualMachineScanV2, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, id) + ret0, _ := ret[0].(*storage.VirtualMachineScanV2) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Get indicates an expected call of Get. +func (mr *MockDataStoreMockRecorder) Get(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDataStore)(nil).Get), ctx, id) +} + +// GetBatch mocks base method. +func (m *MockDataStore) GetBatch(ctx context.Context, ids []string) ([]*storage.VirtualMachineScanV2, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBatch", ctx, ids) + ret0, _ := ret[0].([]*storage.VirtualMachineScanV2) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBatch indicates an expected call of GetBatch. +func (mr *MockDataStoreMockRecorder) GetBatch(ctx, ids any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBatch", reflect.TypeOf((*MockDataStore)(nil).GetBatch), ctx, ids) +} + +// Search mocks base method. +func (m *MockDataStore) Search(ctx context.Context, q *v1.Query) ([]search.Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Search", ctx, q) + ret0, _ := ret[0].([]search.Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Search indicates an expected call of Search. +func (mr *MockDataStoreMockRecorder) Search(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockDataStore)(nil).Search), ctx, q) +} + +// SearchRawVMScans mocks base method. +func (m *MockDataStore) SearchRawVMScans(ctx context.Context, q *v1.Query) ([]*storage.VirtualMachineScanV2, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchRawVMScans", ctx, q) + ret0, _ := ret[0].([]*storage.VirtualMachineScanV2) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchRawVMScans indicates an expected call of SearchRawVMScans. +func (mr *MockDataStoreMockRecorder) SearchRawVMScans(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchRawVMScans", reflect.TypeOf((*MockDataStore)(nil).SearchRawVMScans), ctx, q) +} diff --git a/central/virtualmachine/scan/v2/datastore/singleton.go b/central/virtualmachine/scan/v2/datastore/singleton.go new file mode 100644 index 0000000000000..0bd1db7e48849 --- /dev/null +++ b/central/virtualmachine/scan/v2/datastore/singleton.go @@ -0,0 +1,28 @@ +package datastore + +import ( + "github.com/stackrox/rox/central/globaldb" + pgStore "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore/store/postgres" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/sync" +) + +var ( + once sync.Once + + ds DataStore +) + +func initialize() { + storage := pgStore.New(globaldb.GetPostgres()) + ds = New(storage) +} + +// Singleton returns a singleton instance of the VM scan v2 datastore. +func Singleton() DataStore { + if !features.VirtualMachinesEnhancedDataModel.Enabled() { + return nil + } + once.Do(initialize) + return ds +} diff --git a/central/virtualmachine/scan/v2/datastore/store/postgres/gen.go b/central/virtualmachine/scan/v2/datastore/store/postgres/gen.go new file mode 100644 index 0000000000000..dc75013246adc --- /dev/null +++ b/central/virtualmachine/scan/v2/datastore/store/postgres/gen.go @@ -0,0 +1,3 @@ +package postgres + +//go:generate pg-table-bindings-wrapper --type=storage.VirtualMachineScanV2 --search-category VIRTUAL_MACHINE_SCANS_V2 --no-copy-from --search-scope VIRTUAL_MACHINE_VULNERABILITIES_V2,VIRTUAL_MACHINE_COMPONENTS_V2,VIRTUAL_MACHINE_SCANS_V2,VIRTUAL_MACHINES_V2,NAMESPACES,CLUSTERS --references=storage.VirtualMachineV2 diff --git a/central/virtualmachine/scan/v2/datastore/store/postgres/store.go b/central/virtualmachine/scan/v2/datastore/store/postgres/store.go new file mode 100644 index 0000000000000..6a02b41bc71d3 --- /dev/null +++ b/central/virtualmachine/scan/v2/datastore/store/postgres/store.go @@ -0,0 +1,118 @@ +// Code generated by pg-bindings generator. DO NOT EDIT. + +package postgres + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5" + "github.com/stackrox/rox/central/metrics" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/logging" + ops "github.com/stackrox/rox/pkg/metrics" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/pgutils" + pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" + pgSearch "github.com/stackrox/rox/pkg/search/postgres" +) + +const ( + baseTable = "virtual_machine_scan_v2" + storeName = "VirtualMachineScanV2" +) + +var ( + log = logging.LoggerForModule() + schema = pkgSchema.VirtualMachineScanV2Schema + targetResource = resources.VirtualMachine +) + +type ( + storeType = storage.VirtualMachineScanV2 + callback = func(obj *storeType) error +) + +// Store is the interface to interact with the storage for storage.VirtualMachineScanV2 +type Store interface { + Upsert(ctx context.Context, obj *storeType) error + UpsertMany(ctx context.Context, objs []*storeType) error + Delete(ctx context.Context, id string) error + DeleteByQuery(ctx context.Context, q *v1.Query) error + DeleteByQueryWithIDs(ctx context.Context, q *v1.Query) ([]string, error) + DeleteMany(ctx context.Context, identifiers []string) error + PruneMany(ctx context.Context, identifiers []string) error + + Count(ctx context.Context, q *v1.Query) (int, error) + Exists(ctx context.Context, id string) (bool, error) + Search(ctx context.Context, q *v1.Query) ([]search.Result, error) + + Get(ctx context.Context, id string) (*storeType, bool, error) + // Deprecated: use GetByQueryFn instead + GetByQuery(ctx context.Context, query *v1.Query) ([]*storeType, error) + GetByQueryFn(ctx context.Context, query *v1.Query, fn callback) error + GetMany(ctx context.Context, identifiers []string) ([]*storeType, []int, error) + GetIDs(ctx context.Context) ([]string, error) + + Walk(ctx context.Context, fn callback) error + WalkByQuery(ctx context.Context, query *v1.Query, fn callback) error +} + +// New returns a new Store instance using the provided sql instance. +func New(db postgres.DB) Store { + return pgSearch.NewGloballyScopedGenericStore[storeType, *storeType]( + db, + schema, + pkGetter, + insertIntoVirtualMachineScanV2, + nil, + metricsSetAcquireDBConnDuration, + metricsSetPostgresOperationDurationTime, + targetResource, + nil, + nil, + ) +} + +// region Helper functions + +func pkGetter(obj *storeType) string { + return obj.GetId() +} + +func metricsSetPostgresOperationDurationTime(start time.Time, op ops.Op) { + metrics.SetPostgresOperationDurationTime(start, op, storeName) +} + +func metricsSetAcquireDBConnDuration(start time.Time, op ops.Op) { + metrics.SetAcquireDBConnDuration(start, op, storeName) +} + +func insertIntoVirtualMachineScanV2(batch *pgx.Batch, obj *storage.VirtualMachineScanV2) error { + + serialized, marshalErr := obj.MarshalVT() + if marshalErr != nil { + return marshalErr + } + + values := []interface{}{ + // parent primary keys start + pgutils.NilOrUUID(obj.GetId()), + pgutils.NilOrUUID(obj.GetVmV2Id()), + obj.GetScanOs(), + protocompat.NilOrTime(obj.GetScanTime()), + obj.GetTopCvss(), + serialized, + } + + finalStr := "INSERT INTO virtual_machine_scan_v2 (Id, VmV2Id, ScanOs, ScanTime, TopCvss, serialized) VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT(Id) DO UPDATE SET Id = EXCLUDED.Id, VmV2Id = EXCLUDED.VmV2Id, ScanOs = EXCLUDED.ScanOs, ScanTime = EXCLUDED.ScanTime, TopCvss = EXCLUDED.TopCvss, serialized = EXCLUDED.serialized" + batch.Queue(finalStr, values...) + + return nil +} + +// endregion Helper functions diff --git a/central/virtualmachine/scan/v2/datastore/store/postgres/store_test.go b/central/virtualmachine/scan/v2/datastore/store/postgres/store_test.go new file mode 100644 index 0000000000000..6eb7cac536f56 --- /dev/null +++ b/central/virtualmachine/scan/v2/datastore/store/postgres/store_test.go @@ -0,0 +1,55 @@ +// Code generated by pg-bindings generator. DO NOT EDIT. + +//go:build sql_integration + +package postgres + +import ( + "context" + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/testutils" + "github.com/stretchr/testify/suite" +) + +type VirtualMachineScanV2StoreSuite struct { + suite.Suite + store Store + testDB *pgtest.TestPostgres +} + +func TestVirtualMachineScanV2Store(t *testing.T) { + suite.Run(t, new(VirtualMachineScanV2StoreSuite)) +} + +func (s *VirtualMachineScanV2StoreSuite) SetupSuite() { + + s.testDB = pgtest.ForT(s.T()) + s.store = New(s.testDB.DB) +} + +func (s *VirtualMachineScanV2StoreSuite) SetupTest() { + ctx := sac.WithAllAccess(context.Background()) + tag, err := s.testDB.Exec(ctx, "TRUNCATE virtual_machine_scan_v2 CASCADE") + s.T().Log("virtual_machine_scan_v2", tag) + s.store = New(s.testDB.DB) + s.NoError(err) +} + +func (s *VirtualMachineScanV2StoreSuite) TestStore() { + ctx := sac.WithAllAccess(context.Background()) + + store := s.store + + virtualMachineScanV2 := &storage.VirtualMachineScanV2{} + s.NoError(testutils.FullInit(virtualMachineScanV2, testutils.SimpleInitializer(), testutils.JSONFieldsFilter)) + + foundVirtualMachineScanV2, exists, err := store.Get(ctx, virtualMachineScanV2.GetId()) + s.NoError(err) + s.False(exists) + s.Nil(foundVirtualMachineScanV2) + +} diff --git a/central/virtualmachine/service/service_impl.go b/central/virtualmachine/service/service_impl.go index 78c284ae2cbf1..6b883cda04dfe 100644 --- a/central/virtualmachine/service/service_impl.go +++ b/central/virtualmachine/service/service_impl.go @@ -15,6 +15,7 @@ import ( "github.com/stackrox/rox/pkg/sac/resources" "github.com/stackrox/rox/pkg/search" "github.com/stackrox/rox/pkg/search/paginated" + "github.com/stackrox/rox/pkg/uuid" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -58,6 +59,9 @@ func (s *serviceImpl) GetVirtualMachine(ctx context.Context, request *v2.GetVirt if request.GetId() == "" { return nil, status.Error(codes.InvalidArgument, "id must be specified") } + if _, err := uuid.FromString(request.GetId()); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "id must be a valid UUID (got: %q)", request.GetId()) + } vm, exists, err := s.datastore.GetVirtualMachine(ctx, request.GetId()) diff --git a/central/virtualmachine/service/service_impl_test.go b/central/virtualmachine/service/service_impl_test.go index 03bf10c66485d..e1311c1f8139c 100644 --- a/central/virtualmachine/service/service_impl_test.go +++ b/central/virtualmachine/service/service_impl_test.go @@ -9,6 +9,7 @@ import ( datastoreMocks "github.com/stackrox/rox/central/virtualmachine/datastore/mocks" v2 "github.com/stackrox/rox/generated/api/v2" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" "github.com/stackrox/rox/pkg/grpc/testutils" "github.com/stackrox/rox/pkg/protoassert" "github.com/stretchr/testify/assert" @@ -24,7 +25,7 @@ func TestGetVirtualMachine(t *testing.T) { ctx := context.Background() storedTestVM := &storage.VirtualMachine{ - Id: "test-vm-id", + Id: fixtureconsts.VirtualMachine1, Name: "test-vm", Namespace: "test-namespace", } @@ -41,11 +42,11 @@ func TestGetVirtualMachine(t *testing.T) { { name: "successful get", request: &v2.GetVirtualMachineRequest{ - Id: "test-vm-id", + Id: fixtureconsts.VirtualMachine1, }, setupMock: func(mockDS *datastoreMocks.MockDataStore) { mockDS.EXPECT(). - GetVirtualMachine(ctx, "test-vm-id"). + GetVirtualMachine(ctx, fixtureconsts.VirtualMachine1). Return(storedTestVM, true, nil) }, expectedResult: testVM, @@ -60,14 +61,23 @@ func TestGetVirtualMachine(t *testing.T) { expectedResult: nil, expectedError: "id must be specified", }, + { + name: "invalid UUID", + request: &v2.GetVirtualMachineRequest{ + Id: "not-a-uuid", + }, + setupMock: func(mockDS *datastoreMocks.MockDataStore) {}, + expectedResult: nil, + expectedError: "id must be a valid UUID", + }, { name: "virtual machine not found", request: &v2.GetVirtualMachineRequest{ - Id: "non-existent-vm", + Id: fixtureconsts.VirtualMachineFake, }, setupMock: func(mockDS *datastoreMocks.MockDataStore) { mockDS.EXPECT(). - GetVirtualMachine(ctx, "non-existent-vm"). + GetVirtualMachine(ctx, fixtureconsts.VirtualMachineFake). Return(nil, false, nil) }, expectedResult: nil, @@ -76,11 +86,11 @@ func TestGetVirtualMachine(t *testing.T) { { name: "datastore error", request: &v2.GetVirtualMachineRequest{ - Id: "test-vm-id", + Id: fixtureconsts.VirtualMachine1, }, setupMock: func(mockDS *datastoreMocks.MockDataStore) { mockDS.EXPECT(). - GetVirtualMachine(ctx, "test-vm-id"). + GetVirtualMachine(ctx, fixtureconsts.VirtualMachine1). Return(nil, false, errors.New("datastore error")) }, expectedResult: nil, diff --git a/central/virtualmachine/v2/datastore/datastore.go b/central/virtualmachine/v2/datastore/datastore.go new file mode 100644 index 0000000000000..296e00c821dfd --- /dev/null +++ b/central/virtualmachine/v2/datastore/datastore.go @@ -0,0 +1,52 @@ +package datastore + +import ( + "context" + + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/search" +) + +// DataStore is the public interface for the VM v2 datastore. +// +//go:generate mockgen-wrapper +type DataStore interface { + // CountVirtualMachines returns the number of VMs matching the query. + CountVirtualMachines(ctx context.Context, query *v1.Query) (int, error) + + // GetVirtualMachine returns the VM with the given ID. + GetVirtualMachine(ctx context.Context, id string) (*storage.VirtualMachineV2, bool, error) + // GetVirtualMachineWithLatestScan returns the VM and its latest scan together. + GetVirtualMachineWithLatestScan(ctx context.Context, id string) (*common.VMWithLatestScan, bool, error) + // GetManyVirtualMachines returns the VMs with the given IDs. + GetManyVirtualMachines(ctx context.Context, ids []string) ([]*storage.VirtualMachineV2, []int, error) + + // EnsureVirtualMachineExists inserts a minimal VM record if one does not + // already exist. This satisfies FK constraints for scan upserts without + // overwriting richer metadata from the VM pipeline. + EnsureVirtualMachineExists(ctx context.Context, vmID string, clusterID string) error + + // UpsertVirtualMachine upserts a VM. The store performs hash-based change + // detection to avoid unnecessary writes. + UpsertVirtualMachine(ctx context.Context, vm *storage.VirtualMachineV2) error + + // UpsertScan upserts scan data (scan, components, CVEs) for a VM. + // Hash-based change detection avoids unnecessary writes. CVE created_at + // timestamps are preserved across scan replacements. Also stamps + // last_agent_contact: this is the scrape-derived path. + UpsertScan(ctx context.Context, vmID string, parts common.VMScanParts) error + + // DeleteVirtualMachines removes VMs and all associated data (FK cascade). + DeleteVirtualMachines(ctx context.Context, ids ...string) error + + // Search returns search results matching the query. + Search(ctx context.Context, query *v1.Query) ([]search.Result, error) + + // SearchRawVirtualMachines returns VMs matching the query. + SearchRawVirtualMachines(ctx context.Context, query *v1.Query) ([]*storage.VirtualMachineV2, error) + + // Walk iterates over all VMs, calling fn for each. + Walk(ctx context.Context, fn func(vm *storage.VirtualMachineV2) error) error +} diff --git a/central/virtualmachine/v2/datastore/datastore_impl.go b/central/virtualmachine/v2/datastore/datastore_impl.go new file mode 100644 index 0000000000000..d3ef7fdff5509 --- /dev/null +++ b/central/virtualmachine/v2/datastore/datastore_impl.go @@ -0,0 +1,133 @@ +package datastore + +import ( + "context" + "math" + "time" + + "github.com/pkg/errors" + "github.com/stackrox/rox/central/metrics" + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store" + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/search" +) + +const ( + defaultPageSize = 100 + preAllocateCap = math.MaxUint16 +) + +type datastoreImpl struct { + store store.Store +} + +func newDatastoreImpl(store store.Store) DataStore { + return &datastoreImpl{ + store: store, + } +} + +func (ds *datastoreImpl) CountVirtualMachines(ctx context.Context, query *v1.Query) (int, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "CountVirtualMachines") + if query == nil { + query = search.EmptyQuery() + } + return ds.store.Count(ctx, query) +} + +func (ds *datastoreImpl) GetVirtualMachine(ctx context.Context, id string) (*storage.VirtualMachineV2, bool, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "GetVirtualMachine") + return ds.store.Get(ctx, id) +} + +func (ds *datastoreImpl) GetManyVirtualMachines(ctx context.Context, ids []string) ([]*storage.VirtualMachineV2, []int, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "GetManyVirtualMachines") + return ds.store.GetMany(ctx, ids) +} + +func (ds *datastoreImpl) GetVirtualMachineWithLatestScan(ctx context.Context, id string) (*common.VMWithLatestScan, bool, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "GetVirtualMachineWithLatestScan") + return ds.store.GetWithLatestScan(ctx, id) +} + +func (ds *datastoreImpl) EnsureVirtualMachineExists(ctx context.Context, vmID string, clusterID string) error { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "EnsureVirtualMachineExists") + + if vmID == "" { + return errors.New("cannot ensure VM exists without an id") + } + if clusterID == "" { + return errors.New("cannot ensure VM exists without a cluster id") + } + + return ds.store.EnsureExists(ctx, vmID, clusterID) +} + +func (ds *datastoreImpl) UpsertVirtualMachine(ctx context.Context, vm *storage.VirtualMachineV2) error { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "UpsertVirtualMachine") + + if vm.GetId() == "" { + return errors.New("cannot upsert a virtual machine without an id") + } + return ds.store.UpsertVM(ctx, vm) +} + +func (ds *datastoreImpl) UpsertScan(ctx context.Context, vmID string, parts common.VMScanParts) error { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "UpsertScan") + + if vmID == "" { + return errors.New("cannot upsert scan without a VM id") + } + return ds.store.UpsertScan(ctx, vmID, parts) +} + +func (ds *datastoreImpl) DeleteVirtualMachines(ctx context.Context, ids ...string) error { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "DeleteVirtualMachines") + if len(ids) == 1 { + return ds.store.Delete(ctx, ids[0]) + } + return ds.store.DeleteMany(ctx, ids) +} + +func (ds *datastoreImpl) Search(ctx context.Context, query *v1.Query) ([]search.Result, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "Search") + return ds.store.Search(ctx, query) +} + +func (ds *datastoreImpl) SearchRawVirtualMachines(ctx context.Context, query *v1.Query) ([]*storage.VirtualMachineV2, error) { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "SearchRawVirtualMachines") + + searchQuery := query.CloneVT() + if len(searchQuery.GetPagination().GetSortOptions()) == 0 { + if searchQuery.GetPagination() == nil { + searchQuery.Pagination = &v1.QueryPagination{} + } + searchQuery.Pagination.SortOptions = []*v1.QuerySortOption{ + {Field: search.VirtualMachineName.String()}, + {Field: search.Namespace.String()}, + } + } + pageSize := searchQuery.GetPagination().GetLimit() + if pageSize <= 0 { + pageSize = defaultPageSize + } + if pageSize > preAllocateCap { + pageSize = preAllocateCap + } + results := make([]*storage.VirtualMachineV2, 0, pageSize) + err := ds.store.WalkByQuery(ctx, searchQuery, func(vm *storage.VirtualMachineV2) error { + results = append(results, vm) + return nil + }) + if err != nil { + return nil, err + } + return results, nil +} + +func (ds *datastoreImpl) Walk(ctx context.Context, fn func(vm *storage.VirtualMachineV2) error) error { + defer metrics.SetDatastoreFunctionDuration(time.Now(), "VirtualMachineV2", "Walk") + return ds.store.Walk(ctx, fn) +} diff --git a/central/virtualmachine/v2/datastore/datastore_impl_test.go b/central/virtualmachine/v2/datastore/datastore_impl_test.go new file mode 100644 index 0000000000000..c813186872dc8 --- /dev/null +++ b/central/virtualmachine/v2/datastore/datastore_impl_test.go @@ -0,0 +1,448 @@ +//go:build sql_integration + +package datastore + +import ( + "context" + "fmt" + "testing" + + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" + pgStore "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/postgres" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/sync" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/suite" +) + +func TestVirtualMachineV2DataStore(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + if !features.VirtualMachinesEnhancedDataModel.Enabled() { + t.Skip("VM enhanced data model is not enabled") + } + suite.Run(t, new(VirtualMachineV2DataStoreTestSuite)) +} + +type VirtualMachineV2DataStoreTestSuite struct { + suite.Suite + + testDB *pgtest.TestPostgres + datastore DataStore + ctx context.Context + noSacCtx context.Context +} + +func (s *VirtualMachineV2DataStoreTestSuite) SetupSuite() { + s.testDB = pgtest.ForT(s.T()) + s.ctx = sac.WithGlobalAccessScopeChecker(context.Background(), + sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS, storage.Access_READ_WRITE_ACCESS), + sac.ResourceScopeKeys(resources.VirtualMachine))) + s.noSacCtx = sac.WithGlobalAccessScopeChecker(context.Background(), + sac.DenyAllAccessScopeChecker()) +} + +func (s *VirtualMachineV2DataStoreTestSuite) SetupTest() { + _, err := s.testDB.Exec(s.ctx, "TRUNCATE virtual_machine_v2 CASCADE") + s.Require().NoError(err) + + store := pgStore.New(s.testDB.DB, concurrency.NewKeyFence()) + s.datastore = newDatastoreImpl(store) +} + +func (s *VirtualMachineV2DataStoreTestSuite) TearDownSuite() { + s.testDB.Close() +} + +func createTestVM(i int) *storage.VirtualMachineV2 { + return &storage.VirtualMachineV2{ + Id: uuid.NewTestUUID(i).String(), + Name: fmt.Sprintf("test-vm-%d", i), + Namespace: "default", + ClusterId: uuid.NewV5FromNonUUIDs("cluster", fmt.Sprintf("%d", i)).String(), + ClusterName: "test-cluster", + GuestOs: "rhel9", + State: storage.VirtualMachineV2_RUNNING, + Notes: []storage.VirtualMachineV2_Note{storage.VirtualMachineV2_MISSING_SCAN_DATA}, + VsockCid: int32(42 + i), + } +} + +func createTestScanParts(vmID string) common.VMScanParts { + scanID := uuid.NewV4().String() + compID := uuid.NewV4().String() + cveID := uuid.NewV4().String() + + return common.VMScanParts{ + Scan: &storage.VirtualMachineScanV2{ + Id: scanID, + VmV2Id: vmID, + ScanOs: "rhel9", + }, + Components: []*storage.VirtualMachineComponentV2{ + { + Id: compID, + VmScanId: scanID, + Name: "openssl", + Version: "1.1.1", + Source: storage.SourceType_OS, + OperatingSystem: "rhel:9", + }, + }, + CVEs: []*storage.VirtualMachineCVEV2{ + { + Id: cveID, + VmV2Id: vmID, + VmComponentId: compID, + CveBaseInfo: &storage.CVEInfo{ + Cve: "CVE-2024-0001", + Summary: "test vulnerability", + }, + PreferredCvss: 7.5, + Severity: storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, + IsFixable: true, + HasFixedBy: &storage.VirtualMachineCVEV2_FixedBy{FixedBy: "1.1.2"}, + }, + }, + } +} + +// region Count tests + +func (s *VirtualMachineV2DataStoreTestSuite) TestCountVirtualMachines() { + // Nil query returns zero for empty store. + count, err := s.datastore.CountVirtualMachines(s.ctx, nil) + s.NoError(err) + s.Equal(0, count) + + // Empty query returns zero for empty store. + count, err = s.datastore.CountVirtualMachines(s.ctx, search.EmptyQuery()) + s.NoError(err) + s.Equal(0, count) + + // After upsert, count reflects the new VM. + vm := createTestVM(1) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm)) + + count, err = s.datastore.CountVirtualMachines(s.ctx, nil) + s.NoError(err) + s.Equal(1, count) + + // SAC denied returns 0. + count, err = s.datastore.CountVirtualMachines(s.noSacCtx, nil) + s.NoError(err) + s.Equal(0, count) +} + +// endregion Count tests + +// region Get tests + +func (s *VirtualMachineV2DataStoreTestSuite) TestGetVirtualMachine() { + vm := createTestVM(1) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm)) + + // Get existing VM. + got, found, err := s.datastore.GetVirtualMachine(s.ctx, vm.GetId()) + s.NoError(err) + s.True(found) + s.Equal(vm.GetName(), got.GetName()) + + // Get missing VM. + _, found, err = s.datastore.GetVirtualMachine(s.ctx, uuid.NewTestUUID(99).String()) + s.NoError(err) + s.False(found) + + // SAC denied. + _, found, err = s.datastore.GetVirtualMachine(s.noSacCtx, vm.GetId()) + s.NoError(err) + s.False(found) +} + +func (s *VirtualMachineV2DataStoreTestSuite) TestGetManyVirtualMachines() { + vm1 := createTestVM(1) + vm2 := createTestVM(2) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm1)) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm2)) + + nonExistentID := uuid.NewTestUUID(99).String() + + // Batch get returns found VMs and missing indices. + results, missing, err := s.datastore.GetManyVirtualMachines(s.ctx, []string{vm1.GetId(), nonExistentID, vm2.GetId()}) + s.NoError(err) + s.Len(results, 2) + s.Equal([]int{1}, missing) + + // SAC denied returns nothing. + results, _, err = s.datastore.GetManyVirtualMachines(s.noSacCtx, []string{vm1.GetId()}) + s.NoError(err) + s.Empty(results) +} + +// endregion Get tests + +// region Upsert tests + +func (s *VirtualMachineV2DataStoreTestSuite) TestUpsertVirtualMachine() { + vm := createTestVM(1) + + // Upsert new VM. + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm)) + + got, found, err := s.datastore.GetVirtualMachine(s.ctx, vm.GetId()) + s.NoError(err) + s.True(found) + s.Equal(vm.GetName(), got.GetName()) + + // Upsert update. + vm.Name = "updated-name" + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm)) + + got, found, err = s.datastore.GetVirtualMachine(s.ctx, vm.GetId()) + s.NoError(err) + s.True(found) + s.Equal("updated-name", got.GetName()) + + // Empty ID rejected. + err = s.datastore.UpsertVirtualMachine(s.ctx, &storage.VirtualMachineV2{Name: "no-id"}) + s.Error(err) + s.Contains(err.Error(), "cannot upsert a virtual machine without an id") +} + +func (s *VirtualMachineV2DataStoreTestSuite) TestUpsertScan() { + vm := createTestVM(1) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm)) + + parts := createTestScanParts(vm.GetId()) + s.NoError(s.datastore.UpsertScan(s.ctx, vm.GetId(), parts)) + + // Verify VM still accessible after scan upsert. + got, found, err := s.datastore.GetVirtualMachine(s.ctx, vm.GetId()) + s.NoError(err) + s.True(found) + s.Equal(vm.GetName(), got.GetName()) + + // Empty VM ID rejected. + err = s.datastore.UpsertScan(s.ctx, "", parts) + s.Error(err) + s.Contains(err.Error(), "cannot upsert scan without a VM id") +} + +func (s *VirtualMachineV2DataStoreTestSuite) TestEnsureVirtualMachineExists() { + vmID := uuid.NewTestUUID(1).String() + clusterID := uuid.NewV5FromNonUUIDs("cluster", "1").String() + + // Creates a new minimal VM when it doesn't exist. + s.NoError(s.datastore.EnsureVirtualMachineExists(s.ctx, vmID, clusterID)) + + got, found, err := s.datastore.GetVirtualMachine(s.ctx, vmID) + s.NoError(err) + s.True(found) + s.Equal(vmID, got.GetId()) + s.Equal(clusterID, got.GetClusterId()) + s.Empty(got.GetName(), "minimal VM should have no name") + + // Does NOT overwrite richer metadata on subsequent calls. + richVM := createTestVM(1) + richVM.Id = vmID + richVM.ClusterId = clusterID + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, richVM)) + + s.NoError(s.datastore.EnsureVirtualMachineExists(s.ctx, vmID, clusterID)) + + got, found, err = s.datastore.GetVirtualMachine(s.ctx, vmID) + s.NoError(err) + s.True(found) + s.Equal(richVM.GetName(), got.GetName(), "richer metadata must not be overwritten") + + // Empty vmID returns an error. + err = s.datastore.EnsureVirtualMachineExists(s.ctx, "", clusterID) + s.Error(err) + s.Contains(err.Error(), "cannot ensure VM exists without an id") +} + +// endregion Upsert tests + +// region Delete tests + +func (s *VirtualMachineV2DataStoreTestSuite) TestDeleteVirtualMachines() { + vm1 := createTestVM(1) + vm2 := createTestVM(2) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm1)) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm2)) + + // Delete existing VM. + s.NoError(s.datastore.DeleteVirtualMachines(s.ctx, vm1.GetId())) + + _, found, err := s.datastore.GetVirtualMachine(s.ctx, vm1.GetId()) + s.NoError(err) + s.False(found) + + // Delete missing VM does not error. + s.NoError(s.datastore.DeleteVirtualMachines(s.ctx, uuid.NewTestUUID(99).String())) + + // Delete with cascade: VM with scan data. + parts := createTestScanParts(vm2.GetId()) + s.NoError(s.datastore.UpsertScan(s.ctx, vm2.GetId(), parts)) + + s.NoError(s.datastore.DeleteVirtualMachines(s.ctx, vm2.GetId())) + + _, found, err = s.datastore.GetVirtualMachine(s.ctx, vm2.GetId()) + s.NoError(err) + s.False(found) +} + +// endregion Delete tests + +// region Search tests + +func (s *VirtualMachineV2DataStoreTestSuite) TestSearch() { + vm := createTestVM(1) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm)) + + results, err := s.datastore.Search(s.ctx, search.EmptyQuery()) + s.NoError(err) + s.Len(results, 1) + + // Query filtering. + results, err = s.datastore.Search(s.ctx, search.NewQueryBuilder(). + AddExactMatches(search.VirtualMachineName, "nonexistent").ProtoQuery()) + s.NoError(err) + s.Empty(results) +} + +func (s *VirtualMachineV2DataStoreTestSuite) TestSearchRawVirtualMachines() { + // Insert VMs with distinct names to verify default sort order. + for i := 1; i <= 3; i++ { + vm := createTestVM(i) + vm.Namespace = fmt.Sprintf("ns-%d", 4-i) // reverse namespace order + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm)) + } + + // Default sort: by name, then namespace. + vms, err := s.datastore.SearchRawVirtualMachines(s.ctx, search.EmptyQuery()) + s.NoError(err) + s.Len(vms, 3) + + // Verify name ordering (test-vm-1, test-vm-2, test-vm-3). + for i, vm := range vms { + s.Equal(fmt.Sprintf("test-vm-%d", i+1), vm.GetName()) + } + + // Pagination: limit to 2. + query := search.EmptyQuery() + query.Pagination = &v1.QueryPagination{Limit: 2} + vms, err = s.datastore.SearchRawVirtualMachines(s.ctx, query) + s.NoError(err) + s.Len(vms, 2) +} + +// endregion Search tests + +// region Walk tests + +func (s *VirtualMachineV2DataStoreTestSuite) TestWalk() { + for i := 1; i <= 3; i++ { + vm := createTestVM(i) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm)) + } + + var walked []*storage.VirtualMachineV2 + err := s.datastore.Walk(s.ctx, func(vm *storage.VirtualMachineV2) error { + walked = append(walked, vm) + return nil + }) + s.NoError(err) + s.Len(walked, 3) +} + +// endregion Walk tests + +// region Concurrency tests + +func (s *VirtualMachineV2DataStoreTestSuite) TestConcurrentUpsertVMAndUpsertScan() { + vm := createTestVM(1) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm)) + + const goroutines = 10 + var wg sync.WaitGroup + wg.Add(goroutines) + errs := make([]error, goroutines) + + for i := range goroutines { + go func(idx int) { + defer wg.Done() + if idx%2 == 0 { + // Simulate VM pipeline: upsert full VM metadata. + vmCopy := createTestVM(1) + vmCopy.Id = vm.GetId() + vmCopy.ClusterId = vm.GetClusterId() + vmCopy.Name = fmt.Sprintf("updated-vm-%d", idx) + errs[idx] = s.datastore.UpsertVirtualMachine(s.ctx, vmCopy) + } else { + // Simulate scan-index pipeline: ensure + upsert scan. + if err := s.datastore.EnsureVirtualMachineExists(s.ctx, vm.GetId(), vm.GetClusterId()); err != nil { + errs[idx] = err + return + } + parts := createTestScanParts(vm.GetId()) + errs[idx] = s.datastore.UpsertScan(s.ctx, vm.GetId(), parts) + } + }(i) + } + wg.Wait() + + for i, err := range errs { + s.NoErrorf(err, "goroutine %d failed", i) + } + + // VM must exist with non-empty metadata (not clobbered to minimal). + got, found, err := s.datastore.GetVirtualMachine(s.ctx, vm.GetId()) + s.NoError(err) + s.True(found) + s.NotEmpty(got.GetName(), "VM name must not be empty after concurrent operations") + s.Equal(vm.GetClusterId(), got.GetClusterId()) +} + +func (s *VirtualMachineV2DataStoreTestSuite) TestConcurrentUpsertScanAndDelete() { + vm := createTestVM(1) + s.NoError(s.datastore.UpsertVirtualMachine(s.ctx, vm)) + + parts := createTestScanParts(vm.GetId()) + s.NoError(s.datastore.UpsertScan(s.ctx, vm.GetId(), parts)) + + const goroutines = 6 + var wg sync.WaitGroup + wg.Add(goroutines) + + for i := range goroutines { + go func(idx int) { + defer wg.Done() + if idx%2 == 0 { + // Re-create the VM and upsert a scan. + _ = s.datastore.UpsertVirtualMachine(s.ctx, vm) + scanParts := createTestScanParts(vm.GetId()) + _ = s.datastore.UpsertScan(s.ctx, vm.GetId(), scanParts) + } else { + _ = s.datastore.DeleteVirtualMachines(s.ctx, vm.GetId()) + } + }(i) + } + wg.Wait() + + // Final state: VM either exists with valid data or is fully deleted. + got, found, err := s.datastore.GetVirtualMachine(s.ctx, vm.GetId()) + s.NoError(err) + if found { + s.Equal(vm.GetId(), got.GetId()) + } +} + +// endregion Concurrency tests diff --git a/central/virtualmachine/v2/datastore/datastore_test_constructors.go b/central/virtualmachine/v2/datastore/datastore_test_constructors.go new file mode 100644 index 0000000000000..5aef488adee23 --- /dev/null +++ b/central/virtualmachine/v2/datastore/datastore_test_constructors.go @@ -0,0 +1,14 @@ +package datastore + +import ( + "testing" + + pgStore "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/postgres" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/postgres" +) + +// GetTestPostgresDataStore provides a datastore connected to postgres for testing purposes. +func GetTestPostgresDataStore(_ testing.TB, pool postgres.DB) DataStore { + return newDatastoreImpl(pgStore.New(pool, concurrency.NewKeyFence())) +} diff --git a/central/virtualmachine/v2/datastore/mocks/datastore.go b/central/virtualmachine/v2/datastore/mocks/datastore.go new file mode 100644 index 0000000000000..d2bddff113c49 --- /dev/null +++ b/central/virtualmachine/v2/datastore/mocks/datastore.go @@ -0,0 +1,213 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: datastore.go +// +// Generated by this command: +// +// mockgen -package mocks -destination mocks/datastore.go -source datastore.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + common "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" + v1 "github.com/stackrox/rox/generated/api/v1" + storage "github.com/stackrox/rox/generated/storage" + search "github.com/stackrox/rox/pkg/search" + gomock "go.uber.org/mock/gomock" +) + +// MockDataStore is a mock of DataStore interface. +type MockDataStore struct { + ctrl *gomock.Controller + recorder *MockDataStoreMockRecorder + isgomock struct{} +} + +// MockDataStoreMockRecorder is the mock recorder for MockDataStore. +type MockDataStoreMockRecorder struct { + mock *MockDataStore +} + +// NewMockDataStore creates a new mock instance. +func NewMockDataStore(ctrl *gomock.Controller) *MockDataStore { + mock := &MockDataStore{ctrl: ctrl} + mock.recorder = &MockDataStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataStore) EXPECT() *MockDataStoreMockRecorder { + return m.recorder +} + +// CountVirtualMachines mocks base method. +func (m *MockDataStore) CountVirtualMachines(ctx context.Context, query *v1.Query) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountVirtualMachines", ctx, query) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountVirtualMachines indicates an expected call of CountVirtualMachines. +func (mr *MockDataStoreMockRecorder) CountVirtualMachines(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountVirtualMachines", reflect.TypeOf((*MockDataStore)(nil).CountVirtualMachines), ctx, query) +} + +// DeleteVirtualMachines mocks base method. +func (m *MockDataStore) DeleteVirtualMachines(ctx context.Context, ids ...string) error { + m.ctrl.T.Helper() + varargs := []any{ctx} + for _, a := range ids { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteVirtualMachines", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteVirtualMachines indicates an expected call of DeleteVirtualMachines. +func (mr *MockDataStoreMockRecorder) DeleteVirtualMachines(ctx any, ids ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx}, ids...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVirtualMachines", reflect.TypeOf((*MockDataStore)(nil).DeleteVirtualMachines), varargs...) +} + +// EnsureVirtualMachineExists mocks base method. +func (m *MockDataStore) EnsureVirtualMachineExists(ctx context.Context, vmID, clusterID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureVirtualMachineExists", ctx, vmID, clusterID) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureVirtualMachineExists indicates an expected call of EnsureVirtualMachineExists. +func (mr *MockDataStoreMockRecorder) EnsureVirtualMachineExists(ctx, vmID, clusterID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureVirtualMachineExists", reflect.TypeOf((*MockDataStore)(nil).EnsureVirtualMachineExists), ctx, vmID, clusterID) +} + +// GetManyVirtualMachines mocks base method. +func (m *MockDataStore) GetManyVirtualMachines(ctx context.Context, ids []string) ([]*storage.VirtualMachineV2, []int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetManyVirtualMachines", ctx, ids) + ret0, _ := ret[0].([]*storage.VirtualMachineV2) + ret1, _ := ret[1].([]int) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetManyVirtualMachines indicates an expected call of GetManyVirtualMachines. +func (mr *MockDataStoreMockRecorder) GetManyVirtualMachines(ctx, ids any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetManyVirtualMachines", reflect.TypeOf((*MockDataStore)(nil).GetManyVirtualMachines), ctx, ids) +} + +// GetVirtualMachine mocks base method. +func (m *MockDataStore) GetVirtualMachine(ctx context.Context, id string) (*storage.VirtualMachineV2, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVirtualMachine", ctx, id) + ret0, _ := ret[0].(*storage.VirtualMachineV2) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetVirtualMachine indicates an expected call of GetVirtualMachine. +func (mr *MockDataStoreMockRecorder) GetVirtualMachine(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVirtualMachine", reflect.TypeOf((*MockDataStore)(nil).GetVirtualMachine), ctx, id) +} + +// GetVirtualMachineWithLatestScan mocks base method. +func (m *MockDataStore) GetVirtualMachineWithLatestScan(ctx context.Context, id string) (*common.VMWithLatestScan, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVirtualMachineWithLatestScan", ctx, id) + ret0, _ := ret[0].(*common.VMWithLatestScan) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetVirtualMachineWithLatestScan indicates an expected call of GetVirtualMachineWithLatestScan. +func (mr *MockDataStoreMockRecorder) GetVirtualMachineWithLatestScan(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVirtualMachineWithLatestScan", reflect.TypeOf((*MockDataStore)(nil).GetVirtualMachineWithLatestScan), ctx, id) +} + +// Search mocks base method. +func (m *MockDataStore) Search(ctx context.Context, query *v1.Query) ([]search.Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Search", ctx, query) + ret0, _ := ret[0].([]search.Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Search indicates an expected call of Search. +func (mr *MockDataStoreMockRecorder) Search(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockDataStore)(nil).Search), ctx, query) +} + +// SearchRawVirtualMachines mocks base method. +func (m *MockDataStore) SearchRawVirtualMachines(ctx context.Context, query *v1.Query) ([]*storage.VirtualMachineV2, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SearchRawVirtualMachines", ctx, query) + ret0, _ := ret[0].([]*storage.VirtualMachineV2) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SearchRawVirtualMachines indicates an expected call of SearchRawVirtualMachines. +func (mr *MockDataStoreMockRecorder) SearchRawVirtualMachines(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchRawVirtualMachines", reflect.TypeOf((*MockDataStore)(nil).SearchRawVirtualMachines), ctx, query) +} + +// UpsertScan mocks base method. +func (m *MockDataStore) UpsertScan(ctx context.Context, vmID string, parts common.VMScanParts) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertScan", ctx, vmID, parts) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertScan indicates an expected call of UpsertScan. +func (mr *MockDataStoreMockRecorder) UpsertScan(ctx, vmID, parts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertScan", reflect.TypeOf((*MockDataStore)(nil).UpsertScan), ctx, vmID, parts) +} + +// UpsertVirtualMachine mocks base method. +func (m *MockDataStore) UpsertVirtualMachine(ctx context.Context, vm *storage.VirtualMachineV2) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertVirtualMachine", ctx, vm) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertVirtualMachine indicates an expected call of UpsertVirtualMachine. +func (mr *MockDataStoreMockRecorder) UpsertVirtualMachine(ctx, vm any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertVirtualMachine", reflect.TypeOf((*MockDataStore)(nil).UpsertVirtualMachine), ctx, vm) +} + +// Walk mocks base method. +func (m *MockDataStore) Walk(ctx context.Context, fn func(*storage.VirtualMachineV2) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Walk", ctx, fn) + ret0, _ := ret[0].(error) + return ret0 +} + +// Walk indicates an expected call of Walk. +func (mr *MockDataStoreMockRecorder) Walk(ctx, fn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Walk", reflect.TypeOf((*MockDataStore)(nil).Walk), ctx, fn) +} diff --git a/central/virtualmachine/v2/datastore/singleton.go b/central/virtualmachine/v2/datastore/singleton.go new file mode 100644 index 0000000000000..50b74f902e9b2 --- /dev/null +++ b/central/virtualmachine/v2/datastore/singleton.go @@ -0,0 +1,29 @@ +package datastore + +import ( + "github.com/stackrox/rox/central/globaldb" + pgStore "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/postgres" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/sync" +) + +var ( + once sync.Once + + ds DataStore +) + +func initialize() { + store := pgStore.New(globaldb.GetPostgres(), concurrency.NewKeyFence()) + ds = newDatastoreImpl(store) +} + +// Singleton provides the interface for non-service external interaction. +func Singleton() DataStore { + if !features.VirtualMachinesEnhancedDataModel.Enabled() { + return nil + } + once.Do(initialize) + return ds +} diff --git a/central/virtualmachine/v2/datastore/store/common/parts.go b/central/virtualmachine/v2/datastore/store/common/parts.go new file mode 100644 index 0000000000000..f36d2761b65fd --- /dev/null +++ b/central/virtualmachine/v2/datastore/store/common/parts.go @@ -0,0 +1,23 @@ +package common + +import "github.com/stackrox/rox/generated/storage" + +// VMWithLatestScan contains VM metadata and its latest scan from one database snapshot. +// Scan is nil when the VM has not been scanned. +type VMWithLatestScan struct { + VM *storage.VirtualMachineV2 + Scan *storage.VirtualMachineScanV2 +} + +// VMScanParts groups the scan, components, and CVEs that together represent +// a single VM scan result. UpsertScan consumes this structure. +type VMScanParts struct { + Scan *storage.VirtualMachineScanV2 + Components []*storage.VirtualMachineComponentV2 + CVEs []*storage.VirtualMachineCVEV2 + // SourceComponents holds the original v1 scan components used for hash + // computation. These are the pre-split representations that contain + // embedded vulnerabilities, allowing direct hashing without intermediate + // structs. + SourceComponents []*storage.EmbeddedVirtualMachineScanComponent +} diff --git a/central/virtualmachine/v2/datastore/store/mocks/store.go b/central/virtualmachine/v2/datastore/store/mocks/store.go new file mode 100644 index 0000000000000..8230b9540a9de --- /dev/null +++ b/central/virtualmachine/v2/datastore/store/mocks/store.go @@ -0,0 +1,221 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: store.go +// +// Generated by this command: +// +// mockgen -package mocks -destination mocks/store.go -source store.go +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + common "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" + v1 "github.com/stackrox/rox/generated/api/v1" + storage "github.com/stackrox/rox/generated/storage" + search "github.com/stackrox/rox/pkg/search" + gomock "go.uber.org/mock/gomock" +) + +// MockStore is a mock of Store interface. +type MockStore struct { + ctrl *gomock.Controller + recorder *MockStoreMockRecorder + isgomock struct{} +} + +// MockStoreMockRecorder is the mock recorder for MockStore. +type MockStoreMockRecorder struct { + mock *MockStore +} + +// NewMockStore creates a new mock instance. +func NewMockStore(ctrl *gomock.Controller) *MockStore { + mock := &MockStore{ctrl: ctrl} + mock.recorder = &MockStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStore) EXPECT() *MockStoreMockRecorder { + return m.recorder +} + +// Count mocks base method. +func (m *MockStore) Count(ctx context.Context, q *v1.Query) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Count", ctx, q) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Count indicates an expected call of Count. +func (mr *MockStoreMockRecorder) Count(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockStore)(nil).Count), ctx, q) +} + +// Delete mocks base method. +func (m *MockStore) Delete(ctx context.Context, id string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, id) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockStoreMockRecorder) Delete(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockStore)(nil).Delete), ctx, id) +} + +// DeleteMany mocks base method. +func (m *MockStore) DeleteMany(ctx context.Context, ids []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteMany", ctx, ids) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteMany indicates an expected call of DeleteMany. +func (mr *MockStoreMockRecorder) DeleteMany(ctx, ids any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteMany", reflect.TypeOf((*MockStore)(nil).DeleteMany), ctx, ids) +} + +// EnsureExists mocks base method. +func (m *MockStore) EnsureExists(ctx context.Context, vmID, clusterID string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureExists", ctx, vmID, clusterID) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureExists indicates an expected call of EnsureExists. +func (mr *MockStoreMockRecorder) EnsureExists(ctx, vmID, clusterID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureExists", reflect.TypeOf((*MockStore)(nil).EnsureExists), ctx, vmID, clusterID) +} + +// Get mocks base method. +func (m *MockStore) Get(ctx context.Context, id string) (*storage.VirtualMachineV2, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, id) + ret0, _ := ret[0].(*storage.VirtualMachineV2) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Get indicates an expected call of Get. +func (mr *MockStoreMockRecorder) Get(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockStore)(nil).Get), ctx, id) +} + +// GetMany mocks base method. +func (m *MockStore) GetMany(ctx context.Context, ids []string) ([]*storage.VirtualMachineV2, []int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMany", ctx, ids) + ret0, _ := ret[0].([]*storage.VirtualMachineV2) + ret1, _ := ret[1].([]int) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetMany indicates an expected call of GetMany. +func (mr *MockStoreMockRecorder) GetMany(ctx, ids any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMany", reflect.TypeOf((*MockStore)(nil).GetMany), ctx, ids) +} + +// GetWithLatestScan mocks base method. +func (m *MockStore) GetWithLatestScan(ctx context.Context, id string) (*common.VMWithLatestScan, bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetWithLatestScan", ctx, id) + ret0, _ := ret[0].(*common.VMWithLatestScan) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetWithLatestScan indicates an expected call of GetWithLatestScan. +func (mr *MockStoreMockRecorder) GetWithLatestScan(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWithLatestScan", reflect.TypeOf((*MockStore)(nil).GetWithLatestScan), ctx, id) +} + +// Search mocks base method. +func (m *MockStore) Search(ctx context.Context, q *v1.Query) ([]search.Result, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Search", ctx, q) + ret0, _ := ret[0].([]search.Result) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Search indicates an expected call of Search. +func (mr *MockStoreMockRecorder) Search(ctx, q any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockStore)(nil).Search), ctx, q) +} + +// UpsertScan mocks base method. +func (m *MockStore) UpsertScan(ctx context.Context, vmID string, parts common.VMScanParts) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertScan", ctx, vmID, parts) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertScan indicates an expected call of UpsertScan. +func (mr *MockStoreMockRecorder) UpsertScan(ctx, vmID, parts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertScan", reflect.TypeOf((*MockStore)(nil).UpsertScan), ctx, vmID, parts) +} + +// UpsertVM mocks base method. +func (m *MockStore) UpsertVM(ctx context.Context, vm *storage.VirtualMachineV2) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertVM", ctx, vm) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertVM indicates an expected call of UpsertVM. +func (mr *MockStoreMockRecorder) UpsertVM(ctx, vm any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertVM", reflect.TypeOf((*MockStore)(nil).UpsertVM), ctx, vm) +} + +// Walk mocks base method. +func (m *MockStore) Walk(ctx context.Context, fn func(*storage.VirtualMachineV2) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Walk", ctx, fn) + ret0, _ := ret[0].(error) + return ret0 +} + +// Walk indicates an expected call of Walk. +func (mr *MockStoreMockRecorder) Walk(ctx, fn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Walk", reflect.TypeOf((*MockStore)(nil).Walk), ctx, fn) +} + +// WalkByQuery mocks base method. +func (m *MockStore) WalkByQuery(ctx context.Context, q *v1.Query, fn func(*storage.VirtualMachineV2) error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WalkByQuery", ctx, q, fn) + ret0, _ := ret[0].(error) + return ret0 +} + +// WalkByQuery indicates an expected call of WalkByQuery. +func (mr *MockStoreMockRecorder) WalkByQuery(ctx, q, fn any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WalkByQuery", reflect.TypeOf((*MockStore)(nil).WalkByQuery), ctx, q, fn) +} diff --git a/central/virtualmachine/v2/datastore/store/postgres/gen.go b/central/virtualmachine/v2/datastore/store/postgres/gen.go new file mode 100644 index 0000000000000..362af519d3fdc --- /dev/null +++ b/central/virtualmachine/v2/datastore/store/postgres/gen.go @@ -0,0 +1,6 @@ +package postgres + +// NOTE: This store uses a custom implementation (store.go) instead of the +// auto-generated one. The go:generate directive below only generates the +// schema, not the store. +//go:generate pg-table-bindings-wrapper --type=storage.VirtualMachineV2 --search-category VIRTUAL_MACHINES_V2 --schema-only --search-scope VIRTUAL_MACHINE_VULNERABILITIES_V2,VIRTUAL_MACHINE_COMPONENTS_V2,VIRTUAL_MACHINE_SCANS_V2,VIRTUAL_MACHINES_V2,NAMESPACES,CLUSTERS diff --git a/central/virtualmachine/v2/datastore/store/postgres/get_vm_bench_test.go b/central/virtualmachine/v2/datastore/store/postgres/get_vm_bench_test.go new file mode 100644 index 0000000000000..af78f48867913 --- /dev/null +++ b/central/virtualmachine/v2/datastore/store/postgres/get_vm_bench_test.go @@ -0,0 +1,64 @@ +//go:build sql_integration + +package postgres + +import ( + "context" + "fmt" + "testing" + + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// BenchmarkGetVM measures the joined VM and latest-scan lookup. +// Only VM metadata and scans are seeded; the lookup does not read components or CVEs. +func BenchmarkGetVM(b *testing.B) { + b.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + ctx := sac.WithAllAccess(context.Background()) + testDB := pgtest.ForT(b) + vmStore := New(testDB.DB, concurrency.NewKeyFence()).(*storeImpl) + for _, vmCount := range []int{1000, 3000} { + b.Run(fmt.Sprintf("VMs%d", vmCount), func(b *testing.B) { + _, err := testDB.Exec(ctx, "TRUNCATE virtual_machine_v2 CASCADE") + require.NoError(b, err) + ids := make([]string, vmCount) + clusterID := uuid.NewV4().String() + tx, err := testDB.Begin(ctx) + require.NoError(b, err) + defer func() { _ = tx.Rollback(ctx) }() + for i := range ids { + ids[i] = uuid.NewV4().String() + require.NoError(b, vmStore.insertVM(ctx, tx, &storage.VirtualMachineV2{ + Id: ids[i], Name: fmt.Sprintf("vm-%d", i), Namespace: "default", + ClusterId: clusterID, ClusterName: "test-cluster", GuestOs: "rhel:9", + State: storage.VirtualMachineV2_RUNNING, LastUpdated: timestamppb.Now(), + })) + require.NoError(b, vmStore.insertScan(ctx, tx, &storage.VirtualMachineScanV2{ + Id: uuid.NewV7().String(), VmV2Id: ids[i], ScanOs: "rhel:9", + ScanTime: timestamppb.Now(), TopCvss: 9.8, + })) + } + require.NoError(b, tx.Commit(ctx)) + _, err = testDB.Exec(ctx, "ANALYZE virtual_machine_v2, virtual_machine_scan_v2") + require.NoError(b, err) + b.Run("JoinedQuery", func(b *testing.B) { + b.ReportAllocs() + i := 0 + for b.Loop() { + result, found, err := vmStore.GetWithLatestScan(ctx, ids[i%len(ids)]) + i++ + require.NoError(b, err) + require.True(b, found) + require.NotNil(b, result.Scan) + } + }) + }) + } +} diff --git a/central/virtualmachine/v2/datastore/store/postgres/store.go b/central/virtualmachine/v2/datastore/store/postgres/store.go new file mode 100644 index 0000000000000..b42d8871aebbc --- /dev/null +++ b/central/virtualmachine/v2/datastore/store/postgres/store.go @@ -0,0 +1,799 @@ +package postgres + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5" + "github.com/pkg/errors" + "github.com/stackrox/hashstructure" + "github.com/stackrox/rox/central/metrics" + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store" + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/contextutil" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/errox" + ops "github.com/stackrox/rox/pkg/metrics" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/pgutils" + pkgSchema "github.com/stackrox/rox/pkg/postgres/schema" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" + pgSearch "github.com/stackrox/rox/pkg/search/postgres" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + vmTable = pkgSchema.VirtualMachineV2TableName + scanTable = pkgSchema.VirtualMachineScanV2TableName + componentTable = pkgSchema.VirtualMachineComponentV2TableName + cveTable = pkgSchema.VirtualMachineCvev2TableName + + getVMStmt = "SELECT serialized FROM " + vmTable + " WHERE Id = $1" + getScanStmt = "SELECT serialized FROM " + scanTable + " WHERE VmV2Id = $1" + + // VM facts, notes, and VSOCK CID, plus scan notes, require the serialized blobs. + // LATERAL selects the newest UUIDv7 scan ID; LEFT JOIN retains VMs without scans. + getVMWithLatestScanStmt = "SELECT vm.serialized, scan.serialized FROM " + vmTable + " AS vm" + + " LEFT JOIN LATERAL (SELECT serialized FROM " + scanTable + + " WHERE VmV2Id = vm.Id ORDER BY Id DESC LIMIT 1) AS scan ON true WHERE vm.Id = $1" +) + +var ( + schema = pkgSchema.VirtualMachineV2Schema + vmSAC = sac.ForResource(resources.VirtualMachine) +) + +// New returns a new Store instance using the provided sql instance. +func New(db postgres.DB, keyFence concurrency.KeyFence) store.Store { + return &storeImpl{ + db: db, + keyFence: keyFence, + } +} + +type storeImpl struct { + db postgres.DB + keyFence concurrency.KeyFence +} + +// region EnsureExists + +func (s *storeImpl) EnsureExists(ctx context.Context, vmID, clusterID string) error { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Upsert, "VirtualMachineV2") + + return pgutils.Retry(ctx, func() error { + iTime := time.Now() + vm := &storage.VirtualMachineV2{ + Id: vmID, + ClusterId: clusterID, + LastUpdated: protocompat.ConvertTimeToTimestampOrNil(&iTime), + } + serialized, marshalErr := vm.MarshalVT() + if marshalErr != nil { + return marshalErr + } + + const stmt = "INSERT INTO " + vmTable + + " (Id, Name, Namespace, ClusterId, ClusterName, GuestOs, State, LastUpdated, LastAgentContact, serialized)" + + " VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" + + " ON CONFLICT(Id) DO NOTHING" + + _, err := s.db.Exec(ctx, stmt, + pgutils.NilOrUUID(vmID), + vm.GetName(), + vm.GetNamespace(), + pgutils.NilOrUUID(clusterID), + vm.GetClusterName(), + vm.GetGuestOs(), + vm.GetState(), + protocompat.NilOrTime(vm.GetLastUpdated()), + protocompat.NilOrTime(vm.GetLastAgentContact()), + serialized, + ) + return err + }) +} + +// endregion EnsureExists + +// region UpsertVM + +func (s *storeImpl) UpsertVM(ctx context.Context, vm *storage.VirtualMachineV2) error { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Upsert, "VirtualMachineV2") + + return pgutils.Retry(ctx, func() error { + return s.upsertVM(ctx, vm) + }) +} + +func (s *storeImpl) upsertVM(ctx context.Context, vm *storage.VirtualMachineV2) error { + vm = vm.CloneVT() // Defensive copy — callers may share the pointer across goroutines + + // Calculate hash to see if anything changed + hash, err := hashVM(vm) + if err != nil { + return err + } + + iTime := time.Now() + vm.LastUpdated = protocompat.ConvertTimeToTimestampOrNil(&iTime) + + return s.keyFence.DoStatusWithLock(concurrency.DiscreteKeySet([]byte(vm.GetId())), func() error { + tx, ctx, err := s.begin(ctx) + if err != nil { + return err + } + + existingVM, err := s.getVirtualMachine(ctx, tx, vm.GetId()) + if err != nil { + if rbErr := tx.Rollback(ctx); rbErr != nil { + return errors.Wrapf(rbErr, "rollback after: %v", err) + } + return err + } + + // Sensor informer upserts never carry last_agent_contact; keep the scrape stamp. + if existingVM != nil && vm.GetLastAgentContact() == nil { + vm.LastAgentContact = existingVM.GetLastAgentContact() + } + + if existingVM != nil && existingVM.GetHash() == hash { + // Unchanged: timestamp-only update. Preserve the existing hash. + vm.Hash = hash + if err := s.updateVMTimestamp(ctx, tx, vm); err != nil { + if rbErr := tx.Rollback(ctx); rbErr != nil { + return errors.Wrapf(rbErr, "rollback after: %v", err) + } + return err + } + return tx.Commit(ctx) + } + + // Changed or new: full upsert. + vm.Hash = hash + if err := s.insertVM(ctx, tx, vm); err != nil { + if rbErr := tx.Rollback(ctx); rbErr != nil { + return errors.Wrapf(rbErr, "rollback after: %v", err) + } + return err + } + return tx.Commit(ctx) + }) +} + +func hashVM(vm *storage.VirtualMachineV2) (uint64, error) { + return hashstructure.Hash(vm, &hashstructure.HashOptions{ZeroNil: true}) +} + +func (s *storeImpl) insertVM(ctx context.Context, tx *postgres.Tx, vm *storage.VirtualMachineV2) error { + id := pgutils.NilOrUUID(vm.GetId()) + if id == nil { + return errors.New("VM ID is empty or not a valid UUID") + } + + serialized, err := vm.MarshalVT() + if err != nil { + return err + } + + values := []interface{}{ + id, + vm.GetName(), + vm.GetNamespace(), + pgutils.NilOrUUID(vm.GetClusterId()), + vm.GetClusterName(), + vm.GetGuestOs(), + vm.GetState(), + protocompat.NilOrTime(vm.GetLastUpdated()), + protocompat.NilOrTime(vm.GetLastAgentContact()), + serialized, + } + + const stmt = "INSERT INTO " + vmTable + + " (Id, Name, Namespace, ClusterId, ClusterName, GuestOs, State, LastUpdated, LastAgentContact, serialized)" + + " VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" + + " ON CONFLICT(Id) DO UPDATE SET" + + " Id = EXCLUDED.Id, Name = EXCLUDED.Name, Namespace = EXCLUDED.Namespace," + + " ClusterId = EXCLUDED.ClusterId, ClusterName = EXCLUDED.ClusterName," + + " GuestOs = EXCLUDED.GuestOs, State = EXCLUDED.State," + + " LastUpdated = EXCLUDED.LastUpdated, LastAgentContact = EXCLUDED.LastAgentContact," + + " serialized = EXCLUDED.serialized" + _, err = tx.Exec(ctx, stmt, values...) + return err +} + +func (s *storeImpl) updateVMTimestamp(ctx context.Context, tx *postgres.Tx, vm *storage.VirtualMachineV2) error { + id := pgutils.NilOrUUID(vm.GetId()) + if id == nil { + return errors.New("VM ID is empty or not a valid UUID") + } + + serialized, err := vm.MarshalVT() + if err != nil { + return err + } + + const stmt = "UPDATE " + vmTable + " SET LastUpdated = $2, LastAgentContact = $3, serialized = $4 WHERE Id = $1" + _, err = tx.Exec(ctx, stmt, id, protocompat.NilOrTime(vm.GetLastUpdated()), protocompat.NilOrTime(vm.GetLastAgentContact()), serialized) + return err +} + +func (s *storeImpl) getVirtualMachine(ctx context.Context, tx *postgres.Tx, id string) (*storage.VirtualMachineV2, error) { + row := tx.QueryRow(ctx, getVMStmt, pgutils.NilOrUUID(id)) + var data []byte + if err := row.Scan(&data); err != nil { + return nil, pgutils.ErrNilIfNoRows(err) + } + var vm storage.VirtualMachineV2 + if err := vm.UnmarshalVTUnsafe(data); err != nil { + return nil, err + } + return &vm, nil +} + +// endregion UpsertVM + +// region UpsertScan + +func (s *storeImpl) UpsertScan(ctx context.Context, vmID string, parts common.VMScanParts) error { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Upsert, "VirtualMachineScanV2") + + return pgutils.Retry(ctx, func() error { + return s.upsertScan(ctx, vmID, parts) + }) +} + +// scanHashWrapper hashes the original v1 scan components directly, relying on +// @gotags in the proto definitions to ignore derived/store-set fields +// (top_cvss, risk_score, created_at). This matches the image store's pattern. +type scanHashWrapper struct { + ScanOs string + Components []*storage.EmbeddedVirtualMachineScanComponent `hash:"set"` +} + +func buildScanHash(parts common.VMScanParts) (uint64, error) { + return hashstructure.Hash(scanHashWrapper{ + ScanOs: parts.Scan.GetScanOs(), + Components: parts.SourceComponents, + }, &hashstructure.HashOptions{ZeroNil: true}) +} + +func (s *storeImpl) upsertScan(ctx context.Context, vmID string, parts common.VMScanParts) error { + iTime := time.Now() + + hash, err := buildScanHash(parts) + if err != nil { + return errors.Wrap(err, "computing scan hash") + } + + return s.keyFence.DoStatusWithLock(concurrency.DiscreteKeySet([]byte(vmID)), func() error { + tx, ctx, err := s.begin(ctx) + if err != nil { + return err + } + + // Touch VM last_updated. + if err := s.touchVMLastUpdated(ctx, tx, vmID, iTime); err != nil { + if rbErr := tx.Rollback(ctx); rbErr != nil { + return errors.Wrapf(rbErr, "rollback after: %v", err) + } + return err + } + + // Read existing scan for VM. + existingScan, err := s.getScanForVM(ctx, tx, vmID) + if err != nil { + if rbErr := tx.Rollback(ctx); rbErr != nil { + return errors.Wrapf(rbErr, "rollback after: %v", err) + } + return err + } + + scanTime := protocompat.ConvertTimeToTimestampOrNil(&iTime) + parts.Scan.ScanTime = scanTime + + if existingScan != nil && existingScan.GetHash() == hash { + // Unchanged: scan_time-only update using existing scan's identity. + existingScan.ScanTime = scanTime + if err := s.updateScanTime(ctx, tx, existingScan.GetId(), existingScan); err != nil { + if rbErr := tx.Rollback(ctx); rbErr != nil { + return errors.Wrapf(rbErr, "rollback after: %v", err) + } + return err + } + return tx.Commit(ctx) + } + + // Changed or new: full replace. + parts.Scan.Hash = hash + + if err := s.fullScanReplace(ctx, tx, vmID, parts, iTime); err != nil { + if rbErr := tx.Rollback(ctx); rbErr != nil { + return errors.Wrapf(rbErr, "rollback after: %v", err) + } + return err + } + return tx.Commit(ctx) + }) +} + +func (s *storeImpl) touchVMLastUpdated(ctx context.Context, tx *postgres.Tx, vmID string, t time.Time) error { + // Read the current VM to update its serialized blob with the new timestamp. + existingVM, err := s.getVirtualMachine(ctx, tx, vmID) + if err != nil { + return errors.Wrapf(err, "reading VM %s for timestamp update", vmID) + } + if existingVM == nil { + return errox.NotFound.Newf("VM %s not found", vmID) + } + ts := protocompat.ConvertTimeToTimestampOrNil(&t) + existingVM.LastUpdated = ts + // UpsertScan is the scrape-derived path (index-report SYNC), so last contact moves with last_updated. + existingVM.LastAgentContact = ts + return s.updateVMTimestamp(ctx, tx, existingVM) +} + +func (s *storeImpl) getScanForVM(ctx context.Context, tx *postgres.Tx, vmID string) (*storage.VirtualMachineScanV2, error) { + row := tx.QueryRow(ctx, getScanStmt, pgutils.NilOrUUID(vmID)) + var data []byte + if err := row.Scan(&data); err != nil { + return nil, pgutils.ErrNilIfNoRows(err) + } + var scan storage.VirtualMachineScanV2 + if err := scan.UnmarshalVTUnsafe(data); err != nil { + return nil, err + } + return &scan, nil +} + +func (s *storeImpl) updateScanTime(ctx context.Context, tx *postgres.Tx, scanID string, scan *storage.VirtualMachineScanV2) error { + id := pgutils.NilOrUUID(scanID) + if id == nil { + return errors.New("scan ID is empty or not a valid UUID") + } + + serialized, err := scan.MarshalVT() + if err != nil { + return err + } + + const stmt = "UPDATE " + scanTable + " SET ScanTime = $2, serialized = $3 WHERE Id = $1" + _, err = tx.Exec(ctx, stmt, id, protocompat.NilOrTime(scan.GetScanTime()), serialized) + return err +} + +func (s *storeImpl) fullScanReplace(ctx context.Context, tx *postgres.Tx, vmID string, parts common.VMScanParts, iTime time.Time) error { + // Build CVE time map from incoming CVEs. + cveTimeMap := buildCVETimeMap(parts.CVEs, iTime) + + // Query existing CVE times. + existingTimes, err := s.getExistingCVETimes(ctx, tx, vmID) + if err != nil { + return err + } + + // Merge: keep oldest created_at per CVE. + for cve, existingTime := range existingTimes { + if incoming, ok := cveTimeMap[cve]; ok { + if existingTime.Before(incoming) { + cveTimeMap[cve] = existingTime + } + } + } + + // Apply merged timestamps to incoming CVE objects. + for _, cve := range parts.CVEs { + cveName := cve.GetCveBaseInfo().GetCve() + if t, ok := cveTimeMap[cveName]; ok { + if cve.GetCveBaseInfo() == nil { + cve.CveBaseInfo = &storage.CVEInfo{} + } + cve.CveBaseInfo.CreatedAt = timestamppb.New(t) + } + } + + // Delete old scan (cascade deletes components + CVEs). + if _, err := tx.Exec(ctx, "DELETE FROM "+scanTable+" WHERE VmV2Id = $1", pgutils.NilOrUUID(vmID)); err != nil { + return errors.Wrap(err, "deleting old scan") + } + + // Insert new scan row. + if err := s.insertScan(ctx, tx, parts.Scan); err != nil { + return err + } + + // COPY FROM components (batched). + if err := s.copyFromComponents(ctx, tx, parts.Components); err != nil { + return err + } + + // COPY FROM CVEs (batched). + return s.copyFromCVEs(ctx, tx, parts.CVEs) +} + +func buildCVETimeMap(cves []*storage.VirtualMachineCVEV2, iTime time.Time) map[string]time.Time { + cveTimeMap := make(map[string]time.Time, len(cves)) + for _, cve := range cves { + cveName := cve.GetCveBaseInfo().GetCve() + createdAt := cve.GetCveBaseInfo().GetCreatedAt() + t := iTime + if createdAt != nil { + t = createdAt.AsTime() + } + if existing, ok := cveTimeMap[cveName]; ok { + if t.Before(existing) { + cveTimeMap[cveName] = t + } + } else { + cveTimeMap[cveName] = t + } + } + return cveTimeMap +} + +func (s *storeImpl) getExistingCVETimes(ctx context.Context, tx *postgres.Tx, vmID string) (map[string]time.Time, error) { + rows, err := tx.Query(ctx, "SELECT cvebaseinfo_cve, cvebaseinfo_createdat FROM "+cveTable+" WHERE vmv2id = $1", pgutils.NilOrUUID(vmID)) + if err != nil { + return nil, errors.Wrap(err, "querying existing CVE times") + } + defer rows.Close() + + result := make(map[string]time.Time) + for rows.Next() { + var cveName string + var createdAt *time.Time + if err := rows.Scan(&cveName, &createdAt); err != nil { + return nil, err + } + if createdAt != nil { + if existing, ok := result[cveName]; ok { + if createdAt.Before(existing) { + result[cveName] = *createdAt + } + } else { + result[cveName] = *createdAt + } + } + } + return result, rows.Err() +} + +func (s *storeImpl) insertScan(ctx context.Context, tx *postgres.Tx, scan *storage.VirtualMachineScanV2) error { + id := pgutils.NilOrUUID(scan.GetId()) + if id == nil { + return errors.New("scan ID is empty or not a valid UUID") + } + vmID := pgutils.NilOrUUID(scan.GetVmV2Id()) + if vmID == nil { + return errors.New("scan VM ID is empty or not a valid UUID") + } + + serialized, err := scan.MarshalVT() + if err != nil { + return err + } + + values := []interface{}{ + id, + vmID, + protocompat.NilOrTime(scan.GetScanTime()), + scan.GetTopCvss(), + serialized, + } + + const stmt = "INSERT INTO " + scanTable + + " (Id, VmV2Id, ScanTime, TopCvss, serialized)" + + " VALUES($1, $2, $3, $4, $5)" + _, err = tx.Exec(ctx, stmt, values...) + return err +} + +func (s *storeImpl) copyFromComponents(ctx context.Context, tx *postgres.Tx, components []*storage.VirtualMachineComponentV2) error { + if len(components) == 0 { + return nil + } + + batchSize := min(len(components), pgSearch.MaxBatchSize) + inputRows := make([][]interface{}, 0, batchSize) + + copyCols := []string{ + "id", + "vmscanid", + "name", + "version", + "source", + "operatingsystem", + "topcvss", + "serialized", + } + + for idx, obj := range components { + id := pgutils.NilOrUUID(obj.GetId()) + if id == nil { + return errors.Errorf("component %d ID is empty or not a valid UUID", idx) + } + scanID := pgutils.NilOrUUID(obj.GetVmScanId()) + if scanID == nil { + return errors.Errorf("component %d scan ID is empty or not a valid UUID", idx) + } + + serialized, err := obj.MarshalVT() + if err != nil { + return err + } + + inputRows = append(inputRows, []interface{}{ + id, + scanID, + obj.GetName(), + obj.GetVersion(), + obj.GetSource(), + obj.GetOperatingSystem(), + obj.GetTopCvss(), + serialized, + }) + + if (idx+1)%batchSize == 0 || idx == len(components)-1 { + if _, err := tx.CopyFrom(ctx, pgx.Identifier{componentTable}, copyCols, pgx.CopyFromRows(inputRows)); err != nil { + return err + } + inputRows = inputRows[:0] + } + } + + return nil +} + +func (s *storeImpl) copyFromCVEs(ctx context.Context, tx *postgres.Tx, cves []*storage.VirtualMachineCVEV2) error { + if len(cves) == 0 { + return nil + } + + batchSize := min(len(cves), pgSearch.MaxBatchSize) + inputRows := make([][]interface{}, 0, batchSize) + + copyCols := []string{ + "id", + "vmv2id", + "vmcomponentid", + "cvebaseinfo_cve", + "cvebaseinfo_publishedon", + "cvebaseinfo_createdat", + "cvebaseinfo_epss_epssprobability", + "preferredcvss", + "severity", + "impactscore", + "nvdcvss", + "isfixable", + "fixedby", + "epssprobability", + "advisory_name", + "advisory_link", + "serialized", + } + + for idx, obj := range cves { + id := pgutils.NilOrUUID(obj.GetId()) + if id == nil { + return errors.Errorf("CVE %d ID is empty or not a valid UUID", idx) + } + vmID := pgutils.NilOrUUID(obj.GetVmV2Id()) + if vmID == nil { + return errors.Errorf("CVE %d VM ID is empty or not a valid UUID", idx) + } + compID := pgutils.NilOrUUID(obj.GetVmComponentId()) + if compID == nil { + return errors.Errorf("CVE %d component ID is empty or not a valid UUID", idx) + } + + serialized, err := obj.MarshalVT() + if err != nil { + return err + } + + inputRows = append(inputRows, []interface{}{ + id, + vmID, + compID, + obj.GetCveBaseInfo().GetCve(), + protocompat.NilOrTime(obj.GetCveBaseInfo().GetPublishedOn()), + protocompat.NilOrTime(obj.GetCveBaseInfo().GetCreatedAt()), + obj.GetCveBaseInfo().GetEpss().GetEpssProbability(), + obj.GetPreferredCvss(), + obj.GetSeverity(), + obj.GetImpactScore(), + obj.GetNvdcvss(), + obj.GetIsFixable(), + obj.GetFixedBy(), + obj.GetEpssProbability(), + obj.GetAdvisory().GetName(), + obj.GetAdvisory().GetLink(), + serialized, + }) + + if (idx+1)%batchSize == 0 || idx == len(cves)-1 { + if _, err := tx.CopyFrom(ctx, pgx.Identifier{cveTable}, copyCols, pgx.CopyFromRows(inputRows)); err != nil { + return err + } + inputRows = inputRows[:0] + } + } + + return nil +} + +// endregion UpsertScan + +// region Delete + +func (s *storeImpl) Delete(ctx context.Context, id string) error { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Remove, "VirtualMachineV2") + + return pgutils.Retry(ctx, func() error { + return s.keyFence.DoStatusWithLock(concurrency.DiscreteKeySet([]byte(id)), func() error { + tx, ctx, err := s.begin(ctx) + if err != nil { + return err + } + // FK cascade handles scan, component, CVE deletion. + if _, err := tx.Exec(ctx, "DELETE FROM "+vmTable+" WHERE Id = $1", pgutils.NilOrUUID(id)); err != nil { + if rbErr := tx.Rollback(ctx); rbErr != nil { + return errors.Wrapf(rbErr, "rollback after: %v", err) + } + return err + } + return tx.Commit(ctx) + }) + }) +} + +// DeleteMany removes multiple VMs and all associated data. +// NOTE: This does not acquire per-ID keyFence locks for performance reasons, +// matching the pattern used by other stores (e.g., image store). Callers +// should avoid concurrent upserts on the same IDs during batch deletes. +func (s *storeImpl) DeleteMany(ctx context.Context, ids []string) error { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.RemoveMany, "VirtualMachineV2") + + if len(ids) == 0 { + return nil + } + + return pgutils.Retry(ctx, func() error { + q := search.NewQueryBuilder().AddDocIDs(ids...).ProtoQuery() + return pgSearch.RunDeleteRequestForSchema(ctx, schema, q, s.db) + }) +} + +// endregion Delete + +// region Read operations + +func (s *storeImpl) Count(ctx context.Context, q *v1.Query) (int, error) { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Count, "VirtualMachineV2") + + return pgutils.Retry2(ctx, func() (int, error) { + return pgSearch.RunCountRequestForSchema(ctx, schema, q, s.db) + }) +} + +func (s *storeImpl) Search(ctx context.Context, q *v1.Query) ([]search.Result, error) { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Search, "VirtualMachineV2") + + return pgutils.Retry2(ctx, func() ([]search.Result, error) { + return pgSearch.RunSearchRequestForSchema(ctx, schema, q, s.db) + }) +} + +func (s *storeImpl) Get(ctx context.Context, id string) (*storage.VirtualMachineV2, bool, error) { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Get, "VirtualMachineV2") + + return pgutils.Retry3(ctx, func() (*storage.VirtualMachineV2, bool, error) { + q := search.NewQueryBuilder().AddDocIDs(id).ProtoQuery() + var vm *storage.VirtualMachineV2 + err := pgSearch.RunQueryForSchemaFn[storage.VirtualMachineV2](ctx, schema, q, s.db, func(v *storage.VirtualMachineV2) error { + vm = v + return nil + }) + if err != nil { + return nil, false, err + } + if vm == nil { + return nil, false, nil + } + return vm, true, nil + }) +} + +// GetWithLatestScan reads the VM and its latest scan in one snapshot. +func (s *storeImpl) GetWithLatestScan(ctx context.Context, id string) (*common.VMWithLatestScan, bool, error) { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Get, "VirtualMachineV2WithLatestScan") + queryCtx, cancel := contextutil.ContextWithTimeoutIfNotExists(ctx, env.PostgresVMStatementTimeout.DurationSetting()) + defer cancel() + return pgutils.Retry3(queryCtx, func() (*common.VMWithLatestScan, bool, error) { + var db postgres.Queryable = s.db + if tx, exists := postgres.TxFromContext(queryCtx); exists { + db = tx + } + var vmBytes, scanBytes []byte + err := db.QueryRow(queryCtx, getVMWithLatestScanStmt, pgutils.NilOrUUID(id)).Scan(&vmBytes, &scanBytes) + if errors.Is(err, pgx.ErrNoRows) { + return nil, false, nil + } + if err != nil { + return nil, false, errors.Wrap(err, "reading VM and latest scan") + } + vm := &storage.VirtualMachineV2{} + if err := vm.UnmarshalVT(vmBytes); err != nil { + return nil, false, errors.Wrap(err, "decoding VM") + } + // Match the scope filtering of Get: inaccessible VMs are not found, and + // neither their metadata nor their scan is returned to the caller. + if allowed, err := vmSAC.ReadAllowed(queryCtx, sac.ClusterScopeKey(vm.GetClusterId()), sac.NamespaceScopeKey(vm.GetNamespace())); err != nil || !allowed { + return nil, false, err + } + result := &common.VMWithLatestScan{VM: vm} + if scanBytes != nil { + result.Scan = &storage.VirtualMachineScanV2{} + if err := result.Scan.UnmarshalVT(scanBytes); err != nil { + return nil, false, errors.Wrap(err, "decoding latest VM scan") + } + } + return result, true, nil + }) +} + +func (s *storeImpl) GetMany(ctx context.Context, ids []string) ([]*storage.VirtualMachineV2, []int, error) { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.GetMany, "VirtualMachineV2") + + return pgutils.Retry3(ctx, func() ([]*storage.VirtualMachineV2, []int, error) { + if len(ids) == 0 { + return nil, nil, nil + } + + q := search.NewQueryBuilder().AddDocIDs(ids...).ProtoQuery() + resultsByID := make(map[string]*storage.VirtualMachineV2, len(ids)) + err := pgSearch.RunQueryForSchemaFn[storage.VirtualMachineV2](ctx, schema, q, s.db, func(vm *storage.VirtualMachineV2) error { + resultsByID[vm.GetId()] = vm + return nil + }) + if err != nil { + return nil, nil, err + } + + // Preserve input order and track missing indices. + elems := make([]*storage.VirtualMachineV2, 0, len(resultsByID)) + missingIndices := make([]int, 0, len(ids)-len(resultsByID)) + for i, id := range ids { + if vm, ok := resultsByID[id]; ok { + elems = append(elems, vm) + } else { + missingIndices = append(missingIndices, i) + } + } + return elems, missingIndices, nil + }) +} + +func (s *storeImpl) Walk(ctx context.Context, fn func(vm *storage.VirtualMachineV2) error) error { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.Walk, "VirtualMachineV2") + + return pgSearch.RunCursorQueryForSchemaFn[storage.VirtualMachineV2](ctx, schema, search.EmptyQuery(), s.db, "Walk", fn) +} + +func (s *storeImpl) WalkByQuery(ctx context.Context, q *v1.Query, fn func(vm *storage.VirtualMachineV2) error) error { + defer metrics.SetPostgresOperationDurationTime(time.Now(), ops.WalkByQuery, "VirtualMachineV2") + + return pgSearch.RunCursorQueryForSchemaFn[storage.VirtualMachineV2](ctx, schema, q, s.db, "WalkByQuery", fn) +} + +// endregion Read operations + +func (s *storeImpl) begin(ctx context.Context) (*postgres.Tx, context.Context, error) { + return postgres.GetTransaction(ctx, s.db) +} diff --git a/central/virtualmachine/v2/datastore/store/postgres/store_test.go b/central/virtualmachine/v2/datastore/store/postgres/store_test.go new file mode 100644 index 0000000000000..a62422a05fe48 --- /dev/null +++ b/central/virtualmachine/v2/datastore/store/postgres/store_test.go @@ -0,0 +1,685 @@ +//go:build sql_integration + +package postgres + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/features" + "github.com/stackrox/rox/pkg/postgres" + "github.com/stackrox/rox/pkg/postgres/pgtest" + "github.com/stackrox/rox/pkg/postgres/pgutils" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/uuid" + "github.com/stretchr/testify/suite" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type VMStoreTestSuite struct { + suite.Suite + store *storeImpl + testDB *pgtest.TestPostgres + ctx context.Context +} + +func TestVMStore(t *testing.T) { + t.Setenv(features.VirtualMachinesEnhancedDataModel.EnvVar(), "true") + if !features.VirtualMachinesEnhancedDataModel.Enabled() { + t.Skip("VM enhanced data model is not enabled") + } + suite.Run(t, new(VMStoreTestSuite)) +} + +func (s *VMStoreTestSuite) SetupSuite() { + s.testDB = pgtest.ForT(s.T()) + s.ctx = sac.WithAllAccess(context.Background()) +} + +func (s *VMStoreTestSuite) SetupTest() { + _, err := s.testDB.Exec(s.ctx, "TRUNCATE virtual_machine_v2 CASCADE") + s.Require().NoError(err) + s.store = New(s.testDB.DB, concurrency.NewKeyFence()).(*storeImpl) +} + +func (s *VMStoreTestSuite) TestGetWithLatestScan_MissingVM() { + result, found, err := s.store.GetWithLatestScan(s.ctx, uuid.NewV4().String()) + s.Require().NoError(err) + s.False(found) + s.Nil(result) +} + +func (s *VMStoreTestSuite) TestGetWithLatestScan_NoScan() { + vm := s.newVM() + vm.Facts = map[string]string{"agentVersion": "1.0"} + s.Require().NoError(s.store.UpsertVM(s.ctx, vm)) + expected, found, err := s.store.Get(s.ctx, vm.GetId()) + s.Require().NoError(err) + s.Require().True(found) + result, found, err := s.store.GetWithLatestScan(s.ctx, vm.GetId()) + s.Require().NoError(err) + s.Require().True(found) + protoassert.Equal(s.T(), expected, result.VM) + s.Nil(result.Scan) +} + +func (s *VMStoreTestSuite) TestGetWithLatestScan_UsesScanID() { + vm := s.newVM() + s.Require().NoError(s.store.UpsertVM(s.ctx, vm)) + expected, found, err := s.store.Get(s.ctx, vm.GetId()) + s.Require().NoError(err) + s.Require().True(found) + ids := []string{uuid.NewV7().String(), uuid.NewV7().String()} + slices.Sort(ids) + old := &storage.VirtualMachineScanV2{ + Id: ids[0], VmV2Id: vm.GetId(), ScanTime: timestamppb.Now(), + } + latest := &storage.VirtualMachineScanV2{ + Id: ids[1], VmV2Id: vm.GetId(), ScanOs: "rhel:9", TopCvss: 9.8, + ScanTime: timestamppb.New(time.Now().Add(-time.Hour)), + Notes: []storage.VirtualMachineScanV2_Note{storage.VirtualMachineScanV2_OS_UNKNOWN}, + } + tx, err := s.testDB.Begin(s.ctx) + s.Require().NoError(err) + defer func() { _ = tx.Rollback(s.ctx) }() + s.Require().NoError(s.store.insertScan(s.ctx, tx, old)) + s.Require().NoError(s.store.insertScan(s.ctx, tx, latest)) + s.Require().NoError(tx.Commit(s.ctx)) + result, found, err := s.store.GetWithLatestScan(s.ctx, vm.GetId()) + s.Require().NoError(err) + s.Require().True(found) + protoassert.Equal(s.T(), expected, result.VM) + protoassert.Equal(s.T(), latest, result.Scan) +} + +func (s *VMStoreTestSuite) TestGetWithLatestScan_ScopedAccess() { + vm := s.newVM() + s.Require().NoError(s.store.UpsertVM(s.ctx, vm)) + s.Require().NoError(s.store.UpsertScan(s.ctx, vm.GetId(), s.newScanParts(vm.GetId()))) + allowedScope := func(cluster, namespace string) context.Context { + return sac.WithGlobalAccessScopeChecker(context.Background(), sac.AllowFixedScopes( + sac.AccessModeScopeKeys(storage.Access_READ_ACCESS), + sac.ResourceScopeKeys(resources.VirtualMachine), + sac.ClusterScopeKeys(cluster), + sac.NamespaceScopeKeys(namespace), + )) + } + tests := map[string]struct { + ctx context.Context + found bool + }{ + "all access": {ctx: s.ctx, found: true}, + "own namespace": {ctx: allowedScope(vm.GetClusterId(), vm.GetNamespace()), found: true}, + "different namespace": {ctx: allowedScope(vm.GetClusterId(), "other")}, + "different cluster": {ctx: allowedScope(uuid.NewV4().String(), vm.GetNamespace())}, + "no access": {ctx: sac.WithNoAccess(context.Background())}, + } + for name, tt := range tests { + s.Run(name, func() { + result, found, err := s.store.GetWithLatestScan(tt.ctx, vm.GetId()) + s.Require().NoError(err) + s.Equal(tt.found, found) + if tt.found { + s.NotNil(result.VM) + s.NotNil(result.Scan) + } else { + s.Nil(result, "VM metadata and scan must both be hidden") + } + _, oldFound, err := s.store.Get(tt.ctx, vm.GetId()) + s.Require().NoError(err) + s.Equal(oldFound, found, "joined read must match existing Get scope filtering") + }) + } +} + +func (s *VMStoreTestSuite) TestGetWithLatestScan_TransactionContext() { + vm := s.newVM() + scan := s.newScanParts(vm.GetId()).Scan + tx, txCtx, err := s.store.begin(s.ctx) + s.Require().NoError(err) + defer func() { _ = tx.Rollback(s.ctx) }() + s.Require().NoError(s.store.insertVM(txCtx, tx, vm)) + s.Require().NoError(s.store.insertScan(txCtx, tx, scan)) + + result, found, err := s.store.GetWithLatestScan(txCtx, vm.GetId()) + s.Require().NoError(err) + s.Require().True(found) + protoassert.Equal(s.T(), vm, result.VM) + protoassert.Equal(s.T(), scan, result.Scan) + + s.Require().NoError(tx.Rollback(s.ctx)) + _, found, err = s.store.GetWithLatestScan(s.ctx, vm.GetId()) + s.Require().NoError(err) + s.False(found, "uncommitted VM must only be visible through the transaction context") +} + +func (s *VMStoreTestSuite) newVM() *storage.VirtualMachineV2 { + return &storage.VirtualMachineV2{ + Id: uuid.NewV4().String(), + Name: "test-vm", + Namespace: "default", + ClusterId: uuid.NewV4().String(), + ClusterName: "test-cluster", + GuestOs: "rhel9", + State: storage.VirtualMachineV2_RUNNING, + Notes: []storage.VirtualMachineV2_Note{storage.VirtualMachineV2_MISSING_SCAN_DATA}, + VsockCid: 42, + } +} + +func (s *VMStoreTestSuite) newScanParts(vmID string) common.VMScanParts { + scanID := uuid.NewV4().String() + compID := uuid.NewV4().String() + cveID := uuid.NewV4().String() + + return common.VMScanParts{ + Scan: &storage.VirtualMachineScanV2{ + Id: scanID, + VmV2Id: vmID, + ScanOs: "rhel9", + }, + Components: []*storage.VirtualMachineComponentV2{ + { + Id: compID, + VmScanId: scanID, + Name: "openssl", + Version: "1.1.1", + Source: storage.SourceType_OS, + OperatingSystem: "rhel:9", + }, + }, + CVEs: []*storage.VirtualMachineCVEV2{ + { + Id: cveID, + VmV2Id: vmID, + VmComponentId: compID, + CveBaseInfo: &storage.CVEInfo{ + Cve: "CVE-2024-0001", + Summary: "test vulnerability", + }, + PreferredCvss: 7.5, + Severity: storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, + IsFixable: true, + HasFixedBy: &storage.VirtualMachineCVEV2_FixedBy{FixedBy: "1.1.2"}, + }, + }, + SourceComponents: []*storage.EmbeddedVirtualMachineScanComponent{ + { + Name: "openssl", + Version: "1.1.1", + Source: storage.SourceType_OS, + Vulnerabilities: []*storage.VirtualMachineVulnerability{ + { + CveBaseInfo: &storage.VirtualMachineCVEInfo{ + Cve: "CVE-2024-0001", + Summary: "test vulnerability", + }, + Severity: storage.VulnerabilitySeverity_IMPORTANT_VULNERABILITY_SEVERITY, + SetFixedBy: &storage.VirtualMachineVulnerability_FixedBy{FixedBy: "1.1.2"}, + Cvss: 7.5, + }, + }, + }, + }, + } +} + +// region UpsertVM tests + +func (s *VMStoreTestSuite) TestUpsertVM_New() { + vm := s.newVM() + + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + got, exists, err := s.store.Get(s.ctx, vm.GetId()) + s.NoError(err) + s.True(exists) + s.Equal(vm.GetName(), got.GetName()) + s.NotNil(got.GetLastUpdated()) + s.NotZero(got.GetHash()) +} + +func (s *VMStoreTestSuite) TestUpsertVM_Unchanged() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + firstGet, _, err := s.store.Get(s.ctx, vm.GetId()) + s.NoError(err) + firstUpdated := firstGet.GetLastUpdated().AsTime() + + // Sleep to ensure timestamp difference. + time.Sleep(10 * time.Millisecond) + + // Upsert again with same data (no field changes). + vmCopy := vm.CloneVT() + vmCopy.LastUpdated = nil + vmCopy.Hash = 0 + s.NoError(s.store.UpsertVM(s.ctx, vmCopy)) + + secondGet, _, err := s.store.Get(s.ctx, vm.GetId()) + s.NoError(err) + s.True(secondGet.GetLastUpdated().AsTime().After(firstUpdated), "timestamp should be updated") + // Hash should remain the same since data didn't change. + s.Equal(firstGet.GetHash(), secondGet.GetHash()) +} + +func (s *VMStoreTestSuite) TestUpsertVM_Changed() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + firstGet, _, err := s.store.Get(s.ctx, vm.GetId()) + s.NoError(err) + + // Change a field. + vm.GuestOs = "ubuntu22" + vm.LastUpdated = nil + vm.Hash = 0 + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + secondGet, _, err := s.store.Get(s.ctx, vm.GetId()) + s.NoError(err) + s.Equal("ubuntu22", secondGet.GetGuestOs()) + s.NotEqual(firstGet.GetHash(), secondGet.GetHash(), "hash should change when data changes") +} + +func (s *VMStoreTestSuite) TestUpsertScan_StampsLastAgentContact() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + beforeScan, _, err := s.store.Get(s.ctx, vm.GetId()) + s.NoError(err) + s.Nil(beforeScan.GetLastAgentContact()) + + parts := s.newScanParts(vm.GetId()) + before := time.Now() + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts)) + after := time.Now() + + afterScan, _, err := s.store.Get(s.ctx, vm.GetId()) + s.NoError(err) + s.Require().NotNil(afterScan.GetLastAgentContact()) + contact := afterScan.GetLastAgentContact().AsTime() + s.False(contact.Before(before), "LastAgentContact %v is before UpsertScan started at %v", contact, before) + s.False(contact.After(after), "LastAgentContact %v is after UpsertScan finished at %v", contact, after) +} + +func (s *VMStoreTestSuite) TestUpsertVM_PreservesLastAgentContactWhenInformerOmitsIt() { + for name, mutate := range map[string]func(*storage.VirtualMachineV2){ + "unchanged upsert": func(*storage.VirtualMachineV2) {}, + "changed upsert": func(informerUpdate *storage.VirtualMachineV2) { + informerUpdate.State = storage.VirtualMachineV2_STOPPED + }, + } { + s.Run(name, func() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + parts := s.newScanParts(vm.GetId()) + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts)) + + afterScan, _, err := s.store.Get(s.ctx, vm.GetId()) + s.NoError(err) + s.Require().NotNil(afterScan.GetLastAgentContact()) + contact := afterScan.GetLastAgentContact().AsTime() + + informerUpdate := vm.CloneVT() + informerUpdate.LastUpdated = nil + informerUpdate.Hash = 0 + informerUpdate.LastAgentContact = nil + mutate(informerUpdate) + s.NoError(s.store.UpsertVM(s.ctx, informerUpdate)) + + got, _, err := s.store.Get(s.ctx, vm.GetId()) + s.NoError(err) + s.Equal(contact, got.GetLastAgentContact().AsTime()) + }) + } +} + +// endregion UpsertVM tests + +// region UpsertScan tests + +func (s *VMStoreTestSuite) TestUpsertScan_New() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + parts := s.newScanParts(vm.GetId()) + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts)) + + // Verify scan was inserted. + scan, err := s.getScanForVM(vm.GetId()) + s.NoError(err) + s.NotNil(scan) + s.Equal(vm.GetId(), scan.GetVmV2Id()) + s.NotZero(scan.GetHash()) + + // Verify components. + components, err := s.getComponentsForScan(scan.GetId()) + s.NoError(err) + s.Len(components, 1) + s.Equal("openssl", components[0].GetName()) + + // Verify CVEs. + cves, err := s.getCVEsForVM(vm.GetId()) + s.NoError(err) + s.Len(cves, 1) + s.Equal("CVE-2024-0001", cves[0].GetCveBaseInfo().GetCve()) +} + +func (s *VMStoreTestSuite) TestUpsertScan_Unchanged() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + parts := s.newScanParts(vm.GetId()) + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts)) + + firstScan, err := s.getScanForVM(vm.GetId()) + s.NoError(err) + firstScanTime := firstScan.GetScanTime().AsTime() + + time.Sleep(10 * time.Millisecond) + + // Upsert with same components and CVEs (new scan ID but same content). + // SourceComponents are identical since content hasn't changed. + parts2 := s.newScanParts(vm.GetId()) + + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts2)) + + secondScan, err := s.getScanForVM(vm.GetId()) + s.NoError(err) + // Scan time should be updated. + s.True(secondScan.GetScanTime().AsTime().After(firstScanTime), "scan time should be updated") + // Scan ID should remain the same (timestamp-only update). + s.Equal(firstScan.GetId(), secondScan.GetId()) +} + +func (s *VMStoreTestSuite) TestUpsertScan_Changed() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + parts := s.newScanParts(vm.GetId()) + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts)) + + firstScan, err := s.getScanForVM(vm.GetId()) + s.NoError(err) + + time.Sleep(10 * time.Millisecond) + + // Change scan data: add a second component and CVE. + parts2 := s.newScanParts(vm.GetId()) + newCompID := uuid.NewV4().String() + parts2.Components = append(parts2.Components, &storage.VirtualMachineComponentV2{ + Id: newCompID, + VmScanId: parts2.Scan.GetId(), + Name: "curl", + Version: "7.68.0", + Source: storage.SourceType_OS, + OperatingSystem: "rhel:9", + }) + parts2.CVEs = append(parts2.CVEs, &storage.VirtualMachineCVEV2{ + Id: uuid.NewV4().String(), + VmV2Id: vm.GetId(), + VmComponentId: newCompID, + CveBaseInfo: &storage.CVEInfo{ + Cve: "CVE-2024-0002", + Summary: "another vulnerability", + }, + PreferredCvss: 5.0, + Severity: storage.VulnerabilitySeverity_MODERATE_VULNERABILITY_SEVERITY, + }) + parts2.SourceComponents = append(parts2.SourceComponents, &storage.EmbeddedVirtualMachineScanComponent{ + Name: "curl", + Version: "7.68.0", + Source: storage.SourceType_OS, + Vulnerabilities: []*storage.VirtualMachineVulnerability{ + { + CveBaseInfo: &storage.VirtualMachineCVEInfo{ + Cve: "CVE-2024-0002", + Summary: "another vulnerability", + }, + Severity: storage.VulnerabilitySeverity_MODERATE_VULNERABILITY_SEVERITY, + Cvss: 5.0, + }, + }, + }) + + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts2)) + + secondScan, err := s.getScanForVM(vm.GetId()) + s.NoError(err) + // Hash should differ since content changed. + s.NotEqual(firstScan.GetHash(), secondScan.GetHash()) + // Should be a new scan ID (full replace). + s.NotEqual(firstScan.GetId(), secondScan.GetId()) + + // Verify two components and two CVEs now. + components, err := s.getComponentsForScan(secondScan.GetId()) + s.NoError(err) + s.Len(components, 2) + + cves, err := s.getCVEsForVM(vm.GetId()) + s.NoError(err) + s.Len(cves, 2) +} + +// endregion UpsertScan tests + +// region CVE created_at preservation tests + +func (s *VMStoreTestSuite) TestUpsertScan_CVECreatedAtPreservation() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + // First scan with a specific created_at. + oldTime := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + parts := s.newScanParts(vm.GetId()) + parts.CVEs[0].CveBaseInfo.CreatedAt = timestamppb.New(oldTime) + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts)) + + // Get the stored CVE created_at. + cves, err := s.getCVEsForVM(vm.GetId()) + s.NoError(err) + s.Require().Len(cves, 1) + firstCreatedAt := cves[0].GetCveBaseInfo().GetCreatedAt().AsTime() + + time.Sleep(10 * time.Millisecond) + + // Second scan with same CVE but newer created_at. + parts2 := s.newScanParts(vm.GetId()) + parts2.CVEs[0].CveBaseInfo.Cve = "CVE-2024-0001" // Same CVE + parts2.CVEs[0].CveBaseInfo.CreatedAt = timestamppb.Now() // Newer timestamp + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts2)) + + // Verify created_at was preserved (oldest wins). + cves2, err := s.getCVEsForVM(vm.GetId()) + s.NoError(err) + s.Require().Len(cves2, 1) + s.Equal(firstCreatedAt.UTC(), cves2[0].GetCveBaseInfo().GetCreatedAt().AsTime().UTC(), + "created_at should be preserved from the first scan") +} + +func (s *VMStoreTestSuite) TestUpsertScan_CVENilCveBaseInfo() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + parts := s.newScanParts(vm.GetId()) + // Add a CVE with nil CveBaseInfo — should not panic. + parts.CVEs = append(parts.CVEs, &storage.VirtualMachineCVEV2{ + Id: uuid.NewV4().String(), + VmV2Id: vm.GetId(), + VmComponentId: parts.Components[0].GetId(), + CveBaseInfo: nil, + PreferredCvss: 3.0, + Severity: storage.VulnerabilitySeverity_LOW_VULNERABILITY_SEVERITY, + }) + + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts)) + + cves, err := s.getCVEsForVM(vm.GetId()) + s.NoError(err) + s.Len(cves, 2) +} + +func (s *VMStoreTestSuite) TestUpsertScan_ScanOsChangeTriggersFullReplace() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + parts := s.newScanParts(vm.GetId()) + parts.Scan.ScanOs = "rhel9" + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts)) + + firstScan, err := s.getScanForVM(vm.GetId()) + s.NoError(err) + + time.Sleep(10 * time.Millisecond) + + // Upsert again with same components/CVEs but different ScanOs. + parts2 := s.newScanParts(vm.GetId()) + parts2.Scan.ScanOs = "ubuntu22" + + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts2)) + + secondScan, err := s.getScanForVM(vm.GetId()) + s.NoError(err) + // ScanOs change should trigger full replace — new scan ID. + s.NotEqual(firstScan.GetId(), secondScan.GetId(), "ScanOs change should trigger full scan replace") + s.NotEqual(firstScan.GetHash(), secondScan.GetHash(), "hash should differ when ScanOs changes") + s.Equal("ubuntu22", secondScan.GetScanOs()) +} + +// endregion CVE created_at preservation tests + +// region Delete tests + +func (s *VMStoreTestSuite) TestDelete_Cascade() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + parts := s.newScanParts(vm.GetId()) + s.NoError(s.store.UpsertScan(s.ctx, vm.GetId(), parts)) + + // Delete the VM. + s.NoError(s.store.Delete(s.ctx, vm.GetId())) + + // VM should be gone. + _, exists, err := s.store.Get(s.ctx, vm.GetId()) + s.NoError(err) + s.False(exists) + + // Scan should be cascade deleted. + scan, err := s.getScanForVM(vm.GetId()) + s.NoError(err) + s.Nil(scan) + + // CVEs should be cascade deleted. + cves, err := s.getCVEsForVM(vm.GetId()) + s.NoError(err) + s.Empty(cves) +} + +func (s *VMStoreTestSuite) TestDeleteMany() { + vm1 := s.newVM() + vm2 := s.newVM() + vm2.Name = "vm2" + s.NoError(s.store.UpsertVM(s.ctx, vm1)) + s.NoError(s.store.UpsertVM(s.ctx, vm2)) + + count, err := s.store.Count(s.ctx, search.EmptyQuery()) + s.NoError(err) + s.Equal(2, count) + + s.NoError(s.store.DeleteMany(s.ctx, []string{vm1.GetId(), vm2.GetId()})) + + count, err = s.store.Count(s.ctx, search.EmptyQuery()) + s.NoError(err) + s.Equal(0, count) +} + +// endregion Delete tests + +// region Read operation tests + +func (s *VMStoreTestSuite) TestCount() { + vm1 := s.newVM() + vm2 := s.newVM() + vm2.Name = "vm2" + s.NoError(s.store.UpsertVM(s.ctx, vm1)) + s.NoError(s.store.UpsertVM(s.ctx, vm2)) + + count, err := s.store.Count(s.ctx, search.EmptyQuery()) + s.NoError(err) + s.Equal(2, count) +} + +func (s *VMStoreTestSuite) TestSearch() { + vm := s.newVM() + s.NoError(s.store.UpsertVM(s.ctx, vm)) + + results, err := s.store.Search(s.ctx, search.EmptyQuery()) + s.NoError(err) + s.Len(results, 1) +} + +func (s *VMStoreTestSuite) TestGetMany() { + vm1 := s.newVM() + vm2 := s.newVM() + vm2.Name = "vm2" + s.NoError(s.store.UpsertVM(s.ctx, vm1)) + s.NoError(s.store.UpsertVM(s.ctx, vm2)) + + // Re-fetch to capture server-set fields (LastUpdated, Hash). + expected1, exists, err := s.store.Get(s.ctx, vm1.GetId()) + s.NoError(err) + s.True(exists) + expected2, exists, err := s.store.Get(s.ctx, vm2.GetId()) + s.NoError(err) + s.True(exists) + + nonExistentID := uuid.NewV4().String() + results, missing, err := s.store.GetMany(s.ctx, []string{vm1.GetId(), nonExistentID, vm2.GetId()}) + s.NoError(err) + s.Len(results, 2) + s.Equal([]int{1}, missing) + + protoassert.SlicesEqual(s.T(), []*storage.VirtualMachineV2{expected1, expected2}, results) +} + +// endregion Read operation tests + +// region Helpers + +func (s *VMStoreTestSuite) getScanForVM(vmID string) (*storage.VirtualMachineScanV2, error) { + tx, ctx, err := s.store.begin(s.ctx) + if err != nil { + return nil, err + } + defer postgres.FinishReadOnlyTransaction(tx) + return s.store.getScanForVM(ctx, tx, vmID) +} + +func (s *VMStoreTestSuite) getComponentsForScan(scanID string) ([]*storage.VirtualMachineComponentV2, error) { + rows, err := s.testDB.DB.Query(s.ctx, "SELECT serialized FROM "+componentTable+" WHERE vmscanid = $1", pgutils.NilOrUUID(scanID)) + if err != nil { + return nil, err + } + return pgutils.ScanRows[storage.VirtualMachineComponentV2, *storage.VirtualMachineComponentV2](rows) +} + +func (s *VMStoreTestSuite) getCVEsForVM(vmID string) ([]*storage.VirtualMachineCVEV2, error) { + rows, err := s.testDB.DB.Query(s.ctx, "SELECT serialized FROM "+cveTable+" WHERE vmv2id = $1", pgutils.NilOrUUID(vmID)) + if err != nil { + return nil, err + } + return pgutils.ScanRows[storage.VirtualMachineCVEV2, *storage.VirtualMachineCVEV2](rows) +} + +// endregion Helpers diff --git a/central/virtualmachine/v2/datastore/store/store.go b/central/virtualmachine/v2/datastore/store/store.go new file mode 100644 index 0000000000000..08b3ac8d05729 --- /dev/null +++ b/central/virtualmachine/v2/datastore/store/store.go @@ -0,0 +1,55 @@ +package store + +import ( + "context" + + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" + v1 "github.com/stackrox/rox/generated/api/v1" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/search" +) + +// Store provides storage functionality for VirtualMachineV2 and its related +// scan, component, and CVE data. +// +//go:generate mockgen-wrapper +type Store interface { + // EnsureExists atomically inserts a minimal VM record (id + clusterId) + // if one does not already exist. Uses INSERT ... ON CONFLICT DO NOTHING + // to avoid overwriting richer metadata from the VM pipeline. + EnsureExists(ctx context.Context, vmID, clusterID string) error + + // UpsertVM upserts a VM. Hash-comparison determines whether a full write + // or timestamp-only update is performed. + // NOTE: Mutates vm in-place (LastUpdated, Hash). + UpsertVM(ctx context.Context, vm *storage.VirtualMachineV2) error + + // UpsertScan upserts scan data (scan, components, CVEs) for a VM. + // Hash-comparison determines whether a full replace or scan_time-only + // update is performed. CVE created_at timestamps are preserved across + // delete/re-insert cycles. Also stamps last_agent_contact: this is the + // scrape-derived path. + // NOTE: Mutates parts in-place (Scan.ScanTime, Scan.Hash, CVE CreatedAt). + UpsertScan(ctx context.Context, vmID string, parts common.VMScanParts) error + + // Delete removes a VM and all associated data (FK cascade). + Delete(ctx context.Context, id string) error + // DeleteMany removes multiple VMs and all associated data. + DeleteMany(ctx context.Context, ids []string) error + + // Count returns the number of VMs matching the query. + Count(ctx context.Context, q *v1.Query) (int, error) + // Search returns search results matching the query. + Search(ctx context.Context, q *v1.Query) ([]search.Result, error) + // Get returns the VM with the given ID. + Get(ctx context.Context, id string) (*storage.VirtualMachineV2, bool, error) + // GetWithLatestScan returns a VM and its latest scan in one query. + GetWithLatestScan(ctx context.Context, id string) (*common.VMWithLatestScan, bool, error) + // GetMany returns the VMs with the given IDs. + GetMany(ctx context.Context, ids []string) ([]*storage.VirtualMachineV2, []int, error) + + // Walk iterates over all VMs, calling fn for each. + Walk(ctx context.Context, fn func(vm *storage.VirtualMachineV2) error) error + // WalkByQuery iterates over VMs matching the query, calling fn for each. + WalkByQuery(ctx context.Context, q *v1.Query, fn func(vm *storage.VirtualMachineV2) error) error +} diff --git a/central/virtualmachine/v2/service/service.go b/central/virtualmachine/v2/service/service.go new file mode 100644 index 0000000000000..d0d6d40169b26 --- /dev/null +++ b/central/virtualmachine/v2/service/service.go @@ -0,0 +1,39 @@ +package service + +import ( + "context" + + "github.com/stackrox/rox/central/views/vmcve" + componentDS "github.com/stackrox/rox/central/virtualmachine/component/v2/datastore" + cveDS "github.com/stackrox/rox/central/virtualmachine/cve/v2/datastore" + scanDS "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore" + vmDS "github.com/stackrox/rox/central/virtualmachine/v2/datastore" + v2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/pkg/grpc" +) + +// Service provides the interface to the VirtualMachineV2Service. +type Service interface { + grpc.APIService + + AuthFuncOverride(ctx context.Context, fullMethodName string) (context.Context, error) + + v2.VirtualMachineV2ServiceServer +} + +// New returns a new Service instance using the given datastores and view. +func New( + vmDataStore vmDS.DataStore, + cveDataStore cveDS.DataStore, + componentDataStore componentDS.DataStore, + scanDataStore scanDS.DataStore, + cveView vmcve.CveView, +) Service { + return &serviceImpl{ + vmDS: vmDataStore, + cveDS: cveDataStore, + componentDS: componentDataStore, + scanDS: scanDataStore, + cveView: cveView, + } +} diff --git a/central/virtualmachine/v2/service/service_impl.go b/central/virtualmachine/v2/service/service_impl.go new file mode 100644 index 0000000000000..855bdc9d4db8c --- /dev/null +++ b/central/virtualmachine/v2/service/service_impl.go @@ -0,0 +1,713 @@ +package service + +import ( + "context" + "slices" + "time" + + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/pkg/errors" + "github.com/stackrox/rox/central/convert/storagetov2" + "github.com/stackrox/rox/central/views/common" + "github.com/stackrox/rox/central/views/vmcve" + componentDS "github.com/stackrox/rox/central/virtualmachine/component/v2/datastore" + cveDS "github.com/stackrox/rox/central/virtualmachine/cve/v2/datastore" + scanDS "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore" + vmDS "github.com/stackrox/rox/central/virtualmachine/v2/datastore" + v2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/auth/permissions" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/grpc/authz" + "github.com/stackrox/rox/pkg/grpc/authz/perrpc" + "github.com/stackrox/rox/pkg/grpc/authz/user" + "github.com/stackrox/rox/pkg/protocompat" + "github.com/stackrox/rox/pkg/sac/resources" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/search/paginated" + "github.com/stackrox/rox/pkg/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + defaultPageSize = 100 +) + +var ( + authorizer = perrpc.FromMap(map[authz.Authorizer][]string{ + user.With(permissions.View(resources.VirtualMachine)): { + v2.VirtualMachineV2Service_ListVMs_FullMethodName, + v2.VirtualMachineV2Service_ListVMCVEs_FullMethodName, + v2.VirtualMachineV2Service_GetVMDashboardCounts_FullMethodName, + v2.VirtualMachineV2Service_GetVM_FullMethodName, + v2.VirtualMachineV2Service_GetVMVulnSummary_FullMethodName, + v2.VirtualMachineV2Service_ListVMCVEsByVM_FullMethodName, + v2.VirtualMachineV2Service_GetVMCVEComponents_FullMethodName, + v2.VirtualMachineV2Service_ListVMComponents_FullMethodName, + v2.VirtualMachineV2Service_GetVMCVEDetail_FullMethodName, + v2.VirtualMachineV2Service_ListVMCVEAffectedVMs_FullMethodName, + }, + }) +) + +type serviceImpl struct { + v2.UnimplementedVirtualMachineV2ServiceServer + + vmDS vmDS.DataStore + cveDS cveDS.DataStore + componentDS componentDS.DataStore + scanDS scanDS.DataStore + cveView vmcve.CveView +} + +// RegisterServiceServer registers this service with the given gRPC Server. +func (s *serviceImpl) RegisterServiceServer(grpcServer *grpc.Server) { + v2.RegisterVirtualMachineV2ServiceServer(grpcServer, s) +} + +// RegisterServiceHandler registers this service with the given gRPC Gateway endpoint. +func (s *serviceImpl) RegisterServiceHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { + return v2.RegisterVirtualMachineV2ServiceHandler(ctx, mux, conn) +} + +// AuthFuncOverride specifies the auth criteria for this API. +func (s *serviceImpl) AuthFuncOverride(ctx context.Context, fullMethodName string) (context.Context, error) { + return ctx, authorizer.Authorized(ctx, fullMethodName) +} + +// ListVMs returns a paginated list of VMs with severity counts. +func (s *serviceImpl) ListVMs(ctx context.Context, request *v2.ListVMsRequest) (*v2.ListVMsResponse, error) { + searchQuery, err := search.ParseQuery(request.GetQuery().GetQuery(), search.MatchAllIfEmpty()) + if err != nil { + return nil, errors.Wrap(err, "parsing input query") + } + paginated.FillPaginationV2(searchQuery, request.GetQuery().GetPagination(), defaultPageSize) + + countQuery := searchQuery.CloneVT() + countQuery.Pagination = nil + totalCount, err := s.vmDS.CountVirtualMachines(ctx, countQuery) + if err != nil { + return nil, err + } + + vms, err := s.vmDS.SearchRawVirtualMachines(ctx, searchQuery) + if err != nil { + return nil, err + } + if len(vms) == 0 { + return &v2.ListVMsResponse{TotalCount: int32(totalCount)}, nil + } + + vmIDs := make([]string, 0, len(vms)) + for _, vm := range vms { + vmIDs = append(vmIDs, vm.GetId()) + } + + scanTimeByVM, err := s.latestScanTimeByVM(ctx, vmIDs) + if err != nil { + return nil, err + } + + // Fetch per-VM CVE severity counts via SQL GROUP BY. + vmFilter := search.NewQueryBuilder().AddExactMatches(search.VirtualMachineID, vmIDs...).ProtoQuery() + severityRows, err := s.cveView.CountBySeverityPerVM(ctx, vmFilter) + if err != nil { + return nil, err + } + severityByVM := make(map[string]*v2.VulnCountBySeverity, len(severityRows)) + for _, row := range severityRows { + severityByVM[row.GetVMID()] = storagetov2.SeverityCountsToProto(row.GetSeverityCounts()) + } + + // Batch fetch component scan counts. The component Notes field has no dedicated + // column (no search tag), so SQL-level filtering of scanned vs unscanned is not + // possible and in-memory aggregation is used. + componentCountsByVM, err := s.batchComponentScanCounts(ctx, vmIDs) + if err != nil { + return nil, err + } + + items := make([]*v2.VMListItem, 0, len(vms)) + for _, vm := range vms { + item := storagetov2.VirtualMachineV2ToListItem(vm) + if counts, ok := severityByVM[vm.GetId()]; ok { + item.CveSeverityCounts = counts + } else { + item.CveSeverityCounts = &v2.VulnCountBySeverity{} + } + item.ComponentScanCount = componentCountsByVM[vm.GetId()] + item.ScanTime = scanTimeByVM[vm.GetId()] + items = append(items, item) + } + + return &v2.ListVMsResponse{ + Vms: items, + TotalCount: int32(totalCount), + }, nil +} + +// ListVMCVEs returns a paginated list of CVEs across all VMs. +func (s *serviceImpl) ListVMCVEs(ctx context.Context, request *v2.ListVMCVEsRequest) (*v2.ListVMCVEsResponse, error) { + searchQuery, err := search.ParseQuery(request.GetQuery().GetQuery(), search.MatchAllIfEmpty()) + if err != nil { + return nil, errors.Wrap(err, "parsing input query") + } + paginated.FillPaginationV2(searchQuery, request.GetQuery().GetPagination(), defaultPageSize) + + countQuery := searchQuery.CloneVT() + countQuery.Pagination = nil + totalCount, err := s.cveView.Count(ctx, countQuery) + if err != nil { + return nil, err + } + + cves, err := s.cveView.Get(ctx, searchQuery) + if err != nil { + return nil, err + } + + // Get total VM count for the response. + totalVMs, err := s.vmDS.CountVirtualMachines(ctx, search.EmptyQuery()) + if err != nil { + return nil, err + } + + items := make([]*v2.VMCVEListItem, 0, len(cves)) + for _, cve := range cves { + item := &v2.VMCVEListItem{ + Cve: cve.GetCVE(), + VmSeverityCounts: storagetov2.SeverityCountsToProto(cve.GetVMsBySeverity()), + TopCvss: cve.GetTopCVSS(), + AffectedVmCount: int32(cve.GetAffectedVMCount()), + TotalVmCount: int32(totalVMs), + EpssProbability: cve.GetEPSSProbability(), + PublishedOn: protocompat.ConvertTimeToTimestampOrNil(cve.GetPublishDate()), + } + items = append(items, item) + } + + return &v2.ListVMCVEsResponse{ + Cves: items, + TotalCount: int32(totalCount), + }, nil +} + +// GetVMDashboardCounts returns the total VM and CVE counts for the dashboard. +func (s *serviceImpl) GetVMDashboardCounts(ctx context.Context, request *v2.VMDashboardCountsRequest) (*v2.VMDashboardCountsResponse, error) { + searchQuery, err := search.ParseQuery(request.GetQuery().GetQuery(), search.MatchAllIfEmpty()) + if err != nil { + return nil, errors.Wrap(err, "parsing input query") + } + + vmCount, err := s.vmDS.CountVirtualMachines(ctx, searchQuery) + if err != nil { + return nil, err + } + + cveCount, err := s.cveView.Count(ctx, searchQuery) + if err != nil { + return nil, err + } + + return &v2.VMDashboardCountsResponse{ + VmCount: int32(vmCount), + CveCount: int32(cveCount), + }, nil +} + +// latestScanTimeByVM returns the latest scan timestamp per VM. Scan IDs are +// UUIDv7, so the greatest ID is the latest scan (same rule as GetVM). +func (s *serviceImpl) latestScanTimeByVM(ctx context.Context, vmIDs []string) (map[string]*timestamppb.Timestamp, error) { + result := make(map[string]*timestamppb.Timestamp, len(vmIDs)) + if len(vmIDs) == 0 { + return result, nil + } + + q := search.NewQueryBuilder().AddExactMatches(search.VirtualMachineID, vmIDs...).ProtoQuery() + scans, err := s.scanDS.SearchRawVMScans(ctx, q) + if err != nil { + return nil, err + } + + latestID := make(map[string]string, len(scans)) + for _, scan := range scans { + vmID := scan.GetVmV2Id() + if prevID, ok := latestID[vmID]; ok && scan.GetId() <= prevID { + continue + } + latestID[vmID] = scan.GetId() + result[vmID] = scan.GetScanTime() + } + return result, nil +} + +// batchComponentScanCounts fetches all components for the given VM IDs in one query +// and counts total vs scanned per VM in memory. The component Notes field is not +// search-indexed, so SQL-level aggregation of scanned vs unscanned is not possible. +func (s *serviceImpl) batchComponentScanCounts(ctx context.Context, vmIDs []string) (map[string]*v2.ComponentScanCount, error) { + result := make(map[string]*v2.ComponentScanCount, len(vmIDs)) + for _, id := range vmIDs { + result[id] = &v2.ComponentScanCount{} + } + if len(vmIDs) == 0 { + return result, nil + } + + q := search.NewQueryBuilder().AddExactMatches(search.VirtualMachineID, vmIDs...).ProtoQuery() + components, err := s.componentDS.SearchRawVMComponents(ctx, q) + if err != nil { + return nil, err + } + + // Components link to scans, not VMs directly. We need to resolve the VM ID + // via the CVE table's vm_v2_id FK. Instead, count components per scan and + // map scans to VMs. + // However, SearchRawVMComponents with a VirtualMachineID filter already + // joins through the scan table, so we get the right components. + // We need the scan-to-VM mapping to bucket by VM. + scanIDs := make(map[string]struct{}) + compsByScan := make(map[string][]*storage.VirtualMachineComponentV2) + for _, comp := range components { + scanIDs[comp.GetVmScanId()] = struct{}{} + compsByScan[comp.GetVmScanId()] = append(compsByScan[comp.GetVmScanId()], comp) + } + + scanIDList := make([]string, 0, len(scanIDs)) + for id := range scanIDs { + scanIDList = append(scanIDList, id) + } + scans, err := s.scanDS.GetBatch(ctx, scanIDList) + if err != nil { + return nil, err + } + + scanToVM := make(map[string]string, len(scans)) + for _, scan := range scans { + scanToVM[scan.GetId()] = scan.GetVmV2Id() + } + + for scanID, comps := range compsByScan { + vmID, ok := scanToVM[scanID] + if !ok { + continue + } + counts, ok := result[vmID] + if !ok { + continue + } + for _, comp := range comps { + counts.Total++ + scanned := !slices.Contains(comp.GetNotes(), storage.VirtualMachineComponentV2_UNSCANNED) + if scanned { + counts.Scanned++ + } + } + } + + return result, nil +} + +// GetVMVulnSummary returns vulnerability severity counts for a single VM. +func (s *serviceImpl) GetVMVulnSummary(ctx context.Context, request *v2.GetVMVulnSummaryRequest) (*v2.VMVulnSummary, error) { + if request.GetId() == "" { + return nil, status.Error(codes.InvalidArgument, "id must be specified") + } + if _, err := uuid.FromString(request.GetId()); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "id must be a valid UUID (got: %q)", request.GetId()) + } + + vmQuery := search.NewQueryBuilder().AddExactMatches(search.VirtualMachineID, request.GetId()).ProtoQuery() + count, err := s.vmDS.CountVirtualMachines(ctx, vmQuery) + if err != nil { + return nil, err + } + if count == 0 { + return nil, status.Errorf(codes.NotFound, "virtual machine %q not found", request.GetId()) + } + + vmFilter := vmQuery.CloneVT() + if request.GetQuery().GetQuery() != "" { + additionalQuery, err := search.ParseQuery(request.GetQuery().GetQuery()) + if err != nil { + return nil, errors.Wrap(err, "parsing input query") + } + vmFilter = search.ConjunctionQuery(vmFilter, additionalQuery) + } + + severityCounts, err := s.cveView.CountBySeverity(ctx, vmFilter, search.CVE) + if err != nil { + return nil, err + } + + proto := storagetov2.SeverityCountsToProto(severityCounts) + fixable, notFixable := countFixability(proto) + + return &v2.VMVulnSummary{ + SeverityCounts: proto, + FixableCount: fixable, + NotFixableCount: notFixable, + }, nil +} + +// ListVMCVEsByVM returns a paginated list of CVEs affecting a specific VM. +func (s *serviceImpl) ListVMCVEsByVM(ctx context.Context, request *v2.ListVMCVEsByVMRequest) (*v2.ListVMCVEsByVMResponse, error) { + if request.GetVmId() == "" { + return nil, status.Error(codes.InvalidArgument, "vm_id must be specified") + } + if _, err := uuid.FromString(request.GetVmId()); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "vm_id must be a valid UUID (got: %q)", request.GetVmId()) + } + + searchQuery, err := search.ParseQuery(request.GetQuery().GetQuery(), search.MatchAllIfEmpty()) + if err != nil { + return nil, errors.Wrap(err, "parsing input query") + } + searchQuery = search.ConjunctionQuery( + searchQuery, + search.NewQueryBuilder().AddExactMatches(search.VirtualMachineID, request.GetVmId()).ProtoQuery(), + ) + paginated.FillPaginationV2(searchQuery, request.GetQuery().GetPagination(), defaultPageSize) + + countQuery := searchQuery.CloneVT() + countQuery.Pagination = nil + totalCount, err := s.cveView.Count(ctx, countQuery) + if err != nil { + return nil, err + } + + cves, err := s.cveView.GetCVEsForVM(ctx, searchQuery) + if err != nil { + return nil, err + } + + // Fetch one raw storage row per CVE for metadata not available through + // the view (summary, link). These are stored in the serialized blob. + cveIDs := make([]string, 0, len(cves)) + for _, cve := range cves { + cveIDs = append(cveIDs, cve.GetCVE()) + } + metadataByID, err := s.fetchCVEMetadata(ctx, request.GetVmId(), cveIDs) + if err != nil { + return nil, err + } + + items := make([]*v2.VMCVERow, 0, len(cves)) + for _, cve := range cves { + row := &v2.VMCVERow{ + Cve: cve.GetCVE(), + Severity: v2.VulnerabilitySeverity(cve.GetMaxSeverity()), + IsFixable: cve.GetIsFixable(), + Cvss: cve.GetMaxCVSS(), + NvdCvss: cve.GetMaxNVDCVSS(), + EpssProbability: cve.GetEPSSProbability(), + AffectedComponentCount: int32(cve.GetAffectedComponentCount()), + PublishedOn: protocompat.ConvertTimeToTimestampOrNil(cve.GetPublishDate()), + } + if meta, ok := metadataByID[cve.GetCVE()]; ok { + row.Summary = meta.GetCveBaseInfo().GetSummary() + row.Link = meta.GetCveBaseInfo().GetLink() + if adv := meta.GetAdvisory(); adv != nil && adv.GetName() != "" { + row.Advisory = &v2.Advisory{ + Name: adv.GetName(), + Link: adv.GetLink(), + } + } + } + items = append(items, row) + } + + return &v2.ListVMCVEsByVMResponse{ + Cves: items, + TotalCount: int32(totalCount), + }, nil +} + +// fetchCVEMetadata fetches one raw storage row per CVE for metadata fields +// (summary, link) that aren't available through the SQL view. +func (s *serviceImpl) fetchCVEMetadata(ctx context.Context, vmID string, cveIDs []string) (map[string]*storage.VirtualMachineCVEV2, error) { + if len(cveIDs) == 0 { + return nil, nil + } + q := search.ConjunctionQuery( + search.NewQueryBuilder().AddExactMatches(search.VirtualMachineID, vmID).ProtoQuery(), + search.NewQueryBuilder().AddExactMatches(search.CVE, cveIDs...).ProtoQuery(), + ) + rows, err := s.cveDS.SearchRawVMCVEs(ctx, q) + if err != nil { + return nil, err + } + result := make(map[string]*storage.VirtualMachineCVEV2, len(cveIDs)) + for _, row := range rows { + cveID := row.GetCveBaseInfo().GetCve() + if _, exists := result[cveID]; !exists { + result[cveID] = row + } + } + return result, nil +} + +// GetVMCVEComponents returns components affected by a specific CVE on a specific VM. +func (s *serviceImpl) GetVMCVEComponents(ctx context.Context, request *v2.GetVMCVEComponentsRequest) (*v2.GetVMCVEComponentsResponse, error) { + if request.GetVmId() == "" || request.GetCveId() == "" { + return nil, status.Error(codes.InvalidArgument, "vm_id and cve_id must be specified") + } + if _, err := uuid.FromString(request.GetVmId()); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "vm_id must be a valid UUID (got: %q)", request.GetVmId()) + } + + q := search.ConjunctionQuery( + search.NewQueryBuilder().AddExactMatches(search.VirtualMachineID, request.GetVmId()).ProtoQuery(), + search.NewQueryBuilder().AddExactMatches(search.CVE, request.GetCveId()).ProtoQuery(), + ) + + components, err := s.cveView.GetCVEComponents(ctx, q) + if err != nil { + return nil, err + } + + rows := make([]*v2.VMCVEComponentRow, 0, len(components)) + for _, comp := range components { + var advisory *v2.Advisory + if comp.GetAdvisoryName() != "" { + advisory = &v2.Advisory{ + Name: comp.GetAdvisoryName(), + Link: comp.GetAdvisoryLink(), + } + } + rows = append(rows, &v2.VMCVEComponentRow{ + ComponentName: comp.GetComponentName(), + ComponentVersion: comp.GetComponentVersion(), + Source: v2.SourceType(comp.GetComponentSource()), + FixedBy: comp.GetFixedBy(), + Advisory: advisory, + }) + } + + return &v2.GetVMCVEComponentsResponse{ + Components: rows, + }, nil +} + +// ListVMComponents returns a paginated list of components for a specific VM. +func (s *serviceImpl) ListVMComponents(ctx context.Context, request *v2.ListVMComponentsRequest) (*v2.ListVMComponentsResponse, error) { + if request.GetVmId() == "" { + return nil, status.Error(codes.InvalidArgument, "vm_id must be specified") + } + if _, err := uuid.FromString(request.GetVmId()); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "vm_id must be a valid UUID (got: %q)", request.GetVmId()) + } + + searchQuery, err := search.ParseQuery(request.GetQuery().GetQuery(), search.MatchAllIfEmpty()) + if err != nil { + return nil, errors.Wrap(err, "parsing input query") + } + searchQuery = search.ConjunctionQuery( + searchQuery, + search.NewQueryBuilder().AddExactMatches(search.VirtualMachineID, request.GetVmId()).ProtoQuery(), + ) + paginated.FillPaginationV2(searchQuery, request.GetQuery().GetPagination(), defaultPageSize) + + countQuery := searchQuery.CloneVT() + countQuery.Pagination = nil + totalCount, err := s.componentDS.Count(ctx, countQuery) + if err != nil { + return nil, err + } + + components, err := s.componentDS.SearchRawVMComponents(ctx, searchQuery) + if err != nil { + return nil, err + } + + items := make([]*v2.VMComponentRow, 0, len(components)) + for _, comp := range components { + items = append(items, storagetov2.VirtualMachineComponentV2ToRow(comp)) + } + + return &v2.ListVMComponentsResponse{ + Components: items, + TotalCount: int32(totalCount), + }, nil +} + +// countFixability sums fixable and not-fixable counts across all severity levels. +func countFixability(counts *v2.VulnCountBySeverity) (fixable, notFixable int32) { + for _, sev := range []func() *v2.VulnFixableCount{ + counts.GetCritical, + counts.GetImportant, + counts.GetModerate, + counts.GetLow, + counts.GetUnknown, + } { + c := sev() + fixable += c.GetFixable() + notFixable += c.GetTotal() - c.GetFixable() + } + return +} + +// GetVMCVEDetail returns detailed information about a specific CVE across all VMs. +func (s *serviceImpl) GetVMCVEDetail(ctx context.Context, request *v2.GetVMCVEDetailRequest) (*v2.VMCVEDetail, error) { + if request.GetCveId() == "" { + return nil, status.Error(codes.InvalidArgument, "cve_id must be specified") + } + + // Look up by CVE identifier (e.g. "CVE-2024-1234"), not by internal UUID. + cveFilter := search.NewQueryBuilder().AddExactMatches(search.CVE, request.GetCveId()).ProtoQuery() + cves, err := s.cveDS.SearchRawVMCVEs(ctx, cveFilter) + if err != nil { + return nil, err + } + if len(cves) == 0 { + return nil, status.Errorf(codes.NotFound, "CVE %q not found", request.GetCveId()) + } + cve := cves[0] + + viewFilter := cveFilter.CloneVT() + if request.GetQuery().GetQuery() != "" { + additionalQuery, err := search.ParseQuery(request.GetQuery().GetQuery()) + if err != nil { + return nil, errors.Wrap(err, "parsing input query") + } + viewFilter = search.ConjunctionQuery(viewFilter, additionalQuery) + } + + severityCounts, err := s.cveView.CountBySeverity(ctx, viewFilter, search.VirtualMachineID) + if err != nil { + return nil, err + } + + // Get affected VM count. + affectedVMIDs, err := s.cveView.GetVMIDs(ctx, viewFilter) + if err != nil { + return nil, err + } + + totalVMs, err := s.vmDS.CountVirtualMachines(ctx, search.EmptyQuery()) + if err != nil { + return nil, err + } + + // Count distinct guest OSes among affected VMs. + affectedGuestOSCount := 0 + if len(affectedVMIDs) > 0 { + affectedVMs, _, err := s.vmDS.GetManyVirtualMachines(ctx, affectedVMIDs) + if err != nil { + return nil, err + } + guestOSSet := make(map[string]struct{}) + for _, vm := range affectedVMs { + if os := vm.GetGuestOs(); os != "" { + guestOSSet[os] = struct{}{} + } + } + affectedGuestOSCount = len(guestOSSet) + } + + return &v2.VMCVEDetail{ + Cve: cve.GetCveBaseInfo().GetCve(), + Summary: cve.GetCveBaseInfo().GetSummary(), + Link: cve.GetCveBaseInfo().GetLink(), + EpssProbability: cve.GetEpssProbability(), + PublishedOn: cve.GetCveBaseInfo().GetPublishedOn(), + FirstDiscovered: cve.GetCveBaseInfo().GetCreatedAt(), + AffectedVmCount: int32(len(affectedVMIDs)), + TotalVmCount: int32(totalVMs), + AffectedGuestOsCount: int32(affectedGuestOSCount), + VmSeverityCounts: storagetov2.SeverityCountsToProto(severityCounts), + TopCvss: cve.GetPreferredCvss(), + }, nil +} + +// ListVMCVEAffectedVMs returns VMs affected by a specific CVE with pagination. +func (s *serviceImpl) ListVMCVEAffectedVMs(ctx context.Context, request *v2.ListVMCVEAffectedVMsRequest) (*v2.ListVMCVEAffectedVMsResponse, error) { + if request.GetCveId() == "" { + return nil, status.Error(codes.InvalidArgument, "cve_id must be specified") + } + + searchQuery, err := search.ParseQuery(request.GetQuery().GetQuery(), search.MatchAllIfEmpty()) + if err != nil { + return nil, errors.Wrap(err, "parsing input query") + } + searchQuery = search.ConjunctionQuery( + searchQuery, + search.NewQueryBuilder().AddExactMatches(search.CVE, request.GetCveId()).ProtoQuery(), + ) + paginated.FillPaginationV2(searchQuery, request.GetQuery().GetPagination(), defaultPageSize) + searchQuery = common.UpdateSortAggs(searchQuery) + + countQuery := searchQuery.CloneVT() + countQuery.Pagination = nil + totalCount, err := s.cveView.CountAffectedVMs(ctx, countQuery) + if err != nil { + return nil, err + } + + affectedVMs, err := s.cveView.GetAffectedVMs(ctx, searchQuery) + if err != nil { + return nil, err + } + + rows := make([]*v2.VMCVEAffectedVMRow, 0, len(affectedVMs)) + for _, vm := range affectedVMs { + rows = append(rows, &v2.VMCVEAffectedVMRow{ + VmId: vm.GetVMID(), + VmName: vm.GetVMName(), + Severity: v2.VulnerabilitySeverity(vm.GetMaxSeverity()), + IsFixable: vm.GetIsFixable(), + Cvss: vm.GetMaxCVSS(), + GuestOs: vm.GetGuestOS(), + AffectedComponentCount: int32(vm.GetAffectedComponentCount()), + }) + } + + return &v2.ListVMCVEAffectedVMsResponse{ + Vms: rows, + TotalCount: int32(totalCount), + }, nil +} + +// GetVM returns detailed information about a single VM. +func (s *serviceImpl) GetVM(ctx context.Context, request *v2.GetVMRequest) (*v2.VMDetail, error) { + if request.GetId() == "" { + return nil, status.Error(codes.InvalidArgument, "id must be specified") + } + if _, err := uuid.FromString(request.GetId()); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "id must be a valid UUID (got: %q)", request.GetId()) + } + + vmWithScan, exists, err := s.vmDS.GetVirtualMachineWithLatestScan(ctx, request.GetId()) + if err != nil { + return nil, err + } + if !exists { + return nil, status.Errorf(codes.NotFound, "virtual machine %q not found", request.GetId()) + } + + vm := vmWithScan.VM + detail := storagetov2.VirtualMachineV2ToDetail(vm) + detail.AgentStatus = storagetov2.AgentStatusFromLastContact( + vm.GetLastAgentContact(), + time.Now(), + env.VirtualMachinesAgentStaleAfter.DurationSetting(), + ) + + if scan := vmWithScan.Scan; scan != nil { + scanNotes := make([]v2.VMScanNote, 0, len(scan.GetNotes())) + for _, n := range scan.GetNotes() { + scanNotes = append(scanNotes, storagetov2.ConvertScanNote(n)) + } + detail.LatestScan = &v2.VMScanInfo{ + ScanId: scan.GetId(), + ScanOs: scan.GetScanOs(), + ScanTime: scan.GetScanTime(), + TopCvss: scan.GetTopCvss(), + ScanNotes: scanNotes, + } + } + + return detail, nil +} diff --git a/central/virtualmachine/v2/service/service_impl_test.go b/central/virtualmachine/v2/service/service_impl_test.go new file mode 100644 index 0000000000000..1dd1ae8eb09ea --- /dev/null +++ b/central/virtualmachine/v2/service/service_impl_test.go @@ -0,0 +1,1160 @@ +package service + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stackrox/rox/central/convert/storagetov2" + commonViews "github.com/stackrox/rox/central/views/common" + "github.com/stackrox/rox/central/views/vmcve" + cveViewMocks "github.com/stackrox/rox/central/views/vmcve/mocks" + componentDSMocks "github.com/stackrox/rox/central/virtualmachine/component/v2/datastore/mocks" + cveDSMocks "github.com/stackrox/rox/central/virtualmachine/cve/v2/datastore/mocks" + scanDSMocks "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore/mocks" + vmDSMocks "github.com/stackrox/rox/central/virtualmachine/v2/datastore/mocks" + "github.com/stackrox/rox/central/virtualmachine/v2/datastore/store/common" + v2 "github.com/stackrox/rox/generated/api/v2" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/fixtures/fixtureconsts" + "github.com/stackrox/rox/pkg/grpc/testutils" + "github.com/stackrox/rox/pkg/protoassert" + "github.com/stackrox/rox/pkg/search" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type testCVEForVM struct { + cve string + maxSeverity int32 + isFixable bool + maxCVSS float32 + maxNVDCVSS float32 + epssProbability float32 + componentCount int + publishDate *time.Time +} + +func (t *testCVEForVM) GetCVE() string { return t.cve } +func (t *testCVEForVM) GetMaxSeverity() int32 { return t.maxSeverity } +func (t *testCVEForVM) GetIsFixable() bool { return t.isFixable } +func (t *testCVEForVM) GetMaxCVSS() float32 { return t.maxCVSS } +func (t *testCVEForVM) GetMaxNVDCVSS() float32 { return t.maxNVDCVSS } +func (t *testCVEForVM) GetEPSSProbability() float32 { return t.epssProbability } +func (t *testCVEForVM) GetAffectedComponentCount() int { return t.componentCount } +func (t *testCVEForVM) GetPublishDate() *time.Time { return t.publishDate } + +func TestAuthz(t *testing.T) { + testutils.AssertAuthzWorks(t, &serviceImpl{}) +} + +func TestGetVMDashboardCounts(t *testing.T) { + ctx := context.Background() + + tests := map[string]struct { + setupMock func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) + expectedError string + expectedVM int32 + expectedCVE int32 + }{ + "successful counts": { + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(5, nil) + mockView.EXPECT().Count(ctx, gomock.Any()).Return(10, nil) + }, + expectedVM: 5, + expectedCVE: 10, + }, + "vm count error": { + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(0, errors.New("vm count failed")) + }, + expectedError: "vm count failed", + }, + "cve count error": { + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(5, nil) + mockView.EXPECT().Count(ctx, gomock.Any()).Return(0, errors.New("cve count failed")) + }, + expectedError: "cve count failed", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockVM := vmDSMocks.NewMockDataStore(ctrl) + mockCVE := cveDSMocks.NewMockDataStore(ctrl) + mockComp := componentDSMocks.NewMockDataStore(ctrl) + mockScan := scanDSMocks.NewMockDataStore(ctrl) + mockView := cveViewMocks.NewMockCveView(ctrl) + + service := &serviceImpl{ + vmDS: mockVM, + cveDS: mockCVE, + componentDS: mockComp, + scanDS: mockScan, + cveView: mockView, + } + + tt.setupMock(mockVM, mockCVE, mockComp, mockScan, mockView) + + result, err := service.GetVMDashboardCounts(ctx, &v2.VMDashboardCountsRequest{}) + + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + assert.Nil(t, result) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedVM, result.GetVmCount()) + assert.Equal(t, tt.expectedCVE, result.GetCveCount()) + } + }) + } +} + +func TestGetVMVulnSummary(t *testing.T) { + ctx := context.Background() + + tests := map[string]struct { + request *v2.GetVMVulnSummaryRequest + setupMock func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) + expectedError string + }{ + "empty id": { + request: &v2.GetVMVulnSummaryRequest{ + Id: "", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "id must be specified", + }, + "invalid UUID": { + request: &v2.GetVMVulnSummaryRequest{ + Id: "not-a-uuid", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "id must be a valid UUID", + }, + "vm not found": { + request: &v2.GetVMVulnSummaryRequest{ + Id: fixtureconsts.VirtualMachine1, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(0, nil) + }, + expectedError: "not found", + }, + "successful with severity counts": { + request: &v2.GetVMVulnSummaryRequest{ + Id: fixtureconsts.VirtualMachine1, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(1, nil) + mockView.EXPECT().CountBySeverity(ctx, gomock.Any(), search.CVE).Return(&commonViews.ResourceCountByImageCVESeverity{ + CriticalSeverityCount: 2, + FixableCriticalSeverityCount: 1, + ImportantSeverityCount: 3, + FixableImportantSeverityCount: 2, + }, nil) + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockVM := vmDSMocks.NewMockDataStore(ctrl) + mockCVE := cveDSMocks.NewMockDataStore(ctrl) + mockComp := componentDSMocks.NewMockDataStore(ctrl) + mockScan := scanDSMocks.NewMockDataStore(ctrl) + mockView := cveViewMocks.NewMockCveView(ctrl) + + service := &serviceImpl{ + vmDS: mockVM, + cveDS: mockCVE, + componentDS: mockComp, + scanDS: mockScan, + cveView: mockView, + } + + tt.setupMock(mockVM, mockCVE, mockComp, mockScan, mockView) + + result, err := service.GetVMVulnSummary(ctx, tt.request) + + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + assert.Nil(t, result) + } else { + require.NoError(t, err) + assert.Equal(t, int32(2), result.GetSeverityCounts().GetCritical().GetTotal()) + assert.Equal(t, int32(1), result.GetSeverityCounts().GetCritical().GetFixable()) + assert.Equal(t, int32(3), result.GetSeverityCounts().GetImportant().GetTotal()) + assert.Equal(t, int32(2), result.GetSeverityCounts().GetImportant().GetFixable()) + // fixable: 1 (critical) + 2 (important) = 3 + assert.Equal(t, int32(3), result.GetFixableCount()) + // not fixable: (2-1) + (3-2) = 2 + assert.Equal(t, int32(2), result.GetNotFixableCount()) + } + }) + } +} + +func TestListVMCVEsByVM(t *testing.T) { + ctx := context.Background() + + cve1Raw := &storage.VirtualMachineCVEV2{ + Id: "cve-uuid-1", + VmV2Id: "vm-1", + VmComponentId: "comp-1", + CveBaseInfo: &storage.CVEInfo{ + Cve: "CVE-2024-1234", + Summary: "test vuln 1", + Link: "https://example.com/1", + }, + PreferredCvss: 7.5, + Severity: storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, + IsFixable: true, + EpssProbability: 0.5, + } + + cve1View := &testCVEForVM{ + cve: "CVE-2024-1234", + maxSeverity: int32(storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY), + isFixable: true, + maxCVSS: 7.5, + componentCount: 3, + } + + tests := map[string]struct { + request *v2.ListVMCVEsByVMRequest + setupMock func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) + expectedError string + expectedCount int32 + }{ + "empty vm_id": { + request: &v2.ListVMCVEsByVMRequest{ + VmId: "", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "vm_id must be specified", + }, + "invalid UUID": { + request: &v2.ListVMCVEsByVMRequest{ + VmId: "not-a-uuid", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "vm_id must be a valid UUID", + }, + "successful list with CVEs": { + request: &v2.ListVMCVEsByVMRequest{ + VmId: fixtureconsts.VirtualMachine1, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockView.EXPECT().Count(ctx, gomock.Any()).Return(1, nil) + mockView.EXPECT().GetCVEsForVM(ctx, gomock.Any()).Return([]vmcve.CVEForVMCore{cve1View}, nil) + mockCVE.EXPECT().SearchRawVMCVEs(ctx, gomock.Any()).Return([]*storage.VirtualMachineCVEV2{cve1Raw}, nil) + }, + expectedCount: 1, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockVM := vmDSMocks.NewMockDataStore(ctrl) + mockCVE := cveDSMocks.NewMockDataStore(ctrl) + mockComp := componentDSMocks.NewMockDataStore(ctrl) + mockScan := scanDSMocks.NewMockDataStore(ctrl) + mockView := cveViewMocks.NewMockCveView(ctrl) + + service := &serviceImpl{ + vmDS: mockVM, + cveDS: mockCVE, + componentDS: mockComp, + scanDS: mockScan, + cveView: mockView, + } + + tt.setupMock(mockVM, mockCVE, mockComp, mockScan, mockView) + + result, err := service.ListVMCVEsByVM(ctx, tt.request) + + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + assert.Nil(t, result) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedCount, result.GetTotalCount()) + if len(result.GetCves()) > 0 { + row := result.GetCves()[0] + assert.Equal(t, "CVE-2024-1234", row.GetCve()) + assert.Equal(t, v2.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, row.GetSeverity()) + assert.True(t, row.GetIsFixable()) + assert.Equal(t, float32(7.5), row.GetCvss()) + assert.Equal(t, int32(3), row.GetAffectedComponentCount()) + assert.Equal(t, "test vuln 1", row.GetSummary()) + assert.Equal(t, "https://example.com/1", row.GetLink()) + } + } + }) + } +} + +func TestGetVMCVEComponents(t *testing.T) { + ctx := context.Background() + + tests := map[string]struct { + request *v2.GetVMCVEComponentsRequest + setupMock func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) + expectedError string + }{ + "empty vm_id": { + request: &v2.GetVMCVEComponentsRequest{ + VmId: "", + CveId: "cve-1", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "vm_id and cve_id must be specified", + }, + "empty cve_id": { + request: &v2.GetVMCVEComponentsRequest{ + VmId: fixtureconsts.VirtualMachine1, + CveId: "", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "vm_id and cve_id must be specified", + }, + "invalid UUID for vm_id": { + request: &v2.GetVMCVEComponentsRequest{ + VmId: "not-a-uuid", + CveId: "CVE-2024-1234", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "vm_id must be a valid UUID", + }, + "view error": { + request: &v2.GetVMCVEComponentsRequest{ + VmId: fixtureconsts.VirtualMachine1, + CveId: "CVE-2024-1234", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockView.EXPECT().GetCVEComponents(ctx, gomock.Any()).Return(nil, errors.New("view error")) + }, + expectedError: "view error", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockVM := vmDSMocks.NewMockDataStore(ctrl) + mockCVE := cveDSMocks.NewMockDataStore(ctrl) + mockComp := componentDSMocks.NewMockDataStore(ctrl) + mockScan := scanDSMocks.NewMockDataStore(ctrl) + mockView := cveViewMocks.NewMockCveView(ctrl) + + service := &serviceImpl{ + vmDS: mockVM, + cveDS: mockCVE, + componentDS: mockComp, + scanDS: mockScan, + cveView: mockView, + } + + tt.setupMock(mockVM, mockCVE, mockComp, mockScan, mockView) + + result, err := service.GetVMCVEComponents(ctx, tt.request) + + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + assert.Nil(t, result) + } else { + require.NoError(t, err) + require.NotEmpty(t, result.GetComponents()) + } + }) + } + + // Verify specific field values for the success case. + t.Run("successful with components - field values", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockView := cveViewMocks.NewMockCveView(ctrl) + comp := cveViewMocks.NewMockCVEComponentCore(ctrl) + comp.EXPECT().GetComponentName().Return("openssl").AnyTimes() + comp.EXPECT().GetComponentVersion().Return("1.1.1").AnyTimes() + comp.EXPECT().GetComponentSource().Return(int32(storage.SourceType_OS)).AnyTimes() + comp.EXPECT().GetFixedBy().Return("1.2.3").AnyTimes() + comp.EXPECT().GetAdvisoryName().Return("RHSA-2024:1234").AnyTimes() + comp.EXPECT().GetAdvisoryLink().Return("https://access.redhat.com").AnyTimes() + mockView.EXPECT().GetCVEComponents(ctx, gomock.Any()).Return([]vmcve.CVEComponentCore{comp}, nil) + + svc := &serviceImpl{cveView: mockView} + result, err := svc.GetVMCVEComponents(ctx, &v2.GetVMCVEComponentsRequest{VmId: fixtureconsts.VirtualMachine1, CveId: "CVE-2024-1234"}) + require.NoError(t, err) + require.Len(t, result.GetComponents(), 1) + row := result.GetComponents()[0] + assert.Equal(t, "openssl", row.GetComponentName()) + assert.Equal(t, "1.1.1", row.GetComponentVersion()) + assert.Equal(t, v2.SourceType_OS, row.GetSource()) + assert.Equal(t, "1.2.3", row.GetFixedBy()) + assert.Equal(t, "RHSA-2024:1234", row.GetAdvisory().GetName()) + }) + + // Verify nil advisory when advisory name is empty. + t.Run("nil advisory when empty", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockView := cveViewMocks.NewMockCveView(ctrl) + comp := cveViewMocks.NewMockCVEComponentCore(ctrl) + comp.EXPECT().GetComponentName().Return("curl").AnyTimes() + comp.EXPECT().GetComponentVersion().Return("7.0").AnyTimes() + comp.EXPECT().GetComponentSource().Return(int32(storage.SourceType_OS)).AnyTimes() + comp.EXPECT().GetFixedBy().Return("").AnyTimes() + comp.EXPECT().GetAdvisoryName().Return("").AnyTimes() + comp.EXPECT().GetAdvisoryLink().Return("").AnyTimes() + mockView.EXPECT().GetCVEComponents(ctx, gomock.Any()).Return([]vmcve.CVEComponentCore{comp}, nil) + + svc := &serviceImpl{cveView: mockView} + result, err := svc.GetVMCVEComponents(ctx, &v2.GetVMCVEComponentsRequest{VmId: fixtureconsts.VirtualMachine1, CveId: "CVE-2024-5678"}) + require.NoError(t, err) + require.Len(t, result.GetComponents(), 1) + assert.Nil(t, result.GetComponents()[0].GetAdvisory()) + }) +} + +func TestListVMComponents(t *testing.T) { + ctx := context.Background() + + comp1 := &storage.VirtualMachineComponentV2{ + Id: "comp-1", + Name: "openssl", + Version: "1.1.1", + Source: storage.SourceType_OS, + CveCount: 2, + } + + tests := map[string]struct { + request *v2.ListVMComponentsRequest + setupMock func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) + expectedError string + expectedCount int32 + }{ + "empty vm_id": { + request: &v2.ListVMComponentsRequest{ + VmId: "", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "vm_id must be specified", + }, + "invalid UUID": { + request: &v2.ListVMComponentsRequest{ + VmId: "not-a-uuid", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "vm_id must be a valid UUID", + }, + "successful list": { + request: &v2.ListVMComponentsRequest{ + VmId: fixtureconsts.VirtualMachine1, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockComp.EXPECT().Count(ctx, gomock.Any()).Return(1, nil) + mockComp.EXPECT().SearchRawVMComponents(ctx, gomock.Any()).Return([]*storage.VirtualMachineComponentV2{comp1}, nil) + }, + expectedCount: 1, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockVM := vmDSMocks.NewMockDataStore(ctrl) + mockCVE := cveDSMocks.NewMockDataStore(ctrl) + mockComp := componentDSMocks.NewMockDataStore(ctrl) + mockScan := scanDSMocks.NewMockDataStore(ctrl) + mockView := cveViewMocks.NewMockCveView(ctrl) + + service := &serviceImpl{ + vmDS: mockVM, + cveDS: mockCVE, + componentDS: mockComp, + scanDS: mockScan, + cveView: mockView, + } + + tt.setupMock(mockVM, mockCVE, mockComp, mockScan, mockView) + + result, err := service.ListVMComponents(ctx, tt.request) + + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + assert.Nil(t, result) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedCount, result.GetTotalCount()) + if len(result.GetComponents()) > 0 { + row := result.GetComponents()[0] + assert.Equal(t, "openssl", row.GetName()) + assert.Equal(t, "1.1.1", row.GetVersion()) + assert.Equal(t, v2.SourceType_OS, row.GetSource()) + assert.Equal(t, int32(2), row.GetCveCount()) + } + } + }) + } +} + +func TestGetVMCVEDetail(t *testing.T) { + ctx := context.Background() + + cve1 := &storage.VirtualMachineCVEV2{ + Id: "cve-uuid-1", + CveBaseInfo: &storage.CVEInfo{ + Cve: "CVE-2024-1234", + Summary: "test vulnerability", + Link: "https://example.com", + }, + PreferredCvss: 7.5, + EpssProbability: 0.5, + } + + tests := map[string]struct { + request *v2.GetVMCVEDetailRequest + setupMock func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) + expectedError string + expectedResult *v2.VMCVEDetail + }{ + "empty cve_id": { + request: &v2.GetVMCVEDetailRequest{ + CveId: "", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "cve_id must be specified", + }, + "CVE not found": { + request: &v2.GetVMCVEDetailRequest{ + CveId: "CVE-2024-1234", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockCVE.EXPECT().SearchRawVMCVEs(ctx, gomock.Any()).Return(nil, nil) + }, + expectedError: "not found", + }, + "successful detail": { + request: &v2.GetVMCVEDetailRequest{ + CveId: "CVE-2024-1234", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockCVE.EXPECT().SearchRawVMCVEs(ctx, gomock.Any()).Return([]*storage.VirtualMachineCVEV2{cve1}, nil) + mockView.EXPECT().CountBySeverity(ctx, gomock.Any(), search.VirtualMachineID).Return(&commonViews.ResourceCountByImageCVESeverity{ + CriticalSeverityCount: 2, + }, nil) + mockView.EXPECT().GetVMIDs(ctx, gomock.Any()).Return([]string{"vm-1"}, nil) + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(5, nil) + mockVM.EXPECT().GetManyVirtualMachines(ctx, []string{"vm-1"}).Return([]*storage.VirtualMachineV2{ + {Id: "vm-1", GuestOs: "rhel-9"}, + }, nil, nil) + }, + }, + "successful detail with query": { + request: &v2.GetVMCVEDetailRequest{ + CveId: "CVE-2024-1234", + Query: &v2.RawQuery{Query: "Severity:CRITICAL_VULNERABILITY_SEVERITY"}, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockCVE.EXPECT().SearchRawVMCVEs(ctx, gomock.Any()).Return([]*storage.VirtualMachineCVEV2{cve1}, nil) + mockView.EXPECT().CountBySeverity(ctx, gomock.Any(), search.VirtualMachineID).Return(&commonViews.ResourceCountByImageCVESeverity{ + CriticalSeverityCount: 2, + }, nil) + mockView.EXPECT().GetVMIDs(ctx, gomock.Any()).Return([]string{"vm-1"}, nil) + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(5, nil) + mockVM.EXPECT().GetManyVirtualMachines(ctx, []string{"vm-1"}).Return([]*storage.VirtualMachineV2{ + {Id: "vm-1", GuestOs: "rhel-9"}, + }, nil, nil) + }, + expectedResult: &v2.VMCVEDetail{ + AffectedVmCount: 1, + TotalVmCount: 5, + AffectedGuestOsCount: 1, + VmSeverityCounts: storagetov2.SeverityCountsToProto(&commonViews.ResourceCountByImageCVESeverity{ + CriticalSeverityCount: 2, + }), + }, + }, + "query filters out all results but CVE detail still loads": { + request: &v2.GetVMCVEDetailRequest{ + CveId: "CVE-2024-1234", + Query: &v2.RawQuery{Query: "Severity:CRITICAL_VULNERABILITY_SEVERITY"}, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockCVE.EXPECT().SearchRawVMCVEs(ctx, gomock.Any()).Return([]*storage.VirtualMachineCVEV2{cve1}, nil) + mockView.EXPECT().CountBySeverity(ctx, gomock.Any(), search.VirtualMachineID).Return(&commonViews.ResourceCountByImageCVESeverity{}, nil) + mockView.EXPECT().GetVMIDs(ctx, gomock.Any()).Return(nil, nil) + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(5, nil) + }, + expectedResult: &v2.VMCVEDetail{ + AffectedVmCount: 0, + TotalVmCount: 5, + AffectedGuestOsCount: 0, + VmSeverityCounts: storagetov2.SeverityCountsToProto(&commonViews.ResourceCountByImageCVESeverity{}), + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockVM := vmDSMocks.NewMockDataStore(ctrl) + mockCVE := cveDSMocks.NewMockDataStore(ctrl) + mockComp := componentDSMocks.NewMockDataStore(ctrl) + mockScan := scanDSMocks.NewMockDataStore(ctrl) + mockView := cveViewMocks.NewMockCveView(ctrl) + + service := &serviceImpl{ + vmDS: mockVM, + cveDS: mockCVE, + componentDS: mockComp, + scanDS: mockScan, + cveView: mockView, + } + + tt.setupMock(mockVM, mockCVE, mockComp, mockScan, mockView) + + result, err := service.GetVMCVEDetail(ctx, tt.request) + + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + assert.Nil(t, result) + } else { + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, "CVE-2024-1234", result.GetCve()) + if tt.expectedResult != nil { + assert.Equal(t, tt.expectedResult.GetAffectedVmCount(), result.GetAffectedVmCount()) + assert.Equal(t, tt.expectedResult.GetTotalVmCount(), result.GetTotalVmCount()) + assert.Equal(t, tt.expectedResult.GetAffectedGuestOsCount(), result.GetAffectedGuestOsCount()) + protoassert.Equal(t, tt.expectedResult.GetVmSeverityCounts(), result.GetVmSeverityCounts()) + } + } + }) + } +} + +func TestGetVM(t *testing.T) { + ctx := context.Background() + + testVMID := fixtureconsts.VirtualMachine1 + + vm1 := &storage.VirtualMachineV2{ + Id: testVMID, + Name: "test-vm-1", + Namespace: "ns-1", + ClusterId: "cluster-1", + ClusterName: "test-cluster", + GuestOs: "rhel-9", + State: storage.VirtualMachineV2_RUNNING, + } + + scan1 := &storage.VirtualMachineScanV2{ + Id: "scan-1", + VmV2Id: testVMID, + ScanOs: "rhel-9", + TopCvss: 9.8, + } + + tests := map[string]struct { + request *v2.GetVMRequest + setupMock func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) + expectedError string + expectedResult *v2.VMDetail + }{ + "empty id": { + request: &v2.GetVMRequest{ + Id: "", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "id must be specified", + }, + "invalid UUID": { + request: &v2.GetVMRequest{ + Id: "not-a-uuid", + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + }, + expectedError: "id must be a valid UUID", + }, + "vm not found": { + request: &v2.GetVMRequest{ + Id: testVMID, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().GetVirtualMachineWithLatestScan(ctx, testVMID).Return(nil, false, nil) + }, + expectedError: "not found", + }, + "successful with scan data": { + request: &v2.GetVMRequest{ + Id: testVMID, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().GetVirtualMachineWithLatestScan(ctx, testVMID).Return(&common.VMWithLatestScan{VM: vm1, Scan: scan1}, true, nil) + }, + expectedResult: func() *v2.VMDetail { + detail := storagetov2.VirtualMachineV2ToDetail(vm1) + detail.LatestScan = &v2.VMScanInfo{ + ScanId: "scan-1", + ScanOs: "rhel-9", + TopCvss: 9.8, + ScanNotes: []v2.VMScanNote{}, + } + return detail + }(), + }, + "successful without scan data": { + request: &v2.GetVMRequest{ + Id: testVMID, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().GetVirtualMachineWithLatestScan(ctx, testVMID).Return(&common.VMWithLatestScan{VM: vm1}, true, nil) + }, + expectedResult: storagetov2.VirtualMachineV2ToDetail(vm1), + }, + "never scraped stays unknown with vsock cid and agent version": { + request: &v2.GetVMRequest{ + Id: testVMID, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + neverScraped := &storage.VirtualMachineV2{ + Id: testVMID, + Name: "test-vm-1", + State: storage.VirtualMachineV2_RUNNING, + VsockCid: 42, + Facts: map[string]string{"agentVersion": "5.0.x-174-g6b7ccc2192"}, + } + mockVM.EXPECT().GetVirtualMachineWithLatestScan(ctx, testVMID).Return(&common.VMWithLatestScan{VM: neverScraped}, true, nil) + }, + expectedResult: func() *v2.VMDetail { + detail := storagetov2.VirtualMachineV2ToDetail(&storage.VirtualMachineV2{ + Id: testVMID, + Name: "test-vm-1", + State: storage.VirtualMachineV2_RUNNING, + VsockCid: 42, + Facts: map[string]string{"agentVersion": "5.0.x-174-g6b7ccc2192"}, + }) + detail.AgentStatus = v2.AgentStatus_AGENT_STATUS_UNKNOWN + return detail + }(), + }, + "fresh scrape is active": { + request: &v2.GetVMRequest{ + Id: testVMID, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + fresh := vm1.CloneVT() + fresh.LastAgentContact = timestamppb.New(time.Now().Add(-time.Hour)) + mockVM.EXPECT().GetVirtualMachineWithLatestScan(ctx, testVMID).Return(&common.VMWithLatestScan{VM: fresh}, true, nil) + }, + expectedResult: func() *v2.VMDetail { + detail := storagetov2.VirtualMachineV2ToDetail(vm1) + detail.AgentStatus = v2.AgentStatus_AGENT_STATUS_ACTIVE + return detail + }(), + }, + "activation status does not affect agent status": { + request: &v2.GetVMRequest{ + Id: testVMID, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + fresh := vm1.CloneVT() + fresh.Facts = map[string]string{"activationStatus": "inactive"} + fresh.LastAgentContact = timestamppb.New(time.Now().Add(-time.Hour)) + mockVM.EXPECT().GetVirtualMachineWithLatestScan(ctx, testVMID).Return(&common.VMWithLatestScan{VM: fresh}, true, nil) + }, + expectedResult: func() *v2.VMDetail { + detail := storagetov2.VirtualMachineV2ToDetail(vm1) + detail.Facts = map[string]string{"activationStatus": "inactive"} + detail.AgentStatus = v2.AgentStatus_AGENT_STATUS_ACTIVE + return detail + }(), + }, + "stale scrape is inactive even when agent version is still set": { + request: &v2.GetVMRequest{ + Id: testVMID, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + stale := vm1.CloneVT() + stale.Facts = map[string]string{"agentVersion": "5.0.x-174-g6b7ccc2192"} + stale.LastAgentContact = timestamppb.New(time.Now().Add(-13 * time.Hour)) + mockVM.EXPECT().GetVirtualMachineWithLatestScan(ctx, testVMID).Return(&common.VMWithLatestScan{VM: stale}, true, nil) + }, + expectedResult: func() *v2.VMDetail { + detail := storagetov2.VirtualMachineV2ToDetail(vm1) + detail.Facts = map[string]string{"agentVersion": "5.0.x-174-g6b7ccc2192"} + detail.AgentStatus = v2.AgentStatus_AGENT_STATUS_INACTIVE + return detail + }(), + }, + "datastore error": { + request: &v2.GetVMRequest{ + Id: testVMID, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().GetVirtualMachineWithLatestScan(ctx, testVMID).Return(nil, false, errors.New("db error")) + }, + expectedError: "db error", + }, + "scan decoding error": { + request: &v2.GetVMRequest{ + Id: testVMID, + }, + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().GetVirtualMachineWithLatestScan(ctx, testVMID).Return(nil, false, errors.New("scan error")) + }, + expectedError: "scan error", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockVM := vmDSMocks.NewMockDataStore(ctrl) + mockCVE := cveDSMocks.NewMockDataStore(ctrl) + mockComp := componentDSMocks.NewMockDataStore(ctrl) + mockScan := scanDSMocks.NewMockDataStore(ctrl) + mockView := cveViewMocks.NewMockCveView(ctrl) + + service := &serviceImpl{ + vmDS: mockVM, + cveDS: mockCVE, + componentDS: mockComp, + scanDS: mockScan, + cveView: mockView, + } + + tt.setupMock(mockVM, mockCVE, mockComp, mockScan, mockView) + + result, err := service.GetVM(ctx, tt.request) + + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + assert.Nil(t, result) + } else { + require.NoError(t, err) + protoassert.Equal(t, tt.expectedResult, result) + } + }) + } +} + +func TestListVMs(t *testing.T) { + ctx := context.Background() + + vm1 := &storage.VirtualMachineV2{ + Id: "vm-1", Name: "test-vm-1", Namespace: "ns-1", + ClusterId: "c-1", ClusterName: "cluster-1", GuestOs: "rhel-9", + State: storage.VirtualMachineV2_RUNNING, + } + + comp1 := &storage.VirtualMachineComponentV2{ + Id: "comp-1", VmScanId: "scan-1", Name: "openssl", Version: "1.1.1", + } + compUnscanned := &storage.VirtualMachineComponentV2{ + Id: "comp-2", VmScanId: "scan-1", Name: "curl", Version: "7.0", + Notes: []storage.VirtualMachineComponentV2_Note{storage.VirtualMachineComponentV2_UNSCANNED}, + } + + scanTime := timestamppb.New(time.Date(2025, time.April, 15, 10, 30, 0, 0, time.UTC)) + scan1 := &storage.VirtualMachineScanV2{ + Id: "scan-1", VmV2Id: "vm-1", ScanTime: scanTime, + } + + tests := map[string]struct { + setupMock func(*vmDSMocks.MockDataStore, *cveDSMocks.MockDataStore, *componentDSMocks.MockDataStore, *scanDSMocks.MockDataStore, *cveViewMocks.MockCveView) + expectedError string + expectedCount int32 + checkResult func(t *testing.T, resp *v2.ListVMsResponse) + }{ + "successful list with severity and component counts": { + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(1, nil) + mockVM.EXPECT().SearchRawVirtualMachines(ctx, gomock.Any()).Return([]*storage.VirtualMachineV2{vm1}, nil) + sevCounts := cveViewMocks.NewMockVMSeverityCounts(gomock.NewController(t)) + sevCounts.EXPECT().GetVMID().Return("vm-1").AnyTimes() + sevCounts.EXPECT().GetSeverityCounts().Return(&commonViews.ResourceCountByImageCVESeverity{ + CriticalSeverityCount: 1, FixableCriticalSeverityCount: 1, + }).AnyTimes() + mockView.EXPECT().CountBySeverityPerVM(ctx, gomock.Any()).Return([]vmcve.VMSeverityCounts{sevCounts}, nil) + mockComp.EXPECT().SearchRawVMComponents(ctx, gomock.Any()).Return([]*storage.VirtualMachineComponentV2{comp1, compUnscanned}, nil) + mockScan.EXPECT().SearchRawVMScans(ctx, gomock.Any()).Return([]*storage.VirtualMachineScanV2{scan1}, nil) + mockScan.EXPECT().GetBatch(ctx, gomock.Any()).Return([]*storage.VirtualMachineScanV2{scan1}, nil) + }, + expectedCount: 1, + checkResult: func(t *testing.T, resp *v2.ListVMsResponse) { + require.Len(t, resp.GetVms(), 1) + item := resp.GetVms()[0] + assert.Equal(t, int32(1), item.GetCveSeverityCounts().GetCritical().GetTotal()) + assert.Equal(t, int32(1), item.GetCveSeverityCounts().GetCritical().GetFixable()) + assert.Equal(t, int32(2), item.GetComponentScanCount().GetTotal()) + assert.Equal(t, int32(1), item.GetComponentScanCount().GetScanned()) + require.NotNil(t, item.GetScanTime()) + assert.Equal(t, scanTime.AsTime(), item.GetScanTime().AsTime()) + }, + }, + "empty list": { + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(0, nil) + mockVM.EXPECT().SearchRawVirtualMachines(ctx, gomock.Any()).Return(nil, nil) + }, + expectedCount: 0, + }, + "should use scan time from the greatest scan ID": { + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + // Greatest UUIDv7 ID sits in the middle with the earliest ScanTime + // so first-row, last-row, and max-timestamp picks fail. + lowID := &storage.VirtualMachineScanV2{ + Id: "01800000-0000-7000-8000-000000000001", VmV2Id: "vm-1", + ScanTime: timestamppb.New(time.Date(2025, time.April, 16, 12, 0, 0, 0, time.UTC)), + } + winID := &storage.VirtualMachineScanV2{ + Id: "01900000-0000-7000-8000-000000000003", VmV2Id: "vm-1", + ScanTime: timestamppb.New(time.Date(2025, time.April, 14, 8, 0, 0, 0, time.UTC)), + } + midID := &storage.VirtualMachineScanV2{ + Id: "01880000-0000-7000-8000-000000000002", VmV2Id: "vm-1", + ScanTime: timestamppb.New(time.Date(2025, time.April, 15, 10, 30, 0, 0, time.UTC)), + } + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(1, nil) + mockVM.EXPECT().SearchRawVirtualMachines(ctx, gomock.Any()).Return([]*storage.VirtualMachineV2{vm1}, nil) + mockView.EXPECT().CountBySeverityPerVM(ctx, gomock.Any()).Return(nil, nil) + mockComp.EXPECT().SearchRawVMComponents(ctx, gomock.Any()).Return(nil, nil) + mockScan.EXPECT().SearchRawVMScans(ctx, gomock.Any()).Return([]*storage.VirtualMachineScanV2{lowID, winID, midID}, nil) + mockScan.EXPECT().GetBatch(ctx, gomock.Any()).Return(nil, nil) + }, + expectedCount: 1, + checkResult: func(t *testing.T, resp *v2.ListVMsResponse) { + require.Len(t, resp.GetVms(), 1) + require.NotNil(t, resp.GetVms()[0].GetScanTime()) + assert.Equal(t, time.Date(2025, time.April, 14, 8, 0, 0, 0, time.UTC), resp.GetVms()[0].GetScanTime().AsTime()) + }, + }, + "vm count error": { + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(0, errors.New("count error")) + }, + expectedError: "count error", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockVM := vmDSMocks.NewMockDataStore(ctrl) + mockCVE := cveDSMocks.NewMockDataStore(ctrl) + mockComp := componentDSMocks.NewMockDataStore(ctrl) + mockScan := scanDSMocks.NewMockDataStore(ctrl) + mockView := cveViewMocks.NewMockCveView(ctrl) + + svc := &serviceImpl{ + vmDS: mockVM, cveDS: mockCVE, componentDS: mockComp, + scanDS: mockScan, cveView: mockView, + } + + tt.setupMock(mockVM, mockCVE, mockComp, mockScan, mockView) + + result, err := svc.ListVMs(ctx, &v2.ListVMsRequest{}) + + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedCount, result.GetTotalCount()) + if tt.checkResult != nil { + tt.checkResult(t, result) + } + } + }) + } +} + +func TestListVMCVEs(t *testing.T) { + ctx := context.Background() + + tests := map[string]struct { + setupMock func(*vmDSMocks.MockDataStore, *cveDSMocks.MockDataStore, *componentDSMocks.MockDataStore, *scanDSMocks.MockDataStore, *cveViewMocks.MockCveView) + expectedError string + expectedCount int32 + }{ + "successful list": { + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockView.EXPECT().Count(ctx, gomock.Any()).Return(1, nil) + cveCore := cveViewMocks.NewMockCveCore(gomock.NewController(t)) + cveCore.EXPECT().GetCVE().Return("CVE-2024-1234").AnyTimes() + cveCore.EXPECT().GetVMsBySeverity().Return(&commonViews.ResourceCountByImageCVESeverity{}).AnyTimes() + cveCore.EXPECT().GetTopCVSS().Return(float32(7.5)).AnyTimes() + cveCore.EXPECT().GetAffectedVMCount().Return(3).AnyTimes() + cveCore.EXPECT().GetEPSSProbability().Return(float32(0.5)).AnyTimes() + cveCore.EXPECT().GetPublishDate().Return(nil).AnyTimes() + mockView.EXPECT().Get(ctx, gomock.Any()).Return([]vmcve.CveCore{cveCore}, nil) + mockVM.EXPECT().CountVirtualMachines(ctx, gomock.Any()).Return(10, nil) + }, + expectedCount: 1, + }, + "view count error": { + setupMock: func(mockVM *vmDSMocks.MockDataStore, mockCVE *cveDSMocks.MockDataStore, mockComp *componentDSMocks.MockDataStore, mockScan *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockView.EXPECT().Count(ctx, gomock.Any()).Return(0, errors.New("view error")) + }, + expectedError: "view error", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockVM := vmDSMocks.NewMockDataStore(ctrl) + mockCVE := cveDSMocks.NewMockDataStore(ctrl) + mockComp := componentDSMocks.NewMockDataStore(ctrl) + mockScan := scanDSMocks.NewMockDataStore(ctrl) + mockView := cveViewMocks.NewMockCveView(ctrl) + + svc := &serviceImpl{ + vmDS: mockVM, cveDS: mockCVE, componentDS: mockComp, + scanDS: mockScan, cveView: mockView, + } + + tt.setupMock(mockVM, mockCVE, mockComp, mockScan, mockView) + + result, err := svc.ListVMCVEs(ctx, &v2.ListVMCVEsRequest{}) + + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedCount, result.GetTotalCount()) + } + }) + } +} + +func TestListVMCVEAffectedVMs(t *testing.T) { + ctx := context.Background() + + tests := map[string]struct { + request *v2.ListVMCVEAffectedVMsRequest + setupMock func(*vmDSMocks.MockDataStore, *cveDSMocks.MockDataStore, *componentDSMocks.MockDataStore, *scanDSMocks.MockDataStore, *cveViewMocks.MockCveView) + expectedError string + expectedCount int32 + }{ + "empty cve_id": { + request: &v2.ListVMCVEAffectedVMsRequest{CveId: ""}, + setupMock: func(_ *vmDSMocks.MockDataStore, _ *cveDSMocks.MockDataStore, _ *componentDSMocks.MockDataStore, _ *scanDSMocks.MockDataStore, _ *cveViewMocks.MockCveView) { + }, + expectedError: "cve_id must be specified", + }, + "successful list": { + request: &v2.ListVMCVEAffectedVMsRequest{CveId: "CVE-2024-1234"}, + setupMock: func(_ *vmDSMocks.MockDataStore, _ *cveDSMocks.MockDataStore, _ *componentDSMocks.MockDataStore, _ *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockView.EXPECT().CountAffectedVMs(ctx, gomock.Any()).Return(1, nil) + affectedVM := cveViewMocks.NewMockAffectedVMCore(gomock.NewController(t)) + affectedVM.EXPECT().GetVMID().Return("vm-1").AnyTimes() + affectedVM.EXPECT().GetVMName().Return("test-vm-1").AnyTimes() + affectedVM.EXPECT().GetMaxSeverity().Return(int32(storage.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY)).AnyTimes() + affectedVM.EXPECT().GetIsFixable().Return(true).AnyTimes() + affectedVM.EXPECT().GetMaxCVSS().Return(float32(7.5)).AnyTimes() + affectedVM.EXPECT().GetGuestOS().Return("rhel-9").AnyTimes() + affectedVM.EXPECT().GetAffectedComponentCount().Return(1).AnyTimes() + mockView.EXPECT().GetAffectedVMs(ctx, gomock.Any()).Return([]vmcve.AffectedVMCore{affectedVM}, nil) + }, + expectedCount: 1, + }, + "count error": { + request: &v2.ListVMCVEAffectedVMsRequest{CveId: "CVE-2024-1234"}, + setupMock: func(_ *vmDSMocks.MockDataStore, _ *cveDSMocks.MockDataStore, _ *componentDSMocks.MockDataStore, _ *scanDSMocks.MockDataStore, mockView *cveViewMocks.MockCveView) { + mockView.EXPECT().CountAffectedVMs(ctx, gomock.Any()).Return(0, errors.New("count error")) + }, + expectedError: "count error", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockVM := vmDSMocks.NewMockDataStore(ctrl) + mockCVE := cveDSMocks.NewMockDataStore(ctrl) + mockComp := componentDSMocks.NewMockDataStore(ctrl) + mockScan := scanDSMocks.NewMockDataStore(ctrl) + mockView := cveViewMocks.NewMockCveView(ctrl) + + svc := &serviceImpl{ + vmDS: mockVM, cveDS: mockCVE, componentDS: mockComp, + scanDS: mockScan, cveView: mockView, + } + + tt.setupMock(mockVM, mockCVE, mockComp, mockScan, mockView) + + result, err := svc.ListVMCVEAffectedVMs(ctx, tt.request) + + if tt.expectedError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedCount, result.GetTotalCount()) + if len(result.GetVms()) > 0 { + row := result.GetVms()[0] + assert.Equal(t, "vm-1", row.GetVmId()) + assert.Equal(t, "test-vm-1", row.GetVmName()) + assert.Equal(t, v2.VulnerabilitySeverity_CRITICAL_VULNERABILITY_SEVERITY, row.GetSeverity()) + assert.True(t, row.GetIsFixable()) + assert.Equal(t, "rhel-9", row.GetGuestOs()) + assert.Equal(t, int32(1), row.GetAffectedComponentCount()) + } + } + }) + } +} diff --git a/central/virtualmachine/v2/service/singleton.go b/central/virtualmachine/v2/service/singleton.go new file mode 100644 index 0000000000000..45b365e20478b --- /dev/null +++ b/central/virtualmachine/v2/service/singleton.go @@ -0,0 +1,32 @@ +package service + +import ( + "github.com/stackrox/rox/central/views/vmcve" + componentDS "github.com/stackrox/rox/central/virtualmachine/component/v2/datastore" + cveDS "github.com/stackrox/rox/central/virtualmachine/cve/v2/datastore" + scanDS "github.com/stackrox/rox/central/virtualmachine/scan/v2/datastore" + vmDS "github.com/stackrox/rox/central/virtualmachine/v2/datastore" + "github.com/stackrox/rox/pkg/sync" +) + +var ( + once sync.Once + + svc Service +) + +func initialize() { + svc = New( + vmDS.Singleton(), + cveDS.Singleton(), + componentDS.Singleton(), + scanDS.Singleton(), + vmcve.Singleton(), + ) +} + +// Singleton provides the instance of the Service interface to register. +func Singleton() Service { + once.Do(initialize) + return svc +} diff --git a/central/vulnmgmt/service/service_impl.go b/central/vulnmgmt/service/service_impl.go index b52b6c6a819f3..a7f4a17be787e 100644 --- a/central/vulnmgmt/service/service_impl.go +++ b/central/vulnmgmt/service/service_impl.go @@ -14,9 +14,11 @@ import ( "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/auth/permissions" "github.com/stackrox/rox/pkg/errox" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/grpc/authz" "github.com/stackrox/rox/pkg/grpc/authz/perrpc" "github.com/stackrox/rox/pkg/grpc/authz/user" + "github.com/stackrox/rox/pkg/images/utils" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/postgres" "github.com/stackrox/rox/pkg/sac/resources" @@ -102,7 +104,12 @@ func (s *serviceImpl) VulnMgmtExportWorkloads(req *v1.VulnMgmtExportWorkloadsReq images := make([]*storage.Image, 0, len(containers)) imageIDs := set.NewStringSet() for _, container := range containers { - imgID := container.GetImage().GetId() + var imgID string + if features.FlattenImageData.Enabled() { + imgID = container.GetImage().GetIdV2() + } else { + imgID = container.GetImage().GetId() + } // Deduplicate images by their ID. if imageIDs.Contains(imgID) { continue @@ -120,6 +127,7 @@ func (s *serviceImpl) VulnMgmtExportWorkloads(req *v1.VulnMgmtExportWorkloadsReq continue } if found { + utils.StripDatasourceNoClone(img.GetScan()) images = append(images, img) imageCache.Add(imgID, img) } else { diff --git a/central/vulnmgmt/vulnerabilityrequest/datastore/datastore_impl.go b/central/vulnmgmt/vulnerabilityrequest/datastore/datastore_impl.go index 3a7e23e8a78a5..c29fd38e9d5c5 100644 --- a/central/vulnmgmt/vulnerabilityrequest/datastore/datastore_impl.go +++ b/central/vulnmgmt/vulnerabilityrequest/datastore/datastore_impl.go @@ -532,7 +532,3 @@ func (c *VulnRequestSearchResultConverter) BuildLocation(result *search.Result) func (c *VulnRequestSearchResultConverter) GetCategory() v1.SearchCategory { return v1.SearchCategory_VULN_REQUEST } - -func (c *VulnRequestSearchResultConverter) GetScore(result *search.Result) float64 { - return result.Score -} diff --git a/central/vulnmgmt/vulnerabilityrequest/manager/querymgr/query_manager_impl.go b/central/vulnmgmt/vulnerabilityrequest/manager/querymgr/query_manager_impl.go index 9a40a7a4f088c..70a598c9fed19 100644 --- a/central/vulnmgmt/vulnerabilityrequest/manager/querymgr/query_manager_impl.go +++ b/central/vulnmgmt/vulnerabilityrequest/manager/querymgr/query_manager_impl.go @@ -13,6 +13,7 @@ import ( "github.com/stackrox/rox/central/vulnmgmt/vulnerabilityrequest/utils" v1 "github.com/stackrox/rox/generated/api/v1" "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/features" "github.com/stackrox/rox/pkg/sac" "github.com/stackrox/rox/pkg/sac/resources" ) @@ -45,6 +46,9 @@ func (m *queryManagerImpl) ImageCount(ctx context.Context, requestID string, que if err != nil { return 0, err } + if features.FlattenImageData.Enabled() { + return m.imageV2s.Count(ctx, query) + } return m.images.Count(ctx, query) } diff --git a/central/vulnmgmt/vulnerabilityrequest/manager/querymgr/query_manager_impl_test.go b/central/vulnmgmt/vulnerabilityrequest/manager/querymgr/query_manager_impl_test.go index 0533a8ee1e377..f27183065cab3 100644 --- a/central/vulnmgmt/vulnerabilityrequest/manager/querymgr/query_manager_impl_test.go +++ b/central/vulnmgmt/vulnerabilityrequest/manager/querymgr/query_manager_impl_test.go @@ -43,7 +43,8 @@ type VulnReqQueryManagerTestSuite struct { pendingReqCache vulnReqCache.VulnReqCache activeReqCache vulnReqCache.VulnReqCache - testImage *storage.Image + testImage *storage.Image + testImageV2 *storage.ImageV2 } func (s *VulnReqQueryManagerTestSuite) TearDownTest() { @@ -68,17 +69,16 @@ func (s *VulnReqQueryManagerTestSuite) SetupTest() { } // Insert test image. - s.testImage = fixtures.GetImage() - s.NoError(s.imageDataStore.UpsertImage(allAccessCtx, s.testImage)) + if features.FlattenImageData.Enabled() { + s.testImageV2 = fixtures.GetImageV2() + s.NoError(s.imageV2DataStore.UpsertImage(allAccessCtx, s.testImageV2)) + } else { + s.testImage = fixtures.GetImage() + s.NoError(s.imageDataStore.UpsertImage(allAccessCtx, s.testImage)) + } } func (s *VulnReqQueryManagerTestSuite) createImageDataStore() { - s.imageDataStore = imageDS.NewWithPostgres( - imagePostgresV2.New(s.testDB.DB, false, concurrency.NewKeyFence()), - mockRisks.NewMockDataStore(s.mockCtrl), - ranking.NewRanker(), - ranking.NewRanker(), - ) if features.FlattenImageData.Enabled() { s.imageV2DataStore = imageV2DS.NewWithPostgres( imageV2Postgres.New(s.testDB.DB, false, concurrency.NewKeyFence()), @@ -86,6 +86,13 @@ func (s *VulnReqQueryManagerTestSuite) createImageDataStore() { ranking.NewRanker(), ranking.NewRanker(), ) + } else { + s.imageDataStore = imageDS.NewWithPostgres( + imagePostgresV2.New(s.testDB.DB, false, concurrency.NewKeyFence()), + mockRisks.NewMockDataStore(s.mockCtrl), + ranking.NewRanker(), + ranking.NewRanker(), + ) } } @@ -97,12 +104,12 @@ func (s *VulnReqQueryManagerTestSuite) createVulnRequestDataStore(pendingReqCach func (s *VulnReqQueryManagerTestSuite) TestWithOneVulReq() { scope := common.VulnReqScope{ - Registry: s.testImage.GetName().GetRegistry(), - Remote: s.testImage.GetName().GetRemote(), - Tag: s.testImage.GetName().GetTag(), + Registry: s.testImageName().GetRegistry(), + Remote: s.testImageName().GetRemote(), + Tag: s.testImageName().GetTag(), } - cveToDefer := s.testImage.GetScan().GetComponents()[0].GetVulns()[0].GetCve() - observedCVE := s.testImage.GetScan().GetComponents()[0].GetVulns()[1].GetCve() + cveToDefer := s.testImageScan().GetComponents()[0].GetVulns()[0].GetCve() + observedCVE := s.testImageScan().GetComponents()[0].GetVulns()[1].GetCve() // Add a approved vulnerability request. req := fixtures.GetImageScopeDeferralRequest(scope.Registry, scope.Remote, scope.Tag, cveToDefer) @@ -112,15 +119,28 @@ func (s *VulnReqQueryManagerTestSuite) TestWithOneVulReq() { s.activeReqCache.Add(req) // Verify that image is returned. - img, err := s.mgr.Images(allAccessCtx, req.GetId(), nil) - s.NoError(err) - s.NotNil(img) - s.Equal(s.testImage.GetId(), img[0].GetId()) + if features.FlattenImageData.Enabled() { + img, err := s.mgr.ImageV2s(allAccessCtx, req.GetId(), nil) + s.NoError(err) + s.NotNil(img) + s.Equal(s.testImageV2.GetId(), img[0].GetId()) + } else { + img, err := s.mgr.Images(allAccessCtx, req.GetId(), nil) + s.NoError(err) + s.NotNil(img) + s.Equal(s.testImage.GetId(), img[0].GetId()) + } // Verify that no image is returned when queried for image that is not in DB. - img, err = s.mgr.Images(allAccessCtx, req.GetId(), search.NewQueryBuilder().AddExactMatches(search.ImageName, "unavailable image").ProtoQuery()) - s.NoError(err) - s.Len(img, 0) + if features.FlattenImageData.Enabled() { + img, err := s.mgr.ImageV2s(allAccessCtx, req.GetId(), search.NewQueryBuilder().AddExactMatches(search.ImageName, "unavailable image").ProtoQuery()) + s.NoError(err) + s.Len(img, 0) + } else { + img, err := s.mgr.Images(allAccessCtx, req.GetId(), search.NewQueryBuilder().AddExactMatches(search.ImageName, "unavailable image").ProtoQuery()) + s.NoError(err) + s.Len(img, 0) + } // Verify that vulnerability request is returned for deferred CVE. actualReq, err := s.mgr.EffectiveVulnReq(allAccessCtx, cveToDefer, scope) @@ -146,14 +166,14 @@ func (s *VulnReqQueryManagerTestSuite) TestWithOneVulReq() { func (s *VulnReqQueryManagerTestSuite) TestWithMultipleActiveReqs() { scopeInDB := common.VulnReqScope{ - Registry: s.testImage.GetName().GetRegistry(), - Remote: s.testImage.GetName().GetRemote(), - Tag: s.testImage.GetName().GetTag(), + Registry: s.testImageName().GetRegistry(), + Remote: s.testImageName().GetRemote(), + Tag: s.testImageName().GetTag(), } scopeNotInDB := common.VulnReqScope{Registry: "fake", Remote: "fake", Tag: "fake"} - cve1 := s.testImage.GetScan().GetComponents()[0].GetVulns()[0].GetCve() - cve2 := s.testImage.GetScan().GetComponents()[0].GetVulns()[1].GetCve() + cve1 := s.testImageScan().GetComponents()[0].GetVulns()[0].GetCve() + cve2 := s.testImageScan().GetComponents()[0].GetVulns()[1].GetCve() // Add a approved vuln request. reqForImgInDB := fixtures.GetImageScopeDeferralRequest(scopeInDB.Registry, scopeInDB.Remote, scopeInDB.Tag, cve1) @@ -198,12 +218,12 @@ func (s *VulnReqQueryManagerTestSuite) TestWithMultipleActiveReqs() { func (s *VulnReqQueryManagerTestSuite) TestWithMultipleScopes() { scope := common.VulnReqScope{ - Registry: s.testImage.GetName().GetRegistry(), - Remote: s.testImage.GetName().GetRemote(), - Tag: s.testImage.GetName().GetTag(), + Registry: s.testImageName().GetRegistry(), + Remote: s.testImageName().GetRemote(), + Tag: s.testImageName().GetTag(), } - cve1 := s.testImage.GetScan().GetComponents()[0].GetVulns()[0].GetCve() - cve2 := s.testImage.GetScan().GetComponents()[0].GetVulns()[1].GetCve() + cve1 := s.testImageScan().GetComponents()[0].GetVulns()[0].GetCve() + cve2 := s.testImageScan().GetComponents()[0].GetVulns()[1].GetCve() // Add a approved vuln request. deferralReq1 := fixtures.GetImageScopeDeferralRequest(scope.Registry, scope.Remote, scope.Tag, cve1) @@ -283,12 +303,12 @@ func (s *VulnReqQueryManagerTestSuite) TestWithMultipleScopes() { func (s *VulnReqQueryManagerTestSuite) TestQueriesForActiveAndPendingRequests() { scope := common.VulnReqScope{ - Registry: s.testImage.GetName().GetRegistry(), - Remote: s.testImage.GetName().GetRemote(), - Tag: s.testImage.GetName().GetTag(), + Registry: s.testImageName().GetRegistry(), + Remote: s.testImageName().GetRemote(), + Tag: s.testImageName().GetTag(), } - cve1 := s.testImage.GetScan().GetComponents()[0].GetVulns()[0].GetCve() - cve2 := s.testImage.GetScan().GetComponents()[0].GetVulns()[1].GetCve() + cve1 := s.testImageScan().GetComponents()[0].GetVulns()[0].GetCve() + cve2 := s.testImageScan().GetComponents()[0].GetVulns()[1].GetCve() // Add a pending and approved vuln request for same cve. deferralReq1 := fixtures.GetImageScopeDeferralRequest(scope.Registry, scope.Remote, scope.Tag, cve1) @@ -332,3 +352,17 @@ func (s *VulnReqQueryManagerTestSuite) TestQueriesForActiveAndPendingRequests() } } } + +func (s *VulnReqQueryManagerTestSuite) testImageName() *storage.ImageName { + if features.FlattenImageData.Enabled() { + return s.testImageV2.GetName() + } + return s.testImage.GetName() +} + +func (s *VulnReqQueryManagerTestSuite) testImageScan() *storage.ImageScan { + if features.FlattenImageData.Enabled() { + return s.testImageV2.GetScan() + } + return s.testImage.GetScan() +} diff --git a/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl.go b/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl.go index 676716e892d05..d06f1c1077aff 100644 --- a/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl.go +++ b/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl.go @@ -449,7 +449,7 @@ func (m *managerImpl) reprocessDeployments(requestID string, affectedImages ...s } func (m *managerImpl) reprocessImage(_ string, affectedImages ...string) error { - imageKeys := make([]*central.InvalidateImageCache_ImageKey, 0, len(affectedImages)) + imageKeys := make([]*central.ImageKey, 0, len(affectedImages)) for _, imgID := range affectedImages { if features.FlattenImageData.Enabled() { image, found, err := m.imagesV2.GetImageMetadata(allAccessCtx, imgID) @@ -459,7 +459,7 @@ func (m *managerImpl) reprocessImage(_ string, affectedImages ...string) error { if !found { continue } - imageKeys = append(imageKeys, ¢ral.InvalidateImageCache_ImageKey{ + imageKeys = append(imageKeys, ¢ral.ImageKey{ ImageId: image.GetDigest(), ImageIdV2: image.GetId(), ImageFullName: image.GetName().GetFullName(), @@ -472,7 +472,7 @@ func (m *managerImpl) reprocessImage(_ string, affectedImages ...string) error { if !found { continue } - imageKeys = append(imageKeys, ¢ral.InvalidateImageCache_ImageKey{ + imageKeys = append(imageKeys, ¢ral.ImageKey{ ImageId: imgID, ImageFullName: image.GetName().GetFullName(), }) @@ -815,7 +815,7 @@ func getShortName(user *storage.SlimUser) string { name := strings.ToUpper(user.GetName()) parts := strings.Split(name, " ") - for i := 0; i < len(parts); i++ { + for i := range parts { parts[i] = strings.TrimSpace(parts[i]) } diff --git a/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl_benchmark_test.go b/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl_benchmark_test.go index 5c89980af44ce..b3885724aee02 100644 --- a/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl_benchmark_test.go +++ b/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl_benchmark_test.go @@ -23,7 +23,7 @@ func BenchmarkCreate(b *testing.B) { ctx := sac.WithAllAccess(context.Background()) numExistingReqs := 10000 - for i := 0; i < numExistingReqs; i++ { + for i := range numExistingReqs { req := fixtures.GetImageScopeDeferralRequest("registry", "remote", fmt.Sprintf("%d", i), "cve-2023-xyz") err := vulnReqDataStore.AddRequest(ctx, req) assert.NoError(b, err) @@ -32,7 +32,7 @@ func BenchmarkCreate(b *testing.B) { manager := New(nil, vulnReqDataStore, pendingCache, nil, nil, nil, nil, nil, nil, nil) b.Run("create", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { tag := fmt.Sprintf("%d", rand.Intn(numExistingReqs)) req := fixtures.GetImageScopeDeferralRequest("registry", "remote", tag, "cve-2023-xyz") req.Id = "" diff --git a/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl_cache_test.go b/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl_cache_test.go index 8c8c563dd6d31..372564c11b550 100644 --- a/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl_cache_test.go +++ b/central/vulnmgmt/vulnerabilityrequest/manager/requestmgr/manager_impl_cache_test.go @@ -52,7 +52,7 @@ func (s *VulnRequestManagerCacheTestSuite) TestAdd() { imageScopeAllTagsReq := fixtures.GetImageScopeDeferralRequest("reg", "remote", ".*", "cve1") imageScopeOneTagsReq := fixtures.GetImageScopeDeferralRequest("reg", "remote", "tag", "cve1") - var enforcedRequests []*storage.VulnerabilityRequest + enforcedRequests := make([]*storage.VulnerabilityRequest, 0, 3) for _, req := range []*storage.VulnerabilityRequest{imageScopeOneTagsReq, imageScopeAllTagsReq, globalReq} { req.Id = "" @@ -72,7 +72,7 @@ func (s *VulnRequestManagerCacheTestSuite) TestAddConflicts() { imageScopeAllTagsReq := fixtures.GetImageScopeDeferralRequest("reg", "remote", ".*", "cve1") imageScopeOneTagsReq := fixtures.GetImageScopeDeferralRequest("reg", "remote", "tag", "cve1") - var enforcedRequests []*storage.VulnerabilityRequest + enforcedRequests := make([]*storage.VulnerabilityRequest, 0, 3) for idx, req := range []*storage.VulnerabilityRequest{globalReq, imageScopeAllTagsReq, imageScopeOneTagsReq} { req.Id = "" diff --git a/central/worker/main.go b/central/worker/main.go new file mode 100644 index 0000000000000..0818932478d55 --- /dev/null +++ b/central/worker/main.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "errors" + "math" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/stackrox/rox/central/globaldb" + notifierDS "github.com/stackrox/rox/central/notifier/datastore" + notifierProcessor "github.com/stackrox/rox/central/notifier/processor" + _ "github.com/stackrox/rox/central/notifiers/all" + reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" + vulnReportV2Scheduler "github.com/stackrox/rox/central/reports/scheduler/v2" + reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" + "github.com/stackrox/rox/central/version" + vStore "github.com/stackrox/rox/central/version/store" + "github.com/stackrox/rox/pkg/dblock" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/logging" + pkgMetrics "github.com/stackrox/rox/pkg/metrics" + "github.com/stackrox/rox/pkg/premain" + "github.com/stackrox/rox/pkg/retry" +) + +const ( + healthAddr = ":8082" + defaultWorkerPoolMaxConns = 20 +) + +var ( + log = logging.LoggerForModule() + workerPoolSize = env.RegisterIntegerSetting("ROX_WORKER_DB_POOL_MAX_CONNS", defaultWorkerPoolMaxConns) +) + +func main() { + premain.StartMain() + + log.Infof("Starting central-worker") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + poolVal := workerPoolSize.IntegerSetting() + if poolVal < 1 || poolVal > math.MaxInt32 { + log.Fatalf("ROX_WORKER_DB_POOL_MAX_CONNS must be between 1 and %d, got %d", math.MaxInt32, poolVal) + } + globaldb.InitializePostgresWithPoolSize(ctx, int32(poolVal)) + log.Infof("DB pool initialized with max_conns=%d", poolVal) + + waitForMigrations(ctx) + + startHealthServer() + + go startMetricsServer() + + scheduler := vulnReportV2Scheduler.Singleton() + scheduler.Start(globaldb.GetPostgres()) + log.Infof("Vulnerability report scheduler started") + + collectionDatastore, _ := collectionDS.Singleton() + rl := newReportListener( + globaldb.GetPostgres(), + scheduler, + reportConfigDS.Singleton(), + reportSnapshotDS.Singleton(), + collectionDatastore, + notifierDS.Singleton(), + notifierProcessor.Singleton(), + ) + rl.start(ctx) + log.Infof("Report LISTEN/NOTIFY listener started") + + log.Infof("central-worker is ready") + + waitForTerminationSignal() + + log.Infof("central-worker shutting down") + + scheduler.Stop() + + globaldb.Close() +} + +func waitForMigrations(ctx context.Context) { + versionStore := vStore.NewPostgres(globaldb.GetPostgres()) + err := retry.WithRetry(func() error { + acquired, release, err := dblock.TryAcquireAdvisoryLock(ctx, globaldb.GetPostgres(), dblock.MigrationLockID) + if err != nil { + return retry.MakeRetryable(err) + } + if !acquired { + return retry.MakeRetryable(errMigratorRunning) + } + defer release() + + if err := version.Ensure(versionStore); err != nil { + return retry.MakeRetryable(err) + } + return nil + }, retry.Tries(30), retry.BetweenAttempts(func(attempt int) { + log.Infof("Migrations not yet complete, waiting (attempt %d)...", attempt+1) + time.Sleep(10 * time.Second) + })) + if err != nil { + log.Fatalf("Timed out waiting for migrations to complete: %v", err) + } + log.Infof("Migrations complete, proceeding with startup") +} + +var errMigratorRunning = retryableError("migrator is still running") + +type retryableError string + +func (e retryableError) Error() string { return string(e) } + +func startHealthServer() { + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + srv := &http.Server{ + Addr: healthAddr, + Handler: mux, + } + + errCh := make(chan error, 1) + go func() { + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- err + } + }() + + select { + case err := <-errCh: + log.Fatalf("Health server failed to start: %v", err) + case <-time.After(1 * time.Second): + } + + go func() { + if err := <-errCh; err != nil { + log.Fatalf("Health server failed: %v", err) + } + }() +} + +func startMetricsServer() { + pkgMetrics.NewServer(pkgMetrics.CentralWorkerSubsystem, pkgMetrics.NewTLSConfigurerFromEnv()).RunForever() + pkgMetrics.GatherThrottleMetricsForever(pkgMetrics.CentralWorkerSubsystem.String()) +} + +func waitForTerminationSignal() { + signalsC := make(chan os.Signal, 1) + signal.Notify(signalsC, syscall.SIGINT, syscall.SIGTERM) + sig := <-signalsC + log.Infof("Caught %s signal", sig) +} diff --git a/central/worker/report_listener.go b/central/worker/report_listener.go new file mode 100644 index 0000000000000..2b2310a7a6359 --- /dev/null +++ b/central/worker/report_listener.go @@ -0,0 +1,240 @@ +package main + +import ( + "context" + "time" + + notifierDS "github.com/stackrox/rox/central/notifier/datastore" + "github.com/stackrox/rox/central/reports/common" + reportConfigDS "github.com/stackrox/rox/central/reports/config/datastore" + schedulerV2 "github.com/stackrox/rox/central/reports/scheduler/v2" + reportGen "github.com/stackrox/rox/central/reports/scheduler/v2/reportgenerator" + reportSnapshotDS "github.com/stackrox/rox/central/reports/snapshot/datastore" + collectionDS "github.com/stackrox/rox/central/resourcecollection/datastore" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/env" + "github.com/stackrox/rox/pkg/notifier" + pkgNotifiers "github.com/stackrox/rox/pkg/notifiers" + "github.com/stackrox/rox/pkg/postgres" + pgNotify "github.com/stackrox/rox/pkg/postgres/notify" + "github.com/stackrox/rox/pkg/sac" + "github.com/stackrox/rox/pkg/search" + "github.com/stackrox/rox/pkg/set" + "github.com/stackrox/rox/pkg/sync" +) + +var listenerCtx = sac.WithAllAccess(context.Background()) + +type reportListener struct { + db postgres.DB + scheduler schedulerV2.Scheduler + reportConfigStore reportConfigDS.DataStore + snapshotStore reportSnapshotDS.DataStore + collectionDatastore collectionDS.DataStore + notifierStore notifierDS.DataStore + notifierProcessor notifier.Processor + knownMu sync.Mutex + knownReportIDs set.StringSet +} + +func newReportListener( + db postgres.DB, + scheduler schedulerV2.Scheduler, + reportConfigStore reportConfigDS.DataStore, + snapshotStore reportSnapshotDS.DataStore, + collectionDatastore collectionDS.DataStore, + notifierStore notifierDS.DataStore, + notifierProcessor notifier.Processor, +) *reportListener { + return &reportListener{ + db: db, + scheduler: scheduler, + reportConfigStore: reportConfigStore, + snapshotStore: snapshotStore, + collectionDatastore: collectionDatastore, + notifierStore: notifierStore, + notifierProcessor: notifierProcessor, + knownReportIDs: set.NewStringSet(), + } +} + +func (r *reportListener) start(ctx context.Context) { + listener := pgNotify.NewListener(r.db, r.handleNotification, + pgNotify.NotifierChanged, + pgNotify.ReportConfigChanged, + pgNotify.ReportRequestSubmitted, + pgNotify.ReportRequestCancelled, + ) + go listener.Listen(ctx) + go r.periodicResync(ctx) +} + +func (r *reportListener) handleNotification(channel, payload string) { + switch channel { + case pgNotify.NotifierChanged: + r.handleNotifierChanged(payload) + case pgNotify.ReportConfigChanged: + r.handleConfigChanged(payload) + case pgNotify.ReportRequestSubmitted: + r.handleRequestSubmitted(payload) + case pgNotify.ReportRequestCancelled: + r.handleRequestCancelled(payload) + } +} + +func (r *reportListener) handleNotifierChanged(notifierID string) { + protoNotifier, exists, err := r.notifierStore.GetNotifier(listenerCtx, notifierID) + if err != nil { + log.Errorf("Failed to load notifier %s: %v", notifierID, err) + return + } + if !exists { + r.notifierProcessor.RemoveNotifier(listenerCtx, notifierID) + return + } + n, err := pkgNotifiers.CreateNotifier(protoNotifier) + if err != nil { + log.Errorf("Failed to create notifier %s: %v", notifierID, err) + return + } + r.notifierProcessor.UpdateNotifier(listenerCtx, n) +} + +func (r *reportListener) handleConfigChanged(configID string) { + config, exists, err := r.reportConfigStore.GetReportConfiguration(listenerCtx, configID) + if err != nil { + log.Errorf("Failed to load report config %s: %v", configID, err) + return + } + if !exists { + r.scheduler.RemoveReportSchedule(configID) + return + } + if config.GetSchedule() != nil { + if err := r.scheduler.UpsertReportSchedule(config); err != nil { + log.Errorf("Failed to upsert schedule for config %s: %v", configID, err) + } + } else { + r.scheduler.RemoveReportSchedule(configID) + } +} + +func (r *reportListener) handleRequestSubmitted(snapshotID string) { + snap, exists, err := r.snapshotStore.Get(listenerCtx, snapshotID) + if err != nil || !exists { + log.Errorf("Failed to load report snapshot %s: exists=%v err=%v", snapshotID, exists, err) + return + } + if snap.GetReportStatus().GetRunState() != storage.ReportStatus_WAITING { + return + } + + var collection *storage.ResourceCollection + if snap.GetCollection().GetId() != "" { + collection, exists, err = r.collectionDatastore.Get(listenerCtx, snap.GetCollection().GetId()) + if err != nil || !exists { + log.Errorf("Failed to load collection for snapshot %s: %v", snapshotID, err) + return + } + } + + req := &reportGen.ReportRequest{ + ReportSnapshot: snap, + Collection: collection, + } + r.submitIfNew(snapshotID, req) +} + +func (r *reportListener) handleRequestCancelled(snapshotID string) { + if _, err := r.scheduler.CancelReportRequest(listenerCtx, snapshotID); err != nil { + log.Errorf("Failed to cancel report request %s: %v", snapshotID, err) + } +} + +func (r *reportListener) periodicResync(ctx context.Context) { + ticker := time.NewTicker(time.Duration(env.CentralWorkerResyncIntervalMins.IntegerSetting()) * time.Minute) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + r.resyncConfigs() + r.resyncPendingRequests() + } + } +} + +func (r *reportListener) resyncConfigs() { + query := search.NewQueryBuilder(). + AddExactMatches(search.ReportType, + storage.ReportConfiguration_VULNERABILITY.String(), + storage.ReportConfiguration_NODE_VULNERABILITY.String(), + ). + ProtoQuery() + configs, err := r.reportConfigStore.GetReportConfigurations(listenerCtx, query) + if err != nil { + log.Errorf("Error resyncing report configs: %v", err) + return + } + + activeConfigIDs := set.NewStringSet() + for _, rc := range configs { + if rc.GetSchedule() != nil && common.HasValidResourceScope(rc.GetResourceScope()) { + activeConfigIDs.Add(rc.GetId()) + if err := r.scheduler.UpsertReportSchedule(rc); err != nil { + log.Errorf("Error resyncing schedule for config %s: %v", rc.GetId(), err) + } + } + } + + for _, id := range r.scheduler.GetScheduledConfigIDs() { + if !activeConfigIDs.Contains(id) { + r.scheduler.RemoveReportSchedule(id) + } + } +} + +func (r *reportListener) resyncPendingRequests() { + query := search.NewQueryBuilder(). + AddExactMatches(search.ReportState, storage.ReportStatus_WAITING.String()). + WithPagination(search.NewPagination().AddSortOption(search.NewSortOption(search.ReportQueuedTime))). + ProtoQuery() + snapshots, err := r.snapshotStore.SearchReportSnapshots(listenerCtx, query) + if err != nil { + log.Errorf("Error resyncing pending report requests: %v", err) + return + } + + for _, snap := range snapshots { + var collection *storage.ResourceCollection + if collID := snap.GetCollection().GetId(); collID != "" { + var exists bool + collection, exists, err = r.collectionDatastore.Get(listenerCtx, collID) + if err != nil || !exists { + log.Errorf("Error loading collection for pending snapshot %s: %v", snap.GetReportId(), err) + continue + } + } + + req := &reportGen.ReportRequest{ + ReportSnapshot: snap, + Collection: collection, + } + r.submitIfNew(snap.GetReportId(), req) + } +} + +func (r *reportListener) submitIfNew(reportID string, req *reportGen.ReportRequest) { + r.knownMu.Lock() + defer r.knownMu.Unlock() + + if r.knownReportIDs.Contains(reportID) { + return + } + if _, err := r.scheduler.SubmitReportRequest(listenerCtx, req, true); err != nil { + log.Errorf("Failed to submit report request %s: %v", reportID, err) + return + } + r.knownReportIDs.Add(reportID) +} diff --git a/codecov.yml b/codecov.yml index 975bf2daf7827..1267824ded0c1 100644 --- a/codecov.yml +++ b/codecov.yml @@ -5,5 +5,5 @@ coverage: threshold: 1% codecov: notify: - after_n_builds: 4 + after_n_builds: 2 wait_for_ci: false diff --git a/compliance/ACK_MODEL.md b/compliance/ACK_MODEL.md new file mode 100644 index 0000000000000..9542a39eae570 --- /dev/null +++ b/compliance/ACK_MODEL.md @@ -0,0 +1,98 @@ +# ACK Model for Node Scanning in ACS + +This document describes the ACK model ACS uses for node scanning. + +For the VM index report ACK/retry design, see `sensor/common/virtualmachine/ACK_MODEL.md`. + +## Summary for Busy Readers + +- ACS uses a generic ACK model: + - `SensorACK` for `Central -> Sensor` + - `ComplianceACK` for `Sensor -> Compliance` +- ACKs were introduced for node scanning so Compliance could retry expensive periodic node scans instead of waiting for the next normal interval when only delivery had failed. + +## Design Overview + +The core idea is simple: producing a scan and confirming its delivery are treated as separate steps. The producer keeps just enough state to retry delivery when delivery fails, without requiring a new scan when the data itself is still valid. + +At a high level, ACK handling looks like this: + +```mermaid +sequenceDiagram + participant P as Producer + retry state + participant S as Sensor + participant X as Central + + P->>P: cache payload + ObserveSending(resource) + P->>S: send scan message + S->>X: forward sensor event + X-->>S: SensorACK(ACK|NACK, type, resource_id, reason) + S-->>P: ComplianceACK(ACK|NACK, type, resource_id, reason) + + alt ACK + P->>P: stop tracking / clear cached payload + else NACK or ACK timeout + P->>P: retry last cached payload if available + end +``` + +A few compatibility constraints still shape the design: + +- Generic `SensorACK` sending is capability-gated through `SensorACKSupport`. +- Node scanning still has a legacy compatibility path because older Central versions used `NodeInventoryACK` instead of `SensorACK`. + +One semantic detail also matters: + +- ACK means "do not retry this message anymore". Most of the time that is because the message was processed successfully, but ACK can also be used intentionally when retrying would only create noise, for example when a feature is disabled. + +## Why ACKs Were Introduced + +ACKs first mattered for node scanning. + +Node scans are periodic and relatively expensive. Without ACKs, Compliance had no good way to know whether the last generated scan had actually made it through Sensor and Central. A temporary problem such as: + +- Sensor being offline +- Central being unreachable +- the node not yet being known to Sensor +- Central failing while processing the scan + +could otherwise force Compliance to wait until the next normal scan interval before trying again. + +So the ACK design added three things: + +- cache the last scan message +- track it as unconfirmed +- retry it on NACK or on missing ACK + +## Node Scanning + +Node scanning is the original model, and it already behaves this way. + +Compliance uses a single local resource key, `this-node`, because one Compliance instance handles exactly one node. + +```mermaid +sequenceDiagram + participant C as Compliance + participant U as UMH + cached last scan + participant S as Sensor node handler + participant X as Central node pipeline + + C->>C: run node inventory / node index scan + C->>U: cache last message + ObserveSending("this-node") + C->>S: send node scan + + alt Sensor offline or node unknown + S-->>C: ComplianceACK(NACK) + else forwarded to Central + S->>X: SensorEvent(node_id, ...) + X-->>S: SensorACK(ACK|NACK) + S-->>C: ComplianceACK(ACK|NACK) + end + + alt ACK + C->>U: HandleACK("this-node") + else NACK or no ACK + U-->>C: RetryCommand("this-node") + C->>S: resend cached last scan + end +``` diff --git a/compliance/cmd/compliance/main.go b/compliance/cmd/compliance/main.go index 64180efb53a13..5f9811ac51c5a 100644 --- a/compliance/cmd/compliance/main.go +++ b/compliance/cmd/compliance/main.go @@ -31,7 +31,9 @@ func main() { cfg := index.DefaultNodeIndexerConfig() scanner := inventory.NewNodeInventoryComponentScanner(np) - scanner.Connect(env.NodeScanningEndpoint.Setting()) + if env.NodeInventoryContainerEnabled.BooleanSetting() { + scanner.Connect(env.NodeScanningEndpoint.Setting()) + } cachedNodeIndexer := index.NewCachingNodeIndexer(cfg, env.NodeIndexCacheDuration.DurationSetting(), env.NodeIndexCachePath.Setting()) ctx, cancel := context.WithCancel(context.Background()) diff --git a/compliance/collection/auditlog/auditlog_impl_test.go b/compliance/collection/auditlog/auditlog_impl_test.go index 420f632b0c776..336701e245806 100644 --- a/compliance/collection/auditlog/auditlog_impl_test.go +++ b/compliance/collection/auditlog/auditlog_impl_test.go @@ -36,7 +36,7 @@ type ComplianceAuditLogReaderTestSuite struct { } func (s *ComplianceAuditLogReaderTestSuite) TestCompareK8sResourceLists() { - for _, r := range resourceTypesAllowList.AsSlice() { + for r := range resourceTypesAllowList.All() { if _, ok := auditResourceToKubeResource[fmt.Sprintf("%v", r)]; !ok { s.T().Errorf("Missing entry in auditResourceToKubeResource map for resource %v", r) } @@ -104,7 +104,7 @@ func (s *ComplianceAuditLogReaderTestSuite) TestReaderTailsLog() { // Write a few more log lines and check that they are read and parsed expectedEvents := make([]auditEvent, 0, 3) lines := make([]string, 0, 3) - for i := 0; i < 3; i++ { + for i := range 3 { eventTime = eventTime.Add(60 * time.Second) line, expectedEvent = s.fakeAuditLogLineAtTime("get", "configmaps", fmt.Sprintf("fake-map%d", i), "stackrox", eventTime.Format(time.RFC3339Nano)) expectedEvents = append(expectedEvents, expectedEvent) @@ -112,10 +112,9 @@ func (s *ComplianceAuditLogReaderTestSuite) TestReaderTailsLog() { } wg := sync.WaitGroup{} - wg.Add(1) // Write to the file in parallel to simulate the log file getting written to in parallel - go func() { + wg.Go(func() { for _, line := range lines { s.appendToFile(logPath, line) } @@ -123,8 +122,7 @@ func (s *ComplianceAuditLogReaderTestSuite) TestReaderTailsLog() { // unlikely scenario is real life since the audit log is constantly being written to line, _ := s.fakeAuditLogLineAtTime("get", "configmaps", "extra-map", "stackrox", eventTime.Format(time.RFC3339Nano)) s.appendToFile(logPath, line) - wg.Done() - }() + }) for _, expectedEvent := range expectedEvents { event = s.getSentEvent(sender.sentC) @@ -147,7 +145,7 @@ func (s *ComplianceAuditLogReaderTestSuite) TestReaderOnlySendsEventsThatMatchRe defer s.cleanupFile(f, logPath) // Write a few log lines that should get filtered out due to unsupported resource type - for i := 0; i < 5; i++ { + for range 5 { line, _ := s.fakeAuditLogLine("get", "something-else", "fake-thing", "stackrox") _, err = f.Write([]byte(line)) s.NoError(err) @@ -319,7 +317,7 @@ func (s *ComplianceAuditLogReaderTestSuite) TestReaderStartsSendingEventsAfterSt eventTime := time.Now() var latestEvent auditEvent var line string - for i := 0; i < 5; i++ { + for range 5 { eventTime = eventTime.Add(1 * time.Second) line, latestEvent = s.fakeAuditLogLineAtTime("get", "secrets", "fake-token", "stackrox", eventTime.Format(time.RFC3339Nano)) _, err = f.Write([]byte(line)) @@ -360,7 +358,7 @@ func (s *ComplianceAuditLogReaderTestSuite) TestReaderStartsSendingEventsAtStart eventTime := time.Now().Add(1 * time.Minute) var latestEvent auditEvent var line string - for i := 0; i < 5; i++ { + for range 5 { eventTime = eventTime.Add(1 * time.Second) line, latestEvent = s.fakeAuditLogLineAtTime("get", "secrets", "fake-token", "stackrox", eventTime.Format(time.RFC3339Nano)) _, err = f.Write([]byte(line)) diff --git a/compliance/collection/auditlog/sender_impl_test.go b/compliance/collection/auditlog/sender_impl_test.go index 01f7d6772de8b..4bbb846e1e7b4 100644 --- a/compliance/collection/auditlog/sender_impl_test.go +++ b/compliance/collection/auditlog/sender_impl_test.go @@ -61,7 +61,7 @@ func (s *ComplianceAuditLogSenderTestSuite) TestSendCorrectlySendsEvents() { func (s *ComplianceAuditLogSenderTestSuite) TestSendSendsInOrder() { client, sender := s.getMocks() - for i := 0; i < 5; i++ { + for i := range 5 { mockEvent := s.fakeAuditEvent("get", "configmaps", fmt.Sprintf("map-%d", i), "stackrox") err := sender.Send(context.Background(), &mockEvent) s.NoError(err) diff --git a/compliance/collection/command/command.go b/compliance/collection/command/command.go index e17e105ec9446..dad92b6058ba4 100644 --- a/compliance/collection/command/command.go +++ b/compliance/collection/command/command.go @@ -77,9 +77,9 @@ func getProcessFromCmdLineBytes(cmdlineBytes []byte) string { return "" } processBytes := cmdlineBytes - index := bytes.Index(cmdlineBytes, []byte("\x00")) - if index != -1 { - processBytes = cmdlineBytes[:index] + before, _, ok := bytes.Cut(cmdlineBytes, []byte("\x00")) + if ok { + processBytes = before } return filepath.Base(string(processBytes)) } diff --git a/compliance/collection/compliance_checks/compliance_checks_bench_test.go b/compliance/collection/compliance_checks/compliance_checks_bench_test.go index dcf2e0a3d5d4f..765412602dbce 100644 --- a/compliance/collection/compliance_checks/compliance_checks_bench_test.go +++ b/compliance/collection/compliance_checks/compliance_checks_bench_test.go @@ -27,8 +27,8 @@ func BenchmarkRunChecks(b *testing.B) { conf := &sensor.MsgToCompliance_ScrapeConfig{ ContainerRuntime: storage.ContainerRuntime_DOCKER_CONTAINER_RUNTIME, } - b.ResetTimer() - for i := 0; i < b.N; i++ { + + for b.Loop() { getCheckResults(run, conf, data) } } @@ -47,8 +47,7 @@ func BenchmarkCompressResults(b *testing.B) { } results := getCheckResults(run, conf, data) - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { _, err := cutils.CompressResults(results) if err != nil { panic(err) @@ -68,8 +67,8 @@ func BenchmarkChecksAndCompression(b *testing.B) { conf := &sensor.MsgToCompliance_ScrapeConfig{ ContainerRuntime: storage.ContainerRuntime_DOCKER_CONTAINER_RUNTIME, } - b.ResetTimer() - for i := 0; i < b.N; i++ { + + for b.Loop() { results := getCheckResults(run, conf, data) _, err := cutils.CompressResults(results) if err != nil { diff --git a/compliance/collection/file/user.go b/compliance/collection/file/user.go index b30f43873e4ff..2b7db2bfa8e1b 100644 --- a/compliance/collection/file/user.go +++ b/compliance/collection/file/user.go @@ -17,8 +17,8 @@ func parseSystemUserMap(file string, userMap map[uint32]string) { log.Error(err) return } - lines := strings.Split(string(contents), "\n") - for _, l := range lines { + lines := strings.SplitSeq(string(contents), "\n") + for l := range lines { if spl := strings.Split(l, ":"); len(spl) > 2 { ui, err := strconv.ParseUint(spl[2], 10, 32) if err != nil { diff --git a/compliance/collection/kubernetes/kubelet.go b/compliance/collection/kubernetes/kubelet.go index 2f6dcaeee67d0..280cad150e9bd 100644 --- a/compliance/collection/kubernetes/kubelet.go +++ b/compliance/collection/kubernetes/kubelet.go @@ -6,7 +6,6 @@ import ( "github.com/pkg/errors" complianceUtils "github.com/stackrox/rox/compliance/utils" "github.com/stackrox/rox/pkg/compliance/checks/standards" - "github.com/stackrox/rox/pkg/pointers" "github.com/stackrox/rox/pkg/set" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -27,17 +26,17 @@ func init() { func applyConfigDefaults(kc *v1beta1.KubeletConfiguration) { // --anonymous-auth - kc.Authentication.Anonymous.Enabled = pointers.Bool(true) + kc.Authentication.Anonymous.Enabled = new(true) // --authentication-token-webhook - kc.Authentication.Webhook.Enabled = pointers.Bool(false) + kc.Authentication.Webhook.Enabled = new(false) // --authorization-mode kc.Authorization.Mode = v1beta1.KubeletAuthorizationModeAlwaysAllow // --read-only-port kc.ReadOnlyPort = 10255 kc.StreamingConnectionIdleTimeout = metav1.Duration{Duration: 4 * time.Hour} - kc.MakeIPTablesUtilChains = pointers.Bool(true) - kc.EventRecordQPS = pointers.Int32(5) + kc.MakeIPTablesUtilChains = new(true) + kc.EventRecordQPS = new(int32(5)) } // GatherKubelet gets the KubeletConfiguration diff --git a/compliance/compliance.go b/compliance/compliance.go index b6f78f2b7ac08..9f12eb2093d57 100644 --- a/compliance/compliance.go +++ b/compliance/compliance.go @@ -4,18 +4,16 @@ import ( "context" "os" "os/signal" + "sync/atomic" "syscall" "time" - "github.com/cenkalti/backoff/v3" + "github.com/cenkalti/backoff/v4" "github.com/pkg/errors" "github.com/stackrox/rox/compliance/collection/auditlog" "github.com/stackrox/rox/compliance/collection/compliance_checks" cmetrics "github.com/stackrox/rox/compliance/collection/metrics" "github.com/stackrox/rox/compliance/node" - "github.com/stackrox/rox/compliance/virtualmachines/relay" - "github.com/stackrox/rox/compliance/virtualmachines/relay/sender" - "github.com/stackrox/rox/compliance/virtualmachines/relay/stream" v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4" "github.com/stackrox/rox/generated/internalapi/sensor" "github.com/stackrox/rox/generated/storage" @@ -28,6 +26,7 @@ import ( "github.com/stackrox/rox/pkg/metrics" "github.com/stackrox/rox/pkg/mtls" "github.com/stackrox/rox/pkg/protoutils" + "github.com/stackrox/rox/pkg/retry/handler" "github.com/stackrox/rox/pkg/utils" "github.com/stackrox/rox/pkg/version" "google.golang.org/grpc/metadata" @@ -35,26 +34,32 @@ import ( var log = logging.LoggerForModule() +const ( + // nodeResourceID is the resource ID used for node scanning UMH. + // Compliance handles exactly one node, so a single constant suffices. + nodeResourceID = "this-node" +) + // Compliance represents the Compliance app type Compliance struct { - nodeNameProvider node.NodeNameProvider - nodeScanner node.NodeScanner - nodeIndexer node.NodeIndexer - umhNodeInventory node.UnconfirmedMessageHandler - umhNodeIndex node.UnconfirmedMessageHandler - cache *sensor.MsgFromCompliance + nodeNameProvider node.NodeNameProvider + nodeScanner node.NodeScanner + nodeIndexer node.NodeIndexer + umhNodeInventory handler.UnconfirmedMessageHandler + umhNodeIndex handler.UnconfirmedMessageHandler + nodeInventoryCache atomic.Pointer[sensor.MsgFromCompliance] + nodeIndexCache atomic.Pointer[sensor.MsgFromCompliance] } // NewComplianceApp constructs the Compliance app object func NewComplianceApp(nnp node.NodeNameProvider, scanner node.NodeScanner, nodeIndexer node.NodeIndexer, - umhNodeInv, umhNodeIndex node.UnconfirmedMessageHandler) *Compliance { + umhNodeInv, umhNodeIndex handler.UnconfirmedMessageHandler) *Compliance { return &Compliance{ nodeNameProvider: nnp, nodeScanner: scanner, nodeIndexer: nodeIndexer, umhNodeInventory: umhNodeInv, umhNodeIndex: umhNodeIndex, - cache: nil, } } @@ -68,7 +73,7 @@ func (c *Compliance) Start() { metrics.GatherThrottleMetricsForever(metrics.ComplianceSubsystem.String()) // Set up Compliance <-> Sensor connection - conn, err := clientconn.AuthenticatedGRPCConnection(context.Background(), env.AdvertisedEndpoint.Setting(), mtls.SensorSubject) + conn, err := clientconn.AuthenticatedGRPCConnection(context.Background(), env.SensorEndpointSetting(), mtls.SensorSubject) if err != nil { log.Fatal(err) } @@ -96,7 +101,7 @@ func (c *Compliance) Start() { }() var wg concurrency.WaitGroup - wg.Add(3) + wg.Add(2) go func(ctx context.Context) { defer wg.Add(-1) @@ -122,30 +127,6 @@ func (c *Compliance) Start() { } }(ctx) - // The virtual machine relay (ROX-30476), which reads VM index reports from vsock connections and forwards them to - // sensor, is currently started and run in the compliance container. This enables reusing the existing connection to - // sensor and accelerates initial development. - go func(ctx context.Context) { - defer wg.Add(-1) - if features.VirtualMachines.Enabled() { - log.Infof("Virtual machine relay enabled") - - reportStream, err := stream.New() - if err != nil { - log.Errorf("Error creating report stream: %v", err) - return - } - - sensorClient := sensor.NewVirtualMachineIndexReportServiceClient(conn) - reportSender := sender.New(sensorClient) - - vmRelay := relay.New(reportStream, reportSender) - if err := vmRelay.Run(ctx); err != nil { - log.Errorf("Error running virtual machine relay: %v", err) - } - } - }(ctx) - // Wait for the terminate signal go func() { sig := <-signalsC @@ -179,13 +160,18 @@ func (c *Compliance) manageNodeInventoryScanLoop(ctx context.Context) <-chan *se select { case <-ctx.Done(): return - case _, ok := <-c.umhNodeInventory.RetryCommand(): - if c.cache == nil { - log.Debug("Requested to retry but cache is empty. Resetting scan timer.") + case resourceID, ok := <-c.umhNodeInventory.RetryCommand(): + if !ok { + log.Info("UMH retry channel for node inventory closed; stopping scan loop") + return + } + cachedMsg := c.nodeInventoryCache.Load() + if cachedMsg == nil { + log.Debugf("Requested to retry %s but cache is empty. Resetting scan timer.", resourceID) cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionResendingCacheMiss, cmetrics.ScannerVersionV2) t.Reset(time.Second) - } else if ok { - nodeInventoriesC <- c.cache + } else { + nodeInventoriesC <- cachedMsg cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionResendingCacheHit, cmetrics.ScannerVersionV2) } case <-t.C: @@ -215,13 +201,18 @@ func (c *Compliance) manageNodeIndexScanLoop(ctx context.Context) <-chan *sensor select { case <-ctx.Done(): return - case _, ok := <-c.umhNodeIndex.RetryCommand(): - if c.cache == nil { - log.Debug("Requested to retry but cache is empty. Resetting scan timer.") + case resourceID, ok := <-c.umhNodeIndex.RetryCommand(): + if !ok { + log.Info("UMH retry channel for node index closed; stopping scan loop") + return + } + cachedMsg := c.nodeIndexCache.Load() + if cachedMsg == nil { + log.Debugf("Requested to retry %s but cache is empty. Resetting scan timer.", resourceID) cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionResendingCacheMiss, cmetrics.ScannerVersionV4) t.Reset(time.Second) - } else if ok { - nodeIndexesC <- c.cache + } else { + nodeIndexesC <- cachedMsg cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionResendingCacheHit, cmetrics.ScannerVersionV4) } case <-t.C: @@ -249,8 +240,8 @@ func (c *Compliance) runNodeInventoryScan(ctx context.Context) *sensor.MsgFromCo } cmetrics.ObserveNodeInventoryScan(msg.GetNodeInventory()) cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionScan, cmetrics.ScannerVersionV2) - c.umhNodeInventory.ObserveSending() - c.cache = msg.CloneVT() + c.umhNodeInventory.ObserveSending(nodeResourceID) + c.nodeInventoryCache.Store(msg.CloneVT()) return msg } @@ -266,11 +257,12 @@ func (c *Compliance) runNodeIndex(ctx context.Context) *sensor.MsgFromCompliance log.Errorf("Error creating node index: %v", err) return nil } - c.umhNodeIndex.ObserveSending() + c.umhNodeIndex.ObserveSending(nodeResourceID) cmetrics.ObserveNodeIndexReport(report, nodeName) msg := c.createIndexMsg(report, nodeName) cmetrics.ObserveReportProtobufMessage(msg, cmetrics.ScannerVersionV4) cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionScan, cmetrics.ScannerVersionV4) + c.nodeIndexCache.Store(msg.CloneVT()) return msg } @@ -342,35 +334,51 @@ func (c *Compliance) runRecv(ctx context.Context, client sensor.ComplianceServic log.Warn("Attempting to stop an un-started audit log reader - this is a no-op") } } - case *sensor.MsgToCompliance_Ack: - switch t.Ack.GetAction() { - case sensor.MsgToCompliance_NodeInventoryACK_ACK: - switch t.Ack.GetMessageType() { - case sensor.MsgToCompliance_NodeInventoryACK_NodeInventory: - c.umhNodeInventory.HandleACK() - case sensor.MsgToCompliance_NodeInventoryACK_NodeIndexer: - c.umhNodeIndex.HandleACK() - default: - log.Errorf("Unknown ACK Type: %s", t.Ack.GetMessageType()) - } - case sensor.MsgToCompliance_NodeInventoryACK_NACK: - switch t.Ack.GetMessageType() { - case sensor.MsgToCompliance_NodeInventoryACK_NodeInventory: - c.umhNodeInventory.HandleNACK() - case sensor.MsgToCompliance_NodeInventoryACK_NodeIndexer: - c.umhNodeIndex.HandleNACK() - default: - log.Errorf("Unknown ACK Type: %s", t.Ack.GetMessageType()) - } - default: - log.Errorf("Unknown ACK Action: %s", t.Ack.GetAction()) - } + case *sensor.MsgToCompliance_ComplianceAck: + // New ComplianceACK from Sensor 4.10+ + c.handleComplianceACK(t.ComplianceAck) default: utils.Should(errors.Errorf("Unhandled msg type: %T", t)) } } } +// handleComplianceACK handles the new ComplianceACK message from Sensor 4.10+. +// This is the generic ACK/NACK message that replaces the legacy NodeInventoryACK. +func (c *Compliance) handleComplianceACK(ack *sensor.MsgToCompliance_ComplianceACK) { + if ack == nil { + log.Error("Received nil ComplianceACK") + return + } + + log.Debugf("Received ComplianceACK: type=%s, action=%s, resource_id=%s, reason=%s", + ack.GetMessageType(), ack.GetAction(), ack.GetResourceId(), ack.GetReason()) + + switch ack.GetMessageType() { + case sensor.MsgToCompliance_ComplianceACK_NODE_INVENTORY: + dispatchACK(c.umhNodeInventory, "node inventory", ack.GetAction(), ack.GetReason()) + case sensor.MsgToCompliance_ComplianceACK_NODE_INDEX_REPORT: + dispatchACK(c.umhNodeIndex, "node index", ack.GetAction(), ack.GetReason()) + default: + log.Errorf("Unknown ComplianceACK message type: %s", ack.GetMessageType()) + } +} + +// dispatchACK routes a ComplianceACK action to the appropriate UMH method. +func dispatchACK(umh handler.UnconfirmedMessageHandler, label string, action sensor.MsgToCompliance_ComplianceACK_Action, reason string) { + switch action { + case sensor.MsgToCompliance_ComplianceACK_ACK: + umh.HandleACK(nodeResourceID) + case sensor.MsgToCompliance_ComplianceACK_NACK: + if reason != "" { + log.Infof("%s NACK received: %s", label, reason) + } + umh.HandleNACK(nodeResourceID) + default: + log.Errorf("Unknown ComplianceACK action for %s: %s", label, action) + } +} + func (c *Compliance) startAuditLogCollection(ctx context.Context, client sensor.ComplianceService_CommunicateClient, request *sensor.MsgToCompliance_AuditLogCollectionRequest_StartRequest) auditlog.Reader { if request.GetCollectStartState() == nil { log.Infof("Starting audit log reader on node %s in cluster %s with no saved state", c.nodeNameProvider.GetNodeName(), request.GetClusterId()) @@ -426,7 +434,7 @@ func (c *Compliance) initializeStream(ctx context.Context, cli sensor.Compliance if err != nil { return nil, nil, errors.Wrap(err, "Failed to initialize sensor connection") } - log.Infof("Successfully connected to Sensor at %s", env.AdvertisedEndpoint.Setting()) + log.Infof("Successfully connected to Sensor at %s", env.SensorEndpointSetting()) return client, config, nil } diff --git a/compliance/compliance_ack_test.go b/compliance/compliance_ack_test.go new file mode 100644 index 0000000000000..144c2bec5ffb9 --- /dev/null +++ b/compliance/compliance_ack_test.go @@ -0,0 +1,109 @@ +package compliance + +import ( + "testing" + + "github.com/stackrox/rox/generated/internalapi/sensor" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stretchr/testify/assert" +) + +// fakeUMH is a minimal test double for handler.UnconfirmedMessageHandler. +// Set retryC to a non-nil channel when tests need RetryCommand() to be selectable. +type fakeUMH struct { + ackCount int + nackCount int + retryC chan string +} + +func (f *fakeUMH) HandleACK(string) { f.ackCount++ } +func (f *fakeUMH) HandleNACK(string) { f.nackCount++ } +func (f *fakeUMH) ObserveSending(string) {} +func (f *fakeUMH) OnACK(func(string)) {} + +func (f *fakeUMH) RetryCommand() <-chan string { + if f.retryC != nil { + return f.retryC + } + return nil +} + +func (f *fakeUMH) Stopped() concurrency.ReadOnlyErrorSignal { + s := concurrency.NewStopper() + s.Flow().ReportStopped() + return s.Client().Stopped() +} + +func TestHandleComplianceACK(t *testing.T) { + inv := &fakeUMH{} + idx := &fakeUMH{} + c := &Compliance{ + umhNodeInventory: inv, + umhNodeIndex: idx, + } + + tests := []struct { + name string + ack *sensor.MsgToCompliance_ComplianceACK + wantInvACK int + wantInvNACK int + wantIdxACK int + wantIdxNACK int + }{ + { + name: "node inventory ack", + ack: &sensor.MsgToCompliance_ComplianceACK{ + Action: sensor.MsgToCompliance_ComplianceACK_ACK, + MessageType: sensor.MsgToCompliance_ComplianceACK_NODE_INVENTORY, + }, + wantInvACK: 1, + }, + { + name: "node inventory nack", + ack: &sensor.MsgToCompliance_ComplianceACK{ + Action: sensor.MsgToCompliance_ComplianceACK_NACK, + MessageType: sensor.MsgToCompliance_ComplianceACK_NODE_INVENTORY, + }, + wantInvNACK: 1, + }, + { + name: "node index ack", + ack: &sensor.MsgToCompliance_ComplianceACK{ + Action: sensor.MsgToCompliance_ComplianceACK_ACK, + MessageType: sensor.MsgToCompliance_ComplianceACK_NODE_INDEX_REPORT, + }, + wantIdxACK: 1, + }, + { + name: "node index nack", + ack: &sensor.MsgToCompliance_ComplianceACK{ + Action: sensor.MsgToCompliance_ComplianceACK_NACK, + MessageType: sensor.MsgToCompliance_ComplianceACK_NODE_INDEX_REPORT, + }, + wantIdxNACK: 1, + }, + { + name: "unknown action ignored", + ack: &sensor.MsgToCompliance_ComplianceACK{ + Action: sensor.MsgToCompliance_ComplianceACK_Action(999), + MessageType: sensor.MsgToCompliance_ComplianceACK_NODE_INDEX_REPORT, + }, + }, + { + name: "nil message ignored", + ack: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + inv.ackCount, inv.nackCount = 0, 0 + idx.ackCount, idx.nackCount = 0, 0 + c.handleComplianceACK(tt.ack) + assert.Equal(t, tt.wantInvACK, inv.ackCount) + assert.Equal(t, tt.wantInvNACK, inv.nackCount) + assert.Equal(t, tt.wantIdxACK, idx.ackCount) + assert.Equal(t, tt.wantIdxNACK, idx.nackCount) + }) + } +} diff --git a/compliance/compliance_test.go b/compliance/compliance_test.go new file mode 100644 index 0000000000000..696a4b5d1fe00 --- /dev/null +++ b/compliance/compliance_test.go @@ -0,0 +1,98 @@ +package compliance + +import ( + "testing" + + "github.com/stackrox/rox/generated/internalapi/sensor" + "github.com/stretchr/testify/suite" +) + +func TestCompliance(t *testing.T) { + suite.Run(t, new(ComplianceTestSuite)) +} + +type ComplianceTestSuite struct { + suite.Suite +} + +func (s *ComplianceTestSuite) TestHandleComplianceACK() { + cases := map[string]struct { + ack *sensor.MsgToCompliance_ComplianceACK + expectedInventoryACKs int + expectedInventoryNACKs int + expectedIndexACKs int + expectedIndexNACKs int + }{ + "should handle NODE_INVENTORY ACK": { + ack: &sensor.MsgToCompliance_ComplianceACK{ + Action: sensor.MsgToCompliance_ComplianceACK_ACK, + MessageType: sensor.MsgToCompliance_ComplianceACK_NODE_INVENTORY, + ResourceId: "node-1", + }, + expectedInventoryACKs: 1, + }, + "should handle NODE_INVENTORY NACK": { + ack: &sensor.MsgToCompliance_ComplianceACK{ + Action: sensor.MsgToCompliance_ComplianceACK_NACK, + MessageType: sensor.MsgToCompliance_ComplianceACK_NODE_INVENTORY, + ResourceId: "node-1", + Reason: "rate limit exceeded", + }, + expectedInventoryNACKs: 1, + }, + "should handle NODE_INDEX_REPORT ACK": { + ack: &sensor.MsgToCompliance_ComplianceACK{ + Action: sensor.MsgToCompliance_ComplianceACK_ACK, + MessageType: sensor.MsgToCompliance_ComplianceACK_NODE_INDEX_REPORT, + ResourceId: "node-2", + }, + expectedIndexACKs: 1, + }, + "should handle NODE_INDEX_REPORT NACK": { + ack: &sensor.MsgToCompliance_ComplianceACK{ + Action: sensor.MsgToCompliance_ComplianceACK_NACK, + MessageType: sensor.MsgToCompliance_ComplianceACK_NODE_INDEX_REPORT, + ResourceId: "node-2", + Reason: "central unreachable", + }, + expectedIndexNACKs: 1, + }, + } + + for name, tc := range cases { + s.Run(name, func() { + mockInventory := &fakeUMH{retryC: make(chan string)} + mockIndex := &fakeUMH{retryC: make(chan string)} + + c := &Compliance{ + umhNodeInventory: mockInventory, + umhNodeIndex: mockIndex, + } + + c.handleComplianceACK(tc.ack) + + s.Equal(tc.expectedInventoryACKs, mockInventory.ackCount, "inventory ACK count") + s.Equal(tc.expectedInventoryNACKs, mockInventory.nackCount, "inventory NACK count") + s.Equal(tc.expectedIndexACKs, mockIndex.ackCount, "index ACK count") + s.Equal(tc.expectedIndexNACKs, mockIndex.nackCount, "index NACK count") + }) + } +} + +func (s *ComplianceTestSuite) TestHandleComplianceACK_NilACK() { + mockInventory := &fakeUMH{retryC: make(chan string)} + mockIndex := &fakeUMH{retryC: make(chan string)} + + c := &Compliance{ + umhNodeInventory: mockInventory, + umhNodeIndex: mockIndex, + } + + // Should not panic and should not call any handlers + c.handleComplianceACK(nil) + + s.Equal(0, mockInventory.ackCount) + s.Equal(0, mockInventory.nackCount) + s.Equal(0, mockIndex.ackCount) + s.Equal(0, mockIndex.nackCount) +} diff --git a/compliance/node/index/indexer.go b/compliance/node/index/indexer.go index 521d079abde2c..673e44b5a256a 100644 --- a/compliance/node/index/indexer.go +++ b/compliance/node/index/indexer.go @@ -3,11 +3,13 @@ package index import ( "bytes" "context" - "crypto/tls" "encoding/json" "fmt" "net/http" + "net/url" "os" + "path/filepath" + "slices" "strconv" "strings" "time" @@ -16,28 +18,28 @@ import ( "github.com/quay/claircore" ccindexer "github.com/quay/claircore/indexer" "github.com/quay/claircore/indexer/controller" + "github.com/quay/claircore/osrelease" + "github.com/quay/claircore/pkg/rhctag" "github.com/quay/claircore/rhel" - "github.com/quay/zlog" - "github.com/rs/zerolog" + "github.com/quay/claircore/rhel/rhcc" + "github.com/quay/claircore/toolkit/types" + "github.com/quay/claircore/toolkit/types/cpe" "github.com/stackrox/rox/compliance/node" "github.com/stackrox/rox/compliance/utils" v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4" + "github.com/stackrox/rox/pkg/clientconn" "github.com/stackrox/rox/pkg/env" "github.com/stackrox/rox/pkg/httputil/proxy" "github.com/stackrox/rox/pkg/logging" "github.com/stackrox/rox/pkg/mtls" "github.com/stackrox/rox/pkg/scannerv4/mappers" - "github.com/stackrox/rox/pkg/sync" "github.com/stackrox/rox/pkg/urlfmt" pkgutils "github.com/stackrox/rox/pkg/utils" - "go.uber.org/zap/zapcore" ) const ( layerMediaType = "application/vnd.claircore.filesystem" - rhcosPackageDB = "sqlite:usr/share/rpm" - // scannerDefinitionsRouteInSensor should be in sync with `scannerDefinitionsRoute` in sensor/sensor.go // Direct import is prohibited by import rules scannerDefinitionsRouteInSensor = "/scanner/definitions" @@ -45,14 +47,16 @@ const ( ) var ( - log = logging.LoggerForModule() - zerologLevel = map[zapcore.Level]zerolog.Level{ - zapcore.DebugLevel: zerolog.DebugLevel, - zapcore.InfoLevel: zerolog.InfoLevel, - zapcore.WarnLevel: zerolog.WarnLevel, - zapcore.ErrorLevel: zerolog.ErrorLevel, - zapcore.PanicLevel: zerolog.PanicLevel, - zapcore.FatalLevel: zerolog.FatalLevel, + log = logging.LoggerForModule() + + // rhcosPackageDBs are Claircore PackageDB names for RHCOS RPM databases. + // RHEL 8 uses Berkeley DB; RHEL 9+ uses SQLite. ostree reports the same RPMs + // under usr/share/rpm and usr/lib/sysimage/rpm-ostree-base-db. + rhcosPackageDBs = []string{ + "sqlite:usr/share/rpm", + "sqlite:usr/lib/sysimage/rpm", + "bdb:usr/share/rpm", + "bdb:usr/lib/sysimage/rpm-ostree-base-db", } // layerDigest is a dummy digest solely meant as a workaround to use Claircore. @@ -60,73 +64,84 @@ var ( // so we use a fake one for all nodes. layerDigest = fmt.Sprintf("sha256:%s", strings.Repeat("a", 64)) ccLayerDigest = claircore.MustParseDigest(layerDigest) - - clientOnce sync.Once - defaultClient *http.Client - defaultClientErr error ) -func init() { - // Default to info level. - logLevel := zerolog.InfoLevel - if level, ok := zerologLevel[logging.GetGlobalLogLevel()]; ok { - logLevel = level +// newDefaultClient configures an HTTP client that works with both internal +// mTLS endpoints (Sensor) and external publicly-trusted endpoints. +func newDefaultClient(mappingURL string) (*http.Client, error) { + serverName, err := extractHostname(mappingURL) + if err != nil { + return nil, err + } + // SensorSubject identifies the expected StackRox service for cert verification + // when the endpoint is Sensor (the default). ServerName is set to the actual + // hostname to avoid defaulting to sensor.stackrox, which would fail hostname + // verification for external endpoints. + tlsConf, err := clientconn.TLSConfig(mtls.SensorSubject, clientconn.TLSConfigOptions{ + UseClientCert: clientconn.MustUseClientCert, + ServerName: serverName, + }) + if err != nil { + return nil, errors.Wrap(err, "creating TLS config") } - l := zerolog.New(os.Stderr). - Level(logLevel) - zlog.Set(&l) + // Clear gRPC ALPN protos to avoid HTTP/2 framing issues (see ROX-21861). + tlsConf.NextProtos = nil + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConf, + Proxy: proxy.FromConfig(), + }, + Timeout: 30 * time.Second, + }, nil } -func getDefaultClient() (*http.Client, error) { - clientOnce.Do(func() { - clientCert, err := mtls.LeafCertificateFromFile() - if err != nil { - defaultClientErr = errors.Wrap(err, "obtaining defaultClient certificate") - return - } - defaultClient = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - // TODO: Should this always be set to true...? - InsecureSkipVerify: true, - Certificates: []tls.Certificate{clientCert}, - }, - Proxy: proxy.FromConfig(), - }, - Timeout: 30 * time.Second, - } - }) - return defaultClient, defaultClientErr +func extractHostname(rawURL string) (string, error) { + if !strings.Contains(rawURL, "://") { + rawURL = "https://" + rawURL + } + parsed, err := url.Parse(rawURL) + if err != nil { + return "", errors.Wrapf(err, "parsing URL %q", rawURL) + } + return parsed.Hostname(), nil } // NodeIndexerConfig represents Scanner V4 node indexer configuration parameters. type NodeIndexerConfig struct { // HostPath is the mount point of the read-only host filesystem on the node. HostPath string + // OSReleasePath is the path where os-release can be found for RHCOS detection. + // This may differ from HostPath when the RPM database is mounted separately. + OSReleasePath string // Client is the HTTP client used to reach out to external data sources. // If unset, a default which uses client-side TLS certificates is used. Client *http.Client // Repo2CPEMappingURL can be used to fetch the repo mapping file. // Consulting the mapping file is preferred over the Container API. Repo2CPEMappingURL string + // Repo2CPEMappingFile is a local path to the repo-to-CPE mapping file. + // When set, leave Repo2CPEMappingURL empty: claircore still fetches a + // non-empty URL on every call (its freshly-constructed rate limiter is + // not treated as "already fetched"), so a non-empty URL would make a + // network request on top of the seeded file. + Repo2CPEMappingFile string // Timeout controls the timeout for any remote API calls. Timeout time.Duration - // PackageDBFilter removes irrelevant packages. For node scanning, we are - // currently only interested in the RHCOS RPM database. - // Filters out all packages whose packageDB does not match the filter. - // Empty string corresponds to no filtering. - PackageDBFilter string + // PackageDBFilter keeps RHCOS RPM databases and drops other package DBs + // Claircore finds under the host index mount. Empty means no filtering. + PackageDBFilter []string } // DefaultNodeIndexerConfig provides the default configuration for a node indexer. func DefaultNodeIndexerConfig() NodeIndexerConfig { return NodeIndexerConfig{ - HostPath: env.NodeIndexHostPath.Setting(), + HostPath: env.NodeIndexHostPath.Setting(), + OSReleasePath: env.NodeIndexOSReleasePath.Setting(), // The default, mTLS-capable client will be used. Client: nil, Repo2CPEMappingURL: buildMappingURL(), Timeout: 10 * time.Second, - PackageDBFilter: rhcosPackageDB, + PackageDBFilter: rhcosPackageDBs, } } @@ -134,7 +149,7 @@ func buildMappingURL() string { if len(env.NodeIndexMappingURL.Setting()) > 0 { return urlfmt.FormatURL(env.NodeIndexMappingURL.Setting(), urlfmt.HTTPS, urlfmt.NoTrailingSlash) } - u := env.AdvertisedEndpoint.Setting() + scannerDefinitionsRouteInSensor + "?file=" + sensorMappingsFile + u := env.SensorEndpointSetting() + scannerDefinitionsRouteInSensor + "?file=" + sensorMappingsFile return urlfmt.FormatURL(u, urlfmt.HTTPS, urlfmt.NoTrailingSlash) } @@ -182,6 +197,14 @@ func (l *localNodeIndexer) IndexNode(ctx context.Context) (*v4.IndexReport, erro } log.Debugf("Finished coalescing report. Report contains %d repositories with %d packages", len(ccReport.Repositories), len(ccReport.Packages)) + rhcosRel, err := osRelease(ctx, l.cfg.OSReleasePath) + if err != nil { + log.Debugf("Not adding RHCOS package to index report: %v", err) + } else { + arch := extractArch(ccReport, pkgs) + addRHCOS(rhcosRel, arch, ccReport) + } + ccReport.Success = true ccReport.State = controller.IndexFinished.String() @@ -195,14 +218,27 @@ func (l *localNodeIndexer) IndexNode(ctx context.Context) (*v4.IndexReport, erro func layer(ctx context.Context, digest string, hostPath string) (*claircore.Layer, error) { log.Debugf("Realizing mount path: %s", hostPath) + if hostPath == "" { + return nil, errors.New("host path is empty") + } + + absoluteHostPath, err := filepath.Abs(hostPath) + if err != nil { + return nil, errors.Wrapf(err, "resolving absolute host path %q", hostPath) + } + hostURI := (&url.URL{ + Scheme: "file", + Path: filepath.ToSlash(absoluteHostPath), + }).String() + desc := &claircore.LayerDescription{ Digest: digest, - URI: hostPath, + URI: hostURI, MediaType: layerMediaType, } l := &claircore.Layer{} - err := l.Init(ctx, desc, nil) + err = l.Init(ctx, desc, nil) return l, errors.Wrap(err, "failed to init layer") } @@ -210,9 +246,9 @@ func runRepositoryScanner(ctx context.Context, cfg NodeIndexerConfig, l *clairco client := cfg.Client if client == nil { var err error - client, err = getDefaultClient() + client, err = newDefaultClient(cfg.Repo2CPEMappingURL) if err != nil { - return nil, errors.Wrap(err, "creating repository scanner http client - check TLS config") + return nil, errors.Wrap(err, "creating repository scanner http client") } } @@ -220,9 +256,10 @@ func runRepositoryScanner(ctx context.Context, cfg NodeIndexerConfig, l *clairco config := rhel.RepositoryScannerConfig{ // Do not reach out to the Red Hat Container Catalog API. // We do *not* want to reach out to the internet for node scanning. - DisableAPI: true, - Repo2CPEMappingURL: cfg.Repo2CPEMappingURL, - Timeout: cfg.Timeout, + DisableAPI: true, + Repo2CPEMappingURL: cfg.Repo2CPEMappingURL, + Repo2CPEMappingFile: cfg.Repo2CPEMappingFile, + Timeout: cfg.Timeout, } var buf bytes.Buffer @@ -243,23 +280,14 @@ func runRepositoryScanner(ctx context.Context, cfg NodeIndexerConfig, l *clairco return repos, nil } -func runPackageScanner(ctx context.Context, packageDBFilter string, layer *claircore.Layer) ([]*claircore.Package, error) { +func runPackageScanner(ctx context.Context, packageDBFilter []string, layer *claircore.Layer) ([]*claircore.Package, error) { scanner := rhel.PackageScanner{} pkgs, err := scanner.Scan(ctx, layer) if err != nil { return nil, errors.Wrap(err, "failed to invoke RHEL scanner") } - // Filter out packages in which we are not interested. - filtered := pkgs - if packageDBFilter != "" { - filtered = pkgs[:0] - for _, pkg := range pkgs { - if pkg.PackageDB == packageDBFilter { - filtered = append(filtered, pkg) - } - } - } + filtered := filterPackages(pkgs, packageDBFilter) for i, p := range filtered { p.ID = strconv.Itoa(i) } @@ -267,6 +295,33 @@ func runPackageScanner(ctx context.Context, packageDBFilter string, layer *clair return filtered, nil } +// filterPackages keeps packages whose PackageDB is in packageDBFilter. +// ostree nodes expose the same RPMs in two PackageDBs, so duplicates +// collapse to one per name/version/arch/kind. +func filterPackages(pkgs []*claircore.Package, packageDBFilter []string) []*claircore.Package { + if len(packageDBFilter) == 0 { + return pkgs + } + type ident struct { + name, version, arch string + kind types.PackageKind + } + out := make([]*claircore.Package, 0, len(pkgs)) + seen := make(map[ident]struct{}, len(pkgs)) + for _, pkg := range pkgs { + if !slices.Contains(packageDBFilter, pkg.PackageDB) { + continue + } + id := ident{pkg.Name, pkg.Version, pkg.Arch, pkg.Kind} + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + out = append(out, pkg) + } + return out +} + func runCoalescer(ctx context.Context, layerDigest claircore.Digest, repos []*claircore.Repository, pkgs []*claircore.Package) (*claircore.IndexReport, error) { la := &ccindexer.LayerArtifacts{ Hash: layerDigest, @@ -283,3 +338,153 @@ func runCoalescer(ctx context.Context, layerDigest claircore.Digest, repos []*cl return ir, nil } + +func validateOSRelease(osRel map[string]string) error { + if variant := osRel["VARIANT_ID"]; variant != "coreos" { + return fmt.Errorf("not RHCOS: VARIANT_ID=%q", variant) + } + if osRel["VERSION"] == "" { + return errors.New("VERSION not found in os-release") + } + if osRel["OPENSHIFT_VERSION"] == "" { + return errors.New("OPENSHIFT_VERSION not found in os-release") + } + if osRel["VERSION_ID"] == "" { + return errors.New("VERSION_ID not found in os-release") + } + return nil +} + +func parseOSRelease(ctx context.Context, hostPath string) (map[string]string, error) { + var f *os.File + for _, relPath := range []string{osrelease.Path, osrelease.FallbackPath} { + path := filepath.Join(hostPath, relPath) + var err error + f, err = os.Open(path) + if err != nil { + continue + } + break + } + if f == nil { + return nil, fmt.Errorf("os-release not found in %s", hostPath) + } + defer func() { + _ = f.Close() + }() + return osrelease.Parse(ctx, f) +} + +type rhcosRelease struct { + version string + normVersion claircore.Version + repoCPE cpe.WFN +} + +// osRelease opens, parse and validate the os release file, returning RHCOS +// version and repository CPE. +func osRelease(ctx context.Context, osRelPath string) (rhcosRelease, error) { + osRel, err := parseOSRelease(ctx, osRelPath) + if err != nil { + return rhcosRelease{}, fmt.Errorf("failed to parse os-release: %w", err) + } + if err := validateOSRelease(osRel); err != nil { + return rhcosRelease{}, fmt.Errorf("invalid os-release: %w", err) + } + + rel := rhcosRelease{ + version: osRel["VERSION"], + } + + // Set up the repository CPE. + rhelMajor, _, _ := strings.Cut(osRel["VERSION_ID"], ".") + cpeStr := fmt.Sprintf("cpe:/a:redhat:openshift:%s::el%s", osRel["OPENSHIFT_VERSION"], rhelMajor) + rel.repoCPE, err = cpe.Unbind(cpeStr) + if err != nil { + return rhcosRelease{}, fmt.Errorf("failed to parse RHCOS repository CPE: %w", err) + } + + // Set up the normalized version. + rhctagVersion, err := rhctag.Parse(rel.version) + if err != nil { + log.Warnf("Failed to parse RHCOS version %q: %v", rel.version, err) + return rhcosRelease{}, fmt.Errorf("failed to parse RHCOS version %q: %w", rel.version, err) + } + minorStart := rhctagVersion.MinorStart() + rel.normVersion = minorStart.Version(true) + + return rel, nil +} + +func addRHCOS(rel rhcosRelease, arch string, report *claircore.IndexReport) { + const ( + rhcosPkgID = "rhcos-pkg" + rhcosSrcID = "rhcos-src" + rhcosRepoID = "rhcos-repo" + ) + + srcPkg := &claircore.Package{ + ID: rhcosSrcID, + Name: "rhcos", + Version: rel.version, + Kind: types.SourcePackage, + NormalizedVersion: rel.normVersion, + Arch: arch, + } + binPkg := &claircore.Package{ + ID: rhcosPkgID, + Name: "rhcos", + Version: rel.version, + Kind: types.BinaryPackage, + NormalizedVersion: rel.normVersion, + Source: srcPkg, + PackageDB: "", + Arch: arch, + RepositoryHint: "rhcc", + } + repo := &claircore.Repository{ + ID: rhcosRepoID, + Name: rel.repoCPE.String(), + Key: rhcc.RepositoryKey, + CPE: rel.repoCPE, + } + + if report.Packages == nil { + report.Packages = make(map[string]*claircore.Package) + } + if report.Repositories == nil { + report.Repositories = make(map[string]*claircore.Repository) + } + if report.Environments == nil { + report.Environments = make(map[string][]*claircore.Environment) + } + + report.Packages[srcPkg.ID] = srcPkg + report.Packages[binPkg.ID] = binPkg + report.Repositories[repo.ID] = repo + report.Environments[binPkg.ID] = []*claircore.Environment{ + { + PackageDB: "", + IntroducedIn: ccLayerDigest, + RepositoryIDs: []string{repo.ID}, + }, + } + + log.Debugf("Added RHCOS package: version=%s, cpe=%s", rel.version, rel.repoCPE.String()) +} + +// extractArch attempts to determine the RHCOS architecture from the current list +// of packages, failing open if no good guess is found, returning an empty string. +func extractArch(report *claircore.IndexReport, pkgs []*claircore.Package) string { + for _, d := range report.Distributions { + if d.Arch != "" && d.Arch != "noarch" { + return d.Arch + } + } + for _, p := range pkgs { + if p.Arch != "" && p.Arch != "noarch" { + return p.Arch + } + } + return "" +} diff --git a/compliance/node/index/indexer_test.go b/compliance/node/index/indexer_test.go index 2a281a61ae888..2adcbebe7807e 100644 --- a/compliance/node/index/indexer_test.go +++ b/compliance/node/index/indexer_test.go @@ -3,16 +3,19 @@ package index import ( "context" "crypto/tls" - "crypto/x509" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" + "slices" "strings" "testing" "time" "github.com/quay/claircore" + "github.com/quay/claircore/toolkit/types" + "github.com/stackrox/rox/pkg/certgen" "github.com/stackrox/rox/pkg/mtls" "github.com/stretchr/testify/suite" ) @@ -58,19 +61,29 @@ func (s *nodeIndexerSuite) createTestServer(tlsEnabled bool) *httptest.Server { if !tlsEnabled { server.Start() } else { - serverCert, err := tls.LoadX509KeyPair( - filepath.Join("testdata", "certs", "server-cert.pem"), - filepath.Join("testdata", "certs", "server-key.pem"), - ) + ca, err := certgen.GenerateCA() s.Require().NoError(err) - caCert, err := os.ReadFile(filepath.Join("testdata", "certs", "ca-cert.pem")) + + serverCertIssued, err := ca.IssueCertForSubject(mtls.SensorSubject) + s.Require().NoError(err) + serverCert, err := tls.X509KeyPair(serverCertIssued.CertPEM, serverCertIssued.KeyPEM) + s.Require().NoError(err) + + clientCertIssued, err := ca.IssueCertForSubject(mtls.SensorSubject) s.Require().NoError(err) - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) + + certDir := s.T().TempDir() + s.Require().NoError(os.WriteFile(filepath.Join(certDir, "ca.pem"), ca.CertPEM(), 0644)) + s.Require().NoError(os.WriteFile(filepath.Join(certDir, "client-cert.pem"), clientCertIssued.CertPEM, 0600)) + s.Require().NoError(os.WriteFile(filepath.Join(certDir, "client-key.pem"), clientCertIssued.KeyPEM, 0600)) + s.T().Setenv(mtls.CAFileEnvName, filepath.Join(certDir, "ca.pem")) + s.T().Setenv(mtls.CertFilePathEnvName, filepath.Join(certDir, "client-cert.pem")) + s.T().Setenv(mtls.KeyFileEnvName, filepath.Join(certDir, "client-key.pem")) + server.TLS = &tls.Config{ Certificates: []tls.Certificate{serverCert}, ClientAuth: tls.RequireAndVerifyClientCert, - ClientCAs: caCertPool, + ClientCAs: ca.CertPool(), } server.StartTLS() } @@ -102,7 +115,23 @@ func (s *nodeIndexerSuite) TestLayer() { func (s *nodeIndexerSuite) TestLayerNoURI() { _, err := layer(context.Background(), layerDigest, "") - s.ErrorContains(err, "no URI provided") + s.ErrorContains(err, "host path is empty") +} + +func (s *nodeIndexerSuite) TestLayerUsesFileURI() { + layer, err := layer(context.Background(), layerDigest, "testdata") + s.Require().NoError(err) + s.T().Cleanup(func() { + s.Require().NoError(layer.Close()) + }) + + parsedURI, err := url.Parse(layer.URI) + s.Require().NoError(err) + s.Equal("file", parsedURI.Scheme) + + expectedPath, err := filepath.Abs("testdata") + s.Require().NoError(err) + s.Equal(filepath.ToSlash(expectedPath), parsedURI.Path) } func (s *nodeIndexerSuite) TestLayerIllegalDigest() { @@ -136,7 +165,7 @@ func (s *nodeIndexerSuite) TestRunRepositoryScannerAnyPath() { func (s *nodeIndexerSuite) TestRunPackageScanner() { layer := s.mustCreateLayer("testdata") - packages, err := runPackageScanner(context.Background(), rhcosPackageDB, layer) + packages, err := runPackageScanner(context.Background(), rhcosPackageDBs, layer) s.NoError(err) s.Len(packages, 106) @@ -145,7 +174,7 @@ func (s *nodeIndexerSuite) TestRunPackageScanner() { func (s *nodeIndexerSuite) TestRunPackageScannerWithUnmatchedFilter() { layer := s.mustCreateLayer("testdata") - packages, err := runPackageScanner(context.Background(), "invalidPackageDB", layer) + packages, err := runPackageScanner(context.Background(), []string{"invalidPackageDB"}, layer) s.NoError(err) // All packages are filtered out. @@ -155,67 +184,227 @@ func (s *nodeIndexerSuite) TestRunPackageScannerWithUnmatchedFilter() { func (s *nodeIndexerSuite) TestRunPackageScannerAnyPath() { layer := s.mustCreateLayer(s.T().TempDir()) - packages, err := runPackageScanner(context.Background(), rhcosPackageDB, layer) + packages, err := runPackageScanner(context.Background(), rhcosPackageDBs, layer) s.NoError(err) // The scanner must not error out, but produce 0 results s.Len(packages, 0) } +func (s *nodeIndexerSuite) TestRHCOSPackageDBs() { + tests := map[string]bool{ + "sqlite:usr/share/rpm": true, + "sqlite:usr/lib/sysimage/rpm": true, + "bdb:usr/share/rpm": true, + "bdb:usr/lib/sysimage/rpm-ostree-base-db": true, + "sqlite:var/lib/rpm": false, + } + for db, want := range tests { + s.Run(db, func() { + s.Equal(want, slices.Contains(rhcosPackageDBs, db)) + }) + } +} + +func (s *nodeIndexerSuite) TestFilterPackages() { + pkg := func(db, name, version string) *claircore.Package { + return &claircore.Package{ + PackageDB: db, + Name: name, + Version: version, + Arch: "x86_64", + Kind: types.BinaryPackage, + } + } + share := "bdb:usr/share/rpm" + ostree := "bdb:usr/lib/sysimage/rpm-ostree-base-db" + filter := []string{share, ostree} + + tests := map[string]struct { + pkgs []*claircore.Package + filter []string + want int + }{ + "empty filter keeps all": { + pkgs: []*claircore.Package{pkg(share, "bash", "1"), pkg("other", "zsh", "1")}, + filter: nil, + want: 2, + }, + "unmatched PackageDB dropped": { + pkgs: []*claircore.Package{pkg("sqlite:var/lib/rpm", "bash", "1")}, + filter: filter, + want: 0, + }, + "ostree duplicate PackageDBs collapse": { + pkgs: []*claircore.Package{ + pkg(share, "bash", "5.0"), + pkg(ostree, "bash", "5.0"), + }, + filter: filter, + want: 1, + }, + "distinct NEVRAs kept": { + pkgs: []*claircore.Package{ + pkg(share, "bash", "5.0"), + pkg(share, "coreutils", "8.0"), + }, + filter: filter, + want: 2, + }, + } + for name, tc := range tests { + s.Run(name, func() { + s.Len(filterPackages(tc.pkgs, tc.filter), tc.want) + }) + } + + s.Run("does not alias input backing array", func() { + a := pkg(share, "bash", "5.0") + b := pkg(ostree, "bash", "5.0") + in := []*claircore.Package{a, b} + got := filterPackages(in, filter) + s.Len(got, 1) + s.Same(a, in[0]) + s.Same(b, in[1]) + s.Len(in, 2) + }) +} + func (s *nodeIndexerSuite) TestBuildMappingURL() { tcs := map[string]struct { + sensorEndpointSetting string advertisedEndpointSetting string + useLegacyAdvertised bool mappingURLSetting string expectedURL string }{ "Empty": { - advertisedEndpointSetting: "", - mappingURLSetting: "", - expectedURL: "https://sensor.stackrox.svc:443/scanner/definitions?file=repo2cpe", + sensorEndpointSetting: "", + mappingURLSetting: "", + expectedURL: "https://sensor.stackrox.svc:443/scanner/definitions?file=repo2cpe", }, "Host with port": { - advertisedEndpointSetting: "example.com:8080", - mappingURLSetting: "", - expectedURL: "https://example.com:8080/scanner/definitions?file=repo2cpe", + sensorEndpointSetting: "example.com:8080", + mappingURLSetting: "", + expectedURL: "https://example.com:8080/scanner/definitions?file=repo2cpe", }, "Host without port": { - advertisedEndpointSetting: "sensor.rhacs.svc", - mappingURLSetting: "", - expectedURL: "https://sensor.rhacs.svc/scanner/definitions?file=repo2cpe", + sensorEndpointSetting: "sensor.rhacs.svc", + mappingURLSetting: "", + expectedURL: "https://sensor.rhacs.svc/scanner/definitions?file=repo2cpe", }, "HTTP scheme": { - advertisedEndpointSetting: "http://example.com", + sensorEndpointSetting: "http://example.com", + mappingURLSetting: "", + expectedURL: "https://example.com/scanner/definitions?file=repo2cpe", + }, + "Legacy advertised endpoint only": { + advertisedEndpointSetting: "sensor.legacy.svc:443", + useLegacyAdvertised: true, mappingURLSetting: "", - expectedURL: "https://example.com/scanner/definitions?file=repo2cpe", + expectedURL: "https://sensor.legacy.svc:443/scanner/definitions?file=repo2cpe", }, "Mapping setting provided": { - advertisedEndpointSetting: "sensor.namespace.svc:443", - mappingURLSetting: "https://example.com/download", - expectedURL: "https://example.com/download", + sensorEndpointSetting: "sensor.namespace.svc:443", + mappingURLSetting: "https://example.com/download", + expectedURL: "https://example.com/download", }, "Mapping setting provided with no scheme and trailing slash": { - advertisedEndpointSetting: "sensor.namespace.svc:443", - mappingURLSetting: "example.com/download/", - expectedURL: "https://example.com/download", + sensorEndpointSetting: "sensor.namespace.svc:443", + mappingURLSetting: "example.com/download/", + expectedURL: "https://example.com/download", }, } for name, tc := range tcs { s.T().Run(name, func(t *testing.T) { - s.T().Setenv("ROX_ADVERTISED_ENDPOINT", tc.advertisedEndpointSetting) + s.T().Setenv("ROX_SENSOR_ENDPOINT", tc.sensorEndpointSetting) + if tc.useLegacyAdvertised { + s.T().Setenv("ROX_ADVERTISED_ENDPOINT", tc.advertisedEndpointSetting) + } else { + s.T().Setenv("ROX_ADVERTISED_ENDPOINT", "") + } s.T().Setenv("ROX_NODE_INDEX_MAPPING_URL", tc.mappingURLSetting) s.Equal(tc.expectedURL, buildMappingURL()) }) } } +func (s *nodeIndexerSuite) TestExtractHostname() { + tcs := map[string]struct { + input string + expected string + wantError bool + }{ + "full URL": {"https://sensor.stackrox.svc:443/scanner/definitions?file=repo2cpe", "sensor.stackrox.svc", false}, + "URL without port": {"https://sensor.stackrox.svc/scanner/definitions", "sensor.stackrox.svc", false}, + "URL without path": {"https://example.com", "example.com", false}, + "URL with port": {"https://example.com:8080", "example.com", false}, + "IP address with port": {"https://10.96.0.1:443/path", "10.96.0.1", false}, + "empty string": {"", "", false}, + "just hostname": {"sensor.stackrox.svc", "sensor.stackrox.svc", false}, + "hostname with port": {"sensor.stackrox.svc:443", "sensor.stackrox.svc", false}, + "invalid URL": {"https://[invalid", "", true}, + } + for name, tc := range tcs { + s.Run(name, func() { + got, err := extractHostname(tc.input) + if tc.wantError { + s.Error(err) + } else { + s.NoError(err) + s.Equal(tc.expected, got) + } + }) + } +} + func (s *nodeIndexerSuite) TestIndexerE2E() { - s.T().Setenv(mtls.CertFilePathEnvName, filepath.Join("testdata", "certs", "client-cert.pem")) - s.T().Setenv(mtls.KeyFileEnvName, filepath.Join("testdata", "certs", "client-key.pem")) server := s.createTestServer(true) + + s.Run("override Repo2CPEMappingURL", func() { + cfg := DefaultNodeIndexerConfig() + cfg.HostPath = "testdata" + cfg.Repo2CPEMappingURL = server.URL + cfg.PackageDBFilter = rhcosPackageDBs + + report, err := NewNodeIndexer(cfg).IndexNode(context.Background()) + s.NoError(err) + s.NotNil(report) + s.True(report.GetSuccess()) + s.Len(report.GetContents().GetPackages(), 106) + s.Len(report.GetContents().GetRepositories(), 2) + }) + + s.Run("explicit ROX_NODE_INDEX_MAPPING_URL", func() { + mappingURL := server.URL + "/scanner/definitions?file=repo2cpe" + s.T().Setenv("ROX_NODE_INDEX_MAPPING_URL", mappingURL) + s.T().Setenv("ROX_ADVERTISED_ENDPOINT", "ignored.example.svc:9999") + + cfg := DefaultNodeIndexerConfig() + s.Equal(buildMappingURL(), cfg.Repo2CPEMappingURL) + s.Nil(cfg.Client) + cfg.HostPath = "testdata" + cfg.PackageDBFilter = rhcosPackageDBs + + report, err := NewNodeIndexer(cfg).IndexNode(context.Background()) + s.NoError(err) + s.NotNil(report) + s.True(report.GetSuccess()) + s.Len(report.GetContents().GetPackages(), 106) + s.Len(report.GetContents().GetRepositories(), 2) + }) +} + +func (s *nodeIndexerSuite) TestIndexerE2ESeparateOSReleasePath() { + // Use a plain HTTP mapping server + explicit client so this test does not + // depend on the process-wide mTLS certwatch state left by other suite tests. + server := s.createTestServer(false) cfg := DefaultNodeIndexerConfig() cfg.HostPath = "testdata" + cfg.OSReleasePath = "testdata-rhcos" cfg.Repo2CPEMappingURL = server.URL - cfg.PackageDBFilter = rhcosPackageDB + cfg.Client = server.Client() + cfg.PackageDBFilter = rhcosPackageDBs indexer := NewNodeIndexer(cfg) report, err := indexer.IndexNode(context.Background()) @@ -223,8 +412,20 @@ func (s *nodeIndexerSuite) TestIndexerE2E() { s.NotNil(report) s.True(report.GetSuccess()) - s.Len(report.GetContents().GetPackages(), 106, "Expected number of installed packages differs") - s.Len(report.GetContents().GetRepositories(), 2, "Expected number of discovered repositories differs") + // 106 RPM packages + 2 rhcos packages (binary + source) + s.Len(report.GetContents().GetPackages(), 108, "Expected 106 RPM + 2 rhcos packages") + // 2 RPM repositories + 1 rhcos repository + s.Len(report.GetContents().GetRepositories(), 3, "Expected 2 RPM + 1 rhcos repositories") + + var hasRHCOS bool + for _, pkg := range report.GetContents().GetPackages() { + if pkg.GetName() == "rhcos" { + hasRHCOS = true + s.Equal("9.6.20260324-0", pkg.GetVersion()) + break + } + } + s.True(hasRHCOS, "Expected rhcos package in report") } func (s *nodeIndexerSuite) TestIndexerE2ENoPath() { @@ -233,7 +434,7 @@ func (s *nodeIndexerSuite) TestIndexerE2ENoPath() { cfg.Client = server.Client() cfg.HostPath = "doesnotexist" cfg.Repo2CPEMappingURL = server.URL - cfg.PackageDBFilter = rhcosPackageDB + cfg.PackageDBFilter = rhcosPackageDBs indexer := NewNodeIndexer(cfg) report, err := indexer.IndexNode(context.Background()) @@ -242,3 +443,183 @@ func (s *nodeIndexerSuite) TestIndexerE2ENoPath() { s.ErrorIs(err, os.ErrNotExist) s.Nil(report) } + +func (s *nodeIndexerSuite) TestParseOSRelease() { + osRel, err := parseOSRelease(context.Background(), "testdata-rhcos") + s.Require().NoError(err) + + s.Equal("coreos", osRel["VARIANT_ID"]) + s.Equal("9.6.20260324-0", osRel["VERSION"]) + s.Equal("9.6", osRel["VERSION_ID"]) + s.Equal("4.21", osRel["OPENSHIFT_VERSION"]) +} + +func (s *nodeIndexerSuite) TestParseOSReleaseNotFound() { + _, err := parseOSRelease(context.Background(), s.T().TempDir()) + s.ErrorContains(err, "os-release not found") +} + +func (s *nodeIndexerSuite) TestOSReleaseInvalidVersion() { + tmpDir := s.T().TempDir() + etcDir := filepath.Join(tmpDir, "etc") + s.Require().NoError(os.MkdirAll(etcDir, 0755)) + + osReleaseContent := `VARIANT_ID=coreos +VERSION=invalid-version-format +VERSION_ID=9.6 +OPENSHIFT_VERSION=4.21 +` + s.Require().NoError(os.WriteFile(filepath.Join(etcDir, "os-release"), []byte(osReleaseContent), 0644)) + + _, err := osRelease(context.Background(), tmpDir) + s.ErrorContains(err, "failed to parse RHCOS version") +} + +func (s *nodeIndexerSuite) TestAddRHCOSPackageToReport() { + rel, err := osRelease(context.Background(), "testdata-rhcos") + s.Require().NoError(err) + + report := &claircore.IndexReport{ + Packages: make(map[string]*claircore.Package), + Repositories: make(map[string]*claircore.Repository), + Environments: make(map[string][]*claircore.Environment), + } + + addRHCOS(rel, "x86_64", report) + + s.Len(report.Packages, 2) + s.Len(report.Repositories, 1) + s.Len(report.Environments, 1) + + var binPkg *claircore.Package + for _, p := range report.Packages { + if p.Kind == types.BinaryPackage && p.Name == "rhcos" { + binPkg = p + break + } + } + s.Require().NotNil(binPkg) + s.Equal("rhcos", binPkg.Name) + s.Equal("9.6.20260324-0", binPkg.Version) + s.Equal("x86_64", binPkg.Arch) + s.Equal("rhcc", binPkg.RepositoryHint) + + var repo *claircore.Repository + for _, r := range report.Repositories { + repo = r + break + } + s.Require().NotNil(repo) + s.Equal("rhcc-container-repository", repo.Key) + s.Contains(repo.Name, "cpe:") + s.Contains(repo.Name, "openshift") + s.Contains(repo.Name, "4.21") +} + +func (s *nodeIndexerSuite) TestValidateOSRelease() { + cases := map[string]struct { + osRel map[string]string + expectError string + expectNoError bool + }{ + "valid": { + osRel: map[string]string{ + "VARIANT_ID": "coreos", + "VERSION": "9.6.20260324-0", + "VERSION_ID": "9.6", + "OPENSHIFT_VERSION": "4.21", + }, + expectNoError: true, + }, + "missing VERSION": { + osRel: map[string]string{ + "VARIANT_ID": "coreos", + "VERSION_ID": "9.6", + "OPENSHIFT_VERSION": "4.21", + }, + expectError: "VERSION not found", + }, + "missing OPENSHIFT_VERSION": { + osRel: map[string]string{ + "VARIANT_ID": "coreos", + "VERSION": "9.6.20260324-0", + "VERSION_ID": "9.6", + }, + expectError: "OPENSHIFT_VERSION not found", + }, + "missing VERSION_ID": { + osRel: map[string]string{ + "VARIANT_ID": "coreos", + "VERSION": "9.6.20260324-0", + "OPENSHIFT_VERSION": "4.21", + }, + expectError: "VERSION_ID not found", + }, + "not RHCOS": { + osRel: map[string]string{ + "VARIANT_ID": "server", + "VERSION": "9.6", + "VERSION_ID": "9.6", + "OPENSHIFT_VERSION": "4.21", + }, + expectError: "not RHCOS", + }, + } + + for name, tc := range cases { + s.Run(name, func() { + err := validateOSRelease(tc.osRel) + if tc.expectNoError { + s.NoError(err) + } else { + s.ErrorContains(err, tc.expectError) + } + }) + } +} + +func (s *nodeIndexerSuite) TestExtractArch() { + cases := map[string]struct { + report *claircore.IndexReport + pkgs []*claircore.Package + expected string + }{ + "from distribution": { + report: &claircore.IndexReport{Distributions: map[string]*claircore.Distribution{"1": {Arch: "x86_64"}}}, + pkgs: nil, + expected: "x86_64", + }, + "from packages when no distribution": { + report: &claircore.IndexReport{}, + pkgs: []*claircore.Package{{Arch: "aarch64"}}, + expected: "aarch64", + }, + "skips noarch distribution": { + report: &claircore.IndexReport{Distributions: map[string]*claircore.Distribution{"1": {Arch: "noarch"}}}, + pkgs: []*claircore.Package{{Arch: "x86_64"}}, + expected: "x86_64", + }, + "skips noarch packages": { + report: &claircore.IndexReport{}, + pkgs: []*claircore.Package{{Arch: "noarch"}, {Arch: "x86_64"}}, + expected: "x86_64", + }, + "empty when all noarch": { + report: &claircore.IndexReport{Distributions: map[string]*claircore.Distribution{"1": {Arch: "noarch"}}}, + pkgs: []*claircore.Package{{Arch: "noarch"}}, + expected: "", + }, + "empty when no data": { + report: &claircore.IndexReport{}, + pkgs: nil, + expected: "", + }, + } + + for name, tc := range cases { + s.Run(name, func() { + got := extractArch(tc.report, tc.pkgs) + s.Equal(tc.expected, got) + }) + } +} diff --git a/compliance/node/index/testdata-rhcos/etc/os-release b/compliance/node/index/testdata-rhcos/etc/os-release new file mode 100644 index 0000000000000..6ae4c8f88b616 --- /dev/null +++ b/compliance/node/index/testdata-rhcos/etc/os-release @@ -0,0 +1,21 @@ +NAME="Red Hat Enterprise Linux CoreOS" +ID="rhel" +ID_LIKE="rhel fedora" +VERSION="9.6.20260324-0" +VERSION_ID="9.6" +VARIANT="CoreOS" +VARIANT_ID=coreos +PLATFORM_ID="platform:el9" +PRETTY_NAME="Red Hat Enterprise Linux CoreOS 421.94.202503241034-0 (Plow)" +ANSI_COLOR="0;31" +CPE_NAME="cpe:/o:redhat:enterprise_linux:9::coreos" +HOME_URL="https://www.redhat.com/" +DOCUMENTATION_URL="https://docs.openshift.com/container-platform/4.21/" +BUG_REPORT_URL="https://access.redhat.com/labs/rhir/" +REDHAT_BUGZILLA_PRODUCT="OpenShift Container Platform" +REDHAT_BUGZILLA_PRODUCT_VERSION="4.21" +REDHAT_SUPPORT_PRODUCT="OpenShift Container Platform" +REDHAT_SUPPORT_PRODUCT_VERSION="4.21" +OPENSHIFT_VERSION="4.21" +RHEL_VERSION="9.6" +OSTREE_VERSION='421.94.202503241034-0' diff --git a/compliance/node/index/testdata/certs/README.txt b/compliance/node/index/testdata/certs/README.txt deleted file mode 100644 index dd37486c055ff..0000000000000 --- a/compliance/node/index/testdata/certs/README.txt +++ /dev/null @@ -1 +0,0 @@ -These files were generated following instructions from https://mariadb.com/docs/server/security/data-in-transit-encryption/create-self-signed-certificates-keys-openssl/. diff --git a/compliance/node/index/testdata/certs/ca-cert.pem b/compliance/node/index/testdata/certs/ca-cert.pem deleted file mode 100644 index 7cb00f02015f1..0000000000000 --- a/compliance/node/index/testdata/certs/ca-cert.pem +++ /dev/null @@ -1,24 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEATCCAumgAwIBAgIUW+AjHh3yyIliq5CPgnL6NzaCGigwDQYJKoZIhvcNAQEL -BQAwgY4xCzAJBgNVBAYTAlVTMRcwFQYDVQQIDA5Ob3J0aCBDYXJvbGluYTEQMA4G -A1UEBwwHUmFsZWlnaDEQMA4GA1UECgwHUmVkIEhhdDERMA8GA1UECwwIU3RhY2tS -b3gxETAPBgNVBAMMCFN1cGVyIENBMRwwGgYJKoZIhvcNAQkBFg1jYUBzZXJ2ZXIu -Y29tMCAXDTI0MTEwMjAwMTgzMFoYDzMwMjQwMzA1MDAxODMwWjCBjjELMAkGA1UE -BhMCVVMxFzAVBgNVBAgMDk5vcnRoIENhcm9saW5hMRAwDgYDVQQHDAdSYWxlaWdo -MRAwDgYDVQQKDAdSZWQgSGF0MREwDwYDVQQLDAhTdGFja1JveDERMA8GA1UEAwwI -U3VwZXIgQ0ExHDAaBgkqhkiG9w0BCQEWDWNhQHNlcnZlci5jb20wggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCc/+/G5d3aNtSnjPb0bi3dSg682WmwtMvb -KjCVLMafE7aJi4QxV02zazyetmRAbS1GGz3h94TCL9cQJswTIUsz9qXLIvFl32vT -NH4mV6VQX2W7clOCG/+wis4RWYKhHCOk9q6qyQI8JTft3V1O+0Gy7u7NpicgKbX2 -FTCDJ/A4bKidUCtizi9fkRDHMqKM1/xGSTv2YtH+CDZrsm0G9B4NX1cBKYdyXjTG -IVfC0zcsiSEXuYUDQnQJPrpj9vuUQ/0BCGEjDNZLF+jJl6G0grMjvNzJ70dKUSXg -SKD9sG8raMcX4TKY4MKT6KkjMvBGoeixaGRFO3EyC5vwsRq+2lk5AgMBAAGjUzBR -MB0GA1UdDgQWBBQPERz8uesACDjYzcXHfrtt2v6bcTAfBgNVHSMEGDAWgBQPERz8 -uesACDjYzcXHfrtt2v6bcTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA -A4IBAQApF863JE+6JN677PwaFGZUHThjtV9ArGvpDqtCkLLTtKeFiTEP9GvM57A9 -DCW8XSOZwRihTVB9i/o22+RYNgijLYbQ9Xn5SqrcwejFQTTm3AzWM+QTbRfDj6na -7iZPs8hQpFORGLQHNByH+9NsOf6Q1Y9vQlVqlSS50VxZZfBbnj21A4yT9usjKiC7 -pToPuDAPJTvCdrqJjJwkj6k0hSf8Isa3+ipO/kjCnPR8by6WceiYi6arqVeixh6m -PGLCIxYnCAhlsHDwj5u2eF2AuIYHruGE7bIKEytyka3YydBgv9EC9ErVC4QbgEF6 -VAZ09jYW7rx0HdjyV2ekHBZwdGsU ------END CERTIFICATE----- diff --git a/compliance/node/index/testdata/certs/client-cert.pem b/compliance/node/index/testdata/certs/client-cert.pem deleted file mode 100644 index dc177809165f2..0000000000000 --- a/compliance/node/index/testdata/certs/client-cert.pem +++ /dev/null @@ -1,23 +0,0 @@ ------BEGIN CERTIFICATE----- -MIID5TCCAs2gAwIBAgIBATANBgkqhkiG9w0BAQsFADCBjjELMAkGA1UEBhMCVVMx -FzAVBgNVBAgMDk5vcnRoIENhcm9saW5hMRAwDgYDVQQHDAdSYWxlaWdoMRAwDgYD -VQQKDAdSZWQgSGF0MREwDwYDVQQLDAhTdGFja1JveDERMA8GA1UEAwwIU3VwZXIg -Q0ExHDAaBgkqhkiG9w0BCQEWDWNhQHNlcnZlci5jb20wIBcNMjQxMTAyMDAyMTEw -WhgPMzAyNDAzMDUwMDIxMTBaMIGWMQswCQYDVQQGEwJVUzEXMBUGA1UECAwOTm9y -dGggQ2Fyb2xpbmExEDAOBgNVBAcMB1JhbGVpZ2gxEDAOBgNVBAoMB1JlZCBIYXQx -ETAPBgNVBAsMCFN0YWNrUm94MRUwEwYDVQQDDAxTdXBlciBDbGllbnQxIDAeBgkq -hkiG9w0BCQEWEWNsaWVudEBjbGllbnQuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEAt8PUf6oEAF6cAdrHqjZ1PMY+MkkxJAx0rgUKMHd/ag1+AtLL -SzF7SImQuOz9Gpvs/5xNWlsVxd2ymecbGFoULLfp4pBtRdtlQKASqdoc1NBEfNlm -WG6uu91rA0RWddf2NTuzR7+C0kIBR62Gu3SYgnF8Hl/hkxvOTK3ZheYi40YGCsA4 -NKPd7EyApj+EnKjsXCLW8FKA4zYaepoBiMTeFl3tKAIFNiXeXWepp5aPCxuSFXcv -BLSd9sEwNvhwt2W8gj2JHBN+zDYEefmaR7OJBS7vM/R4bSyF7RU6b1iGDWIAIL+7 -YhtTWl1dEFnyMtCtuYqvbUMjkt8jU5MX46UKhwIDAQABo0IwQDAdBgNVHQ4EFgQU -SsPP4Z71b1dWlZrz6OhcSKOiVfUwHwYDVR0jBBgwFoAUDxEc/LnrAAg42M3Fx367 -bdr+m3EwDQYJKoZIhvcNAQELBQADggEBAFy+Pk/vwZImHwQg2rEW97m2if7X6ItV -VQ16PNoIKN308u1LnhNbAp9Ghyo06qThukQ0gqxN2R8gn28G9akG2IlOwy1vSpai -hHBjmV5NO8GCAAk32nnL93FrPb7cBjCNI2Kyj+mA0sbTeHsojMN5i+WUf4sbBsoi -No72JPmaflYfYRcXtgmNtvSbAi2EWOEXfVYAcSE6cqjqvFt5QwgguIZYDvV+cOhv -Ck+WFNCqhtzjN1ZTMMclaZA1Hzj4+C8jqTVDMKl5/Oy/pmYH2PgR0WzY9ayEY9L2 -F/kRnyokczoR1bD7rFsk7uBomkRHCnQb5OFJQueql5kIoRQMA5Cl0yk= ------END CERTIFICATE----- diff --git a/compliance/node/index/testdata/certs/client-key.pem b/compliance/node/index/testdata/certs/client-key.pem deleted file mode 100644 index eed44064f5b40..0000000000000 --- a/compliance/node/index/testdata/certs/client-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC3w9R/qgQAXpwB -2seqNnU8xj4ySTEkDHSuBQowd39qDX4C0stLMXtIiZC47P0am+z/nE1aWxXF3bKZ -5xsYWhQst+nikG1F22VAoBKp2hzU0ER82WZYbq673WsDRFZ11/Y1O7NHv4LSQgFH -rYa7dJiCcXweX+GTG85MrdmF5iLjRgYKwDg0o93sTICmP4ScqOxcItbwUoDjNhp6 -mgGIxN4WXe0oAgU2Jd5dZ6mnlo8LG5IVdy8EtJ32wTA2+HC3ZbyCPYkcE37MNgR5 -+ZpHs4kFLu8z9HhtLIXtFTpvWIYNYgAgv7tiG1NaXV0QWfIy0K25iq9tQyOS3yNT -kxfjpQqHAgMBAAECggEADi4FOqii0Gh8f0QjfgXfvAXMDNNLAG4ZnsfyxlSiN0u9 -OniLiUzTJ4C98xcOXLxFaF/mNB+tKWx2+Js1owZoYh765Cmi1O4d/QOdDRAwg1di -wYOA+zfnMcXfGQxv4337mJX3uoSTkqfXM3G3YMWZcw9QtYNOg6x0mcQccdGVRZYj -A93wD7Vwe61PFQchckADBnokRIcr6pV6jYPOuTlbIIAWl3l1/+4SK5IDwbXLSlAZ -U0hhRaWRh95H0JkgcdIx19Nv9d0gNOvkpS4hszNkscZZuRYSy2VH1XgQ42G+crt4 -vwbeghJ+Zo+PvpjxqhQTxSrWgDqWD4qOfqwSqntBUQKBgQDviPF3xi3nusMcTn69 -oC3XyLUgaGaaAl8FJiBMbXrc5uiPXKBBskTPVdthkLt4J42inFLAAQ88k3GRWKiq -gAWVBQBZd6aWprOsRXzo8ARFxQMmGzDBghH8Nje7tFv5h9JdcZ23ut3/L3GOEJMi -6xsSyxSOR6OVpBCE1uuU7pdx2QKBgQDEZYMGAVfU0740LvCxc8aYhmj3phIMjdln -sZpyyfwUBj2TRj2yO6L9bk2YCWmxsDP/MpbDP9g/WORq1Y6LZSgurTcfRp3n/gSr -edoMJKbqTDeOM0Am0mVY00UNKRwlbeybA8s4M+umbVcdGiq53ZD2Pilm5uWnCQyU -ZsGqnhZDXwKBgBAMhdk3jwXSrpXr8HP2Enw9t10HgbBZ7lHi/xD/2NaKJhTM6PPd -5NxbbUp5LwZa6BN4E9dHeuTdICPT4+EgmL8mSaK08JVxb+Jf+YGF61NdYtufwMGG -XHwgidVqvSrX3dFi9mVEAPHDFQGmvKT/SZ4VuzfZNu2kw3KTD8+3uK/pAoGBALmG -P0hr+mF1CMdY1Ex9Vsx8pCnVOlWbphTbZFAtWDA+WOqHvv5gca7ABRKrSanuDqBz -VJHq/MeSnGwDgt1RWK0HOfwUhFRD1Ht75yvzb64DA7QT2/UQaWaIZUFyZUFCXvFs -mwtjmKclwNYgAt6DXE4RGKE5WxX4aWbCzw3yuos3AoGAayieU8L4k78kyervdooX -oXunXVb3xRPi7Q4B2E3YQ0IYLtBt0Uaqfzcr/UQ9wMXzUrLCB1tJCpDtOm2wJmm9 -p+bMhDYpewtrRsi4cxR2D9iziW6P9EbCoyKhFmUzysCkoiENcmRdFp07kvp/4fDZ -l9+ZqDm23HWQ2EdzlzwHFZI= ------END PRIVATE KEY----- diff --git a/compliance/node/index/testdata/certs/server-cert.pem b/compliance/node/index/testdata/certs/server-cert.pem deleted file mode 100644 index 4a3acd7d66dc5..0000000000000 --- a/compliance/node/index/testdata/certs/server-cert.pem +++ /dev/null @@ -1,23 +0,0 @@ ------BEGIN CERTIFICATE----- -MIID5TCCAs2gAwIBAgIBATANBgkqhkiG9w0BAQsFADCBjjELMAkGA1UEBhMCVVMx -FzAVBgNVBAgMDk5vcnRoIENhcm9saW5hMRAwDgYDVQQHDAdSYWxlaWdoMRAwDgYD -VQQKDAdSZWQgSGF0MREwDwYDVQQLDAhTdGFja1JveDERMA8GA1UEAwwIU3VwZXIg -Q0ExHDAaBgkqhkiG9w0BCQEWDWNhQHNlcnZlci5jb20wIBcNMjQxMTAyMDAyMDIw -WhgPMzAyNDAzMDUwMDIwMjBaMIGWMQswCQYDVQQGEwJVUzEXMBUGA1UECAwOTm9y -dGggQ2Fyb2xpbmExEDAOBgNVBAcMB1JhbGVpZ2gxEDAOBgNVBAoMB1JlZCBIYXQx -ETAPBgNVBAsMCFN0YWNrUm94MRUwEwYDVQQDDAxTdXBlciBTZXJ2ZXIxIDAeBgkq -hkiG9w0BCQEWEXNlcnZlckBzZXJ2ZXIuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEAoxXRgREvyjFMSwEMzUY1KqktBvDBOWpBHEXE7NKMZTLdntXp -FV+L3VBf6N1oDdTjYrSF4A8fZXlse1j3MmpNR78ZdTyv8lMmmd0TWwwX8qnpjoy7 -SP9wigVb2GEVFcgFKlAz4VFEyJF0uWAR5GHR+bbP7JcOHmcy4JjuLZXXZ2qggy0n -yj6QnQElrwkyNvu892NcfKogJnh5olJ48TDYcId3xTuXYwHGZY0f0+1Qee3vUWl4 -s1oFXI/xFl9zpAow+meWf3jhUvxRPK/J3nTGWVfk4Janvg7Jr1b3wu4gv3+yvmlr -kznlr2+m9Rwqaouz0/q6dF30EtgFlbz7VwTKuQIDAQABo0IwQDAdBgNVHQ4EFgQU -jCcutwV21d2k3rLbEUnaSq4/QjwwHwYDVR0jBBgwFoAUDxEc/LnrAAg42M3Fx367 -bdr+m3EwDQYJKoZIhvcNAQELBQADggEBADD/p1MnzArmMLJH6mk/tZarcaVKq7SQ -Bua4eogayS14PLKvntqTybDgJiyCBn4sIO53sjNQE5XAD0ldJjAwThVtM5eREs0O -uhsDUjKF2AU8guTHcV+ZYlT4GSIL8UFulNqrptCTkbqjP/8i+54hHDfb5cDwLfYQ -klMP/hKN2r4DQPnZY1qdK7ftzifwRXUG0OngCTPW0Tz5gz/0xIJLD/WF/3EBniw3 -ZCGoV5JqVmKep9FsLEMwiAuIFNx9gnb8Qou1RGFLetVu3UNlNtEhJuH8ReBGVt6g -VJV3tFbGa2xo91AkP9ROmEjTBWxM9WuyY7eVB4ov/hf5i8/9NHi162E= ------END CERTIFICATE----- diff --git a/compliance/node/index/testdata/certs/server-key.pem b/compliance/node/index/testdata/certs/server-key.pem deleted file mode 100644 index 48b9ce69dc032..0000000000000 --- a/compliance/node/index/testdata/certs/server-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCjFdGBES/KMUxL -AQzNRjUqqS0G8ME5akEcRcTs0oxlMt2e1ekVX4vdUF/o3WgN1ONitIXgDx9leWx7 -WPcyak1Hvxl1PK/yUyaZ3RNbDBfyqemOjLtI/3CKBVvYYRUVyAUqUDPhUUTIkXS5 -YBHkYdH5ts/slw4eZzLgmO4tlddnaqCDLSfKPpCdASWvCTI2+7z3Y1x8qiAmeHmi -UnjxMNhwh3fFO5djAcZljR/T7VB57e9RaXizWgVcj/EWX3OkCjD6Z5Z/eOFS/FE8 -r8nedMZZV+Tglqe+DsmvVvfC7iC/f7K+aWuTOeWvb6b1HCpqi7PT+rp0XfQS2AWV -vPtXBMq5AgMBAAECggEAJ8C+w85Trt6bcUD9ROBTP2XjkuiGwR8UZfbfzLZ6AIUI -npoX6dzTdAwOhO3LWmvFTZ8hbv9gsNSLS/ngr8pXsiBz7S7w2InKRAEuoafuAnnG -c7f8Qwv4+7vYqoLnp9qM38aTDt5HfPhv0mwHkHsDuvID9vDPQTv4m+TLUhNPYWap -rKKYtkpfCNqe/fgzfSsFGbblg2fEAd5YdexYBameNT+aP6wi007dsdQbH8T2Zd8G -eIG3IqMyddH3JW9iCcTrYlPzAn2p3DPYFWm0FjYXP/3PVL/fOHgMbuY70deLozHZ -I/Iqr9m76Rc4ufV5dCJNyA+4rBacSppOy+lfwnjIvQKBgQDU/gDcl3kBGlBY/2Ic -08ITYwtixrymnieoTjOpVmAIPwZomidb5hh3rvL/NFpbmXsinsKEBX5F21jV7IoL -LaPJ/YBA36Qj4NS4u5KNh8ZyWJZ2QWBmv1MTShYfa0MARbgxZFQyt190c5OcPNJn -pD8uIwPKkREYwNYEvLEdmb5zywKBgQDEBAXcMddHcTqV1shgUJR41ebyeoCYEOhQ -KZfRIISyRto/JJETUnMv1oLYBBDQhMMs+9oAUtzmejSbU0QRw7zOjcPA447df4Y4 -D8GpHjCatEBH+gGtHEuOB38CvlXFErLPxAT+n5/bzZrrRp6K7wGjedivoWqwcy+i -350niYFTCwKBgQCtqh2Rv3/OQmJY7oJU9uQ7YnqaI2kWgcXcGUdOPzs1/g1/pJGL -AOXaUC+Znlo+bbpU1BXOB5yZ4OtiIee8jY4K2EOGodL33g4UQfyQobBQ86Ja8Ztk -q4NVKwc2vwJ2amJxA1kTsSiHjuVAcucr0mQhfjNNK1NBV5opaqEPfZgoWQKBgQCj -XSb1KaTeKgRFtUgy3XWgb8WW+XoLQqwbg07AQHXXjy19yqAOlnzhtco+9sR8xOlV -qGE8O9NXUypc+byVgajuDUDEleMPv6TBPICOzlZ72RZp5ndXK2nc6Wp1guRA5yCY -ACqZe7fjnSwsQiMIb8HR2U86sTEV9ioReISl2vgpmQKBgQC9uXqLBc108VVqxodp -Z9XEnI5gvjr5x1Neutty0rahVS++nd4cd7Z7zWJQsHE76D5lwDDVm88YdHrvt6EV -qZOWIZG6cHU0XpFXFQ2OiBQag+KdEZjPVfEHEhsavsZfYh27O4mPhOAqTfr9KFS4 -uaoiYVoBgMHdX37aw70CBXs8kw== ------END PRIVATE KEY----- diff --git a/compliance/node/interfaces.go b/compliance/node/interfaces.go index 9633eadad2867..010abee20dede 100644 --- a/compliance/node/interfaces.go +++ b/compliance/node/interfaces.go @@ -28,11 +28,3 @@ type NodeIndexer interface { IndexNode(ctx context.Context) (*v4.IndexReport, error) GetIntervals() *utils.NodeScanIntervals } - -// UnconfirmedMessageHandler handles the observation of sending, and ACK/NACK messages -type UnconfirmedMessageHandler interface { - HandleACK() - HandleNACK() - ObserveSending() - RetryCommand() <-chan struct{} -} diff --git a/compliance/virtualmachines/relay/connutil/connutil.go b/compliance/virtualmachines/relay/connutil/connutil.go deleted file mode 100644 index b6cb72025b4cf..0000000000000 --- a/compliance/virtualmachines/relay/connutil/connutil.go +++ /dev/null @@ -1,40 +0,0 @@ -// Package connutil contains helpers for bounded reads on generic net.Conn -// implementations. -package connutil - -import ( - "io" - "net" - "time" - - "github.com/pkg/errors" - "github.com/stackrox/rox/pkg/logging" -) - -var log = logging.LoggerForModule() - -// ReadFromConn reads from conn with a deadline and rejects payloads above -// maxSize bytes. -func ReadFromConn(conn net.Conn, maxSize int, timeout time.Duration) ([]byte, error) { - log.Debugf("Reading from connection (max bytes: %d, timeout: %s)", maxSize, timeout) - - if err := conn.SetReadDeadline(time.Now().Add(timeout)); err != nil { - return nil, errors.Wrap(err, "setting read deadline on connection") - } - - // Even if not strictly required, we limit the amount of data to be read to protect Sensor against large workloads. - // Add 1 to the limit so we can detect oversized data. If we used exactly maxSize, we couldn't tell the difference - // between a valid message of exactly maxSize bytes and an invalid message that's larger than maxSize (both would - // read maxSize bytes). With maxSize+1, reading more than maxSize bytes means the original data was too large. - limitedReader := io.LimitReader(conn, int64(maxSize+1)) - data, err := io.ReadAll(limitedReader) - if err != nil { - return nil, errors.Wrapf(err, "reading data from connection (remote: %v)", conn.RemoteAddr()) - } - - if len(data) > maxSize { - return nil, errors.Errorf("data size exceeds the limit (%d bytes, remote: %v)", maxSize, conn.RemoteAddr()) - } - - return data, nil -} diff --git a/compliance/virtualmachines/relay/connutil/connutil_test.go b/compliance/virtualmachines/relay/connutil/connutil_test.go deleted file mode 100644 index 7da1738a2e6be..0000000000000 --- a/compliance/virtualmachines/relay/connutil/connutil_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package connutil - -import ( - "testing" - "time" - - relaytest "github.com/stackrox/rox/compliance/virtualmachines/relay/testutils" - "github.com/stretchr/testify/suite" -) - -func TestConnutil(t *testing.T) { - suite.Run(t, new(connutilTestSuite)) -} - -type connutilTestSuite struct { - suite.Suite -} - -func (s *connutilTestSuite) TestReadFromConn() { - data := []byte("Hello, world!") - - cases := map[string]struct { - delay time.Duration - maxSize int - readTimeout time.Duration - shouldError bool - }{ - "data smaller than limit succeeds": { - maxSize: 2 * len(data), - readTimeout: 10 * time.Second, - shouldError: false, - }, - "data of equal size as limit succeeds": { - maxSize: len(data), - readTimeout: 10 * time.Second, - shouldError: false, - }, - "data larger than limit fails": { - maxSize: len(data) - 1, - readTimeout: 10 * time.Second, - shouldError: true, - }, - "delay longer than timeout fails": { - maxSize: len(data), - delay: 1 * time.Second, - readTimeout: 100 * time.Millisecond, - shouldError: true, - }, - "delay shorter than timeout succeeds": { - maxSize: len(data), - delay: 100 * time.Millisecond, - readTimeout: 1 * time.Second, - shouldError: false, - }, - } - - for name, c := range cases { - s.Run(name, func() { - conn := relaytest.NewMockVsockConn(s.T()).WithData(data).WithDelay(c.delay) - - readData, err := ReadFromConn(conn, c.maxSize, c.readTimeout) - if c.shouldError { - s.Error(err) - } else { - s.Require().NoError(err) - s.Equal(data, readData) - } - }) - } -} diff --git a/compliance/virtualmachines/relay/metrics/metrics.go b/compliance/virtualmachines/relay/metrics/metrics.go deleted file mode 100644 index 224c179ca1b09..0000000000000 --- a/compliance/virtualmachines/relay/metrics/metrics.go +++ /dev/null @@ -1,90 +0,0 @@ -package metrics - -import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/stackrox/rox/pkg/metrics" -) - -// IndexReportsMismatchingVsockCID is a counter for the number of virtual machine index reports whose reported vsock CID does not -// match the connection's vsock CID. -var IndexReportsMismatchingVsockCID = prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.ComplianceSubsystem.String(), - Name: "virtual_machine_relay_index_reports_mismatching_vsock_cid_total", - Help: "Total number of virtual machine index reports received by this Relay with mismatching vsock CID", - }, -) - -// IndexReportsReceived is a counter for the number of virtual machine index reports received. -var IndexReportsReceived = prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.ComplianceSubsystem.String(), - Name: "virtual_machine_relay_index_reports_received_total", - Help: "Total number of virtual machine index reports received by this Relay", - }, -) - -// IndexReportsSentToSensor is a counter for the number of virtual machine index reports sent to sensor. -var IndexReportsSentToSensor = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.ComplianceSubsystem.String(), - Name: "virtual_machine_relay_index_reports_sent_total", - Help: "Total number of virtual machine index reports sent to sensor by this Relay", - }, - []string{"failed"}, -) - -// ConnectionsAccepted is a counter for the number of connections accepted by this relay. A mismatch between -// this and IndexReportsReceived indicates issues reading or parsing data. -var ConnectionsAccepted = prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.ComplianceSubsystem.String(), - Name: "virtual_machine_relay_connections_accepted_total", - Help: "Total number of connections accepted by this Relay", - }, -) - -// SemaphoreAcquisitionFailures is a counter for the number of times the connection-handling semaphore that limits -// concurrency could not be acquired. A likely and significant reason for that is that the maximum parallel connections -// were reached. -var SemaphoreAcquisitionFailures = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.ComplianceSubsystem.String(), - Name: "virtual_machine_relay_sem_acquisition_failures_total", - Help: "Number of failed attempts to acquire connection-handling semaphore", - }, - []string{"reason"}, -) - -var SemaphoreHoldingSize = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.ComplianceSubsystem.String(), - Name: "virtual_machine_relay_sem_holding_size", - Help: "Number of connections being handled", - }) - -var SemaphoreQueueSize = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.ComplianceSubsystem.String(), - Name: "virtual_machine_relay_sem_queue_size", - Help: "Number of connections waiting to be handled", - }) - -func init() { - prometheus.MustRegister( - IndexReportsMismatchingVsockCID, - IndexReportsReceived, - IndexReportsSentToSensor, - ConnectionsAccepted, - SemaphoreAcquisitionFailures, - SemaphoreHoldingSize, - SemaphoreQueueSize, - ) -} diff --git a/compliance/virtualmachines/relay/relay.go b/compliance/virtualmachines/relay/relay.go deleted file mode 100644 index 4d9a0d0912abf..0000000000000 --- a/compliance/virtualmachines/relay/relay.go +++ /dev/null @@ -1,57 +0,0 @@ -package relay - -import ( - "context" - - "github.com/pkg/errors" - "github.com/stackrox/rox/compliance/virtualmachines/relay/sender" - v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" - "github.com/stackrox/rox/pkg/logging" -) - -var log = logging.LoggerForModule() - -// IndexReportStream manages report collection and produces validated reports. -type IndexReportStream interface { - // Start begins accepting connections and returns a channel of validated reports. - // The channel is currently not closed to avoid races during shutdown. - // TODO: Implement proper shutdown logic that closes the channel. - Start(ctx context.Context) (<-chan *v1.VMReport, error) -} - -type Relay struct { - reportStream IndexReportStream - reportSender sender.IndexReportSender -} - -// New creates a Relay with the given report stream and sender. -func New(reportStream IndexReportStream, reportSender sender.IndexReportSender) *Relay { - return &Relay{ - reportStream: reportStream, - reportSender: reportSender, - } -} - -func (r *Relay) Run(ctx context.Context) error { - log.Info("Starting virtual machine relay") - - reportChan, err := r.reportStream.Start(ctx) - if err != nil { - return errors.Wrap(err, "starting report stream") - } - - for { - select { - case <-ctx.Done(): - return ctx.Err() - case vmReport := <-reportChan: - if vmReport == nil { - log.Warn("Received nil VM report, skipping") - continue - } - if err := r.reportSender.Send(ctx, vmReport); err != nil { - log.Errorf("Failed to send VM report (vsock CID: %s): %v", vmReport.GetIndexReport().GetVsockCid(), err) - } - } - } -} diff --git a/compliance/virtualmachines/relay/relay_test.go b/compliance/virtualmachines/relay/relay_test.go deleted file mode 100644 index d0256371d8a94..0000000000000 --- a/compliance/virtualmachines/relay/relay_test.go +++ /dev/null @@ -1,312 +0,0 @@ -package relay - -import ( - "context" - "errors" - "testing" - "time" - - relaytest "github.com/stackrox/rox/compliance/virtualmachines/relay/testutils" - v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" - "github.com/stackrox/rox/pkg/concurrency" - "github.com/stackrox/rox/pkg/sync" - "github.com/stretchr/testify/suite" -) - -func TestRelay(t *testing.T) { - suite.Run(t, new(relayTestSuite)) -} - -type relayTestSuite struct { - suite.Suite - ctx context.Context -} - -func (s *relayTestSuite) SetupTest() { - s.ctx = context.Background() -} - -// errTestStreamStart is returned by the mock stream's Start method to simulate -// a startup failure. -var errTestStreamStart = errors.New("test stream start failure") - -// errTest is a generic test error for use in mock implementations. -var errTest = errors.New("test error") - -// TestRelay_StartFailure verifies that Relay.Run propagates stream startup -// errors and does not enter the main select loop when initialization fails. -func (s *relayTestSuite) TestRelay_StartFailure() { - // Use a bounded context to ensure the test fails if Relay.Run blocks. - ctx, cancel := context.WithTimeout(s.ctx, 100*time.Millisecond) - defer cancel() - - // Create a stream that fails immediately on Start. - stream := &failingIndexReportStream{} - - // Create a dummy sender. It should never be used in this test because the - // relay is expected to fail before entering its main loop. - sender := &mockIndexReportSender{ - failOnIndex: -1, - expectedCount: 0, - } - - // Construct the relay under test. - relay := New(stream, sender) - - // Run the relay in a goroutine so we can assert it returns promptly and - // does not block in its select loop. - errCh := make(chan error, 1) - go func() { - errCh <- relay.Run(ctx) - }() - - select { - case err := <-errCh: - // Relay.Run should surface the stream startup error (possibly wrapped). - s.Require().Error(err, "Relay.Run should return an error when stream Start fails") - s.Require().ErrorIs(err, errTestStreamStart, "Relay.Run should wrap the stream startup error") - s.Equal(1, stream.startCalled, "stream.Start should be called exactly once") - case <-time.After(100 * time.Millisecond): - s.Fail("Relay.Run did not return promptly on stream Start failure (likely entered select loop)") - } -} - -// TestRelay_Integration tests the interaction between stream, relay, and sender. -// This uses mock implementations to verify the full data flow without real vsock/sensor. -func (s *relayTestSuite) TestRelay_Integration() { - // Create mock sender that signals when reports are received - done := concurrency.NewSignal() - mockIndexReportSender := &mockIndexReportSender{ - failOnIndex: -1, // never fail - done: &done, - expectedCount: 2, - } - - // Create mock stream that produces test messages with discovered data - mockIndexReportStream := &mockIndexReportStream{ - reports: []*v1.VMReport{ - { - IndexReport: &v1.IndexReport{VsockCid: "100"}, - DiscoveredData: &v1.DiscoveredData{ - DetectedOs: v1.DetectedOS_RHEL, - OsVersion: "9.3", - ActivationStatus: v1.ActivationStatus_ACTIVE, - DnfMetadataStatus: v1.DnfMetadataStatus_AVAILABLE, - }, - }, - { - IndexReport: &v1.IndexReport{VsockCid: "200"}, - DiscoveredData: &v1.DiscoveredData{ - DetectedOs: v1.DetectedOS_UNKNOWN, - OsVersion: "unknown", - ActivationStatus: v1.ActivationStatus_INACTIVE, - DnfMetadataStatus: v1.DnfMetadataStatus_UNAVAILABLE, - }, - }, - }, - } - - // Create relay with mock dependencies using the public constructor - relay := New(mockIndexReportStream, mockIndexReportSender) - - // Run relay in background - ctx, cancel := context.WithTimeout(s.ctx, 5*time.Second) - defer cancel() - - errChan := make(chan error, 1) - go func() { - errChan <- relay.Run(ctx) - }() - - // Wait for all reports to be processed (or timeout) - select { - case <-done.Done(): - // All reports processed - case <-time.After(1 * time.Second): - s.Fail("Timeout waiting for reports to be processed") - } - - cancel() - - // Verify all messages were sent with discovered data preserved - mockIndexReportSender.mu.Lock() - s.Require().Len(mockIndexReportSender.sentMessages, 2) - - first := mockIndexReportSender.sentMessages[0] - second := mockIndexReportSender.sentMessages[1] - - // IndexReport fields are preserved - s.Equal("100", first.GetIndexReport().GetVsockCid()) - s.Equal("200", second.GetIndexReport().GetVsockCid()) - - // VM discovered data is preserved - s.Equal(v1.DetectedOS_RHEL, first.GetDiscoveredData().GetDetectedOs()) - s.Equal("9.3", first.GetDiscoveredData().GetOsVersion()) - s.Equal(v1.ActivationStatus_ACTIVE, first.GetDiscoveredData().GetActivationStatus()) - s.Equal(v1.DnfMetadataStatus_AVAILABLE, first.GetDiscoveredData().GetDnfMetadataStatus()) - - s.Equal(v1.DetectedOS_UNKNOWN, second.GetDiscoveredData().GetDetectedOs()) - s.Equal("unknown", second.GetDiscoveredData().GetOsVersion()) - s.Equal(v1.ActivationStatus_INACTIVE, second.GetDiscoveredData().GetActivationStatus()) - s.Equal(v1.DnfMetadataStatus_UNAVAILABLE, second.GetDiscoveredData().GetDnfMetadataStatus()) - - mockIndexReportSender.mu.Unlock() - - // Verify relay exited cleanly - err := <-errChan - s.ErrorIs(err, context.Canceled) -} - -// TestRelay_SenderErrorsDoNotStopProcessing verifies that sender errors don't halt the relay -func (s *relayTestSuite) TestRelay_SenderErrorsDoNotStopProcessing() { - // Sender fails on second report but signals completion - done := concurrency.NewSignal() - mockIndexReportSender := &mockIndexReportSender{ - failOnIndex: 1, // fail on second report - done: &done, - expectedCount: 3, - } - - mockIndexReportStream := &mockIndexReportStream{ - reports: []*v1.VMReport{ - relaytest.NewTestVMReport("100"), - relaytest.NewTestVMReport("200"), - relaytest.NewTestVMReport("300"), - }, - } - - relay := New(mockIndexReportStream, mockIndexReportSender) - - ctx, cancel := context.WithTimeout(s.ctx, 5*time.Second) - defer cancel() - - errChan := make(chan error, 1) - go func() { - errChan <- relay.Run(ctx) - }() - - // Wait for all reports to be attempted - select { - case <-done.Done(): - // All reports attempted - case <-time.After(1 * time.Second): - s.Fail("Timeout waiting for reports to be processed") - } - - cancel() - - // All three messages should have been attempted - mockIndexReportSender.mu.Lock() - s.Require().Len(mockIndexReportSender.sentMessages, 3) - mockIndexReportSender.mu.Unlock() - - err := <-errChan - s.ErrorIs(err, context.Canceled) -} - -// TestRelay_ContextCancellation verifies relay stops on context cancellation -func (s *relayTestSuite) TestRelay_ContextCancellation() { - // The mocked stream signals when first message is sent - started := concurrency.NewSignal() - mockIndexReportStream := &mockIndexReportStream{ - reports: []*v1.VMReport{ - relaytest.NewTestVMReport("100"), - relaytest.NewTestVMReport("200"), // Second message will never be processed - }, - started: &started, - } - - mockIndexReportSender := &mockIndexReportSender{ - failOnIndex: -1, // never fail - } - - relay := New(mockIndexReportStream, mockIndexReportSender) - - ctx, cancel := context.WithCancel(s.ctx) - - errChan := make(chan error, 1) - go func() { - errChan <- relay.Run(ctx) - }() - - // Wait for stream to start sending reports - <-started.Done() - - // Cancel immediately - cancel() - - // Should exit quickly - select { - case err := <-errChan: - s.ErrorIs(err, context.Canceled) - case <-time.After(100 * time.Millisecond): - s.Fail("Relay did not exit after context cancellation") - } -} - -// Mock implementations - -// failingIndexReportStream is a mock IndexReportStream whose Start method -// always fails. It tracks how many times Start is called so tests can assert -// correct behavior. -type failingIndexReportStream struct { - startCalled int -} - -// Start implements IndexReportStream.Start. It always returns a nil channel -// and errTestStreamStart to simulate a stream startup failure. -func (f *failingIndexReportStream) Start(ctx context.Context) (<-chan *v1.VMReport, error) { - f.startCalled++ - return nil, errTestStreamStart -} - -type mockIndexReportStream struct { - reports []*v1.VMReport - started *concurrency.Signal // signals when first report is streamed -} - -func (m *mockIndexReportStream) Start(ctx context.Context) (<-chan *v1.VMReport, error) { - reportChan := make(chan *v1.VMReport, len(m.reports)) - - go func() { - for i, report := range m.reports { - select { - case <-ctx.Done(): - return - case reportChan <- report: - // Signal when first report is streamed - if i == 0 && m.started != nil { - m.started.Signal() - } - } - } - }() - - return reportChan, nil -} - -type mockIndexReportSender struct { - mu sync.Mutex - sentMessages []*v1.VMReport - failOnIndex int // Index to fail on (0-based), use -1 to never fail - done *concurrency.Signal // signals when expectedCount reports are sent - expectedCount int // number of reports expected before signaling done -} - -func (m *mockIndexReportSender) Send(_ context.Context, vmReport *v1.VMReport) error { - m.mu.Lock() - currentIndex := len(m.sentMessages) - m.sentMessages = append(m.sentMessages, vmReport) - - // Signal done when we've sent expected count - if m.done != nil && len(m.sentMessages) == m.expectedCount { - m.done.Signal() - } - m.mu.Unlock() - - // Fail on the specified index - if currentIndex == m.failOnIndex { - return errTest - } - return nil -} diff --git a/compliance/virtualmachines/relay/sender/index_report_sender.go b/compliance/virtualmachines/relay/sender/index_report_sender.go deleted file mode 100644 index cd9270b915fc6..0000000000000 --- a/compliance/virtualmachines/relay/sender/index_report_sender.go +++ /dev/null @@ -1,113 +0,0 @@ -// Package sender handles sending index reports to sensor and retrying on errors -package sender - -import ( - "context" - "strconv" - "time" - - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/stackrox/rox/compliance/virtualmachines/relay/metrics" - "github.com/stackrox/rox/generated/internalapi/sensor" - v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" - "github.com/stackrox/rox/pkg/logging" - "github.com/stackrox/rox/pkg/retry" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -var log = logging.LoggerForModule() - -// IndexReportSender sends index reports to Sensor. -type IndexReportSender interface { - Send(ctx context.Context, vmReport *v1.VMReport) error -} - -type sensorIndexReportSender struct { - sensorClient sensor.VirtualMachineIndexReportServiceClient -} - -var _ IndexReportSender = (*sensorIndexReportSender)(nil) - -// New creates an IndexReportSender that sends reports to Sensor with retry logic. -func New(sensorClient sensor.VirtualMachineIndexReportServiceClient) IndexReportSender { - return &sensorIndexReportSender{ - sensorClient: sensorClient, - } -} - -// Send sends the VM report to Sensor, retrying on transient errors. -func (s *sensorIndexReportSender) Send(ctx context.Context, vmReport *v1.VMReport) error { - indexReport := vmReport.GetIndexReport() - if indexReport == nil { - return errors.New("VM report missing required index_report field") - } - - log.Infof("Sending VM report to sensor (vsockCID: %s)", indexReport.GetVsockCid()) - - // This is the sending logic that will be retried if needed - sendFunc := func() error { - sendToSensorCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - req := &sensor.UpsertVirtualMachineIndexReportRequest{ - IndexReport: indexReport, - DiscoveredData: vmReport.GetDiscoveredData(), - } - - resp, err := s.sensorClient.UpsertVirtualMachineIndexReport(sendToSensorCtx, req) - - if resp != nil && !resp.GetSuccess() { - // This can't happen as of this writing (Success is only false when an error is returned) but is - // theoretically possible, let's add retries too. - if err == nil { - log.Errorf("Sending index report didn't return an error but response indicated failure: %v", resp) - err = retry.MakeRetryable(errors.New("sensor failed to handle virtual machine index report")) - } - } - - if isRetryableGRPCError(err) { - err = retry.MakeRetryable(err) - } - - return err - } - - onFailedAttemptsFunc := func(e error) { - log.Warnf("Error sending index report to sensor, retrying. Error was: %v", e) - } - - tries := 10 // With default backoff logic in pkg/retry, this takes around 50 s (without considering timeouts) - - // Considering a timeout of 5 seconds and 10 tries with exponential backoff, the maximum time until running out of - // tries is around 1 min 40 s. Given that each virtual machine sends an index report every 4 hours, these retries - // seem reasonable and are unlikely to cause issues. - err := retry.WithRetry( - sendFunc, - retry.WithContext(ctx), - retry.OnFailedAttempts(onFailedAttemptsFunc), - retry.Tries(tries), - retry.OnlyRetryableErrors(), - retry.WithExponentialBackoff()) - - metrics.IndexReportsSentToSensor.With(prometheus.Labels{"failed": strconv.FormatBool(err != nil)}).Inc() - - return err -} - -func isRetryableGRPCError(err error) bool { - grpcErr, ok := status.FromError(err) - if !ok { - return false - } - code := grpcErr.Code() - switch code { - case codes.DeadlineExceeded: - return !errors.Is(err, context.Canceled) - case codes.Unavailable, codes.ResourceExhausted, codes.Internal: - return true - default: - return false - } -} diff --git a/compliance/virtualmachines/relay/sender/index_report_sender_test.go b/compliance/virtualmachines/relay/sender/index_report_sender_test.go deleted file mode 100644 index faaf5df8b04b3..0000000000000 --- a/compliance/virtualmachines/relay/sender/index_report_sender_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package sender - -import ( - "context" - "testing" - "time" - - relaytest "github.com/stackrox/rox/compliance/virtualmachines/relay/testutils" - v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" - "github.com/stackrox/rox/pkg/errox" - "github.com/stretchr/testify/suite" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestSender(t *testing.T) { - suite.Run(t, new(senderTestSuite)) -} - -type senderTestSuite struct { - suite.Suite - - ctx context.Context -} - -func (s *senderTestSuite) SetupTest() { - s.ctx = context.Background() -} - -func (s *senderTestSuite) TestSend_HandlesContextCancellation() { - client := relaytest.NewMockSensorClient(s.T()) - sender := New(client) - ctx, cancel := context.WithCancel(s.ctx) - cancel() - - err := sender.Send(ctx, relaytest.NewTestVMReport("42")) - s.Require().Error(err) - s.Contains(err.Error(), "context canceled") -} - -func (s *senderTestSuite) TestSend_RetriesOnRetryableErrors() { - cases := map[string]struct { - err error - respSuccess bool - shouldRetry bool - }{ - "retryable error is retried": { - err: status.Error(codes.ResourceExhausted, "retryable error"), - respSuccess: false, - shouldRetry: true, - }, - "non-retryable error is not retried": { - err: errox.NotImplemented, - respSuccess: false, - shouldRetry: false, - }, - "Unsuccessful request is retried": { - err: nil, - respSuccess: false, - shouldRetry: true, - }, - } - for name, c := range cases { - s.Run(name, func() { - client := relaytest.NewMockSensorClient(s.T()).WithError(c.err) - if !c.respSuccess { - client = client.WithUnsuccessfulResponse() - } - sender := New(client) - - // The retry logic uses withExponentialBackoff, which currently has an initial delay between retries of - // 100 ms, therefore after 500 ms the failing call has been retried already - ctx, cancel := context.WithTimeout(s.ctx, 500*time.Millisecond) - defer cancel() - - err := sender.Send(ctx, relaytest.NewTestVMReport("42")) - s.Require().Error(err) - - retried := len(client.CapturedRequests()) > 1 - s.Equal(c.shouldRetry, retried) - }) - } -} - -func (s *senderTestSuite) TestReportSender_Send() { - client := relaytest.NewMockSensorClient(s.T()) - sender := New(client) - - msg := &v1.VMReport{ - IndexReport: &v1.IndexReport{VsockCid: "42"}, - DiscoveredData: &v1.DiscoveredData{ - DetectedOs: v1.DetectedOS_RHEL, - OsVersion: "9.2", - ActivationStatus: v1.ActivationStatus_ACTIVE, - DnfMetadataStatus: v1.DnfMetadataStatus_AVAILABLE, - }, - } - - err := sender.Send(s.ctx, msg) - s.Require().NoError(err) - s.Require().Len(client.CapturedRequests(), 1) - - // Verify that discovered data fields are correctly forwarded into the request - req := client.CapturedRequests()[0] - s.Equal(msg.GetDiscoveredData().GetDetectedOs(), req.GetDiscoveredData().GetDetectedOs()) - s.Equal(msg.GetDiscoveredData().GetActivationStatus(), req.GetDiscoveredData().GetActivationStatus()) - s.Equal(msg.GetDiscoveredData().GetDnfMetadataStatus(), req.GetDiscoveredData().GetDnfMetadataStatus()) -} - -func (s *senderTestSuite) TestReportSender_SendHandlesErrors() { - client := relaytest.NewMockSensorClient(s.T()).WithError(errox.NotImplemented) - sender := New(client) - - err := sender.Send(s.ctx, relaytest.NewTestVMReport("42")) - s.Require().Error(err) -} - -func (s *senderTestSuite) TestReportSender_SendMissingIndexReport() { - client := relaytest.NewMockSensorClient(s.T()) - sender := New(client) - - msg := &v1.VMReport{ - DiscoveredData: &v1.DiscoveredData{ - DetectedOs: v1.DetectedOS_RHEL, - OsVersion: "9.2", - ActivationStatus: v1.ActivationStatus_ACTIVE, - DnfMetadataStatus: v1.DnfMetadataStatus_AVAILABLE, - }, - } - - err := sender.Send(s.ctx, msg) - s.Require().Error(err) - s.Require().Empty(client.CapturedRequests()) -} diff --git a/compliance/virtualmachines/relay/stream/vsock_index_report_stream.go b/compliance/virtualmachines/relay/stream/vsock_index_report_stream.go deleted file mode 100644 index 22011519e97ac..0000000000000 --- a/compliance/virtualmachines/relay/stream/vsock_index_report_stream.go +++ /dev/null @@ -1,259 +0,0 @@ -// Package stream manages the vsock server and produces validated index reports. -// It combines connection management, parsing, and validation before forwarding -// reports to the relay for sending to Sensor. -package stream - -import ( - "context" - "net" - "strconv" - "time" - - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/stackrox/rox/compliance/virtualmachines/relay/connutil" - "github.com/stackrox/rox/compliance/virtualmachines/relay/metrics" - "github.com/stackrox/rox/compliance/virtualmachines/relay/vsock" - v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" - "github.com/stackrox/rox/pkg/env" - "github.com/stackrox/rox/pkg/logging" - "github.com/stackrox/rox/pkg/sync" - "golang.org/x/sync/semaphore" - "google.golang.org/protobuf/proto" -) - -var log = logging.LoggerForModule() - -type VsockIndexReportStream struct { - listener net.Listener - listenerMu sync.Mutex - semaphore *semaphore.Weighted - maxConcurrentConnections int - semaphoreTimeout time.Duration - connectionReadTimeout time.Duration - waitAfterFailedAccept time.Duration - maxSizeBytes int -} - -// New creates a VsockIndexReportStream with a vsock listener. -// Concurrency limits are read from env vars VirtualMachinesMaxConcurrentVsockConnections -// and VirtualMachinesConcurrencyTimeout. -func New() (*VsockIndexReportStream, error) { - listener, err := vsock.NewListener() - if err != nil { - return nil, errors.Wrap(err, "creating vsock listener") - } - - maxConcurrentConnections := env.VirtualMachinesMaxConcurrentVsockConnections.IntegerSetting() - semaphoreTimeout := env.VirtualMachinesConcurrencyTimeout.DurationSetting() - maxSizeBytes := env.VirtualMachinesVsockConnMaxSizeKB.IntegerSetting() * 1024 - - return &VsockIndexReportStream{ - listener: listener, - semaphore: semaphore.NewWeighted(int64(maxConcurrentConnections)), - maxConcurrentConnections: maxConcurrentConnections, - semaphoreTimeout: semaphoreTimeout, - connectionReadTimeout: 10 * time.Second, - waitAfterFailedAccept: time.Second, - maxSizeBytes: maxSizeBytes, - }, nil -} - -// Start begins accepting vsock connections and returns a channel of validated reports. -// The stream spawns goroutines to handle each connection concurrently (up to the -// configured limit). Reports are validated before being sent to the channel. -func (p *VsockIndexReportStream) Start(ctx context.Context) (<-chan *v1.VMReport, error) { - log.Info("Starting report stream") - - if p.listener == nil { - return nil, errors.New("listener is nil") - } - - // Buffer size = concurrency limit to allow stream goroutines to complete - // without blocking on sender. Use the already-derived maxConcurrentConnections - // to keep this as the single source of truth. - reportChan := make(chan *v1.VMReport, p.maxConcurrentConnections) - - // Single place that shuts down the listener when the context is done. - go func() { - <-ctx.Done() - p.stop() - }() - - // Start the accept loop in a goroutine - go p.acceptLoop(ctx, reportChan) - - return reportChan, nil -} - -func (p *VsockIndexReportStream) acceptLoop(ctx context.Context, reportChan chan<- *v1.VMReport) { - for { - // Accept() is blocking, but it will return when ctx is cancelled and the goroutine in Start() calls p.stop() - conn, err := p.listener.Accept() - if err != nil { - if ctx.Err() != nil { - log.Info("Stopping report stream") - return - } - - // We deliberately don't kill the listener on errors. The only way to stop that is to cancel the context. - // If we had return here on fatal errors, then compliance would continue working without the relay - // and that would make it an invisible problem to the user. - log.Errorf("Error accepting connection: %v", err) - - select { - case <-time.After(p.waitAfterFailedAccept): - case <-ctx.Done(): - return - } - continue - } - metrics.ConnectionsAccepted.Inc() - - if err := p.acquireSemaphore(ctx); err != nil { - if ctx.Err() != nil { - log.Info("Stopping report stream") - return - } - - // This log is rate-limited because when the concurrency limit is reached it is emitted every - // semaphoreTimeout, which is user-configurable (min: 1 second). - logging.GetRateLimitedLogger().WarnL( - "relay semaphore timeout", - "Failed to acquire semaphore to handle connection: %v", - err, - ) - - // When the concurrency limit is reached, the semaphore cannot be acquired. We close the connection and - // continue to listen. In this case, there is no need to add an extra wait to prevent a busy loop, because - // we already waited semaphoreTimeout - if err := conn.Close(); err != nil { - log.Warnf("Failed to close connection after failing to acquire semaphore: %v", err) - } - - continue - } - - go p.handleConnection(ctx, conn, reportChan) - } -} - -func (p *VsockIndexReportStream) handleConnection(ctx context.Context, conn net.Conn, reportChan chan<- *v1.VMReport) { - defer p.releaseSemaphore() - - defer func(conn net.Conn) { - if err := conn.Close(); err != nil { - log.Errorf("Failed to close connection: %v", err) - } - }(conn) - - log.Infof("Handling connection from %s", conn.RemoteAddr()) - - vmReport, err := p.receiveAndValidateVMReport(conn) - if err != nil { - log.Errorf("Error handling connection from %v: %v", conn.RemoteAddr(), err) - return - } - - log.Infof("Finished handling connection from %s", conn.RemoteAddr()) - - // Send validated message to channel. Use select to avoid blocking during shutdown - // when the relay stops reading from the channel. - select { - case reportChan <- vmReport: - // Report sent successfully - case <-ctx.Done(): - // Context cancelled during send - exit without blocking to allow defers to execute - log.Debug("Context cancelled while sending report, skipping send") - return - } -} - -func (p *VsockIndexReportStream) receiveAndValidateVMReport(conn net.Conn) (*v1.VMReport, error) { - vsockCID, err := vsock.ExtractVsockCIDFromConnection(conn) - if err != nil { - return nil, errors.Wrap(err, "extracting vsock CID") - } - - data, err := connutil.ReadFromConn(conn, p.maxSizeBytes, p.connectionReadTimeout) - if err != nil { - return nil, errors.Wrapf(err, "reading from connection (vsock CID: %d)", vsockCID) - } - - log.Debugf("Parsing VM report (vsock CID: %d)", vsockCID) - vmReport, err := parseVMReport(data) - if err != nil { - return nil, errors.Wrapf(err, "parsing VM report data (vsock CID: %d)", vsockCID) - } - - err = validateReportedVsockCID(vmReport, vsockCID) - if err != nil { - log.Debugf("Error validating reported vsock CID: %v", err) - return nil, errors.Wrap(err, "validating reported vsock CID") - } - - metrics.IndexReportsReceived.Inc() - - return vmReport, nil -} - -func parseVMReport(data []byte) (*v1.VMReport, error) { - msg := &v1.VMReport{} - - if err := proto.Unmarshal(data, msg); err != nil { - return nil, errors.Wrap(err, "unmarshalling data") - } - return msg, nil -} - -// validateReportedVsockCID ensures the message's vsock CID matches the connection. -func validateReportedVsockCID(vmReport *v1.VMReport, connVsockCID uint32) error { - reportedCID := vmReport.GetIndexReport().GetVsockCid() - if reportedCID != strconv.FormatUint(uint64(connVsockCID), 10) { - metrics.IndexReportsMismatchingVsockCID.Inc() - return errors.Errorf("mismatch between reported (%s) and real (%d) vsock CIDs", reportedCID, connVsockCID) - } - return nil -} - -func (p *VsockIndexReportStream) stop() { - p.listenerMu.Lock() - defer p.listenerMu.Unlock() - - if p.listener == nil { - return - } - - log.Info("Stopping index report stream") - if err := p.listener.Close(); err != nil { - log.Errorf("Error closing listener: %v", err) - } - p.listener = nil -} - -func (p *VsockIndexReportStream) acquireSemaphore(parentCtx context.Context) error { - semCtx, cancel := context.WithTimeout(parentCtx, p.semaphoreTimeout) - defer cancel() - - metrics.SemaphoreQueueSize.Inc() - defer metrics.SemaphoreQueueSize.Dec() - if err := p.semaphore.Acquire(semCtx, 1); err != nil { - reason := "unknown" - if errors.Is(err, context.DeadlineExceeded) { - log.Debug("Could not acquire semaphore, too many concurrent connections") - reason = "concurrency_limit" - } else if errors.Is(err, context.Canceled) { - log.Debug("Could not acquire semaphore, the context was canceled") - reason = "context_canceled" - } - metrics.SemaphoreAcquisitionFailures.With(prometheus.Labels{"reason": reason}).Inc() - return errors.Wrap(err, "failed to acquire semaphore") - } - metrics.SemaphoreHoldingSize.Inc() - return nil -} - -func (p *VsockIndexReportStream) releaseSemaphore() { - p.semaphore.Release(1) - metrics.SemaphoreHoldingSize.Dec() -} diff --git a/compliance/virtualmachines/relay/stream/vsock_index_report_stream_test.go b/compliance/virtualmachines/relay/stream/vsock_index_report_stream_test.go deleted file mode 100644 index bf1062b0530f6..0000000000000 --- a/compliance/virtualmachines/relay/stream/vsock_index_report_stream_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package stream - -import ( - "testing" - - v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" - "github.com/stretchr/testify/suite" - "google.golang.org/protobuf/proto" -) - -func TestStream(t *testing.T) { - suite.Run(t, new(streamTestSuite)) -} - -type streamTestSuite struct { - suite.Suite -} - -func (s *streamTestSuite) TestParseVMReport() { - data := []byte("malformed-data") - parsedVMReport, err := parseVMReport(data) - s.Require().Error(err) - s.Require().Nil(parsedVMReport) - - s.Run("should parse message without discovered data", func() { - indexReportOnly := &v1.VMReport{ - IndexReport: &v1.IndexReport{}, - } - marshaledIndexReportOnly, err := proto.Marshal(indexReportOnly) - s.Require().NoError(err) - - parsedVMReport, err := parseVMReport(marshaledIndexReportOnly) - s.Require().NoError(err) - s.Require().NotNil(parsedVMReport) - s.Require().Nil(parsedVMReport.GetDiscoveredData()) - }) - - validVMReport := &v1.VMReport{ - IndexReport: &v1.IndexReport{VsockCid: "42"}, - DiscoveredData: &v1.DiscoveredData{ - DetectedOs: v1.DetectedOS_UNKNOWN, - OsVersion: "", - ActivationStatus: v1.ActivationStatus_ACTIVATION_UNSPECIFIED, - DnfMetadataStatus: v1.DnfMetadataStatus_DNF_METADATA_UNSPECIFIED, - }, - } - data, err = proto.Marshal(validVMReport) - s.Require().NoError(err) - parsedVMReport, err = parseVMReport(data) - s.Require().NoError(err) - s.Require().True(proto.Equal(validVMReport, parsedVMReport)) -} - -func (s *streamTestSuite) TestValidateVsockCID() { - s.Run("missing index report does not panic", func() { - vmReport := &v1.VMReport{} - connVsockCID := uint32(42) - s.NotPanics(func() { - err := validateReportedVsockCID(vmReport, connVsockCID) - s.Require().Error(err) - }) - }) - - // Reported CID is 42 - vmReport := &v1.VMReport{ - IndexReport: &v1.IndexReport{VsockCid: "42"}, - } - - // Real (connection) CID is 99 - does not match, should return error - connVsockCID := uint32(99) - err := validateReportedVsockCID(vmReport, connVsockCID) - s.Require().Error(err) - - // Real (connection) CID is 42 - matches, should return nil - connVsockCID = uint32(42) - err = validateReportedVsockCID(vmReport, connVsockCID) - s.Require().NoError(err) -} diff --git a/compliance/virtualmachines/relay/testutils/testutils.go b/compliance/virtualmachines/relay/testutils/testutils.go deleted file mode 100644 index f4eddb6b6d005..0000000000000 --- a/compliance/virtualmachines/relay/testutils/testutils.go +++ /dev/null @@ -1,152 +0,0 @@ -package relaytest - -import ( - "context" - "io" - "net" - "os" - "testing" - "time" - - "github.com/mdlayher/vsock" - "github.com/stackrox/rox/generated/internalapi/sensor" - v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" - "google.golang.org/grpc" - "google.golang.org/protobuf/proto" -) - -// MockVsockConn implements net.Conn and allows tests to craft vsock-backed connections easily. -type MockVsockConn struct { - closed bool - data []byte - delay time.Duration - readErr error - remoteAddr net.Addr - readDeadline time.Time -} - -func NewMockVsockConn(_ testing.TB) *MockVsockConn { - return &MockVsockConn{} -} - -func (c *MockVsockConn) WithVsockCID(vsockCID uint32) *MockVsockConn { - c.remoteAddr = &vsock.Addr{ContextID: vsockCID} - return c -} - -func (c *MockVsockConn) WithData(data []byte) *MockVsockConn { - c.data = data - return c -} - -func (c *MockVsockConn) WithVMReport(vmReport *v1.VMReport) (*MockVsockConn, error) { - data, err := proto.Marshal(vmReport) - if err != nil { - return nil, err - } - c.data = data - return c, nil -} - -// Deprecated: Use WithVMReport instead -func (c *MockVsockConn) WithIndexReport(indexReport *v1.IndexReport) (*MockVsockConn, error) { - vmReport := &v1.VMReport{ - IndexReport: indexReport, - } - return c.WithVMReport(vmReport) -} - -// NewTestVMReport creates a VMReport with default test values. -func NewTestVMReport(vsockCID string) *v1.VMReport { - return &v1.VMReport{ - IndexReport: &v1.IndexReport{VsockCid: vsockCID}, - DiscoveredData: &v1.DiscoveredData{ - DetectedOs: v1.DetectedOS_UNKNOWN, - OsVersion: "", - ActivationStatus: v1.ActivationStatus_ACTIVATION_UNSPECIFIED, - DnfMetadataStatus: v1.DnfMetadataStatus_DNF_METADATA_UNSPECIFIED, - }, - } -} - -func (c *MockVsockConn) WithDelay(delay time.Duration) *MockVsockConn { - c.delay = delay - return c -} - -func (c *MockVsockConn) SetRemoteAddr(addr net.Addr) { - c.remoteAddr = addr -} - -func (c *MockVsockConn) Read(b []byte) (int, error) { - time.Sleep(c.delay) - if !c.readDeadline.IsZero() && time.Now().After(c.readDeadline) { - return 0, os.ErrDeadlineExceeded - } - if c.readErr != nil { - return 0, c.readErr - } - n := copy(b, c.data) - if n == len(c.data) { - return n, io.EOF - } - return n, nil -} - -func (c *MockVsockConn) RemoteAddr() net.Addr { - return c.remoteAddr -} - -func (c *MockVsockConn) Close() error { - c.closed = true - return nil -} - -func (c *MockVsockConn) Write([]byte) (int, error) { return 0, nil } -func (c *MockVsockConn) LocalAddr() net.Addr { return nil } -func (c *MockVsockConn) SetDeadline(time.Time) error { return nil } -func (c *MockVsockConn) SetReadDeadline(t time.Time) error { c.readDeadline = t; return nil } -func (c *MockVsockConn) SetWriteDeadline(time.Time) error { return nil } - -// MockSensorClient captures UpsertVirtualMachineIndexReport calls and allows injecting delays or errors. -type MockSensorClient struct { - capturedRequests []*sensor.UpsertVirtualMachineIndexReportRequest - delay time.Duration - err error - response *sensor.UpsertVirtualMachineIndexReportResponse -} - -func NewMockSensorClient(_ testing.TB) *MockSensorClient { - return &MockSensorClient{ - response: &sensor.UpsertVirtualMachineIndexReportResponse{Success: true}, - } -} - -func (m *MockSensorClient) WithDelay(delay time.Duration) *MockSensorClient { - m.delay = delay - return m -} - -func (m *MockSensorClient) WithError(err error) *MockSensorClient { - m.err = err - return m -} - -func (m *MockSensorClient) WithUnsuccessfulResponse() *MockSensorClient { - m.response = &sensor.UpsertVirtualMachineIndexReportResponse{Success: false} - return m -} - -func (m *MockSensorClient) CapturedRequests() []*sensor.UpsertVirtualMachineIndexReportRequest { - return m.capturedRequests -} - -func (m *MockSensorClient) UpsertVirtualMachineIndexReport(ctx context.Context, req *sensor.UpsertVirtualMachineIndexReportRequest, _ ...grpc.CallOption) (*sensor.UpsertVirtualMachineIndexReportResponse, error) { - select { - case <-time.After(m.delay): - m.capturedRequests = append(m.capturedRequests, req) - return m.response, m.err - case <-ctx.Done(): - return nil, ctx.Err() - } -} diff --git a/compliance/virtualmachines/relay/testutils/testutils_test.go b/compliance/virtualmachines/relay/testutils/testutils_test.go deleted file mode 100644 index d59c6eda8058e..0000000000000 --- a/compliance/virtualmachines/relay/testutils/testutils_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package relaytest - -import ( - "context" - "io" - "testing" - "time" - - "github.com/mdlayher/vsock" - "github.com/stackrox/rox/generated/internalapi/sensor" - v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" - "github.com/stretchr/testify/require" -) - -func TestMockVsockConn(t *testing.T) { - conn := NewMockVsockConn(t). - WithVsockCID(1234). - WithData([]byte("hello")). - WithDelay(10 * time.Millisecond) - - require.Equal(t, &vsock.Addr{ContextID: 1234}, conn.RemoteAddr()) - - buf := make([]byte, 5) - n, err := conn.Read(buf) - require.ErrorAs(t, err, &io.EOF) - require.Equal(t, 5, n) - require.Equal(t, []byte("hello"), buf) -} - -func TestMockSensorClient(t *testing.T) { - client := NewMockSensorClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - req := &sensor.UpsertVirtualMachineIndexReportRequest{ - IndexReport: &v1.IndexReport{VsockCid: "1"}, - } - - // Ensure first request is captured. - resp, err := client.UpsertVirtualMachineIndexReport(ctx, req) - require.NoError(t, err) - require.True(t, resp.GetSuccess()) - require.Len(t, client.CapturedRequests(), 1) -} diff --git a/compliance/virtualmachines/relay/vsock/vsock.go b/compliance/virtualmachines/relay/vsock/vsock.go deleted file mode 100644 index 17d36359f42e3..0000000000000 --- a/compliance/virtualmachines/relay/vsock/vsock.go +++ /dev/null @@ -1,40 +0,0 @@ -// Package vsock provides utilities for working with vsock (VM sockets) connections. -// It handles vsock-specific operations like creating listeners and extracting context IDs. -package vsock - -import ( - "fmt" - "net" - - "github.com/mdlayher/vsock" - "github.com/pkg/errors" - "github.com/stackrox/rox/pkg/env" -) - -// ExtractVsockCIDFromConnection extracts and validates the vsock context ID. -// Rejects CIDs ≤2 which are reserved (0=ANY, 1=LOCAL, 2=HOST per vsock spec). -func ExtractVsockCIDFromConnection(conn net.Conn) (uint32, error) { - remoteAddr, ok := conn.RemoteAddr().(*vsock.Addr) - if !ok { - return 0, fmt.Errorf("failed to extract remote address from vsock connection: unexpected type %T, value: %v", - conn.RemoteAddr(), conn.RemoteAddr()) - } - - // Reject invalid values according to the vsock spec (https://www.man7.org/linux/man-pages/man7/vsock.7.html) - if remoteAddr.ContextID <= 2 { - return 0, fmt.Errorf("received an invalid vsock context ID: %d (values <=2 are reserved)", remoteAddr.ContextID) - } - - return remoteAddr.ContextID, nil -} - -// NewListener creates a vsock listener on the host context ID (vsock.Host) using the port -// from VirtualMachinesVsockPort env var. Caller must close the returned listener. -func NewListener() (net.Listener, error) { - port := env.VirtualMachinesVsockPort.IntegerSetting() - listener, err := vsock.ListenContextID(vsock.Host, uint32(port), nil) - if err != nil { - return nil, errors.Wrapf(err, "listening on vsock port %d", port) - } - return listener, nil -} diff --git a/compliance/virtualmachines/relay/vsock/vsock_test.go b/compliance/virtualmachines/relay/vsock/vsock_test.go deleted file mode 100644 index 390afa25eac39..0000000000000 --- a/compliance/virtualmachines/relay/vsock/vsock_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package vsock - -import ( - "net" - "testing" - - relaytest "github.com/stackrox/rox/compliance/virtualmachines/relay/testutils" - "github.com/stretchr/testify/suite" -) - -func TestVsock(t *testing.T) { - suite.Run(t, new(vsockTestSuite)) -} - -type vsockTestSuite struct { - suite.Suite -} - -func (s *vsockTestSuite) TestExtractVsockCIDFromConnection() { - - connWrongAddrType := relaytest.NewMockVsockConn(s.T()).WithVsockCID(1234) - connWrongAddrType.SetRemoteAddr(&net.TCPAddr{}) - - cases := map[string]struct { - conn net.Conn - shouldError bool - expectedVsockCID uint32 - }{ - "wrong type fails": { - conn: connWrongAddrType, - shouldError: true, - expectedVsockCID: 0, - }, - "reserved vsock CID fails": { - conn: relaytest.NewMockVsockConn(s.T()).WithVsockCID(2), - shouldError: true, - expectedVsockCID: 0, - }, - "valid vsock CID succeeds": { - conn: relaytest.NewMockVsockConn(s.T()).WithVsockCID(42), - shouldError: false, - expectedVsockCID: 42, - }, - } - - for name, c := range cases { - s.Run(name, func() { - vsockCID, err := ExtractVsockCIDFromConnection(c.conn) - if c.shouldError { - s.Require().Error(err) - } else { - s.Require().NoError(err) - s.Equal(c.expectedVsockCID, vsockCID) - } - }) - } -} diff --git a/compliance/virtualmachines/roxagent/README.md b/compliance/virtualmachines/roxagent/README.md index b7fca2fdbc523..4dafcb651f3dd 100644 --- a/compliance/virtualmachines/roxagent/README.md +++ b/compliance/virtualmachines/roxagent/README.md @@ -1,66 +1,105 @@ -# VM Agent +# roxagent -Runs inside VMs to scan for vulnerabilities and report back to the host via vsock. -While not directly related to the `compliance` feature, the agent utilizes compliance -node scanning code for package scanning in the virtual machine. +roxagent runs inside KubeVirt VMs. It scans installed packages for vulnerabilities and serves index reports to Sensor over VSOCK. + +Sensor's VM scraper dials the agent, pulls a cached report, and forwards it to Central for vulnerability matching. Start the agent with `roxagent serve`. + +The sources live under `compliance/` because the agent reuses the Scanner V4 node indexer for RPM/DNF package databases. ## What it does -Scans the VM for installed packages (`rpm`/`dnf` databases), creates vulnerability reports, and sends them to the host over vsock. Can run once or continuously in daemon mode. Requires `sudo` privileges to scan packages. +1. Listens on a VSOCK port with mandatory mTLS. Startup does not wait for a mapping or a first scan. +2. Obtains a repository-to-CPE mapping (Sensor push over VSOCK by default, or a download URL). Until one is ready, GetReport returns `MAPPING_REQUIRED` and no scan runs. +3. Scans the VM for installed packages and caches the index report with a content-hash token. +4. On each Sensor connection, handles a framed `VMServiceRequest` / `VMServiceResponse` (for example `get_report` and `sync_repo_cpe_mapping`). +5. Periodically rescans and atomically swaps the cached report. ## Usage ```bash -# Single scan -sudo ./roxagent - -# Daemon mode (scans every 4 hours by default) -sudo ./roxagent --daemon +sudo ./roxagent serve -# Custom settings -sudo ./roxagent --daemon --index-interval 10m --host-path /custom/path --port 2048 +# Example with common overrides +sudo ./roxagent serve --port 818 --host-path / --rescan-interval 4h ``` -## Flags +## Flags (`serve`) -- `--daemon` - Run continuously (default: false). -- `--index-interval` - Time between scans in daemon mode (default: 4h). -- `--host-path` - Where to look for package databases (default: /). -- `--port` - VSock port (default: 818). -- `--repo-cpe-url` - URL for the repository to CPE mapping. -- `--timeout` - VSock client timeout when sending index reports. -- `--verbose` - Prints the index reports to stdout. +| Flag | Default | Notes | +|------|---------|-------| +| `--port` | `818` | VSOCK listen port | +| `--host-path` | `/` | Root filesystem path for package indexing | +| `--repo-cpe-url` | empty (Sensor-managed) | Mapping download URL. Empty means Sensor pushes the mapping over VSOCK. | +| `--repo-cpe-bundled-path` | empty | Optional seed mapping file for Sensor-managed agents. Ignored when `--repo-cpe-url` is set. | +| `--rescan-interval` | `4h` | Must be between `5m` and `168h` (7d) | +| `--ca-fetch-timeout` | `10s` | Timeout for one KubeVirt CA fetch over VSOCK | +| `--conn-deadline` | `30s` | Max time for one connection's TLS handshake plus request/response (`5s`-`5m`) | ## How it works -1. Scans filesystem for `rpm`/`dnf` package databases. -2. Pulls repo-to-CPE mappings from Red Hat. Network connection to the public Internet or to Sensor is required. -3. Creates protobuf index report. -4. Sends report to host via vsock. +1. **Mapping file:** Every scan reads a local cache file kept fresh by a mapping provider. With `--repo-cpe-url` empty (the default), Sensor pushes the mapping over VSOCK via `sync_repo_cpe_mapping`. `--repo-cpe-bundled-path` is an optional seed for that Sensor-managed path when there is no cache yet. With a URL set, the agent downloads in the background and never accepts a Sensor push or a bundled seed: a URL that never succeeds stays not-ready rather than looking configured. A failed later URL refresh keeps the last-good cache from a previous successful fetch of that URL. + + A URL-backed agent needs outbound HTTPS to that host (or a proxy). Isolated networks either rely on Sensor (optionally with a bundled seed), or host a copy of `repository-to-cpe.json` the VM can reach and pass `--repo-cpe-url`. +2. **Listen first:** VSOCK comes up immediately. The first scan runs when a mapping is ready (cached from a previous run, a bundled seed, a successful URL fetch, or a Sensor push), not at process start. +3. **Serving reports:** Sensor connects and the agent serves the cached report. If Sensor already has the current content-hash token, the agent omits the payload. Sensor can still force a full report by sending an empty token (first request, NACK retry, or the 4h refresh). With no mapping yet, the response is `MAPPING_REQUIRED`. +4. **Rescan:** On `--rescan-interval`, or sooner when the mapping changes, re-indexes the rpm/dnf database and swaps the cache. The token stays the same when the report and facts are unchanged. + +### TLS (mandatory) -The host receives these reports and forwards them to StackRox Central for vulnerability analysis. +Sensor always dials with TLS. The agent always requires TLS; there is no plaintext mode. -## Building +- On each handshake, the agent fetches the KubeVirt CA from the host (VSOCK CID 2, port 1) via virt-handler's `System.CABundle` RPC. In namespace-isolated VSOCK mode that CA service is only up for the duration of the dial/handshake, so the fetch runs inside the handshake. +- If a fetch fails but a CA was cached earlier, the agent reuses the cache. If there is no cache yet, the handshake fails. +- Sensor (via virt-handler) presents a client cert signed by the KubeVirt CA; the agent verifies it. +- The agent uses a self-signed server cert (virt-handler does not verify the server cert). + +## Deployment + +### Native systemd (CI / GitHub Action) + +CI and the [Add VMs to Cluster](../../../.github/workflows/add-vms-to-cluster.yml) workflow install through `scripts/ci/add-vms/`: + +1. Workflow → composite action `scripts/ci/add-vms` +2. `add-vms.sh` deploys VMs, then calls `install-agent-native.sh` +3. `install-agent-native.sh` cross-compiles roxagent, copies it with `virtctl scp`, and installs two units: + - `roxagent-prep.service` builds the `/tmp/roxroot` mount-point skeleton and copies `/var/lib/rpm` to `/tmp/roxagent-rpm` (writable, for SQLite WAL - see [quadlet/README.md](quadlet/README.md#why-copy-the-rpm-database)) + - `roxagent-serve.service` bind-mounts the real host paths into `/tmp/roxroot` and runs: + +```bash +ExecStart=/usr/local/bin/roxagent serve --port 818 --host-path /tmp/roxroot +``` + +Verification waits until `roxagent-serve.service` is enabled and active. That is the same path developers get when they run the workflow against an infra cluster. + +### Quadlet (RHEL VMs) + +See [quadlet/README.md](quadlet/README.md) for Podman Quadlet deployment (`Exec=serve`, systemd unit `roxagent.service`). + +### Building from source ```bash go build -o roxagent . -# For Linux VMs -GOOS=linux GOARCH=amd64 go build -o roxagent-linux . +GOOS=linux GOARCH=amd64 go build -o roxagent . ``` ## Troubleshooting -**Can't connect to host** -- Check if vsock is enabled in the VM. -- Verify the port isn't in use. -- Make sure vsock kernel modules are loaded. +### Can't connect / dial failures from Sensor + +- Confirm VSOCK is enabled on the VMI (`spec.domain.devices.autoattachVSOCK`). +- Confirm the VSOCK port is free inside the VM and matches Sensor's config. +- Confirm Sensor has RBAC for `virtualmachineinstances/vsock` on `subresources.kubevirt.io`. + +### No packages found (zero-package reports) + +- Confirm `--host-path` points at the real root (native CI uses `/tmp/roxroot` with bind mounts). +- Confirm `rpm`/`dnf` databases exist and are readable under that path. +- Check Sensor logs for `reportcheck` warnings. -**No packages found** -- Check `--host-path` points to the right place. -- Verify `rpm`/`dnf` databases exist and are readable. -- Use `--verbose` to examine the index report and compare with the content from `rpm`/`dnf` databases. +### TLS handshake failures -**Scan failures** -- Check internet access for repo-to-CPE downloads. -- Look at logs for specific errors. +- Confirm virt-handler is serving the CA on CID 2 port 1 during the dial. +- On first connect with a cold CA cache, a failed CA fetch fails the handshake (no plaintext fallback). +- Look for `Rejected plaintext connection` if the peer is not speaking TLS. +- Tune `--ca-fetch-timeout` / `--conn-deadline` if handshakes time out under load. diff --git a/compliance/virtualmachines/roxagent/cmd/cmd.go b/compliance/virtualmachines/roxagent/cmd/cmd.go index 197d55064ee37..ded5aa003db5e 100644 --- a/compliance/virtualmachines/roxagent/cmd/cmd.go +++ b/compliance/virtualmachines/roxagent/cmd/cmd.go @@ -2,17 +2,11 @@ package cmd import ( "context" - "time" "github.com/spf13/cobra" - "github.com/stackrox/rox/compliance/virtualmachines/roxagent/common" - "github.com/stackrox/rox/compliance/virtualmachines/roxagent/index" - "github.com/stackrox/rox/compliance/virtualmachines/roxagent/vsock" "github.com/stackrox/rox/pkg/logging" ) -const repoToCPEMappingURL = "https://security.access.redhat.com/data/metrics/repository-to-cpe.json" - var log = logging.LoggerForModule() func RootCmd(ctx context.Context) *cobra.Command { @@ -22,39 +16,6 @@ func RootCmd(ctx context.Context) *cobra.Command { SilenceUsage: true, } cmd.SetContext(ctx) - cfg := &common.Config{} - cmd.Flags().BoolVar(&cfg.DaemonMode, "daemon", false, - "Run in daemon mode. Sends index reports continuously.", - ) - cmd.Flags().DurationVar(&cfg.IndexInterval, "index-interval", 240*time.Minute, - "Interval duration in which index reports are sent in daemon mode.", - ) - cmd.Flags().StringVar(&cfg.IndexHostPath, "host-path", "/", - "Path where the indexer starts searching for the RPM and DNF databases.", - ) - cmd.Flags().StringVar(&cfg.RepoToCPEMappingURL, "repo-cpe-url", repoToCPEMappingURL, - "URL for the repository to CPE mapping.", - ) - cmd.Flags().DurationVar(&cfg.Timeout, "timeout", 30*time.Second, - "VSock client timeout when sending index reports.", - ) - cmd.Flags().BoolVar(&cfg.Verbose, "verbose", false, - "Prints the index reports to stdout.", - ) - cmd.Flags().Uint32Var(&cfg.VsockPort, "port", 818, - "VSock port to connect with the virtual machine host.", - ) - cmd.Run = func(cmd *cobra.Command, _ []string) { - client := &vsock.Client{Port: cfg.VsockPort, Timeout: cfg.Timeout, Verbose: cfg.Verbose} - if cfg.DaemonMode { - if err := index.RunDaemon(ctx, cfg, client); err != nil { - log.Errorf("Running indexer daemon: %v", err) - } - return - } - if err := index.RunSingle(ctx, cfg, client); err != nil { - log.Errorf("Running indexer: %v", err) - } - } + cmd.AddCommand(ServeCmd(ctx)) return &cmd } diff --git a/compliance/virtualmachines/roxagent/cmd/rescanner.go b/compliance/virtualmachines/roxagent/cmd/rescanner.go new file mode 100644 index 0000000000000..98af9a7281b77 --- /dev/null +++ b/compliance/virtualmachines/roxagent/cmd/rescanner.go @@ -0,0 +1,158 @@ +package cmd + +import ( + "context" + "fmt" + "sync/atomic" + "time" + + "github.com/stackrox/rox/compliance/virtualmachines/roxagent/vsockserver" + v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4" +) + +// Retry policy for rescanner.Run's schedule: how soon to try again after a +// rescan fails, instead of leaving Sensor with a stale cached report for up +// to the full (potentially hours-long) rescan interval. +const ( + rescanRetryBaseBackoff = 2 * time.Minute + rescanRetryMaxBackoff = 30 * time.Minute +) + +// rescanner owns the scan-and-cache-update concern: periodically rescanning +// the VM filesystem and publishing results to cache, independent of how the +// cached report is served over VSOCK. Kept separate from Server/CARefresher +// wiring in runServe so its retry/backoff policy is easier to reason about +// and test on its own. Scanning does not make a network call; provider +// keeps the mapping file fresh independently, on its own schedule, so only +// the schedule for the next rescan attempt is retried here, not the scan +// itself. +type rescanner struct { + cache *vsockserver.ReportCache + hostPath string + provider vsockserver.MappingProvider + interval time.Duration + + // scanFn defaults to the package scan function; tests override it to + // inject failures. factsFn defaults to the package discoverFacts + // function; tests override it to avoid exercising the real + // filesystem, since discoverFacts otherwise reads real host paths + // (e.g. hostPath="" resolves to "/etc/pki/entitlement" et al., not a + // no-op). + scanFn func(ctx context.Context, hostPath, mappingFilePath string) (*v4.IndexReport, error) + factsFn func(hostPath string) map[string]string + + // timer is the next scan deadline. OnMappingChanged Reset(0) fires it + // immediately; a Reset(0) during scanOnce is received on the next + // select because t.C is unbuffered (Go 1.23+). Nil after Run returns + // so a later Reset is a no-op. + timer atomic.Pointer[time.Timer] +} + +func newRescanner(cache *vsockserver.ReportCache, hostPath string, provider vsockserver.MappingProvider, interval time.Duration) *rescanner { + r := &rescanner{ + cache: cache, + hostPath: hostPath, + provider: provider, + interval: interval, + scanFn: scan, + factsFn: discoverFacts, + } + r.timer.Store(time.NewTimer(interval)) + return r +} + +// OnMappingChanged is the updater's onChange callback: it schedules an +// immediate scan attempt. +func (r *rescanner) OnMappingChanged() { + t := r.timer.Load() + if t == nil { + return + } + t.Reset(0) + log.Info("Mapping changed, scheduling immediate rescan") +} + +// Run rescans every r.interval, publishing each successful result to cache. +// If a rescan fails, the next attempt is scheduled sooner than the full +// interval - with exponential backoff, capped at both rescanRetryMaxBackoff +// and r.interval itself, so a retry is never slower than just waiting for +// the next scheduled rescan would be. Blocks until ctx is cancelled. +func (r *rescanner) Run(ctx context.Context) { + t := r.timer.Load() + defer func() { + r.timer.Store(nil) + t.Stop() + }() + + backoff := rescanRetryBaseBackoff + for { + select { + case <-ctx.Done(): + return + case <-t.C: + // Schedule the default next scan before scanOnce so a + // mapping change during the scan (Reset(0)) is not + // overwritten by the post-scan Reset. + t.Reset(r.interval) + if err := r.scanOnce(ctx); err != nil { + retryIn := min(backoff, r.interval) + log.Errorf("Rescan failed: %v; trying again in %v", err, retryIn) + backoff = min(backoff*2, rescanRetryMaxBackoff) + t.Reset(retryIn) + continue + } + backoff = rescanRetryBaseBackoff + } + } +} + +// scanOnce keeps the mapping file unchanged while this scan reads it, then +// applies any pending Sync so a follow-up scan can use the new mapping. +func (r *rescanner) scanOnce(ctx context.Context) error { + if !r.provider.Ready() { + log.Info("Skipping rescan: repository-to-CPE mapping not yet available") + return nil + } + + gate, hasGate := r.provider.(vsockserver.ScanBusyGate) + if hasGate { + gate.MarkScanBusy() + defer gate.MarkScanIdleAndApplyPending() + } + + path, err := r.provider.Path() + if err != nil { + return fmt.Errorf("resolving mapping file path: %w", err) + } + + // Snapshot the hash before scanFn: a URL-backed provider has no busy + // gate, so a background fetch can replace the live mapping mid-scan. + mappingHash := r.provider.Hash() + log.Infof("Starting rescan (mapping hash=%s)", mappingHash) + report, err := r.scanFn(ctx, r.hostPath, path) + if err != nil { + return err + } + + r.cache.SetReport(report, r.factsFn(r.hostPath), mappingHash) + log.Infof("Rescan complete, report updated (token=%s, packages=%d)", r.cache.Token(), len(report.GetContents().GetPackages())) + return nil +} + +// runAsync starts Run in a goroutine and returns a channel that is closed +// when Run returns. Callers that cancel ctx should wait on stopped so +// assertions do not race Run's last scanOnce. +func (r *rescanner) runAsync(ctx context.Context) (stopped <-chan struct{}) { + return startRun(ctx, r.Run) +} + +// startRun starts run in a goroutine and returns a channel that is closed +// when run returns. +func startRun(ctx context.Context, run func(context.Context)) (stopped <-chan struct{}) { + ch := make(chan struct{}) + go func() { + defer close(ch) + run(ctx) + }() + return ch +} diff --git a/compliance/virtualmachines/roxagent/cmd/rescanner_test.go b/compliance/virtualmachines/roxagent/cmd/rescanner_test.go new file mode 100644 index 0000000000000..4839d828e207f --- /dev/null +++ b/compliance/virtualmachines/roxagent/cmd/rescanner_test.go @@ -0,0 +1,332 @@ +package cmd + +import ( + "context" + "errors" + "net" + "testing" + "testing/synctest" + "time" + + "github.com/stackrox/rox/compliance/virtualmachines/roxagent/vsockserver" + v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4" + pb "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/sync" + "github.com/stackrox/rox/pkg/vsockframing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +// fakeProvider is a vsockserver.MappingProvider test double for rescanner +// tests; UpdatePath/Bytes are unused by rescanner and always zero. +type fakeProvider struct { + ready bool + path string + pathErr error + hash string +} + +func (f *fakeProvider) Ready() bool { return f.ready } +func (f *fakeProvider) Hash() string { return f.hash } +func (f *fakeProvider) UpdatePath() pb.RepoCPEMappingUpdatePath { + return pb.RepoCPEMappingUpdatePath_REPO_CPE_MAPPING_UPDATE_PATH_UNSPECIFIED +} +func (f *fakeProvider) Bytes() ([]byte, error) { return nil, errors.New("not implemented") } +func (f *fakeProvider) Path() (string, error) { return f.path, f.pathErr } + +var _ vsockserver.MappingProvider = (*fakeProvider)(nil) + +// fakeGatedProvider adds ScanBusyGate call tracking on top of fakeProvider, +// mirroring SensorUpdater (gated) as opposed to URLUpdater (ungated) so +// tests can verify scanOnce only invokes the gate for providers that +// implement it. +type fakeGatedProvider struct { + fakeProvider + busyCalls int + idleCalls int +} + +func (f *fakeGatedProvider) MarkScanBusy() { f.busyCalls++ } +func (f *fakeGatedProvider) MarkScanIdleAndApplyPending() { f.idleCalls++ } + +var ( + _ vsockserver.MappingProvider = (*fakeGatedProvider)(nil) + _ vsockserver.ScanBusyGate = (*fakeGatedProvider)(nil) +) + +// testRescanner returns a rescanner with a long default interval so tests +// that don't care about the periodic loop never trigger it by accident. +// factsFn is stubbed out so Run never exercises the real discoverFacts, +// which - unlike scanFn - has no test-friendly no-op input; hostPath="" +// resolves to real absolute host paths (e.g. "/etc/pki/entitlement"), not +// a safe default. provider defaults to always-ready when nil, so existing +// tests that don't care about readiness are unaffected. +func testRescanner(provider vsockserver.MappingProvider) *rescanner { + if provider == nil { + provider = &fakeProvider{ready: true} + } + r := newRescanner(&vsockserver.ReportCache{}, "", provider, time.Hour) + r.factsFn = func(string) map[string]string { return nil } + return r +} + +func TestRescanner_Run(t *testing.T) { + t.Run("should publish to cache on each tick", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r := testRescanner(nil) + var mu sync.Mutex + var calls int + r.scanFn = func(_ context.Context, _, _ string) (*v4.IndexReport, error) { + return concurrency.WithLock2(&mu, func() (*v4.IndexReport, error) { + calls++ + return &v4.IndexReport{HashId: "ok"}, nil + }) + } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := r.runAsync(ctx) + defer func() { + cancel() + <-stopped + }() + synctest.Wait() + + time.Sleep(r.interval) + synctest.Wait() + assert.Equal(t, 1, concurrency.WithLock1(&mu, func() int { return calls })) + + time.Sleep(r.interval) + synctest.Wait() + assert.Equal(t, 2, concurrency.WithLock1(&mu, func() int { return calls }), "should rescan again on the next tick") + }) + }) + + t.Run("should retry sooner than the full interval after a failed rescan", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r := testRescanner(nil) + var mu sync.Mutex + var calls int + r.scanFn = func(_ context.Context, _, _ string) (*v4.IndexReport, error) { + return concurrency.WithLock2(&mu, func() (*v4.IndexReport, error) { + calls++ + if calls == 1 { + return nil, errors.New("transient scan error") + } + return &v4.IndexReport{HashId: "ok"}, nil + }) + } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := r.runAsync(ctx) + defer func() { + cancel() + <-stopped + }() + synctest.Wait() + + time.Sleep(r.interval) + synctest.Wait() + assert.Equal(t, 1, concurrency.WithLock1(&mu, func() int { return calls })) + + time.Sleep(rescanRetryBaseBackoff) + synctest.Wait() + assert.Equal(t, 2, concurrency.WithLock1(&mu, func() int { return calls }), "the failed rescan was never retried") + + time.Sleep(rescanRetryBaseBackoff) + synctest.Wait() + assert.Equal(t, 2, concurrency.WithLock1(&mu, func() int { return calls }), "a successful rescan should wait the full interval, not keep the retry backoff") + }) + }) + + t.Run("should stop promptly when the context is cancelled", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r := testRescanner(nil) + + ctx, cancel := context.WithCancel(t.Context()) + stopped := r.runAsync(ctx) + + // Run must return promptly once ctx is cancelled: if it + // doesn't, the bubble deadlocks on the blocked <-stopped below + // (nothing left to advance the fake clock), and synctest.Test + // fails the test on deadlock automatically. + cancel() + <-stopped + // Updaters can still fire onChange after Run has returned. + r.OnMappingChanged() + }) + }) + + t.Run("a periodic tick is a no-op when the mapping is not yet ready", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r := testRescanner(&fakeProvider{ready: false}) + var calls int + r.scanFn = func(context.Context, string, string) (*v4.IndexReport, error) { + calls++ + return &v4.IndexReport{}, nil + } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := r.runAsync(ctx) + defer func() { + cancel() + <-stopped + }() + synctest.Wait() + + time.Sleep(r.interval) + synctest.Wait() + + assert.Zero(t, calls, "scanFn must not be called while the mapping provider isn't ready") + }) + }) + + t.Run("OnMappingChanged triggers an immediate scan attempt", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r := testRescanner(&fakeProvider{ready: true}) + var calls int + r.scanFn = func(context.Context, string, string) (*v4.IndexReport, error) { + calls++ + return &v4.IndexReport{}, nil + } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := r.runAsync(ctx) + defer func() { + cancel() + <-stopped + }() + synctest.Wait() + + r.OnMappingChanged() + synctest.Wait() + + assert.Equal(t, 1, calls, "OnMappingChanged should trigger a scan without waiting for the next tick") + }) + }) + + t.Run("OnMappingChanged failure retries at backoff not the full interval", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r := testRescanner(nil) + var mu sync.Mutex + var calls int + r.scanFn = func(_ context.Context, _, _ string) (*v4.IndexReport, error) { + return concurrency.WithLock2(&mu, func() (*v4.IndexReport, error) { + calls++ + if calls == 1 { + return nil, errors.New("transient scan error") + } + return &v4.IndexReport{HashId: "ok"}, nil + }) + } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := r.runAsync(ctx) + defer func() { + cancel() + <-stopped + }() + synctest.Wait() + + r.OnMappingChanged() + synctest.Wait() + assert.Equal(t, 1, concurrency.WithLock1(&mu, func() int { return calls }), "OnMappingChanged should trigger a scan immediately") + + time.Sleep(rescanRetryBaseBackoff) + synctest.Wait() + assert.Equal(t, 2, concurrency.WithLock1(&mu, func() int { return calls }), "a failed mapping-triggered scan should retry after the backoff") + + time.Sleep(rescanRetryBaseBackoff) + synctest.Wait() + assert.Equal(t, 2, concurrency.WithLock1(&mu, func() int { return calls }), "a successful retry should wait the full interval") + }) + }) + + t.Run("a failed scan clears busy via idle-and-apply-pending", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + provider := &fakeGatedProvider{fakeProvider: fakeProvider{ready: true}} + r := testRescanner(provider) + r.scanFn = func(context.Context, string, string) (*v4.IndexReport, error) { + return nil, errors.New("scan failed") + } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := r.runAsync(ctx) + defer func() { + cancel() + <-stopped + }() + synctest.Wait() + + time.Sleep(r.interval) + synctest.Wait() + + assert.Equal(t, 1, provider.busyCalls) + assert.Equal(t, 1, provider.idleCalls, "a failed scan must not leave a Sync stuck behind busy forever") + }) + }) + + t.Run("a successful scan clears busy without waiting for GetReport", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + provider := &fakeGatedProvider{fakeProvider: fakeProvider{ready: true}} + r := testRescanner(provider) + r.scanFn = func(context.Context, string, string) (*v4.IndexReport, error) { + return &v4.IndexReport{HashId: "ok"}, nil + } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := r.runAsync(ctx) + defer func() { + cancel() + <-stopped + }() + synctest.Wait() + + time.Sleep(r.interval) + synctest.Wait() + + assert.Equal(t, 1, provider.busyCalls) + assert.Equal(t, 1, provider.idleCalls, "a successful scan must apply a deferred Sync itself; GetReport can be hours away") + }) + }) +} + +// TestScanOnce_StoresHashFromBeforeTheScan covers a URL refresh that +// replaces the live mapping while scanFn runs: SetReport must keep the +// hash from before the scan. +func TestScanOnce_StoresHashFromBeforeTheScan(t *testing.T) { + const hashAtScan = "hash-at-scan-start" + provider := &fakeProvider{ready: true, hash: hashAtScan} + r := testRescanner(provider) + r.scanFn = func(context.Context, string, string) (*v4.IndexReport, error) { + provider.hash = "hash-after-url-refresh" + return &v4.IndexReport{HashId: "ok"}, nil + } + + require.NoError(t, r.scanOnce(t.Context())) + assert.Equal(t, "hash-after-url-refresh", provider.Hash(), "the live mapping may have moved on") + assert.Equal(t, hashAtScan, mappingHashFromCache(t, r.cache, provider)) +} + +func mappingHashFromCache(t *testing.T, cache *vsockserver.ReportCache, provider vsockserver.MappingProvider) string { + t.Helper() + handler := vsockserver.NewHandler(cache, "test", provider, nil) + clientConn, serverConn := net.Pipe() + go handler.HandleConn(serverConn) + + req, err := proto.Marshal(&pb.VMServiceRequest{ + Method: &pb.VMServiceRequest_GetReport{GetReport: &pb.GetReportRequest{}}, + }) + require.NoError(t, err) + require.NoError(t, vsockframing.WriteFrame(clientConn, req)) + + respData, err := vsockframing.ReadFrame(clientConn, 10<<20) + require.NoError(t, err) + _ = clientConn.Close() + + var resp pb.VMServiceResponse + require.NoError(t, proto.Unmarshal(respData, &resp)) + require.NotNil(t, resp.GetGetReport()) + return resp.GetMeta().GetRepoCpeMappingHash() +} diff --git a/compliance/virtualmachines/roxagent/cmd/sdnotify.go b/compliance/virtualmachines/roxagent/cmd/sdnotify.go new file mode 100644 index 0000000000000..9ad3a4df5bfc2 --- /dev/null +++ b/compliance/virtualmachines/roxagent/cmd/sdnotify.go @@ -0,0 +1,20 @@ +package cmd + +import ( + "fmt" + + "github.com/coreos/go-systemd/v22/daemon" +) + +// notifySystemdReady sends READY=1 on NOTIFY_SOCKET when set (Type=notify / +// Quadlet Notify=true). No-op when unset so bare `roxagent serve` still works. +func notifySystemdReady() error { + sent, err := daemon.SdNotify(false, daemon.SdNotifyReady) + if err != nil { + return fmt.Errorf("sd_notify READY=1: %w", err) + } + if sent { + log.Info("Signaled systemd readiness (READY=1)") + } + return nil +} diff --git a/compliance/virtualmachines/roxagent/cmd/sdnotify_test.go b/compliance/virtualmachines/roxagent/cmd/sdnotify_test.go new file mode 100644 index 0000000000000..f3cae0a4c2a23 --- /dev/null +++ b/compliance/virtualmachines/roxagent/cmd/sdnotify_test.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "net" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestNotifySystemdReady_NoSocketIsNoop covers running without systemd +// (NOTIFY_SOCKET unset): readiness notify must not error. +func TestNotifySystemdReady_NoSocketIsNoop(t *testing.T) { + t.Setenv("NOTIFY_SOCKET", "") + require.NoError(t, notifySystemdReady()) +} + +// TestNotifySystemdReady_WritesReady covers the sd_notify protocol against a +// fake NOTIFY_SOCKET: READY=1 must be written as a unixgram datagram. +func TestNotifySystemdReady_WritesReady(t *testing.T) { + // Prefer a short path under /tmp: macOS AF_UNIX sun_path is small, and + // t.TempDir() paths are often too long to bind. + f, err := os.CreateTemp("", "rox-sdn-*.sock") + require.NoError(t, err) + sockPath := f.Name() + require.NoError(t, f.Close()) + require.NoError(t, os.Remove(sockPath)) + t.Cleanup(func() { _ = os.Remove(sockPath) }) + + addr := &net.UnixAddr{Name: sockPath, Net: "unixgram"} + ln, err := net.ListenUnixgram("unixgram", addr) + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + + t.Setenv("NOTIFY_SOCKET", sockPath) + + require.NoError(t, notifySystemdReady()) + + _ = ln.SetReadDeadline(time.Now().Add(2 * time.Second)) + buf := make([]byte, 64) + n, _, err := ln.ReadFromUnix(buf) + require.NoError(t, err) + assert.Equal(t, "READY=1", string(buf[:n])) +} diff --git a/compliance/virtualmachines/roxagent/cmd/serve.go b/compliance/virtualmachines/roxagent/cmd/serve.go new file mode 100644 index 0000000000000..61871ed0cf39b --- /dev/null +++ b/compliance/virtualmachines/roxagent/cmd/serve.go @@ -0,0 +1,255 @@ +package cmd + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "math/big" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/mdlayher/vsock" + "github.com/spf13/cobra" + "github.com/stackrox/rox/compliance/node/index" + "github.com/stackrox/rox/compliance/virtualmachines/roxagent/discovery" + "github.com/stackrox/rox/compliance/virtualmachines/roxagent/vsockserver" + v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4" + "github.com/stackrox/rox/pkg/httputil/proxy" + "github.com/stackrox/rox/pkg/sync" +) + +// mappingCachePath is where the repository-to-CPE mapping file is cached; +// scans always read from this file, so a slow or unavailable mapping +// endpoint never blocks or fails a scan directly. Hardcoded for now; make +// it a --mapping-cache-path flag later if needed. +var mappingCachePath = filepath.Join(os.TempDir(), "roxagent-repo2cpe.json") + +// Set via -ldflags at build time. +var agentVersion = "development" //XDef:STABLE_MAIN_VERSION + +// Bounds enforced on the corresponding CLI flags in serveConfig.validate. +const ( + // minRescanInterval guards against a misconfigured, too-frequent + // rescan cadence hammering the VM's disk; it has no effect on ACS + // itself, only on load imposed on the scanned VM. + minRescanInterval = 5 * time.Minute + // maxRescanInterval caps how stale a cached report can get from a + // misconfigured interval (e.g. years). Zero is rejected, not treated + // as "disable periodic rescans" (can be changed in the future). + maxRescanInterval = 7 * 24 * time.Hour + + // connDeadline bounds one connection's TLS handshake plus + // request/response. Its range balances tolerating slow-but-legitimate + // connections (e.g. under host resource contention) against limiting + // how long a stalled or malicious peer can occupy the agent's single + // in-flight-connection slot (see vsockserver.WithConnDeadline). + minConnDeadline = 5 * time.Second + maxConnDeadline = 5 * time.Minute +) + +// ServeCmd returns the "serve" cobra subcommand for pull-mode operation. +func ServeCmd(ctx context.Context) *cobra.Command { + var cfg serveConfig + cmd := &cobra.Command{ + Use: "serve", + Short: "Scan and serve report over VSOCK (pull mode).", + RunE: func(_ *cobra.Command, _ []string) error { + return runServe(ctx, cfg) + }, + } + cmd.Flags().Uint32Var(&cfg.port, "port", 818, "VSOCK port to listen on") + cmd.Flags().StringVar(&cfg.hostPath, "host-path", "/", "Root filesystem path for indexing") + cmd.Flags().StringVar(&cfg.repoCPEURL, "repo-cpe-url", "", + "Repository to CPE mapping URL. Empty (default) means Sensor pushes the mapping over VSOCK instead.") + cmd.Flags().StringVar(&cfg.repoCPEBundledPath, "repo-cpe-bundled-path", "", + "Optional seed mapping file for Sensor-managed agents. Ignored when --repo-cpe-url is set.") + cmd.Flags().DurationVar(&cfg.rescanInterval, "rescan-interval", 4*time.Hour, + fmt.Sprintf("Interval between rescans (range %v-%v)", minRescanInterval, maxRescanInterval)) + cmd.Flags().DurationVar(&cfg.caFetchTimeout, "ca-fetch-timeout", 10*time.Second, + "Timeout for each KubeVirt CA fetch attempt over VSOCK") + cmd.Flags().DurationVar(&cfg.connDeadline, "conn-deadline", vsockserver.DefaultConnDeadline, + fmt.Sprintf("Max time allowed for one connection's TLS handshake and request/response "+ + "(range %v-%v). Raising it tolerates slower legitimate connections (e.g. under host "+ + "resource contention) at the cost of letting a stalled or malicious peer occupy the "+ + "agent's single in-flight-connection slot for longer; lowering it does the opposite.", + minConnDeadline, maxConnDeadline)) + return cmd +} + +// serveConfig holds runServe's inputs, validated together by validate. +type serveConfig struct { + port uint32 + hostPath string + repoCPEURL string + repoCPEBundledPath string + rescanInterval time.Duration + caFetchTimeout time.Duration + connDeadline time.Duration +} + +func (c serveConfig) validate() error { + if c.rescanInterval < minRescanInterval || c.rescanInterval > maxRescanInterval { + return fmt.Errorf("rescan-interval must be between %v and %v (got %v)", minRescanInterval, maxRescanInterval, c.rescanInterval) + } + if c.caFetchTimeout <= 0 { + return errors.New("ca-fetch-timeout must be greater than 0") + } + if c.connDeadline < minConnDeadline || c.connDeadline > maxConnDeadline { + return fmt.Errorf("conn-deadline must be between %v and %v (got %v)", minConnDeadline, maxConnDeadline, c.connDeadline) + } + return nil +} + +// newRescannerAndProvider builds the rescanner and its mapping provider. +// OnMappingChanged does not read provider, so passing it to the +// provider's constructor before provider is assigned below is safe. +func newRescannerAndProvider(cache *vsockserver.ReportCache, cfg serveConfig) (vmRescanner *rescanner, provider vsockserver.MappingProvider, updater vsockserver.MappingUpdater, urlUpdater *vsockserver.URLUpdater) { + vmRescanner = newRescanner(cache, cfg.hostPath, nil, cfg.rescanInterval) + + if cfg.repoCPEURL != "" { + urlUpdater = vsockserver.NewURLUpdater(cfg.repoCPEURL, mappingCachePath, vmRescanner.OnMappingChanged) + provider = urlUpdater + } else { + sensorUpdater := vsockserver.NewSensorUpdater(mappingCachePath, cfg.repoCPEBundledPath, vmRescanner.OnMappingChanged) + provider, updater = sensorUpdater, sensorUpdater + } + vmRescanner.provider = provider + + return vmRescanner, provider, updater, urlUpdater +} + +func runServe(ctx context.Context, cfg serveConfig) error { + if err := cfg.validate(); err != nil { + return err + } + + // serveCtx lets early-return paths (systemd notify failure after listen) + // stop Serve instead of leaking the accept loop until process exit. + serveCtx, cancelServe := context.WithCancel(ctx) + defer cancelServe() + + cache := &vsockserver.ReportCache{} + vmRescanner, provider, updater, urlUpdater := newRescannerAndProvider(cache, cfg) + + handler := vsockserver.NewHandler(cache, agentVersion, provider, updater) + + // TLS is mandatory: sensor always dials with TLS, so a plaintext agent is + // unreachable. The KubeVirt CA (served by virt-handler on CID 2, port 1) + // is fetched on demand, during each incoming handshake, whenever it + // isn't already cached — see CARefresher.TLSConfig. + // In KubeVirt's namespace-isolated VSOCK mode the CA service exists + // only for the duration of an in-flight handshake, so it cannot be + // warmed up independently ahead of time. + caRefresher := vsockserver.NewCARefresher(vsockserver.WithFetchTimeout(cfg.caFetchTimeout)) + serverCert, err := selfSignedCert() + if err != nil { + return fmt.Errorf("generating server certificate: %w", err) + } + tlsCfg := caRefresher.TLSConfig(serverCert) + log.Info("TLS enabled with KubeVirt CA (fetched on demand if not yet cached)") + + srv := vsockserver.NewServer(handler, tlsCfg, vsockserver.WithConnDeadline(cfg.connDeadline)) + + ln, err := vsock.Listen(cfg.port, nil) + if err != nil { + return fmt.Errorf("listening on VSOCK port %d: %w", cfg.port, err) + } + log.Infof("Listening on VSOCK port %d (pull mode)", cfg.port) + + if urlUpdater != nil { + if err := urlUpdater.Start(serveCtx); err != nil { + return fmt.Errorf("starting repository-to-CPE mapping downloader: %w", err) + } + defer urlUpdater.Stop() + } + + var wg sync.WaitGroup + wg.Go(func() { srv.Serve(serveCtx, ln) }) + wg.Go(func() { vmRescanner.Run(serveCtx) }) + + // Report systemd readiness once VSOCK is up. + if err := notifySystemdReady(); err != nil { + // Exiting, as systemd would kill this service anyway after TimeoutStartSec. + cancelServe() + wg.Wait() + return err + } + + <-serveCtx.Done() + // Wait for Serve's graceful drain (in-flight connections) and the + // rescan loop to finish before returning, so the process doesn't exit + // mid-drain or mid-scan. + wg.Wait() + return nil +} + +// scan indexes the VM filesystem at hostPath, consulting the +// repository-to-CPE mapping data cached at mappingFilePath. +// The mapping provider keeps mappingFilePath fresh independently. +func scan(ctx context.Context, hostPath, mappingFilePath string) (*v4.IndexReport, error) { + cfg := index.NodeIndexerConfig{ + HostPath: hostPath, + Client: &http.Client{Transport: proxy.RoundTripper()}, + Repo2CPEMappingFile: mappingFilePath, + } + return index.NewNodeIndexer(cfg).IndexNode(ctx) +} + +func discoverFacts(hostPath string) map[string]string { + d := discovery.DiscoverVMData(hostPath) + return map[string]string{ + "detected_os": d.GetDetectedOs().String(), + "os_version": d.GetOsVersion(), + "activation_status": d.GetActivationStatus().String(), + "dnf_metadata_status": d.GetDnfMetadataStatus().String(), + } +} + +// selfSignedCert generates a self-signed ECDSA TLS certificate. +// +// This cert exists solely to satisfy TLS protocol requirements: a server MUST +// present a certificate so the key exchange can establish an encrypted channel. +// No party in the connection path validates this cert's identity or expiry — +// virt-handler connects with InsecureSkipVerify: true and no VerifyPeerCertificate +// callback (see kubevirt/kubevirt pkg/virt-handler/rest/console.go, VSOCKHandler). +// Authentication is handled in the opposite direction: the agent verifies +// virt-handler's client cert against the KubeVirt CA via RequireAndVerifyClientCert. +// +// The cert is ephemeral: regenerated on every agent start, never persisted. +func selfSignedCert() (tls.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generating ECDSA key: %w", err) + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generating serial: %w", err) + } + + tmpl := &x509.Certificate{ + SerialNumber: serial, + NotBefore: time.Now().Add(-1 * time.Minute), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, fmt.Errorf("creating certificate: %w", err) + } + + return tls.Certificate{ + Certificate: [][]byte{certDER}, + PrivateKey: key, + }, nil +} diff --git a/compliance/virtualmachines/roxagent/cmd/serve_test.go b/compliance/virtualmachines/roxagent/cmd/serve_test.go new file mode 100644 index 0000000000000..5585594363f96 --- /dev/null +++ b/compliance/virtualmachines/roxagent/cmd/serve_test.go @@ -0,0 +1,471 @@ +package cmd + +import ( + "context" + "crypto/x509" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/stackrox/rox/compliance/virtualmachines/roxagent/vsockserver" + v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4" + pb "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" + "github.com/stackrox/rox/pkg/concurrency" + "github.com/stackrox/rox/pkg/sync" + "github.com/stackrox/rox/pkg/virtualmachine/cpemapping" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// validMappingJSON is the minimal content cpemapping.ValidateMapping accepts. +// otherValidMappingJSON is a distinct valid payload for tests that need two +// different hashes (e.g. deferred-apply across a simulated scan). +const ( + validMappingJSON = `{"data":{}}` + otherValidMappingJSON = `{"data":{"rhel-9-server-rpms":{"cpes":["cpe:/o:redhat:enterprise_linux:9"]}}}` +) + +// waitTimeout/waitTick bound assert.Eventually polling for background +// effects of a real (non-fake-clock) mapping update, e.g. onChange firing. +const ( + waitTimeout = 2 * time.Second + waitTick = 10 * time.Millisecond +) + +// withMappingCachePath points the package-level mappingCachePath at a fresh +// file under a per-test temp dir, so tests never share state through it or +// touch the real hardcoded path, and restores it afterward. +func withMappingCachePath(t *testing.T) string { + t.Helper() + original := mappingCachePath + path := filepath.Join(t.TempDir(), "cache.json") + mappingCachePath = path + t.Cleanup(func() { mappingCachePath = original }) + return path +} + +// registerSensorPersistDrain waits for SensorUpdater's fire-and-forget cache +// writes before t.TempDir() cleanup, so AtomicWriteFile's sibling .tmp files +// cannot race directory removal. +func registerSensorPersistDrain(t *testing.T, updater vsockserver.MappingUpdater) { + t.Helper() + u, ok := updater.(*vsockserver.SensorUpdater) + require.True(t, ok, "registerSensorPersistDrain requires a *SensorUpdater") + t.Cleanup(u.WaitPersist) +} + +func TestNewRescannerAndProvider_URLSelection(t *testing.T) { + withMappingCachePath(t) + + tests := map[string]struct { + repoCPEURL string + wantUpdatePath pb.RepoCPEMappingUpdatePath + wantUpdaterNil bool + wantURLUpdater bool + }{ + "empty repo-cpe-url selects the Sensor-managed path": { + repoCPEURL: "", + wantUpdatePath: pb.RepoCPEMappingUpdatePath_REPO_CPE_MAPPING_UPDATE_PATH_SENSOR, + wantUpdaterNil: false, + wantURLUpdater: false, + }, + "non-empty repo-cpe-url selects the URL-managed path": { + repoCPEURL: "https://example.invalid/repo-to-cpe.json", + wantUpdatePath: pb.RepoCPEMappingUpdatePath_REPO_CPE_MAPPING_UPDATE_PATH_URL, + wantUpdaterNil: true, + wantURLUpdater: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + cfg := serveConfig{repoCPEURL: tt.repoCPEURL} + _, provider, updater, urlUpdater := newRescannerAndProvider(&vsockserver.ReportCache{}, cfg) + + assert.Equal(t, tt.wantUpdatePath, provider.UpdatePath()) + assert.Equal(t, tt.wantUpdaterNil, updater == nil) + assert.Equal(t, tt.wantURLUpdater, urlUpdater != nil) + }) + } +} + +// TestNewRescannerAndProvider_NoMappingStaysIdle covers agent startup with +// no mapping file present yet: the provider must report not-Ready instead +// of treating a missing file as a fetch error, so runServe never attempts +// a scan before a mapping actually arrives. +func TestNewRescannerAndProvider_NoMappingStaysIdle(t *testing.T) { + for name, repoCPEURL := range map[string]string{"Sensor-managed": "", "URL-managed": "https://example.invalid/repo-to-cpe.json"} { + t.Run(name, func(t *testing.T) { + withMappingCachePath(t) + vmRescanner, provider, _, _ := newRescannerAndProvider(&vsockserver.ReportCache{}, serveConfig{repoCPEURL: repoCPEURL}) + + assert.False(t, provider.Ready(), "no cached mapping file must leave the provider not-Ready, not erroring") + + var scanCalls int + vmRescanner.scanFn = func(context.Context, string, string) (*v4.IndexReport, error) { + scanCalls++ + return &v4.IndexReport{}, nil + } + require.NoError(t, vmRescanner.scanOnce(t.Context())) + assert.Zero(t, scanCalls, "a not-Ready provider must never trigger a disk scan") + }) + } +} + +// TestNewRescannerAndProvider_CallbackWiredAfterConstruction pins down the +// construction order: the rescanner's provider field is only assigned +// after the provider itself is built, but OnMappingChanged - already bound +// to the rescanner - must be safe to call throughout, and the assignment +// must have actually taken effect by the time construction returns. +func TestNewRescannerAndProvider_CallbackWiredAfterConstruction(t *testing.T) { + withMappingCachePath(t) + vmRescanner, provider, _, _ := newRescannerAndProvider(&vsockserver.ReportCache{}, serveConfig{}) + + assert.Same(t, provider, vmRescanner.provider, "the rescanner must observe the same provider instance constructed alongside it") + assert.NotPanics(t, vmRescanner.OnMappingChanged) +} + +// TestNewRescannerAndProvider_SyncKicksFirstScan simulates a Sensor-pushed +// mapping arriving (without any real VSOCK connection) and checks it +// triggers an immediate scan attempt via the onChange wiring. +// Runs on the real clock (not synctest): SensorUpdater.Update's fire-and- +// forget cache-persistence goroutine is real background I/O outside the +// rescanner's own fake-clock-driven loop. +func TestNewRescannerAndProvider_SyncKicksFirstScan(t *testing.T) { + cachePath := withMappingCachePath(t) + vmRescanner, provider, updater, _ := newRescannerAndProvider(&vsockserver.ReportCache{}, serveConfig{rescanInterval: time.Hour}) + require.False(t, provider.Ready()) + require.NotNil(t, updater, "Sensor path must produce a non-nil updater") + registerSensorPersistDrain(t, updater) + + var mu sync.Mutex + var calls int + vmRescanner.scanFn = func(context.Context, string, string) (*v4.IndexReport, error) { + concurrency.WithLock(&mu, func() { calls++ }) + return &v4.IndexReport{}, nil + } + vmRescanner.factsFn = func(string) map[string]string { return nil } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := vmRescanner.runAsync(ctx) + defer func() { + cancel() + <-stopped + }() + + updated, err := updater.Update([]byte(validMappingJSON)) + require.NoError(t, err) + require.True(t, updated) + + assert.Eventually(t, func() bool { + return concurrency.WithLock1(&mu, func() int { return calls }) == 1 + }, waitTimeout, waitTick, "a Sync-style mapping update should kick an immediate scan attempt") + + // SensorUpdater.Update persists to disk via a fire-and-forget goroutine + // (see vsockserver.SensorUpdater.persistAndNotify), so the on-disk + // content lands independently of, and possibly after, the scan kick + // asserted above. + assert.Eventually(t, func() bool { + content, err := os.ReadFile(cachePath) + return err == nil && string(content) == validMappingJSON + }, waitTimeout, waitTick, "a synced mapping should also land in the on-disk cache") +} + +// TestNewRescannerAndProvider_BundledPathSeedsSensor covers cmd passing +// --repo-cpe-bundled-path through to NewSensorUpdater: a seed file is +// enough to become Ready and trigger the first scan, without a Sensor push. +func TestNewRescannerAndProvider_BundledPathSeedsSensor(t *testing.T) { + withMappingCachePath(t) + bundledPath := filepath.Join(t.TempDir(), "bundled.json") + require.NoError(t, os.WriteFile(bundledPath, []byte(validMappingJSON), 0o600)) + + vmRescanner, provider, updater, urlUpdater := newRescannerAndProvider(&vsockserver.ReportCache{}, serveConfig{ + repoCPEBundledPath: bundledPath, + rescanInterval: time.Hour, + }) + require.True(t, provider.Ready(), "a Sensor-managed seed file must bootstrap the provider Ready") + require.NotNil(t, updater) + require.Nil(t, urlUpdater) + assert.Equal(t, cpemapping.HashMapping([]byte(validMappingJSON)), provider.Hash()) + + var mu sync.Mutex + var calls int + vmRescanner.scanFn = func(context.Context, string, string) (*v4.IndexReport, error) { + concurrency.WithLock(&mu, func() { calls++ }) + return &v4.IndexReport{}, nil + } + vmRescanner.factsFn = func(string) map[string]string { return nil } + + ctx, cancel := context.WithCancel(t.Context()) + stopped := vmRescanner.runAsync(ctx) + defer func() { + cancel() + <-stopped + }() + + assert.Eventually(t, func() bool { + return concurrency.WithLock1(&mu, func() int { return calls }) == 1 + }, waitTimeout, waitTick, "bootstrapping from the bundled seed should kick an immediate scan attempt") +} + +// TestNewRescannerAndProvider_URLIgnoresBundledPath covers URL-managed +// plus --repo-cpe-bundled-path: the seed is Sensor-only, so a URL that +// has never succeeded stays not-ready instead of looking configured. +func TestNewRescannerAndProvider_URLIgnoresBundledPath(t *testing.T) { + withMappingCachePath(t) + bundledPath := filepath.Join(t.TempDir(), "bundled.json") + require.NoError(t, os.WriteFile(bundledPath, []byte(validMappingJSON), 0o600)) + + _, provider, updater, urlUpdater := newRescannerAndProvider(&vsockserver.ReportCache{}, serveConfig{ + repoCPEURL: "https://example.invalid/repo-to-cpe.json", + repoCPEBundledPath: bundledPath, + }) + require.NotNil(t, urlUpdater) + require.Nil(t, updater) + assert.False(t, provider.Ready(), "URL-managed must not seed from --repo-cpe-bundled-path") + assert.Equal(t, pb.RepoCPEMappingUpdatePath_REPO_CPE_MAPPING_UPDATE_PATH_URL, provider.UpdatePath()) +} + +// TestNewRescannerAndProvider_URLNeverSucceeds_StaysNotSensorManaged covers +// a configured --repo-cpe-url that never produces a usable mapping: the +// update path must stay URL, with no fallback to Sensor, even once the +// downloader has actually attempted and failed a fetch. Log-level (ERROR +// vs WARN) is not asserted here: the codebase has no lightweight log- +// capture convention (see report), and URLUpdater.onDownloadComplete only +// ever logs WARN today regardless of whether a mapping was ever +// successfully fetched — see the task report for this gap. +func TestNewRescannerAndProvider_URLNeverSucceeds_StaysNotSensorManaged(t *testing.T) { + withMappingCachePath(t) + + var requests atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + _, provider, updater, urlUpdater := newRescannerAndProvider(&vsockserver.ReportCache{}, serveConfig{repoCPEURL: srv.URL}) + require.Nil(t, updater) + require.NotNil(t, urlUpdater) + + ctx, cancel := context.WithCancel(t.Context()) + require.NoError(t, urlUpdater.Start(ctx)) + defer func() { + cancel() + urlUpdater.Stop() + }() + + assert.Eventually(t, func() bool { return requests.Load() > 0 }, waitTimeout, waitTick, + "the downloader should have attempted at least one fetch against the configured URL") + + assert.False(t, provider.Ready(), "a URL that never succeeds must never make the provider Ready") + assert.Equal(t, pb.RepoCPEMappingUpdatePath_REPO_CPE_MAPPING_UPDATE_PATH_URL, provider.UpdatePath(), + "a failing URL must not fall back to Sensor management") +} + +// TestNewRescannerAndProvider_RestartSwitchesUpdatePath simulates an +// operator adding or removing --repo-cpe-url between two agent restarts +// that share the same on-disk mappingCachePath. Per the design, the +// update path is re-derived purely from cfg.repoCPEURL on every restart, +// never persisted; this table only pins that re-derivation, not whether +// the previous updater's cached bytes get re-read (see report: cmd wires +// both updaters to the same mappingCachePath file, so unlike the design's +// "own cache file" wording, content bootstrapped by one updater is in +// fact visible to the other after a switch). +func TestNewRescannerAndProvider_RestartSwitchesUpdatePath(t *testing.T) { + tests := map[string]struct { + firstURL string + secondURL string + wantFirst pb.RepoCPEMappingUpdatePath + wantAfter pb.RepoCPEMappingUpdatePath + }{ + "adding --repo-cpe-url switches Sensor-managed to URL-managed": { + firstURL: "", + secondURL: "https://example.invalid/repo-to-cpe.json", + wantFirst: pb.RepoCPEMappingUpdatePath_REPO_CPE_MAPPING_UPDATE_PATH_SENSOR, + wantAfter: pb.RepoCPEMappingUpdatePath_REPO_CPE_MAPPING_UPDATE_PATH_URL, + }, + "removing --repo-cpe-url switches URL-managed to Sensor-managed": { + firstURL: "https://example.invalid/repo-to-cpe.json", + secondURL: "", + wantFirst: pb.RepoCPEMappingUpdatePath_REPO_CPE_MAPPING_UPDATE_PATH_URL, + wantAfter: pb.RepoCPEMappingUpdatePath_REPO_CPE_MAPPING_UPDATE_PATH_SENSOR, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + withMappingCachePath(t) + + _, firstProvider, _, _ := newRescannerAndProvider(&vsockserver.ReportCache{}, serveConfig{repoCPEURL: tt.firstURL}) + require.Equal(t, tt.wantFirst, firstProvider.UpdatePath()) + + _, secondProvider, _, _ := newRescannerAndProvider(&vsockserver.ReportCache{}, serveConfig{repoCPEURL: tt.secondURL}) + assert.Equal(t, tt.wantAfter, secondProvider.UpdatePath(), + "a fresh restart must re-derive the update path from the current --repo-cpe-url, not remember the prior one") + }) + } +} + +// TestNewRescannerAndProvider_SensorSync_DefersUnderScanThenApplies covers +// the cmd-level wiring for deferred apply: SensorUpdater's own deferred- +// apply logic is unit-tested in vsockserver/sensor_updater_test.go, so this +// only checks that newRescannerAndProvider's returned updater/provider +// pair exposes that behavior end-to-end through the ScanBusyGate interface. +func TestNewRescannerAndProvider_SensorSync_DefersUnderScanThenApplies(t *testing.T) { + cachePath := withMappingCachePath(t) + require.NoError(t, os.WriteFile(cachePath, []byte(validMappingJSON), 0o600)) + + _, provider, updater, _ := newRescannerAndProvider(&vsockserver.ReportCache{}, serveConfig{}) + require.True(t, provider.Ready(), "a pre-seeded Sensor cache must bootstrap the provider Ready") + require.Equal(t, cpemapping.HashMapping([]byte(validMappingJSON)), provider.Hash()) + registerSensorPersistDrain(t, updater) + + gate, ok := updater.(vsockserver.ScanBusyGate) + require.True(t, ok, "the Sensor updater must implement ScanBusyGate") + gate.MarkScanBusy() + + updated, err := updater.Update([]byte(otherValidMappingJSON)) + require.NoError(t, err) + require.True(t, updated) + + assert.Equal(t, cpemapping.HashMapping([]byte(validMappingJSON)), provider.Hash(), + "the active mapping must stay the bootstrapped content while a scan is in flight") + + gate.MarkScanIdleAndApplyPending() + + assert.Equal(t, cpemapping.HashMapping([]byte(otherValidMappingJSON)), provider.Hash(), + "the pending mapping should apply once the scan is marked idle") + + // MarkScanIdleAndApplyPending's persist is also fire-and-forget; wait + // for it so the background write can't still be touching cachePath's + // directory when t.TempDir() cleans it up. + assert.Eventually(t, func() bool { + content, err := os.ReadFile(cachePath) + return err == nil && string(content) == otherValidMappingJSON + }, waitTimeout, waitTick, "the applied mapping should also land in the on-disk cache") +} + +// TestNewRescannerAndProvider_SensorUpdate_InvalidContentKeepsLastGood +// covers the cmd-level wiring only; SensorUpdater's own validation +// behavior is unit-tested in vsockserver/sensor_updater_test.go. +func TestNewRescannerAndProvider_SensorUpdate_InvalidContentKeepsLastGood(t *testing.T) { + cachePath := withMappingCachePath(t) + _, provider, updater, _ := newRescannerAndProvider(&vsockserver.ReportCache{}, serveConfig{}) + registerSensorPersistDrain(t, updater) + + updated, err := updater.Update([]byte(validMappingJSON)) + require.NoError(t, err) + require.True(t, updated) + wantHash := provider.Hash() + // Drain the first Update's fire-and-forget persist before the test ends, + // so its background write can't still be touching cachePath's directory + // when t.TempDir() cleans it up. + assert.Eventually(t, func() bool { + content, err := os.ReadFile(cachePath) + return err == nil && string(content) == validMappingJSON + }, waitTimeout, waitTick, "the initial valid mapping should land in the on-disk cache") + + updated, err = updater.Update([]byte(`{not valid json`)) + assert.Error(t, err) + assert.False(t, updated) + assert.Equal(t, wantHash, provider.Hash(), "invalid content must not replace the last-known-good mapping") +} + +// validServeConfig returns a serveConfig that passes validate(), so each +// test case in TestRunServe_ValidatesFlags only needs to override the one +// field it's exercising. +func validServeConfig() serveConfig { + return serveConfig{ + rescanInterval: minRescanInterval, + caFetchTimeout: time.Second, + connDeadline: minConnDeadline, + } +} + +// TestRunServe_ValidatesFlags exercises runServe's argument validation, +// which - unlike the rest of runServe - never touches the filesystem, +// network, or VSOCK, so it is cheap to cover without a real host/agent +// environment. port/hostPath/repoCPEURL are irrelevant to these cases, +// since validation returns before any of them are used. +func TestRunServe_ValidatesFlags(t *testing.T) { + tests := map[string]struct { + mutate func(*serveConfig) + errContains string + }{ + "should error when rescan interval is zero": { + mutate: func(c *serveConfig) { c.rescanInterval = 0 }, + errContains: "rescan-interval", + }, + "should error when rescan interval is negative": { + mutate: func(c *serveConfig) { c.rescanInterval = -time.Second }, + errContains: "rescan-interval", + }, + "should error when rescan interval is below the minimum": { + mutate: func(c *serveConfig) { c.rescanInterval = minRescanInterval - time.Second }, + errContains: "rescan-interval", + }, + "should error when rescan interval is above the maximum": { + mutate: func(c *serveConfig) { c.rescanInterval = maxRescanInterval + time.Hour }, + errContains: "rescan-interval", + }, + "should error when ca fetch timeout is zero": { + mutate: func(c *serveConfig) { c.caFetchTimeout = 0 }, + errContains: "ca-fetch-timeout", + }, + "should error when ca fetch timeout is negative": { + mutate: func(c *serveConfig) { c.caFetchTimeout = -time.Second }, + errContains: "ca-fetch-timeout", + }, + "should error when conn deadline is below the minimum": { + mutate: func(c *serveConfig) { c.connDeadline = minConnDeadline - time.Second }, + errContains: "conn-deadline", + }, + "should error when conn deadline is above the maximum": { + mutate: func(c *serveConfig) { c.connDeadline = maxConnDeadline + time.Second }, + errContains: "conn-deadline", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + cfg := validServeConfig() + tt.mutate(&cfg) + err := runServe(t.Context(), cfg) + assert.ErrorContains(t, err, tt.errContains) + }) + } +} + +func TestDiscoverFacts(t *testing.T) { + facts := discoverFacts(t.TempDir()) + + assert.Contains(t, facts, "detected_os") + assert.Contains(t, facts, "os_version") + assert.Contains(t, facts, "activation_status") + assert.Contains(t, facts, "dnf_metadata_status") +} + +func TestSelfSignedCert(t *testing.T) { + cert, err := selfSignedCert() + require.NoError(t, err) + require.Len(t, cert.Certificate, 1) + require.NotNil(t, cert.PrivateKey) + + parsed, err := x509.ParseCertificate(cert.Certificate[0]) + require.NoError(t, err) + assert.Contains(t, parsed.ExtKeyUsage, x509.ExtKeyUsageServerAuth) + assert.True(t, parsed.NotBefore.Before(time.Now()), "cert should already be valid") + assert.True(t, parsed.NotAfter.After(time.Now()), "cert should not be expired") + + // Self-signed: issuer and subject are the same key, so it must verify + // against a pool containing only itself. + pool := x509.NewCertPool() + pool.AddCert(parsed) + _, err = parsed.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}) + assert.NoError(t, err) +} diff --git a/compliance/virtualmachines/roxagent/common/config.go b/compliance/virtualmachines/roxagent/common/config.go deleted file mode 100644 index 969a328897adf..0000000000000 --- a/compliance/virtualmachines/roxagent/common/config.go +++ /dev/null @@ -1,13 +0,0 @@ -package common - -import "time" - -type Config struct { - DaemonMode bool - IndexHostPath string - IndexInterval time.Duration - RepoToCPEMappingURL string - Timeout time.Duration - Verbose bool - VsockPort uint32 -} diff --git a/compliance/virtualmachines/roxagent/discovery/discovery.go b/compliance/virtualmachines/roxagent/discovery/discovery.go new file mode 100644 index 0000000000000..8c32d34bd0501 --- /dev/null +++ b/compliance/virtualmachines/roxagent/discovery/discovery.go @@ -0,0 +1,329 @@ +package discovery + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/hashicorp/go-multierror" + v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" + "github.com/stackrox/rox/pkg/logging" + "github.com/stackrox/rox/pkg/set" +) + +var log = logging.LoggerForModule() + +const ( + osReleasePath = "/etc/os-release" + entitlementDirPath = "/etc/pki/entitlement" + dnfCacheDirPath = "/var/cache/dnf" + entitlementKeySuffix = "-key.pem" + entitlementCertSuffix = ".pem" + // osReleaseIDKey is the key name for the ID field in /etc/os-release. + osReleaseIDKey = "ID" + // osReleaseVersionIDKey is the key name for the VERSION_ID field in /etc/os-release. + osReleaseVersionIDKey = "VERSION_ID" + // rhelOSID is the value of the ID field in /etc/os-release for Red Hat Enterprise Linux. + rhelOSID = "rhel" +) + +var defaultReposDirs = []string{ + "/etc/yum.repos.d", + "/etc/yum/repos.d", + "/etc/distro.repos.d", +} + +// DiscoverVMData discovers VM metadata from the host system. +// Returns discovered data with defaults (UNKNOWN/UNSPECIFIED) if discovery fails. +func DiscoverVMData(hostPath string) *v1.DiscoveredData { + result := &v1.DiscoveredData{} + + // Discover OS and version from /etc/os-release. + // Currently assumes RHEL systems: reads /etc/os-release, checks if ID field equals "rhel" to detect RHEL, + // and extracts VERSION_ID field as the OS version. Falls back to UNKNOWN for non-RHEL systems. + // This behavior is based on assumptions about /etc/os-release format and RHEL-specific values. + // Future improvements may include support for other OS types and more robust version detection. + detectedOS, osVersion, err := discoverOSAndVersionWithPath(hostPathFor(hostPath, osReleasePath)) + if err != nil { + log.Infof("Unable to discover OS and version: %v", err) + } else { + result.DetectedOs = detectedOS + result.OsVersion = osVersion + } + + // Discover activation status from /etc/pki/entitlement. + // Currently assumes RHEL entitlement certificates: checks for matching cert/key pairs by filename + // (e.g., "123-key.pem" and "123.pem"). System is considered ACTIVATED if at least one matching pair exists, + // otherwise INACTIVE. This behavior is based on assumptions about RHEL entitlement certificate naming + // conventions and file structure. Future improvements may include actual certificate validation and + // support for other activation mechanisms. + activationStatus, err := discoverActivationStatusWithPath(hostPathFor(hostPath, entitlementDirPath)) + if err != nil { + log.Infof("Observations during discovering activation status: %v", err) + } + // Some errors are of a warning nature, so we still set the discovery result. + result.ActivationStatus = activationStatus + + // Discover DNF metadata status. + dnfStatus, err := discoverDnfMetadataStatusWithPaths( + hostPath, + defaultReposDirs, + dnfCacheDirPath, + ) + if err != nil { + log.Infof("Observations during discovering DNF metadata status: %v", err) + } + // Some errors are of a warning nature, so we still set the discovery result. + result.DnfMetadataStatus = dnfStatus + + return result +} + +func hostPathFor(hostPath, path string) string { + if hostPath == "" || hostPath == string(os.PathSeparator) { + return path + } + // This join+clean approach is safe (no escape from hostPath) only when + // the input path is absolute (e.g., "/etc/os-release"). For example, + // hostPath="/host", path="/../etc/os-release" would clean to "/etc/os-release". + trimmedPath := strings.TrimPrefix(path, string(os.PathSeparator)) + return filepath.Clean(filepath.Join(hostPath, trimmedPath)) +} + +// discoverOSAndVersionWithPath reads os-release from the given path and returns DetectedOS and OSVersion. +func discoverOSAndVersionWithPath(path string) (v1.DetectedOS, string, error) { + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return v1.DetectedOS_UNKNOWN, "", fmt.Errorf("unsupported OS detected: missing %s: %w", path, err) + } + return v1.DetectedOS_UNKNOWN, "", fmt.Errorf("opening %s: %w", path, err) + } + defer func() { + if err := file.Close(); err != nil { + log.Warnf("Failed to close %s: %v", path, err) + } + }() + + osRelease, err := parseOSRelease(file) + if err != nil { + return v1.DetectedOS_UNKNOWN, "", fmt.Errorf("reading %s: %w", path, err) + } + + // Determine DetectedOS based on ID field + var detectedOS v1.DetectedOS + if id, ok := osRelease[osReleaseIDKey]; ok && strings.TrimSpace(id) == rhelOSID { + detectedOS = v1.DetectedOS_RHEL + } else { + detectedOS = v1.DetectedOS_UNKNOWN + } + + // Get OS version from VERSION_ID + var osVersion string + if versionID, ok := osRelease[osReleaseVersionIDKey]; ok { + if detectedOS != v1.DetectedOS_UNKNOWN { + return detectedOS, strings.TrimSpace(versionID), nil + } + // For non-RHEL systems, store the name of the OS (ID) and version (VERSION_ID) together. + // The version field is only informative and used for debugging in case of problems with scanning; + // we want to know which OS and version caused a potential issue. + osID := strings.TrimSpace(osRelease[osReleaseIDKey]) + if osID == "" { + osID = "unknown-OS" + } + osVersion = fmt.Sprintf("%s %s", osID, versionID) + } + + return detectedOS, osVersion, nil +} + +// parseOSRelease parses /etc/os-release key-value pairs. +// +// We copy ClairCore's os-release parser instead of importing it to avoid pulling +// in heavy scanner/indexer dependencies into roxagent. As Rob Pike put it, +// "a little bit of copying is better than a little bit of dependency." +// +// Source (copied, adapted to our usage): +// https://github.com/quay/claircore/blob/9f69181a1555935c8840a9191c91567e55b9cf0c/osrelease/scanner.go +func parseOSRelease(r io.Reader) (map[string]string, error) { + osRelease := make(map[string]string) + scanner := bufio.NewScanner(r) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + b := bytes.TrimSpace(scanner.Bytes()) + switch { + case len(b) == 0: + continue + case b[0] == '#': + continue + } + if !bytes.ContainsRune(b, '=') { + return nil, fmt.Errorf("osrelease: malformed line %q", scanner.Text()) + } + key, value, _ := strings.Cut(string(b), "=") + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + switch { + case len(value) == 0: + case value[0] == '\'': + value = strings.TrimFunc(value, func(r rune) bool { return r == '\'' }) + value = strings.ReplaceAll(value, `'\''`, `'`) + case value[0] == '"': + value = strings.TrimFunc(value, func(r rune) bool { return r == '"' }) + value = osReleaseDQReplacer.Replace(value) + default: + } + osRelease[key] = value + } + if err := scanner.Err(); err != nil { + return nil, err + } + return osRelease, nil +} + +var osReleaseDQReplacer = strings.NewReplacer( + "\\`", "`", + `\\`, `\`, + `\"`, `"`, + `\$`, `$`, +) + +// discoverActivationStatusWithPath checks the given path for matching cert/key pairs. +func discoverActivationStatusWithPath(path string) (v1.ActivationStatus, error) { + entries, err := os.ReadDir(path) + if err != nil { + return v1.ActivationStatus_ACTIVATION_UNSPECIFIED, fmt.Errorf("reading %s: %w", path, err) + } + + // Build sets of base names (without suffix) for keys and certs + keyBases := set.NewStringSet() + certBases := set.NewStringSet() + + // The `entries` are already sorted by name, so optimistically we just need to check two files. + // We can stop when first matching pair is found. + for _, entry := range entries { + if entry.IsDir() { + continue + } + + name := entry.Name() + if before, ok := strings.CutSuffix(name, entitlementKeySuffix); ok { + base := before + keyBases.Add(base) + if certBases.Contains(base) { + return v1.ActivationStatus_ACTIVE, nil + } + } else if before, ok := strings.CutSuffix(name, entitlementCertSuffix); ok { + base := before + certBases.Add(base) + if keyBases.Contains(base) { + return v1.ActivationStatus_ACTIVE, nil + } + } + } + + return v1.ActivationStatus_INACTIVE, nil +} + +// discoverDnfMetadataStatusWithPaths checks both repos and cache directories. +// Currently assumes RHEL DNF setup: checks for both (1) at least one *.repo file in a default reposdir +// (/etc/yum.repos.d, /etc/yum/repos.d, /etc/distro.repos.d) and (2) at least one directory in +// /var/cache/dnf containing "-rpms-" in its name. Metadata is considered AVAILABLE only if both +// conditions are met. +func discoverDnfMetadataStatusWithPaths(hostPath string, reposDirPaths []string, cacheDirPath string) (v1.DnfMetadataStatus, error) { + hasRepoDir, hasRepoFile, err := discoverDnfRepoFilePresence(hostPath, reposDirPaths) + if !hasRepoDir { + return v1.DnfMetadataStatus_DNF_METADATA_UNSPECIFIED, err + } + if !hasRepoFile { + return v1.DnfMetadataStatus_UNAVAILABLE, err + } + + hasCacheRepoDir, err := discoverDnfCacheRepoDirPresence(hostPath, cacheDirPath) + if err != nil { + return v1.DnfMetadataStatus_DNF_METADATA_UNSPECIFIED, err + } + + if hasCacheRepoDir { + return v1.DnfMetadataStatus_AVAILABLE, nil + } + return v1.DnfMetadataStatus_UNAVAILABLE, nil +} + +// discoverDnfRepoFilePresence reports whether any default reposdir contains a *.repo file. +// Assumptions and behavior: +// - Uses the default DNF reposdir locations: /etc/yum.repos.d, /etc/yum/repos.d, /etc/distro.repos.d. +// - Returns hasDir=true when at least one reposdir is readable. +// - Returns hasRepo=true as soon as a *.repo file is found in any reposdir. +// - If reposdirs are unreadable, or contain no *.repo files, returns an aggregated error describing each failure. +// - There is no support for DNF 5 defaults currently. +// - Tested against DNF 4 defaults (libdnf ConfigMain.cpp): +// https://github.com/rpm-software-management/libdnf/blob/53839f5bd88f378e57a1f1671b3db48d29984e24/libdnf/conf/ConfigMain.cpp +// - DNF 5 uses a different reposdir list (/etc/yum.repos.d, /etc/distro.repos.d, /usr/share/dnf5/repos.d), +// so this logic may miss repos configured only in the DNF 5 default path: +// https://github.com/rpm-software-management/dnf5/blob/185eaef1e0ad663bdb827a2179ab1df574a27d88/include/libdnf5/conf/const.hpp +func discoverDnfRepoFilePresence(hostPath string, reposDirPaths []string) (hasDir, hasRepo bool, err error) { + if len(reposDirPaths) == 0 { + return false, false, errors.New("missing repository directories") + } + + // Check for repo files in default reposdir locations. + var repoDirErrs *multierror.Error + for _, reposDirPath := range reposDirPaths { + reposPath := hostPathFor(hostPath, reposDirPath) + repoEntries, err := os.ReadDir(reposPath) + if err != nil { + repoDirErrs = multierror.Append(repoDirErrs, fmt.Errorf("reading %q: %w", reposPath, err)) + continue + } + // If at least one directory exists and is readable, we don't need to return DNF_METADATA_UNSPECIFIED. + hasDir = true + + for _, entry := range repoEntries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".repo") { + return true, true, nil + } + } + repoDirErrs = multierror.Append(repoDirErrs, fmt.Errorf("no .repo files found in %q", reposPath)) + + } + + return hasDir, hasRepo, repoDirErrs.ErrorOrNil() +} + +// discoverDnfCacheRepoDirPresence reports whether the DNF cache contains repo directories. +// Assumptions and behavior: +// - Uses the default system cachedir at /var/cache/dnf (libdnf Const.hpp): +// https://github.com/rpm-software-management/libdnf/blob/53839f5bd88f378e57a1f1671b3db48d29984e24/libdnf/conf/Const.hpp +// - Treats any subdirectory containing "-rpms-" as a repo cache directory. +// - Returns true as soon as a matching directory is found. +// - If the cache directory is missing, returns an "unsupported OS detected" error. +// - If the cache directory exists but cannot be read, returns a read error. +// - There is no support for DNF 5 defaults currently. +// - Tested against DNF 4 defaults; DNF 5 uses /var/cache/libdnf5: +// https://github.com/rpm-software-management/dnf5/blob/185eaef1e0ad663bdb827a2179ab1df574a27d88/include/libdnf5/conf/const.hpp +func discoverDnfCacheRepoDirPresence(hostPath, cacheDirPath string) (bool, error) { + cachePath := hostPathFor(hostPath, cacheDirPath) + cacheEntries, err := os.ReadDir(cachePath) + if err != nil { + if os.IsNotExist(err) { + return false, fmt.Errorf("unsupported OS detected: missing %s: %w", cachePath, err) + } + return false, fmt.Errorf("reading %s: %w", cachePath, err) + } + + for _, entry := range cacheEntries { + if entry.IsDir() { + // Check if it looks like a repo directory (contains "-rpms-" pattern) + if strings.Contains(entry.Name(), "-rpms-") { + return true, nil + } + } + } + return false, nil +} diff --git a/compliance/virtualmachines/roxagent/discovery/discovery_test.go b/compliance/virtualmachines/roxagent/discovery/discovery_test.go new file mode 100644 index 0000000000000..1b7e02855939e --- /dev/null +++ b/compliance/virtualmachines/roxagent/discovery/discovery_test.go @@ -0,0 +1,559 @@ +package discovery + +import ( + "os" + "path/filepath" + "strings" + "testing" + + v1 "github.com/stackrox/rox/generated/internalapi/virtualmachine/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiscoverOSAndVersion(t *testing.T) { + tests := []struct { + name string + osRelease string + expectedOS v1.DetectedOS + expectedVersion string + }{ + { + name: "RHEL 8.10", + osRelease: `NAME="Red Hat Enterprise Linux" +VERSION="8.10 (Ootpa)" +ID="rhel" +ID_LIKE="fedora" +VERSION_ID="8.10" +PLATFORM_ID="platform:el8" +PRETTY_NAME="Red Hat Enterprise Linux 8.10 (Ootpa)" +ANSI_COLOR="0;31" +CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" +HOME_URL="https://www.redhat.com/" +DOCUMENTATION_URL="https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8" +BUG_REPORT_URL="https://issues.redhat.com/" + +REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" +REDHAT_BUGZILLA_PRODUCT_VERSION=8.10 +REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" +REDHAT_SUPPORT_PRODUCT_VERSION="8.10"`, + expectedOS: v1.DetectedOS_RHEL, + expectedVersion: "8.10", + }, + { + name: "RHEL 9", + osRelease: `ID="rhel" +VERSION_ID="9.2"`, + expectedOS: v1.DetectedOS_RHEL, + expectedVersion: "9.2", + }, + { + name: "Non-RHEL OS", + osRelease: `ID="debian" +VERSION_ID="12"`, + expectedOS: v1.DetectedOS_UNKNOWN, + expectedVersion: "debian 12", + }, + { + name: "ID field missing but VERSION_ID is present", + osRelease: `VERSION_ID="12"`, + expectedOS: v1.DetectedOS_UNKNOWN, + expectedVersion: "unknown-OS 12", + }, + { + name: "ID and VERSION_ID fields missing", + osRelease: ``, + expectedOS: v1.DetectedOS_UNKNOWN, + expectedVersion: "", + }, + { + name: "Unknown OS with version only", + osRelease: `VERSION_ID="10"`, + expectedOS: v1.DetectedOS_UNKNOWN, + expectedVersion: "unknown-OS 10", + }, + { + name: "Missing VERSION_ID", + osRelease: `ID="rhel"`, + expectedOS: v1.DetectedOS_RHEL, + expectedVersion: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + testOsReleasePath := filepath.Join(tmpDir, "os-release") + + err := os.WriteFile(testOsReleasePath, []byte(tt.osRelease), 0644) + require.NoError(t, err) + + // Use a testable version that accepts a path + detectedOS, osVersion, err := discoverOSAndVersionWithPath(testOsReleasePath) + require.NoError(t, err) + assert.Equal(t, tt.expectedOS, detectedOS) + assert.Equal(t, tt.expectedVersion, osVersion) + }) + } +} + +func TestDiscoverOSAndVersion_MissingFile(t *testing.T) { + tmpDir := t.TempDir() + missingPath := filepath.Join(tmpDir, "nonexistent") + + detectedOS, osVersion, err := discoverOSAndVersionWithPath(missingPath) + assert.Error(t, err) + assert.Equal(t, v1.DetectedOS_UNKNOWN, detectedOS) + assert.Equal(t, "", osVersion) +} + +func TestDiscoverOSAndVersion_MalformedOSRelease(t *testing.T) { + tmpDir := t.TempDir() + testOsReleasePath := filepath.Join(tmpDir, "os-release") + + err := os.WriteFile(testOsReleasePath, []byte("ID=rhel\nMALFORMED_LINE"), 0644) + require.NoError(t, err) + + detectedOS, osVersion, err := discoverOSAndVersionWithPath(testOsReleasePath) + assert.Error(t, err) + assert.Equal(t, v1.DetectedOS_UNKNOWN, detectedOS) + assert.Equal(t, "", osVersion) +} + +func TestDiscoverDnfRepoFilePresence(t *testing.T) { + tests := map[string]struct { + setup func(t *testing.T) (string, []string) + expectedHasDir bool + expectedHasRepo bool + expectedErrParts []string + }{ + "should return true when repo file exists in reposdir": { + setup: func(t *testing.T) (string, []string) { + hostPath := t.TempDir() + reposDirPath := "/etc/yum.repos.d" + reposPath := hostPathFor(hostPath, reposDirPath) + require.NoError(t, os.MkdirAll(reposPath, 0750)) + repoFilePath := filepath.Join(reposPath, "test.repo") + require.NoError(t, os.WriteFile(repoFilePath, []byte("content"), 0600)) + return hostPath, []string{reposDirPath} + }, + expectedHasDir: true, + expectedHasRepo: true, + }, + "should return error when all reposdirs are unreadable": { + setup: func(t *testing.T) (string, []string) { + hostPath := t.TempDir() + return hostPath, []string{"/etc/yum.repos.d", "/etc/yum/repos.d"} + }, + expectedHasDir: false, + expectedHasRepo: false, + expectedErrParts: []string{"reading", "no such file or directory"}, + }, + "should return error when reposdirs are readable but no repo files exist": { + setup: func(t *testing.T) (string, []string) { + hostPath := t.TempDir() + reposDirPaths := []string{"/etc/yum.repos.d", "/etc/yum/repos.d"} + for _, reposDirPath := range reposDirPaths { + reposPath := hostPathFor(hostPath, reposDirPath) + require.NoError(t, os.MkdirAll(reposPath, 0750)) + require.NoError(t, os.WriteFile(filepath.Join(reposPath, "not-a-repo.txt"), []byte("content"), 0600)) + } + return hostPath, reposDirPaths + }, + expectedHasDir: true, + expectedHasRepo: false, + expectedErrParts: []string{"no .repo files found"}, + }, + "should return true when repo file exists even if other reposdir is missing": { + setup: func(t *testing.T) (string, []string) { + hostPath := t.TempDir() + reposDirPaths := []string{"/etc/yum.repos.d", "/etc/yum/repos.d"} + reposPath := hostPathFor(hostPath, reposDirPaths[0]) + require.NoError(t, os.MkdirAll(reposPath, 0750)) + require.NoError(t, os.WriteFile(filepath.Join(reposPath, "example.repo"), []byte("content"), 0600)) + return hostPath, reposDirPaths + }, + expectedHasDir: true, + expectedHasRepo: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + hostPath, reposDirPaths := tt.setup(t) + hasDir, hasRepo, err := discoverDnfRepoFilePresence(hostPath, reposDirPaths) + assert.Equal(t, tt.expectedHasDir, hasDir) + assert.Equal(t, tt.expectedHasRepo, hasRepo) + if len(tt.expectedErrParts) == 0 { + assert.NoError(t, err) + return + } + require.Error(t, err) + for _, part := range tt.expectedErrParts { + assert.ErrorContains(t, err, part) + } + }) + } +} + +func TestDiscoverDnfCacheRepoDirPresence(t *testing.T) { + tests := map[string]struct { + setup func(t *testing.T) (string, string) + expectedFound bool + expectedErrParts []string + }{ + "should return true when cache dir contains repo directory": { + setup: func(t *testing.T) (string, string) { + hostPath := t.TempDir() + cacheDirPath := "/var/cache/dnf" + cachePath := hostPathFor(hostPath, cacheDirPath) + require.NoError(t, os.MkdirAll(filepath.Join(cachePath, "rhel-9-for-x86_64-appstream-rpms-123"), 0750)) + return hostPath, cacheDirPath + }, + expectedFound: true, + }, + "should return false when cache dir has no repo directories": { + setup: func(t *testing.T) (string, string) { + hostPath := t.TempDir() + cacheDirPath := "/var/cache/dnf" + cachePath := hostPathFor(hostPath, cacheDirPath) + require.NoError(t, os.MkdirAll(filepath.Join(cachePath, "some-other-dir"), 0750)) + return hostPath, cacheDirPath + }, + expectedFound: false, + }, + "should return error when cache dir is missing": { + setup: func(t *testing.T) (string, string) { + hostPath := t.TempDir() + return hostPath, "/var/cache/dnf" + }, + expectedFound: false, + expectedErrParts: []string{"unsupported OS detected: missing"}, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + hostPath, cacheDirPath := tt.setup(t) + found, err := discoverDnfCacheRepoDirPresence(hostPath, cacheDirPath) + assert.Equal(t, tt.expectedFound, found) + if len(tt.expectedErrParts) == 0 { + assert.NoError(t, err) + return + } + require.Error(t, err) + for _, part := range tt.expectedErrParts { + assert.ErrorContains(t, err, part) + } + }) + } +} + +func TestParseOSRelease_QuotedValues(t *testing.T) { + input := strings.NewReader(`# comment +ID='rhel' +VERSION_ID="9.4" +NAME="Red Hat \$NAME" +`) + + values, err := parseOSRelease(input) + require.NoError(t, err) + assert.Equal(t, "rhel", values["ID"]) + assert.Equal(t, "9.4", values["VERSION_ID"]) + assert.Equal(t, "Red Hat $NAME", values["NAME"]) +} + +func TestHostPathFor(t *testing.T) { + tests := []struct { + name string + hostPath string + path string + expected string + }{ + { + name: "Empty host path uses original path", + hostPath: "", + path: "/etc/os-release", + expected: "/etc/os-release", + }, + { + name: "Root host path uses original path", + hostPath: "/", + path: "/etc/os-release", + expected: "/etc/os-release", + }, + { + name: "Prefix host path joins with absolute path", + hostPath: "/host", + path: "/etc/os-release", + expected: "/host/etc/os-release", + }, + { + name: "Prefix host path joins with nested path", + hostPath: "/host/rootfs", + path: "/var/cache/dnf", + expected: "/host/rootfs/var/cache/dnf", + }, + { + name: "Cleaned path removes dot segments", + hostPath: "/root/../host", + path: "/var/lib/../cache//dnf/", + expected: "/host/var/cache/dnf", + }, + { + name: "Cleaned path collapses extra slashes", + hostPath: "/host//", + path: "/etc/os-release", + expected: "/host/etc/os-release", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, hostPathFor(tt.hostPath, tt.path)) + }) + } +} + +func TestDiscoverActivationStatus(t *testing.T) { + tests := []struct { + name string + files []string + expectedStatus v1.ActivationStatus + }{ + { + name: "Activated - single pair", + files: []string{ + "3341241341658386286-key.pem", + "3341241341658386286.pem", + }, + expectedStatus: v1.ActivationStatus_ACTIVE, + }, + { + name: "Activated - multiple pairs", + files: []string{ + "111-key.pem", + "111.pem", + "222-key.pem", + "222.pem", + }, + expectedStatus: v1.ActivationStatus_ACTIVE, + }, + { + name: "Unactivated - empty directory", + files: []string{}, + expectedStatus: v1.ActivationStatus_INACTIVE, + }, + { + name: "Unactivated - only key file", + files: []string{ + "3341241341658386286-key.pem", + }, + expectedStatus: v1.ActivationStatus_INACTIVE, + }, + { + name: "Unactivated - only cert file", + files: []string{ + "3341241341658386286.pem", + }, + expectedStatus: v1.ActivationStatus_INACTIVE, + }, + { + name: "Unactivated - mismatched names", + files: []string{ + "111-key.pem", + "222.pem", + }, + expectedStatus: v1.ActivationStatus_INACTIVE, + }, + { + name: "Unactivated - other files", + files: []string{ + "some-other-file.txt", + }, + expectedStatus: v1.ActivationStatus_INACTIVE, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + + for _, filename := range tt.files { + filePath := filepath.Join(tmpDir, filename) + err := os.WriteFile(filePath, []byte("test content"), 0644) + require.NoError(t, err) + } + + activationStatus, err := discoverActivationStatusWithPath(tmpDir) + assert.NoError(t, err) + assert.Equal(t, tt.expectedStatus, activationStatus) + }) + } +} + +func TestDiscoverActivationStatus_MissingDir(t *testing.T) { + tmpDir := t.TempDir() + missingPath := filepath.Join(tmpDir, "nonexistent") + + activationStatus, err := discoverActivationStatusWithPath(missingPath) + assert.Error(t, err) + assert.Equal(t, v1.ActivationStatus_ACTIVATION_UNSPECIFIED, activationStatus) +} + +func TestDiscoverDnfMetadataStatus(t *testing.T) { + tests := map[string]struct { + reposDirs []string + repoDirFiles map[string][]string + cacheDirs []string + expectedStatus v1.DnfMetadataStatus + expectedErrs []string + }{ + "should report available when repo file and cache dir exist": { + reposDirs: []string{"yum.repos.d"}, + repoDirFiles: map[string][]string{"yum.repos.d": {"rhel9.repo"}}, + cacheDirs: []string{"rhel-9-for-x86_64-appstream-rpms-3dc6dc0880df5476"}, + expectedStatus: v1.DnfMetadataStatus_AVAILABLE, + }, + "should report available when repo file is in second reposdir": { + reposDirs: []string{"yum.repos.d", "yum/repos.d"}, + repoDirFiles: map[string][]string{ + "yum.repos.d": {}, + "yum/repos.d": {"rhel9.repo"}, + }, + cacheDirs: []string{"rhel-9-for-x86_64-appstream-rpms-3dc6dc0880df5476"}, + expectedStatus: v1.DnfMetadataStatus_AVAILABLE, + }, + "should report available with multiple repo files and cache dirs": { + reposDirs: []string{"yum.repos.d"}, + repoDirFiles: map[string][]string{ + "yum.repos.d": {"baseos.repo", "appstream.repo"}, + }, + cacheDirs: []string{ + "rhel-9-for-x86_64-appstream-rpms-3dc6dc0880df5476", + "rhel-9-for-x86_64-baseos-rpms-a2cdae14f4ed6c20", + }, + expectedStatus: v1.DnfMetadataStatus_AVAILABLE, + }, + "should report unavailable when no repo files exist": { + reposDirs: []string{"yum.repos.d"}, + repoDirFiles: map[string][]string{"yum.repos.d": {}}, + cacheDirs: []string{"rhel-9-for-x86_64-appstream-rpms-3dc6dc0880df5476"}, + expectedStatus: v1.DnfMetadataStatus_UNAVAILABLE, + expectedErrs: []string{"no .repo files found"}, + }, + "should report unavailable when no cache dirs exist": { + reposDirs: []string{"yum.repos.d"}, + repoDirFiles: map[string][]string{"yum.repos.d": {"rhel9.repo"}}, + cacheDirs: []string{}, + expectedStatus: v1.DnfMetadataStatus_UNAVAILABLE, + }, + "should report unavailable when cache dir lacks -rpms- pattern": { + reposDirs: []string{"yum.repos.d"}, + repoDirFiles: map[string][]string{"yum.repos.d": {"rhel9.repo"}}, + cacheDirs: []string{"some-other-dir"}, + expectedStatus: v1.DnfMetadataStatus_UNAVAILABLE, + }, + "should report unavailable for empty directories": { + reposDirs: []string{"yum.repos.d"}, + repoDirFiles: map[string][]string{"yum.repos.d": {}}, + cacheDirs: []string{}, + expectedStatus: v1.DnfMetadataStatus_UNAVAILABLE, + expectedErrs: []string{"no .repo files found"}, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "dnf") + require.NoError(t, os.MkdirAll(cacheDir, 0755)) + + reposDirPaths := make([]string, 0, len(tt.reposDirs)) + for _, dir := range tt.reposDirs { + dirPath := filepath.Join(tmpDir, dir) + reposDirPaths = append(reposDirPaths, dirPath) + if files, ok := tt.repoDirFiles[dir]; ok { + require.NoError(t, os.MkdirAll(dirPath, 0755)) + for _, filename := range files { + filePath := filepath.Join(dirPath, filename) + err := os.WriteFile(filePath, []byte("[repo]\nname=test"), 0644) + require.NoError(t, err) + } + } + } + + for _, dirname := range tt.cacheDirs { + dirPath := filepath.Join(cacheDir, dirname) + err := os.MkdirAll(dirPath, 0755) + require.NoError(t, err) + } + + dnfStatus, err := discoverDnfMetadataStatusWithPaths("", reposDirPaths, cacheDir) + if len(tt.expectedErrs) == 0 { + assert.NoError(t, err) + } else { + require.Error(t, err) + for _, expectedErr := range tt.expectedErrs { + assert.ErrorContains(t, err, expectedErr) + } + } + assert.Equal(t, tt.expectedStatus, dnfStatus) + }) + } +} + +func TestDiscoverDnfMetadataStatus_MissingDirs(t *testing.T) { + tests := map[string]struct { + reposDirs []string + repoDirFiles map[string][]string + cacheDir string + setupCache func(string) error + errorContains string + }{ + "should return error when repos dir is missing": { + reposDirs: []string{"nonexistent-repos"}, + repoDirFiles: map[string][]string{}, + cacheDir: "dnf", + setupCache: func(path string) error { return os.MkdirAll(path, 0755) }, + errorContains: "reading", + }, + "should return error when cache dir is missing": { + reposDirs: []string{"yum.repos.d"}, + repoDirFiles: map[string][]string{ + "yum.repos.d": {"test.repo"}, + }, + cacheDir: "nonexistent-cache", + setupCache: nil, + errorContains: "unsupported OS detected: missing", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + tmpDir := t.TempDir() + cacheDirPath := filepath.Join(tmpDir, tt.cacheDir) + if tt.setupCache != nil { + require.NoError(t, tt.setupCache(cacheDirPath)) + } + + reposDirPaths := make([]string, 0, len(tt.reposDirs)) + for _, dir := range tt.reposDirs { + dirPath := filepath.Join(tmpDir, dir) + reposDirPaths = append(reposDirPaths, dirPath) + if files, ok := tt.repoDirFiles[dir]; ok { + require.NoError(t, os.MkdirAll(dirPath, 0755)) + for _, filename := range files { + filePath := filepath.Join(dirPath, filename) + require.NoError(t, os.WriteFile(filePath, []byte("[repo]"), 0644)) + } + } + } + + dnfStatus, err := discoverDnfMetadataStatusWithPaths("", reposDirPaths, cacheDirPath) + assert.Error(t, err) + assert.ErrorContains(t, err, tt.errorContains) + assert.Equal(t, v1.DnfMetadataStatus_DNF_METADATA_UNSPECIFIED, dnfStatus) + }) + } +} diff --git a/compliance/virtualmachines/roxagent/index/index.go b/compliance/virtualmachines/roxagent/index/index.go deleted file mode 100644 index 87c67d66dc9b6..0000000000000 --- a/compliance/virtualmachines/roxagent/index/index.go +++ /dev/null @@ -1,77 +0,0 @@ -package index - -import ( - "context" - "fmt" - "net/http" - "time" - - "github.com/stackrox/rox/compliance/node/index" - "github.com/stackrox/rox/compliance/virtualmachines/roxagent/common" - "github.com/stackrox/rox/compliance/virtualmachines/roxagent/vsock" - v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4" - "github.com/stackrox/rox/pkg/httputil/proxy" - "github.com/stackrox/rox/pkg/logging" -) - -var log = logging.LoggerForModule() - -const ( - mappingClientTimeout = 30 * time.Second -) - -func RunDaemon(ctx context.Context, cfg *common.Config, client *vsock.Client) error { - // Create the initial index report immediately. - if err := RunSingle(ctx, cfg, client); err != nil { - log.Errorf("Failed to run initial index: %v", err) - } - - ticker := time.NewTicker(cfg.IndexInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - if err := RunSingle(ctx, cfg, client); err != nil { - log.Errorf("Failed to run index: %v", err) - } - } - } -} - -func RunSingle(ctx context.Context, cfg *common.Config, client *vsock.Client) error { - report, err := runIndexer(ctx, cfg) - if err != nil { - return fmt.Errorf("creating index report: %w", err) - } - if !report.GetSuccess() { - return fmt.Errorf("failed index report: %s", report.GetErr()) - } - if err := client.SendIndexReport(report); err != nil { - return fmt.Errorf("sending index report: %w", err) - } - return nil -} - -func runIndexer(ctx context.Context, cfg *common.Config) (*v4.IndexReport, error) { - indexerCfg := index.NodeIndexerConfig{ - HostPath: cfg.IndexHostPath, - // Client used to fetch the repo to cpe mapping json. - Client: &http.Client{Transport: proxy.RoundTripper()}, - // URL where to get the repo to cpe mapping json from. - // In ACS, we fetch it internally from the cluster (to prevent Collector from accessing the Internet): - // "https://sensor.stackrox.svc:443/scanner/definitions?file=repo2cpe" - Repo2CPEMappingURL: cfg.RepoToCPEMappingURL, - Timeout: mappingClientTimeout, - // Disable package filtering. - PackageDBFilter: "", - } - - report, err := index.NewNodeIndexer(indexerCfg).IndexNode(ctx) - if err != nil { - return nil, err - } - return report, nil -} diff --git a/compliance/virtualmachines/roxagent/quadlet/README.md b/compliance/virtualmachines/roxagent/quadlet/README.md new file mode 100644 index 0000000000000..9f12e3469a8b4 --- /dev/null +++ b/compliance/virtualmachines/roxagent/quadlet/README.md @@ -0,0 +1,196 @@ +# Quadlet Deployment for roxagent + +Run roxagent on RHEL VMs with Podman Quadlet: a systemd-managed container that serves index reports to Sensor over VSOCK. + +## Overview + +[Podman Quadlet](https://docs.podman.io/en/latest/markdown/podman-systemd.unit.5.html) turns `roxagent.container` into a systemd unit that runs `roxagent serve`. The agent scans installed packages, caches the report, and listens on a VSOCK port. Sensor dials in when it needs a report. The agent rescans on its own schedule (default: every 4 hours). + +`Notify=true` makes Podman pass systemd's notify socket into the container, so `systemctl status roxagent` becomes `active` only after the agent has bound the VSOCK listener (not merely when the container process starts). `TimeoutStartSec=600` covers a cold pull of the main image, which can exceed systemd's 90s default. + +### Components + +| File | Description | +|------|-------------| +| `roxagent.container` | Quadlet container unit that runs `roxagent serve` | +| `install.sh` | Installation script for local or remote deployment | + +## Prerequisites + +* RHEL 8, 9, or 10 VM running on KubeVirt with VSOCK enabled +* Podman installed (`dnf install -y podman`) +* StackRox deployed with VM scanning enabled (`ROX_VIRTUAL_MACHINES=true`) +* Network access to pull the StackRox main image. For a private registry, see [Private Registries](#private-registries). + +## Installation + +### 1. Configure the Image Tag + +Edit `roxagent.container` and set the image tag to match your StackRox Secured Cluster deployments: + +```ini +Image=quay.io/stackrox-io/main:4.11.0 +``` + +### 2. Install the Units + +**Local installation:** + +```bash +./install.sh +``` + +**Remote installation via SSH:** + +```bash +./install.sh --ssh user@hostname +./install.sh --ssh user@hostname 2222 # Custom SSH port +``` + +**Remote installation via virtctl:** + +```bash +./install.sh --virtctl -n openshift-cnv cloud-user@vmi/rhel10-1 +``` + +### 3. Verify Installation + +```bash +# Check service status +sudo systemctl status roxagent.service + +# View logs +sudo journalctl -u roxagent.service -f + +# After editing roxagent.container, reload then restart +sudo systemctl daemon-reload +sudo systemctl restart roxagent.service +``` + +## How It Works + +``` +┌─────────────────────────────────────────────────────────────┐ +│ RHEL VM │ +│ ┌────────────────────────┐ │ +│ │ roxagent container │ │ +│ │ - mounts live RPM/DNF │ │ +│ │ DBs read-only │ │ +│ │ - listens on VSOCK │ │ +│ │ - rescans every 4h │ │ +│ └───────────┬────────────┘ │ +└──────────────┼──────────────────────────────────────────────┘ + ▲ vsock +┌──────────────┼─────────────────────────────────────────────┐ +│ Kubernetes Host │ │ +│ ┌────────────────────────────────────────────┐ │ +│ │ Sensor │ │ +│ │ - connects over VSOCK and fetches reports │ │ +│ │ - forwards to Central │ │ +│ └────────────────────────────────────────────┘ │ +└────────────────────────────────────────────────────────────┘ +``` + +### Why Mount the RPM Database Read-Only? + +The container bind-mounts the live host `/var/lib/rpm` (and DNF history paths) read-only under `/host`. Claircore copies each SQLite DB to a temp spool before opening it with `query_only`, so WAL sidecars are created next to the spool inside the container, never on the host. That keeps package inventory fresh across rescans without copying the DB at start, and without giving the agent a writable host RPM directory. + +Optional DNF paths (`/var/lib/dnf`, `/usr/lib/sysimage/libdnf5`, `/var/cache/dnf`, repo dirs) are stripped by `install.sh` when missing on the guest. + +## Configuration + +### Rescan Interval + +To change how often the agent rescans, edit the `Exec=` line in `roxagent.container`: + +```ini +Exec=serve --host-path /host --rescan-interval 2h +``` + +### VSOCK Port + +```ini +Exec=serve --host-path /host --port 2048 +``` + +The port must match the StackRox Sensor configuration. + +### Private Registries + +Quadlet pulls the image as a systemd service. That pull does not use `/etc/containers/auth.json` unless you pass the path into Podman. + +RHEL 8 Quadlet (Podman 4.9) rejects `AuthFile=` in the `[Container]` section (`unsupported key 'AuthFile'`), so set a systemd environment variable instead. + +If the StackRox main image is in a private registry: + +1. Put registry credentials in `/etc/containers/auth.json`. Create that file with `sudo podman login` or by copying a Docker config JSON. Each registry entry needs an `auth` field (`base64(username:password)`); RHEL 8 Podman ignores username/password-only files. +2. Add this line to the `[Service]` section of `roxagent.container`: + +```ini +Environment=REGISTRY_AUTH_FILE=/etc/containers/auth.json +``` + +On Podman 5 and later you can set `AuthFile=/etc/containers/auth.json` in `[Container]` instead. + +3. Reload systemd and restart the unit: + +```bash +sudo systemctl daemon-reload +sudo systemctl restart roxagent.service +``` + +## Troubleshooting + +### No Packages Found + +Confirm the live RPM database is mounted and readable inside the container (host path is `/var/lib/rpm`, container path is `/host/var/lib/rpm`): + +```bash +sudo podman exec systemd-roxagent ls -la /host/var/lib/rpm/ +``` + +Check agent logs on the host: + +```bash +sudo journalctl -u roxagent.service +``` + +Make sure that your VM guest OS is activated and has executed at least a single DNF transaction (e.g., dnf install, dnf update). + +### VSOCK Connection Failed + +Verify VSOCK is enabled in the VM: + +```bash +ls -la /dev/vsock +lsmod | grep vsock +``` + +### Image Pull Unauthorized + +If `journalctl -u roxagent.service` reports `unauthorized` while pulling the main image, Quadlet is not using your registry credentials. Set `REGISTRY_AUTH_FILE` as described in [Private Registries](#private-registries). If `quadlet --dryrun` reports `unsupported key 'AuthFile'`, use the `[Service]` `Environment=` form, not `AuthFile=`. + +### Container Fails to Start + +Check Quadlet generation: + +```bash +/usr/libexec/podman/quadlet --dryrun +sudo journalctl -u roxagent.service +``` + +### VM Not Appearing in Central + +1. Verify `ROX_VIRTUAL_MACHINES=true` is set on Central and Sensor +2. Check Sensor logs for VSOCK scraper activity +3. Verify Sensor can reach Central + +## Uninstallation + +```bash +sudo systemctl stop roxagent.service +sudo rm /etc/containers/systemd/roxagent.container +sudo systemctl daemon-reload +``` + +Quadlet units are generated under `/run/systemd/generator/`, so `systemctl enable` / `disable` do not apply. Boot start comes from `[Install] WantedBy=multi-user.target` in `roxagent.container`; removing that file and reloading drops the service. diff --git a/compliance/virtualmachines/roxagent/quadlet/install.sh b/compliance/virtualmachines/roxagent/quadlet/install.sh new file mode 100755 index 0000000000000..ecf96d0d4d849 --- /dev/null +++ b/compliance/virtualmachines/roxagent/quadlet/install.sh @@ -0,0 +1,267 @@ +#!/bin/bash +# Install roxagent Quadlet units on a RHEL VM +# +# Usage: +# ./install.sh # Install locally +# ./install.sh --virtctl -n openshift-cnv cloud-user@vmi/rhel10-1 # Via virtctl +# ./install.sh --ssh user@host # SSH (port 22) +# ./install.sh --ssh user@host 2222 # SSH with custom port + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Directory containing the Quadlet unit files (roxagent.container, etc.) +# to install. Defaults to the script's own directory; override for testing. +QUADLET_FILES_DIR="${QUADLET_FILES_DIR:-${SCRIPT_DIR}}" + +# Host paths that may not exist on all RHEL versions (e.g. DNF paths on +# yum-only RHEL 8). Volume= lines referencing these are stripped when the +# source path is absent. +OPTIONAL_HOST_PATHS=( + /etc/yum.repos.d + /etc/yum/repos.d + /etc/distro.repos.d + /etc/redhat-release + /etc/system-release-cpe + /var/cache/dnf + /var/lib/dnf + /usr/lib/sysimage/libdnf5 +) + +STAGED_INSTALL_FILES=( + install.sh + roxagent.container +) + +TRANSPORT_KIND="" +SSH_HOST="" +SSH_PORT="" +VIRTCTL_TARGET="" +VIRTCTL_FLAGS=() + +main() { + if [ $# -eq 0 ]; then + setup_transport_local + install_local + elif [ "$1" = "--stage-dir" ]; then + if [ $# -ne 2 ]; then + echo "usage: $0 --stage-dir " >&2 + exit 1 + fi + install_from_stage_dir "$2" + return + elif [ "$1" = "--ssh" ]; then + shift + if [ $# -lt 1 ]; then + usage + exit 1 + fi + setup_transport_ssh "$1" "${2:-22}" + install_remote + elif [ "$1" = "--virtctl" ]; then + shift + setup_transport_virtctl "$@" + install_remote + else + usage + exit 1 + fi + + echo "" + echo "Done! The roxagent serve process is running." + echo "" + echo "To view logs: sudo journalctl -u roxagent.service -f" + echo "To check status: sudo systemctl status roxagent.service" + echo "To restart: sudo systemctl restart roxagent.service" +} + +usage() { + cat >&2 < # Remote install via virtctl + $0 --ssh [port] # Remote install via SSH + +Examples: + $0 --virtctl -n openshift-cnv cloud-user@vmi/rhel10-1 + $0 --ssh root@192.168.1.10 + $0 --ssh root@192.168.1.10 2222 +EOF +} + +remote_copy() { + case "${TRANSPORT_KIND}" in + ssh) + scp -P "${SSH_PORT}" "$1" "${SSH_HOST}:$2" + ;; + virtctl) + virtctl scp "${VIRTCTL_FLAGS[@]}" "$1" "${VIRTCTL_TARGET}:$2" + ;; + *) + echo "remote_copy called without a remote transport" >&2 + return 1 + ;; + esac +} + +remote_exec() { + case "${TRANSPORT_KIND}" in + ssh) + ssh -p "${SSH_PORT}" "${SSH_HOST}" bash -s + ;; + virtctl) + virtctl ssh "${VIRTCTL_FLAGS[@]}" "${VIRTCTL_TARGET}" --command 'bash -s' + ;; + *) + echo "remote_exec called without a remote transport" >&2 + return 1 + ;; + esac +} + +setup_transport_local() { + TRANSPORT_KIND="local" +} + +setup_transport_ssh() { + TRANSPORT_KIND="ssh" + SSH_HOST="$1" + SSH_PORT="${2:-22}" +} + +setup_transport_virtctl() { + if [ $# -eq 0 ]; then + echo "error: virtctl mode requires at least a target (e.g. cloud-user@vmi/rhel10-1)" >&2 + echo "" >&2 + usage + exit 1 + fi + + # Expects virtctl mode arguments after the explicit "--virtctl" selector, e.g.: + # -n openshift-cnv cloud-user@vmi/rhel10-1 + # The last positional arg is the target; everything else are flags. + TRANSPORT_KIND="virtctl" + VIRTCTL_TARGET="${*: -1}" + + local argc=$# + if (( argc > 1 )); then + # Collect all arguments except the last one (the target) as flags. + VIRTCTL_FLAGS=("${@:1:argc-1}") + else + VIRTCTL_FLAGS=() + fi +} + +# --- Install logic (shared) -------------------------------------------------- + +create_remote_stage_dir() { + local output + output="$( + remote_exec <<'SCRIPT' +set -euo pipefail +stage_dir="$(mktemp -d "${TMPDIR:-/var/tmp}/roxagent-install.XXXXXX")" +printf '__STAGE_DIR__=%s\n' "${stage_dir}" +SCRIPT + )" + + local stage_dir + stage_dir="$(printf '%s\n' "${output}" | sed -n 's/^__STAGE_DIR__=//p' | tail -n 1)" + if [ -z "${stage_dir}" ]; then + echo "failed to determine remote stage dir" >&2 + return 1 + fi + printf '%s\n' "${stage_dir}" +} + +copy_remote_stage_files() { + local remote_stage_dir="$1" + local file + for file in "${STAGED_INSTALL_FILES[@]}"; do + remote_copy "${QUADLET_FILES_DIR}/${file}" "${remote_stage_dir}/${file}" + done +} + +validate_stage_dir() { + local stage_dir="$1" + local file + for file in "${STAGED_INSTALL_FILES[@]}"; do + [ "${file}" = "install.sh" ] && continue + if [ ! -f "${stage_dir}/${file}" ]; then + echo "missing staged file: ${stage_dir}/${file}" >&2 + return 1 + fi + done +} + +install_from_stage_dir() { + local stage_dir="$1" + validate_stage_dir "${stage_dir}" + + echo "Installing Quadlet units from ${stage_dir}..." + + # Quadlet container file (strip mounts for paths missing on this host) + sudo mkdir -p /etc/containers/systemd/ + filter_container_file "${stage_dir}/roxagent.container" \ + | sudo tee /etc/containers/systemd/roxagent.container >/dev/null + # restorecon resets SELinux labels so systemd/podman can read the new files. + sudo restorecon -Rv /etc/containers/systemd/ 2>/dev/null || true + + echo "Reloading systemd..." + sudo systemctl daemon-reload + + # Quadlet writes the unit under /run/systemd/generator/, which systemd + # treats as generated: `systemctl enable` is rejected. Boot start comes from + # [Install] WantedBy= in roxagent.container (the generator applies it). + echo "Starting roxagent service..." + sudo systemctl start roxagent.service + + if ! sudo systemctl is-active --quiet roxagent.service; then + echo "roxagent.service failed to become active" >&2 + sudo systemctl status --no-pager roxagent.service || true + return 1 + fi +} + +install_local() { + install_from_stage_dir "${QUADLET_FILES_DIR}" +} + +install_remote() { + local remote_stage_dir + + echo "Preparing stage directory on target..." + remote_stage_dir="$(create_remote_stage_dir)" + + echo "Copying files to target..." + copy_remote_stage_files "${remote_stage_dir}" + + echo "Running install on target..." + remote_exec < diff --git a/ui/apps/platform/package-lock.json b/ui/apps/platform/package-lock.json index dc627cd569594..b46d16429e0c9 100644 --- a/ui/apps/platform/package-lock.json +++ b/ui/apps/platform/package-lock.json @@ -10,20 +10,19 @@ "hasInstallScript": true, "license": "UNLICENSED", "dependencies": { - "@apollo/client": "^3.6.3", + "@apollo/client": "^3.8.10", "@lifeomic/axios-fetch": "^3.1.0", "@openshift-console/dynamic-plugin-sdk": "4.19.0", - "@patternfly/react-charts": "7.2.2", - "@patternfly/react-component-groups": "^5.5.8", - "@patternfly/react-core": "^5.4.14", - "@patternfly/react-icons": "^5.4.2", - "@patternfly/react-styles": "^5.4.1", - "@patternfly/react-table": "^5.4.16", - "@patternfly/react-topology": "^5.4.1", - "@patternfly/react-user-feedback": "^5.0.0", + "@patternfly/patternfly": "^6.4.0", + "@patternfly/react-charts": "^8.4.1", + "@patternfly/react-component-groups": "^6.4.0", + "@patternfly/react-core": "^6.4.1", + "@patternfly/react-icons": "^6.4.0", + "@patternfly/react-styles": "^6.4.0", + "@patternfly/react-table": "^6.4.1", + "@patternfly/react-topology": "^6.4.0", "@segment/analytics-next": "^1.81.1", - "axios": "^1.12.0", - "computed-style-to-inline-style": "^3.0.0", + "axios": "^1.18.0", "connected-react-router": "^6.9.3", "d3-axis": "^1.0.12", "d3-brush": "^3.0.0", @@ -39,18 +38,15 @@ "framer-motion": "^10.0.0", "graphql": "^16.11.0", "history": "^4.10.1", - "html2canvas": "1.0.0-rc.7", "initials": "^3.1.2", "js-base64": "^3.7.2", - "jspdf": "^4.0.0", - "jspdf-autotable": "^5.0.7", - "lodash": "^4.17.21", + "lodash": "^4.17.23", "mobx": "^6.13.7", "mobx-react": "^7.6.0", "object-resolve-path": "^1.1.1", "pluralize": "^8.0.0", "prop-types": "^15.7.2", - "qs": "^6.14.1", + "qs": "^6.16.0", "raven-for-redux": "^1.3.1", "raven-js": "^3.27.0", "react": "^18.2.0", @@ -75,7 +71,6 @@ "react-syntax-highlighter": "^15.6.1", "react-table-6": "^6.11.0", "react-toastify": "^7.0.3", - "react-transition-group": "^4.4.1", "react-truncate": "^2.4.0", "react-vis": "^1.12.1", "redoc": "^2.5.0", @@ -87,48 +82,50 @@ "store": "^2.0.12", "styled-components": "^5.3.1", "use-deep-compare-effect": "^1.8.1", - "yaml": "^2.7.1", + "victory": "^37.3.6", + "yaml": "^2.8.3", "yup": "^1.6.1" }, "devDependencies": { + "@eslint-community/eslint-plugin-eslint-comments": "^4.6.0", "@eslint/css": "^0.14.1", - "@eslint/json": "^0.14.0", + "@eslint/js": "^9.39.3", + "@eslint/json": "^1.0.1", "@eslint/markdown": "^7.5.1", "@openshift-console/dynamic-plugin-sdk-webpack": "4.19.0", "@tailwindcss/forms": "^0.2.1", "@testing-library/cypress": "^10.1.0", - "@testing-library/dom": "^10.4.0", - "@testing-library/jest-dom": "^6.6.3", - "@testing-library/react": "^16.3.0", + "@testing-library/dom": "^10.4.1", + "@testing-library/jest-dom": "^6.9.1", + "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", - "@types/lodash": "^4.17.20", + "@types/lodash": "^4.17.24", "@types/pluralize": "^0.0.33", "@types/qs": "^6.14.0", "@types/react": "^18.3.26", "@types/react-dom": "^18.3.7", "@types/react-redux": "^7.1.34", - "@typescript-eslint/eslint-plugin": "^8.42.0", - "@typescript-eslint/parser": "^8.42.0", - "@vitejs/plugin-basic-ssl": "^1.1.0", - "@vitejs/plugin-react": "^4.4.0", - "@vitest/coverage-v8": "^3.1.1", - "@vitest/eslint-plugin": "^1.3.3", + "@typescript-eslint/eslint-plugin": "^8.56.1", + "@typescript-eslint/parser": "^8.56.1", + "@vitejs/plugin-basic-ssl": "^2.2.0", + "@vitejs/plugin-react": "^5.2.0", + "@vitest/coverage-v8": "^4.1.0", + "@vitest/eslint-plugin": "^1.6.12", "axe-core": "^4.10.3", "copy-webpack-plugin": "^13.0.0", "css-loader": "^7.1.2", - "cypress": "^15.5.0", + "cypress": "^15.14.1", "cypress-axe": "^1.7.0", "cypress-vite": "^1.8.0", - "eslint": "^9.39.1", + "eslint": "^9.39.3", "eslint-import-resolver-typescript": "^4.2.6", - "eslint-plugin-cypress": "^5.2.0", - "eslint-plugin-eslint-comments": "^3.2.0", + "eslint-plugin-cypress": "^6.1.0", "eslint-plugin-import": "^2.32.0", "eslint-plugin-jest-dom": "^5.5.0", "eslint-plugin-prettier": "^5.5.4", "eslint-plugin-react": "^7.37.5", - "eslint-plugin-react-hooks": "^7.0.1", - "eslint-plugin-testing-library": "^7.13.4", + "eslint-plugin-react-hooks": "^7.1.1", + "eslint-plugin-testing-library": "^7.16.0", "globals": "^16.5.0", "jsdom": "^20.0.3", "levenary": "^1.1.1", @@ -137,19 +134,19 @@ "node-fetch": "^2.6.1", "postcss": "^8.4.47", "postcss-cli": "^8.3.1", - "prettier": "^3.6.2", + "prettier": "^3.8.1", "redux-immutable-state-invariant": "^2.1.0", "redux-saga-test-plan": "^3.7.0", "style-loader": "^4.0.0", "tailwindcss": "^2.0.3", "ts-loader": "^9.5.2", "typescript": "^5.9.2", - "vite": "^6.4.1", + "vite": "^6.4.2", "vite-plugin-svgr": "^4.3.0", - "vitest": "^3.0.5", - "webpack": "^5.99.9", + "vitest": "^4.1.0", + "webpack": "^5.105.0", "webpack-cli": "^6.0.1", - "webpack-dev-server": "^5.2.2" + "webpack-dev-server": "^5.2.4" }, "engines": { "node": ">=22.13.0" @@ -175,32 +172,18 @@ "integrity": "sha512-Ff9+ksdQQB3rMncgqDK78uLznstjyfIf2Arnh22pW8kBpLs6rpKDwgnZT46hin5Hl1WzazzK64DOrhSwYpS7bQ==", "dev": true }, - "node_modules/@ampproject/remapping": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", - "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@apollo/client": { - "version": "3.8.7", - "resolved": "https://registry.npmjs.org/@apollo/client/-/client-3.8.7.tgz", - "integrity": "sha1-CQsVGPUTUDuaamkO4+rsSVKYIuE= sha512-DnQtFkQrCyxHTSa9gR84YRLmU/al6HeXcLZazVe+VxKBmx/Hj4rV8xWtzfWYX5ijartsqDR7SJgV037MATEecA==", + "version": "3.8.10", + "resolved": "https://registry.npmjs.org/@apollo/client/-/client-3.8.10.tgz", + "integrity": "sha512-p/22RZ8ehHyvySnC20EHPPe0gdu8Xp6ZCiXOfdEe1ZORw5cUteD/TLc66tfKv8qu8NLIfbiWoa+6s70XnKvxqg==", + "license": "MIT", "dependencies": { "@graphql-typed-document-node/core": "^3.1.1", - "@wry/context": "^0.7.3", "@wry/equality": "^0.5.6", - "@wry/trie": "^0.4.3", + "@wry/trie": "^0.5.0", "graphql-tag": "^2.12.6", "hoist-non-react-statics": "^3.3.2", - "optimism": "^0.17.5", + "optimism": "^0.18.0", "prop-types": "^15.7.2", "response-iterator": "^0.2.6", "symbol-observable": "^4.0.0", @@ -231,12 +214,12 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", - "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", + "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" }, @@ -245,9 +228,9 @@ } }, "node_modules/@babel/compat-data": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.0.tgz", - "integrity": "sha512-60X7qkglvrap8mn1lh2ebxXdZYtUcpd7gsmy9kLaBJ4i/WdY8PqTSdxyA8qraikqKQK5C1KRBKXqznrVapyNaw==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.0.tgz", + "integrity": "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==", "dev": true, "license": "MIT", "engines": { @@ -255,22 +238,22 @@ } }, "node_modules/@babel/core": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.0.tgz", - "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.0", - "@babel/helper-compilation-targets": "^7.27.2", - "@babel/helper-module-transforms": "^7.27.3", - "@babel/helpers": "^7.27.6", - "@babel/parser": "^7.28.0", - "@babel/template": "^7.27.2", - "@babel/traverse": "^7.28.0", - "@babel/types": "^7.28.0", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", + "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-compilation-targets": "^7.28.6", + "@babel/helper-module-transforms": "^7.28.6", + "@babel/helpers": "^7.28.6", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/traverse": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", @@ -292,13 +275,13 @@ "dev": true }, "node_modules/@babel/generator": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.0.tgz", - "integrity": "sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg==", + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", + "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", "license": "MIT", "dependencies": { - "@babel/parser": "^7.28.0", - "@babel/types": "^7.28.0", + "@babel/parser": "^7.29.0", + "@babel/types": "^7.29.0", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" @@ -320,13 +303,13 @@ } }, "node_modules/@babel/helper-compilation-targets": { - "version": "7.27.2", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", - "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", + "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/compat-data": "^7.27.2", + "@babel/compat-data": "^7.28.6", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", @@ -346,28 +329,28 @@ } }, "node_modules/@babel/helper-module-imports": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", - "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", + "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", "license": "MIT", "dependencies": { - "@babel/traverse": "^7.27.1", - "@babel/types": "^7.27.1" + "@babel/traverse": "^7.28.6", + "@babel/types": "^7.28.6" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.27.3", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.27.3.tgz", - "integrity": "sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", + "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-module-imports": "^7.27.1", - "@babel/helper-validator-identifier": "^7.27.1", - "@babel/traverse": "^7.27.3" + "@babel/helper-module-imports": "^7.28.6", + "@babel/helper-validator-identifier": "^7.28.5", + "@babel/traverse": "^7.28.6" }, "engines": { "node": ">=6.9.0" @@ -396,9 +379,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", - "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "license": "MIT", "engines": { "node": ">=6.9.0" @@ -415,26 +398,26 @@ } }, "node_modules/@babel/helpers": { - "version": "7.27.6", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.6.tgz", - "integrity": "sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug==", + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.2.tgz", + "integrity": "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/template": "^7.27.2", - "@babel/types": "^7.27.6" + "@babel/template": "^7.28.6", + "@babel/types": "^7.29.0" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.0.tgz", - "integrity": "sha512-jVZGvOxOuNSsuQuLRTh13nU0AogFlw32w/MT+LV6D3sP5WdbW61E77RnkbaO2dUvmPAYrBDJXGn5gGS6tH4j8g==", + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", + "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", "license": "MIT", "dependencies": { - "@babel/types": "^7.28.0" + "@babel/types": "^7.29.0" }, "bin": { "parser": "bin/babel-parser.js" @@ -485,31 +468,31 @@ } }, "node_modules/@babel/template": { - "version": "7.27.2", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", - "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", + "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.27.1", - "@babel/parser": "^7.27.2", - "@babel/types": "^7.27.1" + "@babel/code-frame": "^7.28.6", + "@babel/parser": "^7.28.6", + "@babel/types": "^7.28.6" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.0.tgz", - "integrity": "sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", + "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.0", + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.28.0", - "@babel/template": "^7.27.2", - "@babel/types": "^7.28.0", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/types": "^7.29.0", "debug": "^4.3.1" }, "engines": { @@ -517,13 +500,13 @@ } }, "node_modules/@babel/types": { - "version": "7.28.2", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.2.tgz", - "integrity": "sha512-ruv7Ae4J5dUYULmeXw1gmb7rYRz57OWCPM57pHojnLq/3Z1CK2lNSLTCVjxVk1F/TZHwOZZrOWi0ur95BbLxNQ==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", "license": "MIT", "dependencies": { "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.27.1" + "@babel/helper-validator-identifier": "^7.28.5" }, "engines": { "node": ">=6.9.0" @@ -560,22 +543,38 @@ } }, "node_modules/@cypress/request/node_modules/form-data": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", - "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", "dev": true, "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "hasown": "^2.0.4", + "mime-types": "^2.1.35" }, "engines": { "node": ">= 6" } }, + "node_modules/@cypress/request/node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/@cypress/request/node_modules/tough-cookie": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", @@ -636,6 +635,78 @@ "node": ">=14.17.0" } }, + "node_modules/@dnd-kit/accessibility": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@dnd-kit/accessibility/-/accessibility-3.1.1.tgz", + "integrity": "sha512-2P+YgaXF+gRsIihwwY1gCsQSYnu9Zyj2py8kY5fFvUM1qm2WA2u639R6YNVfU4GWr+ZM5mqEsfHZZLoRONbemw==", + "license": "MIT", + "peer": true, + "dependencies": { + "tslib": "^2.0.0" + }, + "peerDependencies": { + "react": ">=16.8.0" + } + }, + "node_modules/@dnd-kit/core": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@dnd-kit/core/-/core-6.3.1.tgz", + "integrity": "sha512-xkGBRQQab4RLwgXxoqETICr6S5JlogafbhNsidmrkVv2YRs5MLwpjoF2qpiGjQt8S9AoxtIV603s0GIUpY5eYQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@dnd-kit/accessibility": "^3.1.1", + "@dnd-kit/utilities": "^3.2.2", + "tslib": "^2.0.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@dnd-kit/modifiers": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@dnd-kit/modifiers/-/modifiers-9.0.0.tgz", + "integrity": "sha512-ybiLc66qRGuZoC20wdSSG6pDXFikui/dCNGthxv4Ndy8ylErY0N3KVxY2bgo7AWwIbxDmXDg3ylAFmnrjcbVvw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@dnd-kit/utilities": "^3.2.2", + "tslib": "^2.0.0" + }, + "peerDependencies": { + "@dnd-kit/core": "^6.3.0", + "react": ">=16.8.0" + } + }, + "node_modules/@dnd-kit/sortable": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@dnd-kit/sortable/-/sortable-10.0.0.tgz", + "integrity": "sha512-+xqhmIIzvAYMGfBYYnbKuNicfSsk4RksY2XdmJhT+HAC01nix6fHCztU68jooFiMUB01Ky3F0FyOvhG/BZrWkg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@dnd-kit/utilities": "^3.2.2", + "tslib": "^2.0.0" + }, + "peerDependencies": { + "@dnd-kit/core": "^6.3.0", + "react": ">=16.8.0" + } + }, + "node_modules/@dnd-kit/utilities": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@dnd-kit/utilities/-/utilities-3.2.2.tgz", + "integrity": "sha512-+MKAJEOfaBe5SmV6t34p80MMKhjvUz0vRrvVJbPT0WElzaOJ/1xs+D+KDv+tD/NE5ujfrChEcshd4fLn0wpiqg==", + "license": "MIT", + "peer": true, + "dependencies": { + "tslib": "^2.0.0" + }, + "peerDependencies": { + "react": ">=16.8.0" + } + }, "node_modules/@emnapi/core": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.4.0.tgz", @@ -1312,10 +1383,53 @@ "node": ">=18" } }, + "node_modules/@eslint-community/eslint-plugin-eslint-comments": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-plugin-eslint-comments/-/eslint-plugin-eslint-comments-4.6.0.tgz", + "integrity": "sha512-2EX2bBQq1ez++xz2o9tEeEQkyvfieWgUFMH4rtJJri2q0Azvhja3hZGXsjPXs31R4fQkZDtWzNDDK2zQn5UE5g==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0", + "ignore": "^7.0.5" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0 || ^9.0.0" + } + }, + "node_modules/@eslint-community/eslint-plugin-eslint-comments/node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@eslint-community/eslint-plugin-eslint-comments/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.8.0.tgz", - "integrity": "sha512-MJQFqrZgcW0UNYLGOuQpey/oTN59vyWwplvCGZztn1cKz9agZPPYpJB7h2OMmuu7VLqkvEjN8feFZJmxNF9D+Q==", + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", "dev": true, "license": "MIT", "dependencies": { @@ -1332,9 +1446,9 @@ } }, "node_modules/@eslint-community/regexpp": { - "version": "4.12.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.1.tgz", - "integrity": "sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==", + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", "dev": true, "license": "MIT", "engines": { @@ -1435,13 +1549,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/@eslint/eslintrc/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, "node_modules/@eslint/eslintrc/node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -1455,23 +1562,10 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@eslint/eslintrc/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/@eslint/js": { - "version": "9.39.1", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.1.tgz", - "integrity": "sha512-S26Stp4zCy88tH94QbBv3XCuzRQiZ9yXofEILmglYTh/Ug/a9/umqvgFtYBAo3Lp0nsI/5/qH1CCrbdK3AP1Tw==", + "version": "9.39.3", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.3.tgz", + "integrity": "sha512-1B1VkCq6FuUNlQvlBYb+1jDu/gV297TIs/OeiaSR9l1H27SVW55ONE1e1Vp16NqP683+xEGzxYtv4XCiDPaQiw==", "dev": true, "license": "MIT", "engines": { @@ -1482,19 +1576,46 @@ } }, "node_modules/@eslint/json": { - "version": "0.14.0", - "resolved": "https://registry.npmjs.org/@eslint/json/-/json-0.14.0.tgz", - "integrity": "sha512-rvR/EZtvUG3p9uqrSmcDJPYSH7atmWr0RnFWN6m917MAPx82+zQgPUmDu0whPFG6XTyM0vB/hR6c1Q63OaYtCQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@eslint/json/-/json-1.0.1.tgz", + "integrity": "sha512-bE2nGv8/U+uRvQEJWOgCsZCa65XsCBgxyyx/sXtTHVv0kqdauACLzyp7A1C3yNn7pRaWjIt5acxY+TAbSyIJXw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^0.17.0", - "@eslint/plugin-kit": "^0.4.1", + "@eslint/core": "^1.1.0", + "@eslint/plugin-kit": "^0.6.0", "@humanwhocodes/momoa": "^3.3.10", "natural-compare": "^1.4.0" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/json/node_modules/@eslint/core": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.1.0.tgz", + "integrity": "sha512-/nr9K9wkr3P1EzFTdFdMoLuo1PmIxjmwvPozwoSodjNBdefGujXQUF93u1DDZpEaTuDvMsIQddsd35BwtrW9Xw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/json/node_modules/@eslint/plugin-kit": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.6.0.tgz", + "integrity": "sha512-bIZEUzOI1jkhviX2cp5vNyXQc6olzb2ohewQubuYlMXZ2Q/XjBO0x0XhGPvc9fjSIiUN0vw+0hq53BJ4eQSJKQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.1.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/markdown": { @@ -1651,111 +1772,6 @@ "react": "*" } }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-regex": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", - "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@isaacs/cliui/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.2.tgz", - "integrity": "sha1-JlIL8Jq+SlZEzVQU43ElqJVCQd0= sha512-tsAQNx32a8CoFhjhijUIhI4kccIAgmGhy8LZMZgGfmXcpMbPRUqn5LWmgRttILi6yeGmBJd2xsPkFMs0PzgPCw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.12", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.12.tgz", @@ -1766,6 +1782,17 @@ "@jridgewell/trace-mapping": "^0.3.24" } }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", @@ -1785,15 +1812,15 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", - "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.29", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.29.tgz", - "integrity": "sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==", + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", "license": "MIT", "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", @@ -1910,20 +1937,45 @@ "@tybys/wasm-util": "^0.9.0" } }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz", - "integrity": "sha1-1LNUml213iaD4MEHGrTxQJBLv2k= sha512-33g3pMJk3bg5nXbL/+CY6I2eJDzZAni49PfJnL5fghPTggPvBd/pFNSgJsdAgWptuFu7qq/ERvOYFlhvsLTCKA==", + "node_modules/@noble/hashes": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.4.0.tgz", + "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", "dev": true, - "dependencies": { - "@nodelib/fs.stat": "2.0.4", - "run-parallel": "^1.1.9" - }, + "license": "MIT", "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { + "node": ">= 16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@nodable/entities": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.1.tgz", + "integrity": "sha512-Pig3HxDIoMgjdEH8OCf/dkcTmLFjJRjWuq8jSnklu284/TKOPibSRERmOykiwmyXTtv61mP+44f3GMx0tLAyjg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz", + "integrity": "sha1-1LNUml213iaD4MEHGrTxQJBLv2k= sha512-33g3pMJk3bg5nXbL/+CY6I2eJDzZAni49PfJnL5fghPTggPvBd/pFNSgJsdAgWptuFu7qq/ERvOYFlhvsLTCKA==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.4", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.4.tgz", "integrity": "sha1-o/LdYbq0O424+hCKEhz//kxnZlU= sha512-IYlHJA0clt2+Vg7bccq+TzRdJvv19c2INqBSsoOLp1je7xjtr7J26+WXR72MCdvU9q1qTzIWDfhMf+DRvQJK4Q==", @@ -2096,84 +2148,10 @@ "node": ">=4" } }, - "node_modules/@openshift-console/dynamic-plugin-sdk/node_modules/@patternfly/react-core": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/@patternfly/react-core/-/react-core-6.3.0.tgz", - "integrity": "sha512-TM+pLwLd5DzaDlOQhqeju9H9QUFQypQiNwXQLNIxOV5r3fmKh4NTp2Av/8WmFkpCj8mejDOfp4TNxoU1zdjCkQ==", - "license": "MIT", - "dependencies": { - "@patternfly/react-icons": "^6.3.0", - "@patternfly/react-styles": "^6.3.0", - "@patternfly/react-tokens": "^6.3.0", - "focus-trap": "7.6.4", - "react-dropzone": "^14.3.5", - "tslib": "^2.8.1" - }, - "peerDependencies": { - "react": "^17 || ^18 || ^19", - "react-dom": "^17 || ^18 || ^19" - } - }, - "node_modules/@openshift-console/dynamic-plugin-sdk/node_modules/@patternfly/react-icons": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/@patternfly/react-icons/-/react-icons-6.3.0.tgz", - "integrity": "sha512-W39JyqKW1UL6/YGuinDnpjbhmmLAfuxVrgDcdFBaK4D7D1iqkkqrDMV8zIzmV/RkodJ79xRnucYhYb2RukG4RA==", - "license": "MIT", - "peerDependencies": { - "react": "^17 || ^18 || ^19", - "react-dom": "^17 || ^18 || ^19" - } - }, - "node_modules/@openshift-console/dynamic-plugin-sdk/node_modules/@patternfly/react-styles": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/@patternfly/react-styles/-/react-styles-6.3.0.tgz", - "integrity": "sha512-FvuyNsY2oN8f2dvCl4Hx8CxBWCIF3BC9JE3Ay1lCuVqY1WYkvW4AQn3/0WVRINCxB9FkQxVNkSjARdwHNCEulw==", - "license": "MIT" - }, - "node_modules/@openshift-console/dynamic-plugin-sdk/node_modules/@patternfly/react-tokens": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/@patternfly/react-tokens/-/react-tokens-6.3.0.tgz", - "integrity": "sha512-yWStfkbxg4RWAExFKS/JRGScyadOy35yr4DFispNeHrkZWMp4pwKf0VdwlQZ7+ZtSgEWtzzy1KFxMLmWh3mEqA==", - "license": "MIT" - }, - "node_modules/@openshift-console/dynamic-plugin-sdk/node_modules/@patternfly/react-topology": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/@patternfly/react-topology/-/react-topology-6.3.0.tgz", - "integrity": "sha512-v0P8khAJqS01haEl9abw3L/MzdpIY+PsEVGBAMgA/tBhU/+9Ieco8fmmjW+7CfRmXhSC8NUjYgasl+9ZJVd3rA==", - "license": "MIT", - "dependencies": { - "@dagrejs/dagre": "1.1.2", - "@patternfly/react-core": "^6.0.0", - "@patternfly/react-icons": "^6.0.0", - "@patternfly/react-styles": "^6.0.0", - "@types/d3": "^7.4.0", - "@types/d3-force": "^1.2.1", - "d3": "^7.8.0", - "mobx": "^6.9.0", - "mobx-react": "^7.6.0", - "point-in-svg-path": "^1.0.1", - "popper.js": "^1.16.1", - "tslib": "^2.0.0", - "webcola": "3.4.0" - }, - "peerDependencies": { - "react": "^17 || ^18 || ^19", - "react-dom": "^17 || ^18 || ^19" - } - }, - "node_modules/@openshift-console/dynamic-plugin-sdk/node_modules/focus-trap": { - "version": "7.6.4", - "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-7.6.4.tgz", - "integrity": "sha512-xx560wGBk7seZ6y933idtjJQc1l+ck+pI3sKvhKozdBV1dRZoKhkW5xoCaFv9tQiX5RH1xfSxjuNu6g+lmN/gw==", - "license": "MIT", - "dependencies": { - "tabbable": "^6.2.0" - } - }, "node_modules/@openshift-console/dynamic-plugin-sdk/node_modules/immutable": { - "version": "3.8.2", - "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.2.tgz", - "integrity": "sha512-15gZoQ38eYjEjxkorfbcgBKBL6R7T459OuK+CpcWt7O3KF4uPCx2tD0uFETlUDIyo+1789crbMhTvQBSR5yBMg==", + "version": "3.8.4", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.4.tgz", + "integrity": "sha512-ZQv17KolYrYmsDoDB8N67zhIKsNi9mGYlk96y/yuxyAqJB2hLzSGSM7k/sB0/ZvO4kx27U9+fBeD/XlLGh/uXQ==", "license": "MIT", "engines": { "node": ">=0.10.0" @@ -2257,173 +2235,391 @@ "node": ">=10" } }, + "node_modules/@patternfly/patternfly": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@patternfly/patternfly/-/patternfly-6.4.0.tgz", + "integrity": "sha512-4drFhg74sEc/fftark5wZevODIog17qR4pwLCdB3j5iK3Uu5oMA2SdLhsEeEQggalfnFzve/Km87MdVR0ghhvQ==", + "license": "MIT" + }, "node_modules/@patternfly/react-charts": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@patternfly/react-charts/-/react-charts-7.2.2.tgz", - "integrity": "sha512-1PFuvXz3mm/o/O+BQ2/2e66ncvtV8XIYxFaimurslCLTygodOvjBDDu/D/5tNa3HLxvA+fm2Q58893POGZi+bw==", + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@patternfly/react-charts/-/react-charts-8.4.1.tgz", + "integrity": "sha512-kqP5TFr9F/SDTp+nXfcX8gMuZIQ6Uxl55qqiWYgR2zflo0UZWnyL/d2s1fI6Bfi5IuycNdlvJJ/i8QdOsLr5mw==", "license": "MIT", "dependencies": { - "@patternfly/react-styles": "^5.2.1", - "@patternfly/react-tokens": "^5.2.1", - "hoist-non-react-statics": "^3.3.0", - "lodash": "^4.17.21", - "tslib": "^2.5.0", - "victory-area": "^36.9.1", - "victory-axis": "^36.9.1", - "victory-bar": "^36.9.1", - "victory-box-plot": "^36.9.1", - "victory-chart": "^36.9.1", - "victory-core": "^36.9.1", - "victory-create-container": "^36.9.1", - "victory-cursor-container": "^36.9.1", - "victory-group": "^36.9.1", - "victory-legend": "^36.9.1", - "victory-line": "^36.9.1", - "victory-pie": "^36.9.1", - "victory-scatter": "^36.9.1", - "victory-stack": "^36.9.1", - "victory-tooltip": "^36.9.1", - "victory-voronoi-container": "^36.9.1", - "victory-zoom-container": "^36.9.1" + "@patternfly/react-styles": "^6.4.0", + "@patternfly/react-tokens": "^6.4.0", + "hoist-non-react-statics": "^3.3.2", + "lodash": "^4.17.23", + "tslib": "^2.8.1" }, "peerDependencies": { - "react": "^17 || ^18", - "react-dom": "^17 || ^18" + "echarts": "^5.6.0 || ^6.0.0", + "react": "^17 || ^18 || ^19", + "react-dom": "^17 || ^18 || ^19", + "victory-area": "^37.3.6", + "victory-axis": "^37.3.6", + "victory-bar": "^37.3.6", + "victory-box-plot": "^37.3.6", + "victory-chart": "^37.3.6", + "victory-core": "^37.3.6", + "victory-create-container": "^37.3.6", + "victory-cursor-container": "^37.3.6", + "victory-group": "^37.3.6", + "victory-legend": "^37.3.6", + "victory-line": "^37.3.6", + "victory-pie": "^37.3.6", + "victory-scatter": "^37.3.6", + "victory-stack": "^37.3.6", + "victory-tooltip": "^37.3.6", + "victory-voronoi-container": "^37.3.6", + "victory-zoom-container": "^37.3.6" + }, + "peerDependenciesMeta": { + "echarts": { + "optional": true + }, + "victory-area": { + "optional": true + }, + "victory-axis": { + "optional": true + }, + "victory-bar": { + "optional": true + }, + "victory-box-plot": { + "optional": true + }, + "victory-chart": { + "optional": true + }, + "victory-core": { + "optional": true + }, + "victory-create-container": { + "optional": true + }, + "victory-cursor-container": { + "optional": true + }, + "victory-group": { + "optional": true + }, + "victory-legend": { + "optional": true + }, + "victory-line": { + "optional": true + }, + "victory-pie": { + "optional": true + }, + "victory-scatter": { + "optional": true + }, + "victory-stack": { + "optional": true + }, + "victory-tooltip": { + "optional": true + }, + "victory-voronoi-container": { + "optional": true + }, + "victory-zoom-container": { + "optional": true + } } }, "node_modules/@patternfly/react-component-groups": { - "version": "5.5.8", - "resolved": "https://registry.npmjs.org/@patternfly/react-component-groups/-/react-component-groups-5.5.8.tgz", - "integrity": "sha512-v31FyvzewWiLVMOR8S4GgD1S+ZWqBPEawPkJG6Ja4BpF5dFJeke6lUovdEPTs4NQEUtBytF6Tsww38XGL6NaaA==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@patternfly/react-component-groups/-/react-component-groups-6.4.0.tgz", + "integrity": "sha512-vg0761nQ/7hfggbp6+XowRcQQSd9oIToh77+4lmsyrs41MkA5ppQIPBCZ4lUZW87kmEPhkHqglpJcVfsrrIM/g==", "license": "MIT", "dependencies": { - "@patternfly/react-core": "^5.4.1", - "@patternfly/react-icons": "^5.4.0", - "@patternfly/react-table": "^5.4.1", - "clsx": "^2.1.1", + "@patternfly/react-core": "^6.0.0", + "@patternfly/react-icons": "^6.0.0", + "@patternfly/react-styles": "^6.0.0", + "@patternfly/react-table": "^6.0.0", "react-jss": "^10.10.0" }, "peerDependencies": { - "react": "^17 || ^18", - "react-dom": "^17 || ^18" + "@patternfly/react-drag-drop": "^6.0.0", + "react": "^17 || ^18 || ^19", + "react-dom": "^17 || ^18 || ^19" } }, - "node_modules/@patternfly/react-component-groups/node_modules/clsx": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", - "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "node_modules/@patternfly/react-core": { + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/@patternfly/react-core/-/react-core-6.4.1.tgz", + "integrity": "sha512-EUSV76Eifkt4R3q2JIaiB6/FHeQqOCttK1DQMXNoOCNa3ODkZ7H+KlMdminufMDfRzhwAgTVihZ62K9PFfc8Vg==", "license": "MIT", - "engines": { - "node": ">=6" + "dependencies": { + "@patternfly/react-icons": "^6.4.0", + "@patternfly/react-styles": "^6.4.0", + "@patternfly/react-tokens": "^6.4.0", + "focus-trap": "7.6.4", + "react-dropzone": "^14.3.5", + "tslib": "^2.8.1" + }, + "peerDependencies": { + "react": "^17 || ^18 || ^19", + "react-dom": "^17 || ^18 || ^19" } }, - "node_modules/@patternfly/react-core": { - "version": "5.4.14", - "resolved": "https://registry.npmjs.org/@patternfly/react-core/-/react-core-5.4.14.tgz", - "integrity": "sha512-oXVMzLs9Pa+xmdc39L2u05zbXfY3mWuOFi4GDv44GPdDexZUFy5W69+Nv5P8cwfMim55Nf5kKYpcqmatD2bBXw==", + "node_modules/@patternfly/react-drag-drop": { + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/@patternfly/react-drag-drop/-/react-drag-drop-6.4.1.tgz", + "integrity": "sha512-+zkN+6JO/6qPKb1srsKyC4OpeF/dgQvZ0U4soXeh/n22n2YVcrH9QAUGn1QBB7FmLiAoATHB9vr1awafF5HIYw==", "license": "MIT", + "peer": true, "dependencies": { - "@patternfly/react-icons": "^5.4.2", - "@patternfly/react-styles": "^5.4.1", - "@patternfly/react-tokens": "^5.4.1", - "focus-trap": "7.6.2", - "react-dropzone": "^14.2.3", - "tslib": "^2.7.0" + "@dnd-kit/core": "^6.3.1", + "@dnd-kit/modifiers": "^9.0.0", + "@dnd-kit/sortable": "^10.0.0", + "@patternfly/react-core": "^6.4.1", + "@patternfly/react-icons": "^6.4.0", + "@patternfly/react-styles": "^6.4.0", + "resize-observer-polyfill": "^1.5.1" }, "peerDependencies": { - "react": "^17 || ^18", - "react-dom": "^17 || ^18" + "react": "^17 || ^18 || ^19", + "react-dom": "^17 || ^18 || ^19" } }, "node_modules/@patternfly/react-icons": { - "version": "5.4.2", - "resolved": "https://registry.npmjs.org/@patternfly/react-icons/-/react-icons-5.4.2.tgz", - "integrity": "sha512-CMQ5oHYzW6TPVTs2jpNJmP2vGCAKR/YeTPwHGO9dLkAUej1IcIxtCCWK2Fdo2UJsnBjuZihasyw2b6ehvbUm9Q==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@patternfly/react-icons/-/react-icons-6.4.0.tgz", + "integrity": "sha512-SPjzatm73NUYv/BL6A/cjRA5sFQ15NkiyPAcT8gmklI7HY+ptd6/eg49uBDFmxTQcSwbb5ISW/R6wwCQBY2M+Q==", "license": "MIT", "peerDependencies": { - "react": "^17 || ^18", - "react-dom": "^17 || ^18" + "react": "^17 || ^18 || ^19", + "react-dom": "^17 || ^18 || ^19" } }, "node_modules/@patternfly/react-styles": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/@patternfly/react-styles/-/react-styles-5.4.1.tgz", - "integrity": "sha512-XA8PXksD8uiA3RTwxdUwJXOCf+V6sVd+2HKapWAdRLvtSV+Sdk7NgCvalb4IAQncsddLopjPQD8gAHA298+N8w==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@patternfly/react-styles/-/react-styles-6.4.0.tgz", + "integrity": "sha512-EXmHA67s5sy+Wy/0uxWoUQ52jr9lsH2wV3QcgtvVc5zxpyBX89gShpqv4jfVqaowznHGDoL6fVBBrSe9BYOliQ==", "license": "MIT" }, "node_modules/@patternfly/react-table": { - "version": "5.4.16", - "resolved": "https://registry.npmjs.org/@patternfly/react-table/-/react-table-5.4.16.tgz", - "integrity": "sha512-K/xNdTvFKFyD6WhuBU/GCq/3LTrSaObdXSTmzefWWcPcN0gcf/J3ouiuapj951JIsP13CMmwJBNtEa3aHR8igg==", + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/@patternfly/react-table/-/react-table-6.4.1.tgz", + "integrity": "sha512-dceS5X1I5gmhRfEf6gsLIdFhrQd5hY+SWRIYzEvI19rrdN9VwCVWg+bhSsNv05pQnGs8uOlzYVBI25p7PJtFeA==", "license": "MIT", "dependencies": { - "@patternfly/react-core": "^5.4.14", - "@patternfly/react-icons": "^5.4.2", - "@patternfly/react-styles": "^5.4.1", - "@patternfly/react-tokens": "^5.4.1", - "lodash": "^4.17.21", - "tslib": "^2.7.0" + "@patternfly/react-core": "^6.4.1", + "@patternfly/react-icons": "^6.4.0", + "@patternfly/react-styles": "^6.4.0", + "@patternfly/react-tokens": "^6.4.0", + "lodash": "^4.17.23", + "tslib": "^2.8.1" }, "peerDependencies": { - "react": "^17 || ^18", - "react-dom": "^17 || ^18" + "react": "^17 || ^18 || ^19", + "react-dom": "^17 || ^18 || ^19" } }, "node_modules/@patternfly/react-tokens": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/@patternfly/react-tokens/-/react-tokens-5.4.1.tgz", - "integrity": "sha512-eygdHE7Krta1mijAv/E8RHiKIgysD0eeNTo8EXUYC8/M4e5K6sqpr2p6rQBF8QiRMN8FnbXvZT3K2OQ28pYt9Q==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@patternfly/react-tokens/-/react-tokens-6.4.0.tgz", + "integrity": "sha512-iZthBoXSGQ/+PfGTdPFJVulaJZI3rwE+7A/whOXPGp3Jyq3k6X52pr1+5nlO6WHasbZ9FyeZGqXf4fazUZNjbw==", "license": "MIT" }, "node_modules/@patternfly/react-topology": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/@patternfly/react-topology/-/react-topology-5.4.1.tgz", - "integrity": "sha512-3tgldDj7qKIGFKTQ1gyrwr3a/VDa/I5w0RBPqicppywvFygPrfi2NlK1Mt+3rtGW5R/fG64tzOcyZKx/y2Wpvw==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@patternfly/react-topology/-/react-topology-6.4.0.tgz", + "integrity": "sha512-Uy2ofRnI0apYiPWUYIAlXifl+4QPx/sqsVku1WglCkqjMPwuR8vC/GTIJEq2qW7mucDAcWSTFAPc5+zqyPRrlQ==", "license": "MIT", "dependencies": { "@dagrejs/dagre": "1.1.2", - "@patternfly/react-core": "^5.1.1", - "@patternfly/react-icons": "^5.1.1", - "@patternfly/react-styles": "^5.1.1", + "@patternfly/react-core": "^6.0.0", + "@patternfly/react-icons": "^6.0.0", + "@patternfly/react-styles": "^6.0.0", "@types/d3": "^7.4.0", "@types/d3-force": "^1.2.1", - "@types/react-measure": "^2.0.6", "d3": "^7.8.0", "mobx": "^6.9.0", "mobx-react": "^7.6.0", "point-in-svg-path": "^1.0.1", "popper.js": "^1.16.1", - "react-measure": "^2.3.0", "tslib": "^2.0.0", "webcola": "3.4.0" }, "peerDependencies": { - "react": "^17 || ^18", - "react-dom": "^17 || ^18" + "react": "^17 || ^18 || ^19", + "react-dom": "^17 || ^18 || ^19" } }, - "node_modules/@patternfly/react-user-feedback": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@patternfly/react-user-feedback/-/react-user-feedback-5.0.0.tgz", - "integrity": "sha1-35V3BShPmjaAHH2kfnV0U9tDHd4= sha512-E28AC87z9A2X5blf9ih9YVoHsAo68tDM6fLrOWFU447v3QHmWwBIViHT7/BL+M2B7ZORQ+LtM9M4bn0lq7/Ezw==", + "node_modules/@peculiar/asn1-cms": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-cms/-/asn1-cms-2.7.0.tgz", + "integrity": "sha512-hew63shtzzvBcSHbhm+cyAmKe6AIfinT9hzEqSPjDC6opTTMKmTkQ0gHuN2KsWlvqiKw1S/fS94fhag/FJkioQ==", + "dev": true, + "license": "MIT", "dependencies": { - "@patternfly/react-core": "^5.0.0", - "@patternfly/react-icons": "^5.0.0" - }, - "peerDependencies": { - "react": "^17 || ^18", - "react-dom": "^17 || ^18" + "@peculiar/asn1-schema": "^2.7.0", + "@peculiar/asn1-x509": "^2.7.0", + "@peculiar/asn1-x509-attr": "^2.7.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" } }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "node_modules/@peculiar/asn1-csr": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-csr/-/asn1-csr-2.7.0.tgz", + "integrity": "sha512-VVsAyGqErT9D1SY4aEqozThXMVI+ssVRiv2DDeYuvpBKLIgZ3hYs3Ay3u/VSoKq6ESFi9cf6rf3IOOzfwh7oMA==", "dev": true, "license": "MIT", - "optional": true, + "dependencies": { + "@peculiar/asn1-schema": "^2.7.0", + "@peculiar/asn1-x509": "^2.7.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-ecc": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-ecc/-/asn1-ecc-2.7.0.tgz", + "integrity": "sha512-n7KEs/Q/wrB415cxy4fHOBhegp4NdJ15fkJPwcB/3/8iNBQC2L/N7SChJPKDJPZGYH0jD4Tg4/0vnHmwghnbKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.7.0", + "@peculiar/asn1-x509": "^2.7.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-pfx": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pfx/-/asn1-pfx-2.7.0.tgz", + "integrity": "sha512-V/nrlQVmhg7lYAsM7E13UDL5erAwFv6kCIVFqNaMIHSVi7dngcT839JkRTkQBqznMG98l2XjxYk74ZztAohZzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.7.0", + "@peculiar/asn1-pkcs8": "^2.7.0", + "@peculiar/asn1-rsa": "^2.7.0", + "@peculiar/asn1-schema": "^2.7.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-pkcs8": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pkcs8/-/asn1-pkcs8-2.7.0.tgz", + "integrity": "sha512-9GTl1nE8Mx1kTZ+7QyYatDyKsm34QcWRBFkY1iPvWC3X4Dona5s/tlLiQsx5WzVdZqiMBZNYT0buyw4/vbhnjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.7.0", + "@peculiar/asn1-x509": "^2.7.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-pkcs9": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pkcs9/-/asn1-pkcs9-2.7.0.tgz", + "integrity": "sha512-Bh7m+OuIaSEllPQcSd9OSp93F4ROWH7sbITWV8MI+8dwsjE5111/87VxiWVvYFKyww3vp39geLv9ENqhwWHcew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.7.0", + "@peculiar/asn1-pfx": "^2.7.0", + "@peculiar/asn1-pkcs8": "^2.7.0", + "@peculiar/asn1-schema": "^2.7.0", + "@peculiar/asn1-x509": "^2.7.0", + "@peculiar/asn1-x509-attr": "^2.7.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-rsa": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-rsa/-/asn1-rsa-2.7.0.tgz", + "integrity": "sha512-/qvENQrXyTZURjMqSeofHul0JJt2sNSzSwk36pl2olkHbaioMQgrASDZAlHXl0xUlnVbHj0uGgOrBMTb5x2aJQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.7.0", + "@peculiar/asn1-x509": "^2.7.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-schema": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-schema/-/asn1-schema-2.7.0.tgz", + "integrity": "sha512-W8ZfWzLmQnrcky+eh3tni4IozMdqBDiHWU0N+vve/UGjMaUs8c0L7A2oEdkBXS8rTpWDpK/aoI3DG/L/hxmxPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/utils": "^2.0.2", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-x509": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509/-/asn1-x509-2.7.0.tgz", + "integrity": "sha512-mUn9RRrkGDnG4ALfunDmzyRW5dg+sWCj/pfnCCqEHYbkGxEpvUt6iVJv8Yw1cyp6SWZ26ZE5oSmI5SqEaen15g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.7.0", + "@peculiar/utils": "^2.0.2", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-x509-attr": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509-attr/-/asn1-x509-attr-2.7.0.tgz", + "integrity": "sha512-NS8e7SOgXipkzUPLF/sce7ukpMpWjhxYsH0n6Y+bHYo4TTxOb95Zv7hqwSuL212mj5YxovjdOKQOgH1As3E94w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.7.0", + "@peculiar/asn1-x509": "^2.7.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@peculiar/utils/-/utils-2.0.3.tgz", + "integrity": "sha512-+oL3HPFRIZ1St2K50lWCXiioIgSoxzz7R1J3uF6neO2yl1sgmpgY6XXJH4BdpoDkMWznQTeYF6oWNDZLCdQ4eQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/x509": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@peculiar/x509/-/x509-1.14.3.tgz", + "integrity": "sha512-C2Xj8FZ0uHWeCXXqX5B4/gVFQmtSkiuOolzAgutjTfseNOHT3pUjljDZsTSxXFGgio54bCzVFqmEOUrIVk8RDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.6.0", + "@peculiar/asn1-csr": "^2.6.0", + "@peculiar/asn1-ecc": "^2.6.0", + "@peculiar/asn1-pkcs9": "^2.6.0", + "@peculiar/asn1-rsa": "^2.6.0", + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.0", + "pvtsutils": "^1.3.6", + "reflect-metadata": "^0.2.2", + "tslib": "^2.8.1", + "tsyringe": "^4.10.0" + }, "engines": { - "node": ">=14" + "node": ">=20.0.0" } }, "node_modules/@pkgr/core": { @@ -2506,16 +2702,10 @@ "node": ">= 14" } }, - "node_modules/@redocly/openapi-core/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "license": "Python-2.0" - }, "node_modules/@redocly/openapi-core/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -2534,22 +2724,10 @@ "node": ">= 14" } }, - "node_modules/@redocly/openapi-core/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/@redocly/openapi-core/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "version": "5.1.8", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.8.tgz", + "integrity": "sha512-7RN35vit8DeBclkofOVmBY0eDAZZQd1HzmukRdSyz95CRh8FT54eqnbj0krQr3mrHR6sfRyYkyhwBWjoV5uqlQ==", "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" @@ -2568,9 +2746,9 @@ } }, "node_modules/@rolldown/pluginutils": { - "version": "1.0.0-beta.19", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.19.tgz", - "integrity": "sha512-3FL3mnMbPu0muGOCaKAhhFEYmqv9eTfPSJRJmANrCwtgK8VuxpsZDGK+m0LYAGoyO8+0j5uRe4PeyPDK1yA/hA==", + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.3.tgz", + "integrity": "sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q==", "dev": true, "license": "MIT" }, @@ -2945,6 +3123,13 @@ "@segment/isodate": "^1.0.3" } }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, "node_modules/@tailwindcss/forms": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.2.1.tgz", @@ -2973,18 +3158,19 @@ } }, "node_modules/@testing-library/dom": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz", - "integrity": "sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==", + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", + "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", "dev": true, + "license": "MIT", "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", "@types/aria-query": "^5.0.1", "aria-query": "5.3.0", - "chalk": "^4.1.0", "dom-accessibility-api": "^0.5.9", "lz-string": "^1.5.0", + "picocolors": "1.1.1", "pretty-format": "^27.0.2" }, "engines": { @@ -3030,18 +3216,17 @@ "dev": true }, "node_modules/@testing-library/jest-dom": { - "version": "6.6.3", - "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.6.3.tgz", - "integrity": "sha512-IteBhl4XqYNkM54f4ejhLRJiZNqcSCoXUOG2CPK7qbD322KjQozM4kHQOfkG2oln9b9HTYqs+Sae8vBATubxxA==", + "version": "6.9.1", + "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.9.1.tgz", + "integrity": "sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA==", "dev": true, "license": "MIT", "dependencies": { "@adobe/css-tools": "^4.4.0", "aria-query": "^5.0.0", - "chalk": "^3.0.0", "css.escape": "^1.5.1", "dom-accessibility-api": "^0.6.3", - "lodash": "^4.17.21", + "picocolors": "^1.1.1", "redent": "^3.0.0" }, "engines": { @@ -3050,23 +3235,10 @@ "yarn": ">=1" } }, - "node_modules/@testing-library/jest-dom/node_modules/chalk": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", - "integrity": "sha1-P3PCv1JlkfV0zEksUeJFY0n4ROQ= sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", - "dev": true, - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/@testing-library/react": { - "version": "16.3.0", - "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.3.0.tgz", - "integrity": "sha512-kFSyxiEDwv1WLl2fgsq6pPBbw5aWKrsY2/noi1Id0TK0UParSF62oFQFGHXIyaG4pp2tEub/Zlel+fjjZILDsw==", + "version": "16.3.2", + "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.3.2.tgz", + "integrity": "sha512-XU5/SytQM+ykqMnAnvB2umaJNIOsLF3PVv//1Ew4CTcpz0/BRyy/af40qqrt7SjKpDdT1saBMc42CUok5gaw+g==", "dev": true, "license": "MIT", "dependencies": { @@ -3106,10 +3278,11 @@ } }, "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha1-9UShSNOrNYAcH2M6dEH9h8LkhL8= sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.1.tgz", + "integrity": "sha512-HqmEUIGRJ5fSXchkVgR5F7qn48bDBzv0kWj/Kfu5e6uci4UlEeng4331LnBkWffb++Ei3FOVLxo8JJWMFBDMeQ==", "dev": true, + "license": "MIT", "engines": { "node": ">= 10" } @@ -3195,13 +3368,14 @@ } }, "node_modules/@types/chai": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.2.tgz", - "integrity": "sha512-8kB30R7Hwqf40JPiKhVzodJs2Qc1ZJ5zuT3uzw5Hq/dhNCl3G3l83jfpdI1e20BP348+fV7VIL/+FxaXkqBmWg==", + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", "dev": true, "license": "MIT", "dependencies": { - "@types/deep-eql": "*" + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" } }, "node_modules/@types/connect": { @@ -3498,16 +3672,16 @@ "license": "MIT" }, "node_modules/@types/express": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.23.tgz", - "integrity": "sha512-Crp6WY9aTYP3qPi2wGDo9iUe/rceX01UMhnF1jmwDcKCFM6cx7YhGP/Mpr3y9AASpfHixIG0E6azCcL5OcDHsQ==", + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", + "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", "dev": true, "license": "MIT", "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.33", "@types/qs": "*", - "@types/serve-static": "*" + "@types/serve-static": "^1" } }, "node_modules/@types/express-serve-static-core": { @@ -3575,9 +3749,9 @@ "dev": true }, "node_modules/@types/lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-H3MHACvFUEiujabxhaI/ImO6gUrd8oOurg7LQtS7mbwIXA/cUqWrvBsaeJ23aZEPk1TAYkurjfMbSELfoCXlGA==", + "version": "4.17.24", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.24.tgz", + "integrity": "sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==", "dev": true, "license": "MIT" }, @@ -3623,16 +3797,6 @@ "form-data": "^3.0.0" } }, - "node_modules/@types/node-forge": { - "version": "1.3.14", - "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.14.tgz", - "integrity": "sha512-mhVF2BnD4BO+jtOp7z1CdzaK4mbuK0LLQYAvdOLqHTavxFNq4zA1EmYkpnFjP8HOUzedfQkRnp0E2ulSAYSzAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/normalize-package-data": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", @@ -3640,12 +3804,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/pako": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.4.tgz", - "integrity": "sha512-VWDCbrLeVXJM9fihYodcLiIv0ku+AlOa/TQ1SvYOaBuyrSKgEcro95LJyIsJ4vSo6BXIxOKxiJAat04CmST9Fw==", - "license": "MIT" - }, "node_modules/@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -3669,12 +3827,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/raf": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/@types/raf/-/raf-3.4.0.tgz", - "integrity": "sha1-K3LL1VQF4HHxxNKZkmOOAisgrMI= sha512-taW5/WYqo36N7V39oYyHP9Ipfd5pNFvGTIQsNGj86xV88YQ7GnI30/yMfKDF7Zgin0m3e+ikX88FvImnK4RjGw==", - "optional": true - }, "node_modules/@types/range-parser": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", @@ -3702,14 +3854,6 @@ "@types/react": "^18.0.0" } }, - "node_modules/@types/react-measure": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/@types/react-measure/-/react-measure-2.0.8.tgz", - "integrity": "sha1-NhWVbGhebZ/tg4CqxTWQszP6408= sha512-Pu4/hQ/1AKVN6efoawtcM+l376WYOI8e1fiM6ir4pdLkHilDCkJLjUGvAm0mWKJ0GE6hzu55yCrcJ/xNyEdFwA==", - "dependencies": { - "@types/react": "*" - } - }, "node_modules/@types/react-redux": { "version": "7.1.34", "resolved": "https://registry.npmjs.org/@types/react-redux/-/react-redux-7.1.34.tgz", @@ -3831,21 +3975,20 @@ } }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.42.0.tgz", - "integrity": "sha512-Aq2dPqsQkxHOLfb2OPv43RnIvfj05nw8v/6n3B2NABIPpHnjQnaLo9QGMTvml+tv4korl/Cjfrb/BYhoL8UUTQ==", + "version": "8.56.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.56.1.tgz", + "integrity": "sha512-Jz9ZztpB37dNC+HU2HI28Bs9QXpzCz+y/twHOwhyrIRdbuVDxSytJNDl6z/aAKlaRIwC7y8wJdkBv7FxYGgi0A==", "dev": true, "license": "MIT", "dependencies": { - "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "8.42.0", - "@typescript-eslint/type-utils": "8.42.0", - "@typescript-eslint/utils": "8.42.0", - "@typescript-eslint/visitor-keys": "8.42.0", - "graphemer": "^1.4.0", - "ignore": "^7.0.0", + "@eslint-community/regexpp": "^4.12.2", + "@typescript-eslint/scope-manager": "8.56.1", + "@typescript-eslint/type-utils": "8.56.1", + "@typescript-eslint/utils": "8.56.1", + "@typescript-eslint/visitor-keys": "8.56.1", + "ignore": "^7.0.5", "natural-compare": "^1.4.0", - "ts-api-utils": "^2.1.0" + "ts-api-utils": "^2.4.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3855,8 +3998,8 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.42.0", - "eslint": "^8.57.0 || ^9.0.0", + "@typescript-eslint/parser": "^8.56.1", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.0.0" } }, @@ -3871,17 +4014,17 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.42.0.tgz", - "integrity": "sha512-r1XG74QgShUgXph1BYseJ+KZd17bKQib/yF3SR+demvytiRXrwd12Blnz5eYGm8tXaeRdd4x88MlfwldHoudGg==", + "version": "8.56.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.56.1.tgz", + "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.42.0", - "@typescript-eslint/types": "8.42.0", - "@typescript-eslint/typescript-estree": "8.42.0", - "@typescript-eslint/visitor-keys": "8.42.0", - "debug": "^4.3.4" + "@typescript-eslint/scope-manager": "8.56.1", + "@typescript-eslint/types": "8.56.1", + "@typescript-eslint/typescript-estree": "8.56.1", + "@typescript-eslint/visitor-keys": "8.56.1", + "debug": "^4.4.3" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3891,20 +4034,20 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.42.0.tgz", - "integrity": "sha512-vfVpLHAhbPjilrabtOSNcUDmBboQNrJUiNAGoImkZKnMjs2TIcWG33s4Ds0wY3/50aZmTMqJa6PiwkwezaAklg==", + "version": "8.56.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.56.1.tgz", + "integrity": "sha512-TAdqQTzHNNvlVFfR+hu2PDJrURiwKsUvxFn1M0h95BB8ah5jejas08jUWG4dBA68jDMI988IvtfdAI53JzEHOQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.42.0", - "@typescript-eslint/types": "^8.42.0", - "debug": "^4.3.4" + "@typescript-eslint/tsconfig-utils": "^8.56.1", + "@typescript-eslint/types": "^8.56.1", + "debug": "^4.4.3" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3918,14 +4061,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.42.0.tgz", - "integrity": "sha512-51+x9o78NBAVgQzOPd17DkNTnIzJ8T/O2dmMBLoK9qbY0Gm52XJcdJcCl18ExBMiHo6jPMErUQWUv5RLE51zJw==", + "version": "8.56.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.56.1.tgz", + "integrity": "sha512-YAi4VDKcIZp0O4tz/haYKhmIDZFEUPOreKbfdAN3SzUDMcPhJ8QI99xQXqX+HoUVq8cs85eRKnD+rne2UAnj2w==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.42.0", - "@typescript-eslint/visitor-keys": "8.42.0" + "@typescript-eslint/types": "8.56.1", + "@typescript-eslint/visitor-keys": "8.56.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3936,9 +4079,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.42.0.tgz", - "integrity": "sha512-kHeFUOdwAJfUmYKjR3CLgZSglGHjbNTi1H8sTYRYV2xX6eNz4RyJ2LIgsDLKf8Yi0/GL1WZAC/DgZBeBft8QAQ==", + "version": "8.56.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.56.1.tgz", + "integrity": "sha512-qOtCYzKEeyr3aR9f28mPJqBty7+DBqsdd63eO0yyDwc6vgThj2UjWfJIcsFeSucYydqcuudMOprZ+x1SpF3ZuQ==", "dev": true, "license": "MIT", "engines": { @@ -3953,17 +4096,17 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.42.0.tgz", - "integrity": "sha512-9KChw92sbPTYVFw3JLRH1ockhyR3zqqn9lQXol3/YbI6jVxzWoGcT3AsAW0mu1MY0gYtsXnUGV/AKpkAj5tVlQ==", + "version": "8.56.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.56.1.tgz", + "integrity": "sha512-yB/7dxi7MgTtGhZdaHCemf7PuwrHMenHjmzgUW1aJpO+bBU43OycnM3Wn+DdvDO/8zzA9HlhaJ0AUGuvri4oGg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.42.0", - "@typescript-eslint/typescript-estree": "8.42.0", - "@typescript-eslint/utils": "8.42.0", - "debug": "^4.3.4", - "ts-api-utils": "^2.1.0" + "@typescript-eslint/types": "8.56.1", + "@typescript-eslint/typescript-estree": "8.56.1", + "@typescript-eslint/utils": "8.56.1", + "debug": "^4.4.3", + "ts-api-utils": "^2.4.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3973,14 +4116,14 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.42.0.tgz", - "integrity": "sha512-LdtAWMiFmbRLNP7JNeY0SqEtJvGMYSzfiWBSmx+VSZ1CH+1zyl8Mmw1TT39OrtsRvIYShjJWzTDMPWZJCpwBlw==", + "version": "8.56.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.56.1.tgz", + "integrity": "sha512-dbMkdIUkIkchgGDIv7KLUpa0Mda4IYjo4IAMJUZ+3xNoUXxMsk9YtKpTHSChRS85o+H9ftm51gsK1dZReY9CVw==", "dev": true, "license": "MIT", "engines": { @@ -3992,22 +4135,21 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.42.0.tgz", - "integrity": "sha512-ku/uYtT4QXY8sl9EDJETD27o3Ewdi72hcXg1ah/kkUgBvAYHLwj2ofswFFNXS+FL5G+AGkxBtvGt8pFBHKlHsQ==", + "version": "8.56.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.56.1.tgz", + "integrity": "sha512-qzUL1qgalIvKWAf9C1HpvBjif+Vm6rcT5wZd4VoMb9+Km3iS3Cv9DY6dMRMDtPnwRAFyAi7YXJpTIEXLvdfPxg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.42.0", - "@typescript-eslint/tsconfig-utils": "8.42.0", - "@typescript-eslint/types": "8.42.0", - "@typescript-eslint/visitor-keys": "8.42.0", - "debug": "^4.3.4", - "fast-glob": "^3.3.2", - "is-glob": "^4.0.3", - "minimatch": "^9.0.4", - "semver": "^7.6.0", - "ts-api-utils": "^2.1.0" + "@typescript-eslint/project-service": "8.56.1", + "@typescript-eslint/tsconfig-utils": "8.56.1", + "@typescript-eslint/types": "8.56.1", + "@typescript-eslint/visitor-keys": "8.56.1", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.4.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -4020,36 +4162,49 @@ "typescript": ">=4.8.4 <6.0.0" } }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0" + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.2.tgz", + "integrity": "sha512-+G4CpNBxa5MprY+04MbgOw1v7So6n5JY166pFi9KfYwT78fxScCeSNQSNzp6dpPSW2rONOps6Ocam1wFhCgoVw==", "dev": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^5.0.2" }, "engines": { - "node": ">=16 || 14 >=14.17" + "node": "18 || 20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { - "version": "7.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", - "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", "dev": true, "license": "ISC", "bin": { @@ -4060,16 +4215,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.42.0.tgz", - "integrity": "sha512-JnIzu7H3RH5BrKC4NoZqRfmjqCIS1u3hGZltDYJgkVdqAezl4L9d1ZLw+36huCujtSBSAirGINF/S4UxOcR+/g==", + "version": "8.56.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.56.1.tgz", + "integrity": "sha512-HPAVNIME3tABJ61siYlHzSWCGtOoeP2RTIaHXFMPqjrQKCGB9OgUVdiNgH7TJS2JNIQ5qQ4RsAUDuGaGme/KOA==", "dev": true, "license": "MIT", "dependencies": { - "@eslint-community/eslint-utils": "^4.7.0", - "@typescript-eslint/scope-manager": "8.42.0", - "@typescript-eslint/types": "8.42.0", - "@typescript-eslint/typescript-estree": "8.42.0" + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/scope-manager": "8.56.1", + "@typescript-eslint/types": "8.56.1", + "@typescript-eslint/typescript-estree": "8.56.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -4079,19 +4234,19 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.42.0.tgz", - "integrity": "sha512-3WbiuzoEowaEn8RSnhJBrxSwX8ULYE9CXaPepS2C2W3NSA5NNIvBaslpBSBElPq0UGr0xVJlXFWOAKIkyylydQ==", + "version": "8.56.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.56.1.tgz", + "integrity": "sha512-KiROIzYdEV85YygXw6BI/Dx4fnBlFQu6Mq4QE4MOH9fFnhohw6wX/OAvDY2/C+ut0I3RSPKenvZJIVYqJNkhEw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.42.0", - "eslint-visitor-keys": "^4.2.1" + "@typescript-eslint/types": "8.56.1", + "eslint-visitor-keys": "^5.0.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -4102,13 +4257,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", "dev": true, "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" @@ -4328,192 +4483,96 @@ ] }, "node_modules/@vitejs/plugin-basic-ssl": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-basic-ssl/-/plugin-basic-ssl-1.2.0.tgz", - "integrity": "sha512-mkQnxTkcldAzIsomk1UuLfAu9n+kpQ3JbHcpCp7d2Oo6ITtji8pHS3QToOWjhPFvNQSnhlkAjmGbhv2QvwO/7Q==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-basic-ssl/-/plugin-basic-ssl-2.2.0.tgz", + "integrity": "sha512-nmyQ1HGRkfUxjsv3jw0+hMhEdZdrtkvMTdkzRUaRWfiO6PCWw2V2Pz3gldCq96Tn9S8htcgdTxw/gmbLLEbfYw==", "dev": true, "license": "MIT", "engines": { - "node": ">=14.21.3" + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" }, "peerDependencies": { - "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0" + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "node_modules/@vitejs/plugin-react": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.6.0.tgz", - "integrity": "sha512-5Kgff+m8e2PB+9j51eGHEpn5kUzRKH2Ry0qGoe8ItJg7pqnkPrYPkDQZGgGmTa0EGarHrkjLvOdU3b1fzI8otQ==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.2.0.tgz", + "integrity": "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/core": "^7.27.4", + "@babel/core": "^7.29.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", - "@rolldown/pluginutils": "1.0.0-beta.19", + "@rolldown/pluginutils": "1.0.0-rc.3", "@types/babel__core": "^7.20.5", - "react-refresh": "^0.17.0" + "react-refresh": "^0.18.0" }, "engines": { - "node": "^14.18.0 || >=16.0.0" + "node": "^20.19.0 || >=22.12.0" }, "peerDependencies": { - "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0-beta.0" - } - }, - "node_modules/@vitejs/plugin-react/node_modules/react-refresh": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", - "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "node_modules/@vitest/coverage-v8": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-3.2.4.tgz", - "integrity": "sha512-EyF9SXU6kS5Ku/U82E259WSnvg6c8KTjppUncuNdm5QHpe17mwREHnjDzozC8x9MZ0xfBUFSaLkRv4TMA75ALQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.0.tgz", + "integrity": "sha512-nDWulKeik2bL2Va/Wl4x7DLuTKAXa906iRFooIRPR+huHkcvp9QDkPQ2RJdmjOFrqOqvNfoSQLF68deE3xC3CQ==", "dev": true, "license": "MIT", "dependencies": { - "@ampproject/remapping": "^2.3.0", "@bcoe/v8-coverage": "^1.0.2", - "ast-v8-to-istanbul": "^0.3.3", - "debug": "^4.4.1", + "@vitest/utils": "4.1.0", + "ast-v8-to-istanbul": "^1.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", - "istanbul-lib-source-maps": "^5.0.6", - "istanbul-reports": "^3.1.7", - "magic-string": "^0.30.17", - "magicast": "^0.3.5", - "std-env": "^3.9.0", - "test-exclude": "^7.0.1", - "tinyrainbow": "^2.0.0" + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.2", + "obug": "^2.1.1", + "std-env": "^4.0.0-rc.1", + "tinyrainbow": "^3.0.3" }, "funding": { "url": "https://opencollective.com/vitest" }, - "peerDependencies": { - "@vitest/browser": "3.2.4", - "vitest": "3.2.4" - }, - "peerDependenciesMeta": { - "@vitest/browser": { - "optional": true - } - } - }, - "node_modules/@vitest/coverage-v8/node_modules/@bcoe/v8-coverage": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", - "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@vitest/coverage-v8/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/@vitest/coverage-v8/node_modules/glob": { - "version": "10.4.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", - "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^1.11.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@vitest/coverage-v8/node_modules/istanbul-lib-source-maps": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-5.0.6.tgz", - "integrity": "sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.23", - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@vitest/coverage-v8/node_modules/magic-string": { - "version": "0.30.17", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", - "integrity": "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0" - } - }, - "node_modules/@vitest/coverage-v8/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" + "peerDependencies": { + "@vitest/browser": "4.1.0", + "vitest": "4.1.0" }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } } }, - "node_modules/@vitest/coverage-v8/node_modules/test-exclude": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-7.0.1.tgz", - "integrity": "sha512-pFYqmTw68LXVjeWJMST4+borgQP2AyMNbg1BpZh9LbyhUeNkeaPF9gzfPGUAnSMV3qPYdWUwDIjjCLiSDOl7vg==", + "node_modules/@vitest/coverage-v8/node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^10.4.1", - "minimatch": "^9.0.4" - }, + "license": "MIT", "engines": { "node": ">=18" } }, "node_modules/@vitest/eslint-plugin": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@vitest/eslint-plugin/-/eslint-plugin-1.3.3.tgz", - "integrity": "sha512-zOB4T5f80JXfP5DC2yQl7azRYq8PmGqYle3uxh3a0NnbKc+EaSYSpEcrVAh2r5W97pi3BVv7oRb5NdEQy0cCXA==", + "version": "1.6.12", + "resolved": "https://registry.npmjs.org/@vitest/eslint-plugin/-/eslint-plugin-1.6.12.tgz", + "integrity": "sha512-4kI47BJNFE+EQ5bmPbHzBF+ibNzx2Fj0Jo9xhWsTPxMddlHwIWl6YAxagefh461hrwx/W0QwBZpxGS404kBXyg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/utils": "^8.24.1" + "@typescript-eslint/scope-manager": "^8.55.0", + "@typescript-eslint/utils": "^8.55.0" + }, + "engines": { + "node": ">=18" }, "peerDependencies": { - "eslint": ">= 8.57.0", - "typescript": ">= 5.0.0", + "eslint": ">=8.57.0", + "typescript": ">=5.0.0", "vitest": "*" }, "peerDependenciesMeta": { @@ -4526,39 +4585,40 @@ } }, "node_modules/@vitest/expect": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", - "integrity": "sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.0.tgz", + "integrity": "sha512-EIxG7k4wlWweuCLG9Y5InKFwpMEOyrMb6ZJ1ihYu02LVj/bzUwn2VMU+13PinsjRW75XnITeFrQBMH5+dLvCDA==", "dev": true, "license": "MIT", "dependencies": { + "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "3.2.4", - "@vitest/utils": "3.2.4", - "chai": "^5.2.0", - "tinyrainbow": "^2.0.0" + "@vitest/spy": "4.1.0", + "@vitest/utils": "4.1.0", + "chai": "^6.2.2", + "tinyrainbow": "^3.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/mocker": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.4.tgz", - "integrity": "sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.0.tgz", + "integrity": "sha512-evxREh+Hork43+Y4IOhTo+h5lGmVRyjqI739Rz4RlUPqwrkFFDF6EMvOOYjTx4E8Tl6gyCLRL8Mu7Ry12a13Tw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "3.2.4", + "@vitest/spy": "4.1.0", "estree-walker": "^3.0.3", - "magic-string": "^0.30.17" + "magic-string": "^0.30.21" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { "msw": "^2.4.9", - "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0" }, "peerDependenciesMeta": { "msw": { @@ -4569,107 +4629,81 @@ } } }, - "node_modules/@vitest/mocker/node_modules/estree-walker": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" - } - }, - "node_modules/@vitest/mocker/node_modules/magic-string": { - "version": "0.30.17", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", - "integrity": "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0" - } - }, "node_modules/@vitest/pretty-format": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", - "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.0.tgz", + "integrity": "sha512-3RZLZlh88Ib0J7NQTRATfc/3ZPOnSUn2uDBUoGNn5T36+bALixmzphN26OUD3LRXWkJu4H0s5vvUeqBiw+kS0A==", "dev": true, "license": "MIT", "dependencies": { - "tinyrainbow": "^2.0.0" + "tinyrainbow": "^3.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/runner": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.4.tgz", - "integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.0.tgz", + "integrity": "sha512-Duvx2OzQ7d6OjchL+trw+aSrb9idh7pnNfxrklo14p3zmNL4qPCDeIJAK+eBKYjkIwG96Bc6vYuxhqDXQOWpoQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "3.2.4", - "pathe": "^2.0.3", - "strip-literal": "^3.0.0" + "@vitest/utils": "4.1.0", + "pathe": "^2.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/snapshot": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.4.tgz", - "integrity": "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.0.tgz", + "integrity": "sha512-0Vy9euT1kgsnj1CHttwi9i9o+4rRLEaPRSOJ5gyv579GJkNpgJK+B4HSv/rAWixx2wdAFci1X4CEPjiu2bXIMg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "3.2.4", - "magic-string": "^0.30.17", + "@vitest/pretty-format": "4.1.0", + "@vitest/utils": "4.1.0", + "magic-string": "^0.30.21", "pathe": "^2.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/snapshot/node_modules/magic-string": { - "version": "0.30.17", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", - "integrity": "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0" - } - }, "node_modules/@vitest/spy": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", - "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.0.tgz", + "integrity": "sha512-pz77k+PgNpyMDv2FV6qmk5ZVau6c3R8HC8v342T2xlFxQKTrSeYw9waIJG8KgV9fFwAtTu4ceRzMivPTH6wSxw==", "dev": true, "license": "MIT", - "dependencies": { - "tinyspy": "^4.0.3" - }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/utils": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", - "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.0.tgz", + "integrity": "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "3.2.4", - "loupe": "^3.1.4", - "tinyrainbow": "^2.0.0" + "@vitest/pretty-format": "4.1.0", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, + "node_modules/@vitest/utils/node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, "node_modules/@webassemblyjs/ast": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz", @@ -4878,10 +4912,23 @@ } } }, + "node_modules/@wry/caches": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@wry/caches/-/caches-1.0.1.tgz", + "integrity": "sha512-bXuaUNLVVkD20wcGBWRyo7j9N3TxePEWFZj2Y+r9OoUzfqmavM84+mFykRicNsBqatba5JLay1t48wxaXaWnlA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.3.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/@wry/context": { "version": "0.7.4", "resolved": "https://registry.npmjs.org/@wry/context/-/context-0.7.4.tgz", - "integrity": "sha1-4y11D6B1lVxKss+4xICV4dQtWZA= sha512-jmT7Sb4ZQWI5iyu3lobQxICu2nC/vbUhP0vIdd6tHC9PTfenmRmuIFqktc6GH9cgi+ZHnsLWPvfSvc4DrYmKiQ==", + "integrity": "sha512-jmT7Sb4ZQWI5iyu3lobQxICu2nC/vbUhP0vIdd6tHC9PTfenmRmuIFqktc6GH9cgi+ZHnsLWPvfSvc4DrYmKiQ==", + "license": "MIT", "dependencies": { "tslib": "^2.3.0" }, @@ -4901,9 +4948,10 @@ } }, "node_modules/@wry/trie": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@wry/trie/-/trie-0.4.3.tgz", - "integrity": "sha1-B31SwiNlhxvz/8urjpXLi8VomvQ= sha512-I6bHwH0fSf6RqQcnnXLJKhkSXG45MFral3GxPaY4uAl0LYDZM+YDVDAiU9bYwjTuysy1S0IeecWtmq1SZA3M1w==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@wry/trie/-/trie-0.5.0.tgz", + "integrity": "sha512-FNoYzHawTMk/6KMQoEG5O4PuioX19UbwdQKF44yw0nLfOypfQdjtfZzo/UIJWAJ23sNIFbD1Ug9lbaDGMwbqQA==", + "license": "MIT", "dependencies": { "tslib": "^2.3.0" }, @@ -5052,7 +5100,6 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", "integrity": "sha1-Sf/1hXfP7j83F2/qtMIuAPhtf3c= sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "dev": true, "dependencies": { "debug": "4" }, @@ -5198,6 +5245,18 @@ "node": ">= 8" } }, + "node_modules/anynum": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.0.tgz", + "integrity": "sha512-xjR9/zBVnUOP6ztMIIgShjsxui80nQUQH+5xJnvrYLs+90bF25/KJqaAi8mk+B4RDtX1Nspi6fmp4YTEts8SfA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/arch": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", @@ -5218,6 +5277,12 @@ } ] }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, "node_modules/aria-query": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", @@ -5430,6 +5495,21 @@ "safer-buffer": "~2.1.0" } }, + "node_modules/asn1js": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.0.10.tgz", + "integrity": "sha512-S2s3aOytiKdFRdulw2qPE51MzjzVOisppcVv7jVFR+Kw0kxwvFrDcYA0h7Ndqbmj0HkMIXYWaoj7fli8kgx1eg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "pvtsutils": "^1.3.6", + "pvutils": "^1.1.5", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", @@ -5451,31 +5531,21 @@ } }, "node_modules/ast-v8-to-istanbul": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-0.3.3.tgz", - "integrity": "sha512-MuXMrSLVVoA6sYN/6Hke18vMzrT4TZNbZIj/hvh0fnYFpO+/kFXcLIaiPwXXWaQUPg4yJD8fj+lfJ7/1EBconw==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-1.0.0.tgz", + "integrity": "sha512-1fSfIwuDICFA4LKkCzRPO7F0hzFf0B7+Xqrl27ynQaa+Rh0e1Es0v6kWHPott3lU10AyAr7oKHa65OppjLn3Rg==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/trace-mapping": "^0.3.25", + "@jridgewell/trace-mapping": "^0.3.31", "estree-walker": "^3.0.3", - "js-tokens": "^9.0.1" - } - }, - "node_modules/ast-v8-to-istanbul/node_modules/estree-walker": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" + "js-tokens": "^10.0.0" } }, "node_modules/ast-v8-to-istanbul/node_modules/js-tokens": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", - "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", "dev": true, "license": "MIT" }, @@ -5599,37 +5669,41 @@ } }, "node_modules/axios": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.0.tgz", - "integrity": "sha512-oXTDccv8PcfjZmPGlWsPSwtOJCZ/b6W5jAMCNcfwJbCzDckwG0jrYJFaWH1yvivfCXjVzV/SPDEhMB3Q+DSurg==", + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.0.tgz", + "integrity": "sha512-E32NzpYKp++W7XRe52rHiXV2ehxmh3wbdgO7MHeFM+vqxLBYHzt0ElkiImtOBxtOmyp0yoC8C6uESVV84Y2/hw==", "license": "MIT", "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.4", - "proxy-from-env": "^1.1.0" + "follow-redirects": "^1.16.0", + "form-data": "^4.0.5", + "https-proxy-agent": "^5.0.1", + "proxy-from-env": "^2.1.0" } }, "node_modules/axios/node_modules/form-data": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", - "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "hasown": "^2.0.4", + "mime-types": "^2.1.35" }, "engines": { "node": ">= 6" } }, "node_modules/axios/node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } }, "node_modules/babel-plugin-emotion": { "version": "9.2.11", @@ -5716,14 +5790,6 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha1-6D46fj8wCzTLnYf2FfoMvzV2kO4= sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, - "node_modules/base64-arraybuffer": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.2.0.tgz", - "integrity": "sha1-S5RPrAGRqlkHr+LYyZnMxXzoD0U= sha512-7emyCsu1/xiBXgQZrscw/8KPRT44I4Yq9Pe6EGs3aPRTsWuggML1/1DTuZUuIaJPIm1FTDUVXl4x/yW8s0kQDQ==", - "engines": { - "node": ">= 0.6.0" - } - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -5744,6 +5810,16 @@ } ] }, + "node_modules/baseline-browser-mapping": { + "version": "2.9.19", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz", + "integrity": "sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, "node_modules/batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -5858,6 +5934,22 @@ "dev": true, "license": "MIT" }, + "node_modules/body-parser/node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/body-parser/node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -5880,9 +5972,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -6006,14 +6098,14 @@ "node": ">= 0.8" } }, - "node_modules/cac": { - "version": "6.7.14", - "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", - "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "node_modules/bytestreamjs": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/bytestreamjs/-/bytestreamjs-2.0.1.tgz", + "integrity": "sha512-U1Z/ob71V/bXfVABvNr/Kumf5VyeQRBEm6Txb0PQ6S7V5GpBM3w4Cbqz/xPDicR5tN0uvDifng8C+5qECeGwyQ==", "dev": true, - "license": "MIT", + "license": "BSD-3-Clause", "engines": { - "node": ">=8" + "node": ">=6.0.0" } }, "node_modules/cachedir": { @@ -6112,9 +6204,9 @@ "integrity": "sha1-FkpUg+Yw+kMh5a8HAg5TGDGyYJs=" }, "node_modules/caniuse-lite": { - "version": "1.0.30001731", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001731.tgz", - "integrity": "sha512-lDdp2/wrOmTRWuoB5DpfNkC0rJDU8DqRa6nYL6HK6sytw70QMopt/NIc/9SM7ylItlBWfACXk0tEn37UWM/+mg==", + "version": "1.0.30001768", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001768.tgz", + "integrity": "sha512-qY3aDRZC5nWPgHUgIB84WL+nySuo19wk0VJpp/XI9T34lrvkyhRvNVOFJOp2kxClQhiFBu+TaUSudf6oa3vkSA==", "dev": true, "funding": [ { @@ -6132,33 +6224,6 @@ ], "license": "CC-BY-4.0" }, - "node_modules/canvg": { - "version": "3.0.11", - "resolved": "https://registry.npmjs.org/canvg/-/canvg-3.0.11.tgz", - "integrity": "sha512-5ON+q7jCTgMp9cjpu4Jo6XbvfYwSB2Ow3kzHKfIyJfaCAOHLbdKPQqGKgfED/R5B+3TFFfe8pegYA+b423SRyA==", - "license": "MIT", - "optional": true, - "dependencies": { - "@babel/runtime": "^7.12.5", - "@types/raf": "^3.4.0", - "core-js": "^3.8.3", - "raf": "^3.4.1", - "regenerator-runtime": "^0.13.7", - "rgbcolor": "^1.0.1", - "stackblur-canvas": "^2.0.0", - "svg-pathdata": "^6.0.3" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/canvg/node_modules/regenerator-runtime": { - "version": "0.13.11", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", - "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", - "license": "MIT", - "optional": true - }, "node_modules/caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", @@ -6178,20 +6243,13 @@ } }, "node_modules/chai": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/chai/-/chai-5.2.0.tgz", - "integrity": "sha512-mCuXncKXk5iCLhfhwTc0izo0gtEmpz5CtG2y8GiOINBlMVS6v8TMRc5TaLWKS6692m9+dVVfzgeVxR5UxWHTYw==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", "dev": true, "license": "MIT", - "dependencies": { - "assertion-error": "^2.0.1", - "check-error": "^2.1.1", - "deep-eql": "^5.0.1", - "loupe": "^3.1.0", - "pathval": "^2.0.0" - }, "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/chalk": { @@ -6249,16 +6307,6 @@ "node": "*" } }, - "node_modules/check-error": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.1.tgz", - "integrity": "sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 16" - } - }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -6555,11 +6603,6 @@ "dev": true, "license": "MIT" }, - "node_modules/computed-style-to-inline-style": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/computed-style-to-inline-style/-/computed-style-to-inline-style-3.0.0.tgz", - "integrity": "sha1-klECRGPFVcfhdo13oZ66m49Bivw=" - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -6701,6 +6744,7 @@ "integrity": "sha512-c2KZL9lP4DjkN3hk/an4pWn5b5ZefhRJnAc42n6LJ19kSnbeRbdQZE5dSeE2LBol1OwJD3X1BQvFTAsa8ReeDA==", "hasInstallScript": true, "license": "MIT", + "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/core-js" @@ -6811,14 +6855,6 @@ "jss-preset-default": "^10.10.0" } }, - "node_modules/css-line-break": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/css-line-break/-/css-line-break-1.1.1.tgz", - "integrity": "sha1-1em90peEAJnrBQPHMQ/TSSegJu8= sha512-1feNVaM4Fyzdj4mKPIQNL2n70MmuYzAXZ1aytlROFX1JsOo070OsugwGjj7nl6jnDJWHDM8zRZswkmeYVWZJQA==", - "dependencies": { - "base64-arraybuffer": "^0.2.0" - } - }, "node_modules/css-loader": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-7.1.2.tgz", @@ -6947,14 +6983,14 @@ "integrity": "sha1-w0+CJqlLuxDDLMDXFK/flCKR/EE= sha512-RpFbQGUE74iyPgvr46U9t1xoQBM8T4BL8SxrN66Le2xYAPSaDJJKeztV3awugusb3g3G9iL8StmkBBXhcbbXhg==" }, "node_modules/cypress": { - "version": "15.5.0", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-15.5.0.tgz", - "integrity": "sha512-7jXBsh5hTfjxr9QQONC2IbdTj0nxSyU8x4eiarMZBzXzCj3pedKviUx8JnLcE4vL8e0TsOzp70WSLRORjEssRA==", + "version": "15.14.1", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-15.14.1.tgz", + "integrity": "sha512-AkuiHNSnmm0a+h/horcvbjmY6dWpCe1Ebp1R0LjMP5I6pjMaNA50Mw1YP/d07pLHJ/sV8FZoGecUWFCJ/Nifpw==", "dev": true, "hasInstallScript": true, "license": "MIT", "dependencies": { - "@cypress/request": "^3.0.9", + "@cypress/request": "^3.0.10", "@cypress/xvfb": "^1.2.4", "@types/sinonjs__fake-timers": "8.1.1", "@types/sizzle": "^2.3.2", @@ -6982,7 +7018,7 @@ "hasha": "5.2.2", "is-installed-globally": "~0.4.0", "listr2": "^3.8.3", - "lodash": "^4.17.21", + "lodash": "^4.17.23", "log-symbols": "^4.0.0", "minimist": "^1.2.8", "ospath": "^1.2.2", @@ -6990,11 +7026,11 @@ "process": "^0.11.10", "proxy-from-env": "1.0.0", "request-progress": "^3.0.0", - "semver": "^7.7.1", "supports-color": "^8.1.1", - "systeminformation": "5.27.7", + "systeminformation": "^5.31.1", "tmp": "~0.2.4", "tree-kill": "1.2.2", + "tslib": "1.14.1", "untildify": "^4.0.0", "yauzl": "^2.10.0" }, @@ -7093,19 +7129,6 @@ "node": ">=8.12.0" } }, - "node_modules/cypress/node_modules/semver": { - "version": "7.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", - "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/cypress/node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -7121,6 +7144,13 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/cypress/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "dev": true, + "license": "0BSD" + }, "node_modules/d3": { "version": "7.9.0", "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", @@ -7675,9 +7705,9 @@ "dev": true }, "node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -7740,16 +7770,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/deep-eql": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", - "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/deep-equal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.0.tgz", @@ -8035,15 +8055,6 @@ "integrity": "sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==", "dev": true }, - "node_modules/dom-helpers": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.1.0.tgz", - "integrity": "sha1-V6cm3gSrzCqLv+Zks+IcWEveUU4= sha512-zRRYDhpiKuAJHasOqCm7lBnsd22nrM4+OYI4ASWCxen+ocTMl7BIAKgGag97TlLiTl6rrau5aPe1VGUm9jQBng==", - "dependencies": { - "@babel/runtime": "^7.5.5", - "csstype": "^2.6.6" - } - }, "node_modules/dom-walk": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.1.tgz", @@ -8063,9 +8074,10 @@ } }, "node_modules/dompurify": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.6.tgz", - "integrity": "sha512-/2GogDQlohXPZe6D6NOgQvXLPSYBqIWMnZ8zzOhn09REE4eyAzb+Hed3jhoM9OkuaJ8P6ZGTTVWQKAi8ieIzfQ==", + "version": "3.4.13", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz", + "integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==", + "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" } @@ -8103,13 +8115,6 @@ "node": ">= 0.4" } }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "dev": true, - "license": "MIT" - }, "node_modules/ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -8129,18 +8134,12 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.195", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.195.tgz", - "integrity": "sha512-URclP0iIaDUzqcAyV1v2PgduJ9N0IdXmWsnPzPfelvBmjmZzEy6xJcjb1cXj+TbYqXgtLrjHEoaSIdTYhw4ezg==", + "version": "1.5.286", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.286.tgz", + "integrity": "sha512-9tfDXhJ4RKFNerfjdCcZfufu49vg620741MNs26a9+bhLThdB+plgMeou98CAaHu/WATj2iHOOHTp1hWtABj2A==", "dev": true, "license": "ISC" }, - "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha1-hAyIA7DYBH9P8M+WMXazLU7z7XI= sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true - }, "node_modules/emotion": { "version": "9.2.12", "resolved": "https://registry.npmjs.org/emotion/-/emotion-9.2.12.tgz", @@ -8170,14 +8169,14 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.18.2", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.18.2.tgz", - "integrity": "sha512-6Jw4sE1maoRJo3q8MsSIn2onJFbLTOjY9hlx4DZXmOKvLRd1Ok2kXmAGXaafL2+ijsJZ1ClYbl/pmqr9+k4iUQ==", + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.19.0.tgz", + "integrity": "sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg==", "dev": true, "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", - "tapable": "^2.2.0" + "tapable": "^2.3.0" }, "engines": { "node": ">=10.13.0" @@ -8331,9 +8330,9 @@ } }, "node_modules/es-module-lexer": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", - "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz", + "integrity": "sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==", "dev": true, "license": "MIT" }, @@ -8491,9 +8490,9 @@ } }, "node_modules/eslint": { - "version": "9.39.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.1.tgz", - "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", + "version": "9.39.3", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.3.tgz", + "integrity": "sha512-VmQ+sifHUbI/IcSopBCF/HO3YiHQx/AVd3UVyYL6weuwW+HvON9VYn5l6Zl1WZzPWXPNZrSQpxwkkZ/VuvJZzg==", "dev": true, "license": "MIT", "dependencies": { @@ -8503,7 +8502,7 @@ "@eslint/config-helpers": "^0.4.2", "@eslint/core": "^0.17.0", "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.39.1", + "@eslint/js": "9.39.3", "@eslint/plugin-kit": "^0.4.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", @@ -8633,35 +8632,29 @@ } }, "node_modules/eslint-plugin-cypress": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-cypress/-/eslint-plugin-cypress-5.2.0.tgz", - "integrity": "sha512-vuCUBQloUSILxtJrUWV39vNIQPlbg0L7cTunEAzvaUzv9LFZZym+KFLH18n9j2cZuFPdlxOqTubCvg5se0DyGw==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-cypress/-/eslint-plugin-cypress-6.1.0.tgz", + "integrity": "sha512-B8sxtNpINDxFkmsu1qKYjg70VsP8SGneEXgLcZMk1bUZcW08S+JyaiMdof1x6dmt02FgOD7YkT4wOaOD5HotJw==", "dev": true, "license": "MIT", "dependencies": { - "globals": "^16.2.0" + "globals": "^17.3.0" }, "peerDependencies": { "eslint": ">=9" } }, - "node_modules/eslint-plugin-eslint-comments": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-eslint-comments/-/eslint-plugin-eslint-comments-3.2.0.tgz", - "integrity": "sha1-nhzXtEE1JquzE5MwcderoFyhL/o= sha512-0jkOl0hfojIHHmEHgmNdqv4fmh7300NdpA9FFpF7zaoLvB/QeXOGNLIo86oAveJFrfB1p05kC8hpEMHM8DwWVQ==", + "node_modules/eslint-plugin-cypress/node_modules/globals": { + "version": "17.3.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.3.0.tgz", + "integrity": "sha512-yMqGUQVVCkD4tqjOJf3TnrvaaHDMYp4VlUSObbkIiuCPe/ofdMBFIAcBbCSRFWOnos6qRiTVStDwqPLUclaxIw==", "dev": true, - "dependencies": { - "escape-string-regexp": "^1.0.5", - "ignore": "^5.0.5" - }, + "license": "MIT", "engines": { - "node": ">=6.5.0" + "node": ">=18" }, "funding": { - "url": "https://github.com/sponsors/mysticatea" - }, - "peerDependencies": { - "eslint": ">=4.19.1" + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/eslint-plugin-import": { @@ -8797,9 +8790,9 @@ } }, "node_modules/eslint-plugin-react-hooks": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.0.1.tgz", - "integrity": "sha512-O0d0m04evaNzEPoSW+59Mezf8Qt0InfgGIBJnpC0h3NH/WjUAR7BIKUfysC6todmtiZ/A0oUVS8Gce0WhBrHsA==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.1.1.tgz", + "integrity": "sha512-f2I7Gw6JbvCexzIInuSbZpfdQ44D7iqdWX01FKLvrPgqxoE7oMj8clOfto8U6vYiz4yd5oKu39rRSVOe1zRu0g==", "dev": true, "license": "MIT", "dependencies": { @@ -8813,7 +8806,7 @@ "node": ">=18" }, "peerDependencies": { - "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0" + "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 || ^10.0.0" } }, "node_modules/eslint-plugin-react/node_modules/resolve": { @@ -8834,21 +8827,20 @@ } }, "node_modules/eslint-plugin-testing-library": { - "version": "7.13.4", - "resolved": "https://registry.npmjs.org/eslint-plugin-testing-library/-/eslint-plugin-testing-library-7.13.4.tgz", - "integrity": "sha512-OOrjrJrFsp20xULIy2wDJWjW7qmH2nV0zrs3Y790Ld3+XZFDbtgKvnE//xDANIBna3M9sk9GPYbzkCLI7XhYmg==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-testing-library/-/eslint-plugin-testing-library-7.16.0.tgz", + "integrity": "sha512-lHZI6/Olb2oZqxd1+s1nOLCtL2PXKrc1ERz6oDbUKS0xZAMFH3Fy6wJo75z3pXTop3BV6+loPi2MSjIYt3vpAg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "^8.15.0", - "@typescript-eslint/utils": "^8.15.0" + "@typescript-eslint/scope-manager": "^8.56.0", + "@typescript-eslint/utils": "^8.56.0" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0", - "pnpm": "^9.14.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0" + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0" } }, "node_modules/eslint-scope": { @@ -9019,6 +9011,16 @@ "node": ">=4.0" } }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -9076,9 +9078,9 @@ "integrity": "sha1-KueOhdmJQVhnCwPUe+wfA72Ru50=" }, "node_modules/expect-type": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.2.1.tgz", - "integrity": "sha512-/kP8CAwxzLVEeFrMm4kMmy4CCDlpipyA7MYLVrdJIkV0fYF0UaigQHRsxHiuY/GEea+bh4KSv3TIlgr+2UL6bw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", "dev": true, "license": "Apache-2.0", "engines": { @@ -9156,6 +9158,22 @@ "dev": true, "license": "MIT" }, + "node_modules/express/node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -9248,26 +9266,15 @@ "dev": true, "license": "MIT" }, - "node_modules/fast-png": { - "version": "6.4.0", - "resolved": "https://registry.npmjs.org/fast-png/-/fast-png-6.4.0.tgz", - "integrity": "sha512-kAqZq1TlgBjZcLr5mcN6NP5Rv4V2f22z00c3g8vRrwkcqjerx7BEhPbOnWCPqaHUl2XWQBJQvOT/FQhdMT7X/Q==", - "license": "MIT", - "dependencies": { - "@types/pako": "^2.0.3", - "iobuffer": "^5.3.2", - "pako": "^2.1.0" - } - }, "node_modules/fast-safe-stringify": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", "integrity": "sha1-EkqohYmSYfaK7bQqfAgN6dpgh0M= sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" }, "node_modules/fast-uri": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", - "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", + "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", "dev": true, "funding": [ { @@ -9281,10 +9288,26 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", + "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.5.0", + "xml-naming": "^0.1.0" + } + }, "node_modules/fast-xml-parser": { - "version": "4.5.3", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.3.tgz", - "integrity": "sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig==", + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.8.0.tgz", + "integrity": "sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==", "funding": [ { "type": "github", @@ -9293,7 +9316,11 @@ ], "license": "MIT", "dependencies": { - "strnum": "^1.1.1" + "@nodable/entities": "^2.1.0", + "fast-xml-builder": "^1.2.0", + "path-expression-matcher": "^1.5.0", + "strnum": "^2.3.0", + "xml-naming": "^0.1.0" }, "bin": { "fxparser": "src/cli/cli.js" @@ -9353,11 +9380,6 @@ "pend": "~1.2.0" } }, - "node_modules/fflate": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", - "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==" - }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -9503,24 +9525,25 @@ "license": "ISC" }, "node_modules/focus-trap": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-7.6.2.tgz", - "integrity": "sha512-9FhUxK1hVju2+AiQIDJ5Dd//9R2n2RAfJ0qfhF4IHGHgcoEUTMpbTeG/zbEuwaiYXfuAH6XE0/aCyxDdRM+W5w==", + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-7.6.4.tgz", + "integrity": "sha512-xx560wGBk7seZ6y933idtjJQc1l+ck+pI3sKvhKozdBV1dRZoKhkW5xoCaFv9tQiX5RH1xfSxjuNu6g+lmN/gw==", "license": "MIT", "dependencies": { "tabbable": "^6.2.0" } }, "node_modules/follow-redirects": { - "version": "1.15.6", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", - "integrity": "sha1-f4FcDNpCScdP8J6V75fCO1/QOZs= sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", "funding": [ { "type": "individual", "url": "https://github.com/sponsors/RubenVerborgh" } ], + "license": "MIT", "engines": { "node": ">=4.0" }, @@ -9551,36 +9574,6 @@ "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.4.tgz", "integrity": "sha1-zF0NiuHUbMmlVcJoL5EJd4WZNd8=" }, - "node_modules/foreground-child": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", - "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", - "dev": true, - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.6", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/foreground-child/node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -9592,15 +9585,15 @@ } }, "node_modules/form-data": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.4.tgz", - "integrity": "sha512-f0cRzm6dkyVYV3nPoooP8XlccPQukegwhAnpoLcXy+X+A8KfpGOoXwDr9FLZd3wzgLaBGQBE3lY93Zm/i1JvIQ==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.5.tgz", + "integrity": "sha512-j23EibVLnp4zNXGW7LjryXYa2X6U/M96yoOX+ybZxwkYajdxRNEqYY3zhh7y0i6kfISKS2jr+EJq1YTUDEv5+w==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", + "hasown": "^2.0.4", "mime-types": "^2.1.35" }, "engines": { @@ -9833,11 +9826,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-node-dimensions": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/get-node-dimensions/-/get-node-dimensions-1.2.1.tgz", - "integrity": "sha1-+3tLtXBg+0JH3VHJ1pDfvsVrCCM= sha512-2MSPMu7S1iOTL+BOa6K1S62hB2zUAYNF/lV0gSVlOaacd087lc6nR1H1r0e3B1CerTo+RceOmi1iJW+vp21xcQ==" - }, "node_modules/get-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", @@ -10044,13 +10032,6 @@ "dev": true, "license": "ISC" }, - "node_modules/graphemer": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", - "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", - "dev": true, - "license": "MIT" - }, "node_modules/graphql": { "version": "16.11.0", "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.11.0.tgz", @@ -10184,9 +10165,10 @@ } }, "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", "dependencies": { "function-bind": "^1.1.2" }, @@ -10358,10 +10340,11 @@ } }, "node_modules/html-escaper": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.0.tgz", - "integrity": "sha1-ceh/kx3j/gnlZmGrmimq3scHtJE= sha512-a4u9BeERWGu/S8JiWEAQcdrg9v4QArtP9keViQjGMdff20fBdd8waotXaNmODqBe6uZ3Nafi7K/ho4gCQHV3Ig==", - "dev": true + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" }, "node_modules/html-parse-stringify": { "version": "3.0.1", @@ -10381,17 +10364,6 @@ "node": ">=8" } }, - "node_modules/html2canvas": { - "version": "1.0.0-rc.7", - "resolved": "https://registry.npmjs.org/html2canvas/-/html2canvas-1.0.0-rc.7.tgz", - "integrity": "sha1-cMFZzg5jlUqRFpUxiU0IrVYnrJg= sha512-yvPNZGejB2KOyKleZspjK/NruXVQuowu8NnV2HYG7gW7ytzl+umffbtUI62v2dCHQLDdsK6HIDtyJZ0W3neerA==", - "dependencies": { - "css-line-break": "1.1.1" - }, - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/http-deceiver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", @@ -10508,7 +10480,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", "integrity": "sha1-xZ7yJKBP6LdU89sAY6Jeow0ABdY= sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "dev": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -10618,9 +10589,10 @@ } }, "node_modules/immutable": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.0.0.tgz", - "integrity": "sha1-uG943mre82CDle+yaakUYnl+LCM= sha512-zIE9hX70qew5qTUjSS7wi1iwj/l7+m54KWU247nhM3v806UdGj1yDndXj+IOYxxtW9zyLI+xqFNZjTuDaLUqFw==", + "version": "4.3.9", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.9.tgz", + "integrity": "sha512-ObHy4YN7ycwZOUCLI1/6svfyAFu7vL8RhAvVu/bh/RZW9EPlOyDaQ9jDQWCtdqzaXUjgXZCW1migtHE7YI7UGQ==", + "license": "MIT", "optional": true }, "node_modules/import-cwd": { @@ -10785,12 +10757,6 @@ "loose-envify": "^1.0.0" } }, - "node_modules/iobuffer": { - "version": "5.4.0", - "resolved": "https://registry.npmjs.org/iobuffer/-/iobuffer-5.4.0.tgz", - "integrity": "sha512-DRebOWuqDvxunfkNJAlc3IzWIPD5xVxwUNbHr7xKB8E6aLJxIPfNX3CoMJghcFjpv6RWQsrcJbghtEwSPoJqMA==", - "license": "MIT" - }, "node_modules/ipaddr.js": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz", @@ -11505,39 +11471,10 @@ "node": ">=10" } }, - "node_modules/istanbul-lib-report/node_modules/make-dir": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", - "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^7.5.3" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/istanbul-lib-report/node_modules/semver": { - "version": "7.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", - "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/istanbul-reports": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", - "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -11566,22 +11503,6 @@ "node": ">= 0.4" } }, - "node_modules/jackspeak": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", - "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, "node_modules/jest-worker": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", @@ -11641,6 +11562,28 @@ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha1-GSA/tZmR35jjoocFDUZHzerzJJk= sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", @@ -11694,17 +11637,17 @@ } }, "node_modules/jsdom/node_modules/form-data": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", - "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", "dev": true, "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "hasown": "^2.0.4", + "mime-types": "^2.1.35" }, "engines": { "node": ">= 6" @@ -11798,32 +11741,6 @@ "graceful-fs": "^4.1.6" } }, - "node_modules/jspdf": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jspdf/-/jspdf-4.0.0.tgz", - "integrity": "sha512-w12U97Z6edKd2tXDn3LzTLg7C7QLJlx0BPfM3ecjK2BckUl9/81vZ+r5gK4/3KQdhAcEZhENUxRhtgYBj75MqQ==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.28.4", - "fast-png": "^6.2.0", - "fflate": "^0.8.1" - }, - "optionalDependencies": { - "canvg": "^3.0.11", - "core-js": "^3.6.0", - "dompurify": "^3.2.4", - "html2canvas": "^1.0.0-rc.5" - } - }, - "node_modules/jspdf-autotable": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/jspdf-autotable/-/jspdf-autotable-5.0.7.tgz", - "integrity": "sha512-2wr7H6liNDBYNwt25hMQwXkEWFOEopgKIvR1Eukuw6Zmprm/ZcnmLTQEjW7Xx3FCbD3v7pflLcnMAv/h1jFDQw==", - "license": "MIT", - "peerDependencies": { - "jspdf": "^2 || ^3 || ^4" - } - }, "node_modules/jsprim": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz", @@ -12016,9 +11933,9 @@ } }, "node_modules/jss/node_modules/csstype": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", - "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "license": "MIT" }, "node_modules/jsx-ast-utils": { @@ -12129,13 +12046,17 @@ } }, "node_modules/loader-runner": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", - "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.1.tgz", + "integrity": "sha512-IWqP2SCPhyVFTBtRcgMHdzlf9ul25NwaFx4wCEH/KjAXuuHY4yNjvPXsBokp8jCB936PyWRaPKUNh8NvylLp2Q==", "dev": true, "license": "MIT", "engines": { "node": ">=6.11.5" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" } }, "node_modules/locate-path": { @@ -12154,14 +12075,16 @@ } }, "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha1-Z5WRxWTDv/quhFTPCz3zcMPWkRw= sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" }, "node_modules/lodash-es": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", - "integrity": "sha1-Q+YmxG5lkbd1C+srUBFzkMYJ4+4= sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", + "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", + "license": "MIT" }, "node_modules/lodash.difference": { "version": "4.5.0", @@ -12344,13 +12267,6 @@ "loose-envify": "cli.js" } }, - "node_modules/loupe": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.4.tgz", - "integrity": "sha512-wJzkKwJrheKtknCOKNEtDK4iqg/MxmZheEMtSTYvnzRdEYaZzmgH976nenp8WdJRdx5Vc1X/9MO0Oszl6ezeXg==", - "dev": true, - "license": "MIT" - }, "node_modules/lower-case": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", @@ -12377,8 +12293,9 @@ "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha1-HaJ+ZxAnGUdpXa9oSOhH8B2EuSA= sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", "dev": true, + "license": "ISC", "dependencies": { "yallist": "^3.0.2" } @@ -12397,16 +12314,55 @@ "lz-string": "bin/bin.js" } }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, "node_modules/magicast": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.3.5.tgz", - "integrity": "sha512-L0WhttDl+2BOsybvEOLK7fW3UA0OQ0IQ2d6Zl2x/a6vVRs3bAY0ECOSHHeL5jD+SbOpOCUEi0y1DgHEn9Qn1AQ==", + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.2.tgz", + "integrity": "sha512-E3ZJh4J3S9KfwdjZhe2afj6R9lGIN5Pher1pF39UGrXRqq/VDaGVIGN13BjHd2u8B61hArAGOnso7nBOouW3TQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.0", + "@babel/types": "^7.29.0", + "source-map-js": "^1.2.1" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.25.4", - "@babel/types": "^7.25.4", - "source-map-js": "^1.2.0" + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" } }, "node_modules/mark.js": { @@ -13505,9 +13461,10 @@ } }, "node_modules/min-document": { - "version": "2.19.0", - "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", - "integrity": "sha1-e9KC4/WELtKVu3SM3Z8f+iyCRoU=", + "version": "2.19.2", + "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.2.tgz", + "integrity": "sha512-8S5I8db/uZN8r9HSLFVWPdJCvYOejMcEC82VIzNUc6Zkklf/d1gg2psfE79/vyhWOj4+J8MtwmoOz3TmvaGu5A==", + "license": "MIT", "dependencies": { "dom-walk": "^0.1.0" } @@ -13535,9 +13492,9 @@ "license": "ISC" }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.4.tgz", + "integrity": "sha512-twmL+S8+7yIsE9wsqgzU3E8/LumN3M3QELrBZ20OdmQ9jB2JvW5oZtBEmft84k/Gs5CG9mqtWc6Y9vW+JEzGxw==", "dev": true, "license": "ISC", "dependencies": { @@ -13555,16 +13512,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, "node_modules/mkdirp": { "version": "0.5.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", @@ -13711,17 +13658,10 @@ "mocha": ">=3.1.2" } }, - "node_modules/mocha/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "peer": true - }, "node_modules/mocha/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", "dev": true, "license": "MIT", "peer": true, @@ -13775,24 +13715,12 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/mocha/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "peer": true, - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/mocha/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "version": "5.1.8", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.8.tgz", + "integrity": "sha512-7RN35vit8DeBclkofOVmBY0eDAZZQd1HzmukRdSyz95CRh8FT54eqnbj0krQr3mrHR6sfRyYkyhwBWjoV5uqlQ==", "dev": true, + "license": "ISC", "peer": true, "dependencies": { "brace-expansion": "^2.0.1" @@ -14003,16 +13931,6 @@ "webidl-conversions": "^3.0.0" } }, - "node_modules/node-forge": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.3.tgz", - "integrity": "sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg==", - "dev": true, - "license": "(BSD-3-Clause OR GPL-2.0)", - "engines": { - "node": ">= 6.13.0" - } - }, "node_modules/node-readfiles": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/node-readfiles/-/node-readfiles-0.2.0.tgz", @@ -14022,9 +13940,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.19", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", - "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", "dev": true, "license": "MIT" }, @@ -14345,6 +14263,17 @@ "dev": true, "license": "MIT" }, + "node_modules/obug": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", + "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT" + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -14409,23 +14338,25 @@ } }, "node_modules/openapi-sampler": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/openapi-sampler/-/openapi-sampler-1.6.1.tgz", - "integrity": "sha512-s1cIatOqrrhSj2tmJ4abFYZQK6l5v+V4toO5q1Pa0DyN8mtyqy2I+Qrj5W9vOELEtybIMQs/TBZGVO/DtTFK8w==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/openapi-sampler/-/openapi-sampler-1.7.0.tgz", + "integrity": "sha512-fWq32F5vqGpgRJYIarC/9Y1wC9tKnRDcCOjsDJ7MIcSv2HsE7kNifcXIZ8FVtNStBUWxYrEk/MKqVF0SwZ5gog==", "license": "MIT", "dependencies": { "@types/json-schema": "^7.0.7", - "fast-xml-parser": "^4.5.0", + "fast-xml-parser": "^5.3.4", "json-pointer": "0.6.2" } }, "node_modules/optimism": { - "version": "0.17.5", - "resolved": "https://registry.npmjs.org/optimism/-/optimism-0.17.5.tgz", - "integrity": "sha1-pMeLOtEsWGI6vtvrtPLywZuOiBY= sha512-TEcp8ZwK1RczmvMnvktxHSF2tKgMWjJ71xEFGX5ApLh67VsMSTy1ZUlipJw8W+KaqgOmQ+4pqwkeivY89j+4Vw==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/optimism/-/optimism-0.18.1.tgz", + "integrity": "sha512-mLXNwWPa9dgFyDqkNi54sjDyNJ9/fTI6WGBLgnXku1vdKY/jovHfZT5r+aiVeFFLOz+foPNOm5YJ4mqgld2GBQ==", + "license": "MIT", "dependencies": { + "@wry/caches": "^1.0.0", "@wry/context": "^0.7.0", - "@wry/trie": "^0.4.3", + "@wry/trie": "^0.5.0", "tslib": "^2.3.0" } }, @@ -14544,19 +14475,6 @@ "node": ">=6" } }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "dev": true, - "license": "BlueOak-1.0.0" - }, - "node_modules/pako": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", - "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==", - "license": "(MIT AND Zlib)" - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -14648,7 +14566,22 @@ "integrity": "sha1-UTvb4tO5XXdi6METfvoZXGxhtbM= sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true, "engines": { - "node": ">=8" + "node": ">=8" + } + }, + "node_modules/path-expression-matcher": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", + "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" } }, "node_modules/path-is-absolute": { @@ -14674,30 +14607,6 @@ "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha1-+8EUtgykKzDZ2vWFjkvWi77bZzU= sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, - "node_modules/path-scurry": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", - "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/path-scurry/node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "dev": true, - "license": "ISC" - }, "node_modules/path-to-regexp": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.9.0.tgz", @@ -14722,16 +14631,6 @@ "dev": true, "license": "MIT" }, - "node_modules/pathval": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", - "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14.16" - } - }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -14844,6 +14743,24 @@ "node": ">=8" } }, + "node_modules/pkijs": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pkijs/-/pkijs-3.4.0.tgz", + "integrity": "sha512-emEcLuomt2j03vxD54giVB4SxTjnsqkU692xZOZXHDVoYyypEm+b3jpiTcc+Cf+myooc+/Ly0z01jqeNHVgJGw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@noble/hashes": "1.4.0", + "asn1js": "^3.0.6", + "bytestreamjs": "^2.0.1", + "pvtsutils": "^1.3.6", + "pvutils": "^1.1.3", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/pluralize": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", @@ -15264,9 +15181,9 @@ } }, "node_modules/prettier": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", - "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", + "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", "dev": true, "license": "MIT", "bin": { @@ -15435,13 +15352,34 @@ "purgecss": "bin/purgecss.js" } }, + "node_modules/pvtsutils": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/pvtsutils/-/pvtsutils-1.3.6.tgz", + "integrity": "sha512-PLgQXQ6H2FWCaeRak8vvk1GW462lMxB5s3Jm673N82zI4vqtVUPuZdffdZbPDFRoU8kAhItWFtPCWiPpp4/EDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.8.1" + } + }, + "node_modules/pvutils": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/pvutils/-/pvutils-1.1.5.tgz", + "integrity": "sha512-KTqnxsgGiQ6ZAzZCVlJH5eOjSnvlyEgx1m8bkRJfOhmGRqfo5KLvmAlACQkrjEtOQ4B7wF9TdSLIs9O90MX9xA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/qs": { - "version": "6.14.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", - "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.1.0" + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" }, "engines": { "node": ">=0.6" @@ -15817,21 +15755,6 @@ "resolved": "https://registry.npmjs.org/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz", "integrity": "sha1-TxonOv38jzSIqMUWv9p4+HI1I2I= sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA==" }, - "node_modules/react-measure": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/react-measure/-/react-measure-2.5.2.tgz", - "integrity": "sha1-T/xBDoucuDbZRVqf8Y/B8Pymf4k= sha512-M+rpbTLWJ3FD6FXvYV6YEGvQ5tMayQ3fGrZhRPHrE9bVlBYfDCLuDcgNttYfk8IqfOI03jz6cbpqMRTUclQnaA==", - "dependencies": { - "@babel/runtime": "^7.2.0", - "get-node-dimensions": "^1.2.1", - "prop-types": "^15.6.2", - "resize-observer-polyfill": "^1.5.0" - }, - "peerDependencies": { - "react": ">0.13.0", - "react-dom": ">0.13.0" - } - }, "node_modules/react-modal": { "version": "3.12.1", "resolved": "https://registry.npmjs.org/react-modal/-/react-modal-3.12.1.tgz", @@ -15924,6 +15847,16 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha1-5pHUqOnHiTZWVVOas3J2Kw77VPA= sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==" }, + "node_modules/react-refresh": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.18.0.tgz", + "integrity": "sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/react-responsive": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/react-responsive/-/react-responsive-9.0.2.tgz", @@ -16159,21 +16092,6 @@ "react-dom": ">=16" } }, - "node_modules/react-transition-group": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.1.tgz", - "integrity": "sha1-Y4aPkyWjjqXulTXYKDJ/hXczRck= sha512-Djqr7OQ2aPUiYurhPalTrVy9ddmFCCzwhqQmtN+J3+3DzLO209Fdr70QrN8Z3DsglWql6iY1lDWAfpFiBtuKGw==", - "dependencies": { - "@babel/runtime": "^7.5.5", - "dom-helpers": "^5.0.1", - "loose-envify": "^1.4.0", - "prop-types": "^15.6.2" - }, - "peerDependencies": { - "react": ">=16.6.0", - "react-dom": ">=16.6.0" - } - }, "node_modules/react-truncate": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/react-truncate/-/react-truncate-2.4.0.tgz", @@ -16507,6 +16425,13 @@ "redux": "^4" } }, + "node_modules/reflect-metadata": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", + "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", @@ -16640,7 +16565,9 @@ "node_modules/resize-observer-polyfill": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz", - "integrity": "sha1-DpAg3T0hAkRY1OvSfiPkAmmBBGQ= sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==" + "integrity": "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==", + "license": "MIT", + "peer": true }, "node_modules/resolve": { "version": "1.22.10", @@ -16741,15 +16668,6 @@ "node": ">=0.10.0" } }, - "node_modules/rgbcolor": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/rgbcolor/-/rgbcolor-1.0.1.tgz", - "integrity": "sha1-1lBezbMEplldom+ktDMHMGd1lF0=", - "optional": true, - "engines": { - "node": ">= 0.8.15" - } - }, "node_modules/robust-predicates": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", @@ -16921,9 +16839,9 @@ } }, "node_modules/schema-utils": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.2.tgz", - "integrity": "sha512-Gn/JaSk/Mt9gYubxTtSn/QCV4em9mpAPiR1rqy/Ocu19u/G9J5WWdNoUT4SiV6mFC3y6cxyFcFwdzPM3FgxGAQ==", + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.3.tgz", + "integrity": "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA==", "dev": true, "license": "MIT", "dependencies": { @@ -16985,17 +16903,17 @@ "license": "MIT" }, "node_modules/selfsigned": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.4.1.tgz", - "integrity": "sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==", + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-5.5.0.tgz", + "integrity": "sha512-ftnu3TW4+3eBfLRFnDEkzGxSF/10BJBkaLJuBHZX0kiPS7bRdlpZGu6YGt4KngMkdTwJE6MbjavFpqHvqVt+Ew==", "dev": true, "license": "MIT", "dependencies": { - "@types/node-forge": "^1.3.0", - "node-forge": "^1" + "@peculiar/x509": "^1.14.2", + "pkijs": "^3.3.3" }, "engines": { - "node": ">=10" + "node": ">=18" } }, "node_modules/semver": { @@ -17328,14 +17246,14 @@ "integrity": "sha1-+w1xM49TKjoUkhNjni0yy+qLyyg= sha512-oXF8tfxx5cDk8r2kYqlkUJzZpDBqVY/II2WhvU0n9Y3XYvAYRmeaf1PvvIvTgPnv4KJ+ES5M0PyDq5Jp+Ygy2g==" }, "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" }, @@ -17347,13 +17265,13 @@ } }, "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" + "object-inspect": "^1.13.4" }, "engines": { "node": ">= 0.4" @@ -17617,15 +17535,6 @@ "dev": true, "license": "MIT" }, - "node_modules/stackblur-canvas": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/stackblur-canvas/-/stackblur-canvas-2.4.0.tgz", - "integrity": "sha1-Ky66kQy0b2/q6RjhxAL4Y9YCwBs= sha512-Z+HixfgYV0ss3C342DxPwc+UvN1SYWqoz7Wsi3xEDWEnaBkSCL3Ey21gF4io+WlLm8/RIrSnCrDBIEcH4O+q5Q==", - "optional": true, - "engines": { - "node": ">=0.1.14" - } - }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -17637,9 +17546,9 @@ } }, "node_modules/std-env": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.9.0.tgz", - "integrity": "sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.0.0.tgz", + "integrity": "sha512-zUMPtQ/HBY3/50VbpkupYHbRroTRZJPRLvreamgErJVys0ceuzMkD44J/QjqhHjOzK42GQ3QZIeFG1OYfOtKqQ==", "dev": true, "license": "MIT" }, @@ -17693,29 +17602,6 @@ "node": ">=8" } }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, "node_modules/string-width/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -17828,20 +17714,6 @@ "node": ">=8" } }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", @@ -17884,37 +17756,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strip-literal": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.0.0.tgz", - "integrity": "sha512-TcccoMhJOM3OebGhSBEmp3UZ2SfDMZUEBdRA/9ynfLi8yYajyWX3JiXArcJt4Umh4vISpspkQIY8ZZoCqjbviA==", - "dev": true, - "license": "MIT", - "dependencies": { - "js-tokens": "^9.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, - "node_modules/strip-literal/node_modules/js-tokens": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", - "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", - "dev": true, - "license": "MIT" - }, "node_modules/strnum": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz", - "integrity": "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.0.tgz", + "integrity": "sha512-sHrVyWWdq28RbhjuJdZsA1SnGRJV6NiXbk6AXBxDOsgAcA+lmpUZCYjOdLBxkXMwis6RRe7dlZt4VlIWFVzkmg==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } ], - "license": "MIT" + "license": "MIT", + "dependencies": { + "anynum": "^1.0.0" + } }, "node_modules/style-loader": { "version": "4.0.0", @@ -18033,16 +17888,6 @@ "integrity": "sha1-/cLinhOVFzYUC3bLEiyO5mMOtrU= sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==", "dev": true }, - "node_modules/svg-pathdata": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/svg-pathdata/-/svg-pathdata-6.0.3.tgz", - "integrity": "sha512-qsjeeq5YjBZ5eMdFuUa4ZosMLxgr5RZ+F+Y1OrDhuOCEInRMA3x74XdBtggJcj9kOeInz0WE+LgCPDkZFlBYJw==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=12.0.0" - } - }, "node_modules/swagger2openapi": { "version": "7.0.8", "resolved": "https://registry.npmjs.org/swagger2openapi/-/swagger2openapi-7.0.8.tgz", @@ -18109,9 +17954,9 @@ } }, "node_modules/systeminformation": { - "version": "5.27.7", - "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.27.7.tgz", - "integrity": "sha512-saaqOoVEEFaux4v0K8Q7caiauRwjXC4XbD2eH60dxHXbpKxQ8kH9Rf7Jh+nryKpOUSEFxtCdBlSUx0/lO6rwRg==", + "version": "5.31.6", + "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.31.6.tgz", + "integrity": "sha512-Uv2b2uGGM6ns+26czgW2cYRabYdnswM0ddSOOlryHOaelzsmDSet1iM/NT7VOYxW8x/BW+HkY+b1Ve2pLTSGSA==", "dev": true, "license": "MIT", "os": [ @@ -18136,9 +17981,9 @@ } }, "node_modules/tabbable": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz", - "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.4.0.tgz", + "integrity": "sha512-05PUHKSNE8ou2dwIxTngl4EzcnsCDZGJ/iCLtDflR/SHB/ny14rXc+qU5P4mG9JkusiV7EivzY9Mhm55AzAvCg==", "license": "MIT" }, "node_modules/tailwindcss": { @@ -18181,13 +18026,17 @@ } }, "node_modules/tapable": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.2.tgz", - "integrity": "sha512-Re10+NauLTMCudc7T5WLFLAwDhQ0JWdrMK+9B2M8zR5hRExKmsRDCBA7/aV/pNJFltmBFO5BAMlQFi/vq3nKOg==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.0.tgz", + "integrity": "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==", "dev": true, "license": "MIT", "engines": { "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" } }, "node_modules/terser": { @@ -18209,9 +18058,9 @@ } }, "node_modules/terser-webpack-plugin": { - "version": "5.3.14", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz", - "integrity": "sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==", + "version": "5.3.16", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.16.tgz", + "integrity": "sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q==", "dev": true, "license": "MIT", "dependencies": { @@ -18341,21 +18190,24 @@ } }, "node_modules/tinyexec": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", - "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.4.tgz", + "integrity": "sha512-u9r3uZC0bdpGOXtlxUIdwf9pkmvhqJdrVCH9fapQtgy/OeTTMZ1nqH7agtvEfmGui6e1XxjcdrlxvxJvc3sMqw==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=18" + } }, "node_modules/tinyglobby": { - "version": "0.2.14", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.14.tgz", - "integrity": "sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==", + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", "dev": true, "license": "MIT", "dependencies": { - "fdir": "^6.4.4", - "picomatch": "^4.0.2" + "fdir": "^6.5.0", + "picomatch": "^4.0.3" }, "engines": { "node": ">=12.0.0" @@ -18365,11 +18217,14 @@ } }, "node_modules/tinyglobby/node_modules/fdir": { - "version": "6.4.6", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.6.tgz", - "integrity": "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==", + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", "dev": true, "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, "peerDependencies": { "picomatch": "^3 || ^4" }, @@ -18380,9 +18235,9 @@ } }, "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", "engines": { @@ -18392,30 +18247,10 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/tinypool": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", - "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.0.0 || >=20.0.0" - } - }, "node_modules/tinyrainbow": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", - "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/tinyspy": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.3.tgz", - "integrity": "sha512-t2T/WLB2WRgZ9EpE4jgPJ9w+i66UZfDc8wHh0xrwiRNN+UwH98GIJkTeZqX9rg0i0ptwzqW+uYeIF0T4F8LR7A==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", "dev": true, "license": "MIT", "engines": { @@ -18443,9 +18278,9 @@ "license": "MIT" }, "node_modules/tmp": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", - "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.6.tgz", + "integrity": "sha512-5sJPdPjfI5Kx+qbrDesxkglRBxW//g7hCsqspEjwkewGvBMGIKMOTKzLt1hFVJzyadba3lDUN20O9qhvbQUSTA==", "dev": true, "license": "MIT", "engines": { @@ -18562,9 +18397,9 @@ } }, "node_modules/ts-api-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", - "integrity": "sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.4.0.tgz", + "integrity": "sha512-3TaVTaAv2gTiMB35i3FiGJaRfwb3Pyn/j3m/bfAvGe8FB7CF6u+LMYqYlDh7reQf7UNvoTvdfAqHGmPGOSsPmA==", "dev": true, "license": "MIT", "engines": { @@ -18659,6 +18494,26 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tsyringe": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/tsyringe/-/tsyringe-4.10.0.tgz", + "integrity": "sha512-axr3IdNuVIxnaK5XGEUFTu3YmAQ6lllgrvqfEoR16g/HGnYY/6We4oWENtAnzK6/LpJ2ur9PAb80RBt7/U4ugw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^1.9.3" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/tsyringe/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "dev": true, + "license": "0BSD" + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -19167,279 +19022,462 @@ "extsprintf": "^1.2.0" } }, - "node_modules/victory-area": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-area/-/victory-area-36.9.2.tgz", - "integrity": "sha512-32aharvPf2RgdQB+/u1j3/ajYFNH/7ugLX9ZRpdd65gP6QEbtXL+58gS6CxvFw6gr/y8a0xMlkMKkpDVacXLpw==", + "node_modules/victory": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory/-/victory-37.3.6.tgz", + "integrity": "sha512-CZ1vjvra0R1U3T2dMI4EsjI8Ng+JmQ2ox/EweSzjkTnHfW/Vn5ylryadawDiYjDMcBvABjO3uODsIlSEm4d/Sw==", + "license": "MIT", + "dependencies": { + "victory-area": "37.3.6", + "victory-axis": "37.3.6", + "victory-bar": "37.3.6", + "victory-box-plot": "37.3.6", + "victory-brush-container": "37.3.6", + "victory-brush-line": "37.3.6", + "victory-candlestick": "37.3.6", + "victory-canvas": "37.3.6", + "victory-chart": "37.3.6", + "victory-core": "37.3.6", + "victory-create-container": "37.3.6", + "victory-cursor-container": "37.3.6", + "victory-errorbar": "37.3.6", + "victory-group": "37.3.6", + "victory-histogram": "37.3.6", + "victory-legend": "37.3.6", + "victory-line": "37.3.6", + "victory-pie": "37.3.6", + "victory-polar-axis": "37.3.6", + "victory-scatter": "37.3.6", + "victory-selection-container": "37.3.6", + "victory-shared-events": "37.3.6", + "victory-stack": "37.3.6", + "victory-tooltip": "37.3.6", + "victory-voronoi": "37.3.6", + "victory-voronoi-container": "37.3.6", + "victory-zoom-container": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "react": ">=16.6.0" + } + }, + "node_modules/victory-area": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-area/-/victory-area-37.3.6.tgz", + "integrity": "sha512-wVC8LKrZJLiSySNuJLRCB449qZTsPiRyzLlNoJwe21y+XA/a2HJbmJSeywmo8P153aX8viKe1H8ygDsTFXQhHw==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.19", + "victory-core": "37.3.6", + "victory-vendor": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "react": ">=16.6.0" + } + }, + "node_modules/victory-axis": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-axis/-/victory-axis-37.3.6.tgz", + "integrity": "sha512-Vi0dZvgmXmnCdoqc49WckeG5cMXnl7FTtqVhXu9JweA9cgCnkZabBd5mRvAjblb3Lo4j0HZCSPKHYWUPW70qZg==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.19", + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "react": ">=16.6.0" + } + }, + "node_modules/victory-bar": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-bar/-/victory-bar-37.3.6.tgz", + "integrity": "sha512-jdATFRWL1LUW/yEpKWx/aId2BiU2o1pPF9+Kh1TFISBduJoI4ZqvZD90H1QK4f/z50PikqiqiDECaKoKM1jfOQ==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.19", + "victory-core": "37.3.6", + "victory-vendor": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "react": ">=16.6.0" + } + }, + "node_modules/victory-box-plot": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-box-plot/-/victory-box-plot-37.3.6.tgz", + "integrity": "sha512-GOucnD63h14ScBuISC/nd1GBTEx6gIZfLE+0P0gyeH1poBKq0trTTvpQDvAMuGR8zICfEETG3ltmUMCwRrFyUg==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2", - "victory-vendor": "^36.9.2" + "victory-core": "37.3.6", + "victory-vendor": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, - "node_modules/victory-axis": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-axis/-/victory-axis-36.9.2.tgz", - "integrity": "sha512-4Odws+IAjprJtBg2b2ZCxEPgrQ6LgIOa22cFkGghzOSfTyNayN4M3AauNB44RZyn2O/hDiM1gdBkEg1g9YDevQ==", + "node_modules/victory-brush-container": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-brush-container/-/victory-brush-container-37.3.6.tgz", + "integrity": "sha512-LfZ2CgX1cYAqCtYxcSB68OfZS2v0T2VLXoEArd0lCXfRBY1Gya7GacCUcuo7GoK9XOXeslx7S/U95aVutt1VLg==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2" + "react-fast-compare": "^3.2.0", + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, - "node_modules/victory-bar": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-bar/-/victory-bar-36.9.2.tgz", - "integrity": "sha512-R3LFoR91FzwWcnyGK2P8DHNVv9gsaWhl5pSr2KdeNtvLbZVEIvUkTeVN9RMBMzterSFPw0mbWhS1Asb3sV6PPw==", + "node_modules/victory-brush-line": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-brush-line/-/victory-brush-line-37.3.6.tgz", + "integrity": "sha512-zsZJfF1fUj4F7mUoIMV+h73qoTClPA4bKM1terlYrDBD8l/c/f0KBbEotu3E1X+n4QMmDRruswaB/YUdqK5QLA==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2", - "victory-vendor": "^36.9.2" + "react-fast-compare": "^3.2.0", + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, - "node_modules/victory-box-plot": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-box-plot/-/victory-box-plot-36.9.2.tgz", - "integrity": "sha512-nUD45V/YHDkAKZyak7YDsz+Vk1F9N0ica3jWQe0AY0JqD9DleHa8RY/olSVws26kLyEj1I+fQqva6GodcLaIqQ==", + "node_modules/victory-candlestick": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-candlestick/-/victory-candlestick-37.3.6.tgz", + "integrity": "sha512-h/mOmkCrsWrirn4dFnpLxJPXpxT+uHxuYxnXGrAyH+YUOrVj3iKaDJlEiVlz5vy30syE5j5hzTQCMsZ/hzHNdg==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2", - "victory-vendor": "^36.9.2" + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, - "node_modules/victory-brush-container": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-brush-container/-/victory-brush-container-36.9.2.tgz", - "integrity": "sha512-KcQjzFeo40tn52cJf1A02l5MqeR9GKkk3loDqM3T2hfi1PCyUrZXEUjGN5HNlLizDRvtcemaAHNAWlb70HbG/g==", + "node_modules/victory-canvas": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-canvas/-/victory-canvas-37.3.6.tgz", + "integrity": "sha512-1CD4S0uZ92sUGGSIEQferEfSqd/z9EXw9G6zkzPIoJeTKFshpfqCjUkNRx9Iu9Upxt3fUpId8Qwl1YfchmbrFg==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "react-fast-compare": "^3.2.0", - "victory-core": "^36.9.2" + "victory-bar": "37.3.6", + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-chart": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-chart/-/victory-chart-36.9.2.tgz", - "integrity": "sha512-dMNcS0BpqL3YiGvI4BSEmPR76FCksCgf3K4CSZ7C/MGyrElqB6wWwzk7afnlB1Qr71YIHXDmdwsPNAl/iEwTtA==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-chart/-/victory-chart-37.3.6.tgz", + "integrity": "sha512-IkPo/W4AJ7bPu902TGER09OseR9ODm+FQAKfOBw4JsdEhZZ7BiG9zgd/25+x0r5EsTLu81CYGQVkBa+ZazcOlA==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", "react-fast-compare": "^3.2.0", - "victory-axis": "^36.9.2", - "victory-core": "^36.9.2", - "victory-polar-axis": "^36.9.2", - "victory-shared-events": "^36.9.2" + "victory-axis": "37.3.6", + "victory-core": "37.3.6", + "victory-polar-axis": "37.3.6", + "victory-shared-events": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-core": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-core/-/victory-core-36.9.2.tgz", - "integrity": "sha512-AzmMy+9MYMaaRmmZZovc/Po9urHne3R3oX7bbXeQdVuK/uMBrlPiv11gVJnuEH2SXLVyep43jlKgaBp8ef9stQ==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-core/-/victory-core-37.3.6.tgz", + "integrity": "sha512-aFgO6KokxPbUCPznZP5UPhOdI22pMuwDXKDt6eoQOnkVim66Ia+K95TQar2nwVKGYV5j26aKVf/n9blwphGJRw==", "license": "MIT", "dependencies": { "lodash": "^4.17.21", "react-fast-compare": "^3.2.0", - "victory-vendor": "^36.9.2" + "victory-vendor": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-create-container": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-create-container/-/victory-create-container-36.9.2.tgz", - "integrity": "sha512-uA0dh1R0YDzuXyE/7StZvq4qshet+WYceY7R1UR5mR/F9079xy+iQsa2Ca4h97/GtVZoLO6r1eKLWBt9TN+U7A==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-create-container/-/victory-create-container-37.3.6.tgz", + "integrity": "sha512-Uf5bFQvqUsXCjqpvBW4LhrdrHkM6dBqxYgub6FCsBb86f84xZQ3vY7jFkg/JfvF0oGKMoWXYYrYLC1sk+fcWVA==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-brush-container": "^36.9.2", - "victory-core": "^36.9.2", - "victory-cursor-container": "^36.9.2", - "victory-selection-container": "^36.9.2", - "victory-voronoi-container": "^36.9.2", - "victory-zoom-container": "^36.9.2" + "victory-brush-container": "37.3.6", + "victory-core": "37.3.6", + "victory-cursor-container": "37.3.6", + "victory-selection-container": "37.3.6", + "victory-voronoi-container": "37.3.6", + "victory-zoom-container": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-cursor-container": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-cursor-container/-/victory-cursor-container-36.9.2.tgz", - "integrity": "sha512-jidab4j3MaciF3fGX70jTj4H9rrLcY8o2LUrhJ67ZLvEFGGmnPtph+p8Fe97Umrag7E/DszjNxQZolpwlgUh3g==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-cursor-container/-/victory-cursor-container-37.3.6.tgz", + "integrity": "sha512-+Oiw57d5nE+iq8As8RvepknzmNtKq1Gsc50u1X3IRd4jXtX8zqZrgXGlVZ+BP/tkLsWnGYVjKulwKBf2oaEUuw==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.19", + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "react": ">=16.6.0" + } + }, + "node_modules/victory-errorbar": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-errorbar/-/victory-errorbar-37.3.6.tgz", + "integrity": "sha512-WGAv/qizOlfmwKv+Yfxr4q6pDgTfloNQwi3Z3M0h8povjMZt74tHYkvi/TASSRYr3zv5kjUqUJ28qAyGMWwryQ==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2" + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-group": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-group/-/victory-group-36.9.2.tgz", - "integrity": "sha512-wBmpsjBTKva8mxHvHNY3b8RE58KtnpLLItEyyAHaYkmExwt3Uj8Cld3sF3vmeuijn2iR64NPKeMbgMbfZJzycw==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-group/-/victory-group-37.3.6.tgz", + "integrity": "sha512-kgy/Azl5BxwlJAV0KDPGypv35TMrOD1J2ZxnJW2Wyyq+e8i0GGBIv5MoBzou64BRsDlS9V0CYRIjnkHgrBpB5w==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", "react-fast-compare": "^3.2.0", - "victory-core": "^36.9.2", - "victory-shared-events": "^36.9.2" + "victory-core": "37.3.6", + "victory-shared-events": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "react": ">=16.6.0" + } + }, + "node_modules/victory-histogram": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-histogram/-/victory-histogram-37.3.6.tgz", + "integrity": "sha512-K4d43MpXHYnGCLEMzfRpJ+lCRRDKALPi/juxfMGVzBPzSMgjC8h9x6hKdxaejiTd/E04UdzNO7J24plL3Uz8rA==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.19", + "react-fast-compare": "^3.2.0", + "victory-bar": "37.3.6", + "victory-core": "37.3.6", + "victory-vendor": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-legend": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-legend/-/victory-legend-36.9.2.tgz", - "integrity": "sha512-cucFJpv6fty+yXp5pElQFQnHBk1TqA4guGUMI+XF/wLlnuM4bhdAtASobRIIBkz0mHGBaCAAV4PzL9azPU/9dg==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-legend/-/victory-legend-37.3.6.tgz", + "integrity": "sha512-vRRrhj3/ENqKVLdaBMzEmR83N6BOjox1bthYT1eJjN2H5SIK35bxn30IkiV/Pz3y627EqZe4TAWaxc0jiJlCiA==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2" + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-line": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-line/-/victory-line-36.9.2.tgz", - "integrity": "sha512-kmYFZUo0o2xC8cXRsmt/oUBRQSZJVT2IJnAkboUepypoj09e6CY5tRH4TSdfEDGkBk23xQkn7d4IFgl4kAGnSA==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-line/-/victory-line-37.3.6.tgz", + "integrity": "sha512-Ke817uf/qFbN9jU7Dba7CrcHXYO5wAZuKKnyeHJmLDeQeFST0773xejnIuC+dBgZipjFr4KIbSd+VcUafFNE1g==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2", - "victory-vendor": "^36.9.2" + "victory-core": "37.3.6", + "victory-vendor": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-pie": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-pie/-/victory-pie-36.9.2.tgz", - "integrity": "sha512-i3zWezvy5wQEkhXKt4rS9ILGH7Vr9Q5eF9fKO4GMwDPBdYOTE3Dh2tVaSrfDC8g9zFIc0DKzOtVoJRTb+0AkPg==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-pie/-/victory-pie-37.3.6.tgz", + "integrity": "sha512-tvdgAZ/HQWlo3KDDe0XAVbizHuaNMbgkkiF7zfA7Ww+3bHSs+0P9dsDtK2xP365D8gBCOv8pWmuzvKRhzNbqeA==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2", - "victory-vendor": "^36.9.2" + "victory-core": "37.3.6", + "victory-vendor": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-polar-axis": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-polar-axis/-/victory-polar-axis-36.9.2.tgz", - "integrity": "sha512-HBR90FF4M56yf/atXjSmy3DMps1vSAaLXmdVXLM/A5g+0pUS7HO719r5x6dsR3I6Rm+8x6Kk8xJs0qgpnGQIEw==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-polar-axis/-/victory-polar-axis-37.3.6.tgz", + "integrity": "sha512-RpFsCkzHezJq5P+C/wtVdjEHX25JIFsSgs6qYSnfr/hayaFbWgK5HhRFpriQm5hg61cx47WxAOLyHvzf0nasvw==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2" + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-scatter": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-scatter/-/victory-scatter-36.9.2.tgz", - "integrity": "sha512-hK9AtbJQfaW05i8BH7Lf1HK7vWMAfQofj23039HEQJqTKbCL77YT+Q0LhZw1a1BRCpC/5aSg9EuqblhfIYw2wg==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-scatter/-/victory-scatter-37.3.6.tgz", + "integrity": "sha512-fp95zMTPXgW1cmTowzDXhn+KxePMVDrzU0lotsHQMdBV7eB+ioXdu9hORlx4VHmMYg2ihsGwRTF+VAZ7rGxphA==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2" + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-selection-container": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-selection-container/-/victory-selection-container-36.9.2.tgz", - "integrity": "sha512-chboroEwqqVlMB60kveXM2WznJ33ZM00PWkFVCoJDzHHlYs7TCADxzhqet2S67SbZGSyvSprY2YztSxX8kZ+XQ==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-selection-container/-/victory-selection-container-37.3.6.tgz", + "integrity": "sha512-gd3qODDlBtLEJM7+2jCXk2YcLBUmIpYEEHswytMhwc6zihxXipGBUHRulhLj/I05mKay2gaOAg5ewiJHd4Awgw==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2" + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-shared-events": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-shared-events/-/victory-shared-events-36.9.2.tgz", - "integrity": "sha512-W/atiw3Or6MnpBuhluFv6007YrixIRh5NtiRvtFLGxNuQJLYjaSh6koRAih5xJer5Pj7YUx0tL9x67jTRcJ6Dg==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-shared-events/-/victory-shared-events-37.3.6.tgz", + "integrity": "sha512-ygrbOtzLUTbtKebacZKyQRekhSAROnAvMkVI/PKsAGsz0ClY9P7qDEJG7eTUUygjO6ax0tI6WNE6JogQzeD1gw==", "license": "MIT", "dependencies": { "json-stringify-safe": "^5.0.1", "lodash": "^4.17.19", "react-fast-compare": "^3.2.0", - "victory-core": "^36.9.2" + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-stack": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-stack/-/victory-stack-36.9.2.tgz", - "integrity": "sha512-imR6FniVlDFlBa/B3Est8kTryNhWj2ZNpivmVOebVDxkKcVlLaDg3LotCUOI7NzOhBQaro0UzeE9KmZV93JcYA==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-stack/-/victory-stack-37.3.6.tgz", + "integrity": "sha512-ldod04RdqGJGH5p5eWXCofdTkbhZqIp3iwW7NpxSbMDLs8zPQIVvDFVtuJgMwQiC5vnIpbhMmxVeFbr8m64ZKA==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", "react-fast-compare": "^3.2.0", - "victory-core": "^36.9.2", - "victory-shared-events": "^36.9.2" + "victory-core": "37.3.6", + "victory-shared-events": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-tooltip": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-tooltip/-/victory-tooltip-36.9.2.tgz", - "integrity": "sha512-76seo4TWD1WfZHJQH87IP3tlawv38DuwrUxpnTn8+uW6/CUex82poQiVevYdmJzhataS9jjyCWv3w7pOmLBCLg==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-tooltip/-/victory-tooltip-37.3.6.tgz", + "integrity": "sha512-vqaJS9noauOqDDBBAV9Ln9duOY/i17h1DCfCPAqhwPFyvFbwKvAub9zPTeYWAm/14VvWX5O/0yekFCVbcC7hjg==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2" + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-vendor": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz", - "integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-37.3.6.tgz", + "integrity": "sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ==", "license": "MIT AND ISC", "dependencies": { "@types/d3-array": "^3.0.3", @@ -19458,39 +19496,62 @@ "d3-timer": "^3.0.1" } }, + "node_modules/victory-voronoi": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-voronoi/-/victory-voronoi-37.3.6.tgz", + "integrity": "sha512-Q+1FWHp8IAbmDL9pGWS0y0N4Cb5qmD9OOgxoxCfIDsLlhGvd6LddhRoknWsN7WnreaK+XiwjSfQkdMTCZ4hdhQ==", + "license": "MIT", + "dependencies": { + "d3-voronoi": "^1.1.4", + "lodash": "^4.17.19", + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "react": ">=16.6.0" + } + }, "node_modules/victory-voronoi-container": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-voronoi-container/-/victory-voronoi-container-36.9.2.tgz", - "integrity": "sha512-NIVYqck9N4OQnEz9mgQ4wILsci3OBWWK7RLuITGHyoD7Ne/+WH1i0Pv2y9eIx+f55rc928FUTugPPhkHvXyH3A==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-voronoi-container/-/victory-voronoi-container-37.3.6.tgz", + "integrity": "sha512-qAAG0rMuK7A4EoJ4cyUk5wNdOW+HuCXNKPOko+hYK6wWOYXJvFhiglYyA85a695YyAXECc6JyJS/crm4IOEFag==", "license": "MIT", "dependencies": { "delaunay-find": "0.0.6", "lodash": "^4.17.19", "react-fast-compare": "^3.2.0", - "victory-core": "^36.9.2", - "victory-tooltip": "^36.9.2" + "victory-core": "37.3.6", + "victory-tooltip": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/victory-zoom-container": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-zoom-container/-/victory-zoom-container-36.9.2.tgz", - "integrity": "sha512-pXa2Ji6EX/pIarKT6Hcmmu2n7IG/x8Vs0D2eACQ/nbpvZa+DXWIxCRW4hcg2Va35fmXcDIEpGaX3/soXzZ+pbw==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-zoom-container/-/victory-zoom-container-37.3.6.tgz", + "integrity": "sha512-AGL+k20mI44OL5b0VgIxlmnNSefIoFmbbim5NraPmIxbtns9qQW/56ivIncJcYomBungIx99gUpsEpcQaMNHgQ==", "license": "MIT", "dependencies": { "lodash": "^4.17.19", - "victory-core": "^36.9.2" + "victory-core": "37.3.6" + }, + "engines": { + "node": ">=18.0.0" }, "peerDependencies": { "react": ">=16.6.0" } }, "node_modules/vite": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", - "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz", + "integrity": "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==", "dev": true, "license": "MIT", "dependencies": { @@ -19562,29 +19623,6 @@ } } }, - "node_modules/vite-node": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", - "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cac": "^6.7.14", - "debug": "^4.4.1", - "es-module-lexer": "^1.7.0", - "pathe": "^2.0.3", - "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" - }, - "bin": { - "vite-node": "vite-node.mjs" - }, - "engines": { - "node": "^18.0.0 || ^20.0.0 || >=22.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, "node_modules/vite-plugin-svgr": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/vite-plugin-svgr/-/vite-plugin-svgr-4.3.0.tgz", @@ -19848,13 +19886,6 @@ "@svgr/core": "*" } }, - "node_modules/vite-plugin-svgr/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, "node_modules/vite-plugin-svgr/node_modules/cosmiconfig": { "version": "8.3.6", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", @@ -19902,19 +19933,6 @@ "dev": true, "license": "MIT" }, - "node_modules/vite-plugin-svgr/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/vite-plugin-svgr/node_modules/picomatch": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", @@ -19997,65 +20015,71 @@ } }, "node_modules/vitest": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.4.tgz", - "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/chai": "^5.2.2", - "@vitest/expect": "3.2.4", - "@vitest/mocker": "3.2.4", - "@vitest/pretty-format": "^3.2.4", - "@vitest/runner": "3.2.4", - "@vitest/snapshot": "3.2.4", - "@vitest/spy": "3.2.4", - "@vitest/utils": "3.2.4", - "chai": "^5.2.0", - "debug": "^4.4.1", - "expect-type": "^1.2.1", - "magic-string": "^0.30.17", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.0.tgz", + "integrity": "sha512-YbDrMF9jM2Lqc++2530UourxZHmkKLxrs4+mYhEwqWS97WJ7wOYEkcr+QfRgJ3PW9wz3odRijLZjHEaRLTNbqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.0", + "@vitest/mocker": "4.1.0", + "@vitest/pretty-format": "4.1.0", + "@vitest/runner": "4.1.0", + "@vitest/snapshot": "4.1.0", + "@vitest/spy": "4.1.0", + "@vitest/utils": "4.1.0", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", "pathe": "^2.0.3", - "picomatch": "^4.0.2", - "std-env": "^3.9.0", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", - "tinyexec": "^0.3.2", - "tinyglobby": "^0.2.14", - "tinypool": "^1.1.1", - "tinyrainbow": "^2.0.0", - "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", - "vite-node": "3.2.4", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.0.3", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0", "why-is-node-running": "^2.3.0" }, "bin": { "vitest": "vitest.mjs" }, "engines": { - "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { "@edge-runtime/vm": "*", - "@types/debug": "^4.1.12", - "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", - "@vitest/browser": "3.2.4", - "@vitest/ui": "3.2.4", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.0", + "@vitest/browser-preview": "4.1.0", + "@vitest/browser-webdriverio": "4.1.0", + "@vitest/ui": "4.1.0", "happy-dom": "*", - "jsdom": "*" + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0" }, "peerDependenciesMeta": { "@edge-runtime/vm": { "optional": true }, - "@types/debug": { + "@opentelemetry/api": { "optional": true }, "@types/node": { "optional": true }, - "@vitest/browser": { + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { "optional": true }, "@vitest/ui": { @@ -20066,23 +20090,16 @@ }, "jsdom": { "optional": true + }, + "vite": { + "optional": false } } }, - "node_modules/vitest/node_modules/magic-string": { - "version": "0.30.17", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", - "integrity": "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0" - } - }, "node_modules/vitest/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", "engines": { @@ -20122,9 +20139,9 @@ } }, "node_modules/watchpack": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.4.tgz", - "integrity": "sha512-c5EGNOiyxxV5qmTtAB7rbiXxi1ooX1pQKMLX/MIabJjRA0SJBQOjKF+KSVfHkr9U1cADPon0mRiVe/riyaiDUA==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.1.tgz", + "integrity": "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg==", "dev": true, "license": "MIT", "dependencies": { @@ -20198,9 +20215,9 @@ } }, "node_modules/webpack": { - "version": "5.100.2", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.100.2.tgz", - "integrity": "sha512-QaNKAvGCDRh3wW1dsDjeMdDXwZm2vqq3zn6Pvq4rHOEOGSaUMgOOjG2Y9ZbIGzpfkJk9ZYTHpDqgDfeBDcnLaw==", + "version": "5.105.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz", + "integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==", "dev": true, "license": "MIT", "dependencies": { @@ -20212,22 +20229,22 @@ "@webassemblyjs/wasm-parser": "^1.14.1", "acorn": "^8.15.0", "acorn-import-phases": "^1.0.3", - "browserslist": "^4.24.0", + "browserslist": "^4.28.1", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.17.2", - "es-module-lexer": "^1.2.1", + "enhanced-resolve": "^5.19.0", + "es-module-lexer": "^2.0.0", "eslint-scope": "5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.11", "json-parse-even-better-errors": "^2.3.1", - "loader-runner": "^4.2.0", + "loader-runner": "^4.3.1", "mime-types": "^2.1.27", "neo-async": "^2.6.2", - "schema-utils": "^4.3.2", - "tapable": "^2.1.1", - "terser-webpack-plugin": "^5.3.11", - "watchpack": "^2.4.1", + "schema-utils": "^4.3.3", + "tapable": "^2.3.0", + "terser-webpack-plugin": "^5.3.16", + "watchpack": "^2.5.1", "webpack-sources": "^3.3.3" }, "bin": { @@ -20344,15 +20361,15 @@ "license": "MIT" }, "node_modules/webpack-dev-server": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-5.2.2.tgz", - "integrity": "sha512-QcQ72gh8a+7JO63TAx/6XZf/CWhgMzu5m0QirvPfGvptOusAxG12w2+aua1Jkjr7hzaWDnJ2n6JFeexMHI+Zjg==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-5.2.4.tgz", + "integrity": "sha512-GqDPGZN9bRqKBTkp4aWkobDDHMsrXKoGSdOH56smIri8qR0JG8gfL8/v/f/OZR3/OKXjG8uwJbFVhKm/FNU/UA==", "dev": true, "license": "MIT", "dependencies": { "@types/bonjour": "^3.5.13", "@types/connect-history-api-fallback": "^1.5.4", - "@types/express": "^4.17.21", + "@types/express": "^4.17.25", "@types/express-serve-static-core": "^4.17.21", "@types/serve-index": "^1.9.4", "@types/serve-static": "^1.15.5", @@ -20362,9 +20379,9 @@ "bonjour-service": "^1.2.1", "chokidar": "^3.6.0", "colorette": "^2.0.10", - "compression": "^1.7.4", + "compression": "^1.8.1", "connect-history-api-fallback": "^2.0.0", - "express": "^4.21.2", + "express": "^4.22.1", "graceful-fs": "^4.2.6", "http-proxy-middleware": "^2.0.9", "ipaddr.js": "^2.1.0", @@ -20372,7 +20389,7 @@ "open": "^10.0.3", "p-retry": "^6.2.0", "schema-utils": "^4.2.0", - "selfsigned": "^2.4.1", + "selfsigned": "^5.5.0", "serve-index": "^1.9.1", "sockjs": "^0.3.24", "spdy": "^4.0.2", @@ -20433,6 +20450,71 @@ "node": ">=10.13.0" } }, + "node_modules/webpack/node_modules/browserslist": { + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", + "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.9.0", + "caniuse-lite": "^1.0.30001759", + "electron-to-chromium": "^1.5.263", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.2.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/webpack/node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, "node_modules/websocket-driver": { "version": "0.7.4", "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz", @@ -20645,25 +20727,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -20723,6 +20786,21 @@ "node": ">=12" } }, + "node_modules/xml-naming": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", + "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", @@ -20748,19 +20826,23 @@ "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha1-27fa+b/YusmrRev2ArjLrQ1dCP0= sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" }, "node_modules/yaml": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.0.tgz", - "integrity": "sha512-4lLa/EcQCB0cJkyts+FpIRx5G/llPxfP6VQU5KByHEhLxY3IJCH0f0Hy1MHI8sClTvsIb8qwRJ6R/ZdlDJ/leQ==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", "license": "ISC", "bin": { "yaml": "bin.mjs" }, "engines": { "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" } }, "node_modules/yaml-ast-parser": { diff --git a/ui/apps/platform/package.json b/ui/apps/platform/package.json index ddcc832297af0..00001ee218471 100644 --- a/ui/apps/platform/package.json +++ b/ui/apps/platform/package.json @@ -13,20 +13,19 @@ }, "license": "UNLICENSED", "dependencies": { - "@apollo/client": "^3.6.3", + "@apollo/client": "^3.8.10", "@lifeomic/axios-fetch": "^3.1.0", "@openshift-console/dynamic-plugin-sdk": "4.19.0", - "@patternfly/react-charts": "7.2.2", - "@patternfly/react-component-groups": "^5.5.8", - "@patternfly/react-core": "^5.4.14", - "@patternfly/react-icons": "^5.4.2", - "@patternfly/react-styles": "^5.4.1", - "@patternfly/react-table": "^5.4.16", - "@patternfly/react-topology": "^5.4.1", - "@patternfly/react-user-feedback": "^5.0.0", + "@patternfly/patternfly": "^6.4.0", + "@patternfly/react-charts": "^8.4.1", + "@patternfly/react-component-groups": "^6.4.0", + "@patternfly/react-core": "^6.4.1", + "@patternfly/react-icons": "^6.4.0", + "@patternfly/react-styles": "^6.4.0", + "@patternfly/react-table": "^6.4.1", + "@patternfly/react-topology": "^6.4.0", "@segment/analytics-next": "^1.81.1", - "axios": "^1.12.0", - "computed-style-to-inline-style": "^3.0.0", + "axios": "^1.18.0", "connected-react-router": "^6.9.3", "d3-axis": "^1.0.12", "d3-brush": "^3.0.0", @@ -42,18 +41,15 @@ "framer-motion": "^10.0.0", "graphql": "^16.11.0", "history": "^4.10.1", - "html2canvas": "1.0.0-rc.7", "initials": "^3.1.2", "js-base64": "^3.7.2", - "jspdf": "^4.0.0", - "jspdf-autotable": "^5.0.7", - "lodash": "^4.17.21", + "lodash": "^4.17.23", "mobx": "^6.13.7", "mobx-react": "^7.6.0", "object-resolve-path": "^1.1.1", "pluralize": "^8.0.0", "prop-types": "^15.7.2", - "qs": "^6.14.1", + "qs": "^6.16.0", "raven-for-redux": "^1.3.1", "raven-js": "^3.27.0", "react": "^18.2.0", @@ -78,7 +74,6 @@ "react-syntax-highlighter": "^15.6.1", "react-table-6": "^6.11.0", "react-toastify": "^7.0.3", - "react-transition-group": "^4.4.1", "react-truncate": "^2.4.0", "react-vis": "^1.12.1", "redoc": "^2.5.0", @@ -90,7 +85,8 @@ "store": "^2.0.12", "styled-components": "^5.3.1", "use-deep-compare-effect": "^1.8.1", - "yaml": "^2.7.1", + "victory": "^37.3.6", + "yaml": "^2.8.3", "yup": "^1.6.1" }, "scripts": { @@ -102,7 +98,7 @@ "build:ocp-plugin": "tsc && NODE_ENV=production webpack --config webpack.ocp-plugin.config.js", "test": "TZ=UTC vitest", "test-coverage": "TZ=UTC vitest run --coverage", - "lint": "eslint --quiet", + "lint": "eslint --quiet --cache --cache-strategy content", "lint:fast-dev": "eslint --rule 'import/namespace: off' --rule 'import/no-cycle: off'", "lint:fast-dev-fix": "eslint --fix --rule 'import/namespace: off' --rule 'import/no-cycle: off'", "lint:fix": "eslint --fix", @@ -120,44 +116,45 @@ "postinstall": "node scripts/check-optional-dependency-versions.js" }, "devDependencies": { + "@eslint-community/eslint-plugin-eslint-comments": "^4.6.0", "@eslint/css": "^0.14.1", - "@eslint/json": "^0.14.0", + "@eslint/js": "^9.39.3", + "@eslint/json": "^1.0.1", "@eslint/markdown": "^7.5.1", "@openshift-console/dynamic-plugin-sdk-webpack": "4.19.0", "@tailwindcss/forms": "^0.2.1", "@testing-library/cypress": "^10.1.0", - "@testing-library/dom": "^10.4.0", - "@testing-library/jest-dom": "^6.6.3", - "@testing-library/react": "^16.3.0", + "@testing-library/dom": "^10.4.1", + "@testing-library/jest-dom": "^6.9.1", + "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", - "@types/lodash": "^4.17.20", + "@types/lodash": "^4.17.24", "@types/pluralize": "^0.0.33", "@types/qs": "^6.14.0", "@types/react": "^18.3.26", "@types/react-dom": "^18.3.7", "@types/react-redux": "^7.1.34", - "@typescript-eslint/eslint-plugin": "^8.42.0", - "@typescript-eslint/parser": "^8.42.0", - "@vitejs/plugin-basic-ssl": "^1.1.0", - "@vitejs/plugin-react": "^4.4.0", - "@vitest/coverage-v8": "^3.1.1", - "@vitest/eslint-plugin": "^1.3.3", + "@typescript-eslint/eslint-plugin": "^8.56.1", + "@typescript-eslint/parser": "^8.56.1", + "@vitejs/plugin-basic-ssl": "^2.2.0", + "@vitejs/plugin-react": "^5.2.0", + "@vitest/coverage-v8": "^4.1.0", + "@vitest/eslint-plugin": "^1.6.12", "axe-core": "^4.10.3", "copy-webpack-plugin": "^13.0.0", "css-loader": "^7.1.2", - "cypress": "^15.5.0", + "cypress": "^15.14.1", "cypress-axe": "^1.7.0", "cypress-vite": "^1.8.0", - "eslint": "^9.39.1", + "eslint": "^9.39.3", "eslint-import-resolver-typescript": "^4.2.6", - "eslint-plugin-cypress": "^5.2.0", - "eslint-plugin-eslint-comments": "^3.2.0", + "eslint-plugin-cypress": "^6.1.0", "eslint-plugin-import": "^2.32.0", "eslint-plugin-jest-dom": "^5.5.0", "eslint-plugin-prettier": "^5.5.4", "eslint-plugin-react": "^7.37.5", - "eslint-plugin-react-hooks": "^7.0.1", - "eslint-plugin-testing-library": "^7.13.4", + "eslint-plugin-react-hooks": "^7.1.1", + "eslint-plugin-testing-library": "^7.16.0", "globals": "^16.5.0", "jsdom": "^20.0.3", "levenary": "^1.1.1", @@ -166,19 +163,19 @@ "node-fetch": "^2.6.1", "postcss": "^8.4.47", "postcss-cli": "^8.3.1", - "prettier": "^3.6.2", + "prettier": "^3.8.1", "redux-immutable-state-invariant": "^2.1.0", "redux-saga-test-plan": "^3.7.0", "style-loader": "^4.0.0", "tailwindcss": "^2.0.3", "ts-loader": "^9.5.2", "typescript": "^5.9.2", - "vite": "^6.4.1", + "vite": "^6.4.2", "vite-plugin-svgr": "^4.3.0", - "vitest": "^3.0.5", - "webpack": "^5.99.9", + "vitest": "^4.1.0", + "webpack": "^5.105.0", "webpack-cli": "^6.0.1", - "webpack-dev-server": "^5.2.2" + "webpack-dev-server": "^5.2.4" }, "optionalDependencies": { "@esbuild/darwin-arm64": "^0.25.5", @@ -202,7 +199,8 @@ "connected-react-router": { "react": "^18.0.0" }, - "react-dropzone": "14.3.2" + "react-dropzone": "14.3.2", + "js-yaml": "^4.3.2" }, "browserslist": [ "Chrome >= 80" diff --git a/ui/apps/platform/scripts/cypress-ocp.sh b/ui/apps/platform/scripts/cypress-ocp.sh index 36dc51a24b6fb..c232af74b2fa1 100755 --- a/ui/apps/platform/scripts/cypress-ocp.sh +++ b/ui/apps/platform/scripts/cypress-ocp.sh @@ -1,16 +1,60 @@ #!/usr/bin/env bash +# Only runs OpenShift Console plugin E2E tests when: +# - ORCHESTRATOR_FLAVOR is 'openshift' +# - OpenShift version is a supported version +# +# Version detection from CLUSTER_FLAVOR_VARIANT format: openshift-4-ocp/-. +# See scripts/ci/jobs/ocp_qa_e2e_tests.py +# Example: openshift-4-ocp/gcp-4.16 -> major=4, minor=16 +# +# Edge cases default to RUNNING tests to avoid silently skipping them: +# - If version cannot be parsed, tests run +# - If CLUSTER_FLAVOR_VARIANT is empty, tests run +# Only skips if we definitively know the version is too old to prevent breakages in CI resulting in no test coverage. + +set -eo pipefail + export CYPRESS_ORCHESTRATOR_FLAVOR="${ORCHESTRATOR_FLAVOR}" -# exit if ORCHESTRATOR_FLAVOR is not 'openshift' - these tests are only relevant for openshift -if [ "${ORCHESTRATOR_FLAVOR}" != "openshift" ]; then - echo "ORCHESTRATOR_FLAVOR is not 'openshift', skipping cypress-ocp" + +if [[ "${ORCHESTRATOR_FLAVOR}" != "openshift" ]]; then + echo "ORCHESTRATOR_FLAVOR is not 'openshift', skipping cypress-ocp tests" exit 0 fi +MIN_MAJOR=4 +MIN_MINOR=19 + +should_run=true +major="" +minor="" + +if [[ -z "${CLUSTER_FLAVOR_VARIANT:-}" ]]; then + echo "CLUSTER_FLAVOR_VARIANT is not set, running cypress-ocp (no version check possible)" +elif [[ "${CLUSTER_FLAVOR_VARIANT}" =~ openshift-4-ocp/[^-]+-([0-9]+)\.([0-9]+) ]]; then + major="${BASH_REMATCH[1]}" + minor="${BASH_REMATCH[2]}" + export CYPRESS_OSCI_OPENSHIFT_VERSION="${major}.${minor}" + + echo "Detected OpenShift version: ${CYPRESS_OSCI_OPENSHIFT_VERSION}" + + # Only skip if we know the version is too old + if [[ "${major}" -lt ${MIN_MAJOR} ]] || { [[ "${major}" -eq ${MIN_MAJOR} ]] && [[ "${minor}" -lt ${MIN_MINOR} ]]; }; then + should_run=false + fi +else + echo "CLUSTER_FLAVOR_VARIANT format not recognized: ${CLUSTER_FLAVOR_VARIANT}" + echo "Running anyway (defaulting to run on parse failure)" +fi + +if [[ "${should_run}" == "false" ]]; then + echo "OpenShift ${CYPRESS_OSCI_OPENSHIFT_VERSION} < ${MIN_MAJOR}.${MIN_MINOR}, skipping cypress-ocp tests" + exit 0 +fi # Opens cypress with environment variables for feature flags and auth OPENSHIFT_CONSOLE_URL="${OPENSHIFT_CONSOLE_URL:-http://localhost:9000}" -API_PROXY_BASE_URL="${OPENSHIFT_CONSOLE_URL}/api/proxy/plugin/advanced-cluster-security/api-service/proxy/central" +api_endpoint="${UI_BASE_URL:-https://localhost:8000}" if [[ -z "$OCP_BRIDGE_AUTH_DISABLED" && ( -z "$OPENSHIFT_CONSOLE_USERNAME" || -z "$OPENSHIFT_CONSOLE_PASSWORD" ) ]]; then echo "OPENSHIFT_CONSOLE_USERNAME and OPENSHIFT_CONSOLE_PASSWORD must be set if OCP_BRIDGE_AUTH_DISABLED is not true" @@ -21,8 +65,15 @@ curl_cfg() { # Use built-in echo to not expose $2 in the process list. echo -n "$1 = \"${2//[\"\\]/\\&}\"" } -if [[ -n "$OPENSHIFT_CONSOLE_PASSWORD" || "$OCP_BRIDGE_AUTH_DISABLED" == "true" ]]; then - readarray -t arr < <(curl -sk --config <(curl_cfg user "$OPENSHIFT_CONSOLE_USERNAME:$OPENSHIFT_CONSOLE_PASSWORD") "${API_PROXY_BASE_URL}"/v1/featureflags | jq -cr '.featureFlags[] | {name: .envVar, enabled: .enabled}') +if [[ -z "$ROX_USERNAME" || -z "$ROX_ADMIN_PASSWORD" ]]; then + # basic auth creds weren't set (e.g. by CI), assume local k8s deployment + # shellcheck source=../../../../scripts/k8s/export-basic-auth-creds.sh + source ../../../scripts/k8s/export-basic-auth-creds.sh ../../../deploy/k8s +fi + +# Fetch feature flags directly from Central rather than through the OCP Console proxy +if [[ -n "$ROX_ADMIN_PASSWORD" ]]; then + readarray -t arr < <(curl -sk --config <(curl_cfg user "admin:$ROX_ADMIN_PASSWORD") "${api_endpoint}"/v1/featureflags | jq -cr '.featureFlags[] | {name: .envVar, enabled: .enabled}') for i in "${arr[@]}"; do name=$(echo "$i" | jq -rc .name) val=$(echo "$i" | jq -rc .enabled) diff --git a/ui/apps/platform/scripts/expose-sensor-proxy.sh b/ui/apps/platform/scripts/expose-sensor-proxy.sh new file mode 100755 index 0000000000000..a4df5fdd4b356 --- /dev/null +++ b/ui/apps/platform/scripts/expose-sensor-proxy.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Default values +NAMESPACE="${1:-stackrox}" +HOURS="${2:-8}" + +LB_SERVICE_NAME="sensor-proxy-dev-lb" +NETPOL_NAME="sensor-proxy-dev-allow-external" +CRONJOB_NAME="sensor-proxy-dev-cleanup" +SERVICE_NAME="sensor-proxy" + +echo "Exposing sensor-proxy in namespace: $NAMESPACE" +echo "LoadBalancer will expire in: $HOURS hours" + +# Validate namespace exists +if ! oc get namespace "$NAMESPACE" &>/dev/null; then + echo "Error: Namespace '$NAMESPACE' does not exist." + echo "Please create the namespace or specify a different namespace." + exit 1 +fi + +# Validate sensor-proxy service exists +if ! oc -n "$NAMESPACE" get service "$SERVICE_NAME" &>/dev/null; then + echo "Error: Service '$SERVICE_NAME' not found in namespace '$NAMESPACE'." + echo "Please ensure StackRox secured cluster services are installed with the sensor-proxy enabled." + exit 1 +fi + +# Calculate expiry time in ISO8601 format +EXPIRES_AT=$(date -u -v+"${HOURS}H" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -d "+${HOURS} hours" +"%Y-%m-%dT%H:%M:%SZ") + +echo "Expiry time: $EXPIRES_AT" + +# Create or update NetworkPolicy to allow external access to port 9444 +echo "Creating/updating NetworkPolicy '$NETPOL_NAME'..." + +cat </dev/null || echo "") + + if [ -z "\$EXPIRES_AT" ]; then + echo "LoadBalancer Service '\$LB_SERVICE_NAME' not found or missing expiry annotation. Exiting." + exit 0 + fi + + echo "LoadBalancer expires at: \$EXPIRES_AT" + + # Convert expiry time to epoch seconds + EXPIRES_EPOCH=\$(date -d "\$EXPIRES_AT" +%s 2>/dev/null || echo "0") + CURRENT_EPOCH=\$(date +%s) + + if [ "\$EXPIRES_EPOCH" -eq 0 ]; then + echo "Error: Could not parse expiry time '\$EXPIRES_AT'" + exit 1 + fi + + if [ "\$CURRENT_EPOCH" -ge "\$EXPIRES_EPOCH" ]; then + echo "LoadBalancer has expired. Deleting resources..." + oc delete service "\$LB_SERVICE_NAME" -n "\$NAMESPACE" --ignore-not-found=true + oc delete networkpolicy "\$NETPOL_NAME" -n "\$NAMESPACE" --ignore-not-found=true + oc delete cronjob "\$CRONJOB_NAME" -n "\$NAMESPACE" --ignore-not-found=true + echo "Cleanup complete." + else + TIME_LEFT=\$((EXPIRES_EPOCH - CURRENT_EPOCH)) + HOURS_LEFT=\$((TIME_LEFT / 3600)) + MINUTES_LEFT=\$(((TIME_LEFT % 3600) / 60)) + echo "LoadBalancer has not expired yet. Time remaining: \${HOURS_LEFT}h \${MINUTES_LEFT}m" + fi +YAML + +# Wait for LoadBalancer to get an external IP +echo "Waiting for LoadBalancer to get an external IP..." +TIMEOUT=60 +ELAPSED=0 +EXTERNAL_IP="" + +while [ $ELAPSED -lt $TIMEOUT ]; do + EXTERNAL_IP=$(oc -n "$NAMESPACE" get service "$LB_SERVICE_NAME" -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "") + + if [ -n "$EXTERNAL_IP" ]; then + break + fi + + sleep 2 + ELAPSED=$((ELAPSED + 2)) +done + +if [ -z "$EXTERNAL_IP" ]; then + echo "Error: LoadBalancer did not receive an external IP within ${TIMEOUT} seconds." + echo "Check service status: oc -n $NAMESPACE get service $LB_SERVICE_NAME" + exit 1 +fi + +echo "" +echo "✓ Successfully created/updated LoadBalancer!" +echo "" +echo "The LoadBalancer and NetworkPolicy will expire at: $EXPIRES_AT" +echo "A CronJob will automatically clean up all resources when expired." diff --git a/ui/apps/platform/scripts/start-ocp-console.sh b/ui/apps/platform/scripts/start-ocp-console.sh index 23a26d37e32e4..de71a80be9884 100755 --- a/ui/apps/platform/scripts/start-ocp-console.sh +++ b/ui/apps/platform/scripts/start-ocp-console.sh @@ -5,22 +5,34 @@ set -euo pipefail CONSOLE_IMAGE=${CONSOLE_IMAGE:="quay.io/openshift/origin-console:latest"} CONSOLE_PORT=${CONSOLE_PORT:=9000} CONSOLE_IMAGE_PLATFORM=${CONSOLE_IMAGE_PLATFORM:="linux/amd64"} +SENSOR_PROXY_NAMESPACE=${SENSOR_PROXY_NAMESPACE:="stackrox"} +SENSOR_PROXY_EXPIRY_HOURS=${SENSOR_PROXY_EXPIRY_HOURS:=8} -# The ACS backend service URL that will receive the console's proxied requests. -ACS_API_SERVICE_URL=${ACS_API_SERVICE_URL:=https://$(oc -n stackrox get route central -o jsonpath='{.spec.host}')} +# Plugin metadata is declared in package.json +PLUGIN_NAME="advanced-cluster-security" -# Whether to inject the OCP auth token into the ACS backend service. -ACS_INJECT_OCP_AUTH_TOKEN=${ACS_INJECT_OCP_AUTH_TOKEN:=true} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# The base path to append onto the console proxy base URL. When following a true path via the sensor-proxy, default to `proxy/central`. -ACS_PROXY_BASE_PATH=${ACS_PROXY_BASE_PATH:="/proxy/central"} -ACS_PROXY_BASE_PATH="${ACS_PROXY_BASE_PATH%/}" # Remove trailing slash if present, we will add it back when constructing the proxy URL. +# Expose sensor-proxy via LoadBalancer +echo "Exposing sensor-proxy via LoadBalancer..." +"$SCRIPT_DIR/expose-sensor-proxy.sh" "$SENSOR_PROXY_NAMESPACE" "$SENSOR_PROXY_EXPIRY_HOURS" -# Plugin metadata is declared in package.json -PLUGIN_NAME="advanced-cluster-security" +# Get the external IP directly from the LoadBalancer service +EXTERNAL_IP=$(oc -n "$SENSOR_PROXY_NAMESPACE" get service sensor-proxy-dev-lb -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null) + +if [ -z "$EXTERNAL_IP" ]; then + echo "Error: Failed to get external IP from LoadBalancer service" + echo "Check service status: oc -n $SENSOR_PROXY_NAMESPACE get service sensor-proxy-dev-lb" + exit 1 +fi + +echo "" +echo "Using sensor-proxy at: $EXTERNAL_IP" + +# Construct the ACS API Service URL using the external IP +ACS_API_SERVICE_URL="https://${EXTERNAL_IP}" -echo "Starting local OpenShift console using ACS API Service URL: $ACS_API_SERVICE_URL" -echo "Injecting OCP auth token: $ACS_INJECT_OCP_AUTH_TOKEN" +echo "ACS API Service URL: $ACS_API_SERVICE_URL" export BRIDGE_USER_AUTH="disabled" export BRIDGE_K8S_MODE="off-cluster" @@ -44,7 +56,7 @@ if [ -n "$GITOPS_HOSTNAME" ]; then export BRIDGE_K8S_MODE_OFF_CLUSTER_GITOPS="https://$GITOPS_HOSTNAME" fi -export BRIDGE_PLUGIN_PROXY='{"services":[{"consoleAPIPath":"/api/proxy/plugin/advanced-cluster-security/api-service'$ACS_PROXY_BASE_PATH'/","endpoint":"'$ACS_API_SERVICE_URL'", "authorize":'$ACS_INJECT_OCP_AUTH_TOKEN'}]}' +export BRIDGE_PLUGIN_PROXY='{"services":[{"consoleAPIPath":"/api/proxy/plugin/advanced-cluster-security/api-service/","endpoint":"'$ACS_API_SERVICE_URL'", "authorize":true}]}' echo "API Server: $BRIDGE_K8S_MODE_OFF_CLUSTER_ENDPOINT" @@ -52,7 +64,7 @@ echo "Console Image: $CONSOLE_IMAGE" echo "Console URL: http://localhost:${CONSOLE_PORT}" echo "Console Platform: $CONSOLE_IMAGE_PLATFORM" -# Prefer podman if installed. Otherwise, fall back to docker. +# Run console container with appropriate configuration if [ -x "$(command -v podman)" ]; then if [ "$(uname -s)" = "Linux" ]; then # Use host networking on Linux since host.containers.internal is unreachable in some environments. diff --git a/ui/apps/platform/src/Components/BinderTabs/Binder.test.tsx b/ui/apps/platform/src/Components/BinderTabs/Binder.test.tsx deleted file mode 100644 index 65c6dff22e6b4..0000000000000 --- a/ui/apps/platform/src/Components/BinderTabs/Binder.test.tsx +++ /dev/null @@ -1,39 +0,0 @@ -import { act, render, screen } from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; - -import BinderTabs from './BinderTabs'; -import Tab from '../Tab'; - -describe('BinderTabs', () => { - test("renders the first tab's contents", () => { - render( - - - Tab 1 Content - - - Tab 2 Content - - - ); - expect(screen.getByText('Tab 1 Content')).toBeInTheDocument(); - }); - - test("selecting a new tab render's the new tab's contents", async () => { - const user = userEvent.setup(); - render( - - - Tab 1 Content - - - Tab 2 Content - - - ); - - await act(() => user.click(screen.getByText('tab 2'))); - - expect(screen.getByText('Tab 2 Content')).toBeInTheDocument(); - }); -}); diff --git a/ui/apps/platform/src/Components/CVEStackedPill/CVEStackedPill.jsx b/ui/apps/platform/src/Components/CVEStackedPill/CVEStackedPill.jsx index 20f8ac1eb494a..54d1bfa17c5af 100644 --- a/ui/apps/platform/src/Components/CVEStackedPill/CVEStackedPill.jsx +++ b/ui/apps/platform/src/Components/CVEStackedPill/CVEStackedPill.jsx @@ -99,7 +99,7 @@ const CVEStackedPill = ({ > )} diff --git a/ui/apps/platform/src/Components/CodeViewer.cy.jsx b/ui/apps/platform/src/Components/CodeViewer.cy.jsx index 9457d6d90c8ac..fe3075f8f31c4 100644 --- a/ui/apps/platform/src/Components/CodeViewer.cy.jsx +++ b/ui/apps/platform/src/Components/CodeViewer.cy.jsx @@ -1,4 +1,4 @@ -import CodeViewer, { CodeViewerThemeProvider } from './CodeViewer'; +import CodeViewer from './CodeViewer'; const sampleYaml = `apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -37,40 +37,4 @@ describe(Cypress.spec.relative, () => { }); }); }); - - it('should toggle light and dark editor themes', () => { - cy.mount(); - - // Default to light mode - cy.get('.pf-v5-theme-dark').should('not.exist'); - - cy.get('button[aria-label="Set dark theme"]').click(); - cy.get('.pf-v5-theme-dark'); - - cy.get('button[aria-label="Set light theme"]').click(); - cy.get('.pf-v5-theme-dark').should('not.exist'); - }); - - it('should share theme state across multiple instances', () => { - cy.mount( - - - - - ); - - cy.get('.pf-v5-theme-dark').should('not.exist'); - - cy.get('button[aria-label="Set dark theme"]').eq(0); - cy.get('button[aria-label="Set dark theme"]').eq(1); - cy.get('button[aria-label="Set dark theme"]').eq(0).click(); - - cy.get('.pf-v5-theme-dark').should('have.length', 2); - - cy.get('button[aria-label="Set light theme"]').eq(0); - cy.get('button[aria-label="Set light theme"]').eq(1); - cy.get('button[aria-label="Set light theme"]').eq(1).click(); - - cy.get('.pf-v5-theme-dark').should('not.exist'); - }); }); diff --git a/ui/apps/platform/src/Components/CodeViewer.tsx b/ui/apps/platform/src/Components/CodeViewer.tsx index ea6fbd93961e9..233d6a1cbd78c 100644 --- a/ui/apps/platform/src/Components/CodeViewer.tsx +++ b/ui/apps/platform/src/Components/CodeViewer.tsx @@ -1,7 +1,5 @@ -import { createContext, useContext, useState } from 'react'; -import type { CSSProperties, Dispatch, ReactElement, ReactNode, SetStateAction } from 'react'; -import { Button, ClipboardCopyButton, CodeBlock, CodeBlockAction } from '@patternfly/react-core'; -import { MoonIcon, SunIcon } from '@patternfly/react-icons'; +import type { CSSProperties, ReactElement, ReactNode } from 'react'; +import { ClipboardCopyButton, CodeBlock, CodeBlockAction } from '@patternfly/react-core'; import { PrismLight as SyntaxHighlighter } from 'react-syntax-highlighter'; import lightTheme from 'react-syntax-highlighter/dist/esm/styles/prism/one-light'; @@ -9,42 +7,13 @@ import darkTheme from 'react-syntax-highlighter/dist/esm/styles/prism/one-dark'; import yaml from 'react-syntax-highlighter/dist/esm/languages/prism/yaml'; import useClipboardCopy from 'hooks/useClipboardCopy'; +import { useTheme } from 'hooks/useTheme'; SyntaxHighlighter.registerLanguage('yaml', yaml); -const CodeViewerThemeContext = createContext< - ['light' | 'dark', Dispatch>] | undefined ->(undefined); - -export const CodeViewerThemeProvider = ({ children }: { children: ReactNode }) => { - const [state, setState] = useState<'light' | 'dark'>('light'); - - return ( - - {children} - - ); -}; - -export const useCodeViewerThemeContext = () => { - const context = useContext(CodeViewerThemeContext); - // Fallback state provides the ability to toggle theme for a single instance if no provider is detected uptree - const fallbackState = useState<'light' | 'dark'>('light'); - return context ?? fallbackState; -}; - // When adding to the supported languages, the correct language definition must be imported and registered as well type SupportedLanguages = 'yaml'; -const defaultStyle = { - '--pf-v5-u-max-height--MaxHeight': '300px', - '--pf-v5-c-code-block__content--PaddingTop': '0', - '--pf-v5-c-code-block__content--PaddingBottom': '0', - '--pf-v5-c-code-block__content--PaddingLeft': '0', - '--pf-v5-c-code-block__content--PaddingRight': '0', - overflowY: 'auto', -} as const; - export type CodeViewerProps = { code: string; language?: SupportedLanguages; @@ -61,18 +30,13 @@ export default function CodeViewer({ additionalControls, }: CodeViewerProps): ReactElement { const { wasCopied, setWasCopied, copyToClipboard } = useClipboardCopy(); - const [theme, setTheme] = useCodeViewerThemeContext(); - - function toggleTheme() { - setTheme((prevValue) => (prevValue === 'light' ? 'dark' : 'light')); - } + const theme = useTheme(); const actions = ( <> copyToClipboard(code)} exitDelay={wasCopied ? 1500 : 600} @@ -82,14 +46,6 @@ export default function CodeViewer({ {wasCopied ? 'Successfully copied to clipboard!' : 'Copy to clipboard'} - - + /> )} @@ -310,8 +329,7 @@ function AutocompleteSelect({ isSelected={false} className={option.className} onClick={() => { - onChange(option.value); - onSearch(option.value); + applySelectedText(option.value); }} id={`select-typeahead-${option?.value?.replace(' ', '-space-')}`} {...option} diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilter.cy.jsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilter.cy.jsx index 580f006bac239..6eb8fc1e24472 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilter.cy.jsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilter.cy.jsx @@ -82,7 +82,7 @@ const imageNameResponseMock = { function Wrapper({ config, searchFilter, onSearch }) { return ( -
+
); @@ -131,15 +131,15 @@ function selectDatePickerDate(month, day, year) { cy.get('button[aria-label="Filter by date toggle"]').click(); // Select a month - cy.get('div.pf-v5-c-calendar-month__header-month button').click(); - cy.get(`button.pf-v5-c-menu__item:contains("${month}")`).click(); + cy.get('div.pf-v6-c-calendar-month__header-month button').click(); + cy.get(`button.pf-v6-c-menu__item:contains("${month}")`).click(); // Select a year cy.get('input[aria-label="Select year"]').clear(); cy.get('input[aria-label="Select year"]').type(year); // Select a day - cy.get(`button.pf-v5-c-calendar-month__date:contains("${day}")`).click(); + cy.get(`button.pf-v6-c-calendar-month__date:contains("${day}")`).click(); } describe(Cypress.spec.relative, () => { @@ -217,13 +217,14 @@ describe(Cypress.spec.relative, () => { cy.get(selectors.attributeSelectToggle).click(); - cy.get(selectors.attributeSelectItems).should('have.length', 5); + cy.get(selectors.attributeSelectItems).should('have.length', 6); // Attributes are in alphabetical order by displayName property. - cy.get(selectors.attributeSelectItems).eq(0).should('have.text', 'Label'); - cy.get(selectors.attributeSelectItems).eq(1).should('have.text', 'Name'); - cy.get(selectors.attributeSelectItems).eq(2).should('have.text', 'Operating system'); - cy.get(selectors.attributeSelectItems).eq(3).should('have.text', 'Registry'); - cy.get(selectors.attributeSelectItems).eq(4).should('have.text', 'Tag'); + cy.get(selectors.attributeSelectItems).eq(0).should('have.text', 'Created time'); + cy.get(selectors.attributeSelectItems).eq(1).should('have.text', 'Label'); + cy.get(selectors.attributeSelectItems).eq(2).should('have.text', 'Name'); + cy.get(selectors.attributeSelectItems).eq(3).should('have.text', 'Operating system'); + cy.get(selectors.attributeSelectItems).eq(4).should('have.text', 'Registry'); + cy.get(selectors.attributeSelectItems).eq(5).should('have.text', 'Tag'); }); it('should display Deployment attributes in the attribute selector', () => { @@ -240,13 +241,14 @@ describe(Cypress.spec.relative, () => { cy.get(selectors.attributeSelectToggle).click(); - cy.get(selectors.attributeSelectItems).should('have.length', 5); + cy.get(selectors.attributeSelectItems).should('have.length', 6); // Attributes are in alphabetical order by displayName property. cy.get(selectors.attributeSelectItems).eq(0).should('have.text', 'Annotation'); - cy.get(selectors.attributeSelectItems).eq(1).should('have.text', 'ID'); - cy.get(selectors.attributeSelectItems).eq(2).should('have.text', 'Label'); - cy.get(selectors.attributeSelectItems).eq(3).should('have.text', 'Name'); - cy.get(selectors.attributeSelectItems).eq(4).should('have.text', 'Status'); + cy.get(selectors.attributeSelectItems).eq(1).should('have.text', 'Container type'); + cy.get(selectors.attributeSelectItems).eq(2).should('have.text', 'ID'); + cy.get(selectors.attributeSelectItems).eq(3).should('have.text', 'Label'); + cy.get(selectors.attributeSelectItems).eq(4).should('have.text', 'Name'); + cy.get(selectors.attributeSelectItems).eq(5).should('have.text', 'Status'); }); it('should display the autocomplete input and correctly search for image tags', () => { @@ -302,9 +304,9 @@ describe(Cypress.spec.relative, () => { cy.get(imageComponenSourceSelectItems).eq(1).should('have.text', 'Python'); cy.get(imageComponenSourceSelectItems).eq(2).should('have.text', 'Java'); cy.get(imageComponenSourceSelectItems).eq(3).should('have.text', 'Ruby'); - cy.get(imageComponenSourceSelectItems).eq(4).should('have.text', 'Node js'); + cy.get(imageComponenSourceSelectItems).eq(4).should('have.text', 'Node.js'); cy.get(imageComponenSourceSelectItems).eq(5).should('have.text', 'Go'); - cy.get(imageComponenSourceSelectItems).eq(6).should('have.text', 'Dotnet Core Runtime'); + cy.get(imageComponenSourceSelectItems).eq(6).should('have.text', '.NET Core Runtime'); cy.get(imageComponenSourceSelectItems).eq(7).should('have.text', 'Infrastructure'); cy.get(imageComponenSourceSelectItems).eq(1).click(); @@ -474,11 +476,12 @@ describe(Cypress.spec.relative, () => { cy.get(autocompleteMenuItems).eq(0).click(); + // Autocomplete selections are wrapped in quotes for exact-match search cy.get('@onSearch').should('have.been.calledWithExactly', [ { action: 'APPEND', category: 'Image', - value: 'docker.io/library/centos:7', + value: '"docker.io/library/centos:7"', }, ]); @@ -516,7 +519,7 @@ describe(Cypress.spec.relative, () => { -
+
+ {entity && ( { setSelectedEntity(ensureString(value)); setSelectedAttribute(undefined); @@ -63,9 +61,10 @@ function CompoundSearchFilter({ entity.attributes.length !== 0 && attribute && ( { setSelectedAttribute(ensureString(value)); }} @@ -78,6 +77,7 @@ function CompoundSearchFilter({ key={`${entity.displayName} ${attribute.displayName}`} entity={entity} attribute={attribute} + isDisabled={isDisabled} searchFilter={searchFilter} additionalContextFilter={additionalContextFilter} onSearch={(payload) => { @@ -92,7 +92,7 @@ function CompoundSearchFilter({ }} /> )} - + ); } diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterDescriptionListGroups.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterDescriptionListGroups.tsx new file mode 100644 index 0000000000000..cfb14b78e586c --- /dev/null +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterDescriptionListGroups.tsx @@ -0,0 +1,52 @@ +import type { ReactElement } from 'react'; +import { + DescriptionListDescription, + DescriptionListGroup, + DescriptionListTerm, + Flex, + FlexItem, +} from '@patternfly/react-core'; + +import type { SearchFilter } from 'types/search'; + +import type { CompoundSearchFilterAttribute } from '../types'; +import { getCompoundSearchFilterLabelDescriptions } from '../utils/utils'; + +export type CompoundSearchFilterDescriptionListGroupsProps = { + attributes: CompoundSearchFilterAttribute[]; + searchFilter: SearchFilter; +}; + +// For maximum composability and reusability: +// Render description list groups instead of description list. +// If rules array is empty, caller is responsible for conditional rendering, like a warning alert. +function CompoundSearchFilterDescriptionListGroups({ + attributes, + searchFilter, +}: CompoundSearchFilterDescriptionListGroupsProps): ReactElement { + const labelGroupDescriptions = getCompoundSearchFilterLabelDescriptions( + attributes, + searchFilter + ); + + return ( + <> + {labelGroupDescriptions.map(({ group, items }) => { + return ( + + {group.label} + + + {items.map((item) => ( + {item.label} + ))} + + + + ); + })} + + ); +} + +export default CompoundSearchFilterDescriptionListGroups; diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterInputField.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterInputField.tsx index 66ae8a186bd5e..919f4922859e0 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterInputField.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterInputField.tsx @@ -6,18 +6,17 @@ import type { CompoundSearchFilterEntity, OnSearchCallback, } from '../types'; +import CompoundSearchFilterSelectInputField from './CompoundSearchFilterSelectInputField'; import SearchFilterConditionDate from './SearchFilterConditionDate'; import SearchFilterConditionNumber from './SearchFilterConditionNumber'; import SearchFilterConditionText from './SearchFilterConditionText'; import SearchFilterAutocompleteSelect from './SearchFilterAutocompleteSelect'; -import SearchFilterSelectExclusiveDouble from './SearchFilterSelectExclusiveDouble'; -import SearchFilterSelectExclusiveSingle from './SearchFilterSelectExclusiveSingle'; -import SearchFilterSelectInclusive from './SearchFilterSelectInclusive'; import SearchFilterText from './SearchFilterText'; export type CompoundSearchFilterInputFieldProps = { entity: CompoundSearchFilterEntity; attribute: CompoundSearchFilterAttribute; + isDisabled?: boolean; onSearch: OnSearchCallback; searchFilter: SearchFilter; additionalContextFilter?: SearchFilter; @@ -26,6 +25,7 @@ export type CompoundSearchFilterInputFieldProps = { function CompoundSearchFilterInputField({ entity, attribute, + isDisabled = false, onSearch, searchFilter, additionalContextFilter, @@ -33,43 +33,55 @@ function CompoundSearchFilterInputField({ const { inputType } = attribute; switch (inputType) { case 'text': - return ; + return ( + + ); case 'date-picker': - return ; + return ( + + ); case 'condition-number': - return ; - case 'condition-text': - return ; - case 'autocomplete': return ( - ); - case 'select-exclusive-double': + case 'condition-text': return ( - ); - case 'select-exclusive-single': + case 'autocomplete': return ( - ); + case 'select-exclusive-double': + case 'select-exclusive-single': case 'select': return ( - diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterLabels.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterLabels.tsx index ec1cfd5776173..9c0eb54b3c9c8 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterLabels.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterLabels.tsx @@ -5,18 +5,15 @@ import { Globe } from 'react-feather'; // eslint-disable-line limited/no-feather import type { SearchFilter } from 'types/search'; import type { CompoundSearchFilterAttribute, CompoundSearchFilterConfig } from '../types'; -import { getCompoundSearchFilterLabelDescriptionOrNull, updateSearchFilter } from '../utils/utils'; -import type { CompoundSearchFilterLabelDescription, IsGlobalPredicate } from '../utils/utils'; - -import './SearchFilterChips.css'; - -const isGlobalPredicateFalse: IsGlobalPredicate = () => false; +import { getCompoundSearchFilterLabelDescriptions, updateSearchFilter } from '../utils/utils'; +import type { IsGlobalPredicate } from '../utils/utils'; const iconGlobe = ; export type CompoundSearchFilterLabelsProps = { attributesSeparateFromConfig: CompoundSearchFilterAttribute[]; config: CompoundSearchFilterConfig; + hasClearFilters?: boolean; isGlobalPredicate?: IsGlobalPredicate; // for certain values in AdvancedFilterToolbar.tsx file onFilterChange?: (searchFilter: SearchFilter) => void; // omit for view-based report details searchFilter: SearchFilter; @@ -25,7 +22,8 @@ export type CompoundSearchFilterLabelsProps = { function CompoundSearchFilterLabels({ attributesSeparateFromConfig, config, - isGlobalPredicate = isGlobalPredicateFalse, + hasClearFilters = true, + isGlobalPredicate, onFilterChange, searchFilter, }: CompoundSearchFilterLabelsProps): ReactElement { @@ -33,21 +31,14 @@ function CompoundSearchFilterLabels({ const attributesFromConfig = config.flatMap(({ attributes }) => attributes); const attributes = [...attributesSeparateFromConfig, ...attributesFromConfig]; - const labelGroupDescriptions: CompoundSearchFilterLabelDescription[] = []; - attributes.forEach((attribute) => { - const labelDescriptionOrNull = getCompoundSearchFilterLabelDescriptionOrNull( - attribute, - searchFilter, - isGlobalPredicate - ); - if (labelDescriptionOrNull !== null) { - // Attribute has one or more values in the search filter. - labelGroupDescriptions.push(labelDescriptionOrNull); - } - }); + const labelGroupDescriptions = getCompoundSearchFilterLabelDescriptions( + attributes, + searchFilter, + isGlobalPredicate + ); return ( - + {labelGroupDescriptions.map(({ group, items }) => { return ( @@ -81,7 +72,7 @@ function CompoundSearchFilterLabels({ ); })} - {labelGroupDescriptions.length !== 0 && onFilterChange && ( + {hasClearFilters && labelGroupDescriptions.length !== 0 && onFilterChange && ( diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterSelectInputField.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterSelectInputField.tsx new file mode 100644 index 0000000000000..69cacc2c08d11 --- /dev/null +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/CompoundSearchFilterSelectInputField.tsx @@ -0,0 +1,56 @@ +import type { SearchFilter } from 'types/search'; +import { ensureExhaustive } from 'utils/type.utils'; + +import type { GenericSelectSearchFilterAttribute, OnSearchCallback } from '../types'; +import SearchFilterSelectExclusiveDouble from './SearchFilterSelectExclusiveDouble'; +import SearchFilterSelectExclusiveSingle from './SearchFilterSelectExclusiveSingle'; +import SearchFilterSelectInclusive from './SearchFilterSelectInclusive'; + +export type CompoundSearchFilterSelectInputFieldProps = { + attribute: GenericSelectSearchFilterAttribute; + isDisabled?: boolean; + onSearch: OnSearchCallback; + searchFilter: SearchFilter; +}; + +function CompoundSearchFilterSelectInputField({ + attribute, + isDisabled = false, + onSearch, + searchFilter, +}: CompoundSearchFilterSelectInputFieldProps) { + const { inputType } = attribute; + switch (inputType) { + case 'select-exclusive-double': + return ( + + ); + case 'select-exclusive-single': + return ( + + ); + case 'select': + return ( + + ); + default: + return ensureExhaustive(inputType); + } +} + +export default CompoundSearchFilterSelectInputField; diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/EntitySelector.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/EntitySelector.tsx index 9f01b6ebe0bff..2eee1d6eb00a8 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/EntitySelector.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/EntitySelector.tsx @@ -1,4 +1,4 @@ -import { SelectOption } from '@patternfly/react-core'; +import { SelectOption, ToolbarItem } from '@patternfly/react-core'; import SimpleSelect from './SimpleSelect'; import type { CompoundSearchFilterConfig, CompoundSearchFilterEntity } from '../types'; @@ -8,28 +8,38 @@ export type EntitySelectorOnChange = (value: string | number | undefined) => voi export type EntitySelectorProps = { entity: CompoundSearchFilterEntity; + isDisabled: boolean; onChange: EntitySelectorOnChange; config: CompoundSearchFilterConfig; menuToggleClassName?: string; }; -function EntitySelector({ entity, onChange, config, menuToggleClassName }: EntitySelectorProps) { +function EntitySelector({ + entity, + isDisabled = false, + onChange, + config, + menuToggleClassName, +}: EntitySelectorProps) { return ( - - {config.map(({ displayName }) => { - return ( - - {displayName} - - ); - })} - + + + {config.map(({ displayName }) => { + return ( + + {displayName} + + ); + })} + + ); } diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterAutocompleteSelect.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterAutocompleteSelect.tsx index dc6c4fdbf586a..724eae38a0056 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterAutocompleteSelect.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterAutocompleteSelect.tsx @@ -1,5 +1,5 @@ import { useState } from 'react'; -import { Button } from '@patternfly/react-core'; +import { Button, ToolbarItem } from '@patternfly/react-core'; import { ArrowRightIcon } from '@patternfly/react-icons'; import type { SearchFilter } from 'types/search'; @@ -11,6 +11,7 @@ import AutocompleteSelect from './AutocompleteSelect'; export type SearchFilterAutocompleteSelectProps = { additionalContextFilter?: SearchFilter; attribute: GenericSearchFilterAttribute; + isDisabled?: boolean; onSearch: OnSearchCallback; searchCategory: string; searchFilter: SearchFilter; @@ -19,6 +20,7 @@ export type SearchFilterAutocompleteSelectProps = { function SearchFilterAutocompleteSelect({ additionalContextFilter, attribute, + isDisabled = false, onSearch, searchCategory, searchFilter, @@ -41,23 +43,28 @@ function SearchFilterAutocompleteSelect({ return ( <> - - + + + + + - )} - - ); -} - -export default SearchFilterChips; diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionDate.cy.jsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionDate.cy.jsx new file mode 100644 index 0000000000000..0e46ac57d2ea9 --- /dev/null +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionDate.cy.jsx @@ -0,0 +1,301 @@ +import { Toolbar, ToolbarContent } from '@patternfly/react-core'; + +import SearchFilterConditionDate from './SearchFilterConditionDate'; + +const attribute = { + displayName: 'Discovered time', + filterChipLabel: 'CVE discovered time', + searchTerm: 'CVE Created Time', + inputType: 'date-picker', +}; + +const selectors = { + conditionSelectToggle: 'button[aria-label="Condition selector toggle"]', + conditionSelectItems: '[aria-label="Condition selector menu"] li', + singleDateInput: 'input[aria-label="Filter by date"]', + startDateInput: 'input[aria-label="Filter by start date"]', + endDateInput: 'input[aria-label="Filter by end date"]', + applyButton: 'button[aria-label="Apply condition and date input to search"]', + relativeDaysInput: 'input[aria-label="Number of days"]', + relativeApplyButton: 'button[aria-label="Apply relative date filter"]', + relativeMinDaysInput: 'input[aria-label="Minimum days ago"]', + relativeMaxDaysInput: 'input[aria-label="Maximum days ago"]', + relativeRangeApplyButton: 'button[aria-label="Apply relative date range filter"]', +}; + +const endBeforeStartErrorText = 'The end date must be on or after the start date'; + +function setup() { + const onSearch = cy.stub().as('onSearch'); + + cy.mount( + + + + + + ); +} + +function selectCondition(condition) { + const escapedCondition = condition.replace(/[()]/g, '\\$&'); + cy.get(selectors.conditionSelectToggle).click(); + cy.get(`${selectors.conditionSelectItems} button`) + .contains(new RegExp(`^${escapedCondition}$`, 'i')) + .click(); +} + +describe(Cypress.spec.relative, () => { + it('should list all conditions in the expected order', () => { + setup(); + + cy.get(selectors.conditionSelectToggle).click(); + + cy.get(selectors.conditionSelectItems).should('have.length', 6); + cy.get(selectors.conditionSelectItems).eq(0).should('have.text', 'Before'); + cy.get(selectors.conditionSelectItems).eq(1).should('have.text', 'On'); + cy.get(selectors.conditionSelectItems).eq(2).should('have.text', 'After'); + cy.get(selectors.conditionSelectItems).eq(3).should('have.text', 'More than (days ago)'); + cy.get(selectors.conditionSelectItems).eq(4).should('have.text', 'Between'); + cy.get(selectors.conditionSelectItems).eq(5).should('have.text', 'Between (days ago)'); + }); + + it('should keep single-date apply behavior for the After condition', () => { + setup(); + + selectCondition('After'); + + cy.get(selectors.singleDateInput).type('01/15/2034'); + cy.get(selectors.applyButton).click(); + + cy.get('@onSearch').should('have.been.calledWithExactly', [ + { + action: 'APPEND', + category: 'CVE Created Time', + value: '>01/15/2034', + }, + ]); + cy.get(selectors.singleDateInput).should('have.value', ''); + }); + + it('should apply the On condition as a bare date with no prefix', () => { + setup(); + + selectCondition('On'); + + cy.get(selectors.singleDateInput).type('01/15/2034'); + cy.get(selectors.applyButton).click(); + + cy.get('@onSearch').should('have.been.calledWithExactly', [ + { + action: 'APPEND', + category: 'CVE Created Time', + value: '01/15/2034', + }, + ]); + cy.get(selectors.singleDateInput).should('have.value', ''); + }); + + it('should reveal start and end date inputs when Between is selected', () => { + setup(); + + selectCondition('Between'); + + cy.get(selectors.singleDateInput).should('not.exist'); + cy.get(selectors.startDateInput).should('exist'); + cy.get(selectors.endDateInput).should('exist'); + }); + + it('should disable the end date input until the start date is valid', () => { + setup(); + + selectCondition('Between'); + + cy.get(selectors.endDateInput).should('be.disabled'); + + cy.get(selectors.startDateInput).type('01/15/2034'); + + cy.get(selectors.endDateInput).should('be.enabled'); + // End date defaults to the day after the start date. + cy.get(selectors.endDateInput).should('have.value', '01/16/2034'); + }); + + it('should keep a chosen end date when the start date changes to an earlier date', () => { + setup(); + + selectCondition('Between'); + + cy.get(selectors.startDateInput).type('01/15/2034'); + cy.get(selectors.endDateInput).clear(); + cy.get(selectors.endDateInput).type('03/20/2034'); + + cy.get(selectors.startDateInput).clear(); + cy.get(selectors.startDateInput).type('01/10/2034'); + + cy.get(selectors.endDateInput).should('have.value', '03/20/2034'); + }); + + it('should re-default the end date when the start date moves past it', () => { + setup(); + + selectCondition('Between'); + + cy.get(selectors.startDateInput).type('01/15/2034'); + // The defaulted end date is before the new start date below. + cy.get(selectors.endDateInput).should('have.value', '01/16/2034'); + cy.get(selectors.startDateInput).clear(); + cy.get(selectors.startDateInput).type('02/01/2034'); + + cy.get(selectors.endDateInput).should('have.value', '02/02/2034'); + }); + + it('should apply a valid range as a tr/- value and clear the inputs', () => { + setup(); + + selectCondition('Between'); + + cy.get(selectors.startDateInput).type('01/15/2034'); + cy.get(selectors.endDateInput).clear(); + cy.get(selectors.endDateInput).type('01/20/2034'); + + cy.get(selectors.applyButton).click(); + + const startMs = new Date(2034, 0, 15, 0, 0, 0, 0).getTime(); + const endMs = new Date(2034, 0, 20, 23, 59, 59, 999).getTime(); + cy.get('@onSearch').should('have.been.calledWithExactly', [ + { + action: 'APPEND', + category: 'CVE Created Time', + value: `tr/${startMs}-${endMs}`, + }, + ]); + cy.get(selectors.startDateInput).should('have.value', ''); + cy.get(selectors.endDateInput).should('have.value', ''); + }); + + it('should apply a same-day range', () => { + setup(); + + selectCondition('Between'); + + cy.get(selectors.startDateInput).type('01/15/2034'); + cy.get(selectors.endDateInput).clear(); + cy.get(selectors.endDateInput).type('01/15/2034'); + + cy.get(selectors.applyButton).click(); + + const startMs = new Date(2034, 0, 15, 0, 0, 0, 0).getTime(); + const endMs = new Date(2034, 0, 15, 23, 59, 59, 999).getTime(); + cy.get('@onSearch').should('have.been.calledWithExactly', [ + { + action: 'APPEND', + category: 'CVE Created Time', + value: `tr/${startMs}-${endMs}`, + }, + ]); + }); + + it('should show an inline error and not emit when the end date is before the start date', () => { + setup(); + + selectCondition('Between'); + + cy.get(selectors.startDateInput).type('01/15/2034'); + cy.get(selectors.endDateInput).clear(); + cy.get(selectors.endDateInput).type('01/10/2034'); + + cy.contains(endBeforeStartErrorText).should('exist'); + + cy.get(selectors.applyButton).click(); + + cy.get('@onSearch').should('not.have.been.called'); + }); + + it('should not emit when the end date is empty', () => { + setup(); + + selectCondition('Between'); + + cy.get(selectors.startDateInput).type('01/15/2034'); + cy.get(selectors.endDateInput).clear(); + + cy.get(selectors.applyButton).click(); + + cy.get('@onSearch').should('not.have.been.called'); + }); + + it('should apply the "More than (days ago)" condition as >Nd format and clear the input', () => { + setup(); + + selectCondition('More than (days ago)'); + + cy.get(selectors.relativeDaysInput).clear(); + cy.get(selectors.relativeDaysInput).type('365'); + cy.get(selectors.relativeApplyButton).click(); + + cy.get('@onSearch').should('have.been.calledWithExactly', [ + { + action: 'APPEND', + category: 'CVE Created Time', + value: '>365d', + }, + ]); + cy.get(selectors.relativeDaysInput).should('have.value', ''); + }); + + it('should apply the "Between (days ago)" condition as Nd-Md format and clear the inputs', () => { + setup(); + + selectCondition('Between (days ago)'); + + cy.get(selectors.relativeMinDaysInput).clear(); + cy.get(selectors.relativeMinDaysInput).type('30'); + cy.get(selectors.relativeMaxDaysInput).clear(); + cy.get(selectors.relativeMaxDaysInput).type('90'); + cy.get(selectors.relativeRangeApplyButton).click(); + + cy.get('@onSearch').should('have.been.calledWithExactly', [ + { + action: 'APPEND', + category: 'CVE Created Time', + value: '30d-90d', + }, + ]); + cy.get(selectors.relativeMinDaysInput).should('have.value', ''); + cy.get(selectors.relativeMaxDaysInput).should('have.value', ''); + }); + + it('should not emit "More than (days ago)" when the input is empty', () => { + setup(); + + selectCondition('More than (days ago)'); + + cy.get(selectors.relativeApplyButton).click(); + + cy.get('@onSearch').should('not.have.been.called'); + }); + + it('should not emit "Between (days ago)" when inputs are empty', () => { + setup(); + + selectCondition('Between (days ago)'); + + cy.get(selectors.relativeRangeApplyButton).click(); + + cy.get('@onSearch').should('not.have.been.called'); + }); + + it('should not emit "Between (days ago)" when min exceeds max', () => { + setup(); + + selectCondition('Between (days ago)'); + + cy.get(selectors.relativeMinDaysInput).clear(); + cy.get(selectors.relativeMinDaysInput).type('90'); + cy.get(selectors.relativeMaxDaysInput).clear(); + cy.get(selectors.relativeMaxDaysInput).type('30'); + cy.get(selectors.relativeRangeApplyButton).click(); + + cy.get('@onSearch').should('not.have.been.called'); + }); +}); diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionDate.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionDate.tsx index 54bf4be1c4846..e580997fd35cc 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionDate.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionDate.tsx @@ -1,96 +1,122 @@ import { useState } from 'react'; -import { Button, DatePicker, Flex, SelectOption } from '@patternfly/react-core'; -import { ArrowRightIcon } from '@patternfly/react-icons'; -import { format } from 'date-fns'; +import { SelectOption, ToolbarItem } from '@patternfly/react-core'; -import { dateConditionMap, dateConditions } from '../utils/utils'; +import { + dateConditionMap, + dateConditions, + dateRangeCondition, + dateRelativeOlderThanCondition, + dateRelativeRangeCondition, +} from '../utils/utils'; import type { GenericSearchFilterAttribute, OnSearchCallback } from '../types'; +import SearchFilterDateRange from './SearchFilterDateRange'; +import SearchFilterDateRelativeOlderThan from './SearchFilterDateRelativeOlderThan'; +import SearchFilterDateRelativeRange from './SearchFilterDateRelativeRange'; +import SearchFilterDateSingle from './SearchFilterDateSingle'; import SimpleSelect from './SimpleSelect'; -function dateFormat(date: Date): string { - return format(date, 'MM/DD/YYYY'); -} +const conditions = [ + ...dateConditions, + dateRelativeOlderThanCondition, + dateRangeCondition, + dateRelativeRangeCondition, +] as const; -function dateParse(date: string): Date { - const split = date.split('/'); - if (split.length !== 3) { - return new Date('Invalid Date'); - } - const month = split[0]; - const day = split[1]; - const year = split[2]; - if (month.length !== 2 || day.length !== 2 || year.length !== 4) { - return new Date('Invalid Date'); - } - return new Date( - `${year.padStart(4, '0')}-${month.padStart(2, '0')}-${day.padStart(2, '0')}T00:00:00` - ); -} +type DateCondition = (typeof conditions)[number]; export type SearchFilterConditionDateProps = { attribute: GenericSearchFilterAttribute; + isDisabled?: boolean; onSearch: OnSearchCallback; // does not depend on searchFilter }; -function SearchFilterConditionDate({ attribute, onSearch }: SearchFilterConditionDateProps) { +function DateConditionInput({ + condition, + category, + isDisabled, + onSearch, +}: { + condition: DateCondition; + category: string; + isDisabled: boolean; + onSearch: OnSearchCallback; +}) { + switch (condition) { + case dateRelativeOlderThanCondition: + return ( + + ); + case dateRangeCondition: + return ( + + ); + case dateRelativeRangeCondition: + return ( + + ); + default: + return ( + + ); + } +} + +function SearchFilterConditionDate({ + attribute, + isDisabled = false, + onSearch, +}: SearchFilterConditionDateProps) { const { searchTerm: category } = attribute; - const [conditionExternal, setConditionExternal] = useState(dateConditions[1]); - const [dateString, setDateString] = useState(''); + const [conditionExternal, setConditionExternal] = useState('On'); return ( - - - setConditionExternal(conditionSelected as (typeof dateConditions)[number]) - } - ariaLabelMenu="Condition selector menu" - ariaLabelToggle="Condition selector toggle" - > - {dateConditions.map((condition) => { - return ( - - {condition} - - ); - })} - - { - setDateString(datePicked); - }} - dateFormat={dateFormat} - dateParse={dateParse} - placeholder="MM/DD/YYYY" - invalidFormatText="Enter valid date: MM/DD/YYYY" - /> - - + ariaLabelMenu="Condition selector menu" + ariaLabelToggle="Condition selector toggle" + > + {conditions.map((condition) => { + return ( + + {condition} + + ); + })} + + + + ); } diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionNumber.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionNumber.tsx index 3c6ed9a2ac367..86a111548dd71 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionNumber.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionNumber.tsx @@ -1,6 +1,6 @@ import { useState } from 'react'; import type { FormEvent } from 'react'; -import { Button, NumberInput, SelectOption } from '@patternfly/react-core'; +import { Button, NumberInput, SelectOption, ToolbarItem } from '@patternfly/react-core'; import { ArrowRightIcon } from '@patternfly/react-icons'; import clamp from 'lodash/clamp'; @@ -22,11 +22,16 @@ function roundToOneDecimalPlace(value: number) { export type SearchFilterConditionNumberProps = { attribute: GenericSearchFilterAttribute; + isDisabled?: boolean; onSearch: OnSearchCallback; // does not depend on searchFilter }; -function SearchFilterConditionNumber({ attribute, onSearch }: SearchFilterConditionNumberProps) { +function SearchFilterConditionNumber({ + attribute, + isDisabled = false, + onSearch, +}: SearchFilterConditionNumberProps) { const { searchTerm: category } = attribute; const [conditionExternal, setConditionExternal] = useState(conditions[0]); @@ -50,65 +55,72 @@ function SearchFilterConditionNumber({ attribute, onSearch }: SearchFilterCondit return ( <> - - setConditionExternal(conditionSelected as (typeof conditions)[number]) - } - ariaLabelMenu="Condition selector menu" - ariaLabelToggle="Condition selector toggle" - > - {conditions.map((condition) => { - return ( - - {condition} - - ); - })} - - ) => { - const { value: valueChanged } = event.target as HTMLInputElement; - setValue(Number(valueChanged)); - }} - onBlur={(event: FormEvent) => { - const target = event.target as HTMLInputElement; - const normalizedNumber = clamp( - Number.isNaN(+target.value) - ? 0 - : roundToOneDecimalPlace(Number(target.value)), - minValue, - maxValue - ); - setValue(normalizedNumber); - }} - onMinus={onMinus} - onPlus={onPlus} - minusBtnAriaLabel="Condition value minus button" - plusBtnAriaLabel="Condition value plus button" - /> - + ariaLabelMenu="Condition selector menu" + ariaLabelToggle="Condition selector toggle" + > + {conditions.map((condition) => { + return ( + + {condition} + + ); + })} + + + + ) => { + const { value: valueChanged } = event.target as HTMLInputElement; + setValue(Number(valueChanged)); + }} + onBlur={(event: FormEvent) => { + const target = event.target as HTMLInputElement; + const normalizedNumber = clamp( + Number.isNaN(+target.value) + ? 0 + : roundToOneDecimalPlace(Number(target.value)), + minValue, + maxValue + ); + setValue(normalizedNumber); + }} + onMinus={onMinus} + onPlus={onPlus} + minusBtnAriaLabel="Condition value minus button" + plusBtnAriaLabel="Condition value plus button" + /> + + + + ); } diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionText.css b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionText.css index d6e6dd49b5068..e2f7f129408a1 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionText.css +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionText.css @@ -1,4 +1,4 @@ /* 100.000% rounded up to 10em overrides 100% */ -.pf-v5-c-form-control.ConditionTextInput { - --pf-v5-c-form-control--Width: 8em; +.pf-v6-c-form-control.ConditionTextInput { + --pf-v6-c-form-control--Width: 8em; } diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionText.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionText.tsx index e49549f7b6445..6e2640ce8f547 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionText.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterConditionText.tsx @@ -7,6 +7,7 @@ import { SelectList, SelectOption, TextInput, + ToolbarItem, } from '@patternfly/react-core'; import type { MenuToggleElement } from '@patternfly/react-core'; import { ArrowRightIcon } from '@patternfly/react-icons'; @@ -77,11 +78,16 @@ export function convertFromInternalToExternalConditionText( export type SearchFilterConditionTextProps = { attribute: ConditionTextFilterAttribute; + isDisabled?: boolean; onSearch: OnSearchCallback; // does not depend on searchFilter }; -function SearchFilterConditionText({ attribute, onSearch }: SearchFilterConditionTextProps) { +function SearchFilterConditionText({ + attribute, + isDisabled = false, + onSearch, +}: SearchFilterConditionTextProps) { const { inputProps, searchTerm: category } = attribute; const { conditionProps: { conditionEntries }, @@ -113,10 +119,11 @@ function SearchFilterConditionText({ attribute, onSearch }: SearchFilterConditio const toggle = (toggleRef: Ref) => ( {conditionSelected[1]} @@ -125,49 +132,55 @@ function SearchFilterConditionText({ attribute, onSearch }: SearchFilterConditio return ( <> - - ) => { - const { value: changedText } = event.target as HTMLInputElement; - setExternalText(changedText); - }} - validated={validateExternalText(externalText) ? 'success' : 'error'} - value={externalText} - /> - + + + + + ) => { + const { value: changedText } = event.target as HTMLInputElement; + setExternalText(changedText); + }} + validated={validateExternalText(externalText) ? 'success' : 'error'} + value={externalText} + /> + + + + ); } diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterDateRange.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterDateRange.tsx new file mode 100644 index 0000000000000..3954eabf41173 --- /dev/null +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterDateRange.tsx @@ -0,0 +1,122 @@ +import { useState } from 'react'; +import { Button, DatePicker, ToolbarItem } from '@patternfly/react-core'; +import { ArrowRightIcon } from '@patternfly/react-icons'; + +import { serializeAbsoluteDateRange } from '../utils/utils'; +import { dateFormat, dateParse, isValidDate } from '../utils/dateInput'; +import type { OnSearchCallback } from '../types'; + +export type SearchFilterDateRangeProps = { + category: string; + isDisabled?: boolean; + onSearch: OnSearchCallback; +}; + +/** + * Date-range body for the date-picker filter input (the Between condition). + * Serializes the picked range to the backend time-range format ("tr/-"). + */ +function SearchFilterDateRange({ + category, + isDisabled = false, + onSearch, +}: SearchFilterDateRangeProps) { + const [startDateString, setStartDateString] = useState(''); + const [endDateString, setEndDateString] = useState(''); + + const startDate = dateParse(startDateString); + + function onStartDateChange(value: string) { + setStartDateString(value); + const datePicked = dateParse(value); + if (!isValidDate(datePicked)) { + return; + } + const endDate = dateParse(endDateString); + if (isValidDate(endDate) && endDate.getTime() >= datePicked.getTime()) { + // Keep a user-chosen end date that is still on or after the new start date. + return; + } + // Default the end date to the day after the start date (PatternFly date-range pattern). + const dayAfterStart = new Date(datePicked); + dayAfterStart.setDate(dayAfterStart.getDate() + 1); + setEndDateString(dateFormat(dayAfterStart)); + } + + function endDateValidator(date: Date): string { + if (!isValidDate(startDate)) { + return ''; + } + return date.getTime() >= startDate.getTime() + ? '' + : 'The end date must be on or after the start date'; + } + + function onApply() { + const end = dateParse(endDateString); + if (!isValidDate(startDate) || !isValidDate(end)) { + return; + } + // null when the start date falls after the end date + const value = serializeAbsoluteDateRange(startDate.getTime(), end.getTime()); + if (value !== null) { + onSearch([ + { + action: 'APPEND', + category, + value, + }, + ]); + setStartDateString(''); + setEndDateString(''); + } + } + + return ( + <> + + { + onStartDateChange(datePicked); + }} + dateFormat={dateFormat} + dateParse={dateParse} + placeholder="MM/DD/YYYY" + invalidFormatText="Enter valid date: MM/DD/YYYY" + /> + + to + + { + setEndDateString(datePicked); + }} + rangeStart={isValidDate(startDate) ? startDate : undefined} + validators={[endDateValidator]} + dateFormat={dateFormat} + dateParse={dateParse} + placeholder="MM/DD/YYYY" + invalidFormatText="Enter valid date: MM/DD/YYYY" + /> + + + + + + ); +} + +export default SearchFilterDateRange; diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterDateRelativeOlderThan.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterDateRelativeOlderThan.tsx new file mode 100644 index 0000000000000..51b3ce2787c84 --- /dev/null +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterDateRelativeOlderThan.tsx @@ -0,0 +1,55 @@ +import { useState } from 'react'; +import { Button, TextInput, ToolbarItem } from '@patternfly/react-core'; +import { ArrowRightIcon } from '@patternfly/react-icons'; + +import { serializeRelativeOlderThan } from '../utils/utils'; +import type { OnSearchCallback } from '../types'; + +export type SearchFilterDateRelativeOlderThanProps = { + category: string; + isDisabled?: boolean; + onSearch: OnSearchCallback; +}; + +function SearchFilterDateRelativeOlderThan({ + category, + isDisabled = false, + onSearch, +}: SearchFilterDateRelativeOlderThanProps) { + const [days, setDays] = useState(''); + + function onApply() { + const value = serializeRelativeOlderThan(Number(days)); + if (days === '' || value === null) { + return; + } + onSearch([{ action: 'APPEND', category, value }]); + setDays(''); + } + + return ( + <> + + setDays(value)} + placeholder="days" + /> + + + + + + ); +} + +export default SearchFilterDateSingle; diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectExclusiveDouble.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectExclusiveDouble.tsx index 88d9dcbc04613..7213e4d60d761 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectExclusiveDouble.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectExclusiveDouble.tsx @@ -1,5 +1,5 @@ import type { ReactElement } from 'react'; -import { SelectOption } from '@patternfly/react-core'; +import { SelectOption, ToolbarItem } from '@patternfly/react-core'; import SelectSingle from 'Components/SelectSingle'; import type { SearchFilter } from 'types/search'; @@ -63,6 +63,7 @@ function getLabelOfSelectedOption( export type SearchFilterSelectExclusiveDoubleProps = { attribute: SelectExclusiveDoubleSearchFilterAttribute; + isDisabled?: boolean; isSeparate?: boolean; // default false if within CompoundSearchFilter onSearch: OnSearchCallback; searchFilter: SearchFilter; @@ -70,6 +71,7 @@ export type SearchFilterSelectExclusiveDoubleProps = { function SearchFilterSelectExclusiveDouble({ attribute, + isDisabled = false, isSeparate = false, onSearch, searchFilter, @@ -93,20 +95,23 @@ function SearchFilterSelectExclusiveDouble({ ); return ( - - {options.map(({ label }) => ( - - {label} - - ))} - + + + {options.map(({ label }) => ( + + {label} + + ))} + + ); } diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectExclusiveSingle.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectExclusiveSingle.tsx index c550787ea67ad..a58e4e773acd9 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectExclusiveSingle.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectExclusiveSingle.tsx @@ -1,5 +1,5 @@ import type { ReactElement } from 'react'; -import { SelectOption } from '@patternfly/react-core'; +import { SelectOption, ToolbarItem } from '@patternfly/react-core'; import SelectSingle from 'Components/SelectSingle'; import type { SearchFilter } from 'types/search'; @@ -9,6 +9,7 @@ import type { OnSearchCallback, SelectExclusiveSingleSearchFilterAttribute } fro export type SearchFilterSelectExclusiveSingleProps = { attribute: SelectExclusiveSingleSearchFilterAttribute; + isDisabled?: boolean; isSeparate?: boolean; // default false if within CompoundSearchFilter onSearch: OnSearchCallback; searchFilter: SearchFilter; @@ -16,6 +17,7 @@ export type SearchFilterSelectExclusiveSingleProps = { function SearchFilterSelectExclusiveSingle({ attribute, + isDisabled = false, isSeparate = false, onSearch, searchFilter, @@ -38,20 +40,23 @@ function SearchFilterSelectExclusiveSingle({ const value = values.length === 1 ? values[0] : ''; return ( - - {options.map(({ label, value }) => ( - - {label} - - ))} - + + + {options.map(({ label, value }) => ( + + {label} + + ))} + + ); } diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectInclusive.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectInclusive.tsx index 07833a6eaf98d..9e2007d0fedd3 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectInclusive.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterSelectInclusive.tsx @@ -1,16 +1,22 @@ import { Fragment } from 'react'; import type { ReactElement } from 'react'; -import { Divider, SelectGroup, SelectList, SelectOption } from '@patternfly/react-core'; +import { + Divider, + SelectGroup, + SelectList, + SelectOption, + ToolbarItem, +} from '@patternfly/react-core'; import CheckboxSelect from 'Components/CheckboxSelect'; import type { SearchFilter } from 'types/search'; import { searchValueAsArray } from 'utils/searchUtils'; import type { OnSearchCallback, SelectSearchFilterAttribute } from '../types'; -import { hasGroupedSelectOptions, hasSelectOptions } from '../utils/utils'; export type SearchFilterSelectInclusiveProps = { attribute: SelectSearchFilterAttribute; + isDisabled?: boolean; isSeparate?: boolean; // default false if within CompoundSearchFilter onSearch: OnSearchCallback; searchFilter: SearchFilter; @@ -18,6 +24,7 @@ export type SearchFilterSelectInclusiveProps = { function SearchFilterSelectInclusive({ attribute, + isDisabled = false, isSeparate = false, onSearch, searchFilter, @@ -32,7 +39,7 @@ function SearchFilterSelectInclusive({ ); - if (hasGroupedSelectOptions(inputProps) && inputProps.groupOptions.length !== 0) { + if ('groupOptions' in inputProps && inputProps.groupOptions.length !== 0) { content = inputProps.groupOptions.map(({ name, options }, index) => { return ( @@ -54,7 +61,7 @@ function SearchFilterSelectInclusive({ ); }); - } else if (hasSelectOptions(inputProps) && inputProps.options.length !== 0) { + } else if ('options' in inputProps && inputProps.options.length !== 0) { content = ( {inputProps.options.map((option) => ( @@ -72,22 +79,25 @@ function SearchFilterSelectInclusive({ } return ( - { - onSearch([ - { - action: checked ? 'SELECT_INCLUSIVE' : 'REMOVE', - category, - value: _value, - }, - ]); - }} - ariaLabelMenu={`${toggleLabel} select menu`} - toggleLabel={toggleLabel} - > - {content} - + + { + onSearch([ + { + action: checked ? 'SELECT_INCLUSIVE' : 'REMOVE', + category, + value: _value, + }, + ]); + }} + ariaLabelMenu={`${toggleLabel} select menu`} + toggleLabel={toggleLabel} + > + {content} + + ); } diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterText.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterText.tsx index 8765bc9e63f7e..64fef1b4c70db 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterText.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SearchFilterText.tsx @@ -1,40 +1,48 @@ import { useState } from 'react'; import type { ReactElement } from 'react'; -import { SearchInput } from '@patternfly/react-core'; +import { SearchInput, ToolbarItem } from '@patternfly/react-core'; import type { GenericSearchFilterAttribute, OnSearchCallback } from '../types'; export type SearchFilterTextProps = { attribute: GenericSearchFilterAttribute; + isDisabled?: boolean; onSearch: OnSearchCallback; // does not depend on searchFilter }; -function SearchFilterText({ attribute, onSearch }: SearchFilterTextProps): ReactElement { +function SearchFilterText({ + attribute, + isDisabled = false, + onSearch, +}: SearchFilterTextProps): ReactElement { const { filterChipLabel, searchTerm: category } = attribute; const textLabel = `Filter results by ${filterChipLabel}`; const [value, setValue] = useState(''); return ( - setValue(_value)} - onSearch={(_event, _value) => { - onSearch([ - { - action: 'APPEND', - category, - value: _value, - }, - ]); - setValue(''); - }} - onClear={() => setValue('')} - submitSearchButtonLabel="Apply text input to search" - /> + + setValue(_value)} + onSearch={(_event, _value) => { + onSearch([ + { + action: 'APPEND', + category, + value: _value, + }, + ]); + setValue(''); + }} + onClear={() => setValue('')} + submitSearchButtonLabel="Apply text input to search" + /> + ); } diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/components/SelectExclusiveSingleTabs.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SelectExclusiveSingleTabs.tsx new file mode 100644 index 0000000000000..98946d153b16d --- /dev/null +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/components/SelectExclusiveSingleTabs.tsx @@ -0,0 +1,50 @@ +import type { ReactElement } from 'react'; +import { Tab, TabTitleText, Tabs } from '@patternfly/react-core'; + +import type { SearchFilter } from 'types/search'; +import { searchValueAsArray } from 'utils/searchUtils'; + +import type { OnSearchPayload, SelectExclusiveSingleSearchFilterAttribute } from '../types'; + +export type SelectExclusiveSingleTabsProps = { + attribute: SelectExclusiveSingleSearchFilterAttribute; + onSelectTab: (payload: OnSearchPayload) => void; + searchFilter: SearchFilter; + tabContentId: string; + usePageInsets?: boolean; +}; + +// Generic presentation of search filter attribute as tabs. +// For example, CVE Snoozed for Nodes Results pages. +// Callback function might need to reset pagination in addition to update search filter. +function SelectExclusiveSingleTabs({ + attribute, + onSelectTab, + searchFilter, + tabContentId, + usePageInsets = true, +}: SelectExclusiveSingleTabsProps): ReactElement { + const { inputProps, searchTerm } = attribute; + const { options } = inputProps; + + const activeKey = searchValueAsArray(searchFilter[searchTerm])[0] ?? options[0].value; + + function onSelect(_, eventKey) { + onSelectTab([{ action: 'SELECT_EXCLUSIVE', category: searchTerm, value: eventKey }]); + } + + return ( + + {options.map(({ label, value }) => ( + {label}} + /> + ))} + + ); +} + +export default SelectExclusiveSingleTabs; diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/types.ts b/ui/apps/platform/src/Components/CompoundSearchFilter/types.ts index 877af980ba7e7..389c00724e63e 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/types.ts +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/types.ts @@ -1,5 +1,6 @@ import type { SearchCategory } from 'services/SearchService'; import type { FeatureFlagEnvVar } from 'types/featureFlag'; +import type { SearchFieldLabel } from 'types/searchOptions'; import type { NonEmptyArray } from 'utils/type.utils'; import type { ConditionTextInputProps } from './components/SearchFilterConditionText'; @@ -13,12 +14,12 @@ export type InputType = | 'select-exclusive-double' | 'select-exclusive-single'; -export type SelectSearchFilterOption = { +export type SelectSearchFilterOption = { label: string; - value: string; + value: T; }; -export type SelectSearchFilterOptions = { - options: SelectSearchFilterOption[]; +export type SelectSearchFilterOptions = { + options: SelectSearchFilterOption[]; }; export type SelectSearchFilterGroupedOption = { @@ -31,7 +32,7 @@ export type SelectSearchFilterGroupedOptions = { type BaseSearchFilterAttribute = { displayName: string; filterChipLabel: string; - searchTerm: string; + searchTerm: SearchFieldLabel; inputType: InputType; featureFlagDependency?: FeatureFlagEnvVar[]; }; @@ -55,23 +56,30 @@ export type SelectExclusiveSingleSearchFilterAttribute = { inputProps: SelectSearchFilterOptions; } & BaseSearchFilterAttribute; +export type SelectSingleSearchFilterAttribute = + | SelectSearchFilterAttribute + | SelectExclusiveSingleSearchFilterAttribute; + export type SelectExclusiveDoubleSearchFilterAttribute = { inputType: 'select-exclusive-double'; inputProps: SelectExclusiveDoubleSearchFilterInputProps; } & BaseSearchFilterAttribute; export type SelectExclusiveDoubleSearchFilterInputProps = { - category2: string; + category2: SearchFieldLabel; options: NonEmptyArray; }; export type SelectExclusiveDoubleSearchFilterOption = { - category: string; + category: SearchFieldLabel; } & SelectSearchFilterOption; export type CompoundSearchFilterAttribute = | ConditionTextFilterAttribute | GenericSearchFilterAttribute + | GenericSelectSearchFilterAttribute; + +export type GenericSelectSearchFilterAttribute = | SelectSearchFilterAttribute | SelectExclusiveDoubleSearchFilterAttribute | SelectExclusiveSingleSearchFilterAttribute; diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/utils/dateInput.ts b/ui/apps/platform/src/Components/CompoundSearchFilter/utils/dateInput.ts new file mode 100644 index 0000000000000..0be76aaa5c710 --- /dev/null +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/utils/dateInput.ts @@ -0,0 +1,36 @@ +import { format } from 'date-fns'; + +/** + * Formats a date for display in the date-picker inputs. + * @param date - The date to format + * @returns Date string like "01/15/2034" + */ +export function dateFormat(date: Date): string { + return format(date, 'MM/DD/YYYY'); +} + +/** + * Parses a strict MM/DD/YYYY date string into a local-time Date. + * @param date - Date string to parse + * @returns The parsed date, or an invalid Date if the string does not match the format + */ +export function dateParse(date: string): Date { + const split = date.split('/'); + if (split.length !== 3) { + return new Date('Invalid Date'); + } + const [month, day, year] = split; + if (month.length !== 2 || day.length !== 2 || year.length !== 4) { + return new Date('Invalid Date'); + } + return new Date(`${year}-${month}-${day}T00:00:00`); +} + +/** + * Returns whether a Date holds a real date value. + * @param date - The date to check + * @returns True if the date is valid + */ +export function isValidDate(date: Date): boolean { + return !Number.isNaN(date.getTime()); +} diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/utils/utils.test.ts b/ui/apps/platform/src/Components/CompoundSearchFilter/utils/utils.test.ts index c5fa9d3fb9b70..b31623091561b 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/utils/utils.test.ts +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/utils/utils.test.ts @@ -1,33 +1,9 @@ -import { deploymentAttributes } from '../attributes/deployment'; -import { imageAttributes } from '../attributes/image'; -import { imageCVEAttributes } from '../attributes/imageCVE'; -import { makeFilterChipDescriptors } from '../components/SearchFilterChips'; import { convertFromInternalToExternalDatePicker, - getDefaultAttributeName, - getDefaultEntityName, - getEntityAttributes, - getSearchFilterConfigWithFeatureFlagDependency, + serializeAbsoluteDateRange, + serializeRelativeDateRange, + serializeRelativeOlderThan, } from './utils'; -import type { CompoundSearchFilterEntity } from '../types'; - -const imageSearchFilterConfig: CompoundSearchFilterEntity = { - displayName: 'Image', - searchCategory: 'IMAGES', - attributes: imageAttributes, -}; - -const deploymentSearchFilterConfig: CompoundSearchFilterEntity = { - displayName: 'Deployment', - searchCategory: 'DEPLOYMENTS', - attributes: deploymentAttributes, -}; - -const imageCVESearchFilterConfig: CompoundSearchFilterEntity = { - displayName: 'Image CVE', - searchCategory: 'IMAGE_VULNERABILITIES_V2', // flat CVE data model - attributes: imageCVEAttributes, -}; describe('utils', () => { describe('convertFromInternalToExternalDatePicker', () => { @@ -69,93 +45,120 @@ describe('utils', () => { expect(() => convertFromInternalToExternalDatePicker('>2024-13-50')).not.toThrow(); expect(convertFromInternalToExternalDatePicker('>2024-13-50')).toEqual('>2024-13-50'); }); - }); - describe('getEntityAttributes', () => { - it('should get the attributes of an entity in a config object', () => { - // Omit EPSSProbability object that has been added to imageCVE attributes. - const config = getSearchFilterConfigWithFeatureFlagDependency( - () => false, - [imageSearchFilterConfig, deploymentSearchFilterConfig, imageCVESearchFilterConfig] + it('formats an absolute date range correctly', () => { + const startMs = new Date(2025, 0, 1).getTime(); + const endMs = new Date(2025, 2, 31).getTime(); + expect(convertFromInternalToExternalDatePicker(`tr/${startMs}-${endMs}`)).toEqual( + 'Between Jan 01, 2025 and Mar 31, 2025' + ); + }); + + it('formats a relative date range correctly', () => { + expect(convertFromInternalToExternalDatePicker('30d-90d')).toEqual( + 'Between 30 and 90 days ago' ); + }); + + it('formats an open-ended relative range correctly', () => { + expect(convertFromInternalToExternalDatePicker('>365d')).toEqual( + 'More than 365 days ago' + ); + }); - const result = getEntityAttributes(config, 'Image CVE'); - - expect(result).toStrictEqual([ - { - displayName: 'CVSS', - filterChipLabel: 'CVSS', - searchTerm: 'CVSS', - inputType: 'condition-number', - }, - { - displayName: 'Discovered time', - filterChipLabel: 'Image CVE discovered time', - searchTerm: 'CVE Created Time', - inputType: 'date-picker', - }, - { - displayName: 'Name', - filterChipLabel: 'Image CVE', - searchTerm: 'CVE', - inputType: 'autocomplete', - }, - ]); + it('returns original value for malformed absolute range', () => { + expect(convertFromInternalToExternalDatePicker('tr/abc-def')).toEqual('tr/abc-def'); + expect(convertFromInternalToExternalDatePicker('tr/123')).toEqual('tr/123'); + expect(convertFromInternalToExternalDatePicker('tr/')).toEqual('tr/'); }); }); - describe('getDefaultEntity', () => { - it('should get the default (first) entity in a config object', () => { - const config = [ - imageSearchFilterConfig, - deploymentSearchFilterConfig, - imageCVESearchFilterConfig, - ]; + describe('serializeAbsoluteDateRange', () => { + it('serializes to tr/- with start-of-day and end-of-day boundaries', () => { + const startMs = new Date(2025, 0, 1, 10, 30).getTime(); + const endMs = new Date(2025, 2, 31, 8, 0).getTime(); + const expectedStartMs = new Date(2025, 0, 1, 0, 0, 0, 0).getTime(); + const expectedEndMs = new Date(2025, 2, 31, 23, 59, 59, 999).getTime(); + expect(serializeAbsoluteDateRange(startMs, endMs)).toEqual( + `tr/${expectedStartMs}-${expectedEndMs}` + ); + }); + + it('serializes a same-day range spanning the whole day', () => { + const dayMs = new Date(2025, 5, 15, 12, 0).getTime(); + const expectedStartMs = new Date(2025, 5, 15, 0, 0, 0, 0).getTime(); + const expectedEndMs = new Date(2025, 5, 15, 23, 59, 59, 999).getTime(); + expect(serializeAbsoluteDateRange(dayMs, dayMs)).toEqual( + `tr/${expectedStartMs}-${expectedEndMs}` + ); + }); - const result = getDefaultEntityName(config); + it('returns null when either input is not a valid date', () => { + const validMs = new Date(2025, 0, 1).getTime(); + expect(serializeAbsoluteDateRange(NaN, validMs)).toBeNull(); + expect(serializeAbsoluteDateRange(validMs, NaN)).toBeNull(); + }); - expect(result).toStrictEqual('Image'); + it('returns null when start date is after end date', () => { + const startMs = new Date(2025, 2, 31).getTime(); + const endMs = new Date(2025, 0, 1).getTime(); + expect(serializeAbsoluteDateRange(startMs, endMs)).toBeNull(); + }); + + it('round-trips through convertFromInternalToExternalDatePicker', () => { + const startMs = new Date(2025, 0, 1).getTime(); + const endMs = new Date(2025, 2, 31).getTime(); + const serialized = serializeAbsoluteDateRange(startMs, endMs); + expect(serialized).not.toBeNull(); + expect(convertFromInternalToExternalDatePicker(serialized as string)).toEqual( + 'Between Jan 01, 2025 and Mar 31, 2025' + ); }); }); - describe('getDefaultAttribute', () => { - it('should get the default (first) attribute of a specific entity in a config object', () => { - const config = [ - imageSearchFilterConfig, - deploymentSearchFilterConfig, - imageCVESearchFilterConfig, - ]; + describe('serializeRelativeOlderThan', () => { + it('serializes a positive integer', () => { + expect(serializeRelativeOlderThan(365)).toEqual('>365d'); + }); + + it('serializes zero', () => { + expect(serializeRelativeOlderThan(0)).toEqual('>0d'); + }); - const result = getDefaultAttributeName(config, 'Image CVE'); + it('returns null for a negative number', () => { + expect(serializeRelativeOlderThan(-1)).toBeNull(); + }); - expect(result).toStrictEqual('CVSS'); + it('returns null for a non-integer', () => { + expect(serializeRelativeOlderThan(1.5)).toBeNull(); }); }); - describe('makeFilterChipDescriptors', () => { - it('should create an array of FilterChipGroupDescriptor objects from a config object', () => { - // Omit EPSSProbability object that has been added to imageCVE attributes. - const config = getSearchFilterConfigWithFeatureFlagDependency( - () => false, - [imageCVESearchFilterConfig] - ); + describe('serializeRelativeDateRange', () => { + it('serializes a valid range', () => { + expect(serializeRelativeDateRange(30, 90)).toEqual('30d-90d'); + }); + + it('serializes equal min and max', () => { + expect(serializeRelativeDateRange(7, 7)).toEqual('7d-7d'); + }); + + it('serializes zero values', () => { + expect(serializeRelativeDateRange(0, 30)).toEqual('0d-30d'); + }); + + it('returns null when min exceeds max', () => { + expect(serializeRelativeDateRange(90, 30)).toBeNull(); + }); + + it('returns null for a negative number', () => { + expect(serializeRelativeDateRange(-1, 30)).toBeNull(); + expect(serializeRelativeDateRange(0, -1)).toBeNull(); + }); - const result = makeFilterChipDescriptors(config); - - expect(result).toStrictEqual([ - { - displayName: 'CVSS', - searchFilterName: 'CVSS', - }, - { - displayName: 'Image CVE discovered time', - searchFilterName: 'CVE Created Time', - }, - { - displayName: 'Image CVE', - searchFilterName: 'CVE', - }, - ]); + it('returns null for a non-integer', () => { + expect(serializeRelativeDateRange(1.5, 30)).toBeNull(); + expect(serializeRelativeDateRange(0, 2.5)).toBeNull(); }); }); }); diff --git a/ui/apps/platform/src/Components/CompoundSearchFilter/utils/utils.tsx b/ui/apps/platform/src/Components/CompoundSearchFilter/utils/utils.tsx index d895384dde9f6..cc115bdae45e8 100644 --- a/ui/apps/platform/src/Components/CompoundSearchFilter/utils/utils.tsx +++ b/ui/apps/platform/src/Components/CompoundSearchFilter/utils/utils.tsx @@ -12,10 +12,7 @@ import type { OnSearchPayload, OnSearchPayloadItem, OnSearchPayloadItemAdd, - SelectSearchFilterAttribute, - SelectSearchFilterGroupedOptions, SelectSearchFilterOption, - SelectSearchFilterOptions, } from '../types'; export const conditionMap = { @@ -38,13 +35,85 @@ export const dateConditions = Object.keys( dateConditionMap ) as unknown as (keyof typeof dateConditionMap)[]; +export const dateRelativeOlderThanCondition = 'More than (days ago)' as const; +export const dateRangeCondition = 'Between' as const; +export const dateRelativeRangeCondition = 'Between (days ago)' as const; + +const absoluteDateRangeRegex = /^tr\/(\d+)-(\d+)$/; +const relativeDateRangeRegex = /^(\d+)d-(\d+)d$/; +const relativeDateOlderThanRegex = /^>(\d+)d$/; + +function isNonNegativeInteger(n: number): boolean { + return Number.isInteger(n) && n >= 0; +} + +export function serializeRelativeOlderThan(days: number): string | null { + return isNonNegativeInteger(days) ? `>${days}d` : null; +} + +export function serializeRelativeDateRange(minDays: number, maxDays: number): string | null { + if (!isNonNegativeInteger(minDays) || !isNonNegativeInteger(maxDays) || minDays > maxDays) { + return null; + } + return `${minDays}d-${maxDays}d`; +} + +/** + * Serializes an absolute date range into the backend time-range query format. + * The start is widened to the start of its day and the end to the end of its day + * (local time) so the range covers the full calendar days the user selected. + * @param startMs - Epoch ms within the first day of the range + * @param endMs - Epoch ms within the last day of the range + * @returns Value like "tr/1735689600000-1743465599999", or null if either input is not + * a valid date or the start date falls after the end date + */ +export function serializeAbsoluteDateRange(startMs: number, endMs: number): string | null { + if (Number.isNaN(startMs) || Number.isNaN(endMs)) { + return null; + } + + const start = new Date(startMs); + start.setHours(0, 0, 0, 0); + const end = new Date(endMs); + end.setHours(23, 59, 59, 999); + + if (start.getTime() > end.getTime()) { + return null; + } + + return `tr/${start.getTime()}-${end.getTime()}`; +} + /** * Formats date picker value like CVE discovered time filter values into user-friendly text - * @param value - Filter value like ">2024-01-01" or "2024-01-01" - * @returns Formatted string like "After January 1, 2024" + * @param value - Filter value like ">2024-01-01", "2024-01-01", "tr/-", "30d-90d", or ">365d" + * @returns Formatted string like "After January 1, 2024" or "Between 30 and 90 days ago" */ export function convertFromInternalToExternalDatePicker(value: string): string { try { + if (value.startsWith('tr/')) { + const absoluteRangeMatch = value.match(absoluteDateRangeRegex); + if (!absoluteRangeMatch) { + return value; // Return original if the time-range value is malformed + } + const startDate = new Date(Number(absoluteRangeMatch[1])); + const endDate = new Date(Number(absoluteRangeMatch[2])); + if (Number.isNaN(startDate.getTime()) || Number.isNaN(endDate.getTime())) { + return value; // Return original if epoch values are invalid + } + return `Between ${getDate(startDate)} and ${getDate(endDate)}`; + } + + const relativeRangeMatch = value.match(relativeDateRangeRegex); + if (relativeRangeMatch) { + return `Between ${relativeRangeMatch[1]} and ${relativeRangeMatch[2]} days ago`; + } + + const olderThanMatch = value.match(relativeDateOlderThanRegex); + if (olderThanMatch) { + return `More than ${olderThanMatch[1]} days ago`; + } + // Parse the condition prefix and date const match = value.match(/^([<>]?)(.+)$/); if (!match) { @@ -102,111 +171,6 @@ export function getAttributeFromEntity( return attributeFound ?? entity?.attributes?.[0]; // default to first attribute } -export function getEntity( - config: CompoundSearchFilterConfig, - entityName: string -): CompoundSearchFilterEntity | undefined { - if (!config || !Array.isArray(config)) { - return undefined; - } - const entity = config.find((entity) => { - return entity.displayName === entityName; - }); - return entity; -} - -export function getAttribute( - config: CompoundSearchFilterConfig, - entityName: string, - attributeName: string -): CompoundSearchFilterAttribute | undefined { - const entity = getEntity(config, entityName); - return entity?.attributes?.find((attribute) => { - return attribute.displayName === attributeName; - }); -} - -export function getDefaultEntityName(config: CompoundSearchFilterConfig): string | undefined { - if (!config || !Array.isArray(config)) { - return undefined; - } - return config?.[0]?.displayName; -} - -export function getEntityAttributes( - config: CompoundSearchFilterConfig, - entityName: string -): CompoundSearchFilterAttribute[] { - const entity = getEntity(config, entityName); - return entity?.attributes ?? []; -} - -export function getDefaultAttributeName( - config: CompoundSearchFilterConfig, - entityName: string -): string | undefined { - const attributes = getEntityAttributes(config, entityName); - return attributes?.[0]?.displayName; -} - -export function ensureConditionNumber(value: unknown): { condition: string; number: number } { - if ( - typeof value === 'object' && - value !== null && - 'condition' in value && - 'number' in value && - typeof value.condition === 'string' && - typeof value.number === 'number' - ) { - return { - condition: value.condition, - number: value.number, - }; - } - return { - condition: conditions[0], - number: 0, - }; -} - -export function ensureConditionDate(value: unknown): { condition: string; date: string } { - if ( - typeof value === 'object' && - value !== null && - 'condition' in value && - 'date' in value && - typeof value.condition === 'string' && - typeof value.date === 'string' - ) { - return { - condition: value.condition, - date: value.date, - }; - } - return { - condition: dateConditions[1], - date: '', - }; -} - -export function isSelectType( - attribute: CompoundSearchFilterAttribute -): attribute is SelectSearchFilterAttribute { - return attribute.inputType === 'select'; -} - -export function hasGroupedSelectOptions( - inputProps: SelectSearchFilterAttribute['inputProps'] -): inputProps is SelectSearchFilterGroupedOptions { - return 'groupOptions' in inputProps; -} - -export function hasSelectOptions( - inputProps: SelectSearchFilterAttribute['inputProps'] -): inputProps is SelectSearchFilterOptions { - return 'options' in inputProps; -} - // Pure function returns searchFilter updated according to payload from interactions. // Assume that update is needed because payload has already been filtered and is non-empty. export function updateSearchFilter( @@ -308,7 +272,7 @@ export type CompoundSearchFilterLabelDescription = { export type IsGlobalPredicate = (category: string, value: string) => boolean; // If attribute has any values in search filter return description else null. -export function getCompoundSearchFilterLabelDescriptionOrNull( +function getCompoundSearchFilterLabelDescriptionOrNull( attribute: CompoundSearchFilterAttribute, searchFilter: SearchFilter, isGlobalPredicate: IsGlobalPredicate @@ -456,6 +420,31 @@ function getLabelForOption(option: SelectSearchFilterOption | undefined, value: return option ? option.label : value; } +const isGlobalPredicateFalse: IsGlobalPredicate = () => false; + +// Return descriptions for compound search filter and also for report configurations. +export function getCompoundSearchFilterLabelDescriptions( + attributes: CompoundSearchFilterAttribute[], + searchFilter: SearchFilter, + isGlobalPredicate: IsGlobalPredicate = isGlobalPredicateFalse // for certain values in AdvancedFilterToolbar.tsx file +): CompoundSearchFilterLabelDescription[] { + const labelGroupDescriptions: CompoundSearchFilterLabelDescription[] = []; + + attributes.forEach((attribute) => { + const labelDescriptionOrNull = getCompoundSearchFilterLabelDescriptionOrNull( + attribute, + searchFilter, + isGlobalPredicate + ); + if (labelDescriptionOrNull !== null) { + // Attribute has one or more values in the search filter. + labelGroupDescriptions.push(labelDescriptionOrNull); + } + }); + + return labelGroupDescriptions; +} + // Given predicate function from useFeatureFlags hook in component // and searchFilterConfig in which some attributes might have featureFlagDependency property, // return config to render search filter. diff --git a/ui/apps/platform/src/Components/ConditionalTooltip.tsx b/ui/apps/platform/src/Components/ConditionalTooltip.tsx new file mode 100644 index 0000000000000..df1519188e076 --- /dev/null +++ b/ui/apps/platform/src/Components/ConditionalTooltip.tsx @@ -0,0 +1,15 @@ +import { Tooltip } from '@patternfly/react-core'; +import type { TooltipProps } from '@patternfly/react-core'; + +export type ConditionalTooltipProps = TooltipProps & { + renderTooltip: boolean; +}; + +function ConditionalTooltip({ renderTooltip, children, ...props }: ConditionalTooltipProps) { + if (renderTooltip) { + return {children}; + } + return children; +} + +export default ConditionalTooltip; diff --git a/ui/apps/platform/src/Components/ContainerArgumentsInfo.tsx b/ui/apps/platform/src/Components/ContainerArgumentsInfo.tsx index cc7501bfd57ef..65ee01e7840af 100644 --- a/ui/apps/platform/src/Components/ContainerArgumentsInfo.tsx +++ b/ui/apps/platform/src/Components/ContainerArgumentsInfo.tsx @@ -1,24 +1,19 @@ -import type { CSSProperties, ReactElement } from 'react'; +import type { ReactElement } from 'react'; import { Card, CardBody, CardTitle, EmptyState, List, ListItem } from '@patternfly/react-core'; type ContainerArgumentsInfoProps = { args: string[]; }; -const styleConstant = { - overflow: 'scroll', - '--pf-v5-u-max-height--MaxHeight': '12ch', -} as CSSProperties; - function ContainerArgumentsInfo({ args }: ContainerArgumentsInfoProps): ReactElement { return ( Arguments {args.length > 0 ? ( - - + + {args.map((arg) => ( - --{arg} + {arg} ))} diff --git a/ui/apps/platform/src/Components/ContainerImageInfo.tsx b/ui/apps/platform/src/Components/ContainerImageInfo.tsx index 2e9cc10a024a6..63587302e8ca1 100644 --- a/ui/apps/platform/src/Components/ContainerImageInfo.tsx +++ b/ui/apps/platform/src/Components/ContainerImageInfo.tsx @@ -1,17 +1,16 @@ import { Link } from 'react-router-dom-v5-compat'; import { Card, CardBody, CardTitle } from '@patternfly/react-core'; -import { vulnerabilitiesAllImagesPath } from 'routePaths'; -import { getWorkloadEntityPagePath } from 'Containers/Vulnerabilities/utils/searchUtils'; import type { ContainerImage } from 'types/deployment.proto'; type ContainerImageInfoProps = { - image: ContainerImage; // note: the k8s API, and our data of it, use singular "command" for this array + image: ContainerImage; + getImageUrl: (imageId: string) => string; }; -function ContainerImageInfo({ image }: ContainerImageInfoProps) { +function ContainerImageInfo({ image, getImageUrl }: ContainerImageInfoProps) { const imageId = image.idV2 && image.idV2 !== '' ? image.idV2 : image.id; - const imageDetailsPageURL = `${vulnerabilitiesAllImagesPath}/${getWorkloadEntityPagePath('Image', imageId, 'OBSERVED')}`; + const imageDetailsPageURL = getImageUrl(imageId); if (imageId === '' || image.notPullable) { const unavailableText = image.notPullable diff --git a/ui/apps/platform/src/Components/ContainerResourcesInfo.tsx b/ui/apps/platform/src/Components/ContainerResourcesInfo.tsx index 14a37364d18cd..f76628a157f8c 100644 --- a/ui/apps/platform/src/Components/ContainerResourcesInfo.tsx +++ b/ui/apps/platform/src/Components/ContainerResourcesInfo.tsx @@ -18,7 +18,7 @@ function ContainerResourcesInfo({ resources }: ContainerResourcesInfoProps) { return ( Resources - + CPU requests (cores) diff --git a/ui/apps/platform/src/Components/ContainerSecretsInfo.tsx b/ui/apps/platform/src/Components/ContainerSecretsInfo.tsx index 54f34b3457067..0c87822ac9d68 100644 --- a/ui/apps/platform/src/Components/ContainerSecretsInfo.tsx +++ b/ui/apps/platform/src/Components/ContainerSecretsInfo.tsx @@ -1,4 +1,3 @@ -import { useState } from 'react'; import { Card, CardBody, @@ -7,8 +6,9 @@ import { DescriptionListDescription, DescriptionListGroup, DescriptionListTerm, + Divider, EmptyState, - ExpandableSection, + Label, Stack, StackItem, } from '@patternfly/react-core'; @@ -20,48 +20,44 @@ type ContainerSecretsInfoProps = { }; function ContainerSecretsInfo({ secrets }: ContainerSecretsInfoProps) { - const initialToggleValues = Array.from({ length: secrets.length }, () => true); - const [secretToggles, setSecretToggles] = useState(initialToggleValues); - - function setToggleAtIndex(i: number) { - const newToggles = [...secretToggles]; - newToggles[i] = !newToggles[i]; - - setSecretToggles(newToggles); - } - return ( Secrets - - {secrets.length > 0 ? ( - secrets.map((secret, index) => ( + {secrets.length > 0 ? ( + + {secrets.map((secret, index) => ( - setToggleAtIndex(index)} - isExpanded={secretToggles[index]} - className="pf-expandable-not-large" - > - - - Source - - {secret.path} - - - - + + + + + + + + + Container path + + + {secret.path} + + + + + {index < secrets.length - 1 && ( + + + + )} + - )) - ) : ( - No secrets - )} - + ))} + + ) : ( + No secrets + )} ); diff --git a/ui/apps/platform/src/Components/ContainerVolumesInfo.tsx b/ui/apps/platform/src/Components/ContainerVolumesInfo.tsx index 7f0dfaccb59ff..17ee2e156b07b 100644 --- a/ui/apps/platform/src/Components/ContainerVolumesInfo.tsx +++ b/ui/apps/platform/src/Components/ContainerVolumesInfo.tsx @@ -1,4 +1,3 @@ -import { useState } from 'react'; import { Card, CardBody, @@ -7,8 +6,9 @@ import { DescriptionListDescription, DescriptionListGroup, DescriptionListTerm, + Divider, EmptyState, - ExpandableSection, + Label, Stack, StackItem, } from '@patternfly/react-core'; @@ -20,75 +20,73 @@ type ContainerVolumesInfoProps = { }; function ContainerVolumesInfo({ volumes }: ContainerVolumesInfoProps) { - const initialToggleValues = Array.from({ length: volumes.length }, () => true); - const [volumeToggles, setVolumeToggles] = useState(initialToggleValues); - - function setToggleAtIndex(i: number) { - const newToggles = [...volumeToggles]; - newToggles[i] = !newToggles[i]; - - setVolumeToggles(newToggles); - } - return ( Volumes - - {volumes.length > 0 ? ( - volumes.map((volume, index) => ( + {volumes.length > 0 ? ( + + {volumes.map((volume, index) => ( - setToggleAtIndex(index)} - isExpanded={volumeToggles[index]} - className="pf-expandable-not-large" - > - - - Source - - {volume.source || '-'} - - - - Destination - - {volume.destination || '-'} - - - - Read only - - {volume.readOnly || 'false'} - - - - Type - - {volume.type} - - - - - Mount propagation - - - {volume.mountPropagation} - - - - + + + + + + + + Source + + {volume.source || '-'} + + + + + Destination + + + {volume.destination || '-'} + + + + Read only + + {volume.readOnly ? 'true' : 'false'} + + + + Type + + {volume.type} + + + + + Mount propagation + + + {volume.mountPropagation} + + + + + {index < volumes.length - 1 && ( + + + + )} + - )) - ) : ( - No volumes - )} - + ))} + + ) : ( + No volumes + )} ); diff --git a/ui/apps/platform/src/Components/ControlDetails.jsx b/ui/apps/platform/src/Components/ControlDetails.jsx deleted file mode 100644 index 347017c03c5c6..0000000000000 --- a/ui/apps/platform/src/Components/ControlDetails.jsx +++ /dev/null @@ -1,43 +0,0 @@ -import PropTypes from 'prop-types'; - -import { standardLabels } from 'messages/standards'; - -import Widget from 'Components/Widget'; - -const ControlDetails = ({ standardId, standardName, control, description, className }) => ( - -
-
- Standard: - - {standardLabels[standardId] || standardName} - -
-
- Control: - {control} -
-
-
{description}
- -); - -ControlDetails.propTypes = { - standardId: PropTypes.string.isRequired, - control: PropTypes.string.isRequired, - description: PropTypes.string.isRequired, - className: PropTypes.string, - standardName: PropTypes.string, -}; - -ControlDetails.defaultProps = { - className: '', - standardName: '', -}; - -export default ControlDetails; diff --git a/ui/apps/platform/src/Components/CountWidget.jsx b/ui/apps/platform/src/Components/CountWidget.jsx deleted file mode 100644 index 606f477897b01..0000000000000 --- a/ui/apps/platform/src/Components/CountWidget.jsx +++ /dev/null @@ -1,54 +0,0 @@ -import { connect } from 'react-redux'; -import PropTypes from 'prop-types'; -import { Link } from 'react-router-dom-v5-compat'; -import Widget from 'Components/Widget'; - -const CountWidget = ({ title, count, description, linkUrl }) => { - const countNode = ( -
0 ? `hover:bg-tertiary-200` : `` - } text-6xl px-5 py-8 h-full flex items-center justify-center`} - > - {count} -
- ); - const descNode = description &&
{description}
; - return ( - - {linkUrl ? ( - - {countNode} - {descNode} - - ) : ( - <> - {countNode} - {descNode} - - )} - - ); -}; - -CountWidget.propTypes = { - title: PropTypes.string.isRequired, - count: PropTypes.number, - description: PropTypes.string, - linkUrl: PropTypes.string, -}; - -CountWidget.defaultProps = { - description: null, - linkUrl: null, - count: 0, -}; - -export default connect()(CountWidget); diff --git a/ui/apps/platform/src/Components/CveType/CveType.test.jsx b/ui/apps/platform/src/Components/CveType/CveType.test.jsx deleted file mode 100644 index 58551ac93bd72..0000000000000 --- a/ui/apps/platform/src/Components/CveType/CveType.test.jsx +++ /dev/null @@ -1,55 +0,0 @@ -import { render } from '@testing-library/react'; - -import CveType from './CveType'; - -describe('CveType', () => { - describe('default style', () => { - test('should show an image CVE type in default style', async () => { - const { asFragment } = render(); - - expect(asFragment()).toMatchSnapshot(); - }); - - test('should show a node CVE type in default style', async () => { - const { asFragment } = render(); - - expect(asFragment()).toMatchSnapshot(); - }); - - test('should show a Kubernetes CVE type in default style', async () => { - const { asFragment } = render(); - - expect(asFragment()).toMatchSnapshot(); - }); - - test('should show an OpenShift CVE type in default style', async () => { - const { asFragment } = render(); - - expect(asFragment()).toMatchSnapshot(); - }); - - test('should show an Istio CVE type in default style', async () => { - const { asFragment } = render(); - - expect(asFragment()).toMatchSnapshot(); - }); - - test('should show an Image and Node CVE type in default style', async () => { - const { asFragment } = render(); - - expect(asFragment()).toMatchSnapshot(); - }); - - test('should show a Node and Kubernetes CVE type in default style', async () => { - const { asFragment } = render(); - - expect(asFragment()).toMatchSnapshot(); - }); - - test('should show a Kubernetes and OpenShift CVE type in default style', async () => { - const { asFragment } = render(); - - expect(asFragment()).toMatchSnapshot(); - }); - }); -}); diff --git a/ui/apps/platform/src/Components/CveType/__snapshots__/CveType.test.jsx.snap b/ui/apps/platform/src/Components/CveType/__snapshots__/CveType.test.jsx.snap deleted file mode 100644 index 07cd5ecbc92b8..0000000000000 --- a/ui/apps/platform/src/Components/CveType/__snapshots__/CveType.test.jsx.snap +++ /dev/null @@ -1,144 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`CveType > default style > should show a Kubernetes CVE type in default style 1`] = ` - - -
-
- Kubernetes CVE -
-
- - -`; - -exports[`CveType > default style > should show a Kubernetes and OpenShift CVE type in default style 1`] = ` - - -
-
- Kubernetes CVE -
-
- OpenShift CVE -
-
- - -`; - -exports[`CveType > default style > should show a Node and Kubernetes CVE type in default style 1`] = ` - - -
-
- Kubernetes CVE -
-
- Node CVE -
-
- - -`; - -exports[`CveType > default style > should show a node CVE type in default style 1`] = ` - - -
-
- Node CVE -
-
- - -`; - -exports[`CveType > default style > should show an Image and Node CVE type in default style 1`] = ` - - -
-
- Image CVE -
-
- Node CVE -
-
- - -`; - -exports[`CveType > default style > should show an Istio CVE type in default style 1`] = ` - - -
-
- Istio CVE -
-
- - -`; - -exports[`CveType > default style > should show an OpenShift CVE type in default style 1`] = ` - - -
-
- OpenShift CVE -
-
- - -`; - -exports[`CveType > default style > should show an image CVE type in default style 1`] = ` - - -
-
- Image CVE -
-
- - -`; diff --git a/ui/apps/platform/src/Components/DashboardLayout/DashboardLayout.jsx b/ui/apps/platform/src/Components/DashboardLayout/DashboardLayout.jsx index 04ee362c48806..dad65fd2b65fc 100644 --- a/ui/apps/platform/src/Components/DashboardLayout/DashboardLayout.jsx +++ b/ui/apps/platform/src/Components/DashboardLayout/DashboardLayout.jsx @@ -10,10 +10,7 @@ const DashboardLayout = ({ headerText, headerComponents, children, banner }) =>
{headerComponents}
-
+
string; }; -function DeploymentContainerConfig({ container }: DeploymentContainerConfigProps) { +function DeploymentContainerConfig({ container, getImageUrl }: DeploymentContainerConfigProps) { const [isExpanded, setIsExpanded] = useState(false); const onToggle = (_isExpanded: boolean) => { setIsExpanded(_isExpanded); }; - const toggleText = container.name; + const toggleContent = ( + + {container.name} + {container.type === 'INIT' && ( + + + + )} + + ); return ( onToggle(_isExpanded)} isExpanded={isExpanded} - displaySize="lg" isWidthLimited data-testid="deployment-container-config" > - - - - + + {container.resources && ( + + + + )} @@ -46,14 +66,16 @@ function DeploymentContainerConfig({ container }: DeploymentContainerConfigProps - - - - + - + + {container.securityContext && ( + + + + )} ); diff --git a/ui/apps/platform/src/Components/DeploymentContainersCard.tsx b/ui/apps/platform/src/Components/DeploymentContainersCard.tsx new file mode 100644 index 0000000000000..cfcbbc7ab6caa --- /dev/null +++ b/ui/apps/platform/src/Components/DeploymentContainersCard.tsx @@ -0,0 +1,69 @@ +import { Card, CardBody, CardTitle, Stack, StackItem } from '@patternfly/react-core'; + +import useContainerGroups from 'hooks/useContainerGroups'; +import type { Container } from 'types/deployment.proto'; +import DeploymentContainerConfig from './DeploymentContainerConfig'; + +type DeploymentContainersCardProps = { + containers: Container[] | null | undefined; + title: string; + getImageUrl: (imageId: string) => string; +}; + +function ContainerGroup({ + containers, + getImageUrl, +}: { + containers: Container[]; + getImageUrl: (imageId: string) => string; +}) { + return ( + + {containers.map((container) => ( + + + + ))} + + ); +} + +function DeploymentContainersCard({ + containers, + title, + getImageUrl, +}: DeploymentContainersCardProps) { + const { regularContainers, initContainers } = useContainerGroups(containers); + + return ( + + + + {title} + + {regularContainers.length > 0 ? ( + + ) : ( + 'None' + )} + + + + {initContainers.length > 0 && ( + + + Init container configuration + + + + + + )} + + ); +} + +export default DeploymentContainersCard; diff --git a/ui/apps/platform/src/Components/DeploymentPortConfig.tsx b/ui/apps/platform/src/Components/DeploymentPortConfig.tsx index 970ab6ac9a420..7872062b9e039 100644 --- a/ui/apps/platform/src/Components/DeploymentPortConfig.tsx +++ b/ui/apps/platform/src/Components/DeploymentPortConfig.tsx @@ -33,7 +33,6 @@ function DeploymentPortConfig({ port }: DeploymentPortConfigProps) { toggleText={toggleText} onToggle={(_event, _isExpanded: boolean) => onToggle(_isExpanded)} isExpanded={isExpanded} - displaySize="lg" isWidthLimited > @@ -66,7 +65,7 @@ function DeploymentPortConfig({ port }: DeploymentPortConfigProps) { {port.exposureInfos.map((exposureInfo) => { return ( - + - {children} - - ); -} -/* eslint-enable generic/react-props-name */ - -export default DescriptionListCompact; diff --git a/ui/apps/platform/src/Components/DescriptionListCompact/index.ts b/ui/apps/platform/src/Components/DescriptionListCompact/index.ts deleted file mode 100644 index a23ca65cf98ec..0000000000000 --- a/ui/apps/platform/src/Components/DescriptionListCompact/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from './DescriptionListCompact'; diff --git a/ui/apps/platform/src/Components/DetailedTooltipContent.tsx b/ui/apps/platform/src/Components/DetailedTooltipContent.tsx index 33d23de167fa9..b5e5aca926164 100644 --- a/ui/apps/platform/src/Components/DetailedTooltipContent.tsx +++ b/ui/apps/platform/src/Components/DetailedTooltipContent.tsx @@ -1,5 +1,5 @@ import type { CSSProperties, ReactElement, ReactNode } from 'react'; -import { Divider, Text, TextVariants } from '@patternfly/react-core'; +import { Content, ContentVariants, Divider } from '@patternfly/react-core'; export type DetailedTooltipContentProps = { title: string; @@ -26,19 +26,19 @@ function DetailedTooltipContent({ const styleConstant = { overflow: 'scroll', - '--pf-v5-u-max-height--MaxHeight': '75vh', + '--pf-v6-u-max-height--MaxHeight': '75vh', } as CSSProperties; return ( -
+
- {title} - + {!!subtitle && {subtitle}}
diff --git a/ui/apps/platform/src/Components/DynamicIcon.tsx b/ui/apps/platform/src/Components/DynamicIcon.tsx index 69cf1bad1f5bc..a63bd1182482e 100644 --- a/ui/apps/platform/src/Components/DynamicIcon.tsx +++ b/ui/apps/platform/src/Components/DynamicIcon.tsx @@ -5,13 +5,13 @@ import type { SVGIconProps } from '@patternfly/react-icons/dist/esm/createIcon'; export function DynamicIcon(props: SVGIconProps) { return ( - + ); } export function DynamicColumnIcon() { - return ; + return ; } export function DynamicTableLabel() { diff --git a/ui/apps/platform/src/Components/EmailNotifier/EmailNotifierForm.tsx b/ui/apps/platform/src/Components/EmailNotifier/EmailNotifierForm.tsx index 84f80259bdc56..3afafc7f3f614 100644 --- a/ui/apps/platform/src/Components/EmailNotifier/EmailNotifierForm.tsx +++ b/ui/apps/platform/src/Components/EmailNotifier/EmailNotifierForm.tsx @@ -74,7 +74,7 @@ function EmailNotifierForm({ return ( <> - +
<> -
+
e.preventDefault()} - className="pf-v5-c-form__group-label-help" + className="pf-v6-c-form__group-label-help" > @@ -143,7 +143,7 @@ function EmailNotifierForm({
{allowUnauthenticatedSmtp && ( + Specifies the email FROM header } @@ -227,7 +227,7 @@ function EmailNotifierForm({ touched={touched} errors={errors} helperText={ - + Specifies the email SENDER header } @@ -261,7 +261,7 @@ function EmailNotifierForm({ } + labelHelp={} fieldId="notifier.labelKey" touched={touched} errors={errors} @@ -320,7 +320,7 @@ function EmailNotifierForm({ label="Hostname for SMTP HELO/EHLO" fieldId="notifier.email.hostnameHeloEhlo" helperText={ - + If left blank, localhost will be used } diff --git a/ui/apps/platform/src/Components/EmailNotifier/EmailNotifierModal.tsx b/ui/apps/platform/src/Components/EmailNotifier/EmailNotifierModal.tsx index 3334316da8ff7..0cd66e0c3c2f2 100644 --- a/ui/apps/platform/src/Components/EmailNotifier/EmailNotifierModal.tsx +++ b/ui/apps/platform/src/Components/EmailNotifier/EmailNotifierModal.tsx @@ -1,5 +1,6 @@ import type { ReactElement } from 'react'; -import { Flex, Modal, Title } from '@patternfly/react-core'; +import { Flex, Title } from '@patternfly/react-core'; +import { Modal } from '@patternfly/react-core/deprecated'; import FormSaveButton from 'Components/PatternFly/FormSaveButton'; import FormCancelButton from 'Components/PatternFly/FormCancelButton'; diff --git a/ui/apps/platform/src/Components/EmailTemplate/EmailTemplateForm.tsx b/ui/apps/platform/src/Components/EmailTemplate/EmailTemplateForm.tsx index 1bd3eefc24327..3329b191709da 100644 --- a/ui/apps/platform/src/Components/EmailTemplate/EmailTemplateForm.tsx +++ b/ui/apps/platform/src/Components/EmailTemplate/EmailTemplateForm.tsx @@ -1,6 +1,7 @@ import type { ReactElement } from 'react'; import { Button, + Content, Flex, FlexItem, Form, @@ -8,9 +9,7 @@ import { FormHelperText, HelperText, HelperTextItem, - Text, TextArea, - TextContent, TextInput, } from '@patternfly/react-core'; import type { FormikContextType } from 'formik'; @@ -46,7 +45,7 @@ function EmailTemplateForm({ const variantForSubject = errors.customSubject && touched.customSubject ? 'error' : 'default'; return ( - + - - + + {values.customSubject.length} / {maxCustomSubjectLength} characters - - + + {!isReadOnly && ( , - , ] : [ - , ]; diff --git a/ui/apps/platform/src/Components/EmptyStateTemplate/EmptyStateTemplate.tsx b/ui/apps/platform/src/Components/EmptyStateTemplate/EmptyStateTemplate.tsx index 7563dccc00043..484424207caca 100644 --- a/ui/apps/platform/src/Components/EmptyStateTemplate/EmptyStateTemplate.tsx +++ b/ui/apps/platform/src/Components/EmptyStateTemplate/EmptyStateTemplate.tsx @@ -1,10 +1,6 @@ import type { ComponentType, PropsWithChildren, ReactElement, ReactNode } from 'react'; -import { - EmptyState, - EmptyStateBody, - EmptyStateHeader, - EmptyStateIcon, -} from '@patternfly/react-core'; +import { EmptyState, EmptyStateBody } from '@patternfly/react-core'; +import type { EmptyStateProps } from '@patternfly/react-core'; import { CubesIcon } from '@patternfly/react-icons'; export type EmptyStateTemplateProps = { @@ -12,7 +8,7 @@ export type EmptyStateTemplateProps = { title: string; headingLevel: 'h1' | 'h2' | 'h3' | 'h4'; icon?: ComponentType>; - iconClassName?: string; + status?: EmptyStateProps['status']; }; function EmptyStateTemplate({ @@ -20,15 +16,16 @@ function EmptyStateTemplate({ title, headingLevel, icon = CubesIcon, - iconClassName = '', + status, }: EmptyStateTemplateProps): ReactElement { return ( - - {title}} - icon={} - headingLevel={headingLevel} - /> + {title}} + variant="lg" + > {children} ); diff --git a/ui/apps/platform/src/Components/EntityIcon.jsx b/ui/apps/platform/src/Components/EntityIcon.jsx index 545764667c7c7..11503a90aba0e 100644 --- a/ui/apps/platform/src/Components/EntityIcon.jsx +++ b/ui/apps/platform/src/Components/EntityIcon.jsx @@ -12,7 +12,6 @@ import policy from 'images/side-panel-icons/policy.svg'; import role from 'images/side-panel-icons/role.svg'; import secrets from 'images/side-panel-icons/secrets.svg'; import serviceAccount from 'images/side-panel-icons/service-account.svg'; -import control from 'images/side-panel-icons/control.svg'; const imageMap = { [entityTypes.CLUSTER]: cluster, @@ -32,7 +31,6 @@ const imageMap = { [entityTypes.ROLE]: role, [entityTypes.SECRET]: secrets, [entityTypes.SERVICE_ACCOUNT]: serviceAccount, - [entityTypes.CONTROL]: control, }; const EntityIcon = ({ className, entityType }) => ( diff --git a/ui/apps/platform/src/Components/EntityScope/EntityScopeCompoundSearchFilter.tsx b/ui/apps/platform/src/Components/EntityScope/EntityScopeCompoundSearchFilter.tsx new file mode 100644 index 0000000000000..147f56941fa7a --- /dev/null +++ b/ui/apps/platform/src/Components/EntityScope/EntityScopeCompoundSearchFilter.tsx @@ -0,0 +1,73 @@ +import { useState } from 'react'; +import type { ReactElement } from 'react'; +import { Flex } from '@patternfly/react-core'; + +import CompoundSearchFilter from 'Components/CompoundSearchFilter/components/CompoundSearchFilter'; +import CompoundSearchFilterLabels from 'Components/CompoundSearchFilter/components/CompoundSearchFilterLabels'; +import type { + CompoundSearchFilterConfig, + OnSearchPayload, +} from 'Components/CompoundSearchFilter/types'; +import { updateSearchFilter } from 'Components/CompoundSearchFilter/utils/utils'; +import type { EntityScope, EntityScopeRule } from 'services/ReportsService.types'; +import type { SearchFilter } from 'types/search'; + +// Why searchFilterConfig and get functions as props? +// Because nodes have cluster but not namespace nor deployment as entities. +// Why not formik object as prop like FiltersQuery? +// To allow possible reuse for saved entity scope in addition to report configuration. +export type EntityScopeCompoundSearchFilterProps = { + entityScope: EntityScope | null; + getEntityScopeRulesFromSearchFilter: (searchFilter: SearchFilter) => EntityScopeRule[]; + getSearchFilterFromEntityScopeRules: (rules: EntityScopeRule[]) => SearchFilter; + searchFilterConfig: CompoundSearchFilterConfig; + setEntityScope: (entityScope: EntityScope) => void; +}; + +// Although entity scope has structured rules instead of query string, +// interaction via compound search filter is similar for resources and filters. +// Caller is responsible for conditional rendering if no rules. +// For example, rules are expected for Deployed imaged but not necessarily for Watched images. +function EntityScopeCompoundSearchFilter({ + entityScope, + getEntityScopeRulesFromSearchFilter, + getSearchFilterFromEntityScopeRules, + searchFilterConfig, + setEntityScope, +}: EntityScopeCompoundSearchFilterProps): ReactElement { + const [searchFilter, setSearchFilter] = useState( + entityScope ? getSearchFilterFromEntityScopeRules(entityScope.rules) : {} + ); + + function onFilterChange(searchFilterChanged: SearchFilter) { + const rules = getEntityScopeRulesFromSearchFilter(searchFilterChanged); + setEntityScope({ rules }); + setSearchFilter(searchFilterChanged); + } + + function onSearch(payload: OnSearchPayload) { + onFilterChange(updateSearchFilter(searchFilter, payload)); + } + + return ( + + + {entityScope && ( + + )} + + ); +} + +export default EntityScopeCompoundSearchFilter; diff --git a/ui/apps/platform/src/Components/EntityScope/EntityScopeDescriptionListGroups.tsx b/ui/apps/platform/src/Components/EntityScope/EntityScopeDescriptionListGroups.tsx new file mode 100644 index 0000000000000..6dc5194691f3c --- /dev/null +++ b/ui/apps/platform/src/Components/EntityScope/EntityScopeDescriptionListGroups.tsx @@ -0,0 +1,48 @@ +import type { ReactElement } from 'react'; +import { + DescriptionListDescription, + DescriptionListGroup, + DescriptionListTerm, + Flex, + FlexItem, +} from '@patternfly/react-core'; + +import type { EntityScope } from 'services/ReportsService.types'; + +import { ruleEntityFieldText, ruleValueText } from './entityScopeText'; + +export type EntityScopeDescriptionListGroupsProps = { + entityScope: EntityScope; +}; + +// For maximum composability and reusability: +// Render description list groups instead of description list. +// If rules array is empty, caller is responsible for conditional rendering, like a warning alert. +function EntityScopeDescriptionListGroups({ + entityScope, +}: EntityScopeDescriptionListGroupsProps): ReactElement { + /* eslint-disable react/no-array-index-key */ + return ( + <> + {entityScope.rules.map((rule, indexOfRule) => { + return ( + + {ruleEntityFieldText(rule)} + + + {rule.values.map((ruleValue, indexOfValue) => ( + + {ruleValueText(ruleValue)} + + ))} + + + + ); + })} + + ); + /* eslint-enable react/no-array-index-key */ +} + +export default EntityScopeDescriptionListGroups; diff --git a/ui/apps/platform/src/Components/EntityScope/entityScopeText.ts b/ui/apps/platform/src/Components/EntityScope/entityScopeText.ts new file mode 100644 index 0000000000000..a5081245d6e77 --- /dev/null +++ b/ui/apps/platform/src/Components/EntityScope/entityScopeText.ts @@ -0,0 +1,38 @@ +import type { + EntityScopeRule, + MatchType, + RuleValue, + ScopeEntity, + ScopeField, +} from 'services/ReportsService.types'; + +const entityTextMap: Record = { + SCOPE_ENTITY_UNSET: '', + SCOPE_ENTITY_DEPLOYMENT: 'Deployment', + SCOPE_ENTITY_NAMESPACE: 'Namespace', + SCOPE_ENTITY_CLUSTER: 'Cluster', +} as const; + +const fieldTextMap: Record = { + FIELD_UNSET: '', + FIELD_ID: 'ID', + FIELD_NAME: 'name', + FIELD_LABEL: 'label', + FIELD_ANNOTATION: 'annotation', +} as const; + +export function ruleEntityFieldText({ entity, field }: EntityScopeRule) { + const entityText = entityTextMap[entity] ?? ''; + const fieldText = fieldTextMap[field] ?? ''; + + return entityText && fieldText ? `${entityText} ${fieldText}` : 'Resource or field unset'; +} + +const matchTypeSuffixMap: Record = { + EXACT: ' (exact match)', + REGEX: ' (regex match)', +} as const; + +export function ruleValueText({ value, matchType }: RuleValue) { + return `${value}${matchTypeSuffixMap[matchType] ?? ''}`; +} diff --git a/ui/apps/platform/src/Components/EntityScope/searchFilterConfig.ts b/ui/apps/platform/src/Components/EntityScope/searchFilterConfig.ts new file mode 100644 index 0000000000000..480d85a3dd06b --- /dev/null +++ b/ui/apps/platform/src/Components/EntityScope/searchFilterConfig.ts @@ -0,0 +1,67 @@ +import { + clusterIdAttribute, + clusterLabelAttribute, + clusterNameAttribute, +} from 'Components/CompoundSearchFilter/attributes/cluster'; +import { + Annotation as deploymentAnnotationAttribute, + ID as deploymentIdAttribute, + Label as deploymentLabelAttribute, + Name as deploymentNameAttribute, +} from 'Components/CompoundSearchFilter/attributes/deployment'; +import { + Annotation as namespaceAnnotationAttribute, + ID as namespaceIdAttribute, + Label as namespaceLabelAttribute, + Name as namespaceNameAttribute, +} from 'Components/CompoundSearchFilter/attributes/namespace'; +import type { CompoundSearchFilterEntity } from 'Components/CompoundSearchFilter/types'; + +const searchFilterEntityForCluster: CompoundSearchFilterEntity = { + displayName: 'Cluster', + searchCategory: 'CLUSTERS', + attributes: [ + // 'Cluster Annotation' is not a search filter + clusterIdAttribute, + clusterLabelAttribute, + clusterNameAttribute, + ], +}; + +const searchFilterEntityForDeployment: CompoundSearchFilterEntity = { + displayName: 'Deployment', + searchCategory: 'DEPLOYMENTS', + attributes: [ + deploymentAnnotationAttribute, + deploymentIdAttribute, + deploymentLabelAttribute, + deploymentNameAttribute, + ], +}; + +const searchFilterEntityForNamespace: CompoundSearchFilterEntity = { + displayName: 'Namespace', + searchCategory: 'NAMESPACES', + attributes: [ + namespaceAnnotationAttribute, + namespaceIdAttribute, + namespaceLabelAttribute, + namespaceNameAttribute, + ], +}; + +// Node vulnerability report configuration. +export const searchFilterConfigForCluster = [searchFilterEntityForCluster]; + +// Virtual machine vulnerability report configuration. +export const searchFilterConfigForClusterNamespace = [ + searchFilterEntityForCluster, + searchFilterEntityForNamespace, +]; + +// Image vulnerability and violation report configuration. +export const searchFilterConfigForClusterNamespaceDeployment = [ + searchFilterEntityForCluster, + searchFilterEntityForDeployment, + searchFilterEntityForNamespace, +]; diff --git a/ui/apps/platform/src/Components/EntityScope/utils.test.ts b/ui/apps/platform/src/Components/EntityScope/utils.test.ts new file mode 100644 index 0000000000000..55a771ca6b0c5 --- /dev/null +++ b/ui/apps/platform/src/Components/EntityScope/utils.test.ts @@ -0,0 +1,469 @@ +import type { EntityScopeRule } from 'services/ReportsService.types'; +import { + collapseMapRuleValues, + getEntityScopeRulesFromSearchFilterForClusterNamespaceDeployment, + getSearchFilterFromEntityScopeRules, + getSearchFilterWithoutEntityScope, + ruleValueToSearchValue, + searchFieldLabelMapForClusterNamespaceDeployment, + searchValueToMapRuleValues, + searchValueToRuleValue, +} from './utils'; + +describe('EntityScope utils', () => { + describe('searchValueToRuleValue', () => { + it('maps a quoted string to EXACT match with unquoted value', () => { + expect(searchValueToRuleValue('"production"')).toEqual({ + matchType: 'EXACT', + value: 'production', + }); + }); + + it('maps an unquoted string to REGEX match with raw value', () => { + expect(searchValueToRuleValue('prod.*')).toEqual({ + matchType: 'REGEX', + value: 'prod.*', + }); + }); + + it('does not treat a single-quote-delimited string as exact', () => { + expect(searchValueToRuleValue("'production'")).toEqual({ + matchType: 'REGEX', + value: "'production'", + }); + }); + + it('handles quoted string with internal quotes', () => { + expect(searchValueToRuleValue('"my"cluster"')).toEqual({ + matchType: 'EXACT', + value: 'my"cluster', + }); + }); + + it('handles empty quoted string', () => { + expect(searchValueToRuleValue('""')).toEqual({ + matchType: 'EXACT', + value: '', + }); + }); + + it('handles key=value label format as REGEX', () => { + expect(searchValueToRuleValue('app=frontend')).toEqual({ + matchType: 'REGEX', + value: 'app=frontend', + }); + }); + + it('handles quoted key=value label format as EXACT', () => { + expect(searchValueToRuleValue('"app=frontend"')).toEqual({ + matchType: 'EXACT', + value: 'app=frontend', + }); + }); + }); + + describe('ruleValueToSearchValue', () => { + it('wraps EXACT match value in quotes for SearchFilter convention', () => { + expect(ruleValueToSearchValue({ matchType: 'EXACT', value: 'production' })).toBe( + '"production"' + ); + }); + + it('returns REGEX match value as-is without r/ prefix', () => { + expect(ruleValueToSearchValue({ matchType: 'REGEX', value: 'prod.*' })).toBe('prod.*'); + }); + + it('does not escape internal quotes in EXACT match values', () => { + expect(ruleValueToSearchValue({ matchType: 'EXACT', value: 'my"cluster' })).toBe( + '"my"cluster"' + ); + }); + + it('returns empty string for REGEX with empty value', () => { + expect(ruleValueToSearchValue({ matchType: 'REGEX', value: '' })).toBe(''); + }); + + it('handles EXACT with key=value label format', () => { + expect(ruleValueToSearchValue({ matchType: 'EXACT', value: 'app=frontend' })).toBe( + '"app=frontend"' + ); + }); + + it('handles REGEX with key=value label format', () => { + expect(ruleValueToSearchValue({ matchType: 'REGEX', value: 'app=front.*' })).toBe( + 'app=front.*' + ); + }); + }); + + describe('getSearchFilterFromEntityScopeRules', () => { + const map = searchFieldLabelMapForClusterNamespaceDeployment; + + it('converts EXACT rule to quoted SearchFilter value', () => { + const result = getSearchFilterFromEntityScopeRules( + [ + { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_NAME', + values: [{ matchType: 'EXACT', value: 'production' }], + }, + ], + map + ); + expect(result).toEqual({ Cluster: ['"production"'] }); + }); + + it('converts REGEX rule to unquoted SearchFilter value', () => { + const result = getSearchFilterFromEntityScopeRules( + [ + { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_NAME', + values: [{ matchType: 'REGEX', value: 'prod.*' }], + }, + ], + map + ); + expect(result).toEqual({ Cluster: ['prod.*'] }); + }); + + it('merges multiple values into a single SearchFilter key', () => { + const result = getSearchFilterFromEntityScopeRules( + [ + { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_NAME', + values: [ + { matchType: 'EXACT', value: 'production' }, + { matchType: 'REGEX', value: 'staging.*' }, + ], + }, + ], + map + ); + expect(result).toEqual({ Cluster: ['"production"', 'staging.*'] }); + }); + + it('converts rules across all entity types', () => { + const result = getSearchFilterFromEntityScopeRules( + [ + { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_NAME', + values: [{ matchType: 'EXACT', value: 'prod' }], + }, + { + entity: 'SCOPE_ENTITY_NAMESPACE', + field: 'FIELD_NAME', + values: [{ matchType: 'REGEX', value: 'default.*' }], + }, + { + entity: 'SCOPE_ENTITY_DEPLOYMENT', + field: 'FIELD_LABEL', + values: [{ matchType: 'EXACT', value: 'app=web' }], + }, + ], + map + ); + expect(result).toEqual({ + Cluster: ['"prod"'], + Namespace: ['default.*'], + 'Deployment Label': ['"app=web"'], + }); + }); + + it('accumulates values from multiple rules for the same field', () => { + const result = getSearchFilterFromEntityScopeRules( + [ + { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_NAME', + values: [{ matchType: 'EXACT', value: 'prod' }], + }, + { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_NAME', + values: [{ matchType: 'REGEX', value: 'staging.*' }], + }, + ], + map + ); + expect(result).toEqual({ Cluster: ['"prod"', 'staging.*'] }); + }); + + it('ignores rules for entities not in the map', () => { + const result = getSearchFilterFromEntityScopeRules( + [ + { + entity: 'SCOPE_ENTITY_NAMESPACE', + field: 'FIELD_NAME', + values: [{ matchType: 'REGEX', value: 'default' }], + }, + ], + { Cluster: { entity: 'SCOPE_ENTITY_CLUSTER', field: 'FIELD_NAME' } } + ); + expect(result).toEqual({}); + }); + + it('returns empty object for empty rules', () => { + expect(getSearchFilterFromEntityScopeRules([], map)).toEqual({}); + }); + }); + + describe('getSearchFilterWithoutEntityScope', () => { + it('removes all entity scope fields from SearchFilter', () => { + const result = getSearchFilterWithoutEntityScope({ + Cluster: '"production"', + Namespace: 'default', + Deployment: 'nginx', + Severity: 'Critical', + CVE: 'CVE-2024-1234', + }); + expect(result).toEqual({ + Severity: 'Critical', + CVE: 'CVE-2024-1234', + }); + }); + + it('removes label, annotation, and ID fields', () => { + const result = getSearchFilterWithoutEntityScope({ + 'Cluster Label': 'env=prod', + 'Namespace Annotation': 'owner=sre', + 'Deployment Label': 'app=web', + 'Cluster ID': 'abc', + 'Namespace ID': 'def', + 'Deployment ID': 'ghi', + Fixable: 'true', + }); + expect(result).toEqual({ Fixable: 'true' }); + }); + + it('returns empty object when all fields are entity scope', () => { + const result = getSearchFilterWithoutEntityScope({ + Cluster: '"production"', + Namespace: 'default', + }); + expect(result).toEqual({}); + }); + + it('returns unchanged object when no entity scope fields present', () => { + const input = { Severity: 'Critical', Fixable: 'true' }; + const result = getSearchFilterWithoutEntityScope(input); + expect(result).toEqual(input); + }); + + it('does not mutate the input', () => { + const input = { Cluster: '"production"', Severity: 'Critical' }; + const inputCopy = { ...input }; + getSearchFilterWithoutEntityScope(input); + expect(input).toEqual(inputCopy); + }); + + it('is case-insensitive for field labels', () => { + const result = getSearchFilterWithoutEntityScope({ + cluster: '"production"', + namespace: 'default', + Severity: 'Critical', + }); + expect(result).toEqual({ Severity: 'Critical' }); + }); + }); + + describe('SearchFilter and EntityScopeRule round-trip', () => { + it('round-trips a mixed EXACT/REGEX SearchFilter through rules and back', () => { + const original = { + Cluster: ['"production"', 'staging.*'], + 'Deployment Label': ['"app=web"'], + Deployment: ['nginx.*'], + }; + + const rules = + getEntityScopeRulesFromSearchFilterForClusterNamespaceDeployment(original); + const result = getSearchFilterFromEntityScopeRules( + rules, + searchFieldLabelMapForClusterNamespaceDeployment + ); + expect(result).toEqual(original); + }); + + it('round-trips entity scope rules through SearchFilter and back', () => { + const originalRules: EntityScopeRule[] = [ + { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_NAME', + values: [ + { matchType: 'EXACT', value: 'production' }, + { matchType: 'REGEX', value: 'staging.*' }, + ], + }, + { + entity: 'SCOPE_ENTITY_DEPLOYMENT', + field: 'FIELD_LABEL', + values: [{ matchType: 'EXACT', value: 'app=web' }], + }, + ]; + const searchFilter = getSearchFilterFromEntityScopeRules( + originalRules, + searchFieldLabelMapForClusterNamespaceDeployment + ); + const result = + getEntityScopeRulesFromSearchFilterForClusterNamespaceDeployment(searchFilter); + expect(result).toEqual(originalRules); + }); + }); + + describe('searchValueToMapRuleValues', () => { + it('expands equal-less regex to two rule values', () => { + expect(searchValueToMapRuleValues('visa')).toEqual([ + { matchType: 'REGEX', value: 'visa=.*' }, + { matchType: 'REGEX', value: '.*=visa' }, + ]); + }); + + it('does not expand key=value regex', () => { + expect(searchValueToMapRuleValues('app=reporting')).toEqual([ + { matchType: 'REGEX', value: 'app=reporting' }, + ]); + }); + + it('does not expand key=value exact', () => { + expect(searchValueToMapRuleValues('"app=reporting"')).toEqual([ + { matchType: 'EXACT', value: 'app=reporting' }, + ]); + }); + }); + + describe('collapseMapRuleValues', () => { + it('collapses complementary regex pair to single value', () => { + expect( + collapseMapRuleValues([ + { matchType: 'REGEX', value: 'visa=.*' }, + { matchType: 'REGEX', value: '.*=visa' }, + ]) + ).toEqual(['visa']); + }); + + it('passes through key=value without collapsing', () => { + expect(collapseMapRuleValues([{ matchType: 'REGEX', value: 'app=reporting' }])).toEqual( + ['app=reporting'] + ); + }); + + it('does not collapse when complement is missing', () => { + expect(collapseMapRuleValues([{ matchType: 'REGEX', value: 'visa=.*' }])).toEqual([ + 'visa=.*', + ]); + }); + }); + + describe('searchValueToRuleValue and ruleValueToSearchValue round-trip', () => { + it('round-trips an EXACT value', () => { + const original = '"production"'; + const ruleValue = searchValueToRuleValue(original); + const result = ruleValueToSearchValue(ruleValue); + expect(result).toBe(original); + }); + + it('round-trips a REGEX value', () => { + const original = 'prod.*'; + const ruleValue = searchValueToRuleValue(original); + const result = ruleValueToSearchValue(ruleValue); + expect(result).toBe(original); + }); + + it('round-trips a key=value label', () => { + const original = 'app=frontend'; + const ruleValue = searchValueToRuleValue(original); + const result = ruleValueToSearchValue(ruleValue); + expect(result).toBe(original); + }); + + it('round-trips a quoted key=value label', () => { + const original = '"app=frontend"'; + const ruleValue = searchValueToRuleValue(original); + const result = ruleValueToSearchValue(ruleValue); + expect(result).toBe(original); + }); + }); + + describe('equal-less label round-trip through full conversion', () => { + it('round-trips equal-less regex label through SearchFilter → rules → SearchFilter', () => { + const original = { 'Deployment Label': ['visa'] }; + const rules = + getEntityScopeRulesFromSearchFilterForClusterNamespaceDeployment(original); + const result = getSearchFilterFromEntityScopeRules( + rules, + searchFieldLabelMapForClusterNamespaceDeployment + ); + expect(result).toEqual(original); + }); + + it('round-trips mixed equal-less and key=value labels', () => { + const original = { + 'Deployment Label': ['visa', '"app=web"', 'env=prod'], + Cluster: ['"production"'], + }; + const rules = + getEntityScopeRulesFromSearchFilterForClusterNamespaceDeployment(original); + const result = getSearchFilterFromEntityScopeRules( + rules, + searchFieldLabelMapForClusterNamespaceDeployment + ); + expect(result).toEqual(original); + }); + + it('does not expand equal-less values for non-map fields', () => { + const original = { Cluster: ['production'] }; + const rules = + getEntityScopeRulesFromSearchFilterForClusterNamespaceDeployment(original); + expect(rules[0].values).toEqual([{ matchType: 'REGEX', value: 'production' }]); + }); + + it('round-trips equal-less regex label through rules → SearchFilter → rules', () => { + const originalRules: EntityScopeRule[] = [ + { + entity: 'SCOPE_ENTITY_DEPLOYMENT', + field: 'FIELD_LABEL', + values: [ + { matchType: 'REGEX', value: 'visa=.*' }, + { matchType: 'REGEX', value: '.*=visa' }, + ], + }, + ]; + const searchFilter = getSearchFilterFromEntityScopeRules( + originalRules, + searchFieldLabelMapForClusterNamespaceDeployment + ); + const result = + getEntityScopeRulesFromSearchFilterForClusterNamespaceDeployment(searchFilter); + expect(result).toEqual(originalRules); + }); + + it('round-trips mixed equal-less and key=value labels through rules → SearchFilter → rules', () => { + const originalRules: EntityScopeRule[] = [ + { + entity: 'SCOPE_ENTITY_DEPLOYMENT', + field: 'FIELD_LABEL', + values: [ + { matchType: 'REGEX', value: 'visa=.*' }, + { matchType: 'REGEX', value: '.*=visa' }, + { matchType: 'EXACT', value: 'app=web' }, + { matchType: 'REGEX', value: 'env=prod' }, + ], + }, + { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_NAME', + values: [{ matchType: 'EXACT', value: 'production' }], + }, + ]; + const searchFilter = getSearchFilterFromEntityScopeRules( + originalRules, + searchFieldLabelMapForClusterNamespaceDeployment + ); + const result = + getEntityScopeRulesFromSearchFilterForClusterNamespaceDeployment(searchFilter); + expect(result).toEqual(originalRules); + }); + }); +}); diff --git a/ui/apps/platform/src/Components/EntityScope/utils.ts b/ui/apps/platform/src/Components/EntityScope/utils.ts new file mode 100644 index 0000000000000..44f5c5b6d3c73 --- /dev/null +++ b/ui/apps/platform/src/Components/EntityScope/utils.ts @@ -0,0 +1,300 @@ +import cloneDeep from 'lodash/cloneDeep'; + +import type { + EntityScopeRule, + RuleValue, + ScopeEntity, + ScopeField, +} from 'services/ReportsService.types'; +import type { SearchFilter } from 'types/search'; +import type { SearchFieldLabel } from 'types/searchOptions'; +import { + getValueByCaseInsensitiveKey, + isQuotedString, + searchValueAsArray, + wrapInQuotes, +} from 'utils/searchUtils'; + +type EntityScopeSearchFieldLabelForCluster = Extract< + SearchFieldLabel, + 'Cluster ID' | 'Cluster' | 'Cluster Label' +>; + +type EntityScopeSearchFieldLabelForClusterNamespace = Extract< + SearchFieldLabel, + | 'Cluster ID' + | 'Cluster' + | 'Cluster Label' + | 'Namespace ID' + | 'Namespace' + | 'Namespace Label' + | 'Namespace Annotation' +>; + +type EntityScopeSearchFieldLabelForClusterNamespaceDeployment = Extract< + SearchFieldLabel, + | 'Cluster ID' + | 'Cluster' + | 'Cluster Label' + | 'Namespace ID' + | 'Namespace' + | 'Namespace Label' + | 'Namespace Annotation' + | 'Deployment ID' + | 'Deployment' + | 'Deployment Label' + | 'Deployment Annotation' +>; + +type EntityScopeRuleWithoutValues = { + entity: Exclude; + field: Exclude; +}; + +const searchFieldLabelMapForCluster: Record< + EntityScopeSearchFieldLabelForCluster, + EntityScopeRuleWithoutValues +> = { + 'Cluster ID': { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_ID', + }, + Cluster: { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_NAME', + }, + 'Cluster Label': { + entity: 'SCOPE_ENTITY_CLUSTER', + field: 'FIELD_LABEL', + }, + // 'Cluster Annotation' is not a search filter +} as const; + +const searchFieldLabelMapForClusterNamespace: Record< + EntityScopeSearchFieldLabelForClusterNamespace, + EntityScopeRuleWithoutValues +> = { + ...searchFieldLabelMapForCluster, + 'Namespace ID': { + entity: 'SCOPE_ENTITY_NAMESPACE', + field: 'FIELD_ID', + }, + Namespace: { + entity: 'SCOPE_ENTITY_NAMESPACE', + field: 'FIELD_NAME', + }, + 'Namespace Label': { + entity: 'SCOPE_ENTITY_NAMESPACE', + field: 'FIELD_LABEL', + }, + 'Namespace Annotation': { + entity: 'SCOPE_ENTITY_NAMESPACE', + field: 'FIELD_ANNOTATION', + }, +} as const; + +export const searchFieldLabelMapForClusterNamespaceDeployment: Record< + EntityScopeSearchFieldLabelForClusterNamespaceDeployment, + EntityScopeRuleWithoutValues +> = { + ...searchFieldLabelMapForClusterNamespace, + 'Deployment ID': { + entity: 'SCOPE_ENTITY_DEPLOYMENT', + field: 'FIELD_ID', + }, + Deployment: { + entity: 'SCOPE_ENTITY_DEPLOYMENT', + field: 'FIELD_NAME', + }, + 'Deployment Label': { + entity: 'SCOPE_ENTITY_DEPLOYMENT', + field: 'FIELD_LABEL', + }, + 'Deployment Annotation': { + entity: 'SCOPE_ENTITY_DEPLOYMENT', + field: 'FIELD_ANNOTATION', + }, +} as const; + +// One size omits all of cluster, namespace, deployment, for simplicity. +// For initial query string when ?action=createFromFilters +export function getSearchFilterWithoutEntityScope( + searchFilterWithEntityScopeRules: SearchFilter +): SearchFilter { + const searchFilterWithoutEntityScopeRules: SearchFilter = cloneDeep( + searchFilterWithEntityScopeRules + ); + + Object.entries(searchFilterWithEntityScopeRules).forEach(([searchFieldLabel]) => { + if ( + getValueByCaseInsensitiveKey( + searchFieldLabelMapForClusterNamespaceDeployment, + searchFieldLabel + ) + ) { + delete searchFilterWithoutEntityScopeRules[searchFieldLabel]; + } + }); + + return searchFilterWithoutEntityScopeRules; +} + +function isLabelOrAnnotationField(field: string): boolean { + return field === 'FIELD_LABEL' || field === 'FIELD_ANNOTATION'; +} + +export const searchValueToRuleValue = (value: string): RuleValue => + isQuotedString(value) + ? { matchType: 'EXACT', value: value.slice(1, -1) } + : { matchType: 'REGEX', value }; + +// For label/annotation fields, equal-less values expand to two rule values +// so the backend queries both the key and value sides of the map field. +export function searchValueToMapRuleValues(value: string): RuleValue[] { + const raw = isQuotedString(value) ? value.slice(1, -1) : value; + const matchType = isQuotedString(value) ? 'EXACT' : 'REGEX'; + + if (raw.includes('=')) { + return [{ matchType, value: raw }]; + } + + return [ + { matchType: 'REGEX', value: `${raw}=.*` }, + { matchType: 'REGEX', value: `.*=${raw}` }, + ]; +} + +/** + * Return initial entity scope rules for corresponding search fields + * when user creates scheduled report configuration from results page. + */ +function getEntityScopeRulesFromSearchFilter( + searchFilter: SearchFilter, + searchFieldLabelMap: Record +) { + const rules: EntityScopeRule[] = []; + + Object.entries(searchFilter).forEach(([searchFieldLabel, searchFieldValue]) => { + const ruleWithoutValues = getValueByCaseInsensitiveKey( + searchFieldLabelMap, + searchFieldLabel + ); + const searchFieldValues = searchValueAsArray(searchFieldValue); + + if (ruleWithoutValues && searchFieldValues.length !== 0) { + rules.push({ + ...ruleWithoutValues, + values: searchFieldValues.flatMap((v) => + isLabelOrAnnotationField(ruleWithoutValues.field) + ? searchValueToMapRuleValues(v) + : [searchValueToRuleValue(v)] + ), + }); + } + }); + + return rules; +} + +export const ruleValueToSearchValue = ({ matchType, value }: RuleValue): string => + matchType === 'EXACT' ? wrapInQuotes(value) : value; + +// Collapses complementary equal-less pairs ([`value=.*`, `.*=value`]) back to a single +// search value. Inverse of searchValueToMapRuleValues. +export function collapseMapRuleValues(ruleValues: RuleValue[]): string[] { + const pairKeys = new Set(ruleValues.map(({ matchType, value }) => `${matchType}:${value}`)); + + return ruleValues.flatMap(({ matchType, value }) => { + if (value.startsWith('.*=')) { + if (pairKeys.has(`${matchType}:${value.slice(3)}=.*`)) { + return []; + } + } + + if (value.endsWith('=.*') && !value.startsWith('.*=')) { + const raw = value.slice(0, -3); + if (pairKeys.has(`${matchType}:.*=${raw}`)) { + return [raw]; + } + } + + return [ruleValueToSearchValue({ matchType, value })]; + }); +} + +/** + * Return search filter in EntityScopeCompoundSearchFilter component. + */ +export function getSearchFilterFromEntityScopeRules( + rules: EntityScopeRule[], + searchFieldLabelMap: Record +) { + const searchFilter: SearchFilter = {}; + + rules.forEach((rule) => { + const found = Object.entries(searchFieldLabelMap).find( + ([, { entity, field }]) => entity === rule.entity && field === rule.field + ); + + if (found) { + const [searchFieldLabel] = found; + const searchFilterValue = getValueByCaseInsensitiveKey(searchFilter, searchFieldLabel); + const searchFilterValues = searchValueAsArray(searchFilterValue); + const ruleValues = isLabelOrAnnotationField(rule.field) + ? collapseMapRuleValues(rule.values) + : rule.values.map(ruleValueToSearchValue); + searchFilter[searchFieldLabel] = [...searchFilterValues, ...ruleValues]; + } + }); + + return searchFilter; +} + +/** + * Return initial entity scope rules for corresponding search fields + * when user creates node vulnerability report configuration from results page. + */ +export function getEntityScopeRulesFromSearchFilterForCluster(searchFilter: SearchFilter) { + return getEntityScopeRulesFromSearchFilter(searchFilter, searchFieldLabelMapForCluster); +} + +export function getSearchFilterFromEntityScopeRulesForCluster(rules: EntityScopeRule[]) { + return getSearchFilterFromEntityScopeRules(rules, searchFieldLabelMapForCluster); +} + +/** + * Return initial entity scope rules for corresponding search fields + * when user creates either virtual machine vulnerability report configuration from results page. + */ +export function getEntityScopeRulesFromSearchFilterForClusterNamespace(searchFilter: SearchFilter) { + return getEntityScopeRulesFromSearchFilter( + searchFilter, + searchFieldLabelMapForClusterNamespace + ); +} + +export function getSearchFilterFromEntityScopeRulesForClusterNamespace(rules: EntityScopeRule[]) { + return getSearchFilterFromEntityScopeRules(rules, searchFieldLabelMapForClusterNamespace); +} + +/** + * Return initial entity scope rules for corresponding search fields + * when user creates either violation or image vulnerability report configuration from results page. + */ +export function getEntityScopeRulesFromSearchFilterForClusterNamespaceDeployment( + searchFilter: SearchFilter +) { + return getEntityScopeRulesFromSearchFilter( + searchFilter, + searchFieldLabelMapForClusterNamespaceDeployment + ); +} + +export function getSearchFilterFromEntityScopeRulesForClusterNamespaceDeployment( + rules: EntityScopeRule[] +) { + return getSearchFilterFromEntityScopeRules( + rules, + searchFieldLabelMapForClusterNamespaceDeployment + ); +} diff --git a/ui/apps/platform/src/Components/ExpandRowTh.tsx b/ui/apps/platform/src/Components/ExpandRowTh.tsx index 16fc7ae5f2378..e991c25df2f5e 100644 --- a/ui/apps/platform/src/Components/ExpandRowTh.tsx +++ b/ui/apps/platform/src/Components/ExpandRowTh.tsx @@ -2,20 +2,19 @@ import { Th } from '@patternfly/react-table'; import type { ThProps } from '@patternfly/react-table'; const expandButtonWidth = '1em'; -const expandButtonPaddingX = '(var(--pf-v5-global--spacer--md) * 2)'; -const firstTableCellPadding = 'var(--pf-v5-c-table--cell--PaddingLeft)'; +const expandButtonPaddingX = 'var(--pf-t--global--spacer--md)'; +const firstTableCellPadding = 'var(--pf-v6-c-table--cell--PaddingBlockStart)'; export default function ExpandRowTh(props: ThProps) { return ( - Row expansion - + /> ); } diff --git a/ui/apps/platform/src/Components/ExportButton.jsx b/ui/apps/platform/src/Components/ExportButton.jsx deleted file mode 100644 index 1d49f4560f71d..0000000000000 --- a/ui/apps/platform/src/Components/ExportButton.jsx +++ /dev/null @@ -1,202 +0,0 @@ -import { Component } from 'react'; -import PropTypes from 'prop-types'; -import { FileText } from 'react-feather'; -import onClickOutside from 'react-onclickoutside'; -import { toast } from 'react-toastify'; -import { Button } from '@patternfly/react-core'; - -import downloadCSV from 'services/CSVDownloadService'; -import WorkflowPDFExportButton from 'Components/WorkflowPDFExportButton'; -import ButtonClassic from 'Components/Button'; -import useCaseTypes from 'constants/useCaseTypes'; -import entityTypes from 'constants/entityTypes'; -import { addBrandedTimestampToString } from 'utils/dateUtils'; - -const queryParamMap = { - [entityTypes.CLUSTER]: 'clusterId', - [entityTypes.STANDARD]: 'standardId', - ALL: '', -}; - -const complianceDownloadUrl = '/api/compliance/export/csv'; - -function checkVulnMgmtSupport(page, type) { - return ( - page === useCaseTypes.VULN_MANAGEMENT && - (type === entityTypes.CVE || - type === entityTypes.IMAGE_CVE || - type === entityTypes.NODE_CVE || - type === entityTypes.CLUSTER_CVE) - ); -} - -function checkComplianceSupport(page, type) { - return page === useCaseTypes.COMPLIANCE && Object.keys(queryParamMap).includes(type); -} -class ExportButton extends Component { - static propTypes = { - className: PropTypes.string, - textClass: PropTypes.string, - fileName: PropTypes.string, - type: PropTypes.string, - id: PropTypes.string, - pdfId: PropTypes.string, - tableOptions: PropTypes.shape({}), - customCsvExportHandler: PropTypes.func, - page: PropTypes.string, - disabled: PropTypes.bool, - isExporting: PropTypes.bool.isRequired, - setIsExporting: PropTypes.func.isRequired, - }; - - static defaultProps = { - className: 'btn btn-base h-10', - textClass: null, - fileName: 'compliance', - type: null, - id: '', - pdfId: '', - tableOptions: {}, - customCsvExportHandler: null, - page: '', - disabled: false, - }; - - constructor(props) { - super(props); - - this.state = { - toggleWidget: false, - csvIsDownloading: false, - }; - } - - handleClickOutside = () => this.setState({ toggleWidget: false }); - - downloadCSVFile = () => { - const { id, fileName, page, type, customCsvExportHandler } = this.props; - const csvName = addBrandedTimestampToString(fileName); - - if (checkVulnMgmtSupport(page, type)) { - if (!customCsvExportHandler) { - throw new Error('A CSV export handler was not supplied to handle CSV export'); - } - - this.setState({ csvIsDownloading: true }); - customCsvExportHandler(csvName) - .catch((err) => { - toast(`An error occurred while trying to export: ${err}`); - }) - .finally(() => { - this.setState({ toggleWidget: false, csvIsDownloading: false }); - }); - } else { - // otherwise, use legacy compliance CSV export - let queryStr = ''; - let value = null; - - // Support for StandardId & ClusterId only - if (queryParamMap[type]) { - if (id) { - value = id; - } - queryStr = `${queryParamMap[type]}=${value}`; - } - - downloadCSV(csvName, complianceDownloadUrl, queryStr); - } - }; - - isCsvSupported = () => { - const { page, type } = this.props; - - const isVulnMgmtSupportedPage = checkVulnMgmtSupport(page, type); - const isComplianceSupportedPage = checkComplianceSupport(page, type); - - return isVulnMgmtSupportedPage || isComplianceSupportedPage; - }; - - renderContent = () => { - const csvButtonText = - this.props.type === entityTypes.CVE - ? 'Download CVES as CSV' - : 'Download Evidence as CSV'; - const { toggleWidget, csvIsDownloading } = this.state; - if (!toggleWidget) { - return null; - } - - const headerText = this.props.fileName; - - const fileName = addBrandedTimestampToString(headerText); - - const wrapperClass = !!this.props.pdfId && !this.isCsvSupported() ? 'min-w-64' : ''; - - return !!this.props.pdfId || this.isCsvSupported() ? ( -
-
-
    -
  • -
    - {!!this.props.pdfId && ( - - )} - {this.isCsvSupported() && ( - - )} -
    -
    • -
  • - or share to -
    Slack
    -
    • -
-
- ) : null; - }; - - toggleWidget = () => { - this.setState(({ toggleWidget }) => ({ toggleWidget: !toggleWidget })); - }; - - render() { - return ( -
- } - onClick={this.toggleWidget} - /> - {this.renderContent()} -
- ); - } -} - -export default onClickOutside(ExportButton); diff --git a/ui/apps/platform/src/Components/FeatureDisabledMessage.tsx b/ui/apps/platform/src/Components/FeatureDisabledMessage.tsx new file mode 100644 index 0000000000000..eb1a5c3fe1744 --- /dev/null +++ b/ui/apps/platform/src/Components/FeatureDisabledMessage.tsx @@ -0,0 +1,32 @@ +import type { ReactElement } from 'react'; +import { Link } from 'react-router-dom-v5-compat'; +import { Bullseye, EmptyState, EmptyStateBody, EmptyStateFooter } from '@patternfly/react-core'; +import { BanIcon } from '@patternfly/react-icons'; + +export type FeatureDisabledMessageProps = { + title: string; + message?: string; + actionText?: string; + url?: string; +}; + +const FeatureDisabledMessage = ({ + title, + message, + actionText, + url, +}: FeatureDisabledMessageProps): ReactElement => { + const isLinkVisible = actionText && url; + return ( + + + + {message && {message}} + {isLinkVisible && {actionText}} + + + + ); +}; + +export default FeatureDisabledMessage; diff --git a/ui/apps/platform/src/Components/FilteredWorkflowViewSelector/types.ts b/ui/apps/platform/src/Components/FilteredWorkflowViewSelector/types.ts index 2dad779e13788..ba0e910f5577d 100644 --- a/ui/apps/platform/src/Components/FilteredWorkflowViewSelector/types.ts +++ b/ui/apps/platform/src/Components/FilteredWorkflowViewSelector/types.ts @@ -2,11 +2,13 @@ import type { HistoryAction } from 'hooks/useURLParameter'; export const userWorkloadWorkflowView = 'Applications view'; export const platformWorkflowView = 'Platform view'; +export const nodeWorkflowView = 'Node view'; export const fullWorkflowView = 'Full view'; export const filteredWorkflowViews = [ userWorkloadWorkflowView, platformWorkflowView, + nodeWorkflowView, fullWorkflowView, ] as const; diff --git a/ui/apps/platform/src/Components/HelpIconTh.tsx b/ui/apps/platform/src/Components/HelpIconTh.tsx index 8d81d649c5875..c9c8b42b7e228 100644 --- a/ui/apps/platform/src/Components/HelpIconTh.tsx +++ b/ui/apps/platform/src/Components/HelpIconTh.tsx @@ -22,13 +22,12 @@ function HelpIconTh({ children, sort, popoverContent }: HelpIconThProps) { + className="pf-v6-u-p-0" + /> diff --git a/ui/apps/platform/src/Components/IconWidget.jsx b/ui/apps/platform/src/Components/IconWidget.jsx deleted file mode 100644 index c81b78d283ad5..0000000000000 --- a/ui/apps/platform/src/Components/IconWidget.jsx +++ /dev/null @@ -1,59 +0,0 @@ -import { connect } from 'react-redux'; -import PropTypes from 'prop-types'; -import { Link } from 'react-router-dom-v5-compat'; -import Widget from 'Components/Widget'; -import Loader from 'Components/Loader'; - -const IconWidget = ({ title, icon, description, linkUrl, loading, textSizeClass }) => { - const contents = loading ? ( - - ) : ( - <> -
- {title} -
-
- {description} -
- - ); - - return ( - - {linkUrl && !loading ? ( - - {contents} - - ) : ( -
{contents}
- )} - - ); -}; - -IconWidget.propTypes = { - title: PropTypes.string.isRequired, - icon: PropTypes.node.isRequired, - description: PropTypes.string, - textSizeClass: PropTypes.string, - linkUrl: PropTypes.string, - loading: PropTypes.bool, -}; - -IconWidget.defaultProps = { - description: null, - linkUrl: null, - textSizeClass: 'text-2xl', - loading: false, -}; - -export default connect()(IconWidget); diff --git a/ui/apps/platform/src/Components/InfoWidget.jsx b/ui/apps/platform/src/Components/InfoWidget.jsx deleted file mode 100644 index b14be1fbe5976..0000000000000 --- a/ui/apps/platform/src/Components/InfoWidget.jsx +++ /dev/null @@ -1,51 +0,0 @@ -import { connect } from 'react-redux'; -import PropTypes from 'prop-types'; -import { Link } from 'react-router-dom-v5-compat'; -import Widget from 'Components/Widget'; -import Loader from 'Components/Loader'; - -const InfoWidget = ({ title, headline, description, linkUrl, loading }) => { - const contents = loading ? ( - - ) : ( -
-
{headline}
-
{description}
-
- ); - return ( - - {linkUrl && !loading ? ( - - {contents} - - ) : ( - contents - )} - - ); -}; - -InfoWidget.propTypes = { - title: PropTypes.string.isRequired, - headline: PropTypes.string, - description: PropTypes.string, - linkUrl: PropTypes.string, - loading: PropTypes.bool, -}; - -InfoWidget.defaultProps = { - description: null, - headline: null, - linkUrl: null, - loading: false, -}; - -export default connect()(InfoWidget); diff --git a/ui/apps/platform/src/Components/KeyValueListModal.tsx b/ui/apps/platform/src/Components/KeyValueListModal.tsx index 5aa468b16abde..63e2dff76c883 100644 --- a/ui/apps/platform/src/Components/KeyValueListModal.tsx +++ b/ui/apps/platform/src/Components/KeyValueListModal.tsx @@ -6,10 +6,10 @@ import { FlexItem, List, ListItem, - Modal, SearchInput, pluralize, } from '@patternfly/react-core'; +import { Modal } from '@patternfly/react-core/deprecated'; import useModal from 'hooks/useModal'; @@ -78,7 +78,7 @@ function KeyValueListModal({ type, keyValues }: KeyValueListModalProps) { {filteredKeyValues.length === 0 && No results} {filteredKeyValues.length !== 0 && ( - + {filteredKeyValues.map(({ key, value }) => { const labelText = `${key}: ${value}`; return {labelText}; diff --git a/ui/apps/platform/src/Components/NotFoundMessage/NotFoundMessage.tsx b/ui/apps/platform/src/Components/NotFoundMessage/NotFoundMessage.tsx index ddf344a3cf001..00c589c882677 100644 --- a/ui/apps/platform/src/Components/NotFoundMessage/NotFoundMessage.tsx +++ b/ui/apps/platform/src/Components/NotFoundMessage/NotFoundMessage.tsx @@ -6,7 +6,6 @@ import { EmptyState, EmptyStateBody, EmptyStateFooter, - EmptyStateHeader, } from '@patternfly/react-core'; export type NotFoundMessageProps = { @@ -27,9 +26,8 @@ const NotFoundMessage = ({ const isButtonVisible = actionText && onClick; const isLinkVisible = actionText && url; return ( - - - + + {message && {message}} {isButtonVisible && ( diff --git a/ui/apps/platform/src/Components/NotifierConfiguration/NotifierConfigurationForm.tsx b/ui/apps/platform/src/Components/NotifierConfiguration/NotifierConfigurationForm.tsx index c1f8f57afce18..3ad5a1548bede 100644 --- a/ui/apps/platform/src/Components/NotifierConfiguration/NotifierConfigurationForm.tsx +++ b/ui/apps/platform/src/Components/NotifierConfiguration/NotifierConfigurationForm.tsx @@ -1,12 +1,9 @@ import { useEffect, useState } from 'react'; import type { ReactElement } from 'react'; -import { Button, Card, CardBody, CardTitle, Flex, FlexItem, Tooltip } from '@patternfly/react-core'; -import { HelpIcon, PencilAltIcon, PlusCircleIcon, TrashIcon } from '@patternfly/react-icons'; -import type { FormikErrors } from 'formik'; +import { Button, Flex, FormSection, TextArea, TextInput } from '@patternfly/react-core'; +import { PlusCircleIcon, TrashIcon } from '@patternfly/react-icons'; +import type { FormikErrors, FormikTouched } from 'formik'; -import { isDefaultEmailTemplate } from 'Components/EmailTemplate/EmailTemplate.utils'; -import EmailTemplateModal from 'Components/EmailTemplate/EmailTemplateModal'; -import type { TemplatePreviewArgs } from 'Components/EmailTemplate/EmailTemplateModal'; import FormLabelGroup from 'Components/PatternFly/FormLabelGroup'; import useIndexKey from 'hooks/useIndexKey'; import { fetchNotifierIntegrations } from 'services/NotifierIntegrationsService'; @@ -27,8 +24,6 @@ function splitAndTrimMailingListsString(mailingListsString: string): string[] { } export type NotifierConfigurationFormProps = { - customBodyDefault: string; - customSubjectDefault: string; errors: FormikErrors; // Caller provides name of property in formik.values and PatternFly fieldId props. // For example: @@ -38,26 +33,22 @@ export type NotifierConfigurationFormProps = { hasWriteAccessForIntegration: boolean; notifierConfigurations: NotifierConfiguration[]; onDeleteLastNotifierConfiguration?: () => void; - renderTemplatePreview?: (args: TemplatePreviewArgs) => ReactElement; setFieldValue: (fieldId: string, value: unknown) => void; + touched?: FormikTouched; }; function NotifierConfigurationForm({ - customBodyDefault, - customSubjectDefault, errors, fieldIdPrefixForFormikAndPatternFly, hasWriteAccessForIntegration, notifierConfigurations, onDeleteLastNotifierConfiguration, - renderTemplatePreview, setFieldValue, + touched, }: NotifierConfigurationFormProps): ReactElement { const { keyFor } = useIndexKey(); const [notifiers, setNotifiers] = useState([]); const [isLoadingNotifiers, setIsLoadingNotifiers] = useState(false); - const [notifierConfigurationSelected, setNotifierConfigurationSelected] = - useState(null); useEffect(() => { setIsLoadingNotifiers(true); @@ -98,202 +89,135 @@ function NotifierConfigurationForm({ return ( <> -
    - {notifierConfigurations.map((notifierConfiguration, index) => { - const { emailConfig, notifierName } = notifierConfiguration; - const { customBody, customSubject, mailingLists, notifierId } = emailConfig; - const fieldId = `${fieldIdPrefixForFormikAndPatternFly}[${index}]`; - const isDefaultEmailTemplateApplied = isDefaultEmailTemplate({ - customBody, - customSubject, - }); - return ( -
  • - - - - - Delivery destination - - - - - - - - { - setFieldValue( - `${fieldId}.emailConfig.mailingLists`, - splitAndTrimMailingListsString(mailingListsString) - ); - updateMailingListsString(index, mailingListsString); - }} - setNotifier={(notifier: NotifierIntegrationBase) => { - setFieldValue(fieldId, { - emailConfig: { - ...emailConfig, - notifierId: notifier.id, - mailingLists: - mailingLists.length === 0 - ? splitAndTrimMailingListsString( - notifier.labelDefault - ) - : mailingLists, - }, - notifierName: notifier.name, - }); - updateMailingListsString(index, notifier.labelDefault); - }} - setNotifiers={setNotifiers} - /> -
    - - Default template applied. Edit to - customize. -
    - ) : ( -
    - Custom template applied. Edit to - customize. -
    - ) - } - > - - - } - fieldId={`${fieldId}.customSubject`} - errors={errors} - isRequired - > - - -
- - - - ); - })} -
  • - -
    • - - {notifierConfigurationSelected && ( - { - const index = notifierConfigurations.indexOf(notifierConfigurationSelected); - if (index >= 0) { - const { emailConfig } = notifierConfigurationSelected; - setFieldValue(`${fieldIdPrefixForFormikAndPatternFly}[${index}]`, { - ...notifierConfigurationSelected, - emailConfig: { - ...emailConfig, - customSubject, - customBody, - }, - }); - } - }} - onClose={() => { - setNotifierConfigurationSelected(null); + {notifierConfigurations.map((notifierConfiguration, index) => { + const { emailConfig, notifierName } = notifierConfiguration; + const { customBody, customSubject, mailingLists, notifierId } = emailConfig; + const fieldId = `${fieldIdPrefixForFormikAndPatternFly}[${index}]`; + + return ( + + + + + { + setFieldValue( + `${fieldId}.emailConfig.mailingLists`, + splitAndTrimMailingListsString(mailingListsString) + ); + updateMailingListsString(index, mailingListsString); + }} + setNotifier={(notifier: NotifierIntegrationBase) => { + setFieldValue(fieldId, { + emailConfig: { + ...emailConfig, + notifierId: notifier.id, + mailingLists: + mailingLists.length === 0 + ? splitAndTrimMailingListsString( + notifier.labelDefault + ) + : mailingLists, + }, + notifierName: notifier.name, + }); + updateMailingListsString(index, notifier.labelDefault); + }} + setNotifiers={setNotifiers} + /> + + { + setFieldValue(`${fieldId}.emailConfig.customSubject`, value); + }} + /> + + +