From ced11ae42a9995c466b0124d152216af7ef0fb96 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 8 Apr 2026 18:13:49 -0400 Subject: [PATCH 01/11] Bump incremental version Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- pyproject.toml | 2 +- socketsecurity/__init__.py | 2 +- uv.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6b37502..c912d72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.2.80" +version = "2.2.81" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 92eb029..d4a1870 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,3 +1,3 @@ __author__ = 'socket.dev' -__version__ = '2.2.80' +__version__ = '2.2.81' USER_AGENT = f'SocketPythonCLI/{__version__}' diff --git a/uv.lock b/uv.lock index 4c6607f..4d82501 100644 --- a/uv.lock +++ b/uv.lock @@ -1168,7 +1168,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.2.78" +version = "2.2.81" source = { editable = "." } dependencies = [ { name = "bs4" }, From 4020285647bcbae7fe2886c1338a2753fff6d6e7 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 8 Apr 2026 18:14:52 -0400 Subject: [PATCH 02/11] Fix gitlab security report schema validation errors Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- socketsecurity/core/messages.py | 45 +++++++++-- tests/unit/test_gitlab_format.py | 131 ++++++++++++++++++++++++++++++- 2 files changed, 168 insertions(+), 8 deletions(-) diff --git a/socketsecurity/core/messages.py b/socketsecurity/core/messages.py index db62a6b..1d0431a 100644 --- a/socketsecurity/core/messages.py +++ b/socketsecurity/core/messages.py @@ -3,7 +3,7 @@ import os import re import uuid -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from mdutils import MdUtils from prettytable import PrettyTable @@ -593,6 +593,20 @@ def create_security_comment_json(diff: Diff) -> dict: output["new_alerts"].append(json.loads(str(alert))) return output + @staticmethod + def _pkg_type_to_package_manager(pkg_type: str) -> str: + """Map Socket pkg_type to GitLab package_manager name for dependency_files.""" + mapping = { + "npm": "npm", + "pypi": "pip", + "go": "go", + "maven": "maven", + "gem": "bundler", + "nuget": "nuget", + "cargo": "cargo", + } + return mapping.get(pkg_type, pkg_type or "unknown") + @staticmethod def map_socket_severity_to_gitlab(severity: str) -> str: """ @@ -743,14 +757,16 @@ def create_security_comment_gitlab(diff: Diff) -> dict: } }, "type": "dependency_scanning", - "start_time": datetime.utcnow().isoformat() + "Z", - "end_time": datetime.utcnow().isoformat() + "Z", + "start_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S"), + "end_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S"), "status": "success" }, - "vulnerabilities": [] + "vulnerabilities": [], + "dependency_files": [] } - # Process each alert + dep_files_map: dict = {} + for alert in diff.new_alerts: vulnerability = { "id": Messages.generate_uuid_from_alert_gitlab(alert), @@ -764,12 +780,29 @@ def create_security_comment_gitlab(diff: Diff) -> dict: "location": Messages.extract_location_gitlab(alert) } - # Add solution if available if hasattr(alert, 'suggestion') and alert.suggestion: vulnerability["solution"] = alert.suggestion gitlab_report["vulnerabilities"].append(vulnerability) + file_path = vulnerability["location"]["file"] + if file_path != "unknown": + pkg_manager = Messages._pkg_type_to_package_manager( + alert.pkg_type if hasattr(alert, 'pkg_type') else "" + ) + if file_path not in dep_files_map: + dep_files_map[file_path] = { + "path": file_path, + "package_manager": pkg_manager, + "dependencies": [] + } + dep_files_map[file_path]["dependencies"].append({ + "package": {"name": alert.pkg_name}, + "version": alert.pkg_version + }) + + gitlab_report["dependency_files"] = list(dep_files_map.values()) + return gitlab_report @staticmethod diff --git a/tests/unit/test_gitlab_format.py b/tests/unit/test_gitlab_format.py index 1799dcf..9ed9bb1 100644 --- a/tests/unit/test_gitlab_format.py +++ b/tests/unit/test_gitlab_format.py @@ -1,3 +1,5 @@ +import re + import pytest from socketsecurity.core.messages import Messages from socketsecurity.core.classes import Diff, Issue @@ -6,8 +8,11 @@ class TestGitLabFormat: """Test suite for GitLab Security Dashboard format generation""" + # GitLab v15.0.0 schema: ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$ + GITLAB_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$") + def test_gitlab_report_structure(self): - """Test basic GitLab report structure is valid""" + """Test basic GitLab report structure matches v15.0.0 schema requirements""" diff = Diff() diff.new_alerts = [] diff.id = "test-scan-id" @@ -15,10 +20,11 @@ def test_gitlab_report_structure(self): report = Messages.create_security_comment_gitlab(diff) - # Verify required top-level fields + # All four root-level keys required by v15.0.0 schema assert "version" in report assert "scan" in report assert "vulnerabilities" in report + assert "dependency_files" in report # Verify scan structure assert report["scan"]["type"] == "dependency_scanning" @@ -28,6 +34,12 @@ def test_gitlab_report_structure(self): assert report["scan"]["scanner"]["id"] == "socket-cli" assert report["scan"]["status"] == "success" + # Timestamps must match GitLab pattern (no microseconds, no trailing Z) + assert self.GITLAB_TIMESTAMP_RE.match(report["scan"]["start_time"]), \ + f"start_time '{report['scan']['start_time']}' doesn't match GitLab pattern" + assert self.GITLAB_TIMESTAMP_RE.match(report["scan"]["end_time"]), \ + f"end_time '{report['scan']['end_time']}' doesn't match GitLab pattern" + def test_vulnerability_mapping(self): """Test Socket Issue maps correctly to GitLab vulnerability""" diff = Diff() @@ -391,3 +403,118 @@ def test_manifests_attribute_fallback(self): vuln = report["vulnerabilities"][0] assert vuln["location"]["file"] == "requirements.txt" + + def test_dependency_files_populated_from_alerts(self): + """Test dependency_files is built from alert locations per v15.0.0 schema""" + diff = Diff() + diff.id = "test-scan-id" + diff.diff_url = "https://socket.dev/test" + + diff.new_alerts = [ + Issue( + pkg_name="pkg-a", + pkg_version="1.0.0", + type="malware", + severity="high", + title="Alert A", + manifests="package.json", + pkg_type="npm", + key="key-a", + purl="pkg:npm/pkg-a@1.0.0" + ), + Issue( + pkg_name="pkg-b", + pkg_version="2.0.0", + type="vulnerability", + severity="medium", + title="Alert B", + manifests="requirements.txt", + pkg_type="pypi", + key="key-b", + purl="pkg:pypi/pkg-b@2.0.0" + ), + ] + + report = Messages.create_security_comment_gitlab(diff) + + assert "dependency_files" in report + dep_files = report["dependency_files"] + assert len(dep_files) == 2 + + paths = {df["path"] for df in dep_files} + assert "package.json" in paths + assert "requirements.txt" in paths + + for df in dep_files: + assert "path" in df + assert "package_manager" in df + assert "dependencies" in df + assert isinstance(df["dependencies"], list) + assert len(df["dependencies"]) >= 1 + for dep in df["dependencies"]: + assert "package" in dep + assert "name" in dep["package"] + assert "version" in dep + + def test_dependency_files_groups_by_manifest(self): + """Test multiple alerts from the same manifest are grouped into one dependency_file""" + diff = Diff() + diff.id = "test-scan-id" + diff.diff_url = "https://socket.dev/test" + + diff.new_alerts = [ + Issue( + pkg_name="pkg-a", pkg_version="1.0.0", type="malware", severity="high", + title="A", manifests="package.json", pkg_type="npm", key="k1", purl="pkg:npm/pkg-a@1.0.0" + ), + Issue( + pkg_name="pkg-b", pkg_version="2.0.0", type="vulnerability", severity="low", + title="B", manifests="package.json", pkg_type="npm", key="k2", purl="pkg:npm/pkg-b@2.0.0" + ), + ] + + report = Messages.create_security_comment_gitlab(diff) + dep_files = report["dependency_files"] + + assert len(dep_files) == 1 + assert dep_files[0]["path"] == "package.json" + assert dep_files[0]["package_manager"] == "npm" + assert len(dep_files[0]["dependencies"]) == 2 + + def test_dependency_files_empty_when_no_alerts(self): + """Test dependency_files is an empty array when there are no alerts""" + diff = Diff() + diff.id = "test-scan-id" + diff.diff_url = "https://socket.dev/test" + diff.new_alerts = [] + + report = Messages.create_security_comment_gitlab(diff) + assert report["dependency_files"] == [] + + def test_dependency_files_skips_unknown_manifest(self): + """Test alerts with unknown manifest don't produce dependency_file entries""" + diff = Diff() + diff.id = "test-scan-id" + diff.diff_url = "https://socket.dev/test" + + diff.new_alerts = [ + Issue( + pkg_name="pkg-a", pkg_version="1.0.0", type="malware", severity="high", + title="A", pkg_type="npm", key="k1", purl="pkg:npm/pkg-a@1.0.0" + ), + ] + + report = Messages.create_security_comment_gitlab(diff) + assert report["dependency_files"] == [] + + def test_pkg_type_to_package_manager_mapping(self): + """Test package manager mapping covers common ecosystems""" + assert Messages._pkg_type_to_package_manager("npm") == "npm" + assert Messages._pkg_type_to_package_manager("pypi") == "pip" + assert Messages._pkg_type_to_package_manager("go") == "go" + assert Messages._pkg_type_to_package_manager("maven") == "maven" + assert Messages._pkg_type_to_package_manager("gem") == "bundler" + assert Messages._pkg_type_to_package_manager("nuget") == "nuget" + assert Messages._pkg_type_to_package_manager("cargo") == "cargo" + assert Messages._pkg_type_to_package_manager("") == "unknown" + assert Messages._pkg_type_to_package_manager("swift") == "swift" From 5430d3f3f8d948ce84e812ee0d2d02190decf7d8 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 8 Apr 2026 18:15:44 -0400 Subject: [PATCH 03/11] Populate gitlab security report with alerts for full scans Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- docs/cli-reference.md | 14 ++++++++--- socketsecurity/core/__init__.py | 43 +++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 0c807f7..a5c06f2 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -702,15 +702,22 @@ All alert types are included in the GitLab report if they're marked as `error` o ### Report Schema -Socket CLI generates reports compliant with [GitLab Dependency Scanning schema version 15.0.0](https://docs.gitlab.com/ee/development/integrations/secure.html). The reports include: +Socket CLI generates reports compliant with [GitLab Dependency Scanning schema version 15.0.0](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/v15.0.0/dist/dependency-scanning-report-format.json). The reports include: -- **Scan metadata**: Analyzer and scanner information +- **Scan metadata**: Analyzer and scanner information with ISO 8601 timestamps - **Vulnerabilities**: Detailed vulnerability data with: - Unique deterministic UUIDs for tracking - Package location and dependency information - Severity levels mapped from Socket's analysis - Socket-specific alert types and CVE identifiers - Links to Socket.dev for detailed analysis +- **Dependency files**: Manifest files and their dependencies discovered during the scan + +**Schema compatibility:** The v15.0.0 schema is supported across all GitLab versions 12.0+ (both self-hosted and cloud). The report includes the `dependency_files` field, which is required by v15.0.0 and accepted as an optional extra by newer schema versions, ensuring maximum compatibility across GitLab instances. + +### Performance Notes + +When `--enable-gitlab-security` (or `--enable-json` / `--enable-sarif`) is used with a full scan (non-diff mode), the CLI fetches package and alert data from the scan results to populate the report. This adds time proportional to the number of packages in the scan. Without these output flags, no additional data is fetched and scan performance is unchanged. ### Requirements @@ -726,7 +733,8 @@ Socket CLI generates reports compliant with [GitLab Dependency Scanning schema v - Ensure the report file follows the correct schema format **Empty vulnerabilities array:** -- This is normal if no new security issues were detected +- This is normal if no new security issues were detected in diff mode +- For full scans, ensure you are using `--enable-gitlab-security` so alert data is fetched - Check Socket.dev dashboard for full analysis details ## Development diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index edd2814..3580705 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -659,9 +659,48 @@ def create_full_scan_with_report_url( diff.report_url = f"{base_socket}/{self.config.org_slug}/sbom/{new_full_scan.id}" diff.diff_url = diff.report_url diff.id = new_full_scan.id - diff.packages = {} - # Return result in the format expected by the user + needs_alerts = ( + self.cli_config is not None + and ( + self.cli_config.enable_gitlab_security + or self.cli_config.enable_json + or self.cli_config.enable_sarif + ) + ) + + if needs_alerts: + log.info("Output format requires alerts, fetching SBOM data for full scan") + sbom_start = time.time() + sbom_artifacts_dict = self.get_sbom_data(new_full_scan.id) + sbom_artifacts = self.get_sbom_data_list(sbom_artifacts_dict) + packages = self.create_packages_dict(sbom_artifacts) + diff.packages = packages + + all_alerts_collection: Dict[str, List[Issue]] = {} + for package_id, package in packages.items(): + self.add_package_alerts_to_collection( + package=package, + alerts_collection=all_alerts_collection, + packages=packages + ) + + consolidated: Set[str] = set() + for alert_key, alerts in all_alerts_collection.items(): + for alert in alerts: + alert_str = f"{alert.purl},{alert.type}" + if (alert.error or alert.warn) and alert_str not in consolidated: + diff.new_alerts.append(alert) + consolidated.add(alert_str) + + sbom_end = time.time() + log.info( + f"Fetched {len(packages)} packages and {len(diff.new_alerts)} alerts " + f"in {sbom_end - sbom_start:.2f}s" + ) + else: + diff.packages = {} + return diff def get_full_scan(self, full_scan_id: str) -> FullScan: From abf416d4dc3be734cb8a421284af9c4a3a2e8a41 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 8 Apr 2026 18:45:48 -0400 Subject: [PATCH 04/11] Skip license-metadata API call when fetching full scan alerts Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- socketsecurity/core/__init__.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 3580705..daaa9a8 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -674,7 +674,7 @@ def create_full_scan_with_report_url( sbom_start = time.time() sbom_artifacts_dict = self.get_sbom_data(new_full_scan.id) sbom_artifacts = self.get_sbom_data_list(sbom_artifacts_dict) - packages = self.create_packages_dict(sbom_artifacts) + packages = self._create_packages_dict_without_license_text(sbom_artifacts) diff.packages = packages all_alerts_collection: Dict[str, List[Issue]] = {} @@ -751,6 +751,30 @@ def create_packages_dict(self, sbom_artifacts: list[SocketArtifact]) -> dict[str return packages + @staticmethod + def _create_packages_dict_without_license_text( + sbom_artifacts: list[SocketArtifact], + ) -> dict[str, Package]: + """Like create_packages_dict but skips the license-metadata API call. + + Used when we only need packages for alert extraction (e.g. populating + GitLab/JSON/SARIF reports from a full scan) and don't need license text. + """ + packages: dict[str, Package] = {} + top_level_count: dict[str, int] = {} + for artifact in sbom_artifacts: + package = Package.from_socket_artifact(asdict(artifact)) + if package.id not in packages: + packages[package.id] = package + if package.topLevelAncestors: + for top_id in package.topLevelAncestors: + top_level_count[top_id] = top_level_count.get(top_id, 0) + 1 + + for package_id, package in packages.items(): + package.transitives = top_level_count.get(package_id, 0) + + return packages + def get_package_license_text(self, package: Package) -> str: """ Gets the license text for a package if available. From 227e73bec33f774b94430d9b8148717fa6fb1581 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 8 Apr 2026 18:46:27 -0400 Subject: [PATCH 05/11] Consolidate e2e test workflows, add additional coverage Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- .github/workflows/e2e-test.yml | 216 +++++------------- tests/e2e/fixtures/simple-npm/package.json | 2 +- .../e2e/fixtures/simple-pypi/requirements.txt | 3 + tests/e2e/validate-gitlab.sh | 63 +++++ tests/e2e/validate-json.sh | 29 +++ tests/e2e/validate-reachability.sh | 65 ++++++ tests/e2e/validate-sarif.sh | 19 ++ tests/e2e/validate-scan.sh | 16 ++ 8 files changed, 253 insertions(+), 160 deletions(-) create mode 100644 tests/e2e/fixtures/simple-pypi/requirements.txt create mode 100755 tests/e2e/validate-gitlab.sh create mode 100755 tests/e2e/validate-json.sh create mode 100755 tests/e2e/validate-reachability.sh create mode 100755 tests/e2e/validate-sarif.sh create mode 100755 tests/e2e/validate-scan.sh diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 03584f6..95e93c9 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -1,4 +1,4 @@ -name: E2E Test +name: E2E Tests on: push: @@ -10,93 +10,58 @@ permissions: contents: read jobs: - e2e-scan: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 - with: - python-version: '3.12' - - - name: Install CLI from local repo - run: | - python -m pip install --upgrade pip - pip install . - - - name: Run Socket CLI scan - env: - SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }} - run: | - set -o pipefail - socketcli \ - --target-path tests/e2e/fixtures/simple-npm \ - --disable-blocking \ - --enable-debug \ - 2>&1 | tee /tmp/scan-output.log - - - name: Verify scan produced a report - run: | - if grep -q "Full scan report URL: https://socket.dev/" /tmp/scan-output.log; then - echo "PASS: Full scan report URL found" - grep "Full scan report URL:" /tmp/scan-output.log - elif grep -q "Diff Url: https://socket.dev/" /tmp/scan-output.log; then - echo "PASS: Diff URL found" - grep "Diff Url:" /tmp/scan-output.log - else - echo "FAIL: No report URL found in scan output" - cat /tmp/scan-output.log - exit 1 - fi - - e2e-sarif: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 - with: - python-version: '3.12' - - - name: Install CLI from local repo - run: | - python -m pip install --upgrade pip - pip install . - - - name: Run Socket CLI scan with --sarif-file - env: - SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }} - run: | - set -o pipefail - socketcli \ - --target-path tests/e2e/fixtures/simple-npm \ - --sarif-file /tmp/results.sarif \ - --disable-blocking \ - 2>&1 | tee /tmp/sarif-output.log - - - name: Verify SARIF file is valid - run: | - python3 -c " - import json, sys - with open('/tmp/results.sarif') as f: - data = json.load(f) - assert data['version'] == '2.1.0', f'Invalid version: {data[\"version\"]}' - assert '\$schema' in data, 'Missing \$schema' - count = len(data['runs'][0]['results']) - print(f'PASS: Valid SARIF 2.1.0 with {count} result(s)') - " - - e2e-reachability: + e2e: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: scan + args: >- + --target-path tests/e2e/fixtures/simple-npm + --disable-blocking + --enable-debug + validate: tests/e2e/validate-scan.sh + + - name: sarif + args: >- + --target-path tests/e2e/fixtures/simple-npm + --sarif-file /tmp/results.sarif + --disable-blocking + validate: tests/e2e/validate-sarif.sh + + - name: reachability + args: >- + --target-path tests/e2e/fixtures/simple-npm + --reach + --disable-blocking + --enable-debug + validate: tests/e2e/validate-reachability.sh + setup-node: "true" + + - name: gitlab + args: >- + --target-path tests/e2e/fixtures/simple-npm + --enable-gitlab-security + --disable-blocking + validate: tests/e2e/validate-gitlab.sh + + - name: json + args: >- + --target-path tests/e2e/fixtures/simple-npm + --enable-json + --disable-blocking + validate: tests/e2e/validate-json.sh + + - name: pypi + args: >- + --target-path tests/e2e/fixtures/simple-pypi + --disable-blocking + --enable-debug + validate: tests/e2e/validate-scan.sh + + name: e2e-${{ matrix.name }} steps: - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 with: @@ -108,6 +73,7 @@ jobs: python-version: '3.12' - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af + if: matrix.setup-node == 'true' with: node-version: '20' @@ -117,85 +83,17 @@ jobs: pip install . - name: Install uv + if: matrix.setup-node == 'true' run: pip install uv - - name: Run Socket CLI with reachability + - name: Run Socket CLI env: SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }} run: | set -o pipefail - socketcli \ - --target-path tests/e2e/fixtures/simple-npm \ - --reach \ - --disable-blocking \ - --enable-debug \ - 2>&1 | tee /tmp/reach-output.log - - - name: Verify reachability analysis completed - run: | - if grep -q "Reachability analysis completed successfully" /tmp/reach-output.log; then - echo "PASS: Reachability analysis completed" - grep "Reachability analysis completed successfully" /tmp/reach-output.log - grep "Results written to:" /tmp/reach-output.log || true - else - echo "FAIL: Reachability analysis did not complete successfully" - cat /tmp/reach-output.log - exit 1 - fi - - - name: Verify scan produced a report - run: | - if grep -q "Full scan report URL: https://socket.dev/" /tmp/reach-output.log; then - echo "PASS: Full scan report URL found" - grep "Full scan report URL:" /tmp/reach-output.log - elif grep -q "Diff Url: https://socket.dev/" /tmp/reach-output.log; then - echo "PASS: Diff URL found" - grep "Diff Url:" /tmp/reach-output.log - else - echo "FAIL: No report URL found in scan output" - cat /tmp/reach-output.log - exit 1 - fi + socketcli ${{ matrix.args }} 2>&1 | tee /tmp/e2e-output.log - - name: Run scan with --sarif-file (all results) + - name: Validate results env: SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }} - run: | - socketcli \ - --target-path tests/e2e/fixtures/simple-npm \ - --reach \ - --sarif-file /tmp/sarif-all.sarif \ - --sarif-scope full \ - --sarif-reachability all \ - --disable-blocking \ - 2>/dev/null - - - name: Run scan with --sarif-file --sarif-reachability reachable (filtered results) - env: - SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }} - run: | - socketcli \ - --target-path tests/e2e/fixtures/simple-npm \ - --reach \ - --sarif-file /tmp/sarif-reachable.sarif \ - --sarif-scope full \ - --sarif-reachability reachable \ - --disable-blocking \ - 2>/dev/null - - - name: Verify reachable-only results are a subset of all results - run: | - test -f /tmp/sarif-all.sarif - test -f /tmp/sarif-reachable.sarif - python3 -c " - import json - with open('/tmp/sarif-all.sarif') as f: - all_data = json.load(f) - with open('/tmp/sarif-reachable.sarif') as f: - reach_data = json.load(f) - all_count = len(all_data['runs'][0]['results']) - reach_count = len(reach_data['runs'][0]['results']) - print(f'All results: {all_count}, Reachable-only results: {reach_count}') - assert reach_count <= all_count, f'FAIL: reachable ({reach_count}) > all ({all_count})' - print('PASS: Reachable-only results is a subset of all results') - " + run: bash ${{ matrix.validate }} diff --git a/tests/e2e/fixtures/simple-npm/package.json b/tests/e2e/fixtures/simple-npm/package.json index cf70416..3dd5fa9 100644 --- a/tests/e2e/fixtures/simple-npm/package.json +++ b/tests/e2e/fixtures/simple-npm/package.json @@ -4,7 +4,7 @@ "description": "Test fixture for reachability analysis", "main": "index.js", "dependencies": { - "lodash": "4.17.23", + "lodash": "4.18.1", "express": "4.22.0", "axios": "1.13.5" }, diff --git a/tests/e2e/fixtures/simple-pypi/requirements.txt b/tests/e2e/fixtures/simple-pypi/requirements.txt new file mode 100644 index 0000000..acce997 --- /dev/null +++ b/tests/e2e/fixtures/simple-pypi/requirements.txt @@ -0,0 +1,3 @@ +requests==2.31.0 +flask==3.0.0 +pyyaml==6.0.1 diff --git a/tests/e2e/validate-gitlab.sh b/tests/e2e/validate-gitlab.sh new file mode 100755 index 0000000..259d07f --- /dev/null +++ b/tests/e2e/validate-gitlab.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPORT="gl-dependency-scanning-report.json" + +if [ ! -f "$REPORT" ]; then + echo "FAIL: GitLab report not found at $REPORT" + exit 1 +fi + +python3 -c " +import json, re, sys + +with open('$REPORT') as f: + data = json.load(f) + +errors = [] + +# v15.0.0 required root-level keys +for key in ('version', 'scan', 'vulnerabilities', 'dependency_files'): + if key not in data: + errors.append(f'Missing required root key: {key}') + +if 'scan' in data: + scan = data['scan'] + + # Timestamp format: YYYY-MM-DDTHH:MM:SS (no microseconds, no trailing Z) + ts_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$') + for field in ('start_time', 'end_time'): + val = scan.get(field, '') + if not ts_pattern.match(val): + errors.append(f'scan.{field} \"{val}\" does not match pattern YYYY-MM-DDTHH:MM:SS') + + if scan.get('type') != 'dependency_scanning': + errors.append(f'scan.type is \"{scan.get(\"type\")}\" expected \"dependency_scanning\"') + + analyzer_id = scan.get('analyzer', {}).get('id', '') + if analyzer_id != 'socket-security': + errors.append(f'scan.analyzer.id is \"{analyzer_id}\" expected \"socket-security\"') + + scanner_id = scan.get('scanner', {}).get('id', '') + if scanner_id != 'socket-cli': + errors.append(f'scan.scanner.id is \"{scanner_id}\" expected \"socket-cli\"') + + if scan.get('status') != 'success': + errors.append(f'scan.status is \"{scan.get(\"status\")}\" expected \"success\"') + +# dependency_files structure check +if 'dependency_files' in data: + for i, df in enumerate(data['dependency_files']): + for req in ('path', 'package_manager', 'dependencies'): + if req not in df: + errors.append(f'dependency_files[{i}] missing required key: {req}') + +if errors: + for e in errors: + print(f'FAIL: {e}') + sys.exit(1) + +vuln_count = len(data.get('vulnerabilities', [])) +dep_file_count = len(data.get('dependency_files', [])) +print(f'PASS: Valid GitLab v15.0.0 report with {vuln_count} vulnerability(ies) and {dep_file_count} dependency file(s)') +" diff --git a/tests/e2e/validate-json.sh b/tests/e2e/validate-json.sh new file mode 100755 index 0000000..6c78a16 --- /dev/null +++ b/tests/e2e/validate-json.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +LOG="/tmp/e2e-output.log" + +python3 -c " +import json, sys + +# The JSON output is on stdout; the log may also contain stderr debug lines. +# Find the first line that parses as valid JSON. +found = False +with open('$LOG') as f: + for line in f: + line = line.strip() + if not line: + continue + try: + data = json.loads(line) + if isinstance(data, dict): + found = True + print(f'PASS: Valid JSON output with {len(data)} top-level key(s)') + break + except json.JSONDecodeError: + continue + +if not found: + print('FAIL: No valid JSON object found in output') + sys.exit(1) +" diff --git a/tests/e2e/validate-reachability.sh b/tests/e2e/validate-reachability.sh new file mode 100755 index 0000000..e6e365e --- /dev/null +++ b/tests/e2e/validate-reachability.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +LOG="/tmp/e2e-output.log" + +# 1. Verify reachability analysis completed +if grep -q "Reachability analysis completed successfully" "$LOG"; then + echo "PASS: Reachability analysis completed" + grep "Reachability analysis completed successfully" "$LOG" + grep "Results written to:" "$LOG" || true +else + echo "FAIL: Reachability analysis did not complete successfully" + cat "$LOG" + exit 1 +fi + +# 2. Verify scan produced a report URL +if grep -q "Full scan report URL: https://socket.dev/" "$LOG"; then + echo "PASS: Full scan report URL found" + grep "Full scan report URL:" "$LOG" +elif grep -q "Diff Url: https://socket.dev/" "$LOG"; then + echo "PASS: Diff URL found" + grep "Diff Url:" "$LOG" +else + echo "FAIL: No report URL found in scan output" + cat "$LOG" + exit 1 +fi + +# 3. Run SARIF with --sarif-reachability all +socketcli \ + --target-path tests/e2e/fixtures/simple-npm \ + --reach \ + --sarif-file /tmp/sarif-all.sarif \ + --sarif-scope full \ + --sarif-reachability all \ + --disable-blocking \ + 2>/dev/null + +# 4. Run SARIF with --sarif-reachability reachable (filtered) +socketcli \ + --target-path tests/e2e/fixtures/simple-npm \ + --reach \ + --sarif-file /tmp/sarif-reachable.sarif \ + --sarif-scope full \ + --sarif-reachability reachable \ + --disable-blocking \ + 2>/dev/null + +# 5. Verify reachable-only results are a subset of all results +test -f /tmp/sarif-all.sarif +test -f /tmp/sarif-reachable.sarif + +python3 -c " +import json +with open('/tmp/sarif-all.sarif') as f: + all_data = json.load(f) +with open('/tmp/sarif-reachable.sarif') as f: + reach_data = json.load(f) +all_count = len(all_data['runs'][0]['results']) +reach_count = len(reach_data['runs'][0]['results']) +print(f'All results: {all_count}, Reachable-only results: {reach_count}') +assert reach_count <= all_count, f'FAIL: reachable ({reach_count}) > all ({all_count})' +print('PASS: Reachable-only results is a subset of all results') +" diff --git a/tests/e2e/validate-sarif.sh b/tests/e2e/validate-sarif.sh new file mode 100755 index 0000000..3846c0f --- /dev/null +++ b/tests/e2e/validate-sarif.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +SARIF="/tmp/results.sarif" + +if [ ! -f "$SARIF" ]; then + echo "FAIL: SARIF file not found at $SARIF" + exit 1 +fi + +python3 -c " +import json, sys +with open('$SARIF') as f: + data = json.load(f) +assert data['version'] == '2.1.0', f'Invalid version: {data[\"version\"]}' +assert '\$schema' in data, 'Missing \$schema' +count = len(data['runs'][0]['results']) +print(f'PASS: Valid SARIF 2.1.0 with {count} result(s)') +" diff --git a/tests/e2e/validate-scan.sh b/tests/e2e/validate-scan.sh new file mode 100755 index 0000000..fd8eccf --- /dev/null +++ b/tests/e2e/validate-scan.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +LOG="/tmp/e2e-output.log" + +if grep -q "Full scan report URL: https://socket.dev/" "$LOG"; then + echo "PASS: Full scan report URL found" + grep "Full scan report URL:" "$LOG" +elif grep -q "Diff Url: https://socket.dev/" "$LOG"; then + echo "PASS: Diff URL found" + grep "Diff Url:" "$LOG" +else + echo "FAIL: No report URL found in scan output" + cat "$LOG" + exit 1 +fi From 8159f3a7b2e27485e42c0feedb6d34751dc70e92 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 8 Apr 2026 18:50:22 -0400 Subject: [PATCH 06/11] Strip logger timestamp prefix to fix e2e test Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- tests/e2e/validate-json.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/e2e/validate-json.sh b/tests/e2e/validate-json.sh index 6c78a16..d4f5302 100755 --- a/tests/e2e/validate-json.sh +++ b/tests/e2e/validate-json.sh @@ -6,22 +6,26 @@ LOG="/tmp/e2e-output.log" python3 -c " import json, sys -# The JSON output is on stdout; the log may also contain stderr debug lines. -# Find the first line that parses as valid JSON. +# The JSON output may be prefixed with a logger timestamp (e.g. '2026-04-08 22:46:50,580: {...}'). +# Try parsing the full line first, then from the first '{' character. found = False with open('$LOG') as f: for line in f: line = line.strip() - if not line: - continue - try: - data = json.loads(line) - if isinstance(data, dict): - found = True - print(f'PASS: Valid JSON output with {len(data)} top-level key(s)') - break - except json.JSONDecodeError: + if not line or '{' not in line: continue + # Try full line first, then from the first brace + for candidate in (line, line[line.index('{'):]): + try: + data = json.loads(candidate) + if isinstance(data, dict): + found = True + print(f'PASS: Valid JSON output with {len(data)} top-level key(s)') + break + except json.JSONDecodeError: + continue + if found: + break if not found: print('FAIL: No valid JSON object found in output') From 92c03357ededa9b9086c9862c98b723f00dc7a52 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Thu, 9 Apr 2026 13:40:24 -0400 Subject: [PATCH 07/11] Include unchanged alerts in GitLab report Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- socketsecurity/core/messages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/socketsecurity/core/messages.py b/socketsecurity/core/messages.py index 1d0431a..0f3b62e 100644 --- a/socketsecurity/core/messages.py +++ b/socketsecurity/core/messages.py @@ -767,7 +767,8 @@ def create_security_comment_gitlab(diff: Diff) -> dict: dep_files_map: dict = {} - for alert in diff.new_alerts: + all_alerts = list(diff.new_alerts) + list(getattr(diff, 'unchanged_alerts', [])) + for alert in all_alerts: vulnerability = { "id": Messages.generate_uuid_from_alert_gitlab(alert), "category": "dependency_scanning", From d68aa6ec78814de09aa287290a08e7c5e295d6b5 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Thu, 9 Apr 2026 13:41:12 -0400 Subject: [PATCH 08/11] Fix commit status to count new + unchanged alerts when strict blocking enabled Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- socketsecurity/socketcli.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/socketsecurity/socketcli.py b/socketsecurity/socketcli.py index 6f17f58..a8de637 100644 --- a/socketsecurity/socketcli.py +++ b/socketsecurity/socketcli.py @@ -649,11 +649,20 @@ def main_code(): scm.enable_merge_pipeline_check() passed = output_handler.report_pass(diff) state = "success" if passed else "failed" - blocking_count = sum(1 for a in diff.new_alerts if a.error) + new_blocking = sum(1 for a in diff.new_alerts if a.error) + unchanged_blocking = 0 + if config.strict_blocking and hasattr(diff, 'unchanged_alerts'): + unchanged_blocking = sum(1 for a in diff.unchanged_alerts if a.error) + blocking_count = new_blocking + unchanged_blocking if passed: description = "No blocking issues" else: - description = f"{blocking_count} blocking alert(s) found" + parts = [] + if new_blocking: + parts.append(f"{new_blocking} new") + if unchanged_blocking: + parts.append(f"{unchanged_blocking} existing") + description = f"{blocking_count} blocking alert(s) found ({', '.join(parts)})" target_url = diff.report_url or diff.diff_url or "" scm.set_commit_status(state, description, target_url) From 417d7476e6293fd9578392ad4d92ca5bca75fdf9 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Thu, 9 Apr 2026 13:42:11 -0400 Subject: [PATCH 09/11] Add comparison table of alert behaviors between GitLab + JSON/SARIF Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- docs/cli-reference.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/cli-reference.md b/docs/cli-reference.md index a5c06f2..b3529d8 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -700,6 +700,26 @@ The GitLab report includes **actionable security alerts** based on your Socket p All alert types are included in the GitLab report if they're marked as `error` or `warn` by your Socket Security policy, ensuring the Security Dashboard shows only actionable findings. +### Alert Population: GitLab vs JSON/SARIF + +The GitLab Security Dashboard report and the JSON/SARIF diff outputs use different alert selection strategies, reflecting their distinct purposes: + +| Output Format | Default Alerts | With `--strict-blocking` | +|:---|:---|:---| +| `--enable-gitlab-security` | **All** alerts (new + existing) | All alerts (same) | +| `--enable-json` | New alerts only | New + existing alerts | +| `--enable-sarif` (diff scope) | New alerts only | New + existing alerts | + +**Why the difference?** GitLab's Security Dashboard is designed to present the full security posture of a project. An empty dashboard on a scan with no dependency changes would be misleading -- the vulnerabilities still exist, they just didn't change. By contrast, JSON and SARIF in diff scope are designed to answer "what changed?" and only include existing alerts when `--strict-blocking` explicitly requests it. + +> **Tip:** If you use `--enable-json` alongside `--enable-gitlab-security`, the GitLab report may contain more vulnerabilities than the JSON output. This is expected. To make JSON output match, add `--strict-blocking`. + +### Alert Ignoring via PR/MR Comments + +When using the CLI with SCM integration (`--scm github` or `--scm gitlab`), users can ignore specific alerts by reacting to Socket's PR/MR comments. Ignored alerts are removed from `--enable-json`, `--enable-sarif`, and console output. + +However, the GitLab Security Dashboard report includes **all** alerts matching your security policy (new and existing), regardless of comment-based ignores. This ensures the Security Dashboard always reflects the full set of known issues. To suppress a vulnerability from the GitLab report, adjust the alert's policy in Socket's dashboard rather than ignoring it via a PR comment. + ### Report Schema Socket CLI generates reports compliant with [GitLab Dependency Scanning schema version 15.0.0](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/v15.0.0/dist/dependency-scanning-report-format.json). The reports include: From 802515c04e57fa424b53831aacd42fc5e63b5e0f Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Thu, 9 Apr 2026 13:42:35 -0400 Subject: [PATCH 10/11] Document comment-based ignore behavior differences Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- docs/cli-reference.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/cli-reference.md b/docs/cli-reference.md index b3529d8..0fdee6c 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -753,8 +753,9 @@ When `--enable-gitlab-security` (or `--enable-json` / `--enable-sarif`) is used - Ensure the report file follows the correct schema format **Empty vulnerabilities array:** -- This is normal if no new security issues were detected in diff mode -- For full scans, ensure you are using `--enable-gitlab-security` so alert data is fetched +- The GitLab report includes both new and existing alerts, so repeated scans of the same repo should still populate the report as long as Socket detects actionable issues +- If the report is empty, verify the Socket dashboard shows alerts for the scanned packages -- an empty report means no error/warn-level alerts exist +- For full scans (non-diff mode), ensure you are using `--enable-gitlab-security` so alert data is fetched - Check Socket.dev dashboard for full analysis details ## Development From 11ac5c416c7e9c86600cfcf338840bbaaadb9f91 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Thu, 9 Apr 2026 13:42:53 -0400 Subject: [PATCH 11/11] Add new unit tests for unchanged alerts Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --- tests/unit/test_gitlab_format.py | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/unit/test_gitlab_format.py b/tests/unit/test_gitlab_format.py index 9ed9bb1..96218e4 100644 --- a/tests/unit/test_gitlab_format.py +++ b/tests/unit/test_gitlab_format.py @@ -507,6 +507,52 @@ def test_dependency_files_skips_unknown_manifest(self): report = Messages.create_security_comment_gitlab(diff) assert report["dependency_files"] == [] + def test_unchanged_alerts_included_in_report(self): + """Test that unchanged_alerts are included alongside new_alerts in the GitLab report""" + diff = Diff() + diff.id = "test-scan-id" + diff.diff_url = "https://socket.dev/test" + + diff.new_alerts = [ + Issue( + pkg_name="new-pkg", pkg_version="1.0.0", type="malware", severity="high", + title="New Alert", manifests="package.json", pkg_type="npm", key="k1", purl="pkg:npm/new-pkg@1.0.0" + ), + ] + diff.unchanged_alerts = [ + Issue( + pkg_name="existing-pkg", pkg_version="2.0.0", type="vulnerability", severity="medium", + title="Existing Alert", manifests="package.json", pkg_type="npm", key="k2", purl="pkg:npm/existing-pkg@2.0.0" + ), + ] + + report = Messages.create_security_comment_gitlab(diff) + assert len(report["vulnerabilities"]) == 2 + + names = {v["name"] for v in report["vulnerabilities"]} + assert "New Alert" in names + assert "Existing Alert" in names + + def test_only_unchanged_alerts_produces_nonempty_report(self): + """Test that a diff with no new alerts but unchanged alerts still populates the report""" + diff = Diff() + diff.id = "test-scan-id" + diff.diff_url = "https://socket.dev/test" + + diff.new_alerts = [] + diff.unchanged_alerts = [ + Issue( + pkg_name="stable-pkg", pkg_version="3.0.0", type="vulnerability", severity="critical", + title="Known Issue", manifests="requirements.txt", pkg_type="pypi", key="k1", purl="pkg:pypi/stable-pkg@3.0.0" + ), + ] + + report = Messages.create_security_comment_gitlab(diff) + assert len(report["vulnerabilities"]) == 1 + assert report["vulnerabilities"][0]["name"] == "Known Issue" + assert len(report["dependency_files"]) == 1 + assert report["dependency_files"][0]["path"] == "requirements.txt" + def test_pkg_type_to_package_manager_mapping(self): """Test package manager mapping covers common ecosystems""" assert Messages._pkg_type_to_package_manager("npm") == "npm"