Skip to content

Commit dc8f3eb

Browse files
RTannRTann
authored
ROX-28353: use NVD 2.0 JSON feeds (#16257)
1 parent 6acfdd4 commit dc8f3eb

3 files changed

Lines changed: 104 additions & 63 deletions

File tree

‎.github/workflows/scanner-nvd-update.yaml‎

Lines changed: 59 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,6 @@ jobs:
2929
python-version: '3.10'
3030

3131
- name: Fetch NVD Feeds
32-
env:
33-
SCANNER_NVD_API_KEY: ${{ secrets.NVD_API_KEY }}
34-
SCANNER_NVD_URL: https://services.nvd.nist.gov/rest/json/cves/2.0
3532
run: |
3633
set -eu
3734
dir=$(mktemp -d)
@@ -93,6 +90,65 @@ jobs:
9390
path: nvd-api.zip
9491
if-no-files-found: error
9592

93+
upload-pr-nvd-feeds:
94+
needs:
95+
- fetch-nvd-feeds
96+
runs-on: ubuntu-latest
97+
if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-nvd')
98+
steps:
99+
# Checkout master to get the latest local actions
100+
- name: Checkout repository
101+
uses: actions/checkout@v4
102+
with:
103+
fetch-depth: 0
104+
105+
- uses: ./.github/actions/job-preamble
106+
with:
107+
free-disk-space: 50
108+
gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
109+
110+
- name: Checkout specific reference
111+
uses: actions/checkout@v4
112+
with:
113+
fetch-depth: 0
114+
ref: ${{ github.event.pull_request.head.sha }}
115+
116+
- name: Authenticate with test GCS bucket
117+
uses: google-github-actions/auth@v2
118+
with:
119+
credentials_json: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }}
120+
121+
- name: Set up Cloud SDK
122+
uses: google-github-actions/setup-gcloud@v2
123+
124+
- name: Download NVD
125+
uses: ./.github/actions/download-artifact-with-retry
126+
with:
127+
name: nvd-feeds
128+
path: .
129+
130+
# PR owner is responsible for verifying NVD bundle is generated
131+
- name: Upload NVD Feeds to Google Cloud Storage
132+
run: |
133+
branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
134+
# Replace / with -, so the branch name isn't truncated when pushed to GCS.
135+
dir=${branch////-}
136+
case "$dir" in
137+
dev|1.0.0)
138+
echo "Error: branch $dir is protected. Choose a different branch name."
139+
exit 1
140+
esac
141+
mkdir -p "$dir"
142+
cp nvd-feeds.zip "$dir/"
143+
gsutil -m cp -r "$dir" "gs://scanner-v4-test/nvd/"
144+
145+
# Checkout again to get the latest local actions for cleanup phase
146+
- name: Checkout repository
147+
uses: actions/checkout@v4
148+
if: always()
149+
with:
150+
fetch-depth: 0
151+
96152
upload-nvd-feeds:
97153
needs:
98154
- fetch-nvd-feeds

‎.github/workflows/scanner-versioned-definitions-update.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,39 @@ jobs:
5050
with:
5151
fetch-depth: 0
5252

53+
- name: Download NVD PR
54+
if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-nvd')
55+
run: |
56+
set -eu
57+
since_time=$(date -u -d '24 hours ago' '+%a, %d %b %Y %H:%M:%S GMT')
58+
NVD_BUNDLE_TYPE=${{ github.event.inputs.job || 'nvd-feeds' }}
59+
case "$NVD_BUNDLE_TYPE" in
60+
nvd-api)
61+
nvd_file=nvd-api.zip
62+
;;
63+
nvd-feeds)
64+
nvd_file=nvd-feeds.zip
65+
;;
66+
*)
67+
echo >&2 "Warning: invalid NVD bundle type '$NVD_BUNDLE_TYPE'"
68+
exit 1
69+
esac
70+
branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
71+
# Replace / with -, so the branch name isn't truncated when pushed to GCS.
72+
dir=${branch////-}
73+
url="https://storage.googleapis.com/scanner-v4-test/nvd/$dir/$nvd_file"
74+
75+
code=$(curl \
76+
-o nvd.zip \
77+
-w "%{http_code}" \
78+
-H "If-Modified-Since: $since_time" \
79+
"$url")
80+
81+
echo "code: $code"
82+
echo "$code" | grep -q 200
83+
5384
- name: Download NVD
85+
if: github.event_name != 'pull_request'
5486
run: |
5587
set -eu
5688
since_time=$(date -u -d '24 hours ago' '+%a, %d %b %Y %H:%M:%S GMT')

‎.github/workflows/scripts/scanner-update-nvd-feeds.py‎

Lines changed: 13 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
#!/usr/bin/env python
22

33
"""
4-
Fetch NVD data from the JSON 1.1 feed archives, parse them into API 2.0
5-
schema and save into files.
4+
Fetch NVD data from the JSON 2.0 feed archives and save into files.
65
"""
76

87
import gzip
@@ -17,70 +16,23 @@
1716
level=logging.INFO,
1817
format="%(asctime)s: %(levelname)s: %(name)s %(message)s")
1918

19+
DEFAULT_BASE_URL = "https://nvd.nist.gov/feeds/json/cve/2.0/"
2020

21-
def convert_cve_feed_to_api(cve_feed):
22-
"""Convert NVDCVEFeedJSON10DefCVEItem to CVEAPIJSON20DefCVEItem."""
21+
class FeedLoader:
2322

24-
# Required fields.
23+
log = logging.getLogger("FeedLoader")
2524

26-
cve_api = {"id": cve_feed['cve']['CVE_data_meta']['ID'],
27-
"published": cve_feed['publishedDate'],
28-
"lastModified": cve_feed['lastModifiedDate'],
29-
"descriptions": [],
30-
"metrics": {}}
31-
32-
# Description.
33-
34-
for desc in cve_feed['cve'].get('description', {}).get('description_data', []):
35-
cve_api["descriptions"].append({"lang": desc["lang"],
36-
"value": desc["value"]})
37-
38-
# Metrics.
39-
40-
impact = cve_feed.get('impact', {})
41-
cvssMetricV2 = impact.get('baseMetricV2', {}).get('cvssV2', {})
42-
cvssMetricV3 = impact.get('baseMetricV3', {}).get('cvssV3', {})
43-
44-
if cvssMetricV2:
45-
key = "cvssMetricV2"
46-
metric = {"type": "Primary",
47-
"cvssData": {
48-
"version": "2.0",
49-
"vectorString": cvssMetricV2["vectorString"],
50-
"baseScore": cvssMetricV2["baseScore"]}}
51-
cve_api["metrics"][key] = [metric]
52-
53-
if cvssMetricV3:
54-
version = cvssMetricV3["version"]
55-
key = "cvssMetricV30" if version == "3.0" else "cvssMetricV31"
56-
metric = {"type": "Primary",
57-
"cvssData": {
58-
"version": version,
59-
"vectorString": cvssMetricV3["vectorString"],
60-
"baseScore": cvssMetricV3["baseScore"]}}
61-
cve_api["metrics"][key] = [metric]
62-
63-
return {"cve": cve_api}
64-
65-
66-
class LegacyLoader:
67-
68-
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.1/"
69-
70-
log = logging.getLogger("LegacyLoader")
71-
72-
def __init__(self, base_url=None):
73-
self._base_url = base_url or self.BASE_URL
25+
def __init__(self, base_url):
26+
self._base_url = base_url
7427

7528
def fetch(self, year):
76-
url = parse.urljoin(self._base_url, f"nvdcve-1.1-{year}.json.gz")
29+
url = parse.urljoin(self._base_url, f"nvdcve-2.0-{year}.json.gz")
7730
self.log.info("fetching and decompressing: %s", url)
7831
with request.urlopen(url, timeout=60) as resp:
7932
with gzip.GzipFile(fileobj=resp) as gz:
80-
cve_feed = json.load(gz)
81-
items = cve_feed.get("CVE_Items", [])
82-
self.log.info("cve count: %d", len(items))
83-
yield from (convert_cve_feed_to_api(c) for c in items)
33+
data = json.load(gz)
34+
yield from (v for v in data["vulnerabilities"]
35+
if v["cve"]["vulnStatus"].lower() != "rejected")
8436

8537

8638
def parse_args():
@@ -110,7 +62,8 @@ def dirpath(s):
11062
default=date.today().year)
11163
parser.add_argument(
11264
"--base-url",
113-
help="URL for the NVD CVE feed (version 1.1).")
65+
help="URL for the NVD CVE feed (version 2.0).",
66+
default=DEFAULT_BASE_URL)
11467
parser.add_argument(
11568
"--file-pattern",
11669
help="File name pattern of the NVD feed files.",
@@ -120,7 +73,7 @@ def dirpath(s):
12073

12174

12275
def main(args):
123-
loader = LegacyLoader(args.base_url)
76+
loader = FeedLoader(args.base_url)
12477
for year in range(args.start_year, args.end_year + 1):
12578
path = os.path.join(args.output, args.file_pattern % {'year': year})
12679
logging.info("fetching '%d' to '%s'", year, path)

0 commit comments

Comments
 (0)