diff --git a/hooks/persistence-elastic/README.md b/hooks/persistence-elastic/README.md index a2d9bd1974..e37cb5ab57 100644 --- a/hooks/persistence-elastic/README.md +++ b/hooks/persistence-elastic/README.md @@ -19,6 +19,12 @@ Installing the Elasticsearch persistenceProvider hook will add a _ReadOnly Hook_ helm upgrade --install elkh secureCodeBox/persistence-elastic ``` +## Elasticsearch Indexing + +For the elasticsearch `indexSuffix` you can provide a date format pattern. We use [Luxon](https://moment.github.io/luxon/) to format the date. So checkout +the [Luxon documentation](https://moment.github.io/luxon/docs/manual/formatting.html#table-of-tokens) to see what kind of format patterns you can use for the +`indexSuffix`. Default pattern is `yyyy-MM-dd` + ## Chart Configuration | Key | Type | Default | Description | @@ -40,7 +46,9 @@ helm upgrade --install elkh secureCodeBox/persistence-elastic | image.repository | string | `"docker.io/securecodebox/persistence-elastic"` | Image repository for the dashboard importer job | | image.tag | string | defaults to the charts version | Image tag for the dashboard importer job | | imagePullSecrets | list | `[]` | | +| indexAppendNamespace | bool | `true` | Define if the name of the namespace where this hook is deployed to must be added to the index name. The namespace can be used to separate index by tenants (namespaces). | | indexPrefix | string | `"scbv2"` | Define a specific index prefix used for all elasticsearch indices. | +| indexSuffix | string | `"“yyyy-MM-dd”"` | Define a specific index suffix based on date pattern (YEAR (yyyy), MONTH (yyyy-MM), WEEK (yyyy-'W'W), DATE (yyyy-MM-dd)). We use Luxon for date formatting (https://moment.github.io/luxon/docs/manual/formatting.html#table-of-tokens) | | kibana | object | `{"enabled":true}` | Configures included Elasticsearch subchart | | kibana.enabled | bool | `true` | Enable if you want to deploy an kibana service (see: https://github.com/elastic/helm-charts/tree/master/kibana) | | nameOverride | string | `""` | | @@ -50,4 +58,4 @@ helm upgrade --install elkh secureCodeBox/persistence-elastic | securityContext | object | `{}` | | | tolerations | list | `[]` | | -[elastic.io]: https://www.elastic.co/products/elasticsearch \ No newline at end of file +[elastic.io]: https://www.elastic.co/products/elasticsearch diff --git a/hooks/persistence-elastic/README.md.gotmpl b/hooks/persistence-elastic/README.md.gotmpl index 619c763d7f..fbbfac87d1 100644 --- a/hooks/persistence-elastic/README.md.gotmpl +++ b/hooks/persistence-elastic/README.md.gotmpl @@ -19,9 +19,16 @@ Installing the Elasticsearch persistenceProvider hook will add a _ReadOnly Hook_ helm upgrade --install elkh secureCodeBox/persistence-elastic ``` +## Elasticsearch Indexing + +For the elasticsearch `indexSuffix` you can provide a date format pattern. We use [Luxon](https://moment.github.io/luxon/) to format the date. So checkout +the [Luxon documentation](https://moment.github.io/luxon/docs/manual/formatting.html#table-of-tokens) to see what kind of format patterns you can use for the +`indexSuffix`. Default pattern is `yyyy-MM-dd` + + ## Chart Configuration {{ template "chart.valuesTable" . }} -[elastic.io]: https://www.elastic.co/products/elasticsearch \ No newline at end of file +[elastic.io]: https://www.elastic.co/products/elasticsearch diff --git a/hooks/persistence-elastic/hook.js b/hooks/persistence-elastic/hook.js index 00f7172aa5..e633455e88 100644 --- a/hooks/persistence-elastic/hook.js +++ b/hooks/persistence-elastic/hook.js @@ -3,6 +3,8 @@ const { Client } = require("@elastic/elasticsearch"); const flatMap = require("lodash.flatmap"); const chunk = require("lodash.chunk"); +const { DateTime } = require("luxon"); + const authParams = {}; const username = process.env["ELASTICSEARCH_USERNAME"]; @@ -10,6 +12,8 @@ const password = process.env["ELASTICSEARCH_PASSWORD"]; const apiKeyId = process.env["ELASTICSEARCH_APIKEY_ID"]; const apiKey = process.env["ELASTICSEARCH_APIKEY"]; +const defaultDateFormat = 'yyyy-MM-dd'; + if (apiKeyId && apiKey) { console.log("Using API Key for Authentication"); authParams.auth = { @@ -39,6 +43,8 @@ async function handle({ now = new Date(), tenant = process.env["NAMESPACE"], indexPrefix = process.env["ELASTICSEARCH_INDEX_PREFIX"] || "scbv2", + indexSuffix = process.env["ELASTICSEARCH_INDEX_SUFFIX"] || defaultDateFormat, + appendNamespace = process.env['ELASTICSEARCH_INDEX_APPEND_NAMESPACE'] || false }) { const findings = await getFindings(); @@ -47,8 +53,8 @@ async function handle({ `Using Elasticsearch Instance at "${process.env["ELASTICSEARCH_ADDRESS"]}"` ); - const timeStamp = now.toISOString().substr(0, 10); - const indexName = `${indexPrefix}_${tenant}_${timeStamp}`; + let indexName = appendNamespace ? `${indexPrefix}_${tenant}_` : `${indexPrefix}_`; + indexName += DateTime.fromJSDate(now).toFormat(indexSuffix) await client.indices.create( { diff --git a/hooks/persistence-elastic/hook.test.js b/hooks/persistence-elastic/hook.test.js index 1a73fbb13f..993e71cac0 100644 --- a/hooks/persistence-elastic/hook.test.js +++ b/hooks/persistence-elastic/hook.test.js @@ -5,33 +5,33 @@ beforeEach(() => { elasticClient.bulk.mockClear(); }); +const scan = { + metadata: { + uid: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc", + name: "demo-scan", + labels: { + company: "iteratec", + }, + }, + spec: { + scanType: "Nmap", + parameters: ["-Pn", "localhost"], + }, +}; + +const testDate = new Date('2020-11-11'); + test("should only send scan summary document if no findings are passing in", async () => { const findings = []; const getFindings = async () => findings; - const scan = { - metadata: { - uid: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc", - name: "demo-scan", - labels: { - company: "iteratec", - }, - }, - spec: { - scanType: "Nmap", - parameters: ["-Pn", "localhost"], - }, - }; - - const now = new Date(); - - await handle({ getFindings, scan, now, tenant: "default" }); + await handle({ getFindings, scan, now: testDate, tenant: "default", appendNamespace: true }); expect(elasticClient.index).toBeCalledTimes(1); expect(elasticClient.index).toBeCalledWith({ body: { - "@timestamp": now, + "@timestamp": testDate, id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc", labels: { company: "iteratec", @@ -41,12 +41,12 @@ test("should only send scan summary document if no findings are passing in", asy scan_type: "Nmap", type: "scan", }, - index: `scbv2_default_${now.toISOString().substr(0, 10)}`, + index: `scbv2_default_2020-11-11`, }); expect(elasticClient.bulk).not.toBeCalled(); }); -test("should send findings to elasticsearch", async () => { +test("should send findings to elasticsearch with given prefix", async () => { const findings = [ { id: "4560b3e6-1219-4f5f-9b44-6579f5a32407", @@ -57,28 +57,12 @@ test("should send findings to elasticsearch", async () => { const getFindings = async () => findings; - const scan = { - metadata: { - uid: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc", - name: "demo-scan", - labels: { - company: "iteratec", - }, - }, - spec: { - scanType: "Nmap", - parameters: ["-Pn", "localhost"], - }, - }; - - const now = new Date(); - - await handle({ getFindings, scan, now, tenant: "default" }); + await handle({ getFindings, scan, now: testDate, tenant: "default", indexPrefix: "myPrefix", appendNamespace: true }); expect(elasticClient.index).toBeCalledTimes(1); expect(elasticClient.index).toBeCalledWith({ body: { - "@timestamp": now, + "@timestamp": testDate, id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc", labels: { company: "iteratec", @@ -88,7 +72,7 @@ test("should send findings to elasticsearch", async () => { scan_type: "Nmap", type: "scan", }, - index: `scbv2_default_${now.toISOString().substr(0, 10)}`, + index: `myPrefix_default_2020-11-11`, }); expect(elasticClient.bulk).toBeCalledTimes(1); @@ -97,11 +81,11 @@ test("should send findings to elasticsearch", async () => { body: [ { index: { - _index: `scbv2_default_${now.toISOString().substr(0, 10)}`, + _index: `myPrefix_default_2020-11-11`, }, }, { - "@timestamp": now, + "@timestamp": testDate, category: "Open Port", id: "4560b3e6-1219-4f5f-9b44-6579f5a32407", name: "Port 5601 is open", @@ -116,3 +100,75 @@ test("should send findings to elasticsearch", async () => { ], }); }); + +test("should not append namespace if 'appendNamespace' is null", async () => { + const findings = []; + + const getFindings = async () => findings; + + await handle({ getFindings, scan, now: testDate, tenant: "default" }); + + expect(elasticClient.index).toBeCalledTimes(1); + expect(elasticClient.index).toBeCalledWith({ + body: { + "@timestamp": testDate, + id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc", + labels: { + company: "iteratec", + }, + name: "demo-scan", + parameters: ["-Pn", "localhost"], + scan_type: "Nmap", + type: "scan", + }, + index: `scbv2_2020-11-11`, + }); +}); + +test("should append date format yyyy", async () => { + const findings = []; + + const getFindings = async () => findings; + + await handle({ getFindings, scan, now: testDate, tenant: "default", indexSuffix: "yyyy" }); + + expect(elasticClient.index).toBeCalledTimes(1); + expect(elasticClient.index).toBeCalledWith({ + body: { + "@timestamp": testDate, + id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc", + labels: { + company: "iteratec", + }, + name: "demo-scan", + parameters: ["-Pn", "localhost"], + scan_type: "Nmap", + type: "scan", + }, + index: `scbv2_2020`, + }); +}); + +test("should append week format like yyyy/'W'W -> 2020/W46", async () => { + const findings = []; + + const getFindings = async () => findings; + + await handle({ getFindings, scan, now: testDate, tenant: "default", indexSuffix: "yyyy/'W'W" }); + + expect(elasticClient.index).toBeCalledTimes(1); + expect(elasticClient.index).toBeCalledWith({ + body: { + "@timestamp": testDate, + id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc", + labels: { + company: "iteratec", + }, + name: "demo-scan", + parameters: ["-Pn", "localhost"], + scan_type: "Nmap", + type: "scan", + }, + index: `scbv2_2020/W46`, + }); +}); diff --git a/hooks/persistence-elastic/package-lock.json b/hooks/persistence-elastic/package-lock.json index d40f69034a..22463b8c90 100644 --- a/hooks/persistence-elastic/package-lock.json +++ b/hooks/persistence-elastic/package-lock.json @@ -2774,6 +2774,12 @@ "@sinonjs/commons": "^1.7.0" } }, + "luxon": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.25.0.tgz", + "integrity": "sha512-hEgLurSH8kQRjY6i4YLey+mcKVAWXbDNlZRmM6AgWDJ1cY3atl8Ztf5wEY7VBReFbmGnwQPz7KYJblL8B2k0jQ==", + "dev": true + }, "make-dir": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.2.tgz", diff --git a/hooks/persistence-elastic/package.json b/hooks/persistence-elastic/package.json index 2d8ad109c5..957e152ed9 100644 --- a/hooks/persistence-elastic/package.json +++ b/hooks/persistence-elastic/package.json @@ -24,13 +24,13 @@ "url": "https://www.iteratec.com" }, "contributors": [ - { - "name" : "Jannik Hollenbach", - "url" : "https://github.com/J12934" + { + "name": "Jannik Hollenbach", + "url": "https://github.com/J12934" }, - { - "name" : "Robert Seedorff", - "url" : "https://github.com/rseedorff" + { + "name": "Robert Seedorff", + "url": "https://github.com/rseedorff" } ], "bugs": { @@ -43,6 +43,7 @@ "lodash.flatmap": "^4.5.0" }, "devDependencies": { - "jest": "^25.1.0" + "jest": "^25.1.0", + "luxon": "^1.25.0" } } diff --git a/hooks/persistence-elastic/templates/persistence-provider.yaml b/hooks/persistence-elastic/templates/persistence-provider.yaml index e444e30f67..99e5401b73 100644 --- a/hooks/persistence-elastic/templates/persistence-provider.yaml +++ b/hooks/persistence-elastic/templates/persistence-provider.yaml @@ -10,6 +10,10 @@ spec: env: - name: ELASTICSEARCH_INDEX_PREFIX value: {{ .Values.indexPrefix | quote }} + - name: ELASTICSEARCH_INDEX_SUFFIX + value: {{ .Values.indexSuffix | quote }} + - name: ELASTICSEARCH_INDEX_APPEND_NAMESPACE + value: {{ .Values.indexAppendNamespace | quote }} {{- if .Values.externalElasticStack.enabled }} - name: ELASTICSEARCH_ADDRESS value: {{ .Values.externalElasticStack.elasticsearchAddress | quote }} @@ -39,4 +43,4 @@ spec: secretKeyRef: name: {{ .Values.authentication.apiKeySecret }} key: id -{{- end }} \ No newline at end of file +{{- end }} diff --git a/hooks/persistence-elastic/values.yaml b/hooks/persistence-elastic/values.yaml index 1dd3ee8684..f0a84741ae 100644 --- a/hooks/persistence-elastic/values.yaml +++ b/hooks/persistence-elastic/values.yaml @@ -12,6 +12,10 @@ image: # indexPrefix -- Define a specific index prefix used for all elasticsearch indices. indexPrefix: "scbv2" +# indexSuffix -- Define a specific index suffix based on date pattern (YEAR (yyyy), MONTH (yyyy-MM), WEEK (yyyy-'W'W), DATE (yyyy-MM-dd)). We use Luxon for date formatting (https://moment.github.io/luxon/docs/manual/formatting.html#table-of-tokens) +indexSuffix: “yyyy-MM-dd” +# indexAppendNamespace -- Define if the name of the namespace where this hook is deployed to must be added to the index name. The namespace can be used to separate index by tenants (namespaces). +indexAppendNamespace: true externalElasticStack: # externalElasticStack.enabled -- Enable this when you already have an Elastic Stack running to which you want to send your results