Merge pull request savon-noir#45 from savon-noir/es

savon-noir · savon-noir · commit 4f5c3fba4e70 · 2014-11-22T22:36:27.000+01:00
Merge of Elasticsearch plugin and example + added cpe_list
diff --git a/README.rst b/README.rst
@@ -85,6 +85,12 @@ Examples
 
 Some codes samples are available in the examples directory or in the `documentation`_.
 
+Among other example, you notice an sample code pushing nmap scan reports in an ElasticSearch instance and allowing you to create fancy dashboards in Kibana like the screenshot below:
+
+.. image:: https://github.com/savon-noir/python-libnmap/blob/es/examples/kibanalibnmap.png
+    :alt: Kibanane
+    :align: center
+
 Contributors
 ------------
 
diff --git a/examples/elastikibana.py b/examples/elastikibana.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from libnmap.parser import NmapParser
+from elasticsearch import Elasticsearch
+from datetime import datetime
+import pygeoip
+
+
+def store_report(nmap_report, database, index):
+    rval = True
+    for nmap_host in nmap_report.hosts:
+        rv = store_reportitem(nmap_host, database, index)
+        if rv is False:
+            print("Failed to store host {0} in "
+                  "elasticsearch".format(nmap_host.address))
+            rval = False
+
+    return rval
+
+
+def get_os(nmap_host):
+    rval = {'vendor': 'unknown', 'product': 'unknown'}
+    if nmap_host.is_up() and nmap_host.os_fingerprinted:
+        cpelist = nmap_host.os.os_cpelist()
+        if len(cpelist):
+            mcpe = cpelist.pop()
+            rval.update({'vendor': mcpe.get_vendor(),
+                         'product': mcpe.get_product()})
+    return rval
+
+
+def get_geoip_code(address):
+    gi = pygeoip.GeoIP('/usr/share/GeoIP/GeoIP.dat')
+    return gi.country_code_by_addr(address)
+
+
+def store_reportitem(nmap_host, database, index):
+    host_keys = ["starttime", "endtime", "address", "hostnames",
+                 "ipv4", "ipv6", "mac", "status"]
+    jhost = {}
+    for hkey in host_keys:
+        if hkey == "starttime" or hkey == "endtime":
+            val = getattr(nmap_host, hkey)
+            jhost[hkey] = datetime.fromtimestamp(int(val) if len(val) else 0)
+        else:
+            jhost[hkey] = getattr(nmap_host, hkey)
+
+    jhost.update({'country': get_geoip_code(nmap_host.address)})
+    jhost.update(get_os(nmap_host))
+    for nmap_service in nmap_host.services:
+        reportitems = get_item(nmap_service)
+
+        for ritem in reportitems:
+            ritem.update(jhost)
+            database.index(index=index,
+                           doc_type="NmapItem",
+                           body=ritem)
+    return jhost
+
+
+def get_item(nmap_service):
+    service_keys = ["port", "protocol", "state"]
+    ritems = []
+
+    # create report item for basic port scan
+    jservice = {}
+    for skey in service_keys:
+        jservice[skey] = getattr(nmap_service, skey)
+    jservice['type'] = 'port-scan'
+    jservice['service'] = nmap_service.service
+    jservice['service-data'] = nmap_service.banner
+    ritems.append(jservice)
+
+    # create report items from nse script output
+    for nse_item in nmap_service.scripts_results:
+        jnse = {}
+        for skey in service_keys:
+            jnse[skey] = getattr(nmap_service, skey)
+        jnse['type'] = 'nse-script'
+        jnse['service'] = nse_item['id']
+        jnse['service-data'] = nse_item['output']
+        ritems.append(jnse)
+
+    return ritems
+
+
+xmlscans = ['../libnmap/test/files/1_hosts.xml',
+            '../libnmap/test/files/full_sudo6.xml',
+            '/vagrant/nmap_switches.xml',
+            '/vagrant/nmap-5hosts.xml']
+
+for xmlscan in xmlscans:
+    nmap_report = NmapParser.parse_fromfile(xmlscan)
+
+    if nmap_report:
+        rep_date = datetime.fromtimestamp(int(nmap_report.started))
+        index = "nmap-{0}".format(rep_date.strftime('%Y-%m-%d'))
+        db = Elasticsearch()
+        j = store_report(nmap_report, db, index)
diff --git a/examples/es_plugin.py b/examples/es_plugin.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+from libnmap.parser import NmapParser
+from libnmap.reportjson import ReportDecoder
+from libnmap.plugins.es import NmapElasticsearchPlugin
+from datetime import datetime
+import json
+
+nmap_report = NmapParser.parse_fromfile('libnmap/test/files/1_hosts.xml')
+mindex = datetime.fromtimestamp(nmap_report.started).strftime('%Y-%m-%d')
+db = NmapElasticsearchPlugin(index=mindex)
+dbid = db.insert(nmap_report)
+nmap_json = db.get(dbid)
+
+nmap_obj = json.loads(json.dumps(nmap_json), cls=ReportDecoder)
+print(nmap_obj)
+#print(db.getall())
+
diff --git a/examples/kibanalibnmap.png b/examples/kibanalibnmap.png
diff --git a/libnmap/objects/os.py b/libnmap/objects/os.py
@@ -376,6 +376,13 @@ def osclass(self, min_accuracy=90):
                     os_array.append(_ftstr)
         return os_array
 
+    def os_cpelist(self):
+        cpelist = []
+        for _osmatch in self.osmatches:
+            for oclass in _osmatch.osclasses:
+                cpelist.extend(oclass.cpelist)
+        return cpelist
+
     def __repr__(self):
         rval = ""
         for _osmatch in self.osmatches:
diff --git a/libnmap/plugins/es.py b/libnmap/plugins/es.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+import json
+from libnmap.reportjson import ReportEncoder
+from libnmap.plugins.backendplugin import NmapBackendPlugin
+from elasticsearch import Elasticsearch
+from datetime import datetime
+
+
+class NmapElasticsearchPlugin(NmapBackendPlugin):
+    """
+        This class enables the user to store and manipulate nmap reports \
+        in a elastic search db.
+    """
+    def __init__(self, index=None):
+        if index is None:
+            self.index = "nmap.{0}".format(datetime.now().strftime('%Y-%m-%d'))
+        else:
+            self.index = index
+        self._esapi = Elasticsearch()
+
+    def insert(self, report, doc_type=None):
+        """
+            insert NmapReport in the backend
+            :param NmapReport:
+            :return: str the ident of the object in the backend for
+            future usage
+            or None
+        """
+        if doc_type is None:
+            doc_type = 'NmapReport'
+        j = json.dumps(report, cls=ReportEncoder)
+        res = self._esapi.index(
+            index=self.index,
+            doc_type=doc_type,
+            body=json.loads(j))
+        rc = res['_id']
+        return rc
+
+    def delete(self, id):
+        """
+            delete NmapReport if the backend
+            :param id: str
+        """
+        raise NotImplementedError
+
+    def get(self, id):
+        """
+            retreive a NmapReport from the backend
+            :param id: str
+            :return: NmapReport
+        """
+        res = self._esapi.get(index=self.index,
+                              doc_type="NmapReport",
+                              id=id)['_source']
+        return res
+
+    def getall(self, filter=None):
+        """
+            :return: collection of tuple (id,NmapReport)
+            :param filter: Nice to have implement a filter capability
+        """
+        rsearch = self._esapi.search(index=self.index,
+                                     body={"query": {"match_all": {}}})
+        print("--------------------")
+        print(type(rsearch))
+        print(rsearch)
+        print("------------")
