From f3fdd9db37827a6b6b353ef8a7718e52c801ee36 Mon Sep 17 00:00:00 2001 From: Gvald Date: Thu, 9 Aug 2018 16:16:36 +0300 Subject: [PATCH] CSV plugin added, minor changes in report, service and host objects --- libnmap/objects/host.py | 3 +- libnmap/objects/report.py | 96 ++++++++++++++++++++- libnmap/objects/service.py | 13 ++- libnmap/parser.py | 46 +++++++--- libnmap/plugins/csv.py | 74 ++++++++++++++++ libnmap/test/test_backend_plugin_factory.py | 30 ++++--- 6 files changed, 232 insertions(+), 30 deletions(-) create mode 100644 libnmap/plugins/csv.py diff --git a/libnmap/objects/host.py b/libnmap/objects/host.py index 671f6f4..fd249fd 100644 --- a/libnmap/objects/host.py +++ b/libnmap/objects/host.py @@ -480,8 +480,7 @@ def get_dict(self): :return dict """ - d = dict([("{0}::{1}".format(s.__class__.__name__, str(s.id)), - hash(s)) + d = dict([(s.report_format, s.state) for s in self.services]) d.update({'address': self.address, 'status': self.status, diff --git a/libnmap/objects/report.py b/libnmap/objects/report.py index 395fc31..15b39ee 100644 --- a/libnmap/objects/report.py +++ b/libnmap/objects/report.py @@ -20,7 +20,7 @@ class NmapReport(object): end user of the lib. NmapReport is certainly the output interface for the end user of the lib. """ - def __init__(self, raw_data=None): + def __init__(self, raw_data=None, source_filename=None): """ Constructor for NmapReport object. @@ -29,7 +29,9 @@ def __init__(self, raw_data=None): self._nmaprun = {} self._scaninfo = {} self._hosts = [] + self._services = {} self._runstats = {} + self._source_filename = source_filename if raw_data is not None: self.__set_raw_data(raw_data) @@ -123,6 +125,72 @@ def hosts(self): """ return self._hosts + @property + def services(self): + """ + Accessor returning an array of scanned services among all hosts. + + Scanned hosts are NmapHost objects. + + :return: list of report-formatted services + """ + return list(self._services.keys()) + + def get_hosts_byservice(self, report_formatted_service): + """ + Gets a host addresses from services list. + + :param report_formatted_service: service to look up + :type report_formatted_service: report_format service + + :return: NmapHost + """ + if report_formatted_service in self._services: + return self._services[report_formatted_service] + else: + return None + + def get_hosts_by_services_list(self, services_list): + """ + Gets a host addresses from services list. + + :param report_formatted_service: service to look up + :type report_formatted_service: report_format service + + :return: NmapHost + """ + res_hosts = {} + for service in self._services: + if '.' in service: + port, sname = service.split('.') + if '/' in sname: + sname = sname.split('/')[0] + if port in services_list or sname in services_list: + hosts = [] + for host in self._services[service]: + for addr_dict in host: + host_str = addr_dict['addr'] + if port: + host_str += ':' + port + hosts.append(host_str) + res_hosts[service] = hosts + return res_hosts + + def export_web_hosts(self): + """ + Gets a host addresses from services list. + + :param report_formatted_service: service to look up + :type report_formatted_service: report_format service + + :return: NmapHost + """ + web_ports = ['80', '443', '8080', '8443'] + #web_ports = ['22'] + #web_services = ['http'] + web_services = ['http', 'https'] + return self.get_hosts_by_services_list(web_ports + web_services) + def get_host_byid(self, host_id): """ Gets a NmapHost object directly from the host array @@ -252,6 +320,31 @@ def hosts_total(self): rval = -1 return rval + @property + def is_from_file(self): + """ + Returning whether the NmapReport object was created from + existing file or not. + + :return: boolean + """ + return True if self._source_filename is not None else False + + + @property + def filename(self): + """ + Returning the filenmae that NmapReport object was created from. + Return empty string if none. + + :return: str + """ + if self.is_from_file: + return self._source_filename + else: + return '' + + def get_raw_data(self): """ Returns a dict representing the NmapReport object. @@ -269,6 +362,7 @@ def __set_raw_data(self, raw_data): self._nmaprun = raw_data['_nmaprun'] self._scaninfo = raw_data['_scaninfo'] self._hosts = raw_data['_hosts'] + self._services = raw_data['_services'] self._runstats = raw_data['_runstats'] def is_consistent(self): diff --git a/libnmap/objects/service.py b/libnmap/objects/service.py index 11d4726..386932d 100644 --- a/libnmap/objects/service.py +++ b/libnmap/objects/service.py @@ -281,10 +281,21 @@ def id(self): This is used for diff()ing NmapService object via NmapDiff. - :return: tuple + :return: string """ return "{0}.{1}".format(self.protocol, self.port) + @property + def report_format(self): + """ + Accessor for the report representation of NmapService. + + Used in csv fieldnames. + + :return: string + """ + return "{0.port}.{0.service}/{0.protocol}".format(self) + def get_dict(self): """ Return a python dict representation of the NmapService object. diff --git a/libnmap/parser.py b/libnmap/parser.py index 8331ef9..8e14531 100644 --- a/libnmap/parser.py +++ b/libnmap/parser.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- - +import os +from collections import defaultdict try: import xml.etree.cElementTree as ET @@ -10,7 +11,8 @@ class NmapParser(object): @classmethod - def parse(cls, nmap_data=None, data_type='XML', incomplete=False): + def parse(cls, nmap_data=None, data_type='XML', + incomplete=False, source_filename=None): """ Generic class method of NmapParser class. @@ -31,6 +33,10 @@ def parse(cls, nmap_data=None, data_type='XML', incomplete=False): the end of the scan. :type incomplete: boolean + :param source_filename: if parsing from nmap report file \ + this will hold the file name of the source report. + :type source_filename: string + As of today, only XML parsing is supported. :return: NmapObject (NmapHost, NmapService or NmapReport) @@ -38,7 +44,7 @@ def parse(cls, nmap_data=None, data_type='XML', incomplete=False): nmapobj = None if data_type == "XML": - nmapobj = cls._parse_xml(nmap_data, incomplete) + nmapobj = cls._parse_xml(nmap_data, incomplete, source_filename) else: raise NmapParserException("Unknown data type provided. " "Please check documentation for " @@ -46,7 +52,7 @@ def parse(cls, nmap_data=None, data_type='XML', incomplete=False): return nmapobj @classmethod - def _parse_xml(cls, nmap_data=None, incomplete=False): + def _parse_xml(cls, nmap_data=None, incomplete=False, source_filename=None): """ Protected class method used to process a specific data type. In this case: XML. This method is called by cls.parse class @@ -72,6 +78,10 @@ def _parse_xml(cls, nmap_data=None, incomplete=False): the end of the scan. :type incomplete: boolean + :param source_filename: if parsing from nmap report file \ + this will hold the file name of the source report. + :type source_filename: string + :return: NmapObject (NmapHost, NmapService or NmapReport) \ or a list of NmapObject """ @@ -93,7 +103,7 @@ def _parse_xml(cls, nmap_data=None, incomplete=False): nmapobj = None if root.tag == 'nmaprun': - nmapobj = cls._parse_xml_report(root) + nmapobj = cls._parse_xml_report(root, source_filename) elif root.tag == 'host': nmapobj = cls._parse_xml_host(root) elif root.tag == 'ports': @@ -106,7 +116,7 @@ def _parse_xml(cls, nmap_data=None, incomplete=False): return nmapobj @classmethod - def _parse_xml_report(cls, root=None): + def _parse_xml_report(cls, root=None, source_filename=None): """ This method parses out a full nmap scan report from its XML root node: . @@ -114,11 +124,16 @@ def _parse_xml_report(cls, root=None): :param root: Element from xml.ElementTree (top of XML the document) :type root: Element + :param source_filename: if parsing from nmap report file \ + this will hold the file name of the source report. + :type source_filename: string + :return: NmapReport object """ nmap_scan = {'_nmaprun': {}, '_scaninfo': {}, - '_hosts': [], '_runstats': {}} + '_hosts': [], '_runstats': {}, + '_services': defaultdict(list)} if root is None: raise NmapParserException("No root node provided to parse XML " @@ -129,14 +144,15 @@ def _parse_xml_report(cls, root=None): if el.tag == 'scaninfo': nmap_scan['_scaninfo'] = cls.__parse_scaninfo(el) elif el.tag == 'host': - nmap_scan['_hosts'].append(cls._parse_xml_host(el)) + nmap_scan['_hosts'].append(cls._parse_xml_host(el, + nmap_scan['_services'])) elif el.tag == 'runstats': nmap_scan['_runstats'] = cls.__parse_runstats(el) # else: # print "struct pparse unknown attr: {0} value: {1}".format( # el.tag, # el.get(el.tag)) - return NmapReport(nmap_scan) + return NmapReport(nmap_scan, source_filename) @classmethod def parse_fromstring(cls, nmap_data, data_type="XML", incomplete=False): @@ -188,9 +204,12 @@ def parse_fromfile(cls, nmap_report_path, """ try: + filename = os.path.basename(nmap_report_path) + filename = os.path.splitext(filename)[0] with open(nmap_report_path, 'r') as fileobj: fdata = fileobj.read() - rval = cls.parse(fdata, data_type, incomplete) + rval = cls.parse(fdata, data_type, incomplete, + filename) except IOError: raise return rval @@ -254,7 +273,7 @@ def __parse_scaninfo(cls, scaninfo_data): return cls.__format_attributes(xelement) @classmethod - def _parse_xml_host(cls, scanhost_data): + def _parse_xml_host(cls, scanhost_data, report_services_dict): """ Protected method parsing a portion of a nmap scan result. Receives a XML tag representing a scanned host with @@ -283,6 +302,7 @@ def _parse_xml_host(cls, scanhost_data): ports_dict = cls._parse_xml_ports(xh) for port in ports_dict['ports']: _services.append(port) + report_services_dict[port.report_format].append(_addresses) _host_extras['extraports'] = ports_dict['extraports'] elif xh.tag == 'status': _status = cls.__format_attributes(xh) @@ -329,7 +349,9 @@ def __parse_hostnames(cls, scanhostnames_data): hostnames = [] for hname in xelement: if hname.tag == 'hostname': - hostnames.append(hname.get('name')) + name = hname.get('name') + if name not in hostnames: + hostnames.append(name) return hostnames @classmethod diff --git a/libnmap/plugins/csv.py b/libnmap/plugins/csv.py new file mode 100644 index 0000000..774c29a --- /dev/null +++ b/libnmap/plugins/csv.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +import csv +from datetime import datetime +from libnmap.objects import NmapService +from libnmap.plugins.backendplugin import NmapBackendPlugin + +class NmapCsvPlugin(NmapBackendPlugin): + """ + CSV plugin class for libnmap + All subclass MUST at least implement the following methods + """ + def __init__(self): + NmapBackendPlugin.__init__(self) + self.dict = {} + + def insert(self, report, doc_type=None): + """ + insert NmapReport in the backend + :param NmapReport: + :return: str the filenmae of the object in the backend for + future usage + or None + """ + if doc_type is None: + doc_type = 'NmapReport' + if report.is_from_file: + id = report.filename + else: + rep_date = datetime.fromtimestamp(int(report.started)) + id = "nmap-{0}".format(rep_date.strftime('%Y-%m-%d_%H-%M')) + with open(str(id) + '.csv', 'w', newline='') as csvfile: + fieldnames = ['address', 'hostnames', 'status'] + fieldnames = fieldnames + report.services + # servicesset = set() + # serviceformat = '{0.id}/{0.service}' + # servicesort = lambda p: p[p.find('.'):p.find('/')] + # for host in report.hosts: + # for service in host.services: + # servicesset.add(serviceformat.format(service)) + # servicefilenames = sorted(list(servicesset), + # key = servicesort) + # fieldnames = fieldnames + servicefilenames + csvwriter = csv.DictWriter(csvfile, fieldnames=fieldnames) + csvwriter.writeheader() + for host in report.hosts: + # data = {key: value for key, value in host.get_dict().items() + # if key in fieldnames} + # for service in host.services: + # data[serviceformat.format(service)] = service.state + csvwriter.writerow(host.get_dict()) + return id + + + def delete(self, id): + """ + delete NmapReport if the backend + :param id: str, filenmae + """ + raise NotImplementedError + + def get(self, id): + """ + retreive a NmapReport from the backend + :param id: str, filenmae + :return: NmapReport + """ + raise NotImplementedError + + def getall(self, filter): + """ + :return: collection of tuple (id,NmapReport) + :param filter: Nice to have implement a filter capability + """ + raise NotImplementedError diff --git a/libnmap/test/test_backend_plugin_factory.py b/libnmap/test/test_backend_plugin_factory.py index 2573c67..3fb15a7 100644 --- a/libnmap/test/test_backend_plugin_factory.py +++ b/libnmap/test/test_backend_plugin_factory.py @@ -48,21 +48,23 @@ def setUp(self): #build a list of NmapReport self.reportList = [] for testfile in self.flist: - fd = open(testfile['file'], 'r') - s = fd.read() - fd.close() - nrp = NmapParser.parse(s) - self.reportList.append(nrp) + # fd = open(testfile['file'], 'r') + # s = fd.read() + # fd.close() + # nrp = NmapParser.parse(s) + # self.reportList.append(nrp) + self.reportList.append(NmapParser.parse_fromfile(testfile['file'])) - self.urls = [{'plugin_name': "mongodb"}, - #{'plugin_name':'sql','url':'sqlite://','echo':'debug'}, - {'plugin_name': 'sql', - 'url': 'sqlite:////tmp/reportdb.sql', - 'echo': False}, - {'plugin_name': 'sql', - #'url': 'mysql+mysqldb://root@localhost/poulet', (mySQL-Python not supporting python3) - 'url': 'mysql+pymysql://root@localhost/poulet', - 'echo': False}, + self.urls = [ {'plugin_name': "csv"}, + # {'plugin_name': "mongodb"}, + # #{'plugin_name':'sql','url':'sqlite://','echo':'debug'}, + # {'plugin_name': 'sql', + # 'url': 'sqlite:////tmp/reportdb.sql', + # 'echo': False}, + # {'plugin_name': 'sql', + # #'url': 'mysql+mysqldb://root@localhost/poulet', (mySQL-Python not supporting python3) + # 'url': 'mysql+pymysql://root@localhost/poulet', + # 'echo': False}, #Walrus ###{'plugin_name': 's3', ### 'aws_access_key_id': 'UU72FLVJCAYRATLXI70YH',