From e8048a3cf0bdd1035dd9351ce60e16ac7a7fcc02 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sun, 3 Jan 2021 15:28:40 +0100 Subject: [PATCH 001/243] First 2021 dev commit --- lib/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index ed84999597c..ab1c823ee42 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5" +VERSION = "1.5.1.2" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) From f5b6b0eb4b8b64a7223cf7c6636716fc222ed9dc Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sun, 3 Jan 2021 16:56:35 +0100 Subject: [PATCH 002/243] Fixes #4504 --- lib/core/settings.py | 2 +- plugins/dbms/mysql/takeover.py | 2 +- plugins/dbms/postgresql/takeover.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index ab1c823ee42..b19971d3bf0 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.2" +VERSION = "1.5.1.3" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/plugins/dbms/mysql/takeover.py b/plugins/dbms/mysql/takeover.py index 12c73e9013e..23c391302a1 100644 --- a/plugins/dbms/mysql/takeover.py +++ b/plugins/dbms/mysql/takeover.py @@ -38,7 +38,7 @@ def udfSetRemotePath(self): banVer = kb.bannerFp["dbmsVersion"] - if distutils.version.LooseVersion(banVer) >= distutils.version.LooseVersion("5.0.67"): + if banVer and distutils.version.LooseVersion(banVer) >= distutils.version.LooseVersion("5.0.67"): if self.__plugindir is None: logger.info("retrieving MySQL plugin directory absolute path") self.__plugindir = unArrayizeValue(inject.getValue("SELECT @@plugin_dir")) diff --git a/plugins/dbms/postgresql/takeover.py b/plugins/dbms/postgresql/takeover.py index 946fe5e1ce4..e1f6241a3eb 100644 --- a/plugins/dbms/postgresql/takeover.py +++ b/plugins/dbms/postgresql/takeover.py @@ -51,7 +51,10 @@ def udfSetLocalPaths(self): banVer = kb.bannerFp["dbmsVersion"] - if distutils.version.LooseVersion(banVer) >= distutils.version.LooseVersion("10"): + if not banVer: + errMsg = "unsupported feature on unknown version of PostgreSQL" + raise SqlmapUnsupportedFeatureException(errMsg) + elif distutils.version.LooseVersion(banVer) >= distutils.version.LooseVersion("10"): majorVer = banVer.split('.')[0] elif distutils.version.LooseVersion(banVer) >= distutils.version.LooseVersion("8.2") and '.' in banVer: majorVer = '.'.join(banVer.split('.')[:2]) From 0c3d8c9bd177ab6d2bd41878b7180e7ef828fdfa Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 4 Jan 2021 13:25:29 +0100 Subject: [PATCH 003/243] Fixes #4505 --- lib/core/settings.py | 2 +- lib/techniques/union/use.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index b19971d3bf0..7b347c497a2 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.3" +VERSION = "1.5.1.4" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index 2f2069c2a90..41e057a208e 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -85,8 +85,9 @@ def _oneShotUnionUse(expression, unpack=True, limited=False): query = agent.forgeUnionQuery(injExpression, vector[0], vector[1], vector[2], vector[3], vector[4], vector[5], vector[6], None, limited) where = PAYLOAD.WHERE.NEGATIVE if conf.limitStart or conf.limitStop else vector[6] else: + injExpression = unescaper.escape(expression) where = vector[6] - query = agent.forgeUnionQuery(expression, vector[0], vector[1], vector[2], vector[3], vector[4], vector[5], vector[6], None, False) + query = agent.forgeUnionQuery(injExpression, vector[0], vector[1], vector[2], vector[3], vector[4], vector[5], vector[6], None, False) payload = agent.payload(newValue=query, where=where) From 091678b9d482eb7c5d3708aae5722eb2016b76c3 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 4 Jan 2021 13:45:38 +0100 Subject: [PATCH 004/243] Some fine tuning (#4505 - in case of --no-escape) --- lib/core/settings.py | 4 ++-- lib/techniques/union/use.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 7b347c497a2..43acc66b74e 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.4" +VERSION = "1.5.1.5" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -490,7 +490,7 @@ REFLECTED_BORDER_REGEX = r"[^A-Za-z]+" # Regular expression used for replacing non-alphanum characters -REFLECTED_REPLACEMENT_REGEX = r"[^\n]{1,100}" +REFLECTED_REPLACEMENT_REGEX = r"[^\n]{1,168}" # Maximum time (in seconds) spent per reflective value(s) replacement REFLECTED_REPLACEMENT_TIMEOUT = 3 diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index 41e057a208e..188861be658 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -98,7 +98,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False): if kb.jsonAggMode: if Backend.isDbms(DBMS.MSSQL): - output = extractRegexResult(r"%s(?P.*)%s" % (kb.chars.start, kb.chars.stop), page or "") + output = extractRegexResult(r"%s(?P.*)%s" % (kb.chars.start, kb.chars.stop), removeReflectiveValues(page or "", payload)) if output: try: retVal = "" @@ -110,11 +110,11 @@ def _oneShotUnionUse(expression, unpack=True, limited=False): else: retVal = getUnicode(retVal) elif Backend.isDbms(DBMS.PGSQL): - output = extractRegexResult(r"(?P%s.*%s)" % (kb.chars.start, kb.chars.stop), page or "") + output = extractRegexResult(r"(?P%s.*%s)" % (kb.chars.start, kb.chars.stop), removeReflectiveValues(page or "", payload)) if output: retVal = output else: - output = extractRegexResult(r"%s(?P.*?)%s" % (kb.chars.start, kb.chars.stop), page or "") + output = extractRegexResult(r"%s(?P.*?)%s" % (kb.chars.start, kb.chars.stop), removeReflectiveValues(page or "", payload)) if output: try: retVal = "" From b36d8248a44af2e5d241aba4517c82d183c2793b Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 4 Jan 2021 13:51:51 +0100 Subject: [PATCH 005/243] More fine-tuning for #4505 (in case of --no-escape) --- lib/core/dicts.py | 1 + lib/core/settings.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/core/dicts.py b/lib/core/dicts.py index 7d6d0f04dfe..29389b33348 100644 --- a/lib/core/dicts.py +++ b/lib/core/dicts.py @@ -412,6 +412,7 @@ HTML_ENTITIES = { "quot": 34, "amp": 38, + "apos": 39, "lt": 60, "gt": 62, "nbsp": 160, diff --git a/lib/core/settings.py b/lib/core/settings.py index 43acc66b74e..35479c61ee3 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.5" +VERSION = "1.5.1.6" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) From 91045aab605028d9f9aad99b7b6b3f2b4502c23d Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 5 Jan 2021 13:36:04 +0100 Subject: [PATCH 006/243] Fixes #4510 --- lib/core/common.py | 2 +- lib/core/settings.py | 2 +- lib/parse/banner.py | 4 ++-- lib/parse/handler.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index a62107f2546..fe7ba906bc4 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -1272,7 +1272,7 @@ def sanitizeStr(value): True """ - return getUnicode(value).replace("\n", " ").replace("\r", "") + return getUnicode(value).replace("\n", " ").replace("\r", "") if value else value def getHeader(headers, key): """ diff --git a/lib/core/settings.py b/lib/core/settings.py index 35479c61ee3..fec293570ef 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.6" +VERSION = "1.5.1.7" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/parse/banner.py b/lib/parse/banner.py index 10cba0c8d52..6c5b8fe5041 100644 --- a/lib/parse/banner.py +++ b/lib/parse/banner.py @@ -26,7 +26,7 @@ class MSSQLBannerHandler(ContentHandler): def __init__(self, banner, info): ContentHandler.__init__(self) - self._banner = sanitizeStr(banner) + self._banner = sanitizeStr(banner or "") self._inVersion = False self._inServicePack = False self._release = None @@ -62,7 +62,7 @@ def characters(self, content): def endElement(self, name): if name == "signature": for version in (self._version, self._versionAlt): - if version and re.search(r" %s[\.\ ]+" % re.escape(version), self._banner): + if version and self._banner and re.search(r" %s[\.\ ]+" % re.escape(version), self._banner): self._feedInfo("dbmsRelease", self._release) self._feedInfo("dbmsVersion", self._version) self._feedInfo("dbmsServicePack", self._servicePack) diff --git a/lib/parse/handler.py b/lib/parse/handler.py index d88b79a31d6..8976e31c4e6 100644 --- a/lib/parse/handler.py +++ b/lib/parse/handler.py @@ -20,7 +20,7 @@ class FingerprintHandler(ContentHandler): def __init__(self, banner, info): ContentHandler.__init__(self) - self._banner = sanitizeStr(banner) + self._banner = sanitizeStr(banner or "") self._regexp = None self._match = None self._dbmsVersion = None @@ -47,7 +47,7 @@ def startElement(self, name, attrs): self._regexp = sanitizeStr(attrs.get("value")) _ = re.match(r"\A[A-Za-z0-9]+", self._regexp) # minor trick avoiding compiling of large amount of regexes - if _ and _.group(0).lower() in self._banner.lower() or not _: + if _ and self._banner and _.group(0).lower() in self._banner.lower() or not _: self._match = re.search(self._regexp, self._banner, re.I | re.M) else: self._match = None From c4e6c3e8548e79a5dbb762427e467d64cbd369ff Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 5 Jan 2021 14:50:54 +0100 Subject: [PATCH 007/243] Fixes #4513 --- lib/core/common.py | 4 +++- lib/core/convert.py | 2 ++ lib/core/settings.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index fe7ba906bc4..3f643e99298 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -1270,9 +1270,11 @@ def sanitizeStr(value): >>> sanitizeStr('foo\\n\\rbar') == 'foo bar' True + >>> sanitizeStr(None) == 'None' + True """ - return getUnicode(value).replace("\n", " ").replace("\r", "") if value else value + return getUnicode(value).replace("\n", " ").replace("\r", "") def getHeader(headers, key): """ diff --git a/lib/core/convert.py b/lib/core/convert.py index ec338eb4b36..d48bdfee8db 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -330,6 +330,8 @@ def getUnicode(value, encoding=None, noneToNull=False): True >>> getUnicode(1) == u'1' True + >>> getUnicode(None) == 'None' + True """ if noneToNull and value is None: diff --git a/lib/core/settings.py b/lib/core/settings.py index fec293570ef..a2235b6f50b 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.7" +VERSION = "1.5.1.8" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) From bd59dd4a0fd61e9bccd7cba37f80cb42ffcde96b Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 6 Jan 2021 15:21:33 +0100 Subject: [PATCH 008/243] Trivial update --- lib/core/settings.py | 2 +- lib/request/connect.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index a2235b6f50b..460895273db 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.8" +VERSION = "1.5.1.9" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/connect.py b/lib/request/connect.py index d32847c632d..18beb1902c4 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -182,7 +182,7 @@ def _retryProxy(**kwargs): warnMsg += "you could successfully use " warnMsg += "switch '--tor' " if IS_WIN: - warnMsg += "(e.g. 'https://www.torproject.org/download/download.html.en')" + warnMsg += "(e.g. 'https://www.torproject.org/download/')" else: warnMsg += "(e.g. 'https://help.ubuntu.com/community/Tor')" else: From 9af2e68b7efce9c1cff8b2f79586d55c44af21ba Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 6 Jan 2021 15:39:51 +0100 Subject: [PATCH 009/243] Update for #4525 --- lib/core/settings.py | 2 +- lib/request/connect.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 460895273db..a32ca3e8fa9 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.9" +VERSION = "1.5.1.10" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/connect.py b/lib/request/connect.py index 18beb1902c4..e228b36673c 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -6,6 +6,7 @@ """ import binascii +import inspect import logging import os import random @@ -13,6 +14,7 @@ import socket import string import struct +import sys import time import traceback @@ -145,6 +147,10 @@ class Connect(object): @staticmethod def _getPageProxy(**kwargs): + if (len(inspect.stack()) > sys.getrecursionlimit() // 2): # Note: https://github.com/sqlmapproject/sqlmap/issues/4525 + warnMsg = "unable to connect to the target URL" + raise SqlmapConnectionException(warnMsg) + try: return Connect.getPage(**kwargs) except RuntimeError: From 779b352f6b710504598abe6d032f2b157a3577b9 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 6 Jan 2021 15:47:50 +0100 Subject: [PATCH 010/243] Minor cosmetics --- lib/core/settings.py | 2 +- lib/request/connect.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index a32ca3e8fa9..f7c13078778 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.10" +VERSION = "1.5.1.11" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/connect.py b/lib/request/connect.py index e228b36673c..66be54d27ec 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -193,11 +193,18 @@ def _retryProxy(**kwargs): warnMsg += "(e.g. 'https://help.ubuntu.com/community/Tor')" else: warnMsg = "if the problem persists please check that the provided " - warnMsg += "target URL is reachable. In case that it is, " - warnMsg += "you can try to rerun with " + warnMsg += "target URL is reachable" + + items = [] if not conf.randomAgent: - warnMsg += "switch '--random-agent' and/or " - warnMsg += "proxy switches ('--ignore-proxy', '--proxy',...)" + items.append("switch '--random-agent'") + if not any((conf.proxy, conf.proxyFile, conf.tor)): + items.append("proxy switches ('--proxy', '--proxy-file'...)") + if items: + warnMsg += ". In case that it is, " + warnMsg += "you can try to rerun with " + warnMsg += " and/or ".join(items) + singleTimeWarnMessage(warnMsg) elif conf.threads > 1: From f84ec1072be73bde88ebebab2fb8eef612ce9d7c Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 6 Jan 2021 16:09:40 +0100 Subject: [PATCH 011/243] Patch related to #4527 (drei) --- lib/core/settings.py | 2 +- lib/request/basic.py | 2 +- lib/request/connect.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index f7c13078778..75b13d7ff08 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.11" +VERSION = "1.5.1.12" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/basic.py b/lib/request/basic.py index 1e05a8f2138..db10990c347 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -303,7 +303,7 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True): page = data.read() except Exception as ex: - if " Date: Thu, 7 Jan 2021 11:44:00 +0100 Subject: [PATCH 012/243] First patch for #4530 --- lib/core/settings.py | 4 +++- lib/request/basic.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 75b13d7ff08..6c09af2dfce 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.12" +VERSION = "1.5.1.13" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -757,6 +757,8 @@ # For preventing MemoryError exceptions (caused when using large sequences in difflib.SequenceMatcher) MAX_DIFFLIB_SEQUENCE_LENGTH = 10 * 1024 * 1024 +MAX_CHAR_HEURISTICS_SIZE = 10000 + # Maximum (multi-threaded) length of entry in bisection algorithm MAX_BISECTION_LENGTH = 50 * 1024 * 1024 diff --git a/lib/request/basic.py b/lib/request/basic.py index db10990c347..ead947cadeb 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -44,6 +44,7 @@ from lib.core.settings import DEFAULT_COOKIE_DELIMITER from lib.core.settings import EVENTVALIDATION_REGEX from lib.core.settings import IDENTYWAF_PARSE_LIMIT +from lib.core.settings import MAX_CHAR_HEURISTICS_SIZE from lib.core.settings import MAX_CONNECTION_TOTAL_SIZE from lib.core.settings import META_CHARSET_REGEX from lib.core.settings import PARSE_HEADERS_LIMIT @@ -258,7 +259,7 @@ def getHeuristicCharEncoding(page): """ key = hash(page) - retVal = kb.cache.encoding.get(key) or detect(page)["encoding"] + retVal = kb.cache.encoding.get(key) or detect(page[:MAX_CHAR_HEURISTICS_SIZE])["encoding"] kb.cache.encoding[key] = retVal if retVal and retVal.lower().replace('-', "") == UNICODE_ENCODING.lower().replace('-', ""): From 0d3889730d8bfa3edd1683aa57c3c6b9799c176d Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 7 Jan 2021 11:56:49 +0100 Subject: [PATCH 013/243] Another patch for #4530 --- lib/core/settings.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 6c09af2dfce..b10a4e0913c 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.13" +VERSION = "1.5.1.14" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -397,14 +397,14 @@ # Regular expressions used for parsing error messages (--parse-errors) ERROR_PARSING_REGEXES = ( r"\[Microsoft\]\[ODBC SQL Server Driver\]\[SQL Server\](?P[^<]+)", - r"[^<]*(fatal|error|warning|exception)[^<]*:?\s*(?P[^<]+)", - r"(?m)^\s*(fatal|error|warning|exception):?\s*(?P[^\n]+?)$", + r"[^<]{0,100}(fatal|error|warning|exception)[^<]*:?\s*(?P[^<]+)", + r"(?m)^\s{0,100}(fatal|error|warning|exception):?\s*(?P[^\n]+?)$", r"(sql|dbc)[^>'\"]{0,32}(fatal|error|warning|exception)()?:\s*(?P[^<>]+)", - r"(?P[^\n>]*SQL Syntax[^\n<]+)", + r"(?P[^\n>]{0,100}SQL Syntax[^\n<]+)", r"(?s)
  • Error Type:
    (?P.+?)
    • ", r"CDbCommand (?P[^<>\n]*SQL[^<>\n]+)", r"error '[0-9a-f]{8}'((<[^>]+>)|\s)+(?P[^<>]+)", - r"\[[^\n\]]+(ODBC|JDBC)[^\n\]]+\](\[[^\]]+\])?(?P[^\n]+(in query expression|\(SQL| at /[^ ]+pdo)[^\n<]+)", + r"\[[^\n\]]{1,100}(ODBC|JDBC)[^\n\]]+\](\[[^\]]+\])?(?P[^\n]+(in query expression|\(SQL| at /[^ ]+pdo)[^\n<]+)", r"(?Pquery error: SELECT[^<>]+)" ) @@ -757,6 +757,7 @@ # For preventing MemoryError exceptions (caused when using large sequences in difflib.SequenceMatcher) MAX_DIFFLIB_SEQUENCE_LENGTH = 10 * 1024 * 1024 +# Maximum size used per page content in getHeuristicCharEncoding() MAX_CHAR_HEURISTICS_SIZE = 10000 # Maximum (multi-threaded) length of entry in bisection algorithm From ebd2a940cb39b8400788fd563bf94171d8523719 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 7 Jan 2021 13:52:38 +0100 Subject: [PATCH 014/243] Patching profile capabilities --- doc/THANKS.md | 5 - doc/THIRD-PARTY.md | 4 - lib/core/profiling.py | 71 +- lib/core/settings.py | 2 +- sqlmap.py | 2 +- thirdparty/gprof2dot/__init__.py | 19 - thirdparty/gprof2dot/gprof2dot.py | 2619 ----------------------------- thirdparty/xdot/__init__.py | 19 - thirdparty/xdot/xdot.py | 2432 --------------------------- 9 files changed, 4 insertions(+), 5169 deletions(-) delete mode 100644 thirdparty/gprof2dot/__init__.py delete mode 100644 thirdparty/gprof2dot/gprof2dot.py delete mode 100644 thirdparty/xdot/__init__.py delete mode 100644 thirdparty/xdot/xdot.py diff --git a/doc/THANKS.md b/doc/THANKS.md index 0fe779b7682..619ac891f55 100644 --- a/doc/THANKS.md +++ b/doc/THANKS.md @@ -151,11 +151,6 @@ Giorgio Fedon, Kasper Fons, * for reporting several bugs -Jose Fonseca, -* for his Gprof2Dot utility for converting profiler output to dot graph(s) and for his XDot utility to render nicely dot graph(s), both included in sqlmap tree inside extra folder. These libraries are used for sqlmap development purposes only - http://code.google.com/p/jrfonseca/wiki/Gprof2Dot - http://code.google.com/p/jrfonseca/wiki/XDot - Alan Franzoni, * for helping out with Python subprocess library diff --git a/doc/THIRD-PARTY.md b/doc/THIRD-PARTY.md index 04d558f613c..76d9e8fe350 100644 --- a/doc/THIRD-PARTY.md +++ b/doc/THIRD-PARTY.md @@ -48,14 +48,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * The `Chardet` library located under `thirdparty/chardet/`. Copyright (C) 2008, Mark Pilgrim. -* The `Gprof2dot` library located under `thirdparty/gprof2dot/`. - Copyright (C) 2008-2009, Jose Fonseca. * The `KeepAlive` library located under `thirdparty/keepalive/`. Copyright (C) 2002-2003, Michael D. Stenner. * The `MultipartPost` library located under `thirdparty/multipart/`. Copyright (C) 2006, Will Holcomb. -* The `XDot` library located under `thirdparty/xdot/` - Copyright (C) 2008, Jose Fonseca. * The `icmpsh` tool located under `extra/icmpsh/`. Copyright (C) 2010, Nico Leidecker, Bernardo Damele. diff --git a/lib/core/profiling.py b/lib/core/profiling.py index cb29d5d90bf..8d4f643a011 100644 --- a/lib/core/profiling.py +++ b/lib/core/profiling.py @@ -14,86 +14,19 @@ from lib.core.data import paths from lib.core.settings import UNICODE_ENCODING -def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None): +def profile(profileOutputFile=None): """ This will run the program and present profiling data in a nice looking graph """ - try: - __import__("gobject") - from thirdparty.gprof2dot import gprof2dot - from thirdparty.xdot import xdot - import gtk - import pydot - except ImportError as ex: - errMsg = "profiling requires third-party libraries ('%s') " % getSafeExString(ex) - errMsg += "(Hint: 'sudo apt install python-pydot python-pyparsing python-profiler graphviz')" - logger.error(errMsg) - - return - if profileOutputFile is None: profileOutputFile = os.path.join(paths.SQLMAP_OUTPUT_PATH, "sqlmap_profile.raw") - if dotOutputFile is None: - dotOutputFile = os.path.join(paths.SQLMAP_OUTPUT_PATH, "sqlmap_profile.dot") - - if imageOutputFile is None: - imageOutputFile = os.path.join(paths.SQLMAP_OUTPUT_PATH, "sqlmap_profile.png") - if os.path.exists(profileOutputFile): os.remove(profileOutputFile) - if os.path.exists(dotOutputFile): - os.remove(dotOutputFile) - - if os.path.exists(imageOutputFile): - os.remove(imageOutputFile) - - infoMsg = "profiling the execution into file '%s'" % profileOutputFile - logger.info(infoMsg) - # Start sqlmap main function and generate a raw profile file cProfile.run("start()", profileOutputFile) - infoMsg = "converting profile data into a dot file '%s'" % dotOutputFile + infoMsg = "execution profiled and stored into file '%s' (e.g. 'gprof2dot -f pstats %s | dot -Tpng -o /tmp/sqlmap_profile.png')" % (profileOutputFile, profileOutputFile) logger.info(infoMsg) - - # Create dot file by using extra/gprof2dot/gprof2dot.py - # http://code.google.com/p/jrfonseca/wiki/Gprof2Dot - dotFilePointer = codecs.open(dotOutputFile, 'wt', UNICODE_ENCODING) - parser = gprof2dot.PstatsParser(profileOutputFile) - profile = parser.parse() - profile.prune(0.5 / 100.0, 0.1 / 100.0) - dot = gprof2dot.DotWriter(dotFilePointer) - dot.graph(profile, gprof2dot.TEMPERATURE_COLORMAP) - dotFilePointer.close() - - infoMsg = "converting dot file into a graph image '%s'" % imageOutputFile - logger.info(infoMsg) - - # Create graph image (png) by using pydot (python-pydot) - # http://code.google.com/p/pydot/ - pydotGraph = pydot.graph_from_dot_file(dotOutputFile) - - # Reference: http://stackoverflow.com/questions/38176472/graph-write-pdfiris-pdf-attributeerror-list-object-has-no-attribute-writ - if isinstance(pydotGraph, list): - pydotGraph = pydotGraph[0] - - try: - pydotGraph.write_png(imageOutputFile) - except OSError: - errMsg = "profiling requires graphviz installed " - errMsg += "(Hint: 'sudo apt install graphviz')" - logger.error(errMsg) - else: - infoMsg = "displaying interactive graph with xdot library" - logger.info(infoMsg) - - # Display interactive Graphviz dot file by using extra/xdot/xdot.py - # http://code.google.com/p/jrfonseca/wiki/XDot - win = xdot.DotWindow() - win.connect('destroy', gtk.main_quit) - win.set_filter("dot") - win.open_file(dotOutputFile) - gtk.main() diff --git a/lib/core/settings.py b/lib/core/settings.py index b10a4e0913c..9f076ec8bed 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.14" +VERSION = "1.5.1.15" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/sqlmap.py b/sqlmap.py index 4c695dae7a8..18e1d58acc7 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -182,7 +182,7 @@ def main(): fuzzTest() else: from lib.controller.controller import start - if conf.profile and six.PY2: + if conf.profile: from lib.core.profiling import profile globals()["start"] = start profile() diff --git a/thirdparty/gprof2dot/__init__.py b/thirdparty/gprof2dot/__init__.py deleted file mode 100644 index 5ef1f2dc104..00000000000 --- a/thirdparty/gprof2dot/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python2 -# -# Copyright 2008-2009 Jose Fonseca -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . -# - -pass diff --git a/thirdparty/gprof2dot/gprof2dot.py b/thirdparty/gprof2dot/gprof2dot.py deleted file mode 100644 index 9eaaadeb23e..00000000000 --- a/thirdparty/gprof2dot/gprof2dot.py +++ /dev/null @@ -1,2619 +0,0 @@ -#!/usr/bin/env python2 -# -# Copyright 2008-2009 Jose Fonseca -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . -# - -"""Generate a dot graph from the output of several profilers.""" - -__author__ = "Jose Fonseca" - -__version__ = "1.0" - - -import sys -import math -import os.path -import re -import textwrap -import optparse -import sys -import xml.parsers.expat - -if sys.version_info >= (3, 0): - xrange = range - -try: - # Debugging helper module - import debug -except ImportError: - pass - - -def times(x): - return u"%u\xd7" % (x,) - -def percentage(p): - return "%.02f%%" % (p*100.0,) - -def add(a, b): - return a + b - -def equal(a, b): - if a == b: - return a - else: - return None - -def fail(a, b): - assert False - - -tol = 2 ** -23 - -def ratio(numerator, denominator): - try: - ratio = float(numerator)/float(denominator) - except ZeroDivisionError: - # 0/0 is undefined, but 1.0 yields more useful results - return 1.0 - if ratio < 0.0: - if ratio < -tol: - sys.stderr.write('warning: negative ratio (%s/%s)\n' % (numerator, denominator)) - return 0.0 - if ratio > 1.0: - if ratio > 1.0 + tol: - sys.stderr.write('warning: ratio greater than one (%s/%s)\n' % (numerator, denominator)) - return 1.0 - return ratio - - -class UndefinedEvent(Exception): - """Raised when attempting to get an event which is undefined.""" - - def __init__(self, event): - Exception.__init__(self) - self.event = event - - def __str__(self): - return 'unspecified event %s' % self.event.name - - -class Event(object): - """Describe a kind of event, and its basic operations.""" - - def __init__(self, name, null, aggregator, formatter = str): - self.name = name - self._null = null - self._aggregator = aggregator - self._formatter = formatter - - def __eq__(self, other): - return self is other - - def __hash__(self): - return id(self) - - def null(self): - return self._null - - def aggregate(self, val1, val2): - """Aggregate two event values.""" - assert val1 is not None - assert val2 is not None - return self._aggregator(val1, val2) - - def format(self, val): - """Format an event value.""" - assert val is not None - return self._formatter(val) - - -CALLS = Event("Calls", 0, add, times) -SAMPLES = Event("Samples", 0, add) -SAMPLES2 = Event("Samples", 0, add) - -TIME = Event("Time", 0.0, add, lambda x: '(' + str(x) + ')') -TIME_RATIO = Event("Time ratio", 0.0, add, lambda x: '(' + percentage(x) + ')') -TOTAL_TIME = Event("Total time", 0.0, fail) -TOTAL_TIME_RATIO = Event("Total time ratio", 0.0, fail, percentage) - - -class Object(object): - """Base class for all objects in profile which can store events.""" - - def __init__(self, events=None): - if events is None: - self.events = {} - else: - self.events = events - - def __hash__(self): - return id(self) - - def __eq__(self, other): - return self is other - - def __contains__(self, event): - return event in self.events - - def __getitem__(self, event): - try: - return self.events[event] - except KeyError: - raise UndefinedEvent(event) - - def __setitem__(self, event, value): - if value is None: - if event in self.events: - del self.events[event] - else: - self.events[event] = value - - -class Call(Object): - """A call between functions. - - There should be at most one call object for every pair of functions. - """ - - def __init__(self, callee_id): - Object.__init__(self) - self.callee_id = callee_id - self.ratio = None - self.weight = None - - -class Function(Object): - """A function.""" - - def __init__(self, id, name): - Object.__init__(self) - self.id = id - self.name = name - self.module = None - self.process = None - self.calls = {} - self.called = None - self.weight = None - self.cycle = None - - def add_call(self, call): - if call.callee_id in self.calls: - sys.stderr.write('warning: overwriting call from function %s to %s\n' % (str(self.id), str(call.callee_id))) - self.calls[call.callee_id] = call - - # TODO: write utility functions - - def __repr__(self): - return self.name - - -class Cycle(Object): - """A cycle made from recursive function calls.""" - - def __init__(self): - Object.__init__(self) - # XXX: Do cycles need an id? - self.functions = set() - - def add_function(self, function): - assert function not in self.functions - self.functions.add(function) - # XXX: Aggregate events? - if function.cycle is not None: - for other in function.cycle.functions: - if function not in self.functions: - self.add_function(other) - function.cycle = self - - -class Profile(Object): - """The whole profile.""" - - def __init__(self): - Object.__init__(self) - self.functions = {} - self.cycles = [] - - def add_function(self, function): - if function.id in self.functions: - sys.stderr.write('warning: overwriting function %s (id %s)\n' % (function.name, str(function.id))) - self.functions[function.id] = function - - def add_cycle(self, cycle): - self.cycles.append(cycle) - - def validate(self): - """Validate the edges.""" - - for function in self.functions.itervalues(): - for callee_id in function.calls.keys(): - assert function.calls[callee_id].callee_id == callee_id - if callee_id not in self.functions: - sys.stderr.write('warning: call to undefined function %s from function %s\n' % (str(callee_id), function.name)) - del function.calls[callee_id] - - def find_cycles(self): - """Find cycles using Tarjan's strongly connected components algorithm.""" - - # Apply the Tarjan's algorithm successively until all functions are visited - visited = set() - for function in self.functions.itervalues(): - if function not in visited: - self._tarjan(function, 0, [], {}, {}, visited) - cycles = [] - for function in self.functions.itervalues(): - if function.cycle is not None and function.cycle not in cycles: - cycles.append(function.cycle) - self.cycles = cycles - if 0: - for cycle in cycles: - sys.stderr.write("Cycle:\n") - for member in cycle.functions: - sys.stderr.write("\tFunction %s\n" % member.name) - - def _tarjan(self, function, order, stack, orders, lowlinks, visited): - """Tarjan's strongly connected components algorithm. - - See also: - - http://en.wikipedia.org/wiki/Tarjan's_strongly_connected_components_algorithm - """ - - visited.add(function) - orders[function] = order - lowlinks[function] = order - order += 1 - pos = len(stack) - stack.append(function) - for call in function.calls.itervalues(): - callee = self.functions[call.callee_id] - # TODO: use a set to optimize lookup - if callee not in orders: - order = self._tarjan(callee, order, stack, orders, lowlinks, visited) - lowlinks[function] = min(lowlinks[function], lowlinks[callee]) - elif callee in stack: - lowlinks[function] = min(lowlinks[function], orders[callee]) - if lowlinks[function] == orders[function]: - # Strongly connected component found - members = stack[pos:] - del stack[pos:] - if len(members) > 1: - cycle = Cycle() - for member in members: - cycle.add_function(member) - return order - - def call_ratios(self, event): - # Aggregate for incoming calls - cycle_totals = {} - for cycle in self.cycles: - cycle_totals[cycle] = 0.0 - function_totals = {} - for function in self.functions.itervalues(): - function_totals[function] = 0.0 - for function in self.functions.itervalues(): - for call in function.calls.itervalues(): - if call.callee_id != function.id: - callee = self.functions[call.callee_id] - function_totals[callee] += call[event] - if callee.cycle is not None and callee.cycle is not function.cycle: - cycle_totals[callee.cycle] += call[event] - - # Compute the ratios - for function in self.functions.itervalues(): - for call in function.calls.itervalues(): - assert call.ratio is None - if call.callee_id != function.id: - callee = self.functions[call.callee_id] - if callee.cycle is not None and callee.cycle is not function.cycle: - total = cycle_totals[callee.cycle] - else: - total = function_totals[callee] - call.ratio = ratio(call[event], total) - - def integrate(self, outevent, inevent): - """Propagate function time ratio allong the function calls. - - Must be called after finding the cycles. - - See also: - - http://citeseer.ist.psu.edu/graham82gprof.html - """ - - # Sanity checking - assert outevent not in self - for function in self.functions.itervalues(): - assert outevent not in function - assert inevent in function - for call in function.calls.itervalues(): - assert outevent not in call - if call.callee_id != function.id: - assert call.ratio is not None - - # Aggregate the input for each cycle - for cycle in self.cycles: - total = inevent.null() - for function in self.functions.itervalues(): - total = inevent.aggregate(total, function[inevent]) - self[inevent] = total - - # Integrate along the edges - total = inevent.null() - for function in self.functions.itervalues(): - total = inevent.aggregate(total, function[inevent]) - self._integrate_function(function, outevent, inevent) - self[outevent] = total - - def _integrate_function(self, function, outevent, inevent): - if function.cycle is not None: - return self._integrate_cycle(function.cycle, outevent, inevent) - else: - if outevent not in function: - total = function[inevent] - for call in function.calls.itervalues(): - if call.callee_id != function.id: - total += self._integrate_call(call, outevent, inevent) - function[outevent] = total - return function[outevent] - - def _integrate_call(self, call, outevent, inevent): - assert outevent not in call - assert call.ratio is not None - callee = self.functions[call.callee_id] - subtotal = call.ratio *self._integrate_function(callee, outevent, inevent) - call[outevent] = subtotal - return subtotal - - def _integrate_cycle(self, cycle, outevent, inevent): - if outevent not in cycle: - - # Compute the outevent for the whole cycle - total = inevent.null() - for member in cycle.functions: - subtotal = member[inevent] - for call in member.calls.itervalues(): - callee = self.functions[call.callee_id] - if callee.cycle is not cycle: - subtotal += self._integrate_call(call, outevent, inevent) - total += subtotal - cycle[outevent] = total - - # Compute the time propagated to callers of this cycle - callees = {} - for function in self.functions.itervalues(): - if function.cycle is not cycle: - for call in function.calls.itervalues(): - callee = self.functions[call.callee_id] - if callee.cycle is cycle: - try: - callees[callee] += call.ratio - except KeyError: - callees[callee] = call.ratio - - for member in cycle.functions: - member[outevent] = outevent.null() - - for callee, call_ratio in callees.iteritems(): - ranks = {} - call_ratios = {} - partials = {} - self._rank_cycle_function(cycle, callee, 0, ranks) - self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set()) - partial = self._integrate_cycle_function(cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent) - assert partial == max(partials.values()) - assert not total or abs(1.0 - partial/(call_ratio*total)) <= 0.001 - - return cycle[outevent] - - def _rank_cycle_function(self, cycle, function, rank, ranks): - if function not in ranks or ranks[function] > rank: - ranks[function] = rank - for call in function.calls.itervalues(): - if call.callee_id != function.id: - callee = self.functions[call.callee_id] - if callee.cycle is cycle: - self._rank_cycle_function(cycle, callee, rank + 1, ranks) - - def _call_ratios_cycle(self, cycle, function, ranks, call_ratios, visited): - if function not in visited: - visited.add(function) - for call in function.calls.itervalues(): - if call.callee_id != function.id: - callee = self.functions[call.callee_id] - if callee.cycle is cycle: - if ranks[callee] > ranks[function]: - call_ratios[callee] = call_ratios.get(callee, 0.0) + call.ratio - self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited) - - def _integrate_cycle_function(self, cycle, function, partial_ratio, partials, ranks, call_ratios, outevent, inevent): - if function not in partials: - partial = partial_ratio*function[inevent] - for call in function.calls.itervalues(): - if call.callee_id != function.id: - callee = self.functions[call.callee_id] - if callee.cycle is not cycle: - assert outevent in call - partial += partial_ratio*call[outevent] - else: - if ranks[callee] > ranks[function]: - callee_partial = self._integrate_cycle_function(cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent) - call_ratio = ratio(call.ratio, call_ratios[callee]) - call_partial = call_ratio*callee_partial - try: - call[outevent] += call_partial - except UndefinedEvent: - call[outevent] = call_partial - partial += call_partial - partials[function] = partial - try: - function[outevent] += partial - except UndefinedEvent: - function[outevent] = partial - return partials[function] - - def aggregate(self, event): - """Aggregate an event for the whole profile.""" - - total = event.null() - for function in self.functions.itervalues(): - try: - total = event.aggregate(total, function[event]) - except UndefinedEvent: - return - self[event] = total - - def ratio(self, outevent, inevent): - assert outevent not in self - assert inevent in self - for function in self.functions.itervalues(): - assert outevent not in function - assert inevent in function - function[outevent] = ratio(function[inevent], self[inevent]) - for call in function.calls.itervalues(): - assert outevent not in call - if inevent in call: - call[outevent] = ratio(call[inevent], self[inevent]) - self[outevent] = 1.0 - - def prune(self, node_thres, edge_thres): - """Prune the profile""" - - # compute the prune ratios - for function in self.functions.itervalues(): - try: - function.weight = function[TOTAL_TIME_RATIO] - except UndefinedEvent: - pass - - for call in function.calls.itervalues(): - callee = self.functions[call.callee_id] - - if TOTAL_TIME_RATIO in call: - # handle exact cases first - call.weight = call[TOTAL_TIME_RATIO] - else: - try: - # make a safe estimate - call.weight = min(function[TOTAL_TIME_RATIO], callee[TOTAL_TIME_RATIO]) - except UndefinedEvent: - pass - - # prune the nodes - for function_id in self.functions.keys(): - function = self.functions[function_id] - if function.weight is not None: - if function.weight < node_thres: - del self.functions[function_id] - - # prune the egdes - for function in self.functions.itervalues(): - for callee_id in function.calls.keys(): - call = function.calls[callee_id] - if callee_id not in self.functions or call.weight is not None and call.weight < edge_thres: - del function.calls[callee_id] - - def dump(self): - for function in self.functions.itervalues(): - sys.stderr.write('Function %s:\n' % (function.name,)) - self._dump_events(function.events) - for call in function.calls.itervalues(): - callee = self.functions[call.callee_id] - sys.stderr.write(' Call %s:\n' % (callee.name,)) - self._dump_events(call.events) - for cycle in self.cycles: - sys.stderr.write('Cycle:\n') - self._dump_events(cycle.events) - for function in cycle.functions: - sys.stderr.write(' Function %s\n' % (function.name,)) - - def _dump_events(self, events): - for event, value in events.iteritems(): - sys.stderr.write(' %s: %s\n' % (event.name, event.format(value))) - - -class Struct: - """Masquerade a dictionary with a structure-like behavior.""" - - def __init__(self, attrs = None): - if attrs is None: - attrs = {} - self.__dict__['_attrs'] = attrs - - def __getattr__(self, name): - try: - return self._attrs[name] - except KeyError: - raise AttributeError(name) - - def __setattr__(self, name, value): - self._attrs[name] = value - - def __str__(self): - return str(self._attrs) - - def __repr__(self): - return repr(self._attrs) - - -class ParseError(Exception): - """Raised when parsing to signal mismatches.""" - - def __init__(self, msg, line): - self.msg = msg - # TODO: store more source line information - self.line = line - - def __str__(self): - return '%s: %r' % (self.msg, self.line) - - -class Parser: - """Parser interface.""" - - def __init__(self): - pass - - def parse(self): - raise NotImplementedError - - -class LineParser(Parser): - """Base class for parsers that read line-based formats.""" - - def __init__(self, file): - Parser.__init__(self) - self._file = file - self.__line = None - self.__eof = False - - def readline(self): - line = self._file.readline() - if not line: - self.__line = '' - self.__eof = True - self.__line = line.rstrip('\r\n') - - def lookahead(self): - assert self.__line is not None - return self.__line - - def consume(self): - assert self.__line is not None - line = self.__line - self.readline() - return line - - def eof(self): - assert self.__line is not None - return self.__eof - - -XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF = range(4) - - -class XmlToken: - - def __init__(self, type, name_or_data, attrs = None, line = None, column = None): - assert type in (XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF) - self.type = type - self.name_or_data = name_or_data - self.attrs = attrs - self.line = line - self.column = column - - def __str__(self): - if self.type == XML_ELEMENT_START: - return '<' + self.name_or_data + ' ...>' - if self.type == XML_ELEMENT_END: - return '' - if self.type == XML_CHARACTER_DATA: - return self.name_or_data - if self.type == XML_EOF: - return 'end of file' - assert 0 - - -class XmlTokenizer: - """Expat based XML tokenizer.""" - - def __init__(self, fp, skip_ws = True): - self.fp = fp - self.tokens = [] - self.index = 0 - self.final = False - self.skip_ws = skip_ws - - self.character_pos = 0, 0 - self.character_data = '' - - self.parser = xml.parsers.expat.ParserCreate() - self.parser.StartElementHandler = self.handle_element_start - self.parser.EndElementHandler = self.handle_element_end - self.parser.CharacterDataHandler = self.handle_character_data - - def handle_element_start(self, name, attributes): - self.finish_character_data() - line, column = self.pos() - token = XmlToken(XML_ELEMENT_START, name, attributes, line, column) - self.tokens.append(token) - - def handle_element_end(self, name): - self.finish_character_data() - line, column = self.pos() - token = XmlToken(XML_ELEMENT_END, name, None, line, column) - self.tokens.append(token) - - def handle_character_data(self, data): - if not self.character_data: - self.character_pos = self.pos() - self.character_data += data - - def finish_character_data(self): - if self.character_data: - if not self.skip_ws or not self.character_data.isspace(): - line, column = self.character_pos - token = XmlToken(XML_CHARACTER_DATA, self.character_data, None, line, column) - self.tokens.append(token) - self.character_data = '' - - def next(self): - size = 16*1024 - while self.index >= len(self.tokens) and not self.final: - self.tokens = [] - self.index = 0 - data = self.fp.read(size) - self.final = len(data) < size - try: - self.parser.Parse(data, self.final) - except xml.parsers.expat.ExpatError as e: - #if e.code == xml.parsers.expat.errors.XML_ERROR_NO_ELEMENTS: - if e.code == 3: - pass - else: - raise e - if self.index >= len(self.tokens): - line, column = self.pos() - token = XmlToken(XML_EOF, None, None, line, column) - else: - token = self.tokens[self.index] - self.index += 1 - return token - - def pos(self): - return self.parser.CurrentLineNumber, self.parser.CurrentColumnNumber - - -class XmlTokenMismatch(Exception): - - def __init__(self, expected, found): - self.expected = expected - self.found = found - - def __str__(self): - return '%u:%u: %s expected, %s found' % (self.found.line, self.found.column, str(self.expected), str(self.found)) - - -class XmlParser(Parser): - """Base XML document parser.""" - - def __init__(self, fp): - Parser.__init__(self) - self.tokenizer = XmlTokenizer(fp) - self.consume() - - def consume(self): - self.token = next(self.tokenizer) - - def match_element_start(self, name): - return self.token.type == XML_ELEMENT_START and self.token.name_or_data == name - - def match_element_end(self, name): - return self.token.type == XML_ELEMENT_END and self.token.name_or_data == name - - def element_start(self, name): - while self.token.type == XML_CHARACTER_DATA: - self.consume() - if self.token.type != XML_ELEMENT_START: - raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token) - if self.token.name_or_data != name: - raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token) - attrs = self.token.attrs - self.consume() - return attrs - - def element_end(self, name): - while self.token.type == XML_CHARACTER_DATA: - self.consume() - if self.token.type != XML_ELEMENT_END: - raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token) - if self.token.name_or_data != name: - raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token) - self.consume() - - def character_data(self, strip = True): - data = '' - while self.token.type == XML_CHARACTER_DATA: - data += self.token.name_or_data - self.consume() - if strip: - data = data.strip() - return data - - -class GprofParser(Parser): - """Parser for GNU gprof output. - - See also: - - Chapter "Interpreting gprof's Output" from the GNU gprof manual - http://sourceware.org/binutils/docs-2.18/gprof/Call-Graph.html#Call-Graph - - File "cg_print.c" from the GNU gprof source code - http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/src/gprof/cg_print.c?rev=1.12&cvsroot=src - """ - - def __init__(self, fp): - Parser.__init__(self) - self.fp = fp - self.functions = {} - self.cycles = {} - - def readline(self): - line = self.fp.readline() - if not line: - sys.stderr.write('error: unexpected end of file\n') - sys.exit(1) - line = line.rstrip('\r\n') - return line - - _int_re = re.compile(r'^\d+$') - _float_re = re.compile(r'^\d+\.\d+$') - - def translate(self, mo): - """Extract a structure from a match object, while translating the types in the process.""" - attrs = {} - groupdict = mo.groupdict() - for name, value in groupdict.iteritems(): - if value is None: - value = None - elif self._int_re.match(value): - value = int(value) - elif self._float_re.match(value): - value = float(value) - attrs[name] = (value) - return Struct(attrs) - - _cg_header_re = re.compile( - # original gprof header - r'^\s+called/total\s+parents\s*$|' + - r'^index\s+%time\s+self\s+descendents\s+called\+self\s+name\s+index\s*$|' + - r'^\s+called/total\s+children\s*$|' + - # GNU gprof header - r'^index\s+%\s+time\s+self\s+children\s+called\s+name\s*$' - ) - - _cg_ignore_re = re.compile( - # spontaneous - r'^\s+\s*$|' - # internal calls (such as "mcount") - r'^.*\((\d+)\)$' - ) - - _cg_primary_re = re.compile( - r'^\[(?P\d+)\]?' + - r'\s+(?P\d+\.\d+)' + - r'\s+(?P\d+\.\d+)' + - r'\s+(?P\d+\.\d+)' + - r'\s+(?:(?P\d+)(?:\+(?P\d+))?)?' + - r'\s+(?P\S.*?)' + - r'(?:\s+\d+)>)?' + - r'\s\[(\d+)\]$' - ) - - _cg_parent_re = re.compile( - r'^\s+(?P\d+\.\d+)?' + - r'\s+(?P\d+\.\d+)?' + - r'\s+(?P\d+)(?:/(?P\d+))?' + - r'\s+(?P\S.*?)' + - r'(?:\s+\d+)>)?' + - r'\s\[(?P\d+)\]$' - ) - - _cg_child_re = _cg_parent_re - - _cg_cycle_header_re = re.compile( - r'^\[(?P\d+)\]?' + - r'\s+(?P\d+\.\d+)' + - r'\s+(?P\d+\.\d+)' + - r'\s+(?P\d+\.\d+)' + - r'\s+(?:(?P\d+)(?:\+(?P\d+))?)?' + - r'\s+\d+)\sas\sa\swhole>' + - r'\s\[(\d+)\]$' - ) - - _cg_cycle_member_re = re.compile( - r'^\s+(?P\d+\.\d+)?' + - r'\s+(?P\d+\.\d+)?' + - r'\s+(?P\d+)(?:\+(?P\d+))?' + - r'\s+(?P\S.*?)' + - r'(?:\s+\d+)>)?' + - r'\s\[(?P\d+)\]$' - ) - - _cg_sep_re = re.compile(r'^--+$') - - def parse_function_entry(self, lines): - parents = [] - children = [] - - while True: - if not lines: - sys.stderr.write('warning: unexpected end of entry\n') - line = lines.pop(0) - if line.startswith('['): - break - - # read function parent line - mo = self._cg_parent_re.match(line) - if not mo: - if self._cg_ignore_re.match(line): - continue - sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line) - else: - parent = self.translate(mo) - parents.append(parent) - - # read primary line - mo = self._cg_primary_re.match(line) - if not mo: - sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line) - return - else: - function = self.translate(mo) - - while lines: - line = lines.pop(0) - - # read function subroutine line - mo = self._cg_child_re.match(line) - if not mo: - if self._cg_ignore_re.match(line): - continue - sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line) - else: - child = self.translate(mo) - children.append(child) - - function.parents = parents - function.children = children - - self.functions[function.index] = function - - def parse_cycle_entry(self, lines): - - # read cycle header line - line = lines[0] - mo = self._cg_cycle_header_re.match(line) - if not mo: - sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line) - return - cycle = self.translate(mo) - - # read cycle member lines - cycle.functions = [] - for line in lines[1:]: - mo = self._cg_cycle_member_re.match(line) - if not mo: - sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line) - continue - call = self.translate(mo) - cycle.functions.append(call) - - self.cycles[cycle.cycle] = cycle - - def parse_cg_entry(self, lines): - if lines[0].startswith("["): - self.parse_cycle_entry(lines) - else: - self.parse_function_entry(lines) - - def parse_cg(self): - """Parse the call graph.""" - - # skip call graph header - while not self._cg_header_re.match(self.readline()): - pass - line = self.readline() - while self._cg_header_re.match(line): - line = self.readline() - - # process call graph entries - entry_lines = [] - while line != '\014': # form feed - if line and not line.isspace(): - if self._cg_sep_re.match(line): - self.parse_cg_entry(entry_lines) - entry_lines = [] - else: - entry_lines.append(line) - line = self.readline() - - def parse(self): - self.parse_cg() - self.fp.close() - - profile = Profile() - profile[TIME] = 0.0 - - cycles = {} - for index in self.cycles.iterkeys(): - cycles[index] = Cycle() - - for entry in self.functions.itervalues(): - # populate the function - function = Function(entry.index, entry.name) - function[TIME] = entry.self - if entry.called is not None: - function.called = entry.called - if entry.called_self is not None: - call = Call(entry.index) - call[CALLS] = entry.called_self - function.called += entry.called_self - - # populate the function calls - for child in entry.children: - call = Call(child.index) - - assert child.called is not None - call[CALLS] = child.called - - if child.index not in self.functions: - # NOTE: functions that were never called but were discovered by gprof's - # static call graph analysis dont have a call graph entry so we need - # to add them here - missing = Function(child.index, child.name) - function[TIME] = 0.0 - function.called = 0 - profile.add_function(missing) - - function.add_call(call) - - profile.add_function(function) - - if entry.cycle is not None: - try: - cycle = cycles[entry.cycle] - except KeyError: - sys.stderr.write('warning: entry missing\n' % entry.cycle) - cycle = Cycle() - cycles[entry.cycle] = cycle - cycle.add_function(function) - - profile[TIME] = profile[TIME] + function[TIME] - - for cycle in cycles.itervalues(): - profile.add_cycle(cycle) - - # Compute derived events - profile.validate() - profile.ratio(TIME_RATIO, TIME) - profile.call_ratios(CALLS) - profile.integrate(TOTAL_TIME, TIME) - profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME) - - return profile - - -class CallgrindParser(LineParser): - """Parser for valgrind's callgrind tool. - - See also: - - https://web.archive.org/web/20190415231603/http://valgrind.org/docs/manual/cl-Format.html - """ - - _call_re = re.compile(r'^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$') - - def __init__(self, infile): - LineParser.__init__(self, infile) - - # Textual positions - self.position_ids = {} - self.positions = {} - - # Numeric positions - self.num_positions = 1 - self.cost_positions = ['line'] - self.last_positions = [0] - - # Events - self.num_events = 0 - self.cost_events = [] - - self.profile = Profile() - self.profile[SAMPLES] = 0 - - def parse(self): - # read lookahead - self.readline() - - self.parse_key('version') - self.parse_key('creator') - self.parse_part() - - # compute derived data - self.profile.validate() - self.profile.find_cycles() - self.profile.ratio(TIME_RATIO, SAMPLES) - self.profile.call_ratios(CALLS) - self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO) - - return self.profile - - def parse_part(self): - while self.parse_header_line(): - pass - while self.parse_body_line(): - pass - return True - - def parse_header_line(self): - return \ - self.parse_empty() or \ - self.parse_comment() or \ - self.parse_part_detail() or \ - self.parse_description() or \ - self.parse_event_specification() or \ - self.parse_cost_line_def() or \ - self.parse_cost_summary() - - _detail_keys = set(('cmd', 'pid', 'thread', 'part')) - - def parse_part_detail(self): - return self.parse_keys(self._detail_keys) - - def parse_description(self): - return self.parse_key('desc') is not None - - def parse_event_specification(self): - event = self.parse_key('event') - if event is None: - return False - return True - - def parse_cost_line_def(self): - pair = self.parse_keys(('events', 'positions')) - if pair is None: - return False - key, value = pair - items = value.split() - if key == 'events': - self.num_events = len(items) - self.cost_events = items - if key == 'positions': - self.num_positions = len(items) - self.cost_positions = items - self.last_positions = [0]*self.num_positions - return True - - def parse_cost_summary(self): - pair = self.parse_keys(('summary', 'totals')) - if pair is None: - return False - return True - - def parse_body_line(self): - return \ - self.parse_empty() or \ - self.parse_comment() or \ - self.parse_cost_line() or \ - self.parse_position_spec() or \ - self.parse_association_spec() - - _cost_re = re.compile(r'^(\d+|\+\d+|-\d+|\*)( \d+)+$') - - def parse_cost_line(self, calls=None): - line = self.lookahead() - mo = self._cost_re.match(line) - if not mo: - return False - - function = self.get_function() - - values = line.split(' ') - assert len(values) == self.num_positions + self.num_events - - positions = values[0 : self.num_positions] - events = values[self.num_positions : ] - - for i in range(self.num_positions): - position = positions[i] - if position == '*': - position = self.last_positions[i] - elif position[0] in '-+': - position = self.last_positions[i] + int(position) - else: - position = int(position) - self.last_positions[i] = position - - events = map(float, events) - - if calls is None: - function[SAMPLES] += events[0] - self.profile[SAMPLES] += events[0] - else: - callee = self.get_callee() - callee.called += calls - - try: - call = function.calls[callee.id] - except KeyError: - call = Call(callee.id) - call[CALLS] = calls - call[SAMPLES] = events[0] - function.add_call(call) - else: - call[CALLS] += calls - call[SAMPLES] += events[0] - - self.consume() - return True - - def parse_association_spec(self): - line = self.lookahead() - if not line.startswith('calls='): - return False - - _, values = line.split('=', 1) - values = values.strip().split() - calls = int(values[0]) - call_position = values[1:] - self.consume() - - self.parse_cost_line(calls) - - return True - - _position_re = re.compile(r'^(?Pc?(?:ob|fl|fi|fe|fn))=\s*(?:\((?P\d+)\))?(?:\s*(?P.+))?') - - _position_table_map = { - 'ob': 'ob', - 'fl': 'fl', - 'fi': 'fl', - 'fe': 'fl', - 'fn': 'fn', - 'cob': 'ob', - 'cfl': 'fl', - 'cfi': 'fl', - 'cfe': 'fl', - 'cfn': 'fn', - } - - _position_map = { - 'ob': 'ob', - 'fl': 'fl', - 'fi': 'fl', - 'fe': 'fl', - 'fn': 'fn', - 'cob': 'cob', - 'cfl': 'cfl', - 'cfi': 'cfl', - 'cfe': 'cfl', - 'cfn': 'cfn', - } - - def parse_position_spec(self): - line = self.lookahead() - mo = self._position_re.match(line) - if not mo: - return False - - position, id, name = mo.groups() - if id: - table = self._position_table_map[position] - if name: - self.position_ids[(table, id)] = name - else: - name = self.position_ids.get((table, id), '') - self.positions[self._position_map[position]] = name - self.consume() - return True - - def parse_empty(self): - line = self.lookahead() - if line.strip(): - return False - self.consume() - return True - - def parse_comment(self): - line = self.lookahead() - if not line.startswith('#'): - return False - self.consume() - return True - - _key_re = re.compile(r'^(\w+):') - - def parse_key(self, key): - pair = self.parse_keys((key,)) - if not pair: - return None - key, value = pair - return value - - def parse_keys(self, keys): - line = self.lookahead() - mo = self._key_re.match(line) - if not mo: - return None - key, value = line.split(':', 1) - if key not in keys: - return None - value = value.strip() - self.consume() - return key, value - - def make_function(self, module, filename, name): - # FIXME: module and filename are not being tracked reliably - #id = '|'.join((module, filename, name)) - id = name - try: - function = self.profile.functions[id] - except KeyError: - function = Function(id, name) - function[SAMPLES] = 0 - function.called = 0 - self.profile.add_function(function) - return function - - def get_function(self): - module = self.positions.get('ob', '') - filename = self.positions.get('fl', '') - function = self.positions.get('fn', '') - return self.make_function(module, filename, function) - - def get_callee(self): - module = self.positions.get('cob', '') - filename = self.positions.get('cfi', '') - function = self.positions.get('cfn', '') - return self.make_function(module, filename, function) - - -class OprofileParser(LineParser): - """Parser for oprofile callgraph output. - - See also: - - http://oprofile.sourceforge.net/doc/opreport.html#opreport-callgraph - """ - - _fields_re = { - 'samples': r'(\d+)', - '%': r'(\S+)', - 'linenr info': r'(?P\(no location information\)|\S+:\d+)', - 'image name': r'(?P\S+(?:\s\(tgid:[^)]*\))?)', - 'app name': r'(?P\S+)', - 'symbol name': r'(?P\(no symbols\)|.+?)', - } - - def __init__(self, infile): - LineParser.__init__(self, infile) - self.entries = {} - self.entry_re = None - - def add_entry(self, callers, function, callees): - try: - entry = self.entries[function.id] - except KeyError: - self.entries[function.id] = (callers, function, callees) - else: - callers_total, function_total, callees_total = entry - self.update_subentries_dict(callers_total, callers) - function_total.samples += function.samples - self.update_subentries_dict(callees_total, callees) - - def update_subentries_dict(self, totals, partials): - for partial in partials.itervalues(): - try: - total = totals[partial.id] - except KeyError: - totals[partial.id] = partial - else: - total.samples += partial.samples - - def parse(self): - # read lookahead - self.readline() - - self.parse_header() - while self.lookahead(): - self.parse_entry() - - profile = Profile() - - reverse_call_samples = {} - - # populate the profile - profile[SAMPLES] = 0 - for _callers, _function, _callees in self.entries.itervalues(): - function = Function(_function.id, _function.name) - function[SAMPLES] = _function.samples - profile.add_function(function) - profile[SAMPLES] += _function.samples - - if _function.application: - function.process = os.path.basename(_function.application) - if _function.image: - function.module = os.path.basename(_function.image) - - total_callee_samples = 0 - for _callee in _callees.itervalues(): - total_callee_samples += _callee.samples - - for _callee in _callees.itervalues(): - if not _callee.self: - call = Call(_callee.id) - call[SAMPLES2] = _callee.samples - function.add_call(call) - - # compute derived data - profile.validate() - profile.find_cycles() - profile.ratio(TIME_RATIO, SAMPLES) - profile.call_ratios(SAMPLES2) - profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO) - - return profile - - def parse_header(self): - while not self.match_header(): - self.consume() - line = self.lookahead() - fields = re.split(r'\s\s+', line) - entry_re = r'^\s*' + r'\s+'.join([self._fields_re[field] for field in fields]) + r'(?P\s+\[self\])?$' - self.entry_re = re.compile(entry_re) - self.skip_separator() - - def parse_entry(self): - callers = self.parse_subentries() - if self.match_primary(): - function = self.parse_subentry() - if function is not None: - callees = self.parse_subentries() - self.add_entry(callers, function, callees) - self.skip_separator() - - def parse_subentries(self): - subentries = {} - while self.match_secondary(): - subentry = self.parse_subentry() - subentries[subentry.id] = subentry - return subentries - - def parse_subentry(self): - entry = Struct() - line = self.consume() - mo = self.entry_re.match(line) - if not mo: - raise ParseError('failed to parse', line) - fields = mo.groupdict() - entry.samples = int(mo.group(1)) - if 'source' in fields and fields['source'] != '(no location information)': - source = fields['source'] - filename, lineno = source.split(':') - entry.filename = filename - entry.lineno = int(lineno) - else: - source = '' - entry.filename = None - entry.lineno = None - entry.image = fields.get('image', '') - entry.application = fields.get('application', '') - if 'symbol' in fields and fields['symbol'] != '(no symbols)': - entry.symbol = fields['symbol'] - else: - entry.symbol = '' - if entry.symbol.startswith('"') and entry.symbol.endswith('"'): - entry.symbol = entry.symbol[1:-1] - entry.id = ':'.join((entry.application, entry.image, source, entry.symbol)) - entry.self = fields.get('self', None) != None - if entry.self: - entry.id += ':self' - if entry.symbol: - entry.name = entry.symbol - else: - entry.name = entry.image - return entry - - def skip_separator(self): - while not self.match_separator(): - self.consume() - self.consume() - - def match_header(self): - line = self.lookahead() - return line.startswith('samples') - - def match_separator(self): - line = self.lookahead() - return line == '-'*len(line) - - def match_primary(self): - line = self.lookahead() - return not line[:1].isspace() - - def match_secondary(self): - line = self.lookahead() - return line[:1].isspace() - - -class SysprofParser(XmlParser): - - def __init__(self, stream): - XmlParser.__init__(self, stream) - - def parse(self): - objects = {} - nodes = {} - - self.element_start('profile') - while self.token.type == XML_ELEMENT_START: - if self.token.name_or_data == 'objects': - assert not objects - objects = self.parse_items('objects') - elif self.token.name_or_data == 'nodes': - assert not nodes - nodes = self.parse_items('nodes') - else: - self.parse_value(self.token.name_or_data) - self.element_end('profile') - - return self.build_profile(objects, nodes) - - def parse_items(self, name): - assert name[-1] == 's' - items = {} - self.element_start(name) - while self.token.type == XML_ELEMENT_START: - id, values = self.parse_item(name[:-1]) - assert id not in items - items[id] = values - self.element_end(name) - return items - - def parse_item(self, name): - attrs = self.element_start(name) - id = int(attrs['id']) - values = self.parse_values() - self.element_end(name) - return id, values - - def parse_values(self): - values = {} - while self.token.type == XML_ELEMENT_START: - name = self.token.name_or_data - value = self.parse_value(name) - assert name not in values - values[name] = value - return values - - def parse_value(self, tag): - self.element_start(tag) - value = self.character_data() - self.element_end(tag) - if value.isdigit(): - return int(value) - if value.startswith('"') and value.endswith('"'): - return value[1:-1] - return value - - def build_profile(self, objects, nodes): - profile = Profile() - - profile[SAMPLES] = 0 - for id, object in objects.iteritems(): - # Ignore fake objects (process names, modules, "Everything", "kernel", etc.) - if object['self'] == 0: - continue - - function = Function(id, object['name']) - function[SAMPLES] = object['self'] - profile.add_function(function) - profile[SAMPLES] += function[SAMPLES] - - for id, node in nodes.iteritems(): - # Ignore fake calls - if node['self'] == 0: - continue - - # Find a non-ignored parent - parent_id = node['parent'] - while parent_id != 0: - parent = nodes[parent_id] - caller_id = parent['object'] - if objects[caller_id]['self'] != 0: - break - parent_id = parent['parent'] - if parent_id == 0: - continue - - callee_id = node['object'] - - assert objects[caller_id]['self'] - assert objects[callee_id]['self'] - - function = profile.functions[caller_id] - - samples = node['self'] - try: - call = function.calls[callee_id] - except KeyError: - call = Call(callee_id) - call[SAMPLES2] = samples - function.add_call(call) - else: - call[SAMPLES2] += samples - - # Compute derived events - profile.validate() - profile.find_cycles() - profile.ratio(TIME_RATIO, SAMPLES) - profile.call_ratios(SAMPLES2) - profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO) - - return profile - - -class SharkParser(LineParser): - """Parser for MacOSX Shark output. - - Author: tom@dbservice.com - """ - - def __init__(self, infile): - LineParser.__init__(self, infile) - self.stack = [] - self.entries = {} - - def add_entry(self, function): - try: - entry = self.entries[function.id] - except KeyError: - self.entries[function.id] = (function, { }) - else: - function_total, callees_total = entry - function_total.samples += function.samples - - def add_callee(self, function, callee): - func, callees = self.entries[function.id] - try: - entry = callees[callee.id] - except KeyError: - callees[callee.id] = callee - else: - entry.samples += callee.samples - - def parse(self): - self.readline() - self.readline() - self.readline() - self.readline() - - match = re.compile(r'(?P[|+ ]*)(?P\d+), (?P[^,]+), (?P.*)') - - while self.lookahead(): - line = self.consume() - mo = match.match(line) - if not mo: - raise ParseError('failed to parse', line) - - fields = mo.groupdict() - prefix = len(fields.get('prefix', 0)) / 2 - 1 - - symbol = str(fields.get('symbol', 0)) - image = str(fields.get('image', 0)) - - entry = Struct() - entry.id = ':'.join([symbol, image]) - entry.samples = int(fields.get('samples', 0)) - - entry.name = symbol - entry.image = image - - # adjust the callstack - if prefix < len(self.stack): - del self.stack[prefix:] - - if prefix == len(self.stack): - self.stack.append(entry) - - # if the callstack has had an entry, it's this functions caller - if prefix > 0: - self.add_callee(self.stack[prefix - 1], entry) - - self.add_entry(entry) - - profile = Profile() - profile[SAMPLES] = 0 - for _function, _callees in self.entries.itervalues(): - function = Function(_function.id, _function.name) - function[SAMPLES] = _function.samples - profile.add_function(function) - profile[SAMPLES] += _function.samples - - if _function.image: - function.module = os.path.basename(_function.image) - - for _callee in _callees.itervalues(): - call = Call(_callee.id) - call[SAMPLES] = _callee.samples - function.add_call(call) - - # compute derived data - profile.validate() - profile.find_cycles() - profile.ratio(TIME_RATIO, SAMPLES) - profile.call_ratios(SAMPLES) - profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO) - - return profile - - -class XPerfParser(Parser): - """Parser for CSVs generted by XPerf, from Microsoft Windows Performance Tools. - """ - - def __init__(self, stream): - Parser.__init__(self) - self.stream = stream - self.profile = Profile() - self.profile[SAMPLES] = 0 - self.column = {} - - def parse(self): - import csv - reader = csv.reader( - self.stream, - delimiter = ',', - quotechar = None, - escapechar = None, - doublequote = False, - skipinitialspace = True, - lineterminator = '\r\n', - quoting = csv.QUOTE_NONE) - it = iter(reader) - row = next(reader) - self.parse_header(row) - for row in it: - self.parse_row(row) - - # compute derived data - self.profile.validate() - self.profile.find_cycles() - self.profile.ratio(TIME_RATIO, SAMPLES) - self.profile.call_ratios(SAMPLES2) - self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO) - - return self.profile - - def parse_header(self, row): - for column in range(len(row)): - name = row[column] - assert name not in self.column - self.column[name] = column - - def parse_row(self, row): - fields = {} - for name, column in self.column.iteritems(): - value = row[column] - for factory in int, float: - try: - value = factory(value) - except ValueError: - pass - else: - break - fields[name] = value - - process = fields['Process Name'] - symbol = fields['Module'] + '!' + fields['Function'] - weight = fields['Weight'] - count = fields['Count'] - - function = self.get_function(process, symbol) - function[SAMPLES] += weight * count - self.profile[SAMPLES] += weight * count - - stack = fields['Stack'] - if stack != '?': - stack = stack.split('/') - assert stack[0] == '[Root]' - if stack[-1] != symbol: - # XXX: some cases the sampled function does not appear in the stack - stack.append(symbol) - caller = None - for symbol in stack[1:]: - callee = self.get_function(process, symbol) - if caller is not None: - try: - call = caller.calls[callee.id] - except KeyError: - call = Call(callee.id) - call[SAMPLES2] = count - caller.add_call(call) - else: - call[SAMPLES2] += count - caller = callee - - def get_function(self, process, symbol): - function_id = process + '!' + symbol - - try: - function = self.profile.functions[function_id] - except KeyError: - module, name = symbol.split('!') - function = Function(function_id, name) - function.process = process - function.module = module - function[SAMPLES] = 0 - self.profile.add_function(function) - - return function - - -class SleepyParser(Parser): - """Parser for GNU gprof output. - - See also: - - http://www.codersnotes.com/sleepy/ - - http://sleepygraph.sourceforge.net/ - """ - - def __init__(self, filename): - Parser.__init__(self) - - from zipfile import ZipFile - - self.database = ZipFile(filename) - - self.symbols = {} - self.calls = {} - - self.profile = Profile() - - _symbol_re = re.compile( - r'^(?P\w+)' + - r'\s+"(?P[^"]*)"' + - r'\s+"(?P[^"]*)"' + - r'\s+"(?P[^"]*)"' + - r'\s+(?P\d+)$' - ) - - def parse_symbols(self): - lines = self.database.read('symbols.txt').splitlines() - for line in lines: - mo = self._symbol_re.match(line) - if mo: - symbol_id, module, procname, sourcefile, sourceline = mo.groups() - - function_id = ':'.join([module, procname]) - - try: - function = self.profile.functions[function_id] - except KeyError: - function = Function(function_id, procname) - function.module = module - function[SAMPLES] = 0 - self.profile.add_function(function) - - self.symbols[symbol_id] = function - - def parse_callstacks(self): - lines = self.database.read("callstacks.txt").splitlines() - for line in lines: - fields = line.split() - samples = int(fields[0]) - callstack = fields[1:] - - callstack = [self.symbols[symbol_id] for symbol_id in callstack] - - callee = callstack[0] - - callee[SAMPLES] += samples - self.profile[SAMPLES] += samples - - for caller in callstack[1:]: - try: - call = caller.calls[callee.id] - except KeyError: - call = Call(callee.id) - call[SAMPLES2] = samples - caller.add_call(call) - else: - call[SAMPLES2] += samples - - callee = caller - - def parse(self): - profile = self.profile - profile[SAMPLES] = 0 - - self.parse_symbols() - self.parse_callstacks() - - # Compute derived events - profile.validate() - profile.find_cycles() - profile.ratio(TIME_RATIO, SAMPLES) - profile.call_ratios(SAMPLES2) - profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO) - - return profile - - -class AQtimeTable: - - def __init__(self, name, fields): - self.name = name - - self.fields = fields - self.field_column = {} - for column in range(len(fields)): - self.field_column[fields[column]] = column - self.rows = [] - - def __len__(self): - return len(self.rows) - - def __iter__(self): - for values, children in self.rows: - fields = {} - for name, value in zip(self.fields, values): - fields[name] = value - children = dict([(child.name, child) for child in children]) - yield fields, children - raise StopIteration - - def add_row(self, values, children=()): - self.rows.append((values, children)) - - -class AQtimeParser(XmlParser): - - def __init__(self, stream): - XmlParser.__init__(self, stream) - self.tables = {} - - def parse(self): - self.element_start('AQtime_Results') - self.parse_headers() - results = self.parse_results() - self.element_end('AQtime_Results') - return self.build_profile(results) - - def parse_headers(self): - self.element_start('HEADERS') - while self.token.type == XML_ELEMENT_START: - self.parse_table_header() - self.element_end('HEADERS') - - def parse_table_header(self): - attrs = self.element_start('TABLE_HEADER') - name = attrs['NAME'] - id = int(attrs['ID']) - field_types = [] - field_names = [] - while self.token.type == XML_ELEMENT_START: - field_type, field_name = self.parse_table_field() - field_types.append(field_type) - field_names.append(field_name) - self.element_end('TABLE_HEADER') - self.tables[id] = name, field_types, field_names - - def parse_table_field(self): - attrs = self.element_start('TABLE_FIELD') - type = attrs['TYPE'] - name = self.character_data() - self.element_end('TABLE_FIELD') - return type, name - - def parse_results(self): - self.element_start('RESULTS') - table = self.parse_data() - self.element_end('RESULTS') - return table - - def parse_data(self): - rows = [] - attrs = self.element_start('DATA') - table_id = int(attrs['TABLE_ID']) - table_name, field_types, field_names = self.tables[table_id] - table = AQtimeTable(table_name, field_names) - while self.token.type == XML_ELEMENT_START: - row, children = self.parse_row(field_types) - table.add_row(row, children) - self.element_end('DATA') - return table - - def parse_row(self, field_types): - row = [None]*len(field_types) - children = [] - self.element_start('ROW') - while self.token.type == XML_ELEMENT_START: - if self.token.name_or_data == 'FIELD': - field_id, field_value = self.parse_field(field_types) - row[field_id] = field_value - elif self.token.name_or_data == 'CHILDREN': - children = self.parse_children() - else: - raise XmlTokenMismatch(" or ", self.token) - self.element_end('ROW') - return row, children - - def parse_field(self, field_types): - attrs = self.element_start('FIELD') - id = int(attrs['ID']) - type = field_types[id] - value = self.character_data() - if type == 'Integer': - value = int(value) - elif type == 'Float': - value = float(value) - elif type == 'Address': - value = int(value) - elif type == 'String': - pass - else: - assert False - self.element_end('FIELD') - return id, value - - def parse_children(self): - children = [] - self.element_start('CHILDREN') - while self.token.type == XML_ELEMENT_START: - table = self.parse_data() - assert table.name not in children - children.append(table) - self.element_end('CHILDREN') - return children - - def build_profile(self, results): - assert results.name == 'Routines' - profile = Profile() - profile[TIME] = 0.0 - for fields, tables in results: - function = self.build_function(fields) - children = tables['Children'] - for fields, _ in children: - call = self.build_call(fields) - function.add_call(call) - profile.add_function(function) - profile[TIME] = profile[TIME] + function[TIME] - profile[TOTAL_TIME] = profile[TIME] - profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME) - return profile - - def build_function(self, fields): - function = Function(self.build_id(fields), self.build_name(fields)) - function[TIME] = fields['Time'] - function[TOTAL_TIME] = fields['Time with Children'] - #function[TIME_RATIO] = fields['% Time']/100.0 - #function[TOTAL_TIME_RATIO] = fields['% with Children']/100.0 - return function - - def build_call(self, fields): - call = Call(self.build_id(fields)) - call[TIME] = fields['Time'] - call[TOTAL_TIME] = fields['Time with Children'] - #call[TIME_RATIO] = fields['% Time']/100.0 - #call[TOTAL_TIME_RATIO] = fields['% with Children']/100.0 - return call - - def build_id(self, fields): - return ':'.join([fields['Module Name'], fields['Unit Name'], fields['Routine Name']]) - - def build_name(self, fields): - # TODO: use more fields - return fields['Routine Name'] - - -class PstatsParser: - """Parser python profiling statistics saved with te pstats module.""" - - def __init__(self, *filename): - import pstats - try: - self.stats = pstats.Stats(*filename) - except ValueError: - import hotshot.stats - self.stats = hotshot.stats.load(filename[0]) - self.profile = Profile() - self.function_ids = {} - - def get_function_name(self, args): - filename, line, name = args - module = os.path.splitext(filename)[0] - module = os.path.basename(module) - return "%s:%d:%s" % (module, line, name) - - def get_function(self, key): - try: - id = self.function_ids[key] - except KeyError: - id = len(self.function_ids) - name = self.get_function_name(key) - function = Function(id, name) - self.profile.functions[id] = function - self.function_ids[key] = id - else: - function = self.profile.functions[id] - return function - - def parse(self): - self.profile[TIME] = 0.0 - self.profile[TOTAL_TIME] = self.stats.total_tt - for fn, (cc, nc, tt, ct, callers) in self.stats.stats.iteritems(): - callee = self.get_function(fn) - callee.called = nc - callee[TOTAL_TIME] = ct - callee[TIME] = tt - self.profile[TIME] += tt - self.profile[TOTAL_TIME] = max(self.profile[TOTAL_TIME], ct) - for fn, value in callers.iteritems(): - caller = self.get_function(fn) - call = Call(callee.id) - if isinstance(value, tuple): - for i in xrange(0, len(value), 4): - nc, cc, tt, ct = value[i:i+4] - if CALLS in call: - call[CALLS] += cc - else: - call[CALLS] = cc - - if TOTAL_TIME in call: - call[TOTAL_TIME] += ct - else: - call[TOTAL_TIME] = ct - - else: - call[CALLS] = value - call[TOTAL_TIME] = ratio(value, nc)*ct - - caller.add_call(call) - #self.stats.print_stats() - #self.stats.print_callees() - - # Compute derived events - self.profile.validate() - self.profile.ratio(TIME_RATIO, TIME) - self.profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME) - - return self.profile - - -class Theme: - - def __init__(self, - bgcolor = (0.0, 0.0, 1.0), - mincolor = (0.0, 0.0, 0.0), - maxcolor = (0.0, 0.0, 1.0), - fontname = "Arial", - minfontsize = 10.0, - maxfontsize = 10.0, - minpenwidth = 0.5, - maxpenwidth = 4.0, - gamma = 2.2, - skew = 1.0): - self.bgcolor = bgcolor - self.mincolor = mincolor - self.maxcolor = maxcolor - self.fontname = fontname - self.minfontsize = minfontsize - self.maxfontsize = maxfontsize - self.minpenwidth = minpenwidth - self.maxpenwidth = maxpenwidth - self.gamma = gamma - self.skew = skew - - def graph_bgcolor(self): - return self.hsl_to_rgb(*self.bgcolor) - - def graph_fontname(self): - return self.fontname - - def graph_fontsize(self): - return self.minfontsize - - def node_bgcolor(self, weight): - return self.color(weight) - - def node_fgcolor(self, weight): - return self.graph_bgcolor() - - def node_fontsize(self, weight): - return self.fontsize(weight) - - def edge_color(self, weight): - return self.color(weight) - - def edge_fontsize(self, weight): - return self.fontsize(weight) - - def edge_penwidth(self, weight): - return max(weight*self.maxpenwidth, self.minpenwidth) - - def edge_arrowsize(self, weight): - return 0.5 * math.sqrt(self.edge_penwidth(weight)) - - def fontsize(self, weight): - return max(weight**2 * self.maxfontsize, self.minfontsize) - - def color(self, weight): - weight = min(max(weight, 0.0), 1.0) - - hmin, smin, lmin = self.mincolor - hmax, smax, lmax = self.maxcolor - - if self.skew < 0: - raise ValueError("Skew must be greater than 0") - elif self.skew == 1.0: - h = hmin + weight*(hmax - hmin) - s = smin + weight*(smax - smin) - l = lmin + weight*(lmax - lmin) - else: - base = self.skew - h = hmin + ((hmax-hmin)*(-1.0 + (base ** weight)) / (base - 1.0)) - s = smin + ((smax-smin)*(-1.0 + (base ** weight)) / (base - 1.0)) - l = lmin + ((lmax-lmin)*(-1.0 + (base ** weight)) / (base - 1.0)) - - return self.hsl_to_rgb(h, s, l) - - def hsl_to_rgb(self, h, s, l): - """Convert a color from HSL color-model to RGB. - - See also: - - http://www.w3.org/TR/css3-color/#hsl-color - """ - - h = h % 1.0 - s = min(max(s, 0.0), 1.0) - l = min(max(l, 0.0), 1.0) - - if l <= 0.5: - m2 = l*(s + 1.0) - else: - m2 = l + s - l*s - m1 = l*2.0 - m2 - r = self._hue_to_rgb(m1, m2, h + 1.0/3.0) - g = self._hue_to_rgb(m1, m2, h) - b = self._hue_to_rgb(m1, m2, h - 1.0/3.0) - - # Apply gamma correction - r **= self.gamma - g **= self.gamma - b **= self.gamma - - return (r, g, b) - - def _hue_to_rgb(self, m1, m2, h): - if h < 0.0: - h += 1.0 - elif h > 1.0: - h -= 1.0 - if h*6 < 1.0: - return m1 + (m2 - m1)*h*6.0 - elif h*2 < 1.0: - return m2 - elif h*3 < 2.0: - return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0 - else: - return m1 - - -TEMPERATURE_COLORMAP = Theme( - mincolor = (2.0/3.0, 0.80, 0.25), # dark blue - maxcolor = (0.0, 1.0, 0.5), # satured red - gamma = 1.0 -) - -PINK_COLORMAP = Theme( - mincolor = (0.0, 1.0, 0.90), # pink - maxcolor = (0.0, 1.0, 0.5), # satured red -) - -GRAY_COLORMAP = Theme( - mincolor = (0.0, 0.0, 0.85), # light gray - maxcolor = (0.0, 0.0, 0.0), # black -) - -BW_COLORMAP = Theme( - minfontsize = 8.0, - maxfontsize = 24.0, - mincolor = (0.0, 0.0, 0.0), # black - maxcolor = (0.0, 0.0, 0.0), # black - minpenwidth = 0.1, - maxpenwidth = 8.0, -) - - -class DotWriter: - """Writer for the DOT language. - - See also: - - "The DOT Language" specification - http://www.graphviz.org/doc/info/lang.html - """ - - def __init__(self, fp): - self.fp = fp - - def graph(self, profile, theme): - self.begin_graph() - - fontname = theme.graph_fontname() - - self.attr('graph', fontname=fontname, ranksep=0.25, nodesep=0.125) - self.attr('node', fontname=fontname, shape="box", style="filled", fontcolor="white", width=0, height=0) - self.attr('edge', fontname=fontname) - - for function in profile.functions.itervalues(): - labels = [] - if function.process is not None: - labels.append(function.process) - if function.module is not None: - labels.append(function.module) - labels.append(function.name) - for event in TOTAL_TIME_RATIO, TIME_RATIO: - if event in function.events: - label = event.format(function[event]) - labels.append(label) - if function.called is not None: - labels.append(u"%u\xd7" % (function.called,)) - - if function.weight is not None: - weight = function.weight - else: - weight = 0.0 - - label = '\n'.join(labels) - self.node(function.id, - label = label, - color = self.color(theme.node_bgcolor(weight)), - fontcolor = self.color(theme.node_fgcolor(weight)), - fontsize = "%.2f" % theme.node_fontsize(weight), - ) - - for call in function.calls.itervalues(): - callee = profile.functions[call.callee_id] - - labels = [] - for event in TOTAL_TIME_RATIO, CALLS: - if event in call.events: - label = event.format(call[event]) - labels.append(label) - - if call.weight is not None: - weight = call.weight - elif callee.weight is not None: - weight = callee.weight - else: - weight = 0.0 - - label = '\n'.join(labels) - - self.edge(function.id, call.callee_id, - label = label, - color = self.color(theme.edge_color(weight)), - fontcolor = self.color(theme.edge_color(weight)), - fontsize = "%.2f" % theme.edge_fontsize(weight), - penwidth = "%.2f" % theme.edge_penwidth(weight), - labeldistance = "%.2f" % theme.edge_penwidth(weight), - arrowsize = "%.2f" % theme.edge_arrowsize(weight), - ) - - self.end_graph() - - def begin_graph(self): - self.write('digraph {\n') - - def end_graph(self): - self.write('}\n') - - def attr(self, what, **attrs): - self.write("\t") - self.write(what) - self.attr_list(attrs) - self.write(";\n") - - def node(self, node, **attrs): - self.write("\t") - self.id(node) - self.attr_list(attrs) - self.write(";\n") - - def edge(self, src, dst, **attrs): - self.write("\t") - self.id(src) - self.write(" -> ") - self.id(dst) - self.attr_list(attrs) - self.write(";\n") - - def attr_list(self, attrs): - if not attrs: - return - self.write(' [') - first = True - for name, value in attrs.iteritems(): - if first: - first = False - else: - self.write(", ") - self.id(name) - self.write('=') - self.id(value) - self.write(']') - - def id(self, id): - if isinstance(id, (int, float)): - s = str(id) - elif isinstance(id, basestring): - if id.isalnum() and not id.startswith('0x'): - s = id - else: - s = self.escape(id) - else: - raise TypeError - self.write(s) - - def color(self, args): - r, g, b = args - - def float2int(f): - if f <= 0.0: - return 0 - if f >= 1.0: - return 255 - return int(255.0*f + 0.5) - - return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)]) - - def escape(self, s): - s = s.encode('utf-8') - s = s.replace('\\', r'\\') - s = s.replace('\n', r'\n') - s = s.replace('\t', r'\t') - s = s.replace('"', r'\"') - return '"' + s + '"' - - def write(self, s): - self.fp.write(s) - - -class Main: - """Main program.""" - - themes = { - "color": TEMPERATURE_COLORMAP, - "pink": PINK_COLORMAP, - "gray": GRAY_COLORMAP, - "bw": BW_COLORMAP, - } - - def main(self): - """Main program.""" - - parser = optparse.OptionParser( - usage="\n\t%prog [options] [file] ...", - version="%%prog %s" % __version__) - parser.add_option( - '-o', '--output', metavar='FILE', - type="string", dest="output", - help="output filename [stdout]") - parser.add_option( - '-n', '--node-thres', metavar='PERCENTAGE', - type="float", dest="node_thres", default=0.5, - help="eliminate nodes below this threshold [default: %default]") - parser.add_option( - '-e', '--edge-thres', metavar='PERCENTAGE', - type="float", dest="edge_thres", default=0.1, - help="eliminate edges below this threshold [default: %default]") - parser.add_option( - '-f', '--format', - type="choice", choices=('prof', 'callgrind', 'oprofile', 'sysprof', 'pstats', 'shark', 'sleepy', 'aqtime', 'xperf'), - dest="format", default="prof", - help="profile format: prof, callgrind, oprofile, sysprof, shark, sleepy, aqtime, pstats, or xperf [default: %default]") - parser.add_option( - '-c', '--colormap', - type="choice", choices=('color', 'pink', 'gray', 'bw'), - dest="theme", default="color", - help="color map: color, pink, gray, or bw [default: %default]") - parser.add_option( - '-s', '--strip', - action="store_true", - dest="strip", default=False, - help="strip function parameters, template parameters, and const modifiers from demangled C++ function names") - parser.add_option( - '-w', '--wrap', - action="store_true", - dest="wrap", default=False, - help="wrap function names") - # add a new option to control skew of the colorization curve - parser.add_option( - '--skew', - type="float", dest="theme_skew", default=1.0, - help="skew the colorization curve. Values < 1.0 give more variety to lower percentages. Value > 1.0 give less variety to lower percentages") - (self.options, self.args) = parser.parse_args(sys.argv[1:]) - - if len(self.args) > 1 and self.options.format != 'pstats': - parser.error('incorrect number of arguments') - - try: - self.theme = self.themes[self.options.theme] - except KeyError: - parser.error('invalid colormap \'%s\'' % self.options.theme) - - # set skew on the theme now that it has been picked. - if self.options.theme_skew: - self.theme.skew = self.options.theme_skew - - if self.options.format == 'prof': - if not self.args: - fp = sys.stdin - else: - fp = open(self.args[0], 'rt') - parser = GprofParser(fp) - elif self.options.format == 'callgrind': - if not self.args: - fp = sys.stdin - else: - fp = open(self.args[0], 'rt') - parser = CallgrindParser(fp) - elif self.options.format == 'oprofile': - if not self.args: - fp = sys.stdin - else: - fp = open(self.args[0], 'rt') - parser = OprofileParser(fp) - elif self.options.format == 'sysprof': - if not self.args: - fp = sys.stdin - else: - fp = open(self.args[0], 'rt') - parser = SysprofParser(fp) - elif self.options.format == 'pstats': - if not self.args: - parser.error('at least a file must be specified for pstats input') - parser = PstatsParser(*self.args) - elif self.options.format == 'xperf': - if not self.args: - fp = sys.stdin - else: - fp = open(self.args[0], 'rt') - parser = XPerfParser(fp) - elif self.options.format == 'shark': - if not self.args: - fp = sys.stdin - else: - fp = open(self.args[0], 'rt') - parser = SharkParser(fp) - elif self.options.format == 'sleepy': - if len(self.args) != 1: - parser.error('exactly one file must be specified for sleepy input') - parser = SleepyParser(self.args[0]) - elif self.options.format == 'aqtime': - if not self.args: - fp = sys.stdin - else: - fp = open(self.args[0], 'rt') - parser = AQtimeParser(fp) - else: - parser.error('invalid format \'%s\'' % self.options.format) - - self.profile = parser.parse() - - if self.options.output is None: - self.output = sys.stdout - else: - self.output = open(self.options.output, 'wt') - - self.write_graph() - - _parenthesis_re = re.compile(r'\([^()]*\)') - _angles_re = re.compile(r'<[^<>]*>') - _const_re = re.compile(r'\s+const$') - - def strip_function_name(self, name): - """Remove extraneous information from C++ demangled function names.""" - - # Strip function parameters from name by recursively removing paired parenthesis - while True: - name, n = self._parenthesis_re.subn('', name) - if not n: - break - - # Strip const qualifier - name = self._const_re.sub('', name) - - # Strip template parameters from name by recursively removing paired angles - while True: - name, n = self._angles_re.subn('', name) - if not n: - break - - return name - - def wrap_function_name(self, name): - """Split the function name on multiple lines.""" - - if len(name) > 32: - ratio = 2.0/3.0 - height = max(int(len(name)/(1.0 - ratio) + 0.5), 1) - width = max(len(name)/height, 32) - # TODO: break lines in symbols - name = textwrap.fill(name, width, break_long_words=False) - - # Take away spaces - name = name.replace(", ", ",") - name = name.replace("> >", ">>") - name = name.replace("> >", ">>") # catch consecutive - - return name - - def compress_function_name(self, name): - """Compress function name according to the user preferences.""" - - if self.options.strip: - name = self.strip_function_name(name) - - if self.options.wrap: - name = self.wrap_function_name(name) - - # TODO: merge functions with same resulting name - - return name - - def write_graph(self): - dot = DotWriter(self.output) - profile = self.profile - profile.prune(self.options.node_thres/100.0, self.options.edge_thres/100.0) - - for function in profile.functions.itervalues(): - function.name = self.compress_function_name(function.name) - - dot.graph(profile, self.theme) - - -if __name__ == '__main__': - Main().main() diff --git a/thirdparty/xdot/__init__.py b/thirdparty/xdot/__init__.py deleted file mode 100644 index 5ef1f2dc104..00000000000 --- a/thirdparty/xdot/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python2 -# -# Copyright 2008-2009 Jose Fonseca -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . -# - -pass diff --git a/thirdparty/xdot/xdot.py b/thirdparty/xdot/xdot.py deleted file mode 100644 index 5b43d6c5128..00000000000 --- a/thirdparty/xdot/xdot.py +++ /dev/null @@ -1,2432 +0,0 @@ -#!/usr/bin/env python2 -# -# Copyright 2008 Jose Fonseca -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . -# - -'''Visualize dot graphs via the xdot format.''' - -__author__ = "Jose Fonseca et al" - - -import os -import sys -import subprocess -import math -import colorsys -import time -import re -import optparse -import sys - -import gobject -import gtk -import gtk.gdk -import gtk.keysyms -import cairo -import pango -import pangocairo - -if sys.version_info >= (3, 0): - xrange = range - -# See http://www.graphviz.org/pub/scm/graphviz-cairo/plugin/cairo/gvrender_cairo.c - -# For pygtk inspiration and guidance see: -# - http://mirageiv.berlios.de/ -# - http://comix.sourceforge.net/ - - -class Pen: - """Store pen attributes.""" - - def __init__(self): - # set default attributes - self.color = (0.0, 0.0, 0.0, 1.0) - self.fillcolor = (0.0, 0.0, 0.0, 1.0) - self.linewidth = 1.0 - self.fontsize = 14.0 - self.fontname = "Times-Roman" - self.dash = () - - def copy(self): - """Create a copy of this pen.""" - pen = Pen() - pen.__dict__ = self.__dict__.copy() - return pen - - def highlighted(self): - pen = self.copy() - pen.color = (1, 0, 0, 1) - pen.fillcolor = (1, .8, .8, 1) - return pen - - -class Shape: - """Abstract base class for all the drawing shapes.""" - - def __init__(self): - pass - - def draw(self, cr, highlight=False): - """Draw this shape with the given cairo context""" - raise NotImplementedError - - def select_pen(self, highlight): - if highlight: - if not hasattr(self, 'highlight_pen'): - self.highlight_pen = self.pen.highlighted() - return self.highlight_pen - else: - return self.pen - - def search_text(self, regexp): - return False - - -class TextShape(Shape): - - LEFT, CENTER, RIGHT = -1, 0, 1 - - def __init__(self, pen, x, y, j, w, t): - Shape.__init__(self) - self.pen = pen.copy() - self.x = x - self.y = y - self.j = j - self.w = w - self.t = t - - def draw(self, cr, highlight=False): - - try: - layout = self.layout - except AttributeError: - layout = cr.create_layout() - - # set font options - # see http://lists.freedesktop.org/archives/cairo/2007-February/009688.html - context = layout.get_context() - fo = cairo.FontOptions() - fo.set_antialias(cairo.ANTIALIAS_DEFAULT) - fo.set_hint_style(cairo.HINT_STYLE_NONE) - fo.set_hint_metrics(cairo.HINT_METRICS_OFF) - try: - pangocairo.context_set_font_options(context, fo) - except TypeError: - # XXX: Some broken pangocairo bindings show the error - # 'TypeError: font_options must be a cairo.FontOptions or None' - pass - - # set font - font = pango.FontDescription() - font.set_family(self.pen.fontname) - font.set_absolute_size(self.pen.fontsize*pango.SCALE) - layout.set_font_description(font) - - # set text - layout.set_text(self.t) - - # cache it - self.layout = layout - else: - cr.update_layout(layout) - - descent = 2 # XXX get descender from font metrics - - width, height = layout.get_size() - width = float(width)/pango.SCALE - height = float(height)/pango.SCALE - # we know the width that dot thinks this text should have - # we do not necessarily have a font with the same metrics - # scale it so that the text fits inside its box - if width > self.w: - f = self.w / width - width = self.w # equivalent to width *= f - height *= f - descent *= f - else: - f = 1.0 - - if self.j == self.LEFT: - x = self.x - elif self.j == self.CENTER: - x = self.x - 0.5*width - elif self.j == self.RIGHT: - x = self.x - width - else: - assert 0 - - y = self.y - height + descent - - cr.move_to(x, y) - - cr.save() - cr.scale(f, f) - cr.set_source_rgba(*self.select_pen(highlight).color) - cr.show_layout(layout) - cr.restore() - - if 0: # DEBUG - # show where dot thinks the text should appear - cr.set_source_rgba(1, 0, 0, .9) - if self.j == self.LEFT: - x = self.x - elif self.j == self.CENTER: - x = self.x - 0.5*self.w - elif self.j == self.RIGHT: - x = self.x - self.w - cr.move_to(x, self.y) - cr.line_to(x+self.w, self.y) - cr.stroke() - - def search_text(self, regexp): - return regexp.search(self.t) is not None - - -class ImageShape(Shape): - - def __init__(self, pen, x0, y0, w, h, path): - Shape.__init__(self) - self.pen = pen.copy() - self.x0 = x0 - self.y0 = y0 - self.w = w - self.h = h - self.path = path - - def draw(self, cr, highlight=False): - cr2 = gtk.gdk.CairoContext(cr) - pixbuf = gtk.gdk.pixbuf_new_from_file(self.path) - sx = float(self.w)/float(pixbuf.get_width()) - sy = float(self.h)/float(pixbuf.get_height()) - cr.save() - cr.translate(self.x0, self.y0 - self.h) - cr.scale(sx, sy) - cr2.set_source_pixbuf(pixbuf, 0, 0) - cr2.paint() - cr.restore() - - -class EllipseShape(Shape): - - def __init__(self, pen, x0, y0, w, h, filled=False): - Shape.__init__(self) - self.pen = pen.copy() - self.x0 = x0 - self.y0 = y0 - self.w = w - self.h = h - self.filled = filled - - def draw(self, cr, highlight=False): - cr.save() - cr.translate(self.x0, self.y0) - cr.scale(self.w, self.h) - cr.move_to(1.0, 0.0) - cr.arc(0.0, 0.0, 1.0, 0, 2.0*math.pi) - cr.restore() - pen = self.select_pen(highlight) - if self.filled: - cr.set_source_rgba(*pen.fillcolor) - cr.fill() - else: - cr.set_dash(pen.dash) - cr.set_line_width(pen.linewidth) - cr.set_source_rgba(*pen.color) - cr.stroke() - - -class PolygonShape(Shape): - - def __init__(self, pen, points, filled=False): - Shape.__init__(self) - self.pen = pen.copy() - self.points = points - self.filled = filled - - def draw(self, cr, highlight=False): - x0, y0 = self.points[-1] - cr.move_to(x0, y0) - for x, y in self.points: - cr.line_to(x, y) - cr.close_path() - pen = self.select_pen(highlight) - if self.filled: - cr.set_source_rgba(*pen.fillcolor) - cr.fill_preserve() - cr.fill() - else: - cr.set_dash(pen.dash) - cr.set_line_width(pen.linewidth) - cr.set_source_rgba(*pen.color) - cr.stroke() - - -class LineShape(Shape): - - def __init__(self, pen, points): - Shape.__init__(self) - self.pen = pen.copy() - self.points = points - - def draw(self, cr, highlight=False): - x0, y0 = self.points[0] - cr.move_to(x0, y0) - for x1, y1 in self.points[1:]: - cr.line_to(x1, y1) - pen = self.select_pen(highlight) - cr.set_dash(pen.dash) - cr.set_line_width(pen.linewidth) - cr.set_source_rgba(*pen.color) - cr.stroke() - - -class BezierShape(Shape): - - def __init__(self, pen, points, filled=False): - Shape.__init__(self) - self.pen = pen.copy() - self.points = points - self.filled = filled - - def draw(self, cr, highlight=False): - x0, y0 = self.points[0] - cr.move_to(x0, y0) - for i in xrange(1, len(self.points), 3): - x1, y1 = self.points[i] - x2, y2 = self.points[i + 1] - x3, y3 = self.points[i + 2] - cr.curve_to(x1, y1, x2, y2, x3, y3) - pen = self.select_pen(highlight) - if self.filled: - cr.set_source_rgba(*pen.fillcolor) - cr.fill_preserve() - cr.fill() - else: - cr.set_dash(pen.dash) - cr.set_line_width(pen.linewidth) - cr.set_source_rgba(*pen.color) - cr.stroke() - - -class CompoundShape(Shape): - - def __init__(self, shapes): - Shape.__init__(self) - self.shapes = shapes - - def draw(self, cr, highlight=False): - for shape in self.shapes: - shape.draw(cr, highlight=highlight) - - def search_text(self, regexp): - for shape in self.shapes: - if shape.search_text(regexp): - return True - return False - - -class Url(object): - - def __init__(self, item, url, highlight=None): - self.item = item - self.url = url - if highlight is None: - highlight = set([item]) - self.highlight = highlight - - -class Jump(object): - - def __init__(self, item, x, y, highlight=None): - self.item = item - self.x = x - self.y = y - if highlight is None: - highlight = set([item]) - self.highlight = highlight - - -class Element(CompoundShape): - """Base class for graph nodes and edges.""" - - def __init__(self, shapes): - CompoundShape.__init__(self, shapes) - - def is_inside(self, x, y): - return False - - def get_url(self, x, y): - return None - - def get_jump(self, x, y): - return None - - -class Node(Element): - - def __init__(self, id, x, y, w, h, shapes, url): - Element.__init__(self, shapes) - - self.id = id - self.x = x - self.y = y - - self.x1 = x - 0.5*w - self.y1 = y - 0.5*h - self.x2 = x + 0.5*w - self.y2 = y + 0.5*h - - self.url = url - - def is_inside(self, x, y): - return self.x1 <= x and x <= self.x2 and self.y1 <= y and y <= self.y2 - - def get_url(self, x, y): - if self.url is None: - return None - if self.is_inside(x, y): - return Url(self, self.url) - return None - - def get_jump(self, x, y): - if self.is_inside(x, y): - return Jump(self, self.x, self.y) - return None - - def __repr__(self): - return "" % self.id - - -def square_distance(x1, y1, x2, y2): - deltax = x2 - x1 - deltay = y2 - y1 - return deltax*deltax + deltay*deltay - - -class Edge(Element): - - def __init__(self, src, dst, points, shapes): - Element.__init__(self, shapes) - self.src = src - self.dst = dst - self.points = points - - RADIUS = 10 - - def is_inside_begin(self, x, y): - return square_distance(x, y, *self.points[0]) <= self.RADIUS*self.RADIUS - - def is_inside_end(self, x, y): - return square_distance(x, y, *self.points[-1]) <= self.RADIUS*self.RADIUS - - def is_inside(self, x, y): - if self.is_inside_begin(x, y): - return True - if self.is_inside_end(x, y): - return True - return False - - def get_jump(self, x, y): - if self.is_inside_begin(x, y): - return Jump(self, self.dst.x, self.dst.y, highlight=set([self, self.dst])) - if self.is_inside_end(x, y): - return Jump(self, self.src.x, self.src.y, highlight=set([self, self.src])) - return None - - def __repr__(self): - return " %s>" % (self.src, self.dst) - - -class Graph(Shape): - - def __init__(self, width=1, height=1, shapes=(), nodes=(), edges=()): - Shape.__init__(self) - - self.width = width - self.height = height - self.shapes = shapes - self.nodes = nodes - self.edges = edges - - def get_size(self): - return self.width, self.height - - def draw(self, cr, highlight_items=None): - if highlight_items is None: - highlight_items = () - cr.set_source_rgba(0.0, 0.0, 0.0, 1.0) - - cr.set_line_cap(cairo.LINE_CAP_BUTT) - cr.set_line_join(cairo.LINE_JOIN_MITER) - - for shape in self.shapes: - shape.draw(cr) - for edge in self.edges: - edge.draw(cr, highlight=(edge in highlight_items)) - for node in self.nodes: - node.draw(cr, highlight=(node in highlight_items)) - - def get_element(self, x, y): - for node in self.nodes: - if node.is_inside(x, y): - return node - for edge in self.edges: - if edge.is_inside(x, y): - return edge - - def get_url(self, x, y): - for node in self.nodes: - url = node.get_url(x, y) - if url is not None: - return url - return None - - def get_jump(self, x, y): - for edge in self.edges: - jump = edge.get_jump(x, y) - if jump is not None: - return jump - for node in self.nodes: - jump = node.get_jump(x, y) - if jump is not None: - return jump - return None - - -BOLD = 1 -ITALIC = 2 -UNDERLINE = 4 -SUPERSCRIPT = 8 -SUBSCRIPT = 16 -STRIKE_THROUGH = 32 - - -class XDotAttrParser: - """Parser for xdot drawing attributes. - See also: - - http://www.graphviz.org/doc/info/output.html#d:xdot - """ - - def __init__(self, parser, buf): - self.parser = parser - self.buf = buf - self.pos = 0 - - self.pen = Pen() - self.shapes = [] - - def __nonzero__(self): - return self.pos < len(self.buf) - - def read_code(self): - pos = self.buf.find(" ", self.pos) - res = self.buf[self.pos:pos] - self.pos = pos + 1 - while self.pos < len(self.buf) and self.buf[self.pos].isspace(): - self.pos += 1 - return res - - def read_int(self): - return int(self.read_code()) - - def read_float(self): - return float(self.read_code()) - - def read_point(self): - x = self.read_float() - y = self.read_float() - return self.transform(x, y) - - def read_text(self): - num = self.read_int() - pos = self.buf.find("-", self.pos) + 1 - self.pos = pos + num - res = self.buf[pos:self.pos] - while self.pos < len(self.buf) and self.buf[self.pos].isspace(): - self.pos += 1 - return res - - def read_polygon(self): - n = self.read_int() - p = [] - for i in range(n): - x, y = self.read_point() - p.append((x, y)) - return p - - def read_color(self): - # See http://www.graphviz.org/doc/info/attrs.html#k:color - c = self.read_text() - c1 = c[:1] - if c1 == '#': - hex2float = lambda h: float(int(h, 16)/255.0) - r = hex2float(c[1:3]) - g = hex2float(c[3:5]) - b = hex2float(c[5:7]) - try: - a = hex2float(c[7:9]) - except (IndexError, ValueError): - a = 1.0 - return r, g, b, a - elif c1.isdigit() or c1 == ".": - # "H,S,V" or "H S V" or "H, S, V" or any other variation - h, s, v = map(float, c.replace(",", " ").split()) - r, g, b = colorsys.hsv_to_rgb(h, s, v) - a = 1.0 - return r, g, b, a - elif c1 == "[": - sys.stderr.write('warning: color gradients not supported yet\n') - return None - else: - return self.lookup_color(c) - - def lookup_color(self, c): - try: - color = gtk.gdk.color_parse(c) - except ValueError: - pass - else: - s = 1.0/65535.0 - r = color.red*s - g = color.green*s - b = color.blue*s - a = 1.0 - return r, g, b, a - - try: - dummy, scheme, index = c.split('/') - r, g, b = brewer_colors[scheme][int(index)] - except (ValueError, KeyError): - pass - else: - s = 1.0/255.0 - r = r*s - g = g*s - b = b*s - a = 1.0 - return r, g, b, a - - sys.stderr.write("warning: unknown color '%s'\n" % c) - return None - - def parse(self): - s = self - - while s: - op = s.read_code() - if op == "c": - color = s.read_color() - if color is not None: - self.handle_color(color, filled=False) - elif op == "C": - color = s.read_color() - if color is not None: - self.handle_color(color, filled=True) - elif op == "S": - # http://www.graphviz.org/doc/info/attrs.html#k:style - style = s.read_text() - if style.startswith("setlinewidth("): - lw = style.split("(")[1].split(")")[0] - lw = float(lw) - self.handle_linewidth(lw) - elif style in ("solid", "dashed", "dotted"): - self.handle_linestyle(style) - elif op == "F": - size = s.read_float() - name = s.read_text() - self.handle_font(size, name) - elif op == "T": - x, y = s.read_point() - j = s.read_int() - w = s.read_float() - t = s.read_text() - self.handle_text(x, y, j, w, t) - elif op == "t": - f = s.read_int() - self.handle_font_characteristics(f) - elif op == "E": - x0, y0 = s.read_point() - w = s.read_float() - h = s.read_float() - self.handle_ellipse(x0, y0, w, h, filled=True) - elif op == "e": - x0, y0 = s.read_point() - w = s.read_float() - h = s.read_float() - self.handle_ellipse(x0, y0, w, h, filled=False) - elif op == "L": - points = self.read_polygon() - self.handle_line(points) - elif op == "B": - points = self.read_polygon() - self.handle_bezier(points, filled=False) - elif op == "b": - points = self.read_polygon() - self.handle_bezier(points, filled=True) - elif op == "P": - points = self.read_polygon() - self.handle_polygon(points, filled=True) - elif op == "p": - points = self.read_polygon() - self.handle_polygon(points, filled=False) - elif op == "I": - x0, y0 = s.read_point() - w = s.read_float() - h = s.read_float() - path = s.read_text() - self.handle_image(x0, y0, w, h, path) - else: - sys.stderr.write("error: unknown xdot opcode '%s'\n" % op) - sys.exit(1) - - return self.shapes - - def transform(self, x, y): - return self.parser.transform(x, y) - - def handle_color(self, color, filled=False): - if filled: - self.pen.fillcolor = color - else: - self.pen.color = color - - def handle_linewidth(self, linewidth): - self.pen.linewidth = linewidth - - def handle_linestyle(self, style): - if style == "solid": - self.pen.dash = () - elif style == "dashed": - self.pen.dash = (6, ) # 6pt on, 6pt off - elif style == "dotted": - self.pen.dash = (2, 4) # 2pt on, 4pt off - - def handle_font(self, size, name): - self.pen.fontsize = size - self.pen.fontname = name - - def handle_font_characteristics(self, flags): - # TODO - if flags != 0: - sys.stderr.write("warning: font characteristics not supported yet\n") - - def handle_text(self, x, y, j, w, t): - self.shapes.append(TextShape(self.pen, x, y, j, w, t)) - - def handle_ellipse(self, x0, y0, w, h, filled=False): - if filled: - # xdot uses this to mean "draw a filled shape with an outline" - self.shapes.append(EllipseShape(self.pen, x0, y0, w, h, filled=True)) - self.shapes.append(EllipseShape(self.pen, x0, y0, w, h)) - - def handle_image(self, x0, y0, w, h, path): - self.shapes.append(ImageShape(self.pen, x0, y0, w, h, path)) - - def handle_line(self, points): - self.shapes.append(LineShape(self.pen, points)) - - def handle_bezier(self, points, filled=False): - if filled: - # xdot uses this to mean "draw a filled shape with an outline" - self.shapes.append(BezierShape(self.pen, points, filled=True)) - self.shapes.append(BezierShape(self.pen, points)) - - def handle_polygon(self, points, filled=False): - if filled: - # xdot uses this to mean "draw a filled shape with an outline" - self.shapes.append(PolygonShape(self.pen, points, filled=True)) - self.shapes.append(PolygonShape(self.pen, points)) - - -EOF = -1 -SKIP = -2 - - -class ParseError(Exception): - - def __init__(self, msg=None, filename=None, line=None, col=None): - self.msg = msg - self.filename = filename - self.line = line - self.col = col - - def __str__(self): - return ':'.join([str(part) for part in (self.filename, self.line, self.col, self.msg) if part != None]) - - -class Scanner: - """Stateless scanner.""" - - # should be overriden by derived classes - tokens = [] - symbols = {} - literals = {} - ignorecase = False - - def __init__(self): - flags = re.DOTALL - if self.ignorecase: - flags |= re.IGNORECASE - self.tokens_re = re.compile( - '|'.join(['(' + regexp + ')' for type, regexp, test_lit in self.tokens]), - flags - ) - - def next(self, buf, pos): - if pos >= len(buf): - return EOF, '', pos - mo = self.tokens_re.match(buf, pos) - if mo: - text = mo.group() - type, regexp, test_lit = self.tokens[mo.lastindex - 1] - pos = mo.end() - if test_lit: - type = self.literals.get(text, type) - return type, text, pos - else: - c = buf[pos] - return self.symbols.get(c, None), c, pos + 1 - - -class Token: - - def __init__(self, type, text, line, col): - self.type = type - self.text = text - self.line = line - self.col = col - - -class Lexer: - - # should be overriden by derived classes - scanner = None - tabsize = 8 - - newline_re = re.compile(r'\r\n?|\n') - - def __init__(self, buf = None, pos = 0, filename = None, fp = None): - if fp is not None: - try: - fileno = fp.fileno() - length = os.path.getsize(fp.name) - import mmap - except: - # read whole file into memory - buf = fp.read() - pos = 0 - else: - # map the whole file into memory - if length: - # length must not be zero - buf = mmap.mmap(fileno, length, access = mmap.ACCESS_READ) - pos = os.lseek(fileno, 0, 1) - else: - buf = '' - pos = 0 - - if filename is None: - try: - filename = fp.name - except AttributeError: - filename = None - - self.buf = buf - self.pos = pos - self.line = 1 - self.col = 1 - self.filename = filename - - def next(self): - while True: - # save state - pos = self.pos - line = self.line - col = self.col - - type, text, endpos = self.scanner.next(self.buf, pos) - assert pos + len(text) == endpos - self.consume(text) - type, text = self.filter(type, text) - self.pos = endpos - - if type == SKIP: - continue - elif type is None: - msg = 'unexpected char ' - if text >= ' ' and text <= '~': - msg += "'%s'" % text - else: - msg += "0x%X" % ord(text) - raise ParseError(msg, self.filename, line, col) - else: - break - return Token(type = type, text = text, line = line, col = col) - - def consume(self, text): - # update line number - pos = 0 - for mo in self.newline_re.finditer(text, pos): - self.line += 1 - self.col = 1 - pos = mo.end() - - # update column number - while True: - tabpos = text.find('\t', pos) - if tabpos == -1: - break - self.col += tabpos - pos - self.col = ((self.col - 1)//self.tabsize + 1)*self.tabsize + 1 - pos = tabpos + 1 - self.col += len(text) - pos - - -class Parser: - - def __init__(self, lexer): - self.lexer = lexer - self.lookahead = next(self.lexer) - - def match(self, type): - if self.lookahead.type != type: - raise ParseError( - msg = 'unexpected token %r' % self.lookahead.text, - filename = self.lexer.filename, - line = self.lookahead.line, - col = self.lookahead.col) - - def skip(self, type): - while self.lookahead.type != type: - self.consume() - - def consume(self): - token = self.lookahead - self.lookahead = next(self.lexer) - return token - - -ID = 0 -STR_ID = 1 -HTML_ID = 2 -EDGE_OP = 3 - -LSQUARE = 4 -RSQUARE = 5 -LCURLY = 6 -RCURLY = 7 -COMMA = 8 -COLON = 9 -SEMI = 10 -EQUAL = 11 -PLUS = 12 - -STRICT = 13 -GRAPH = 14 -DIGRAPH = 15 -NODE = 16 -EDGE = 17 -SUBGRAPH = 18 - - -class DotScanner(Scanner): - - # token regular expression table - tokens = [ - # whitespace and comments - (SKIP, - r'[ \t\f\r\n\v]+|' - r'//[^\r\n]*|' - r'/\*.*?\*/|' - r'#[^\r\n]*', - False), - - # Alphanumeric IDs - (ID, r'[a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*', True), - - # Numeric IDs - (ID, r'-?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)', False), - - # String IDs - (STR_ID, r'"[^"\\]*(?:\\.[^"\\]*)*"', False), - - # HTML IDs - (HTML_ID, r'<[^<>]*(?:<[^<>]*>[^<>]*)*>', False), - - # Edge operators - (EDGE_OP, r'-[>-]', False), - ] - - # symbol table - symbols = { - '[': LSQUARE, - ']': RSQUARE, - '{': LCURLY, - '}': RCURLY, - ',': COMMA, - ':': COLON, - ';': SEMI, - '=': EQUAL, - '+': PLUS, - } - - # literal table - literals = { - 'strict': STRICT, - 'graph': GRAPH, - 'digraph': DIGRAPH, - 'node': NODE, - 'edge': EDGE, - 'subgraph': SUBGRAPH, - } - - ignorecase = True - - -class DotLexer(Lexer): - - scanner = DotScanner() - - def filter(self, type, text): - # TODO: handle charset - if type == STR_ID: - text = text[1:-1] - - # line continuations - text = text.replace('\\\r\n', '') - text = text.replace('\\\r', '') - text = text.replace('\\\n', '') - - # quotes - text = text.replace('\\"', '"') - - # layout engines recognize other escape codes (many non-standard) - # but we don't translate them here - - type = ID - - elif type == HTML_ID: - text = text[1:-1] - type = ID - - return type, text - - -class DotParser(Parser): - - def __init__(self, lexer): - Parser.__init__(self, lexer) - self.graph_attrs = {} - self.node_attrs = {} - self.edge_attrs = {} - - def parse(self): - self.parse_graph() - self.match(EOF) - - def parse_graph(self): - if self.lookahead.type == STRICT: - self.consume() - self.skip(LCURLY) - self.consume() - while self.lookahead.type != RCURLY: - self.parse_stmt() - self.consume() - - def parse_subgraph(self): - id = None - if self.lookahead.type == SUBGRAPH: - self.consume() - if self.lookahead.type == ID: - id = self.lookahead.text - self.consume() - if self.lookahead.type == LCURLY: - self.consume() - while self.lookahead.type != RCURLY: - self.parse_stmt() - self.consume() - return id - - def parse_stmt(self): - if self.lookahead.type == GRAPH: - self.consume() - attrs = self.parse_attrs() - self.graph_attrs.update(attrs) - self.handle_graph(attrs) - elif self.lookahead.type == NODE: - self.consume() - self.node_attrs.update(self.parse_attrs()) - elif self.lookahead.type == EDGE: - self.consume() - self.edge_attrs.update(self.parse_attrs()) - elif self.lookahead.type in (SUBGRAPH, LCURLY): - self.parse_subgraph() - else: - id = self.parse_node_id() - if self.lookahead.type == EDGE_OP: - self.consume() - node_ids = [id, self.parse_node_id()] - while self.lookahead.type == EDGE_OP: - node_ids.append(self.parse_node_id()) - attrs = self.parse_attrs() - for i in range(0, len(node_ids) - 1): - self.handle_edge(node_ids[i], node_ids[i + 1], attrs) - elif self.lookahead.type == EQUAL: - self.consume() - self.parse_id() - else: - attrs = self.parse_attrs() - self.handle_node(id, attrs) - if self.lookahead.type == SEMI: - self.consume() - - def parse_attrs(self): - attrs = {} - while self.lookahead.type == LSQUARE: - self.consume() - while self.lookahead.type != RSQUARE: - name, value = self.parse_attr() - attrs[name] = value - if self.lookahead.type == COMMA: - self.consume() - self.consume() - return attrs - - def parse_attr(self): - name = self.parse_id() - if self.lookahead.type == EQUAL: - self.consume() - value = self.parse_id() - else: - value = 'true' - return name, value - - def parse_node_id(self): - node_id = self.parse_id() - if self.lookahead.type == COLON: - self.consume() - port = self.parse_id() - if self.lookahead.type == COLON: - self.consume() - compass_pt = self.parse_id() - else: - compass_pt = None - else: - port = None - compass_pt = None - # XXX: we don't really care about port and compass point values when parsing xdot - return node_id - - def parse_id(self): - self.match(ID) - id = self.lookahead.text - self.consume() - return id - - def handle_graph(self, attrs): - pass - - def handle_node(self, id, attrs): - pass - - def handle_edge(self, src_id, dst_id, attrs): - pass - - -class XDotParser(DotParser): - - XDOTVERSION = '1.6' - - def __init__(self, xdotcode): - lexer = DotLexer(buf = xdotcode) - DotParser.__init__(self, lexer) - - self.nodes = [] - self.edges = [] - self.shapes = [] - self.node_by_name = {} - self.top_graph = True - - def handle_graph(self, attrs): - if self.top_graph: - # Check xdot version - try: - xdotversion = attrs['xdotversion'] - except KeyError: - pass - else: - if float(xdotversion) > float(self.XDOTVERSION): - sys.stderr.write('warning: xdot version %s, but supported is %s\n' % (xdotversion, self.XDOTVERSION)) - - # Parse bounding box - try: - bb = attrs['bb'] - except KeyError: - return - - if bb: - xmin, ymin, xmax, ymax = map(float, bb.split(",")) - - self.xoffset = -xmin - self.yoffset = -ymax - self.xscale = 1.0 - self.yscale = -1.0 - # FIXME: scale from points to pixels - - self.width = max(xmax - xmin, 1) - self.height = max(ymax - ymin, 1) - - self.top_graph = False - - for attr in ("_draw_", "_ldraw_", "_hdraw_", "_tdraw_", "_hldraw_", "_tldraw_"): - if attr in attrs: - parser = XDotAttrParser(self, attrs[attr]) - self.shapes.extend(parser.parse()) - - def handle_node(self, id, attrs): - try: - pos = attrs['pos'] - except KeyError: - return - - x, y = self.parse_node_pos(pos) - w = float(attrs.get('width', 0))*72 - h = float(attrs.get('height', 0))*72 - shapes = [] - for attr in ("_draw_", "_ldraw_"): - if attr in attrs: - parser = XDotAttrParser(self, attrs[attr]) - shapes.extend(parser.parse()) - url = attrs.get('URL', None) - node = Node(id, x, y, w, h, shapes, url) - self.node_by_name[id] = node - if shapes: - self.nodes.append(node) - - def handle_edge(self, src_id, dst_id, attrs): - try: - pos = attrs['pos'] - except KeyError: - return - - points = self.parse_edge_pos(pos) - shapes = [] - for attr in ("_draw_", "_ldraw_", "_hdraw_", "_tdraw_", "_hldraw_", "_tldraw_"): - if attr in attrs: - parser = XDotAttrParser(self, attrs[attr]) - shapes.extend(parser.parse()) - if shapes: - src = self.node_by_name[src_id] - dst = self.node_by_name[dst_id] - self.edges.append(Edge(src, dst, points, shapes)) - - def parse(self): - DotParser.parse(self) - - return Graph(self.width, self.height, self.shapes, self.nodes, self.edges) - - def parse_node_pos(self, pos): - x, y = pos.split(",") - return self.transform(float(x), float(y)) - - def parse_edge_pos(self, pos): - points = [] - for entry in pos.split(' '): - fields = entry.split(',') - try: - x, y = fields - except ValueError: - # TODO: handle start/end points - continue - else: - points.append(self.transform(float(x), float(y))) - return points - - def transform(self, x, y): - # XXX: this is not the right place for this code - x = (x + self.xoffset)*self.xscale - y = (y + self.yoffset)*self.yscale - return x, y - - -class Animation(object): - - step = 0.03 # seconds - - def __init__(self, dot_widget): - self.dot_widget = dot_widget - self.timeout_id = None - - def start(self): - self.timeout_id = gobject.timeout_add(int(self.step * 1000), self.tick) - - def stop(self): - self.dot_widget.animation = NoAnimation(self.dot_widget) - if self.timeout_id is not None: - gobject.source_remove(self.timeout_id) - self.timeout_id = None - - def tick(self): - self.stop() - - -class NoAnimation(Animation): - - def start(self): - pass - - def stop(self): - pass - - -class LinearAnimation(Animation): - - duration = 0.6 - - def start(self): - self.started = time.time() - Animation.start(self) - - def tick(self): - t = (time.time() - self.started) / self.duration - self.animate(max(0, min(t, 1))) - return (t < 1) - - def animate(self, t): - pass - - -class MoveToAnimation(LinearAnimation): - - def __init__(self, dot_widget, target_x, target_y): - Animation.__init__(self, dot_widget) - self.source_x = dot_widget.x - self.source_y = dot_widget.y - self.target_x = target_x - self.target_y = target_y - - def animate(self, t): - sx, sy = self.source_x, self.source_y - tx, ty = self.target_x, self.target_y - self.dot_widget.x = tx * t + sx * (1-t) - self.dot_widget.y = ty * t + sy * (1-t) - self.dot_widget.queue_draw() - - -class ZoomToAnimation(MoveToAnimation): - - def __init__(self, dot_widget, target_x, target_y): - MoveToAnimation.__init__(self, dot_widget, target_x, target_y) - self.source_zoom = dot_widget.zoom_ratio - self.target_zoom = self.source_zoom - self.extra_zoom = 0 - - middle_zoom = 0.5 * (self.source_zoom + self.target_zoom) - - distance = math.hypot(self.source_x - self.target_x, - self.source_y - self.target_y) - rect = self.dot_widget.get_allocation() - visible = min(rect.width, rect.height) / self.dot_widget.zoom_ratio - visible *= 0.9 - if distance > 0: - desired_middle_zoom = visible / distance - self.extra_zoom = min(0, 4 * (desired_middle_zoom - middle_zoom)) - - def animate(self, t): - a, b, c = self.source_zoom, self.extra_zoom, self.target_zoom - self.dot_widget.zoom_ratio = c*t + b*t*(1-t) + a*(1-t) - self.dot_widget.zoom_to_fit_on_resize = False - MoveToAnimation.animate(self, t) - - -class DragAction(object): - - def __init__(self, dot_widget): - self.dot_widget = dot_widget - - def on_button_press(self, event): - self.startmousex = self.prevmousex = event.x - self.startmousey = self.prevmousey = event.y - self.start() - - def on_motion_notify(self, event): - if event.is_hint: - x, y, state = event.window.get_pointer() - else: - x, y, state = event.x, event.y, event.state - deltax = self.prevmousex - x - deltay = self.prevmousey - y - self.drag(deltax, deltay) - self.prevmousex = x - self.prevmousey = y - - def on_button_release(self, event): - self.stopmousex = event.x - self.stopmousey = event.y - self.stop() - - def draw(self, cr): - pass - - def start(self): - pass - - def drag(self, deltax, deltay): - pass - - def stop(self): - pass - - def abort(self): - pass - - -class NullAction(DragAction): - - def on_motion_notify(self, event): - if event.is_hint: - x, y, state = event.window.get_pointer() - else: - x, y, state = event.x, event.y, event.state - dot_widget = self.dot_widget - item = dot_widget.get_url(x, y) - if item is None: - item = dot_widget.get_jump(x, y) - if item is not None: - dot_widget.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.HAND2)) - dot_widget.set_highlight(item.highlight) - else: - dot_widget.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.ARROW)) - dot_widget.set_highlight(None) - - -class PanAction(DragAction): - - def start(self): - self.dot_widget.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.FLEUR)) - - def drag(self, deltax, deltay): - self.dot_widget.x += deltax / self.dot_widget.zoom_ratio - self.dot_widget.y += deltay / self.dot_widget.zoom_ratio - self.dot_widget.queue_draw() - - def stop(self): - self.dot_widget.window.set_cursor(gtk.gdk.Cursor(gtk.gdk.ARROW)) - - abort = stop - - -class ZoomAction(DragAction): - - def drag(self, deltax, deltay): - self.dot_widget.zoom_ratio *= 1.005 ** (deltax + deltay) - self.dot_widget.zoom_to_fit_on_resize = False - self.dot_widget.queue_draw() - - def stop(self): - self.dot_widget.queue_draw() - - -class ZoomAreaAction(DragAction): - - def drag(self, deltax, deltay): - self.dot_widget.queue_draw() - - def draw(self, cr): - cr.save() - cr.set_source_rgba(.5, .5, 1.0, 0.25) - cr.rectangle(self.startmousex, self.startmousey, - self.prevmousex - self.startmousex, - self.prevmousey - self.startmousey) - cr.fill() - cr.set_source_rgba(.5, .5, 1.0, 1.0) - cr.set_line_width(1) - cr.rectangle(self.startmousex - .5, self.startmousey - .5, - self.prevmousex - self.startmousex + 1, - self.prevmousey - self.startmousey + 1) - cr.stroke() - cr.restore() - - def stop(self): - x1, y1 = self.dot_widget.window2graph(self.startmousex, - self.startmousey) - x2, y2 = self.dot_widget.window2graph(self.stopmousex, - self.stopmousey) - self.dot_widget.zoom_to_area(x1, y1, x2, y2) - - def abort(self): - self.dot_widget.queue_draw() - - -class DotWidget(gtk.DrawingArea): - """PyGTK widget that draws dot graphs.""" - - __gsignals__ = { - 'expose-event': 'override', - 'clicked' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, (gobject.TYPE_STRING, gtk.gdk.Event)) - } - - filter = 'dot' - - def __init__(self): - gtk.DrawingArea.__init__(self) - - self.graph = Graph() - self.openfilename = None - - self.set_flags(gtk.CAN_FOCUS) - - self.add_events(gtk.gdk.BUTTON_PRESS_MASK | gtk.gdk.BUTTON_RELEASE_MASK) - self.connect("button-press-event", self.on_area_button_press) - self.connect("button-release-event", self.on_area_button_release) - self.add_events(gtk.gdk.POINTER_MOTION_MASK | gtk.gdk.POINTER_MOTION_HINT_MASK | gtk.gdk.BUTTON_RELEASE_MASK) - self.connect("motion-notify-event", self.on_area_motion_notify) - self.connect("scroll-event", self.on_area_scroll_event) - self.connect("size-allocate", self.on_area_size_allocate) - - self.connect('key-press-event', self.on_key_press_event) - self.last_mtime = None - - gobject.timeout_add(1000, self.update) - - self.x, self.y = 0.0, 0.0 - self.zoom_ratio = 1.0 - self.zoom_to_fit_on_resize = False - self.animation = NoAnimation(self) - self.drag_action = NullAction(self) - self.presstime = None - self.highlight = None - - def set_filter(self, filter): - self.filter = filter - - def run_filter(self, dotcode): - if not self.filter: - return dotcode - p = subprocess.Popen( - [self.filter, '-Txdot'], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False, - universal_newlines=True - ) - xdotcode, error = p.communicate(dotcode) - sys.stderr.write(error) - if p.returncode != 0: - dialog = gtk.MessageDialog(type=gtk.MESSAGE_ERROR, - message_format=error, - buttons=gtk.BUTTONS_OK) - dialog.set_title('Dot Viewer') - dialog.run() - dialog.destroy() - return None - return xdotcode - - def set_dotcode(self, dotcode, filename=None): - self.openfilename = None - if isinstance(dotcode, unicode): - dotcode = dotcode.encode('utf8') - xdotcode = self.run_filter(dotcode) - if xdotcode is None: - return False - try: - self.set_xdotcode(xdotcode) - except ParseError as ex: - dialog = gtk.MessageDialog(type=gtk.MESSAGE_ERROR, - message_format=str(ex), - buttons=gtk.BUTTONS_OK) - dialog.set_title('Dot Viewer') - dialog.run() - dialog.destroy() - return False - else: - if filename is None: - self.last_mtime = None - else: - self.last_mtime = os.stat(filename).st_mtime - self.openfilename = filename - return True - - def set_xdotcode(self, xdotcode): - parser = XDotParser(xdotcode) - self.graph = parser.parse() - self.zoom_image(self.zoom_ratio, center=True) - - def reload(self): - if self.openfilename is not None: - try: - fp = file(self.openfilename, 'rt') - self.set_dotcode(fp.read(), self.openfilename) - fp.close() - except IOError: - pass - - def update(self): - if self.openfilename is not None: - current_mtime = os.stat(self.openfilename).st_mtime - if current_mtime != self.last_mtime: - self.last_mtime = current_mtime - self.reload() - return True - - def do_expose_event(self, event): - cr = self.window.cairo_create() - - # set a clip region for the expose event - cr.rectangle( - event.area.x, event.area.y, - event.area.width, event.area.height - ) - cr.clip() - - cr.set_source_rgba(1.0, 1.0, 1.0, 1.0) - cr.paint() - - cr.save() - rect = self.get_allocation() - cr.translate(0.5*rect.width, 0.5*rect.height) - cr.scale(self.zoom_ratio, self.zoom_ratio) - cr.translate(-self.x, -self.y) - - self.graph.draw(cr, highlight_items=self.highlight) - cr.restore() - - self.drag_action.draw(cr) - - return False - - def get_current_pos(self): - return self.x, self.y - - def set_current_pos(self, x, y): - self.x = x - self.y = y - self.queue_draw() - - def set_highlight(self, items): - if self.highlight != items: - self.highlight = items - self.queue_draw() - - def zoom_image(self, zoom_ratio, center=False, pos=None): - # Constrain zoom ratio to a sane range to prevent numeric instability. - zoom_ratio = min(zoom_ratio, 1E4) - zoom_ratio = max(zoom_ratio, 1E-6) - - if center: - self.x = self.graph.width/2 - self.y = self.graph.height/2 - elif pos is not None: - rect = self.get_allocation() - x, y = pos - x -= 0.5*rect.width - y -= 0.5*rect.height - self.x += x / self.zoom_ratio - x / zoom_ratio - self.y += y / self.zoom_ratio - y / zoom_ratio - self.zoom_ratio = zoom_ratio - self.zoom_to_fit_on_resize = False - self.queue_draw() - - def zoom_to_area(self, x1, y1, x2, y2): - rect = self.get_allocation() - width = abs(x1 - x2) - height = abs(y1 - y2) - if width == 0 and height == 0: - self.zoom_ratio *= self.ZOOM_INCREMENT - else: - self.zoom_ratio = min( - float(rect.width)/float(width), - float(rect.height)/float(height) - ) - self.zoom_to_fit_on_resize = False - self.x = (x1 + x2) / 2 - self.y = (y1 + y2) / 2 - self.queue_draw() - - def zoom_to_fit(self): - rect = self.get_allocation() - rect.x += self.ZOOM_TO_FIT_MARGIN - rect.y += self.ZOOM_TO_FIT_MARGIN - rect.width -= 2 * self.ZOOM_TO_FIT_MARGIN - rect.height -= 2 * self.ZOOM_TO_FIT_MARGIN - zoom_ratio = min( - float(rect.width)/float(self.graph.width), - float(rect.height)/float(self.graph.height) - ) - self.zoom_image(zoom_ratio, center=True) - self.zoom_to_fit_on_resize = True - - ZOOM_INCREMENT = 1.25 - ZOOM_TO_FIT_MARGIN = 12 - - def on_zoom_in(self, action): - self.zoom_image(self.zoom_ratio * self.ZOOM_INCREMENT) - - def on_zoom_out(self, action): - self.zoom_image(self.zoom_ratio / self.ZOOM_INCREMENT) - - def on_zoom_fit(self, action): - self.zoom_to_fit() - - def on_zoom_100(self, action): - self.zoom_image(1.0) - - POS_INCREMENT = 100 - - def on_key_press_event(self, widget, event): - if event.keyval == gtk.keysyms.Left: - self.x -= self.POS_INCREMENT/self.zoom_ratio - self.queue_draw() - return True - if event.keyval == gtk.keysyms.Right: - self.x += self.POS_INCREMENT/self.zoom_ratio - self.queue_draw() - return True - if event.keyval == gtk.keysyms.Up: - self.y -= self.POS_INCREMENT/self.zoom_ratio - self.queue_draw() - return True - if event.keyval == gtk.keysyms.Down: - self.y += self.POS_INCREMENT/self.zoom_ratio - self.queue_draw() - return True - if event.keyval in (gtk.keysyms.Page_Up, - gtk.keysyms.plus, - gtk.keysyms.equal, - gtk.keysyms.KP_Add): - self.zoom_image(self.zoom_ratio * self.ZOOM_INCREMENT) - self.queue_draw() - return True - if event.keyval in (gtk.keysyms.Page_Down, - gtk.keysyms.minus, - gtk.keysyms.KP_Subtract): - self.zoom_image(self.zoom_ratio / self.ZOOM_INCREMENT) - self.queue_draw() - return True - if event.keyval == gtk.keysyms.Escape: - self.drag_action.abort() - self.drag_action = NullAction(self) - return True - if event.keyval == gtk.keysyms.r: - self.reload() - return True - if event.keyval == gtk.keysyms.f: - win = widget.get_toplevel() - find_toolitem = win.uimanager.get_widget('/ToolBar/Find') - textentry = find_toolitem.get_children() - win.set_focus(textentry[0]) - return True - if event.keyval == gtk.keysyms.q: - gtk.main_quit() - return True - if event.keyval == gtk.keysyms.p: - self.on_print() - return True - return False - - print_settings = None - def on_print(self, action=None): - print_op = gtk.PrintOperation() - - if self.print_settings != None: - print_op.set_print_settings(self.print_settings) - - print_op.connect("begin_print", self.begin_print) - print_op.connect("draw_page", self.draw_page) - - res = print_op.run(gtk.PRINT_OPERATION_ACTION_PRINT_DIALOG, self.parent.parent) - - if res == gtk.PRINT_OPERATION_RESULT_APPLY: - print_settings = print_op.get_print_settings() - - def begin_print(self, operation, context): - operation.set_n_pages(1) - return True - - def draw_page(self, operation, context, page_nr): - cr = context.get_cairo_context() - - rect = self.get_allocation() - cr.translate(0.5*rect.width, 0.5*rect.height) - cr.scale(self.zoom_ratio, self.zoom_ratio) - cr.translate(-self.x, -self.y) - - self.graph.draw(cr, highlight_items=self.highlight) - - def get_drag_action(self, event): - state = event.state - if event.button in (1, 2): # left or middle button - if state & gtk.gdk.CONTROL_MASK: - return ZoomAction - elif state & gtk.gdk.SHIFT_MASK: - return ZoomAreaAction - else: - return PanAction - return NullAction - - def on_area_button_press(self, area, event): - self.animation.stop() - self.drag_action.abort() - action_type = self.get_drag_action(event) - self.drag_action = action_type(self) - self.drag_action.on_button_press(event) - self.presstime = time.time() - self.pressx = event.x - self.pressy = event.y - return False - - def is_click(self, event, click_fuzz=4, click_timeout=1.0): - assert event.type == gtk.gdk.BUTTON_RELEASE - if self.presstime is None: - # got a button release without seeing the press? - return False - # XXX instead of doing this complicated logic, shouldn't we listen - # for gtk's clicked event instead? - deltax = self.pressx - event.x - deltay = self.pressy - event.y - return (time.time() < self.presstime + click_timeout - and math.hypot(deltax, deltay) < click_fuzz) - - def on_click(self, element, event): - """Override this method in subclass to process - click events. Note that element can be None - (click on empty space).""" - return False - - def on_area_button_release(self, area, event): - self.drag_action.on_button_release(event) - self.drag_action = NullAction(self) - x, y = int(event.x), int(event.y) - if self.is_click(event): - el = self.get_element(x, y) - if self.on_click(el, event): - return True - - if event.button == 1: - url = self.get_url(x, y) - if url is not None: - self.emit('clicked', unicode(url.url), event) - else: - jump = self.get_jump(x, y) - if jump is not None: - self.animate_to(jump.x, jump.y) - - return True - - if event.button == 1 or event.button == 2: - return True - return False - - def on_area_scroll_event(self, area, event): - if event.direction == gtk.gdk.SCROLL_UP: - self.zoom_image(self.zoom_ratio * self.ZOOM_INCREMENT, - pos=(event.x, event.y)) - return True - if event.direction == gtk.gdk.SCROLL_DOWN: - self.zoom_image(self.zoom_ratio / self.ZOOM_INCREMENT, - pos=(event.x, event.y)) - return True - return False - - def on_area_motion_notify(self, area, event): - self.drag_action.on_motion_notify(event) - return True - - def on_area_size_allocate(self, area, allocation): - if self.zoom_to_fit_on_resize: - self.zoom_to_fit() - - def animate_to(self, x, y): - self.animation = ZoomToAnimation(self, x, y) - self.animation.start() - - def window2graph(self, x, y): - rect = self.get_allocation() - x -= 0.5*rect.width - y -= 0.5*rect.height - x /= self.zoom_ratio - y /= self.zoom_ratio - x += self.x - y += self.y - return x, y - - def get_element(self, x, y): - x, y = self.window2graph(x, y) - return self.graph.get_element(x, y) - - def get_url(self, x, y): - x, y = self.window2graph(x, y) - return self.graph.get_url(x, y) - - def get_jump(self, x, y): - x, y = self.window2graph(x, y) - return self.graph.get_jump(x, y) - - -class FindMenuToolAction(gtk.Action): - __gtype_name__ = "FindMenuToolAction" - - def __init__(self, *args, **kw): - gtk.Action.__init__(self, *args, **kw) - self.set_tool_item_type(gtk.ToolItem) - - -class DotWindow(gtk.Window): - - ui = ''' - - - - - - - - - - - - - - - ''' - - base_title = 'Dot Viewer' - - def __init__(self, widget=None): - gtk.Window.__init__(self) - - self.graph = Graph() - - window = self - - window.set_title(self.base_title) - window.set_default_size(512, 512) - vbox = gtk.VBox() - window.add(vbox) - - self.widget = widget or DotWidget() - - # Create a UIManager instance - uimanager = self.uimanager = gtk.UIManager() - - # Add the accelerator group to the toplevel window - accelgroup = uimanager.get_accel_group() - window.add_accel_group(accelgroup) - - # Create an ActionGroup - actiongroup = gtk.ActionGroup('Actions') - self.actiongroup = actiongroup - - # Create actions - actiongroup.add_actions(( - ('Open', gtk.STOCK_OPEN, None, None, None, self.on_open), - ('Reload', gtk.STOCK_REFRESH, None, None, None, self.on_reload), - ('Print', gtk.STOCK_PRINT, None, None, "Prints the currently visible part of the graph", self.widget.on_print), - ('ZoomIn', gtk.STOCK_ZOOM_IN, None, None, None, self.widget.on_zoom_in), - ('ZoomOut', gtk.STOCK_ZOOM_OUT, None, None, None, self.widget.on_zoom_out), - ('ZoomFit', gtk.STOCK_ZOOM_FIT, None, None, None, self.widget.on_zoom_fit), - ('Zoom100', gtk.STOCK_ZOOM_100, None, None, None, self.widget.on_zoom_100), - )) - - find_action = FindMenuToolAction("Find", None, - "Find a node by name", None) - actiongroup.add_action(find_action) - - # Add the actiongroup to the uimanager - uimanager.insert_action_group(actiongroup, 0) - - # Add a UI descrption - uimanager.add_ui_from_string(self.ui) - - # Create a Toolbar - toolbar = uimanager.get_widget('/ToolBar') - vbox.pack_start(toolbar, False) - - vbox.pack_start(self.widget) - - self.last_open_dir = "." - - self.set_focus(self.widget) - - # Add Find text search - find_toolitem = uimanager.get_widget('/ToolBar/Find') - self.textentry = gtk.Entry(max=20) - self.textentry.set_icon_from_stock(0, gtk.STOCK_FIND) - find_toolitem.add(self.textentry) - - self.textentry.set_activates_default(True) - self.textentry.connect ("activate", self.textentry_activate, self.textentry); - self.textentry.connect ("changed", self.textentry_changed, self.textentry); - - self.show_all() - - def find_text(self, entry_text): - found_items = [] - dot_widget = self.widget - regexp = re.compile(entry_text) - for node in dot_widget.graph.nodes: - if node.search_text(regexp): - found_items.append(node) - return found_items - - def textentry_changed(self, widget, entry): - entry_text = entry.get_text() - dot_widget = self.widget - if not entry_text: - dot_widget.set_highlight(None) - return - - found_items = self.find_text(entry_text) - dot_widget.set_highlight(found_items) - - def textentry_activate(self, widget, entry): - entry_text = entry.get_text() - dot_widget = self.widget - if not entry_text: - dot_widget.set_highlight(None) - return; - - found_items = self.find_text(entry_text) - dot_widget.set_highlight(found_items) - if(len(found_items) == 1): - dot_widget.animate_to(found_items[0].x, found_items[0].y) - - def set_filter(self, filter): - self.widget.set_filter(filter) - - def set_dotcode(self, dotcode, filename=None): - if self.widget.set_dotcode(dotcode, filename): - self.update_title(filename) - self.widget.zoom_to_fit() - - def set_xdotcode(self, xdotcode, filename=None): - if self.widget.set_xdotcode(xdotcode): - self.update_title(filename) - self.widget.zoom_to_fit() - - def update_title(self, filename=None): - if filename is None: - self.set_title(self.base_title) - else: - self.set_title(os.path.basename(filename) + ' - ' + self.base_title) - - def open_file(self, filename): - try: - fp = file(filename, 'rt') - self.set_dotcode(fp.read(), filename) - fp.close() - except IOError as ex: - dlg = gtk.MessageDialog(type=gtk.MESSAGE_ERROR, - message_format=str(ex), - buttons=gtk.BUTTONS_OK) - dlg.set_title(self.base_title) - dlg.run() - dlg.destroy() - - def on_open(self, action): - chooser = gtk.FileChooserDialog(title="Open dot File", - action=gtk.FILE_CHOOSER_ACTION_OPEN, - buttons=(gtk.STOCK_CANCEL, - gtk.RESPONSE_CANCEL, - gtk.STOCK_OPEN, - gtk.RESPONSE_OK)) - chooser.set_default_response(gtk.RESPONSE_OK) - chooser.set_current_folder(self.last_open_dir) - filter = gtk.FileFilter() - filter.set_name("Graphviz dot files") - filter.add_pattern("*.dot") - chooser.add_filter(filter) - filter = gtk.FileFilter() - filter.set_name("All files") - filter.add_pattern("*") - chooser.add_filter(filter) - if chooser.run() == gtk.RESPONSE_OK: - filename = chooser.get_filename() - self.last_open_dir = chooser.get_current_folder() - chooser.destroy() - self.open_file(filename) - else: - chooser.destroy() - - def on_reload(self, action): - self.widget.reload() - - -class OptionParser(optparse.OptionParser): - - def format_epilog(self, formatter): - # Prevent stripping the newlines in epilog message - # http://stackoverflow.com/questions/1857346/python-optparse-how-to-include-additional-info-in-usage-output - return self.epilog - - -def main(): - - parser = OptionParser( - usage='\n\t%prog [file]', - epilog=''' -Shortcuts: - Up, Down, Left, Right scroll - PageUp, +, = zoom in - PageDown, - zoom out - R reload dot file - F find - Q quit - P print - Escape halt animation - Ctrl-drag zoom in/out - Shift-drag zooms an area -''' - ) - parser.add_option( - '-f', '--filter', - type='choice', choices=('dot', 'neato', 'twopi', 'circo', 'fdp'), - dest='filter', default='dot', - help='graphviz filter: dot, neato, twopi, circo, or fdp [default: %default]') - parser.add_option( - '-n', '--no-filter', - action='store_const', const=None, dest='filter', - help='assume input is already filtered into xdot format (use e.g. dot -Txdot)') - - (options, args) = parser.parse_args(sys.argv[1:]) - if len(args) > 1: - parser.error('incorrect number of arguments') - - win = DotWindow() - win.connect('destroy', gtk.main_quit) - win.set_filter(options.filter) - if len(args) == 0: - if not sys.stdin.isatty(): - win.set_dotcode(sys.stdin.read()) - else: - if args[0] == '-': - win.set_dotcode(sys.stdin.read()) - else: - win.open_file(args[0]) - gtk.main() - - -# Apache-Style Software License for ColorBrewer software and ColorBrewer Color -# Schemes, Version 1.1 -# -# Copyright (c) 2002 Cynthia Brewer, Mark Harrower, and The Pennsylvania State -# University. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions as source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. The end-user documentation included with the redistribution, if any, -# must include the following acknowledgment: -# -# This product includes color specifications and designs developed by -# Cynthia Brewer (http://colorbrewer.org/). -# -# Alternately, this acknowledgment may appear in the software itself, if and -# wherever such third-party acknowledgments normally appear. -# -# 3. The name "ColorBrewer" must not be used to endorse or promote products -# derived from this software without prior written permission. For written -# permission, please contact Cynthia Brewer at cbrewer@psu.edu. -# -# 4. Products derived from this software may not be called "ColorBrewer", -# nor may "ColorBrewer" appear in their name, without prior written -# permission of Cynthia Brewer. -# -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESSED OR IMPLIED WARRANTIES, -# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CYNTHIA -# BREWER, MARK HARROWER, OR THE PENNSYLVANIA STATE UNIVERSITY BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -brewer_colors = { - 'accent3': [(127, 201, 127), (190, 174, 212), (253, 192, 134)], - 'accent4': [(127, 201, 127), (190, 174, 212), (253, 192, 134), (255, 255, 153)], - 'accent5': [(127, 201, 127), (190, 174, 212), (253, 192, 134), (255, 255, 153), (56, 108, 176)], - 'accent6': [(127, 201, 127), (190, 174, 212), (253, 192, 134), (255, 255, 153), (56, 108, 176), (240, 2, 127)], - 'accent7': [(127, 201, 127), (190, 174, 212), (253, 192, 134), (255, 255, 153), (56, 108, 176), (240, 2, 127), (191, 91, 23)], - 'accent8': [(127, 201, 127), (190, 174, 212), (253, 192, 134), (255, 255, 153), (56, 108, 176), (240, 2, 127), (191, 91, 23), (102, 102, 102)], - 'blues3': [(222, 235, 247), (158, 202, 225), (49, 130, 189)], - 'blues4': [(239, 243, 255), (189, 215, 231), (107, 174, 214), (33, 113, 181)], - 'blues5': [(239, 243, 255), (189, 215, 231), (107, 174, 214), (49, 130, 189), (8, 81, 156)], - 'blues6': [(239, 243, 255), (198, 219, 239), (158, 202, 225), (107, 174, 214), (49, 130, 189), (8, 81, 156)], - 'blues7': [(239, 243, 255), (198, 219, 239), (158, 202, 225), (107, 174, 214), (66, 146, 198), (33, 113, 181), (8, 69, 148)], - 'blues8': [(247, 251, 255), (222, 235, 247), (198, 219, 239), (158, 202, 225), (107, 174, 214), (66, 146, 198), (33, 113, 181), (8, 69, 148)], - 'blues9': [(247, 251, 255), (222, 235, 247), (198, 219, 239), (158, 202, 225), (107, 174, 214), (66, 146, 198), (33, 113, 181), (8, 81, 156), (8, 48, 107)], - 'brbg10': [(84, 48, 5), (0, 60, 48), (140, 81, 10), (191, 129, 45), (223, 194, 125), (246, 232, 195), (199, 234, 229), (128, 205, 193), (53, 151, 143), (1, 102, 94)], - 'brbg11': [(84, 48, 5), (1, 102, 94), (0, 60, 48), (140, 81, 10), (191, 129, 45), (223, 194, 125), (246, 232, 195), (245, 245, 245), (199, 234, 229), (128, 205, 193), (53, 151, 143)], - 'brbg3': [(216, 179, 101), (245, 245, 245), (90, 180, 172)], - 'brbg4': [(166, 97, 26), (223, 194, 125), (128, 205, 193), (1, 133, 113)], - 'brbg5': [(166, 97, 26), (223, 194, 125), (245, 245, 245), (128, 205, 193), (1, 133, 113)], - 'brbg6': [(140, 81, 10), (216, 179, 101), (246, 232, 195), (199, 234, 229), (90, 180, 172), (1, 102, 94)], - 'brbg7': [(140, 81, 10), (216, 179, 101), (246, 232, 195), (245, 245, 245), (199, 234, 229), (90, 180, 172), (1, 102, 94)], - 'brbg8': [(140, 81, 10), (191, 129, 45), (223, 194, 125), (246, 232, 195), (199, 234, 229), (128, 205, 193), (53, 151, 143), (1, 102, 94)], - 'brbg9': [(140, 81, 10), (191, 129, 45), (223, 194, 125), (246, 232, 195), (245, 245, 245), (199, 234, 229), (128, 205, 193), (53, 151, 143), (1, 102, 94)], - 'bugn3': [(229, 245, 249), (153, 216, 201), (44, 162, 95)], - 'bugn4': [(237, 248, 251), (178, 226, 226), (102, 194, 164), (35, 139, 69)], - 'bugn5': [(237, 248, 251), (178, 226, 226), (102, 194, 164), (44, 162, 95), (0, 109, 44)], - 'bugn6': [(237, 248, 251), (204, 236, 230), (153, 216, 201), (102, 194, 164), (44, 162, 95), (0, 109, 44)], - 'bugn7': [(237, 248, 251), (204, 236, 230), (153, 216, 201), (102, 194, 164), (65, 174, 118), (35, 139, 69), (0, 88, 36)], - 'bugn8': [(247, 252, 253), (229, 245, 249), (204, 236, 230), (153, 216, 201), (102, 194, 164), (65, 174, 118), (35, 139, 69), (0, 88, 36)], - 'bugn9': [(247, 252, 253), (229, 245, 249), (204, 236, 230), (153, 216, 201), (102, 194, 164), (65, 174, 118), (35, 139, 69), (0, 109, 44), (0, 68, 27)], - 'bupu3': [(224, 236, 244), (158, 188, 218), (136, 86, 167)], - 'bupu4': [(237, 248, 251), (179, 205, 227), (140, 150, 198), (136, 65, 157)], - 'bupu5': [(237, 248, 251), (179, 205, 227), (140, 150, 198), (136, 86, 167), (129, 15, 124)], - 'bupu6': [(237, 248, 251), (191, 211, 230), (158, 188, 218), (140, 150, 198), (136, 86, 167), (129, 15, 124)], - 'bupu7': [(237, 248, 251), (191, 211, 230), (158, 188, 218), (140, 150, 198), (140, 107, 177), (136, 65, 157), (110, 1, 107)], - 'bupu8': [(247, 252, 253), (224, 236, 244), (191, 211, 230), (158, 188, 218), (140, 150, 198), (140, 107, 177), (136, 65, 157), (110, 1, 107)], - 'bupu9': [(247, 252, 253), (224, 236, 244), (191, 211, 230), (158, 188, 218), (140, 150, 198), (140, 107, 177), (136, 65, 157), (129, 15, 124), (77, 0, 75)], - 'dark23': [(27, 158, 119), (217, 95, 2), (117, 112, 179)], - 'dark24': [(27, 158, 119), (217, 95, 2), (117, 112, 179), (231, 41, 138)], - 'dark25': [(27, 158, 119), (217, 95, 2), (117, 112, 179), (231, 41, 138), (102, 166, 30)], - 'dark26': [(27, 158, 119), (217, 95, 2), (117, 112, 179), (231, 41, 138), (102, 166, 30), (230, 171, 2)], - 'dark27': [(27, 158, 119), (217, 95, 2), (117, 112, 179), (231, 41, 138), (102, 166, 30), (230, 171, 2), (166, 118, 29)], - 'dark28': [(27, 158, 119), (217, 95, 2), (117, 112, 179), (231, 41, 138), (102, 166, 30), (230, 171, 2), (166, 118, 29), (102, 102, 102)], - 'gnbu3': [(224, 243, 219), (168, 221, 181), (67, 162, 202)], - 'gnbu4': [(240, 249, 232), (186, 228, 188), (123, 204, 196), (43, 140, 190)], - 'gnbu5': [(240, 249, 232), (186, 228, 188), (123, 204, 196), (67, 162, 202), (8, 104, 172)], - 'gnbu6': [(240, 249, 232), (204, 235, 197), (168, 221, 181), (123, 204, 196), (67, 162, 202), (8, 104, 172)], - 'gnbu7': [(240, 249, 232), (204, 235, 197), (168, 221, 181), (123, 204, 196), (78, 179, 211), (43, 140, 190), (8, 88, 158)], - 'gnbu8': [(247, 252, 240), (224, 243, 219), (204, 235, 197), (168, 221, 181), (123, 204, 196), (78, 179, 211), (43, 140, 190), (8, 88, 158)], - 'gnbu9': [(247, 252, 240), (224, 243, 219), (204, 235, 197), (168, 221, 181), (123, 204, 196), (78, 179, 211), (43, 140, 190), (8, 104, 172), (8, 64, 129)], - 'greens3': [(229, 245, 224), (161, 217, 155), (49, 163, 84)], - 'greens4': [(237, 248, 233), (186, 228, 179), (116, 196, 118), (35, 139, 69)], - 'greens5': [(237, 248, 233), (186, 228, 179), (116, 196, 118), (49, 163, 84), (0, 109, 44)], - 'greens6': [(237, 248, 233), (199, 233, 192), (161, 217, 155), (116, 196, 118), (49, 163, 84), (0, 109, 44)], - 'greens7': [(237, 248, 233), (199, 233, 192), (161, 217, 155), (116, 196, 118), (65, 171, 93), (35, 139, 69), (0, 90, 50)], - 'greens8': [(247, 252, 245), (229, 245, 224), (199, 233, 192), (161, 217, 155), (116, 196, 118), (65, 171, 93), (35, 139, 69), (0, 90, 50)], - 'greens9': [(247, 252, 245), (229, 245, 224), (199, 233, 192), (161, 217, 155), (116, 196, 118), (65, 171, 93), (35, 139, 69), (0, 109, 44), (0, 68, 27)], - 'greys3': [(240, 240, 240), (189, 189, 189), (99, 99, 99)], - 'greys4': [(247, 247, 247), (204, 204, 204), (150, 150, 150), (82, 82, 82)], - 'greys5': [(247, 247, 247), (204, 204, 204), (150, 150, 150), (99, 99, 99), (37, 37, 37)], - 'greys6': [(247, 247, 247), (217, 217, 217), (189, 189, 189), (150, 150, 150), (99, 99, 99), (37, 37, 37)], - 'greys7': [(247, 247, 247), (217, 217, 217), (189, 189, 189), (150, 150, 150), (115, 115, 115), (82, 82, 82), (37, 37, 37)], - 'greys8': [(255, 255, 255), (240, 240, 240), (217, 217, 217), (189, 189, 189), (150, 150, 150), (115, 115, 115), (82, 82, 82), (37, 37, 37)], - 'greys9': [(255, 255, 255), (240, 240, 240), (217, 217, 217), (189, 189, 189), (150, 150, 150), (115, 115, 115), (82, 82, 82), (37, 37, 37), (0, 0, 0)], - 'oranges3': [(254, 230, 206), (253, 174, 107), (230, 85, 13)], - 'oranges4': [(254, 237, 222), (253, 190, 133), (253, 141, 60), (217, 71, 1)], - 'oranges5': [(254, 237, 222), (253, 190, 133), (253, 141, 60), (230, 85, 13), (166, 54, 3)], - 'oranges6': [(254, 237, 222), (253, 208, 162), (253, 174, 107), (253, 141, 60), (230, 85, 13), (166, 54, 3)], - 'oranges7': [(254, 237, 222), (253, 208, 162), (253, 174, 107), (253, 141, 60), (241, 105, 19), (217, 72, 1), (140, 45, 4)], - 'oranges8': [(255, 245, 235), (254, 230, 206), (253, 208, 162), (253, 174, 107), (253, 141, 60), (241, 105, 19), (217, 72, 1), (140, 45, 4)], - 'oranges9': [(255, 245, 235), (254, 230, 206), (253, 208, 162), (253, 174, 107), (253, 141, 60), (241, 105, 19), (217, 72, 1), (166, 54, 3), (127, 39, 4)], - 'orrd3': [(254, 232, 200), (253, 187, 132), (227, 74, 51)], - 'orrd4': [(254, 240, 217), (253, 204, 138), (252, 141, 89), (215, 48, 31)], - 'orrd5': [(254, 240, 217), (253, 204, 138), (252, 141, 89), (227, 74, 51), (179, 0, 0)], - 'orrd6': [(254, 240, 217), (253, 212, 158), (253, 187, 132), (252, 141, 89), (227, 74, 51), (179, 0, 0)], - 'orrd7': [(254, 240, 217), (253, 212, 158), (253, 187, 132), (252, 141, 89), (239, 101, 72), (215, 48, 31), (153, 0, 0)], - 'orrd8': [(255, 247, 236), (254, 232, 200), (253, 212, 158), (253, 187, 132), (252, 141, 89), (239, 101, 72), (215, 48, 31), (153, 0, 0)], - 'orrd9': [(255, 247, 236), (254, 232, 200), (253, 212, 158), (253, 187, 132), (252, 141, 89), (239, 101, 72), (215, 48, 31), (179, 0, 0), (127, 0, 0)], - 'paired10': [(166, 206, 227), (106, 61, 154), (31, 120, 180), (178, 223, 138), (51, 160, 44), (251, 154, 153), (227, 26, 28), (253, 191, 111), (255, 127, 0), (202, 178, 214)], - 'paired11': [(166, 206, 227), (106, 61, 154), (255, 255, 153), (31, 120, 180), (178, 223, 138), (51, 160, 44), (251, 154, 153), (227, 26, 28), (253, 191, 111), (255, 127, 0), (202, 178, 214)], - 'paired12': [(166, 206, 227), (106, 61, 154), (255, 255, 153), (177, 89, 40), (31, 120, 180), (178, 223, 138), (51, 160, 44), (251, 154, 153), (227, 26, 28), (253, 191, 111), (255, 127, 0), (202, 178, 214)], - 'paired3': [(166, 206, 227), (31, 120, 180), (178, 223, 138)], - 'paired4': [(166, 206, 227), (31, 120, 180), (178, 223, 138), (51, 160, 44)], - 'paired5': [(166, 206, 227), (31, 120, 180), (178, 223, 138), (51, 160, 44), (251, 154, 153)], - 'paired6': [(166, 206, 227), (31, 120, 180), (178, 223, 138), (51, 160, 44), (251, 154, 153), (227, 26, 28)], - 'paired7': [(166, 206, 227), (31, 120, 180), (178, 223, 138), (51, 160, 44), (251, 154, 153), (227, 26, 28), (253, 191, 111)], - 'paired8': [(166, 206, 227), (31, 120, 180), (178, 223, 138), (51, 160, 44), (251, 154, 153), (227, 26, 28), (253, 191, 111), (255, 127, 0)], - 'paired9': [(166, 206, 227), (31, 120, 180), (178, 223, 138), (51, 160, 44), (251, 154, 153), (227, 26, 28), (253, 191, 111), (255, 127, 0), (202, 178, 214)], - 'pastel13': [(251, 180, 174), (179, 205, 227), (204, 235, 197)], - 'pastel14': [(251, 180, 174), (179, 205, 227), (204, 235, 197), (222, 203, 228)], - 'pastel15': [(251, 180, 174), (179, 205, 227), (204, 235, 197), (222, 203, 228), (254, 217, 166)], - 'pastel16': [(251, 180, 174), (179, 205, 227), (204, 235, 197), (222, 203, 228), (254, 217, 166), (255, 255, 204)], - 'pastel17': [(251, 180, 174), (179, 205, 227), (204, 235, 197), (222, 203, 228), (254, 217, 166), (255, 255, 204), (229, 216, 189)], - 'pastel18': [(251, 180, 174), (179, 205, 227), (204, 235, 197), (222, 203, 228), (254, 217, 166), (255, 255, 204), (229, 216, 189), (253, 218, 236)], - 'pastel19': [(251, 180, 174), (179, 205, 227), (204, 235, 197), (222, 203, 228), (254, 217, 166), (255, 255, 204), (229, 216, 189), (253, 218, 236), (242, 242, 242)], - 'pastel23': [(179, 226, 205), (253, 205, 172), (203, 213, 232)], - 'pastel24': [(179, 226, 205), (253, 205, 172), (203, 213, 232), (244, 202, 228)], - 'pastel25': [(179, 226, 205), (253, 205, 172), (203, 213, 232), (244, 202, 228), (230, 245, 201)], - 'pastel26': [(179, 226, 205), (253, 205, 172), (203, 213, 232), (244, 202, 228), (230, 245, 201), (255, 242, 174)], - 'pastel27': [(179, 226, 205), (253, 205, 172), (203, 213, 232), (244, 202, 228), (230, 245, 201), (255, 242, 174), (241, 226, 204)], - 'pastel28': [(179, 226, 205), (253, 205, 172), (203, 213, 232), (244, 202, 228), (230, 245, 201), (255, 242, 174), (241, 226, 204), (204, 204, 204)], - 'piyg10': [(142, 1, 82), (39, 100, 25), (197, 27, 125), (222, 119, 174), (241, 182, 218), (253, 224, 239), (230, 245, 208), (184, 225, 134), (127, 188, 65), (77, 146, 33)], - 'piyg11': [(142, 1, 82), (77, 146, 33), (39, 100, 25), (197, 27, 125), (222, 119, 174), (241, 182, 218), (253, 224, 239), (247, 247, 247), (230, 245, 208), (184, 225, 134), (127, 188, 65)], - 'piyg3': [(233, 163, 201), (247, 247, 247), (161, 215, 106)], - 'piyg4': [(208, 28, 139), (241, 182, 218), (184, 225, 134), (77, 172, 38)], - 'piyg5': [(208, 28, 139), (241, 182, 218), (247, 247, 247), (184, 225, 134), (77, 172, 38)], - 'piyg6': [(197, 27, 125), (233, 163, 201), (253, 224, 239), (230, 245, 208), (161, 215, 106), (77, 146, 33)], - 'piyg7': [(197, 27, 125), (233, 163, 201), (253, 224, 239), (247, 247, 247), (230, 245, 208), (161, 215, 106), (77, 146, 33)], - 'piyg8': [(197, 27, 125), (222, 119, 174), (241, 182, 218), (253, 224, 239), (230, 245, 208), (184, 225, 134), (127, 188, 65), (77, 146, 33)], - 'piyg9': [(197, 27, 125), (222, 119, 174), (241, 182, 218), (253, 224, 239), (247, 247, 247), (230, 245, 208), (184, 225, 134), (127, 188, 65), (77, 146, 33)], - 'prgn10': [(64, 0, 75), (0, 68, 27), (118, 42, 131), (153, 112, 171), (194, 165, 207), (231, 212, 232), (217, 240, 211), (166, 219, 160), (90, 174, 97), (27, 120, 55)], - 'prgn11': [(64, 0, 75), (27, 120, 55), (0, 68, 27), (118, 42, 131), (153, 112, 171), (194, 165, 207), (231, 212, 232), (247, 247, 247), (217, 240, 211), (166, 219, 160), (90, 174, 97)], - 'prgn3': [(175, 141, 195), (247, 247, 247), (127, 191, 123)], - 'prgn4': [(123, 50, 148), (194, 165, 207), (166, 219, 160), (0, 136, 55)], - 'prgn5': [(123, 50, 148), (194, 165, 207), (247, 247, 247), (166, 219, 160), (0, 136, 55)], - 'prgn6': [(118, 42, 131), (175, 141, 195), (231, 212, 232), (217, 240, 211), (127, 191, 123), (27, 120, 55)], - 'prgn7': [(118, 42, 131), (175, 141, 195), (231, 212, 232), (247, 247, 247), (217, 240, 211), (127, 191, 123), (27, 120, 55)], - 'prgn8': [(118, 42, 131), (153, 112, 171), (194, 165, 207), (231, 212, 232), (217, 240, 211), (166, 219, 160), (90, 174, 97), (27, 120, 55)], - 'prgn9': [(118, 42, 131), (153, 112, 171), (194, 165, 207), (231, 212, 232), (247, 247, 247), (217, 240, 211), (166, 219, 160), (90, 174, 97), (27, 120, 55)], - 'pubu3': [(236, 231, 242), (166, 189, 219), (43, 140, 190)], - 'pubu4': [(241, 238, 246), (189, 201, 225), (116, 169, 207), (5, 112, 176)], - 'pubu5': [(241, 238, 246), (189, 201, 225), (116, 169, 207), (43, 140, 190), (4, 90, 141)], - 'pubu6': [(241, 238, 246), (208, 209, 230), (166, 189, 219), (116, 169, 207), (43, 140, 190), (4, 90, 141)], - 'pubu7': [(241, 238, 246), (208, 209, 230), (166, 189, 219), (116, 169, 207), (54, 144, 192), (5, 112, 176), (3, 78, 123)], - 'pubu8': [(255, 247, 251), (236, 231, 242), (208, 209, 230), (166, 189, 219), (116, 169, 207), (54, 144, 192), (5, 112, 176), (3, 78, 123)], - 'pubu9': [(255, 247, 251), (236, 231, 242), (208, 209, 230), (166, 189, 219), (116, 169, 207), (54, 144, 192), (5, 112, 176), (4, 90, 141), (2, 56, 88)], - 'pubugn3': [(236, 226, 240), (166, 189, 219), (28, 144, 153)], - 'pubugn4': [(246, 239, 247), (189, 201, 225), (103, 169, 207), (2, 129, 138)], - 'pubugn5': [(246, 239, 247), (189, 201, 225), (103, 169, 207), (28, 144, 153), (1, 108, 89)], - 'pubugn6': [(246, 239, 247), (208, 209, 230), (166, 189, 219), (103, 169, 207), (28, 144, 153), (1, 108, 89)], - 'pubugn7': [(246, 239, 247), (208, 209, 230), (166, 189, 219), (103, 169, 207), (54, 144, 192), (2, 129, 138), (1, 100, 80)], - 'pubugn8': [(255, 247, 251), (236, 226, 240), (208, 209, 230), (166, 189, 219), (103, 169, 207), (54, 144, 192), (2, 129, 138), (1, 100, 80)], - 'pubugn9': [(255, 247, 251), (236, 226, 240), (208, 209, 230), (166, 189, 219), (103, 169, 207), (54, 144, 192), (2, 129, 138), (1, 108, 89), (1, 70, 54)], - 'puor10': [(127, 59, 8), (45, 0, 75), (179, 88, 6), (224, 130, 20), (253, 184, 99), (254, 224, 182), (216, 218, 235), (178, 171, 210), (128, 115, 172), (84, 39, 136)], - 'puor11': [(127, 59, 8), (84, 39, 136), (45, 0, 75), (179, 88, 6), (224, 130, 20), (253, 184, 99), (254, 224, 182), (247, 247, 247), (216, 218, 235), (178, 171, 210), (128, 115, 172)], - 'puor3': [(241, 163, 64), (247, 247, 247), (153, 142, 195)], - 'puor4': [(230, 97, 1), (253, 184, 99), (178, 171, 210), (94, 60, 153)], - 'puor5': [(230, 97, 1), (253, 184, 99), (247, 247, 247), (178, 171, 210), (94, 60, 153)], - 'puor6': [(179, 88, 6), (241, 163, 64), (254, 224, 182), (216, 218, 235), (153, 142, 195), (84, 39, 136)], - 'puor7': [(179, 88, 6), (241, 163, 64), (254, 224, 182), (247, 247, 247), (216, 218, 235), (153, 142, 195), (84, 39, 136)], - 'puor8': [(179, 88, 6), (224, 130, 20), (253, 184, 99), (254, 224, 182), (216, 218, 235), (178, 171, 210), (128, 115, 172), (84, 39, 136)], - 'puor9': [(179, 88, 6), (224, 130, 20), (253, 184, 99), (254, 224, 182), (247, 247, 247), (216, 218, 235), (178, 171, 210), (128, 115, 172), (84, 39, 136)], - 'purd3': [(231, 225, 239), (201, 148, 199), (221, 28, 119)], - 'purd4': [(241, 238, 246), (215, 181, 216), (223, 101, 176), (206, 18, 86)], - 'purd5': [(241, 238, 246), (215, 181, 216), (223, 101, 176), (221, 28, 119), (152, 0, 67)], - 'purd6': [(241, 238, 246), (212, 185, 218), (201, 148, 199), (223, 101, 176), (221, 28, 119), (152, 0, 67)], - 'purd7': [(241, 238, 246), (212, 185, 218), (201, 148, 199), (223, 101, 176), (231, 41, 138), (206, 18, 86), (145, 0, 63)], - 'purd8': [(247, 244, 249), (231, 225, 239), (212, 185, 218), (201, 148, 199), (223, 101, 176), (231, 41, 138), (206, 18, 86), (145, 0, 63)], - 'purd9': [(247, 244, 249), (231, 225, 239), (212, 185, 218), (201, 148, 199), (223, 101, 176), (231, 41, 138), (206, 18, 86), (152, 0, 67), (103, 0, 31)], - 'purples3': [(239, 237, 245), (188, 189, 220), (117, 107, 177)], - 'purples4': [(242, 240, 247), (203, 201, 226), (158, 154, 200), (106, 81, 163)], - 'purples5': [(242, 240, 247), (203, 201, 226), (158, 154, 200), (117, 107, 177), (84, 39, 143)], - 'purples6': [(242, 240, 247), (218, 218, 235), (188, 189, 220), (158, 154, 200), (117, 107, 177), (84, 39, 143)], - 'purples7': [(242, 240, 247), (218, 218, 235), (188, 189, 220), (158, 154, 200), (128, 125, 186), (106, 81, 163), (74, 20, 134)], - 'purples8': [(252, 251, 253), (239, 237, 245), (218, 218, 235), (188, 189, 220), (158, 154, 200), (128, 125, 186), (106, 81, 163), (74, 20, 134)], - 'purples9': [(252, 251, 253), (239, 237, 245), (218, 218, 235), (188, 189, 220), (158, 154, 200), (128, 125, 186), (106, 81, 163), (84, 39, 143), (63, 0, 125)], - 'rdbu10': [(103, 0, 31), (5, 48, 97), (178, 24, 43), (214, 96, 77), (244, 165, 130), (253, 219, 199), (209, 229, 240), (146, 197, 222), (67, 147, 195), (33, 102, 172)], - 'rdbu11': [(103, 0, 31), (33, 102, 172), (5, 48, 97), (178, 24, 43), (214, 96, 77), (244, 165, 130), (253, 219, 199), (247, 247, 247), (209, 229, 240), (146, 197, 222), (67, 147, 195)], - 'rdbu3': [(239, 138, 98), (247, 247, 247), (103, 169, 207)], - 'rdbu4': [(202, 0, 32), (244, 165, 130), (146, 197, 222), (5, 113, 176)], - 'rdbu5': [(202, 0, 32), (244, 165, 130), (247, 247, 247), (146, 197, 222), (5, 113, 176)], - 'rdbu6': [(178, 24, 43), (239, 138, 98), (253, 219, 199), (209, 229, 240), (103, 169, 207), (33, 102, 172)], - 'rdbu7': [(178, 24, 43), (239, 138, 98), (253, 219, 199), (247, 247, 247), (209, 229, 240), (103, 169, 207), (33, 102, 172)], - 'rdbu8': [(178, 24, 43), (214, 96, 77), (244, 165, 130), (253, 219, 199), (209, 229, 240), (146, 197, 222), (67, 147, 195), (33, 102, 172)], - 'rdbu9': [(178, 24, 43), (214, 96, 77), (244, 165, 130), (253, 219, 199), (247, 247, 247), (209, 229, 240), (146, 197, 222), (67, 147, 195), (33, 102, 172)], - 'rdgy10': [(103, 0, 31), (26, 26, 26), (178, 24, 43), (214, 96, 77), (244, 165, 130), (253, 219, 199), (224, 224, 224), (186, 186, 186), (135, 135, 135), (77, 77, 77)], - 'rdgy11': [(103, 0, 31), (77, 77, 77), (26, 26, 26), (178, 24, 43), (214, 96, 77), (244, 165, 130), (253, 219, 199), (255, 255, 255), (224, 224, 224), (186, 186, 186), (135, 135, 135)], - 'rdgy3': [(239, 138, 98), (255, 255, 255), (153, 153, 153)], - 'rdgy4': [(202, 0, 32), (244, 165, 130), (186, 186, 186), (64, 64, 64)], - 'rdgy5': [(202, 0, 32), (244, 165, 130), (255, 255, 255), (186, 186, 186), (64, 64, 64)], - 'rdgy6': [(178, 24, 43), (239, 138, 98), (253, 219, 199), (224, 224, 224), (153, 153, 153), (77, 77, 77)], - 'rdgy7': [(178, 24, 43), (239, 138, 98), (253, 219, 199), (255, 255, 255), (224, 224, 224), (153, 153, 153), (77, 77, 77)], - 'rdgy8': [(178, 24, 43), (214, 96, 77), (244, 165, 130), (253, 219, 199), (224, 224, 224), (186, 186, 186), (135, 135, 135), (77, 77, 77)], - 'rdgy9': [(178, 24, 43), (214, 96, 77), (244, 165, 130), (253, 219, 199), (255, 255, 255), (224, 224, 224), (186, 186, 186), (135, 135, 135), (77, 77, 77)], - 'rdpu3': [(253, 224, 221), (250, 159, 181), (197, 27, 138)], - 'rdpu4': [(254, 235, 226), (251, 180, 185), (247, 104, 161), (174, 1, 126)], - 'rdpu5': [(254, 235, 226), (251, 180, 185), (247, 104, 161), (197, 27, 138), (122, 1, 119)], - 'rdpu6': [(254, 235, 226), (252, 197, 192), (250, 159, 181), (247, 104, 161), (197, 27, 138), (122, 1, 119)], - 'rdpu7': [(254, 235, 226), (252, 197, 192), (250, 159, 181), (247, 104, 161), (221, 52, 151), (174, 1, 126), (122, 1, 119)], - 'rdpu8': [(255, 247, 243), (253, 224, 221), (252, 197, 192), (250, 159, 181), (247, 104, 161), (221, 52, 151), (174, 1, 126), (122, 1, 119)], - 'rdpu9': [(255, 247, 243), (253, 224, 221), (252, 197, 192), (250, 159, 181), (247, 104, 161), (221, 52, 151), (174, 1, 126), (122, 1, 119), (73, 0, 106)], - 'rdylbu10': [(165, 0, 38), (49, 54, 149), (215, 48, 39), (244, 109, 67), (253, 174, 97), (254, 224, 144), (224, 243, 248), (171, 217, 233), (116, 173, 209), (69, 117, 180)], - 'rdylbu11': [(165, 0, 38), (69, 117, 180), (49, 54, 149), (215, 48, 39), (244, 109, 67), (253, 174, 97), (254, 224, 144), (255, 255, 191), (224, 243, 248), (171, 217, 233), (116, 173, 209)], - 'rdylbu3': [(252, 141, 89), (255, 255, 191), (145, 191, 219)], - 'rdylbu4': [(215, 25, 28), (253, 174, 97), (171, 217, 233), (44, 123, 182)], - 'rdylbu5': [(215, 25, 28), (253, 174, 97), (255, 255, 191), (171, 217, 233), (44, 123, 182)], - 'rdylbu6': [(215, 48, 39), (252, 141, 89), (254, 224, 144), (224, 243, 248), (145, 191, 219), (69, 117, 180)], - 'rdylbu7': [(215, 48, 39), (252, 141, 89), (254, 224, 144), (255, 255, 191), (224, 243, 248), (145, 191, 219), (69, 117, 180)], - 'rdylbu8': [(215, 48, 39), (244, 109, 67), (253, 174, 97), (254, 224, 144), (224, 243, 248), (171, 217, 233), (116, 173, 209), (69, 117, 180)], - 'rdylbu9': [(215, 48, 39), (244, 109, 67), (253, 174, 97), (254, 224, 144), (255, 255, 191), (224, 243, 248), (171, 217, 233), (116, 173, 209), (69, 117, 180)], - 'rdylgn10': [(165, 0, 38), (0, 104, 55), (215, 48, 39), (244, 109, 67), (253, 174, 97), (254, 224, 139), (217, 239, 139), (166, 217, 106), (102, 189, 99), (26, 152, 80)], - 'rdylgn11': [(165, 0, 38), (26, 152, 80), (0, 104, 55), (215, 48, 39), (244, 109, 67), (253, 174, 97), (254, 224, 139), (255, 255, 191), (217, 239, 139), (166, 217, 106), (102, 189, 99)], - 'rdylgn3': [(252, 141, 89), (255, 255, 191), (145, 207, 96)], - 'rdylgn4': [(215, 25, 28), (253, 174, 97), (166, 217, 106), (26, 150, 65)], - 'rdylgn5': [(215, 25, 28), (253, 174, 97), (255, 255, 191), (166, 217, 106), (26, 150, 65)], - 'rdylgn6': [(215, 48, 39), (252, 141, 89), (254, 224, 139), (217, 239, 139), (145, 207, 96), (26, 152, 80)], - 'rdylgn7': [(215, 48, 39), (252, 141, 89), (254, 224, 139), (255, 255, 191), (217, 239, 139), (145, 207, 96), (26, 152, 80)], - 'rdylgn8': [(215, 48, 39), (244, 109, 67), (253, 174, 97), (254, 224, 139), (217, 239, 139), (166, 217, 106), (102, 189, 99), (26, 152, 80)], - 'rdylgn9': [(215, 48, 39), (244, 109, 67), (253, 174, 97), (254, 224, 139), (255, 255, 191), (217, 239, 139), (166, 217, 106), (102, 189, 99), (26, 152, 80)], - 'reds3': [(254, 224, 210), (252, 146, 114), (222, 45, 38)], - 'reds4': [(254, 229, 217), (252, 174, 145), (251, 106, 74), (203, 24, 29)], - 'reds5': [(254, 229, 217), (252, 174, 145), (251, 106, 74), (222, 45, 38), (165, 15, 21)], - 'reds6': [(254, 229, 217), (252, 187, 161), (252, 146, 114), (251, 106, 74), (222, 45, 38), (165, 15, 21)], - 'reds7': [(254, 229, 217), (252, 187, 161), (252, 146, 114), (251, 106, 74), (239, 59, 44), (203, 24, 29), (153, 0, 13)], - 'reds8': [(255, 245, 240), (254, 224, 210), (252, 187, 161), (252, 146, 114), (251, 106, 74), (239, 59, 44), (203, 24, 29), (153, 0, 13)], - 'reds9': [(255, 245, 240), (254, 224, 210), (252, 187, 161), (252, 146, 114), (251, 106, 74), (239, 59, 44), (203, 24, 29), (165, 15, 21), (103, 0, 13)], - 'set13': [(228, 26, 28), (55, 126, 184), (77, 175, 74)], - 'set14': [(228, 26, 28), (55, 126, 184), (77, 175, 74), (152, 78, 163)], - 'set15': [(228, 26, 28), (55, 126, 184), (77, 175, 74), (152, 78, 163), (255, 127, 0)], - 'set16': [(228, 26, 28), (55, 126, 184), (77, 175, 74), (152, 78, 163), (255, 127, 0), (255, 255, 51)], - 'set17': [(228, 26, 28), (55, 126, 184), (77, 175, 74), (152, 78, 163), (255, 127, 0), (255, 255, 51), (166, 86, 40)], - 'set18': [(228, 26, 28), (55, 126, 184), (77, 175, 74), (152, 78, 163), (255, 127, 0), (255, 255, 51), (166, 86, 40), (247, 129, 191)], - 'set19': [(228, 26, 28), (55, 126, 184), (77, 175, 74), (152, 78, 163), (255, 127, 0), (255, 255, 51), (166, 86, 40), (247, 129, 191), (153, 153, 153)], - 'set23': [(102, 194, 165), (252, 141, 98), (141, 160, 203)], - 'set24': [(102, 194, 165), (252, 141, 98), (141, 160, 203), (231, 138, 195)], - 'set25': [(102, 194, 165), (252, 141, 98), (141, 160, 203), (231, 138, 195), (166, 216, 84)], - 'set26': [(102, 194, 165), (252, 141, 98), (141, 160, 203), (231, 138, 195), (166, 216, 84), (255, 217, 47)], - 'set27': [(102, 194, 165), (252, 141, 98), (141, 160, 203), (231, 138, 195), (166, 216, 84), (255, 217, 47), (229, 196, 148)], - 'set28': [(102, 194, 165), (252, 141, 98), (141, 160, 203), (231, 138, 195), (166, 216, 84), (255, 217, 47), (229, 196, 148), (179, 179, 179)], - 'set310': [(141, 211, 199), (188, 128, 189), (255, 255, 179), (190, 186, 218), (251, 128, 114), (128, 177, 211), (253, 180, 98), (179, 222, 105), (252, 205, 229), (217, 217, 217)], - 'set311': [(141, 211, 199), (188, 128, 189), (204, 235, 197), (255, 255, 179), (190, 186, 218), (251, 128, 114), (128, 177, 211), (253, 180, 98), (179, 222, 105), (252, 205, 229), (217, 217, 217)], - 'set312': [(141, 211, 199), (188, 128, 189), (204, 235, 197), (255, 237, 111), (255, 255, 179), (190, 186, 218), (251, 128, 114), (128, 177, 211), (253, 180, 98), (179, 222, 105), (252, 205, 229), (217, 217, 217)], - 'set33': [(141, 211, 199), (255, 255, 179), (190, 186, 218)], - 'set34': [(141, 211, 199), (255, 255, 179), (190, 186, 218), (251, 128, 114)], - 'set35': [(141, 211, 199), (255, 255, 179), (190, 186, 218), (251, 128, 114), (128, 177, 211)], - 'set36': [(141, 211, 199), (255, 255, 179), (190, 186, 218), (251, 128, 114), (128, 177, 211), (253, 180, 98)], - 'set37': [(141, 211, 199), (255, 255, 179), (190, 186, 218), (251, 128, 114), (128, 177, 211), (253, 180, 98), (179, 222, 105)], - 'set38': [(141, 211, 199), (255, 255, 179), (190, 186, 218), (251, 128, 114), (128, 177, 211), (253, 180, 98), (179, 222, 105), (252, 205, 229)], - 'set39': [(141, 211, 199), (255, 255, 179), (190, 186, 218), (251, 128, 114), (128, 177, 211), (253, 180, 98), (179, 222, 105), (252, 205, 229), (217, 217, 217)], - 'spectral10': [(158, 1, 66), (94, 79, 162), (213, 62, 79), (244, 109, 67), (253, 174, 97), (254, 224, 139), (230, 245, 152), (171, 221, 164), (102, 194, 165), (50, 136, 189)], - 'spectral11': [(158, 1, 66), (50, 136, 189), (94, 79, 162), (213, 62, 79), (244, 109, 67), (253, 174, 97), (254, 224, 139), (255, 255, 191), (230, 245, 152), (171, 221, 164), (102, 194, 165)], - 'spectral3': [(252, 141, 89), (255, 255, 191), (153, 213, 148)], - 'spectral4': [(215, 25, 28), (253, 174, 97), (171, 221, 164), (43, 131, 186)], - 'spectral5': [(215, 25, 28), (253, 174, 97), (255, 255, 191), (171, 221, 164), (43, 131, 186)], - 'spectral6': [(213, 62, 79), (252, 141, 89), (254, 224, 139), (230, 245, 152), (153, 213, 148), (50, 136, 189)], - 'spectral7': [(213, 62, 79), (252, 141, 89), (254, 224, 139), (255, 255, 191), (230, 245, 152), (153, 213, 148), (50, 136, 189)], - 'spectral8': [(213, 62, 79), (244, 109, 67), (253, 174, 97), (254, 224, 139), (230, 245, 152), (171, 221, 164), (102, 194, 165), (50, 136, 189)], - 'spectral9': [(213, 62, 79), (244, 109, 67), (253, 174, 97), (254, 224, 139), (255, 255, 191), (230, 245, 152), (171, 221, 164), (102, 194, 165), (50, 136, 189)], - 'ylgn3': [(247, 252, 185), (173, 221, 142), (49, 163, 84)], - 'ylgn4': [(255, 255, 204), (194, 230, 153), (120, 198, 121), (35, 132, 67)], - 'ylgn5': [(255, 255, 204), (194, 230, 153), (120, 198, 121), (49, 163, 84), (0, 104, 55)], - 'ylgn6': [(255, 255, 204), (217, 240, 163), (173, 221, 142), (120, 198, 121), (49, 163, 84), (0, 104, 55)], - 'ylgn7': [(255, 255, 204), (217, 240, 163), (173, 221, 142), (120, 198, 121), (65, 171, 93), (35, 132, 67), (0, 90, 50)], - 'ylgn8': [(255, 255, 229), (247, 252, 185), (217, 240, 163), (173, 221, 142), (120, 198, 121), (65, 171, 93), (35, 132, 67), (0, 90, 50)], - 'ylgn9': [(255, 255, 229), (247, 252, 185), (217, 240, 163), (173, 221, 142), (120, 198, 121), (65, 171, 93), (35, 132, 67), (0, 104, 55), (0, 69, 41)], - 'ylgnbu3': [(237, 248, 177), (127, 205, 187), (44, 127, 184)], - 'ylgnbu4': [(255, 255, 204), (161, 218, 180), (65, 182, 196), (34, 94, 168)], - 'ylgnbu5': [(255, 255, 204), (161, 218, 180), (65, 182, 196), (44, 127, 184), (37, 52, 148)], - 'ylgnbu6': [(255, 255, 204), (199, 233, 180), (127, 205, 187), (65, 182, 196), (44, 127, 184), (37, 52, 148)], - 'ylgnbu7': [(255, 255, 204), (199, 233, 180), (127, 205, 187), (65, 182, 196), (29, 145, 192), (34, 94, 168), (12, 44, 132)], - 'ylgnbu8': [(255, 255, 217), (237, 248, 177), (199, 233, 180), (127, 205, 187), (65, 182, 196), (29, 145, 192), (34, 94, 168), (12, 44, 132)], - 'ylgnbu9': [(255, 255, 217), (237, 248, 177), (199, 233, 180), (127, 205, 187), (65, 182, 196), (29, 145, 192), (34, 94, 168), (37, 52, 148), (8, 29, 88)], - 'ylorbr3': [(255, 247, 188), (254, 196, 79), (217, 95, 14)], - 'ylorbr4': [(255, 255, 212), (254, 217, 142), (254, 153, 41), (204, 76, 2)], - 'ylorbr5': [(255, 255, 212), (254, 217, 142), (254, 153, 41), (217, 95, 14), (153, 52, 4)], - 'ylorbr6': [(255, 255, 212), (254, 227, 145), (254, 196, 79), (254, 153, 41), (217, 95, 14), (153, 52, 4)], - 'ylorbr7': [(255, 255, 212), (254, 227, 145), (254, 196, 79), (254, 153, 41), (236, 112, 20), (204, 76, 2), (140, 45, 4)], - 'ylorbr8': [(255, 255, 229), (255, 247, 188), (254, 227, 145), (254, 196, 79), (254, 153, 41), (236, 112, 20), (204, 76, 2), (140, 45, 4)], - 'ylorbr9': [(255, 255, 229), (255, 247, 188), (254, 227, 145), (254, 196, 79), (254, 153, 41), (236, 112, 20), (204, 76, 2), (153, 52, 4), (102, 37, 6)], - 'ylorrd3': [(255, 237, 160), (254, 178, 76), (240, 59, 32)], - 'ylorrd4': [(255, 255, 178), (254, 204, 92), (253, 141, 60), (227, 26, 28)], - 'ylorrd5': [(255, 255, 178), (254, 204, 92), (253, 141, 60), (240, 59, 32), (189, 0, 38)], - 'ylorrd6': [(255, 255, 178), (254, 217, 118), (254, 178, 76), (253, 141, 60), (240, 59, 32), (189, 0, 38)], - 'ylorrd7': [(255, 255, 178), (254, 217, 118), (254, 178, 76), (253, 141, 60), (252, 78, 42), (227, 26, 28), (177, 0, 38)], - 'ylorrd8': [(255, 255, 204), (255, 237, 160), (254, 217, 118), (254, 178, 76), (253, 141, 60), (252, 78, 42), (227, 26, 28), (177, 0, 38)], -} - - -if __name__ == '__main__': - main() From ccf9e7de542d4b4b1bd833bf4be311e64fba8b4a Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 7 Jan 2021 14:00:10 +0100 Subject: [PATCH 015/243] Another patch regarding #4530 --- lib/core/settings.py | 4 ++-- lib/request/basic.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 9f076ec8bed..afa14eb932d 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.15" +VERSION = "1.5.1.16" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -757,7 +757,7 @@ # For preventing MemoryError exceptions (caused when using large sequences in difflib.SequenceMatcher) MAX_DIFFLIB_SEQUENCE_LENGTH = 10 * 1024 * 1024 -# Maximum size used per page content in getHeuristicCharEncoding() +# Maximum size used per page content in getHeuristicCharEncoding() and identYwaf MAX_CHAR_HEURISTICS_SIZE = 10000 # Maximum (multi-threaded) length of entry in bisection algorithm diff --git a/lib/request/basic.py b/lib/request/basic.py index ead947cadeb..be19290dcb1 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -396,7 +396,7 @@ def processResponse(page, responseHeaders, code=None, status=None): logger.warning("parsed DBMS error message: '%s'" % msg.rstrip('.')) if not conf.skipWaf and kb.processResponseCounter < IDENTYWAF_PARSE_LIMIT: - rawResponse = "%s %s %s\n%s\n%s" % (_http_client.HTTPConnection._http_vsn_str, code or "", status or "", "".join(getUnicode(responseHeaders.headers if responseHeaders else [])), page) + rawResponse = "%s %s %s\n%s\n%s" % (_http_client.HTTPConnection._http_vsn_str, code or "", status or "", "".join(getUnicode(responseHeaders.headers if responseHeaders else [])), page[:MAX_CHAR_HEURISTICS_SIZE]) identYwaf.non_blind.clear() if identYwaf.non_blind_check(rawResponse, silent=True): From 1f39dbd06dc9c7414625029a33ef59f68ca4a676 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 7 Jan 2021 14:20:03 +0100 Subject: [PATCH 016/243] Another patch regarding #4530 --- lib/core/settings.py | 6 +++--- lib/parse/html.py | 3 +++ lib/request/basic.py | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index afa14eb932d..01443d3c233 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.16" +VERSION = "1.5.1.17" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -757,8 +757,8 @@ # For preventing MemoryError exceptions (caused when using large sequences in difflib.SequenceMatcher) MAX_DIFFLIB_SEQUENCE_LENGTH = 10 * 1024 * 1024 -# Maximum size used per page content in getHeuristicCharEncoding() and identYwaf -MAX_CHAR_HEURISTICS_SIZE = 10000 +# Page size threshold used in heuristic checks (e.g. getHeuristicCharEncoding(), identYwaf, htmlParser, etc.) +HEURISTIC_PAGE_SIZE_THRESHOLD = 64 * 1024 # Maximum (multi-threaded) length of entry in bisection algorithm MAX_BISECTION_LENGTH = 50 * 1024 * 1024 diff --git a/lib/parse/html.py b/lib/parse/html.py index 833ad9eda40..4de60831339 100644 --- a/lib/parse/html.py +++ b/lib/parse/html.py @@ -13,6 +13,7 @@ from lib.core.common import parseXmlFile from lib.core.data import kb from lib.core.data import paths +from lib.core.settings import HEURISTIC_PAGE_SIZE_THRESHOLD from lib.core.threads import getCurrentThreadData class HTMLHandler(ContentHandler): @@ -69,6 +70,8 @@ def htmlParser(page): >>> threadData.lastErrorPage = None """ + page = page[:HEURISTIC_PAGE_SIZE_THRESHOLD] + xmlfile = paths.ERRORS_XML handler = HTMLHandler(page) key = hash(page) diff --git a/lib/request/basic.py b/lib/request/basic.py index be19290dcb1..cb1cbc57fd7 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -43,8 +43,8 @@ from lib.core.settings import BLOCKED_IP_REGEX from lib.core.settings import DEFAULT_COOKIE_DELIMITER from lib.core.settings import EVENTVALIDATION_REGEX +from lib.core.settings import HEURISTIC_PAGE_SIZE_THRESHOLD from lib.core.settings import IDENTYWAF_PARSE_LIMIT -from lib.core.settings import MAX_CHAR_HEURISTICS_SIZE from lib.core.settings import MAX_CONNECTION_TOTAL_SIZE from lib.core.settings import META_CHARSET_REGEX from lib.core.settings import PARSE_HEADERS_LIMIT @@ -259,7 +259,7 @@ def getHeuristicCharEncoding(page): """ key = hash(page) - retVal = kb.cache.encoding.get(key) or detect(page[:MAX_CHAR_HEURISTICS_SIZE])["encoding"] + retVal = kb.cache.encoding.get(key) or detect(page[:HEURISTIC_PAGE_SIZE_THRESHOLD])["encoding"] kb.cache.encoding[key] = retVal if retVal and retVal.lower().replace('-', "") == UNICODE_ENCODING.lower().replace('-', ""): @@ -396,7 +396,7 @@ def processResponse(page, responseHeaders, code=None, status=None): logger.warning("parsed DBMS error message: '%s'" % msg.rstrip('.')) if not conf.skipWaf and kb.processResponseCounter < IDENTYWAF_PARSE_LIMIT: - rawResponse = "%s %s %s\n%s\n%s" % (_http_client.HTTPConnection._http_vsn_str, code or "", status or "", "".join(getUnicode(responseHeaders.headers if responseHeaders else [])), page[:MAX_CHAR_HEURISTICS_SIZE]) + rawResponse = "%s %s %s\n%s\n%s" % (_http_client.HTTPConnection._http_vsn_str, code or "", status or "", "".join(getUnicode(responseHeaders.headers if responseHeaders else [])), page[:HEURISTIC_PAGE_SIZE_THRESHOLD]) identYwaf.non_blind.clear() if identYwaf.non_blind_check(rawResponse, silent=True): From e3028f195e0d29e1f5bb6d976c168fc7e6156c06 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 11 Jan 2021 17:36:23 +0100 Subject: [PATCH 017/243] Support for Raima Database Manager DBMS --- data/xml/errors.xml | 5 ++ data/xml/queries.xml | 39 +++++++++++++ lib/controller/handler.py | 4 ++ lib/core/agent.py | 9 ++- lib/core/common.py | 4 +- lib/core/dicts.py | 3 + lib/core/dump.py | 8 +-- lib/core/enums.py | 2 + lib/core/profiling.py | 3 - lib/core/settings.py | 10 ++-- lib/request/inject.py | 2 +- lib/utils/brute.py | 2 +- lib/utils/pivotdumptable.py | 3 +- plugins/dbms/raima/__init__.py | 29 ++++++++++ plugins/dbms/raima/connector.py | 15 +++++ plugins/dbms/raima/enumeration.py | 84 ++++++++++++++++++++++++++++ plugins/dbms/raima/filesystem.py | 18 ++++++ plugins/dbms/raima/fingerprint.py | 93 +++++++++++++++++++++++++++++++ plugins/dbms/raima/syntax.py | 22 ++++++++ plugins/dbms/raima/takeover.py | 28 ++++++++++ plugins/generic/databases.py | 34 ++++++----- plugins/generic/entries.py | 23 ++++---- sqlmap.py | 1 - 23 files changed, 397 insertions(+), 44 deletions(-) create mode 100644 plugins/dbms/raima/__init__.py create mode 100644 plugins/dbms/raima/connector.py create mode 100644 plugins/dbms/raima/enumeration.py create mode 100644 plugins/dbms/raima/filesystem.py create mode 100644 plugins/dbms/raima/fingerprint.py create mode 100644 plugins/dbms/raima/syntax.py create mode 100644 plugins/dbms/raima/takeover.py diff --git a/data/xml/errors.xml b/data/xml/errors.xml index 568b61bbe0d..99af382037f 100644 --- a/data/xml/errors.xml +++ b/data/xml/errors.xml @@ -217,4 +217,9 @@ + + + + + diff --git a/data/xml/queries.xml b/data/xml/queries.xml index b61bdb7c654..75a6f4b53ec 100644 --- a/data/xml/queries.xml +++ b/data/xml/queries.xml @@ -1626,4 +1626,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/controller/handler.py b/lib/controller/handler.py index 106b8051f32..a54a13140b3 100644 --- a/lib/controller/handler.py +++ b/lib/controller/handler.py @@ -33,6 +33,7 @@ from lib.core.settings import ORACLE_ALIASES from lib.core.settings import PGSQL_ALIASES from lib.core.settings import PRESTO_ALIASES +from lib.core.settings import RAIMA_ALIASES from lib.core.settings import SQLITE_ALIASES from lib.core.settings import SYBASE_ALIASES from lib.core.settings import VERTICA_ALIASES @@ -82,6 +83,8 @@ from plugins.dbms.postgresql import PostgreSQLMap from plugins.dbms.presto.connector import Connector as PrestoConn from plugins.dbms.presto import PrestoMap +from plugins.dbms.raima.connector import Connector as RaimaConn +from plugins.dbms.raima import RaimaMap from plugins.dbms.sqlite.connector import Connector as SQLiteConn from plugins.dbms.sqlite import SQLiteMap from plugins.dbms.sybase.connector import Connector as SybaseConn @@ -121,6 +124,7 @@ def setHandler(): (DBMS.CACHE, CACHE_ALIASES, CacheMap, CacheConn), (DBMS.EXTREMEDB, EXTREMEDB_ALIASES, ExtremeDBMap, ExtremeDBConn), (DBMS.FRONTBASE, FRONTBASE_ALIASES, FrontBaseMap, FrontBaseConn), + (DBMS.RAIMA, RAIMA_ALIASES, RaimaMap, RaimaConn), ] _ = max(_ if (conf.get("dbms") or Backend.getIdentifiedDbms() or kb.heuristicExtendedDbms or "").lower() in _[1] else () for _ in items) diff --git a/lib/core/agent.py b/lib/core/agent.py index c17407c4843..d0df1a1ccbf 100644 --- a/lib/core/agent.py +++ b/lib/core/agent.py @@ -487,7 +487,10 @@ def nullAndCastField(self, field): else: if not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')): nulledCastedField = rootQuery.cast.query % field - if Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI): + + if re.search("COUNT\(", field) and Backend.getIdentifiedDbms() in (DBMS.RAIMA,): + pass + elif Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI): nulledCastedField = rootQuery.isnull.query % (nulledCastedField, nulledCastedField) else: nulledCastedField = rootQuery.isnull.query % nulledCastedField @@ -696,7 +699,7 @@ def concatQuery(self, query, unpack=True): elif fieldsNoSelect: concatenatedQuery = "CONCAT('%s',%s,'%s')" % (kb.chars.start, concatenatedQuery, kb.chars.stop) - elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE, DBMS.SQLITE, DBMS.DB2, DBMS.FIREBIRD, DBMS.HSQLDB, DBMS.H2, DBMS.MONETDB, DBMS.DERBY, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.ALTIBASE, DBMS.MIMERSQL, DBMS.CRATEDB, DBMS.CUBRID, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE): + elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE, DBMS.SQLITE, DBMS.DB2, DBMS.FIREBIRD, DBMS.HSQLDB, DBMS.H2, DBMS.MONETDB, DBMS.DERBY, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.ALTIBASE, DBMS.MIMERSQL, DBMS.CRATEDB, DBMS.CUBRID, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE, DBMS.RAIMA): if fieldsExists: concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'||" % kb.chars.start, 1) concatenatedQuery += "||'%s'" % kb.chars.stop @@ -1006,7 +1009,7 @@ def limitQuery(self, num, query, field=None, uniqueField=None): fromFrom = limitedQuery[fromIndex + 1:] orderBy = None - if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL, DBMS.SQLITE, DBMS.H2, DBMS.VERTICA, DBMS.PRESTO, DBMS.MIMERSQL, DBMS.CUBRID, DBMS.EXTREMEDB): + if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL, DBMS.SQLITE, DBMS.H2, DBMS.VERTICA, DBMS.PRESTO, DBMS.MIMERSQL, DBMS.CUBRID, DBMS.EXTREMEDB, DBMS.RAIMA): limitStr = queries[Backend.getIdentifiedDbms()].limit.query % (num, 1) limitedQuery += " %s" % limitStr diff --git a/lib/core/common.py b/lib/core/common.py index 3f643e99298..d161feeee59 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -4193,7 +4193,7 @@ def safeSQLIdentificatorNaming(name, isTable=False): if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ACCESS, DBMS.CUBRID, DBMS.SQLITE): # Note: in SQLite double-quotes are treated as string if column/identifier is non-existent (e.g. SELECT "foobar" FROM users) retVal = "`%s`" % retVal - elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX, DBMS.MONETDB, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE): + elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX, DBMS.MONETDB, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE, DBMS.RAIMA): retVal = "\"%s\"" % retVal elif Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.ALTIBASE, DBMS.MIMERSQL): retVal = "\"%s\"" % retVal.upper() @@ -4231,7 +4231,7 @@ def unsafeSQLIdentificatorNaming(name): if isinstance(name, six.string_types): if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ACCESS, DBMS.CUBRID, DBMS.SQLITE): retVal = name.replace("`", "") - elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX, DBMS.MONETDB, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE): + elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX, DBMS.MONETDB, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE, DBMS.RAIMA): retVal = name.replace("\"", "") elif Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.ALTIBASE, DBMS.MIMERSQL): retVal = name.replace("\"", "").upper() diff --git a/lib/core/dicts.py b/lib/core/dicts.py index 29389b33348..5aa1bdda9ec 100644 --- a/lib/core/dicts.py +++ b/lib/core/dicts.py @@ -33,6 +33,7 @@ from lib.core.settings import ORACLE_ALIASES from lib.core.settings import PGSQL_ALIASES from lib.core.settings import PRESTO_ALIASES +from lib.core.settings import RAIMA_ALIASES from lib.core.settings import SQLITE_ALIASES from lib.core.settings import SYBASE_ALIASES from lib.core.settings import VERTICA_ALIASES @@ -244,6 +245,7 @@ DBMS.CACHE: (CACHE_ALIASES, "python jaydebeapi & python-jpype", "https://pypi.python.org/pypi/JayDeBeApi/ & http://jpype.sourceforge.net/", None), DBMS.EXTREMEDB: (EXTREMEDB_ALIASES, None, None, None), DBMS.FRONTBASE: (FRONTBASE_ALIASES, None, None, None), + DBMS.RAIMA: (RAIMA_ALIASES, None, None, None), } # Reference: https://blog.jooq.org/tag/sysibm-sysdummy1/ @@ -280,6 +282,7 @@ DBMS.CUBRID: "(NULL SETEQ NULL)", DBMS.CACHE: "%SQLUPPER NULL", DBMS.EXTREMEDB: "NULLIFZERO(hashcode(NULL))", + DBMS.RAIMA: "IF(ROWNUMBER()>0,CONVERT(NULL,TINYINT),NULL))", } SQL_STATEMENTS = { diff --git a/lib/core/dump.py b/lib/core/dump.py index a9d0d81d42d..89d4b0ac0ed 100644 --- a/lib/core/dump.py +++ b/lib/core/dump.py @@ -243,7 +243,7 @@ def dbTables(self, dbTables): for db, tables in dbTables.items(): tables = sorted(filter(None, tables)) - self._write("Database: %s" % unsafeSQLIdentificatorNaming(db) if db else "Current database") + self._write("Database: %s" % unsafeSQLIdentificatorNaming(db) if db and METADB_SUFFIX not in db else "") if len(tables) == 1: self._write("[1 table]") @@ -298,7 +298,7 @@ def dbTableColumns(self, tableColumns, content_type=None): maxlength2 = max(maxlength2, len("TYPE")) lines2 = "-" * (maxlength2 + 2) - self._write("Database: %s\nTable: %s" % (unsafeSQLIdentificatorNaming(db) if db else "Current database", unsafeSQLIdentificatorNaming(table))) + self._write("Database: %s\nTable: %s" % (unsafeSQLIdentificatorNaming(db) if db and METADB_SUFFIX not in db else "", unsafeSQLIdentificatorNaming(table))) if len(columns) == 1: self._write("[1 column]") @@ -353,7 +353,7 @@ def dbTablesCount(self, dbTables): maxlength1 = max(maxlength1, getConsoleLength(getUnicode(table))) for db, counts in dbTables.items(): - self._write("Database: %s" % unsafeSQLIdentificatorNaming(db) if db else "Current database") + self._write("Database: %s" % unsafeSQLIdentificatorNaming(db) if db and METADB_SUFFIX not in db else "") lines1 = "-" * (maxlength1 + 2) blank1 = " " * (maxlength1 - len("Table")) @@ -479,7 +479,7 @@ def dbTableValues(self, tableValues): separator += "+%s" % lines separator += "+" - self._write("Database: %s\nTable: %s" % (unsafeSQLIdentificatorNaming(db) if db else "Current database", unsafeSQLIdentificatorNaming(table))) + self._write("Database: %s\nTable: %s" % (unsafeSQLIdentificatorNaming(db) if db and METADB_SUFFIX not in db else "", unsafeSQLIdentificatorNaming(table))) if conf.dumpFormat == DUMP_FORMAT.SQLITE: cols = [] diff --git a/lib/core/enums.py b/lib/core/enums.py index aedf1934ee0..deb2beacf49 100644 --- a/lib/core/enums.py +++ b/lib/core/enums.py @@ -57,6 +57,7 @@ class DBMS(object): CACHE = "InterSystems Cache" EXTREMEDB = "eXtremeDB" FRONTBASE = "FrontBase" + RAIMA = "Raima Database Manager" class DBMS_DIRECTORY_NAME(object): ACCESS = "access" @@ -84,6 +85,7 @@ class DBMS_DIRECTORY_NAME(object): CACHE = "cache" EXTREMEDB = "extremedb" FRONTBASE = "frontbase" + RAIMA = "raima" class FORK(object): MARIADB = "MariaDB" diff --git a/lib/core/profiling.py b/lib/core/profiling.py index 8d4f643a011..84a8558e15c 100644 --- a/lib/core/profiling.py +++ b/lib/core/profiling.py @@ -5,14 +5,11 @@ See the file 'LICENSE' for copying permission """ -import codecs import cProfile import os -from lib.core.common import getSafeExString from lib.core.data import logger from lib.core.data import paths -from lib.core.settings import UNICODE_ENCODING def profile(profileOutputFile=None): """ diff --git a/lib/core/settings.py b/lib/core/settings.py index 01443d3c233..f0d5a11ee21 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.17" +VERSION = "1.5.1.18" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -282,6 +282,7 @@ CACHE_SYSTEM_DBS = ("%Dictionary", "INFORMATION_SCHEMA", "%SYS") EXTREMEDB_SYSTEM_DBS = ("",) FRONTBASE_SYSTEM_DBS = ("DEFINITION_SCHEMA", "INFORMATION_SCHEMA") +RAIMA_SYSTEM_DBS = ("",) # Note: () + () MSSQL_ALIASES = ("microsoft sql server", "mssqlserver", "mssql", "ms") @@ -289,7 +290,7 @@ PGSQL_ALIASES = ("postgresql", "postgres", "pgsql", "psql", "pg") + ("cockroach", "cockroachdb", "redshift", "greenplum", "yellowbrick", "enterprisedb", "aurora") ORACLE_ALIASES = ("oracle", "orcl", "ora", "or") SQLITE_ALIASES = ("sqlite", "sqlite3") -ACCESS_ALIASES = ("msaccess", "access", "jet", "microsoft access") +ACCESS_ALIASES = ("microsoft access", "msaccess", "access", "jet") FIREBIRD_ALIASES = ("firebird", "mozilla firebird", "interbase", "ibase", "fb") MAXDB_ALIASES = ("max", "maxdb", "sap maxdb", "sap db") SYBASE_ALIASES = ("sybase", "sybase sql server") @@ -309,13 +310,14 @@ CACHE_ALIASES = ("intersystems cache", "cachedb", "cache") EXTREMEDB_ALIASES = ("extremedb", "extreme") FRONTBASE_ALIASES = ("frontbase",) +RAIMA_ALIASES = ("raima database manager", "raima", "raimadb", "raimadm", "rdm", "rds", "velocis") DBMS_DIRECTORY_DICT = dict((getattr(DBMS, _), getattr(DBMS_DIRECTORY_NAME, _)) for _ in dir(DBMS) if not _.startswith("_")) -SUPPORTED_DBMS = set(MSSQL_ALIASES + MYSQL_ALIASES + PGSQL_ALIASES + ORACLE_ALIASES + SQLITE_ALIASES + ACCESS_ALIASES + FIREBIRD_ALIASES + MAXDB_ALIASES + SYBASE_ALIASES + DB2_ALIASES + HSQLDB_ALIASES + H2_ALIASES + INFORMIX_ALIASES + MONETDB_ALIASES + DERBY_ALIASES + VERTICA_ALIASES + MCKOI_ALIASES + PRESTO_ALIASES + ALTIBASE_ALIASES + MIMERSQL_ALIASES + CRATEDB_ALIASES + CUBRID_ALIASES + CACHE_ALIASES + EXTREMEDB_ALIASES) +SUPPORTED_DBMS = set(MSSQL_ALIASES + MYSQL_ALIASES + PGSQL_ALIASES + ORACLE_ALIASES + SQLITE_ALIASES + ACCESS_ALIASES + FIREBIRD_ALIASES + MAXDB_ALIASES + SYBASE_ALIASES + DB2_ALIASES + HSQLDB_ALIASES + H2_ALIASES + INFORMIX_ALIASES + MONETDB_ALIASES + DERBY_ALIASES + VERTICA_ALIASES + MCKOI_ALIASES + PRESTO_ALIASES + ALTIBASE_ALIASES + MIMERSQL_ALIASES + CRATEDB_ALIASES + CUBRID_ALIASES + CACHE_ALIASES + EXTREMEDB_ALIASES + RAIMA_ALIASES) SUPPORTED_OS = ("linux", "windows") -DBMS_ALIASES = ((DBMS.MSSQL, MSSQL_ALIASES), (DBMS.MYSQL, MYSQL_ALIASES), (DBMS.PGSQL, PGSQL_ALIASES), (DBMS.ORACLE, ORACLE_ALIASES), (DBMS.SQLITE, SQLITE_ALIASES), (DBMS.ACCESS, ACCESS_ALIASES), (DBMS.FIREBIRD, FIREBIRD_ALIASES), (DBMS.MAXDB, MAXDB_ALIASES), (DBMS.SYBASE, SYBASE_ALIASES), (DBMS.DB2, DB2_ALIASES), (DBMS.HSQLDB, HSQLDB_ALIASES), (DBMS.H2, H2_ALIASES), (DBMS.INFORMIX, INFORMIX_ALIASES), (DBMS.MONETDB, MONETDB_ALIASES), (DBMS.DERBY, DERBY_ALIASES), (DBMS.VERTICA, VERTICA_ALIASES), (DBMS.MCKOI, MCKOI_ALIASES), (DBMS.PRESTO, PRESTO_ALIASES), (DBMS.ALTIBASE, ALTIBASE_ALIASES), (DBMS.MIMERSQL, MIMERSQL_ALIASES), (DBMS.CRATEDB, CRATEDB_ALIASES), (DBMS.CUBRID, CUBRID_ALIASES), (DBMS.CACHE, CACHE_ALIASES), (DBMS.EXTREMEDB, EXTREMEDB_ALIASES), (DBMS.FRONTBASE, FRONTBASE_ALIASES)) +DBMS_ALIASES = ((DBMS.MSSQL, MSSQL_ALIASES), (DBMS.MYSQL, MYSQL_ALIASES), (DBMS.PGSQL, PGSQL_ALIASES), (DBMS.ORACLE, ORACLE_ALIASES), (DBMS.SQLITE, SQLITE_ALIASES), (DBMS.ACCESS, ACCESS_ALIASES), (DBMS.FIREBIRD, FIREBIRD_ALIASES), (DBMS.MAXDB, MAXDB_ALIASES), (DBMS.SYBASE, SYBASE_ALIASES), (DBMS.DB2, DB2_ALIASES), (DBMS.HSQLDB, HSQLDB_ALIASES), (DBMS.H2, H2_ALIASES), (DBMS.INFORMIX, INFORMIX_ALIASES), (DBMS.MONETDB, MONETDB_ALIASES), (DBMS.DERBY, DERBY_ALIASES), (DBMS.VERTICA, VERTICA_ALIASES), (DBMS.MCKOI, MCKOI_ALIASES), (DBMS.PRESTO, PRESTO_ALIASES), (DBMS.ALTIBASE, ALTIBASE_ALIASES), (DBMS.MIMERSQL, MIMERSQL_ALIASES), (DBMS.CRATEDB, CRATEDB_ALIASES), (DBMS.CUBRID, CUBRID_ALIASES), (DBMS.CACHE, CACHE_ALIASES), (DBMS.EXTREMEDB, EXTREMEDB_ALIASES), (DBMS.FRONTBASE, FRONTBASE_ALIASES), (DBMS.RAIMA, RAIMA_ALIASES)) USER_AGENT_ALIASES = ("ua", "useragent", "user-agent") REFERER_ALIASES = ("ref", "referer", "referrer") diff --git a/lib/request/inject.py b/lib/request/inject.py index 66871a96033..2de0e5e93b5 100644 --- a/lib/request/inject.py +++ b/lib/request/inject.py @@ -113,7 +113,7 @@ def _goInference(payload, expression, charsetType=None, firstChar=None, lastChar else: expression = "SELECT %s FROM (%s)" % (field, expression) - if field and conf.hexConvert or conf.binaryFields and field in conf.binaryFields: + if field and conf.hexConvert or conf.binaryFields and field in conf.binaryFields or Backend.getIdentifiedDbms() in (DBMS.RAIMA,): nulledCastedField = agent.nullAndCastField(field) injExpression = expression.replace(field, nulledCastedField, 1) else: diff --git a/lib/utils/brute.py b/lib/utils/brute.py index a03fe1b40c7..7fdc140e2a8 100644 --- a/lib/utils/brute.py +++ b/lib/utils/brute.py @@ -104,7 +104,7 @@ def tableExists(tableFile, regex=None): tables = filterListValue(tables, regex) for conf.db in (conf.db.split(',') if conf.db else [conf.db]): - if conf.db: + if conf.db and METADB_SUFFIX not in conf.db: infoMsg = "checking database '%s'" % conf.db logger.info(infoMsg) diff --git a/lib/utils/pivotdumptable.py b/lib/utils/pivotdumptable.py index fd9128d943f..1d33797fdc1 100644 --- a/lib/utils/pivotdumptable.py +++ b/lib/utils/pivotdumptable.py @@ -30,6 +30,7 @@ from lib.core.exception import SqlmapNoneDataException from lib.core.settings import MAX_INT from lib.core.settings import NULL +from lib.core.settings import SINGLE_QUOTE_MARKER from lib.core.unescaper import unescaper from lib.request import inject from lib.utils.safe2bin import safechardecode @@ -128,7 +129,7 @@ def _(column, pivotValue): if column == colList[0]: query = dumpNode.query.replace("'%s'" if unescaper.escape(pivotValue, False) != pivotValue else "%s", "%s") % (agent.preprocessField(table, column), table, agent.preprocessField(table, column), unescaper.escape(pivotValue, False)) else: - query = dumpNode.query2.replace("'%s'" if unescaper.escape(pivotValue, False) != pivotValue else "%s", "%s") % (agent.preprocessField(table, column), table, agent.preprocessField(table, colList[0]), unescaper.escape(pivotValue, False)) + query = dumpNode.query2.replace("'%s'" if unescaper.escape(pivotValue, False) != pivotValue else "%s", "%s") % (agent.preprocessField(table, column), table, agent.preprocessField(table, colList[0]), unescaper.escape(pivotValue, False) if SINGLE_QUOTE_MARKER not in dumpNode.query2 else pivotValue) query = agent.whereQuery(query) return unArrayizeValue(inject.getValue(query, blind=blind, time=blind, union=not blind, error=not blind)) diff --git a/plugins/dbms/raima/__init__.py b/plugins/dbms/raima/__init__.py new file mode 100644 index 00000000000..565c306292a --- /dev/null +++ b/plugins/dbms/raima/__init__.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2021 sqlmap developers (http://sqlmap.org/) +See the file 'LICENSE' for copying permission +""" + +from lib.core.enums import DBMS +from lib.core.settings import RAIMA_SYSTEM_DBS +from lib.core.unescaper import unescaper +from plugins.dbms.raima.enumeration import Enumeration +from plugins.dbms.raima.filesystem import Filesystem +from plugins.dbms.raima.fingerprint import Fingerprint +from plugins.dbms.raima.syntax import Syntax +from plugins.dbms.raima.takeover import Takeover +from plugins.generic.misc import Miscellaneous + +class RaimaMap(Syntax, Fingerprint, Enumeration, Filesystem, Miscellaneous, Takeover): + """ + This class defines Raima methods + """ + + def __init__(self): + self.excludeDbsList = RAIMA_SYSTEM_DBS + + for cls in self.__class__.__bases__: + cls.__init__(self) + + unescaper[DBMS.RAIMA] = Syntax.escape diff --git a/plugins/dbms/raima/connector.py b/plugins/dbms/raima/connector.py new file mode 100644 index 00000000000..1ddfb7181a2 --- /dev/null +++ b/plugins/dbms/raima/connector.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2021 sqlmap developers (http://sqlmap.org/) +See the file 'LICENSE' for copying permission +""" + +from lib.core.exception import SqlmapUnsupportedFeatureException +from plugins.generic.connector import Connector as GenericConnector + +class Connector(GenericConnector): + def connect(self): + errMsg = "on Raima it is not (currently) possible to establish a " + errMsg += "direct connection" + raise SqlmapUnsupportedFeatureException(errMsg) diff --git a/plugins/dbms/raima/enumeration.py b/plugins/dbms/raima/enumeration.py new file mode 100644 index 00000000000..980d2e3a20f --- /dev/null +++ b/plugins/dbms/raima/enumeration.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2021 sqlmap developers (http://sqlmap.org/) +See the file 'LICENSE' for copying permission +""" + +from lib.core.data import logger +from plugins.generic.enumeration import Enumeration as GenericEnumeration + +class Enumeration(GenericEnumeration): + def getBanner(self): + warnMsg = "on Raima it is not possible to get the banner" + logger.warn(warnMsg) + + return None + + def getCurrentUser(self): + warnMsg = "on Raima it is not possible to enumerate the current user" + logger.warn(warnMsg) + + def getCurrentDb(self): + warnMsg = "on Raima it is not possible to get name of the current database" + logger.warn(warnMsg) + + def isDba(self, user=None): + warnMsg = "on Raima it is not possible to test if current user is DBA" + logger.warn(warnMsg) + + def getUsers(self): + warnMsg = "on Raima it is not possible to enumerate the users" + logger.warn(warnMsg) + + return [] + + def getPasswordHashes(self): + warnMsg = "on Raima it is not possible to enumerate the user password hashes" + logger.warn(warnMsg) + + return {} + + def getPrivileges(self, *args, **kwargs): + warnMsg = "on Raima it is not possible to enumerate the user privileges" + logger.warn(warnMsg) + + return {} + + def getDbs(self): + warnMsg = "on Raima it is not possible to enumerate databases (use only '--tables')" + logger.warn(warnMsg) + + return [] + + def searchDb(self): + warnMsg = "on Raima it is not possible to search databases" + logger.warn(warnMsg) + + return [] + + def searchTable(self): + warnMsg = "on Raima it is not possible to search tables" + logger.warn(warnMsg) + + return [] + + def searchColumn(self): + warnMsg = "on Raima it is not possible to search columns" + logger.warn(warnMsg) + + return [] + + def search(self): + warnMsg = "on Raima search option is not available" + logger.warn(warnMsg) + + def getHostname(self): + warnMsg = "on Raima it is not possible to enumerate the hostname" + logger.warn(warnMsg) + + def getStatements(self): + warnMsg = "on Raima it is not possible to enumerate the SQL statements" + logger.warn(warnMsg) + + return [] diff --git a/plugins/dbms/raima/filesystem.py b/plugins/dbms/raima/filesystem.py new file mode 100644 index 00000000000..f9982e22ce3 --- /dev/null +++ b/plugins/dbms/raima/filesystem.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2021 sqlmap developers (http://sqlmap.org/) +See the file 'LICENSE' for copying permission +""" + +from lib.core.exception import SqlmapUnsupportedFeatureException +from plugins.generic.filesystem import Filesystem as GenericFilesystem + +class Filesystem(GenericFilesystem): + def readFile(self, remoteFile): + errMsg = "on Raima it is not possible to read files" + raise SqlmapUnsupportedFeatureException(errMsg) + + def writeFile(self, localFile, remoteFile, fileType=None, forceCheck=False): + errMsg = "on Raima it is not possible to write files" + raise SqlmapUnsupportedFeatureException(errMsg) diff --git a/plugins/dbms/raima/fingerprint.py b/plugins/dbms/raima/fingerprint.py new file mode 100644 index 00000000000..12b17937c72 --- /dev/null +++ b/plugins/dbms/raima/fingerprint.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2021 sqlmap developers (http://sqlmap.org/) +See the file 'LICENSE' for copying permission +""" + +from lib.core.common import Backend +from lib.core.common import Format +from lib.core.data import conf +from lib.core.data import kb +from lib.core.data import logger +from lib.core.enums import DBMS +from lib.core.session import setDbms +from lib.core.settings import METADB_SUFFIX +from lib.core.settings import RAIMA_ALIASES +from lib.request import inject +from plugins.generic.fingerprint import Fingerprint as GenericFingerprint + +class Fingerprint(GenericFingerprint): + def __init__(self): + GenericFingerprint.__init__(self, DBMS.RAIMA) + + def getFingerprint(self): + value = "" + wsOsFp = Format.getOs("web server", kb.headersFp) + + if wsOsFp: + value += "%s\n" % wsOsFp + + if kb.data.banner: + dbmsOsFp = Format.getOs("back-end DBMS", kb.bannerFp) + + if dbmsOsFp: + value += "%s\n" % dbmsOsFp + + value += "back-end DBMS: " + + if not conf.extensiveFp: + value += DBMS.RAIMA + return value + + actVer = Format.getDbms() + blank = " " * 15 + value += "active fingerprint: %s" % actVer + + if kb.bannerFp: + banVer = kb.bannerFp.get("dbmsVersion") + + if banVer: + banVer = Format.getDbms([banVer]) + value += "\n%sbanner parsing fingerprint: %s" % (blank, banVer) + + htmlErrorFp = Format.getErrorParsedDBMSes() + + if htmlErrorFp: + value += "\n%shtml error message fingerprint: %s" % (blank, htmlErrorFp) + + return value + + def checkDbms(self): + if not conf.extensiveFp and Backend.isDbmsWithin(RAIMA_ALIASES): + setDbms(DBMS.RAIMA) + return True + + infoMsg = "testing %s" % DBMS.RAIMA + logger.info(infoMsg) + + result = inject.checkBooleanExpression("ROWNUMBER()=ROWNUMBER()") + + if result: + infoMsg = "confirming %s" % DBMS.RAIMA + logger.info(infoMsg) + + result = inject.checkBooleanExpression("INSSTR('[RANDSTR1]',0,0,'[RANDSTR2]') IS NOT NULL") + + if not result: + warnMsg = "the back-end DBMS is not %s" % DBMS.RAIMA + logger.warn(warnMsg) + + return False + + setDbms(DBMS.RAIMA) + + return True + else: + warnMsg = "the back-end DBMS is not %s" % DBMS.RAIMA + logger.warn(warnMsg) + + return False + + def forceDbmsEnum(self): + conf.db = ("%s%s" % (DBMS.RAIMA, METADB_SUFFIX)).replace(' ', '_') diff --git a/plugins/dbms/raima/syntax.py b/plugins/dbms/raima/syntax.py new file mode 100644 index 00000000000..19853b3e800 --- /dev/null +++ b/plugins/dbms/raima/syntax.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2021 sqlmap developers (http://sqlmap.org/) +See the file 'LICENSE' for copying permission +""" + +from lib.core.convert import getOrds +from plugins.generic.syntax import Syntax as GenericSyntax + +class Syntax(GenericSyntax): + @staticmethod + def escape(expression, quote=True): + """ + >>> Syntax.escape("SELECT 'abcdefgh' FROM foobar") == "SELECT CHAR(97)||CHAR(98)||CHAR(99)||CHAR(100)||CHAR(101)||CHAR(102)||CHAR(103)||CHAR(104) FROM foobar" + True + """ + + def escaper(value): + return "||".join("CHAR(%d)" % _ for _ in getOrds(value)) + + return Syntax._escape(expression, quote, escaper) diff --git a/plugins/dbms/raima/takeover.py b/plugins/dbms/raima/takeover.py new file mode 100644 index 00000000000..17ba3671c17 --- /dev/null +++ b/plugins/dbms/raima/takeover.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2021 sqlmap developers (http://sqlmap.org/) +See the file 'LICENSE' for copying permission +""" + +from lib.core.exception import SqlmapUnsupportedFeatureException +from plugins.generic.takeover import Takeover as GenericTakeover + +class Takeover(GenericTakeover): + def osCmd(self): + errMsg = "on Raima it is not possible to execute commands" + raise SqlmapUnsupportedFeatureException(errMsg) + + def osShell(self): + errMsg = "on Raima it is not possible to execute commands" + raise SqlmapUnsupportedFeatureException(errMsg) + + def osPwn(self): + errMsg = "on Raima it is not possible to establish an " + errMsg += "out-of-band connection" + raise SqlmapUnsupportedFeatureException(errMsg) + + def osSmb(self): + errMsg = "on Raima it is not possible to establish an " + errMsg += "out-of-band connection" + raise SqlmapUnsupportedFeatureException(errMsg) diff --git a/plugins/generic/databases.py b/plugins/generic/databases.py index 289d73454bf..37386a80b09 100644 --- a/plugins/generic/databases.py +++ b/plugins/generic/databases.py @@ -47,6 +47,7 @@ from lib.core.exception import SqlmapNoneDataException from lib.core.exception import SqlmapUserQuitException from lib.core.settings import CURRENT_DB +from lib.core.settings import METADB_SUFFIX from lib.core.settings import PLUS_ONE_DBMSES from lib.core.settings import REFLECTED_VALUE_MARKER from lib.core.settings import UPPER_CASE_DBMSES @@ -222,7 +223,7 @@ def getTables(self, bruteForce=None): logger.warn(warnMsg) bruteForce = True - elif Backend.getIdentifiedDbms() in (DBMS.MCKOI, DBMS.EXTREMEDB): + elif Backend.getIdentifiedDbms() in (DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.RAIMA): bruteForce = True elif Backend.getIdentifiedDbms() in (DBMS.ACCESS,): @@ -336,8 +337,9 @@ def getTables(self, bruteForce=None): comment = unArrayizeValue(inject.getValue(query, blind=False, time=False)) if not isNoneValue(comment): - infoMsg = "retrieved comment '%s' for table '%s' " % (comment, unsafeSQLIdentificatorNaming(table)) - infoMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(db) + infoMsg = "retrieved comment '%s' for table '%s'" % (comment, unsafeSQLIdentificatorNaming(table)) + if METADB_SUFFIX not in db: + infoMsg += " in database '%s'" % unsafeSQLIdentificatorNaming(db) logger.info(infoMsg) else: warnMsg = "on %s it is not " % Backend.getIdentifiedDbms() @@ -418,8 +420,9 @@ def getTables(self, bruteForce=None): comment = unArrayizeValue(inject.getValue(query, union=False, error=False)) if not isNoneValue(comment): - infoMsg = "retrieved comment '%s' for table '%s' " % (comment, unsafeSQLIdentificatorNaming(table)) - infoMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(db) + infoMsg = "retrieved comment '%s' for table '%s'" % (comment, unsafeSQLIdentificatorNaming(table)) + if METADB_SUFFIX not in db: + infoMsg += " in database '%s'" % unsafeSQLIdentificatorNaming(db) logger.info(infoMsg) else: warnMsg = "on %s it is not " % Backend.getIdentifiedDbms() @@ -516,8 +519,9 @@ def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMod tblList = list(tblList) elif not conf.search: - errMsg = "unable to retrieve the tables " - errMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) + errMsg = "unable to retrieve the tables" + if METADB_SUFFIX not in conf.db: + errMsg += " in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) raise SqlmapNoneDataException(errMsg) else: return kb.data.cachedColumns @@ -534,7 +538,7 @@ def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMod logger.warn(warnMsg) bruteForce = True - elif Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB): + elif Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.RAIMA): warnMsg = "cannot retrieve column names, " warnMsg += "back-end DBMS is %s" % Backend.getIdentifiedDbms() logger.warn(warnMsg) @@ -549,7 +553,7 @@ def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMod resumeAvailable = True break - if resumeAvailable and not conf.freshQueries: + if resumeAvailable and not (conf.freshQueries and not colList): columns = {} for column in colList: @@ -632,7 +636,8 @@ def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMod values = [(_,) for _ in colList] else: infoMsg += "for table '%s' " % unsafeSQLIdentificatorNaming(tbl) - infoMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) + if METADB_SUFFIX not in conf.db: + infoMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) logger.info(infoMsg) values = None @@ -781,7 +786,8 @@ def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMod columns[safeSQLIdentificatorNaming(value)] = None else: infoMsg += "for table '%s' " % unsafeSQLIdentificatorNaming(tbl) - infoMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) + if METADB_SUFFIX not in conf.db: + infoMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) logger.info(infoMsg) count = inject.getValue(query, union=False, error=False, expected=EXPECTED.INT, charsetType=CHARSET_TYPE.DIGITS) @@ -802,7 +808,8 @@ def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMod if not columns: errMsg = "unable to retrieve the %scolumns " % ("number of " if not Backend.isDbms(DBMS.MSSQL) else "") errMsg += "for table '%s' " % unsafeSQLIdentificatorNaming(tbl) - errMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) + if METADB_SUFFIX not in conf.db: + errMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) logger.error(errMsg) continue @@ -904,7 +911,8 @@ def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMod if not kb.data.cachedColumns: warnMsg = "unable to retrieve column names for " warnMsg += ("table '%s' " % unsafeSQLIdentificatorNaming(unArrayizeValue(tblList))) if len(tblList) == 1 else "any table " - warnMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) + if METADB_SUFFIX not in conf.db: + warnMsg += "in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) logger.warn(warnMsg) if bruteForce is None: diff --git a/plugins/generic/entries.py b/plugins/generic/entries.py index 8cc32b5385c..3cb50a0fd25 100644 --- a/plugins/generic/entries.py +++ b/plugins/generic/entries.py @@ -42,6 +42,7 @@ from lib.core.exception import SqlmapUnsupportedFeatureException from lib.core.settings import CHECK_ZERO_COLUMNS_THRESHOLD from lib.core.settings import CURRENT_DB +from lib.core.settings import METADB_SUFFIX from lib.core.settings import NULL from lib.core.settings import PLUS_ONE_DBMSES from lib.core.settings import UPPER_CASE_DBMSES @@ -137,9 +138,9 @@ def dumpTable(self, foundData=None): kb.dumpTable = "%s.%s" % (conf.db, tbl) if safeSQLIdentificatorNaming(conf.db) not in kb.data.cachedColumns or safeSQLIdentificatorNaming(tbl, True) not in kb.data.cachedColumns[safeSQLIdentificatorNaming(conf.db)] or not kb.data.cachedColumns[safeSQLIdentificatorNaming(conf.db)][safeSQLIdentificatorNaming(tbl, True)]: - warnMsg = "unable to enumerate the columns for table " - warnMsg += "'%s' in database" % unsafeSQLIdentificatorNaming(tbl) - warnMsg += " '%s'" % unsafeSQLIdentificatorNaming(conf.db) + warnMsg = "unable to enumerate the columns for table '%s'" % unsafeSQLIdentificatorNaming(tbl) + if METADB_SUFFIX not in conf.db: + warnMsg += " in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) warnMsg += ", skipping" if len(tblList) > 1 else "" logger.warn(warnMsg) @@ -153,7 +154,8 @@ def dumpTable(self, foundData=None): if not colList: warnMsg = "skipping table '%s'" % unsafeSQLIdentificatorNaming(tbl) - warnMsg += " in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) + if METADB_SUFFIX not in conf.db: + warnMsg += " in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) warnMsg += " (no usable column names)" logger.warn(warnMsg) continue @@ -166,7 +168,8 @@ def dumpTable(self, foundData=None): if conf.col: infoMsg += " of column(s) '%s'" % colNames infoMsg += " for table '%s'" % unsafeSQLIdentificatorNaming(tbl) - infoMsg += " in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) + if METADB_SUFFIX not in conf.db: + infoMsg += " in database '%s'" % unsafeSQLIdentificatorNaming(conf.db) logger.info(infoMsg) for column in colList: @@ -182,7 +185,7 @@ def dumpTable(self, foundData=None): if Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2, DBMS.DERBY, DBMS.ALTIBASE, DBMS.MIMERSQL): query = rootQuery.inband.query % (colString, tbl.upper() if not conf.db else ("%s.%s" % (conf.db.upper(), tbl.upper()))) - elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.ACCESS, DBMS.FIREBIRD, DBMS.MAXDB, DBMS.MCKOI, DBMS.EXTREMEDB): + elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.ACCESS, DBMS.FIREBIRD, DBMS.MAXDB, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.RAIMA): query = rootQuery.inband.query % (colString, tbl) elif Backend.getIdentifiedDbms() in (DBMS.SYBASE, DBMS.MSSQL): # Partial inband and error @@ -291,12 +294,10 @@ def dumpTable(self, foundData=None): if Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2, DBMS.DERBY, DBMS.ALTIBASE, DBMS.MIMERSQL): query = rootQuery.blind.count % (tbl.upper() if not conf.db else ("%s.%s" % (conf.db.upper(), tbl.upper()))) - elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.ACCESS, DBMS.FIREBIRD, DBMS.MCKOI, DBMS.EXTREMEDB): + elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.MAXDB, DBMS.ACCESS, DBMS.FIREBIRD, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.RAIMA): query = rootQuery.blind.count % tbl elif Backend.getIdentifiedDbms() in (DBMS.SYBASE, DBMS.MSSQL): query = rootQuery.blind.count % ("%s.%s" % (conf.db, tbl)) - elif Backend.isDbms(DBMS.MAXDB): - query = rootQuery.blind.count % tbl elif Backend.isDbms(DBMS.INFORMIX): query = rootQuery.blind.count % (conf.db, tbl) else: @@ -329,8 +330,8 @@ def dumpTable(self, foundData=None): continue - elif Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.SYBASE, DBMS.MAXDB, DBMS.MSSQL, DBMS.INFORMIX, DBMS.MCKOI): - if Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB): + elif Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.SYBASE, DBMS.MAXDB, DBMS.MSSQL, DBMS.INFORMIX, DBMS.MCKOI, DBMS.RAIMA): + if Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI, DBMS.RAIMA): table = tbl elif Backend.getIdentifiedDbms() in (DBMS.SYBASE, DBMS.MSSQL, DBMS.MAXDB): table = "%s.%s" % (conf.db, tbl) diff --git a/sqlmap.py b/sqlmap.py index 18e1d58acc7..f18df7fbe0d 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -80,7 +80,6 @@ from lib.core.settings import VERSION from lib.parse.cmdline import cmdLineParser from lib.utils.crawler import crawl - from thirdparty import six except KeyboardInterrupt: errMsg = "user aborted" From 2ef07c80dbd851715ffa3b59accd201cc7de7278 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 12 Jan 2021 13:21:51 +0100 Subject: [PATCH 018/243] Some more refactoring --- lib/controller/checks.py | 12 ++++++------ lib/core/datatype.py | 8 ++++++-- lib/core/option.py | 7 +------ lib/core/settings.py | 2 +- lib/request/connect.py | 16 ++++++++-------- lib/request/redirecthandler.py | 8 ++++---- lib/utils/brute.py | 12 ++++++------ lib/utils/hash.py | 6 +++--- lib/utils/search.py | 6 +++--- plugins/dbms/raima/connector.py | 2 +- plugins/dbms/raima/enumeration.py | 28 ++++++++++++++-------------- plugins/dbms/raima/filesystem.py | 4 ++-- plugins/dbms/raima/takeover.py | 8 ++++---- plugins/generic/databases.py | 19 ++++++++++++------- 14 files changed, 71 insertions(+), 67 deletions(-) diff --git a/lib/controller/checks.py b/lib/controller/checks.py index 2119233efd4..a435917c0ab 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -1167,7 +1167,7 @@ def checkDynParam(place, parameter, value): dynamicity might depend on another parameter. """ - if kb.redirectChoice: + if kb.choices.redirect: return None kb.matchRatio = None @@ -1268,7 +1268,7 @@ def checkStability(): secondPage, _, _ = Request.queryPage(content=True, noteResponseTime=False, raise404=False) - if kb.redirectChoice: + if kb.choices.redirect: return None kb.pageStable = (firstPage == secondPage) @@ -1415,11 +1415,11 @@ def checkWaf(): value = "" if not conf.parameters.get(PLACE.GET) else conf.parameters[PLACE.GET] + DEFAULT_GET_POST_DELIMITER value += "%s=%s" % (randomStr(), agent.addPayloadDelimiters(payload)) - pushValue(kb.redirectChoice) + pushValue(kb.choices.redirect) pushValue(kb.resendPostOnRedirect) pushValue(conf.timeout) - kb.redirectChoice = REDIRECTION.YES + kb.choices.redirect = REDIRECTION.YES kb.resendPostOnRedirect = False conf.timeout = IPS_WAF_CHECK_TIMEOUT @@ -1432,7 +1432,7 @@ def checkWaf(): conf.timeout = popValue() kb.resendPostOnRedirect = popValue() - kb.redirectChoice = popValue() + kb.choices.redirect = popValue() hashDBWrite(HASHDB_KEYS.CHECK_WAF_RESULT, retVal, True) @@ -1565,7 +1565,7 @@ def checkConnection(suppressOutput=False): else: kb.errorIsNone = True - if kb.redirectChoice == REDIRECTION.YES and threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID: + if kb.choices.redirect == REDIRECTION.YES and threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID: if (threadData.lastRedirectURL[1] or "").startswith("https://") and conf.hostname in getUnicode(threadData.lastRedirectURL[1]): conf.url = re.sub(r"https?://", "https://", conf.url) match = re.search(r":(\d+)", threadData.lastRedirectURL[1]) diff --git a/lib/core/datatype.py b/lib/core/datatype.py index 9b675443117..ab3032dee3f 100644 --- a/lib/core/datatype.py +++ b/lib/core/datatype.py @@ -21,13 +21,14 @@ class AttribDict(dict): 1 """ - def __init__(self, indict=None, attribute=None): + def __init__(self, indict=None, attribute=None, keycheck=True): if indict is None: indict = {} # Set any attributes here - before initialisation # these remain as normal attributes self.attribute = attribute + self.keycheck = keycheck dict.__init__(self, indict) self.__initialised = True @@ -43,7 +44,10 @@ def __getattr__(self, item): try: return self.__getitem__(item) except KeyError: - raise AttributeError("unable to access item '%s'" % item) + if self.keycheck: + raise AttributeError("unable to access item '%s'" % item) + else: + return None def __setattr__(self, item, value): """ diff --git a/lib/core/option.py b/lib/core/option.py index 1ef78a367b1..e4cba0cc975 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -2013,12 +2013,10 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.chars.stop = "%s%s%s" % (KB_CHARS_BOUNDARY_CHAR, randomStr(length=3, alphabet=KB_CHARS_LOW_FREQUENCY_ALPHABET), KB_CHARS_BOUNDARY_CHAR) kb.chars.at, kb.chars.space, kb.chars.dollar, kb.chars.hash_ = ("%s%s%s" % (KB_CHARS_BOUNDARY_CHAR, _, KB_CHARS_BOUNDARY_CHAR) for _ in randomStr(length=4, lowercase=True)) + kb.choices = AttribDict(keycheck=False) kb.codePage = None - kb.columnExistsChoice = None kb.commonOutputs = None - kb.connErrorChoice = None kb.connErrorCounter = 0 - kb.cookieEncodeChoice = None kb.copyExecTest = None kb.counters = {} kb.customInjectionMark = CUSTOM_INJECTION_MARK_CHAR @@ -2122,7 +2120,6 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.proxyAuthHeader = None kb.queryCounter = 0 kb.randomPool = {} - kb.redirectChoice = None kb.reflectiveMechanism = True kb.reflectiveCounters = {REFLECTIVE_COUNTER.MISS: 0, REFLECTIVE_COUNTER.HIT: 0} kb.requestCounter = 0 @@ -2142,9 +2139,7 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.reduceTests = None kb.sslSuccess = False kb.stickyDBMS = False - kb.storeHashesChoice = None kb.suppressResumeInfo = False - kb.tableExistsChoice = None kb.tableFrom = None kb.technique = None kb.tempDir = None diff --git a/lib/core/settings.py b/lib/core/settings.py index f0d5a11ee21..36e25999850 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.18" +VERSION = "1.5.1.19" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/connect.py b/lib/request/connect.py index 881266855b1..bf4bc1257db 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -610,8 +610,8 @@ class _(dict): # Get HTTP response if hasattr(conn, "redurl"): - page = (threadData.lastRedirectMsg[1] if kb.redirectChoice == REDIRECTION.NO else Connect._connReadProxy(conn)) if not skipRead else None - skipLogTraffic = kb.redirectChoice == REDIRECTION.NO + page = (threadData.lastRedirectMsg[1] if kb.choices.redirect == REDIRECTION.NO else Connect._connReadProxy(conn)) if not skipRead else None + skipLogTraffic = kb.choices.redirect == REDIRECTION.NO code = conn.redcode if not finalCode else code else: page = Connect._connReadProxy(conn) if not skipRead else None @@ -844,13 +844,13 @@ class _(dict): with kb.locks.connError: kb.connErrorCounter += 1 - if kb.connErrorCounter >= MAX_CONSECUTIVE_CONNECTION_ERRORS and kb.connErrorChoice is None: + if kb.connErrorCounter >= MAX_CONSECUTIVE_CONNECTION_ERRORS and kb.choices.connError is None: message = "there seems to be a continuous problem with connection to the target. " message += "Are you sure that you want to continue? [y/N] " - kb.connErrorChoice = readInput(message, default='N', boolean=True) + kb.choices.connError = readInput(message, default='N', boolean=True) - if kb.connErrorChoice is False: + if kb.choices.connError is False: raise SqlmapSkipTargetException if "forcibly closed" in tbMsg: @@ -1025,10 +1025,10 @@ def queryPage(value=None, place=None, content=False, getRatioValue=False, silent skip = False if place == PLACE.COOKIE or place == PLACE.CUSTOM_HEADER and value.split(',')[0].upper() == HTTP_HEADER.COOKIE.upper(): - if kb.cookieEncodeChoice is None: + if kb.choices.cookieEncode is None: msg = "do you want to URL encode cookie values (implementation specific)? %s" % ("[Y/n]" if not conf.url.endswith(".aspx") else "[y/N]") # Reference: https://support.microsoft.com/en-us/kb/313282 - kb.cookieEncodeChoice = readInput(msg, default='Y' if not conf.url.endswith(".aspx") else 'N', boolean=True) - if not kb.cookieEncodeChoice: + kb.choices.cookieEncode = readInput(msg, default='Y' if not conf.url.endswith(".aspx") else 'N', boolean=True) + if not kb.choices.cookieEncode: skip = True if not skip: diff --git a/lib/request/redirecthandler.py b/lib/request/redirecthandler.py index e2de7536f55..ffb5605b9be 100644 --- a/lib/request/redirecthandler.py +++ b/lib/request/redirecthandler.py @@ -48,13 +48,13 @@ def _get_header_redirect(self, headers): def _ask_redirect_choice(self, redcode, redurl, method): with kb.locks.redirect: - if kb.redirectChoice is None: + if kb.choices.redirect is None: msg = "got a %d redirect to " % redcode msg += "'%s'. Do you want to follow? [Y/n] " % redurl - kb.redirectChoice = REDIRECTION.YES if readInput(msg, default='Y', boolean=True) else REDIRECTION.NO + kb.choices.redirect = REDIRECTION.YES if readInput(msg, default='Y', boolean=True) else REDIRECTION.NO - if kb.redirectChoice == REDIRECTION.YES and method == HTTPMETHOD.POST and kb.resendPostOnRedirect is None: + if kb.choices.redirect == REDIRECTION.YES and method == HTTPMETHOD.POST and kb.resendPostOnRedirect is None: msg = "redirect is a result of a " msg += "POST request. Do you want to " msg += "resend original POST data to a new " @@ -116,7 +116,7 @@ def http_error_302(self, req, fp, code, msg, headers): redurl = None result = fp - if redurl and kb.redirectChoice == REDIRECTION.YES: + if redurl and kb.choices.redirect == REDIRECTION.YES: parseResponse(content, headers) req.headers[HTTP_HEADER.HOST] = getHostHeader(redurl) diff --git a/lib/utils/brute.py b/lib/utils/brute.py index 7fdc140e2a8..1d8a3e2902a 100644 --- a/lib/utils/brute.py +++ b/lib/utils/brute.py @@ -63,15 +63,15 @@ def _addPageTextWords(): @stackedmethod def tableExists(tableFile, regex=None): - if kb.tableExistsChoice is None and not any(_ for _ in kb.injection.data if _ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)) and not conf.direct: + if kb.choices.tableExists is None and not any(_ for _ in kb.injection.data if _ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)) and not conf.direct: warnMsg = "it's not recommended to use '%s' and/or '%s' " % (PAYLOAD.SQLINJECTION[PAYLOAD.TECHNIQUE.TIME], PAYLOAD.SQLINJECTION[PAYLOAD.TECHNIQUE.STACKED]) warnMsg += "for common table existence check" logger.warn(warnMsg) message = "are you sure you want to continue? [y/N] " - kb.tableExistsChoice = readInput(message, default='N', boolean=True) + kb.choices.tableExists = readInput(message, default='N', boolean=True) - if not kb.tableExistsChoice: + if not kb.choices.tableExists: return None result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), randomStr()))) @@ -187,15 +187,15 @@ def tableExistsThread(): return kb.data.cachedTables def columnExists(columnFile, regex=None): - if kb.columnExistsChoice is None and not any(_ for _ in kb.injection.data if _ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)) and not conf.direct: + if kb.choices.columnExists is None and not any(_ for _ in kb.injection.data if _ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)) and not conf.direct: warnMsg = "it's not recommended to use '%s' and/or '%s' " % (PAYLOAD.SQLINJECTION[PAYLOAD.TECHNIQUE.TIME], PAYLOAD.SQLINJECTION[PAYLOAD.TECHNIQUE.STACKED]) warnMsg += "for common column existence check" logger.warn(warnMsg) message = "are you sure you want to continue? [y/N] " - kb.columnExistsChoice = readInput(message, default='N', boolean=True) + kb.choices.columnExists = readInput(message, default='N', boolean=True) - if not kb.columnExistsChoice: + if not kb.choices.columnExists: return None if not conf.tbl: diff --git a/lib/utils/hash.py b/lib/utils/hash.py index a69d96d2269..7510104a512 100644 --- a/lib/utils/hash.py +++ b/lib/utils/hash.py @@ -637,13 +637,13 @@ def storeHashesToFile(attack_dict): if item and item not in items: items.add(item) - if kb.storeHashesChoice is None: + if kb.choices.storeHashes is None: message = "do you want to store hashes to a temporary file " message += "for eventual further processing with other tools [y/N] " - kb.storeHashesChoice = readInput(message, default='N', boolean=True) + kb.choices.storeHashes = readInput(message, default='N', boolean=True) - if items and kb.storeHashesChoice: + if items and kb.choices.storeHashes: handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.HASHES, suffix=".txt") os.close(handle) diff --git a/lib/utils/search.py b/lib/utils/search.py index 009f5797e88..2b7aee5eab4 100644 --- a/lib/utils/search.py +++ b/lib/utils/search.py @@ -184,8 +184,8 @@ def _search(dork): @stackedmethod def search(dork): - pushValue(kb.redirectChoice) - kb.redirectChoice = REDIRECTION.YES + pushValue(kb.choices.redirect) + kb.choices.redirect = REDIRECTION.YES try: return _search(dork) @@ -203,7 +203,7 @@ def search(dork): else: raise finally: - kb.redirectChoice = popValue() + kb.choices.redirect = popValue() def setHTTPHandlers(): # Cross-referenced function raise NotImplementedError diff --git a/plugins/dbms/raima/connector.py b/plugins/dbms/raima/connector.py index 1ddfb7181a2..3274991f6ad 100644 --- a/plugins/dbms/raima/connector.py +++ b/plugins/dbms/raima/connector.py @@ -10,6 +10,6 @@ class Connector(GenericConnector): def connect(self): - errMsg = "on Raima it is not (currently) possible to establish a " + errMsg = "on Raima Database Manager it is not (currently) possible to establish a " errMsg += "direct connection" raise SqlmapUnsupportedFeatureException(errMsg) diff --git a/plugins/dbms/raima/enumeration.py b/plugins/dbms/raima/enumeration.py index 980d2e3a20f..34d4fa36855 100644 --- a/plugins/dbms/raima/enumeration.py +++ b/plugins/dbms/raima/enumeration.py @@ -10,75 +10,75 @@ class Enumeration(GenericEnumeration): def getBanner(self): - warnMsg = "on Raima it is not possible to get the banner" + warnMsg = "on Raima Database Manager it is not possible to get the banner" logger.warn(warnMsg) return None def getCurrentUser(self): - warnMsg = "on Raima it is not possible to enumerate the current user" + warnMsg = "on Raima Database Manager it is not possible to enumerate the current user" logger.warn(warnMsg) def getCurrentDb(self): - warnMsg = "on Raima it is not possible to get name of the current database" + warnMsg = "on Raima Database Manager it is not possible to get name of the current database" logger.warn(warnMsg) def isDba(self, user=None): - warnMsg = "on Raima it is not possible to test if current user is DBA" + warnMsg = "on Raima Database Manager it is not possible to test if current user is DBA" logger.warn(warnMsg) def getUsers(self): - warnMsg = "on Raima it is not possible to enumerate the users" + warnMsg = "on Raima Database Manager it is not possible to enumerate the users" logger.warn(warnMsg) return [] def getPasswordHashes(self): - warnMsg = "on Raima it is not possible to enumerate the user password hashes" + warnMsg = "on Raima Database Manager it is not possible to enumerate the user password hashes" logger.warn(warnMsg) return {} def getPrivileges(self, *args, **kwargs): - warnMsg = "on Raima it is not possible to enumerate the user privileges" + warnMsg = "on Raima Database Manager it is not possible to enumerate the user privileges" logger.warn(warnMsg) return {} def getDbs(self): - warnMsg = "on Raima it is not possible to enumerate databases (use only '--tables')" + warnMsg = "on Raima Database Manager it is not possible to enumerate databases (use only '--tables')" logger.warn(warnMsg) return [] def searchDb(self): - warnMsg = "on Raima it is not possible to search databases" + warnMsg = "on Raima Database Manager it is not possible to search databases" logger.warn(warnMsg) return [] def searchTable(self): - warnMsg = "on Raima it is not possible to search tables" + warnMsg = "on Raima Database Manager it is not possible to search tables" logger.warn(warnMsg) return [] def searchColumn(self): - warnMsg = "on Raima it is not possible to search columns" + warnMsg = "on Raima Database Manager it is not possible to search columns" logger.warn(warnMsg) return [] def search(self): - warnMsg = "on Raima search option is not available" + warnMsg = "on Raima Database Manager search option is not available" logger.warn(warnMsg) def getHostname(self): - warnMsg = "on Raima it is not possible to enumerate the hostname" + warnMsg = "on Raima Database Manager it is not possible to enumerate the hostname" logger.warn(warnMsg) def getStatements(self): - warnMsg = "on Raima it is not possible to enumerate the SQL statements" + warnMsg = "on Raima Database Manager it is not possible to enumerate the SQL statements" logger.warn(warnMsg) return [] diff --git a/plugins/dbms/raima/filesystem.py b/plugins/dbms/raima/filesystem.py index f9982e22ce3..da17a0d1ec1 100644 --- a/plugins/dbms/raima/filesystem.py +++ b/plugins/dbms/raima/filesystem.py @@ -10,9 +10,9 @@ class Filesystem(GenericFilesystem): def readFile(self, remoteFile): - errMsg = "on Raima it is not possible to read files" + errMsg = "on Raima Database Manager it is not possible to read files" raise SqlmapUnsupportedFeatureException(errMsg) def writeFile(self, localFile, remoteFile, fileType=None, forceCheck=False): - errMsg = "on Raima it is not possible to write files" + errMsg = "on Raima Database Manager it is not possible to write files" raise SqlmapUnsupportedFeatureException(errMsg) diff --git a/plugins/dbms/raima/takeover.py b/plugins/dbms/raima/takeover.py index 17ba3671c17..1ab01334704 100644 --- a/plugins/dbms/raima/takeover.py +++ b/plugins/dbms/raima/takeover.py @@ -10,19 +10,19 @@ class Takeover(GenericTakeover): def osCmd(self): - errMsg = "on Raima it is not possible to execute commands" + errMsg = "on Raima Database Manager it is not possible to execute commands" raise SqlmapUnsupportedFeatureException(errMsg) def osShell(self): - errMsg = "on Raima it is not possible to execute commands" + errMsg = "on Raima Database Manager it is not possible to execute commands" raise SqlmapUnsupportedFeatureException(errMsg) def osPwn(self): - errMsg = "on Raima it is not possible to establish an " + errMsg = "on Raima Database Manager it is not possible to establish an " errMsg += "out-of-band connection" raise SqlmapUnsupportedFeatureException(errMsg) def osSmb(self): - errMsg = "on Raima it is not possible to establish an " + errMsg = "on Raima Database Manager it is not possible to establish an " errMsg += "out-of-band connection" raise SqlmapUnsupportedFeatureException(errMsg) diff --git a/plugins/generic/databases.py b/plugins/generic/databases.py index 37386a80b09..86853c58e41 100644 --- a/plugins/generic/databases.py +++ b/plugins/generic/databases.py @@ -540,8 +540,8 @@ def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMod elif Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.RAIMA): warnMsg = "cannot retrieve column names, " - warnMsg += "back-end DBMS is %s" % Backend.getIdentifiedDbms() - logger.warn(warnMsg) + warnMsg += "back-end DBMS is '%s'" % Backend.getIdentifiedDbms() + singleTimeWarnMessage(warnMsg) bruteForce = True if bruteForce: @@ -571,12 +571,17 @@ def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMod return kb.data.cachedColumns - message = "do you want to use common column existence check? %s" % ("[Y/n/q]" if Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB) else "[y/N/q]") - choice = readInput(message, default='Y' if 'Y' in message else 'N').upper() + if kb.choices.columnExists is None: + message = "do you want to use common column existence check? %s" % ("[Y/n/q]" if Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB) else "[y/N/q]") + kb.choices.columnExists = readInput(message, default='Y' if 'Y' in message else 'N').upper() - if choice == 'N': - return - elif choice == 'Q': + if kb.choices.columnExists == 'N': + if dumpMode and colList: + kb.data.cachedColumns[safeSQLIdentificatorNaming(conf.db)] = {safeSQLIdentificatorNaming(tbl, True): dict((_, None) for _ in colList)} + return kb.data.cachedColumns + else: + return None + elif kb.choices.columnExists == 'Q': raise SqlmapUserQuitException else: return columnExists(paths.COMMON_COLUMNS) From 3002f92e344d05dc0c2ce2f23ba553818f7884de Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 12 Jan 2021 13:23:08 +0100 Subject: [PATCH 019/243] Trivial update --- lib/core/settings.py | 2 +- plugins/generic/databases.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 36e25999850..7860818d217 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.19" +VERSION = "1.5.1.20" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/plugins/generic/databases.py b/plugins/generic/databases.py index 86853c58e41..58ee2fb00a4 100644 --- a/plugins/generic/databases.py +++ b/plugins/generic/databases.py @@ -540,7 +540,7 @@ def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMod elif Backend.getIdentifiedDbms() in (DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.RAIMA): warnMsg = "cannot retrieve column names, " - warnMsg += "back-end DBMS is '%s'" % Backend.getIdentifiedDbms() + warnMsg += "back-end DBMS is %s" % Backend.getIdentifiedDbms() singleTimeWarnMessage(warnMsg) bruteForce = True From 66a3118f819b55a77434d17e9b506bacad66bb42 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 12 Jan 2021 15:36:02 +0100 Subject: [PATCH 020/243] Adding support for JSON aggregation on SQLite --- lib/core/settings.py | 2 +- lib/techniques/union/use.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 7860818d217..166db6b5753 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.20" +VERSION = "1.5.1.21" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index 188861be658..2740c969233 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -242,14 +242,16 @@ def unionUse(expression, unpack=True, dump=False): debugMsg += "it does not play well with UNION query SQL injection" singleTimeDebugMessage(debugMsg) - if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL) and expressionFields and not conf.binaryFields: + if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL, DBMS.SQLITE) and expressionFields and not conf.binaryFields: match = re.search(r"SELECT\s*(.+?)\bFROM", expression, re.I) - if match and not (Backend.isDbms(DBMS.ORACLE) and FROM_DUMMY_TABLE[DBMS.ORACLE] in expression): + if match and not (Backend.isDbms(DBMS.ORACLE) and FROM_DUMMY_TABLE[DBMS.ORACLE] in expression) and not re.search(r"\b(MIN|MAX|COUNT)\(", expression): kb.jsonAggMode = True if Backend.isDbms(DBMS.MYSQL): query = expression.replace(expressionFields, "CONCAT('%s',JSON_ARRAYAGG(CONCAT_WS('%s',%s)),'%s')" % (kb.chars.start, kb.chars.delimiter, expressionFields, kb.chars.stop), 1) elif Backend.isDbms(DBMS.ORACLE): query = expression.replace(expressionFields, "'%s'||JSON_ARRAYAGG(%s)||'%s'" % (kb.chars.start, ("||'%s'||" % kb.chars.delimiter).join(expressionFieldsList), kb.chars.stop), 1) + elif Backend.isDbms(DBMS.SQLITE): + query = expression.replace(expressionFields, "'%s'||JSON_GROUP_ARRAY(%s)||'%s'" % (kb.chars.start, ("||'%s'||" % kb.chars.delimiter).join("COALESCE(%s,' ')" % field for field in expressionFieldsList), kb.chars.stop), 1) elif Backend.isDbms(DBMS.PGSQL): # Note: ARRAY_AGG does CSV alike output, thus enclosing start/end inside each item query = expression.replace(expressionFields, "ARRAY_AGG('%s'||%s||'%s')::text" % (kb.chars.start, ("||'%s'||" % kb.chars.delimiter).join("COALESCE(%s::text,' ')" % field for field in expressionFieldsList), kb.chars.stop), 1) elif Backend.isDbms(DBMS.MSSQL): From a697e6c3072cfcfdeef7dc956d530fa05f1553be Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 12 Jan 2021 15:57:12 +0100 Subject: [PATCH 021/243] All of a sudden PY2 _configparser doesn't support \tkey = value --- lib/core/common.py | 6 +++--- lib/core/settings.py | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index d161feeee59..2b85b4cff7b 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -216,7 +216,7 @@ def write(self, fp): fp.write("[%s]\n" % _configparser.DEFAULTSECT) for (key, value) in self._defaults.items(): - fp.write("\t%s = %s" % (key, getUnicode(value, UNICODE_ENCODING))) + fp.write("%s = %s" % (key, getUnicode(value, UNICODE_ENCODING))) fp.write("\n") @@ -226,9 +226,9 @@ def write(self, fp): for (key, value) in self._sections[section].items(): if key != "__name__": if value is None: - fp.write("\t%s\n" % (key)) + fp.write("%s\n" % (key)) elif not isListLike(value): - fp.write("\t%s = %s\n" % (key, getUnicode(value, UNICODE_ENCODING))) + fp.write("%s = %s\n" % (key, getUnicode(value, UNICODE_ENCODING))) fp.write("\n") diff --git a/lib/core/settings.py b/lib/core/settings.py index 166db6b5753..f90edc863fd 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.21" +VERSION = "1.5.1.22" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -813,6 +813,9 @@ # Default REST-JSON API server listen port RESTAPI_DEFAULT_PORT = 8775 +# Unsupported options by REST-JSON API server +RESTAPI_UNSUPPORTED_OPTIONS = {"sqlShell", "wizard"} + # Use "Supplementary Private Use Area-A" INVALID_UNICODE_PRIVATE_AREA = False From c8dc375fb57aa3ac8b27d9cdd180ec406fa1139d Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 12 Jan 2021 15:58:40 +0100 Subject: [PATCH 022/243] Update for #4537 --- lib/core/settings.py | 2 +- lib/utils/api.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index f90edc863fd..9dc519460e9 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.22" +VERSION = "1.5.1.23" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/utils/api.py b/lib/utils/api.py index 33a15460407..a478c8bbafd 100644 --- a/lib/utils/api.py +++ b/lib/utils/api.py @@ -49,6 +49,7 @@ from lib.core.settings import RESTAPI_DEFAULT_ADAPTER from lib.core.settings import RESTAPI_DEFAULT_ADDRESS from lib.core.settings import RESTAPI_DEFAULT_PORT +from lib.core.settings import RESTAPI_UNSUPPORTED_OPTIONS from lib.core.settings import VERSION_STRING from lib.core.shell import autoCompletion from lib.core.subprocessng import Popen @@ -502,6 +503,11 @@ def scan_start(taskid): logger.warning("[%s] Invalid JSON options provided to scan_start()" % taskid) return jsonize({"success": False, "message": "Invalid JSON options"}) + for key in request.json: + if key in RESTAPI_UNSUPPORTED_OPTIONS: + logger.warning("[%s] Unsupported option '%s' provided to scan_start()" % (taskid, key)) + return jsonize({"success": False, "message": "Unsupported option '%s'" % key}) + # Initialize sqlmap engine's options with user's provided options, if any for option, value in request.json.items(): DataStore.tasks[taskid].set_option(option, value) @@ -836,7 +842,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non raw = _client("%s/task/new" % addr) res = dejsonize(raw) if not res["success"]: - logger.error("Failed to create new task") + logger.error("Failed to create new task ('%s')" % res.get("message", "")) continue taskid = res["taskid"] logger.info("New task ID is '%s'" % taskid) @@ -844,7 +850,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non raw = _client("%s/scan/%s/start" % (addr, taskid), cmdLineOptions) res = dejsonize(raw) if not res["success"]: - logger.error("Failed to start scan") + logger.error("Failed to start scan ('%s')" % res.get("message", "")) continue logger.info("Scanning started") From 0b1b64b6827ad7a403b050af02cbcfd47ac58030 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 12 Jan 2021 16:21:43 +0100 Subject: [PATCH 023/243] Thank you Travis CI in discovering of a cluster of f*cks --- lib/core/settings.py | 2 +- lib/core/testing.py | 3 ++- lib/techniques/union/use.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 9dc519460e9..066b285a4ea 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.23" +VERSION = "1.5.1.24" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/core/testing.py b/lib/core/testing.py index 769caa9d42d..164de8abe72 100644 --- a/lib/core/testing.py +++ b/lib/core/testing.py @@ -46,7 +46,8 @@ def vulnTest(): ("--dummy", ("all tested parameters do not appear to be injectable", "does not seem to be injectable", "there is not at least one", "~might be injectable")), ("-u '&id2=1' -p id2 -v 5 --flush-session --level=5 --test-filter='AND boolean-based blind - WHERE or HAVING clause (MySQL comment)'", ("~1AND",)), ("--list-tampers", ("between", "MySQL", "xforwardedfor")), - ("-r --flush-session -v 5 --test-skip='heavy' --save=", ("CloudFlare", "possible DBMS: 'SQLite'", "User-agent: foobar", "~Type: time-based blind")), + ("-r --flush-session -v 5 --test-skip='heavy' --save=", ("CloudFlare", "possible DBMS: 'SQLite'", "User-agent: foobar", "~Type: time-based blind", "saved command line options to the configuration file")), + ("-c ", ("CloudFlare", "possible DBMS: 'SQLite'", "User-agent: foobar", "~Type: time-based blind")), (" -r -l --flush-session --banner --technique=B", ("banner: '3.", "STDIN")), ("-l --flush-session --keep-alive --skip-waf -v 5 --technique=U --union-from=users --banner --parse-errors", ("banner: '3.", "ORDER BY term out of range", "~xp_cmdshell", "Connection: keep-alive")), ("-l --offline --banner -v 5", ("banner: '3.", "~[TRAFFIC OUT]")), diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index 2740c969233..eaf4f394f07 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -242,7 +242,7 @@ def unionUse(expression, unpack=True, dump=False): debugMsg += "it does not play well with UNION query SQL injection" singleTimeDebugMessage(debugMsg) - if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL, DBMS.SQLITE) and expressionFields and not conf.binaryFields: + if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL, DBMS.SQLITE) and expressionFields and not any((conf.binaryFields, conf.limitStart, conf.limitStop, conf.forcePartial)): match = re.search(r"SELECT\s*(.+?)\bFROM", expression, re.I) if match and not (Backend.isDbms(DBMS.ORACLE) and FROM_DUMMY_TABLE[DBMS.ORACLE] in expression) and not re.search(r"\b(MIN|MAX|COUNT)\(", expression): kb.jsonAggMode = True From 12238019ddf320a69f3a87b1a24cd8b00911d27c Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 12 Jan 2021 16:28:50 +0100 Subject: [PATCH 024/243] Patch for Python2.6 --- lib/core/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 066b285a4ea..00a8789c2f2 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.24" +VERSION = "1.5.1.25" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -814,7 +814,7 @@ RESTAPI_DEFAULT_PORT = 8775 # Unsupported options by REST-JSON API server -RESTAPI_UNSUPPORTED_OPTIONS = {"sqlShell", "wizard"} +RESTAPI_UNSUPPORTED_OPTIONS = ("sqlShell", "wizard") # Use "Supplementary Private Use Area-A" INVALID_UNICODE_PRIVATE_AREA = False From fa93f5e099718de0e67cd5d861f74cad5f49cd23 Mon Sep 17 00:00:00 2001 From: shelld3v <59408894+shelld3v@users.noreply.github.com> Date: Tue, 12 Jan 2021 22:34:17 +0700 Subject: [PATCH 025/243] New translation: Vietnamese (#4509) * Create README-vi-VN.md * Update README.md * Consistency update (same style as in other translations) Co-authored-by: Miroslav Stampar --- README.md | 3 +- doc/translations/README-vi-VN.md | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 doc/translations/README-vi-VN.md diff --git a/README.md b/README.md index 3bca18a0a31..4b0079b7ed0 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ You can visit the [collection of screenshots](https://github.com/sqlmapproject/s Installation ---- -You can download the latest tarball by clicking [here](https://github.com/sqlmapproject/sqlmap/tarball/master) or latest zipball by clicking [here](https://github.com/sqlmapproject/sqlmap/zipball/master). +You can download the latest tarball by clicking [here](https://github.com/sqlmapproject/sqlmap/tarball/master) or latest zipball by clicking [here](https://github.com/sqlmapproject/sqlmap/zipball/master). Preferably, you can download sqlmap by cloning the [Git](https://github.com/sqlmapproject/sqlmap) repository: @@ -71,3 +71,4 @@ Translations * [Spanish](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-es-MX.md) * [Turkish](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-tr-TR.md) * [Ukrainian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-uk-UA.md) +* [Vietnamese](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-vi-VN.md) diff --git a/doc/translations/README-vi-VN.md b/doc/translations/README-vi-VN.md new file mode 100644 index 00000000000..71216af6135 --- /dev/null +++ b/doc/translations/README-vi-VN.md @@ -0,0 +1,53 @@ +# sqlmap ![](https://i.imgur.com/fe85aVR.png) + +[![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/twitter-@sqlmap-blue.svg)](https://twitter.com/sqlmap) + +sqlmap là một công cụ kiểm tra thâm nhập mã nguồn mở, nhằm tự động hóa quá trình phát hiện, khai thác lỗ hổng tiêm SQL và tiếp quản các máy chủ cơ sở dữ liệu. Nó đi kèm với +một hệ thống phát hiện mạnh mẽ, nhiều tính năng thích hợp cho người kiểm tra thâm nhập và một loạt các tùy chọn bao gồm lấy dấu cơ sở dữ liệu, truy xuất dữ liệu từ cơ sở dữ +liệu, truy cập tệp của hệ thống và thực hiện các lệnh trên hệ điều hành thông qua kết nối ngoài. + +Ảnh chụp màn hình +---- + +![Screenshot](https://raw.github.com/wiki/sqlmapproject/sqlmap/images/sqlmap_screenshot.png) + +Bạn có thể truy cập vào [bộ sưu tập ảnh chụp màn hình](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots), chúng trình bày một số tính năng trên wiki. + +Cài đặt +---- + + +Bạn có thể tải xuống tập tin nén tar mới nhất bằng cách nhấp vào [đây](https://github.com/sqlmapproject/sqlmap/tarball/master) hoặc tập tin nén zip mới nhất bằng cách nhấp vào [đây](https://github.com/sqlmapproject/sqlmap/zipball/master). + +Tốt hơn là bạn có thể tải xuống sqlmap bằng cách clone với [Git](https://github.com/sqlmapproject/sqlmap): + + git clone --depth 1 https://github.com/sqlmapproject/sqlmap.git sqlmap-dev + +sqlmap hoạt động hiệu quả với [Python](http://www.python.org/download/) phiên bản **2.6**, **2.7** và **3.x** trên bất kì nền tảng nào. + +Sử dụng +---- + +Để có được danh sách các tùy chọn cơ bản, hãy sử dụng: + + python sqlmap.py -h + +Để có được danh sách tất cả các tùy chọn, hãy sử dụng: + + python sqlmap.py -hh + +Bạn có thể tìm thấy video chạy mẫu [tại đây](https://asciinema.org/a/46601). +Để có cái nhìn tổng quan về các khả năng của sqlmap, danh sách các tính năng được hỗ trợ và mô tả về tất cả các tùy chọn, cùng với các ví dụ, bạn nên tham khảo [hướng dẫn sử dụng](https://github.com/sqlmapproject/sqlmap/wiki/Usage) (Tiếng Anh). + +Liên kết +---- + +* Trang chủ: http://sqlmap.org +* Tải xuống: [.tar.gz](https://github.com/sqlmapproject/sqlmap/tarball/master) hoặc [.zip](https://github.com/sqlmapproject/sqlmap/zipball/master) +* Lịch sử thay nguồn đổi cấp dữ liệu RSS: https://github.com/sqlmapproject/sqlmap/commits/master.atom +* Theo dõi vấn đề: https://github.com/sqlmapproject/sqlmap/issues +* Hướng dẫn sử dụng: https://github.com/sqlmapproject/sqlmap/wiki +* Các câu hỏi thường gặp (FAQ): https://github.com/sqlmapproject/sqlmap/wiki/FAQ +* Twitter: [@sqlmap](https://twitter.com/sqlmap) +* Demo: [http://www.youtube.com/user/inquisb/videos](http://www.youtube.com/user/inquisb/videos) +* Ảnh chụp màn hình: https://github.com/sqlmapproject/sqlmap/wiki/Screenshots From 29a7646334084e2aab056ff68a2b5eee4eca7252 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 12 Jan 2021 16:55:15 +0100 Subject: [PATCH 026/243] Trivial update --- lib/core/common.py | 6 ++++++ lib/core/settings.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/core/common.py b/lib/core/common.py index 2b85b4cff7b..47647b83d2b 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -1831,6 +1831,9 @@ def getLimitRange(count, plusOne=False): def parseUnionPage(page): """ Returns resulting items from UNION query inside provided page content + + >>> parseUnionPage("%sfoo%s%sbar%s" % (kb.chars.start, kb.chars.stop, kb.chars.start, kb.chars.stop)) + ['foo', 'bar'] """ if page is None: @@ -2064,6 +2067,9 @@ def safeFilepathEncode(filepath): def safeExpandUser(filepath): """ Patch for a Python Issue18171 (http://bugs.python.org/issue18171) + + >>> os.path.basename(__file__) in safeExpandUser(__file__) + True """ retVal = filepath diff --git a/lib/core/settings.py b/lib/core/settings.py index 00a8789c2f2..cb6081dfe15 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.25" +VERSION = "1.5.1.26" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) From 5d3bd6ea7a703d7949d47ad63d70334c17f2d19f Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 13 Jan 2021 11:47:10 +0100 Subject: [PATCH 027/243] Adding support for YugabyteDB --- lib/core/enums.py | 1 + lib/core/settings.py | 2 +- plugins/dbms/postgresql/fingerprint.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/core/enums.py b/lib/core/enums.py index deb2beacf49..cf4eabb0afb 100644 --- a/lib/core/enums.py +++ b/lib/core/enums.py @@ -101,6 +101,7 @@ class FORK(object): ENTERPRISEDB = "EnterpriseDB" YELLOWBRICK = "Yellowbrick" IRIS = "Iris" + YUGABYTEDB = "YugabyteDB" class CUSTOM_LOGGING(object): PAYLOAD = 9 diff --git a/lib/core/settings.py b/lib/core/settings.py index cb6081dfe15..14dbd831868 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.26" +VERSION = "1.5.1.27" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/plugins/dbms/postgresql/fingerprint.py b/plugins/dbms/postgresql/fingerprint.py index 2d487cf9ab4..9cf301db1c7 100644 --- a/plugins/dbms/postgresql/fingerprint.py +++ b/plugins/dbms/postgresql/fingerprint.py @@ -39,6 +39,8 @@ def getFingerprint(self): fork = FORK.YELLOWBRICK elif inject.checkBooleanExpression("VERSION() LIKE '%EnterpriseDB%'"): # Reference: https://www.enterprisedb.com/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/11/EDB_Postgres_Advanced_Server_Guide.1.087.html fork = FORK.ENTERPRISEDB + elif inject.checkBooleanExpression("VERSION() LIKE '%YB-%'"): # Reference: https://github.com/yugabyte/yugabyte-db/issues/2447#issue-499562926 + fork = FORK.YUGABYTEDB elif inject.checkBooleanExpression("AURORA_VERSION() LIKE '%'"): # Reference: https://aws.amazon.com/premiumsupport/knowledge-center/aurora-version-number/ fork = FORK.AURORA else: From 1a0c53362644b9df4b7bb7ffa1813cda78bf1000 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 13 Jan 2021 13:17:46 +0100 Subject: [PATCH 028/243] Fixes #4541 --- lib/core/settings.py | 2 +- lib/takeover/xp_cmdshell.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 14dbd831868..6883d3c0353 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.27" +VERSION = "1.5.1.28" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/takeover/xp_cmdshell.py b/lib/takeover/xp_cmdshell.py index b143794bd14..c0bc064fd5e 100644 --- a/lib/takeover/xp_cmdshell.py +++ b/lib/takeover/xp_cmdshell.py @@ -166,9 +166,12 @@ def xpCmdshellForgeCmd(self, cmd, insertIntoTable=None): # Obfuscate the command to execute, also useful to bypass filters # on single-quotes self._randStr = randomStr(lowercase=True) - self._cmd = "0x%s" % encodeHex(cmd, binary=False) self._forgedCmd = "DECLARE @%s VARCHAR(8000);" % self._randStr - self._forgedCmd += "SET @%s=%s;" % (self._randStr, self._cmd) + + try: + self._forgedCmd += "SET @%s=%s;" % (self._randStr, "0x%s" % encodeHex(cmd, binary=False)) + except UnicodeError: + self._forgedCmd += "SET @%s='%s';" % (self._randStr, cmd) # Insert the command standard output into a support table, # 'sqlmapoutput', except when DBMS credentials are provided because From 42884d3f094a09a710b2dc7a8b028d07554d8f36 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 13 Jan 2021 17:32:57 +0100 Subject: [PATCH 029/243] Problems with badge.fury.io --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b0079b7ed0..5a5d1f69ff5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # sqlmap ![](https://i.imgur.com/fe85aVR.png) -[![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/twitter-@sqlmap-blue.svg)](https://twitter.com/sqlmap) +[![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/twitter-@sqlmap-blue.svg)](https://twitter.com/sqlmap) sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester, and a broad range of switches including database fingerprinting, over data fetching from the database, accessing the underlying file system, and executing commands on the operating system via out-of-band connections. From 6fa5922fbdfc8b4cd8af49ef0eb5cfb329926053 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 14 Jan 2021 22:48:57 +0100 Subject: [PATCH 030/243] Bug fix (error heuristics not working in heavy dynamicity cases) --- data/xml/errors.xml | 3 ++- lib/controller/checks.py | 13 +++++++------ lib/core/settings.py | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/data/xml/errors.xml b/data/xml/errors.xml index 99af382037f..29bb39a2db6 100644 --- a/data/xml/errors.xml +++ b/data/xml/errors.xml @@ -42,7 +42,7 @@ - + @@ -55,6 +55,7 @@ + diff --git a/lib/controller/checks.py b/lib/controller/checks.py index a435917c0ab..2bc73f90cb1 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -157,6 +157,7 @@ def checkSqlInjection(place, parameter, value): # error message, simple heuristic check or via DBMS-specific # payload), ask the user to limit the tests to the fingerprinted # DBMS + if kb.reduceTests is None and not conf.testFilter and (intersect(Backend.getErrorParsedDBMSes(), SUPPORTED_DBMS, True) or kb.heuristicDbms or injection.dbms): msg = "it looks like the back-end DBMS is '%s'. " % (Format.getErrorParsedDBMSes() or kb.heuristicDbms or joinValue(injection.dbms, '/')) msg += "Do you want to skip test payloads specific for other DBMSes? [Y/n]" @@ -1041,11 +1042,6 @@ def heuristicCheckSqlInjection(place, parameter): if conf.skipHeuristics: return None - if kb.heavilyDynamic: - debugMsg = "heuristic check skipped because of heavy dynamicity" - logger.debug(debugMsg) - return None - origValue = conf.paramDict[place][parameter] paramType = conf.method if conf.method not in (None, HTTPMETHOD.GET, HTTPMETHOD.POST) else place @@ -1082,7 +1078,7 @@ def _(page): casting = _(page) and not _(kb.originalPage) - if not casting and not result and kb.dynamicParameter and origValue.isdigit(): + if not casting and not result and kb.dynamicParameter and origValue.isdigit() and not kb.heavilyDynamic: randInt = int(randomInt()) payload = "%s%s%s" % (prefix, "%d-%d" % (int(origValue) + randInt, randInt), suffix) payload = agent.payload(place, parameter, newValue=payload, where=PAYLOAD.WHERE.REPLACE) @@ -1096,6 +1092,11 @@ def _(page): kb.heuristicTest = HEURISTIC_TEST.CASTED if casting else HEURISTIC_TEST.NEGATIVE if not result else HEURISTIC_TEST.POSITIVE + if kb.heavilyDynamic: + debugMsg = "heuristic check stopped because of heavy dynamicity" + logger.debug(debugMsg) + return kb.heuristicTest + if casting: errMsg = "possible %s casting detected (e.g. '" % ("integer" if origValue.isdigit() else "type") diff --git a/lib/core/settings.py b/lib/core/settings.py index 6883d3c0353..54f820865c6 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.28" +VERSION = "1.5.1.29" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) From fd137b49ef1cd51f17ce85c75d7c85b72018bf72 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Fri, 15 Jan 2021 09:17:33 +0100 Subject: [PATCH 031/243] Fixing Python3.10 incompatibility --- lib/core/datatype.py | 3 ++- lib/core/settings.py | 2 +- thirdparty/six/__init__.py | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/core/datatype.py b/lib/core/datatype.py index ab3032dee3f..12838b977fe 100644 --- a/lib/core/datatype.py +++ b/lib/core/datatype.py @@ -10,6 +10,7 @@ import types from thirdparty.odict import OrderedDict +from thirdparty.six.moves import collections_abc class AttribDict(dict): """ @@ -159,7 +160,7 @@ def keys(self): return self.cache.keys() # Reference: https://code.activestate.com/recipes/576694/ -class OrderedSet(collections.MutableSet): +class OrderedSet(collections_abc.MutableSet): """ This class defines the set with ordered (as added) items diff --git a/lib/core/settings.py b/lib/core/settings.py index 54f820865c6..c6ae1e02e78 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.29" +VERSION = "1.5.1.30" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/thirdparty/six/__init__.py b/thirdparty/six/__init__.py index 83f69783d1a..d162d09cdb2 100644 --- a/thirdparty/six/__init__.py +++ b/thirdparty/six/__init__.py @@ -223,6 +223,12 @@ def get_code(self, fullname): return None get_source = get_code # same as get_code + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + _importer = _SixMetaPathImporter(__name__) From 4490d5548257f3d9727215e87a2f2d3f9ba07ce2 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Fri, 15 Jan 2021 09:18:21 +0100 Subject: [PATCH 032/243] Removing leftover --- lib/core/datatype.py | 1 - lib/core/settings.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/core/datatype.py b/lib/core/datatype.py index 12838b977fe..739780d5b04 100644 --- a/lib/core/datatype.py +++ b/lib/core/datatype.py @@ -5,7 +5,6 @@ See the file 'LICENSE' for copying permission """ -import collections import copy import types diff --git a/lib/core/settings.py b/lib/core/settings.py index c6ae1e02e78..5f3c766cd24 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.30" +VERSION = "1.5.1.31" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) From d78590213bc1af0a39ada2033da2179f2481afa0 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Fri, 15 Jan 2021 09:31:57 +0100 Subject: [PATCH 033/243] Brrrrr incompatibilities with every new Python minor revision --- lib/core/common.py | 6 +++--- lib/core/convert.py | 4 ++-- lib/core/datatype.py | 4 ++-- lib/core/option.py | 4 ++-- lib/core/settings.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index 47647b83d2b..0478c2c1e6f 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -9,7 +9,6 @@ import binascii import codecs -import collections import contextlib import copy import distutils.version @@ -194,6 +193,7 @@ from thirdparty.magic import magic from thirdparty.odict import OrderedDict from thirdparty.six import unichr as _unichr +from thirdparty.six.moves import collections_abc as _collections from thirdparty.six.moves import configparser as _configparser from thirdparty.six.moves import http_client as _http_client from thirdparty.six.moves import input as _input @@ -3267,7 +3267,7 @@ def filterNone(values): retVal = values - if isinstance(values, collections.Iterable): + if isinstance(values, _collections.Iterable): retVal = [_ for _ in values if _] return retVal @@ -3558,7 +3558,7 @@ def arrayizeValue(value): ['1'] """ - if isinstance(value, collections.KeysView): + if isinstance(value, _collections.KeysView): value = [_ for _ in value] elif not isListLike(value): value = [value] diff --git a/lib/core/convert.py b/lib/core/convert.py index d48bdfee8db..520c306a4c8 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -13,7 +13,6 @@ import base64 import binascii import codecs -import collections import json import re import sys @@ -31,6 +30,7 @@ from lib.core.settings import UNICODE_ENCODING from thirdparty import six from thirdparty.six import unichr as _unichr +from thirdparty.six.moves import collections_abc as _collections try: from html import escape as htmlEscape @@ -106,7 +106,7 @@ def singleTimeWarnMessage(message): # Cross-referenced function sys.stdout.flush() def filterNone(values): # Cross-referenced function - return [_ for _ in values if _] if isinstance(values, collections.Iterable) else values + return [_ for _ in values if _] if isinstance(values, _collections.Iterable) else values def isListLike(value): # Cross-referenced function return isinstance(value, (list, tuple, set, BigArray)) diff --git a/lib/core/datatype.py b/lib/core/datatype.py index 739780d5b04..412d545439b 100644 --- a/lib/core/datatype.py +++ b/lib/core/datatype.py @@ -9,7 +9,7 @@ import types from thirdparty.odict import OrderedDict -from thirdparty.six.moves import collections_abc +from thirdparty.six.moves import collections_abc as _collections class AttribDict(dict): """ @@ -159,7 +159,7 @@ def keys(self): return self.cache.keys() # Reference: https://code.activestate.com/recipes/576694/ -class OrderedSet(collections_abc.MutableSet): +class OrderedSet(_collections.MutableSet): """ This class defines the set with ordered (as added) items diff --git a/lib/core/option.py b/lib/core/option.py index e4cba0cc975..d307799638d 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -8,7 +8,6 @@ from __future__ import division import codecs -import collections import functools import glob import inspect @@ -154,6 +153,7 @@ from thirdparty import six from thirdparty.keepalive import keepalive from thirdparty.multipart import multipartpost +from thirdparty.six.moves import collections_abc as _collections from thirdparty.six.moves import http_client as _http_client from thirdparty.six.moves import http_cookiejar as _http_cookiejar from thirdparty.six.moves import urllib as _urllib @@ -415,7 +415,7 @@ def retrieve(): conf.googlePage += 1 def _setStdinPipeTargets(): - if isinstance(conf.stdinPipe, collections.Iterable): + if isinstance(conf.stdinPipe, _collections.Iterable): infoMsg = "using 'STDIN' for parsing targets list" logger.info(infoMsg) diff --git a/lib/core/settings.py b/lib/core/settings.py index 5f3c766cd24..0f89cf9fc03 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.31" +VERSION = "1.5.1.32" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) From 6244850749edd307e5a9834ef3f8e2e4de684495 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Fri, 15 Jan 2021 10:51:49 +0100 Subject: [PATCH 034/243] More DREI stuff --- lib/core/settings.py | 2 +- thirdparty/bottle/bottle.py | 6 +++++- thirdparty/odict/ordereddict.py | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 0f89cf9fc03..a48b21c41e8 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.32" +VERSION = "1.5.1.33" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/thirdparty/bottle/bottle.py b/thirdparty/bottle/bottle.py index aeb6b92b455..de959cd370c 100644 --- a/thirdparty/bottle/bottle.py +++ b/thirdparty/bottle/bottle.py @@ -134,13 +134,17 @@ def _e(): # Lots of stdlib and builtin differences. if py3k: + try: + from collections.abc import MutableMapping as DictMixin + except ImportError: + from collections import MutableMapping as DictMixin + import http.client as httplib import _thread as thread from urllib.parse import urljoin, SplitResult as UrlSplitResult from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote urlunquote = functools.partial(urlunquote, encoding='latin1') from http.cookies import SimpleCookie - from collections import MutableMapping as DictMixin import pickle from io import BytesIO from configparser import ConfigParser, Error as ConfigParserError diff --git a/thirdparty/odict/ordereddict.py b/thirdparty/odict/ordereddict.py index 4f87050c643..1cdd6f46edc 100644 --- a/thirdparty/odict/ordereddict.py +++ b/thirdparty/odict/ordereddict.py @@ -23,7 +23,10 @@ try: from UserDict import DictMixin except ImportError: - from collections import MutableMapping as DictMixin + try: + from collections.abc import MutableMapping as DictMixin + except ImportError: + from collections import MutableMapping as DictMixin class OrderedDict(dict, DictMixin): From ed0909fe579935db7f953d1bd16810c047eea73e Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 18 Jan 2021 12:04:47 +0100 Subject: [PATCH 035/243] Fixes #4547 --- lib/core/common.py | 2 +- lib/core/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index 0478c2c1e6f..7b52e85c780 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -4603,7 +4603,7 @@ def geturl(self): if filtered and filtered != content: try: forms = ParseResponse(filtered, backwards_compat=False) - except ParseError: + except: errMsg = "no success" if raise_: raise SqlmapGenericException(errMsg) diff --git a/lib/core/settings.py b/lib/core/settings.py index a48b21c41e8..1066c8026bc 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.33" +VERSION = "1.5.1.34" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) From e5086ef61a9761c91125a011036748409c01edd3 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 18 Jan 2021 12:29:35 +0100 Subject: [PATCH 036/243] Fixes #4545 --- lib/core/common.py | 3 ++- lib/core/settings.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index 7b52e85c780..f604d3a0796 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -1059,7 +1059,8 @@ def dataToDumpFile(dumpFile, data): errMsg = "permission denied when flushing dump data" logger.error(errMsg) else: - raise + errMsg = "error occurred when writing dump data to file ('%s')" % getUnicode(ex) + logger.error(errMsg) def dataToOutFile(filename, data): """ diff --git a/lib/core/settings.py b/lib/core/settings.py index 1066c8026bc..c0e2f3d3534 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.34" +VERSION = "1.5.1.35" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) From f3e8039145e80381f39f06b4585ea6a383d24179 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 19 Jan 2021 12:25:03 +0100 Subject: [PATCH 037/243] Fixes #4549 --- lib/core/settings.py | 2 +- plugins/generic/misc.py | 22 ++++++---------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index c0e2f3d3534..674e5688b4d 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.35" +VERSION = "1.5.1.36" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/plugins/generic/misc.py b/plugins/generic/misc.py index 0c46dd99085..3ca193bd504 100644 --- a/plugins/generic/misc.py +++ b/plugins/generic/misc.py @@ -25,7 +25,6 @@ from lib.core.enums import HASHDB_KEYS from lib.core.enums import OS from lib.core.exception import SqlmapNoneDataException -from lib.core.exception import SqlmapUnsupportedFeatureException from lib.request import inject class Miscellaneous(object): @@ -83,25 +82,16 @@ def getVersionFromBanner(self): infoMsg = "detecting back-end DBMS version from its banner" logger.info(infoMsg) - if Backend.isDbms(DBMS.MYSQL): - first, last = 1, 6 - - elif Backend.isDbms(DBMS.PGSQL): - first, last = 12, 6 - - elif Backend.isDbms(DBMS.MSSQL): - first, last = 29, 9 - - else: - raise SqlmapUnsupportedFeatureException("unsupported DBMS") - - query = queries[Backend.getIdentifiedDbms()].substring.query % (queries[Backend.getIdentifiedDbms()].banner.query, first, last) + query = queries[Backend.getIdentifiedDbms()].banner.query if conf.direct: query = "SELECT %s" % query - kb.bannerFp["dbmsVersion"] = unArrayizeValue(inject.getValue(query)) - kb.bannerFp["dbmsVersion"] = (kb.bannerFp["dbmsVersion"] or "").replace(',', "").replace('-', "").replace(' ', "") + kb.bannerFp["dbmsVersion"] = unArrayizeValue(inject.getValue(query)) or "" + + match = re.search(r"\d[\d.-]*", kb.bannerFp["dbmsVersion"]) + if match: + kb.bannerFp["dbmsVersion"] = match.group(0) def delRemoteFile(self, filename): if not filename: From 9568ee3c4db4011fb1984fcb223d443f32b394cb Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 19 Jan 2021 12:32:32 +0100 Subject: [PATCH 038/243] Fixes #4548 --- lib/core/settings.py | 2 +- plugins/dbms/mssqlserver/filesystem.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 674e5688b4d..b69e4b9b38d 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.36" +VERSION = "1.5.1.37" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/plugins/dbms/mssqlserver/filesystem.py b/plugins/dbms/mssqlserver/filesystem.py index 2a4fd28ecbd..e3e466b0e0b 100644 --- a/plugins/dbms/mssqlserver/filesystem.py +++ b/plugins/dbms/mssqlserver/filesystem.py @@ -8,6 +8,7 @@ import ntpath import os +from lib.core.common import checkFile from lib.core.common import getLimitRange from lib.core.common import isNumPosStrValue from lib.core.common import isTechniqueAvailable @@ -384,13 +385,13 @@ def stackedWriteFile(self, localFile, remoteFile, fileType, forceCheck=False): # procedure to write a file on the back-end Microsoft SQL Server # file system self.initEnv() - self.getRemoteTempPath() tmpPath = posixToNtSlashes(conf.tmpPath) remoteFile = posixToNtSlashes(remoteFile) - with open(localFile, "rb") as f: - localFileContent = f.read() + + checkFile(localFile) + localFileContent = open(localFile, "rb").read() self._stackedWriteFilePS(tmpPath, localFileContent, remoteFile, fileType) written = self.askCheckWrittenFile(localFile, remoteFile, forceCheck) From b2c8ba10cd39120c13845089cdbd0dc5ff62f189 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 20 Jan 2021 16:06:19 +0100 Subject: [PATCH 039/243] Minor just in case update --- lib/core/settings.py | 2 +- lib/parse/handler.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index b69e4b9b38d..124b0bfd0dd 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.5.1.37" +VERSION = "1.5.1.38" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/parse/handler.py b/lib/parse/handler.py index 8976e31c4e6..82fbbbcdb8a 100644 --- a/lib/parse/handler.py +++ b/lib/parse/handler.py @@ -30,7 +30,7 @@ def __init__(self, banner, info): def _feedInfo(self, key, value): value = sanitizeStr(value) - if value in (None, "None"): + if value in (None, "None", ""): return if key == "dbmsVersion": @@ -62,10 +62,10 @@ def startElement(self, name, attrs): self._techVersion = sanitizeStr(attrs.get("tech_version")) self._sp = sanitizeStr(attrs.get("sp")) - if self._dbmsVersion.isdigit(): + if self._dbmsVersion and self._dbmsVersion.isdigit(): self._feedInfo("dbmsVersion", self._match.group(int(self._dbmsVersion))) - if self._techVersion.isdigit(): + if self._techVersion and self._techVersion.isdigit(): self._feedInfo("technology", "%s %s" % (attrs.get("technology"), self._match.group(int(self._techVersion)))) else: self._feedInfo("technology", attrs.get("technology")) From 4a12493d85339a0afb9f6aadb30462d901eff153 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 20 Jan 2021 16:29:52 +0100 Subject: [PATCH 040/243] Python3 fix (parsing of headers) --- extra/vulnserver/vulnserver.py | 1 + lib/core/settings.py | 2 +- lib/core/testing.py | 2 +- lib/parse/headers.py | 3 +-- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extra/vulnserver/vulnserver.py b/extra/vulnserver/vulnserver.py index f38728838b1..458e90bc3ba 100644 --- a/extra/vulnserver/vulnserver.py +++ b/extra/vulnserver/vulnserver.py @@ -98,6 +98,7 @@ def do_REQUEST(self): if "