From 829351421f98406901bfc93a5ba36296c56ae447 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 25 Nov 2015 10:12:07 +0100 Subject: [PATCH 01/23] Minor cosmetics --- lib/techniques/union/use.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index cc4a93aaa..ce34dcc28 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -56,7 +56,7 @@ from lib.utils.progress import ProgressBar from thirdparty.odict.odict import OrderedDict def _oneShotUnionUse(expression, unpack=True, limited=False): - retVal = hashDBRetrieve("%s%s" % (conf.hexConvert, expression), checkConf=True) # as union data is stored raw unconverted + retVal = hashDBRetrieve("%s%s" % (conf.hexConvert or False, expression), checkConf=True) # as union data is stored raw unconverted threadData = getCurrentThreadData() threadData.resumed = retVal is not None @@ -102,7 +102,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False): if Backend.isDbms(DBMS.MSSQL) and wasLastResponseDBMSError(): retVal = htmlunescape(retVal).replace("
", "\n") - hashDBWrite("%s%s" % (conf.hexConvert, expression), retVal) + hashDBWrite("%s%s" % (conf.hexConvert or False, expression), retVal) else: trimmed = _("%s(?P.*?)<" % (kb.chars.start)) From a18c69d78bdc4b65cacc4bc7d7f670e6f2f00673 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 25 Nov 2015 10:21:32 +0100 Subject: [PATCH 02/23] Fixes #1564 --- lib/techniques/union/use.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index ce34dcc28..a85e01d12 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -111,6 +111,9 @@ def _oneShotUnionUse(expression, unpack=True, limited=False): warnMsg += "(probably due to its length and/or content): " warnMsg += safecharencode(trimmed) logger.warn(warnMsg) + else: + vector = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector + kb.unionDuplicates = vector[7] return retVal From c7ec1534a61a7a855c6faf7b68a4225e4bc7df55 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 25 Nov 2015 13:04:34 +0100 Subject: [PATCH 03/23] Patch related to #1256 --- lib/request/httpshandler.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/request/httpshandler.py b/lib/request/httpshandler.py index 0ade4aca4..432c190f5 100644 --- a/lib/request/httpshandler.py +++ b/lib/request/httpshandler.py @@ -43,11 +43,14 @@ class HTTPSConnection(httplib.HTTPSConnection): success = False - if not kb.tlsSNI: - for protocol in _protocols: + # Reference(s): https://docs.python.org/2/library/ssl.html#ssl.SSLContext + # https://www.mnot.net/blog/2014/12/27/python_2_and_tls_sni + if kb.tlsSNI != False and hasattr(ssl, "SSLContext"): + for protocol in filter(lambda _: _ >= ssl.PROTOCOL_TLSv1, _protocols): try: sock = create_sock() - _ = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=protocol) + context = ssl.SSLContext(protocol) + _ = context.wrap_socket(sock, do_handshake_on_connect=False, server_hostname=self.host) if _: success = True self.sock = _ @@ -60,16 +63,16 @@ class HTTPSConnection(httplib.HTTPSConnection): self._tunnel_host = None logger.debug("SSL connection error occurred ('%s')" % getSafeExString(ex)) - # Reference(s): https://docs.python.org/2/library/ssl.html#ssl.SSLContext - # https://www.mnot.net/blog/2014/12/27/python_2_and_tls_sni - if not success and hasattr(ssl, "SSLContext"): - for protocol in filter(lambda _: _ >= ssl.PROTOCOL_TLSv1, _protocols): + if kb.tlsSNI is None: + kb.tlsSNI = success + + if not success: + for protocol in _protocols: try: sock = create_sock() - context = ssl.SSLContext(protocol) - _ = context.wrap_socket(sock, do_handshake_on_connect=False, server_hostname=self.host) + _ = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=protocol) if _: - kb.tlsSNI = success = True + success = True self.sock = _ _protocols.remove(protocol) _protocols.insert(0, protocol) From f9da29284c674c7344e7bec6f35ff1925702ecde Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Fri, 27 Nov 2015 18:35:58 +0100 Subject: [PATCH 04/23] Minor bug fix (reported via email) --- lib/core/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/common.py b/lib/core/common.py index 1f28b5f25..9558b69f9 100755 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -2700,7 +2700,7 @@ def parseSqliteTableSchema(value): table = {} columns = {} - for match in re.finditer(r"(\w+)[\"'`]?\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|TEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b", value, re.I): + for match in re.finditer(r"(\w+)[\"'`]?\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|LONGTEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b", value, re.I): columns[match.group(1)] = match.group(2) table[conf.tbl] = columns From 5f2c31f8ec66c5be32b23054dbe28fdf07eb878f Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sat, 28 Nov 2015 22:42:25 +0100 Subject: [PATCH 05/23] Minor consistency patch --- lib/takeover/web.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/takeover/web.py b/lib/takeover/web.py index 504fb4a2c..2aefba410 100644 --- a/lib/takeover/web.py +++ b/lib/takeover/web.py @@ -9,10 +9,9 @@ import os import posixpath import re import StringIO +import tempfile import urlparse -from tempfile import mkstemp - from extra.cloak.cloak import decloak from lib.core.agent import agent from lib.core.common import arrayizeValue @@ -257,7 +256,7 @@ class Web: stagerName = "tmpu%s.%s" % (randomStr(lowercase=True), self.webApi) self.webStagerFilePath = posixpath.join(ntToPosixSlashes(directory), stagerName) - handle, filename = mkstemp() + handle, filename = tempfile.mkstemp() os.fdopen(handle).close() # close low level handle (causing problems later) with open(filename, "w+") as f: From 795777b7c5ba8ec7bd1abdba334b41b9d00e536b Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sat, 28 Nov 2015 22:44:42 +0100 Subject: [PATCH 06/23] Minor patch --- lib/takeover/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/takeover/web.py b/lib/takeover/web.py index 2aefba410..9da5bcbcb 100644 --- a/lib/takeover/web.py +++ b/lib/takeover/web.py @@ -257,9 +257,9 @@ class Web: self.webStagerFilePath = posixpath.join(ntToPosixSlashes(directory), stagerName) handle, filename = tempfile.mkstemp() - os.fdopen(handle).close() # close low level handle (causing problems later) + os.close(handle) - with open(filename, "w+") as f: + with open(filename, "w+b") as f: _ = decloak(os.path.join(paths.SQLMAP_SHELL_PATH, "stager.%s_" % self.webApi)) _ = _.replace("WRITABLE_DIR", utf8encode(directory.replace('/', '\\\\') if Backend.isOs(OS.WINDOWS) else directory)) f.write(_) From a219ff9a927d1fe5b10f3921d60f5a258f0976d4 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sun, 29 Nov 2015 19:40:14 +0100 Subject: [PATCH 07/23] Fixes #1572 --- lib/core/wordlist.py | 4 ---- lib/utils/hash.py | 6 +++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/core/wordlist.py b/lib/core/wordlist.py index e7c902beb..30f2c277d 100644 --- a/lib/core/wordlist.py +++ b/lib/core/wordlist.py @@ -77,10 +77,6 @@ class Wordlist(object): except StopIteration: self.adjust() retVal = self.iter.next().rstrip() - try: - retVal = retVal.decode(UNICODE_ENCODING) - except UnicodeDecodeError: - continue if not self.proc_count or self.counter % self.proc_count == self.proc_id: break return retVal diff --git a/lib/utils/hash.py b/lib/utils/hash.py index 7ab48c26f..271485550 100644 --- a/lib/utils/hash.py +++ b/lib/utils/hash.py @@ -209,6 +209,9 @@ def oracle_old_passwd(password, username, uppercase=True): # prior to version ' if isinstance(username, unicode): username = unicode.encode(username, UNICODE_ENCODING) # pyDes has issues with unicode strings + if isinstance(password, unicode): + password = unicode.encode(password, UNICODE_ENCODING) + unistr = "".join("\0%s" % c for c in (username + password).upper()) cipher = des(hexdecode("0123456789ABCDEF"), CBC, IV, pad) @@ -327,7 +330,8 @@ def wordpress_passwd(password, salt, count, prefix, uppercase=False): return output - password = password.encode(UNICODE_ENCODING) + if isinstance(password, unicode): + password = password.encode(UNICODE_ENCODING) cipher = md5(salt) cipher.update(password) From 80d3ff67064f612527c33c0a4dbb84e5ad483a67 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 2 Dec 2015 12:05:40 +0100 Subject: [PATCH 08/23] Adding hidden switch for disabling socket preconnect (debugging purposes) --- lib/core/option.py | 3 +++ lib/parse/cmdline.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/core/option.py b/lib/core/option.py index 7fe25005d..d3a7e64dd 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1024,6 +1024,9 @@ def _setSocketPreConnect(): Makes a pre-connect version of socket.connect """ + if conf.disablePrecon: + return + def _(): while kb.threadContinue: try: diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index 68bad2f3f..131421903 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -754,6 +754,9 @@ def cmdLineParser(argv=None): parser.add_option("--pickled-options", dest="pickledOptions", help=SUPPRESS_HELP) + parser.add_option("--disable-precon", dest="disablePrecon", action="store_true", + help=SUPPRESS_HELP) + parser.add_option("--profile", dest="profile", action="store_true", help=SUPPRESS_HELP) @@ -780,9 +783,6 @@ def cmdLineParser(argv=None): parser.add_option("--run-case", dest="runCase", help=SUPPRESS_HELP) - parser.add_option("--nnc5ed", dest="nnc5ed", action="store_true", - help=SUPPRESS_HELP) # temporary hidden switch :) - parser.add_option_group(target) parser.add_option_group(request) parser.add_option_group(optimization) From 6397704456ab64f40b105491d58fd0b592241920 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 3 Dec 2015 01:43:37 +0100 Subject: [PATCH 09/23] Patch for an Issue #1578 --- lib/core/option.py | 2 +- lib/request/connect.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/core/option.py b/lib/core/option.py index d3a7e64dd..795924866 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1376,7 +1376,7 @@ def _setHTTPExtraHeaders(): errMsg = "invalid header value: %s. Valid header format is 'name:value'" % repr(headerValue).lstrip('u') raise SqlmapSyntaxException(errMsg) - elif not conf.httpHeaders or len(conf.httpHeaders) == 1: + elif not conf.requestFile and len(conf.httpHeaders or []) < 2: conf.httpHeaders.append((HTTP_HEADER.ACCEPT_LANGUAGE, "en-us,en;q=0.5")) if not conf.charset: conf.httpHeaders.append((HTTP_HEADER.ACCEPT_CHARSET, "ISO-8859-15,utf-8;q=0.7,*;q=0.7")) diff --git a/lib/request/connect.py b/lib/request/connect.py index 40e873fe3..93414961e 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -343,6 +343,9 @@ class Connect(object): # Prepare HTTP headers headers = forgeHeaders({HTTP_HEADER.COOKIE: cookie, HTTP_HEADER.USER_AGENT: ua, HTTP_HEADER.REFERER: referer, HTTP_HEADER.HOST: host}) + if HTTP_HEADER.COOKIE in headers: + cookie = headers[HTTP_HEADER.COOKIE] + if kb.authHeader: headers[HTTP_HEADER.AUTHORIZATION] = kb.authHeader @@ -370,6 +373,12 @@ class Connect(object): if boundary: headers[HTTP_HEADER.CONTENT_TYPE] = "%s; boundary=%s" % (headers[HTTP_HEADER.CONTENT_TYPE], boundary) + # Reset header values to original in case of provided request file + if target and conf.requestFile: + headers = OrderedDict(conf.httpHeaders) + if cookie: + headers[HTTP_HEADER.COOKIE] = cookie + if auxHeaders: for key, value in auxHeaders.items(): for _ in headers.keys(): From a7c135174cbc2c3113f8b1265bef32411dcbfcc9 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 3 Dec 2015 02:00:16 +0100 Subject: [PATCH 10/23] Fixes #1579 --- lib/core/common.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index 9558b69f9..77fc7bfa4 100755 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -3770,8 +3770,12 @@ def decodeHexValue(value, raw=False): def _(value): retVal = value - if value and isinstance(value, basestring) and len(value) % 2 == 0: - retVal = hexdecode(retVal) + if value and isinstance(value, basestring): + if len(value) % 2 != 0: + retVal = "%s?" % hexdecode(value[:-1]) + singleTimeWarnMessage("there was a problem decoding value '%s' from expected hexadecimal form" % value) + else: + retVal = hexdecode(value) if not kb.binaryField and not raw: if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"): From c6d42174955b04f77e02dc5eee8ff6ef18f99acb Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 3 Dec 2015 02:08:59 +0100 Subject: [PATCH 11/23] Minor update (just in case) --- lib/controller/checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/controller/checks.py b/lib/controller/checks.py index e03b8909d..276a3379e 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -1034,7 +1034,7 @@ def checkStability(): delay = max(0, min(1, delay)) time.sleep(delay) - secondPage, _ = Request.queryPage(content=True, raise404=False) + secondPage, _ = Request.queryPage(content=True, noteResponseTime=False, raise404=False) if kb.redirectChoice: return None From d50c0b7103da9786b08c73f784f35e2fd60eef6a Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 3 Dec 2015 12:16:00 +0100 Subject: [PATCH 12/23] Fixes #1581 --- lib/core/optiondict.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/core/optiondict.py b/lib/core/optiondict.py index 9eb0d121a..5f1bd1fea 100644 --- a/lib/core/optiondict.py +++ b/lib/core/optiondict.py @@ -227,6 +227,7 @@ optDict = { }, "Hidden": { "dummy": "boolean", + "disablePrecon": "boolean", "binaryFields": "string", "profile": "boolean", "cpuThrottle": "integer", From 7517e64417e46a9223218555ecb4464d8e74dbb2 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sat, 5 Dec 2015 00:52:58 +0100 Subject: [PATCH 13/23] Minor bug fix (reported via email) --- lib/core/option.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/option.py b/lib/core/option.py index 795924866..55f56a1b0 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1576,7 +1576,7 @@ def _cleanupOptions(): conf.progressWidth = width - 46 for key, value in conf.items(): - if value and any(key.endswith(_) for _ in ("Path", "File")): + if value and any(key.endswith(_) for _ in ("Path", "File", "Dir")): conf[key] = safeExpandUser(value) if conf.testParameter: From d5e6be41db069df9bc5315e2d5a5e537d857855f Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sun, 6 Dec 2015 23:24:09 +0100 Subject: [PATCH 14/23] Fixes #1582 --- lib/utils/pivotdumptable.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/utils/pivotdumptable.py b/lib/utils/pivotdumptable.py index 392c3aaf9..6703e5dc7 100644 --- a/lib/utils/pivotdumptable.py +++ b/lib/utils/pivotdumptable.py @@ -64,15 +64,19 @@ def pivotDumpTable(table, colList, count=None, blind=True): colList = filter(None, sorted(colList, key=lambda x: len(x) if x else MAX_INT)) if conf.pivotColumn: - if any(re.search(r"(.+\.)?%s" % re.escape(conf.pivotColumn), _, re.I) for _ in colList): - infoMsg = "using column '%s' as a pivot " % conf.pivotColumn - infoMsg += "for retrieving row data" - logger.info(infoMsg) + for _ in colList: + if re.search(r"(.+\.)?%s" % re.escape(conf.pivotColumn), _, re.I): + infoMsg = "using column '%s' as a pivot " % conf.pivotColumn + infoMsg += "for retrieving row data" + logger.info(infoMsg) - validPivotValue = True - colList.remove(conf.pivotColumn) - colList.insert(0, conf.pivotColumn) - else: + colList.remove(_) + colList.insert(0, _) + + validPivotValue = True + break + + if not validPivotValue: warnMsg = "column '%s' not " % conf.pivotColumn warnMsg += "found in table '%s'" % table logger.warn(warnMsg) From b5b3411f16adf19b5169d35c3465ae1baa68e26b Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sun, 6 Dec 2015 23:49:22 +0100 Subject: [PATCH 15/23] Fixes #1574 --- lib/core/option.py | 2 +- lib/request/httpshandler.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/core/option.py b/lib/core/option.py index 55f56a1b0..a2804d1f5 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1897,7 +1897,7 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.safeReq = AttribDict() kb.singleLogFlags = set() kb.reduceTests = None - kb.tlsSNI = None + kb.tlsSNI = {} kb.stickyDBMS = False kb.stickyLevel = None kb.storeCrawlingChoice = None diff --git a/lib/request/httpshandler.py b/lib/request/httpshandler.py index 432c190f5..5cf0613ca 100644 --- a/lib/request/httpshandler.py +++ b/lib/request/httpshandler.py @@ -45,12 +45,12 @@ class HTTPSConnection(httplib.HTTPSConnection): # Reference(s): https://docs.python.org/2/library/ssl.html#ssl.SSLContext # https://www.mnot.net/blog/2014/12/27/python_2_and_tls_sni - if kb.tlsSNI != False and hasattr(ssl, "SSLContext"): + if kb.tlsSNI.get(self.host) != False and hasattr(ssl, "SSLContext"): for protocol in filter(lambda _: _ >= ssl.PROTOCOL_TLSv1, _protocols): try: sock = create_sock() context = ssl.SSLContext(protocol) - _ = context.wrap_socket(sock, do_handshake_on_connect=False, server_hostname=self.host) + _ = context.wrap_socket(sock, do_handshake_on_connect=True, server_hostname=self.host) if _: success = True self.sock = _ @@ -63,8 +63,8 @@ class HTTPSConnection(httplib.HTTPSConnection): self._tunnel_host = None logger.debug("SSL connection error occurred ('%s')" % getSafeExString(ex)) - if kb.tlsSNI is None: - kb.tlsSNI = success + if kb.tlsSNI.get(self.host) is None: + kb.tlsSNI[self.host] = success if not success: for protocol in _protocols: From af60f11319b9684f2671085c0119c88819cfc24b Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 7 Dec 2015 16:17:28 +0100 Subject: [PATCH 16/23] Fixes #1584 (hello @w3af looking for the patch of this one ;) --- lib/core/profiling.py | 1 - thirdparty/xdot/xdot.py | 478 +++++++++++++++++++++++++++++++--------- 2 files changed, 374 insertions(+), 105 deletions(-) diff --git a/lib/core/profiling.py b/lib/core/profiling.py index c212a0bb5..e93f6b80b 100644 --- a/lib/core/profiling.py +++ b/lib/core/profiling.py @@ -87,5 +87,4 @@ def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None): win.connect('destroy', gtk.main_quit) win.set_filter("dot") win.open_file(dotOutputFile) - gobject.timeout_add(1000, win.update, dotOutputFile) gtk.main() diff --git a/thirdparty/xdot/xdot.py b/thirdparty/xdot/xdot.py index 4bc94640e..77dd86632 100644 --- a/thirdparty/xdot/xdot.py +++ b/thirdparty/xdot/xdot.py @@ -16,11 +16,9 @@ # along with this program. If not, see . # -'''Visualize dot graphs via the xdot Format.''' +'''Visualize dot graphs via the xdot format.''' -__author__ = "Jose Fonseca" - -__version__ = "0.4" +__author__ = "Jose Fonseca et al" import os @@ -30,6 +28,7 @@ import math import colorsys import time import re +import optparse import gobject import gtk @@ -90,13 +89,12 @@ class Shape: else: return self.pen + def search_text(self, regexp): + return False + class TextShape(Shape): - #fontmap = pangocairo.CairoFontMap() - #fontmap.set_resolution(72) - #context = fontmap.create_context() - LEFT, CENTER, RIGHT = -1, 0, 1 def __init__(self, pen, x, y, j, w, t): @@ -191,6 +189,33 @@ class TextShape(Shape): cr.line_to(x+self.w, self.y) cr.stroke() + def search_text(self, regexp): + return regexp.search(self.t) is not None + + +class ImageShape(Shape): + + def __init__(self, pen, x0, y0, w, h, path): + Shape.__init__(self) + self.pen = pen.copy() + self.x0 = x0 + self.y0 = y0 + self.w = w + self.h = h + self.path = path + + def draw(self, cr, highlight=False): + cr2 = gtk.gdk.CairoContext(cr) + pixbuf = gtk.gdk.pixbuf_new_from_file(self.path) + sx = float(self.w)/float(pixbuf.get_width()) + sy = float(self.h)/float(pixbuf.get_height()) + cr.save() + cr.translate(self.x0, self.y0 - self.h) + cr.scale(sx, sy) + cr2.set_source_pixbuf(pixbuf, 0, 0) + cr2.paint() + cr.restore() + class EllipseShape(Shape): @@ -304,6 +329,12 @@ class CompoundShape(Shape): for shape in self.shapes: shape.draw(cr, highlight=highlight) + def search_text(self, regexp): + for shape in self.shapes: + if shape.search_text(regexp): + return True + return False + class Url(object): @@ -332,6 +363,9 @@ class Element(CompoundShape): def __init__(self, shapes): CompoundShape.__init__(self, shapes) + def is_inside(self, x, y): + return False + def get_url(self, x, y): return None @@ -341,9 +375,10 @@ class Element(CompoundShape): class Node(Element): - def __init__(self, x, y, w, h, shapes, url): + def __init__(self, id, x, y, w, h, shapes, url): Element.__init__(self, shapes) + self.id = id self.x = x self.y = y @@ -360,7 +395,6 @@ class Node(Element): def get_url(self, x, y): if self.url is None: return None - #print (x, y), (self.x1, self.y1), "-", (self.x2, self.y2) if self.is_inside(x, y): return Url(self, self.url) return None @@ -370,6 +404,9 @@ class Node(Element): return Jump(self, self.x, self.y) return None + def __repr__(self): + return "" % self.id + def square_distance(x1, y1, x2, y2): deltax = x2 - x1 @@ -387,13 +424,29 @@ class Edge(Element): RADIUS = 10 + def is_inside_begin(self, x, y): + return square_distance(x, y, *self.points[0]) <= self.RADIUS*self.RADIUS + + def is_inside_end(self, x, y): + return square_distance(x, y, *self.points[-1]) <= self.RADIUS*self.RADIUS + + def is_inside(self, x, y): + if self.is_inside_begin(x, y): + return True + if self.is_inside_end(x, y): + return True + return False + def get_jump(self, x, y): - if square_distance(x, y, *self.points[0]) <= self.RADIUS*self.RADIUS: + if self.is_inside_begin(x, y): return Jump(self, self.dst.x, self.dst.y, highlight=set([self, self.dst])) - if square_distance(x, y, *self.points[-1]) <= self.RADIUS*self.RADIUS: + if self.is_inside_end(x, y): return Jump(self, self.src.x, self.src.y, highlight=set([self, self.src])) return None + def __repr__(self): + return " %s>" % (self.src, self.dst) + class Graph(Shape): @@ -424,6 +477,14 @@ class Graph(Shape): for node in self.nodes: node.draw(cr, highlight=(node in highlight_items)) + def get_element(self, x, y): + for node in self.nodes: + if node.is_inside(x, y): + return node + for edge in self.edges: + if edge.is_inside(x, y): + return edge + def get_url(self, x, y): for node in self.nodes: url = node.get_url(x, y) @@ -443,6 +504,14 @@ class Graph(Shape): return None +BOLD = 1 +ITALIC = 2 +UNDERLINE = 4 +SUPERSCRIPT = 8 +SUBSCRIPT = 16 +STRIKE_THROUGH = 32 + + class XDotAttrParser: """Parser for xdot drawing attributes. See also: @@ -451,20 +520,15 @@ class XDotAttrParser: def __init__(self, parser, buf): self.parser = parser - self.buf = self.unescape(buf) + self.buf = buf self.pos = 0 - + self.pen = Pen() self.shapes = [] def __nonzero__(self): return self.pos < len(self.buf) - def unescape(self, buf): - buf = buf.replace('\\"', '"') - buf = buf.replace('\\n', '\n') - return buf - def read_code(self): pos = self.buf.find(" ", self.pos) res = self.buf[self.pos:pos] @@ -473,19 +537,19 @@ class XDotAttrParser: self.pos += 1 return res - def read_number(self): + def read_int(self): return int(self.read_code()) def read_float(self): return float(self.read_code()) def read_point(self): - x = self.read_number() - y = self.read_number() + x = self.read_float() + y = self.read_float() return self.transform(x, y) def read_text(self): - num = self.read_number() + num = self.read_int() pos = self.buf.find("-", self.pos) + 1 self.pos = pos + num res = self.buf[pos:self.pos] @@ -494,7 +558,7 @@ class XDotAttrParser: return res def read_polygon(self): - n = self.read_number() + n = self.read_int() p = [] for i in range(n): x, y = self.read_point() @@ -521,6 +585,9 @@ class XDotAttrParser: r, g, b = colorsys.hsv_to_rgb(h, s, v) a = 1.0 return r, g, b, a + elif c1 == "[": + sys.stderr.write('warning: color gradients not supported yet\n') + return None else: return self.lookup_color(c) @@ -549,8 +616,8 @@ class XDotAttrParser: b = b*s a = 1.0 return r, g, b, a - - sys.stderr.write("unknown color '%s'\n" % c) + + sys.stderr.write("warning: unknown color '%s'\n" % c) return None def parse(self): @@ -573,7 +640,7 @@ class XDotAttrParser: lw = style.split("(")[1].split(")")[0] lw = float(lw) self.handle_linewidth(lw) - elif style in ("solid", "dashed"): + elif style in ("solid", "dashed", "dotted"): self.handle_linestyle(style) elif op == "F": size = s.read_float() @@ -581,19 +648,22 @@ class XDotAttrParser: self.handle_font(size, name) elif op == "T": x, y = s.read_point() - j = s.read_number() - w = s.read_number() + j = s.read_int() + w = s.read_float() t = s.read_text() self.handle_text(x, y, j, w, t) + elif op == "t": + f = s.read_int() + self.handle_font_characteristics(f) elif op == "E": x0, y0 = s.read_point() - w = s.read_number() - h = s.read_number() + w = s.read_float() + h = s.read_float() self.handle_ellipse(x0, y0, w, h, filled=True) elif op == "e": x0, y0 = s.read_point() - w = s.read_number() - h = s.read_number() + w = s.read_float() + h = s.read_float() self.handle_ellipse(x0, y0, w, h, filled=False) elif op == "L": points = self.read_polygon() @@ -610,12 +680,18 @@ class XDotAttrParser: elif op == "p": points = self.read_polygon() self.handle_polygon(points, filled=False) + elif op == "I": + x0, y0 = s.read_point() + w = s.read_float() + h = s.read_float() + path = s.read_text() + self.handle_image(x0, y0, w, h, path) else: - sys.stderr.write("unknown xdot opcode '%s'\n" % op) - break + sys.stderr.write("error: unknown xdot opcode '%s'\n" % op) + sys.exit(1) return self.shapes - + def transform(self, x, y): return self.parser.transform(x, y) @@ -633,11 +709,18 @@ class XDotAttrParser: self.pen.dash = () elif style == "dashed": self.pen.dash = (6, ) # 6pt on, 6pt off + elif style == "dotted": + self.pen.dash = (2, 4) # 2pt on, 4pt off def handle_font(self, size, name): self.pen.fontsize = size self.pen.fontname = name + def handle_font_characteristics(self, flags): + # TODO + if flags != 0: + sys.stderr.write("warning: font characteristics not supported yet\n" % op) + def handle_text(self, x, y, j, w, t): self.shapes.append(TextShape(self.pen, x, y, j, w, t)) @@ -647,6 +730,9 @@ class XDotAttrParser: self.shapes.append(EllipseShape(self.pen, x0, y0, w, h, filled=True)) self.shapes.append(EllipseShape(self.pen, x0, y0, w, h)) + def handle_image(self, x0, y0, w, h, path): + self.shapes.append(ImageShape(self.pen, x0, y0, w, h, path)) + def handle_line(self, points): self.shapes.append(LineShape(self.pen, points)) @@ -677,7 +763,7 @@ class ParseError(Exception): def __str__(self): return ':'.join([str(part) for part in (self.filename, self.line, self.col, self.msg) if part != None]) - + class Scanner: """Stateless scanner.""" @@ -921,11 +1007,12 @@ class DotLexer(Lexer): text = text.replace('\\\r\n', '') text = text.replace('\\\r', '') text = text.replace('\\\n', '') + + # quotes + text = text.replace('\\"', '"') - text = text.replace('\\r', '\r') - text = text.replace('\\n', '\n') - text = text.replace('\\t', '\t') - text = text.replace('\\', '') + # layout engines recognize other escape codes (many non-standard) + # but we don't translate them here type = ID @@ -1059,10 +1146,12 @@ class DotParser(Parser): class XDotParser(DotParser): + XDOTVERSION = '1.6' + def __init__(self, xdotcode): lexer = DotLexer(buf = xdotcode) DotParser.__init__(self, lexer) - + self.nodes = [] self.edges = [] self.shapes = [] @@ -1071,27 +1160,35 @@ class XDotParser(DotParser): def handle_graph(self, attrs): if self.top_graph: + # Check xdot version + try: + xdotversion = attrs['xdotversion'] + except KeyError: + pass + else: + if float(xdotversion) > float(self.XDOTVERSION): + sys.stderr.write('warning: xdot version %s, but supported is %s\n' % (xdotversion, self.XDOTVERSION)) + + # Parse bounding box try: bb = attrs['bb'] except KeyError: return - if not bb: - return + if bb: + xmin, ymin, xmax, ymax = map(float, bb.split(",")) - xmin, ymin, xmax, ymax = map(float, bb.split(",")) + self.xoffset = -xmin + self.yoffset = -ymax + self.xscale = 1.0 + self.yscale = -1.0 + # FIXME: scale from points to pixels - self.xoffset = -xmin - self.yoffset = -ymax - self.xscale = 1.0 - self.yscale = -1.0 - # FIXME: scale from points to pixels - - self.width = xmax - xmin - self.height = ymax - ymin - - self.top_graph = False + self.width = max(xmax - xmin, 1) + self.height = max(ymax - ymin, 1) + self.top_graph = False + for attr in ("_draw_", "_ldraw_", "_hdraw_", "_tdraw_", "_hldraw_", "_tldraw_"): if attr in attrs: parser = XDotAttrParser(self, attrs[attr]) @@ -1104,15 +1201,15 @@ class XDotParser(DotParser): return x, y = self.parse_node_pos(pos) - w = float(attrs['width'])*72 - h = float(attrs['height'])*72 + w = float(attrs.get('width', 0))*72 + h = float(attrs.get('height', 0))*72 shapes = [] for attr in ("_draw_", "_ldraw_"): if attr in attrs: parser = XDotAttrParser(self, attrs[attr]) shapes.extend(parser.parse()) url = attrs.get('URL', None) - node = Node(x, y, w, h, shapes, url) + node = Node(id, x, y, w, h, shapes, url) self.node_by_name[id] = node if shapes: self.nodes.append(node) @@ -1122,7 +1219,7 @@ class XDotParser(DotParser): pos = attrs['pos'] except KeyError: return - + points = self.parse_edge_pos(pos) shapes = [] for attr in ("_draw_", "_ldraw_", "_hdraw_", "_tdraw_", "_hldraw_", "_tldraw_"): @@ -1399,6 +1496,9 @@ class DotWidget(gtk.DrawingArea): self.connect("size-allocate", self.on_area_size_allocate) self.connect('key-press-event', self.on_key_press_event) + self.last_mtime = None + + gobject.timeout_add(1000, self.update) self.x, self.y = 0.0, 0.0 self.zoom_ratio = 1.0 @@ -1411,9 +1511,9 @@ class DotWidget(gtk.DrawingArea): def set_filter(self, filter): self.filter = filter - def set_dotcode(self, dotcode, filename=''): - if isinstance(dotcode, unicode): - dotcode = dotcode.encode('utf8') + def run_filter(self, dotcode): + if not self.filter: + return dotcode p = subprocess.Popen( [self.filter, '-Txdot'], stdin=subprocess.PIPE, @@ -1423,6 +1523,7 @@ class DotWidget(gtk.DrawingArea): universal_newlines=True ) xdotcode, error = p.communicate(dotcode) + sys.stderr.write(error) if p.returncode != 0: dialog = gtk.MessageDialog(type=gtk.MESSAGE_ERROR, message_format=error, @@ -1430,10 +1531,19 @@ class DotWidget(gtk.DrawingArea): dialog.set_title('Dot Viewer') dialog.run() dialog.destroy() + return None + return xdotcode + + def set_dotcode(self, dotcode, filename=None): + self.openfilename = None + if isinstance(dotcode, unicode): + dotcode = dotcode.encode('utf8') + xdotcode = self.run_filter(dotcode) + if xdotcode is None: return False try: self.set_xdotcode(xdotcode) - except ParseError, ex: + except ParseError as ex: dialog = gtk.MessageDialog(type=gtk.MESSAGE_ERROR, message_format=str(ex), buttons=gtk.BUTTONS_OK) @@ -1442,11 +1552,14 @@ class DotWidget(gtk.DrawingArea): dialog.destroy() return False else: + if filename is None: + self.last_mtime = None + else: + self.last_mtime = os.stat(filename).st_mtime self.openfilename = filename return True def set_xdotcode(self, xdotcode): - #print xdotcode parser = XDotParser(xdotcode) self.graph = parser.parse() self.zoom_image(self.zoom_ratio, center=True) @@ -1460,6 +1573,14 @@ class DotWidget(gtk.DrawingArea): except IOError: pass + def update(self): + if self.openfilename is not None: + current_mtime = os.stat(self.openfilename).st_mtime + if current_mtime != self.last_mtime: + self.last_mtime = current_mtime + self.reload() + return True + def do_expose_event(self, event): cr = self.window.cairo_create() @@ -1500,6 +1621,10 @@ class DotWidget(gtk.DrawingArea): self.queue_draw() def zoom_image(self, zoom_ratio, center=False, pos=None): + # Constrain zoom ratio to a sane range to prevent numeric instability. + zoom_ratio = min(zoom_ratio, 1E4) + zoom_ratio = max(zoom_ratio, 1E-6) + if center: self.x = self.graph.width/2 self.y = self.graph.height/2 @@ -1518,10 +1643,13 @@ class DotWidget(gtk.DrawingArea): rect = self.get_allocation() width = abs(x1 - x2) height = abs(y1 - y2) - self.zoom_ratio = min( - float(rect.width)/float(width), - float(rect.height)/float(height) - ) + if width == 0 and height == 0: + self.zoom_ratio *= self.ZOOM_INCREMENT + else: + self.zoom_ratio = min( + float(rect.width)/float(width), + float(rect.height)/float(height) + ) self.zoom_to_fit_on_resize = False self.x = (x1 + x2) / 2 self.y = (y1 + y2) / 2 @@ -1574,11 +1702,16 @@ class DotWidget(gtk.DrawingArea): self.y += self.POS_INCREMENT/self.zoom_ratio self.queue_draw() return True - if event.keyval == gtk.keysyms.Page_Up: + if event.keyval in (gtk.keysyms.Page_Up, + gtk.keysyms.plus, + gtk.keysyms.equal, + gtk.keysyms.KP_Add): self.zoom_image(self.zoom_ratio * self.ZOOM_INCREMENT) self.queue_draw() return True - if event.keyval == gtk.keysyms.Page_Down: + if event.keyval in (gtk.keysyms.Page_Down, + gtk.keysyms.minus, + gtk.keysyms.KP_Subtract): self.zoom_image(self.zoom_ratio / self.ZOOM_INCREMENT) self.queue_draw() return True @@ -1589,11 +1722,49 @@ class DotWidget(gtk.DrawingArea): if event.keyval == gtk.keysyms.r: self.reload() return True + if event.keyval == gtk.keysyms.f: + win = widget.get_toplevel() + find_toolitem = win.uimanager.get_widget('/ToolBar/Find') + textentry = find_toolitem.get_children() + win.set_focus(textentry[0]) + return True if event.keyval == gtk.keysyms.q: gtk.main_quit() return True + if event.keyval == gtk.keysyms.p: + self.on_print() + return True return False + print_settings = None + def on_print(self, action=None): + print_op = gtk.PrintOperation() + + if self.print_settings != None: + print_op.set_print_settings(self.print_settings) + + print_op.connect("begin_print", self.begin_print) + print_op.connect("draw_page", self.draw_page) + + res = print_op.run(gtk.PRINT_OPERATION_ACTION_PRINT_DIALOG, self.parent.parent) + + if res == gtk.PRINT_OPERATION_RESULT_APPLY: + print_settings = print_op.get_print_settings() + + def begin_print(self, operation, context): + operation.set_n_pages(1) + return True + + def draw_page(self, operation, context, page_nr): + cr = context.get_cairo_context() + + rect = self.get_allocation() + cr.translate(0.5*rect.width, 0.5*rect.height) + cr.scale(self.zoom_ratio, self.zoom_ratio) + cr.translate(-self.x, -self.y) + + self.graph.draw(cr, highlight_items=self.highlight) + def get_drag_action(self, event): state = event.state if event.button in (1, 2): # left or middle button @@ -1628,20 +1799,32 @@ class DotWidget(gtk.DrawingArea): return (time.time() < self.presstime + click_timeout and math.hypot(deltax, deltay) < click_fuzz) + def on_click(self, element, event): + """Override this method in subclass to process + click events. Note that element can be None + (click on empty space).""" + return False + def on_area_button_release(self, area, event): self.drag_action.on_button_release(event) self.drag_action = NullAction(self) - if event.button == 1 and self.is_click(event): - x, y = int(event.x), int(event.y) - url = self.get_url(x, y) - if url is not None: - self.emit('clicked', unicode(url.url), event) - else: - jump = self.get_jump(x, y) - if jump is not None: - self.animate_to(jump.x, jump.y) + x, y = int(event.x), int(event.y) + if self.is_click(event): + el = self.get_element(x, y) + if self.on_click(el, event): + return True + + if event.button == 1: + url = self.get_url(x, y) + if url is not None: + self.emit('clicked', unicode(url.url), event) + else: + jump = self.get_jump(x, y) + if jump is not None: + self.animate_to(jump.x, jump.y) + + return True - return True if event.button == 1 or event.button == 2: return True return False @@ -1679,6 +1862,10 @@ class DotWidget(gtk.DrawingArea): y += self.y return x, y + def get_element(self, x, y): + x, y = self.window2graph(x, y) + return self.graph.get_element(x, y) + def get_url(self, x, y): x, y = self.window2graph(x, y) return self.graph.get_url(x, y) @@ -1688,6 +1875,14 @@ class DotWidget(gtk.DrawingArea): return self.graph.get_jump(x, y) +class FindMenuToolAction(gtk.Action): + __gtype_name__ = "FindMenuToolAction" + + def __init__(self, *args, **kw): + gtk.Action.__init__(self, *args, **kw) + self.set_tool_item_type(gtk.ToolItem) + + class DotWindow(gtk.Window): ui = ''' @@ -1695,28 +1890,33 @@ class DotWindow(gtk.Window): + + + ''' - def __init__(self): + base_title = 'Dot Viewer' + + def __init__(self, widget=None): gtk.Window.__init__(self) self.graph = Graph() window = self - window.set_title('Dot Viewer') + window.set_title(self.base_title) window.set_default_size(512, 512) vbox = gtk.VBox() window.add(vbox) - self.widget = DotWidget() + self.widget = widget or DotWidget() # Create a UIManager instance uimanager = self.uimanager = gtk.UIManager() @@ -1733,12 +1933,17 @@ class DotWindow(gtk.Window): actiongroup.add_actions(( ('Open', gtk.STOCK_OPEN, None, None, None, self.on_open), ('Reload', gtk.STOCK_REFRESH, None, None, None, self.on_reload), + ('Print', gtk.STOCK_PRINT, None, None, "Prints the currently visible part of the graph", self.widget.on_print), ('ZoomIn', gtk.STOCK_ZOOM_IN, None, None, None, self.widget.on_zoom_in), ('ZoomOut', gtk.STOCK_ZOOM_OUT, None, None, None, self.widget.on_zoom_out), ('ZoomFit', gtk.STOCK_ZOOM_FIT, None, None, None, self.widget.on_zoom_fit), ('Zoom100', gtk.STOCK_ZOOM_100, None, None, None, self.widget.on_zoom_100), )) + find_action = FindMenuToolAction("Find", None, + "Find a node by name", None) + actiongroup.add_action(find_action) + # Add the actiongroup to the uimanager uimanager.insert_action_group(actiongroup, 0) @@ -1751,45 +1956,82 @@ class DotWindow(gtk.Window): vbox.pack_start(self.widget) + self.last_open_dir = "." + self.set_focus(self.widget) + # Add Find text search + find_toolitem = uimanager.get_widget('/ToolBar/Find') + self.textentry = gtk.Entry(max=20) + self.textentry.set_icon_from_stock(0, gtk.STOCK_FIND) + find_toolitem.add(self.textentry) + + self.textentry.set_activates_default(True) + self.textentry.connect ("activate", self.textentry_activate, self.textentry); + self.textentry.connect ("changed", self.textentry_changed, self.textentry); + self.show_all() - def update(self, filename): - import os - if not hasattr(self, "last_mtime"): - self.last_mtime = None + def find_text(self, entry_text): + found_items = [] + dot_widget = self.widget + regexp = re.compile(entry_text) + for node in dot_widget.graph.nodes: + if node.search_text(regexp): + found_items.append(node) + return found_items - current_mtime = os.stat(filename).st_mtime - if current_mtime != self.last_mtime: - self.last_mtime = current_mtime - self.open_file(filename) + def textentry_changed(self, widget, entry): + entry_text = entry.get_text() + dot_widget = self.widget + if not entry_text: + dot_widget.set_highlight(None) + return + + found_items = self.find_text(entry_text) + dot_widget.set_highlight(found_items) - return True + def textentry_activate(self, widget, entry): + entry_text = entry.get_text() + dot_widget = self.widget + if not entry_text: + dot_widget.set_highlight(None) + return; + + found_items = self.find_text(entry_text) + dot_widget.set_highlight(found_items) + if(len(found_items) == 1): + dot_widget.animate_to(found_items[0].x, found_items[0].y) def set_filter(self, filter): self.widget.set_filter(filter) - def set_dotcode(self, dotcode, filename=''): + def set_dotcode(self, dotcode, filename=None): if self.widget.set_dotcode(dotcode, filename): - self.set_title(os.path.basename(filename) + ' - Dot Viewer') + self.update_title(filename) self.widget.zoom_to_fit() - def set_xdotcode(self, xdotcode, filename=''): + def set_xdotcode(self, xdotcode, filename=None): if self.widget.set_xdotcode(xdotcode): - self.set_title(os.path.basename(filename) + ' - Dot Viewer') + self.update_title(filename) self.widget.zoom_to_fit() + + def update_title(self, filename=None): + if filename is None: + self.set_title(self.base_title) + else: + self.set_title(os.path.basename(filename) + ' - ' + self.base_title) def open_file(self, filename): try: fp = file(filename, 'rt') self.set_dotcode(fp.read(), filename) fp.close() - except IOError, ex: + except IOError as ex: dlg = gtk.MessageDialog(type=gtk.MESSAGE_ERROR, message_format=str(ex), buttons=gtk.BUTTONS_OK) - dlg.set_title('Dot Viewer') + dlg.set_title(self.base_title) dlg.run() dlg.destroy() @@ -1801,6 +2043,7 @@ class DotWindow(gtk.Window): gtk.STOCK_OPEN, gtk.RESPONSE_OK)) chooser.set_default_response(gtk.RESPONSE_OK) + chooser.set_current_folder(self.last_open_dir) filter = gtk.FileFilter() filter.set_name("Graphviz dot files") filter.add_pattern("*.dot") @@ -1811,6 +2054,7 @@ class DotWindow(gtk.Window): chooser.add_filter(filter) if chooser.run() == gtk.RESPONSE_OK: filename = chooser.get_filename() + self.last_open_dir = chooser.get_current_folder() chooser.destroy() self.open_file(filename) else: @@ -1820,17 +2064,41 @@ class DotWindow(gtk.Window): self.widget.reload() -def main(): - import optparse +class OptionParser(optparse.OptionParser): - parser = optparse.OptionParser( + def format_epilog(self, formatter): + # Prevent stripping the newlines in epilog message + # http://stackoverflow.com/questions/1857346/python-optparse-how-to-include-additional-info-in-usage-output + return self.epilog + + +def main(): + + parser = OptionParser( usage='\n\t%prog [file]', - version='%%prog %s' % __version__) + epilog=''' +Shortcuts: + Up, Down, Left, Right scroll + PageUp, +, = zoom in + PageDown, - zoom out + R reload dot file + F find + Q quit + P print + Escape halt animation + Ctrl-drag zoom in/out + Shift-drag zooms an area +''' + ) parser.add_option( '-f', '--filter', type='choice', choices=('dot', 'neato', 'twopi', 'circo', 'fdp'), dest='filter', default='dot', help='graphviz filter: dot, neato, twopi, circo, or fdp [default: %default]') + parser.add_option( + '-n', '--no-filter', + action='store_const', const=None, dest='filter', + help='assume input is already filtered into xdot format (use e.g. dot -Txdot)') (options, args) = parser.parse_args(sys.argv[1:]) if len(args) > 1: @@ -1839,12 +2107,14 @@ def main(): win = DotWindow() win.connect('destroy', gtk.main_quit) win.set_filter(options.filter) - if len(args) >= 1: + if len(args) == 0: + if not sys.stdin.isatty(): + win.set_dotcode(sys.stdin.read()) + else: if args[0] == '-': win.set_dotcode(sys.stdin.read()) else: win.open_file(args[0]) - gobject.timeout_add(1000, win.update, args[0]) gtk.main() From efc91b015d8b8eab99c166caddd80d36020b0498 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 9 Dec 2015 10:07:37 +0100 Subject: [PATCH 17/23] Fixes #1589 --- lib/core/dump.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/core/dump.py b/lib/core/dump.py index 058f5d9e9..15c0c14e9 100644 --- a/lib/core/dump.py +++ b/lib/core/dump.py @@ -578,7 +578,8 @@ class Dump(object): if not os.path.isdir(dumpDbPath): os.makedirs(dumpDbPath, 0755) - filepath = os.path.join(dumpDbPath, "%s-%d.bin" % (unsafeSQLIdentificatorNaming(column), randomInt(8))) + _ = re.sub(r"[^\w]", "_", normalizeUnicode(unsafeSQLIdentificatorNaming(column))) + filepath = os.path.join(dumpDbPath, "%s-%d.bin" % (_, randomInt(8))) warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath) logger.warn(warnMsg) From 1c5c937507381d4bed5cdb3acab6294fc52cf66c Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 9 Dec 2015 10:14:13 +0100 Subject: [PATCH 18/23] Minor update --- lib/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index bf5439130..9938d9c0f 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -503,7 +503,7 @@ DEFAULT_COOKIE_DELIMITER = ';' FORCE_COOKIE_EXPIRATION_TIME = "9999999999" # Github OAuth token used for creating an automatic Issue for unhandled exceptions -GITHUB_REPORT_OAUTH_TOKEN = "YzQzM2M2YzgzMDExN2I5ZDMyYjAzNTIzODIwZDA2MDFmMmVjODI1Ng==" +GITHUB_REPORT_OAUTH_TOKEN = "YzNkYTgyMTdjYzdjNjZjMjFjMWE5ODI5OGQyNzk2ODM1M2M0MzUyOA==" # Skip unforced HashDB flush requests below the threshold number of cached items HASHDB_FLUSH_THRESHOLD = 32 From 31d250f98e087585dad3af58ff00ca90d1436760 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 9 Dec 2015 12:00:34 +0100 Subject: [PATCH 19/23] Fixes #1592 --- lib/core/convert.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/core/convert.py b/lib/core/convert.py index 8f7123a00..36de453d2 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -8,6 +8,7 @@ See the file 'doc/COPYING' for copying permission import base64 import json import pickle +import StringIO import sys from lib.core.settings import IS_WIN @@ -67,10 +68,23 @@ def base64unpickle(value): retVal = None + def _(self): + if len(self.stack) > 1: + func = self.stack[-2] + if '.' in repr(func) and " 'lib." not in repr(func): + raise Exception, "abusing reduce() is bad, Mkay!" + self.load_reduce() + + def loads(str): + file = StringIO.StringIO(str) + unpickler = pickle.Unpickler(file) + unpickler.dispatch[pickle.REDUCE] = _ + return unpickler.load() + try: - retVal = pickle.loads(base64decode(value)) + retVal = loads(base64decode(value)) except TypeError: - retVal = pickle.loads(base64decode(bytes(value))) + retVal = loads(base64decode(bytes(value))) return retVal From 2eb5f5e8419b00d6f1c688af7b2e5c61bfdc52f0 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 9 Dec 2015 12:20:09 +0100 Subject: [PATCH 20/23] Handling 'address already in use' for sqlmapapi server instance --- lib/utils/api.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/utils/api.py b/lib/utils/api.py index 07367b15c..8bcdd937e 100644 --- a/lib/utils/api.py +++ b/lib/utils/api.py @@ -10,6 +10,7 @@ import logging import os import re import shlex +import socket import sqlite3 import sys import tempfile @@ -654,7 +655,13 @@ def server(host="0.0.0.0", port=RESTAPI_SERVER_PORT): DataStore.current_db.init() # Run RESTful API - run(host=host, port=port, quiet=True, debug=False) + try: + run(host=host, port=port, quiet=True, debug=False) + except socket.error, ex: + if "already in use" in getSafeExString(ex): + logger.error("Address already in use ('%s:%s')" % (host, port)) + else: + raise def _client(url, options=None): From 663c976a3b58da206c20baf11edef3e80a42de92 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 9 Dec 2015 19:53:48 +0100 Subject: [PATCH 21/23] Fixes #1600 --- lib/core/convert.py | 4 +++- lib/core/settings.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/core/convert.py b/lib/core/convert.py index 36de453d2..26b3a49af 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -10,9 +10,11 @@ import json import pickle import StringIO import sys +import types from lib.core.settings import IS_WIN from lib.core.settings import UNICODE_ENCODING +from lib.core.settings import PICKLE_REDUCE_WHITELIST def base64decode(value): """ @@ -71,7 +73,7 @@ def base64unpickle(value): def _(self): if len(self.stack) > 1: func = self.stack[-2] - if '.' in repr(func) and " 'lib." not in repr(func): + if func not in PICKLE_REDUCE_WHITELIST: raise Exception, "abusing reduce() is bad, Mkay!" self.load_reduce() diff --git a/lib/core/settings.py b/lib/core/settings.py index 9938d9c0f..974cbe82b 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -11,7 +11,9 @@ import subprocess import string import sys import time +import types +from lib.core.datatype import AttribDict from lib.core.enums import DBMS from lib.core.enums import DBMS_DIRECTORY_NAME from lib.core.enums import OS @@ -427,6 +429,8 @@ HTML_TITLE_REGEX = "(?P<result>[^<]+)" # Table used for Base64 conversion in WordPress hash cracking routine ITOA64 = "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +PICKLE_REDUCE_WHITELIST = (types.BooleanType, types.DictType, types.FloatType, types.IntType, types.ListType, types.LongType, types.NoneType, types.StringType, types.TupleType, types.UnicodeType, types.XRangeType, type(AttribDict()), type(set())) + # Chars used to quickly distinguish if the user provided tainted parameter values DUMMY_SQL_INJECTION_CHARS = ";()'" From 273679f542a5705c60d1e22efbd9e88b2e953a44 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 10 Dec 2015 13:23:50 +0100 Subject: [PATCH 22/23] Adding new charset replacement (reported via email) --- lib/request/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/request/basic.py b/lib/request/basic.py index 196035148..21ecd2a34 100755 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -150,7 +150,7 @@ def checkCharEncoding(encoding, warn=True): return encoding # Reference: http://www.destructor.de/charsets/index.htm - translate = {"windows-874": "iso-8859-11", "en_us": "utf8", "macintosh": "iso-8859-1", "euc_tw": "big5_tw", "th": "tis-620", "unicode": "utf8", "utc8": "utf8", "ebcdic": "ebcdic-cp-be", "iso-8859": "iso8859-1", "ansi": "ascii", "gbk2312": "gbk", "windows-31j": "cp932"} + translate = {"windows-874": "iso-8859-11", "utf-8859-1": "utf8", "en_us": "utf8", "macintosh": "iso-8859-1", "euc_tw": "big5_tw", "th": "tis-620", "unicode": "utf8", "utc8": "utf8", "ebcdic": "ebcdic-cp-be", "iso-8859": "iso8859-1", "ansi": "ascii", "gbk2312": "gbk", "windows-31j": "cp932"} for delimiter in (';', ',', '('): if delimiter in encoding: From dc7f2a71d2b38e113287c543eb2ef57795b62006 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sat, 12 Dec 2015 23:48:30 +0100 Subject: [PATCH 23/23] Minor refactoring --- lib/core/settings.py | 6 ++++++ lib/utils/api.py | 9 ++++----- sqlmapapi.py | 13 ++++++------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 974cbe82b..654049134 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -593,6 +593,12 @@ EVENTVALIDATION_REGEX = r'(?i)(?P__EVENTVALIDATION[^"]*)[^>]+value="(?P