From 510ceb6e1960bb9bbb325cb4a3d167ef1d631467 Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Wed, 9 Jan 2013 16:04:23 +0000 Subject: [PATCH 01/11] first attempt to have --os-pwn and other takeover switches work across Windows and Linux - issue #28 --- lib/core/common.py | 18 +++++ lib/core/subprocessng.py | 151 +++++++++++++++++++++++++++++++------ lib/core/update.py | 2 +- lib/takeover/metasploit.py | 108 +++++++++++++++----------- 4 files changed, 214 insertions(+), 65 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index 62292b370..7607b645b 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -3309,3 +3309,21 @@ def isNumber(value): return False else: return True + +def pollProcess(process, suppress_errors=False): + while True: + dataToStdout(".") + time.sleep(1) + + returncode = process.poll() + + if returncode is not None: + if not suppress_errors: + if returncode == 0: + dataToStdout(" done\n") + elif returncode < 0: + dataToStdout(" process terminated by signal %d\n" % returncode) + elif returncode > 0: + dataToStdout(" quit unexpectedly with return code %d\n" % returncode) + + break diff --git a/lib/core/subprocessng.py b/lib/core/subprocessng.py index efd3cf34b..9db2af0bd 100644 --- a/lib/core/subprocessng.py +++ b/lib/core/subprocessng.py @@ -7,13 +7,19 @@ See the file 'doc/COPYING' for copying permission import errno import os +import subprocess import sys import time from lib.core.common import dataToStdout from lib.core.settings import IS_WIN -if not IS_WIN: +if IS_WIN: + from win32file import ReadFile, WriteFile + from win32pipe import PeekNamedPipe + import msvcrt +else: + import select import fcntl if (sys.hexversion >> 16) >= 0x202: @@ -61,30 +67,131 @@ def blockingWriteToFD(fd, data): break -def setNonBlocking(fd): - """ - Make a file descriptor non-blocking - """ +# the following code is taken from http://code.activestate.com/recipes/440554-module-to-allow-asynchronous-subprocess-use-on-win/ +class Popen(subprocess.Popen): + def recv(self, maxsize=None): + return self._recv('stdout', maxsize) - if IS_WIN is not True: - flags = fcntl.fcntl(fd, FCNTL.F_GETFL) - flags = flags | os.O_NONBLOCK - fcntl.fcntl(fd, FCNTL.F_SETFL, flags) + def recv_err(self, maxsize=None): + return self._recv('stderr', maxsize) -def pollProcess(process, suppress_errors=False): - while True: - dataToStdout(".") - time.sleep(1) + def send_recv(self, input='', maxsize=None): + return self.send(input), self.recv(maxsize), self.recv_err(maxsize) - returncode = process.poll() + def get_conn_maxsize(self, which, maxsize): + if maxsize is None: + maxsize = 1024 + elif maxsize < 1: + maxsize = 1 + return getattr(self, which), maxsize - if returncode is not None: - if not suppress_errors: - if returncode == 0: - dataToStdout(" done\n") - elif returncode < 0: - dataToStdout(" process terminated by signal %d\n" % returncode) - elif returncode > 0: - dataToStdout(" quit unexpectedly with return code %d\n" % returncode) + def _close(self, which): + getattr(self, which).close() + setattr(self, which, None) + if subprocess.mswindows: + def send(self, input): + if not self.stdin: + return None + + try: + x = msvcrt.get_osfhandle(self.stdin.fileno()) + (errCode, written) = WriteFile(x, input) + except ValueError: + return self._close('stdin') + except (subprocess.pywintypes.error, Exception), why: + if why[0] in (109, errno.ESHUTDOWN): + return self._close('stdin') + raise + + return written + + def _recv(self, which, maxsize): + conn, maxsize = self.get_conn_maxsize(which, maxsize) + if conn is None: + return None + + try: + x = msvcrt.get_osfhandle(conn.fileno()) + (read, nAvail, nMessage) = PeekNamedPipe(x, 0) + if maxsize < nAvail: + nAvail = maxsize + if nAvail > 0: + (errCode, read) = ReadFile(x, nAvail, None) + except ValueError: + return self._close(which) + except (subprocess.pywintypes.error, Exception), why: + if why[0] in (109, errno.ESHUTDOWN): + return self._close(which) + raise + + if self.universal_newlines: + read = self._translate_newlines(read) + return read + else: + def send(self, input): + if not self.stdin: + return None + + if not select.select([], [self.stdin], [], 0)[1]: + return 0 + + try: + written = os.write(self.stdin.fileno(), input) + except OSError, why: + if why[0] == errno.EPIPE: #broken pipe + return self._close('stdin') + raise + + return written + + def _recv(self, which, maxsize): + conn, maxsize = self.get_conn_maxsize(which, maxsize) + if conn is None: + return None + + flags = fcntl.fcntl(conn, fcntl.F_GETFL) + if not conn.closed: + fcntl.fcntl(conn, fcntl.F_SETFL, flags| os.O_NONBLOCK) + + try: + if not select.select([conn], [], [], 0)[0]: + return '' + + r = conn.read(maxsize) + if not r: + return self._close(which) + + if self.universal_newlines: + r = self._translate_newlines(r) + return r + finally: + if not conn.closed: + fcntl.fcntl(conn, fcntl.F_SETFL, flags) + +def recv_some(p, t=.1, e=1, tr=5, stderr=0): + if tr < 1: + tr = 1 + x = time.time()+t + y = [] + r = '' + pr = p.recv + if stderr: + pr = p.recv_err + while time.time() < x or r: + r = pr() + if r is None: break + elif r: + y.append(r) + else: + time.sleep(max((x-time.time())/tr, 0)) + return ''.join(y) + +def send_all(p, data): + if not data: + return + + while len(data): + sent = p.send(data) + data = buffer(data, sent) diff --git a/lib/core/update.py b/lib/core/update.py index f00eba8ef..b04dd7975 100644 --- a/lib/core/update.py +++ b/lib/core/update.py @@ -13,13 +13,13 @@ from subprocess import PIPE from subprocess import Popen as execute from lib.core.common import dataToStdout +from lib.core.common import pollProcess from lib.core.data import conf from lib.core.data import logger from lib.core.data import paths from lib.core.revision import getRevisionNumber from lib.core.settings import GIT_REPOSITORY from lib.core.settings import IS_WIN -from lib.core.subprocessng import pollProcess def update(): if not conf.updateAll: diff --git a/lib/takeover/metasploit.py b/lib/takeover/metasploit.py index acfbea8ac..bb1ba64d2 100644 --- a/lib/takeover/metasploit.py +++ b/lib/takeover/metasploit.py @@ -10,9 +10,7 @@ import re import sys import time -from select import select from subprocess import PIPE -from subprocess import Popen as execute from lib.core.common import dataToStdout from lib.core.common import Backend @@ -21,6 +19,7 @@ from lib.core.common import getRemoteIP from lib.core.common import getUnicode from lib.core.common import normalizePath from lib.core.common import ntToPosixSlashes +from lib.core.common import pollProcess from lib.core.common import randomRange from lib.core.common import randomStr from lib.core.common import readInput @@ -35,9 +34,14 @@ from lib.core.settings import IS_WIN from lib.core.settings import UNICODE_ENCODING from lib.core.subprocessng import blockingReadFromFD from lib.core.subprocessng import blockingWriteToFD -from lib.core.subprocessng import pollProcess -from lib.core.subprocessng import setNonBlocking +from lib.core.subprocessng import Popen as execute +from lib.core.subprocessng import send_all +from lib.core.subprocessng import recv_some +if IS_WIN: + import msvcrt +else: + from select import select class Metasploit: """ @@ -410,14 +414,14 @@ class Metasploit: if not Backend.isOs(OS.WINDOWS): return - proc.stdin.write("use espia\n") - proc.stdin.write("use incognito\n") + send_all(proc, "use espia\n") + send_all(proc, "use incognito\n") # This extension is loaded by default since Metasploit > 3.7 - #proc.stdin.write("use priv\n") + #send_all(proc, "use priv\n") # This extension freezes the connection on 64-bit systems - #proc.stdin.write("use sniffer\n") - proc.stdin.write("sysinfo\n") - proc.stdin.write("getuid\n") + #send_all(proc, "use sniffer\n") + send_all(proc, "sysinfo\n") + send_all(proc, "getuid\n") if conf.privEsc: print @@ -427,7 +431,7 @@ class Metasploit: infoMsg += "techniques, including kitrap0d" logger.info(infoMsg) - proc.stdin.write("getsystem\n") + send_all(proc, "getsystem\n") infoMsg = "displaying the list of Access Tokens availables. " infoMsg += "Choose which user you want to impersonate by " @@ -435,15 +439,11 @@ class Metasploit: infoMsg += "'getsystem' does not success to elevate privileges" logger.info(infoMsg) - proc.stdin.write("list_tokens -u\n") - proc.stdin.write("getuid\n") + send_all(proc, "list_tokens -u\n") + send_all(proc, "getuid\n") def _controlMsfCmd(self, proc, func): stdin_fd = sys.stdin.fileno() - setNonBlocking(stdin_fd) - - proc_out_fd = proc.stdout.fileno() - setNonBlocking(proc_out_fd) while True: returncode = proc.poll() @@ -456,39 +456,63 @@ class Metasploit: return returncode try: - ready_fds = select([stdin_fd, proc_out_fd], [], [], 1) + if IS_WIN: + timeout = 3 - if stdin_fd in ready_fds[0]: - try: - proc.stdin.write(blockingReadFromFD(stdin_fd)) - except IOError: - # Probably the child has exited - pass + inp = "" + start_time = time.time() - if proc_out_fd in ready_fds[0]: - out = blockingReadFromFD(proc_out_fd) - blockingWriteToFD(sys.stdout.fileno(), out) + while True: + if msvcrt.kbhit(): + char = msvcrt.getche() - # For --os-pwn and --os-bof - pwnBofCond = self.connectionStr.startswith("reverse") - pwnBofCond &= "Starting the payload handler" in out + if ord(char) == 13: # enter_key + break + elif ord(char) >= 32: # space_char + inp += char - # For --os-smbrelay - smbRelayCond = "Server started" in out + if len(inp) == 0 and (time.time() - start_time) > timeout: + break - if pwnBofCond or smbRelayCond: - func() + if len(inp) > 0: + try: + send_all(proc, inp) + except IOError: + # Probably the child has exited + pass + else: + ready_fds = select([stdin_fd], [], [], 1) - if "Starting the payload handler" in out and "shell" in self.payloadStr: - if Backend.isOs(OS.WINDOWS): - proc.stdin.write("whoami\n") - else: - proc.stdin.write("uname -a ; id\n") + if stdin_fd in ready_fds[0]: + try: + send_all(proc, blockingReadFromFD(stdin_fd)) + except IOError: + # Probably the child has exited + pass - metSess = re.search("Meterpreter session ([\d]+) opened", out) + out = recv_some(proc, t=.1, e=0) + blockingWriteToFD(sys.stdout.fileno(), out) - if metSess: - self._loadMetExtensions(proc, metSess.group(1)) + # For --os-pwn and --os-bof + pwnBofCond = self.connectionStr.startswith("reverse") + pwnBofCond &= "Starting the payload handler" in out + + # For --os-smbrelay + smbRelayCond = "Server started" in out + + if pwnBofCond or smbRelayCond: + func() + + if "Starting the payload handler" in out and "shell" in self.payloadStr: + if Backend.isOs(OS.WINDOWS): + send_all(proc, "whoami\n") + else: + send_all(proc, "uname -a ; id\n") + + metSess = re.search("Meterpreter session ([\d]+) opened", out) + + if metSess: + self._loadMetExtensions(proc, metSess.group(1)) except EOFError: returncode = proc.wait() From 58a60562ace9241731332e7d5e021569dbfc15a8 Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Wed, 9 Jan 2013 16:05:55 +0000 Subject: [PATCH 02/11] avoid exiting with a traceback for missing dependency, handle properly at some point --- lib/core/subprocessng.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/core/subprocessng.py b/lib/core/subprocessng.py index 9db2af0bd..02db47a23 100644 --- a/lib/core/subprocessng.py +++ b/lib/core/subprocessng.py @@ -15,8 +15,11 @@ from lib.core.common import dataToStdout from lib.core.settings import IS_WIN if IS_WIN: - from win32file import ReadFile, WriteFile - from win32pipe import PeekNamedPipe + try: + from win32file import ReadFile, WriteFile + from win32pipe import PeekNamedPipe + except ImportError: + pass import msvcrt else: import select From d120dc18d134a09ecd104531349cfd950d5c07ad Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Wed, 9 Jan 2013 22:06:27 +0000 Subject: [PATCH 03/11] cleanup --- lib/core/log.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/core/log.py b/lib/core/log.py index f25b9952a..bf6b22cac 100644 --- a/lib/core/log.py +++ b/lib/core/log.py @@ -33,6 +33,3 @@ FORMATTER = logging.Formatter("\r[%(asctime)s] [%(levelname)s] %(message)s", "%H LOGGER_HANDLER.setFormatter(FORMATTER) LOGGER.addHandler(LOGGER_HANDLER) LOGGER.setLevel(logging.WARN) - -# to handle logger with the RESTful API -LOGGER_OUTPUT = StringIO.StringIO() From 794700eb37212ba82c9c704d98d9cfcd518045f1 Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Wed, 9 Jan 2013 22:08:50 +0000 Subject: [PATCH 04/11] preparing to handle logging calls by a separate file descriptor when sqlmap is executed by the REST API - issue #297 --- lib/core/convert.py | 4 ++++ lib/core/option.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/lib/core/convert.py b/lib/core/convert.py index ae1b1f201..ceae15931 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -11,6 +11,7 @@ except: import md5 import sha +import json import pickle import sys import struct @@ -126,3 +127,6 @@ def stdoutencode(data): retVal = data.encode(UNICODE_ENCODING) return retVal + +def jsonize(data): + return json.dumps(data, sort_keys=False, indent=4) diff --git a/lib/core/option.py b/lib/core/option.py index 77347d90b..e1c9f3b9b 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -52,7 +52,9 @@ from lib.core.common import singleTimeWarnMessage from lib.core.common import UnicodeRawConfigParser from lib.core.common import urldecode from lib.core.common import urlencode +from lib.core.convert import base64pickle from lib.core.convert import base64unpickle +from lib.core.convert import jsonize from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger @@ -1804,6 +1806,31 @@ def _mergeOptions(inputOptions, overrideOptions): if hasattr(conf, key) and conf[key] is None: conf[key] = value +# Logger recorder object, which keeps the log structure +class LogRecorder(logging.StreamHandler): + """ + Logging handler class which only records CUSTOM_LOGGING.PAYLOAD entries + to a global list. + """ + loghist = [] + + def emit(self, record): + """ + Simply record the emitted events. + """ + self.loghist.append({'levelname': record.levelname, + 'text': record.message % record.args if record.args else record.message, + 'id': len(self.loghist)}) + + if conf.fdLog: + os.write(conf.fdLog, base64pickle(self.loghist)) + +def _setRestAPILog(): + if hasattr(conf, "fdLog") and conf.fdLog: + #logger.removeHandler(LOGGER_HANDLER) + LOGGER_RECORDER = LogRecorder() + logger.addHandler(LOGGER_RECORDER) + def _setTrafficOutputFP(): if conf.trafficFile: infoMsg = "setting file for logging HTTP traffic" @@ -2069,14 +2096,13 @@ def init(inputOptions=AttribDict(), overrideOptions=False): if not inputOptions.disableColoring: coloramainit() - elif hasattr(LOGGER_HANDLER, "disable_coloring"): - LOGGER_HANDLER.disable_coloring = True _setConfAttributes() _setKnowledgeBaseAttributes() _mergeOptions(inputOptions, overrideOptions) _useWizardInterface() setVerbosity() + _setRestAPILog() _saveCmdline() _setRequestFromFile() _cleanupOptions() From 9766f6025ea0f311953cb9ec505a7e56be3a9bc7 Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Wed, 9 Jan 2013 22:09:50 +0000 Subject: [PATCH 05/11] logging is now handled in a separate file descriptor :) - issue #297 --- lib/parse/cmdline.py | 2 +- lib/utils/api.py | 38 ++++++++++++++++++-------------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index 945fad0ad..f3b02981a 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -664,7 +664,7 @@ def cmdLineParser(): help="Simple wizard interface for beginner users") # Hidden and/or experimental options - parser.add_option("--pickle", dest="pickledOptions", help=SUPPRESS_HELP) + parser.add_option("--pickled-options", dest="pickledOptions", help=SUPPRESS_HELP) parser.add_option("--profile", dest="profile", action="store_true", help=SUPPRESS_HELP) diff --git a/lib/utils/api.py b/lib/utils/api.py index 7106814bc..13341a034 100644 --- a/lib/utils/api.py +++ b/lib/utils/api.py @@ -19,16 +19,15 @@ from subprocess import Popen from lib.controller.controller import start from lib.core.common import unArrayizeValue from lib.core.convert import base64pickle +from lib.core.convert import base64unpickle from lib.core.convert import hexencode +from lib.core.convert import jsonize from lib.core.convert import stdoutencode from lib.core.data import paths -from lib.core.datatype import AttribDict from lib.core.data import kb from lib.core.data import logger +from lib.core.datatype import AttribDict from lib.core.defaults import _defaults -from lib.core.log import FORMATTER -from lib.core.log import LOGGER_HANDLER -from lib.core.log import LOGGER_OUTPUT from lib.core.exception import SqlmapMissingDependence from lib.core.optiondict import optDict from lib.core.option import init @@ -49,13 +48,11 @@ RESTAPI_SERVER_PORT = 8775 # Local global variables adminid = "" +pipes = dict() procs = dict() tasks = AttribDict() # Generic functions -def jsonize(data): - return json.dumps(data, sort_keys=False, indent=4) - def is_admin(taskid): global adminid if adminid != taskid: @@ -254,6 +251,7 @@ def scan_start(taskid): """ global tasks global procs + global pipes if taskid not in tasks: abort(500, "Invalid task ID") @@ -269,8 +267,13 @@ def scan_start(taskid): # Launch sqlmap engine in a separate thread logger.debug("starting a scan for task ID %s" % taskid) - procs[taskid] = Popen("python sqlmap.py --pickle %s" % base64pickle(tasks[taskid]), shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) - stdout, stderr = procs[taskid].communicate() + pipes[taskid] = os.pipe() + + # Provide sqlmap engine with the writable pipe for logging + tasks[taskid]["fdLog"] = pipes[taskid][1] + + # Launch sqlmap engine + procs[taskid] = Popen("python sqlmap.py --pickled-options %s" % base64pickle(tasks[taskid]), shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=False) return jsonize({"success": True}) @@ -279,17 +282,17 @@ def scan_output(taskid): """ Read the standard output of sqlmap core execution """ + global pipes global tasks if taskid not in tasks: abort(500, "Invalid task ID") - sys.stdout.seek(0) - output = sys.stdout.read() - sys.stdout.flush() - sys.stdout.truncate(0) + stdout, stderr = procs[taskid].communicate() - return jsonize({"output": output}) + print "stderr:", stderr + + return jsonize({"stdout": stdout, "stderr": stderr}) @get("/scan//delete") def scan_delete(taskid): @@ -315,12 +318,7 @@ def scan_log(taskid): if taskid not in tasks: abort(500, "Invalid task ID") - LOGGER_OUTPUT.seek(0) - output = LOGGER_OUTPUT.read() - LOGGER_OUTPUT.flush() - LOGGER_OUTPUT.truncate(0) - - return jsonize({"log": output}) + return jsonize({"log": base64unpickle(os.read(pipes[taskid][0], 100000))}) # Function to handle files inside the output directory @get("/download///") From 2126a5ba12916c6336fdcada7d79e119965b7c17 Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Thu, 10 Jan 2013 00:00:00 +0000 Subject: [PATCH 06/11] minor index fix --- lib/core/option.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/option.py b/lib/core/option.py index e1c9f3b9b..4c624308f 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1820,7 +1820,7 @@ class LogRecorder(logging.StreamHandler): """ self.loghist.append({'levelname': record.levelname, 'text': record.message % record.args if record.args else record.message, - 'id': len(self.loghist)}) + 'id': len(self.loghist)+1}) if conf.fdLog: os.write(conf.fdLog, base64pickle(self.loghist)) From ef40779ad369d43b51a7b89fcabfb6a5e91cf44b Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Thu, 10 Jan 2013 00:01:28 +0000 Subject: [PATCH 07/11] upgraded to use custom subprocessng for non-blocking send and read functions for spawned processes. Added new method to display range of log messages, just in case and improved parsing/unpickling of read log messages --- lib/utils/api.py | 51 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/lib/utils/api.py b/lib/utils/api.py index 13341a034..a67c0aa1a 100644 --- a/lib/utils/api.py +++ b/lib/utils/api.py @@ -14,7 +14,7 @@ import tempfile import types from subprocess import PIPE -from subprocess import Popen +from subprocess import STDOUT from lib.controller.controller import start from lib.core.common import unArrayizeValue @@ -32,6 +32,9 @@ from lib.core.exception import SqlmapMissingDependence from lib.core.optiondict import optDict from lib.core.option import init from lib.core.settings import UNICODE_ENCODING +from lib.core.subprocessng import Popen as execute +from lib.core.subprocessng import send_all +from lib.core.subprocessng import recv_some from thirdparty.bottle.bottle import abort from thirdparty.bottle.bottle import error from thirdparty.bottle.bottle import get @@ -273,7 +276,7 @@ def scan_start(taskid): tasks[taskid]["fdLog"] = pipes[taskid][1] # Launch sqlmap engine - procs[taskid] = Popen("python sqlmap.py --pickled-options %s" % base64pickle(tasks[taskid]), shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=False) + procs[taskid] = execute("python sqlmap.py --pickled-options %s" % base64pickle(tasks[taskid]), shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=False) return jsonize({"success": True}) @@ -288,11 +291,9 @@ def scan_output(taskid): if taskid not in tasks: abort(500, "Invalid task ID") - stdout, stderr = procs[taskid].communicate() + stdout = recv_some(procs[taskid], t=1, e=0) - print "stderr:", stderr - - return jsonize({"stdout": stdout, "stderr": stderr}) + return jsonize({"stdout": stdout}) @get("/scan//delete") def scan_delete(taskid): @@ -309,16 +310,50 @@ def scan_delete(taskid): return jsonize({"success": True}) -# Function to handle scans' logs +# Functions to handle scans' logs +@get("/scan//log//") +def scan_log_limited(taskid, start, end): + """ + Retrieve the log messages + """ + log = None + + if taskid not in tasks: + abort(500, "Invalid task ID") + + if not start.isdigit() or not end.isdigit() or end <= start: + abort(500, "Invalid start or end value, must be digits") + + start = max(0, int(start)-1) + end = max(1, int(end)) + pickledLog = os.read(pipes[taskid][0], 100000) + + try: + log = base64unpickle(pickledLog) + log = log[slice(start, end)] + except (KeyError, IndexError, TypeError), e: + logger.error("handled exception when trying to unpickle logger dictionary in scan_log_limited(): %s" % str(e)) + + return jsonize({"log": log}) + @get("/scan//log") def scan_log(taskid): """ Retrieve the log messages """ + log = None + if taskid not in tasks: abort(500, "Invalid task ID") - return jsonize({"log": base64unpickle(os.read(pipes[taskid][0], 100000))}) + pickledLog = os.read(pipes[taskid][0], 100000) + + try: + log = base64unpickle(pickledLog) + except (KeyError, IndexError, TypeError), e: + logger.error("handled exception when trying to unpickle logger dictionary in scan_log(): %s" % str(e)) + + return jsonize({"log": log}) # Function to handle files inside the output directory @get("/download///") From ccc3c3d1a351d57bae1650ec8e6ad24f00351086 Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Thu, 10 Jan 2013 00:51:05 +0000 Subject: [PATCH 08/11] minor fix to distinguish stdout from stderr --- lib/core/subprocessng.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/core/subprocessng.py b/lib/core/subprocessng.py index 02db47a23..9742b8935 100644 --- a/lib/core/subprocessng.py +++ b/lib/core/subprocessng.py @@ -178,9 +178,10 @@ def recv_some(p, t=.1, e=1, tr=5, stderr=0): x = time.time()+t y = [] r = '' - pr = p.recv if stderr: pr = p.recv_err + else: + pr = p.recv while time.time() < x or r: r = pr() if r is None: From 10f109994497e538d82dc828f5063bbc6e3ed0d3 Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Thu, 10 Jan 2013 00:51:56 +0000 Subject: [PATCH 09/11] remove logging handler that shows logging messages to stdout - issue #297 --- lib/core/option.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/core/option.py b/lib/core/option.py index 4c624308f..3fdc408fc 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1819,7 +1819,7 @@ class LogRecorder(logging.StreamHandler): Simply record the emitted events. """ self.loghist.append({'levelname': record.levelname, - 'text': record.message % record.args if record.args else record.message, + 'text': record.msg % record.args if record.args else record.msg, 'id': len(self.loghist)+1}) if conf.fdLog: @@ -1827,7 +1827,7 @@ class LogRecorder(logging.StreamHandler): def _setRestAPILog(): if hasattr(conf, "fdLog") and conf.fdLog: - #logger.removeHandler(LOGGER_HANDLER) + logger.removeHandler(LOGGER_HANDLER) LOGGER_RECORDER = LogRecorder() logger.addHandler(LOGGER_RECORDER) From 8093f3950dc2875821c19a3fae090dd36ff397c7 Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Thu, 10 Jan 2013 00:52:44 +0000 Subject: [PATCH 10/11] properly distinguish stdout from stderr with a separate pipe (tracebacks go to stderr) - issue #297 --- lib/utils/api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/utils/api.py b/lib/utils/api.py index a67c0aa1a..6e0b4fcbc 100644 --- a/lib/utils/api.py +++ b/lib/utils/api.py @@ -14,7 +14,6 @@ import tempfile import types from subprocess import PIPE -from subprocess import STDOUT from lib.controller.controller import start from lib.core.common import unArrayizeValue @@ -276,7 +275,7 @@ def scan_start(taskid): tasks[taskid]["fdLog"] = pipes[taskid][1] # Launch sqlmap engine - procs[taskid] = execute("python sqlmap.py --pickled-options %s" % base64pickle(tasks[taskid]), shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=False) + procs[taskid] = execute("python sqlmap.py --pickled-options %s" % base64pickle(tasks[taskid]), shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=False) return jsonize({"success": True}) @@ -291,9 +290,10 @@ def scan_output(taskid): if taskid not in tasks: abort(500, "Invalid task ID") - stdout = recv_some(procs[taskid], t=1, e=0) + stdout = recv_some(procs[taskid], t=1, e=0, stderr=0) + stderr = recv_some(procs[taskid], t=1, e=0, stderr=1) - return jsonize({"stdout": stdout}) + return jsonize({"stdout": stdout, "stderr": stderr}) @get("/scan//delete") def scan_delete(taskid): From ca337159f578d611079414e9d2bb931ff5de151b Mon Sep 17 00:00:00 2001 From: Bernardo Damele Date: Thu, 10 Jan 2013 01:11:22 +0000 Subject: [PATCH 11/11] added reminder TODO --- lib/core/option.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/core/option.py b/lib/core/option.py index 3fdc408fc..9d6875076 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1823,6 +1823,8 @@ class LogRecorder(logging.StreamHandler): 'id': len(self.loghist)+1}) if conf.fdLog: + # TODO: this is very heavy operation and slows down a lot the + # whole execution of the sqlmap engine, find an alternative os.write(conf.fdLog, base64pickle(self.loghist)) def _setRestAPILog():