diff --git a/lib/core/common.py b/lib/core/common.py index 62292b370..7607b645b 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -3309,3 +3309,21 @@ def isNumber(value): return False else: return True + +def pollProcess(process, suppress_errors=False): + while True: + dataToStdout(".") + time.sleep(1) + + returncode = process.poll() + + if returncode is not None: + if not suppress_errors: + if returncode == 0: + dataToStdout(" done\n") + elif returncode < 0: + dataToStdout(" process terminated by signal %d\n" % returncode) + elif returncode > 0: + dataToStdout(" quit unexpectedly with return code %d\n" % returncode) + + break diff --git a/lib/core/convert.py b/lib/core/convert.py index ae1b1f201..ceae15931 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -11,6 +11,7 @@ except: import md5 import sha +import json import pickle import sys import struct @@ -126,3 +127,6 @@ def stdoutencode(data): retVal = data.encode(UNICODE_ENCODING) return retVal + +def jsonize(data): + return json.dumps(data, sort_keys=False, indent=4) diff --git a/lib/core/log.py b/lib/core/log.py index f25b9952a..bf6b22cac 100644 --- a/lib/core/log.py +++ b/lib/core/log.py @@ -33,6 +33,3 @@ FORMATTER = logging.Formatter("\r[%(asctime)s] [%(levelname)s] %(message)s", "%H LOGGER_HANDLER.setFormatter(FORMATTER) LOGGER.addHandler(LOGGER_HANDLER) LOGGER.setLevel(logging.WARN) - -# to handle logger with the RESTful API -LOGGER_OUTPUT = StringIO.StringIO() diff --git a/lib/core/option.py b/lib/core/option.py index d914cef95..cbd2d4522 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -52,7 +52,9 @@ from lib.core.common import singleTimeWarnMessage from lib.core.common import UnicodeRawConfigParser from lib.core.common import urldecode from lib.core.common import urlencode +from lib.core.convert import base64pickle from lib.core.convert import base64unpickle +from lib.core.convert import jsonize from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger @@ -1804,6 +1806,33 @@ def _mergeOptions(inputOptions, overrideOptions): if hasattr(conf, key) and conf[key] is None: conf[key] = value +# Logger recorder object, which keeps the log structure +class LogRecorder(logging.StreamHandler): + """ + Logging handler class which only records CUSTOM_LOGGING.PAYLOAD entries + to a global list. + """ + loghist = [] + + def emit(self, record): + """ + Simply record the emitted events. + """ + self.loghist.append({'levelname': record.levelname, + 'text': record.msg % record.args if record.args else record.msg, + 'id': len(self.loghist)+1}) + + if conf.fdLog: + # TODO: this is very heavy operation and slows down a lot the + # whole execution of the sqlmap engine, find an alternative + os.write(conf.fdLog, base64pickle(self.loghist)) + +def _setRestAPILog(): + if hasattr(conf, "fdLog") and conf.fdLog: + logger.removeHandler(LOGGER_HANDLER) + LOGGER_RECORDER = LogRecorder() + logger.addHandler(LOGGER_RECORDER) + def _setTrafficOutputFP(): if conf.trafficFile: infoMsg = "setting file for logging HTTP traffic" @@ -2069,14 +2098,13 @@ def init(inputOptions=AttribDict(), overrideOptions=False): if not inputOptions.disableColoring: coloramainit() - elif hasattr(LOGGER_HANDLER, "disable_coloring"): - LOGGER_HANDLER.disable_coloring = True _setConfAttributes() _setKnowledgeBaseAttributes() _mergeOptions(inputOptions, overrideOptions) _useWizardInterface() setVerbosity() + _setRestAPILog() _saveCmdline() _setRequestFromFile() _cleanupOptions() diff --git a/lib/core/subprocessng.py b/lib/core/subprocessng.py index efd3cf34b..9742b8935 100644 --- a/lib/core/subprocessng.py +++ b/lib/core/subprocessng.py @@ -7,13 +7,22 @@ See the file 'doc/COPYING' for copying permission import errno import os +import subprocess import sys import time from lib.core.common import dataToStdout from lib.core.settings import IS_WIN -if not IS_WIN: +if IS_WIN: + try: + from win32file import ReadFile, WriteFile + from win32pipe import PeekNamedPipe + except ImportError: + pass + import msvcrt +else: + import select import fcntl if (sys.hexversion >> 16) >= 0x202: @@ -61,30 +70,132 @@ def blockingWriteToFD(fd, data): break -def setNonBlocking(fd): - """ - Make a file descriptor non-blocking - """ +# the following code is taken from http://code.activestate.com/recipes/440554-module-to-allow-asynchronous-subprocess-use-on-win/ +class Popen(subprocess.Popen): + def recv(self, maxsize=None): + return self._recv('stdout', maxsize) - if IS_WIN is not True: - flags = fcntl.fcntl(fd, FCNTL.F_GETFL) - flags = flags | os.O_NONBLOCK - fcntl.fcntl(fd, FCNTL.F_SETFL, flags) + def recv_err(self, maxsize=None): + return self._recv('stderr', maxsize) -def pollProcess(process, suppress_errors=False): - while True: - dataToStdout(".") - time.sleep(1) + def send_recv(self, input='', maxsize=None): + return self.send(input), self.recv(maxsize), self.recv_err(maxsize) - returncode = process.poll() + def get_conn_maxsize(self, which, maxsize): + if maxsize is None: + maxsize = 1024 + elif maxsize < 1: + maxsize = 1 + return getattr(self, which), maxsize - if returncode is not None: - if not suppress_errors: - if returncode == 0: - dataToStdout(" done\n") - elif returncode < 0: - dataToStdout(" process terminated by signal %d\n" % returncode) - elif returncode > 0: - dataToStdout(" quit unexpectedly with return code %d\n" % returncode) + def _close(self, which): + getattr(self, which).close() + setattr(self, which, None) + if subprocess.mswindows: + def send(self, input): + if not self.stdin: + return None + + try: + x = msvcrt.get_osfhandle(self.stdin.fileno()) + (errCode, written) = WriteFile(x, input) + except ValueError: + return self._close('stdin') + except (subprocess.pywintypes.error, Exception), why: + if why[0] in (109, errno.ESHUTDOWN): + return self._close('stdin') + raise + + return written + + def _recv(self, which, maxsize): + conn, maxsize = self.get_conn_maxsize(which, maxsize) + if conn is None: + return None + + try: + x = msvcrt.get_osfhandle(conn.fileno()) + (read, nAvail, nMessage) = PeekNamedPipe(x, 0) + if maxsize < nAvail: + nAvail = maxsize + if nAvail > 0: + (errCode, read) = ReadFile(x, nAvail, None) + except ValueError: + return self._close(which) + except (subprocess.pywintypes.error, Exception), why: + if why[0] in (109, errno.ESHUTDOWN): + return self._close(which) + raise + + if self.universal_newlines: + read = self._translate_newlines(read) + return read + else: + def send(self, input): + if not self.stdin: + return None + + if not select.select([], [self.stdin], [], 0)[1]: + return 0 + + try: + written = os.write(self.stdin.fileno(), input) + except OSError, why: + if why[0] == errno.EPIPE: #broken pipe + return self._close('stdin') + raise + + return written + + def _recv(self, which, maxsize): + conn, maxsize = self.get_conn_maxsize(which, maxsize) + if conn is None: + return None + + flags = fcntl.fcntl(conn, fcntl.F_GETFL) + if not conn.closed: + fcntl.fcntl(conn, fcntl.F_SETFL, flags| os.O_NONBLOCK) + + try: + if not select.select([conn], [], [], 0)[0]: + return '' + + r = conn.read(maxsize) + if not r: + return self._close(which) + + if self.universal_newlines: + r = self._translate_newlines(r) + return r + finally: + if not conn.closed: + fcntl.fcntl(conn, fcntl.F_SETFL, flags) + +def recv_some(p, t=.1, e=1, tr=5, stderr=0): + if tr < 1: + tr = 1 + x = time.time()+t + y = [] + r = '' + if stderr: + pr = p.recv_err + else: + pr = p.recv + while time.time() < x or r: + r = pr() + if r is None: break + elif r: + y.append(r) + else: + time.sleep(max((x-time.time())/tr, 0)) + return ''.join(y) + +def send_all(p, data): + if not data: + return + + while len(data): + sent = p.send(data) + data = buffer(data, sent) diff --git a/lib/core/update.py b/lib/core/update.py index f00eba8ef..b04dd7975 100644 --- a/lib/core/update.py +++ b/lib/core/update.py @@ -13,13 +13,13 @@ from subprocess import PIPE from subprocess import Popen as execute from lib.core.common import dataToStdout +from lib.core.common import pollProcess from lib.core.data import conf from lib.core.data import logger from lib.core.data import paths from lib.core.revision import getRevisionNumber from lib.core.settings import GIT_REPOSITORY from lib.core.settings import IS_WIN -from lib.core.subprocessng import pollProcess def update(): if not conf.updateAll: diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index 945fad0ad..f3b02981a 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -664,7 +664,7 @@ def cmdLineParser(): help="Simple wizard interface for beginner users") # Hidden and/or experimental options - parser.add_option("--pickle", dest="pickledOptions", help=SUPPRESS_HELP) + parser.add_option("--pickled-options", dest="pickledOptions", help=SUPPRESS_HELP) parser.add_option("--profile", dest="profile", action="store_true", help=SUPPRESS_HELP) diff --git a/lib/takeover/metasploit.py b/lib/takeover/metasploit.py index e98a35600..5037ea075 100644 --- a/lib/takeover/metasploit.py +++ b/lib/takeover/metasploit.py @@ -10,9 +10,7 @@ import re import sys import time -from select import select from subprocess import PIPE -from subprocess import Popen as execute from lib.core.common import dataToStdout from lib.core.common import Backend @@ -21,6 +19,7 @@ from lib.core.common import getRemoteIP from lib.core.common import getUnicode from lib.core.common import normalizePath from lib.core.common import ntToPosixSlashes +from lib.core.common import pollProcess from lib.core.common import randomRange from lib.core.common import randomStr from lib.core.common import readInput @@ -35,9 +34,14 @@ from lib.core.settings import IS_WIN from lib.core.settings import UNICODE_ENCODING from lib.core.subprocessng import blockingReadFromFD from lib.core.subprocessng import blockingWriteToFD -from lib.core.subprocessng import pollProcess -from lib.core.subprocessng import setNonBlocking +from lib.core.subprocessng import Popen as execute +from lib.core.subprocessng import send_all +from lib.core.subprocessng import recv_some +if IS_WIN: + import msvcrt +else: + from select import select class Metasploit: """ @@ -410,14 +414,14 @@ class Metasploit: if not Backend.isOs(OS.WINDOWS): return - proc.stdin.write("use espia\n") - proc.stdin.write("use incognito\n") + send_all(proc, "use espia\n") + send_all(proc, "use incognito\n") # This extension is loaded by default since Metasploit > 3.7 - #proc.stdin.write("use priv\n") + #send_all(proc, "use priv\n") # This extension freezes the connection on 64-bit systems - #proc.stdin.write("use sniffer\n") - proc.stdin.write("sysinfo\n") - proc.stdin.write("getuid\n") + #send_all(proc, "use sniffer\n") + send_all(proc, "sysinfo\n") + send_all(proc, "getuid\n") if conf.privEsc: print @@ -427,7 +431,7 @@ class Metasploit: infoMsg += "techniques, including kitrap0d" logger.info(infoMsg) - proc.stdin.write("getsystem\n") + send_all(proc, "getsystem\n") infoMsg = "displaying the list of Access Tokens availables. " infoMsg += "Choose which user you want to impersonate by " @@ -435,15 +439,11 @@ class Metasploit: infoMsg += "'getsystem' does not success to elevate privileges" logger.info(infoMsg) - proc.stdin.write("list_tokens -u\n") - proc.stdin.write("getuid\n") + send_all(proc, "list_tokens -u\n") + send_all(proc, "getuid\n") def _controlMsfCmd(self, proc, func): stdin_fd = sys.stdin.fileno() - setNonBlocking(stdin_fd) - - proc_out_fd = proc.stdout.fileno() - setNonBlocking(proc_out_fd) while True: returncode = proc.poll() @@ -456,39 +456,63 @@ class Metasploit: return returncode try: - ready_fds = select([stdin_fd, proc_out_fd], [], [], 1) + if IS_WIN: + timeout = 3 - if stdin_fd in ready_fds[0]: - try: - proc.stdin.write(blockingReadFromFD(stdin_fd)) - except IOError: - # Probably the child has exited - pass + inp = "" + start_time = time.time() - if proc_out_fd in ready_fds[0]: - out = blockingReadFromFD(proc_out_fd) - blockingWriteToFD(sys.stdout.fileno(), out) + while True: + if msvcrt.kbhit(): + char = msvcrt.getche() - # For --os-pwn and --os-bof - pwnBofCond = self.connectionStr.startswith("reverse") - pwnBofCond &= "Starting the payload handler" in out + if ord(char) == 13: # enter_key + break + elif ord(char) >= 32: # space_char + inp += char - # For --os-smbrelay - smbRelayCond = "Server started" in out + if len(inp) == 0 and (time.time() - start_time) > timeout: + break - if pwnBofCond or smbRelayCond: - func() + if len(inp) > 0: + try: + send_all(proc, inp) + except IOError: + # Probably the child has exited + pass + else: + ready_fds = select([stdin_fd], [], [], 1) - if "Starting the payload handler" in out and "shell" in self.payloadStr: - if Backend.isOs(OS.WINDOWS): - proc.stdin.write("whoami\n") - else: - proc.stdin.write("uname -a ; id\n") + if stdin_fd in ready_fds[0]: + try: + send_all(proc, blockingReadFromFD(stdin_fd)) + except IOError: + # Probably the child has exited + pass - metSess = re.search("Meterpreter session ([\d]+) opened", out) + out = recv_some(proc, t=.1, e=0) + blockingWriteToFD(sys.stdout.fileno(), out) - if metSess: - self._loadMetExtensions(proc, metSess.group(1)) + # For --os-pwn and --os-bof + pwnBofCond = self.connectionStr.startswith("reverse") + pwnBofCond &= "Starting the payload handler" in out + + # For --os-smbrelay + smbRelayCond = "Server started" in out + + if pwnBofCond or smbRelayCond: + func() + + if "Starting the payload handler" in out and "shell" in self.payloadStr: + if Backend.isOs(OS.WINDOWS): + send_all(proc, "whoami\n") + else: + send_all(proc, "uname -a ; id\n") + + metSess = re.search("Meterpreter session ([\d]+) opened", out) + + if metSess: + self._loadMetExtensions(proc, metSess.group(1)) except EOFError: returncode = proc.wait() diff --git a/lib/utils/api.py b/lib/utils/api.py index 7106814bc..6e0b4fcbc 100644 --- a/lib/utils/api.py +++ b/lib/utils/api.py @@ -14,25 +14,26 @@ import tempfile import types from subprocess import PIPE -from subprocess import Popen from lib.controller.controller import start from lib.core.common import unArrayizeValue from lib.core.convert import base64pickle +from lib.core.convert import base64unpickle from lib.core.convert import hexencode +from lib.core.convert import jsonize from lib.core.convert import stdoutencode from lib.core.data import paths -from lib.core.datatype import AttribDict from lib.core.data import kb from lib.core.data import logger +from lib.core.datatype import AttribDict from lib.core.defaults import _defaults -from lib.core.log import FORMATTER -from lib.core.log import LOGGER_HANDLER -from lib.core.log import LOGGER_OUTPUT from lib.core.exception import SqlmapMissingDependence from lib.core.optiondict import optDict from lib.core.option import init from lib.core.settings import UNICODE_ENCODING +from lib.core.subprocessng import Popen as execute +from lib.core.subprocessng import send_all +from lib.core.subprocessng import recv_some from thirdparty.bottle.bottle import abort from thirdparty.bottle.bottle import error from thirdparty.bottle.bottle import get @@ -49,13 +50,11 @@ RESTAPI_SERVER_PORT = 8775 # Local global variables adminid = "" +pipes = dict() procs = dict() tasks = AttribDict() # Generic functions -def jsonize(data): - return json.dumps(data, sort_keys=False, indent=4) - def is_admin(taskid): global adminid if adminid != taskid: @@ -254,6 +253,7 @@ def scan_start(taskid): """ global tasks global procs + global pipes if taskid not in tasks: abort(500, "Invalid task ID") @@ -269,8 +269,13 @@ def scan_start(taskid): # Launch sqlmap engine in a separate thread logger.debug("starting a scan for task ID %s" % taskid) - procs[taskid] = Popen("python sqlmap.py --pickle %s" % base64pickle(tasks[taskid]), shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) - stdout, stderr = procs[taskid].communicate() + pipes[taskid] = os.pipe() + + # Provide sqlmap engine with the writable pipe for logging + tasks[taskid]["fdLog"] = pipes[taskid][1] + + # Launch sqlmap engine + procs[taskid] = execute("python sqlmap.py --pickled-options %s" % base64pickle(tasks[taskid]), shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=False) return jsonize({"success": True}) @@ -279,17 +284,16 @@ def scan_output(taskid): """ Read the standard output of sqlmap core execution """ + global pipes global tasks if taskid not in tasks: abort(500, "Invalid task ID") - sys.stdout.seek(0) - output = sys.stdout.read() - sys.stdout.flush() - sys.stdout.truncate(0) + stdout = recv_some(procs[taskid], t=1, e=0, stderr=0) + stderr = recv_some(procs[taskid], t=1, e=0, stderr=1) - return jsonize({"output": output}) + return jsonize({"stdout": stdout, "stderr": stderr}) @get("/scan//delete") def scan_delete(taskid): @@ -306,21 +310,50 @@ def scan_delete(taskid): return jsonize({"success": True}) -# Function to handle scans' logs +# Functions to handle scans' logs +@get("/scan//log//") +def scan_log_limited(taskid, start, end): + """ + Retrieve the log messages + """ + log = None + + if taskid not in tasks: + abort(500, "Invalid task ID") + + if not start.isdigit() or not end.isdigit() or end <= start: + abort(500, "Invalid start or end value, must be digits") + + start = max(0, int(start)-1) + end = max(1, int(end)) + pickledLog = os.read(pipes[taskid][0], 100000) + + try: + log = base64unpickle(pickledLog) + log = log[slice(start, end)] + except (KeyError, IndexError, TypeError), e: + logger.error("handled exception when trying to unpickle logger dictionary in scan_log_limited(): %s" % str(e)) + + return jsonize({"log": log}) + @get("/scan//log") def scan_log(taskid): """ Retrieve the log messages """ + log = None + if taskid not in tasks: abort(500, "Invalid task ID") - LOGGER_OUTPUT.seek(0) - output = LOGGER_OUTPUT.read() - LOGGER_OUTPUT.flush() - LOGGER_OUTPUT.truncate(0) + pickledLog = os.read(pipes[taskid][0], 100000) - return jsonize({"log": output}) + try: + log = base64unpickle(pickledLog) + except (KeyError, IndexError, TypeError), e: + logger.error("handled exception when trying to unpickle logger dictionary in scan_log(): %s" % str(e)) + + return jsonize({"log": log}) # Function to handle files inside the output directory @get("/download///")