Merge branch 'master' of github.com:sqlmapproject/sqlmap

lib/core/common.py

@@ -3309,3 +3309,21 @@ def isNumber(value):
         return False
     else:
         return True
+
+def pollProcess(process, suppress_errors=False):
+    while True:
+        dataToStdout(".")
+        time.sleep(1)
+
+        returncode = process.poll()
+
+        if returncode is not None:
+            if not suppress_errors:
+                if returncode == 0:
+                    dataToStdout(" done\n")
+                elif returncode < 0:
+                    dataToStdout(" process terminated by signal %d\n" % returncode)
+                elif returncode > 0:
+                    dataToStdout(" quit unexpectedly with return code %d\n" % returncode)
+
+            break

lib/core/convert.py

@@ -11,6 +11,7 @@ except:
     import md5
     import sha
 
+import json
 import pickle
 import sys
 import struct
@@ -126,3 +127,6 @@ def stdoutencode(data):
         retVal = data.encode(UNICODE_ENCODING)
 
     return retVal
+
+def jsonize(data):
+    return json.dumps(data, sort_keys=False, indent=4)

lib/core/log.py

@@ -33,6 +33,3 @@ FORMATTER = logging.Formatter("\r[%(asctime)s] [%(levelname)s] %(message)s", "%H
 LOGGER_HANDLER.setFormatter(FORMATTER)
 LOGGER.addHandler(LOGGER_HANDLER)
 LOGGER.setLevel(logging.WARN)
-
-# to handle logger with the RESTful API
-LOGGER_OUTPUT = StringIO.StringIO()

lib/core/option.py

@@ -52,7 +52,9 @@ from lib.core.common import singleTimeWarnMessage
 from lib.core.common import UnicodeRawConfigParser
 from lib.core.common import urldecode
 from lib.core.common import urlencode
+from lib.core.convert import base64pickle
 from lib.core.convert import base64unpickle
+from lib.core.convert import jsonize
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -1804,6 +1806,33 @@ def _mergeOptions(inputOptions, overrideOptions):
         if hasattr(conf, key) and conf[key] is None:
             conf[key] = value
 
+# Logger recorder object, which keeps the log structure
+class LogRecorder(logging.StreamHandler):
+    """
+    Logging handler class which only records CUSTOM_LOGGING.PAYLOAD entries
+    to a global list.
+    """
+    loghist = []
+
+    def emit(self, record):
+        """
+        Simply record the emitted events.
+        """
+        self.loghist.append({'levelname': record.levelname,
+                             'text': record.msg % record.args if record.args else record.msg,
+                             'id': len(self.loghist)+1})
+
+        if conf.fdLog:
+            # TODO: this is very heavy operation and slows down a lot the
+            # whole execution of the sqlmap engine, find an alternative
+            os.write(conf.fdLog, base64pickle(self.loghist))
+
+def _setRestAPILog():
+    if hasattr(conf, "fdLog") and conf.fdLog:
+        logger.removeHandler(LOGGER_HANDLER)
+        LOGGER_RECORDER = LogRecorder()
+        logger.addHandler(LOGGER_RECORDER)
+
 def _setTrafficOutputFP():
     if conf.trafficFile:
         infoMsg = "setting file for logging HTTP traffic"
@@ -2069,14 +2098,13 @@ def init(inputOptions=AttribDict(), overrideOptions=False):
 
     if not inputOptions.disableColoring:
         coloramainit()
-    elif hasattr(LOGGER_HANDLER, "disable_coloring"):
-        LOGGER_HANDLER.disable_coloring = True
 
     _setConfAttributes()
     _setKnowledgeBaseAttributes()
     _mergeOptions(inputOptions, overrideOptions)
     _useWizardInterface()
     setVerbosity()
+    _setRestAPILog()
     _saveCmdline()
     _setRequestFromFile()
     _cleanupOptions()

lib/core/subprocessng.py

@@ -7,13 +7,22 @@ See the file 'doc/COPYING' for copying permission
 
 import errno
 import os
+import subprocess
 import sys
 import time
 
 from lib.core.common import dataToStdout
 from lib.core.settings import IS_WIN
 
-if not IS_WIN:
+if IS_WIN:
+    try:
+        from win32file import ReadFile, WriteFile
+        from win32pipe import PeekNamedPipe
+    except ImportError:
+        pass
+    import msvcrt
+else:
+    import select
     import fcntl
 
     if (sys.hexversion >> 16) >= 0x202:
@@ -61,30 +70,132 @@ def blockingWriteToFD(fd, data):
 
         break
 
-def setNonBlocking(fd):
+# the following code is taken from http://code.activestate.com/recipes/440554-module-to-allow-asynchronous-subprocess-use-on-win/
-    """
+class Popen(subprocess.Popen):
-    Make a file descriptor non-blocking
+    def recv(self, maxsize=None):
-    """
+        return self._recv('stdout', maxsize)
 
-    if IS_WIN is not True:
+    def recv_err(self, maxsize=None):
-        flags = fcntl.fcntl(fd, FCNTL.F_GETFL)
+        return self._recv('stderr', maxsize)
-        flags = flags | os.O_NONBLOCK
-        fcntl.fcntl(fd, FCNTL.F_SETFL, flags)
 
-def pollProcess(process, suppress_errors=False):
+    def send_recv(self, input='', maxsize=None):
-    while True:
+        return self.send(input), self.recv(maxsize), self.recv_err(maxsize)
-        dataToStdout(".")
-        time.sleep(1)
 
-        returncode = process.poll()
+    def get_conn_maxsize(self, which, maxsize):
+        if maxsize is None:
+            maxsize = 1024
+        elif maxsize < 1:
+            maxsize = 1
+        return getattr(self, which), maxsize
 
-        if returncode is not None:
+    def _close(self, which):
-            if not suppress_errors:
+        getattr(self, which).close()
-                if returncode == 0:
+        setattr(self, which, None)
-                    dataToStdout(" done\n")
-                elif returncode < 0:
-                    dataToStdout(" process terminated by signal %d\n" % returncode)
-                elif returncode > 0:
-                    dataToStdout(" quit unexpectedly with return code %d\n" % returncode)
 
+    if subprocess.mswindows:
+        def send(self, input):
+            if not self.stdin:
+                return None
+
+            try:
+                x = msvcrt.get_osfhandle(self.stdin.fileno())
+                (errCode, written) = WriteFile(x, input)
+            except ValueError:
+                return self._close('stdin')
+            except (subprocess.pywintypes.error, Exception), why:
+                if why[0] in (109, errno.ESHUTDOWN):
+                    return self._close('stdin')
+                raise
+
+            return written
+
+        def _recv(self, which, maxsize):
+            conn, maxsize = self.get_conn_maxsize(which, maxsize)
+            if conn is None:
+                return None
+
+            try:
+                x = msvcrt.get_osfhandle(conn.fileno())
+                (read, nAvail, nMessage) = PeekNamedPipe(x, 0)
+                if maxsize < nAvail:
+                    nAvail = maxsize
+                if nAvail > 0:
+                    (errCode, read) = ReadFile(x, nAvail, None)
+            except ValueError:
+                return self._close(which)
+            except (subprocess.pywintypes.error, Exception), why:
+                if why[0] in (109, errno.ESHUTDOWN):
+                    return self._close(which)
+                raise
+
+            if self.universal_newlines:
+                read = self._translate_newlines(read)
+            return read
+    else:
+        def send(self, input):
+            if not self.stdin:
+                return None
+
+            if not select.select([], [self.stdin], [], 0)[1]:
+                return 0
+
+            try:
+                written = os.write(self.stdin.fileno(), input)
+            except OSError, why:
+                if why[0] == errno.EPIPE: #broken pipe
+                    return self._close('stdin')
+                raise
+
+            return written
+
+        def _recv(self, which, maxsize):
+            conn, maxsize = self.get_conn_maxsize(which, maxsize)
+            if conn is None:
+                return None
+
+            flags = fcntl.fcntl(conn, fcntl.F_GETFL)
+            if not conn.closed:
+                fcntl.fcntl(conn, fcntl.F_SETFL, flags| os.O_NONBLOCK)
+
+            try:
+                if not select.select([conn], [], [], 0)[0]:
+                    return ''
+
+                r = conn.read(maxsize)
+                if not r:
+                    return self._close(which)
+
+                if self.universal_newlines:
+                    r = self._translate_newlines(r)
+                return r
+            finally:
+                if not conn.closed:
+                    fcntl.fcntl(conn, fcntl.F_SETFL, flags)
+
+def recv_some(p, t=.1, e=1, tr=5, stderr=0):
+    if tr < 1:
+        tr = 1
+    x = time.time()+t
+    y = []
+    r = ''
+    if stderr:
+        pr = p.recv_err
+    else:
+        pr = p.recv
+    while time.time() < x or r:
+        r = pr()
+        if r is None:
             break
+        elif r:
+            y.append(r)
+        else:
+            time.sleep(max((x-time.time())/tr, 0))
+    return ''.join(y)
+
+def send_all(p, data):
+    if not data:
+        return
+
+    while len(data):
+        sent = p.send(data)
+        data = buffer(data, sent)

lib/core/update.py

@@ -13,13 +13,13 @@ from subprocess import PIPE
 from subprocess import Popen as execute
 
 from lib.core.common import dataToStdout
+from lib.core.common import pollProcess
 from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.data import paths
 from lib.core.revision import getRevisionNumber
 from lib.core.settings import GIT_REPOSITORY
 from lib.core.settings import IS_WIN
-from lib.core.subprocessng import pollProcess
 
 def update():
     if not conf.updateAll:

lib/parse/cmdline.py

@@ -664,7 +664,7 @@ def cmdLineParser():
                                   help="Simple wizard interface for beginner users")
 
         # Hidden and/or experimental options
-        parser.add_option("--pickle", dest="pickledOptions", help=SUPPRESS_HELP)
+        parser.add_option("--pickled-options", dest="pickledOptions", help=SUPPRESS_HELP)
 
         parser.add_option("--profile", dest="profile", action="store_true",
                           help=SUPPRESS_HELP)

lib/takeover/metasploit.py

@@ -10,9 +10,7 @@ import re
 import sys
 import time
 
-from select import select
 from subprocess import PIPE
-from subprocess import Popen as execute
 
 from lib.core.common import dataToStdout
 from lib.core.common import Backend
@@ -21,6 +19,7 @@ from lib.core.common import getRemoteIP
 from lib.core.common import getUnicode
 from lib.core.common import normalizePath
 from lib.core.common import ntToPosixSlashes
+from lib.core.common import pollProcess
 from lib.core.common import randomRange
 from lib.core.common import randomStr
 from lib.core.common import readInput
@@ -35,9 +34,14 @@ from lib.core.settings import IS_WIN
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.subprocessng import blockingReadFromFD
 from lib.core.subprocessng import blockingWriteToFD
-from lib.core.subprocessng import pollProcess
+from lib.core.subprocessng import Popen as execute
-from lib.core.subprocessng import setNonBlocking
+from lib.core.subprocessng import send_all
+from lib.core.subprocessng import recv_some
 
+if IS_WIN:
+    import msvcrt
+else:
+    from select import select
 
 class Metasploit:
     """
@@ -410,14 +414,14 @@ class Metasploit:
         if not Backend.isOs(OS.WINDOWS):
             return
 
-        proc.stdin.write("use espia\n")
+        send_all(proc, "use espia\n")
-        proc.stdin.write("use incognito\n")
+        send_all(proc, "use incognito\n")
         # This extension is loaded by default since Metasploit > 3.7
-        #proc.stdin.write("use priv\n")
+        #send_all(proc, "use priv\n")
         # This extension freezes the connection on 64-bit systems
-        #proc.stdin.write("use sniffer\n")
+        #send_all(proc, "use sniffer\n")
-        proc.stdin.write("sysinfo\n")
+        send_all(proc, "sysinfo\n")
-        proc.stdin.write("getuid\n")
+        send_all(proc, "getuid\n")
 
         if conf.privEsc:
             print
@@ -427,7 +431,7 @@ class Metasploit:
             infoMsg += "techniques, including kitrap0d"
             logger.info(infoMsg)
 
-            proc.stdin.write("getsystem\n")
+            send_all(proc, "getsystem\n")
 
             infoMsg = "displaying the list of Access Tokens availables. "
             infoMsg += "Choose which user you want to impersonate by "
@@ -435,15 +439,11 @@ class Metasploit:
             infoMsg += "'getsystem' does not success to elevate privileges"
             logger.info(infoMsg)
 
-            proc.stdin.write("list_tokens -u\n")
+            send_all(proc, "list_tokens -u\n")
-            proc.stdin.write("getuid\n")
+            send_all(proc, "getuid\n")
 
     def _controlMsfCmd(self, proc, func):
         stdin_fd = sys.stdin.fileno()
-        setNonBlocking(stdin_fd)
-
-        proc_out_fd = proc.stdout.fileno()
-        setNonBlocking(proc_out_fd)
 
         while True:
             returncode = proc.poll()
@@ -456,39 +456,63 @@ class Metasploit:
                 return returncode
 
             try:
-                ready_fds = select([stdin_fd, proc_out_fd], [], [], 1)
+                if IS_WIN:
+                    timeout = 3
 
-                if stdin_fd in ready_fds[0]:
+                    inp = ""
-                    try:
+                    start_time = time.time()
-                        proc.stdin.write(blockingReadFromFD(stdin_fd))
-                    except IOError:
-                        # Probably the child has exited
-                        pass
 
-                if proc_out_fd in ready_fds[0]:
+                    while True:
-                    out = blockingReadFromFD(proc_out_fd)
+                        if msvcrt.kbhit():
-                    blockingWriteToFD(sys.stdout.fileno(), out)
+                            char = msvcrt.getche()
 
-                    # For --os-pwn and --os-bof
+                            if ord(char) == 13:     # enter_key
-                    pwnBofCond = self.connectionStr.startswith("reverse")
+                                break
-                    pwnBofCond &= "Starting the payload handler" in out
+                            elif ord(char) >= 32:   # space_char
+                                inp += char
 
-                    # For --os-smbrelay
+                        if len(inp) == 0 and (time.time() - start_time) > timeout:
-                    smbRelayCond = "Server started" in out
+                            break
 
-                    if pwnBofCond or smbRelayCond:
+                    if len(inp) > 0:
-                        func()
+                        try:
+                            send_all(proc, inp)
+                        except IOError:
+                            # Probably the child has exited
+                            pass
+                else:
+                    ready_fds = select([stdin_fd], [], [], 1)
 
-                    if "Starting the payload handler" in out and "shell" in self.payloadStr:
+                    if stdin_fd in ready_fds[0]:
-                        if Backend.isOs(OS.WINDOWS):
+                        try:
-                            proc.stdin.write("whoami\n")
+                            send_all(proc, blockingReadFromFD(stdin_fd))
-                        else:
+                        except IOError:
-                            proc.stdin.write("uname -a ; id\n")
+                            # Probably the child has exited
+                            pass
 
-                    metSess = re.search("Meterpreter session ([\d]+) opened", out)
+                out = recv_some(proc, t=.1, e=0)
+                blockingWriteToFD(sys.stdout.fileno(), out)
 
-                    if metSess:
+                # For --os-pwn and --os-bof
-                        self._loadMetExtensions(proc, metSess.group(1))
+                pwnBofCond = self.connectionStr.startswith("reverse")
+                pwnBofCond &= "Starting the payload handler" in out
+
+                # For --os-smbrelay
+                smbRelayCond = "Server started" in out
+
+                if pwnBofCond or smbRelayCond:
+                    func()
+
+                if "Starting the payload handler" in out and "shell" in self.payloadStr:
+                    if Backend.isOs(OS.WINDOWS):
+                        send_all(proc, "whoami\n")
+                    else:
+                        send_all(proc, "uname -a ; id\n")
+
+                metSess = re.search("Meterpreter session ([\d]+) opened", out)
+
+                if metSess:
+                    self._loadMetExtensions(proc, metSess.group(1))
 
             except EOFError:
                 returncode = proc.wait()

lib/utils/api.py

@@ -14,25 +14,26 @@ import tempfile
 import types
 
 from subprocess import PIPE
-from subprocess import Popen
 
 from lib.controller.controller import start
 from lib.core.common import unArrayizeValue
 from lib.core.convert import base64pickle
+from lib.core.convert import base64unpickle
 from lib.core.convert import hexencode
+from lib.core.convert import jsonize
 from lib.core.convert import stdoutencode
 from lib.core.data import paths
-from lib.core.datatype import AttribDict
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.datatype import AttribDict
 from lib.core.defaults import _defaults
-from lib.core.log import FORMATTER
-from lib.core.log import LOGGER_HANDLER
-from lib.core.log import LOGGER_OUTPUT
 from lib.core.exception import SqlmapMissingDependence
 from lib.core.optiondict import optDict
 from lib.core.option import init
 from lib.core.settings import UNICODE_ENCODING
+from lib.core.subprocessng import Popen as execute
+from lib.core.subprocessng import send_all
+from lib.core.subprocessng import recv_some
 from thirdparty.bottle.bottle import abort
 from thirdparty.bottle.bottle import error
 from thirdparty.bottle.bottle import get
@@ -49,13 +50,11 @@ RESTAPI_SERVER_PORT = 8775
 
 # Local global variables
 adminid = ""
+pipes = dict()
 procs = dict()
 tasks = AttribDict()
 
 # Generic functions
-def jsonize(data):
-    return json.dumps(data, sort_keys=False, indent=4)
-
 def is_admin(taskid):
     global adminid
     if adminid != taskid:
@@ -254,6 +253,7 @@ def scan_start(taskid):
     """
     global tasks
     global procs
+    global pipes
 
     if taskid not in tasks:
         abort(500, "Invalid task ID")
@@ -269,8 +269,13 @@ def scan_start(taskid):
     # Launch sqlmap engine in a separate thread
     logger.debug("starting a scan for task ID %s" % taskid)
 
-    procs[taskid] = Popen("python sqlmap.py --pickle %s" % base64pickle(tasks[taskid]), shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+    pipes[taskid] = os.pipe()
-    stdout, stderr = procs[taskid].communicate()
+
+    # Provide sqlmap engine with the writable pipe for logging
+    tasks[taskid]["fdLog"] = pipes[taskid][1]
+
+    # Launch sqlmap engine
+    procs[taskid] = execute("python sqlmap.py --pickled-options %s" % base64pickle(tasks[taskid]), shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=False)
 
     return jsonize({"success": True})
 
@@ -279,17 +284,16 @@ def scan_output(taskid):
     """
     Read the standard output of sqlmap core execution
     """
+    global pipes
     global tasks
 
     if taskid not in tasks:
         abort(500, "Invalid task ID")
 
-    sys.stdout.seek(0)
+    stdout = recv_some(procs[taskid], t=1, e=0, stderr=0)
-    output = sys.stdout.read()
+    stderr = recv_some(procs[taskid], t=1, e=0, stderr=1)
-    sys.stdout.flush()
-    sys.stdout.truncate(0)
 
-    return jsonize({"output": output})
+    return jsonize({"stdout": stdout, "stderr": stderr})
 
 @get("/scan/<taskid>/delete")
 def scan_delete(taskid):
@@ -306,21 +310,50 @@ def scan_delete(taskid):
 
     return jsonize({"success": True})
 
-# Function to handle scans' logs
+# Functions to handle scans' logs
+@get("/scan/<taskid>/log/<start>/<end>")
+def scan_log_limited(taskid, start, end):
+    """
+    Retrieve the log messages
+    """
+    log = None
+
+    if taskid not in tasks:
+        abort(500, "Invalid task ID")
+
+    if not start.isdigit() or not end.isdigit() or end <= start:
+        abort(500, "Invalid start or end value, must be digits")
+
+    start = max(0, int(start)-1)
+    end = max(1, int(end))
+    pickledLog = os.read(pipes[taskid][0], 100000)
+
+    try:
+        log = base64unpickle(pickledLog)
+        log = log[slice(start, end)]
+    except (KeyError, IndexError, TypeError), e:
+        logger.error("handled exception when trying to unpickle logger dictionary in scan_log_limited(): %s" % str(e))
+
+    return jsonize({"log": log})
+
 @get("/scan/<taskid>/log")
 def scan_log(taskid):
     """
     Retrieve the log messages
     """
+    log = None
+
     if taskid not in tasks:
         abort(500, "Invalid task ID")
 
-    LOGGER_OUTPUT.seek(0)
+    pickledLog = os.read(pipes[taskid][0], 100000)
-    output = LOGGER_OUTPUT.read()
-    LOGGER_OUTPUT.flush()
-    LOGGER_OUTPUT.truncate(0)
 
-    return jsonize({"log": output})
+    try:
+        log = base64unpickle(pickledLog)
+    except (KeyError, IndexError, TypeError), e:
+        logger.error("handled exception when trying to unpickle logger dictionary in scan_log(): %s" % str(e))
+
+    return jsonize({"log": log})
 
 # Function to handle files inside the output directory
 @get("/download/<taskid>/<target>/<filename:path>")