refactoring of MSSQL XML banner parsing

extra/mssqlsig/update.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+
+"""
+$Id: fingerprint.py 2463 2010-11-30 22:40:25Z inquisb $
+
+Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
+See the file 'doc/COPYING' for copying permission
+"""
+
+# Removes duplicate entries in wordlist like files
+
+import codecs
+import difflib
+import os
+import re
+import sys
+import urllib2
+import urlparse
+
+from xml.dom.minidom import Document
+
+MSSQL_XML = os.path.abspath("../../xml/banner/mssql.xml")
+
+# Url to update Microsoft SQL Server XML versions file from
+MSSQL_VERSIONS_URL = "http://www.sqlsecurity.com/FAQs/SQLServerVersionDatabase/tabid/63/Default.aspx"
+
+def updateMSSQLXML():
+    infoMsg = "[INFO] retrieving data from '%s'" % MSSQL_VERSIONS_URL
+    print infoMsg
+
+    try:
+        req = urllib2.Request(MSSQL_VERSIONS_URL)
+        f = urllib2.urlopen(req)
+        mssqlVersionsHtmlString = f.read()
+        f.close()
+    except urllib2.URLError:
+        __mssqlPath     = urlparse.urlsplit(MSSQL_VERSIONS_URL)
+        __mssqlHostname = __mssqlPath[1]
+
+        warnMsg  = "[WARNING] sqlmap was unable to connect to %s," % __mssqlHostname
+        warnMsg += " check your Internet connection and retry"
+        print warnMsg
+
+        return
+
+    releases      = re.findall("class=\"BCC_DV_01DarkBlueTitle\">SQL Server ([\d\.]+) Builds", mssqlVersionsHtmlString, re.I | re.M)
+    releasesCount = len(releases)
+
+    # Create the minidom document
+    doc = Document()
+
+    # Create the <root> base element
+    root = doc.createElement("root")
+    doc.appendChild(root)
+
+    for index in range(0, releasesCount):
+        release = releases[index]
+
+        # Skip Microsoft SQL Server 6.5 because the HTML
+        # table is in another format
+        if release == "6.5":
+            continue
+
+        # Create the <signatures> base element
+        signatures = doc.createElement("signatures")
+        signatures.setAttribute("release", release)
+        root.appendChild(signatures)
+
+        startIdx = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index])
+
+        if index == releasesCount - 1:
+            stopIdx  = len(mssqlVersionsHtmlString)
+        else:
+            stopIdx  = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index + 1])
+
+        mssqlVersionsReleaseString = mssqlVersionsHtmlString[startIdx:stopIdx]
+        servicepackVersion = re.findall("</td><td>[7\.0|2000|2005|2008]*(.*?)</td><td.*?([\d\.]+)</td>[\r]*\n", mssqlVersionsReleaseString, re.I | re.M)
+
+        for servicePack, version in servicepackVersion:
+            if servicePack.startswith(" "):
+                servicePack = servicePack[1:]
+            if "/" in servicePack:
+                servicePack = servicePack[:servicePack.index("/")]
+            if "(" in servicePack:
+                servicePack = servicePack[:servicePack.index("(")]
+            if "-" in servicePack:
+                servicePack = servicePack[:servicePack.index("-")]
+            if "*" in servicePack:
+                servicePack = servicePack[:servicePack.index("*")]
+            if servicePack.startswith("+"):
+                servicePack = "0%s" % servicePack
+
+            servicePack = servicePack.replace("\t", " ")
+            servicePack = servicePack.replace("No SP", "0")
+            servicePack = servicePack.replace("RTM", "0")
+            servicePack = servicePack.replace("SP", "")
+            servicePack = servicePack.replace("Service Pack", "")
+            servicePack = servicePack.replace("<a href=\"http:", "")
+            servicePack = servicePack.replace("  ", " ")
+            servicePack = servicePack.replace("+ ", "+")
+            servicePack = servicePack.replace(" +", "+")
+
+            if servicePack.endswith(" "):
+                servicePack = servicePack[:-1]
+
+            if servicePack and version:
+                # Create the main <card> element
+                signature = doc.createElement("signature")
+                signatures.appendChild(signature)
+
+                # Create a <version> element
+                versionElement = doc.createElement("version")
+                signature.appendChild(versionElement)
+
+                # Give the <version> elemenet some text
+                versionText = doc.createTextNode(version)
+                versionElement.appendChild(versionText)
+
+                # Create a <servicepack> element
+                servicepackElement = doc.createElement("servicepack")
+                signature.appendChild(servicepackElement)
+
+                # Give the <servicepack> elemenet some text
+                servicepackText = doc.createTextNode(servicePack)
+                servicepackElement.appendChild(servicepackText)
+
+    # Save our newly created XML to the signatures file
+    mssqlXml = codecs.open(MSSQL_XML, "w", "utf8")
+    doc.writexml(writer=mssqlXml, addindent="    ", newl="\n")
+    mssqlXml.close()
+
+    infoMsg  = "[INFO] done. retrieved data parsed and saved into '%s'" % MSSQL_XML
+    print infoMsg
+
+if __name__ == "__main__":
+    updateMSSQLXML()

lib/core/settings.py

@@ -100,9 +100,6 @@ IS_WIN             = subprocess.mswindows
 PLATFORM           = os.name
 PYVERSION          = sys.version.split()[0]
 
-# Url to update Microsoft SQL Server XML versions file from
-MSSQL_VERSIONS_URL = "http://www.sqlsecurity.com/FAQs/SQLServerVersionDatabase/tabid/63/Default.aspx"
-
 # Database management system specific variables
 MSSQL_SYSTEM_DBS    = ( "Northwind", "model", "msdb", "pubs", "tempdb" )
 MYSQL_SYSTEM_DBS    = ( "information_schema", "mysql" )                   # Before MySQL 5.0 only "mysql"

lib/core/update.py

@@ -7,17 +7,13 @@ Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
 See the file 'doc/COPYING' for copying permission
 """
 
-import codecs
-import difflib
 import os
 import re
 import shutil
 import sys
 import time
-import urlparse
 
 from distutils.dir_util import mkpath
-from xml.dom.minidom import Document
 
 from subprocess import PIPE
 from subprocess import Popen as execute
@@ -28,166 +24,15 @@ from lib.core.common import readInput
 from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.data import paths
-from lib.core.exception import sqlmapConnectionException
 from lib.core.exception import sqlmapFilePathException
-from lib.core.settings import MSSQL_VERSIONS_URL
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.subprocessng import pollProcess
 from lib.request.connect import Connect as Request
 
-def __updateMSSQLXML():
+def update():
-    infoMsg = "updating Microsoft SQL Server XML versions file"
+    if not conf.updateAll:
-    logger.info(infoMsg)
-
-    try:
-        mssqlVersionsHtmlString, _ = Request.getPage(url=MSSQL_VERSIONS_URL, direct=True)
-    except sqlmapConnectionException, _:
-        __mssqlPath     = urlparse.urlsplit(MSSQL_VERSIONS_URL)
-        __mssqlHostname = __mssqlPath[1]
-
-        warnMsg  = "sqlmap was unable to connect to %s," % __mssqlHostname
-        warnMsg += " check your Internet connection and retry"
-        logger.warn(warnMsg)
-
         return
 
-    releases      = re.findall("class=\"BCC_DV_01DarkBlueTitle\">SQL Server ([\d\.]+) Builds", mssqlVersionsHtmlString, re.I | re.M)
-    releasesCount = len(releases)
-
-    # Create the minidom document
-    doc = Document()
-
-    # Create the <root> base element
-    root = doc.createElement("root")
-    doc.appendChild(root)
-
-    for index in range(0, releasesCount):
-        release = releases[index]
-
-        # Skip Microsoft SQL Server 6.5 because the HTML
-        # table is in another format
-        if release == "6.5":
-            continue
-
-        # Create the <signatures> base element
-        signatures = doc.createElement("signatures")
-        signatures.setAttribute("release", release)
-        root.appendChild(signatures)
-
-        startIdx = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index])
-
-        if index == releasesCount - 1:
-            stopIdx  = len(mssqlVersionsHtmlString)
-        else:
-            stopIdx  = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index + 1])
-
-        mssqlVersionsReleaseString = mssqlVersionsHtmlString[startIdx:stopIdx]
-        servicepackVersion = re.findall("</td><td>[7\.0|2000|2005|2008]*(.*?)</td><td.*?([\d\.]+)</td>[\r]*\n", mssqlVersionsReleaseString, re.I | re.M)
-
-        for servicePack, version in servicepackVersion:
-            if servicePack.startswith(" "):
-                servicePack = servicePack[1:]
-            if "/" in servicePack:
-                servicePack = servicePack[:servicePack.index("/")]
-            if "(" in servicePack:
-                servicePack = servicePack[:servicePack.index("(")]
-            if "-" in servicePack:
-                servicePack = servicePack[:servicePack.index("-")]
-            if "*" in servicePack:
-                servicePack = servicePack[:servicePack.index("*")]
-            if servicePack.startswith("+"):
-                servicePack = "0%s" % servicePack
-
-            servicePack = servicePack.replace("\t", " ")
-            servicePack = servicePack.replace("No SP", "0")
-            servicePack = servicePack.replace("RTM", "0")
-            servicePack = servicePack.replace("SP", "")
-            servicePack = servicePack.replace("Service Pack", "")
-            servicePack = servicePack.replace("<a href=\"http:", "")
-            servicePack = servicePack.replace("  ", " ")
-            servicePack = servicePack.replace("+ ", "+")
-            servicePack = servicePack.replace(" +", "+")
-
-            if servicePack.endswith(" "):
-                servicePack = servicePack[:-1]
-
-            if servicePack and version:
-                # Create the main <card> element
-                signature = doc.createElement("signature")
-                signatures.appendChild(signature)
-
-                # Create a <version> element
-                versionElement = doc.createElement("version")
-                signature.appendChild(versionElement)
-
-                # Give the <version> elemenet some text
-                versionText = doc.createTextNode(version)
-                versionElement.appendChild(versionText)
-
-                # Create a <servicepack> element
-                servicepackElement = doc.createElement("servicepack")
-                signature.appendChild(servicepackElement)
-
-                # Give the <servicepack> elemenet some text
-                servicepackText = doc.createTextNode(servicePack)
-                servicepackElement.appendChild(servicepackText)
-
-    # Get the XML old file content to a local variable
-    mssqlXml = codecs.open(paths.MSSQL_XML, "r", UNICODE_ENCODING)
-    oldMssqlXml = mssqlXml.read()
-    oldMssqlXmlSignatures = oldMssqlXml.count("<signature>")
-    oldMssqlXmlList = oldMssqlXml.splitlines(1)
-    mssqlXml.close()
-
-    # Backup the XML old file
-    shutil.copy(paths.MSSQL_XML, "%s.bak" % paths.MSSQL_XML)
-
-    # Save our newly created XML to the signatures file
-    mssqlXml = codecs.open(paths.MSSQL_XML, "w", UNICODE_ENCODING)
-    doc.writexml(writer=mssqlXml, addindent="    ", newl="\n")
-    mssqlXml.close()
-
-    # Get the XML new file content to a local variable
-    mssqlXml = codecs.open(paths.MSSQL_XML, "r", UNICODE_ENCODING)
-    newMssqlXml = mssqlXml.read()
-    newMssqlXmlSignatures = newMssqlXml.count("<signature>")
-    newMssqlXmlList = newMssqlXml.splitlines(1)
-    mssqlXml.close()
-
-    # If the new XML versions file differs from the old one it probably
-    # means that we have got new Microsoft SQL Server versions
-    if oldMssqlXmlSignatures != newMssqlXmlSignatures:
-        infoMsg  = "Microsoft SQL Server XML versions file updated successfully. "
-
-        if oldMssqlXmlSignatures < newMssqlXmlSignatures:
-            infoMsg += "%d " % (newMssqlXmlSignatures - oldMssqlXmlSignatures)
-            infoMsg += "new signatures added since the last update"
-
-        # NOTE: This should never happen, in this rare case it might
-        # be that the Microsoft SQL Server versions database
-        # (MSSQL_VERSIONS_URL) changed its structure
-        else:
-            infoMsg += "%d " % (oldMssqlXmlSignatures - newMssqlXmlSignatures)
-            infoMsg += "signatures removed since the last update"
-
-        logger.info(infoMsg)
-
-        message = "Do you want to see the differences? [Y/n] "
-        test = readInput(message, default="Y")
-
-        if not test or test[0] in ("y", "Y"):
-            infoMsg = "Differences:"
-            logger.info(infoMsg)
-
-            # Compare the old XML file with the new one
-            diff = difflib.unified_diff(oldMssqlXmlList, newMssqlXmlList, "%s.bak" % paths.MSSQL_XML, paths.MSSQL_XML)
-            sys.stdout.writelines(diff)
-    else:
-        infoMsg  = "no new Microsoft SQL Server versions since the "
-        infoMsg += "last update"
-        logger.info(infoMsg)
-
-def __updateSqlmap():
     rootDir = paths.SQLMAP_ROOT_PATH
 
     infoMsg = "updating sqlmap to latest development version from the "
@@ -240,10 +85,3 @@ def __updateSqlmap():
             revision = re.search("revision\s+([\d]+)", svnStdout, re.I)
             if revision:
                 logger.info('updated to the latest revision %s' % revision.group(1))
-
-def update():
-    if not conf.updateAll:
-        return
-
-    __updateSqlmap()
-    __updateMSSQLXML()