Adding missing module (Issue #674 and Issue #747)

lib/parse/sitemap.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2014 sqlmap developers (http://sqlmap.org/)
+See the file 'doc/COPYING' for copying permission
+"""
+
+import re
+
+from lib.core.common import readInput
+from lib.core.data import kb
+from lib.core.data import logger
+from lib.request.connect import Connect as Request
+from thirdparty.oset.pyoset import oset
+
+abortedFlag = None
+
+def parseSitemap(url, retVal=None):
+    global abortedFlag
+
+    if retVal is not None:
+        logger.debug("parsing sitemap '%s'" % url)
+
+    try:
+        if retVal is None:
+            abortedFlag = False
+            retVal = oset()
+
+        content = Request.getPage(url=url, raise404=True)[0] if not abortedFlag else ""
+        for match in re.finditer(r"<loc>\s*([^<]+)", content):
+            if abortedFlag:
+                break
+            url = match.group(1).strip()
+            if url.endswith(".xml") and "sitemap" in url.lower():
+                if kb.followSitemapRecursion is None:
+                    message = "sitemap recursion detected. Do you want to follow? [y/N] "
+                    test = readInput(message, default="N")
+                    kb.followSitemapRecursion = test[0] in ("y", "Y")
+                if kb.followSitemapRecursion:
+                    parseSitemap(url, retVal)
+            else:
+                retVal.add(url)
+
+    except KeyboardInterrupt:
+        abortedFlag = True
+        warnMsg = "user aborted during sitemap parsing. sqlmap "
+        warnMsg += "will use partial list"
+        logger.warn(warnMsg)
+
+    return retVal