From 508b9cc7639c712776fbb3662c2799c65a7b2376 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sun, 7 Nov 2010 00:12:00 +0000 Subject: [PATCH] dynamicity engine update --- lib/controller/checks.py | 12 +++++++++--- lib/core/common.py | 3 +++ lib/core/convert.py | 4 ++-- lib/request/comparison.py | 5 ++--- lib/request/connect.py | 4 ++++ 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/controller/checks.py b/lib/controller/checks.py index cc8d9033e..4b2400fa1 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -15,7 +15,6 @@ from difflib import SequenceMatcher from lib.core.agent import agent from lib.core.common import beep -from lib.core.common import getFilteredPageContent from lib.core.common import getUnicode from lib.core.common import randomInt from lib.core.common import randomStr @@ -28,6 +27,7 @@ from lib.core.data import kb from lib.core.data import logger from lib.core.data import paths from lib.core.exception import sqlmapConnectionException +from lib.core.exception import sqlmapGenericException from lib.core.exception import sqlmapNoneDataException from lib.core.exception import sqlmapUserQuitException from lib.core.exception import sqlmapSilentQuitException @@ -219,6 +219,8 @@ def checkStability(): time.sleep(1) secondPage, _ = Request.queryPage(content=True) + conf.seqMatcher.set_seq1(firstPage) + kb.pageStable = (firstPage == secondPage) if kb.pageStable: @@ -283,6 +285,11 @@ def checkStability(): else: checkDynamicContent(firstPage, secondPage) + if not Request.queryPage(): + errMsg = "target url is too dynamic. unable to continue. consider using other methods" + logger.error(errMsg) + raise sqlmapSilentQuitException + return kb.pageStable def checkString(): @@ -386,8 +393,7 @@ def checkConnection(): logger.info(infoMsg) try: - page, _ = Request.getPage() - conf.seqMatcher.set_seq1(page if not conf.textOnly else getFilteredPageContent(page)) + Request.getPage() except sqlmapConnectionException, errMsg: errMsg = getUnicode(errMsg) diff --git a/lib/core/common.py b/lib/core/common.py index b89500fd5..9095c0acb 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -41,6 +41,7 @@ from lib.core.data import kb from lib.core.data import logger from lib.core.data import paths from lib.core.data import queries +from lib.core.convert import htmlunescape from lib.core.convert import urlencode from lib.core.exception import sqlmapFilePathException from lib.core.exception import sqlmapGenericException @@ -1102,6 +1103,8 @@ def getFilteredPageContent(page): while retVal.find(" ") != -1: retVal = retVal.replace(" ", " ") + retVal = htmlunescape(retVal) + return retVal def getPageTextWordsSet(page): diff --git a/lib/core/convert.py b/lib/core/convert.py index f0038766c..25f96c9bc 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -93,7 +93,7 @@ def utf8decode(string): return string.decode("utf-8") def htmlescape(string): - return string.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"').replace("'", ''') + return string.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"').replace("'", ''').replace(' ', ' ') def htmlunescape(string): - return string.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"').replace(''', "'") + return string.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"').replace(''', "'").replace(' ', ' ') diff --git a/lib/request/comparison.py b/lib/request/comparison.py index 18e93aa38..089ff782b 100644 --- a/lib/request/comparison.py +++ b/lib/request/comparison.py @@ -9,7 +9,6 @@ See the file 'doc/COPYING' for copying permission import re -from lib.core.common import getFilteredPageContent from lib.core.common import wasLastRequestError from lib.core.data import conf from lib.core.data import kb @@ -50,7 +49,7 @@ def comparison(page, headers=None, getSeqMatcher=False, pageLength=None): if conf.regexp: return re.search(conf.regexp, page, re.I | re.M) is not None - # Dynamic content lines to be excluded before calculating page hash + # Dynamic content lines to be excluded before comparison if not kb.nullConnection: for item in kb.dynamicMarkings: prefix, postfix = item @@ -72,7 +71,7 @@ def comparison(page, headers=None, getSeqMatcher=False, pageLength=None): if ratio > 1.: ratio = 1. / ratio else: - conf.seqMatcher.set_seq2(page if not conf.textOnly else getFilteredPageContent(page)) + conf.seqMatcher.set_seq2(page) ratio = round(conf.seqMatcher.ratio(), 3) if kb.locks.seqLock: diff --git a/lib/request/connect.py b/lib/request/connect.py index 4f1840246..0d52f68e6 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -18,6 +18,7 @@ import traceback from lib.contrib import multipartpost from lib.core.agent import agent from lib.core.common import readInput +from lib.core.common import getFilteredPageContent from lib.core.common import getUnicode from lib.core.convert import urlencode from lib.core.common import urlEncodeCookieValues @@ -366,6 +367,9 @@ class Connect: if not pageLength: page, headers = Connect.getPage(url=uri, get=get, post=post, cookie=cookie, ua=ua, silent=silent, method=method, auxHeaders=auxHeaders, response=response, raise404=raise404) + + if conf.textOnly: + page = getFilteredPageContent(page) if content or response: return page, headers