fix for a bug reported by g@brindi.si (UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position

47: ordinal not in range(128))
This commit is contained in:
Miroslav Stampar 2011-06-24 19:24:11 +00:00
parent f434c3b29e
commit e9286ddd5b
2 changed files with 3 additions and 2 deletions

View File

@ -133,6 +133,7 @@ class Connect:
refreshing = kwargs.get('refreshing', False)
retrying = kwargs.get('retrying', False)
redirecting = kwargs.get('redirecting', False)
crawling = kwargs.get('crawling', False)
if not urlparse.urlsplit(url).netloc:
url = urlparse.urljoin(conf.url, url)
@ -188,7 +189,7 @@ class Connect:
return page
elif refreshing:
elif any ([refreshing, crawling]):
pass
elif target:

View File

@ -55,7 +55,7 @@ class Crawler:
content = None
try:
if current:
content = Request.getPage(url=current, raise404=False)[0]
content = Request.getPage(url=current, crawling=True, raise404=False)[0]
except sqlmapConnectionException, e:
errMsg = "connection exception detected (%s). skipping " % e
errMsg += "url '%s'" % current