sorry Bernardo, i hope your mobile is turned off :)))

This commit is contained in:
Miroslav Stampar 2011-06-20 22:47:24 +00:00
parent 2a4a284a29
commit dfc02d8c3c

View File

@ -54,7 +54,7 @@ class Crawler:
content = None
try:
if current.split('.')[-1].lower() not in CRAWL_EXCLUDE_EXTENSIONS:
if current:
content = Request.getPage(url=current, raise404=False)[0]
except sqlmapConnectionException, e:
errMsg = "connection exception detected (%s). skipping " % e
@ -79,6 +79,7 @@ class Crawler:
elif not target:
continue
if url.split('.')[-1].lower() not in CRAWL_EXCLUDE_EXTENSIONS:
kb.locks.outputs.acquire()
threadData.shared.deeper.add(url)
if re.search(r"(.*?)\?(.+)", url):