minor bug fix

This commit is contained in:
Miroslav Stampar 2011-12-20 10:34:28 +00:00
parent 1b16b5e0f1
commit bdc724cb46
2 changed files with 28 additions and 22 deletions

View File

@ -106,6 +106,9 @@ Alessandro Curio <alessandro.curio@gmail.com>
Alessio Dalla Piazza <alessio.dallapiazza@gmail.com>
for reporting a couple of bugs
Sherif El-Deeb <archeldeeb@gmail.com>
for reporting a minor bug
Stefano Di Paola <stefano.dipaola@wisec.it>
for suggesting good features

View File

@ -72,6 +72,7 @@ class Crawler:
break
if isinstance(content, unicode):
try:
soup = BeautifulSoup(content)
for tag in soup('a'):
if tag.get("href"):
@ -92,7 +93,9 @@ class Crawler:
if re.search(r"(.*?)\?(.+)", url):
threadData.shared.outputs.add(url)
kb.locks.outputs.release()
except UnicodeEncodeError: # for non-HTML files
pass
finally:
if conf.forms:
findPageForms(content, current, False, True)