diff --git a/lib/core/option.py b/lib/core/option.py index abb9efe5c..a1f29a842 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -192,7 +192,7 @@ def __feedTargetsDict(reqFile, addedTargetUrls): continue if conf.scope: - getPostReq &= re.search(conf.scope, url) is not None + getPostReq &= re.search(conf.scope, url, re.I) is not None if getPostReq: if not kb.targetUrls or url not in addedTargetUrls: diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py index 9dece4627..266b4e91f 100644 --- a/lib/utils/crawler.py +++ b/lib/utils/crawler.py @@ -58,14 +58,21 @@ class Crawler: for tag in soup('a'): if tag.get("href"): url = urlparse.urljoin(conf.url, tag.get("href")) + # flag to know if we are dealing with the same target host target = reduce(lambda x, y: x == y, map(lambda x: urlparse.urlparse(x).netloc.split(':')[0], [url, conf.url])) - if target: - kb.locks.outputs.acquire() - threadData.shared.deeper.add(url) - if re.search(r"(.*?)\?(.+)", url): - threadData.shared.outputs.add(url) - kb.locks.outputs.release() + + if conf.scope: + if not re.search(conf.scope, url, re.I): + continue + elif not target: + continue + + kb.locks.outputs.acquire() + threadData.shared.deeper.add(url) + if re.search(r"(.*?)\?(.+)", url): + threadData.shared.outputs.add(url) + kb.locks.outputs.release() threadData.shared.deeper = set() threadData.shared.unprocessed = set([conf.url])