From 2bfdac5ebcdd515859888aadad628c4505df3bc6 Mon Sep 17 00:00:00 2001 From: stamparm Date: Tue, 30 Apr 2013 18:32:46 +0200 Subject: [PATCH] Minor update for crawler --- lib/utils/crawler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py index 0cbe73ea3..ffccf67da 100644 --- a/lib/utils/crawler.py +++ b/lib/utils/crawler.py @@ -27,6 +27,7 @@ from thirdparty.oset.pyoset import oset def crawl(target): try: + visited = set() threadData = getCurrentThreadData() threadData.shared.value = oset() @@ -37,6 +38,10 @@ def crawl(target): with kb.locks.limit: if threadData.shared.unprocessed: current = threadData.shared.unprocessed.pop() + if current in visited: + continue + else: + visited.add(current) else: break