2010-12-09 15:41:16 +03:00
|
|
|
#!/usr/bin/env python
|
|
|
|
|
|
2011-12-22 02:59:23 +04:00
|
|
|
|
2010-12-09 15:41:16 +03:00
|
|
|
"""
|
2011-01-31 14:41:28 +03:00
|
|
|
$Id$
|
2010-12-09 15:41:16 +03:00
|
|
|
|
2011-07-08 00:10:03 +04:00
|
|
|
Copyright (c) 2006-2011 sqlmap developers (http://www.sqlmap.org/)
|
2010-12-09 15:41:16 +03:00
|
|
|
See the file 'doc/COPYING' for copying permission
|
|
|
|
|
|
|
|
|
|
# Removes duplicate entries in wordlist like files
|
|
|
|
|
|
|
|
|
|
import sys
|
|
|
|
|
|
|
|
|
|
if len(sys.argv) > 0:
|
|
|
|
|
|
2011-12-22 02:59:23 +04:00
|
|
|
items = list()
|
|
|
|
|
f = open(sys.argv[1], 'r')
|
|
|
|
|
|
|
|
|
|
for item in f.readlines():
|
|
|
|
|
item = item.strip()
|
|
|
|
|
try:
|
|
|
|
|
str.encode(item)
|
|
|
|
|
if item in items:
|
|
|
|
|
if item:
|
|
|
|
|
print item
|
|
|
|
|
else:
|
|
|
|
|
items.append(item)
|
|
|
|
|
|
|
|
|
|
if not item:
|
|
|
|
|
items.append('')
|
|
|
|
|
except:
|
|
|
|
|
pass
|
|
|
|
|
f.close()
|
|
|
|
|
|
|
|
|
|
f = open(sys.argv[1], 'w+')
|
|
|
|
|
f.writelines("\n".join(items))
|
