removed duplicate entries inside common wordlists (tables & columns) and added a script which does that automatically

extra/shutils/duplicates.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+"""
+$Id: fingerprint.py 2463 2010-11-30 22:40:25Z inquisb $
+
+Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
+See the file 'doc/COPYING' for copying permission
+"""
+
+# Removes duplicate entries in wordlist like files
+
+import sys
+
+if len(sys.argv) > 0:
+
+    items = list()
+    f = open(sys.argv[1], 'r')
+
+    for item in f.readlines():
+        item = item.strip()
+        if item in items:
+            if item:
+                print item
+        items.append(item)
+
+    f.close()
+
+    f = open(sys.argv[1], 'w+')
+    f.writelines("\n".join(items))
+    f.close()

extra/shutils/duplicates.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-# $Id$
-
-# Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
-# See the file 'doc/COPYING' for copying permission
-
-# Lists all duplicate entries inside a file
-cat $1 | uniq -d

txt/common-columns.txt

@@ -722,3 +722,4 @@ usrs
 warez
 xar_name
 xar_pass
+id

txt/common-tables.txt

@@ -730,7 +730,6 @@ forum_vote
 THOT_TYPE
 cmts_track
 bkp_ItemReplication
-User
 hostbenchmarks
 filearchive
 f_spatialcontext
@@ -1195,7 +1194,6 @@ manufacturer
 Tasks
 island
 coupon
-SALES
 webcal_report
 RegistryPackage
 sysmaps_links
@@ -1534,7 +1532,6 @@ ewst_sessioni
 nuke_gallery_media_types
 outdoor_spaces
 po_seq
-files
 salariedEmployees
 grp
 jforum_topics
@@ -1606,7 +1603,6 @@ BORROWER
 phpbb_acl_options
 markers
 Population
-Country
 shipping
 guava_preferences
 rating
@@ -1619,7 +1615,6 @@ SPACE
 geo_Sea
 DATA_ORG
 Contributor
-
 jos_vm_product_download
 jos_vm_coupons
 jos_vm_product_reviews
@@ -1704,7 +1699,6 @@ jos_vm_product_relations
 jos_core_acl_aro_sections
 jos_vm_order_history
 jos_banner
-
 php_users
 ALL_USERS
 banned_users
@@ -1712,7 +1706,6 @@ users_tmp
 users_club
 publicusers
 cmsusers
-
 # List provided by Anastasios Monachos (anastasiosm@gmail.com)
 blacklist
 cost
@@ -1770,10 +1763,7 @@ sort
 _wfspro_admin
 4images_users
 a_admin
-account
-accounts
 adm
-admin
 admin_login
 admin_user
 admin_userinfo
@@ -1784,36 +1774,24 @@ administration
 administrator
 administrators
 adminrights
-admins
 adminuser
 art
 article_admin
 articles
 artikel
 aut
-author
 autore
 backend
 backend_users
 backenduser
 bbs
-book
 chat_config
 chat_messages
 chat_users
-client
-clients
 clubconfig
-company
-config
-contact
-contacts
 content
-control
 cpg_config
 cpg132_users
-customer
-customers
 customers_basket
 dbadmins
 dealer
@@ -1826,15 +1804,12 @@ e107_user
 forum.ibf_members
 fusion_user_groups
 fusion_users
-group
-groups
 ibf_admin_sessions
 ibf_conf_settings
 ibf_members
 ibf_members_converge
 ibf_sessions
 icq
-images
 index
 info
 ipb.ibf_members
@@ -1842,17 +1817,12 @@ ipb_sessions
 joomla_users
 jos_blastchatc_users
 jos_comprofiler_members
-jos_contact_details
 jos_joomblog_users
-jos_messages_cfg
 jos_moschat_users
-jos_users
 knews_lostpass
 korisnici
 kpro_adminlogs
 kpro_user
-links
-login
 login_admin
 login_admins
 login_user
@@ -1873,17 +1843,14 @@ manager
 mb_users
 member
 memberlist
-members
 minibbtable_users
 mitglieder
 movie
-movies
 mybb_users
 mysql
 mysql.user
 name
 names
-news
 news_lostpass
 newsletter
 nuke_authors
@@ -1892,14 +1859,10 @@ nuke_config
 nuke_popsettings
 nuke_users
 obb_profiles
-order
-orders
 parol
-partner
 partners
 passes
 password
-passwords
 perdorues
 perdoruesit
 phorum_session
@@ -1907,11 +1870,9 @@ phorum_user
 phorum_users
 phpads_clients
 phpads_config
-phpbb_users
 phpBB2.forum_users
 phpBB2.phpbb_users
 phpmyadmin.pma_table_info
-pma_table_info
 poll_user
 punbb_users
 pwd
@@ -1921,9 +1882,6 @@ reg_users
 registered
 reguser
 regusers
-session
-sessions
-settings
 shop.cards
 shop.orders
 site_login
@@ -1938,7 +1896,6 @@ statistics
 superuser
 sysadmin
 sysadmins
-system
 sysuser
 sysusers
 table
@@ -1956,13 +1913,10 @@ tbl
 tbl_user
 tbl_users
 tbluser
-tbl_clients
 tbl_client
 tblclients
 tblclient
-test
 usebb_members
-user
 user_admin
 user_info
 user_list
@@ -1971,12 +1925,9 @@ user_logins
 user_names
 usercontrol
 userinfo
-userlist
 userlogins
 username
 usernames
-userrights
-users
 vb_user
 vbulletin_session
 vbulletin_user
@@ -1995,11 +1946,9 @@ yabb_settings
 yabbse_settings
 ACT_INFO
 ActiveDataFeed
-Category
 CategoryGroup
 ChicksPass
 ClickTrack
-Country
 CountryCodes1
 CustomNav
 DataFeedPerformance1
@@ -2088,7 +2037,6 @@ cms_admins
 user_name
 jos_user
 table_user
-email
 mail
 bulletin
 cc_info