another minor update

extra/shutils/duplicates.py

@@ -18,11 +18,18 @@ if len(sys.argv) > 0:
 
     for item in f.readlines():
         item = item.strip()
-        if item in items:
-            if item:
-                print item
-        items.append(item)
+        try:
+            str.encode(item)
+            if item in items:
+                if item:
+                    print item
+            else:
+                items.append(item)
 
+            if not item:
+                items.append('')
+        except:
+            pass
     f.close()
 
     f = open(sys.argv[1], 'w+')

txt/common-columns.txt

@@ -474,21 +474,14 @@ module_addr
 
 # List from schemafuzz.py (http://www.beenuarora.com/code/schemafuzz.py)
 user
-username
-password
-passwd
 pass
 cc_number
-id
-email
 emri
 fjalekalimi
 pwd
-user_name
 customers_email_address
 customers_password
 user_password
-name
 user_pass
 admin_user
 admin_password
@@ -522,9 +515,6 @@ usrpass
 usr_pass
 usrnam
 nc
-uid
-userid
-user_id
 myusername
 mail
 emni
@@ -594,13 +584,11 @@ access
 accnt
 accnts
 account
-accounts
 admin
 adminemail
 adminlogin
 adminmail
 admins
-aid
 aim
 auth
 authenticate
@@ -610,20 +598,16 @@ cc_expires
 cc_owner
 cc_type
 cfg
-cid
 clientname
 clientpassword
 clientusername
-conf
 config
 contact
 converge_pass_hash
 converge_pass_salt
 crack
-customer
 customers
 cvvnumber]
-data
 db_database_name
 db_hostname
 db_password
@@ -632,23 +616,18 @@ download
 e-mail
 emailaddress
 full
-gid
 group
-group_name
 hash
 hashsalt
 homepage
 icq
 icq_number
-id_group
 id_member
 images
 index
 ip_address
 last_ip
 last_login
-lastname
-log
 login_name
 login_pw
 loginkey
@@ -683,8 +662,6 @@ serial
 session_member_id
 session_member_login_key
 sesskey
-setting
-sid
 spacer
 status
 store
@@ -697,7 +674,6 @@ temp_pass
 temp_password
 temppass
 temppasword
-text
 un
 user_email
 user_icq
@@ -722,4 +698,3 @@ usrs
 warez
 xar_name
 xar_pass
-id