From f712d2477e82c60a64049e3bd729d3354c147bbf Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 9 Dec 2010 12:41:16 +0000 Subject: [PATCH] removed duplicate entries inside common wordlists (tables & columns) and added a script which does that automatically --- extra/shutils/duplicates.py | 30 + extra/shutils/duplicates.sh | 9 - extra/shutils/id.sh | 2 +- txt/common-columns.txt | 1449 ++++++++++++++++++----------------- txt/common-tables.txt | 56 +- 5 files changed, 758 insertions(+), 788 deletions(-) create mode 100644 extra/shutils/duplicates.py delete mode 100755 extra/shutils/duplicates.sh diff --git a/extra/shutils/duplicates.py b/extra/shutils/duplicates.py new file mode 100644 index 000000000..7b684ad5c --- /dev/null +++ b/extra/shutils/duplicates.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +""" +$Id: fingerprint.py 2463 2010-11-30 22:40:25Z inquisb $ + +Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/) +See the file 'doc/COPYING' for copying permission +""" + +# Removes duplicate entries in wordlist like files + +import sys + +if len(sys.argv) > 0: + + items = list() + f = open(sys.argv[1], 'r') + + for item in f.readlines(): + item = item.strip() + if item in items: + if item: + print item + items.append(item) + + f.close() + + f = open(sys.argv[1], 'w+') + f.writelines("\n".join(items)) + f.close() diff --git a/extra/shutils/duplicates.sh b/extra/shutils/duplicates.sh deleted file mode 100755 index 9f7844b9d..000000000 --- a/extra/shutils/duplicates.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# $Id$ - -# Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/) -# See the file 'doc/COPYING' for copying permission - -# Lists all duplicate entries inside a file -cat $1 | uniq -d \ No newline at end of file diff --git a/extra/shutils/id.sh b/extra/shutils/id.sh index b29e5df8f..acd5ecccc 100755 --- a/extra/shutils/id.sh +++ b/extra/shutils/id.sh @@ -6,4 +6,4 @@ # See the file 'doc/COPYING' for copying permission # Adds SVN property 'Id' to project files -find ../../. -type f -name "*.py" -exec svn propset svn:keywords "Id" '{}' \; \ No newline at end of file +find ../../. -type f -name "*.py" -exec svn propset svn:keywords "Id" '{}' \; diff --git a/txt/common-columns.txt b/txt/common-columns.txt index d8436a133..edeb9d8d4 100644 --- a/txt/common-columns.txt +++ b/txt/common-columns.txt @@ -1,724 +1,725 @@ -# Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/) -# See the file 'doc/COPYING' for copying permission - -id -name -user_id -description -username -type -title -userid -group_id -first_name -itemid -category_id -firstname -code -pno -nextval -hostid -table_name -cid -email -smtp_helo -platformid -dept_id -album_id -key_ -the -child_cfg -jid -platform -expression -functionid -smtp_server -uid -clock -alarmid -alertid -private_key -actionid -triggerid -triggertemplateid -local_spi -delay -sid -mediaid -peer_cfg -smtp_email -order_id -shared_secret -itemtemplateid -certificate -insertid -role_id -song_id -item_id -product_id -blob_id -distip -artist_id -empno -customer_name -grade -branch_name -portal_id -deptno -data -rid -app_id -class -loan_number -countryid -enabled -fname -country -ename -object_id -idtype -groupid -rowid -accno -account_number -event -passwd -sequence_id -datarow -owner_id -display -pid -venue -locked -eno -serviceid -alias -categoryid -canoccupantsinvite -keyword -channel_id -loginrestrictedtonickname -registrationenabled -logenabled -ip -maxnumber -tag_id -alert_id -cananyonediscoverjid -address -sumdatarow -emp_id -ono -anyone -surname -subdomain -maxusers -ccc -datacol -os -status_id -node_id -essn -last_name -iteration -canchangenickname -canoccupantschangesubject -membersonly -created_by -succ_rate -dnumber -service_id -mid -publicroom -propvalue -empty_days -moderated -customer_id -wdatarow -persistent -authorid -patch_status_id -submitted_by -resolution_id -osvendor -routeid -arch -fid -assigned_to -ns -event_id -problem_code -city -note -channel -element_id -cat_id -position_id -schema_id -area -bug_category_id -session_id -project_id -random -nsprefix -archive_id -nsschema -view_id -pname -bug_group_id -lastname -link_id -langid -catname -bug_id -magic_string -m_id -zip -patch_category_id -custno -idcountry -stopid -identifier -category -isbn -group_project_id -extension_id -state -password -page -extension -spellid -dno -instanceof -network -priority -aname -person_id -ncbofile -student_number -term_id -uno -path_id -aid -location_id -propertyno -course_number -tid -langug_code -variable -dept_desc -orderno -ownerno -partof -clientno -white -macaddr -jobtypeid -direction -md5sum -orga_id -parentcategoryid -beginstateid -mname -qno -src -featurename -client_id -route_id -ticker -version -modulename -maty_id -currentstateid -userinfo_id -column_id -imageinfo_id -staffno -lid -metadatainfoid -context -app_title -dest -attributecategory_id -operation_type -dnum -pers_id_registerer -datasource -connectorid -our_loc -country_name -dname -capital -search_id -statechangeid -rightid -endstateid -distconnectorid -walnut -distmacaddr -pixsize -jobid -revid -match_cid -branchno -prepend_digits -stockno -ncbofileid -object_type -type_id -pubid -qagent -office -db_name -bank -dummy -storyname -col -petty -qname -store_id -inv_id -inventory -gift -cno -item -c_sec_id -row_id -price -loc_id -ssn -c_id -sname -parent -allowance -color -group_name -accounts -vendorid -gifi_accno -movie_id -rate -company -subid -commentpath -protocol_action_id -topic_id -s_id -config_id -long -link -copyright -vehicle -customerid -customer -f_id -chart_id -url -host -loans -charttype -imagefile -data_set_id -guest_ip -biosample_id -affiliation_id -os_id -street_id -book_code -object_name -start_date -form_id -itemno -provincial -confid -ratingid -drinker -qname_id -whatsdom -config_name -ship_id -investigator_id -smilies_id -cal_id -license_id -conf -contact_id -procedure_id -column_name -chromosome_id -tf_key -agent_specialtyid -users_id -gid -publisher_code -setting -format_id -word -slogan -superssn -product -referredby -operationid -ban_ip -p_id -lbl_aom_unaccessible_shipmethod -origin -comment_id -product_version -probe_id -orderdate -ordernumber -data_type_id -publisherid -lake_id -course_id -questionid -student_id -user_name -answerid -hashtag -preference_id -author_num -branch_num -derived_id -factoryid -filterid -log -pnumber -specialtyid -plugin_id -aa -file -dept_number -action_attribute_id -cpr -storeid -progenitor_id -staff_number -deptid -semester -poi_id -part_id -cell_line_id -transaction_id -agentid -regionid -token -serial_no -experimental_data_set_id -cp_id -the_geom -model -o_id -personid -display_name -salesperson_id -dependent_name -license -tablename -employee_id -e_id -id_group -location -bb -languageid -int4 -msg_id -department -book_id -ingredientid -action_type_id -maker -app -id_customer -this -entry_id -county_id -protocol_type_id -empnbr -unit_number -bar -studentid -dbid -title_id -cname -emp_num -owner -course_name -editionnumber -sessionid -mealid -com_id -text -chip_layout_id -watchlistid -qty -data_set_type_id -orderid -module_id -c1 -dlocation -domainid -course_no -mgrssn -id_log -access_control_type_id -account_id -checking -protocol_id -request_id -settingsid -lname -sale_date -module_addr - -# List from schemafuzz.py (http://www.beenuarora.com/code/schemafuzz.py) -user -username -password -passwd -pass -cc_number -id -email -emri -fjalekalimi -pwd -user_name -customers_email_address -customers_password -user_password -name -user_pass -admin_user -admin_password -admin_pass -usern -user_n -users -login -logins -login_user -login_admin -login_username -user_username -user_login -auid -apwd -adminid -admin_id -adminuser -adminuserid -admin_userid -adminusername -admin_username -adminname -admin_name -usr -usr_n -usrname -usr_name -usrpass -usr_pass -usrnam -nc -uid -userid -user_id -myusername -mail -emni -logohu -punonjes -kpro_user -wp_users -emniplote -perdoruesi -perdorimi -punetoret -logini -llogaria -fjalekalimin -kodi -emer -ime -korisnik -korisnici -user1 -administrator -administrator_name -mem_login -login_password -login_pass -login_passwd -login_pwd -sifra -lozinka -psw -pass1word -pass_word -passw -pass_w -user_passwd -userpass -userpassword -userpwd -user_pwd -useradmin -user_admin -mypassword -passwrd -admin_pwd -admin_passwd -mem_password -memlogin -e_mail -usrn -u_name -uname -mempassword -mem_pass -mem_passwd -mem_pwd -p_word -pword -p_assword -myname -my_username -my_name -my_password -my_email -cvvnumber -about -access -accnt -accnts -account -accounts -admin -adminemail -adminlogin -adminmail -admins -aid -aim -auth -authenticate -authentication -blog -cc_expires -cc_owner -cc_type -cfg -cid -clientname -clientpassword -clientusername -conf -config -contact -converge_pass_hash -converge_pass_salt -crack -customer -customers -cvvnumber] -data -db_database_name -db_hostname -db_password -db_username -download -e-mail -emailaddress -full -gid -group -group_name -hash -hashsalt -homepage -icq -icq_number -id_group -id_member -images -index -ip_address -last_ip -last_login -lastname -log -login_name -login_pw -loginkey -loginout -logo -md5hash -member -member_id -member_login_key -member_name -memberid -membername -members -new -news -nick -number -nummer -pass_hash -passwordsalt -passwort -personal_key -phone -privacy -pw -pwrd -salt -search -secretanswer -secretquestion -serial -session_member_id -session_member_login_key -sesskey -setting -sid -spacer -status -store -store1 -store2 -store3 -store4 -table_prefix -temp_pass -temp_password -temppass -temppasword -text -un -user_email -user_icq -user_ip -user_level -user_passw -user_pw -user_pword -user_pwrd -user_un -user_uname -user_usernm -user_usernun -user_usrnm -userip -userlogin -usernm -userpw -usr2 -usrnm -usrs -warez -xar_name -xar_pass +# Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/) +# See the file 'doc/COPYING' for copying permission + +id +name +user_id +description +username +type +title +userid +group_id +first_name +itemid +category_id +firstname +code +pno +nextval +hostid +table_name +cid +email +smtp_helo +platformid +dept_id +album_id +key_ +the +child_cfg +jid +platform +expression +functionid +smtp_server +uid +clock +alarmid +alertid +private_key +actionid +triggerid +triggertemplateid +local_spi +delay +sid +mediaid +peer_cfg +smtp_email +order_id +shared_secret +itemtemplateid +certificate +insertid +role_id +song_id +item_id +product_id +blob_id +distip +artist_id +empno +customer_name +grade +branch_name +portal_id +deptno +data +rid +app_id +class +loan_number +countryid +enabled +fname +country +ename +object_id +idtype +groupid +rowid +accno +account_number +event +passwd +sequence_id +datarow +owner_id +display +pid +venue +locked +eno +serviceid +alias +categoryid +canoccupantsinvite +keyword +channel_id +loginrestrictedtonickname +registrationenabled +logenabled +ip +maxnumber +tag_id +alert_id +cananyonediscoverjid +address +sumdatarow +emp_id +ono +anyone +surname +subdomain +maxusers +ccc +datacol +os +status_id +node_id +essn +last_name +iteration +canchangenickname +canoccupantschangesubject +membersonly +created_by +succ_rate +dnumber +service_id +mid +publicroom +propvalue +empty_days +moderated +customer_id +wdatarow +persistent +authorid +patch_status_id +submitted_by +resolution_id +osvendor +routeid +arch +fid +assigned_to +ns +event_id +problem_code +city +note +channel +element_id +cat_id +position_id +schema_id +area +bug_category_id +session_id +project_id +random +nsprefix +archive_id +nsschema +view_id +pname +bug_group_id +lastname +link_id +langid +catname +bug_id +magic_string +m_id +zip +patch_category_id +custno +idcountry +stopid +identifier +category +isbn +group_project_id +extension_id +state +password +page +extension +spellid +dno +instanceof +network +priority +aname +person_id +ncbofile +student_number +term_id +uno +path_id +aid +location_id +propertyno +course_number +tid +langug_code +variable +dept_desc +orderno +ownerno +partof +clientno +white +macaddr +jobtypeid +direction +md5sum +orga_id +parentcategoryid +beginstateid +mname +qno +src +featurename +client_id +route_id +ticker +version +modulename +maty_id +currentstateid +userinfo_id +column_id +imageinfo_id +staffno +lid +metadatainfoid +context +app_title +dest +attributecategory_id +operation_type +dnum +pers_id_registerer +datasource +connectorid +our_loc +country_name +dname +capital +search_id +statechangeid +rightid +endstateid +distconnectorid +walnut +distmacaddr +pixsize +jobid +revid +match_cid +branchno +prepend_digits +stockno +ncbofileid +object_type +type_id +pubid +qagent +office +db_name +bank +dummy +storyname +col +petty +qname +store_id +inv_id +inventory +gift +cno +item +c_sec_id +row_id +price +loc_id +ssn +c_id +sname +parent +allowance +color +group_name +accounts +vendorid +gifi_accno +movie_id +rate +company +subid +commentpath +protocol_action_id +topic_id +s_id +config_id +long +link +copyright +vehicle +customerid +customer +f_id +chart_id +url +host +loans +charttype +imagefile +data_set_id +guest_ip +biosample_id +affiliation_id +os_id +street_id +book_code +object_name +start_date +form_id +itemno +provincial +confid +ratingid +drinker +qname_id +whatsdom +config_name +ship_id +investigator_id +smilies_id +cal_id +license_id +conf +contact_id +procedure_id +column_name +chromosome_id +tf_key +agent_specialtyid +users_id +gid +publisher_code +setting +format_id +word +slogan +superssn +product +referredby +operationid +ban_ip +p_id +lbl_aom_unaccessible_shipmethod +origin +comment_id +product_version +probe_id +orderdate +ordernumber +data_type_id +publisherid +lake_id +course_id +questionid +student_id +user_name +answerid +hashtag +preference_id +author_num +branch_num +derived_id +factoryid +filterid +log +pnumber +specialtyid +plugin_id +aa +file +dept_number +action_attribute_id +cpr +storeid +progenitor_id +staff_number +deptid +semester +poi_id +part_id +cell_line_id +transaction_id +agentid +regionid +token +serial_no +experimental_data_set_id +cp_id +the_geom +model +o_id +personid +display_name +salesperson_id +dependent_name +license +tablename +employee_id +e_id +id_group +location +bb +languageid +int4 +msg_id +department +book_id +ingredientid +action_type_id +maker +app +id_customer +this +entry_id +county_id +protocol_type_id +empnbr +unit_number +bar +studentid +dbid +title_id +cname +emp_num +owner +course_name +editionnumber +sessionid +mealid +com_id +text +chip_layout_id +watchlistid +qty +data_set_type_id +orderid +module_id +c1 +dlocation +domainid +course_no +mgrssn +id_log +access_control_type_id +account_id +checking +protocol_id +request_id +settingsid +lname +sale_date +module_addr + +# List from schemafuzz.py (http://www.beenuarora.com/code/schemafuzz.py) +user +username +password +passwd +pass +cc_number +id +email +emri +fjalekalimi +pwd +user_name +customers_email_address +customers_password +user_password +name +user_pass +admin_user +admin_password +admin_pass +usern +user_n +users +login +logins +login_user +login_admin +login_username +user_username +user_login +auid +apwd +adminid +admin_id +adminuser +adminuserid +admin_userid +adminusername +admin_username +adminname +admin_name +usr +usr_n +usrname +usr_name +usrpass +usr_pass +usrnam +nc +uid +userid +user_id +myusername +mail +emni +logohu +punonjes +kpro_user +wp_users +emniplote +perdoruesi +perdorimi +punetoret +logini +llogaria +fjalekalimin +kodi +emer +ime +korisnik +korisnici +user1 +administrator +administrator_name +mem_login +login_password +login_pass +login_passwd +login_pwd +sifra +lozinka +psw +pass1word +pass_word +passw +pass_w +user_passwd +userpass +userpassword +userpwd +user_pwd +useradmin +user_admin +mypassword +passwrd +admin_pwd +admin_passwd +mem_password +memlogin +e_mail +usrn +u_name +uname +mempassword +mem_pass +mem_passwd +mem_pwd +p_word +pword +p_assword +myname +my_username +my_name +my_password +my_email +cvvnumber +about +access +accnt +accnts +account +accounts +admin +adminemail +adminlogin +adminmail +admins +aid +aim +auth +authenticate +authentication +blog +cc_expires +cc_owner +cc_type +cfg +cid +clientname +clientpassword +clientusername +conf +config +contact +converge_pass_hash +converge_pass_salt +crack +customer +customers +cvvnumber] +data +db_database_name +db_hostname +db_password +db_username +download +e-mail +emailaddress +full +gid +group +group_name +hash +hashsalt +homepage +icq +icq_number +id_group +id_member +images +index +ip_address +last_ip +last_login +lastname +log +login_name +login_pw +loginkey +loginout +logo +md5hash +member +member_id +member_login_key +member_name +memberid +membername +members +new +news +nick +number +nummer +pass_hash +passwordsalt +passwort +personal_key +phone +privacy +pw +pwrd +salt +search +secretanswer +secretquestion +serial +session_member_id +session_member_login_key +sesskey +setting +sid +spacer +status +store +store1 +store2 +store3 +store4 +table_prefix +temp_pass +temp_password +temppass +temppasword +text +un +user_email +user_icq +user_ip +user_level +user_passw +user_pw +user_pword +user_pwrd +user_un +user_uname +user_usernm +user_usernun +user_usrnm +userip +userlogin +usernm +userpw +usr2 +usrnm +usrs +warez +xar_name +xar_pass +id \ No newline at end of file diff --git a/txt/common-tables.txt b/txt/common-tables.txt index 5b8235526..f46e549b1 100644 --- a/txt/common-tables.txt +++ b/txt/common-tables.txt @@ -730,7 +730,6 @@ forum_vote THOT_TYPE cmts_track bkp_ItemReplication -User hostbenchmarks filearchive f_spatialcontext @@ -1195,7 +1194,6 @@ manufacturer Tasks island coupon -SALES webcal_report RegistryPackage sysmaps_links @@ -1534,7 +1532,6 @@ ewst_sessioni nuke_gallery_media_types outdoor_spaces po_seq -files salariedEmployees grp jforum_topics @@ -1606,7 +1603,6 @@ BORROWER phpbb_acl_options markers Population -Country shipping guava_preferences rating @@ -1619,7 +1615,6 @@ SPACE geo_Sea DATA_ORG Contributor - jos_vm_product_download jos_vm_coupons jos_vm_product_reviews @@ -1704,7 +1699,6 @@ jos_vm_product_relations jos_core_acl_aro_sections jos_vm_order_history jos_banner - php_users ALL_USERS banned_users @@ -1712,7 +1706,6 @@ users_tmp users_club publicusers cmsusers - # List provided by Anastasios Monachos (anastasiosm@gmail.com) blacklist cost @@ -1756,7 +1749,7 @@ MANAGEMENTGROUP SUBSCRIBE TBLUSERS TBLLIST -TBLLOG +TBLLOG TBLPROFILES TBLREPORTS TBLTRANSACTIONS @@ -1770,10 +1763,7 @@ sort _wfspro_admin 4images_users a_admin -account -accounts adm -admin admin_login admin_user admin_userinfo @@ -1784,36 +1774,24 @@ administration administrator administrators adminrights -admins adminuser art article_admin articles artikel aut -author autore backend backend_users backenduser bbs -book chat_config chat_messages chat_users -client -clients clubconfig -company -config -contact -contacts content -control cpg_config cpg132_users -customer -customers customers_basket dbadmins dealer @@ -1826,15 +1804,12 @@ e107_user forum.ibf_members fusion_user_groups fusion_users -group -groups ibf_admin_sessions ibf_conf_settings ibf_members ibf_members_converge ibf_sessions icq -images index info ipb.ibf_members @@ -1842,17 +1817,12 @@ ipb_sessions joomla_users jos_blastchatc_users jos_comprofiler_members -jos_contact_details jos_joomblog_users -jos_messages_cfg jos_moschat_users -jos_users knews_lostpass korisnici kpro_adminlogs kpro_user -links -login login_admin login_admins login_user @@ -1873,17 +1843,14 @@ manager mb_users member memberlist -members minibbtable_users mitglieder movie -movies mybb_users mysql mysql.user name names -news news_lostpass newsletter nuke_authors @@ -1892,14 +1859,10 @@ nuke_config nuke_popsettings nuke_users obb_profiles -order -orders parol -partner partners passes password -passwords perdorues perdoruesit phorum_session @@ -1907,11 +1870,9 @@ phorum_user phorum_users phpads_clients phpads_config -phpbb_users phpBB2.forum_users phpBB2.phpbb_users phpmyadmin.pma_table_info -pma_table_info poll_user punbb_users pwd @@ -1921,9 +1882,6 @@ reg_users registered reguser regusers -session -sessions -settings shop.cards shop.orders site_login @@ -1938,7 +1896,6 @@ statistics superuser sysadmin sysadmins -system sysuser sysusers table @@ -1956,13 +1913,10 @@ tbl tbl_user tbl_users tbluser -tbl_clients tbl_client tblclients tblclient -test usebb_members -user user_admin user_info user_list @@ -1971,12 +1925,9 @@ user_logins user_names usercontrol userinfo -userlist userlogins username usernames -userrights -users vb_user vbulletin_session vbulletin_user @@ -1995,11 +1946,9 @@ yabb_settings yabbse_settings ACT_INFO ActiveDataFeed -Category CategoryGroup ChicksPass ClickTrack -Country CountryCodes1 CustomNav DataFeedPerformance1 @@ -2088,7 +2037,6 @@ cms_admins user_name jos_user table_user -email mail bulletin cc_info @@ -2100,4 +2048,4 @@ Site_Login UserAdmin Admins Login -Logins +Logins \ No newline at end of file