improvement of url encoding technique (implemented failsafe routine for shortening too long GET queries)

This commit is contained in:
Miroslav Stampar 2011-03-09 09:36:56 +00:00
parent 9b2962ff1c
commit 5c97f9a496
3 changed files with 28 additions and 3 deletions

View File

@ -19,7 +19,10 @@ import struct
import urllib
from lib.core.data import conf
from lib.core.data import logger
from lib.core.settings import UNICODE_ENCODING
from lib.core.settings import URLENCODE_CHAR_LIMIT
from lib.core.settings import URLENCODE_FAILSAFE_CHARS
def base64decode(value):
return value.decode("base64")
@ -80,7 +83,7 @@ def urldecode(value, encoding=None):
return result
def urlencode(value, safe="%&=|()", convall=False):
def urlencode(value, safe="%&=", convall=False, limit=False):
if conf.direct or "POSTxml" in conf.paramDict:
return value
@ -92,8 +95,24 @@ def urlencode(value, safe="%&=|()", convall=False):
if convall:
result = urllib.quote(utf8encode(value)) # Reference: http://old.nabble.com/Re:-Problem:-neither-urllib2.quote-nor-urllib.quote-encode-the--unicode-strings-arguments-p19823144.html
else:
count = 0
while True:
result = urllib.quote(utf8encode(value), safe)
if limit and len(result) > URLENCODE_CHAR_LIMIT:
if count >= len(URLENCODE_FAILSAFE_CHARS):
dbgMsg = "failed to fully shorten urlencoding value"
logger.debug(dbgMsg)
break
while count < len(URLENCODE_FAILSAFE_CHARS):
safe += URLENCODE_FAILSAFE_CHARS[count]
count += 1
if safe[-1] in value:
break
else:
break
return result
def utf8encode(value):

View File

@ -271,3 +271,9 @@ REFLECTED_VALUE_MARKER = '__REFLECTED_VALUE__'
# Regular expression used for marking non-alphanum characters
REFLECTED_NON_ALPHA_NUM_REGEX = r'[^<>\\r\\n]+?'
# chars which can be used as a failsafe values in case of too long URL encoding value
URLENCODE_FAILSAFE_CHARS = '()|,'
# maximum length of urlencoded value after which failsafe procedure takes away
URLENCODE_CHAR_LIMIT = 4000

View File

@ -421,7 +421,7 @@ class Connect:
checkPayload(value)
if PLACE.GET in conf.parameters:
get = urlencode(conf.parameters[PLACE.GET] if place != PLACE.GET or not value else value)
get = urlencode(conf.parameters[PLACE.GET] if place != PLACE.GET or not value else value, limit=True)
if PLACE.POST in conf.parameters:
post = urlencode(conf.parameters[PLACE.POST] if place != PLACE.POST or not value else value)