sqlmapproject/sqlmap
1
1
Fork 0
mirror of https://github.com/sqlmapproject/sqlmap.git synced 2025-01-23 15:54:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

Adding compression to BigArray mechanism (to save disk space on big dumps)

Browse Source
This commit is contained in:
Miroslav Stampar 2017-11-17 11:45:02 +01:00
parent f210d66dff
commit d54ec88648
3 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/core/bigarray.py
View File

@ -14,10 +14,12 @@ import itertools
import os
import sys
import tempfile
import zlib
from lib.core.enums import MKSTEMP_PREFIX
from lib.core.exception import SqlmapSystemException
from lib.core.settings import BIGARRAY_CHUNK_SIZE
from lib.core.settings import BIGARRAY_COMPRESS_LEVEL
DEFAULT_SIZE_OF = sys.getsizeof(object())
@ -80,8 +82,8 @@ class BigArray(list):
if len(self.chunks[-1]) < 1:
self.chunks.pop()
try:
with open(self.chunks[-1], "rb") as fp:
self.chunks[-1] = pickle.load(fp)
with open(self.chunks[-1], "rb") as f:
self.chunks[-1] = pickle.loads(zlib.decompress(f.read()))
except IOError, ex:
errMsg = "exception occurred while retrieving data "
errMsg += "from a temporary file ('%s')" % ex.message
@ -101,8 +103,8 @@ class BigArray(list):
handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.BIG_ARRAY)
self.filenames.add(filename)
os.close(handle)
with open(filename, "w+b") as fp:
pickle.dump(chunk, fp, pickle.HIGHEST_PROTOCOL)
with open(filename, "w+b") as f:
f.write(zlib.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
return filename
except (OSError, IOError), ex:
errMsg = "exception occurred while storing data "
@ -119,8 +121,8 @@ class BigArray(list):
if not (self.cache and self.cache.index == index):
try:
with open(self.chunks[index], "rb") as fp:
self.cache = Cache(index, pickle.load(fp), False)
with open(self.chunks[index], "rb") as f:
self.cache = Cache(index, pickle.loads(zlib.decompress(f.read())), False)
except IOError, ex:
errMsg = "exception occurred while retrieving data "
errMsg += "from a temporary file ('%s')" % ex.message

5
lib/core/settings.py
View File

@ -19,7 +19,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME
from lib.core.enums import OS
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
VERSION = "1.1.11.21"
VERSION = "1.1.11.22"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@ -520,6 +520,9 @@ ROTATING_CHARS = ('\\', '|', '|', '/', '-')
# Approximate chunk length (in bytes) used by BigArray objects (only last chunk and cached one are held in memory)
BIGARRAY_CHUNK_SIZE = 1024 * 1024
# Compress (zlib) level used for storing BigArray chunks to disk (0-9)
BIGARRAY_COMPRESS_LEVEL = 9
# Maximum number of socket pre-connects
SOCKET_PRE_CONNECT_QUEUE_SIZE = 3

4
txt/checksum.md5
View File

@ -26,7 +26,7 @@ f96467fc5cd1d87f52dd7966c8ae6e79 extra/shutils/regressiontest.py
d2cdb9e832e18a81e936ca3348144b16 lib/controller/handler.py
5fb9aaf874daa47ea2b672a22740e56b lib/controller/__init__.py
fd69e56ce20a5a49ce10a7a745022378 lib/core/agent.py
d55b4b58019d6dbfddd40ec919f9f172 lib/core/bigarray.py
8d9d771f7e67582c56a96a8d0ccbe4fc lib/core/bigarray.py
16b63aed575a43c1d406df30a9b22afe lib/core/common.py
54326d3a690f8b26fe5a5da1a589b369 lib/core/convert.py
90b1b08368ac8a859300e6fa6a8c796e lib/core/data.py
@ -46,7 +46,7 @@ e1c000db9be27f973569b1a430629037 lib/core/option.py
760d9df2a27ded29109b390ab202e72d lib/core/replication.py
a2466b62e67f8b31736bac4dac590e51 lib/core/revision.py
02d4762140a72fd44668d3dab5eabda9 lib/core/session.py
915cf4eeb4fefcb6ebb8c10cd3c50468 lib/core/settings.py
5ca78784a48256f3b8a7bc98b9ec1978 lib/core/settings.py
35bffbad762eb9e03db9e93b1c991103 lib/core/shell.py
a59ec28371ae067a6fdd8f810edbee3d lib/core/subprocessng.py
d5a04d672a18f78deb2839c3745ff83c lib/core/target.py