Couple of bug fixes for BigArray

data/txt/sha256sums.txt

@@ -165,7 +165,7 @@ eed1db5da17eca4c65a8f999166e2246eef84397687ae820bbe4984ef65a09df  extra/vulnserv
 49bcd74281297c79a6ae5d4b0d1479ddace4476fddaf4383ca682a6977b553e3  lib/controller/handler.py
 4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811  lib/controller/__init__.py
 ac44a343947162532dbf17bd1f9ab424f8008f677367c5ad3f9f7b715a679818  lib/core/agent.py
-fbba89420acafcdb9ba1a95428cf2161b13cfa2d1a7ad7d5e70c14b0e04861f0  lib/core/bigarray.py
+fdb5e3828e14f5bca76bed85747c111a861c972bac3892ac789d0285c1b0e8b3  lib/core/bigarray.py
 f6062e324fdeaacf9df0a289fc3f12f755143e3876a70cb65b38aa2e690f73c1  lib/core/common.py
 11c748cc96ea2bc507bc6c1930a17fe4bc6fdd2dd2a80430df971cb21428eb00  lib/core/compat.py
 39ea62d4224be860befeffb3843c150f2343b64555ad8c438a400222056f6cc0  lib/core/convert.py
@@ -188,7 +188,7 @@ c4bfb493a03caf84dd362aec7c248097841de804b7413d0e1ecb8a90c8550bc0  lib/core/readl
 d1bd70c1a55858495c727fbec91e30af267459c8f64d50fabf9e4ee2c007e920  lib/core/replication.py
 1d0f80b0193ac5204527bfab4bde1a7aee0f693fd008e86b4b29f606d1ef94f3  lib/core/revision.py
 d2eb8e4b05ac93551272b3d4abfaf5b9f2d3ac92499a7704c16ed0b4f200db38  lib/core/session.py
-be3e2b9a16441137d97d366f09dfb14a39d89a258945d70413456fd42e85ca22  lib/core/settings.py
+d9a017b535ac6566d750ec4315e155cbe056fc39538b932071c1c546ec26d535  lib/core/settings.py
 1c5eab9494eb969bc9ce118a2ea6954690c6851cbe54c18373c723b99734bf09  lib/core/shell.py
 4eea6dcf023e41e3c64b210cb5c2efc7ca893b727f5e49d9c924f076bb224053  lib/core/subprocessng.py
 cdd352e1331c6b535e780f6edea79465cb55af53aa2114dcea0e8bf382e56d1a  lib/core/target.py

lib/core/bigarray.py

@@ -12,6 +12,7 @@ except:
 
 import itertools
 import os
+import shutil
 import sys
 import tempfile
 import threading
@@ -28,6 +29,13 @@ try:
 except TypeError:
     DEFAULT_SIZE_OF = 16
 
+try:
+    # Python 2: basestring covers str and unicode
+    STRING_TYPES = (basestring,)
+except NameError:
+    # Python 3: str and bytes are separate
+    STRING_TYPES = (str, bytes)
+
 def _size_of(instance):
     """
     Returns total size of a given instance / object (in bytes)
@@ -35,7 +43,9 @@ def _size_of(instance):
 
     retval = sys.getsizeof(instance, DEFAULT_SIZE_OF)
 
-    if isinstance(instance, dict):
+    if isinstance(instance, STRING_TYPES):
+        return retval
+    elif isinstance(instance, dict):
         retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(instance.items()))
     elif hasattr(instance, "__iter__"):
         retval += sum(_size_of(_) for _ in instance if _ != instance)
@@ -58,12 +68,29 @@ class BigArray(list):
 
     >>> _ = BigArray(xrange(100000))
     >>> _[20] = 0
+    >>> _[-1] = 999
     >>> _[99999]
-    99999
+    999
+    >>> _[100000]
+    Traceback (most recent call last):
+    ...
+    IndexError: BigArray index out of range
     >>> _ += [0]
+    >>> sum(_)
+    4999850980
+    >>> _[len(_) // 2] = 17
+    >>> sum(_)
+    4999800997
     >>> _[100000]
     0
-    >>> _ = _ + [1]
+    >>> _[0] = [None]
+    >>> _.index(0)
+    20
+    >>> import pickle; __ = pickle.loads(pickle.dumps(_))
+    >>> __.append(1)
+    >>> len(_)
+    100001
+    >>> _ = __
     >>> _[-1]
     1
     >>> len([_ for _ in BigArray(xrange(100000))])
@@ -134,15 +161,23 @@ class BigArray(list):
             if self[index] == value:
                 return index
 
-        return ValueError, "%s is not in list" % value
+        raise ValueError("%s is not in list" % value)
+
+    def __reduce__(self):
+        return (self.__class__, (), self.__getstate__())
 
     def close(self):
-        while self.filenames:
+        with self._lock:
-            filename = self.filenames.pop()
+            while self.filenames:
-            try:
+                filename = self.filenames.pop()
-                self._os_remove(filename)
+                try:
-            except OSError:
+                    self._os_remove(filename)
-                pass
+                except OSError:
+                    pass
+            self.chunks = [[]]
+            self.cache = None
+            self.chunk_length = getattr(sys, "maxsize", None)
+            self._size_counter = 0
 
     def __del__(self):
         self.close()
@@ -181,41 +216,89 @@ class BigArray(list):
                 raise SqlmapSystemException(errMsg)
 
     def __getstate__(self):
-        return self.chunks, self.filenames
+        if self.cache and self.cache.dirty:
+            filename = self._dump(self.cache.data)
+            self.chunks[self.cache.index] = filename
+            self.cache.dirty = False
+
+        return self.chunks, self.filenames, self.chunk_length
 
     def __setstate__(self, state):
         self.__init__()
-        self.chunks, self.filenames = state
+        chunks, filenames, self.chunk_length = state
+
+        file_mapping = {}
+        self.filenames = set()
+        self.chunks = []
+
+        for filename in filenames:
+            if not os.path.exists(filename):
+                continue
+
+            try:
+                handle, new_filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.BIG_ARRAY)
+                os.close(handle)
+                shutil.copyfile(filename, new_filename)
+                self.filenames.add(new_filename)
+                file_mapping[filename] = new_filename
+            except (OSError, IOError):
+                pass
+
+        for chunk in chunks:
+            if isinstance(chunk, STRING_TYPES):
+                if chunk in file_mapping:
+                    self.chunks.append(file_mapping[chunk])
+                else:
+                    errMsg = "exception occurred while restoring BigArray chunk "
+                    errMsg += "from file '%s'" % chunk
+                    raise SqlmapSystemException(errMsg)
+            else:
+                self.chunks.append(chunk)
 
     def __getitem__(self, y):
-        length = len(self)
+        with self._lock:
-        if length == 0:
+            length = len(self)
-            raise IndexError("BigArray index out of range")
+            if length == 0:
+                raise IndexError("BigArray index out of range")
 
-        while y < 0:
+            if y < 0:
-            y += length
+                y += length
 
-        index = y // self.chunk_length
+            if y < 0 or y >= length:
-        offset = y % self.chunk_length
+                raise IndexError("BigArray index out of range")
-        chunk = self.chunks[index]
 
-        if isinstance(chunk, list):
+            index = y // self.chunk_length
-            return chunk[offset]
+            offset = y % self.chunk_length
-        else:
+            chunk = self.chunks[index]
-            self._checkcache(index)
+
-            return self.cache.data[offset]
+            if isinstance(chunk, list):
+                return chunk[offset]
+            else:
+                self._checkcache(index)
+                return self.cache.data[offset]
 
     def __setitem__(self, y, value):
-        index = y // self.chunk_length
+        with self._lock:
-        offset = y % self.chunk_length
+            length = len(self)
-        chunk = self.chunks[index]
+            if length == 0:
+                raise IndexError("BigArray index out of range")
 
-        if isinstance(chunk, list):
+            if y < 0:
-            chunk[offset] = value
+                y += length
-        else:
+
-            self._checkcache(index)
+            if y < 0 or y >= length:
-            self.cache.data[offset] = value
+                raise IndexError("BigArray index out of range")
-            self.cache.dirty = True
+
+            index = y // self.chunk_length
+            offset = y % self.chunk_length
+            chunk = self.chunks[index]
+
+            if isinstance(chunk, list):
+                chunk[offset] = value
+            else:
+                self._checkcache(index)
+                self.cache.data[offset] = value
+                self.cache.dirty = True
 
     def __repr__(self):
         return "%s%s" % ("..." if len(self.chunks) > 1 else "", self.chunks[-1].__repr__())

lib/core/settings.py

@@ -19,7 +19,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.9.12.29"
+VERSION = "1.9.12.30"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -658,7 +658,7 @@ ROTATING_CHARS = ('\\', '|', '|', '/', '-')
 BIGARRAY_CHUNK_SIZE = 1024 * 1024
 
 # Compress level used for storing BigArray chunks to disk (0-9)
-BIGARRAY_COMPRESS_LEVEL = 9
+BIGARRAY_COMPRESS_LEVEL = 4
 
 # Maximum number of socket pre-connects
 SOCKET_PRE_CONNECT_QUEUE_SIZE = 3