mirror of
https://github.com/python-pillow/Pillow.git
synced 2024-12-26 01:46:18 +03:00
Merge pull request #2738 from uploadcare/block-storage
Block & array hybrid storage
This commit is contained in:
wiredfool
GitHub
commit
c82f9fe1bb
39
PIL/Image.py
39
PIL/Image.py
|
|
@ -110,6 +110,7 @@ import os
|
|||
import sys
|
||||
import io
|
||||
import struct
|
||||
import atexit
|
||||
|
||||
# type stuff
|
||||
import collections
|
||||
|
|
@ -2836,3 +2837,41 @@ def radial_gradient(mode):
|
|||
:param mode: Input mode.
|
||||
"""
|
||||
return Image()._new(core.radial_gradient(mode))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------
|
||||
# Resources
|
||||
|
||||
def _apply_env_variables(env=None):
|
||||
if env is None:
|
||||
env = os.environ
|
||||
|
||||
for var_name, setter in [
|
||||
('PILLOW_ALIGNMENT', core.set_alignment),
|
||||
('PILLOW_BLOCK_SIZE', core.set_block_size),
|
||||
('PILLOW_BLOCKS_MAX', core.set_blocks_max),
|
||||
]:
|
||||
if var_name not in env:
|
||||
continue
|
||||
|
||||
var = env[var_name].lower()
|
||||
|
||||
units = 1
|
||||
for postfix, mul in [('k', 1024), ('m', 1024*1024)]:
|
||||
if var.endswith(postfix):
|
||||
units = mul
|
||||
var = var[:-len(postfix)]
|
||||
|
||||
try:
|
||||
var = int(var) * units
|
||||
except ValueError:
|
||||
warnings.warn("{0} is not int".format(var_name))
|
||||
continue
|
||||
|
||||
try:
|
||||
setter(var)
|
||||
except ValueError as e:
|
||||
warnings.warn("{0}: {1}".format(var_name, e))
|
||||
|
||||
_apply_env_variables()
|
||||
atexit.register(core.clear_cache)
|
||||
|
|
|
|||
181
Tests/test_core_resources.py
Normal file
181
Tests/test_core_resources.py
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
from __future__ import division, print_function
|
||||
|
||||
import sys
|
||||
|
||||
from helper import unittest, PillowTestCase
|
||||
from PIL import Image
|
||||
|
||||
|
||||
is_pypy = hasattr(sys, 'pypy_version_info')
|
||||
|
||||
|
||||
class TestCoreStats(PillowTestCase):
|
||||
def test_get_stats(self):
|
||||
# Create at least one image
|
||||
Image.new('RGB', (10, 10))
|
||||
|
||||
stats = Image.core.get_stats()
|
||||
self.assertIn('new_count', stats)
|
||||
self.assertIn('reused_blocks', stats)
|
||||
self.assertIn('freed_blocks', stats)
|
||||
self.assertIn('allocated_blocks', stats)
|
||||
self.assertIn('reallocated_blocks', stats)
|
||||
self.assertIn('blocks_cached', stats)
|
||||
|
||||
def test_reset_stats(self):
|
||||
Image.core.reset_stats()
|
||||
|
||||
stats = Image.core.get_stats()
|
||||
self.assertEqual(stats['new_count'], 0)
|
||||
self.assertEqual(stats['reused_blocks'], 0)
|
||||
self.assertEqual(stats['freed_blocks'], 0)
|
||||
self.assertEqual(stats['allocated_blocks'], 0)
|
||||
self.assertEqual(stats['reallocated_blocks'], 0)
|
||||
self.assertEqual(stats['blocks_cached'], 0)
|
||||
|
||||
|
||||
class TestCoreMemory(PillowTestCase):
|
||||
def tearDown(self):
|
||||
# Restore default values
|
||||
Image.core.set_alignment(1)
|
||||
Image.core.set_block_size(1024*1024)
|
||||
Image.core.set_blocks_max(0)
|
||||
Image.core.clear_cache()
|
||||
|
||||
def test_get_alignment(self):
|
||||
alignment = Image.core.get_alignment()
|
||||
|
||||
self.assertGreater(alignment, 0)
|
||||
|
||||
def test_set_alignment(self):
|
||||
for i in [1, 2, 4, 8, 16, 32]:
|
||||
Image.core.set_alignment(i)
|
||||
alignment = Image.core.get_alignment()
|
||||
self.assertEqual(alignment, i)
|
||||
|
||||
# Try to construct new image
|
||||
Image.new('RGB', (10, 10))
|
||||
|
||||
self.assertRaises(ValueError, Image.core.set_alignment, 0)
|
||||
self.assertRaises(ValueError, Image.core.set_alignment, -1)
|
||||
self.assertRaises(ValueError, Image.core.set_alignment, 3)
|
||||
|
||||
def test_get_block_size(self):
|
||||
block_size = Image.core.get_block_size()
|
||||
|
||||
self.assertGreaterEqual(block_size, 4096)
|
||||
|
||||
def test_set_block_size(self):
|
||||
for i in [4096, 2*4096, 3*4096]:
|
||||
Image.core.set_block_size(i)
|
||||
block_size = Image.core.get_block_size()
|
||||
self.assertEqual(block_size, i)
|
||||
|
||||
# Try to construct new image
|
||||
Image.new('RGB', (10, 10))
|
||||
|
||||
self.assertRaises(ValueError, Image.core.set_block_size, 0)
|
||||
self.assertRaises(ValueError, Image.core.set_block_size, -1)
|
||||
self.assertRaises(ValueError, Image.core.set_block_size, 4000)
|
||||
|
||||
def test_set_block_size_stats(self):
|
||||
Image.core.reset_stats()
|
||||
Image.core.set_blocks_max(0)
|
||||
Image.core.set_block_size(4096)
|
||||
Image.new('RGB', (256, 256))
|
||||
|
||||
stats = Image.core.get_stats()
|
||||
self.assertGreaterEqual(stats['new_count'], 1)
|
||||
self.assertGreaterEqual(stats['allocated_blocks'], 64)
|
||||
if not is_pypy:
|
||||
self.assertGreaterEqual(stats['freed_blocks'], 64)
|
||||
|
||||
def test_get_blocks_max(self):
|
||||
blocks_max = Image.core.get_blocks_max()
|
||||
|
||||
self.assertGreaterEqual(blocks_max, 0)
|
||||
|
||||
def test_set_blocks_max(self):
|
||||
for i in [0, 1, 10]:
|
||||
Image.core.set_blocks_max(i)
|
||||
blocks_max = Image.core.get_blocks_max()
|
||||
self.assertEqual(blocks_max, i)
|
||||
|
||||
# Try to construct new image
|
||||
Image.new('RGB', (10, 10))
|
||||
|
||||
self.assertRaises(ValueError, Image.core.set_blocks_max, -1)
|
||||
|
||||
@unittest.skipIf(is_pypy, "images are not collected")
|
||||
def test_set_blocks_max_stats(self):
|
||||
Image.core.reset_stats()
|
||||
Image.core.set_blocks_max(128)
|
||||
Image.core.set_block_size(4096)
|
||||
Image.new('RGB', (256, 256))
|
||||
Image.new('RGB', (256, 256))
|
||||
|
||||
stats = Image.core.get_stats()
|
||||
self.assertGreaterEqual(stats['new_count'], 2)
|
||||
self.assertGreaterEqual(stats['allocated_blocks'], 64)
|
||||
self.assertGreaterEqual(stats['reused_blocks'], 64)
|
||||
self.assertEqual(stats['freed_blocks'], 0)
|
||||
self.assertEqual(stats['blocks_cached'], 64)
|
||||
|
||||
@unittest.skipIf(is_pypy, "images are not collected")
|
||||
def test_clear_cache_stats(self):
|
||||
Image.core.reset_stats()
|
||||
Image.core.clear_cache()
|
||||
Image.core.set_blocks_max(128)
|
||||
Image.core.set_block_size(4096)
|
||||
Image.new('RGB', (256, 256))
|
||||
Image.new('RGB', (256, 256))
|
||||
# Keep 16 blocks in cache
|
||||
Image.core.clear_cache(16)
|
||||
|
||||
stats = Image.core.get_stats()
|
||||
self.assertGreaterEqual(stats['new_count'], 2)
|
||||
self.assertGreaterEqual(stats['allocated_blocks'], 64)
|
||||
self.assertGreaterEqual(stats['reused_blocks'], 64)
|
||||
self.assertGreaterEqual(stats['freed_blocks'], 48)
|
||||
self.assertEqual(stats['blocks_cached'], 16)
|
||||
|
||||
def test_large_images(self):
|
||||
Image.core.reset_stats()
|
||||
Image.core.set_blocks_max(0)
|
||||
Image.core.set_block_size(4096)
|
||||
Image.new('RGB', (2048, 16))
|
||||
Image.core.clear_cache()
|
||||
|
||||
stats = Image.core.get_stats()
|
||||
self.assertGreaterEqual(stats['new_count'], 1)
|
||||
self.assertGreaterEqual(stats['allocated_blocks'], 16)
|
||||
self.assertGreaterEqual(stats['reused_blocks'], 0)
|
||||
self.assertEqual(stats['blocks_cached'], 0)
|
||||
if not is_pypy:
|
||||
self.assertGreaterEqual(stats['freed_blocks'], 16)
|
||||
|
||||
|
||||
class TestEnvVars(PillowTestCase):
|
||||
def tearDown(self):
|
||||
# Restore default values
|
||||
Image.core.set_alignment(1)
|
||||
Image.core.set_block_size(1024*1024)
|
||||
Image.core.set_blocks_max(0)
|
||||
Image.core.clear_cache()
|
||||
|
||||
def test_units(self):
|
||||
Image._apply_env_variables({'PILLOW_BLOCKS_MAX': '2K'})
|
||||
self.assertEqual(Image.core.get_blocks_max(), 2*1024)
|
||||
Image._apply_env_variables({'PILLOW_BLOCK_SIZE': '2m'})
|
||||
self.assertEqual(Image.core.get_block_size(), 2*1024*1024)
|
||||
|
||||
def test_warnings(self):
|
||||
self.assert_warning(
|
||||
UserWarning, Image._apply_env_variables,
|
||||
{'PILLOW_ALIGNMENT': '15'})
|
||||
self.assert_warning(
|
||||
UserWarning, Image._apply_env_variables,
|
||||
{'PILLOW_BLOCK_SIZE': '1024'})
|
||||
self.assert_warning(
|
||||
UserWarning, Image._apply_env_variables,
|
||||
{'PILLOW_BLOCKS_MAX': 'wat'})
|
||||
|
|
@ -383,6 +383,12 @@ class TestImage(PillowTestCase):
|
|||
im = Image.new('L', (0, 0))
|
||||
self.assertEqual(im.size, (0, 0))
|
||||
|
||||
im = Image.new('L', (0, 100))
|
||||
self.assertEqual(im.size, (0, 100))
|
||||
|
||||
im = Image.new('L', (100, 0))
|
||||
self.assertEqual(im.size, (100, 0))
|
||||
|
||||
self.assertTrue(Image.new('RGB', (1, 1)))
|
||||
# Should pass lists too
|
||||
i = Image.new('RGB', [1, 1])
|
||||
|
|
|
|||
|
|
@ -305,9 +305,9 @@ class CoreResampleAlphaCorrectTest(PillowTestCase):
|
|||
class CoreResamplePassesTest(PillowTestCase):
|
||||
@contextmanager
|
||||
def count(self, diff):
|
||||
count = Image.core.getcount()
|
||||
count = Image.core.get_stats()['new_count']
|
||||
yield
|
||||
self.assertEqual(Image.core.getcount() - count, diff)
|
||||
self.assertEqual(Image.core.get_stats()['new_count'] - count, diff)
|
||||
|
||||
def test_horizontal(self):
|
||||
im = hopper('L')
|
||||
|
|
|
|||
180
_imaging.c
180
_imaging.c
|
|
@ -641,15 +641,6 @@ _new_block(PyObject* self, PyObject* args)
|
|||
return PyImagingNew(ImagingNewBlock(mode, xsize, ysize));
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
_getcount(PyObject* self, PyObject* args)
|
||||
{
|
||||
if (!PyArg_ParseTuple(args, ":getcount"))
|
||||
return NULL;
|
||||
|
||||
return PyInt_FromLong(ImagingNewCount);
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
_linear_gradient(PyObject* self, PyObject* args)
|
||||
{
|
||||
|
|
@ -3330,6 +3321,164 @@ static PyTypeObject PixelAccess_Type = {
|
|||
|
||||
/* -------------------------------------------------------------------- */
|
||||
|
||||
static PyObject*
|
||||
_get_stats(PyObject* self, PyObject* args)
|
||||
{
|
||||
PyObject* d;
|
||||
ImagingMemoryArena arena = &ImagingDefaultArena;
|
||||
|
||||
if (!PyArg_ParseTuple(args, ":get_stats"))
|
||||
return NULL;
|
||||
|
||||
d = PyDict_New();
|
||||
if ( ! d)
|
||||
return NULL;
|
||||
PyDict_SetItemString(d, "new_count",
|
||||
PyInt_FromLong(arena->stats_new_count));
|
||||
PyDict_SetItemString(d, "allocated_blocks",
|
||||
PyInt_FromLong(arena->stats_allocated_blocks));
|
||||
PyDict_SetItemString(d, "reused_blocks",
|
||||
PyInt_FromLong(arena->stats_reused_blocks));
|
||||
PyDict_SetItemString(d, "reallocated_blocks",
|
||||
PyInt_FromLong(arena->stats_reallocated_blocks));
|
||||
PyDict_SetItemString(d, "freed_blocks",
|
||||
PyInt_FromLong(arena->stats_freed_blocks));
|
||||
PyDict_SetItemString(d, "blocks_cached",
|
||||
PyInt_FromLong(arena->blocks_cached));
|
||||
return d;
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
_reset_stats(PyObject* self, PyObject* args)
|
||||
{
|
||||
ImagingMemoryArena arena = &ImagingDefaultArena;
|
||||
|
||||
if (!PyArg_ParseTuple(args, ":reset_stats"))
|
||||
return NULL;
|
||||
|
||||
arena->stats_new_count = 0;
|
||||
arena->stats_allocated_blocks = 0;
|
||||
arena->stats_reused_blocks = 0;
|
||||
arena->stats_reallocated_blocks = 0;
|
||||
arena->stats_freed_blocks = 0;
|
||||
|
||||
Py_INCREF(Py_None);
|
||||
return Py_None;
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
_get_alignment(PyObject* self, PyObject* args)
|
||||
{
|
||||
if (!PyArg_ParseTuple(args, ":get_alignment"))
|
||||
return NULL;
|
||||
|
||||
return PyInt_FromLong(ImagingDefaultArena.alignment);
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
_get_block_size(PyObject* self, PyObject* args)
|
||||
{
|
||||
if (!PyArg_ParseTuple(args, ":get_block_size"))
|
||||
return NULL;
|
||||
|
||||
return PyInt_FromLong(ImagingDefaultArena.block_size);
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
_get_blocks_max(PyObject* self, PyObject* args)
|
||||
{
|
||||
if (!PyArg_ParseTuple(args, ":get_blocks_max"))
|
||||
return NULL;
|
||||
|
||||
return PyInt_FromLong(ImagingDefaultArena.blocks_max);
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
_set_alignment(PyObject* self, PyObject* args)
|
||||
{
|
||||
int alignment;
|
||||
if (!PyArg_ParseTuple(args, "i:set_alignment", &alignment))
|
||||
return NULL;
|
||||
|
||||
if (alignment < 1 || alignment > 128) {
|
||||
PyErr_SetString(PyExc_ValueError, "alignment should be from 1 to 128");
|
||||
return NULL;
|
||||
}
|
||||
/* Is power of two */
|
||||
if (alignment & (alignment - 1)) {
|
||||
PyErr_SetString(PyExc_ValueError, "alignment should be power of two");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ImagingDefaultArena.alignment = alignment;
|
||||
|
||||
Py_INCREF(Py_None);
|
||||
return Py_None;
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
_set_block_size(PyObject* self, PyObject* args)
|
||||
{
|
||||
int block_size;
|
||||
if (!PyArg_ParseTuple(args, "i:set_block_size", &block_size))
|
||||
return NULL;
|
||||
|
||||
if (block_size <= 0) {
|
||||
PyErr_SetString(PyExc_ValueError,
|
||||
"block_size should be greater than 0");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (block_size & 0xfff) {
|
||||
PyErr_SetString(PyExc_ValueError,
|
||||
"block_size should be multiple of 4096");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ImagingDefaultArena.block_size = block_size;
|
||||
|
||||
Py_INCREF(Py_None);
|
||||
return Py_None;
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
_set_blocks_max(PyObject* self, PyObject* args)
|
||||
{
|
||||
int blocks_max;
|
||||
if (!PyArg_ParseTuple(args, "i:set_blocks_max", &blocks_max))
|
||||
return NULL;
|
||||
|
||||
if (blocks_max < 0) {
|
||||
PyErr_SetString(PyExc_ValueError,
|
||||
"blocks_max should be greater than 0");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if ( ! ImagingMemorySetBlocksMax(&ImagingDefaultArena, blocks_max)) {
|
||||
ImagingError_MemoryError();
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Py_INCREF(Py_None);
|
||||
return Py_None;
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
_clear_cache(PyObject* self, PyObject* args)
|
||||
{
|
||||
int i = 0;
|
||||
|
||||
if (!PyArg_ParseTuple(args, "|i:clear_cache", &i))
|
||||
return NULL;
|
||||
|
||||
ImagingMemoryClearCache(&ImagingDefaultArena, i);
|
||||
|
||||
Py_INCREF(Py_None);
|
||||
return Py_None;
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------------------- */
|
||||
|
||||
/* FIXME: this is something of a mess. Should replace this with
|
||||
pluggable codecs, but not before PIL 1.2 */
|
||||
|
||||
|
|
@ -3393,8 +3542,6 @@ static PyMethodDef functions[] = {
|
|||
{"new", (PyCFunction)_new, 1},
|
||||
{"merge", (PyCFunction)_merge, 1},
|
||||
|
||||
{"getcount", (PyCFunction)_getcount, 1},
|
||||
|
||||
/* Functions */
|
||||
{"convert", (PyCFunction)_convert2, 1},
|
||||
|
||||
|
|
@ -3484,6 +3631,17 @@ static PyMethodDef functions[] = {
|
|||
{"outline", (PyCFunction)PyOutline_Create, 1},
|
||||
#endif
|
||||
|
||||
/* Resource management */
|
||||
{"get_stats", (PyCFunction)_get_stats, 1},
|
||||
{"reset_stats", (PyCFunction)_reset_stats, 1},
|
||||
{"get_alignment", (PyCFunction)_get_alignment, 1},
|
||||
{"get_block_size", (PyCFunction)_get_block_size, 1},
|
||||
{"get_blocks_max", (PyCFunction)_get_blocks_max, 1},
|
||||
{"set_alignment", (PyCFunction)_set_alignment, 1},
|
||||
{"set_block_size", (PyCFunction)_set_block_size, 1},
|
||||
{"set_blocks_max", (PyCFunction)_set_blocks_max, 1},
|
||||
{"clear_cache", (PyCFunction)_clear_cache, 1},
|
||||
|
||||
{NULL, NULL} /* sentinel */
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,9 @@ identified the format of the clipboard data.
|
|||
The ``PIL.Image.core.copy`` and ``PIL.Image.Image.im.copy2`` methods
|
||||
have been removed.
|
||||
|
||||
The ``PIL.Image.core.getcount`` methods have been removed, use
|
||||
``PIL.Image.core.get_stats()['new_count']`` property instead.
|
||||
|
||||
|
||||
API Additions
|
||||
=============
|
||||
|
|
@ -106,4 +109,3 @@ This release contains several performance improvements:
|
|||
friendly algorithm.
|
||||
* ImageFilters based on Kernel convolution are significantly faster
|
||||
due to the new MultibandFilter feature.
|
||||
|
||||
|
|
|
|||
|
|
@ -75,6 +75,11 @@ typedef struct ImagingPaletteInstance* ImagingPalette;
|
|||
|
||||
#define IMAGING_MODE_LENGTH 6+1 /* Band names ("1", "L", "P", "RGB", "RGBA", "CMYK", "YCbCr", "BGR;xy") */
|
||||
|
||||
typedef struct {
|
||||
char *ptr;
|
||||
int size;
|
||||
} ImagingMemoryBlock;
|
||||
|
||||
struct ImagingMemoryInstance {
|
||||
|
||||
/* Format */
|
||||
|
|
@ -95,6 +100,7 @@ struct ImagingMemoryInstance {
|
|||
/* Internals */
|
||||
char **image; /* Actual raster data. */
|
||||
char *block; /* Set if data is allocated in a single block. */
|
||||
ImagingMemoryBlock *blocks; /* Memory blocks for pixel storage */
|
||||
|
||||
int pixelsize; /* Size of a pixel, in bytes (1, 2 or 4) */
|
||||
int linesize; /* Size of a line, in bytes (xsize * pixelsize) */
|
||||
|
|
@ -152,11 +158,26 @@ struct ImagingPaletteInstance {
|
|||
|
||||
};
|
||||
|
||||
typedef struct ImagingMemoryArena {
|
||||
int alignment; /* Alignment in memory of each line of an image */
|
||||
int block_size; /* Preferred block size, bytes */
|
||||
int blocks_max; /* Maximum number of cached blocks */
|
||||
int blocks_cached; /* Current number of blocks not associated with images */
|
||||
ImagingMemoryBlock *blocks_pool;
|
||||
int stats_new_count; /* Number of new allocated images */
|
||||
int stats_allocated_blocks; /* Number of allocated blocks */
|
||||
int stats_reused_blocks; /* Number of blocks which were retrieved from a pool */
|
||||
int stats_reallocated_blocks; /* Number of blocks which were actually reallocated after retrieving */
|
||||
int stats_freed_blocks; /* Number of freed blocks */
|
||||
} *ImagingMemoryArena;
|
||||
|
||||
|
||||
/* Objects */
|
||||
/* ------- */
|
||||
|
||||
extern int ImagingNewCount;
|
||||
extern struct ImagingMemoryArena ImagingDefaultArena;
|
||||
extern int ImagingMemorySetBlocksMax(ImagingMemoryArena arena, int blocks_max);
|
||||
extern void ImagingMemoryClearCache(ImagingMemoryArena arena, int new_size);
|
||||
|
||||
extern Imaging ImagingNew(const char* mode, int xsize, int ysize);
|
||||
extern Imaging ImagingNewDirty(const char* mode, int xsize, int ysize);
|
||||
|
|
|
|||
|
|
@ -173,7 +173,6 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size)
|
|||
im->linesize = xsize * 4;
|
||||
|
||||
} else if (strcmp(mode, "RGBa") == 0) {
|
||||
/* EXPERIMENTAL */
|
||||
/* 32-bit true colour images with premultiplied alpha */
|
||||
im->bands = im->pixelsize = 4;
|
||||
im->linesize = xsize * 4;
|
||||
|
|
@ -230,7 +229,7 @@ ImagingNewPrologueSubtype(const char *mode, int xsize, int ysize, int size)
|
|||
break;
|
||||
}
|
||||
|
||||
ImagingNewCount++;
|
||||
ImagingDefaultArena.stats_new_count += 1;
|
||||
|
||||
return im;
|
||||
}
|
||||
|
|
@ -265,48 +264,186 @@ ImagingDelete(Imaging im)
|
|||
/* ------------------ */
|
||||
/* Allocate image as an array of line buffers. */
|
||||
|
||||
#define IMAGING_PAGE_SIZE (4096)
|
||||
|
||||
struct ImagingMemoryArena ImagingDefaultArena = {
|
||||
1, // alignment
|
||||
16*1024*1024, // block_size
|
||||
0, // blocks_max
|
||||
0, // blocks_cached
|
||||
NULL, // blocks_pool
|
||||
0, 0, 0, 0, 0 // Stats
|
||||
};
|
||||
|
||||
int
|
||||
ImagingMemorySetBlocksMax(ImagingMemoryArena arena, int blocks_max)
|
||||
{
|
||||
void *p;
|
||||
/* Free already cached blocks */
|
||||
ImagingMemoryClearCache(arena, blocks_max);
|
||||
|
||||
if (blocks_max == 0 && arena->blocks_pool != NULL) {
|
||||
free(arena->blocks_pool);
|
||||
arena->blocks_pool = NULL;
|
||||
} else if (arena->blocks_pool != NULL) {
|
||||
p = realloc(arena->blocks_pool, sizeof(*arena->blocks_pool) * blocks_max);
|
||||
if ( ! p) {
|
||||
// Leave previous blocks_max value
|
||||
return 0;
|
||||
}
|
||||
arena->blocks_pool = p;
|
||||
} else {
|
||||
arena->blocks_pool = calloc(sizeof(*arena->blocks_pool), blocks_max);
|
||||
if ( ! arena->blocks_pool) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
arena->blocks_max = blocks_max;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
void
|
||||
ImagingMemoryClearCache(ImagingMemoryArena arena, int new_size)
|
||||
{
|
||||
while (arena->blocks_cached > new_size) {
|
||||
arena->blocks_cached -= 1;
|
||||
free(arena->blocks_pool[arena->blocks_cached].ptr);
|
||||
arena->stats_freed_blocks += 1;
|
||||
}
|
||||
}
|
||||
|
||||
ImagingMemoryBlock
|
||||
memory_get_block(ImagingMemoryArena arena, int requested_size, int dirty)
|
||||
{
|
||||
ImagingMemoryBlock block = {NULL, 0};
|
||||
|
||||
if (arena->blocks_cached > 0) {
|
||||
// Get block from cache
|
||||
arena->blocks_cached -= 1;
|
||||
block = arena->blocks_pool[arena->blocks_cached];
|
||||
// Reallocate if needed
|
||||
if (block.size != requested_size){
|
||||
block.ptr = realloc(block.ptr, requested_size);
|
||||
}
|
||||
if ( ! block.ptr) {
|
||||
// Can't allocate, free prevous pointer (it is still valid)
|
||||
free(arena->blocks_pool[arena->blocks_cached].ptr);
|
||||
arena->stats_freed_blocks += 1;
|
||||
return block;
|
||||
}
|
||||
if ( ! dirty) {
|
||||
memset(block.ptr, 0, requested_size);
|
||||
}
|
||||
arena->stats_reused_blocks += 1;
|
||||
if (block.ptr != arena->blocks_pool[arena->blocks_cached].ptr) {
|
||||
arena->stats_reallocated_blocks += 1;
|
||||
}
|
||||
} else {
|
||||
if (dirty) {
|
||||
block.ptr = malloc(requested_size);
|
||||
} else {
|
||||
block.ptr = calloc(1, requested_size);
|
||||
}
|
||||
arena->stats_allocated_blocks += 1;
|
||||
}
|
||||
block.size = requested_size;
|
||||
return block;
|
||||
}
|
||||
|
||||
void
|
||||
memory_return_block(ImagingMemoryArena arena, ImagingMemoryBlock block)
|
||||
{
|
||||
if (arena->blocks_cached < arena->blocks_max) {
|
||||
// Reduce block size
|
||||
if (block.size > arena->block_size) {
|
||||
block.size = arena->block_size;
|
||||
block.ptr = realloc(block.ptr, arena->block_size);
|
||||
}
|
||||
arena->blocks_pool[arena->blocks_cached] = block;
|
||||
arena->blocks_cached += 1;
|
||||
} else {
|
||||
free(block.ptr);
|
||||
arena->stats_freed_blocks += 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void
|
||||
ImagingDestroyArray(Imaging im)
|
||||
{
|
||||
int y;
|
||||
int y = 0;
|
||||
|
||||
if (im->image)
|
||||
for (y = 0; y < im->ysize; y++)
|
||||
if (im->image[y])
|
||||
free(im->image[y]);
|
||||
if (im->blocks) {
|
||||
while (im->blocks[y].ptr) {
|
||||
memory_return_block(&ImagingDefaultArena, im->blocks[y]);
|
||||
y += 1;
|
||||
}
|
||||
free(im->blocks);
|
||||
}
|
||||
}
|
||||
|
||||
Imaging
|
||||
ImagingAllocateArray(Imaging im, int dirty)
|
||||
ImagingAllocateArray(Imaging im, int dirty, int block_size)
|
||||
{
|
||||
ImagingSectionCookie cookie;
|
||||
int y, line_in_block, current_block;
|
||||
ImagingMemoryArena arena = &ImagingDefaultArena;
|
||||
ImagingMemoryBlock block = {NULL, 0};
|
||||
int aligned_linesize, lines_per_block, blocks_count;
|
||||
char *aligned_ptr = NULL;
|
||||
|
||||
int y;
|
||||
char* p;
|
||||
|
||||
ImagingSectionEnter(&cookie);
|
||||
|
||||
/* Allocate image as an array of lines */
|
||||
for (y = 0; y < im->ysize; y++) {
|
||||
/* malloc check linesize checked in prologue */
|
||||
if (dirty) {
|
||||
p = (char *) malloc(im->linesize);
|
||||
} else {
|
||||
p = (char *) calloc(1, im->linesize);
|
||||
}
|
||||
if (!p) {
|
||||
ImagingDestroyArray(im);
|
||||
break;
|
||||
}
|
||||
im->image[y] = p;
|
||||
/* 0-width or 0-height image. No need to do anything */
|
||||
if ( ! im->linesize || ! im->ysize) {
|
||||
return im;
|
||||
}
|
||||
|
||||
ImagingSectionLeave(&cookie);
|
||||
aligned_linesize = (im->linesize + arena->alignment - 1) & -arena->alignment;
|
||||
lines_per_block = (block_size - (arena->alignment - 1)) / aligned_linesize;
|
||||
if (lines_per_block == 0)
|
||||
lines_per_block = 1;
|
||||
blocks_count = (im->ysize + lines_per_block - 1) / lines_per_block;
|
||||
// printf("NEW size: %dx%d, ls: %d, lpb: %d, blocks: %d\n",
|
||||
// im->xsize, im->ysize, aligned_linesize, lines_per_block, blocks_count);
|
||||
|
||||
if (y != im->ysize) {
|
||||
/* One extra ponter is always NULL */
|
||||
im->blocks = calloc(sizeof(*im->blocks), blocks_count + 1);
|
||||
if ( ! im->blocks) {
|
||||
return (Imaging) ImagingError_MemoryError();
|
||||
}
|
||||
|
||||
/* Allocate image as an array of lines */
|
||||
line_in_block = 0;
|
||||
current_block = 0;
|
||||
for (y = 0; y < im->ysize; y++) {
|
||||
if (line_in_block == 0) {
|
||||
int required;
|
||||
int lines_remaining = lines_per_block;
|
||||
if (lines_remaining > im->ysize - y) {
|
||||
lines_remaining = im->ysize - y;
|
||||
}
|
||||
required = lines_remaining * aligned_linesize + arena->alignment - 1;
|
||||
block = memory_get_block(arena, required, dirty);
|
||||
if ( ! block.ptr) {
|
||||
ImagingDestroyArray(im);
|
||||
return (Imaging) ImagingError_MemoryError();
|
||||
}
|
||||
im->blocks[current_block] = block;
|
||||
/* Bulletproof code from libc _int_memalign */
|
||||
aligned_ptr = (char *)(
|
||||
((unsigned long) (block.ptr + arena->alignment - 1)) &
|
||||
-((signed long) arena->alignment));
|
||||
}
|
||||
|
||||
im->image[y] = aligned_ptr + aligned_linesize * line_in_block;
|
||||
|
||||
line_in_block += 1;
|
||||
if (line_in_block >= lines_per_block) {
|
||||
/* Reset counter and start new block */
|
||||
line_in_block = 0;
|
||||
current_block += 1;
|
||||
}
|
||||
}
|
||||
|
||||
im->destroy = ImagingDestroyArray;
|
||||
|
||||
return im;
|
||||
|
|
@ -325,17 +462,13 @@ ImagingDestroyBlock(Imaging im)
|
|||
}
|
||||
|
||||
Imaging
|
||||
ImagingAllocateBlock(Imaging im, int dirty)
|
||||
ImagingAllocateBlock(Imaging im)
|
||||
{
|
||||
Py_ssize_t y, i;
|
||||
|
||||
/* We shouldn't overflow, since the threshold defined
|
||||
below says that we're only going to allocate max 4M
|
||||
here before going to the array allocator. Check anyway.
|
||||
*/
|
||||
/* overflow check for malloc */
|
||||
if (im->linesize &&
|
||||
im->ysize > INT_MAX / im->linesize) {
|
||||
/* punt if we're going to overflow */
|
||||
return (Imaging) ImagingError_MemoryError();
|
||||
}
|
||||
|
||||
|
|
@ -345,13 +478,8 @@ ImagingAllocateBlock(Imaging im, int dirty)
|
|||
platforms */
|
||||
im->block = (char *) malloc(1);
|
||||
} else {
|
||||
if (dirty) {
|
||||
/* malloc check ok, overflow check above */
|
||||
im->block = (char *) malloc(im->ysize * im->linesize);
|
||||
} else {
|
||||
/* malloc check ok, overflow check above */
|
||||
im->block = (char *) calloc(im->ysize, im->linesize);
|
||||
}
|
||||
/* malloc check ok, overflow check above */
|
||||
im->block = (char *) calloc(im->ysize, im->linesize);
|
||||
}
|
||||
|
||||
if ( ! im->block) {
|
||||
|
|
@ -371,11 +499,6 @@ ImagingAllocateBlock(Imaging im, int dirty)
|
|||
/* --------------------------------------------------------------------
|
||||
* Create a new, internally allocated, image.
|
||||
*/
|
||||
#if defined(IMAGING_SMALL_MODEL)
|
||||
#define THRESHOLD 16384L
|
||||
#else
|
||||
#define THRESHOLD (2048*2048*4L)
|
||||
#endif
|
||||
|
||||
Imaging
|
||||
ImagingNewInternal(const char* mode, int xsize, int ysize, int dirty)
|
||||
|
|
@ -390,15 +513,14 @@ ImagingNewInternal(const char* mode, int xsize, int ysize, int dirty)
|
|||
if ( ! im)
|
||||
return NULL;
|
||||
|
||||
if (im->ysize && im->linesize <= THRESHOLD / im->ysize) {
|
||||
if (ImagingAllocateBlock(im, dirty)) {
|
||||
return im;
|
||||
}
|
||||
/* assume memory error; try allocating in array mode instead */
|
||||
ImagingError_Clear();
|
||||
if (ImagingAllocateArray(im, dirty, ImagingDefaultArena.block_size)) {
|
||||
return im;
|
||||
}
|
||||
|
||||
if (ImagingAllocateArray(im, dirty)) {
|
||||
ImagingError_Clear();
|
||||
|
||||
// Try to allocate the image once more with smallest possible block size
|
||||
if (ImagingAllocateArray(im, dirty, IMAGING_PAGE_SIZE)) {
|
||||
return im;
|
||||
}
|
||||
|
||||
|
|
@ -423,11 +545,15 @@ ImagingNewBlock(const char* mode, int xsize, int ysize)
|
|||
{
|
||||
Imaging im;
|
||||
|
||||
if (xsize < 0 || ysize < 0) {
|
||||
return (Imaging) ImagingError_ValueError("bad image size");
|
||||
}
|
||||
|
||||
im = ImagingNewPrologue(mode, xsize, ysize);
|
||||
if ( ! im)
|
||||
return NULL;
|
||||
|
||||
if (ImagingAllocateBlock(im, 0)) {
|
||||
if (ImagingAllocateBlock(im)) {
|
||||
return im;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user