diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c33309bbf..6bfd0b7ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,15 +51,19 @@ jobs: include: - { python-version: "3.11", PYTHONOPTIMIZE: 1, REVERSE: "--reverse" } - { python-version: "3.10", PYTHONOPTIMIZE: 2 } + # SIMD-accelerated builds for x86 + - { os: "ubuntu-latest", python-version: "3.9", acceleration: "sse4"} + - { os: "ubuntu-latest", python-version: "3.12", acceleration: "avx2"} # Free-threaded - { os: "ubuntu-latest", python-version: "3.13-dev", disable-gil: true } # M1 only available for 3.10+ - { os: "macos-13", python-version: "3.9" } + - { os: "macos-13", python-version: "3.9", acceleration: "avx2"} exclude: - { os: "macos-latest", python-version: "3.9" } runs-on: ${{ matrix.os }} - name: ${{ matrix.os }} Python ${{ matrix.python-version }} ${{ matrix.disable-gil && 'free-threaded' || '' }} + name: ${{ matrix.os }} Python ${{ matrix.python-version }} ${{ matrix.acceleration }} ${{ matrix.disable-gil && 'free-threaded' || '' }} steps: - uses: actions/checkout@v4 @@ -109,7 +113,7 @@ jobs: GHA_LIBIMAGEQUANT_CACHE_HIT: ${{ steps.cache-libimagequant.outputs.cache-hit }} - name: Install macOS dependencies - if: startsWith(matrix.os, 'macOS') + if: startsWith(matrix.os, 'macos') run: | .github/workflows/macos-install.sh env: @@ -119,6 +123,11 @@ jobs: if: "matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'" run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Set compiler options for optimization + if: ${{ matrix.acceleration }} + run: | + echo "CC=cc -m${{ matrix.acceleration }}" >> $GITHUB_ENV + - name: Build run: | .ci/build.sh diff --git a/Tests/test_features.py b/Tests/test_features.py index ed7929973..b6cf1edb7 100644 --- a/Tests/test_features.py +++ b/Tests/test_features.py @@ -35,7 +35,9 @@ def test_version() -> None: assert version is None else: assert function(name) == version - if name != "PIL": + if name == "acceleration": + assert version in ("avx2", "sse4", "sse2", "neon", None) + elif name != "PIL": if name == "zlib" and version is not None: version = re.sub(".zlib-ng$", "", version) elif name == "libtiff" and version is not None: diff --git a/src/PIL/features.py b/src/PIL/features.py index 75d59e01c..f2cfb8dff 100644 --- a/src/PIL/features.py +++ b/src/PIL/features.py @@ -129,6 +129,7 @@ features: dict[str, tuple[str, str | bool, str | None]] = { "libjpeg_turbo": ("PIL._imaging", "HAVE_LIBJPEGTURBO", "libjpeg_turbo_version"), "libimagequant": ("PIL._imaging", "HAVE_LIBIMAGEQUANT", "imagequant_version"), "xcb": ("PIL._imaging", "HAVE_XCB", None), + "acceleration": ("PIL._imaging", "acceleration", "acceleration"), } @@ -282,6 +283,7 @@ def pilinfo(out: IO[str] | None = None, supported_formats: bool = True) -> None: for name, feature in [ ("pil", "PIL CORE"), + ("acceleration", "Acceleration"), ("tkinter", "TKINTER"), ("freetype2", "FREETYPE2"), ("littlecms2", "LITTLECMS2"), @@ -303,7 +305,7 @@ def pilinfo(out: IO[str] | None = None, supported_formats: bool = True) -> None: if v is None: v = version(name) if v is not None: - version_static = name in ("pil", "jpg") + version_static = name in ("pil", "jpg", "acceleration") if name == "littlecms2": # this check is also in src/_imagingcms.c:setup_module() version_static = tuple(int(x) for x in v.split(".")) < (2, 7) diff --git a/src/_imaging.c b/src/_imaging.c index 2db4486b2..e604bb72e 100644 --- a/src/_imaging.c +++ b/src/_imaging.c @@ -4425,6 +4425,19 @@ setup_module(PyObject *m) { Py_INCREF(have_xcb); PyModule_AddObject(m, "HAVE_XCB", have_xcb); +#ifdef __AVX2__ + PyModule_AddStringConstant(m, "acceleration", "avx2"); +#elif defined(__SSE4__) + PyModule_AddStringConstant(m, "acceleration", "sse4"); +#elif defined(__SSE2__) + PyModule_AddStringConstant(m, "acceleration", "sse2"); +#elif defined(__NEON__) + PyModule_AddStringConstant(m, "acceleration", "neon"); +#else + Py_INCREF(Py_False); + PyModule_AddObject(m, "acceleration", Py_False); +#endif + PyObject *pillow_version = PyUnicode_FromString(version); PyDict_SetItemString( d, "PILLOW_VERSION", pillow_version ? pillow_version : Py_None diff --git a/src/libImaging/Bands.c b/src/libImaging/Bands.c index e1b16b34a..0ab3156ed 100644 --- a/src/libImaging/Bands.c +++ b/src/libImaging/Bands.c @@ -21,6 +21,9 @@ Imaging ImagingGetBand(Imaging imIn, int band) { Imaging imOut; int x, y; +#ifdef __SSE4__ + __m128i shuffle_mask; +#endif /* Check arguments */ if (!imIn || imIn->type != IMAGING_TYPE_UINT8) { @@ -46,14 +49,41 @@ ImagingGetBand(Imaging imIn, int band) { return NULL; } +#ifdef __SSE4__ + shuffle_mask = _mm_set_epi8( + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + 12 + band, + 8 + band, + 4 + band, + 0 + band + ); +#endif + /* Extract band from image */ for (y = 0; y < imIn->ysize; y++) { UINT8 *in = (UINT8 *)imIn->image[y] + band; UINT8 *out = imOut->image8[y]; x = 0; for (; x < imIn->xsize - 3; x += 4) { +#ifdef __SSE4__ + __m128i source = _mm_loadu_si128((__m128i *)(in - band)); + *((UINT32 *)(out + x)) = + _mm_cvtsi128_si32(_mm_shuffle_epi8(source, shuffle_mask)); +#else UINT32 v = MAKE_UINT32(in[0], in[4], in[8], in[12]); memcpy(out + x, &v, sizeof(v)); +#endif in += 16; } for (; x < imIn->xsize; x++) { @@ -99,10 +129,20 @@ ImagingSplit(Imaging imIn, Imaging bands[4]) { UINT8 *out1 = bands[1]->image8[y]; x = 0; for (; x < imIn->xsize - 3; x += 4) { +#ifdef __SSE4__ + __m128i source = _mm_loadu_si128((__m128i *)in); + source = _mm_shuffle_epi8( + source, + _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0) + ); + *((UINT32 *)(out0 + x)) = _mm_cvtsi128_si32(source); + *((UINT32 *)(out1 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 12)); +#else UINT32 v = MAKE_UINT32(in[0], in[4], in[8], in[12]); memcpy(out0 + x, &v, sizeof(v)); v = MAKE_UINT32(in[0 + 3], in[4 + 3], in[8 + 3], in[12 + 3]); memcpy(out1 + x, &v, sizeof(v)); +#endif in += 16; } for (; x < imIn->xsize; x++) { @@ -119,12 +159,23 @@ ImagingSplit(Imaging imIn, Imaging bands[4]) { UINT8 *out2 = bands[2]->image8[y]; x = 0; for (; x < imIn->xsize - 3; x += 4) { +#ifdef __SSE4__ + __m128i source = _mm_loadu_si128((__m128i *)in); + source = _mm_shuffle_epi8( + source, + _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0) + ); + *((UINT32 *)(out0 + x)) = _mm_cvtsi128_si32(source); + *((UINT32 *)(out1 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 4)); + *((UINT32 *)(out2 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 8)); +#else UINT32 v = MAKE_UINT32(in[0], in[4], in[8], in[12]); memcpy(out0 + x, &v, sizeof(v)); v = MAKE_UINT32(in[0 + 1], in[4 + 1], in[8 + 1], in[12 + 1]); memcpy(out1 + x, &v, sizeof(v)); v = MAKE_UINT32(in[0 + 2], in[4 + 2], in[8 + 2], in[12 + 2]); memcpy(out2 + x, &v, sizeof(v)); +#endif in += 16; } for (; x < imIn->xsize; x++) { @@ -143,6 +194,17 @@ ImagingSplit(Imaging imIn, Imaging bands[4]) { UINT8 *out3 = bands[3]->image8[y]; x = 0; for (; x < imIn->xsize - 3; x += 4) { +#ifdef __SSE4__ + __m128i source = _mm_loadu_si128((__m128i *)in); + source = _mm_shuffle_epi8( + source, + _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0) + ); + *((UINT32 *)(out0 + x)) = _mm_cvtsi128_si32(source); + *((UINT32 *)(out1 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 4)); + *((UINT32 *)(out2 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 8)); + *((UINT32 *)(out3 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 12)); +#else UINT32 v = MAKE_UINT32(in[0], in[4], in[8], in[12]); memcpy(out0 + x, &v, sizeof(v)); v = MAKE_UINT32(in[0 + 1], in[4 + 1], in[8 + 1], in[12 + 1]); @@ -151,6 +213,7 @@ ImagingSplit(Imaging imIn, Imaging bands[4]) { memcpy(out2 + x, &v, sizeof(v)); v = MAKE_UINT32(in[0 + 3], in[4 + 3], in[8 + 3], in[12 + 3]); memcpy(out3 + x, &v, sizeof(v)); +#endif in += 16; } for (; x < imIn->xsize; x++) { diff --git a/src/libImaging/BoxBlur.c b/src/libImaging/BoxBlur.c index ed91541fe..956b59f16 100644 --- a/src/libImaging/BoxBlur.c +++ b/src/libImaging/BoxBlur.c @@ -5,6 +5,184 @@ typedef UINT8 pixel[4]; +#if defined(__SSE4__) + +void static inline ImagingLineBoxBlur32_SSE4( + UINT32 *lineOut, + UINT32 *lineIn, + int lastx, + int radius, + int edgeA, + int edgeB, + UINT32 ww, + UINT32 fw +) { + int x; + __m128i acc; + __m128i bulk; + __m128i wws = _mm_set1_epi32(ww); + __m128i fws = _mm_set1_epi32(fw); + __m128i b23 = _mm_set1_epi32(1 << 23); + __m128i shiftmask = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 11, 15); + + // #define MOVE_ACC(acc, subtract, add) + // acc[0] += lineIn[add][0] - lineIn[subtract][0]; + +#define MOVE_ACC(acc, subtract, add) \ + { \ + __m128i tmp1 = mm_cvtepu8_epi32(&lineIn[add]); \ + __m128i tmp2 = mm_cvtepu8_epi32(&lineIn[subtract]); \ + acc = _mm_sub_epi32(_mm_add_epi32(acc, tmp1), tmp2); \ + } + + // #define ADD_FAR(bulk, acc, left, right) + // bulk[0] = (acc[0] * ww) + (lineIn[left][0] + lineIn[right][0]) * fw; + +#define ADD_FAR(bulk, acc, left, right) \ + { \ + __m128i tmp3 = mm_cvtepu8_epi32(&lineIn[left]); \ + __m128i tmp4 = mm_cvtepu8_epi32(&lineIn[right]); \ + __m128i tmp5 = _mm_mullo_epi32(_mm_add_epi32(tmp3, tmp4), fws); \ + __m128i tmp6 = _mm_mullo_epi32(acc, wws); \ + bulk = _mm_add_epi32(tmp5, tmp6); \ + } + + // #define SAVE(x, bulk) + // lineOut[x][0] = (UINT8)((bulk[0] + (1 << 23)) >> 24); + +#define SAVE(x, bulk) \ + lineOut[x] = \ + _mm_cvtsi128_si32(_mm_shuffle_epi8(_mm_add_epi32(bulk, b23), shiftmask)) + + /* Compute acc for -1 pixel (outside of image): + From "-radius-1" to "-1" get first pixel, + then from "0" to "radius-1". */ + // acc[0] = lineIn[0][0] * (radius + 1); + acc = _mm_mullo_epi32(mm_cvtepu8_epi32(&lineIn[0]), _mm_set1_epi32(radius + 1)); + + /* As radius can be bigger than xsize, iterate to edgeA -1. */ + for (x = 0; x < edgeA - 1; x++) { + // acc[0] += lineIn[x][0]; + acc = _mm_add_epi32(acc, mm_cvtepu8_epi32(&lineIn[x])); + } + /* Then multiply remainder to last x. */ + // acc[0] += lineIn[lastx][0] * (radius - edgeA + 1); + acc = _mm_add_epi32( + acc, + _mm_mullo_epi32( + mm_cvtepu8_epi32(&lineIn[x]), _mm_set1_epi32(radius - edgeA + 1) + ) + ); + + if (edgeA <= edgeB) { + /* Subtract pixel from left ("0"). + Add pixels from radius. */ + for (x = 0; x < edgeA; x++) { + MOVE_ACC(acc, 0, x + radius); + ADD_FAR(bulk, acc, 0, x + radius + 1); + SAVE(x, bulk); + } + /* Subtract previous pixel from "-radius". + Add pixels from radius. */ + for (x = edgeA; x < edgeB; x++) { + MOVE_ACC(acc, x - radius - 1, x + radius); + ADD_FAR(bulk, acc, x - radius - 1, x + radius + 1); + SAVE(x, bulk); + } + /* Subtract previous pixel from "-radius". + Add last pixel. */ + for (x = edgeB; x <= lastx; x++) { + MOVE_ACC(acc, x - radius - 1, lastx); + ADD_FAR(bulk, acc, x - radius - 1, lastx); + SAVE(x, bulk); + } + } else { + for (x = 0; x < edgeB; x++) { + MOVE_ACC(acc, 0, x + radius); + ADD_FAR(bulk, acc, 0, x + radius + 1); + SAVE(x, bulk); + } + for (x = edgeB; x < edgeA; x++) { + MOVE_ACC(acc, 0, lastx); + ADD_FAR(bulk, acc, 0, lastx); + SAVE(x, bulk); + } + for (x = edgeA; x <= lastx; x++) { + MOVE_ACC(acc, x - radius - 1, lastx); + ADD_FAR(bulk, acc, x - radius - 1, lastx); + SAVE(x, bulk); + } + } + +#undef MOVE_ACC +#undef ADD_FAR +#undef SAVE +} + +void static inline ImagingLineBoxBlurZero32_SSE4( + UINT32 *lineOut, + UINT32 *lineIn, + int lastx, + int edgeA, + int edgeB, + UINT32 ww, + UINT32 fw +) { + int x; + __m128i acc; + __m128i bulk; + __m128i wws = _mm_set1_epi32(ww); + __m128i fws = _mm_set1_epi32(fw); + __m128i b23 = _mm_set1_epi32(1 << 23); + __m128i shiftmask = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 11, 15); + + // #define ADD_FAR(bulk, acc, left, right) + // bulk[0] = (acc[0] * ww) + (lineIn[left][0] + lineIn[right][0]) * fw; + +#define ADD_FAR(bulk, acc, left, right) \ + { \ + __m128i tmp3 = mm_cvtepu8_epi32(&lineIn[left]); \ + __m128i tmp4 = mm_cvtepu8_epi32(&lineIn[right]); \ + __m128i tmp5 = _mm_mullo_epi32(_mm_add_epi32(tmp3, tmp4), fws); \ + __m128i tmp6 = _mm_mullo_epi32(acc, wws); \ + bulk = _mm_add_epi32(tmp5, tmp6); \ + } + + // #define SAVE(x, bulk) + // lineOut[x][0] = (UINT8)((bulk[0] + (1 << 23)) >> 24); + +#define SAVE(x, bulk) \ + lineOut[x] = \ + _mm_cvtsi128_si32(_mm_shuffle_epi8(_mm_add_epi32(bulk, b23), shiftmask)) + + /* Subtract pixel from left ("0"). + Add pixels from radius. */ + for (x = 0; x < edgeA; x++) { + acc = mm_cvtepu8_epi32(&lineIn[0]); + ADD_FAR(bulk, acc, 0, x + 1); + SAVE(x, bulk); + } + /* Subtract previous pixel from "-radius". + Add pixels from radius. */ + for (x = edgeA; x < edgeB; x++) { + acc = mm_cvtepu8_epi32(&lineIn[x]); + ADD_FAR(bulk, acc, x - 1, x + 1); + SAVE(x, bulk); + } + /* Subtract previous pixel from "-radius". + Add last pixel. */ + for (x = edgeB; x <= lastx; x++) { + acc = mm_cvtepu8_epi32(&lineIn[lastx]); + ADD_FAR(bulk, acc, x - 1, lastx); + SAVE(x, bulk); + } + +#undef ADD_FAR +#undef SAVE +} + +#endif // defined(__SSE4__) + void static inline ImagingLineBoxBlur32( pixel *lineOut, pixel *lineIn, @@ -168,6 +346,41 @@ void static inline ImagingLineBoxBlur8( #undef SAVE } +void static inline ImagingLineBoxBlurZero8( + UINT8 *lineOut, UINT8 *lineIn, int lastx, int edgeA, int edgeB, UINT32 ww, UINT32 fw +) { + int x; + UINT32 acc; + UINT32 bulk; + +#define MOVE_ACC(acc, subtract, add) acc += lineIn[add] - lineIn[subtract]; + +#define ADD_FAR(bulk, acc, left, right) \ + bulk = (acc * ww) + (lineIn[left] + lineIn[right]) * fw; + +#define SAVE(x, bulk) lineOut[x] = (UINT8)((bulk + (1 << 23)) >> 24) + + for (x = 0; x < edgeA; x++) { + acc = lineIn[0]; + ADD_FAR(bulk, acc, 0, x + 1); + SAVE(x, bulk); + } + for (x = edgeA; x < edgeB; x++) { + acc = lineIn[x]; + ADD_FAR(bulk, acc, x - 1, x + 1); + SAVE(x, bulk); + } + for (x = edgeB; x <= lastx; x++) { + acc = lineIn[lastx]; + ADD_FAR(bulk, acc, x - 1, lastx); + SAVE(x, bulk); + } + +#undef MOVE_ACC +#undef ADD_FAR +#undef SAVE +} + Imaging ImagingHorizontalBoxBlur(Imaging imOut, Imaging imIn, float floatRadius) { ImagingSectionCookie cookie; @@ -186,22 +399,35 @@ ImagingHorizontalBoxBlur(Imaging imOut, Imaging imIn, float floatRadius) { return ImagingError_MemoryError(); } - // printf(">>> %d %d %d\n", radius, ww, fw); + // printf(">>> radius: %d edges: %d %d, weights: %d %d\n", + // radius, edgeA, edgeB, ww, fw); ImagingSectionEnter(&cookie); if (imIn->image8) { for (y = 0; y < imIn->ysize; y++) { - ImagingLineBoxBlur8( - (imIn == imOut ? (UINT8 *)lineOut : imOut->image8[y]), - imIn->image8[y], - imIn->xsize - 1, - radius, - edgeA, - edgeB, - ww, - fw - ); + if (radius) { + ImagingLineBoxBlur8( + (imIn == imOut ? (UINT8 *)lineOut : imOut->image8[y]), + imIn->image8[y], + imIn->xsize - 1, + radius, + edgeA, + edgeB, + ww, + fw + ); + } else { + ImagingLineBoxBlurZero8( + (imIn == imOut ? (UINT8 *)lineOut : imOut->image8[y]), + imIn->image8[y], + imIn->xsize - 1, + edgeA, + edgeB, + ww, + fw + ); + } if (imIn == imOut) { // Commit. memcpy(imOut->image8[y], lineOut, imIn->xsize); @@ -209,6 +435,30 @@ ImagingHorizontalBoxBlur(Imaging imOut, Imaging imIn, float floatRadius) { } } else { for (y = 0; y < imIn->ysize; y++) { +#if defined(__SSE4__) + if (radius) { + ImagingLineBoxBlur32_SSE4( + imIn == imOut ? (UINT32 *)lineOut : (UINT32 *)imOut->image32[y], + (UINT32 *)imIn->image32[y], + imIn->xsize - 1, + radius, + edgeA, + edgeB, + ww, + fw + ); + } else { + ImagingLineBoxBlurZero32_SSE4( + imIn == imOut ? (UINT32 *)lineOut : (UINT32 *)imOut->image32[y], + (UINT32 *)imIn->image32[y], + imIn->xsize - 1, + edgeA, + edgeB, + ww, + fw + ); + } +#else // defined(__SSE4__) ImagingLineBoxBlur32( imIn == imOut ? (pixel *)lineOut : (pixel *)imOut->image32[y], (pixel *)imIn->image32[y], @@ -219,6 +469,7 @@ ImagingHorizontalBoxBlur(Imaging imOut, Imaging imIn, float floatRadius) { ww, fw ); +#endif // defined(__SSE4__) if (imIn == imOut) { // Commit. memcpy(imOut->image32[y], lineOut, imIn->xsize * 4); diff --git a/src/libImaging/ImPlatform.h b/src/libImaging/ImPlatform.h index c9b7e43b4..d5abe6627 100644 --- a/src/libImaging/ImPlatform.h +++ b/src/libImaging/ImPlatform.h @@ -97,3 +97,5 @@ typedef signed __int64 int64_t; #ifdef __GNUC__ #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) #endif + +#include "ImagingSIMD.h" diff --git a/src/libImaging/ImagingSIMD.h b/src/libImaging/ImagingSIMD.h new file mode 100644 index 000000000..fbf612868 --- /dev/null +++ b/src/libImaging/ImagingSIMD.h @@ -0,0 +1,49 @@ +/* Microsoft compiler doesn't limit intrinsics for an architecture. + This macro is set only on x86 and means SSE2 and above including AVX2. */ +#if defined(_M_X64) || _M_IX86_FP == 2 +#define __SSE2__ +/* However, Microsoft compiler set __AVX2__ if /arch:AVX2 option is set */ +#ifdef __AVX2__ +#define __SSE4_2__ +#endif +#endif + +/* For better readability */ +#ifdef __SSE4_2__ +#define __SSE4__ +#endif +#ifdef __aarch64__ +#define __NEON__ +#endif + +#ifdef __SSE2__ +#include // MMX +#include // SSE +#include // SSE2 +#endif +#ifdef __SSE4__ +#include // SSE3 +#include // SSSE3 +#include // SSE4.1 +#include // SSE4.2 +#endif +#ifdef __AVX2__ +#include // AVX, AVX2 +#endif +#ifdef __NEON__ +#include // ARM NEON +#endif + +#ifdef __SSE4__ +static inline __m128i +mm_cvtepu8_epi32(void *ptr) { + return _mm_cvtepu8_epi32(_mm_cvtsi32_si128(*(INT32 *)ptr)); +} +#endif + +#ifdef __AVX2__ +static inline __m256i +mm256_cvtepu8_epi32(void *ptr) { + return _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i *)ptr)); +} +#endif