Merge 25ae464421 into 09d75436f3

.github/workflows/test.yml

@@ -51,15 +51,19 @@ jobs:
         include:
         - { python-version: "3.11", PYTHONOPTIMIZE: 1, REVERSE: "--reverse" }
         - { python-version: "3.10", PYTHONOPTIMIZE: 2 }
+        # SIMD-accelerated builds for x86
+        - { os: "ubuntu-latest", python-version: "3.9", acceleration: "sse4"}
+        - { os: "ubuntu-latest", python-version: "3.12", acceleration: "avx2"}
         # Free-threaded
         - { os: "ubuntu-latest", python-version: "3.13-dev", disable-gil: true }
         # M1 only available for 3.10+
         - { os: "macos-13", python-version: "3.9" }
+        - { os: "macos-13", python-version: "3.9", acceleration: "avx2"}
         exclude:
         - { os: "macos-latest", python-version: "3.9" }
 
     runs-on: ${{ matrix.os }}
-    name: ${{ matrix.os }} Python ${{ matrix.python-version }} ${{ matrix.disable-gil && 'free-threaded' || '' }}
+    name: ${{ matrix.os }} Python ${{ matrix.python-version }} ${{ matrix.acceleration }} ${{ matrix.disable-gil && 'free-threaded' || '' }}
 
     steps:
     - uses: actions/checkout@v4
@@ -109,7 +113,7 @@ jobs:
         GHA_LIBIMAGEQUANT_CACHE_HIT: ${{ steps.cache-libimagequant.outputs.cache-hit }}
 
     - name: Install macOS dependencies
-      if: startsWith(matrix.os, 'macOS')
+      if: startsWith(matrix.os, 'macos')
       run: |
         .github/workflows/macos-install.sh
       env:
@@ -119,6 +123,11 @@ jobs:
       if: "matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'"
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
 
+    - name: Set compiler options for optimization
+      if: ${{ matrix.acceleration }}
+      run: |
+        echo "CC=cc -m${{ matrix.acceleration }}" >> $GITHUB_ENV
+
     - name: Build
       run: |
         .ci/build.sh

Tests/test_features.py

@@ -35,7 +35,9 @@ def test_version() -> None:
             assert version is None
         else:
             assert function(name) == version
-            if name != "PIL":
+            if name == "acceleration":
+                assert version in ("avx2", "sse4", "sse2", "neon", None)
+            elif name != "PIL":
                 if name == "zlib" and version is not None:
                     version = re.sub(".zlib-ng$", "", version)
                 elif name == "libtiff" and version is not None:

src/PIL/features.py

@@ -129,6 +129,7 @@ features: dict[str, tuple[str, str | bool, str | None]] = {
     "libjpeg_turbo": ("PIL._imaging", "HAVE_LIBJPEGTURBO", "libjpeg_turbo_version"),
     "libimagequant": ("PIL._imaging", "HAVE_LIBIMAGEQUANT", "imagequant_version"),
     "xcb": ("PIL._imaging", "HAVE_XCB", None),
+    "acceleration": ("PIL._imaging", "acceleration", "acceleration"),
 }
 
 
@@ -282,6 +283,7 @@ def pilinfo(out: IO[str] | None = None, supported_formats: bool = True) -> None:
 
     for name, feature in [
         ("pil", "PIL CORE"),
+        ("acceleration", "Acceleration"),
         ("tkinter", "TKINTER"),
         ("freetype2", "FREETYPE2"),
         ("littlecms2", "LITTLECMS2"),
@@ -303,7 +305,7 @@ def pilinfo(out: IO[str] | None = None, supported_formats: bool = True) -> None:
             if v is None:
                 v = version(name)
             if v is not None:
-                version_static = name in ("pil", "jpg")
+                version_static = name in ("pil", "jpg", "acceleration")
                 if name == "littlecms2":
                     # this check is also in src/_imagingcms.c:setup_module()
                     version_static = tuple(int(x) for x in v.split(".")) < (2, 7)

src/_imaging.c

@@ -4425,6 +4425,19 @@ setup_module(PyObject *m) {
     Py_INCREF(have_xcb);
     PyModule_AddObject(m, "HAVE_XCB", have_xcb);
 
+#ifdef __AVX2__
+    PyModule_AddStringConstant(m, "acceleration", "avx2");
+#elif defined(__SSE4__)
+    PyModule_AddStringConstant(m, "acceleration", "sse4");
+#elif defined(__SSE2__)
+    PyModule_AddStringConstant(m, "acceleration", "sse2");
+#elif defined(__NEON__)
+    PyModule_AddStringConstant(m, "acceleration", "neon");
+#else
+    Py_INCREF(Py_False);
+    PyModule_AddObject(m, "acceleration", Py_False);
+#endif
+
     PyObject *pillow_version = PyUnicode_FromString(version);
     PyDict_SetItemString(
         d, "PILLOW_VERSION", pillow_version ? pillow_version : Py_None

src/libImaging/Bands.c

@@ -21,6 +21,9 @@ Imaging
 ImagingGetBand(Imaging imIn, int band) {
     Imaging imOut;
     int x, y;
+#ifdef __SSE4__
+    __m128i shuffle_mask;
+#endif
 
     /* Check arguments */
     if (!imIn || imIn->type != IMAGING_TYPE_UINT8) {
@@ -46,14 +49,41 @@ ImagingGetBand(Imaging imIn, int band) {
         return NULL;
     }
 
+#ifdef __SSE4__
+    shuffle_mask = _mm_set_epi8(
+        -1,
+        -1,
+        -1,
+        -1,
+        -1,
+        -1,
+        -1,
+        -1,
+        -1,
+        -1,
+        -1,
+        -1,
+        12 + band,
+        8 + band,
+        4 + band,
+        0 + band
+    );
+#endif
+
     /* Extract band from image */
     for (y = 0; y < imIn->ysize; y++) {
         UINT8 *in = (UINT8 *)imIn->image[y] + band;
         UINT8 *out = imOut->image8[y];
         x = 0;
         for (; x < imIn->xsize - 3; x += 4) {
+#ifdef __SSE4__
+            __m128i source = _mm_loadu_si128((__m128i *)(in - band));
+            *((UINT32 *)(out + x)) =
+                _mm_cvtsi128_si32(_mm_shuffle_epi8(source, shuffle_mask));
+#else
             UINT32 v = MAKE_UINT32(in[0], in[4], in[8], in[12]);
             memcpy(out + x, &v, sizeof(v));
+#endif
             in += 16;
         }
         for (; x < imIn->xsize; x++) {
@@ -99,10 +129,20 @@ ImagingSplit(Imaging imIn, Imaging bands[4]) {
             UINT8 *out1 = bands[1]->image8[y];
             x = 0;
             for (; x < imIn->xsize - 3; x += 4) {
+#ifdef __SSE4__
+                __m128i source = _mm_loadu_si128((__m128i *)in);
+                source = _mm_shuffle_epi8(
+                    source,
+                    _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0)
+                );
+                *((UINT32 *)(out0 + x)) = _mm_cvtsi128_si32(source);
+                *((UINT32 *)(out1 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 12));
+#else
                 UINT32 v = MAKE_UINT32(in[0], in[4], in[8], in[12]);
                 memcpy(out0 + x, &v, sizeof(v));
                 v = MAKE_UINT32(in[0 + 3], in[4 + 3], in[8 + 3], in[12 + 3]);
                 memcpy(out1 + x, &v, sizeof(v));
+#endif
                 in += 16;
             }
             for (; x < imIn->xsize; x++) {
@@ -119,12 +159,23 @@ ImagingSplit(Imaging imIn, Imaging bands[4]) {
             UINT8 *out2 = bands[2]->image8[y];
             x = 0;
             for (; x < imIn->xsize - 3; x += 4) {
+#ifdef __SSE4__
+                __m128i source = _mm_loadu_si128((__m128i *)in);
+                source = _mm_shuffle_epi8(
+                    source,
+                    _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0)
+                );
+                *((UINT32 *)(out0 + x)) = _mm_cvtsi128_si32(source);
+                *((UINT32 *)(out1 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 4));
+                *((UINT32 *)(out2 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 8));
+#else
                 UINT32 v = MAKE_UINT32(in[0], in[4], in[8], in[12]);
                 memcpy(out0 + x, &v, sizeof(v));
                 v = MAKE_UINT32(in[0 + 1], in[4 + 1], in[8 + 1], in[12 + 1]);
                 memcpy(out1 + x, &v, sizeof(v));
                 v = MAKE_UINT32(in[0 + 2], in[4 + 2], in[8 + 2], in[12 + 2]);
                 memcpy(out2 + x, &v, sizeof(v));
+#endif
                 in += 16;
             }
             for (; x < imIn->xsize; x++) {
@@ -143,6 +194,17 @@ ImagingSplit(Imaging imIn, Imaging bands[4]) {
             UINT8 *out3 = bands[3]->image8[y];
             x = 0;
             for (; x < imIn->xsize - 3; x += 4) {
+#ifdef __SSE4__
+                __m128i source = _mm_loadu_si128((__m128i *)in);
+                source = _mm_shuffle_epi8(
+                    source,
+                    _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0)
+                );
+                *((UINT32 *)(out0 + x)) = _mm_cvtsi128_si32(source);
+                *((UINT32 *)(out1 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 4));
+                *((UINT32 *)(out2 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 8));
+                *((UINT32 *)(out3 + x)) = _mm_cvtsi128_si32(_mm_srli_si128(source, 12));
+#else
                 UINT32 v = MAKE_UINT32(in[0], in[4], in[8], in[12]);
                 memcpy(out0 + x, &v, sizeof(v));
                 v = MAKE_UINT32(in[0 + 1], in[4 + 1], in[8 + 1], in[12 + 1]);
@@ -151,6 +213,7 @@ ImagingSplit(Imaging imIn, Imaging bands[4]) {
                 memcpy(out2 + x, &v, sizeof(v));
                 v = MAKE_UINT32(in[0 + 3], in[4 + 3], in[8 + 3], in[12 + 3]);
                 memcpy(out3 + x, &v, sizeof(v));
+#endif
                 in += 16;
             }
             for (; x < imIn->xsize; x++) {

src/libImaging/BoxBlur.c

@@ -5,6 +5,184 @@
 
 typedef UINT8 pixel[4];
 
+#if defined(__SSE4__)
+
+void static inline ImagingLineBoxBlur32_SSE4(
+    UINT32 *lineOut,
+    UINT32 *lineIn,
+    int lastx,
+    int radius,
+    int edgeA,
+    int edgeB,
+    UINT32 ww,
+    UINT32 fw
+) {
+    int x;
+    __m128i acc;
+    __m128i bulk;
+    __m128i wws = _mm_set1_epi32(ww);
+    __m128i fws = _mm_set1_epi32(fw);
+    __m128i b23 = _mm_set1_epi32(1 << 23);
+    __m128i shiftmask = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 11, 15);
+
+    // #define MOVE_ACC(acc, subtract, add)
+    //     acc[0] += lineIn[add][0] - lineIn[subtract][0];
+
+#define MOVE_ACC(acc, subtract, add)                         \
+    {                                                        \
+        __m128i tmp1 = mm_cvtepu8_epi32(&lineIn[add]);       \
+        __m128i tmp2 = mm_cvtepu8_epi32(&lineIn[subtract]);  \
+        acc = _mm_sub_epi32(_mm_add_epi32(acc, tmp1), tmp2); \
+    }
+
+    // #define ADD_FAR(bulk, acc, left, right)
+    //     bulk[0] = (acc[0] * ww) + (lineIn[left][0] + lineIn[right][0]) * fw;
+
+#define ADD_FAR(bulk, acc, left, right)                                 \
+    {                                                                   \
+        __m128i tmp3 = mm_cvtepu8_epi32(&lineIn[left]);                 \
+        __m128i tmp4 = mm_cvtepu8_epi32(&lineIn[right]);                \
+        __m128i tmp5 = _mm_mullo_epi32(_mm_add_epi32(tmp3, tmp4), fws); \
+        __m128i tmp6 = _mm_mullo_epi32(acc, wws);                       \
+        bulk = _mm_add_epi32(tmp5, tmp6);                               \
+    }
+
+    // #define SAVE(x, bulk)
+    //    lineOut[x][0] = (UINT8)((bulk[0] + (1 << 23)) >> 24);
+
+#define SAVE(x, bulk) \
+    lineOut[x] =      \
+        _mm_cvtsi128_si32(_mm_shuffle_epi8(_mm_add_epi32(bulk, b23), shiftmask))
+
+    /* Compute acc for -1 pixel (outside of image):
+       From "-radius-1" to "-1" get first pixel,
+       then from "0" to "radius-1". */
+    // acc[0] = lineIn[0][0] * (radius + 1);
+    acc = _mm_mullo_epi32(mm_cvtepu8_epi32(&lineIn[0]), _mm_set1_epi32(radius + 1));
+
+    /* As radius can be bigger than xsize, iterate to edgeA -1. */
+    for (x = 0; x < edgeA - 1; x++) {
+        // acc[0] += lineIn[x][0];
+        acc = _mm_add_epi32(acc, mm_cvtepu8_epi32(&lineIn[x]));
+    }
+    /* Then multiply remainder to last x. */
+    // acc[0] += lineIn[lastx][0] * (radius - edgeA + 1);
+    acc = _mm_add_epi32(
+        acc,
+        _mm_mullo_epi32(
+            mm_cvtepu8_epi32(&lineIn[x]), _mm_set1_epi32(radius - edgeA + 1)
+        )
+    );
+
+    if (edgeA <= edgeB) {
+        /* Subtract pixel from left ("0").
+           Add pixels from radius. */
+        for (x = 0; x < edgeA; x++) {
+            MOVE_ACC(acc, 0, x + radius);
+            ADD_FAR(bulk, acc, 0, x + radius + 1);
+            SAVE(x, bulk);
+        }
+        /* Subtract previous pixel from "-radius".
+           Add pixels from radius. */
+        for (x = edgeA; x < edgeB; x++) {
+            MOVE_ACC(acc, x - radius - 1, x + radius);
+            ADD_FAR(bulk, acc, x - radius - 1, x + radius + 1);
+            SAVE(x, bulk);
+        }
+        /* Subtract previous pixel from "-radius".
+           Add last pixel. */
+        for (x = edgeB; x <= lastx; x++) {
+            MOVE_ACC(acc, x - radius - 1, lastx);
+            ADD_FAR(bulk, acc, x - radius - 1, lastx);
+            SAVE(x, bulk);
+        }
+    } else {
+        for (x = 0; x < edgeB; x++) {
+            MOVE_ACC(acc, 0, x + radius);
+            ADD_FAR(bulk, acc, 0, x + radius + 1);
+            SAVE(x, bulk);
+        }
+        for (x = edgeB; x < edgeA; x++) {
+            MOVE_ACC(acc, 0, lastx);
+            ADD_FAR(bulk, acc, 0, lastx);
+            SAVE(x, bulk);
+        }
+        for (x = edgeA; x <= lastx; x++) {
+            MOVE_ACC(acc, x - radius - 1, lastx);
+            ADD_FAR(bulk, acc, x - radius - 1, lastx);
+            SAVE(x, bulk);
+        }
+    }
+
+#undef MOVE_ACC
+#undef ADD_FAR
+#undef SAVE
+}
+
+void static inline ImagingLineBoxBlurZero32_SSE4(
+    UINT32 *lineOut,
+    UINT32 *lineIn,
+    int lastx,
+    int edgeA,
+    int edgeB,
+    UINT32 ww,
+    UINT32 fw
+) {
+    int x;
+    __m128i acc;
+    __m128i bulk;
+    __m128i wws = _mm_set1_epi32(ww);
+    __m128i fws = _mm_set1_epi32(fw);
+    __m128i b23 = _mm_set1_epi32(1 << 23);
+    __m128i shiftmask = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 11, 15);
+
+    // #define ADD_FAR(bulk, acc, left, right)
+    //     bulk[0] = (acc[0] * ww) + (lineIn[left][0] + lineIn[right][0]) * fw;
+
+#define ADD_FAR(bulk, acc, left, right)                                 \
+    {                                                                   \
+        __m128i tmp3 = mm_cvtepu8_epi32(&lineIn[left]);                 \
+        __m128i tmp4 = mm_cvtepu8_epi32(&lineIn[right]);                \
+        __m128i tmp5 = _mm_mullo_epi32(_mm_add_epi32(tmp3, tmp4), fws); \
+        __m128i tmp6 = _mm_mullo_epi32(acc, wws);                       \
+        bulk = _mm_add_epi32(tmp5, tmp6);                               \
+    }
+
+    // #define SAVE(x, bulk)
+    //    lineOut[x][0] = (UINT8)((bulk[0] + (1 << 23)) >> 24);
+
+#define SAVE(x, bulk) \
+    lineOut[x] =      \
+        _mm_cvtsi128_si32(_mm_shuffle_epi8(_mm_add_epi32(bulk, b23), shiftmask))
+
+    /* Subtract pixel from left ("0").
+       Add pixels from radius. */
+    for (x = 0; x < edgeA; x++) {
+        acc = mm_cvtepu8_epi32(&lineIn[0]);
+        ADD_FAR(bulk, acc, 0, x + 1);
+        SAVE(x, bulk);
+    }
+    /* Subtract previous pixel from "-radius".
+       Add pixels from radius. */
+    for (x = edgeA; x < edgeB; x++) {
+        acc = mm_cvtepu8_epi32(&lineIn[x]);
+        ADD_FAR(bulk, acc, x - 1, x + 1);
+        SAVE(x, bulk);
+    }
+    /* Subtract previous pixel from "-radius".
+       Add last pixel. */
+    for (x = edgeB; x <= lastx; x++) {
+        acc = mm_cvtepu8_epi32(&lineIn[lastx]);
+        ADD_FAR(bulk, acc, x - 1, lastx);
+        SAVE(x, bulk);
+    }
+
+#undef ADD_FAR
+#undef SAVE
+}
+
+#endif  // defined(__SSE4__)
+
 void static inline ImagingLineBoxBlur32(
     pixel *lineOut,
     pixel *lineIn,
@@ -168,6 +346,41 @@ void static inline ImagingLineBoxBlur8(
 #undef SAVE
 }
 
+void static inline ImagingLineBoxBlurZero8(
+    UINT8 *lineOut, UINT8 *lineIn, int lastx, int edgeA, int edgeB, UINT32 ww, UINT32 fw
+) {
+    int x;
+    UINT32 acc;
+    UINT32 bulk;
+
+#define MOVE_ACC(acc, subtract, add) acc += lineIn[add] - lineIn[subtract];
+
+#define ADD_FAR(bulk, acc, left, right) \
+    bulk = (acc * ww) + (lineIn[left] + lineIn[right]) * fw;
+
+#define SAVE(x, bulk) lineOut[x] = (UINT8)((bulk + (1 << 23)) >> 24)
+
+    for (x = 0; x < edgeA; x++) {
+        acc = lineIn[0];
+        ADD_FAR(bulk, acc, 0, x + 1);
+        SAVE(x, bulk);
+    }
+    for (x = edgeA; x < edgeB; x++) {
+        acc = lineIn[x];
+        ADD_FAR(bulk, acc, x - 1, x + 1);
+        SAVE(x, bulk);
+    }
+    for (x = edgeB; x <= lastx; x++) {
+        acc = lineIn[lastx];
+        ADD_FAR(bulk, acc, x - 1, lastx);
+        SAVE(x, bulk);
+    }
+
+#undef MOVE_ACC
+#undef ADD_FAR
+#undef SAVE
+}
+
 Imaging
 ImagingHorizontalBoxBlur(Imaging imOut, Imaging imIn, float floatRadius) {
     ImagingSectionCookie cookie;
@@ -186,22 +399,35 @@ ImagingHorizontalBoxBlur(Imaging imOut, Imaging imIn, float floatRadius) {
         return ImagingError_MemoryError();
     }
 
-    // printf(">>> %d %d %d\n", radius, ww, fw);
+    // printf(">>> radius: %d edges: %d %d, weights: %d %d\n",
+    //     radius, edgeA, edgeB, ww, fw);
 
     ImagingSectionEnter(&cookie);
 
     if (imIn->image8) {
         for (y = 0; y < imIn->ysize; y++) {
-            ImagingLineBoxBlur8(
-                (imIn == imOut ? (UINT8 *)lineOut : imOut->image8[y]),
-                imIn->image8[y],
-                imIn->xsize - 1,
-                radius,
-                edgeA,
-                edgeB,
-                ww,
-                fw
-            );
+            if (radius) {
+                ImagingLineBoxBlur8(
+                    (imIn == imOut ? (UINT8 *)lineOut : imOut->image8[y]),
+                    imIn->image8[y],
+                    imIn->xsize - 1,
+                    radius,
+                    edgeA,
+                    edgeB,
+                    ww,
+                    fw
+                );
+            } else {
+                ImagingLineBoxBlurZero8(
+                    (imIn == imOut ? (UINT8 *)lineOut : imOut->image8[y]),
+                    imIn->image8[y],
+                    imIn->xsize - 1,
+                    edgeA,
+                    edgeB,
+                    ww,
+                    fw
+                );
+            }
             if (imIn == imOut) {
                 // Commit.
                 memcpy(imOut->image8[y], lineOut, imIn->xsize);
@@ -209,6 +435,30 @@ ImagingHorizontalBoxBlur(Imaging imOut, Imaging imIn, float floatRadius) {
         }
     } else {
         for (y = 0; y < imIn->ysize; y++) {
+#if defined(__SSE4__)
+            if (radius) {
+                ImagingLineBoxBlur32_SSE4(
+                    imIn == imOut ? (UINT32 *)lineOut : (UINT32 *)imOut->image32[y],
+                    (UINT32 *)imIn->image32[y],
+                    imIn->xsize - 1,
+                    radius,
+                    edgeA,
+                    edgeB,
+                    ww,
+                    fw
+                );
+            } else {
+                ImagingLineBoxBlurZero32_SSE4(
+                    imIn == imOut ? (UINT32 *)lineOut : (UINT32 *)imOut->image32[y],
+                    (UINT32 *)imIn->image32[y],
+                    imIn->xsize - 1,
+                    edgeA,
+                    edgeB,
+                    ww,
+                    fw
+                );
+            }
+#else   // defined(__SSE4__)
             ImagingLineBoxBlur32(
                 imIn == imOut ? (pixel *)lineOut : (pixel *)imOut->image32[y],
                 (pixel *)imIn->image32[y],
@@ -219,6 +469,7 @@ ImagingHorizontalBoxBlur(Imaging imOut, Imaging imIn, float floatRadius) {
                 ww,
                 fw
             );
+#endif  // defined(__SSE4__)
             if (imIn == imOut) {
                 // Commit.
                 memcpy(imOut->image32[y], lineOut, imIn->xsize * 4);

src/libImaging/ImPlatform.h

@@ -97,3 +97,5 @@ typedef signed __int64 int64_t;
 #ifdef __GNUC__
 #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
 #endif
+
+#include "ImagingSIMD.h"

src/libImaging/ImagingSIMD.h

@@ -0,0 +1,49 @@
+/* Microsoft compiler doesn't limit intrinsics for an architecture.
+   This macro is set only on x86 and means SSE2 and above including AVX2. */
+#if defined(_M_X64) || _M_IX86_FP == 2
+#define __SSE2__
+/* However, Microsoft compiler set __AVX2__ if /arch:AVX2 option is set */
+#ifdef __AVX2__
+#define __SSE4_2__
+#endif
+#endif
+
+/* For better readability */
+#ifdef __SSE4_2__
+#define __SSE4__
+#endif
+#ifdef __aarch64__
+#define __NEON__
+#endif
+
+#ifdef __SSE2__
+#include <mmintrin.h>   // MMX
+#include <xmmintrin.h>  // SSE
+#include <emmintrin.h>  // SSE2
+#endif
+#ifdef __SSE4__
+#include <pmmintrin.h>  // SSE3
+#include <tmmintrin.h>  // SSSE3
+#include <smmintrin.h>  // SSE4.1
+#include <nmmintrin.h>  // SSE4.2
+#endif
+#ifdef __AVX2__
+#include <immintrin.h>  // AVX, AVX2
+#endif
+#ifdef __NEON__
+#include <arm_neon.h>  // ARM NEON
+#endif
+
+#ifdef __SSE4__
+static inline __m128i
+mm_cvtepu8_epi32(void *ptr) {
+    return _mm_cvtepu8_epi32(_mm_cvtsi32_si128(*(INT32 *)ptr));
+}
+#endif
+
+#ifdef __AVX2__
+static inline __m256i
+mm256_cvtepu8_epi32(void *ptr) {
+    return _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i *)ptr));
+}
+#endif