From b23fc5a958172d9019749e0c3b7c98e01fc042e4 Mon Sep 17 00:00:00 2001 From: Alexander Date: Wed, 13 Sep 2017 20:38:32 +0300 Subject: [PATCH] SIMD Filter. 5x5i_4u8 SSE4 --- libImaging/FilterSIMD_3x3i_4u8.c | 2 +- libImaging/FilterSIMD_5x5i_4u8.c | 135 +++++++++++++++++++++++++++++++ src/libImaging/Filter.c | 7 +- 3 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 libImaging/FilterSIMD_5x5i_4u8.c diff --git a/libImaging/FilterSIMD_3x3i_4u8.c b/libImaging/FilterSIMD_3x3i_4u8.c index 261e1adcc..8e970a31d 100644 --- a/libImaging/FilterSIMD_3x3i_4u8.c +++ b/libImaging/FilterSIMD_3x3i_4u8.c @@ -113,7 +113,7 @@ ImagingFilter3x3i_4u8(Imaging imOut, Imaging im, const INT16* kernel, ss = _mm_add_epi32(ss, _mm_madd_epi16( \ pix1##row, _mm_shuffle_epi32(kernel10, kctl))); \ ss = _mm_add_epi32(ss, _mm_madd_epi16( \ - pix2##row, _mm_shuffle_epi32(kernel20, kctl))); \ + pix2##row, _mm_shuffle_epi32(kernel20, kctl))); __m128i shuffle = _mm_set_epi8(15,11,14,10,13,9,12,8, 7,3,6,2,5,1,4,0); __m128i kernel00 = _mm_set_epi16( diff --git a/libImaging/FilterSIMD_5x5i_4u8.c b/libImaging/FilterSIMD_5x5i_4u8.c new file mode 100644 index 000000000..d1fb60fc5 --- /dev/null +++ b/libImaging/FilterSIMD_5x5i_4u8.c @@ -0,0 +1,135 @@ +void +ImagingFilter5x5i_4u8(Imaging imOut, Imaging im, const INT16* kernel, + INT32 offset) +{ +#define MM_KERNEL_LOAD(row, x) \ + pix0##row = _mm_cvtepi32_ps(_mm_cvtepu8_epi32(*(__m128i*) &in2[x])); \ + pix1##row = _mm_cvtepi32_ps(_mm_cvtepu8_epi32(*(__m128i*) &in1[x])); \ + pix2##row = _mm_cvtepi32_ps(_mm_cvtepu8_epi32(*(__m128i*) &in0[x])); \ + pix3##row = _mm_cvtepi32_ps(_mm_cvtepu8_epi32(*(__m128i*) &in_1[x])); \ + pix4##row = _mm_cvtepi32_ps(_mm_cvtepu8_epi32(*(__m128i*) &in_2[x])); + +#define MM_KERNEL_SUM(row, kindex) \ + ss = _mm_add_ps(ss, _mm_mul_ps(pix0##row, _mm_set1_ps(kernel[0 + kindex]))); \ + ss = _mm_add_ps(ss, _mm_mul_ps(pix1##row, _mm_set1_ps(kernel[5 + kindex]))); \ + ss = _mm_add_ps(ss, _mm_mul_ps(pix2##row, _mm_set1_ps(kernel[10 + kindex]))); \ + ss = _mm_add_ps(ss, _mm_mul_ps(pix3##row, _mm_set1_ps(kernel[15 + kindex]))); \ + ss = _mm_add_ps(ss, _mm_mul_ps(pix4##row, _mm_set1_ps(kernel[20 + kindex]))); + + int x, y; + + offset += 1 << (PRECISION_BITS-1); + + memcpy(imOut->image32[0], im->image32[0], im->linesize); + memcpy(imOut->image32[1], im->image32[1], im->linesize); + for (y = 2; y < im->ysize-2; y++) { + INT32* in_2 = im->image32[y-2]; + INT32* in_1 = im->image32[y-1]; + INT32* in0 = im->image32[y]; + INT32* in1 = im->image32[y+1]; + INT32* in2 = im->image32[y+2]; + INT32* out = imOut->image32[y]; + __m128i shuffle = _mm_set_epi8(15,11,14,10,13,9,12,8, 7,3,6,2,5,1,4,0); + __m128i kernel00 = _mm_set_epi16( + 0, 0, 0, kernel[4], + kernel[3], kernel[2], kernel[1], kernel[0]); + __m128i kernel10 = _mm_set_epi16( + 0, 0, 0, kernel[9], + kernel[8], kernel[7], kernel[6], kernel[5]); + __m128i kernel20 = _mm_set_epi16( + 0, 0, 0, kernel[14], + kernel[13], kernel[12], kernel[11], kernel[10]); + __m128i kernel30 = _mm_set_epi16( + 0, 0, 0, kernel[19], + kernel[18], kernel[17], kernel[16], kernel[15]); + __m128i kernel40 = _mm_set_epi16( + 0, 0, 0, kernel[24], + kernel[23], kernel[22], kernel[21], kernel[20]); + __m128i zero = _mm_setzero_si128(); + __m128i pix00, pix10, pix20, pix30, pix40; + __m128i pix01, pix11, pix21, pix31, pix41; + + + out[0] = in0[0]; + out[1] = in0[1]; + x = 2; + for (; x < im->xsize-2; x++) { + __m128i ss0; + + pix00 = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*) &in2[x-2]), shuffle); + pix10 = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*) &in1[x-2]), shuffle); + pix20 = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*) &in0[x-2]), shuffle); + pix30 = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*) &in_1[x-2]), shuffle); + pix40 = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*) &in_2[x-2]), shuffle); + + pix01 = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*) &in2[x+2]), shuffle); + pix11 = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*) &in1[x+2]), shuffle); + pix21 = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*) &in0[x+2]), shuffle); + pix31 = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*) &in_1[x+2]), shuffle); + pix41 = _mm_shuffle_epi8(_mm_loadu_si128((__m128i*) &in_2[x+2]), shuffle); + + ss0 = _mm_set1_epi32(offset); + + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpacklo_epi8(pix00, zero), + _mm_shuffle_epi32(kernel00, 0x00))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpacklo_epi8(pix10, zero), + _mm_shuffle_epi32(kernel10, 0x00))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpacklo_epi8(pix20, zero), + _mm_shuffle_epi32(kernel20, 0x00))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpacklo_epi8(pix30, zero), + _mm_shuffle_epi32(kernel30, 0x00))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpacklo_epi8(pix40, zero), + _mm_shuffle_epi32(kernel40, 0x00))); + + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpackhi_epi8(pix00, zero), + _mm_shuffle_epi32(kernel00, 0x55))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpackhi_epi8(pix10, zero), + _mm_shuffle_epi32(kernel10, 0x55))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpackhi_epi8(pix20, zero), + _mm_shuffle_epi32(kernel20, 0x55))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpackhi_epi8(pix30, zero), + _mm_shuffle_epi32(kernel30, 0x55))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpackhi_epi8(pix40, zero), + _mm_shuffle_epi32(kernel40, 0x55))); + + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpacklo_epi8(pix01, zero), + _mm_shuffle_epi32(kernel00, 0xaa))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpacklo_epi8(pix11, zero), + _mm_shuffle_epi32(kernel10, 0xaa))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpacklo_epi8(pix21, zero), + _mm_shuffle_epi32(kernel20, 0xaa))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpacklo_epi8(pix31, zero), + _mm_shuffle_epi32(kernel30, 0xaa))); + ss0 = _mm_add_epi32(ss0, _mm_madd_epi16( + _mm_unpacklo_epi8(pix41, zero), + _mm_shuffle_epi32(kernel40, 0xaa))); + + ss0 = _mm_srai_epi32(ss0, PRECISION_BITS); + + ss0 = _mm_packs_epi32(ss0, ss0); + ss0 = _mm_packus_epi16(ss0, ss0); + out[x] = _mm_cvtsi128_si32(ss0); + } + out[x] = in0[x]; + out[x+1] = in0[x+1]; + } + memcpy(imOut->image32[y], im->image32[y], im->linesize); + memcpy(imOut->image32[y+1], im->image32[y+1], im->linesize); + +#undef MM_KERNEL_LOAD +#undef MM_KERNEL_SUM +} diff --git a/src/libImaging/Filter.c b/src/libImaging/Filter.c index 1c5b8b21f..b66436da6 100644 --- a/src/libImaging/Filter.c +++ b/src/libImaging/Filter.c @@ -47,6 +47,7 @@ #include "FilterSIMD_3x3i_4u8.c" #include "FilterSIMD_5x5f_u8.c" #include "FilterSIMD_5x5f_4u8.c" +#include "FilterSIMD_5x5i_4u8.c" Imaging @@ -169,7 +170,11 @@ ImagingFilter(Imaging im, int xsize, int ysize, const FLOAT32* kernel, if (im->image8) { ImagingFilter5x5f_u8(imOut, im, kernel, offset); } else { - ImagingFilter5x5f_4u8(imOut, im, kernel, offset); + if (fast_path) { + ImagingFilter5x5i_4u8(imOut, im, norm_kernel, norm_offset); + } else { + ImagingFilter5x5f_4u8(imOut, im, kernel, offset); + } } } ImagingSectionLeave(&cookie);