/* * Copyright (C) 2011 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include /* Unsigned extend packed 8-bit integer (in LBS) into packed 32-bit integer */ static inline __m128i cvtepu8_epi32(__m128i x) { #if defined(__SSE4_1__) return _mm_cvtepu8_epi32(x); #elif defined(__SSSE3__) const __m128i M8to32 = _mm_set_epi32(0xffffff03, 0xffffff02, 0xffffff01, 0xffffff00); x = _mm_shuffle_epi8(x, M8to32); return x; #else # error "Require at least SSSE3" #endif } static inline __m128i packus_epi32(__m128i lo, __m128i hi) { #if defined(__SSE4_1__) return _mm_packus_epi32(lo, hi); #elif defined(__SSSE3__) const __m128i C0 = _mm_set_epi32(0x0000, 0x0000, 0x0000, 0x0000); const __m128i C1 = _mm_set_epi32(0xffff, 0xffff, 0xffff, 0xffff); const __m128i M32to16L = _mm_set_epi32(0xffffffff, 0xffffffff, 0x0d0c0908, 0x05040100); const __m128i M32to16H = _mm_set_epi32(0x0d0c0908, 0x05040100, 0xffffffff, 0xffffffff); lo = _mm_and_si128(lo, _mm_cmpgt_epi32(lo, C0)); lo = _mm_or_si128(lo, _mm_cmpgt_epi32(lo, C1)); hi = _mm_and_si128(hi, _mm_cmpgt_epi32(hi, C0)); hi = _mm_or_si128(hi, _mm_cmpgt_epi32(hi, C1)); return _mm_or_si128(_mm_shuffle_epi8(lo, M32to16L), _mm_shuffle_epi8(hi, M32to16H)); #else # error "Require at least SSSE3" #endif } static inline __m128i mullo_epi32(__m128i x, __m128i y) { #if defined(__SSE4_1__) return _mm_mullo_epi32(x, y); #elif defined(__SSSE3__) const __m128i Meven = _mm_set_epi32(0x00000000, 0xffffffff, 0x00000000, 0xffffffff); __m128i even = _mm_mul_epu32(x, y); __m128i odd = _mm_mul_epu32(_mm_srli_si128(x, 4), _mm_srli_si128(y, 4)); even = _mm_and_si128(even, Meven); odd = _mm_and_si128(odd, Meven); return _mm_or_si128(even, _mm_slli_si128(odd, 4)); #else # error "Require at least SSSE3" #endif } /* 'mask' must packed 8-bit of 0x00 or 0xff */ static inline __m128i blendv_epi8(__m128i x, __m128i y, __m128i mask) { #if defined(__SSE4_1__) return _mm_blendv_epi8(x, y, mask); #elif defined(__SSSE3__) return _mm_or_si128(_mm_andnot_si128(mask, x), _mm_and_si128(y, mask)); #else # error "Require at least SSSE3" #endif } extern "C" void rsdIntrinsicConvolve3x3_K(void *dst, const void *y0, const void *y1, const void *y2, const short *coef, uint32_t count) { __m128i x; __m128i c0, c2, c4, c6, c8; __m128i r0, r1, r2; __m128i p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11; __m128i o0, o1; uint32_t i; x = _mm_loadl_epi64((const __m128i *)(coef+0)); c0 = _mm_shuffle_epi32(x, 0x00); c2 = _mm_shuffle_epi32(x, 0x55); x = _mm_loadl_epi64((const __m128i *)(coef+4)); c4 = _mm_shuffle_epi32(x, 0x00); c6 = _mm_shuffle_epi32(x, 0x55); x = _mm_loadl_epi64((const __m128i *)(coef+8)); c8 = _mm_shuffle_epi32(x, 0x00); for (i = 0; i < count; ++i) { p0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0)), _mm_setzero_si128()); p1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0+1)), _mm_setzero_si128()); p2 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0+2)), _mm_setzero_si128()); p3 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0+3)), _mm_setzero_si128()); p4 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1)), _mm_setzero_si128()); p5 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1+1)), _mm_setzero_si128()); p6 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1+2)), _mm_setzero_si128()); p7 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1+3)), _mm_setzero_si128()); p8 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2)), _mm_setzero_si128()); p9 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2+1)), _mm_setzero_si128()); p10 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2+2)), _mm_setzero_si128()); p11 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2+3)), _mm_setzero_si128()); o0 = _mm_madd_epi16(_mm_unpacklo_epi16(p0, p1), c0); o1 = _mm_madd_epi16(_mm_unpacklo_epi16(p1, p2), c0); o0 = _mm_add_epi32(o0, _mm_madd_epi16(_mm_unpacklo_epi16(p2, p4), c2)); o1 = _mm_add_epi32(o1, _mm_madd_epi16(_mm_unpacklo_epi16(p3, p5), c2)); o0 = _mm_add_epi32(o0, _mm_madd_epi16(_mm_unpacklo_epi16(p5, p6), c4)); o1 = _mm_add_epi32(o1, _mm_madd_epi16(_mm_unpacklo_epi16(p6, p7), c4)); o0 = _mm_add_epi32(o0, _mm_madd_epi16(_mm_unpacklo_epi16(p8, p9), c6)); o1 = _mm_add_epi32(o1, _mm_madd_epi16(_mm_unpacklo_epi16(p9, p10), c6)); o0 = _mm_add_epi32(o0, _mm_madd_epi16(_mm_unpacklo_epi16(p10, _mm_setzero_si128()), c8)); o1 = _mm_add_epi32(o1, _mm_madd_epi16(_mm_unpacklo_epi16(p11, _mm_setzero_si128()), c8)); o0 = _mm_srai_epi32(o0, 8); o1 = _mm_srai_epi32(o1, 8); o0 = packus_epi32(o0, o1); o0 = _mm_packus_epi16(o0, o0); _mm_storel_epi64((__m128i *)dst, o0); y0 = (const char *)y0 + 8; y1 = (const char *)y1 + 8; y2 = (const char *)y2 + 8; dst = (char *)dst + 8; } } void rsdIntrinsicColorMatrix4x4_K(void *dst, const void *src, const short *coef, uint32_t count) { const __m128i T4x4 = _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0); const __m128i Mxy = _mm_set_epi32(0xff0dff0c, 0xff09ff08, 0xff05ff04, 0xff01ff00); const __m128i Mzw = _mm_set_epi32(0xff0fff0e, 0xff0bff0a, 0xff07ff06, 0xff03ff02); __m128i c0, c1, c2, c3; __m128i i4, o4; __m128i xy, zw; __m128i x2, y2, z2, w2; uint32_t i; c0 = _mm_loadl_epi64((const __m128i *)(coef+0)); c1 = _mm_loadl_epi64((const __m128i *)(coef+4)); c0 = _mm_unpacklo_epi16(c0, c1); c2 = _mm_loadl_epi64((const __m128i *)(coef+8)); c3 = _mm_loadl_epi64((const __m128i *)(coef+12)); c2 = _mm_unpacklo_epi16(c2, c3); for (i = 0; i < count; ++i) { i4 = _mm_load_si128((const __m128i *)src); xy = _mm_shuffle_epi8(i4, Mxy); zw = _mm_shuffle_epi8(i4, Mzw); x2 = _mm_madd_epi16(xy, _mm_shuffle_epi32(c0, 0x00)); y2 = _mm_madd_epi16(xy, _mm_shuffle_epi32(c0, 0x55)); z2 = _mm_madd_epi16(xy, _mm_shuffle_epi32(c0, 0xaa)); w2 = _mm_madd_epi16(xy, _mm_shuffle_epi32(c0, 0xff)); x2 = _mm_add_epi32(x2, _mm_madd_epi16(zw, _mm_shuffle_epi32(c2, 0x00))); y2 = _mm_add_epi32(y2, _mm_madd_epi16(zw, _mm_shuffle_epi32(c2, 0x55))); z2 = _mm_add_epi32(z2, _mm_madd_epi16(zw, _mm_shuffle_epi32(c2, 0xaa))); w2 = _mm_add_epi32(w2, _mm_madd_epi16(zw, _mm_shuffle_epi32(c2, 0xff))); x2 = _mm_srai_epi32(x2, 8); y2 = _mm_srai_epi32(y2, 8); z2 = _mm_srai_epi32(z2, 8); w2 = _mm_srai_epi32(w2, 8); x2 = packus_epi32(x2, y2); z2 = packus_epi32(z2, w2); o4 = _mm_packus_epi16(x2, z2); o4 = _mm_shuffle_epi8(o4, T4x4); _mm_storeu_si128((__m128i *)dst, o4); src = (const char *)src + 16; dst = (char *)dst + 16; } } void rsdIntrinsicColorMatrix3x3_K(void *dst, const void *src, const short *coef, uint32_t count) { const __m128i T4x4 = _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0); const __m128i Mxy = _mm_set_epi32(0xff0dff0c, 0xff09ff08, 0xff05ff04, 0xff01ff00); const __m128i Mzw = _mm_set_epi32(0xff0fff0e, 0xff0bff0a, 0xff07ff06, 0xff03ff02); __m128i c0, c1, c2, c3; __m128i i4, o4; __m128i xy, zw; __m128i x2, y2, z2, w2; uint32_t i; c0 = _mm_loadl_epi64((const __m128i *)(coef+0)); c1 = _mm_loadl_epi64((const __m128i *)(coef+4)); c0 = _mm_unpacklo_epi16(c0, c1); c2 = _mm_loadl_epi64((const __m128i *)(coef+8)); c3 = _mm_loadl_epi64((const __m128i *)(coef+12)); c2 = _mm_unpacklo_epi16(c2, c3); for (i = 0; i < count; ++i) { i4 = _mm_loadu_si128((const __m128i *)src); xy = _mm_shuffle_epi8(i4, Mxy); zw = _mm_shuffle_epi8(i4, Mzw); x2 = _mm_madd_epi16(xy, _mm_shuffle_epi32(c0, 0x00)); y2 = _mm_madd_epi16(xy, _mm_shuffle_epi32(c0, 0x55)); z2 = _mm_madd_epi16(xy, _mm_shuffle_epi32(c0, 0xaa)); x2 = _mm_add_epi32(x2, _mm_madd_epi16(zw, _mm_shuffle_epi32(c2, 0x00))); y2 = _mm_add_epi32(y2, _mm_madd_epi16(zw, _mm_shuffle_epi32(c2, 0x55))); z2 = _mm_add_epi32(z2, _mm_madd_epi16(zw, _mm_shuffle_epi32(c2, 0xaa))); x2 = _mm_srai_epi32(x2, 8); y2 = _mm_srai_epi32(y2, 8); z2 = _mm_srai_epi32(z2, 8); w2 = _mm_srli_epi32(zw, 16); x2 = packus_epi32(x2, y2); z2 = packus_epi32(z2, w2); o4 = _mm_packus_epi16(x2, z2); o4 = _mm_shuffle_epi8(o4, T4x4); _mm_storeu_si128((__m128i *)dst, o4); src = (const char *)src + 16; dst = (char *)dst + 16; } } void rsdIntrinsicColorMatrixDot_K(void *dst, const void *src, const short *coef, uint32_t count) { const __m128i T4x4 = _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0); const __m128i Mxy = _mm_set_epi32(0xff0dff0c, 0xff09ff08, 0xff05ff04, 0xff01ff00); const __m128i Mzw = _mm_set_epi32(0xff0fff0e, 0xff0bff0a, 0xff07ff06, 0xff03ff02); __m128i c0, c1, c2, c3; __m128i i4, o4; __m128i xy, zw; __m128i x2, y2, z2, w2; uint32_t i; c0 = _mm_loadl_epi64((const __m128i *)(coef+0)); c0 = _mm_shufflelo_epi16(c0, 0); c1 = _mm_loadl_epi64((const __m128i *)(coef+4)); c1 = _mm_shufflelo_epi16(c1, 0); c0 = _mm_unpacklo_epi16(c0, c1); c2 = _mm_loadl_epi64((const __m128i *)(coef+8)); c2 = _mm_shufflelo_epi16(c2, 0); c3 = _mm_loadl_epi64((const __m128i *)(coef+12)); c3 = _mm_shufflelo_epi16(c3, 0); c2 = _mm_unpacklo_epi16(c2, c3); for (i = 0; i < count; ++i) { i4 = _mm_loadu_si128((const __m128i *)src); xy = _mm_shuffle_epi8(i4, Mxy); zw = _mm_shuffle_epi8(i4, Mzw); x2 = _mm_madd_epi16(xy, c0); x2 = _mm_add_epi32(x2, _mm_madd_epi16(zw, c2)); x2 = _mm_srai_epi32(x2, 8); y2 = x2; z2 = x2; w2 = _mm_srli_epi32(zw, 16); x2 = packus_epi32(x2, y2); z2 = packus_epi32(z2, w2); o4 = _mm_packus_epi16(x2, z2); o4 = _mm_shuffle_epi8(o4, T4x4); _mm_storeu_si128((__m128i *)dst, o4); src = (const char *)src + 16; dst = (char *)dst + 16; } } void rsdIntrinsicBlurVFU4_K(void *dst, const void *pin, int stride, const void *gptr, int rct, int x1, int x2) { const char *pi; __m128i pi0, pi1; __m128 pf0, pf1; __m128 bp0, bp1; __m128 x; int r; for (; x1 < x2; x1 += 2) { pi = (const char *)pin + (x1 << 2); bp0 = _mm_setzero_ps(); bp1 = _mm_setzero_ps(); for (r = 0; r < rct; ++r) { x = _mm_load_ss((const float *)gptr + r); x = _mm_shuffle_ps(x, x, _MM_SHUFFLE(0, 0, 0, 0)); pi0 = _mm_cvtsi32_si128(*(const int *)pi); pi1 = _mm_cvtsi32_si128(*((const int *)pi + 1)); pf0 = _mm_cvtepi32_ps(cvtepu8_epi32(pi0)); pf1 = _mm_cvtepi32_ps(cvtepu8_epi32(pi1)); bp0 = _mm_add_ps(bp0, _mm_mul_ps(pf0, x)); bp1 = _mm_add_ps(bp1, _mm_mul_ps(pf1, x)); pi += stride; } _mm_storeu_ps((float *)dst, bp0); _mm_storeu_ps((float *)dst + 4, bp1); dst = (char *)dst + 32; } } void rsdIntrinsicBlurHFU4_K(void *dst, const void *pin, const void *gptr, int rct, int x1, int x2) { const __m128i Mu8 = _mm_set_epi32(0xffffffff, 0xffffffff, 0xffffffff, 0x0c080400); const float *pi; __m128 pf, x, y; __m128i o; int r; for (; x1 < x2; ++x1) { /* rct is define as 2*r+1 by the caller */ x = _mm_load_ss((const float *)gptr); x = _mm_shuffle_ps(x, x, _MM_SHUFFLE(0, 0, 0, 0)); pi = (const float *)pin + (x1 << 2); pf = _mm_mul_ps(x, _mm_load_ps(pi)); for (r = 1; r < rct; r += 2) { x = _mm_load_ss((const float *)gptr + r); y = _mm_load_ss((const float *)gptr + r + 1); x = _mm_shuffle_ps(x, x, _MM_SHUFFLE(0, 0, 0, 0)); y = _mm_shuffle_ps(y, y, _MM_SHUFFLE(0, 0, 0, 0)); pf = _mm_add_ps(pf, _mm_mul_ps(x, _mm_load_ps(pi + (r << 2)))); pf = _mm_add_ps(pf, _mm_mul_ps(y, _mm_load_ps(pi + (r << 2) + 4))); } o = _mm_cvtps_epi32(pf); *(int *)dst = _mm_cvtsi128_si32(_mm_shuffle_epi8(o, Mu8)); dst = (char *)dst + 4; } } void rsdIntrinsicBlurHFU1_K(void *dst, const void *pin, const void *gptr, int rct, int x1, int x2) { const __m128i Mu8 = _mm_set_epi32(0xffffffff, 0xffffffff, 0xffffffff, 0x0c080400); const float *pi; __m128 pf, g0, g1, g2, g3, gx, p0, p1; __m128i o; int r; for (; x1 < x2; x1+=4) { g0 = _mm_load_ss((const float *)gptr); g0 = _mm_shuffle_ps(g0, g0, _MM_SHUFFLE(0, 0, 0, 0)); pi = (const float *)pin + x1; pf = _mm_mul_ps(g0, _mm_loadu_ps(pi)); for (r = 1; r < rct; r += 4) { gx = _mm_loadu_ps((const float *)gptr + r); p0 = _mm_loadu_ps(pi + r); p1 = _mm_loadu_ps(pi + r + 4); g0 = _mm_shuffle_ps(gx, gx, _MM_SHUFFLE(0, 0, 0, 0)); pf = _mm_add_ps(pf, _mm_mul_ps(g0, p0)); g1 = _mm_shuffle_ps(gx, gx, _MM_SHUFFLE(1, 1, 1, 1)); pf = _mm_add_ps(pf, _mm_mul_ps(g1, _mm_alignr_epi8(p1, p0, 4))); g2 = _mm_shuffle_ps(gx, gx, _MM_SHUFFLE(2, 2, 2, 2)); pf = _mm_add_ps(pf, _mm_mul_ps(g2, _mm_alignr_epi8(p1, p0, 8))); g3 = _mm_shuffle_ps(gx, gx, _MM_SHUFFLE(3, 3, 3, 3)); pf = _mm_add_ps(pf, _mm_mul_ps(g3, _mm_alignr_epi8(p1, p0, 12))); } o = _mm_cvtps_epi32(pf); *(int *)dst = _mm_cvtsi128_si32(_mm_shuffle_epi8(o, Mu8)); dst = (char *)dst + 4; } } void rsdIntrinsicYuv_K(void *dst, const unsigned char *pY, const unsigned char *pUV, uint32_t count, const short *param) { __m128i biasY, biasUV; __m128i c0, c1, c2, c3, c4; biasY = _mm_set1_epi32(param[8]); /* 16 */ biasUV = _mm_set1_epi32(param[16]); /* 128 */ c0 = _mm_set1_epi32(param[0]); /* 298 */ c1 = _mm_set1_epi32(param[1]); /* 409 */ c2 = _mm_set1_epi32(param[2]); /* -100 */ c3 = _mm_set1_epi32(param[3]); /* 516 */ c4 = _mm_set1_epi32(param[4]); /* -208 */ __m128i Y, UV, U, V, R, G, B, A; A = _mm_set1_epi32(255); uint32_t i; for (i = 0; i < (count << 1); ++i) { Y = cvtepu8_epi32(_mm_set1_epi32(*(const int *)pY)); UV = cvtepu8_epi32(_mm_set1_epi32(*(const int *)pUV)); Y = _mm_sub_epi32(Y, biasY); UV = _mm_sub_epi32(UV, biasUV); U = _mm_shuffle_epi32(UV, 0xf5); V = _mm_shuffle_epi32(UV, 0xa0); Y = mullo_epi32(Y, c0); R = _mm_add_epi32(Y, mullo_epi32(V, c1)); R = _mm_add_epi32(R, biasUV); R = _mm_srai_epi32(R, 8); G = _mm_add_epi32(Y, mullo_epi32(U, c2)); G = _mm_add_epi32(G, mullo_epi32(V, c4)); G = _mm_add_epi32(G, biasUV); G = _mm_srai_epi32(G, 8); B = _mm_add_epi32(Y, mullo_epi32(U, c3)); B = _mm_add_epi32(B, biasUV); B = _mm_srai_epi32(B, 8); __m128i y1, y2, y3, y4; y1 = packus_epi32(R, G); y2 = packus_epi32(B, A); y3 = _mm_packus_epi16(y1, y2); const __m128i T4x4 = _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0); y4 = _mm_shuffle_epi8(y3, T4x4); _mm_storeu_si128((__m128i *)dst, y4); pY += 4; pUV += 4; dst = (__m128i *)dst + 1; } } void rsdIntrinsicYuvR_K(void *dst, const unsigned char *pY, const unsigned char *pUV, uint32_t count, const short *param) { __m128i biasY, biasUV; __m128i c0, c1, c2, c3, c4; biasY = _mm_set1_epi32(param[8]); /* 16 */ biasUV = _mm_set1_epi32(param[16]); /* 128 */ c0 = _mm_set1_epi32(param[0]); /* 298 */ c1 = _mm_set1_epi32(param[1]); /* 409 */ c2 = _mm_set1_epi32(param[2]); /* -100 */ c3 = _mm_set1_epi32(param[3]); /* 516 */ c4 = _mm_set1_epi32(param[4]); /* -208 */ __m128i Y, UV, U, V, R, G, B, A; A = _mm_set1_epi32(255); uint32_t i; for (i = 0; i < (count << 1); ++i) { Y = cvtepu8_epi32(_mm_set1_epi32(*(const int *)pY)); UV = cvtepu8_epi32(_mm_set1_epi32(*(const int *)pUV)); Y = _mm_sub_epi32(Y, biasY); UV = _mm_sub_epi32(UV, biasUV); V = _mm_shuffle_epi32(UV, 0xf5); U = _mm_shuffle_epi32(UV, 0xa0); Y = mullo_epi32(Y, c0); R = _mm_add_epi32(Y, mullo_epi32(V, c1)); R = _mm_add_epi32(R, biasUV); R = _mm_srai_epi32(R, 8); G = _mm_add_epi32(Y, mullo_epi32(U, c2)); G = _mm_add_epi32(G, mullo_epi32(V, c4)); G = _mm_add_epi32(G, biasUV); G = _mm_srai_epi32(G, 8); B = _mm_add_epi32(Y, mullo_epi32(U, c3)); B = _mm_add_epi32(B, biasUV); B = _mm_srai_epi32(B, 8); __m128i y1, y2, y3, y4; y1 = packus_epi32(R, G); y2 = packus_epi32(B, A); y3 = _mm_packus_epi16(y1, y2); const __m128i T4x4 = _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0); y4 = _mm_shuffle_epi8(y3, T4x4); _mm_storeu_si128((__m128i *)dst, y4); pY += 4; pUV += 4; dst = (__m128i *)dst + 1; } } void rsdIntrinsicYuv2_K(void *dst, const unsigned char *pY, const unsigned char *pU, const unsigned char *pV, uint32_t count, const short *param) { __m128i biasY, biasUV; __m128i c0, c1, c2, c3, c4; biasY = _mm_set1_epi32(param[8]); /* 16 */ biasUV = _mm_set1_epi32(param[16]); /* 128 */ c0 = _mm_set1_epi32(param[0]); /* 298 */ c1 = _mm_set1_epi32(param[1]); /* 409 */ c2 = _mm_set1_epi32(param[2]); /* -100 */ c3 = _mm_set1_epi32(param[3]); /* 516 */ c4 = _mm_set1_epi32(param[4]); /* -208 */ __m128i Y, U, V, R, G, B, A; A = _mm_set1_epi32(255); uint32_t i; for (i = 0; i < (count << 1); ++i) { Y = cvtepu8_epi32(_mm_set1_epi32(*(const int *)pY)); U = cvtepu8_epi32(_mm_set1_epi32(*(const int *)pU)); V = cvtepu8_epi32(_mm_set1_epi32(*(const int *)pV)); Y = _mm_sub_epi32(Y, biasY); U = _mm_sub_epi32(U, biasUV); V = _mm_sub_epi32(V, biasUV); Y = mullo_epi32(Y, c0); R = _mm_add_epi32(Y, mullo_epi32(V, c1)); R = _mm_add_epi32(R, biasUV); R = _mm_srai_epi32(R, 8); G = _mm_add_epi32(Y, mullo_epi32(U, c2)); G = _mm_add_epi32(G, mullo_epi32(V, c4)); G = _mm_add_epi32(G, biasUV); G = _mm_srai_epi32(G, 8); B = _mm_add_epi32(Y, mullo_epi32(U, c3)); B = _mm_add_epi32(B, biasUV); B = _mm_srai_epi32(B, 8); __m128i y1, y2, y3, y4; y1 = packus_epi32(R, G); y2 = packus_epi32(B, A); y3 = _mm_packus_epi16(y1, y2); const __m128i T4x4 = _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0); y4 = _mm_shuffle_epi8(y3, T4x4); _mm_storeu_si128((__m128i *)dst, y4); pY += 4; pU += 4; pV += 4; dst = (__m128i *)dst + 1; } } extern "C" void rsdIntrinsicConvolve5x5_K(void *dst, const void *y0, const void *y1, const void *y2, const void *y3, const void *y4, const short *coef, uint32_t count) { __m128i x; __m128i c0, c2, c4, c6, c8, c10, c12; __m128i c14, c16, c18, c20, c22, c24; __m128i r0, r1, r2, r3, r4, r5, r6, r7, r8, r9; __m128i p0, p1, p2, p3, p4, p5, p6, p7; __m128i p8, p9, p10, p11, p12, p13, p14, p15; __m128i p16, p17, p18, p19, p20, p21, p22, p23; __m128i p24, p25, p26, p27, p28, p29, p30, p31; __m128i p32, p33, p34, p35, p36, p37, p38, p39; __m128i o0, o1, o2, o3; uint32_t i; x = _mm_loadl_epi64((const __m128i *)(coef+0)); c0 = _mm_shuffle_epi32(x, 0x00); c2 = _mm_shuffle_epi32(x, 0x55); x = _mm_loadl_epi64((const __m128i *)(coef+4)); c4 = _mm_shuffle_epi32(x, 0x00); c6 = _mm_shuffle_epi32(x, 0x55); x = _mm_loadl_epi64((const __m128i *)(coef+8)); c8 = _mm_shuffle_epi32(x, 0x00); c10 = _mm_shuffle_epi32(x, 0x55); x = _mm_loadl_epi64((const __m128i *)(coef+12)); c12 = _mm_shuffle_epi32(x, 0x00); c14 = _mm_shuffle_epi32(x, 0x55); x = _mm_loadl_epi64((const __m128i *)(coef+16)); c16 = _mm_shuffle_epi32(x, 0x00); c18 = _mm_shuffle_epi32(x, 0x55); x = _mm_loadl_epi64((const __m128i *)(coef+20)); c20 = _mm_shuffle_epi32(x, 0x00); c22 = _mm_shuffle_epi32(x, 0x55); x = _mm_loadl_epi64((const __m128i *)(coef+24)); c24 = _mm_shuffle_epi32(x, 0x00); for (i = 0; i < count; ++i) { p0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*(int32_t *)y0), _mm_setzero_si128()); p1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0+1)), _mm_setzero_si128()); p2 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0+2)), _mm_setzero_si128()); p3 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0+3)), _mm_setzero_si128()); p4 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0+4)), _mm_setzero_si128()); p5 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0+5)), _mm_setzero_si128()); p6 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0+6)), _mm_setzero_si128()); p7 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y0+7)), _mm_setzero_si128()); p8 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1)), _mm_setzero_si128()); p9 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1+1)), _mm_setzero_si128()); p10 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1+2)), _mm_setzero_si128()); p11 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1+3)), _mm_setzero_si128()); p12 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1+4)), _mm_setzero_si128()); p13 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1+5)), _mm_setzero_si128()); p14 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1+6)), _mm_setzero_si128()); p15 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y1+7)), _mm_setzero_si128()); p16 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2)), _mm_setzero_si128()); p17 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2+1)), _mm_setzero_si128()); p18 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2+2)), _mm_setzero_si128()); p19 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2+3)), _mm_setzero_si128()); p20 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2+4)), _mm_setzero_si128()); p21 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2+5)), _mm_setzero_si128()); p22 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2+6)), _mm_setzero_si128()); p23 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y2+7)), _mm_setzero_si128()); p24 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y3)), _mm_setzero_si128()); p25 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y3+1)), _mm_setzero_si128()); p26 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y3+2)), _mm_setzero_si128()); p27 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y3+3)), _mm_setzero_si128()); p28 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y3+4)), _mm_setzero_si128()); p29 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y3+5)), _mm_setzero_si128()); p30 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y3+6)), _mm_setzero_si128()); p31 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y3+7)), _mm_setzero_si128()); p32 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y4)), _mm_setzero_si128()); p33 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y4+1)), _mm_setzero_si128()); p34 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y4+2)), _mm_setzero_si128()); p35 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y4+3)), _mm_setzero_si128()); p36 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y4+4)), _mm_setzero_si128()); p37 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y4+5)), _mm_setzero_si128()); p38 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y4+6)), _mm_setzero_si128()); p39 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*((int32_t *)y4+7)), _mm_setzero_si128()); o0 = _mm_madd_epi16( _mm_unpacklo_epi16(p0, p1), c0); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p2, p3), c2)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p4, p8), c4)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p9,p10), c6)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p11, p12), c8)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p16, p17), c10)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p18, p19), c12)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p20, p24), c14)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p25,p26), c16)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p27, p28), c18)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p32, p33), c20)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p34, p35), c22)); o0 = _mm_add_epi32(o0, _mm_madd_epi16( _mm_unpacklo_epi16(p36, _mm_setzero_si128()), c24)); o0 = _mm_srai_epi32(o0, 8); o1 = _mm_madd_epi16( _mm_unpacklo_epi16(p1, p2), c0); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p3,p4), c2)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p5, p9), c4)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p10,p11), c6)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p12,p13), c8)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p17,p18), c10)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p19,p20), c12)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p21,p25), c14)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p26, p27), c16)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p28, p29), c18)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p33, p34), c20)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p35, p36), c22)); o1 = _mm_add_epi32(o1, _mm_madd_epi16( _mm_unpacklo_epi16(p37, _mm_setzero_si128()), c24)); o1 = _mm_srai_epi32(o1, 8); o2 = _mm_madd_epi16( _mm_unpacklo_epi16(p2,p3), c0); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p4, p5), c2)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p6, p10), c4)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p11, p12), c6)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p13, p14), c8)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p18, p19), c10)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p20, p21), c12)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p22, p26), c14)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p27, p28), c16)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p29, p30), c18)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p34, p35), c20)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p36, p37), c22)); o2 = _mm_add_epi32(o2, _mm_madd_epi16( _mm_unpacklo_epi16(p38, _mm_setzero_si128()), c24)); o2 = _mm_srai_epi32(o2, 8); o3 = _mm_madd_epi16( _mm_unpacklo_epi16(p3,p4), c0); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p5, p6), c2)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p7, p11), c4)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p12, p13), c6)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p14, p15), c8)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p19, p20), c10)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p21, p22), c12)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p23, p27), c14)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p28, p29), c16)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p30, p31), c18)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p35, p36), c20)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p37,p38), c22)); o3 = _mm_add_epi32(o3, _mm_madd_epi16( _mm_unpacklo_epi16(p39, _mm_setzero_si128()), c24)); o3 = _mm_srai_epi32(o3, 8); o0 = packus_epi32(o0, o1); o2 = packus_epi32(o2, o3); o0 = _mm_packus_epi16(o0, o2); _mm_storeu_si128((__m128i *)dst, o0); y0 = (const char *)y0 + 16; y1 = (const char *)y1 + 16; y2 = (const char *)y2 + 16; y3 = (const char *)y3 + 16; y4 = (const char *)y4 + 16; dst = (char *)dst + 16; } } void rsdIntrinsicBlendSrcOver_K(void *dst, const void *src, uint32_t count8) { __m128i all1s, ina, ins; __m128i in0, in1, out0, out1; __m128i t0, t1, t2, t3; uint32_t i; all1s = _mm_set1_epi16(255); for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); ins = _mm_unpacklo_epi8(in0, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t0 = _mm_unpacklo_epi8(out0, _mm_setzero_si128()); t0 = _mm_mullo_epi16(t0, _mm_sub_epi16(all1s, ina)); t0 = _mm_srli_epi16(t0, 8); t0 = _mm_add_epi16(t0, ins); ins = _mm_unpackhi_epi8(in0, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t1 = _mm_unpackhi_epi8(out0, _mm_setzero_si128()); t1 = _mm_mullo_epi16(t1, _mm_sub_epi16(all1s, ina)); t1 = _mm_srli_epi16(t1, 8); t1 = _mm_add_epi16(t1, ins); ins = _mm_unpacklo_epi8(in1, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t2 = _mm_unpacklo_epi8(out1, _mm_setzero_si128()); t2 = _mm_mullo_epi16(t2, _mm_sub_epi16(all1s, ina)); t2 = _mm_srli_epi16(t2, 8); t2 = _mm_add_epi16(t2, ins); ins = _mm_unpackhi_epi8(in1, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t3 = _mm_unpackhi_epi8(out1, _mm_setzero_si128()); t3 = _mm_mullo_epi16(t3, _mm_sub_epi16(all1s, ina)); t3 = _mm_srli_epi16(t3, 8); t3 = _mm_add_epi16(t3, ins); t0 = _mm_packus_epi16(t0, t1); t2 = _mm_packus_epi16(t2, t3); _mm_storeu_si128((__m128i *)dst, t0); _mm_storeu_si128((__m128i *)dst + 1, t2); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendDstOver_K(void *dst, const void *src, uint32_t count8) { __m128i all1s, outa, outs; __m128i in0, in1, out0, out1; __m128i t0, t1, t2, t3; uint32_t i; all1s = _mm_set1_epi16(255); for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); outs = _mm_unpacklo_epi8(out0, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t0 = _mm_unpacklo_epi8(in0, _mm_setzero_si128()); t0 = _mm_mullo_epi16(t0, _mm_sub_epi16(all1s, outa)); t0 = _mm_srli_epi16(t0, 8); t0 = _mm_add_epi16(t0, outs); outs = _mm_unpackhi_epi8(out0, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t1 = _mm_unpackhi_epi8(in0, _mm_setzero_si128()); t1 = _mm_mullo_epi16(t1, _mm_sub_epi16(all1s, outa)); t1 = _mm_srli_epi16(t1, 8); t1 = _mm_add_epi16(t1, outs); outs = _mm_unpacklo_epi8(out1, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t2 = _mm_unpacklo_epi8(in1, _mm_setzero_si128()); t2 = _mm_mullo_epi16(t2, _mm_sub_epi16(all1s, outa)); t2 = _mm_srli_epi16(t2, 8); t2 = _mm_add_epi16(t2, outs); outs = _mm_unpackhi_epi8(out1, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t3 = _mm_unpackhi_epi8(in1, _mm_setzero_si128()); t3 = _mm_mullo_epi16(t3, _mm_sub_epi16(all1s, outa)); t3 = _mm_srli_epi16(t3, 8); t3 = _mm_add_epi16(t3, outs); t0 = _mm_packus_epi16(t0, t1); t2 = _mm_packus_epi16(t2, t3); _mm_storeu_si128((__m128i *)dst, t0); _mm_storeu_si128((__m128i *)dst + 1, t2); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendSrcIn_K(void *dst, const void *src, uint32_t count8) { __m128i outa; __m128i in0, in1, out0, out1; __m128i t0, t1, t2, t3; uint32_t i; for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); outa = _mm_unpacklo_epi8(out0, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outa, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t0 = _mm_unpacklo_epi8(in0, _mm_setzero_si128()); t0 = _mm_mullo_epi16(t0, outa); t0 = _mm_srli_epi16(t0, 8); outa = _mm_unpackhi_epi8(out0, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outa, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t1 = _mm_unpackhi_epi8(in0, _mm_setzero_si128()); t1 = _mm_mullo_epi16(t1, outa); t1 = _mm_srli_epi16(t1, 8); outa = _mm_unpacklo_epi8(out1, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outa, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t2 = _mm_unpacklo_epi8(in1, _mm_setzero_si128()); t2 = _mm_mullo_epi16(t2, outa); t2 = _mm_srli_epi16(t2, 8); outa = _mm_unpackhi_epi8(out1, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outa, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t3 = _mm_unpackhi_epi8(in1, _mm_setzero_si128()); t3 = _mm_mullo_epi16(t3, outa); t3 = _mm_srli_epi16(t3, 8); t0 = _mm_packus_epi16(t0, t1); t2 = _mm_packus_epi16(t2, t3); _mm_storeu_si128((__m128i *)dst, t0); _mm_storeu_si128((__m128i *)dst + 1, t2); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendDstIn_K(void *dst, const void *src, uint32_t count8) { __m128i ina; __m128i in0, in1, out0, out1; __m128i t0, t1, t2, t3; uint32_t i; for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); ina = _mm_unpacklo_epi8(in0, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ina, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t0 = _mm_unpacklo_epi8(out0, _mm_setzero_si128()); t0 = _mm_mullo_epi16(t0, ina); t0 = _mm_srli_epi16(t0, 8); ina = _mm_unpackhi_epi8(in0, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ina, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t1 = _mm_unpackhi_epi8(out0, _mm_setzero_si128()); t1 = _mm_mullo_epi16(t1, ina); t1 = _mm_srli_epi16(t1, 8); ina = _mm_unpacklo_epi8(in1, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ina, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t2 = _mm_unpacklo_epi8(out1, _mm_setzero_si128()); t2 = _mm_mullo_epi16(t2, ina); t2 = _mm_srli_epi16(t2, 8); ina = _mm_unpackhi_epi8(in1, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ina, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t3 = _mm_unpackhi_epi8(out1, _mm_setzero_si128()); t3 = _mm_mullo_epi16(t3, ina); t3 = _mm_srli_epi16(t3, 8); t0 = _mm_packus_epi16(t0, t1); t2 = _mm_packus_epi16(t2, t3); _mm_storeu_si128((__m128i *)dst, t0); _mm_storeu_si128((__m128i *)dst + 1, t2); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendSrcOut_K(void *dst, const void *src, uint32_t count8) { __m128i all1s, outa; __m128i in0, in1, out0, out1; __m128i t0, t1, t2, t3; uint32_t i; all1s = _mm_set1_epi16(255); for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); outa = _mm_unpacklo_epi8(out0, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outa, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t0 = _mm_unpacklo_epi8(in0, _mm_setzero_si128()); t0 = _mm_mullo_epi16(t0, _mm_sub_epi16(all1s, outa)); t0 = _mm_srli_epi16(t0, 8); outa = _mm_unpackhi_epi8(out0, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outa, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t1 = _mm_unpackhi_epi8(in0, _mm_setzero_si128()); t1 = _mm_mullo_epi16(t1, _mm_sub_epi16(all1s, outa)); t1 = _mm_srli_epi16(t1, 8); outa = _mm_unpacklo_epi8(out1, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outa, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t2 = _mm_unpacklo_epi8(in1, _mm_setzero_si128()); t2 = _mm_mullo_epi16(t2, _mm_sub_epi16(all1s, outa)); t2 = _mm_srli_epi16(t2, 8); outa = _mm_unpackhi_epi8(out1, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outa, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t3 = _mm_unpackhi_epi8(in1, _mm_setzero_si128()); t3 = _mm_mullo_epi16(t3, _mm_sub_epi16(all1s, outa)); t3 = _mm_srli_epi16(t3, 8); t0 = _mm_packus_epi16(t0, t1); t2 = _mm_packus_epi16(t2, t3); _mm_storeu_si128((__m128i *)dst, t0); _mm_storeu_si128((__m128i *)dst + 1, t2); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendDstOut_K(void *dst, const void *src, uint32_t count8) { __m128i all1s, ina; __m128i in0, in1, out0, out1; __m128i t0, t1, t2, t3; uint32_t i; all1s = _mm_set1_epi16(255); for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); ina = _mm_unpacklo_epi8(in0, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ina, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t0 = _mm_unpacklo_epi8(out0, _mm_setzero_si128()); t0 = _mm_mullo_epi16(t0, _mm_sub_epi16(all1s, ina)); t0 = _mm_srli_epi16(t0, 8); ina = _mm_unpackhi_epi8(in0, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ina, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t1 = _mm_unpackhi_epi8(out0, _mm_setzero_si128()); t1 = _mm_mullo_epi16(t1, _mm_sub_epi16(all1s, ina)); t1 = _mm_srli_epi16(t1, 8); ina = _mm_unpacklo_epi8(in1, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ina, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t2 = _mm_unpacklo_epi8(out1, _mm_setzero_si128()); t2 = _mm_mullo_epi16(t2, _mm_sub_epi16(all1s, ina)); t2 = _mm_srli_epi16(t2, 8); ina = _mm_unpackhi_epi8(in1, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ina, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); t3 = _mm_unpackhi_epi8(out1, _mm_setzero_si128()); t3 = _mm_mullo_epi16(t3, _mm_sub_epi16(all1s, ina)); t3 = _mm_srli_epi16(t3, 8); t0 = _mm_packus_epi16(t0, t1); t2 = _mm_packus_epi16(t2, t3); _mm_storeu_si128((__m128i *)dst, t0); _mm_storeu_si128((__m128i *)dst + 1, t2); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendSrcAtop_K(void *dst, const void *src, uint32_t count8) { const __m128i M0001 = _mm_set_epi32(0xff000000, 0xff000000, 0xff000000, 0xff000000); __m128i all1s, ina, outa, ins, outs; __m128i in0, in1, out0, out1; __m128i t0, t1, t2, t3; uint32_t i; all1s = _mm_set1_epi16(255); for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); ins = _mm_unpacklo_epi8(in0, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); outs = _mm_unpacklo_epi8(out0, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t0 = _mm_sub_epi16(all1s, ina); t0 = _mm_mullo_epi16(t0, outs); t0 = _mm_adds_epu16(t0, _mm_mullo_epi16(outa, ins)); t0 = _mm_srli_epi16(t0, 8); ins = _mm_unpackhi_epi8(in0, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); outs = _mm_unpackhi_epi8(out0, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t1 = _mm_sub_epi16(all1s, ina); t1 = _mm_mullo_epi16(t1, outs); t1 = _mm_adds_epu16(t1, _mm_mullo_epi16(outa, ins)); t1 = _mm_srli_epi16(t1, 8); ins = _mm_unpacklo_epi8(in1, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); outs = _mm_unpacklo_epi8(out1, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t2 = _mm_sub_epi16(all1s, ina); t2 = _mm_mullo_epi16(t2, outs); t2 = _mm_adds_epu16(t2, _mm_mullo_epi16(outa, ins)); t2 = _mm_srli_epi16(t2, 8); ins = _mm_unpackhi_epi8(in1, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); outs = _mm_unpackhi_epi8(out1, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t3 = _mm_sub_epi16(all1s, ina); t3 = _mm_mullo_epi16(t3, outs); t3 = _mm_adds_epu16(t3, _mm_mullo_epi16(outa, ins)); t3 = _mm_srli_epi16(t3, 8); t0 = _mm_packus_epi16(t0, t1); t0 = blendv_epi8(t0, out0, M0001); t2 = _mm_packus_epi16(t2, t3); t2 = blendv_epi8(t2, out1, M0001); _mm_storeu_si128((__m128i *)dst, t0); _mm_storeu_si128((__m128i *)dst + 1, t2); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendDstAtop_K(void *dst, const void *src, uint32_t count8) { const __m128i M0001 = _mm_set_epi32(0xff000000, 0xff000000, 0xff000000, 0xff000000); __m128i all1s, ina, ins, outa, outs; __m128i in0, in1, out0, out1; __m128i t0, t1, t2, t3; uint32_t i; all1s = _mm_set1_epi16(255); for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); ins = _mm_unpacklo_epi8(in0, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); outs = _mm_unpacklo_epi8(out0, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t0 = _mm_sub_epi16(all1s, outa); t0 = _mm_mullo_epi16(t0, ins); t0 = _mm_adds_epu16(t0, _mm_mullo_epi16(ina, outs)); t0 = _mm_srli_epi16(t0, 8); ins = _mm_unpackhi_epi8(in0, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); outs = _mm_unpackhi_epi8(out0, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t1 = _mm_sub_epi16(all1s, outa); t1 = _mm_mullo_epi16(t1, ins); t1 = _mm_adds_epu16(t1, _mm_mullo_epi16(ina, outs)); t1 = _mm_srli_epi16(t1, 8); ins = _mm_unpacklo_epi8(in1, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); outs = _mm_unpacklo_epi8(out1, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t2 = _mm_sub_epi16(all1s, outa); t2 = _mm_mullo_epi16(t2, ins); t2 = _mm_adds_epu16(t2, _mm_mullo_epi16(ina, outs)); t2 = _mm_srli_epi16(t2, 8); ins = _mm_unpackhi_epi8(in1, _mm_setzero_si128()); ina = _mm_shufflelo_epi16(ins, 0xFF); ina = _mm_shufflehi_epi16(ina, 0xFF); outs = _mm_unpackhi_epi8(out1, _mm_setzero_si128()); outa = _mm_shufflelo_epi16(outs, 0xFF); outa = _mm_shufflehi_epi16(outa, 0xFF); t3 = _mm_sub_epi16(all1s, outa); t3 = _mm_mullo_epi16(t3, ins); t3 = _mm_adds_epu16(t3, _mm_mullo_epi16(ina, outs)); t3 = _mm_srli_epi16(t3, 8); t0 = _mm_packus_epi16(t0, t1); t0 = blendv_epi8(t0, in0, M0001); t2 = _mm_packus_epi16(t2, t3); t2 = blendv_epi8(t2, in1, M0001); _mm_storeu_si128((__m128i *)dst, t0); _mm_storeu_si128((__m128i *)dst + 1, t2); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendXor_K(void *dst, const void *src, uint32_t count8) { __m128i in0, in1, out0, out1; uint32_t i; for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); out0 = _mm_xor_si128(out0, in0); out1 = _mm_xor_si128(out1, in1); _mm_storeu_si128((__m128i *)dst, out0); _mm_storeu_si128((__m128i *)dst + 1, out1); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendMultiply_K(void *dst, const void *src, uint32_t count8) { __m128i in0, in1, out0, out1; __m128i t0, t1, t2, t3; uint32_t i; for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); t0 = _mm_unpacklo_epi8(in0, _mm_setzero_si128()); t0 = _mm_mullo_epi16(t0, _mm_unpacklo_epi8(out0, _mm_setzero_si128())); t0 = _mm_srli_epi16(t0, 8); t1 = _mm_unpackhi_epi8(in0, _mm_setzero_si128()); t1 = _mm_mullo_epi16(t1, _mm_unpackhi_epi8(out0, _mm_setzero_si128())); t1 = _mm_srli_epi16(t1, 8); t2 = _mm_unpacklo_epi8(in1, _mm_setzero_si128()); t2 = _mm_mullo_epi16(t2, _mm_unpacklo_epi8(out1, _mm_setzero_si128())); t2 = _mm_srli_epi16(t2, 8); t3 = _mm_unpackhi_epi8(in1, _mm_setzero_si128()); t3 = _mm_mullo_epi16(t3, _mm_unpackhi_epi8(out1, _mm_setzero_si128())); t3 = _mm_srli_epi16(t3, 8); t0 = _mm_packus_epi16(t0, t1); t2 = _mm_packus_epi16(t2, t3); _mm_storeu_si128((__m128i *)dst, t0); _mm_storeu_si128((__m128i *)dst + 1, t2); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendAdd_K(void *dst, const void *src, uint32_t count8) { __m128i in0, in1, out0, out1; uint32_t i; for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); out0 = _mm_adds_epu8(out0, in0); out1 = _mm_adds_epu8(out1, in1); _mm_storeu_si128((__m128i *)dst, out0); _mm_storeu_si128((__m128i *)dst + 1, out1); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } } void rsdIntrinsicBlendSub_K(void *dst, const void *src, uint32_t count8) { __m128i in0, in1, out0, out1; uint32_t i; for (i = 0; i < count8; ++i) { in0 = _mm_loadu_si128((const __m128i *)src); in1 = _mm_loadu_si128((const __m128i *)src + 1); out0 = _mm_loadu_si128((const __m128i *)dst); out1 = _mm_loadu_si128((const __m128i *)dst + 1); out0 = _mm_subs_epu8(out0, in0); out1 = _mm_subs_epu8(out1, in1); _mm_storeu_si128((__m128i *)dst, out0); _mm_storeu_si128((__m128i *)dst + 1, out1); src = (const __m128i *)src + 2; dst = (__m128i *)dst + 2; } }