From c065d71a2fec192085a5741195135f7d4f5f31f7 Mon Sep 17 00:00:00 2001 From: Recep Aslantas Date: Fri, 30 Apr 2021 21:12:17 +0300 Subject: [PATCH] simd, sse: optimize mat4 inv with sse * reduce a few shufflees * re-oder instructions for ILP --- include/cglm/simd/sse2/mat4.h | 160 ++++++++++++++++++++++------------ 1 file changed, 104 insertions(+), 56 deletions(-) diff --git a/include/cglm/simd/sse2/mat4.h b/include/cglm/simd/sse2/mat4.h index f5d7135..8545958 100644 --- a/include/cglm/simd/sse2/mat4.h +++ b/include/cglm/simd/sse2/mat4.h @@ -143,97 +143,121 @@ glm_mat4_inv_fast_sse2(mat4 mat, mat4 dest) { v0, v1, v2, v3, t0, t1, t2, t3, t4, t5, x0, x1, x2, x3, x4, x5, x6, x7, x8, x9; - + x8 = _mm_set_ps(-0.f, 0.f, -0.f, 0.f); x9 = glmm_shuff1(x8, 2, 1, 2, 1); - + /* 127 <- 0 */ r0 = glmm_load(mat[0]); /* d c b a */ r1 = glmm_load(mat[1]); /* h g f e */ r2 = glmm_load(mat[2]); /* l k j i */ r3 = glmm_load(mat[3]); /* p o n m */ - - x0 = _mm_shuffle_ps(r2, r3, _MM_SHUFFLE(3, 2, 3, 2)); /* p o l k */ - x1 = glmm_shuff1(x0, 1, 3, 3, 3); /* l p p p */ + + x0 = _mm_movehl_ps(r3, r2); /* p o l k */ + x3 = _mm_movelh_ps(r2, r3); /* n m j i */ + x1 = glmm_shuff1(x0, 1, 3, 3 ,3); /* l p p p */ x2 = glmm_shuff1(x0, 0, 2, 2, 2); /* k o o o */ - x0 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(3, 3, 3, 3)); /* h h l l */ - x3 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(2, 2, 2, 2)); /* g g k k */ + x4 = glmm_shuff1(x3, 1, 3, 3, 3); /* j n n n */ + x7 = glmm_shuff1(x3, 0, 2, 2, 2); /* i m m m */ + x6 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(0, 0, 0, 0)); /* e e i i */ + x5 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(1, 1, 1, 1)); /* f f j j */ + x3 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(2, 2, 2, 2)); /* g g k k */ + x0 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(3, 3, 3, 3)); /* h h l l */ + + t0 = _mm_mul_ps(x3, x1); + t1 = _mm_mul_ps(x5, x1); + t2 = _mm_mul_ps(x5, x2); + t3 = _mm_mul_ps(x6, x1); + t4 = _mm_mul_ps(x6, x2); + t5 = _mm_mul_ps(x6, x4); + /* t1[0] = k * p - o * l; t1[0] = k * p - o * l; t2[0] = g * p - o * h; t3[0] = g * l - k * h; */ - t0 = glmm_fnmadd(x2, x0, _mm_mul_ps(x3, x1)); + t0 = glmm_fnmadd(x2, x0, t0); - x4 = _mm_shuffle_ps(r2, r3, _MM_SHUFFLE(2, 1, 2, 1)); /* o n k j */ - x4 = glmm_shuff1(x4, 0, 2, 2, 2); /* j n n n */ - x5 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(1, 1, 1, 1)); /* f f j j */ - /* t1[1] = j * p - n * l; t1[1] = j * p - n * l; t2[1] = f * p - n * h; t3[1] = f * l - j * h; */ - t1 = glmm_fnmadd(x4, x0, _mm_mul_ps(x5, x1)); + t1 = glmm_fnmadd(x4, x0, t1); /* t1[2] = j * o - n * k t1[2] = j * o - n * k; t2[2] = f * o - n * g; t3[2] = f * k - j * g; */ - t2 = glmm_fnmadd(x4, x3, _mm_mul_ps(x5, x2)); + t2 = glmm_fnmadd(x4, x3, t2); - x6 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(0, 0, 0, 0)); /* e e i i */ - x7 = glmm_shuff2(r3, r2, 0, 0, 0, 0, 2, 0, 0, 0); /* i m m m */ - /* t1[3] = i * p - m * l; t1[3] = i * p - m * l; t2[3] = e * p - m * h; t3[3] = e * l - i * h; */ - t3 = glmm_fnmadd(x7, x0, _mm_mul_ps(x6, x1)); + t3 = glmm_fnmadd(x7, x0, t3); /* t1[4] = i * o - m * k; t1[4] = i * o - m * k; t2[4] = e * o - m * g; t3[4] = e * k - i * g; */ - t4 = glmm_fnmadd(x7, x3, _mm_mul_ps(x6, x2)); + t4 = glmm_fnmadd(x7, x3, t4); /* t1[5] = i * n - m * j; t1[5] = i * n - m * j; t2[5] = e * n - m * f; t3[5] = e * j - i * f; */ - t5 = glmm_fnmadd(x7, x5, _mm_mul_ps(x6, x4)); + t5 = glmm_fnmadd(x7, x5, t5); - x0 = glmm_shuff2(r1, r0, 0, 0, 0, 0, 2, 2, 2, 0); /* a a a e */ - x1 = glmm_shuff2(r1, r0, 1, 1, 1, 1, 2, 2, 2, 0); /* b b b f */ - x2 = glmm_shuff2(r1, r0, 2, 2, 2, 2, 2, 2, 2, 0); /* c c c g */ - x3 = glmm_shuff2(r1, r0, 3, 3, 3, 3, 2, 2, 2, 0); /* d d d h */ + x4 = _mm_movelh_ps(r0, r1); /* f e b a */ + x5 = _mm_movehl_ps(r1, r0); /* h g d c */ + + x0 = glmm_shuff1(x4, 0, 0, 0, 2); /* a a a e */ + x1 = glmm_shuff1(x4, 1, 1, 1, 3); /* b b b f */ + x2 = glmm_shuff1(x5, 0, 0, 0, 2); /* c c c g */ + x3 = glmm_shuff1(x5, 1, 1, 1, 3); /* d d d h */ + + v2 = _mm_mul_ps(x0, t1); + v1 = _mm_mul_ps(x0, t0); + v3 = _mm_mul_ps(x0, t2); + v0 = _mm_mul_ps(x1, t0); + + v2 = glmm_fnmadd(x1, t3, v2); + v3 = glmm_fnmadd(x1, t4, v3); + v0 = glmm_fnmadd(x2, t1, v0); + v1 = glmm_fnmadd(x2, t3, v1); + + v3 = glmm_fmadd(x2, t5, v3); + v0 = glmm_fmadd(x3, t2, v0); + v2 = glmm_fmadd(x3, t5, v2); + v1 = glmm_fmadd(x3, t4, v1); /* dest[0][0] = f * t1[0] - g * t1[1] + h * t1[2]; dest[0][1] =-(b * t1[0] - c * t1[1] + d * t1[2]); dest[0][2] = b * t2[0] - c * t2[1] + d * t2[2]; dest[0][3] =-(b * t3[0] - c * t3[1] + d * t3[2]); */ - v0 = _mm_xor_ps(glmm_fmadd(x3, t2, glmm_fnmadd(x2, t1, _mm_mul_ps(x1, t0))), x8); + v0 = _mm_xor_ps(v0, x8); /* dest[2][0] = e * t1[1] - f * t1[3] + h * t1[5]; dest[2][1] =-(a * t1[1] - b * t1[3] + d * t1[5]); dest[2][2] = a * t2[1] - b * t2[3] + d * t2[5]; dest[2][3] =-(a * t3[1] - b * t3[3] + d * t3[5]);*/ - v2 = _mm_xor_ps(glmm_fmadd(x3, t5, glmm_fnmadd(x1, t3, _mm_mul_ps(x0, t1))), x8); + v2 = _mm_xor_ps(v2, x8); /* dest[1][0] =-(e * t1[0] - g * t1[3] + h * t1[4]); dest[1][1] = a * t1[0] - c * t1[3] + d * t1[4]; dest[1][2] =-(a * t2[0] - c * t2[3] + d * t2[4]); dest[1][3] = a * t3[0] - c * t3[3] + d * t3[4]; */ - v1 = _mm_xor_ps(glmm_fmadd(x3, t4, glmm_fnmadd(x2, t3, _mm_mul_ps(x0, t0))), x9); + v1 = _mm_xor_ps(v1, x9); /* dest[3][0] =-(e * t1[2] - f * t1[4] + g * t1[5]); dest[3][1] = a * t1[2] - b * t1[4] + c * t1[5]; dest[3][2] =-(a * t2[2] - b * t2[4] + c * t2[5]); dest[3][3] = a * t3[2] - b * t3[4] + c * t3[5]; */ - v3 = _mm_xor_ps(glmm_fmadd(x2, t5, glmm_fnmadd(x1, t4, _mm_mul_ps(x0, t2))), x9); + v3 = _mm_xor_ps(v3, x9); /* determinant */ x0 = _mm_shuffle_ps(v0, v1, _MM_SHUFFLE(0, 0, 0, 0)); @@ -255,97 +279,121 @@ glm_mat4_inv_sse2(mat4 mat, mat4 dest) { v0, v1, v2, v3, t0, t1, t2, t3, t4, t5, x0, x1, x2, x3, x4, x5, x6, x7, x8, x9; - + x8 = _mm_set_ps(-0.f, 0.f, -0.f, 0.f); x9 = glmm_shuff1(x8, 2, 1, 2, 1); - + /* 127 <- 0 */ r0 = glmm_load(mat[0]); /* d c b a */ r1 = glmm_load(mat[1]); /* h g f e */ r2 = glmm_load(mat[2]); /* l k j i */ r3 = glmm_load(mat[3]); /* p o n m */ - - x0 = _mm_shuffle_ps(r2, r3, _MM_SHUFFLE(3, 2, 3, 2)); /* p o l k */ - x1 = glmm_shuff1(x0, 1, 3, 3, 3); /* l p p p */ + + x0 = _mm_movehl_ps(r3, r2); /* p o l k */ + x3 = _mm_movelh_ps(r2, r3); /* n m j i */ + x1 = glmm_shuff1(x0, 1, 3, 3 ,3); /* l p p p */ x2 = glmm_shuff1(x0, 0, 2, 2, 2); /* k o o o */ - x0 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(3, 3, 3, 3)); /* h h l l */ - x3 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(2, 2, 2, 2)); /* g g k k */ + x4 = glmm_shuff1(x3, 1, 3, 3, 3); /* j n n n */ + x7 = glmm_shuff1(x3, 0, 2, 2, 2); /* i m m m */ + x6 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(0, 0, 0, 0)); /* e e i i */ + x5 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(1, 1, 1, 1)); /* f f j j */ + x3 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(2, 2, 2, 2)); /* g g k k */ + x0 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(3, 3, 3, 3)); /* h h l l */ + + t0 = _mm_mul_ps(x3, x1); + t1 = _mm_mul_ps(x5, x1); + t2 = _mm_mul_ps(x5, x2); + t3 = _mm_mul_ps(x6, x1); + t4 = _mm_mul_ps(x6, x2); + t5 = _mm_mul_ps(x6, x4); + /* t1[0] = k * p - o * l; t1[0] = k * p - o * l; t2[0] = g * p - o * h; t3[0] = g * l - k * h; */ - t0 = glmm_fnmadd(x2, x0, _mm_mul_ps(x3, x1)); + t0 = glmm_fnmadd(x2, x0, t0); - x4 = _mm_shuffle_ps(r2, r3, _MM_SHUFFLE(2, 1, 2, 1)); /* o n k j */ - x4 = glmm_shuff1(x4, 0, 2, 2, 2); /* j n n n */ - x5 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(1, 1, 1, 1)); /* f f j j */ - /* t1[1] = j * p - n * l; t1[1] = j * p - n * l; t2[1] = f * p - n * h; t3[1] = f * l - j * h; */ - t1 = glmm_fnmadd(x4, x0, _mm_mul_ps(x5, x1)); + t1 = glmm_fnmadd(x4, x0, t1); /* t1[2] = j * o - n * k t1[2] = j * o - n * k; t2[2] = f * o - n * g; t3[2] = f * k - j * g; */ - t2 = glmm_fnmadd(x4, x3, _mm_mul_ps(x5, x2)); + t2 = glmm_fnmadd(x4, x3, t2); - x6 = _mm_shuffle_ps(r2, r1, _MM_SHUFFLE(0, 0, 0, 0)); /* e e i i */ - x7 = glmm_shuff2(r3, r2, 0, 0, 0, 0, 2, 0, 0, 0); /* i m m m */ - /* t1[3] = i * p - m * l; t1[3] = i * p - m * l; t2[3] = e * p - m * h; t3[3] = e * l - i * h; */ - t3 = glmm_fnmadd(x7, x0, _mm_mul_ps(x6, x1)); + t3 = glmm_fnmadd(x7, x0, t3); /* t1[4] = i * o - m * k; t1[4] = i * o - m * k; t2[4] = e * o - m * g; t3[4] = e * k - i * g; */ - t4 = glmm_fnmadd(x7, x3, _mm_mul_ps(x6, x2)); + t4 = glmm_fnmadd(x7, x3, t4); /* t1[5] = i * n - m * j; t1[5] = i * n - m * j; t2[5] = e * n - m * f; t3[5] = e * j - i * f; */ - t5 = glmm_fnmadd(x7, x5, _mm_mul_ps(x6, x4)); + t5 = glmm_fnmadd(x7, x5, t5); - x0 = glmm_shuff2(r1, r0, 0, 0, 0, 0, 2, 2, 2, 0); /* a a a e */ - x1 = glmm_shuff2(r1, r0, 1, 1, 1, 1, 2, 2, 2, 0); /* b b b f */ - x2 = glmm_shuff2(r1, r0, 2, 2, 2, 2, 2, 2, 2, 0); /* c c c g */ - x3 = glmm_shuff2(r1, r0, 3, 3, 3, 3, 2, 2, 2, 0); /* d d d h */ + x4 = _mm_movelh_ps(r0, r1); /* f e b a */ + x5 = _mm_movehl_ps(r1, r0); /* h g d c */ + + x0 = glmm_shuff1(x4, 0, 0, 0, 2); /* a a a e */ + x1 = glmm_shuff1(x4, 1, 1, 1, 3); /* b b b f */ + x2 = glmm_shuff1(x5, 0, 0, 0, 2); /* c c c g */ + x3 = glmm_shuff1(x5, 1, 1, 1, 3); /* d d d h */ + + v2 = _mm_mul_ps(x0, t1); + v1 = _mm_mul_ps(x0, t0); + v3 = _mm_mul_ps(x0, t2); + v0 = _mm_mul_ps(x1, t0); + + v2 = glmm_fnmadd(x1, t3, v2); + v3 = glmm_fnmadd(x1, t4, v3); + v0 = glmm_fnmadd(x2, t1, v0); + v1 = glmm_fnmadd(x2, t3, v1); + + v3 = glmm_fmadd(x2, t5, v3); + v0 = glmm_fmadd(x3, t2, v0); + v2 = glmm_fmadd(x3, t5, v2); + v1 = glmm_fmadd(x3, t4, v1); /* dest[0][0] = f * t1[0] - g * t1[1] + h * t1[2]; dest[0][1] =-(b * t1[0] - c * t1[1] + d * t1[2]); dest[0][2] = b * t2[0] - c * t2[1] + d * t2[2]; dest[0][3] =-(b * t3[0] - c * t3[1] + d * t3[2]); */ - v0 = _mm_xor_ps(glmm_fmadd(x3, t2, glmm_fnmadd(x2, t1, _mm_mul_ps(x1, t0))), x8); + v0 = _mm_xor_ps(v0, x8); /* dest[2][0] = e * t1[1] - f * t1[3] + h * t1[5]; dest[2][1] =-(a * t1[1] - b * t1[3] + d * t1[5]); dest[2][2] = a * t2[1] - b * t2[3] + d * t2[5]; dest[2][3] =-(a * t3[1] - b * t3[3] + d * t3[5]);*/ - v2 = _mm_xor_ps(glmm_fmadd(x3, t5, glmm_fnmadd(x1, t3, _mm_mul_ps(x0, t1))), x8); + v2 = _mm_xor_ps(v2, x8); /* dest[1][0] =-(e * t1[0] - g * t1[3] + h * t1[4]); dest[1][1] = a * t1[0] - c * t1[3] + d * t1[4]; dest[1][2] =-(a * t2[0] - c * t2[3] + d * t2[4]); dest[1][3] = a * t3[0] - c * t3[3] + d * t3[4]; */ - v1 = _mm_xor_ps(glmm_fmadd(x3, t4, glmm_fnmadd(x2, t3, _mm_mul_ps(x0, t0))), x9); + v1 = _mm_xor_ps(v1, x9); /* dest[3][0] =-(e * t1[2] - f * t1[4] + g * t1[5]); dest[3][1] = a * t1[2] - b * t1[4] + c * t1[5]; dest[3][2] =-(a * t2[2] - b * t2[4] + c * t2[5]); dest[3][3] = a * t3[2] - b * t3[4] + c * t3[5]; */ - v3 = _mm_xor_ps(glmm_fmadd(x2, t5, glmm_fnmadd(x1, t4, _mm_mul_ps(x0, t2))), x9); + v3 = _mm_xor_ps(v3, x9); /* determinant */ x0 = _mm_shuffle_ps(v0, v1, _MM_SHUFFLE(0, 0, 0, 0));