APPBROWSER-323: Transpose performance optimization

Change-Id: Ib678dee9de43690e4cfb7be1e7ccf7a7ab38233d
Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/112085
Reviewed-by: Joel Liang <joel.liang@arm.com>
Tested-by: BSG Visual Compute Jenkins server to access repositories on http://mpd-gerrit.cambridge.arm.com <bsgcomp@arm.com>
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>

1 files changed, 168 insertions, 153 deletions

diff --git a/src/core/GLES_COMPUTE/cs_shaders/transpose.cs b/src/core/GLES_COMPUTE/cs_shaders/transpose.cs
index c251d95292..f8ad30312c 100755..100644
--- a/src/core/GLES_COMPUTE/cs_shaders/transpose.cs
+++ b/src/core/GLES_COMPUTE/cs_shaders/transpose.cs
@@ -24,33 +24,42 @@
 layout(local_size_x = LOCAL_SIZE_X, local_size_y = LOCAL_SIZE_Y, local_size_z = LOCAL_SIZE_Z) in;
 #include "helpers.h"
 
-#ifdef DATA_TYPE_FP32
-precision highp float;
-
-BUFFER_DECLARATION(src, 1, float, readonly);
-BUFFER_DECLARATION(dst, 2, float, writeonly);
-
-layout(std140) uniform shader_params
-{
-    IMAGE_PARAM_DECLARATION(src);
-    IMAGE_PARAM_DECLARATION(dst);
-};
+#define SWAP_ROW_func(u0, l0) \
+    {                         \
+        tmp_swap = u0;        \
+        u0       = l0;        \
+        l0       = tmp_swap;  \
+    }
 
-#define LOAD16(r, name, offset)          \
-    r.x = LOAD4(name, offset);           \
-    r.y = LOAD4(name, offset + uint(1)); \
-    r.z = LOAD4(name, offset + uint(2)); \
-    r.w = LOAD4(name, offset + uint(3))
+#define SWAP_4x4_func(u0, u1, u2, u3, l0, l1, l2, l3) \
+    {                                                 \
+        vec4 tmp_swap;                                \
+        SWAP_ROW_func(u0, l0);                        \
+        SWAP_ROW_func(u1, l1);                        \
+        SWAP_ROW_func(u2, l2);                        \
+        SWAP_ROW_func(u3, l3);                        \
+    }
 
-#define STORE16(name, offset, r)         \
-    STORE4(name, offset, r.x);           \
-    STORE4(name, offset + uint(1), r.y); \
-    STORE4(name, offset + uint(2), r.z); \
-    STORE4(name, offset + uint(3), r.w)
+#define TRANSPOSE_4x4_func(u0, u1, u2, u3) \
+    {                                      \
+        mat4x4 matin, matout;              \
+        matin[0] = u0;                     \
+        matin[1] = u1;                     \
+        matin[2] = u2;                     \
+        matin[3] = u3;                     \
+        matout   = transpose(matin);       \
+        u0       = matout[0];              \
+        u1       = matout[1];              \
+        u2       = matout[2];              \
+        u3       = matout[3];              \
+    }
 
 /** This OpenGL ES kernel computes the matrix transposition of input matrix
  *
- * @param[in]  src_ptr                           Pointer to the source matrix. Supported data types: F32
+ * @note The data type must be passed at compile time using "#define DATA_TYPE_NAME". e.g. "#define DATA_TYPE_FP32"
+ * @note Optimization name must be passed using "#define OPTIMIZATION_NAME" for F16. e.g. "#define TRANSPOSE_8X8"
+ *
+ * @param[in]  src_ptr                           Pointer to the source matrix. Supported data types: F32/F16
  * @param[in]  src_stride_x                      Stride of the source matrix in X dimension (in bytes)
  * @param[in]  src_step_x                        src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in]  src_stride_y                      Stride of the source matrix in Y dimension (in bytes)
@@ -63,13 +72,42 @@ layout(std140) uniform shader_params
  * @param[in]  dst_step_y                        dst_gx_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in]  dst_offset_first_element_in_bytes The offset of the first element in the destination matrix
  */
+
+layout(std140) uniform shader_params
+{
+    IMAGE_PARAM_DECLARATION(src);
+    IMAGE_PARAM_DECLARATION(dst);
+};
+
+#ifdef DATA_TYPE_FP32
+precision highp float;
+
+BUFFER_DECLARATION(src, 1, float, readonly);
+BUFFER_DECLARATION(dst, 2, float, writeonly);
+
+#define LOAD16(r, name, offset)              \
+    {                                        \
+        r.x = LOAD4(name, offset);           \
+        r.y = LOAD4(name, offset + uint(1)); \
+        r.z = LOAD4(name, offset + uint(2)); \
+        r.w = LOAD4(name, offset + uint(3)); \
+    }
+
+#define STORE16(name, offset, r)             \
+    {                                        \
+        STORE4(name, offset, r.x);           \
+        STORE4(name, offset + uint(1), r.y); \
+        STORE4(name, offset + uint(2), r.z); \
+        STORE4(name, offset + uint(3), r.w); \
+    }
+
 void main(void)
 {
-    // Compute source address
+    // compute source address
     Image src = CONVERT_TO_IMAGE_STRUCT(src);
     Image dst = CONVERT_TO_IMAGE_STRUCT(dst);
 
-    // Load the NxN block at (x, y)
+    // load the NxN block at (x, y)
     vec4 u0;
     vec4 u1;
     vec4 u2;
@@ -79,25 +117,10 @@ void main(void)
     LOAD16(u2, src, offset(src, 0, 2));
     LOAD16(u3, src, offset(src, 0, 3));
 
-    // Transpose the block
-    vec4 tmp;
-    tmp.xyz = u0.yzw;
-    u0.y    = u1.x;
-    u0.z    = u2.x;
-    u0.w    = u3.x;
-    u1.x    = tmp.x;
-    u2.x    = tmp.y;
-    u3.x    = tmp.z;
-    tmp.xy  = u1.zw;
-    u1.z    = u2.y;
-    u1.w    = u3.y;
-    u2.y    = tmp.x;
-    u3.y    = tmp.y;
-    tmp.x   = u2.w;
-    u2.w    = u3.z;
-    u3.z    = tmp.x;
-
-    // Store the block at (y, x)
+    // transpose the block
+    TRANSPOSE_4x4_func(u0, u1, u2, u3);
+
+    // store the block at (y, x)
     uint dst_offset_in_bytes = uint(16) * uint(gl_GlobalInvocationID.y) + uint(4) * uint(gl_GlobalInvocationID.x) * (dst.stride_y) + (dst.offset_first_element_in_bytes);
 
     STORE16(dst, uint((dst_offset_in_bytes + uint(0) * dst.stride_y) >> 2), u0);
@@ -106,148 +129,67 @@ void main(void)
     STORE16(dst, uint((dst_offset_in_bytes + uint(3) * dst.stride_y) >> 2), u3);
 }
 
-#elif defined(DATA_TYPE_FP16)
+#elif defined(DATA_TYPE_FP16) /* DATA_TYPE_FP32 */
 precision mediump float;
 
-layout(std140) uniform shader_params
-{
-    IMAGE_PARAM_DECLARATION(src);
-    IMAGE_PARAM_DECLARATION(dst);
-};
-
 #if defined(TRANSPOSE_4X4)
+
 BUFFER_DECLARATION(src, 1, uvec2, readonly);
 BUFFER_DECLARATION(dst, 2, uvec2, writeonly);
 
-/** This OpenGL ES kernel computes the matrix transposition of input matrix
- *
- * @param[in]  src_ptr                           Pointer to the source matrix. Supported data types: F16
- * @param[in]  src_stride_x                      Stride of the source matrix in X dimension (in bytes)
- * @param[in]  src_step_x                        src_stride_x * number of elements along X processed per workitem(in bytes)
- * @param[in]  src_stride_y                      Stride of the source matrix in Y dimension (in bytes)
- * @param[in]  src_step_y                        src_stride_y * number of elements along Y processed per workitem(in bytes)
- * @param[in]  src_offset_first_element_in_bytes The offset of the first element in the source matrix
- * @param[out] dst_ptr                           Pointer to the destination matrix Supported data type: same as src_ptr
- * @param[in]  dst_stride_x                      Stride of the destination matrix in X dimension (in bytes)
- * @param[in]  dst_step_x                        dst_gx_stride_x * number of elements along X processed per workitem(in bytes)
- * @param[in]  dst_stride_y                      Stride of the destination matrix in Y dimension (in bytes)
- * @param[in]  dst_step_y                        dst_gx_stride_y * number of elements along Y processed per workitem(in bytes)
- * @param[in]  dst_offset_first_element_in_bytes The offset of the first element in the destination matrix
- */
 void main(void)
 {
-    // Compute source address
+    // compute source address
     Image src = GC_CONVERT_TO_IMAGE_STRUCT(src);
     Image dst = GC_CONVERT_TO_IMAGE_STRUCT(dst);
 
-    // Load the NxN block at (x, y)
+    // load the NxN block at (x, y)
     vec4  u0;
     vec4  u1;
     vec4  u2;
     vec4  u3;
     uvec2 packed_s[4];
+
     GC_LOAD1_2D_OFFSET(packed_s[0], src, 0, 0);
     GC_LOAD1_2D_OFFSET(packed_s[1], src, 0, 1);
     GC_LOAD1_2D_OFFSET(packed_s[2], src, 0, 2);
     GC_LOAD1_2D_OFFSET(packed_s[3], src, 0, 3);
+
     u0 = vec4(unpackHalf2x16(packed_s[0].x), unpackHalf2x16(packed_s[0].y));
     u1 = vec4(unpackHalf2x16(packed_s[1].x), unpackHalf2x16(packed_s[1].y));
     u2 = vec4(unpackHalf2x16(packed_s[2].x), unpackHalf2x16(packed_s[2].y));
     u3 = vec4(unpackHalf2x16(packed_s[3].x), unpackHalf2x16(packed_s[3].y));
 
-    // Transpose the block
-    vec4 tmp;
-    tmp.xyz = u0.yzw;
-    u0.y    = u1.x;
-    u0.z    = u2.x;
-    u0.w    = u3.x;
-    u1.x    = tmp.x;
-    u2.x    = tmp.y;
-    u3.x    = tmp.z;
-    tmp.xy  = u1.zw;
-    u1.z    = u2.y;
-    u1.w    = u3.y;
-    u2.y    = tmp.x;
-    u3.y    = tmp.y;
-    tmp.x   = u2.w;
-    u2.w    = u3.z;
-    u3.z    = tmp.x;
-
-    // Store the block at (y, x)
+    // transpose the block
+    TRANSPOSE_4x4_func(u0, u1, u2, u3);
+
+    // store the block at (y, x)
     uint dst_offset_in_bytes = uint(8) * uint(gl_GlobalInvocationID.y) + uint(gl_GlobalInvocationID.x) * (dst_step_y) + (dst.offset_first_element_in_bytes);
+    dst.current_offset       = dst_offset_in_bytes;
 
     packed_s[0] = uvec2(packHalf2x16(u0.xy), packHalf2x16(u0.zw));
     packed_s[1] = uvec2(packHalf2x16(u1.xy), packHalf2x16(u1.zw));
     packed_s[2] = uvec2(packHalf2x16(u2.xy), packHalf2x16(u2.zw));
     packed_s[3] = uvec2(packHalf2x16(u3.xy), packHalf2x16(u3.zw));
-    GC_STORE1(packed_s[0], dst, uint((dst_offset_in_bytes + uint(0) * dst_stride_y) >> 3));
-    GC_STORE1(packed_s[1], dst, uint((dst_offset_in_bytes + uint(1) * dst_stride_y) >> 3));
-    GC_STORE1(packed_s[2], dst, uint((dst_offset_in_bytes + uint(2) * dst_stride_y) >> 3));
-    GC_STORE1(packed_s[3], dst, uint((dst_offset_in_bytes + uint(3) * dst_stride_y) >> 3));
+
+    GC_STORE1_2D_OFFSET(packed_s[0], dst, 0, 0);
+    GC_STORE1_2D_OFFSET(packed_s[1], dst, 0, 1);
+    GC_STORE1_2D_OFFSET(packed_s[2], dst, 0, 2);
+    GC_STORE1_2D_OFFSET(packed_s[3], dst, 0, 3);
 }
+
 #elif defined(TRANSPOSE_8X8) /* TRANSPOSE_4X4 */
+
 BUFFER_DECLARATION(src, 1, uvec4, readonly);
 BUFFER_DECLARATION(dst, 2, uvec4, writeonly);
 
-#define SWAP_ROW(u0, l0)     \
-    {                        \
-        tmp_swap = u0;       \
-        u0       = l0;       \
-        l0       = tmp_swap; \
-    }
-
-#define SWAP_4x4(u0, u1, u2, u3, l0, l1, l2, l3) \
-    {                                            \
-        vec4 tmp_swap;                           \
-        SWAP_ROW(u0, l0);                        \
-        SWAP_ROW(u1, l1);                        \
-        SWAP_ROW(u2, l2);                        \
-        SWAP_ROW(u3, l3);                        \
-    }
-
-#define TRANSPOSE_4x4(u0, u1, u2, u3) \
-    {                                 \
-        vec4 tmp;                     \
-        tmp.xyz = u0.yzw;             \
-        u0.y    = u1.x;               \
-        u0.z    = u2.x;               \
-        u0.w    = u3.x;               \
-        u1.x    = tmp.x;              \
-        u2.x    = tmp.y;              \
-        u3.x    = tmp.z;              \
-        tmp.xy  = u1.zw;              \
-        u1.z    = u2.y;               \
-        u1.w    = u3.y;               \
-        u2.y    = tmp.x;              \
-        u3.y    = tmp.y;              \
-        tmp.x   = u2.w;               \
-        u2.w    = u3.z;               \
-        u3.z    = tmp.x;              \
-    }
-
-/** This OpenGL ES kernel computes the matrix transposition of input matrix
- *
- * @param[in]  src_ptr                           Pointer to the source matrix. Supported data types:F16
- * @param[in]  src_stride_x                      Stride of the source matrix in X dimension (in bytes)
- * @param[in]  src_step_x                        src_stride_x * number of elements along X processed per workitem(in bytes)
- * @param[in]  src_stride_y                      Stride of the source matrix in Y dimension (in bytes)
- * @param[in]  src_step_y                        src_stride_y * number of elements along Y processed per workitem(in bytes)
- * @param[in]  src_offset_first_element_in_bytes The offset of the first element in the source matrix
- * @param[out] dst_ptr                           Pointer to the destination matrix Supported data type: same as src_ptr
- * @param[in]  dst_stride_x                      Stride of the destination matrix in X dimension (in bytes)
- * @param[in]  dst_step_x                        dst_gx_stride_x * number of elements along X processed per workitem(in bytes)
- * @param[in]  dst_stride_y                      Stride of the destination matrix in Y dimension (in bytes)
- * @param[in]  dst_step_y                        dst_gx_stride_y * number of elements along Y processed per workitem(in bytes)
- * @param[in]  dst_offset_first_element_in_bytes The offset of the first element in the destination matrix
- */
 void main(void)
 {
-    // Compute source address
+    // compute source address
     Image src = GC_CONVERT_TO_IMAGE_STRUCT(src);
     Image dst = GC_CONVERT_TO_IMAGE_STRUCT(dst);
 
-    vec4 u[8][2];
-
+    vec4  u[8][2];
     uvec4 packed_s[8];
 
     for(int i = 0; i < 8; i++)
@@ -257,21 +199,94 @@ void main(void)
         u[i][1] = vec4(unpackHalf2x16(packed_s[i].z), unpackHalf2x16(packed_s[i].w));
     }
 
-    // Transpose the block
-    TRANSPOSE_4x4(u[0][0], u[1][0], u[2][0], u[3][0]);
-    TRANSPOSE_4x4(u[0][1], u[1][1], u[2][1], u[3][1]);
-    TRANSPOSE_4x4(u[4][0], u[5][0], u[6][0], u[7][0]);
-    TRANSPOSE_4x4(u[4][1], u[5][1], u[6][1], u[7][1]);
-    SWAP_4x4(u[0][1], u[1][1], u[2][1], u[3][1], u[4][0], u[5][0], u[6][0], u[7][0]);
+    // transpose the block
+    TRANSPOSE_4x4_func(u[0][0], u[1][0], u[2][0], u[3][0]);
+    TRANSPOSE_4x4_func(u[0][1], u[1][1], u[2][1], u[3][1]);
+    TRANSPOSE_4x4_func(u[4][0], u[5][0], u[6][0], u[7][0]);
+    TRANSPOSE_4x4_func(u[4][1], u[5][1], u[6][1], u[7][1]);
+    SWAP_4x4_func(u[0][1], u[1][1], u[2][1], u[3][1], u[4][0], u[5][0], u[6][0], u[7][0]);
 
-    // Store the block at (y, x)
+    // store the block at (y, x)
     uint dst_offset_in_bytes = uint(16) * uint(gl_GlobalInvocationID.y) + uint(gl_GlobalInvocationID.x) * (dst_step_y) + (dst.offset_first_element_in_bytes);
+    dst.current_offset       = dst_offset_in_bytes;
 
     for(int i = 0; i < 8; i++)
     {
         packed_s[i] = uvec4(packHalf2x16(u[i][0].xy), packHalf2x16(u[i][0].zw), packHalf2x16(u[i][1].xy), packHalf2x16(u[i][1].zw));
-        GC_STORE1(packed_s[i], dst, uint((dst_offset_in_bytes + uint(i) * dst_stride_y) >> 4));
+        GC_STORE1_2D_OFFSET(packed_s[i], dst, 0, i);
+    }
+}
+
+#elif defined(TRANSPOSE_8X8_SQUARE) /* TRANSPOSE_4X4 */
+
+BUFFER_DECLARATION(src, 1, uvec4, readonly);
+BUFFER_DECLARATION(dst, 2, uvec4, writeonly);
+
+void main(void)
+{
+    Image src = GC_CONVERT_TO_IMAGE_STRUCT(src);
+    Image dst = GC_CONVERT_TO_IMAGE_STRUCT(dst);
+
+    if(gl_GlobalInvocationID.x <= gl_GlobalInvocationID.y)
+    {
+        uint blk1_offset_in_bytes = src.current_offset;
+        uint blk2_offset_in_bytes = uint(16) * uint(gl_GlobalInvocationID.y) + uint(gl_GlobalInvocationID.x) * (dst_step_y) + (dst.offset_first_element_in_bytes);
+
+        // load block1
+        vec4  u1[8][2];
+        uvec4 packed_s[8];
+
+        src.current_offset = blk1_offset_in_bytes;
+        for(int i = 0; i < 8; i++)
+        {
+            GC_LOAD1_2D_OFFSET(packed_s[i], src, 0, i);
+            u1[i][0] = vec4(unpackHalf2x16(packed_s[i].x), unpackHalf2x16(packed_s[i].y));
+            u1[i][1] = vec4(unpackHalf2x16(packed_s[i].z), unpackHalf2x16(packed_s[i].w));
+        }
+
+        // transpose block1
+        TRANSPOSE_4x4_func(u1[0][0], u1[1][0], u1[2][0], u1[3][0]);
+        TRANSPOSE_4x4_func(u1[0][1], u1[1][1], u1[2][1], u1[3][1]);
+        TRANSPOSE_4x4_func(u1[4][0], u1[5][0], u1[6][0], u1[7][0]);
+        TRANSPOSE_4x4_func(u1[4][1], u1[5][1], u1[6][1], u1[7][1]);
+        SWAP_4x4_func(u1[0][1], u1[1][1], u1[2][1], u1[3][1], u1[4][0], u1[5][0], u1[6][0], u1[7][0]);
+
+        // write to block2
+        dst.current_offset = blk2_offset_in_bytes;
+        for(int i = 0; i < 8; i++)
+        {
+            packed_s[i] = uvec4(packHalf2x16(u1[i][0].xy), packHalf2x16(u1[i][0].zw), packHalf2x16(u1[i][1].xy), packHalf2x16(u1[i][1].zw));
+            GC_STORE1_2D_OFFSET(packed_s[i], dst, 0, i);
+        }
+
+        // load block2
+        vec4 u2[8][2];
+
+        src.current_offset = blk2_offset_in_bytes;
+        for(int i = 0; i < 8; i++)
+        {
+            GC_LOAD1_2D_OFFSET(packed_s[i], src, 0, i);
+            u2[i][0] = vec4(unpackHalf2x16(packed_s[i].x), unpackHalf2x16(packed_s[i].y));
+            u2[i][1] = vec4(unpackHalf2x16(packed_s[i].z), unpackHalf2x16(packed_s[i].w));
+        }
+
+        // transpose block2
+        TRANSPOSE_4x4_func(u2[0][0], u2[1][0], u2[2][0], u2[3][0]);
+        TRANSPOSE_4x4_func(u2[0][1], u2[1][1], u2[2][1], u2[3][1]);
+        TRANSPOSE_4x4_func(u2[4][0], u2[5][0], u2[6][0], u2[7][0]);
+        TRANSPOSE_4x4_func(u2[4][1], u2[5][1], u2[6][1], u2[7][1]);
+        SWAP_4x4_func(u2[0][1], u2[1][1], u2[2][1], u2[3][1], u2[4][0], u2[5][0], u2[6][0], u2[7][0]);
+
+        // write to block1
+        dst.current_offset = blk1_offset_in_bytes;
+        for(int i = 0; i < 8; i++)
+        {
+            packed_s[i] = uvec4(packHalf2x16(u2[i][0].xy), packHalf2x16(u2[i][0].zw), packHalf2x16(u2[i][1].xy), packHalf2x16(u2[i][1].zw));
+            GC_STORE1_2D_OFFSET(packed_s[i], dst, 0, i);
+        }
     }
 }
+
 #endif /* TRANSPOSE_4X4 */
-#endif /*ARM_COMPUTE_ENABLE_FP16*/
+
+#endif /* DATA_TYPE_FP32 */