COMPMID-1704: Collapse the 4th dimension in CLPoolingLayerKernel

Change-Id: I76e57af6608b55b6f59a5d06aecc30063ee4c3cc
Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/155733
Tested-by: bsgcomp <bsgcomp@arm.com>
Reviewed-by: Michele DiGiorgio <michele.digiorgio@arm.com>
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>

3 files changed, 78 insertions, 26 deletions

diff --git a/src/core/CL/cl_kernels/pooling_layer.cl b/src/core/CL/cl_kernels/pooling_layer.cl
index 080835348d..7d15d100e9 100644
--- a/src/core/CL/cl_kernels/pooling_layer.cl
+++ b/src/core/CL/cl_kernels/pooling_layer.cl
@@ -489,7 +489,11 @@ DATA_TYPE calculate_avg_scale_nhwc(const int pool_size_x, const int pool_size_y,
                                    const int pad_x, const int pad_y, const int stride_x, const int stride_y)
 {
     int start_x = get_global_id(1) * stride_x - pad_x;
+#if defined(DST_DEPTH)
+    int start_y = (get_global_id(2) % DST_DEPTH) * stride_y - pad_y;
+#else  /* defined(DST_DEPTH) */
     int start_y = get_global_id(2) * stride_y - pad_y;
+#endif /* defined(DST_DEPTH) */
 
 #if !defined(EXCLUDE_PADDING)
     upper_bound_w += pad_x;
@@ -522,30 +526,43 @@ DATA_TYPE calculate_avg_scale_nhwc(const int pool_size_x, const int pool_size_y,
  * @param[in]  input_step_y                         input_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in]  input_stride_z                       Stride of the source tensor in Z dimension (in bytes)
  * @param[in]  input_step_z                         input_stride_z * number of elements along Z processed per workitem(in bytes)
+ * @param[in]  input_stride_w                       Stride of the source tensor in W dimension (in bytes)
+ * @param[in]  input_step_w                         input_stride_w * number of elements along W processed per workitem(in bytes)
  * @param[in]  input_offset_first_element_in_bytes  The offset of the first element in the source image
  * @param[out] output_ptr                           Pointer to the destination image. Supported data types: same as @p input_ptr
- * @param[in]  output_stride_x                      Stride of the destination image in X dimension (in bytes)
+ * @param[in]  output_stride_x                      Stride of the destination tensor in X dimension (in bytes)
  * @param[in]  output_step_x                        output_stride_x * number of elements along X processed per workitem(in bytes)
- * @param[in]  output_stride_y                      Stride of the destination image in Y dimension (in bytes)
+ * @param[in]  output_stride_y                      Stride of the destination tensor in Y dimension (in bytes)
  * @param[in]  output_step_y                        output_stride_y * number of elements along Y processed per workitem(in bytes)
- * @param[in]  output_stride_z                      Stride of the source tensor in Z dimension (in bytes)
+ * @param[in]  output_stride_z                      Stride of the destination tensor in Z dimension (in bytes)
  * @param[in]  output_step_z                        output_stride_z * number of elements along Z processed per workitem(in bytes)
+ * @param[in]  output_stride_w                      Stride of the destination tensor in W dimension (in bytes)
+ * @param[in]  output_step_w                        output_stride_w * number of elements along W processed per workitem(in bytes)
  * @param[in]  output_offset_first_element_in_bytes The offset of the first element in the destination image
  */
 __kernel void pooling_layer_MxN_nhwc(
-    TENSOR3D_DECLARATION(input),
-    TENSOR3D_DECLARATION(output))
+    TENSOR4D_DECLARATION(input),
+    TENSOR4D_DECLARATION(output))
 {
     // Get pixels pointer
-    Tensor3D input  = CONVERT_TO_TENSOR3D_STRUCT(input);
-    Tensor3D output = CONVERT_TO_TENSOR3D_STRUCT(output);
+#if defined(DST_DEPTH)
+    Tensor4D input  = CONVERT_TO_TENSOR4D_STRUCT(input, DST_DEPTH);
+    Tensor4D output = CONVERT_TO_TENSOR4D_STRUCT(output, DST_DEPTH);
+#else  /* defined(DST_DEPTH) */
+    Tensor3D  input      = CONVERT_TO_TENSOR3D_STRUCT(input);
+    Tensor3D  output     = CONVERT_TO_TENSOR3D_STRUCT(output);
+#endif /* defined(DST_DEPTH) */
 
     VEC_DATA_TYPE(DATA_TYPE, 8)
     vdata           = INITIAL_VALUE;
     DATA_TYPE sdata = INITIAL_VALUE;
 
-    const int idx_width  = get_global_id(1) * STRIDE_X;
+    const int idx_width = get_global_id(1) * STRIDE_X;
+#if defined(DST_DEPTH)
+    const int idx_height = (get_global_id(2) % DST_DEPTH) * STRIDE_Y;
+#else  /* defined(DST_DEPTH) */
     const int idx_height = get_global_id(2) * STRIDE_Y;
+#endif /* defined(DST_DEPTH) */
 
     for(int y = 0; y < POOL_SIZE_Y; ++y)
     {
@@ -555,8 +572,14 @@ __kernel void pooling_layer_MxN_nhwc(
             int x1 = select(x, PAD_X - idx_width - 1, x + idx_width - PAD_X < 0 || x + idx_width - PAD_X >= MAX_WIDTH);
             x1     = select(x1, PAD_X - idx_width - 1, y != y1);
 
+#if defined(DST_DEPTH)
+            VEC_DATA_TYPE(DATA_TYPE, 8)
+            data0 = vload8(0, (__global DATA_TYPE *)tensor4D_offset(&input, 0, x1 - PAD_X, y1 - PAD_Y, 0));
+#else  /* defined(DST_DEPTH) */
             VEC_DATA_TYPE(DATA_TYPE, 8)
             data0 = vload8(0, (__global DATA_TYPE *)tensor3D_offset(&input, 0, x1 - PAD_X, y1 - PAD_Y));
+#endif /* defined(DST_DEPTH) */
+
 #if defined(POOL_L2)
             // Raise to power of 2 for L2 Pooling
             data0 *= data0;
diff --git a/src/core/CL/cl_kernels/pooling_layer_quantized.cl b/src/core/CL/cl_kernels/pooling_layer_quantized.cl
index 17d893a013..58d89871e3 100644
--- a/src/core/CL/cl_kernels/pooling_layer_quantized.cl
+++ b/src/core/CL/cl_kernels/pooling_layer_quantized.cl
@@ -126,7 +126,11 @@ int calculate_avg_scale_nhwc(const int pool_size_x, const int pool_size_y, int u
                              const int pad_x, const int pad_y, const int stride_x, const int stride_y)
 {
     int start_x = get_global_id(1) * stride_x - pad_x;
-    int start_y = get_global_id(2) * stride_y - pad_y;
+#if defined(DST_DEPTH)
+    int start_y = (get_global_id(2) % DST_DEPTH) * stride_y - pad_y;
+#else  /* defined(DST_DEPTH) */
+    int  start_y = get_global_id(2) * stride_y - pad_y;
+#endif /* defined(DST_DEPTH) */
 
     const int end_x = min(start_x + pool_size_x, upper_bound_w);
     const int end_y = min(start_y + pool_size_y, upper_bound_h);
@@ -153,39 +157,58 @@ int calculate_avg_scale_nhwc(const int pool_size_x, const int pool_size_y, int u
  * @param[in]  input_step_y                         input_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in]  input_stride_z                       Stride of the source tensor in Z dimension (in bytes)
  * @param[in]  input_step_z                         input_stride_z * number of elements along Z processed per workitem(in bytes)
+ * @param[in]  input_stride_w                       Stride of the source tensor in W dimension (in bytes)
+ * @param[in]  input_step_w                         input_stride_w * number of elements along W processed per workitem(in bytes)
  * @param[in]  input_offset_first_element_in_bytes  The offset of the first element in the source image
  * @param[out] output_ptr                           Pointer to the destination image. Supported data types: same as @p input_ptr
- * @param[in]  output_stride_x                      Stride of the destination image in X dimension (in bytes)
+ * @param[in]  output_stride_x                      Stride of the destination tensor in X dimension (in bytes)
  * @param[in]  output_step_x                        output_stride_x * number of elements along X processed per workitem(in bytes)
- * @param[in]  output_stride_y                      Stride of the destination image in Y dimension (in bytes)
+ * @param[in]  output_stride_y                      Stride of the destination tensor in Y dimension (in bytes)
  * @param[in]  output_step_y                        output_stride_y * number of elements along Y processed per workitem(in bytes)
- * @param[in]  output_stride_z                      Stride of the source tensor in Z dimension (in bytes)
+ * @param[in]  output_stride_z                      Stride of the destination tensor in Z dimension (in bytes)
  * @param[in]  output_step_z                        output_stride_z * number of elements along Z processed per workitem(in bytes)
+ * @param[in]  output_stride_w                      Stride of the destination tensor in W dimension (in bytes)
+ * @param[in]  output_step_w                        output_stride_w * number of elements along W processed per workitem(in bytes)
  * @param[in]  output_offset_first_element_in_bytes The offset of the first element in the destination image
  */
 __kernel void pooling_layer_MxN_quantized_nhwc(
-    TENSOR3D_DECLARATION(input),
-    TENSOR3D_DECLARATION(output))
+    TENSOR4D_DECLARATION(input),
+    TENSOR4D_DECLARATION(output))
 {
     // Get pixels pointer
+#if defined(DST_DEPTH)
+    Tensor4D input  = CONVERT_TO_TENSOR4D_STRUCT(input, DST_DEPTH);
+    Tensor4D output = CONVERT_TO_TENSOR4D_STRUCT(output, DST_DEPTH);
+#else  /* defined(DST_DEPTH) */
     Tensor3D input  = CONVERT_TO_TENSOR3D_STRUCT(input);
     Tensor3D output = CONVERT_TO_TENSOR3D_STRUCT(output);
+#endif /* defined(DST_DEPTH) */
 
     int8 vdata = 0;
 
-    const int idx_width  = get_global_id(1) * STRIDE_X;
+    const int idx_width = get_global_id(1) * STRIDE_X;
+#if defined(DST_DEPTH)
+    const int idx_height = (get_global_id(2) % DST_DEPTH) * STRIDE_Y;
+#else  /* defined(DST_DEPTH) */
     const int idx_height = get_global_id(2) * STRIDE_Y;
+#endif /* defined(DST_DEPTH) */
 
     for(int y = 0; y < POOL_SIZE_Y; ++y)
     {
         int y1 = select(y, PAD_Y - idx_height, y + idx_height < PAD_Y || y + idx_height > MAX_HEIGHT);
         for(int x = 0; x < POOL_SIZE_X; ++x)
         {
-            int x1      = select(x, PAD_X - idx_width - 1, x + idx_width < PAD_X || x + idx_width > MAX_WIDTH);
-            x1          = select(x1, PAD_X - idx_width - 1, y != y1);
+            int x1 = select(x, PAD_X - idx_width - 1, x + idx_width < PAD_X || x + idx_width > MAX_WIDTH);
+            x1     = select(x1, PAD_X - idx_width - 1, y != y1);
+
+#if defined(DST_DEPTH)
+            uchar8 data = vload8(0, (__global uchar *)tensor4D_offset(&input, 0, x1 - PAD_X, y1 - PAD_Y, 0));
+#else  /* defined(DST_DEPTH) */
             uchar8 data = vload8(0, (__global uchar *)tensor3D_offset(&input, 0, x1 - PAD_X, y1 - PAD_Y));
-            int8 data0  = convert_int8(data);
-            vdata       = POOL_OP(vdata, data0);
+#endif /* defined(DST_DEPTH) */
+
+            int8 data0 = convert_int8(data);
+            vdata      = POOL_OP(vdata, data0);
         }
     }
 
diff --git a/src/core/CL/kernels/CLPoolingLayerKernel.cpp b/src/core/CL/kernels/CLPoolingLayerKernel.cpp
index df13068239..bd21ea0a6c 100644
--- a/src/core/CL/kernels/CLPoolingLayerKernel.cpp
+++ b/src/core/CL/kernels/CLPoolingLayerKernel.cpp
@@ -257,6 +257,8 @@ void CLPoolingLayerKernel::configure(const ICLTensor *input, ICLTensor *output,
             build_opts.add_option_if(exclude_padding, "-DEXCLUDE_PADDING");
             build_opts.add_option("-DMAX_WIDTH=" + support::cpp11::to_string(input->info()->dimension(idx_width)));
             build_opts.add_option("-DMAX_HEIGHT=" + support::cpp11::to_string(input->info()->dimension(idx_height)));
+            build_opts.add_option_if(output->info()->tensor_shape().total_size_upper(3) > 1,
+                                     "-DDST_DEPTH=" + support::cpp11::to_string(output->info()->dimension(idx_height)));
             std::string kernel_name = is_data_type_quantized_asymmetric(data_type) ? "pooling_layer_MxN_quantized_nhwc" : "pooling_layer_MxN_nhwc";
             _kernel                 = static_cast<cl::Kernel>(CLKernelLibrary::get().create_kernel(kernel_name, build_opts.options()));
             break;
@@ -315,12 +317,14 @@ void CLPoolingLayerKernel::run(const Window &window, cl::CommandQueue &queue)
     unsigned int pool_stride_y = 0;
     std::tie(pool_stride_x, pool_stride_y) = _pool_info.pad_stride_info().stride();
 
+    // Collapse window
+    Window window_collapsed = window.collapse_if_possible(ICLKernel::window(), Window::DimZ);
+
     switch(_input->info()->data_layout())
     {
         case DataLayout::NCHW:
         {
-            Window window_collapsed = window.collapse_if_possible(ICLKernel::window(), Window::DimZ);
-            Window slice            = window_collapsed.first_slice_window_3D();
+            Window slice = window_collapsed.first_slice_window_3D();
             do
             {
                 // Upsample input by pool size
@@ -343,21 +347,23 @@ void CLPoolingLayerKernel::run(const Window &window, cl::CommandQueue &queue)
         }
         case DataLayout::NHWC:
         {
-            Window slice = window.first_slice_window_3D();
+            const size_t total_batches = _output->info()->tensor_shape().total_size_upper(3);
 
-            Window in_slice = window.first_slice_window_3D();
+            Window slice    = window_collapsed.first_slice_window_4D();
+            Window in_slice = window_collapsed.first_slice_window_4D();
             in_slice.set(Window::DimX, Window::Dimension(0, _input->info()->dimension(0), _num_elems_processed_per_iteration));
             in_slice.set(Window::DimY, Window::Dimension(0, _input->info()->dimension(1), pool_stride_x));
             in_slice.set(Window::DimZ, Window::Dimension(0, _input->info()->dimension(2), pool_stride_y));
+            in_slice.set(3, Window::Dimension(0, total_batches, 1));
             do
             {
                 // Set inputs
                 unsigned int idx = 0;
-                add_3D_tensor_argument(idx, _input, in_slice);
-                add_3D_tensor_argument(idx, _output, slice);
+                add_4D_tensor_argument(idx, _input, in_slice);
+                add_4D_tensor_argument(idx, _output, slice);
                 enqueue(queue, *this, slice, lws_hint());
             }
-            while(window.slide_window_slice_3D(slice) && window.slide_window_slice_3D(in_slice));
+            while(window.slide_window_slice_4D(slice) && window.slide_window_slice_4D(in_slice));
             break;
         }
         default: