COMPMID-1413 - Improve the performance of GEMMLowp with 8 bit dot product on OpenCL

COMPMID-1424 - Add dot product support for CLDepthwise QASYMM8 3x3 NHWC non-unit stride

With this patch we are able to improve the performance of MobileNet v1-qasymm8 by 37 %
Tried to use the dot product instruction in CLDepthwise QASYMM8 3x3 NHWC non-unit stride
but I have not seen any benefit (maybe because we have few arithemtic operation and we
do not have more load instructions). However Depthwise convolution has been improved by
30%

Change-Id: Id768a99c2e53a04276707e427af5d0ec93419ada
Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/155082
Tested-by: bsgcomp <bsgcomp@arm.com>
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>

4 files changed, 71 insertions, 58 deletions

diff --git a/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h b/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h
index 48b880174d..fbf0c08b36 100644
--- a/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h
+++ b/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h
@@ -157,43 +157,48 @@ public:
 private:
     /** Configures the appropriate matrix multiply routine
      *
-     * @param[in]      input         Input tensor. Data types supported: QASYMM8/F16/F32.
-     * @param[in]      weights       Weights tensor. Data type supported: Same as @p input.
-     * @param[in, out] output        Output tensor. Data types supported: Same as @p input,
-     *                               except for input of QASYMM8 type where output should be of S32 type.
-     * @param[in]      gemm_3d_depth (Optional) Depth of GEMM 3D (Defaults to 1)
+     * @param[in]      input                 Input tensor. Data types supported: QASYMM8/F16/F32.
+     * @param[in]      weights               Weights tensor. Data type supported: Same as @p input.
+     * @param[in]      biases                Biases tensor. Shared biases supported. Biases are 1D tensor with dimensions [OFM].
+     *                                       Data type supported: Should match @p input data type, except for input of QASYMM8 type where biases should be of S32 type.
+     * @param[in, out] output                Output tensor. Data types supported: Same as @p input,
+     *                                       except for input of QASYMM8 type where output should be of S32 type.
+     * @param[in]      gemmlowp_output_stage GEMMLowp output stage info
+     * @param[in]      gemm_3d_depth         (Optional) Depth of GEMM 3D (Defaults to 1)
      */
-    void configure_mm(const ICLTensor *input, const ICLTensor *weights, ICLTensor *output, int gemm_3d_depth = 1);
+    void configure_mm(const ICLTensor *input, const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, const GEMMLowpOutputStageInfo &gemmlowp_output_stage, int gemm_3d_depth = 1);
     /** Static function to check if given info will lead to a valid configuration of @ref CLGEMMConvolutionLayer matrix multiply routines
      *
-     * @param[in] input         Input tensor. Data types supported: QASYMM8/F16/F32.
-     * @param[in] weights       Weights tensor. Data type supported: Same as @p input.
-     * @param[in] output        Output tensor. Data types supported: Same as @p input,
-     *                          except for input of QASYMM8 type where output should be of S32 type.
-     * @param[in] gemm_3d_depth (Optional) Depth of GEMM 3D (Defaults to 1)
-     * @param[in] skip_im2col   (Optional) Flag which specifies if im2col has to be skipped. i.e. 1x1 convolution with NHWC data layout. (Default to false)
+     * @param[in] input                 Input tensor. Data types supported: QASYMM8/F16/F32.
+     * @param[in] weights               Weights tensor. Data type supported: Same as @p input.
+     * @param[in] output                Output tensor. Data types supported: Same as @p input,
+     *                                  except for input of QASYMM8 type where output should be of S32 type.
+     * @param[in] biases                Biases tensor. Shared biases supported. Biases are 1D tensor with dimensions [OFM].
+     *                                  Data type supported: Should match @p input data type, except for input of QASYMM8 type where biases should be of S32 type.
+     * @param[in] gemmlowp_output_stage GEMMLowp output stage info
+     * @param[in] gemm_3d_depth         (Optional) Depth of GEMM 3D (Defaults to 1)
+     * @param[in] skip_im2col           (Optional) Flag which specifies if im2col has to be skipped. i.e. 1x1 convolution with NHWC data layout. (Default to false)
      *
      * @return a status
      */
-    static Status validate_mm(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *output, int gemm_3d_depth = 1, bool skip_im2col = false);
+    static Status validate_mm(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const GEMMLowpOutputStageInfo &gemmlowp_output_stage,
+                              int gemm_3d_depth = 1, bool skip_im2col = false);
 
 private:
-    CLMemoryGroup                                  _memory_group;
-    CLConvolutionLayerReshapeWeights               _reshape_weights;
-    CLIm2ColKernel                                 _im2col_kernel;
-    CLGEMM                                         _mm_gemm;
-    CLGEMMLowpMatrixMultiplyCore                   _mm_gemmlowp;
-    CLGEMMLowpQuantizeDownInt32ToUint8ScaleByFloat _gemmlowp_output_stage;
-    CLCol2ImKernel                                 _col2im_kernel;
-    CLActivationLayer                              _activationlayer_function;
-    CLArithmeticAdditionKernel                     _add_bias_kernel;
+    CLMemoryGroup                    _memory_group;
+    CLConvolutionLayerReshapeWeights _reshape_weights;
+    CLIm2ColKernel                   _im2col_kernel;
+    CLGEMM                           _mm_gemm;
+    CLGEMMLowpMatrixMultiplyCore     _mm_gemmlowp;
+    CLCol2ImKernel                   _col2im_kernel;
+    CLActivationLayer                _activationlayer_function;
+    CLArithmeticAdditionKernel       _add_bias_kernel;
 
     const ICLTensor *_original_weights;
 
     CLTensor _im2col_output;
     CLTensor _weights_reshaped;
     CLTensor _gemm_output;
-    CLTensor _tmp_output;
 
     DataLayout _data_layout;
 
diff --git a/arm_compute/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.h b/arm_compute/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.h
index f404ccdf4c..82f307a773 100644
--- a/arm_compute/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.h
+++ b/arm_compute/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.h
@@ -27,6 +27,7 @@
 #include "arm_compute/core/CL/kernels/CLGEMMInterleave4x4Kernel.h"
 #include "arm_compute/core/CL/kernels/CLGEMMLowpMatrixMultiplyKernel.h"
 #include "arm_compute/core/CL/kernels/CLGEMMLowpOffsetContributionKernel.h"
+#include "arm_compute/core/CL/kernels/CLGEMMLowpOffsetContributionOutputStageKernel.h"
 #include "arm_compute/core/CL/kernels/CLGEMMLowpReductionKernel.h"
 #include "arm_compute/core/CL/kernels/CLGEMMTranspose1xWKernel.h"
 #include "arm_compute/runtime/CL/CLMemoryGroup.h"
@@ -45,7 +46,8 @@ class ICLTensor;
  *  -# @ref CLGEMMLowpMatrixMultiplyKernel
  *  -# @ref CLGEMMLowpMatrixAReductionKernel (if the offset of matrix B is not 0)
  *  -# @ref CLGEMMLowpMatrixBReductionKernel (if the offset of matrix A is not 0)
- *  -# @ref CLGEMMLowpOffsetContributionKernel
+ *  -# @ref CLGEMMLowpOffsetContributionKernel (if gemm_info.gemmlowp_output_stage == NONE)
+ *  -# @ref CLGEMMLowpOffsetContributionOutputStageKernel (if gemm_info.gemmlowp_output_stage != NONE)
  *
 */
 class CLGEMMLowpMatrixMultiplyCore : public IFunction
@@ -63,54 +65,60 @@ public:
     CLGEMMLowpMatrixMultiplyCore &operator=(CLGEMMLowpMatrixMultiplyCore &&) = default;
     /** Initialise the kernel's inputs, output
      *
-     * @note GEMM_LOWP:  low precision GEMM kernel
+     * @note GEMMLowp:  low precision GEMM kernel. [A * B + C]
      *  This kernel performs the following computations:
      *
      *  -# Convert a values from QASYMM8 to int32 and add a_offset to each of them.
      *  -# Convert b values from QASYMM8 to int32 add b_offset to each of them.
      *  -# Compute the matrix product of the resulting a * b in int32.
+     *  -# Quantize to uint8 if gemm_info.gemmlowp_output_stage != NONE
      *
      * @param[in]  a         First input tensor  (Matrix A). Data type supported: QASYMM8.
      * @param[in]  b         Second input tensor (Matrix B). Data type supported: same as @p a
-     * @param[out] output    Output tensor. Data type supported: Data type supported: S32
+     * @param[in]  c         Third input tensor  (Matrix C). It can be a nullptr. Data type supported: S32
+     * @param[out] output    Output tensor. Data type supported: S32 or QASYMM8 if gemm_info.gemmlowp_output_stage != NONE
      * @param[in]  gemm_info (Optional) Specifies if the matrix A and/or matrix B have been reshaped and
      *                       if the reshape of matrix B should be executed only for the first run
      */
-    void configure(const ICLTensor *a, const ICLTensor *b, ICLTensor *output, const GEMMInfo &gemm_info = GEMMInfo());
+    void configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor *c, ICLTensor *output, const GEMMInfo &gemm_info = GEMMInfo());
     /** Static function to check if given info will lead to a valid configuration of @ref CLGEMMLowpMatrixMultiplyCore
      *
      * @param[in] a         First input tensor  (Matrix A). Data type supported: QASYMM8.
      * @param[in] b         Second input tensor (Matrix B). Data type supported: same as @p a
-     * @param[in] output    Output tensor. Data type supported: Data type supported: S32
+     * @param[in] c         Third input tensor  (Matrix C). It can be a nullptr. Data type supported: S32
+     * @param[in] output    Output tensor. Data type supported: S32 or QASYMM8 if gemm_info.gemmlowp_output_stage != NONE
      * @param[in] gemm_info (Optional) Specifies if the matrix A and/or matrix B have been reshaped and
      *                      if the reshape of matrix B should be executed only for the first run
      *
      * @return a status
      */
-    static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *output, const GEMMInfo &gemm_info = GEMMInfo());
+    static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *output, const GEMMInfo &gemm_info = GEMMInfo());
 
     // Inherited methods overridden:
     void run() override;
     void prepare() override;
 
 private:
-    CLMemoryGroup                      _memory_group;
-    CLGEMMLowpMatrixMultiplyKernel     _mm_kernel;
-    CLGEMMInterleave4x4Kernel          _mtx_a_reshape_kernel;
-    CLGEMMTranspose1xWKernel           _mtx_b_reshape_kernel;
-    CLGEMMLowpMatrixAReductionKernel   _mtx_a_reduction_kernel;
-    CLGEMMLowpMatrixBReductionKernel   _mtx_b_reduction_kernel;
-    CLGEMMLowpOffsetContributionKernel _offset_contribution_kernel;
-    CLTensor                           _vector_sum_col;
-    CLTensor                           _vector_sum_row;
-    CLTensor                           _tmp_a;
-    CLTensor                           _tmp_b;
-    const ICLTensor                   *_original_b;
-    int32_t                            _a_offset;
-    int32_t                            _b_offset;
-    bool                               _is_interleaved_transposed;
-    bool                               _reshape_b_only_on_first_run;
-    bool                               _is_prepared;
+    CLMemoryGroup                                 _memory_group;
+    CLGEMMLowpMatrixMultiplyKernel                _mm_kernel;
+    CLGEMMInterleave4x4Kernel                     _mtx_a_reshape_kernel;
+    CLGEMMTranspose1xWKernel                      _mtx_b_reshape_kernel;
+    CLGEMMLowpMatrixAReductionKernel              _mtx_a_reduction_kernel;
+    CLGEMMLowpMatrixBReductionKernel              _mtx_b_reduction_kernel;
+    CLGEMMLowpOffsetContributionKernel            _offset_contribution_kernel;
+    CLGEMMLowpOffsetContributionOutputStageKernel _offset_contribution_output_stage_kernel;
+    CLTensor                                      _vector_sum_col;
+    CLTensor                                      _vector_sum_row;
+    CLTensor                                      _tmp_a;
+    CLTensor                                      _tmp_b;
+    CLTensor                                      _mm_result_s32;
+    const ICLTensor                              *_original_b;
+    int32_t                                       _a_offset;
+    int32_t                                       _b_offset;
+    bool                                          _is_interleaved_transposed;
+    bool                                          _reshape_b_only_on_first_run;
+    bool                                          _is_prepared;
+    bool                                          _fuse_output_stage;
 };
 }
 #endif /*__ARM_COMPUTE_CLGEMMLOWPMATRIXMULTIPLYCORE_H__ */
diff --git a/arm_compute/runtime/CL/functions/CLGEMMLowpOutputStage.h b/arm_compute/runtime/CL/functions/CLGEMMLowpOutputStage.h
index 51fcbe9392..3330b40d8a 100644
--- a/arm_compute/runtime/CL/functions/CLGEMMLowpOutputStage.h
+++ b/arm_compute/runtime/CL/functions/CLGEMMLowpOutputStage.h
@@ -131,24 +131,22 @@ public:
      * @param[in]  min                          (Optional) Min value used to saturate down the output result before converting back to QASYMM8
      * @param[in]  max                          (Optional) Max value used to saturate up the output result before converting back to QASYMM8,
      *                                          Along with @p min, this value can be used to implement "rectified linear unit" activation functions
-     * @param[in]  output_3d_depth              (Optional) Depth of output in 3D (Defaults to 1)
      */
     void configure(const ICLTensor *input, const ICLTensor *bias, ICLTensor *output, int result_fixedpoint_multiplier, int result_shift, int result_offset_after_shift,
-                   int min = 0, int max = 0, unsigned int output_3d_depth = 1);
+                   int min = 0, int max = 0);
     /** Static function to check if given info will lead to a valid configuration of @ref CLGEMMLowpQuantizeDownInt32ToUint8ScaleByFixedPoint
      *
-     * @param[in] input           Input tensor. It is the output of @ref CLGEMMLowpMatrixMultiplyCore function. Data type supported: S32
-     * @param[in] bias            Biases tensor. Only shared biases supported and it can be a nullptr if the addition of biases is not required.
-     *                            Biases are 1D tensor with dimensions [OFM]. Data type supported: Same as @p input.
-     * @param[in] output          Output tensor. Data type supported: Data type supported: QASYMM8
-     * @param[in] min             (Optional) Min value used to saturate down the output result before converting back to QASYMM8
-     * @param[in] max             (Optional) Max value used to saturate up the output result before converting back to QASYMM8,
+     * @param[in] input  Input tensor. It is the output of @ref CLGEMMLowpMatrixMultiplyCore function. Data type supported: S32
+     * @param[in] bias   Biases tensor. Only shared biases supported and it can be a nullptr if the addition of biases is not required.
+     *                   Biases are 1D tensor with dimensions [OFM]. Data type supported: Same as @p input.
+     * @param[in] output Output tensor. Data type supported: Data type supported: QASYMM8
+     * @param[in] min    (Optional) Min value used to saturate down the output result before converting back to QASYMM8
+     * @param[in] max    (Optional) Max value used to saturate up the output result before converting back to QASYMM8,
      *                            Along with @p min, this value can be used to implement "rectified linear unit" activation functions
-     * @param[in] output_3d_depth (Optional) Depth of output in 3D (Defaults to 1)
      *
      * @return a status
      */
-    static Status validate(const ITensorInfo *input, const ITensorInfo *bias, const ITensorInfo *output, int min = 0, int max = 0, unsigned int output_3d_depth = 1);
+    static Status validate(const ITensorInfo *input, const ITensorInfo *bias, const ITensorInfo *output, int min = 0, int max = 0);
 };
 
 /** Basic function to execute CLGEMMLowpQuantizeDownInt32ToUint8ScaleByFloat on OpenCL.
diff --git a/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h b/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h
index 3db76f423c..682475c824 100644
--- a/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h
+++ b/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h
@@ -75,22 +75,24 @@ public:
      *
      * @param[in]  a         First input tensor  (Matrix A). Data type supported: QASYMM8.
      * @param[in]  b         Second input tensor (Matrix B). Data type supported: same as @p a
+     * @param[in]  c         Third input tensor  (Matrix C). It can be a nullptr. Data type supported: S32
      * @param[out] output    Output tensor. Data type supported: Data type supported: S32
      * @param[in]  gemm_info (Optional) Specifies if the matrix A and/or matrix B have been reshaped and
      *                       if the reshape of matrix B should be executed only for the first run
      */
-    void configure(const ITensor *a, const ITensor *b, ITensor *output, const GEMMInfo &gemm_info = GEMMInfo());
+    void configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *output, const GEMMInfo &gemm_info = GEMMInfo());
     /** Static function to check if given info will lead to a valid configuration of @ref NEGEMMLowpMatrixMultiplyCore
      *
      * @param[in] a         First input tensor  (Matrix A). Data type supported: QASYMM8.
      * @param[in] b         Second input tensor (Matrix B). Data type supported: same as @p a
+     * @param[in] c         Third input tensor  (Matrix C). It can be a nullptr. Data type supported: S32
      * @param[in] output    Output tensor. Data type supported: Data type supported: S32
      * @param[in] gemm_info (Optional) Specifies if the matrix A and/or matrix B have been reshaped and
      *                      if the reshape of matrix B should be executed only for the first run
      *
      * @return a status
      */
-    static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *output, const GEMMInfo &gemm_info = GEMMInfo());
+    static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *output, const GEMMInfo &gemm_info = GEMMInfo());
 
     // Inherited methods overridden
     void run() override;