From b54ba2848515bf0aee0619c760518481f58c7525 Mon Sep 17 00:00:00 2001 From: Michele Di Giorgio Date: Tue, 14 Jan 2020 15:31:55 +0000 Subject: COMPMID-2847: Fuse output stage in GEMMLowpMatrixMultiplyReshapedOnlyRHS Change-Id: Icd60eb368a34295434e8c141885b4666973a92a1 Signed-off-by: Michele Di Giorgio Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/2732 Tested-by: Arm Jenkins Reviewed-by: Georgios Pinitas Reviewed-by: Gian Marco Iodice Comments-Addressed: Arm Jenkins --- src/core/CL/cl_kernels/repeat.h | 104 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) (limited to 'src/core/CL/cl_kernels/repeat.h') diff --git a/src/core/CL/cl_kernels/repeat.h b/src/core/CL/cl_kernels/repeat.h index 691f7aea01..4e761db073 100644 --- a/src/core/CL/cl_kernels/repeat.h +++ b/src/core/CL/cl_kernels/repeat.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2019-2020 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #ifndef ARM_COMPUTE_REPEAT_H #define ARM_COMPUTE_REPEAT_H +#include "helpers.h" + /** Macros that help in loop unrolling */ //Repeat macros with 3 param, excluding the implicit ID param #define REPEAT_3_1(P_X, P_A, P_B, P_C) P_X##_DEF(0, P_A, P_B, P_C) @@ -76,8 +78,106 @@ #define REPEAT_DEF_3_N(P_NUM, P_OP, P_A, P_B, P_C) REPEAT_3_##P_NUM(P_OP, P_A, P_B, P_C) //One level of indirection to ensure order of expansion does not affect preprocessing P_NUM #define REPEAT_3_N(P_NUM, P_OP, P_A, P_B, P_C) REPEAT_DEF_3_N(P_NUM, P_OP, P_A, P_B, P_C) -//Macro for initializing N variables. generates N statements that defines VAR##N = RHS_ACCESSOR_DEF(...) +// Repeat macros with 4 param, excluding the implicit ID param +#define REPEAT_4_1(P_X, P_A, P_B, P_C, P_D) P_X##_DEF(0, P_A, P_B, P_C, P_D) +#define REPEAT_4_2(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(1, P_A, P_B, P_C, P_D); \ + REPEAT_4_1(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_3(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(2, P_A, P_B, P_C, P_D); \ + REPEAT_4_2(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_4(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(3, P_A, P_B, P_C, P_D); \ + REPEAT_4_3(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_5(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(4, P_A, P_B, P_C, P_D); \ + REPEAT_4_4(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_6(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(5, P_A, P_B, P_C, P_D); \ + REPEAT_4_5(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_7(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(6, P_A, P_B, P_C, P_D); \ + REPEAT_4_6(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_8(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(7, P_A, P_B, P_C, P_D); \ + REPEAT_4_7(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_9(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(8, P_A, P_B, P_C, P_D); \ + REPEAT_4_8(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_10(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(9, P_A, P_B, P_C, P_D); \ + REPEAT_4_9(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_11(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(A, P_A, P_B, P_C, P_D); \ + REPEAT_4_10(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_12(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(B, P_A, P_B, P_C, P_D); \ + REPEAT_4_11(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_13(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(C, P_A, P_B, P_C, P_D); \ + REPEAT_4_12(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_14(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(D, P_A, P_B, P_C, P_D); \ + REPEAT_4_13(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_15(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(E, P_A, P_B, P_C, P_D); \ + REPEAT_4_14(P_X, P_A, P_B, P_C, P_D) +#define REPEAT_4_16(P_X, P_A, P_B, P_C, P_D) \ + P_X##_DEF(F, P_A, P_B, P_C, P_D); \ + REPEAT_4_15(P_X, P_A, P_B, P_C, P_D) + +#define REPEAT_DEF_4_N(P_NUM, P_OP, P_A, P_B, P_C, P_D) REPEAT_4_##P_NUM(P_OP, P_A, P_B, P_C, P_D) //One level of indirection to ensure order of expansion does not affect preprocessing P_NUM +#define REPEAT_4_N(P_NUM, P_OP, P_A, P_B, P_C, P_D) REPEAT_DEF_4_N(P_NUM, P_OP, P_A, P_B, P_C, P_D) + +// Macro for initializing N variables. Generates N statements that defines VAR##N = RHS_ACCESSOR_DEF(...) #define VAR_INIT_TO_CONST_DEF(ID, TYPE, VAR, VAL) TYPE VAR##ID = VAL #define REPEAT_VAR_INIT_TO_CONST(N, TYPE, VAR, VAL) REPEAT_3_N(N, VAR_INIT_TO_CONST, TYPE, VAR, VAL) +// Macro for initializing N variables by converting the data type. Generates N statements that defines VAR##N = RHS_ACCESSOR_DEF(...) +#define VAR_INIT_CONVERT_SAT_DEF(ID, TYPE_OUT, VAR_IN, VAR_OUT) TYPE_OUT VAR_OUT##ID = CONVERT_SAT(VAR_IN##ID, TYPE_OUT) +#define REPEAT_VAR_INIT_CONVERT_SAT(N, TYPE_OUT, VAR_IN, VAR_OUT) REPEAT_3_N(N, VAR_INIT_CONVERT_SAT, TYPE_OUT, VAR_IN, VAR_OUT) + +// Macro for adding a constant to N variables. Generates N statements that defines VAR##N =RHS_ACCESSOR_DEF(...) +#define ADD_CONST_TO_VAR_DEF(ID, TYPE, VAR, VAL) VAR##ID += (TYPE)VAL +#define REPEAT_ADD_CONST_TO_VAR(N, TYPE, VAR, VAL) REPEAT_3_N(N, ADD_CONST_TO_VAR, TYPE, VAR, VAL) + +// Macro for multiplying N variables (VAR_B) by a constant (VAL) and adding to other N variables (VAR_A). Generates N statements that defines VAR_A##N =RHS_ACCESSOR_DEF(...) +#define MLA_VAR_WITH_CONST_VEC_DEF(ID, VAR_A, VAR_B, VAL) VAR_A##ID += VAR_B##ID * VAL +#define REPEAT_MLA_VAR_WITH_CONST_VEC(N, VAR_A, VAR_B, VAL) REPEAT_3_N(N, MLA_VAR_WITH_CONST_VEC, VAR_A, VAR_B, VAL) + +// Macro for adding a vector to N-variables. Generates N statements that defines VAR##N =RHS_ACCESSOR_DEF(...) +#define ADD_VECTOR_TO_VAR_DEF(ID, TYPE, VAR, VEC) VAR##ID += VEC +#define REPEAT_ADD_VECTOR_TO_VAR(N, VAR, VEC) REPEAT_3_N(N, ADD_VECTOR_TO_VAR, "", VAR, VEC) + +// Macro for adding a two N-variables. Generates N statements that defines VAR##N =RHS_ACCESSOR_DEF(...) +#define ADD_TWO_VARS_DEF(ID, TYPE, VAR_A, VAR_B) VAR_A##ID += VAR_B##ID +#define REPEAT_ADD_TWO_VARS(N, VAR_A, VAR_B) REPEAT_3_N(N, ADD_TWO_VARS, "", VAR_A, VAR_B) + +// Macro for performing Max between a constant and N variables. Generates N statements that defines VAR##N =RHS_ACCESSOR_DEF(...) +#define MAX_CONST_VAR_DEF(ID, TYPE, VAR, VAL) VAR##ID = max(VAR##ID, (TYPE)VAL) +#define REPEAT_MAX_CONST_VAR(N, TYPE, VAR, VAL) REPEAT_3_N(N, MAX_CONST_VAR, TYPE, VAR, VAL) + +// Macro for performing Min between a constant and N variables. Generates N statements that defines VAR##N =RHS_ACCESSOR_DEF(...) +#define MIN_CONST_VAR_DEF(ID, TYPE, VAR, VAL) VAR##ID = min(VAR##ID, (TYPE)VAL) +#define REPEAT_MIN_CONST_VAR(N, TYPE, VAR, VAL) REPEAT_3_N(N, MIN_CONST_VAR, TYPE, VAR, VAL) + +// Macro for performing ASYMM_MULT_BY_QUANT_MULTIPLIER_GREATER_THAN_ONE to N variables. Generates N statements that defines VAR##N =RHS_ACCESSOR_DEF(...) +#define ASYMM_MULT_BY_QUANT_MULTIPLIER_GREATER_THAN_ONE_DEF(ID, SIZE, VAR, RES_MUL, RES_SHIFT) VAR##ID = ASYMM_MULT_BY_QUANT_MULTIPLIER_GREATER_THAN_ONE(VAR##ID, RES_MUL, RES_SHIFT, SIZE) +#define REPEAT_ASYMM_MULT_BY_QUANT_MULTIPLIER_GREATER_THAN_ONE(N, SIZE, VAR, RES_MUL, RES_SHIFT) REPEAT_4_N(N, ASYMM_MULT_BY_QUANT_MULTIPLIER_GREATER_THAN_ONE, SIZE, VAR, RES_MUL, RES_SHIFT) + +// Macro for performing ASYMM_MULT_BY_QUANT_MULTIPLIER_LESS_THAN_ONE to N variables. Generates N statements that defines VAR##N =RHS_ACCESSOR_DEF(...) +#define ASYMM_MULT_BY_QUANT_MULTIPLIER_LESS_THAN_ONE_DEF(ID, SIZE, VAR, RES_MUL, RES_SHIFT) VAR##ID = ASYMM_MULT_BY_QUANT_MULTIPLIER_LESS_THAN_ONE(VAR##ID, RES_MUL, RES_SHIFT, SIZE) +#define REPEAT_ASYMM_MULT_BY_QUANT_MULTIPLIER_LESS_THAN_ONE(N, SIZE, VAR, RES_MUL, RES_SHIFT) REPEAT_4_N(N, ASYMM_MULT_BY_QUANT_MULTIPLIER_LESS_THAN_ONE, SIZE, VAR, RES_MUL, RES_SHIFT) + +// Macro for performing per-channel ASYMM_MULT_BY_QUANT_MULTIPLIER to N variables. +#define ASYMM_MULT_BY_QUANT_MULTIPLIER_PER_CHANNEL_DEF(ID, SIZE, VAR, RES_MUL, RES_SHIFT) \ + ({ \ + VEC_DATA_TYPE(int, N0) \ + VAR##ID_shift_lt0 = ASYMM_MULT_BY_QUANT_MULTIPLIER_GREATER_THAN_ONE(VAR##ID, RES_MUL, RES_SHIFT, N0); \ + VEC_DATA_TYPE(int, N0) \ + VAR##ID_shift_gt0 = ASYMM_MULT_BY_QUANT_MULTIPLIER_LESS_THAN_ONE(VAR##ID, RES_MUL, RES_SHIFT, N0); \ + VAR##ID = select(VAR##ID_shift_lt0, VAR##ID_shift_gt0, RES_SHIFT >= 0); \ + }) +#define REPEAT_ASYMM_MULT_BY_QUANT_MULTIPLIER_PER_CHANNEL(N, SIZE, VAR, RES_MUL, RES_SHIFT) REPEAT_4_N(N, ASYMM_MULT_BY_QUANT_MULTIPLIER_PER_CHANNEL, SIZE, VAR, RES_MUL, RES_SHIFT) + #endif // ARM_COMPUTE_REPEAT_H -- cgit v1.2.1