From beb2d45ed515a2d0f0727c038ff837f21c61d2dd Mon Sep 17 00:00:00 2001 From: Michele Di Giorgio Date: Mon, 11 May 2020 16:17:51 +0100 Subject: COMPMID-3470: Modify NE/CLQLSTMLayer interface to provide 3 outputs Change-Id: I895b697c89c9a7509d48a54ac1effb7fbd8cca19 Signed-off-by: Michele Di Giorgio Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/3174 Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins Reviewed-by: Sang-Hoon Park --- arm_compute/runtime/CL/functions/CLQLSTMLayer.h | 102 ++++++++++++---------- arm_compute/runtime/NEON/functions/NEQLSTMLayer.h | 92 ++++++++++--------- src/runtime/CL/functions/CLQLSTMLayer.cpp | 24 +++-- src/runtime/NEON/functions/NEQLSTMLayer.cpp | 17 +++- 4 files changed, 133 insertions(+), 102 deletions(-) diff --git a/arm_compute/runtime/CL/functions/CLQLSTMLayer.h b/arm_compute/runtime/CL/functions/CLQLSTMLayer.h index 722275e269..219f46ee48 100644 --- a/arm_compute/runtime/CL/functions/CLQLSTMLayer.h +++ b/arm_compute/runtime/CL/functions/CLQLSTMLayer.h @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CLQLSTMLAYER_H #define ARM_COMPUTE_CLQLSTMLAYER_H +#include "arm_compute/core/CL/kernels/CLCopyKernel.h" #include "arm_compute/core/CL/kernels/CLElementwiseOperationKernel.h" #include "arm_compute/core/CL/kernels/CLGEMMLowpReductionKernel.h" #include "arm_compute/core/CL/kernels/CLPixelWiseMultiplicationKernel.h" @@ -46,6 +47,7 @@ class ICLTensor; * This function calls the following CL functions/kernels: * * -# @ref CLActivationLayer Activation functions (tanh and logistic) + * -# @ref CLCopyKernel Copy kernel for copying output_state_out to output * -# @ref CLSaturatedArithmeticOperationKernel Elementwise addition and subtraction * -# @ref CLGEMMLowpMatrixMultiplyCore Quantized matrix multiplication core. Accumulators are 32-bit integers * -# @ref CLGEMMLowpQuantizeDownInt32ToInt16ScaleByFixedPoint Convert 32-bit integers into QSYMM16 @@ -78,10 +80,11 @@ public: * @param[in] forget_gate_bias 1D weights tensor with dimensions [num_units]. Data type supported: S32. * @param[in] cell_bias 1D weights tensor with dimensions [num_units]. Data type supported: S32. * @param[in] output_gate_bias 1D weights tensor with dimensions [num_units]. Data type supported: S32. - * @param[in] cell_state_in 2D tensor with dimensions [output_size, batch_size]. Data type supported: QSYMM16. - * @param[in] output_state_in 2D tensor with dimensions [num_units, batch_size]. Data type supported: Same as @p input. - * @param[out] cell_state_out Destination tensor. Output is a 2D tensor with dimensions [output_size, batch_size]. Data type supported: QSYMM16. - * @param[out] output_state_out Destination tensor. Output is a 2D tensor with dimensions [num_units, batch_size].Data types supported: Same as @p input. + * @param[in] cell_state_in 2D tensor with dimensions [num_units, batch_size]. Data type supported: QSYMM16. + * @param[in] output_state_in 2D tensor with dimensions [output_size, batch_size]. Data type supported: Same as @p input. + * @param[out] cell_state_out Destination tensor. Output is a 2D tensor with dimensions [num_units, batch_size]. Data type supported: QSYMM16. + * @param[out] output_state_out Destination tensor. Output is a 2D tensor with dimensions [output_size, batch_size].Data types supported: Same as @p input. + * @param[out] output Destination tensor. Output is a 2D tensor with dimensions [output_size, batch_size].Data types supported: Same as @p input. * @param[in] lstm_params Weights tensors used in peephole, CIFG and layer normalization optimizations: * input_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at input gate. * forget_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at forget gate. @@ -111,7 +114,7 @@ public: const ICLTensor *recurrent_to_forget_weights, const ICLTensor *recurrent_to_cell_weights, const ICLTensor *recurrent_to_output_weights, const ICLTensor *forget_gate_bias, const ICLTensor *cell_bias, const ICLTensor *output_gate_bias, const ICLTensor *cell_state_in, const ICLTensor *output_state_in, - ICLTensor *cell_state_out, ICLTensor *output_state_out, + ICLTensor *cell_state_out, ICLTensor *output_state_out, ICLTensor *output, const LSTMParams &lstm_params); /** Initialize function's tensors. @@ -127,10 +130,11 @@ public: * @param[in] forget_gate_bias 1D weights tensor with dimensions [num_units]. Data type supported: S32. * @param[in] cell_bias 1D weights tensor with dimensions [num_units]. Data type supported: S32. * @param[in] output_gate_bias 1D weights tensor with dimensions [num_units]. Data type supported: S32. - * @param[in] cell_state_in 2D tensor with dimensions [output_size, batch_size]. Data type supported: QSYMM16. - * @param[in] output_state_in 2D tensor with dimensions [num_units, batch_size]. Data type supported: Same as @p input. - * @param[out] cell_state_out Destination tensor. Output is a 2D tensor with dimensions [output_size, batch_size]. Data type supported: QSYMM16. - * @param[out] output_state_out Destination tensor. Output is a 2D tensor with dimensions [num_units, batch_size].Data types supported: Same as @p input. + * @param[in] cell_state_in 2D tensor with dimensions [num_units, batch_size]. Data type supported: QSYMM16. + * @param[in] output_state_in 2D tensor with dimensions [output_size, batch_size]. Data type supported: Same as @p input. + * @param[out] cell_state_out Destination tensor. Output is a 2D tensor with dimensions [num_units, batch_size]. Data type supported: QSYMM16. + * @param[out] output_state_out Destination tensor. Output is a 2D tensor with dimensions [output_size, batch_size].Data types supported: Same as @p input. + * @param[out] output Destination tensor. Output is a 2D tensor with dimensions [output_size, batch_size].Data types supported: Same as @p input. * @param[in] lstm_params Weights tensors used in peephole, CIFG and layer normalization optimizations: * input_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at input gate. * forget_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at forget gate. @@ -160,48 +164,49 @@ public: const ICLTensor *recurrent_to_forget_weights, const ICLTensor *recurrent_to_cell_weights, const ICLTensor *recurrent_to_output_weights, const ICLTensor *forget_gate_bias, const ICLTensor *cell_bias, const ICLTensor *output_gate_bias, const ICLTensor *cell_state_in, const ICLTensor *output_state_in, - ICLTensor *cell_state_out, ICLTensor *output_state_out, + ICLTensor *cell_state_out, ICLTensor *output_state_out, ICLTensor *output, const LSTMParams &lstm_params); /** Static function to check if given info will lead to a valid configuration of @ref CLQLSTMLayer * - * @param[in] input Source tensor info. Input is a 2D tensor info with dimensions [input_size, batch_size]. Data types supported: QASYMM8_SIGNED. - * @param[in] input_to_forget_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. - * @param[in] input_to_cell_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. - * @param[in] input_to_output_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. - * @param[in] recurrent_to_forget_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. - * @param[in] recurrent_to_cell_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. - * @param[in] recurrent_to_output_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. - * @param[in] forget_gate_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. - * @param[in] cell_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. - * @param[in] output_gate_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. - * @param[in] cell_state_in 2D tensor info with dimensions [num_units, batch_size]. Data type supported: QSYMM16. - * @param[in] output_state_in 2D tensor info with dimensions [output_size, batch_size]. Data type supported: Same as @p input. - * @param[out] cell_state_out Destination tensor info. Output is a 2D tensor info with dimensions [num_units, batch_size]. Data type supported: QSYMM16. - * @param[out] output_state_out Destination tensor info. Output is a 2D tensor info with dimensions [output_size, batch_size].Data types supported: Same as @p input. - * @param[in] lstm_params Weights tensors info used in peephole, CIFG and layer normalization optimizations: - * input_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at input gate. - * forget_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at forget gate. - * cell_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at cell gate. - * output_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at output gate. - * hidden_state_zero The zero point of the hidden state. - * hidden_state_scale The scale of the hidden state. - * input_to_input_weights (Optional) 2D weights tensor with dimensions [input_size, num_units]. Data type supported: QSYMM8. - * recurrent_to_input_weights (Optional) 2D weights tensor with dimensions [output_size, num_units]. Data type supported: QSYMM8. - * cell_to_input_weights (Optional) 1D weights tensor with dimensions [num_units]. Can be nullptr. Data type supported: QSYMM16. - * cell_to_forget_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * cell_to_output_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * input_gate_bias (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: S32. - * projection_weights (Optional) 2D weights tensor with dimensions [output_size, num_units]. Data type supported: QSYMM8. - * projection_bias (Optional) 1D weights tensor with dimensions [output_size]. S32. - * input_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * forget_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * cell_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * output_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * cell_threshold (Optional) The clipping threshold for the cell state, such that values are bound within [-cell_clip, cell_clip]. - * If set to 0.0 then clipping is disabled. - * projection_threshold (Optional) The clipping threshold for the output from the projection layer, such that values are bound within - * [-proj_clip, proj_clip]. If set to 0.0 then clipping is disabled. + * @param[in] input Source tensor info. Input is a 2D tensor info with dimensions [input_size, batch_size]. Data types supported: QASYMM8_SIGNED. + * @param[in] input_to_forget_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. + * @param[in] input_to_cell_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. + * @param[in] input_to_output_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. + * @param[in] recurrent_to_forget_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. + * @param[in] recurrent_to_cell_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. + * @param[in] recurrent_to_output_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. + * @param[in] forget_gate_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. + * @param[in] cell_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. + * @param[in] output_gate_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. + * @param[in] cell_state_in 2D tensor info with dimensions [num_units, batch_size]. Data type supported: QSYMM16. + * @param[in] output_state_in 2D tensor info with dimensions [output_size, batch_size]. Data type supported: Same as @p input. + * @param[in] cell_state_out Destination tensor info. Output is a 2D tensor info with dimensions [num_units, batch_size]. Data type supported: QSYMM16. + * @param[in] output_state_out Destination tensor info. Output is a 2D tensor info with dimensions [output_size, batch_size].Data types supported: Same as @p input. + * @param[in] output Destination tensor info. Output is a 2D tensor info with dimensions [output_size, batch_size].Data types supported: Same as @p input. + * @param[in] lstm_params Weights tensors info used in peephole, CIFG and layer normalization optimizations: + * input_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at input gate. + * forget_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at forget gate. + * cell_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at cell gate. + * output_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at output gate. + * hidden_state_zero The zero point of the hidden state. + * hidden_state_scale The scale of the hidden state. + * input_to_input_weights (Optional) 2D weights tensor with dimensions [input_size, num_units]. Data type supported: QSYMM8. + * recurrent_to_input_weights (Optional) 2D weights tensor with dimensions [output_size, num_units]. Data type supported: QSYMM8. + * cell_to_input_weights (Optional) 1D weights tensor with dimensions [num_units]. Can be nullptr. Data type supported: QSYMM16. + * cell_to_forget_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * cell_to_output_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * input_gate_bias (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: S32. + * projection_weights (Optional) 2D weights tensor with dimensions [output_size, num_units]. Data type supported: QSYMM8. + * projection_bias (Optional) 1D weights tensor with dimensions [output_size]. S32. + * input_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * forget_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * cell_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * output_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * cell_threshold (Optional) The clipping threshold for the cell state, such that values are bound within [-cell_clip, cell_clip]. + * If set to 0.0 then clipping is disabled. + * projection_threshold (Optional) The clipping threshold for the output from the projection layer, such that values are bound within + * [-proj_clip, proj_clip]. If set to 0.0 then clipping is disabled. * @return a status */ static Status validate(const ITensorInfo *input, @@ -209,7 +214,7 @@ public: const ITensorInfo *recurrent_to_forget_weights, const ITensorInfo *recurrent_to_cell_weights, const ITensorInfo *recurrent_to_output_weights, const ITensorInfo *forget_gate_bias, const ITensorInfo *cell_bias, const ITensorInfo *output_gate_bias, const ITensorInfo *cell_state_in, const ITensorInfo *output_state_in, - const ITensorInfo *cell_state_out, const ITensorInfo *output_state_out, + const ITensorInfo *cell_state_out, const ITensorInfo *output_state_out, const ITensorInfo *output, const LSTMParams &lstm_params); // Inherited methods overridden: @@ -314,6 +319,7 @@ private: CLSaturatedArithmeticOperationKernel _accumulate_projection{}; CLActivationLayer _projection_clip{}; std::array _layer_norms{ {} }; + CLCopyKernel _copy_output{}; // Tensor pointers const ICLTensor *_input_to_input_weights diff --git a/arm_compute/runtime/NEON/functions/NEQLSTMLayer.h b/arm_compute/runtime/NEON/functions/NEQLSTMLayer.h index 9eb0654cfe..4dde85e895 100644 --- a/arm_compute/runtime/NEON/functions/NEQLSTMLayer.h +++ b/arm_compute/runtime/NEON/functions/NEQLSTMLayer.h @@ -26,6 +26,7 @@ #include "arm_compute/core/NEON/kernels/NEArithmeticAdditionKernel.h" #include "arm_compute/core/NEON/kernels/NEArithmeticSubtractionKernel.h" +#include "arm_compute/core/NEON/kernels/NECopyKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMLowpReductionKernel.h" #include "arm_compute/core/NEON/kernels/NEPixelWiseMultiplicationKernel.h" #include "arm_compute/core/NEON/kernels/NEQLSTMLayerNormalizationKernel.h" @@ -49,6 +50,7 @@ class ITensor; * -# @ref NEActivationLayer Activation functions (tanh and logistic) * -# @ref NEArithmeticAdditionKernel Elementwise addition * -# @ref NEArithmeticSubtractionKernel Elementwise subtraction + * -# @ref NECopyKernel Copy kernel for copying output_state_out to output * -# @ref NEGEMMLowpMatrixMultiplyCore Quantized matrix multiplication core. Accumulators are 32-bit integers * -# @ref NEGEMMLowpQuantizeDownInt32ToInt16ScaleByFixedPoint Convert 32-bit integers into QSYMM16 * -# @ref NEGEMMLowpMatrixAReductionKernel For precomputing effective biases to use @@ -80,10 +82,11 @@ public: * @param[in] forget_gate_bias 1D weights tensor with dimensions [num_units]. Data type supported: S32. * @param[in] cell_bias 1D weights tensor with dimensions [num_units]. Data type supported: S32. * @param[in] output_gate_bias 1D weights tensor with dimensions [num_units]. Data type supported: S32. - * @param[in] cell_state_in 2D tensor with dimensions [output_size, batch_size]. Data type supported: QSYMM16. - * @param[in] output_state_in 2D tensor with dimensions [num_units, batch_size]. Data type supported: Same as @p input. - * @param[out] cell_state_out Destination tensor. Output is a 2D tensor with dimensions [output_size, batch_size]. Data type supported: QSYMM16. - * @param[out] output_state_out Destination tensor. Output is a 2D tensor with dimensions [num_units, batch_size].Data types supported: Same as @p input. + * @param[in] cell_state_in 2D tensor with dimensions [num_units, batch_size]. Data type supported: QSYMM16. + * @param[in] output_state_in 2D tensor with dimensions [output_size, batch_size]. Data type supported: Same as @p input. + * @param[out] cell_state_out Destination tensor. Output is a 2D tensor with dimensions [num_units, batch_size]. Data type supported: QSYMM16. + * @param[out] output_state_out Destination tensor. Output is a 2D tensor with dimensions [output_size, batch_size].Data types supported: Same as @p input. + * @param[out] output Destination tensor. Output is a 2D tensor with dimensions [output_size, batch_size].Data types supported: Same as @p input. * @param[in] lstm_params Weights tensors used in peephole, CIFG and layer normalization optimizations: * input_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at input gate. * forget_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at forget gate. @@ -113,48 +116,49 @@ public: const ITensor *recurrent_to_forget_weights, const ITensor *recurrent_to_cell_weights, const ITensor *recurrent_to_output_weights, const ITensor *forget_gate_bias, const ITensor *cell_bias, const ITensor *output_gate_bias, const ITensor *cell_state_in, const ITensor *output_state_in, - ITensor *cell_state_out, ITensor *output_state_out, + ITensor *cell_state_out, ITensor *output_state_out, ITensor *output, const LSTMParams &lstm_params); /** Static function to check if given info will lead to a valid configuration of @ref NEQLSTMLayer * - * @param[in] input Source tensor info. Input is a 2D tensor info with dimensions [input_size, batch_size]. Data types supported: QASYMM8_SIGNED. - * @param[in] input_to_forget_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. - * @param[in] input_to_cell_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. - * @param[in] input_to_output_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. - * @param[in] recurrent_to_forget_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. - * @param[in] recurrent_to_cell_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. - * @param[in] recurrent_to_output_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. - * @param[in] forget_gate_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. - * @param[in] cell_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. - * @param[in] output_gate_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. - * @param[in] cell_state_in 2D tensor info with dimensions [num_units, batch_size]. Data type supported: QSYMM16. - * @param[in] output_state_in 2D tensor info with dimensions [output_size, batch_size]. Data type supported: Same as @p input. - * @param[out] cell_state_out Destination tensor info. Output is a 2D tensor info with dimensions [num_units, batch_size]. Data type supported: QSYMM16. - * @param[out] output_state_out Destination tensor info. Output is a 2D tensor info with dimensions [output_size, batch_size].Data types supported: Same as @p input. - * @param[in] lstm_params Weights tensors info used in peephole, CIFG and layer normalization optimizations: - * input_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at input gate. - * forget_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at forget gate. - * cell_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at cell gate. - * output_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at output gate. - * hidden_state_zero The zero point of the hidden state. - * hidden_state_scale The scale of the hidden state. - * input_to_input_weights (Optional) 2D weights tensor with dimensions [input_size, num_units]. Data type supported: QSYMM8. - * recurrent_to_input_weights (Optional) 2D weights tensor with dimensions [output_size, num_units]. Data type supported: QSYMM8. - * cell_to_input_weights (Optional) 1D weights tensor with dimensions [num_units]. Can be nullptr. Data type supported: QSYMM16. - * cell_to_forget_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * cell_to_output_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * input_gate_bias (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: S32. - * projection_weights (Optional) 2D weights tensor with dimensions [output_size, num_units]. Data type supported: QSYMM8. - * projection_bias (Optional) 1D weights tensor with dimensions [output_size]. S32. - * input_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * forget_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * cell_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * output_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. - * cell_threshold (Optional) The clipping threshold for the cell state, such that values are bound within [-cell_clip, cell_clip]. - * If set to 0.0 then clipping is disabled. - * projection_threshold (Optional) The clipping threshold for the output from the projection layer, such that values are bound within - * [-proj_clip, proj_clip]. If set to 0.0 then clipping is disabled. + * @param[in] input Source tensor info. Input is a 2D tensor info with dimensions [input_size, batch_size]. Data types supported: QASYMM8_SIGNED. + * @param[in] input_to_forget_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. + * @param[in] input_to_cell_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. + * @param[in] input_to_output_weights 2D weights tensor info with dimensions [input_size, num_units]. Data type supported: QSYMM8. + * @param[in] recurrent_to_forget_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. + * @param[in] recurrent_to_cell_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. + * @param[in] recurrent_to_output_weights 2D weights tensor info with dimensions [output_size, num_units]. Data type supported: QSYMM8. + * @param[in] forget_gate_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. + * @param[in] cell_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. + * @param[in] output_gate_bias 1D weights tensor info with dimensions [num_units]. Data type supported: S32. + * @param[in] cell_state_in 2D tensor info with dimensions [num_units, batch_size]. Data type supported: QSYMM16. + * @param[in] output_state_in 2D tensor info with dimensions [output_size, batch_size]. Data type supported: Same as @p input. + * @param[in] cell_state_out Destination tensor info. Output is a 2D tensor info with dimensions [num_units, batch_size]. Data type supported: QSYMM16. + * @param[in] output_state_out Destination tensor info. Output is a 2D tensor info with dimensions [output_size, batch_size].Data types supported: Same as @p input. + * @param[in] output Destination tensor info. Output is a 2D tensor info with dimensions [output_size, batch_size].Data types supported: Same as @p input. + * @param[in] lstm_params Weights tensors info used in peephole, CIFG and layer normalization optimizations: + * input_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at input gate. + * forget_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at forget gate. + * cell_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at cell gate. + * output_intermediate_scale Scale of the intermediate result of matmul, i.e. input to layer normalization, at output gate. + * hidden_state_zero The zero point of the hidden state. + * hidden_state_scale The scale of the hidden state. + * input_to_input_weights (Optional) 2D weights tensor with dimensions [input_size, num_units]. Data type supported: QSYMM8. + * recurrent_to_input_weights (Optional) 2D weights tensor with dimensions [output_size, num_units]. Data type supported: QSYMM8. + * cell_to_input_weights (Optional) 1D weights tensor with dimensions [num_units]. Can be nullptr. Data type supported: QSYMM16. + * cell_to_forget_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * cell_to_output_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * input_gate_bias (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: S32. + * projection_weights (Optional) 2D weights tensor with dimensions [output_size, num_units]. Data type supported: QSYMM8. + * projection_bias (Optional) 1D weights tensor with dimensions [output_size]. S32. + * input_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * forget_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * cell_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * output_layer_norm_weights (Optional) 1D weights tensor with dimensions [num_units]. Data type supported: QSYMM16. + * cell_threshold (Optional) The clipping threshold for the cell state, such that values are bound within [-cell_clip, cell_clip]. + * If set to 0.0 then clipping is disabled. + * projection_threshold (Optional) The clipping threshold for the output from the projection layer, such that values are bound within + * [-proj_clip, proj_clip]. If set to 0.0 then clipping is disabled. * @return a status */ static Status validate(const ITensorInfo *input, @@ -162,7 +166,7 @@ public: const ITensorInfo *recurrent_to_forget_weights, const ITensorInfo *recurrent_to_cell_weights, const ITensorInfo *recurrent_to_output_weights, const ITensorInfo *forget_gate_bias, const ITensorInfo *cell_bias, const ITensorInfo *output_gate_bias, const ITensorInfo *cell_state_in, const ITensorInfo *output_state_in, - const ITensorInfo *cell_state_out, const ITensorInfo *output_state_out, + const ITensorInfo *cell_state_out, const ITensorInfo *output_state_out, const ITensorInfo *output, const LSTMParams &lstm_params); // Inherited methods overridden: @@ -304,6 +308,8 @@ private: std::array _layer_norms{ {} }; + NECopyKernel _copy_output{}; + // Tensor pointers const ITensor *_input_to_input_weights{ nullptr }; const ITensor *_recurrent_to_input_weights{ nullptr }; diff --git a/src/runtime/CL/functions/CLQLSTMLayer.cpp b/src/runtime/CL/functions/CLQLSTMLayer.cpp index d9b5c7c64d..a20ffc6f37 100644 --- a/src/runtime/CL/functions/CLQLSTMLayer.cpp +++ b/src/runtime/CL/functions/CLQLSTMLayer.cpp @@ -76,12 +76,12 @@ void CLQLSTMLayer::configure(const ICLTensor *input, const ICLTensor *recurrent_to_forget_weights, const ICLTensor *recurrent_to_cell_weights, const ICLTensor *recurrent_to_output_weights, const ICLTensor *forget_gate_bias, const ICLTensor *cell_bias, const ICLTensor *output_gate_bias, const ICLTensor *cell_state_in, const ICLTensor *output_state_in, - ICLTensor *cell_state_out, ICLTensor *output_state_out, + ICLTensor *cell_state_out, ICLTensor *output_state_out, ICLTensor *output, const LSTMParams &lstm_params) { configure(CLKernelLibrary::get().get_compile_context(), input, input_to_forget_weights, input_to_cell_weights, input_to_output_weights, recurrent_to_forget_weights, recurrent_to_cell_weights, recurrent_to_output_weights, forget_gate_bias, cell_bias, output_gate_bias, - cell_state_in, output_state_in, cell_state_out, output_state_out, lstm_params); + cell_state_in, output_state_in, cell_state_out, output, output_state_out, lstm_params); } void CLQLSTMLayer::configure(const CLCompileContext &compile_context, const ICLTensor *input, @@ -89,12 +89,13 @@ void CLQLSTMLayer::configure(const CLCompileContext &compile_context, const ICLT const ICLTensor *recurrent_to_forget_weights, const ICLTensor *recurrent_to_cell_weights, const ICLTensor *recurrent_to_output_weights, const ICLTensor *forget_gate_bias, const ICLTensor *cell_bias, const ICLTensor *output_gate_bias, const ICLTensor *cell_state_in, const ICLTensor *output_state_in, - ICLTensor *cell_state_out, ICLTensor *output_state_out, + ICLTensor *cell_state_out, ICLTensor *output_state_out, ICLTensor *output, const LSTMParams &lstm_params) { ARM_COMPUTE_ERROR_ON_NULLPTR(input, input_to_forget_weights, input_to_cell_weights, input_to_output_weights, recurrent_to_forget_weights, recurrent_to_cell_weights, recurrent_to_output_weights, - forget_gate_bias, cell_bias, output_gate_bias, cell_state_in, output_state_in, cell_state_out, output_state_out); + forget_gate_bias, cell_bias, output_gate_bias, cell_state_in, output_state_in, + cell_state_out, output_state_out, output); // Set lstm parameters LSTMParams lstm_params_info{}; @@ -104,7 +105,8 @@ void CLQLSTMLayer::configure(const CLCompileContext &compile_context, const ICLT ARM_COMPUTE_ERROR_THROW_ON(CLQLSTMLayer::validate(input->info(), input_to_forget_weights->info(), input_to_cell_weights->info(), input_to_output_weights->info(), recurrent_to_forget_weights->info(), recurrent_to_cell_weights->info(), recurrent_to_output_weights->info(), forget_gate_bias->info(), cell_bias->info(), output_gate_bias->info(), - cell_state_in->info(), output_state_in->info(), cell_state_out->info(), output_state_out->info(), lstm_params_info)); + cell_state_in->info(), output_state_in->info(), cell_state_out->info(), output_state_out->info(), output->info(), + lstm_params_info)); const int batch_size = input->info()->dimension(1); const int num_units = input_to_output_weights->info()->dimension(1); @@ -446,6 +448,9 @@ void CLQLSTMLayer::configure(const CLCompileContext &compile_context, const ICLT _has_projection_clipping = true; } } + + // Copy output_state_out to output + _copy_output.configure(compile_context, output_state_out, output); } Status CLQLSTMLayer::validate(const ITensorInfo *input, @@ -453,11 +458,12 @@ Status CLQLSTMLayer::validate(const ITensorInfo *input, const ITensorInfo *recurrent_to_forget_weights, const ITensorInfo *recurrent_to_cell_weights, const ITensorInfo *recurrent_to_output_weights, const ITensorInfo *forget_gate_bias, const ITensorInfo *cell_bias, const ITensorInfo *output_gate_bias, const ITensorInfo *cell_state_in, const ITensorInfo *output_state_in, - const ITensorInfo *cell_state_out, const ITensorInfo *output_state_out, + const ITensorInfo *cell_state_out, const ITensorInfo *output_state_out, const ITensorInfo *output, const LSTMParams &lstm_params) { ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, input_to_forget_weights, input_to_cell_weights, input_to_output_weights, recurrent_to_forget_weights, recurrent_to_cell_weights, - recurrent_to_output_weights, forget_gate_bias, cell_bias, output_gate_bias, cell_state_in, output_state_in, cell_state_out, output_state_out); + recurrent_to_output_weights, forget_gate_bias, cell_bias, output_gate_bias, cell_state_in, output_state_in, + cell_state_out, output_state_out, output); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8_SIGNED); ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->num_dimensions() != 2, "Input must have exactly 2 dimensions"); @@ -768,6 +774,7 @@ Status CLQLSTMLayer::validate(const ITensorInfo *input, ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(output_state_in, output_state_out); } + ARM_COMPUTE_RETURN_ON_ERROR(CLCopyKernel::validate(output_state_out, output)); return Status{}; } @@ -887,6 +894,9 @@ void CLQLSTMLayer::run() _projection_clip.run(); } } + + // Copy output_state_out to output + CLScheduler::get().enqueue(_copy_output); } void CLQLSTMLayer::prepare() diff --git a/src/runtime/NEON/functions/NEQLSTMLayer.cpp b/src/runtime/NEON/functions/NEQLSTMLayer.cpp index 9c78ea8b75..466c41307b 100644 --- a/src/runtime/NEON/functions/NEQLSTMLayer.cpp +++ b/src/runtime/NEON/functions/NEQLSTMLayer.cpp @@ -106,7 +106,7 @@ void NEQLSTMLayer::configure(const ITensor *input, const ITensor *recurrent_to_forget_weights, const ITensor *recurrent_to_cell_weights, const ITensor *recurrent_to_output_weights, const ITensor *forget_gate_bias, const ITensor *cell_bias, const ITensor *output_gate_bias, const ITensor *cell_state_in, const ITensor *output_state_in, - ITensor *cell_state_out, ITensor *output_state_out, + ITensor *cell_state_out, ITensor *output_state_out, ITensor *output, const LSTMParams &lstm_params) { ARM_COMPUTE_ERROR_ON_NULLPTR(input, input_to_forget_weights, input_to_cell_weights, input_to_output_weights, @@ -121,7 +121,8 @@ void NEQLSTMLayer::configure(const ITensor *input, ARM_COMPUTE_ERROR_THROW_ON(NEQLSTMLayer::validate(input->info(), input_to_forget_weights->info(), input_to_cell_weights->info(), input_to_output_weights->info(), recurrent_to_forget_weights->info(), recurrent_to_cell_weights->info(), recurrent_to_output_weights->info(), forget_gate_bias->info(), cell_bias->info(), output_gate_bias->info(), - cell_state_in->info(), output_state_in->info(), cell_state_out->info(), output_state_out->info(), lstm_params_info)); + cell_state_in->info(), output_state_in->info(), cell_state_out->info(), output_state_out->info(), output->info(), + lstm_params_info)); const int batch_size = input->info()->dimension(1); const int num_units = input_to_output_weights->info()->dimension(1); @@ -515,6 +516,9 @@ void NEQLSTMLayer::configure(const ITensor *input, _hidden_gate.allocator()->allocate(); } } + + // Copy output_state_out to output + _copy_output.configure(output_state_out, output); } Status NEQLSTMLayer::validate(const ITensorInfo *input, @@ -522,11 +526,12 @@ Status NEQLSTMLayer::validate(const ITensorInfo *input, const ITensorInfo *recurrent_to_forget_weights, const ITensorInfo *recurrent_to_cell_weights, const ITensorInfo *recurrent_to_output_weights, const ITensorInfo *forget_gate_bias, const ITensorInfo *cell_bias, const ITensorInfo *output_gate_bias, const ITensorInfo *cell_state_in, const ITensorInfo *output_state_in, - const ITensorInfo *cell_state_out, const ITensorInfo *output_state_out, + const ITensorInfo *cell_state_out, const ITensorInfo *output_state_out, const ITensorInfo *output, const LSTMParams &lstm_params) { ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, input_to_forget_weights, input_to_cell_weights, input_to_output_weights, recurrent_to_forget_weights, recurrent_to_cell_weights, - recurrent_to_output_weights, forget_gate_bias, cell_bias, output_gate_bias, cell_state_in, output_state_in, cell_state_out, output_state_out); + recurrent_to_output_weights, forget_gate_bias, cell_bias, output_gate_bias, cell_state_in, output_state_in, + cell_state_out, output_state_out, output); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8_SIGNED); ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->num_dimensions() != 2, "Input must have exactly 2 dimensions"); @@ -867,6 +872,7 @@ Status NEQLSTMLayer::validate(const ITensorInfo *input, ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(output_state_in, output_state_out); } + ARM_COMPUTE_RETURN_ON_ERROR(NECopyKernel::validate(output_state_out, output)); return Status{}; } @@ -1011,6 +1017,9 @@ void NEQLSTMLayer::run() _hidden_to_output_copy.run(); } } + + // Copy output_state_out to output + NEScheduler::get().schedule(&_copy_output, Window::DimY); } void NEQLSTMLayer::prepare() -- cgit v1.2.1