diff options
| -rw-r--r-- | arm_compute/core/CL/kernels/CLCol2ImKernel.h | 12 | ||||
| -rw-r--r-- | arm_compute/core/CL/kernels/CLGEMMMatrixAdditionKernel.h | 11 | ||||
| -rw-r--r-- | arm_compute/core/CL/kernels/CLIm2ColKernel.h | 6 | ||||
| -rw-r--r-- | arm_compute/core/CL/kernels/CLWeightsReshapeKernel.h | 15 | ||||
| -rw-r--r-- | arm_compute/core/utils/misc/ShapeCalculator.h | 20 | ||||
| -rw-r--r-- | arm_compute/runtime/CL/functions/CLGEMM.h | 14 | ||||
| -rw-r--r-- | arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h | 56 | ||||
| -rw-r--r-- | arm_compute/runtime/NEON/functions/NEGEMM.h | 2 | ||||
| -rw-r--r-- | src/core/CL/kernels/CLCol2ImKernel.cpp | 85 | ||||
| -rw-r--r-- | src/core/CL/kernels/CLGEMMMatrixAdditionKernel.cpp | 68 | ||||
| -rw-r--r-- | src/core/CL/kernels/CLGEMMMatrixMultiplyKernel.cpp | 1 | ||||
| -rw-r--r-- | src/core/CL/kernels/CLWeightsReshapeKernel.cpp | 73 | ||||
| -rw-r--r-- | src/runtime/CL/functions/CLConvolutionLayer.cpp | 8 | ||||
| -rw-r--r-- | src/runtime/CL/functions/CLDeconvolutionLayer.cpp | 2 | ||||
| -rw-r--r-- | src/runtime/CL/functions/CLGEMM.cpp | 48 | ||||
| -rw-r--r-- | src/runtime/CL/functions/CLGEMMConvolutionLayer.cpp | 345 |
16 files changed, 495 insertions, 271 deletions
diff --git a/arm_compute/core/CL/kernels/CLCol2ImKernel.h b/arm_compute/core/CL/kernels/CLCol2ImKernel.h index bd86da1b5e..24d0fdd914 100644 --- a/arm_compute/core/CL/kernels/CLCol2ImKernel.h +++ b/arm_compute/core/CL/kernels/CLCol2ImKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -72,6 +72,16 @@ public: * @param[in] convolved_dims Output convolved dimensions. */ void configure(const ICLTensor *input, ICLTensor *output, std::pair<unsigned int, unsigned int> convolved_dims); + /** Static function to check if given info will lead to a valid configuration of @ref CLCol2ImKernel + * + * @param[in] input The input tensor to convert. Data types supported: QS8/QS16/QASYMM8/F16/F32 + * @param[in] output The output tensor. 3 lower dimensions represent a single output [width, height, OFM], + * while the rest represent batch of outputs. Data types supported: Same as @p input + * @param[in] convolved_dims Output convolved dimensions. + * + * @return a status + */ + static Status validate(const ITensorInfo *input, const ITensorInfo *output, std::pair<unsigned int, unsigned int> convolved_dims); // Inherited methods overridden: void run(const Window &window, cl::CommandQueue &queue) override; diff --git a/arm_compute/core/CL/kernels/CLGEMMMatrixAdditionKernel.h b/arm_compute/core/CL/kernels/CLGEMMMatrixAdditionKernel.h index 8f73d8c2c3..dc84a40ca8 100644 --- a/arm_compute/core/CL/kernels/CLGEMMMatrixAdditionKernel.h +++ b/arm_compute/core/CL/kernels/CLGEMMMatrixAdditionKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -57,6 +57,15 @@ public: * @param[in] beta Weight of matrix C */ void configure(const ICLTensor *input, ICLTensor *output, float beta); + /** Static function to check if given info will lead to a valid configuration of @ref CLGEMMMatrixAdditionKernel. + * + * @param[in] input Input tensor (Matrix C). Data types supported: QS8/QS16/F16/F32 + * @param[in] output Output tensor. If this kernel is used to finalize the GEMM result (alpha * AB + beta * C), output must contain the result obtained by @ref CLGEMMMatrixMultiplyKernel. Data type supported: same as @p input + * @param[in] beta Weight of matrix C + * + * @return a status + */ + static Status validate(const ITensorInfo *input, const ITensorInfo *output, const float beta); // Inherited methods overridden: void run(const Window &window, cl::CommandQueue &queue) override; diff --git a/arm_compute/core/CL/kernels/CLIm2ColKernel.h b/arm_compute/core/CL/kernels/CLIm2ColKernel.h index e38e7e8a49..1ad302eedb 100644 --- a/arm_compute/core/CL/kernels/CLIm2ColKernel.h +++ b/arm_compute/core/CL/kernels/CLIm2ColKernel.h @@ -77,9 +77,6 @@ public: * @param[in] has_bias In case biases are provided expands the matrix with 1. */ void configure(const ICLTensor *input, ICLTensor *output, const Size2D &kernel_dims, const PadStrideInfo &conv_info, bool has_bias); - - // Inherited methods overridden: - void run(const Window &window, cl::CommandQueue &queue) override; /** Static function to check if given info will lead to a valid configuration of @ref CLIm2ColKernel * * @param[in] input The input tensor to convert. 3 lower dimensions represent a single input [width, height, IFM], @@ -94,6 +91,9 @@ public: */ static Status validate(const ITensorInfo *input, const ITensorInfo *output, const Size2D &kernel_dims, const PadStrideInfo &conv_info, bool has_bias); + // Inherited methods overridden: + void run(const Window &window, cl::CommandQueue &queue) override; + private: /** Run the reshape kernel optimised for the special case (stride is 1, padding is 0 and kernel's low 3 dimensions are same as input) * diff --git a/arm_compute/core/CL/kernels/CLWeightsReshapeKernel.h b/arm_compute/core/CL/kernels/CLWeightsReshapeKernel.h index 6c84ded49e..b9ede12e3d 100644 --- a/arm_compute/core/CL/kernels/CLWeightsReshapeKernel.h +++ b/arm_compute/core/CL/kernels/CLWeightsReshapeKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -43,7 +43,6 @@ public: CLWeightsReshapeKernel &operator=(CLWeightsReshapeKernel &&) = default; /** Default destructor */ ~CLWeightsReshapeKernel() = default; - /** Set the input and output of the kernel. * * @param[in] input The input tensor to convert. Weights are 4D tensor with dimensions [kernel_x, kernel_y, IFM, OFM] if shared, @@ -54,6 +53,18 @@ public: * @param[out] output The output tensor. Should be a 2D Tensor. Data types supported: Same as @p input */ void configure(const ICLTensor *input, const ICLTensor *biases, ICLTensor *output); + /** Static function to check if given info will lead to a valid configuration of @ref CLWeightsReshapeKernel + * + * @param[in] input The input tensor to convert. Weights are 4D tensor with dimensions [kernel_x, kernel_y, IFM, OFM] if shared, + * and 5D tensor with dimensions [kernel_x, kernel_y, IFM, OFM, num_patches] if unshared. Data types supported: QS8/QS16/QASYMM8/F16/F32 + * @param[in] biases The shared biases tensor to append. Bias is 1D tensor with dimensions [OFM] if shared and 2D tensor with + * dimensions [OFM, num_patches] if unshared. Data types supported: Same as @p input + * @warning Appending biases to weights reshaped matrix is not supported for quantized asymmetric types. + * @param[in] output The output tensor. Should be a 2D Tensor. Data types supported: Same as @p input + * + * @return a status + */ + static Status validate(const ITensorInfo *input, const ITensorInfo *biases, const ITensorInfo *output); // Inherited methods overridden: void run(const Window &window, cl::CommandQueue &queue) override; diff --git a/arm_compute/core/utils/misc/ShapeCalculator.h b/arm_compute/core/utils/misc/ShapeCalculator.h index e51c6bbe98..c53ac4c71f 100644 --- a/arm_compute/core/utils/misc/ShapeCalculator.h +++ b/arm_compute/core/utils/misc/ShapeCalculator.h @@ -40,6 +40,17 @@ inline TensorShape compute_permutation_output_shape(const ITensorInfo &input, co permute(output_shape, perm); return output_shape; } +inline TensorShape compute_weights_reshaped_shape(const ITensorInfo &weights, bool has_bias = false) +{ + // Calculate output shape + TensorShape weights_reshaped{ weights.tensor_shape() }; + weights_reshaped.collapse(3); + const size_t tmp_dim = weights_reshaped[0]; + weights_reshaped.set(0, weights_reshaped[1]); + weights_reshaped.set(1, tmp_dim + (has_bias ? 1 : 0)); + + return weights_reshaped; +} inline TensorShape compute_interleaved_shape(const ITensorInfo &a, int mult_interleave4x4_height = 1) { // The interleaved output matrix will have the following shape: [ a_height * W, ceil(a_width / W) ] where W = 4 * mult_interleave4x4_height @@ -101,6 +112,15 @@ inline TensorShape compute_im2col_shape(const ITensorInfo &input) return shape_im2col; } +inline TensorShape compute_col2im_shape(const ITensorInfo &input, std::pair<unsigned int, unsigned int> convolved_dims) +{ + TensorShape col2im_shape{ input.tensor_shape() }; + col2im_shape.set(0, convolved_dims.first); + col2im_shape.set(1, convolved_dims.second); + col2im_shape.set(2, input.tensor_shape()[0]); + + return col2im_shape; +} inline TensorShape compute_transposed_shape(const ITensorInfo &input) { TensorShape shape_transposed{ input.tensor_shape() }; diff --git a/arm_compute/runtime/CL/functions/CLGEMM.h b/arm_compute/runtime/CL/functions/CLGEMM.h index 0f144915d7..2e82457ee2 100644 --- a/arm_compute/runtime/CL/functions/CLGEMM.h +++ b/arm_compute/runtime/CL/functions/CLGEMM.h @@ -72,6 +72,20 @@ public: * in case matrix A and matrix B have been already transformed. */ void configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor *c, ICLTensor *output, float alpha, float beta, const GEMMInfo &gemm_info = GEMMInfo()); + /** Static function to check if given info will lead to a valid configuration of @ref CLGEMM. + * + * @param[in] a First input tensor (Matrix or Vector A). Data types supported: QS8/QS16/F16/F32 + * @param[in] b Second input tensor (Matrix B). Data type supported: same as @p a. + * @param[in] c Third input tensor (Matrix C). It can be a nullptr if just the multiplication between @p a and @p b is needed. Data type supported: same as @p a. + * @param[out] output Output tensor. Data type supported: same as @p a + * @param[in] alpha Weight of the matrix product + * @param[in] beta Weight of matrix C + * @param[in] gemm_info (Optional) Specifies if the matrix A and/or matrix B have been reshaped and + * if the reshape of matrix B should happen only for the first run + * + * @return a status + */ + static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ICLTensor *c, const ITensorInfo *output, const float alpha, const float beta, const GEMMInfo &gemm_info = GEMMInfo()); // Inherited methods overridden: void run() override; diff --git a/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h b/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h index 7126688f8b..24029509b8 100644 --- a/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h +++ b/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h @@ -58,14 +58,22 @@ public: CLConvolutionLayerReshapeWeights(std::shared_ptr<IMemoryManager> memory_manager = nullptr); /** Set the input and output tensors. * - * @param[in] weights Weights tensor. Weights are 4D tensor with dimensions [kernel_x, kernel_y, IFM, OFM]. - * Data type supported: QS8/QASYMM8/QS16/F16/F32. - * @param[in] biases Biases tensor. Shared biases supported. Biases are 1D tensor with dimensions [OFM]. Data type supported: Same as @p weights. - * @param[out] output Destination tensor. Data types supported: Same as @p weights. - * @param[in] transpose1xW True if the weights are to undergo a 1xW transposition after reshaping (in case of GEMM operation), false otherwise. - * Data types supported: Same as @p weights. + * @param[in] weights Weights tensor. Weights are 4D tensor with dimensions [kernel_x, kernel_y, IFM, OFM]. + * Data type supported: QS8/QASYMM8/QS16/F16/F32. + * @param[in] biases Biases tensor. Shared biases supported. Biases are 1D tensor with dimensions [OFM]. Data type supported: Same as @p weights. + * @param[out] output Destination tensor. Data types supported: Same as @p weights. */ - void configure(const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, bool transpose1xW); + void configure(const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output); + /** Static function to check if given info will lead to a valid configuration of @ref CLConvolutionLayerReshapeWeights + * + * @param[in] weights Weights tensor. Weights are 4D tensor with dimensions [kernel_x, kernel_y, IFM, OFM]. + * Data type supported: QS8/QASYMM8/QS16/F16/F32. + * @param[in] biases Biases tensor. Shared biases supported. Biases are 1D tensor with dimensions [OFM]. Data type supported: Same as @p weights. + * @param[in] output Destination tensor. Data types supported: Same as @p weights. + * + * @return a status + */ + static Status validate(const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output); // Inherited methods overridden: void run() override; @@ -74,7 +82,6 @@ private: CLWeightsReshapeKernel _weights_reshape_kernel; CLGEMMTranspose1xWKernel _weights_transposed_kernel; CLTensor _weights_reshaped; - bool _transpose1xW; }; /** Basic function to compute the convolution layer. This function calls the following OpenCL kernels/functions: @@ -112,6 +119,22 @@ public: * tensor has also been transposed with CLGEMMTranspose1xWKernel. Data type supported: Same as @p input. */ void configure(const ICLTensor *input, const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, const PadStrideInfo &conv_info, const WeightsInfo &weights_info = WeightsInfo()); + /** Static function to check if given info will lead to a valid configuration of @ref CLGEMMConvolutionLayer. + * + * @param[in] input Source tensor. 3 lower dimensions represent a single input [width, height, IFM], + * while every optional dimension from 4 and above represent a batch of inputs. + * Data types supported: QS8/QASYMM8/QS16/F16/F32. + * @param[in] weights Weights tensor. Weights are 4D tensor with dimensions [kernel_x, kernel_y, IFM, OFM]. Data type supported: Same as @p input. + * @param[in] biases Biases tensor. Shared biases supported. Biases are 1D tensor with dimensions [OFM]. + * Data type supported: Should match @p input data type, except for input of QASYMM8 type where biases should be of S32 type. + * @param[out] output Destination tensor. 3 lower dimensions represent a single output [width, height, OFM], while the rest represent batch of outputs. + * Data types supported: Same as @p input. + * @param[in] conv_info Contains padding and stride information described in @ref PadStrideInfo. + * @param[in] weights_info Specifies if the weights tensor has been reshaped with CLWeightsReshapeKernel. If this is not part of the fully connected layer the weights + * tensor has also been transposed with CLGEMMTranspose1xWKernel. Data type supported: Same as @p input. + */ + static Status validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const PadStrideInfo &conv_info, + const WeightsInfo &weights_info = WeightsInfo()); // Inherited methods overridden: void run() override; @@ -123,16 +146,23 @@ private: * @param weights Weights tensor. Data type supported: Same as @p input. * @param output Output tensor. Data types supported: Same as @p input, * except for input of QASYMM8 type where output should be of S32 type. - * @param is_interleaved_transposed Flag that signals if matrix is interleaved transposed */ - void configure_mm(const ICLTensor *input, const ICLTensor *weights, ICLTensor *output, bool is_interleaved_transposed, bool are_weights_reshaped); + void configure_mm(const ICLTensor *input, const ICLTensor *weights, ICLTensor *output); + /** Static function to check if given info will lead to a valid configuration of @ref CLGEMMConvolutionLayer matrix multiply routines + * + * @param[in] input Input tensor. Data types supported: QS8/QASYMM8/QS16/F16/F32. + * @param[in] weights Weights tensor. Data type supported: Same as @p input. + * @param[in] output Output tensor. Data types supported: Same as @p input, + * except for input of QASYMM8 type where output should be of S32 type. + * + * @return a status + */ + static Status validate_mm(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *output); private: CLMemoryGroup _memory_group; CLConvolutionLayerReshapeWeights _reshape_weights; CLIm2ColKernel _im2col_kernel; - CLGEMMInterleave4x4Kernel _interleave_kernel; - CLGEMMMatrixMultiplyKernel _mm_kernel; CLGEMM _mm_gemm; CLGEMMLowpMatrixMultiplyCore _mm_gemmlowp; CLGEMMLowpQuantizeDownInt32ToUint8ScaleByFixedPoint _gemmlowp_output_stage; @@ -145,9 +175,7 @@ private: CLTensor _gemm_output; CLTensor _tmp_output; - bool _are_weights_reshaped; bool _is_quantized; - bool _is_interleaved_transposed; }; } #endif /* __ARM_COMPUTE_CLGEMMCONVOLUTIONLAYER_H__ */ diff --git a/arm_compute/runtime/NEON/functions/NEGEMM.h b/arm_compute/runtime/NEON/functions/NEGEMM.h index 4b0614badc..f2b6ef77bd 100644 --- a/arm_compute/runtime/NEON/functions/NEGEMM.h +++ b/arm_compute/runtime/NEON/functions/NEGEMM.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * diff --git a/src/core/CL/kernels/CLCol2ImKernel.cpp b/src/core/CL/kernels/CLCol2ImKernel.cpp index c8005ec0f6..eacfa4c110 100644 --- a/src/core/CL/kernels/CLCol2ImKernel.cpp +++ b/src/core/CL/kernels/CLCol2ImKernel.cpp @@ -31,10 +31,55 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/misc/ShapeCalculator.h" #include <cmath> using namespace arm_compute; +using namespace arm_compute::misc::shape_calculator; + +namespace +{ +Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output, std::pair<unsigned int, unsigned int> convolved_dims) +{ + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, output); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32); + + // Checks performed when output is configured + if(output->total_size() != 0) + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(output->tensor_shape(), compute_col2im_shape(*input, convolved_dims)); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(input, output); + ARM_COMPUTE_ERROR_ON_MISMATCHING_QUANTIZATION_INFO(input, output); + } + + return Status{}; +} + +std::pair<Status, Window> validate_and_configure_window(ITensorInfo *input, ITensorInfo *output, std::pair<unsigned int, unsigned int> convolved_dims) +{ + ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); + // Output auto inizialitation if not yet initialized + auto_init_if_empty(*output, input->clone()->set_tensor_shape(compute_col2im_shape(*input, convolved_dims))); + + const unsigned int num_elems_read_per_iteration = is_data_type_fixed_point(input->data_type()) ? 1 : 8; + + // Configure window + Window win = calculate_max_window(*input, Steps(num_elems_read_per_iteration)); + + // Update window and padding just for the input tensor as we cannot access out-of-bounds elements in the output one + AccessWindowHorizontal input_access(input, 0, num_elems_read_per_iteration); + bool window_changed = update_window_and_padding(win, input_access); + + Coordinates coord; + coord.set_num_dimensions(output->num_dimensions()); + output->set_valid_region(ValidRegion(coord, output->tensor_shape())); + + Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{}; + return std::make_pair(err, win); +} +} // namespace CLCol2ImKernel::CLCol2ImKernel() : _input(nullptr), _output(nullptr), _convolved_dims() @@ -43,20 +88,10 @@ CLCol2ImKernel::CLCol2ImKernel() void CLCol2ImKernel::configure(const ICLTensor *input, ICLTensor *output, std::pair<unsigned int, unsigned int> convolved_dims) { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32); - ARM_COMPUTE_ERROR_ON_NULLPTR(output); - - TensorShape output_shape = input->info()->tensor_shape(); - output_shape.set(0, convolved_dims.first); - output_shape.set(1, convolved_dims.second); - output_shape.set(2, input->info()->tensor_shape()[0]); + ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); - // Output auto inizialitation if not yet initialized - auto_init_if_empty(*output->info(), input->info()->clone()->set_tensor_shape(output_shape)); - - ARM_COMPUTE_ERROR_ON_MISMATCHING_DIMENSIONS(output->info()->tensor_shape(), output_shape); - ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); - ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, output); + // Perform validation step + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), output->info(), convolved_dims)); _input = input; _output = output; @@ -89,20 +124,10 @@ void CLCol2ImKernel::configure(const ICLTensor *input, ICLTensor *output, std::p } } - const unsigned int num_elems_read_per_iteration = is_data_type_fixed_point(data_type) ? 1 : 8; - - // Configure window - Window win = calculate_max_window(*input->info(), Steps(num_elems_read_per_iteration)); - - // Update window and padding just for the input tensor as we cannot access out-of-bounds elements in the output one - AccessWindowHorizontal input_access(input->info(), 0, num_elems_read_per_iteration); - update_window_and_padding(win, input_access); - - Coordinates coord; - coord.set_num_dimensions(output->info()->num_dimensions()); - output->info()->set_valid_region(ValidRegion(coord, output->info()->tensor_shape())); - - ICLKernel::configure(win); + // Configure kernel window + auto win_config = validate_and_configure_window(input->info(), output->info(), _convolved_dims); + ARM_COMPUTE_ERROR_THROW_ON(win_config.first); + ICLKernel::configure(win_config.second); // Set config_id for enabling LWS tuning _config_id = "col2im_"; @@ -117,6 +142,12 @@ void CLCol2ImKernel::configure(const ICLTensor *input, ICLTensor *output, std::p _config_id += support::cpp11::to_string(output->info()->dimension(1)); } +Status CLCol2ImKernel::validate(const ITensorInfo *input, const ITensorInfo *output, std::pair<unsigned int, unsigned int> convolved_dims) +{ + ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, output, convolved_dims)); + return Status{}; +} + void CLCol2ImKernel::run(const Window &window, cl::CommandQueue &queue) { ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); diff --git a/src/core/CL/kernels/CLGEMMMatrixAdditionKernel.cpp b/src/core/CL/kernels/CLGEMMMatrixAdditionKernel.cpp index 1499df0bec..3fe956d759 100644 --- a/src/core/CL/kernels/CLGEMMMatrixAdditionKernel.cpp +++ b/src/core/CL/kernels/CLGEMMMatrixAdditionKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -36,6 +36,42 @@ using namespace arm_compute; +namespace +{ +std::pair<Status, Window> validate_and_configure_window(ITensorInfo *input, ITensorInfo *output) +{ + const unsigned int num_elems_processed_per_iteration = max_cl_vector_width / data_size_from_type(input->data_type()); + // Configure kernel window + Window win = calculate_max_window(*input, Steps(num_elems_processed_per_iteration)); + + AccessWindowHorizontal input_access(input, 0, num_elems_processed_per_iteration); + AccessWindowHorizontal output_access(output, 0, num_elems_processed_per_iteration); + + bool window_changed = update_window_and_padding(win, input_access, output_access); + + output_access.set_valid_region(win, input->valid_region()); + + Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{}; + return std::make_pair(err, win); +} +} // namespace + +namespace +{ +Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output, const float beta) +{ + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, output); + + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); + ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(0) != output->dimension(0)); + ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(1) != output->dimension(1)); + + ARM_COMPUTE_UNUSED(beta); + return Status{}; +} +} // namespace + CLGEMMMatrixAdditionKernel::CLGEMMMatrixAdditionKernel() : _input(nullptr), _output(nullptr) { @@ -43,14 +79,13 @@ CLGEMMMatrixAdditionKernel::CLGEMMMatrixAdditionKernel() void CLGEMMMatrixAdditionKernel::configure(const ICLTensor *input, ICLTensor *output, float beta) { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32); - ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); - ARM_COMPUTE_ERROR_ON(input->info()->dimension(0) != output->info()->dimension(0)); - ARM_COMPUTE_ERROR_ON(input->info()->dimension(1) != output->info()->dimension(1)); + ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); + + // Perform validation step + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), output->info(), beta)); - _input = input; - _output = output; - const unsigned int num_elems_processed_per_iteration = max_cl_vector_width / data_size_from_type(input->info()->data_type()); + _input = input; + _output = output; std::ostringstream ma_arguments; if(is_data_type_fixed_point(input->info()->data_type())) @@ -74,16 +109,15 @@ void CLGEMMMatrixAdditionKernel::configure(const ICLTensor *input, ICLTensor *ou _kernel = static_cast<cl::Kernel>(CLKernelLibrary::get().create_kernel(("gemm_ma_" + data_type_name), build_opts)); // Configure kernel window - Window win = calculate_max_window(*_input->info(), Steps(num_elems_processed_per_iteration)); - - AccessWindowHorizontal input_access(input->info(), 0, num_elems_processed_per_iteration); - AccessWindowHorizontal output_access(output->info(), 0, num_elems_processed_per_iteration); - - update_window_and_padding(win, input_access, output_access); - - output_access.set_valid_region(win, input->info()->valid_region()); + auto win_config = validate_and_configure_window(input->info(), output->info()); + ARM_COMPUTE_ERROR_THROW_ON(win_config.first); + ICLKernel::configure(win_config.second); +} - ICLKernel::configure(win); +Status CLGEMMMatrixAdditionKernel::validate(const ITensorInfo *input, const ITensorInfo *output, const float beta) +{ + ARM_COMPUTE_RETURN_ERROR_ON(validate_arguments(input, output, beta)); + return Status{}; } void CLGEMMMatrixAdditionKernel::run(const Window &window, cl::CommandQueue &queue) diff --git a/src/core/CL/kernels/CLGEMMMatrixMultiplyKernel.cpp b/src/core/CL/kernels/CLGEMMMatrixMultiplyKernel.cpp index e23feb269a..6c31e371da 100644 --- a/src/core/CL/kernels/CLGEMMMatrixMultiplyKernel.cpp +++ b/src/core/CL/kernels/CLGEMMMatrixMultiplyKernel.cpp @@ -50,6 +50,7 @@ using ElementsProcessed = Steps; inline Status validate_arguments(const ITensorInfo *input0, const ITensorInfo *input1, const ITensorInfo *output, bool is_interleaved_transposed, const GEMMReshapeInfo &reshape_info) { + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input0, input1, output); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input0, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input0, input1); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(input0, input1); diff --git a/src/core/CL/kernels/CLWeightsReshapeKernel.cpp b/src/core/CL/kernels/CLWeightsReshapeKernel.cpp index 3a9a32e58f..f5eaa5afb2 100644 --- a/src/core/CL/kernels/CLWeightsReshapeKernel.cpp +++ b/src/core/CL/kernels/CLWeightsReshapeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -31,8 +31,41 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/misc/ShapeCalculator.h" using namespace arm_compute; +using namespace arm_compute::misc::shape_calculator; + +namespace +{ +Status validate_arguments(const ITensorInfo *input, const ITensorInfo *biases, const ITensorInfo *output) +{ + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, output); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32); + + if(biases != nullptr) + { + ARM_COMPUTE_RETURN_ERROR_ON(is_data_type_quantized_asymmetric(input->data_type())); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(input, biases); + ARM_COMPUTE_RETURN_ERROR_ON((input->num_dimensions() == 4) && (biases->num_dimensions() != 1)); + ARM_COMPUTE_RETURN_ERROR_ON((input->num_dimensions() == 5) && (biases->num_dimensions() != 2)); + ARM_COMPUTE_RETURN_ERROR_ON((input->num_dimensions() == 4) && (biases->dimension(0) != input->tensor_shape()[3])); + ARM_COMPUTE_RETURN_ERROR_ON((input->num_dimensions() == 5) && (biases->dimension(0) != input->tensor_shape()[3] || biases->dimension(1) != input->tensor_shape()[4])); + } + + // Checks performed when output is configured + if(output->total_size() != 0) + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(output->tensor_shape(), compute_weights_reshaped_shape(*input, biases != nullptr)); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(input, output); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_QUANTIZATION_INFO(input, output); + } + + return Status{}; +} +} // namespace CLWeightsReshapeKernel::CLWeightsReshapeKernel() : _input(nullptr), _biases(nullptr), _output(nullptr) @@ -41,35 +74,17 @@ CLWeightsReshapeKernel::CLWeightsReshapeKernel() void CLWeightsReshapeKernel::configure(const ICLTensor *input, const ICLTensor *biases, ICLTensor *output) { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32); - ARM_COMPUTE_ERROR_ON_NULLPTR(output); - - const DataType data_type = input->info()->data_type(); - - // Calculate output shape - TensorShape output_shape{ input->info()->tensor_shape() }; - output_shape.collapse(3); - const size_t tmp_dim = output_shape[0]; - output_shape.set(0, output_shape[1]); - output_shape.set(1, tmp_dim + (biases != nullptr ? 1 : 0)); + ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); // Output tensor auto inizialitation if not yet initialized - auto_init_if_empty(*output->info(), input->info()->clone()->set_tensor_shape(output_shape)); + auto_init_if_empty(*output->info(), input->info()->clone()->set_tensor_shape(compute_weights_reshaped_shape(*input->info(), (biases != nullptr)))); - ARM_COMPUTE_ERROR_ON_MISMATCHING_DIMENSIONS(output->info()->tensor_shape(), output_shape); - ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); - ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, output); + // Perform validation step + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), + (biases != nullptr) ? biases->info() : nullptr, + output->info())); - if(biases != nullptr) - { - ARM_COMPUTE_ERROR_ON(is_data_type_quantized_asymmetric(data_type)); - ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases); - ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, biases); - ARM_COMPUTE_ERROR_ON((input->info()->num_dimensions() == 4) && (biases->info()->num_dimensions() != 1)); - ARM_COMPUTE_ERROR_ON((input->info()->num_dimensions() == 5) && (biases->info()->num_dimensions() != 2)); - ARM_COMPUTE_ERROR_ON((input->info()->num_dimensions() == 4) && (biases->info()->dimension(0) != input->info()->tensor_shape()[3])); - ARM_COMPUTE_ERROR_ON((input->info()->num_dimensions() == 5) && (biases->info()->dimension(0) != input->info()->tensor_shape()[3] || biases->info()->dimension(1) != input->info()->tensor_shape()[4])); - } + const DataType data_type = input->info()->data_type(); _biases = biases; _output = output; @@ -99,6 +114,12 @@ void CLWeightsReshapeKernel::configure(const ICLTensor *input, const ICLTensor * ICLKernel::configure(win); } +Status CLWeightsReshapeKernel::validate(const ITensorInfo *input, const ITensorInfo *biases, const ITensorInfo *output) +{ + ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, biases, output)); + return Status{}; +} + void CLWeightsReshapeKernel::run(const Window &window, cl::CommandQueue &queue) { ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); diff --git a/src/runtime/CL/functions/CLConvolutionLayer.cpp b/src/runtime/CL/functions/CLConvolutionLayer.cpp index a0bee520a6..1a486ce5c7 100644 --- a/src/runtime/CL/functions/CLConvolutionLayer.cpp +++ b/src/runtime/CL/functions/CLConvolutionLayer.cpp @@ -26,6 +26,8 @@ #include "arm_compute/core/PixelValue.h" #include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/quantization/AsymmHelpers.h" #include "arm_compute/runtime/CL/CLScheduler.h" #include <cmath> @@ -33,6 +35,7 @@ #include <tuple> using namespace arm_compute; +using namespace arm_compute::misc::shape_calculator; CLConvolutionLayer::CLConvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager) : _memory_manager(std::move(memory_manager)), _function() @@ -70,7 +73,7 @@ void CLConvolutionLayer::configure(ICLTensor *input, const ICLTensor *weights, c Status CLConvolutionLayer::validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const PadStrideInfo &conv_info, const WeightsInfo &weights_info) { - ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output); + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output); //Configure if the parameters match the direct convolution or the gemm-based const GPUTarget gpu_target = CLScheduler::get().target(); @@ -86,8 +89,7 @@ Status CLConvolutionLayer::validate(const ITensorInfo *input, const ITensorInfo case ConvolutionMethod::GEMM: { // Validate gemm-based convolution layer - /* TODO COMPMID-754: Add validation methods for CLGEMMConvolutionLayer - CLGEMMConvolutionLayer::validate(input, weights, biases, output, conv_info, weights_info); */ + CLGEMMConvolutionLayer::validate(input, weights, biases, output, conv_info, weights_info); break; } default: diff --git a/src/runtime/CL/functions/CLDeconvolutionLayer.cpp b/src/runtime/CL/functions/CLDeconvolutionLayer.cpp index e7b546878f..e3bbe0f8be 100644 --- a/src/runtime/CL/functions/CLDeconvolutionLayer.cpp +++ b/src/runtime/CL/functions/CLDeconvolutionLayer.cpp @@ -80,7 +80,7 @@ Status CLDeconvolutionLayer::validate(const ITensorInfo *input, const ITensorInf const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL); ARM_COMPUTE_RETURN_ON_ERROR(CLDeconvolutionLayerUpsample::validate(input, &scale_out_info, BorderSize(inner_border_right, inner_border_top), info)); - // TODO (COMPMID-754): Add validation of CLConvolutionLayer when added. + ARM_COMPUTE_RETURN_ON_ERROR(CLConvolutionLayer::validate(&scale_out_info, weights, bias, output, info, WeightsInfo())); return Status{}; } diff --git a/src/runtime/CL/functions/CLGEMM.cpp b/src/runtime/CL/functions/CLGEMM.cpp index f02eb169b7..da00d2dc64 100644 --- a/src/runtime/CL/functions/CLGEMM.cpp +++ b/src/runtime/CL/functions/CLGEMM.cpp @@ -60,6 +60,31 @@ inline bool is_interleaved_transposed(int m, int n, int k, DataType data_type, b return flag; } + +Status validate_arguments(const ITensorInfo *a, const ITensorInfo *b, const ICLTensor *c, const ITensorInfo *output, const float alpha, const float beta, const GEMMInfo &gemm_info = GEMMInfo()) +{ + ARM_COMPUTE_ERROR_ON_NULLPTR(a, b, output); + + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(a, b, output); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.is_a_reshaped(), "Matrix A already reshaped is not supported"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.is_b_reshaped(), "Matrix B already reshaped is not supported"); + + if(c != nullptr) + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(a, c->info()); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->dimension(1) != c->info()->dimension(1), "The C matrix must have the same number of rows as the matrix A"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(b->dimension(0) != c->info()->dimension(0), "The C matrix must have the same number of columns as the matrix B"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(c->info()->dimension(0) != output->dimension(0), "The C matrix must have the same number of rows as the output matrix"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(c->info()->dimension(1) != output->dimension(1), "The C matrix must have the same number of columns as the output matrix"); + } + + ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->dimension(0) != b->dimension(1), "The product AB is defined only if the number of columns in A is equal to the number of rows in B"); + + ARM_COMPUTE_UNUSED(alpha); + ARM_COMPUTE_UNUSED(beta); + return Status{}; +} } // namespace CLGEMM::CLGEMM(std::shared_ptr<IMemoryManager> memory_manager) @@ -70,21 +95,10 @@ CLGEMM::CLGEMM(std::shared_ptr<IMemoryManager> memory_manager) void CLGEMM::configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor *c, ICLTensor *output, float alpha, float beta, const GEMMInfo &gemm_info) { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32); - ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(a, b, output); - ARM_COMPUTE_ERROR_ON_MSG(gemm_info.is_a_reshaped(), "Matrix A already reshaped is not supported"); - ARM_COMPUTE_ERROR_ON_MSG(gemm_info.is_b_reshaped(), "Matrix B already reshaped is not supported"); - - if(c != nullptr) - { - ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(a, c); - ARM_COMPUTE_ERROR_ON_MSG(a->info()->dimension(1) != c->info()->dimension(1), "The C matrix must have the same number of rows as the matrix A"); - ARM_COMPUTE_ERROR_ON_MSG(b->info()->dimension(0) != c->info()->dimension(0), "The C matrix must have the same number of columns as the matrix B"); - ARM_COMPUTE_ERROR_ON_MSG(c->info()->dimension(0) != output->info()->dimension(0), "The C matrix must have the same number of rows as the output matrix"); - ARM_COMPUTE_ERROR_ON_MSG(c->info()->dimension(1) != output->info()->dimension(1), "The C matrix must have the same number of columns as the output matrix"); - } + ARM_COMPUTE_ERROR_ON_NULLPTR(a, b, output); - ARM_COMPUTE_ERROR_ON_MSG(a->info()->dimension(0) != b->info()->dimension(1), "The product AB is defined only if the number of columns in A is equal to the number of rows in B"); + // Perform validation step + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(a->info(), b->info(), c, output->info(), alpha, beta, gemm_info)); // Check if we need to reshape the matrix B only on the first run _reshape_b_only_on_first_run = gemm_info.reshape_b_only_on_first_run(); @@ -152,6 +166,12 @@ void CLGEMM::configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor * } } +Status CLGEMM::validate(const ITensorInfo *a, const ITensorInfo *b, const ICLTensor *c, const ITensorInfo *output, const float alpha, const float beta, const GEMMInfo &gemm_info) +{ + ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(a, b, c, output, alpha, beta, gemm_info)); + return Status{}; +} + void CLGEMM::run() { _memory_group.acquire(); diff --git a/src/runtime/CL/functions/CLGEMMConvolutionLayer.cpp b/src/runtime/CL/functions/CLGEMMConvolutionLayer.cpp index 60e1bde4e2..23c3050476 100644 --- a/src/runtime/CL/functions/CLGEMMConvolutionLayer.cpp +++ b/src/runtime/CL/functions/CLGEMMConvolutionLayer.cpp @@ -27,6 +27,7 @@ #include "arm_compute/core/Size2D.h" #include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" #include "arm_compute/runtime/CL/CLScheduler.h" @@ -35,53 +36,52 @@ #include <tuple> using namespace arm_compute; +using namespace arm_compute::misc::shape_calculator; CLConvolutionLayerReshapeWeights::CLConvolutionLayerReshapeWeights(std::shared_ptr<IMemoryManager> memory_manager) - : _memory_group(std::move(memory_manager)), _weights_reshape_kernel(), _weights_transposed_kernel(), _weights_reshaped(), _transpose1xW(false) + : _memory_group(std::move(memory_manager)), _weights_reshape_kernel(), _weights_transposed_kernel(), _weights_reshaped() { } -void CLConvolutionLayerReshapeWeights::configure(const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, bool transpose1xW) +void CLConvolutionLayerReshapeWeights::configure(const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output) { + // Perform validation step ARM_COMPUTE_ERROR_ON_NULLPTR(weights, output); - ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 4); - - if(biases != nullptr) - { - ARM_COMPUTE_ERROR_ON(is_data_type_quantized_asymmetric(weights->info()->data_type())); - ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(weights, biases); - ARM_COMPUTE_ERROR_ON(biases->info()->dimension(0) != weights->info()->dimension(3)); - ARM_COMPUTE_ERROR_ON(biases->info()->num_dimensions() > 1); - } + ARM_COMPUTE_ERROR_THROW_ON(CLConvolutionLayerReshapeWeights::validate(weights->info(), + (biases != nullptr) ? biases->info() : nullptr, + output->info())); const bool append_biases = (biases != nullptr) && !is_data_type_quantized_asymmetric(weights->info()->data_type()); - const unsigned bias_element = (append_biases) ? 1 : 0; const ICLTensor *biases_to_use = (append_biases) ? biases : nullptr; - _transpose1xW = transpose1xW; + _weights_reshape_kernel.configure(weights, biases_to_use, output); + + output->info()->set_quantization_info(weights->info()->quantization_info()); +} + +Status CLConvolutionLayerReshapeWeights::validate(const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output) +{ + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(weights); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(weights, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32); + ARM_COMPUTE_RETURN_ERROR_ON(weights->num_dimensions() > 4); - if(transpose1xW) + if(biases != nullptr) { - // Create tensor to store the reshaped weights - const unsigned int mat_weights_cols = weights->info()->dimension(3); - const unsigned int mat_weights_rows = weights->info()->dimension(0) * weights->info()->dimension(1) * weights->info()->dimension(2) + bias_element; - TensorShape shape_wr(mat_weights_cols, mat_weights_rows); - const DataType dt = weights->info()->data_type(); - const int fixed_point_position = weights->info()->fixed_point_position(); - TensorInfo info_wr(shape_wr, 1, dt, fixed_point_position); - - _weights_reshaped.allocator()->init(info_wr); - _memory_group.manage(&_weights_reshaped); - _weights_reshape_kernel.configure(weights, biases_to_use, &_weights_reshaped); - _weights_transposed_kernel.configure(&_weights_reshaped, output); - _weights_reshaped.allocator()->allocate(); + ARM_COMPUTE_RETURN_ERROR_ON(is_data_type_quantized_asymmetric(weights->data_type())); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, biases); + ARM_COMPUTE_RETURN_ERROR_ON(biases->dimension(0) != weights->dimension(3)); + ARM_COMPUTE_RETURN_ERROR_ON(biases->num_dimensions() > 1); } - else + + if((output != nullptr) && (output->total_size() != 0)) { - _weights_reshape_kernel.configure(weights, biases_to_use, output); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, output); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(weights, output); + + CLWeightsReshapeKernel::validate(weights, biases, output); } - output->info()->set_quantization_info(weights->info()->quantization_info()); + return Status{}; } void CLConvolutionLayerReshapeWeights::run() @@ -89,99 +89,92 @@ void CLConvolutionLayerReshapeWeights::run() _memory_group.acquire(); CLScheduler::get().enqueue(_weights_reshape_kernel); - if(_transpose1xW) - { - CLScheduler::get().enqueue(_weights_transposed_kernel); - } _memory_group.release(); } CLGEMMConvolutionLayer::CLGEMMConvolutionLayer(std::shared_ptr<IMemoryManager> memory_manager) - : _memory_group(memory_manager), _reshape_weights(), _im2col_kernel(), _interleave_kernel(), _mm_kernel(), _mm_gemm(memory_manager), _mm_gemmlowp(memory_manager), _gemmlowp_output_stage(), - _col2im_kernel(), _im2col_output(), _interleave_output(), _weights_reshaped(), _weights_transposed(), _gemm_output(), _tmp_output(), _are_weights_reshaped(false), _is_quantized(false), - _is_interleaved_transposed(false) + : _memory_group(memory_manager), _reshape_weights(), _im2col_kernel(), _mm_gemm(memory_manager), _mm_gemmlowp(memory_manager), _gemmlowp_output_stage(), _col2im_kernel(), _im2col_output(), + _interleave_output(), _weights_reshaped(), _weights_transposed(), _gemm_output(), _tmp_output(), _is_quantized(false) { } -void CLGEMMConvolutionLayer::configure_mm(const ICLTensor *input, const ICLTensor *weights, ICLTensor *output, bool is_interleaved_transposed, bool are_weights_reshaped) +void CLGEMMConvolutionLayer::configure_mm(const ICLTensor *input, const ICLTensor *weights, ICLTensor *output) { ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights); + ARM_COMPUTE_ERROR_THROW_ON(validate_mm(input->info(), weights->info(), output->info())); + if(_is_quantized) { - if(are_weights_reshaped) - { - ARM_COMPUTE_ERROR("Weights already reshaped are not suppported with gemmlowp"); - } - else - { - // Since we need negative offsets for computing convolution, we need to change QuantizationInfo() - // Extract and negate input and weights offset - const QuantizationInfo input_quantization_info = input->info()->quantization_info(); - const QuantizationInfo weights_quantization_info = weights->info()->quantization_info(); + // Since we need negative offsets for computing convolution, we need to change QuantizationInfo() + // Extract and negate input and weights offset + const QuantizationInfo input_quantization_info = input->info()->quantization_info(); + const QuantizationInfo weights_quantization_info = weights->info()->quantization_info(); - input->info()->set_quantization_info(QuantizationInfo(input_quantization_info.scale, -input_quantization_info.offset)); - weights->info()->set_quantization_info(QuantizationInfo(weights_quantization_info.scale, -weights_quantization_info.offset)); + input->info()->set_quantization_info(QuantizationInfo(input_quantization_info.scale, -input_quantization_info.offset)); + weights->info()->set_quantization_info(QuantizationInfo(weights_quantization_info.scale, -weights_quantization_info.offset)); - _mm_gemmlowp.configure(input, weights, output, GEMMInfo(false, false, true /* Reshape weights only for the first run*/)); + _mm_gemmlowp.configure(input, weights, output, GEMMInfo(false, false, true /* Reshape weights only for the first run*/)); - // Revert back QuantizatioInfo as input and weights could be used in other convolution layers - input->info()->set_quantization_info(input_quantization_info); - weights->info()->set_quantization_info(weights_quantization_info); - } + // Revert back QuantizatioInfo as input and weights could be used in other convolution layers + input->info()->set_quantization_info(input_quantization_info); + weights->info()->set_quantization_info(weights_quantization_info); } else { - if(are_weights_reshaped) - { - // Configure matrix multiply kernel - _mm_kernel.configure(input, weights, output, 1.f, is_interleaved_transposed); - } - else - { - // Configure matrix multiply function - _mm_gemm.configure(input, weights, nullptr, output, 1.0f, 0.0f, GEMMInfo(false, false, true /* Reshape weights only for the first run*/)); - } + // Configure matrix multiply function + _mm_gemm.configure(input, weights, nullptr, output, 1.0f, 0.0f, GEMMInfo(false, false, true /* Reshape weights only for the first run*/)); } } +Status CLGEMMConvolutionLayer::validate_mm(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *output) +{ + const bool is_quantized = is_data_type_quantized_asymmetric(input->data_type()); + + const GEMMInfo &gemm_info = GEMMInfo(false, false, true /* Reshape weights only for the first run */); + if(is_quantized) + { + // Since we need negative offsets for computing convolution, we need to change QuantizationInfo() + // Extract and negate input and weights offset + const QuantizationInfo input_quantization_info = input->quantization_info(); + const QuantizationInfo weights_quantization_info = weights->quantization_info(); + + std::unique_ptr<ITensorInfo> input_qa = input->clone(); + std::unique_ptr<ITensorInfo> weights_qa = weights->clone(); + input_qa->set_quantization_info(QuantizationInfo(input_quantization_info.scale, -input_quantization_info.offset)); + weights_qa->set_quantization_info(QuantizationInfo(weights_quantization_info.scale, -weights_quantization_info.offset)); + + // Perform validation step on GEMMLowp + CLGEMMLowpMatrixMultiplyCore::validate(input_qa.get(), weights_qa.get(), output, gemm_info); + } + else + { + // Perform validation step on Matrix multiply function + CLGEMM::validate(input, weights, nullptr, output, 1.0f, 0.0f, gemm_info); + } + return Status{}; +} + void CLGEMMConvolutionLayer::configure(const ICLTensor *input, const ICLTensor *weights, const ICLTensor *biases, ICLTensor *output, const PadStrideInfo &conv_info, const WeightsInfo &weights_info) { ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output); - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32); - ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights); - ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, weights); - ARM_COMPUTE_ERROR_ON(weights_info.are_reshaped() && CLScheduler::get().target() == GPUTarget::BIFROST); - ARM_COMPUTE_ERROR_ON(!weights_info.are_reshaped() && weights->info()->dimension(2) != input->info()->dimension(2)); - ARM_COMPUTE_ERROR_ON(weights->info()->num_dimensions() > 4); - ARM_COMPUTE_ERROR_ON(weights_info.are_reshaped() && is_data_type_quantized_asymmetric(input->info()->data_type())); - _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type()); + ARM_COMPUTE_ERROR_THROW_ON(CLGEMMConvolutionLayer::validate(input->info(), + weights->info(), + biases != nullptr ? biases->info() : nullptr, + output->info(), + conv_info, + weights_info)); - if(biases != nullptr) - { - if(_is_quantized) - { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(biases, 1, DataType::S32); - } - else - { - ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases); - } - ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, biases); - ARM_COMPUTE_ERROR_ON(!weights_info.are_reshaped() && biases->info()->dimension(0) != weights->info()->dimension(3)); - ARM_COMPUTE_ERROR_ON(biases->info()->num_dimensions() > 1); - } + _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type()); const DataType dt = input->info()->data_type(); - // Set the GPU target for matrix multiply and im2col and col2im - _mm_kernel.set_target(CLScheduler::get().target()); + // Set the GPU target for im2col and col2im _im2col_kernel.set_target(CLScheduler::get().target()); _col2im_kernel.set_target(CLScheduler::get().target()); const bool append_bias = (biases != nullptr) && (!_is_quantized); - _are_weights_reshaped = weights_info.are_reshaped(); const unsigned bias_element = (append_bias) ? 1 : 0; const ICLTensor *biases_to_use = (append_bias) ? biases : nullptr; @@ -195,41 +188,19 @@ void CLGEMMConvolutionLayer::configure(const ICLTensor *input, const ICLTensor * unsigned int conv_w = 0; unsigned int conv_h = 0; - const unsigned int kernel_width = (_are_weights_reshaped) ? weights_info.kernel_size().first : weights->info()->dimension(0); - const unsigned int kernel_height = (_are_weights_reshaped) ? weights_info.kernel_size().second : weights->info()->dimension(1); + const unsigned int kernel_width = weights->info()->dimension(0); + const unsigned int kernel_height = weights->info()->dimension(1); std::tie(conv_w, conv_h) = scaled_dimensions(input->info()->dimension(0), input->info()->dimension(1), kernel_width, kernel_height, conv_info); - // Check if its a "fully connected" convolution - const bool is_fully_connected_convolution = ((conv_w == 1) && (conv_h == 1)); - _is_interleaved_transposed = (!is_fully_connected_convolution) && (!_is_quantized) && (_are_weights_reshaped); - unsigned int mat_weights_cols = weights->info()->dimension(3); unsigned int mat_weights_rows = weights->info()->dimension(0) * weights->info()->dimension(1) * weights->info()->dimension(2) + bias_element; - // Reshape weights if needed - if(_are_weights_reshaped) - { - if(is_fully_connected_convolution || _is_quantized) - { - mat_weights_cols = weights->info()->dimension(0); - mat_weights_rows = weights->info()->dimension(1); - } - else - { - mat_weights_cols = weights_info.num_kernels(); - const unsigned int quarter_reshaped_cols = weights->info()->dimension(0) / 4; - mat_weights_rows = quarter_reshaped_cols + bias_element; - } - } - else - { - // _weights_reshaped will be auto configured in the kernel. - // Just append biases and do not transpose 1xW as it will be reshaped in CLGEMM - _reshape_weights.configure(weights, biases_to_use, &_weights_reshaped, false); + // _weights_reshaped will be auto configured in the kernel. + // Just append biases and do not transpose 1xW as it will be reshaped in CLGEMM + _reshape_weights.configure(weights, biases_to_use, &_weights_reshaped); - weights = &_weights_reshaped; - } + weights = &_weights_reshaped; // Create tensor to store im2col reshaped inputs const unsigned int mat_input_cols = mat_weights_rows; @@ -259,21 +230,9 @@ void CLGEMMConvolutionLayer::configure(const ICLTensor *input, const ICLTensor * // Configure im2col _im2col_kernel.configure(input, &_im2col_output, Size2D(kernel_width, kernel_height), conv_info, append_bias); - // Configure matrix multiply - if(_is_interleaved_transposed) - { - // Configure GEMMInterleave4x4. _input_interleaved_reshaped will be auto configured in the kernel - _memory_group.manage(&_interleave_output); - _interleave_kernel.configure(&_im2col_output, &_interleave_output); + // Configure GEMM + configure_mm(&_im2col_output, weights, &_gemm_output); - // Configure GEMM - configure_mm(&_interleave_output, weights, &_gemm_output, true, _are_weights_reshaped); - _interleave_output.allocator()->allocate(); - } - else - { - configure_mm(&_im2col_output, weights, &_gemm_output, false, _are_weights_reshaped); - } _im2col_output.allocator()->allocate(); // Configure output stage for quantized case @@ -299,53 +258,117 @@ void CLGEMMConvolutionLayer::configure(const ICLTensor *input, const ICLTensor * ARM_COMPUTE_ERROR_ON_MSG((output->info()->dimension(0) != conv_w) || (output->info()->dimension(1) != conv_h), "Output shape does not match the expected one"); // Allocate intermediate tensor - if(!_are_weights_reshaped) + _weights_reshaped.allocator()->allocate(); + + ARM_COMPUTE_UNUSED(weights_info); +} + +Status CLGEMMConvolutionLayer::validate(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *output, const PadStrideInfo &conv_info, + const WeightsInfo &weights_info) +{ + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(weights_info.are_reshaped(), "Weights already reshaped are not supported!"); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QASYMM8, DataType::QS16, DataType::F16, DataType::F32); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(input, weights); + ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(2) != input->dimension(2)); + ARM_COMPUTE_RETURN_ERROR_ON(weights->num_dimensions() > 4); + + const bool is_quantized = is_data_type_quantized_asymmetric(input->data_type()); + const bool append_bias = (biases != nullptr) && (!is_quantized); + const unsigned bias_element = (append_bias) ? 1 : 0; + const DataType dt = input->data_type(); + + // Get convolved dimensions + unsigned int conv_w = 0; + unsigned int conv_h = 0; + + const unsigned int kernel_width = weights->dimension(0); + const unsigned int kernel_height = weights->dimension(1); + + std::tie(conv_w, conv_h) = scaled_dimensions(input->dimension(0), input->dimension(1), kernel_width, kernel_height, conv_info); + + unsigned int mat_weights_cols = weights->dimension(3); + unsigned int mat_weights_rows = weights->dimension(0) * weights->dimension(1) * weights->dimension(2) + bias_element; + + CLConvolutionLayerReshapeWeights::validate(weights, biases, nullptr); + + // Create tensor info for im2col reshaped inputs + const unsigned int mat_input_cols = mat_weights_rows; + const unsigned int mat_input_rows = conv_w * conv_h; + TensorShape shape_im2col = input->tensor_shape(); + shape_im2col.set(0, mat_input_cols); + shape_im2col.set(1, mat_input_rows); + shape_im2col.set(2, 1); + TensorInfo im2col_reshaped_info(shape_im2col, 1, dt, input->fixed_point_position()); + im2col_reshaped_info.set_quantization_info(input->quantization_info()); + CLIm2ColKernel::validate(input, &im2col_reshaped_info, Size2D(kernel_width, kernel_height), conv_info, append_bias); + + // Create GEMM output tensor + TensorShape shape_gemm = im2col_reshaped_info.tensor_shape(); + shape_gemm.set(0, mat_weights_cols); + shape_gemm.set(1, mat_input_rows); + const DataType gemm_data_type = is_quantized ? DataType::S32 : dt; + // GEMM output should be S32 for acquiring raw integer accumulator without quantized postprocessing for quantized asymmetric input. + TensorInfo info_gemm(shape_gemm, 1, gemm_data_type, input->fixed_point_position()); + info_gemm.set_quantization_info(output->quantization_info()); + + validate_mm(&im2col_reshaped_info, weights, &info_gemm); + + TensorInfo tmp_info(input->tensor_shape(), 1, DataType::QASYMM8, input->fixed_point_position()); + if(is_quantized) { - _weights_reshaped.allocator()->allocate(); + float multiplier = input->quantization_info().scale * weights->quantization_info().scale / output->quantization_info().scale; + int output_multiplier, output_shift; + quantization::calculate_quantized_multiplier_less_than_one(multiplier, &output_multiplier, &output_shift); + // Validate output stage for quantized case + CLGEMMLowpQuantizeDownInt32ToUint8ScaleByFixedPoint::validate(&info_gemm, biases, &tmp_info, output->quantization_info().offset); } + + // Validate Col2Im + CLCol2ImKernel::validate(is_quantized ? &tmp_info : &info_gemm, output, std::make_pair(conv_w, conv_h)); + + if(biases != nullptr) + { + if(is_quantized) + { + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(biases, 1, DataType::S32); + } + else + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases); + } + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT(input, biases); + ARM_COMPUTE_RETURN_ERROR_ON(biases->dimension(0) != weights->dimension(3)); + ARM_COMPUTE_RETURN_ERROR_ON(biases->num_dimensions() > 1); + } + + return Status{}; } void CLGEMMConvolutionLayer::run() { // Run weights reshaping (Runs once for every configure) - if(!_are_weights_reshaped) - { - _are_weights_reshaped = true; - _reshape_weights.run(); - } + _reshape_weights.run(); _memory_group.acquire(); // Run im2col CLScheduler::get().enqueue(_im2col_kernel); - // Note: _is_interleaved_transposed is true only if the weights passed to the function have been passed already reshaped - // and if we do not have QASYMM8 data type. If this flag is true, we need to run the - // gemm kernel instead of gemm function - if(_is_interleaved_transposed) + // Runs CLGEMM or CLGEMMLowpMatrixMultiplyCore functions + if(_is_quantized) { - // Run interleave4x4 kernel - CLScheduler::get().enqueue(_interleave_kernel); + // Run gemmlowp + _mm_gemmlowp.run(); - // Run matrix multiply kernel - CLScheduler::get().enqueue(_mm_kernel); + // Run output stage + _gemmlowp_output_stage.run(); } else { - // Runs CLGEMM or CLGEMMLowpMatrixMultiplyCore functions - if(_is_quantized) - { - // Run gemmlowp - _mm_gemmlowp.run(); - - // Run output stage - _gemmlowp_output_stage.run(); - } - else - { - // Run gemm - _mm_gemm.run(); - } + // Run gemm + _mm_gemm.run(); } // Reshape output matrix |