From 750641dd6aab1e5e62d1875b97b230312bb87959 Mon Sep 17 00:00:00 2001 From: Gian Marco Iodice Date: Tue, 8 May 2018 12:01:57 +0100 Subject: COMPMID-1052 - Rework validate method in CLGEMM Change-Id: Iece5bd6478b5fac5164abff30c1e63e8a77291a9 Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/130374 Reviewed-by: Anthony Barbier Tested-by: Jenkins --- .../core/CL/kernels/CLGEMMMatrixAdditionKernel.h | 2 +- arm_compute/core/utils/misc/ShapeCalculator.h | 8 ++ arm_compute/runtime/CL/functions/CLGEMM.h | 10 +-- src/core/CL/kernels/CLGEMMInterleave4x4Kernel.cpp | 18 ++-- src/core/CL/kernels/CLGEMMMatrixAdditionKernel.cpp | 14 ++-- src/core/CL/kernels/CLGEMMMatrixMultiplyKernel.cpp | 15 ++-- src/core/CL/kernels/CLGEMMTranspose1xWKernel.cpp | 16 ++-- src/runtime/CL/functions/CLGEMM.cpp | 98 ++++++++++++++-------- 8 files changed, 106 insertions(+), 75 deletions(-) diff --git a/arm_compute/core/CL/kernels/CLGEMMMatrixAdditionKernel.h b/arm_compute/core/CL/kernels/CLGEMMMatrixAdditionKernel.h index dc84a40ca8..3755d943c5 100644 --- a/arm_compute/core/CL/kernels/CLGEMMMatrixAdditionKernel.h +++ b/arm_compute/core/CL/kernels/CLGEMMMatrixAdditionKernel.h @@ -65,7 +65,7 @@ public: * * @return a status */ - static Status validate(const ITensorInfo *input, const ITensorInfo *output, const float beta); + static Status validate(const ITensorInfo *input, const ITensorInfo *output, float beta); // Inherited methods overridden: void run(const Window &window, cl::CommandQueue &queue) override; diff --git a/arm_compute/core/utils/misc/ShapeCalculator.h b/arm_compute/core/utils/misc/ShapeCalculator.h index 9543d989b8..30d3f9bb62 100644 --- a/arm_compute/core/utils/misc/ShapeCalculator.h +++ b/arm_compute/core/utils/misc/ShapeCalculator.h @@ -358,6 +358,14 @@ inline TensorShape compute_rnn_shape(const ITensorInfo *input, const unsigned in return output_shape; } +inline TensorShape compute_mm_shape(const ITensorInfo &input0, const ITensorInfo &input1, bool is_interleaved_transposed, const GEMMReshapeInfo &reshape_info) +{ + TensorShape tensor_shape{ input0.tensor_shape() }; + tensor_shape.set(0, is_interleaved_transposed ? reshape_info.n() : input1.dimension(0)); + tensor_shape.set(1, is_interleaved_transposed ? reshape_info.m() : input0.dimension(1)); + + return tensor_shape; +} } // namespace shape_calculator } // namespace misc } // namespace arm_compute diff --git a/arm_compute/runtime/CL/functions/CLGEMM.h b/arm_compute/runtime/CL/functions/CLGEMM.h index 60ff32c6fa..7a145cc183 100644 --- a/arm_compute/runtime/CL/functions/CLGEMM.h +++ b/arm_compute/runtime/CL/functions/CLGEMM.h @@ -85,10 +85,10 @@ public: void configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor *c, ICLTensor *output, float alpha, float beta, const GEMMInfo &gemm_info = GEMMInfo()); /** Static function to check if given info will lead to a valid configuration of @ref CLGEMM. * - * @param[in] a First input tensor (Matrix or Vector A). Data types supported: QS8/QS16/F16/F32 - * @param[in] b Second input tensor (Matrix B). Data type supported: same as @p a. - * @param[in] c Third input tensor (Matrix C). It can be a nullptr if just the multiplication between @p a and @p b is needed. Data type supported: same as @p a. - * @param[out] output Output tensor. Data type supported: same as @p a + * @param[in] a First input tensor info (Matrix or Vector A). Data types supported: QS8/QS16/F16/F32 + * @param[in] b Second input tensor info (Matrix B). Data type supported: same as @p a. + * @param[in] c Third input tensor info (Matrix C). It can be a nullptr if just the multiplication between @p a and @p b is needed. Data type supported: same as @p a. + * @param[out] output Output tensor info. Data type supported: same as @p a * @param[in] alpha Weight of the matrix product * @param[in] beta Weight of matrix C * @param[in] gemm_info (Optional) Specifies if the matrix A and/or matrix B have been reshaped and @@ -96,7 +96,7 @@ public: * * @return a status */ - static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ICLTensor *c, const ITensorInfo *output, const float alpha, const float beta, const GEMMInfo &gemm_info = GEMMInfo()); + static Status validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *output, float alpha, float beta, const GEMMInfo &gemm_info = GEMMInfo()); // Inherited methods overridden: void run() override; diff --git a/src/core/CL/kernels/CLGEMMInterleave4x4Kernel.cpp b/src/core/CL/kernels/CLGEMMInterleave4x4Kernel.cpp index d12255ff24..8f669a9298 100644 --- a/src/core/CL/kernels/CLGEMMInterleave4x4Kernel.cpp +++ b/src/core/CL/kernels/CLGEMMInterleave4x4Kernel.cpp @@ -69,16 +69,16 @@ std::pair validate_and_configure_window(ITensorInfo *input, ITen AccessWindowRectangle input_access(input, 0, 0, num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y); window_changed = window_changed || update_window_and_padding(win, input_access); - // Configure window in case of configured output - if(output->total_size() != 0) - { - const float scale_x = 4.0f * static_cast(mult_interleave4x4_height); - const float scale_y = 1.0f / (scale_x); + // Output auto inizialitation if not yet initialized + auto_init_if_empty(*output, input->clone()->set_tensor_shape(compute_interleaved_shape(*input, mult_interleave4x4_height))); - AccessWindowRectangle output_access(output, 0, 0, num_elems_written_per_iteration, 1, scale_x, scale_y); - window_changed = window_changed || update_window_and_padding(win, output_access); - output_access.set_valid_region(win, input->valid_region()); - } + // Configure window + const float scale_x = 4.0f * static_cast(mult_interleave4x4_height); + const float scale_y = 1.0f / (scale_x); + + AccessWindowRectangle output_access(output, 0, 0, num_elems_written_per_iteration, 1, scale_x, scale_y); + window_changed = window_changed || update_window_and_padding(win, output_access); + output_access.set_valid_region(win, input->valid_region()); // Collapse along the Z direction // This collapse needs to be here in order to tune the Z dimension of LWS diff --git a/src/core/CL/kernels/CLGEMMMatrixAdditionKernel.cpp b/src/core/CL/kernels/CLGEMMMatrixAdditionKernel.cpp index 4b4814f15d..4538812baa 100644 --- a/src/core/CL/kernels/CLGEMMMatrixAdditionKernel.cpp +++ b/src/core/CL/kernels/CLGEMMMatrixAdditionKernel.cpp @@ -58,16 +58,15 @@ std::pair validate_and_configure_window(ITensorInfo *input, ITen namespace { -Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output, const float beta) +Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output, float beta) { ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, output); + ARM_COMPUTE_UNUSED(input, output, beta); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); - ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(0) != output->dimension(0)); - ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(1) != output->dimension(1)); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(input, output); - ARM_COMPUTE_UNUSED(beta); return Status{}; } } // namespace @@ -114,11 +113,10 @@ void CLGEMMMatrixAdditionKernel::configure(const ICLTensor *input, ICLTensor *ou ICLKernel::configure(win_config.second); } -Status CLGEMMMatrixAdditionKernel::validate(const ITensorInfo *input, const ITensorInfo *output, const float beta) +Status CLGEMMMatrixAdditionKernel::validate(const ITensorInfo *input, const ITensorInfo *output, float beta) { - ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); - ARM_COMPUTE_RETURN_ERROR_ON(validate_arguments(input, output, beta)); - ARM_COMPUTE_RETURN_ERROR_ON(validate_and_configure_window(input->clone().get(), output->clone().get()).first); + ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, output, beta)); + ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(input->clone().get(), output->clone().get()).first); return Status{}; } diff --git a/src/core/CL/kernels/CLGEMMMatrixMultiplyKernel.cpp b/src/core/CL/kernels/CLGEMMMatrixMultiplyKernel.cpp index 674937eff0..cc9ae27bed 100644 --- a/src/core/CL/kernels/CLGEMMMatrixMultiplyKernel.cpp +++ b/src/core/CL/kernels/CLGEMMMatrixMultiplyKernel.cpp @@ -107,7 +107,7 @@ inline Status validate_arguments(const ITensorInfo *input0, const ITensorInfo *i } inline std::pair validate_and_configure_window(ITensorInfo *input0, ITensorInfo *input1, ITensorInfo *output, - bool is_interleaved_transposed, GPUTarget gpu_target, + bool is_interleaved_transposed, const GEMMReshapeInfo &reshape_info, GPUTarget gpu_target, ElementsProcessed &num_elements_processed) { bool window_changed = false; @@ -117,6 +117,9 @@ inline std::pair validate_and_configure_window(ITensorInfo *inpu unsigned int &num_elems_processed_per_iteration_x = num_elements_processed[0]; unsigned int &num_elems_processed_per_iteration_y = num_elements_processed[1]; + // Output tensor auto inizialitation if not yet initialized + auto_init_if_empty(*output, input0->clone()->set_tensor_shape(compute_mm_shape(*input0, *input1, is_interleaved_transposed, reshape_info))); + if(is_interleaved_transposed) { // Configure kernel window @@ -182,13 +185,6 @@ void CLGEMMMatrixMultiplyKernel::configure(const ICLTensor *input0, const ICLTen { ARM_COMPUTE_ERROR_ON_NULLPTR(input0, input1, output); - // Output tensor auto inizialitation if not yet initialized - TensorShape tensor_shape{ input0->info()->tensor_shape() }; - tensor_shape.set(0, is_interleaved_transposed ? reshape_info.n() : input1->info()->dimension(0)); - tensor_shape.set(1, is_interleaved_transposed ? reshape_info.m() : input0->info()->dimension(1)); - - auto_init_if_empty(*output->info(), input0->info()->clone()->set_tensor_shape(tensor_shape)); - // Perform validate step ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input0->info(), input1->info(), output->info(), is_interleaved_transposed, reshape_info)); @@ -246,7 +242,7 @@ void CLGEMMMatrixMultiplyKernel::configure(const ICLTensor *input0, const ICLTen ElementsProcessed num_elements_processed{}; // Configure kernel window - auto win_config = validate_and_configure_window(input0->info(), input1->info(), output->info(), is_interleaved_transposed, gpu_target, num_elements_processed); + auto win_config = validate_and_configure_window(input0->info(), input1->info(), output->info(), is_interleaved_transposed, reshape_info, gpu_target, num_elements_processed); ARM_COMPUTE_ERROR_THROW_ON(win_config.first); ICLKernel::configure(win_config.second); @@ -354,6 +350,7 @@ Status CLGEMMMatrixMultiplyKernel::validate(const ITensorInfo *input0, const ITe input1->clone().get(), output->clone().get(), is_interleaved_transposed, + reshape_info, gpu_target, num_elements_processed) .first); diff --git a/src/core/CL/kernels/CLGEMMTranspose1xWKernel.cpp b/src/core/CL/kernels/CLGEMMTranspose1xWKernel.cpp index 54abb1cde0..a9618609e6 100644 --- a/src/core/CL/kernels/CLGEMMTranspose1xWKernel.cpp +++ b/src/core/CL/kernels/CLGEMMTranspose1xWKernel.cpp @@ -78,17 +78,13 @@ std::pair validate_and_configure_window(ITensorInfo *input, ITen AccessWindowHorizontal input_access(input, 0, num_elems_processed_per_iteration); + // Output tensor auto inizialitation if not yet initialized + auto_init_if_empty(*output, input->clone()->set_tensor_shape(compute_transpose1xW_with_element_size_shape(*input, mult_transpose1xW_width))); + // Configure window in case of configured output - if(output->total_size() != 0) - { - AccessWindowStatic output_access(output, 0, 0, ceil_to_multiple(output->dimension(0), scale_x), output->dimension(1)); - window_changed = window_changed || update_window_and_padding(win, input_access, output_access); - output_access.set_valid_region(win, ValidRegion(Coordinates(0, 0), input->tensor_shape())); - } - else - { - window_changed = window_changed || update_window_and_padding(win, input_access); - } + AccessWindowStatic output_access(output, 0, 0, ceil_to_multiple(output->dimension(0), scale_x), output->dimension(1)); + window_changed = window_changed || update_window_and_padding(win, input_access, output_access); + output_access.set_valid_region(win, ValidRegion(Coordinates(0, 0), input->tensor_shape())); // Collapse along the Z direction Window collapsed = win.collapse(win, Window::DimZ); diff --git a/src/runtime/CL/functions/CLGEMM.cpp b/src/runtime/CL/functions/CLGEMM.cpp index 1ee51a0a48..f81da6c0a5 100644 --- a/src/runtime/CL/functions/CLGEMM.cpp +++ b/src/runtime/CL/functions/CLGEMM.cpp @@ -29,15 +29,18 @@ #include "arm_compute/core/CL/kernels/CLGEMMMatrixMultiplyKernel.h" #include "arm_compute/core/CL/kernels/CLGEMMTranspose1xWKernel.h" #include "arm_compute/core/Error.h" +#include "arm_compute/core/GPUTarget.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/runtime/CL/CLScheduler.h" #include "arm_compute/runtime/ITensorAllocator.h" using namespace arm_compute; +using namespace arm_compute::misc::shape_calculator; namespace { @@ -66,35 +69,6 @@ inline bool is_interleaved_transposed(int m, int n, int k, DataType data_type, b return flag; } - -Status validate_arguments(const ITensorInfo *a, const ITensorInfo *b, const ICLTensor *c, const ITensorInfo *output, const float alpha, const float beta, const GEMMInfo &gemm_info = GEMMInfo()) -{ - ARM_COMPUTE_ERROR_ON_NULLPTR(a, b, output); - - ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(a, b); - ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.is_a_reshaped(), "Matrix A already reshaped is not supported"); - ARM_COMPUTE_RETURN_ERROR_ON_MSG(gemm_info.is_b_reshaped(), "Matrix B already reshaped is not supported"); - - if(c != nullptr) - { - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(a, c->info()); - ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->dimension(1) != c->info()->dimension(1), "The matrix C must have the same number of rows as the matrix A"); - ARM_COMPUTE_RETURN_ERROR_ON_MSG(b->dimension(0) != c->info()->dimension(0), "The matrix C must have the same number of columns as the matrix B"); - } - - if(output->total_size() != 0) - { - ARM_COMPUTE_RETURN_ERROR_ON_MSG(b->dimension(0) != output->dimension(0), "The output matrix must have the same number of columns as the matrix B"); - ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->dimension(1) != output->dimension(1), "The output matrix must have the same number of rows as the matrix A"); - } - - ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->dimension(0) != b->dimension(1), "The product AB is defined only if the number of columns in A is equal to the number of rows in B"); - - ARM_COMPUTE_UNUSED(alpha); - ARM_COMPUTE_UNUSED(beta); - return Status{}; -} } // namespace CLGEMM::CLGEMM(std::shared_ptr memory_manager) @@ -108,7 +82,7 @@ void CLGEMM::configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor * ARM_COMPUTE_ERROR_ON_NULLPTR(a, b, output); // Perform validation step - ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(a->info(), b->info(), c, output->info(), alpha, beta, gemm_info)); + ARM_COMPUTE_ERROR_THROW_ON(validate(a->info(), b->info(), c != nullptr ? c->info() : nullptr, output->info(), alpha, beta, gemm_info)); // Store original b matrix _original_b = b; @@ -136,7 +110,7 @@ void CLGEMM::configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor * int mult_transpose1xW_width = 1; int mult_interleave4x4_height = 1; - if(gpu_target_is_in(gpu_target, GPUTarget::G71, GPUTarget::G72, GPUTarget::G51, GPUTarget::G51BIG, GPUTarget::G51LIT, GPUTarget::TNOX)) + if(get_arch_from_target(gpu_target) == GPUTarget::BIFROST) { mult_transpose1xW_width = 4; mult_interleave4x4_height = 2; @@ -185,9 +159,67 @@ void CLGEMM::configure(const ICLTensor *a, const ICLTensor *b, const ICLTensor * } } -Status CLGEMM::validate(const ITensorInfo *a, const ITensorInfo *b, const ICLTensor *c, const ITensorInfo *output, const float alpha, const float beta, const GEMMInfo &gemm_info) +Status CLGEMM::validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *output, float alpha, float beta, const GEMMInfo &gemm_info) { - ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(a, b, c, output, alpha, beta, gemm_info)); + ARM_COMPUTE_UNUSED(alpha); + + // Check if we need to reshape the matrix B only on the first run + const bool reshape_b_only_on_first_run = gemm_info.reshape_b_only_on_first_run(); + + const ITensorInfo *matrix_a_info = a; + const ITensorInfo *matrix_b_info = b; + + TensorInfo tmp_a_info{}; + TensorInfo tmp_b_info{}; + TensorInfo tmp_output_info = *output->clone(); + + // Get the GPU target + const GPUTarget gpu_target = CLScheduler::get().target(); + + // Arguments used by GEMMReshapeInfo + // If we pass the matrix A and matrix B reshaped to CLGEMMMatrixMultiplyKernel, we need to pass m, n, k, mult_transpose1xW_width and mult_interleave4x4_height to CLGEMMReshapeInfo + // in order to know how the matrices have been reshaped + const int m = a->dimension(1); + const int n = b->dimension(0); + const int k = a->dimension(0); + int mult_transpose1xW_width = 1; + int mult_interleave4x4_height = 1; + + if(get_arch_from_target(gpu_target) == GPUTarget::BIFROST) + { + mult_transpose1xW_width = 4; + mult_interleave4x4_height = 2; + } + + const GEMMReshapeInfo reshape_info = GEMMReshapeInfo(m, n, k, mult_transpose1xW_width, mult_interleave4x4_height); + + // Check if we need to reshape the matrix A and matrix B + const bool run_interleave_transpose = is_interleaved_transposed(m, n, k, a->data_type(), reshape_b_only_on_first_run, gpu_target); + + if(run_interleave_transpose) + { + matrix_a_info = &tmp_a_info; + matrix_b_info = &tmp_b_info; + + // Validate interleave kernel + auto_init_if_empty(tmp_a_info, a->clone()->set_tensor_shape(compute_interleaved_shape(*a, mult_interleave4x4_height))); + ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMInterleave4x4Kernel::validate(a, &tmp_a_info, mult_interleave4x4_height)); + + // Validate transpose kernel + auto_init_if_empty(tmp_b_info, b->clone()->set_tensor_shape(compute_transpose1xW_with_element_size_shape(*b, mult_transpose1xW_width))); + ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMTranspose1xWKernel::validate(b, &tmp_b_info, mult_transpose1xW_width)); + } + + // Validate matrix multiply + auto_init_if_empty(tmp_output_info, matrix_a_info->clone()->set_tensor_shape(compute_mm_shape(*matrix_a_info, *matrix_b_info, run_interleave_transpose, reshape_info))); + ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMMatrixMultiplyKernel::validate(matrix_a_info, matrix_b_info, &tmp_output_info, alpha, run_interleave_transpose, reshape_info, gpu_target)); + + if(beta != 0 && c != nullptr) + { + // Validate matrix addition kernel + ARM_COMPUTE_RETURN_ON_ERROR(CLGEMMMatrixAdditionKernel::validate(c, &tmp_output_info, beta)); + } + return Status{}; } -- cgit v1.2.1