From cfa2bba98169cb5ab1945462514be1b6badf7d98 Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Thu, 27 Jun 2019 17:00:52 +0100 Subject: COMPMID-2178: Update GEMM assembly code. Perform offset reduction and requantization within the assembly wrapper. Change-Id: I5d5b3e1f6f9ef4c71805362c57f88ff199c027a3 Signed-off-by: Georgios Pinitas Reviewed-on: https://review.mlplatform.org/c/1541 Comments-Addressed: Pablo Marquez Reviewed-by: Gian Marco Iodice Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins --- src/runtime/NEON/functions/NEGEMM.cpp | 8 +- .../NEON/functions/NEGEMMAssemblyDispatch.cpp | 129 +++++++---- .../NEON/functions/NEGEMMConvolutionLayer.cpp | 4 +- .../NEGEMMLowpAssemblyMatrixMultiplyCore.cpp | 4 +- .../functions/NEGEMMLowpMatrixMultiplyCore.cpp | 251 ++++++++++++--------- 5 files changed, 240 insertions(+), 156 deletions(-) (limited to 'src/runtime/NEON/functions') diff --git a/src/runtime/NEON/functions/NEGEMM.cpp b/src/runtime/NEON/functions/NEGEMM.cpp index 2f36397c8e..37d0e09fc9 100644 --- a/src/runtime/NEON/functions/NEGEMM.cpp +++ b/src/runtime/NEON/functions/NEGEMM.cpp @@ -58,7 +58,7 @@ void NEGEMM::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITe _run_vector_matrix_multiplication = a->info()->dimension(1) < 2; _original_b = b; - bool run_optimised = c == nullptr && bool(NEGEMMAssemblyDispatch::validate(a->info(), b->info(), d->info(), alpha, beta, gemm_info)); + bool run_optimised = c == nullptr && bool(NEGEMMAssemblyDispatch::validate(a->info(), b->info(), c != nullptr ? c->info() : nullptr, d->info(), alpha, beta, gemm_info)); if(run_optimised) { @@ -66,11 +66,11 @@ void NEGEMM::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITe { GEMMInfo gemm_info_ntb = gemm_info; gemm_info_ntb.set_pretranpose_B(false); - _asm_glue.configure(a, b, d, alpha, beta, gemm_info_ntb); + _asm_glue.configure(a, b, c, d, alpha, beta, gemm_info_ntb); } else { - _asm_glue.configure(a, b, d, alpha, beta, gemm_info); + _asm_glue.configure(a, b, c, d, alpha, beta, gemm_info); } ARM_COMPUTE_ERROR_ON(!_asm_glue.is_configured()); } @@ -178,7 +178,7 @@ Status NEGEMM::validate(const ITensorInfo *a, const ITensorInfo *b, const ITenso } // Check if we need to run the optimized assembly kernel - const bool run_optimised = c == nullptr && bool(NEGEMMAssemblyDispatch::validate(a, b, output, alpha, beta, gemm_info)); + const bool run_optimised = c == nullptr && bool(NEGEMMAssemblyDispatch::validate(a, b, c, output, alpha, beta, gemm_info)); if(!run_optimised) { diff --git a/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp b/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp index 2de7d2b279..2a4498b0a9 100644 --- a/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp +++ b/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp @@ -74,7 +74,7 @@ std::unique_ptr create_function_all_types(const arm_gemm::KernelDescr } /** Fallback in case ACL doesn't have a function */ -template +template class Fallback : public NEGEMMAssemblyDispatch::IFallback { public: @@ -82,13 +82,16 @@ public: * * @param[in] a Input tensor containing the Matrix A. * @param[in] b Input tensor containing the Matrix B. + * @param[in] c Input tensor containing the Matrix C. * @param[out] d Output tensor to store the result of matrix multiplication. * @param[in] args Matrix multiplication information. * @param[in] gemm_info GEMM meta-data * @param[in] memory_group Memory group to be used by the function. + * @param[in] os Output stage meta-data. */ - void configure(const ITensor *a, const ITensor *b, ITensor *d, arm_gemm::GemmArgs args, - const GEMMInfo &gemm_info, MemoryGroup &memory_group); + void configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, + arm_gemm::GemmArgs args, const GEMMInfo &gemm_info, + MemoryGroup &memory_group, const OutputStage &os = {}); // Inherited methods overridden: void run() override; @@ -118,6 +121,10 @@ private: { nullptr }; + const ITensor *_c + { + nullptr + }; /** Output */ ITensor *_d{ nullptr }; /** GEMM workspace */ @@ -130,18 +137,19 @@ private: GEMMInfo _gemm_info{}; }; -template -void Fallback::configure(const ITensor *a, const ITensor *b, ITensor *d, arm_gemm::GemmArgs args, - const GEMMInfo &gemm_info, MemoryGroup &memory_group) +template +void Fallback::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, + arm_gemm::GemmArgs args, const GEMMInfo &gemm_info, + MemoryGroup &memory_group, const OutputStage &os) { arm_gemm::GemmConfig gemm_cfg; - const arm_gemm::KernelDescription gemm_kernel_info = arm_gemm::get_gemm_method(args); + const arm_gemm::KernelDescription gemm_kernel_info = arm_gemm::get_gemm_method(args, os); if(gemm_kernel_info.method != arm_gemm::GemmMethod::GEMV_BATCHED) { gemm_cfg.filter = gemm_kernel_info.name; args._cfg = &gemm_cfg; } - _gemm_kernel_asm = arm_gemm::gemm(args); + _gemm_kernel_asm = arm_gemm::gemm(args, os); if(_gemm_kernel_asm == nullptr) { //configuration not supported: Leave function unconfigured: @@ -173,6 +181,7 @@ void Fallback::configure(const ITensor *a, const ITensor _optimised_kernel = std::move(acl_gemm_wrapper); _a = a; _b = b; + _c = c; _d = d; _gemm_info = gemm_info; // Check for pre-transposed support @@ -185,11 +194,17 @@ void Fallback::configure(const ITensor *a, const ITensor } } -template -void Fallback::prepare() +template +void Fallback::prepare() { if(!_is_prepared) { + // Setup up matrix bias in the assembly kernel, it's just a pointer to matrix C. + if(_c && _c->info()->data_type() == DataType::S32) + { + _gemm_kernel_asm->set_quantized_bias(reinterpret_cast(_c->buffer() + _c->info()->offset_first_element_in_bytes())); + } + // Pretranspose B if required if(_gemm_kernel_asm->B_pretranspose_required()) { @@ -207,8 +222,8 @@ void Fallback::prepare() } } -template -void Fallback::allocate_workspace(size_t workspace_size, MemoryGroup &memory_group, size_t alignment) +template +void Fallback::allocate_workspace(size_t workspace_size, MemoryGroup &memory_group, size_t alignment) { ARM_COMPUTE_ERROR_ON_MSG(workspace_size == 0, "size cannot be 0"); _workspace.allocator()->init(TensorInfo(TensorShape{ (workspace_size + alignment /* FIXME: remove alignment after COMPMID-1088 */) }, 1, DataType::S8), alignment); @@ -216,14 +231,14 @@ void Fallback::allocate_workspace(size_t workspace_size, _workspace.allocator()->allocate(); } -template -bool Fallback::is_configured() const +template +bool Fallback::is_configured() const { return _optimised_kernel != nullptr; } -template -void Fallback::run() +template +void Fallback::run() { const int lda = _a->info()->strides_in_bytes().y() / sizeof(TypeInput); int ldb = 0; @@ -277,10 +292,8 @@ void Fallback::run() } template -void create_function_or_arm_gemm(std::unique_ptr &acl_function, - std::unique_ptr &arm_gemm, - MemoryGroup &memory_group, const ITensor *a, const ITensor *b, - ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info, +void create_function_or_arm_gemm(std::unique_ptr &acl_function, std::unique_ptr &arm_gemm, MemoryGroup &memory_group, + const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info, std::shared_ptr memory_manager) { INEGEMMWrapperKernel::Params p = INEGEMMWrapperKernel::extract_parameters(a, b, d, gemm_info); @@ -289,15 +302,51 @@ void create_function_or_arm_gemm(std::unique_ptr arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, gemm_info.pretranpose_B()); - //Try to create an ACL function: - acl_function = create_function_all_types(arm_gemm::get_gemm_method(args), a, b, d, alpha, beta, gemm_info, std::move(memory_manager)); + // Try to create an ACL function: + const arm_gemm::KernelDescription gemm_kernel_info = arm_gemm::get_gemm_method(args); + acl_function = create_function_all_types(gemm_kernel_info, a, b, d, alpha, beta, gemm_info, std::move(memory_manager)); - //If we still don't have an ACL function: + // If we still don't have an ACL function: if(acl_function == nullptr) { //Fallback onto arm_gemm function if ACL doesn't support this method. auto fallback = support::cpp14::make_unique>(); - fallback->configure(a, b, d, args, gemm_info, memory_group); + fallback->configure(a, b, c, d, args, gemm_info, memory_group); + arm_gemm = std::move(fallback); + } +} + +template +void create_function_or_arm_gemm_quant(std::unique_ptr &acl_function, std::unique_ptr &arm_gemm, MemoryGroup &memory_group, + const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info, + std::shared_ptr memory_manager) +{ + INEGEMMWrapperKernel::Params p = INEGEMMWrapperKernel::extract_parameters(a, b, d, gemm_info); + const CPUInfo &ci = NEScheduler::get().cpu_info(); + unsigned int num_threads = NEScheduler::get().num_threads(); + + arm_gemm::GemmArgs args(&ci, p.M, p.N, p.K, p.batches, p.multis, false, false, alpha, beta, num_threads, gemm_info.pretranpose_B()); + + // Configure requantization info + const int32_t a_offset = -a->info()->quantization_info().uniform().offset; + const int32_t b_offset = -b->info()->quantization_info().uniform().offset; + const GEMMLowpOutputStageInfo os_info = gemm_info.gemmlowp_output_stage(); + + const arm_gemm::ARequantizeLayer32 gemm_requant_info(nullptr, + a_offset, b_offset, os_info.gemmlowp_offset, + -os_info.gemmlowp_shift, os_info.gemmlowp_multiplier, + os_info.gemmlowp_min_bound, os_info.gemmlowp_max_bound); + + // Try to create an ACL function: + const arm_gemm::KernelDescription gemm_kernel_info = arm_gemm::get_gemm_method(args, gemm_requant_info); + acl_function = create_function_all_types(gemm_kernel_info, a, b, d, alpha, beta, gemm_info, std::move(memory_manager)); + + // If we still don't have an ACL function: + if(acl_function == nullptr) + { + // Fallback onto arm_gemm function if ACL doesn't support this method. + auto fallback = support::cpp14::make_unique>(); + fallback->configure(a, b, c, d, args, gemm_info, memory_group, gemm_requant_info); arm_gemm = std::move(fallback); } } @@ -309,11 +358,10 @@ NEGEMMAssemblyDispatch::NEGEMMAssemblyDispatch(std::shared_ptr m { } -Status NEGEMMAssemblyDispatch::validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *d, float alpha, float beta, const GEMMInfo &gemm_info) +Status NEGEMMAssemblyDispatch::validate(const ITensorInfo *a, const ITensorInfo *b, const ITensorInfo *c, const ITensorInfo *d, float alpha, float beta, const GEMMInfo &gemm_info) { - ARM_COMPUTE_UNUSED(alpha); - ARM_COMPUTE_UNUSED(beta); - ARM_COMPUTE_UNUSED(gemm_info); + ARM_COMPUTE_UNUSED(alpha, beta, gemm_info); + ARM_COMPUTE_UNUSED(c); ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(a, b, d); ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(a); #ifndef __aarch64__ @@ -324,19 +372,17 @@ Status NEGEMMAssemblyDispatch::validate(const ITensorInfo *a, const ITensorInfo ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::F32 && d->data_type() != DataType::F32, "Only F32 output supported for F32 input"); ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::F16 && d->data_type() != DataType::F16, "Only F16 output supported for F16 input"); ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::U8 && d->data_type() != DataType::U32, "Only U32 output supported for U8 input"); - ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::QASYMM8 && d->data_type() != DataType::S32 && d->data_type() != DataType::U32, "Only U32/S32 output supported for QASYMM8 input"); ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::S8 && d->data_type() != DataType::S32, "Only S32 output supported for S8 input"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::QASYMM8 && d->data_type() != DataType::QASYMM8, "Only QASYMM8 output supported for QASYMM8 input"); return Status{}; } -void NEGEMMAssemblyDispatch::configure(const ITensor *a, const ITensor *b, ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info) +void NEGEMMAssemblyDispatch::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *d, float alpha, float beta, const GEMMInfo &gemm_info) { - ARM_COMPUTE_ERROR_ON_NULLPTR(a); - ARM_COMPUTE_ERROR_ON_NULLPTR(b); - ARM_COMPUTE_ERROR_ON_NULLPTR(d); + ARM_COMPUTE_ERROR_ON_NULLPTR(a, b, d); //If we don't support a combination of data types, silently return: it is the caller's responsibility to check if configure() was successful via is_configured() - if(!NEGEMMAssemblyDispatch::validate(a->info(), b->info(), d->info(), alpha, beta, gemm_info)) + if(!NEGEMMAssemblyDispatch::validate(a->info(), b->info(), c != nullptr ? c->info() : nullptr, d->info(), alpha, beta, gemm_info)) { return; } @@ -344,20 +390,27 @@ void NEGEMMAssemblyDispatch::configure(const ITensor *a, const ITensor *b, ITens switch(a->info()->data_type()) { case DataType::F32: - create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, d, alpha, beta, gemm_info, _memory_manager); + create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, c, d, alpha, beta, gemm_info, _memory_manager); break; #ifdef __aarch64__ case DataType::U8: case DataType::QASYMM8: - create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, d, alpha, beta, gemm_info, _memory_manager); + if(d->info()->data_type() == DataType::S32) + { + create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, c, d, alpha, beta, gemm_info, _memory_manager); + } + else + { + create_function_or_arm_gemm_quant(_function, _arm_gemm, _memory_group, a, b, c, d, alpha, beta, gemm_info, _memory_manager); + } break; case DataType::S8: - create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, d, alpha, beta, gemm_info, _memory_manager); + create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, c, d, alpha, beta, gemm_info, _memory_manager); break; #endif /* __aarch64__ */ #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC case DataType::F16: - create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, d, alpha, beta, gemm_info, _memory_manager); + create_function_or_arm_gemm(_function, _arm_gemm, _memory_group, a, b, c, d, alpha, beta, gemm_info, _memory_manager); break; #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ default: diff --git a/src/runtime/NEON/functions/NEGEMMConvolutionLayer.cpp b/src/runtime/NEON/functions/NEGEMMConvolutionLayer.cpp index c011ddd18f..bd46944f7a 100644 --- a/src/runtime/NEON/functions/NEGEMMConvolutionLayer.cpp +++ b/src/runtime/NEON/functions/NEGEMMConvolutionLayer.cpp @@ -124,7 +124,7 @@ void NEGEMMConvolutionLayer::configure_mm(const ITensor *input, const ITensor *w // Merge activation with output stage int min_activation = 0; - int max_activation = 0; + int max_activation = 255; const std::set supported_acts = { ActivationLayerInfo::ActivationFunction::RELU, ActivationLayerInfo::ActivationFunction::BOUNDED_RELU, @@ -191,7 +191,7 @@ Status NEGEMMConvolutionLayer::validate_mm(const ITensorInfo *input, const ITens // Merge activation with output stage int min_activation = 0; - int max_activation = 0; + int max_activation = 255; const std::set supported_acts = { ActivationLayerInfo::ActivationFunction::RELU, ActivationLayerInfo::ActivationFunction::BOUNDED_RELU, diff --git a/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp b/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp index 5b70c8724c..aa40113c5e 100644 --- a/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp +++ b/src/runtime/NEON/functions/NEGEMMLowpAssemblyMatrixMultiplyCore.cpp @@ -43,7 +43,7 @@ NEGEMMLowpAssemblyMatrixMultiplyCore::NEGEMMLowpAssemblyMatrixMultiplyCore(std:: { } -void NEGEMMLowpAssemblyMatrixMultiplyCore::configure(const ITensor *a, const ITensor *b, ITensor *output) +void NEGEMMLowpAssemblyMatrixMultiplyCore::configure(const ITensor *a, const ITensor *b, const ITensor *c, ITensor *output) { ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(a, 1, DataType::U8, DataType::S8); ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::U32, DataType::S32); @@ -59,7 +59,7 @@ void NEGEMMLowpAssemblyMatrixMultiplyCore::configure(const ITensor *a, const ITe case DataType::QASYMM8: case DataType::U8: { - _asm_glue.configure(a, b, output, 1.f, 0.f, GEMMInfo(false, false, true)); + _asm_glue.configure(a, b, c, output, 1.f, 0.f, GEMMInfo(false, false, true)); run_optimised = _asm_glue.is_configured(); break; } diff --git a/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp b/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp index f10f114287..6dc5dd2a65 100644 --- a/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp +++ b/src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp @@ -43,7 +43,7 @@ using namespace arm_compute::misc::shape_calculator; NEGEMMLowpMatrixMultiplyCore::NEGEMMLowpMatrixMultiplyCore(std::shared_ptr memory_manager) : _memory_group(memory_manager), _asm_glue(memory_manager), _mm_kernel(nullptr), _mtx_a_reshape_kernel(nullptr), _mtx_b_reshape_kernel(nullptr), _mtx_a_reduction_kernel(), _mtx_b_reduction_kernel(), _offset_contribution_kernel(), _offset_contribution_output_stage_kernel(), _vector_sum_col(), _vector_sum_row(), _tmp_a(), _tmp_b(), _mm_result_s32(), _original_b(nullptr), _a_offset(0), _b_offset(0), - _run_vector_matrix_multiplication(false), _dot_product_path(false), _reshape_b_only_on_first_run(false), _is_prepared(false), _fuse_output_stage(false) + _run_vector_matrix_multiplication(false), _assembly_path(false), _fused_assembly_path(false), _reshape_b_only_on_first_run(false), _is_prepared(false), _fuse_output_stage(false) { } @@ -66,17 +66,15 @@ void NEGEMMLowpMatrixMultiplyCore::configure(const ITensor *a, const ITensor *b, _run_vector_matrix_multiplication = a->info()->dimension(1) < 2; _reshape_b_only_on_first_run = gemm_info.reshape_b_only_on_first_run(); _is_prepared = false; + _fused_assembly_path = false; _original_b = b; // If GEMMLowpOutputStage != NONE, fuse the offset contribution with the output stage if(gemm_info.gemmlowp_output_stage().type != GEMMLowpOutputStageType::NONE) { _fuse_output_stage = true; - _memory_group.manage(&_mm_result_s32); - TensorInfo info_mm_result_s32(output->info()->tensor_shape(), 1, DataType::S32); - _mm_result_s32.allocator()->init(info_mm_result_s32); } @@ -87,8 +85,16 @@ void NEGEMMLowpMatrixMultiplyCore::configure(const ITensor *a, const ITensor *b, case DataType::U8: case DataType::S8: { - _asm_glue.configure(a, b, _fuse_output_stage ? &_mm_result_s32 : output, 1.f, 0.f, gemm_info); - _dot_product_path = _asm_glue.is_configured(); + if(a->info()->data_type() == DataType::QASYMM8 && gemm_info.gemmlowp_output_stage().type == GEMMLowpOutputStageType::QUANTIZE_DOWN_FIXEDPOINT) + { + _asm_glue.configure(a, b, c, output, 1.f, 0.f, gemm_info); + _fused_assembly_path = _asm_glue.is_configured(); + } + else + { + _asm_glue.configure(a, b, nullptr, _fuse_output_stage ? &_mm_result_s32 : output, 1.f, 0.f, gemm_info); + } + _assembly_path = _asm_glue.is_configured(); break; } default: @@ -98,7 +104,7 @@ void NEGEMMLowpMatrixMultiplyCore::configure(const ITensor *a, const ITensor *b, } } #endif /* __aarch64__ */ - if(!(_dot_product_path || _run_vector_matrix_multiplication)) + if(!(_assembly_path || _run_vector_matrix_multiplication)) { matrix_a = &_tmp_a; matrix_b = &_tmp_b; @@ -130,63 +136,64 @@ void NEGEMMLowpMatrixMultiplyCore::configure(const ITensor *a, const ITensor *b, } } - // Initialize matrix B reduction kernel only if _a_offset is not equal to 0 - if(_a_offset != 0) + if(!_fused_assembly_path) { - TensorInfo info_vector_sum_col(compute_reductionA_shape(*b->info()), 1, DataType::S32); - - _vector_sum_col.allocator()->init(info_vector_sum_col); - if(!_reshape_b_only_on_first_run) + // Initialize matrix B reduction kernel only if _a_offset is not equal to 0 + if(_a_offset != 0) { - _memory_group.manage(&_vector_sum_col); - } + TensorInfo info_vector_sum_col(compute_reductionA_shape(*b->info()), 1, DataType::S32); - // Configure Matrix B reduction kernel - _mtx_b_reduction_kernel.configure(b, &_vector_sum_col, a->info()->dimension(0), false); - } + _vector_sum_col.allocator()->init(info_vector_sum_col); + if(!_reshape_b_only_on_first_run) + { + _memory_group.manage(&_vector_sum_col); + } - // Initialize Matrix A reduction kernel only if _b_offset is not equal to 0 - if(_b_offset != 0) - { - TensorInfo info_vector_sum_row(compute_reductionB_shape(*a->info()), 1, DataType::S32); + // Configure Matrix B reduction kernel + _mtx_b_reduction_kernel.configure(b, &_vector_sum_col, a->info()->dimension(0), false); + } - _vector_sum_row.allocator()->init(info_vector_sum_row); - _memory_group.manage(&_vector_sum_row); + // Initialize Matrix A reduction kernel only if _b_offset is not equal to 0 + if(_b_offset != 0) + { + TensorInfo info_vector_sum_row(compute_reductionB_shape(*a->info()), 1, DataType::S32); - // Configure matrix A reduction kernel - _mtx_a_reduction_kernel.configure(a, &_vector_sum_row, a->info()->dimension(0), false); - } + _vector_sum_row.allocator()->init(info_vector_sum_row); + _memory_group.manage(&_vector_sum_row); - if(_fuse_output_stage) - { - // Configure matrix multiply kernel - if(!_dot_product_path) - { - auto k = arm_compute::support::cpp14::make_unique(); - k->configure(matrix_a, matrix_b, &_mm_result_s32); - _mm_kernel = std::move(k); + // Configure matrix A reduction kernel + _mtx_a_reduction_kernel.configure(a, &_vector_sum_row, a->info()->dimension(0), false); } - _offset_contribution_output_stage_kernel.configure(&_mm_result_s32, _a_offset == 0 ? nullptr : &_vector_sum_col, _b_offset == 0 ? nullptr : &_vector_sum_row, c, output, a->info()->dimension(0), - _a_offset, _b_offset, gemm_info.gemmlowp_output_stage()); + if(_fuse_output_stage) + { + // Configure matrix multiply kernel + if(!_assembly_path) + { + auto k = arm_compute::support::cpp14::make_unique(); + k->configure(matrix_a, matrix_b, &_mm_result_s32); + _mm_kernel = std::move(k); + } - _mm_result_s32.allocator()->allocate(); - } - else - { - // Configure matrix multiply kernel - if(!_dot_product_path) + _offset_contribution_output_stage_kernel.configure(&_mm_result_s32, _a_offset == 0 ? nullptr : &_vector_sum_col, _b_offset == 0 ? nullptr : &_vector_sum_row, c, output, a->info()->dimension(0), + _a_offset, _b_offset, gemm_info.gemmlowp_output_stage()); + } + else { - auto k = arm_compute::support::cpp14::make_unique(); - k->configure(matrix_a, matrix_b, output); - _mm_kernel = std::move(k); + // Configure matrix multiply kernel + if(!_assembly_path) + { + auto k = arm_compute::support::cpp14::make_unique(); + k->configure(matrix_a, matrix_b, output); + _mm_kernel = std::move(k); + } + // Configure offset contribution kernel + _offset_contribution_kernel.configure(output, _a_offset == 0 ? nullptr : &_vector_sum_col, _b_offset == 0 ? nullptr : &_vector_sum_row, a->info()->dimension(0), _a_offset, _b_offset); } - // Configure offset contribution kernel - _offset_contribution_kernel.configure(output, _a_offset == 0 ? nullptr : &_vector_sum_col, _b_offset == 0 ? nullptr : &_vector_sum_row, a->info()->dimension(0), _a_offset, _b_offset); } // Allocate tensors - if(!_dot_product_path && !_run_vector_matrix_multiplication) + if(!_assembly_path && !_run_vector_matrix_multiplication) { _tmp_a.allocator()->allocate(); if(!_reshape_b_only_on_first_run) @@ -195,14 +202,22 @@ void NEGEMMLowpMatrixMultiplyCore::configure(const ITensor *a, const ITensor *b, } } - if(_a_offset != 0 && !_reshape_b_only_on_first_run) + if(!_fused_assembly_path) { - _vector_sum_col.allocator()->allocate(); + if(_a_offset != 0 && !_reshape_b_only_on_first_run) + { + _vector_sum_col.allocator()->allocate(); + } + + if(_b_offset != 0) + { + _vector_sum_row.allocator()->allocate(); + } } - if(_b_offset != 0) + if(_fuse_output_stage) { - _vector_sum_row.allocator()->allocate(); + _mm_result_s32.allocator()->allocate(); } } @@ -227,14 +242,24 @@ Status NEGEMMLowpMatrixMultiplyCore::validate(const ITensorInfo *a, const ITenso int32_t a_offset = a->quantization_info().uniform().offset; int32_t b_offset = b->quantization_info().uniform().offset; - bool fuse_output_stage = gemm_info.gemmlowp_output_stage().type != GEMMLowpOutputStageType::NONE; + bool fuse_output_stage = gemm_info.gemmlowp_output_stage().type != GEMMLowpOutputStageType::NONE && a->data_type() != DataType::QASYMM8; if(fuse_output_stage) { auto_init_if_empty(mm_result_s32_info, a->clone()->set_tensor_shape(output->tensor_shape()).set_data_type(DataType::S32)); } // Check if we need to run the optimized assembly kernel - const bool run_optimised = bool(NEGEMMAssemblyDispatch::validate(a, b, fuse_output_stage ? &mm_result_s32_info : output, 1.f, 0.f, gemm_info)); + bool run_optimised = false; + bool run_optimised_requantized = false; + if(is_data_type_quantized_asymmetric(a->data_type())) + { + run_optimised = bool(NEGEMMAssemblyDispatch::validate(a, b, c, output, 1.f, 0.f, gemm_info)); + run_optimised_requantized = run_optimised; + } + else + { + run_optimised = bool(NEGEMMAssemblyDispatch::validate(a, b, nullptr, fuse_output_stage ? &mm_result_s32_info : output, 1.f, 0.f, gemm_info)); + } if(run_optimised) { @@ -286,52 +311,55 @@ Status NEGEMMLowpMatrixMultiplyCore::validate(const ITensorInfo *a, const ITenso } } - TensorInfo info_vector_sum_col{}; - TensorInfo info_vector_sum_row{}; - - // Validate matrix B reduction kernel only if _a_offset is not equal to 0 - if(a_offset != 0) + if(!run_optimised_requantized) { - info_vector_sum_col = TensorInfo(compute_reductionA_shape(*b), 1, DataType::S32); + TensorInfo info_vector_sum_col{}; + TensorInfo info_vector_sum_row{}; - // Configure Matrix B reduction kernel - ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpMatrixBReductionKernel::validate(b, &info_vector_sum_col, a->dimension(0), false)); - } - - // Validate Matrix A reduction kernel only if _b_offset is not equal to 0 - if(b_offset != 0) - { - info_vector_sum_row = TensorInfo(compute_reductionB_shape(*a), 1, DataType::S32); + // Validate matrix B reduction kernel only if _a_offset is not equal to 0 + if(a_offset != 0) + { + info_vector_sum_col = TensorInfo(compute_reductionA_shape(*b), 1, DataType::S32); - // Configure matrix A reduction kernel - ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpMatrixAReductionKernel::validate(a, &info_vector_sum_row, a->dimension(0), false)); - } + // Configure Matrix B reduction kernel + ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpMatrixBReductionKernel::validate(b, &info_vector_sum_col, a->dimension(0), false)); + } - if(fuse_output_stage) - { - if(!run_optimised) + // Validate Matrix A reduction kernel only if _b_offset is not equal to 0 + if(b_offset != 0) { - ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpMatrixMultiplyKernel::validate(matrix_a_info, matrix_b_info, &mm_result_s32_info)); + info_vector_sum_row = TensorInfo(compute_reductionB_shape(*a), 1, DataType::S32); + + // Configure matrix A reduction kernel + ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpMatrixAReductionKernel::validate(a, &info_vector_sum_row, a->dimension(0), false)); } - // Validate offset contribution kernel - ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpOffsetContributionOutputStageKernel::validate(&mm_result_s32_info, - a_offset == 0 ? nullptr : &info_vector_sum_col, - b_offset == 0 ? nullptr : &info_vector_sum_row, - c, output, a_offset, b_offset, - gemm_info.gemmlowp_output_stage())); - } - else - { - if(!run_optimised) + if(fuse_output_stage) + { + if(!run_optimised) + { + ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpMatrixMultiplyKernel::validate(matrix_a_info, matrix_b_info, &mm_result_s32_info)); + } + + // Validate offset contribution kernel + ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpOffsetContributionOutputStageKernel::validate(&mm_result_s32_info, + a_offset == 0 ? nullptr : &info_vector_sum_col, + b_offset == 0 ? nullptr : &info_vector_sum_row, + c, output, a_offset, b_offset, + gemm_info.gemmlowp_output_stage())); + } + else { - ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpMatrixMultiplyKernel::validate(matrix_a_info, matrix_b_info, output)); + if(!run_optimised) + { + ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpMatrixMultiplyKernel::validate(matrix_a_info, matrix_b_info, output)); + } + // Validate offset contribution kernel + ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpOffsetContributionKernel::validate(output, + a_offset == 0 ? nullptr : &info_vector_sum_col, + b_offset == 0 ? nullptr : &info_vector_sum_row, + a_offset, b_offset)); } - // Validate offset contribution kernel - ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpOffsetContributionKernel::validate(output, - a_offset == 0 ? nullptr : &info_vector_sum_col, - b_offset == 0 ? nullptr : &info_vector_sum_row, - a_offset, b_offset)); } return Status{}; } @@ -362,27 +390,30 @@ void NEGEMMLowpMatrixMultiplyCore::run() NEScheduler::get().schedule(_mm_kernel.get(), Window::DimY); } - // Run matrix A reduction kernel only if _b_offset is not equal to 0 - if(_b_offset != 0) + if(!_fused_assembly_path) { - NEScheduler::get().schedule(&_mtx_a_reduction_kernel, Window::DimX); - } + // Run matrix A reduction kernel only if _b_offset is not equal to 0 + if(_b_offset != 0) + { + NEScheduler::get().schedule(&_mtx_a_reduction_kernel, Window::DimX); + } - // Run matrix B reduction kernel only if _a_offset is not equal to 0 - if(_a_offset != 0 && !_reshape_b_only_on_first_run) - { - NEScheduler::get().schedule(&_mtx_b_reduction_kernel, Window::DimX); - } + // Run matrix B reduction kernel only if _a_offset is not equal to 0 + if(_a_offset != 0 && !_reshape_b_only_on_first_run) + { + NEScheduler::get().schedule(&_mtx_b_reduction_kernel, Window::DimX); + } - if(_fuse_output_stage) - { - // Run offset contribution kernel - NEScheduler::get().schedule(&_offset_contribution_output_stage_kernel, Window::DimY); - } - else - { - // Run offset contribution kernel - NEScheduler::get().schedule(&_offset_contribution_kernel, Window::DimY); + if(_fuse_output_stage) + { + // Run offset contribution kernel + NEScheduler::get().schedule(&_offset_contribution_output_stage_kernel, Window::DimY); + } + else + { + // Run offset contribution kernel + NEScheduler::get().schedule(&_offset_contribution_kernel, Window::DimY); + } } } -- cgit v1.2.1