From 9e698d55d861b3a2a2deb9486a75dd541e928705 Mon Sep 17 00:00:00 2001 From: David Mansell Date: Tue, 25 Aug 2020 15:02:02 +0100 Subject: COMPMID-3757: (u)int8: Don't select the 16-bit route on A53 for cases with very few rows Also added 2D version of the 16-bit route, and altered the selection heuristic so that 2D mode will be used in cases where 1D mode won't thread well. Change-Id: I0057fde08456771dc0090ac51f50d82f8bb86044 Signed-off-by: Michele Di Giorgio Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/3903 Tested-by: Arm Jenkins Reviewed-by: SiCong Li Reviewed-by: Georgios Pinitas Comments-Addressed: Arm Jenkins --- src/core/NEON/kernels/arm_gemm/gemm_int8.cpp | 14 +++++-- src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp | 24 +++++++---- .../NEON/functions/NEGEMMAssemblyDispatch.cpp | 49 ++++++++++++---------- 3 files changed, 55 insertions(+), 32 deletions(-) diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp index bddcc8dab1..147caeefbd 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp @@ -121,18 +121,26 @@ static const GemmImplementation gemm_s8_methods[] = { nullptr, [](const GemmArgs &args) { return new GemmInterleaved(args); } }, +{ + GemmMethod::GEMM_INTERLEAVED_2D, + "gemm_s16_12x8_2d", + nullptr, + [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4 && (args._Msize / args._maxthreads) < 8; }, + [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d(args); }, +}, { GemmMethod::GEMM_INTERLEAVED, - "gemm_s16_12x8", + "gemm_s16_12x8_1d", nullptr, - [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53; }, + [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4; }, [](const GemmArgs &args) { return new GemmInterleaved(args); }, }, { GemmMethod::GEMM_INTERLEAVED_2D, "gemm_s8_4x4_2d", nullptr, - [](const GemmArgs &args) { return (args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8); }, + [](const GemmArgs &args) { return ((args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8)) || + ((args._Msize / args._maxthreads) < 4); }, [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d(args); } }, { diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp index 88726b1448..06e68cbc43 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp @@ -100,13 +100,6 @@ static const GemmImplementation gemm_u8_methods[] = { nullptr, [](const GemmArgs &args) { return new GemmHybrid(args); } }, -{ - GemmMethod::GEMM_INTERLEAVED, - "gemm_u16_12x8", - nullptr, - [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53; }, - [](const GemmArgs &args) { return new GemmInterleaved(args); }, -}, { GemmMethod::GEMM_HYBRID, "hybrid_u8u32_dot_16x4", @@ -128,11 +121,26 @@ static const GemmImplementation gemm_u8_methods[] = { nullptr, [](const GemmArgs &args) { return new GemmInterleaved(args); } }, +{ + GemmMethod::GEMM_INTERLEAVED_2D, + "gemm_u16_12x8_2d", + nullptr, + [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4 && (args._Msize / args._maxthreads) < 8; }, + [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d(args); }, +}, +{ + GemmMethod::GEMM_INTERLEAVED, + "gemm_u16_12x8_1d", + nullptr, + [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4; }, + [](const GemmArgs &args) { return new GemmInterleaved(args); }, +}, { GemmMethod::GEMM_INTERLEAVED_2D, "gemm_u8_4x4_2d", nullptr, - [](const GemmArgs &args) { return (args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8); }, + [](const GemmArgs &args) { return ((args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8)) || + ((args._Msize / args._maxthreads) < 4); }, [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d(args); } }, { diff --git a/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp b/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp index 30232b4435..ad349cb635 100644 --- a/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp +++ b/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp @@ -415,17 +415,44 @@ void Fallback::run() in1_ptr = reinterpret_cast(_b->buffer() + _b->info()->offset_first_element_in_bytes()); } + IScheduler::Hints scheduling_hint = IScheduler::Hints(Window::DimX); + if(_kernel_info.method == arm_gemm::GemmMethod::GEMM_INTERLEAVED && _d->info()->data_type() == DataType::F32) + { + const int granule_threshold = 200; + scheduling_hint = IScheduler::Hints(Window::DimX, IScheduler::StrategyHint::DYNAMIC, granule_threshold); + } + else if(_kernel_info.method == arm_gemm::GemmMethod::GEMM_INTERLEAVED_2D && (_d->info()->data_type() == DataType::F32 || _d->info()->data_type() == DataType::F16 + || _d->info()->data_type() == DataType::U8 || _d->info()->data_type() == DataType::S8)) + { + //GEMM_INTERLEAVED supports 2D parallelism, IScheduler::split_dimensions_all signals to parallelise over all window dimensions + const int granule_threshold = 200; + scheduling_hint = IScheduler::Hints(IScheduler::split_dimensions_all, IScheduler::StrategyHint::STATIC, granule_threshold); + } + else if(_kernel_info.method == arm_gemm::GemmMethod::QUANTIZE_WRAPPER_2D && (_d->info()->data_type() == DataType::QASYMM8 || _d->info()->data_type() == DataType::QASYMM8_SIGNED)) + { + //special case for QASYMM8 to support 2D parallelism, scheduler here may be tweaked differently compared to FP32 case + const int granule_threshold = 200; + scheduling_hint = IScheduler::Hints(IScheduler::split_dimensions_all, IScheduler::StrategyHint::STATIC, granule_threshold); + } + // Set workspace if needed and reset number of threads as buffer manager gets re-created with max_threads if(_workspace.buffer() != nullptr) { _gemm_kernel_asm->set_working_space(reinterpret_cast(_workspace.buffer())); + const unsigned int split_dim = scheduling_hint.split_dimension(); const unsigned int window_size = _gemm_kernel_asm->get_window_size().total_size(); unsigned int num_threads = NEScheduler::get().num_threads(); if(window_size < num_threads) { num_threads = window_size; - _gemm_kernel_asm->set_nthreads(num_threads); } + if(split_dim != IScheduler::split_dimensions_all) + { + // Make sure the kernel does not expect more threads than we can actually spawn + const unsigned int num_iterations = _optimised_kernel.get()->window().num_iterations(split_dim); + num_threads = std::min(num_iterations, num_threads); + } + _gemm_kernel_asm->set_nthreads(num_threads); } // Prepare assembly kernel @@ -443,26 +470,6 @@ void Fallback::run() out_ptr, ldd, batch_stride_d, multi_stride_d, bias, 0); // Schedule assembly kernel - IScheduler::Hints scheduling_hint = IScheduler::Hints(Window::DimX); - if(_kernel_info.method == arm_gemm::GemmMethod::GEMM_INTERLEAVED && _d->info()->data_type() == DataType::F32) - { - const int granule_threshold = 200; - scheduling_hint = IScheduler::Hints(Window::DimX, IScheduler::StrategyHint::DYNAMIC, granule_threshold); - } - else if(_kernel_info.method == arm_gemm::GemmMethod::GEMM_INTERLEAVED_2D && (_d->info()->data_type() == DataType::F32 || _d->info()->data_type() == DataType::F16 - || _d->info()->data_type() == DataType::U8 || _d->info()->data_type() == DataType::S8)) - { - //GEMM_INTERLEAVED supports 2D parallelism, IScheduler::split_dimensions_all signals to parallelise over all window dimensions - const int granule_threshold = 200; - scheduling_hint = IScheduler::Hints(IScheduler::split_dimensions_all, IScheduler::StrategyHint::STATIC, granule_threshold); - } - else if(_kernel_info.method == arm_gemm::GemmMethod::QUANTIZE_WRAPPER_2D && (_d->info()->data_type() == DataType::QASYMM8 || _d->info()->data_type() == DataType::QASYMM8_SIGNED)) - { - //special case for QASYMM8 to support 2D parallelism, scheduler here may be tweaked differently compared to FP32 case - const int granule_threshold = 200; - scheduling_hint = IScheduler::Hints(IScheduler::split_dimensions_all, IScheduler::StrategyHint::STATIC, granule_threshold); - } - NEScheduler::get().schedule(_optimised_kernel.get(), scheduling_hint); } -- cgit v1.2.1