aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/core/NEON/kernels/arm_gemm/gemm_int8.cpp14
-rw-r--r--src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp24
-rw-r--r--src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp49
3 files changed, 55 insertions, 32 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
index bddcc8dab1..147caeefbd 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
@@ -122,17 +122,25 @@ static const GemmImplementation<int8_t, int32_t> gemm_s8_methods[] = {
[](const GemmArgs &args) { return new GemmInterleaved<gemm_s8_12x8, int8_t, int32_t>(args); }
},
{
+ GemmMethod::GEMM_INTERLEAVED_2D,
+ "gemm_s16_12x8_2d",
+ nullptr,
+ [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4 && (args._Msize / args._maxthreads) < 8; },
+ [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<gemm_s16_12x8, int8_t, int32_t>(args); },
+},
+{
GemmMethod::GEMM_INTERLEAVED,
- "gemm_s16_12x8",
+ "gemm_s16_12x8_1d",
nullptr,
- [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53; },
+ [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4; },
[](const GemmArgs &args) { return new GemmInterleaved<gemm_s16_12x8, int8_t, int32_t>(args); },
},
{
GemmMethod::GEMM_INTERLEAVED_2D,
"gemm_s8_4x4_2d",
nullptr,
- [](const GemmArgs &args) { return (args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8); },
+ [](const GemmArgs &args) { return ((args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8)) ||
+ ((args._Msize / args._maxthreads) < 4); },
[](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<gemm_s8_4x4, int8_t, int32_t>(args); }
},
{
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
index 88726b1448..06e68cbc43 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
@@ -101,13 +101,6 @@ static const GemmImplementation<uint8_t, uint32_t> gemm_u8_methods[] = {
[](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_u8u32_dot_4x6, uint8_t, uint32_t>(args); }
},
{
- GemmMethod::GEMM_INTERLEAVED,
- "gemm_u16_12x8",
- nullptr,
- [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53; },
- [](const GemmArgs &args) { return new GemmInterleaved<gemm_u16_12x8, uint8_t, uint32_t>(args); },
-},
-{
GemmMethod::GEMM_HYBRID,
"hybrid_u8u32_dot_16x4",
[](const GemmArgs &args) { return args._ci->has_dotprod() && args._Ksize>=16; },
@@ -130,9 +123,24 @@ static const GemmImplementation<uint8_t, uint32_t> gemm_u8_methods[] = {
},
{
GemmMethod::GEMM_INTERLEAVED_2D,
+ "gemm_u16_12x8_2d",
+ nullptr,
+ [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4 && (args._Msize / args._maxthreads) < 8; },
+ [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<gemm_u16_12x8, uint8_t, uint32_t>(args); },
+},
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "gemm_u16_12x8_1d",
+ nullptr,
+ [](const GemmArgs &args) { return args._ci->get_cpu_model() == CPUModel::A53 && args._Msize > 4; },
+ [](const GemmArgs &args) { return new GemmInterleaved<gemm_u16_12x8, uint8_t, uint32_t>(args); },
+},
+{
+ GemmMethod::GEMM_INTERLEAVED_2D,
"gemm_u8_4x4_2d",
nullptr,
- [](const GemmArgs &args) { return (args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8); },
+ [](const GemmArgs &args) { return ((args._maxthreads >= 8) && (args._Msize >= 8) && (args._Nsize >= 8)) ||
+ ((args._Msize / args._maxthreads) < 4); },
[](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<gemm_u8_4x4, uint8_t, uint32_t>(args); }
},
{
diff --git a/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp b/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp
index 30232b4435..ad349cb635 100644
--- a/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp
+++ b/src/runtime/NEON/functions/NEGEMMAssemblyDispatch.cpp
@@ -415,17 +415,44 @@ void Fallback<TypeInput, TypeOutput, OutputStage>::run()
in1_ptr = reinterpret_cast<const TypeInput *>(_b->buffer() + _b->info()->offset_first_element_in_bytes());
}
+ IScheduler::Hints scheduling_hint = IScheduler::Hints(Window::DimX);
+ if(_kernel_info.method == arm_gemm::GemmMethod::GEMM_INTERLEAVED && _d->info()->data_type() == DataType::F32)
+ {
+ const int granule_threshold = 200;
+ scheduling_hint = IScheduler::Hints(Window::DimX, IScheduler::StrategyHint::DYNAMIC, granule_threshold);
+ }
+ else if(_kernel_info.method == arm_gemm::GemmMethod::GEMM_INTERLEAVED_2D && (_d->info()->data_type() == DataType::F32 || _d->info()->data_type() == DataType::F16
+ || _d->info()->data_type() == DataType::U8 || _d->info()->data_type() == DataType::S8))
+ {
+ //GEMM_INTERLEAVED supports 2D parallelism, IScheduler::split_dimensions_all signals to parallelise over all window dimensions
+ const int granule_threshold = 200;
+ scheduling_hint = IScheduler::Hints(IScheduler::split_dimensions_all, IScheduler::StrategyHint::STATIC, granule_threshold);
+ }
+ else if(_kernel_info.method == arm_gemm::GemmMethod::QUANTIZE_WRAPPER_2D && (_d->info()->data_type() == DataType::QASYMM8 || _d->info()->data_type() == DataType::QASYMM8_SIGNED))
+ {
+ //special case for QASYMM8 to support 2D parallelism, scheduler here may be tweaked differently compared to FP32 case
+ const int granule_threshold = 200;
+ scheduling_hint = IScheduler::Hints(IScheduler::split_dimensions_all, IScheduler::StrategyHint::STATIC, granule_threshold);
+ }
+
// Set workspace if needed and reset number of threads as buffer manager gets re-created with max_threads
if(_workspace.buffer() != nullptr)
{
_gemm_kernel_asm->set_working_space(reinterpret_cast<void *>(_workspace.buffer()));
+ const unsigned int split_dim = scheduling_hint.split_dimension();
const unsigned int window_size = _gemm_kernel_asm->get_window_size().total_size();
unsigned int num_threads = NEScheduler::get().num_threads();
if(window_size < num_threads)
{
num_threads = window_size;
- _gemm_kernel_asm->set_nthreads(num_threads);
}
+ if(split_dim != IScheduler::split_dimensions_all)
+ {
+ // Make sure the kernel does not expect more threads than we can actually spawn
+ const unsigned int num_iterations = _optimised_kernel.get()->window().num_iterations(split_dim);
+ num_threads = std::min(num_iterations, num_threads);
+ }
+ _gemm_kernel_asm->set_nthreads(num_threads);
}
// Prepare assembly kernel
@@ -443,26 +470,6 @@ void Fallback<TypeInput, TypeOutput, OutputStage>::run()
out_ptr, ldd, batch_stride_d, multi_stride_d,
bias, 0);
// Schedule assembly kernel
- IScheduler::Hints scheduling_hint = IScheduler::Hints(Window::DimX);
- if(_kernel_info.method == arm_gemm::GemmMethod::GEMM_INTERLEAVED && _d->info()->data_type() == DataType::F32)
- {
- const int granule_threshold = 200;
- scheduling_hint = IScheduler::Hints(Window::DimX, IScheduler::StrategyHint::DYNAMIC, granule_threshold);
- }
- else if(_kernel_info.method == arm_gemm::GemmMethod::GEMM_INTERLEAVED_2D && (_d->info()->data_type() == DataType::F32 || _d->info()->data_type() == DataType::F16
- || _d->info()->data_type() == DataType::U8 || _d->info()->data_type() == DataType::S8))
- {
- //GEMM_INTERLEAVED supports 2D parallelism, IScheduler::split_dimensions_all signals to parallelise over all window dimensions
- const int granule_threshold = 200;
- scheduling_hint = IScheduler::Hints(IScheduler::split_dimensions_all, IScheduler::StrategyHint::STATIC, granule_threshold);
- }
- else if(_kernel_info.method == arm_gemm::GemmMethod::QUANTIZE_WRAPPER_2D && (_d->info()->data_type() == DataType::QASYMM8 || _d->info()->data_type() == DataType::QASYMM8_SIGNED))
- {
- //special case for QASYMM8 to support 2D parallelism, scheduler here may be tweaked differently compared to FP32 case
- const int granule_threshold = 200;
- scheduling_hint = IScheduler::Hints(IScheduler::split_dimensions_all, IScheduler::StrategyHint::STATIC, granule_threshold);
- }
-
NEScheduler::get().schedule(_optimised_kernel.get(), scheduling_hint);
}
generated by cgit v1.2.1 (git 2.23.0) at 2024-05-20 11:26:40 +0000