From c0b6f76561580414f08633a804fc548ccad65659 Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Mon, 2 Nov 2020 01:37:17 +0000 Subject: COMPMID-3776: Indirect GEMM Signed-off-by: Georgios Pinitas Change-Id: I51a1b0f098bc3a8c408c50c92221e4df3061e12c Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/4343 Tested-by: Arm Jenkins Reviewed-by: Sang-Hoon Park Reviewed-by: Michele Di Giorgio Comments-Addressed: Arm Jenkins --- src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp | 120 ++++++++++++++------------- 1 file changed, 62 insertions(+), 58 deletions(-) (limited to 'src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp') diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp index ddb438f06c..e9e335f500 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp @@ -24,6 +24,7 @@ #include "arm_gemm.hpp" #include "gemm_common.hpp" #include "gemm_hybrid.hpp" +#include "gemm_hybrid_indirect.hpp" #include "gemm_implementation.hpp" #include "gemm_interleaved.hpp" #include "gemm_interleaved_pretransposed_2d.hpp" @@ -31,127 +32,130 @@ #include "gemv_pretransposed.hpp" #include "kernels/a32_sgemm_8x6.hpp" -#include "kernels/a64_hybrid_fp32_mla_16x4.hpp" -#include "kernels/a64_hybrid_fp32_mla_4x8.hpp" -#include "kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp" -#include "kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp" -#include "kernels/a64_sgemm_12x8.hpp" -#include "kernels/a64_sgemv_pretransposed.hpp" +#include "kernels/a64_gemv_fp32_mla_32.hpp" +#include "kernels/a64_hybrid_fp32_mla_6x16.hpp" +#include "kernels/a64_hybrid_fp32_mla_8x4.hpp" +#include "kernels/a64_sgemm_8x12.hpp" +#include "kernels/a64_smallK_hybrid_fp32_mla_6x4.hpp" +#include "kernels/a64_smallK_hybrid_fp32_mla_8x4.hpp" -#include "kernels/sve_hybrid_fp32_mla_4VLx4.hpp" -#include "kernels/sve_hybrid_fp32_mmla_4VLx4.hpp" -#include "kernels/sve_interleaved_fp32_mla_3VLx8.hpp" -#include "kernels/sve_interleaved_fp32_mmla_3VLx8.hpp" -#include "kernels/sve_smallK_hybrid_fp32_mla_1VLx8.hpp" +#include "kernels/sve_gemv_fp32_mla_8VL.hpp" +#include "kernels/sve_hybrid_fp32_mla_6x4VL.hpp" +#include "kernels/sve_hybrid_fp32_mla_8x1VL.hpp" +#include "kernels/sve_interleaved_fp32_mla_8x3VL.hpp" +#include "kernels/sve_interleaved_fp32_mmla_8x3VL.hpp" +#include "kernels/sve_smallK_hybrid_fp32_mla_8x1VL.hpp" namespace arm_gemm { static const GemmImplementation gemm_fp32_methods[] = { +// GEMV cases - starting with 'gemv_batched' wrapper to turn batched GEMV into GEMM. { GemmMethod::GEMV_BATCHED, "gemv_batched", - [](const GemmArgs &args) { return (args._Msize==1) && (args._nbatches>1); }, + [](const GemmArgs &args) { return args._Msize==1 && args._nbatches>1 && !args._indirect_input; }, nullptr, [](const GemmArgs &args) { return new GemvBatched(args); } }, #ifdef __aarch64__ +#ifdef __ARM_FEATURE_SVE { - GemmMethod::GEMV_PRETRANSPOSED, - "sgemv_pretransposed", - [](const GemmArgs &args) { return (args._Msize==1 && args._nbatches==1); }, + GemmMethod::GEMM_HYBRID, + "sve_gemv_fp32_mla_8VL", + [](const GemmArgs &args) { return args._Msize==1 && args._nbatches==1 && !args._indirect_input; }, nullptr, - [](const GemmArgs &args) { return new GemvPretransposed(args); } + [](const GemmArgs &args) { return new GemvPretransposed(args); } }, -#if defined(__ARM_FEATURE_SVE) && defined(MMLA_FP32) +#endif { GemmMethod::GEMM_HYBRID, - "hybrid_fp32_mmla_4VLx4", - [](const GemmArgs &args) { return (args._Ksize >= 4); }, - [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmHybrid(args); } + "a64_gemv_fp32_mla_32", + [](const GemmArgs &args) { return args._Msize==1 && args._nbatches==1 && !args._indirect_input; }, + nullptr, + [](const GemmArgs &args) { return new GemvPretransposed(args); } }, + +// MMLA next due to higher throughput (SVE only) +#if defined(__ARM_FEATURE_SVE) && defined(MMLA_FP32) { GemmMethod::GEMM_INTERLEAVED, - "interleaved_fp32_mmla_3VLx8", + "sve_interleaved_fp32_mmla_8x3VL", [](const GemmArgs &args) { return (args._Ksize>4); }, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, #endif // __ARM_FEATURE_SVE && MMLA_FP32 #ifdef __ARM_FEATURE_SVE -// SVE smallk / hybrid methods +// SVE smallk / hybrid methods { GemmMethod::GEMM_HYBRID, - "smallK_hybrid_fp32_mla_1VLx8", - [](const GemmArgs &args) { return (args._Ksize <= 24); }, + "sve_smallK_hybrid_fp32_mla_8x1VL", + [](const GemmArgs &args) { return args._Ksize <= 24 && !args._indirect_input; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_HYBRID, - "hybrid_fp32_mla_4VLx4", - [](const GemmArgs &args) { return (args._Ksize >= 4); }, + "sve_hybrid_fp32_mla_8x1VL", + nullptr, + [](const GemmArgs &args) { return (args._Nsize < 12); }, + [](const GemmArgs &args) { return new GemmHybridIndirect(args); } +}, +{ + GemmMethod::GEMM_HYBRID, + "sve_hybrid_fp32_mla_6x4VL", + nullptr, [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybridIndirect(args); } }, #endif // __ARM_FEATURE_SVE // NEON hybrid methods { GemmMethod::GEMM_HYBRID, - "smallK_hybrid_fp32_mla_4x8", - [](const GemmArgs &args) { return (args._Ksize <= 8) && (args._Nsize % 4)==0; }, + "a64_smallK_hybrid_fp32_mla_8x4", + [](const GemmArgs &args) { return args._Ksize <= 8 && (args._Nsize % 4)==0 && !args._indirect_input; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_HYBRID, - "smallK_hybrid_fp32_mla_4x6", - [](const GemmArgs &args) { return (args._Ksize > 8) && (args._Ksize <= 16) && (args._Nsize % 4)==0; }, + "a64_smallK_hybrid_fp32_mla_6x4", + [](const GemmArgs &args) { return (args._Ksize > 8 && args._Ksize <= 16) && (args._Nsize % 4)==0 && !args._indirect_input; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybrid(args); } }, { GemmMethod::GEMM_HYBRID, - "hybrid_fp32_mla_4x8_normal", - [](const GemmArgs &args) { return (args._Ksize >= 4); }, + "a64_hybrid_fp32_mla_8x4", + nullptr, [](const GemmArgs &args) { return (args._Nsize < 12); }, - [](const GemmArgs &args) { return new GemmHybrid(args); } + [](const GemmArgs &args) { return new GemmHybridIndirect(args); } }, GemmImplementation::with_estimate( GemmMethod::GEMM_HYBRID, - "hybrid_fp32_mla_16x4", - [](const GemmArgs &args) { return (args._Ksize >= 4); }, - [](const GemmArgs &args) { return GemmHybrid::estimate_cycles(args, hybrid_fp32_mla_16x4::get_performance_parameters(args._ci)); }, - [](const GemmArgs &args) { return new GemmHybrid(args); } + "a64_hybrid_fp32_mla_6x16", + nullptr, + [](const GemmArgs &args) { return GemmHybridIndirect::estimate_cycles(args, cls_a64_hybrid_fp32_mla_6x16::get_performance_parameters(args._ci)); }, + [](const GemmArgs &args) { return new GemmHybridIndirect(args); } ), - #ifdef __ARM_FEATURE_SVE { GemmMethod::GEMM_INTERLEAVED, - "interleaved_fp32_mla_3VLx8", + "sve_interleaved_fp32_mla_8x3VL", [](const GemmArgs &args) { return (args._Ksize>4); }, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return new GemmInterleaved(args); } }, #endif // __ARM_FEATURE_SVE -// Pretranposed, 2D split -GemmImplementation::with_estimate( - GemmMethod::GEMM_INTERLEAVED_2D, - "sgemm_12x8_2d", - nullptr, - [](const GemmArgs &args) { return GemmInterleavedPretransposed2d::estimate_cycles(args, sgemm_12x8::get_performance_parameters(args._ci)); }, - [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d(args); } -), -// 1D split (with pretransposed or not) GemmImplementation::with_estimate( GemmMethod::GEMM_INTERLEAVED, - "sgemm_12x8_1d", + "a64_sgemm_8x12", nullptr, - [](const GemmArgs &args) { return GemmInterleaved::estimate_cycles(args, sgemm_12x8::get_performance_parameters(args._ci)); }, - [](const GemmArgs &args) { return new GemmInterleaved(args); } + [](const GemmArgs &args) { return GemmInterleaved::estimate_cycles(args, cls_a64_sgemm_8x12::get_performance_parameters(args._ci)); }, + [](const GemmArgs &args) { return new GemmInterleaved(args); } ), #endif // __aarch64__ -- cgit v1.2.1