diff options
Diffstat (limited to 'src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp')
-rw-r--r-- | src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp | 120 |
1 files changed, 62 insertions, 58 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp index ddb438f06c..e9e335f500 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp @@ -24,6 +24,7 @@ #include "arm_gemm.hpp" #include "gemm_common.hpp" #include "gemm_hybrid.hpp" +#include "gemm_hybrid_indirect.hpp" #include "gemm_implementation.hpp" #include "gemm_interleaved.hpp" #include "gemm_interleaved_pretransposed_2d.hpp" @@ -31,127 +32,130 @@ #include "gemv_pretransposed.hpp" #include "kernels/a32_sgemm_8x6.hpp" -#include "kernels/a64_hybrid_fp32_mla_16x4.hpp" -#include "kernels/a64_hybrid_fp32_mla_4x8.hpp" -#include "kernels/a64_smallK_hybrid_fp32_mla_4x6.hpp" -#include "kernels/a64_smallK_hybrid_fp32_mla_4x8.hpp" -#include "kernels/a64_sgemm_12x8.hpp" -#include "kernels/a64_sgemv_pretransposed.hpp" +#include "kernels/a64_gemv_fp32_mla_32.hpp" +#include "kernels/a64_hybrid_fp32_mla_6x16.hpp" +#include "kernels/a64_hybrid_fp32_mla_8x4.hpp" +#include "kernels/a64_sgemm_8x12.hpp" +#include "kernels/a64_smallK_hybrid_fp32_mla_6x4.hpp" +#include "kernels/a64_smallK_hybrid_fp32_mla_8x4.hpp" -#include "kernels/sve_hybrid_fp32_mla_4VLx4.hpp" -#include "kernels/sve_hybrid_fp32_mmla_4VLx4.hpp" -#include "kernels/sve_interleaved_fp32_mla_3VLx8.hpp" -#include "kernels/sve_interleaved_fp32_mmla_3VLx8.hpp" -#include "kernels/sve_smallK_hybrid_fp32_mla_1VLx8.hpp" +#include "kernels/sve_gemv_fp32_mla_8VL.hpp" +#include "kernels/sve_hybrid_fp32_mla_6x4VL.hpp" +#include "kernels/sve_hybrid_fp32_mla_8x1VL.hpp" +#include "kernels/sve_interleaved_fp32_mla_8x3VL.hpp" +#include "kernels/sve_interleaved_fp32_mmla_8x3VL.hpp" +#include "kernels/sve_smallK_hybrid_fp32_mla_8x1VL.hpp" namespace arm_gemm { static const GemmImplementation<float, float> gemm_fp32_methods[] = { +// GEMV cases - starting with 'gemv_batched' wrapper to turn batched GEMV into GEMM. { GemmMethod::GEMV_BATCHED, "gemv_batched", - [](const GemmArgs &args) { return (args._Msize==1) && (args._nbatches>1); }, + [](const GemmArgs &args) { return args._Msize==1 && args._nbatches>1 && !args._indirect_input; }, nullptr, [](const GemmArgs &args) { return new GemvBatched<float, float>(args); } }, #ifdef __aarch64__ +#ifdef __ARM_FEATURE_SVE { - GemmMethod::GEMV_PRETRANSPOSED, - "sgemv_pretransposed", - [](const GemmArgs &args) { return (args._Msize==1 && args._nbatches==1); }, + GemmMethod::GEMM_HYBRID, + "sve_gemv_fp32_mla_8VL", + [](const GemmArgs &args) { return args._Msize==1 && args._nbatches==1 && !args._indirect_input; }, nullptr, - [](const GemmArgs &args) { return new GemvPretransposed<sgemv_pretransposed, float, float>(args); } + [](const GemmArgs &args) { return new GemvPretransposed<cls_sve_gemv_fp32_mla_8VL, float, float>(args); } }, -#if defined(__ARM_FEATURE_SVE) && defined(MMLA_FP32) +#endif { GemmMethod::GEMM_HYBRID, - "hybrid_fp32_mmla_4VLx4", - [](const GemmArgs &args) { return (args._Ksize >= 4); }, - [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mmla_4VLx4, float, float>(args); } + "a64_gemv_fp32_mla_32", + [](const GemmArgs &args) { return args._Msize==1 && args._nbatches==1 && !args._indirect_input; }, + nullptr, + [](const GemmArgs &args) { return new GemvPretransposed<cls_a64_gemv_fp32_mla_32, float, float>(args); } }, + +// MMLA next due to higher throughput (SVE only) +#if defined(__ARM_FEATURE_SVE) && defined(MMLA_FP32) { GemmMethod::GEMM_INTERLEAVED, - "interleaved_fp32_mmla_3VLx8", + "sve_interleaved_fp32_mmla_8x3VL", [](const GemmArgs &args) { return (args._Ksize>4); }, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp32_mmla_3VLx8, float, float>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_fp32_mmla_8x3VL, float, float>(args); } }, #endif // __ARM_FEATURE_SVE && MMLA_FP32 #ifdef __ARM_FEATURE_SVE -// SVE smallk / hybrid methods +// SVE smallk / hybrid methods { GemmMethod::GEMM_HYBRID, - "smallK_hybrid_fp32_mla_1VLx8", - [](const GemmArgs &args) { return (args._Ksize <= 24); }, + "sve_smallK_hybrid_fp32_mla_8x1VL", + [](const GemmArgs &args) { return args._Ksize <= 24 && !args._indirect_input; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_1VLx8, float, float>(args); } + [](const GemmArgs &args) { return new GemmHybrid<cls_sve_smallK_hybrid_fp32_mla_8x1VL, float, float>(args); } }, { GemmMethod::GEMM_HYBRID, - "hybrid_fp32_mla_4VLx4", - [](const GemmArgs &args) { return (args._Ksize >= 4); }, + "sve_hybrid_fp32_mla_8x1VL", + nullptr, + [](const GemmArgs &args) { return (args._Nsize < 12); }, + [](const GemmArgs &args) { return new GemmHybridIndirect<cls_sve_hybrid_fp32_mla_8x1VL, float, float>(args); } +}, +{ + GemmMethod::GEMM_HYBRID, + "sve_hybrid_fp32_mla_6x4VL", + nullptr, [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, - [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_4VLx4, float, float>(args); } + [](const GemmArgs &args) { return new GemmHybridIndirect<cls_sve_hybrid_fp32_mla_6x4VL, float, float>(args); } }, #endif // __ARM_FEATURE_SVE // NEON hybrid methods { GemmMethod::GEMM_HYBRID, - "smallK_hybrid_fp32_mla_4x8", - [](const GemmArgs &args) { return (args._Ksize <= 8) && (args._Nsize % 4)==0; }, + "a64_smallK_hybrid_fp32_mla_8x4", + [](const GemmArgs &args) { return args._Ksize <= 8 && (args._Nsize % 4)==0 && !args._indirect_input; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_4x8, float, float>(args); } + [](const GemmArgs &args) { return new GemmHybrid<cls_a64_smallK_hybrid_fp32_mla_8x4, float, float>(args); } }, { GemmMethod::GEMM_HYBRID, - "smallK_hybrid_fp32_mla_4x6", - [](const GemmArgs &args) { return (args._Ksize > 8) && (args._Ksize <= 16) && (args._Nsize % 4)==0; }, + "a64_smallK_hybrid_fp32_mla_6x4", + [](const GemmArgs &args) { return (args._Ksize > 8 && args._Ksize <= 16) && (args._Nsize % 4)==0 && !args._indirect_input; }, nullptr, - [](const GemmArgs &args) { return new GemmHybrid<smallK_hybrid_fp32_mla_4x6, float, float>(args); } + [](const GemmArgs &args) { return new GemmHybrid<cls_a64_smallK_hybrid_fp32_mla_6x4, float, float>(args); } }, { GemmMethod::GEMM_HYBRID, - "hybrid_fp32_mla_4x8_normal", - [](const GemmArgs &args) { return (args._Ksize >= 4); }, + "a64_hybrid_fp32_mla_8x4", + nullptr, [](const GemmArgs &args) { return (args._Nsize < 12); }, - [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_4x8, float, float>(args); } + [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_fp32_mla_8x4, float, float>(args); } }, GemmImplementation<float, float>::with_estimate( GemmMethod::GEMM_HYBRID, - "hybrid_fp32_mla_16x4", - [](const GemmArgs &args) { return (args._Ksize >= 4); }, - [](const GemmArgs &args) { return GemmHybrid<hybrid_fp32_mla_16x4, float, float>::estimate_cycles(args, hybrid_fp32_mla_16x4::get_performance_parameters(args._ci)); }, - [](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_16x4, float, float>(args); } + "a64_hybrid_fp32_mla_6x16", + nullptr, + [](const GemmArgs &args) { return GemmHybridIndirect<cls_a64_hybrid_fp32_mla_6x16, float, float>::estimate_cycles(args, cls_a64_hybrid_fp32_mla_6x16::get_performance_parameters(args._ci)); }, + [](const GemmArgs &args) { return new GemmHybridIndirect<cls_a64_hybrid_fp32_mla_6x16, float, float>(args); } ), - #ifdef __ARM_FEATURE_SVE { GemmMethod::GEMM_INTERLEAVED, - "interleaved_fp32_mla_3VLx8", + "sve_interleaved_fp32_mla_8x3VL", [](const GemmArgs &args) { return (args._Ksize>4); }, nullptr, - [](const GemmArgs &args) { return new GemmInterleaved<interleaved_fp32_mla_3VLx8, float, float>(args); } + [](const GemmArgs &args) { return new GemmInterleaved<cls_sve_interleaved_fp32_mla_8x3VL, float, float>(args); } }, #endif // __ARM_FEATURE_SVE -// Pretranposed, 2D split -GemmImplementation<float, float>::with_estimate( - GemmMethod::GEMM_INTERLEAVED_2D, - "sgemm_12x8_2d", - nullptr, - [](const GemmArgs &args) { return GemmInterleavedPretransposed2d<sgemm_12x8, float, float>::estimate_cycles(args, sgemm_12x8::get_performance_parameters(args._ci)); }, - [](const GemmArgs &args) { return new GemmInterleavedPretransposed2d<sgemm_12x8, float, float>(args); } -), -// 1D split (with pretransposed or not) GemmImplementation<float, float>::with_estimate( GemmMethod::GEMM_INTERLEAVED, - "sgemm_12x8_1d", + "a64_sgemm_8x12", nullptr, - [](const GemmArgs &args) { return GemmInterleaved<sgemm_12x8, float, float>::estimate_cycles(args, sgemm_12x8::get_performance_parameters(args._ci)); }, - [](const GemmArgs &args) { return new GemmInterleaved<sgemm_12x8, float, float>(args); } + [](const GemmArgs &args) { return GemmInterleaved<cls_a64_sgemm_8x12, float, float>::estimate_cycles(args, cls_a64_sgemm_8x12::get_performance_parameters(args._ci)); }, + [](const GemmArgs &args) { return new GemmInterleaved<cls_a64_sgemm_8x12, float, float>(args); } ), #endif // __aarch64__ |