From 7cd26d4a1b14bc4bf7c61496803416ab3d84791f Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Wed, 9 Jan 2019 18:35:17 +0000 Subject: COMPMID-1867: Add NEON/SVE GEMM Hybrid kernels. Change-Id: Ib40a9921e7f9a6a8be6c38872d6b3a0f24ed0cd3 Reviewed-on: https://review.mlplatform.org/515 Reviewed-by: Anthony Barbier Tested-by: Arm Jenkins --- src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp | 211 +++++++++++++++------------ 1 file changed, 120 insertions(+), 91 deletions(-) (limited to 'src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp') diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp index 7d14971b70..8bc33ccb69 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018 ARM Limited. + * Copyright (c) 2017-2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #include "arm_gemm.hpp" #include "gemm_common.hpp" +#include "gemm_hybrid.hpp" #include "gemm_implementation.hpp" #include "gemm_interleaved.hpp" #include "gemm_native.hpp" @@ -30,112 +31,140 @@ #include "gemv_native_transposed.hpp" #include "gemv_pretransposed.hpp" -#include "kernels/a64_sgemm_12x8.hpp" #include "kernels/a32_sgemm_8x6.hpp" -#include "kernels/a64_sgemv_trans.hpp" -#include "kernels/a64_sgemv_pretransposed.hpp" +#include "kernels/a64_sgemm_12x8.hpp" #include "kernels/a64_sgemm_native_16x4.hpp" +#include "kernels/a64_sgemm_nativeA_pretransposeB_16x4.hpp" +#include "kernels/a64_sgemv_pretransposed.hpp" +#include "kernels/a64_sgemv_trans.hpp" +#include "kernels/sve_hybrid_fp32_mla_4VLx4.hpp" #include "kernels/sve_interleaved_fp32_mla_3VLx8.hpp" +#include "kernels/sve_native_fp32_mla_4VLx4.hpp" +#include "kernels/sve_smallK_fp32_mla_1VLx4.hpp" +#include "kernels/sve_smallK_hybrid_fp32_mla_1VLx4.hpp" namespace arm_gemm { -#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE) -// SGEMM implementations for AArch64 without SVE - -// Pretransposed GEMV -class GemmImpl_sgemm_gemv_pretransposed : public GemmImplementation { -public: - bool is_supported(const GemmArgs &args) override { - return (args._Msize==1 && args._alpha==1.0f && args._pretransposed_hint && args._nbatches==1); - } +static const GemmImplementation gemm_fp32_methods[] = +{ +{ + GemmMethod::GEMV_BATCHED, + "gemv_batched", + [](const GemmArgs &args) { return (args._Msize==1) && (args._nbatches>1); }, + nullptr, + [](const GemmArgs &args) { return new GemvBatched(args); } +}, +#ifdef __aarch64__ +{ + GemmMethod::GEMV_PRETRANSPOSED, + "sgemv_pretransposed", + [](const GemmArgs &args) { return (args._Msize==1 && args._alpha==1.0f && args._pretransposed_hint && args._nbatches==1); }, + nullptr, + [](const GemmArgs &args) { return new GemvPretransposed(args); } +}, +{ + GemmMethod::GEMV_NATIVE_TRANSPOSED, + "sgemv_trans", + [](const GemmArgs &args) { return (args._Msize==1 && args._alpha==1.0f && !args._trA && !args._trB && args._nbatches==1); }, + nullptr, + [](const GemmArgs &args) { return new GemvNativeTransposed(args); } +}, - UniqueGemmCommon instantiate(const GemmArgs &args) override { - return UniqueGemmCommon (new GemvPretransposed(args._ci, args._Nsize, args._Ksize, args._nmulti, args._trB, args._beta)); - } - - GemmImpl_sgemm_gemv_pretransposed() : GemmImplementation(GemmMethod::GEMV_PRETRANSPOSED) { } -}; - -// Native GEMV -class GemmImpl_sgemm_gemv_native_transposed : public GemmImplementation { -public: - bool is_supported(const GemmArgs &args) override { - return (args._Msize==1 && args._alpha==1.0f && !args._trA && !args._trB && args._nbatches==1); - } - - UniqueGemmCommon instantiate(const GemmArgs &args) override { - return UniqueGemmCommon (new GemvNativeTransposed(args._ci, args._Nsize, args._Ksize, args._nmulti, args._beta)); - } - - GemmImpl_sgemm_gemv_native_transposed() : GemmImplementation(GemmMethod::GEMV_NATIVE_TRANSPOSED) { } -}; - -// Native GEMM -class GemmImpl_sgemm_gemm_native : public GemmImplementation { -public: - bool is_supported(const GemmArgs &args) override { - return (args._Ksize>4 && (args._Nsize % 16)==0 && args._alpha==1.0f && !args._trA && !args._trB); - } - - bool is_recommended(const GemmArgs &args) override { - return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); - } - - UniqueGemmCommon instantiate(const GemmArgs &args) override { - return UniqueGemmCommon (new GemmNative(args._ci, args._Msize, args._Nsize, args._Ksize, args._nbatches, args._nmulti, args._beta)); - } - - GemmImpl_sgemm_gemm_native() : GemmImplementation(GemmMethod::GEMM_NATIVE) { } -}; -#endif // __aarch64__ - -// Interleaved GEMM -class GemmImpl_sgemm_gemm_interleaved : public GemmImplementation { -public: - UniqueGemmCommon instantiate(const GemmArgs &args) override { #ifdef __ARM_FEATURE_SVE - return UniqueGemmCommon (new GemmInterleaved(args)); -#elif defined(__aarch64__) - return UniqueGemmCommon (new GemmInterleaved(args)); -#elif defined(__arm__) - return UniqueGemmCommon (new GemmInterleaved(args)); -#else -# error Unknown Architecture. -#endif - } - - GemmImpl_sgemm_gemm_interleaved() : GemmImplementation(GemmMethod::GEMM_INTERLEAVED) { } -}; + // SVE smallk / native / hybrid methods +{ + GemmMethod::GEMM_HYBRID, + "smallK_hybrid_fp32_mla_1VLx4", + [](const GemmArgs &args) { return (args._Ksize <= 24) && !args._trA && args._alpha==1.0f && args._pretransposed_hint; }, + nullptr, + [](const GemmArgs &args) { return new GemmHybrid(args); } +}, +{ + GemmMethod::GEMM_HYBRID, + "hybrid_fp32_mla_4VLx4", + [](const GemmArgs &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid(args); } +}, +{ + GemmMethod::GEMM_NATIVE, + "smallK_fp32_mla_1VLx4", + [](const GemmArgs &args) { return (args._Ksize <= 24) && !args._trA && !args._trB && args._alpha==1.0f; }, + nullptr, + [](const GemmArgs &args) { return new GemmNative(args); } +}, +{ + GemmMethod::GEMM_NATIVE, + "native_fp32_mla_4VLx4", + [](const GemmArgs &args) { return (args._Ksize>4 && args._alpha==1.0f && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmNative(args); } +}, +#endif // __ARM_FEATURE_SVE + +// NEON native / hybrid methods +{ + GemmMethod::GEMM_HYBRID, + "sgemm_nativeA_pretransposeB_16x4", + [](const GemmArgs &args) { return (args._Ksize >= 4) && (args._alpha == 1.0f) && !args._trA && args._pretransposed_hint; }, + [](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmHybrid(args); } +}, +{ + GemmMethod::GEMM_NATIVE, + "sgemm_native_16x4", + [](const GemmArgs &args) { return (args._Ksize>4 && (args._Nsize % 16)==0 && args._alpha==1.0f && !args._trA && !args._trB); }, + [](const GemmArgs &args) { return ((args._Ksize <= 128) && (args._Nsize <= 128)) || ((args._nmulti > 1) && ((args._Msize / args._maxthreads) < 8)); }, + [](const GemmArgs &args) { return new GemmNative(args); } +}, -static GemmImpl_gemv_batched gemv_batched_impl{}; -#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE) -static GemmImpl_sgemm_gemv_pretransposed sgemm_gemv_pretransposed_impl{}; -static GemmImpl_sgemm_gemv_native_transposed sgemm_gemv_native_transposed_impl{}; -static GemmImpl_sgemm_gemm_native sgemm_gemm_native_impl{}; -#endif -static GemmImpl_sgemm_gemm_interleaved sgemm_gemm_interleaved_impl{}; +#ifdef __ARM_FEATURE_SVE + { + GemmMethod::GEMM_INTERLEAVED, + "interleaved_fp32_mla_3VLx8", + [](const GemmArgs &args) { return (args._Ksize>4); }, + nullptr, + [](const GemmArgs &args) { return new GemmInterleaved(args); } +}, +#endif // __ARM_FEATURE_SVE +{ + GemmMethod::GEMM_INTERLEAVED, + "sgemm_12x8", + nullptr, + nullptr, + [](const GemmArgs &args) { return new GemmInterleaved(args); } +}, +#endif // __aarch64__ -/* List of implementations (order matters) */ -static std::vector *> SGemmMethods = { - &gemv_batched_impl, -#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE) - &sgemm_gemv_pretransposed_impl, - &sgemm_gemv_native_transposed_impl, - &sgemm_gemm_native_impl, -#endif - &sgemm_gemm_interleaved_impl +#ifdef __arm__ + { + GemmMethod::GEMM_INTERLEAVED, + "sgemm_8x6", + nullptr, + nullptr, + [](const GemmArgs &args) { return new GemmInterleaved(args); } +}, +#endif // __arm__ +{ + GemmMethod::DEFAULT, + "", + nullptr, + nullptr, + nullptr +} }; /* Templated function to return this list. */ template<> -std::vector *> &gemm_implementation_list() { - return SGemmMethods; +const GemmImplementation *gemm_implementation_list() { + return gemm_fp32_methods; } /* Explicitly instantiate the external functions for these types. */ -template UniqueGemmCommon gemm(GemmArgs &args, GemmConfig *cfg); -template GemmMethod get_gemm_method(GemmArgs &args); -template bool method_is_compatible(GemmMethod method, GemmArgs &args); +template UniqueGemmCommon gemm(const GemmArgs &args); +template KernelDescription get_gemm_method(const GemmArgs &args); +template bool method_is_compatible(GemmMethod method, const GemmArgs &args); +template std::vector get_compatible_kernels (const GemmArgs &args); -} // namespace arm_gemm +} // namespace arm_gemm \ No newline at end of file -- cgit v1.2.1