diff options
Diffstat (limited to 'src/runtime/heuristics/matmul_native')
8 files changed, 382 insertions, 10 deletions
diff --git a/src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultConfigValhall.cpp b/src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultConfigValhall.cpp index 4b923547c4..3a02a60650 100644 --- a/src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultConfigValhall.cpp +++ b/src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultConfigValhall.cpp @@ -62,6 +62,7 @@ ClMatMulNativeDefaultConfigValhall::configure(const ITensorInfo *lhs, const ITen switch (_target) { case GPUTarget::G715: + case GPUTarget::G615: func = configs_G715.get_function(lhs->data_type()); break; case GPUTarget::G710: diff --git a/src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultVariantValhall.cpp b/src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultVariantValhall.cpp new file mode 100644 index 0000000000..3878f698fd --- /dev/null +++ b/src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultVariantValhall.cpp @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultVariantValhall.h" + +#include "arm_compute/core/Error.h" +#include "arm_compute/core/GPUTarget.h" +#include "arm_compute/core/TensorInfo.h" + +namespace arm_compute +{ +namespace cl_matmul +{ +ClMatMulNativeDefaultVariantValhall::ClMatMulNativeDefaultVariantValhall(GPUTarget gpu) + : IClMatMulNativeKernelVariant(gpu) +{ +} + +MatMulKernelType ClMatMulNativeDefaultVariantValhall::select_kernel(const ITensorInfo *lhs, + const ITensorInfo *rhs, + const MatMulInfo &info, + const ActivationLayerInfo &act_info) +{ + ARM_COMPUTE_UNUSED(rhs); + + using VariantFunctionExecutorPtr = + MatMulKernelType (ClMatMulNativeDefaultVariantValhall::*)(int k, bool act_enabled); + + ClMatMulNativeVariantArray<VariantFunctionExecutorPtr> configs_G715( + &ClMatMulNativeDefaultVariantValhall::configure_G715_float, + &ClMatMulNativeDefaultVariantValhall::configure_G715_quantized); + + ClMatMulNativeVariantArray<VariantFunctionExecutorPtr> configs_default( + &ClMatMulNativeDefaultVariantValhall::configure_default_float, + &ClMatMulNativeDefaultVariantValhall::configure_default_quantized); + + VariantFunctionExecutorPtr func = nullptr; + switch (_target) + { + case GPUTarget::G715: + case GPUTarget::G615: + func = configs_G715.get_function(lhs->data_type()); + break; + default: + func = configs_default.get_function(lhs->data_type()); + break; + } + + const int k = info.adj_lhs() ? lhs->tensor_shape().y() : lhs->tensor_shape().x(); + const bool act_enabled = act_info.enabled(); + + ARM_COMPUTE_ERROR_ON_MSG(func == nullptr, "Data type not supported for matmul native"); + return (this->*func)(k, act_enabled); +} + +MatMulKernelType ClMatMulNativeDefaultVariantValhall::configure_G715_float(int k, bool act_enabled) +{ + // MMUL kernel works only when K is a multiple of 4 + if (!act_enabled && k % 4 == 0) + { + return MatMulKernelType::NATIVE_MMUL_FP; + } + + return MatMulKernelType::NATIVE_FP; +} + +MatMulKernelType ClMatMulNativeDefaultVariantValhall::configure_G715_quantized(int k, bool act_enabled) +{ + // MMUL kernel works only when K is a multiple of 16 + if (!act_enabled && k % 16 == 0) + { + return MatMulKernelType::NATIVE_MMUL_QUANTIZED; + } + + return MatMulKernelType::NATIVE_QUANTIZED; +} + +MatMulKernelType ClMatMulNativeDefaultVariantValhall::configure_default_float(int k, bool act_enabled) +{ + ARM_COMPUTE_UNUSED(k, act_enabled); + + return MatMulKernelType::NATIVE_FP; +} + +MatMulKernelType ClMatMulNativeDefaultVariantValhall::configure_default_quantized(int k, bool act_enabled) +{ + ARM_COMPUTE_UNUSED(k, act_enabled); + + return MatMulKernelType::NATIVE_QUANTIZED; +} + +} // namespace cl_matmul +} // namespace arm_compute diff --git a/src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultVariantValhall.h b/src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultVariantValhall.h new file mode 100644 index 0000000000..a202676e98 --- /dev/null +++ b/src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultVariantValhall.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEDEFAULTVARIANTVALHALL_H +#define ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEDEFAULTVARIANTVALHALL_H + +#include "src/runtime/heuristics/matmul_native/IClMatMulNativeKernelVariant.h" + +namespace arm_compute +{ +namespace cl_matmul +{ +/** Valhall based OpenCL matmul configuration */ +class ClMatMulNativeDefaultVariantValhall final : public IClMatMulNativeKernelVariant +{ +public: + /** Constructor + * + * @param[in] gpu GPU target + */ + ClMatMulNativeDefaultVariantValhall(GPUTarget gpu); + + // Inherited overridden method + MatMulKernelType select_kernel(const ITensorInfo *lhs, + const ITensorInfo *rhs, + const MatMulInfo &info, + const ActivationLayerInfo &act_info) override; + +private: + MatMulKernelType configure_G715_float(int k, bool act_enabled); + MatMulKernelType configure_G715_quantized(int k, bool act_enabled); + MatMulKernelType configure_default_float(int k, bool act_enabled); + MatMulKernelType configure_default_quantized(int k, bool act_enabled); +}; +} // namespace cl_matmul +} // namespace arm_compute +#endif // ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEDEFAULTVARIANTVALHALL_H diff --git a/src/runtime/heuristics/matmul_native/ClMatMulNativeHelpers.h b/src/runtime/heuristics/matmul_native/ClMatMulNativeHelpers.h index a114fffa68..699f5fe8c1 100644 --- a/src/runtime/heuristics/matmul_native/ClMatMulNativeHelpers.h +++ b/src/runtime/heuristics/matmul_native/ClMatMulNativeHelpers.h @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifndef SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEHELPERS -#define SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEHELPERS +#ifndef ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEHELPERS_H +#define ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEHELPERS_H #include "arm_compute/core/Types.h" @@ -80,4 +80,4 @@ MatMulKernelInfo find_info(const MatMulNativeConfigsMatrix &configs, unsigned int b); } // namespace cl_matmul } // namespace arm_compute -#endif /* SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEHELPERS */ +#endif // ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEHELPERS_H diff --git a/src/runtime/heuristics/matmul_native/ClMatMulNativeKernelConfig.h b/src/runtime/heuristics/matmul_native/ClMatMulNativeKernelConfig.h index b10018a6d2..e7485bca81 100644 --- a/src/runtime/heuristics/matmul_native/ClMatMulNativeKernelConfig.h +++ b/src/runtime/heuristics/matmul_native/ClMatMulNativeKernelConfig.h @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifndef SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEKERNELCONFIG -#define SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEKERNELCONFIG +#ifndef ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEKERNELCONFIG_H +#define ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEKERNELCONFIG_H #include "src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultConfigValhall.h" #include "src/runtime/heuristics/matmul_native/IClMatMulNativeKernelConfig.h" @@ -50,6 +50,7 @@ public: case GPUTarget::MIDGARD: case GPUTarget::BIFROST: case GPUTarget::VALHALL: + case GPUTarget::FIFTHGEN: return std::make_unique<ClMatMulNativeDefaultConfigValhall>(gpu); default: ARM_COMPUTE_ERROR("Not supported GPU target"); @@ -58,4 +59,4 @@ public: }; } // namespace cl_matmul } // namespace arm_compute -#endif /* SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEKERNELCONFIG */ +#endif // ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEKERNELCONFIG_H diff --git a/src/runtime/heuristics/matmul_native/ClMatMulNativeKernelVariant.h b/src/runtime/heuristics/matmul_native/ClMatMulNativeKernelVariant.h new file mode 100644 index 0000000000..c2895b8919 --- /dev/null +++ b/src/runtime/heuristics/matmul_native/ClMatMulNativeKernelVariant.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEKERNELVARIANT_H +#define ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEKERNELVARIANT_H + +#include "src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultVariantValhall.h" +#include "src/runtime/heuristics/matmul_native/IClMatMulNativeKernelVariant.h" + +#include <memory> + +namespace arm_compute +{ +namespace cl_matmul +{ + +/** ClMatMul variant factory class */ +class ClMatMulNativeKernelVariantFactory final +{ +public: + /** Static method to call the ClMatMul configuration class accordingly with the GPU target + * + * @param[in] gpu GPU target + * + * @return IClMatMulNativeKernelVariant + */ + static std::unique_ptr<IClMatMulNativeKernelVariant> create(GPUTarget gpu) + { + switch (get_arch_from_target(gpu)) + { + case GPUTarget::MIDGARD: + case GPUTarget::BIFROST: + case GPUTarget::VALHALL: + case GPUTarget::FIFTHGEN: + return std::make_unique<ClMatMulNativeDefaultVariantValhall>(gpu); + default: + ARM_COMPUTE_ERROR("Not supported GPU target"); + } + } +}; +} // namespace cl_matmul +} // namespace arm_compute +#endif // ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_CLMATMULNATIVEKERNELVARIANT_H diff --git a/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelConfig.h b/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelConfig.h index b9b091100c..00ba3641d5 100644 --- a/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelConfig.h +++ b/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelConfig.h @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifndef SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_ICLMATMULNATIVEKERNELCONFIG -#define SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_ICLMATMULNATIVEKERNELCONFIG +#ifndef ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_ICLMATMULNATIVEKERNELCONFIG_H +#define ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_ICLMATMULNATIVEKERNELCONFIG_H #include "arm_compute/core/GPUTarget.h" #include "arm_compute/core/KernelDescriptors.h" @@ -85,7 +85,9 @@ private: std::array<T, 3> _configs; }; -/** Basic interface for the matmul native kernel configuration */ +/** Basic interface for the matmul native kernel configuration + * This is the base class that chooses architecture specific kernel configurations. +*/ class IClMatMulNativeKernelConfig { public: @@ -112,4 +114,4 @@ protected: }; } // namespace cl_matmul } // namespace arm_compute -#endif /* SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_ICLMATMULNATIVEKERNELCONFIG */ +#endif // ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_ICLMATMULNATIVEKERNELCONFIG_H diff --git a/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelVariant.h b/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelVariant.h new file mode 100644 index 0000000000..eac41dd6a3 --- /dev/null +++ b/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelVariant.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_ICLMATMULNATIVEKERNELVARIANT_H +#define ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_ICLMATMULNATIVEKERNELVARIANT_H + +#include "arm_compute/core/CoreTypes.h" // DataType +#include "arm_compute/core/GPUTarget.h" +#include "arm_compute/core/ITensorInfo.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" +#include "arm_compute/function_info/MatMulInfo.h" + +#include "src/core/common/Macros.h" + +#include <array> + +namespace arm_compute +{ +namespace cl_matmul +{ +enum class MatMulKernelType +{ + /** Native matrix multiplication for FP types */ + NATIVE_FP, + + /** Native matrix multiplication for quantized types */ + NATIVE_QUANTIZED, + + /** Native matrix multiplication using MMUL extension for FP types */ + NATIVE_MMUL_FP, + + /** Native matrix multiplication using MMUL extension for Quantized types */ + NATIVE_MMUL_QUANTIZED +}; + +/** Basic container for the OpenCL MatMul Native variant functions */ +template <class T> +class ClMatMulNativeVariantArray +{ +public: + /** Alias for Float index */ + static constexpr size_t DT_FLOAT = 0; + /** Alias for Quantized type index */ + static constexpr size_t DT_QUANTIZED = 1; + + /** Constructor + * + * @param[in] func_float Function to call for matmul native float (F32, F16) + * @param[in] func_quantized Function to call for matmul native quantized (QASYMM8, QASYMM8_SIGNED, QSYMM8_PER_CHANNEL) + * + */ + ClMatMulNativeVariantArray(T func_float, T func_quantized) : _configs{func_float, func_quantized} + { + } + + /** Method to return the matmul native variant function based on data type + * + * @param[in] data_type Input data type + * + * @return the valid function otherwise it returns nullptr if the data type is not valid + */ + T get_function(DataType data_type) + { + switch (data_type) + { + case DataType::F32: + case DataType::F16: + return _configs.at(DT_FLOAT); + case DataType::QASYMM8: + case DataType::QASYMM8_SIGNED: + case DataType::QSYMM8_PER_CHANNEL: + return _configs.at(DT_QUANTIZED); + default: + return nullptr; + } + } + +private: + std::array<T, 2> _configs; +}; + +/** Basic interface for the matmul native kernel variant + * This is the base class that chooses architecture specific kernel variants. +*/ +class IClMatMulNativeKernelVariant +{ +public: + /** Constructor + * + * @param[in] arch GPU target + */ + IClMatMulNativeKernelVariant(GPUTarget arch) : _target(arch) + { + } + ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(IClMatMulNativeKernelVariant); + /** Virtual destructor */ + virtual ~IClMatMulNativeKernelVariant() = default; + /** This method returns the @ref MatMulKernelType for the given inputs + * + * @param[in] lhs LHS tensor + * @param[in] rhs RHS tensor + * @param[in] info MatMul info + * @param[in] act_info Activation layer info + */ + virtual MatMulKernelType select_kernel(const ITensorInfo *lhs, + const ITensorInfo *rhs, + const MatMulInfo &info, + const ActivationLayerInfo &act_info) = 0; + +protected: + GPUTarget _target; +}; +} // namespace cl_matmul +} // namespace arm_compute +#endif // ACL_SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_ICLMATMULNATIVEKERNELVARIANT_H |