From 6ff12a0f7765f62b8d0fa8554021e1cac2789f19 Mon Sep 17 00:00:00 2001 From: Pablo Tello Date: Thu, 2 Nov 2017 16:09:35 +0000 Subject: COMPMID-662: Integrated the new a64_s8_gemm_12x8 + dot product kernel into ACL. Change-Id: Id8f919e486a132fc58346c9f84fccbeeb83d19b3 Reviewed-on: http://mpd-gerrit.cambridge.arm.com/94233 Tested-by: Kaizen Reviewed-by: Anthony Barbier Reviewed-by: Gian Marco Iodice --- arm_compute/core/NEON/NEKernels.h | 1 - .../core/NEON/kernels/NEGEMMAssemblyBaseKernel.h | 2 +- .../NEON/kernels/NEGEMMLowpAssemblyBaseKernel.h | 78 ----- .../core/NEON/kernels/NEGEMMLowpFinalizeKernel.h | 2 +- .../NEON/kernels/NEGEMMLowpMatrixMultiplyKernel.h | 8 +- .../core/NEON/kernels/NEGEMMLowpReductionKernel.h | 2 +- .../kernels/arm64/NEGEMMLowpAArch64V8P4Kernel.h | 6 +- .../NEON/kernels/assembly/gemm_interleaved.hpp | 13 +- .../kernels/assembly/kernels/a64_gemm_s8_12x8.hpp | 61 ++++ .../assembly/kernels/a64_gemm_s8_12x8/a55r1.hpp | 367 +++++++++++++++++++++ .../a64_gemm_s8_12x8/dot_toolchain_support.h | 66 ++++ .../assembly/kernels/a64_gemm_s8_12x8/generic.hpp | 363 ++++++++++++++++++++ .../core/NEON/kernels/assembly/profiler.hpp | 18 +- 13 files changed, 886 insertions(+), 101 deletions(-) delete mode 100644 arm_compute/core/NEON/kernels/NEGEMMLowpAssemblyBaseKernel.h create mode 100644 arm_compute/core/NEON/kernels/assembly/kernels/a64_gemm_s8_12x8.hpp create mode 100644 arm_compute/core/NEON/kernels/assembly/kernels/a64_gemm_s8_12x8/a55r1.hpp create mode 100644 arm_compute/core/NEON/kernels/assembly/kernels/a64_gemm_s8_12x8/dot_toolchain_support.h create mode 100644 arm_compute/core/NEON/kernels/assembly/kernels/a64_gemm_s8_12x8/generic.hpp (limited to 'arm_compute/core') diff --git a/arm_compute/core/NEON/NEKernels.h b/arm_compute/core/NEON/NEKernels.h index 918dfc6914..8dedf38b3e 100644 --- a/arm_compute/core/NEON/NEKernels.h +++ b/arm_compute/core/NEON/NEKernels.h @@ -61,7 +61,6 @@ #include "arm_compute/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMInterleave4x4Kernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMInterleaveBlockedKernel.h" -#include "arm_compute/core/NEON/kernels/NEGEMMLowpAssemblyBaseKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMLowpFinalizeKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMLowpMatrixMultiplyKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMLowpReductionKernel.h" diff --git a/arm_compute/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h b/arm_compute/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h index e298bfdebd..9e0fe8059b 100644 --- a/arm_compute/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h +++ b/arm_compute/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h @@ -30,7 +30,7 @@ namespace arm_compute { class ITensor; -/** AssemblyBase/armv7a NEON kernel to multiply two input matrices "A" and "B". */ +/** Base class for GEMM NEON kernels implemented in Assembly. */ class NEGEMMAssemblyBaseKernel : public INEKernel { public: diff --git a/arm_compute/core/NEON/kernels/NEGEMMLowpAssemblyBaseKernel.h b/arm_compute/core/NEON/kernels/NEGEMMLowpAssemblyBaseKernel.h deleted file mode 100644 index 32105ad6d4..0000000000 --- a/arm_compute/core/NEON/kernels/NEGEMMLowpAssemblyBaseKernel.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2017 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef __ARM_COMPUTE_NEGEMMLOWPASSEMBLYBASE_H__ -#define __ARM_COMPUTE_NEGEMMLOWPASSEMBLYBASE_H__ - -#include "arm_compute/core/NEON/INEKernel.h" - -namespace arm_compute -{ -class ITensor; - -/** GEMMLOWP AssemblyBase NEON kernel to multiply two input matrices "A" and "B". */ -class NEGEMMLowpAssemblyBaseKernel : public INEKernel -{ -public: - /** Constructor */ - NEGEMMLowpAssemblyBaseKernel() - : _input0(nullptr), _input1(nullptr), _output(nullptr), _workspace(nullptr), _transform_0(true), _transform_1(true) - { - } - - /** Prevent instances of this class from being copied (As this class contains pointers) */ - NEGEMMLowpAssemblyBaseKernel(const NEGEMMLowpAssemblyBaseKernel &) = delete; - /** Prevent instances of this class from being copied (As this class contains pointers) */ - NEGEMMLowpAssemblyBaseKernel &operator=(const NEGEMMLowpAssemblyBaseKernel &) = delete; - /** Allow instances of this class to be moved */ - NEGEMMLowpAssemblyBaseKernel(NEGEMMLowpAssemblyBaseKernel &&) = default; - /** Allow instances of this class to be moved */ - NEGEMMLowpAssemblyBaseKernel &operator=(NEGEMMLowpAssemblyBaseKernel &&) = default; - - virtual ~NEGEMMLowpAssemblyBaseKernel() = default; - - /** Initialise the kernel's input and output. - * - * The computed function is C = a * AxB + b * C. - * - * @param[in] input0 Input tensor containing the Matrix A. Data types supported: F32 - * @param[in] input1 Input tensor containing the Matrix B. Data types supported: same as @p input0 - * @param[in,out] output Output tensor to store the result of matrix multiplication. If @p beta is not zero the values are multiplied by @p beta before the result is accumulated. Otherwise the values are overwritten by the result. Data types supported: same as @p input0. - */ - void configure(const ITensor *input0, const ITensor *input1, ITensor *output) - { - internal_configure(input0, input1, output); - } - -protected: - virtual void internal_configure(const ITensor *input0, const ITensor *input1, ITensor *output) = 0; - - const ITensor *_input0; - const ITensor *_input1; - ITensor *_output; - ITensor *_workspace; - bool _transform_0; - bool _transform_1; -}; -} // namespace arm_compute -#endif /*__ARM_COMPUTE_NEGEMMLOWPASSEMBLYBASE_H__*/ diff --git a/arm_compute/core/NEON/kernels/NEGEMMLowpFinalizeKernel.h b/arm_compute/core/NEON/kernels/NEGEMMLowpFinalizeKernel.h index 77b2bdc177..8908fabc1e 100644 --- a/arm_compute/core/NEON/kernels/NEGEMMLowpFinalizeKernel.h +++ b/arm_compute/core/NEON/kernels/NEGEMMLowpFinalizeKernel.h @@ -62,7 +62,7 @@ public: * @param[in] vector_sum_row Input row-vector of sums of all the entries in each row of input0. * Note: vector_sum_row can be a nullptr in case b_offset = 0. Data type supported: same as @p vector_sum_col * @param[in] mm_result Input tensor containing the result of @ref NEGEMMLowpMatrixMultiplyKernel. Data type supported: same as @p vector_sum_col - * @param[out] output Output tensor containing the result of GEMMLowP. Data type supported: U8 + * @param[out] output Output tensor containing the result of GEMMLowP. Data type supported: S8 * @param[in] num_mtx_a_cols Number of matrix A columns * @param[in] a_offset Offset to be added to each element of the matrix A. * @param[in] b_offset Offset to be added to each element of the matrix B. diff --git a/arm_compute/core/NEON/kernels/NEGEMMLowpMatrixMultiplyKernel.h b/arm_compute/core/NEON/kernels/NEGEMMLowpMatrixMultiplyKernel.h index 670274b8f3..f145eb6ca3 100644 --- a/arm_compute/core/NEON/kernels/NEGEMMLowpMatrixMultiplyKernel.h +++ b/arm_compute/core/NEON/kernels/NEGEMMLowpMatrixMultiplyKernel.h @@ -35,8 +35,8 @@ class ITensor; * @note @ref NEGEMMLowpMatrixMultiplyKernel low precision matrix product kernel * This kernel performs the following computation: * - * -# Convert a values from uint8 to int32 - * -# Convert b values from uint8 to int32 + * -# Convert a values from int8 to int32 + * -# Convert b values from int8 to int32 * -# Compute the int32 matrix product of the resulting a * b and store the result as int32 * */ @@ -58,7 +58,7 @@ public: * The input matrices @p input0 and @p input1 must be the output of the kernels: @ref NEGEMMInterleave4x4Kernel and @ref NEGEMMTranspose1xWKernel. These two * kernels change the layout of the original matrices to be more cache-friendly. * - * @param[in] input0 Input tensor containing the interleaved Matrix A. Data type supported: U8 + * @param[in] input0 Input tensor containing the interleaved Matrix A. Data type supported: S8 * @param[in] input1 Input tensor containing the transposed Matrix B. Data type supported: same as @p input0 * @param[out] output Output tensor to store the result of matrix multiplication. Data type supported: S32 */ @@ -74,4 +74,4 @@ private: bool _slide_matrix_b; }; } // namespace arm_compute -#endif /*__ARM_COMPUTE_NEGEMMLOWPMATRIXMULTIPLYKERNEL_H__*/ \ No newline at end of file +#endif /*__ARM_COMPUTE_NEGEMMLOWPMATRIXMULTIPLYKERNEL_H__*/ diff --git a/arm_compute/core/NEON/kernels/NEGEMMLowpReductionKernel.h b/arm_compute/core/NEON/kernels/NEGEMMLowpReductionKernel.h index 143e8b917b..a069969681 100644 --- a/arm_compute/core/NEON/kernels/NEGEMMLowpReductionKernel.h +++ b/arm_compute/core/NEON/kernels/NEGEMMLowpReductionKernel.h @@ -48,7 +48,7 @@ public: public: /** Initialise the kernel's input and output. * - * @param[in] input Input tensor containing the interleaved or transposed matrix. Data type supported: U8 + * @param[in] input Input tensor containing the interleaved or transposed matrix. Data type supported: S8 * @param[out] output Output row-vector of sums of all the entries in each row/col of input tensor. Data type supported: S32 * @param[in] k Number of matrix A columns (or matrix B rows) * @param[in] is_reshaped True if the input tensor has been reshaped diff --git a/arm_compute/core/NEON/kernels/arm64/NEGEMMLowpAArch64V8P4Kernel.h b/arm_compute/core/NEON/kernels/arm64/NEGEMMLowpAArch64V8P4Kernel.h index 4eab7f91fc..32779891db 100644 --- a/arm_compute/core/NEON/kernels/arm64/NEGEMMLowpAArch64V8P4Kernel.h +++ b/arm_compute/core/NEON/kernels/arm64/NEGEMMLowpAArch64V8P4Kernel.h @@ -24,7 +24,7 @@ #ifndef __ARM_COMPUTE_NEGEMMLOWPAARCH64V8P4KERNEL_H__ #define __ARM_COMPUTE_NEGEMMLOWPAARCH64V8P4KERNEL_H__ -#include "arm_compute/core/NEON/kernels/NEGEMMLowpAssemblyBaseKernel.h" +#include "arm_compute/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h" // Enable only if compiled for AArch64-V8.2-A targets #ifdef ARM_COMPUTE_AARCH64_V8_2 @@ -34,7 +34,7 @@ namespace arm_compute class ITensor; /** AArch64 NEON kernel to multiply two input matrices "A" and "B". */ -class NEGEMMLowpAArch64V8P4Kernel : public NEGEMMLowpAssemblyBaseKernel +class NEGEMMLowpAArch64V8P4Kernel : public NEGEMMAssemblyBaseKernel { public: // Inherited methods overridden: @@ -42,7 +42,7 @@ public: bool is_parallelisable() const override; protected: - void internal_configure(const ITensor *input0, const ITensor *input1, ITensor *output) override; + void internal_configure(const ITensor *input0, const ITensor *input1, ITensor *output, ITensor *workspace, float alpha, float beta, bool transform_0, bool transform_1) override; }; } // namespace arm_compute #endif /* ARM_COMPUTE_AARCH64_V8_2 */ diff --git a/arm_compute/core/NEON/kernels/assembly/gemm_interleaved.hpp b/arm_compute/core/NEON/kernels/assembly/gemm_interleaved.hpp index a186d88355..659ef837f5 100644 --- a/arm_compute/core/NEON/kernels/assembly/gemm_interleaved.hpp +++ b/arm_compute/core/NEON/kernels/assembly/gemm_interleaved.hpp @@ -24,6 +24,7 @@ #pragma once #include +#include #include "gemm_common.hpp" #include "profiler.hpp" @@ -114,12 +115,13 @@ public: // Work out the rounded size of M - needed for some buffers. Mround = (M + (strat.out_height - 1)) / strat.out_height; Mround *= strat.out_height; + } // Actually execute the GEMM. void execute(const To *A, const int lda, const To *B, const int ldb, Tr *C, const int ldc, const Tr alpha, const Tr beta, void *working_space) const override { + assert(working_space); profiler prof; - int8_t *working_space_bytes = reinterpret_cast(working_space); intptr_t working_space_int = reinterpret_cast(working_space_bytes); size_t diff = 0; @@ -128,7 +130,6 @@ public: diff = 0x10 - (working_space_int & 0xF); } - // TODO: Multithreaded implementations could share the burden of transforming these blocks. Toi * const a_panel = reinterpret_cast(working_space_bytes + diff); Toi * const b_panel = reinterpret_cast(working_space_bytes + get_a_working_size() + diff); Tri * const c_panel = reinterpret_cast(working_space_bytes + get_a_working_size() + get_b_working_size() + diff); @@ -141,7 +142,7 @@ public: int kern_k = ((kmax - k0) + (strat.k_unroll - 1)) / strat.k_unroll; kern_k *= strat.k_unroll; - prof(PROFILE_PREPA, [&](void) { + prof(PROFILE_PREPA, (M * (kmax-k0) * sizeof(Toi)), [&](void) { if (trA ^ strategy::A_transpose) { Transform(a_panel, A, lda, 0, M, k0, kmax); } else { @@ -155,7 +156,7 @@ public: int bblocks = (xmax - x0 + strat.out_width - 1) / strat.out_width; - prof(PROFILE_PREPB, [&](void) { + prof(PROFILE_PREPB, (xmax-x0) * (kmax-k0) * sizeof(Toi), [&](void) { if (trB ^ strategy::B_transpose) { Transform(b_panel, B, ldb, x0, xmax, k0, kmax); } else { @@ -167,8 +168,8 @@ public: unsigned int ymax = y + strat.out_height; if (ymax > M) ymax = M; - prof(PROFILE_KERNEL, [&](void) { strat.kernel(a_panel + (y * kern_k), b_panel, c_panel, 1, bblocks, kern_k); }); - prof(PROFILE_MERGE, [&](void) { MergeResults(C, c_panel, ldc, y, ymax, x0, xmax, alpha, (k0==0 ? beta : static_cast(1))); }); + prof(PROFILE_KERNEL, (strat.out_height * bblocks * strat.out_width * kern_k), [&](void) { strat.kernel(a_panel + (y * kern_k), b_panel, c_panel, 1, bblocks, kern_k); }); + prof(PROFILE_MERGE, (strat.out_height * bblocks * strat.out_width * sizeof(Tr)), [&](void) { MergeResults(C, c_panel, ldc, y, ymax, x0, xmax, alpha, (k0==0 ? beta : static_cast(1))); }); } } } diff --git a/arm_compute/core/NEON/kernels/assembly/kernels/a64_gemm_s8_12x8.hpp b/arm_compute/core/NEON/kernels/assembly/kernels/a64_gemm_s8_12x8.hpp new file mode 100644 index 0000000000..88cbb361b3 --- /dev/null +++ b/arm_compute/core/NEON/kernels/assembly/kernels/a64_gemm_s8_12x8.hpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +// Load the actual kernel +#include "a64_gemm_s8_12x8/generic.hpp" + +class gemm_s8_12x8 { +public: + typedef int8_t operand_type; + typedef int32_t result_type; + + typedef void (*kern_type)(const int8_t *, const int8_t *, int32_t *, int, int, int); + + /* Describes the data layout for A input */ + static const int A_interleave = 8; + static const int A_block = 4; + static const bool A_transpose = false; + + /* Same for B input */ + static const int B_interleave = 12; + static const int B_block = 4; + static const bool B_transpose = true; + + /* Kernel blocking parameters */ + static const int out_width = 12; + static const int out_height = 8; + static const int k_unroll = 4; + + kern_type kernel = nullptr; + + gemm_s8_12x8(const CPUInfo *ci) { + kernel = a64_gemm_s8_12x8; + } +}; + +#endif // __aarch64__ + diff --git a/arm_compute/core/NEON/kernels/assembly/kernels/a64_gemm_s8_12x8/a55r1.hpp b/arm_compute/core/NEON/kernels/assembly/kernels/a64_gemm_s8_12x8/a55r1.hpp new file mode 100644 index 0000000000..5ed930c0b0 --- /dev/null +++ b/arm_compute/core/NEON/kernels/assembly/kernels/a64_gemm_s8_12x8/a55r1.hpp @@ -0,0 +1,367 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#pragma once + +#ifdef __aarch64__ + +#include +#include "dot_toolchain_support.h" +#include + +void a64_gemm_s8_12x8_a55r1(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) { + assert(Apanel); + assert(Bpanel); + assert(Cpanel); + K/=4; + const long int row_jump=0; + const long int block_jump=0; + const int32_t *a_ptr = reinterpret_cast(Apanel); + int32_t *c_ptr = reinterpret_cast(Cpanel); + for (int yb=0; yb(Bpanel); + for (int xb=0; xb +#include "dot_toolchain_support.h" +#include + + +inline void a64_gemm_s8_12x8(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) { + assert(Apanel); + assert(Bpanel); + assert(Cpanel); + K/=4; + const long int row_jump=0; + const long int block_jump=0; + const int32_t *a_ptr = reinterpret_cast(Apanel); + int32_t *c_ptr = reinterpret_cast(Cpanel); + for (int yb=0; yb(Bpanel); + for (int xb=0; xb - void operator() (int i, T func) { + void operator() (int i, unsigned long u, T func) { if (currentevent==maxevents) { func(); } else { + events[currentevent] = i; + units[currentevent] = u; start_counter(countfd); func(); long long cycs = stop_counter(countfd); - events[currentevent] = i; times[currentevent++] = cycs; } } @@ -84,7 +88,7 @@ public: class profiler { public: template - void operator() (int i, T func) { + void operator() (int i, unsigned long u, T func) { func(); } }; @@ -95,3 +99,5 @@ public: #define PROFILE_PREPB 2 #define PROFILE_KERNEL 3 #define PROFILE_MERGE 4 + + -- cgit v1.2.1