Use look up table for fp16 activation

- Enables FP16 lut for logistic activation
- Adds LUTManager to re-use lut where appropriate.

Signed-off-by: Mohammed Suhail Munshi <MohammedSuhail.Munshi@arm.com>
Change-Id: I94667b63b452a8e58a1eb59cb0b5866178954523
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/10864
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Gunes Bayir <gunes.bayir@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Benchmark: Arm Jenkins <bsgcomp@arm.com>

6 files changed, 176 insertions, 17 deletions

diff --git a/src/cpu/kernels/CpuActivationKernel.cpp b/src/cpu/kernels/CpuActivationKernel.cpp
index 3f3d72e8df..7cfa39b286 100644
--- a/src/cpu/kernels/CpuActivationKernel.cpp
+++ b/src/cpu/kernels/CpuActivationKernel.cpp
@@ -83,6 +83,13 @@ static const std::vector<CpuActivationKernel::ActivationKernel> available_kernel
                 data.f != ActivationLayerInfo::ActivationFunction::GELU;
      },
      REGISTER_QSYMM16_SVE2(arm_compute::cpu::sve2_qsymm16_activation)},
+    {"sve_fp16_activation_lut",
+     [](const ActivationDataTypeISASelectorData &data)
+     {
+         return data.dt == DataType::F16 && data.isa.fp16 && data.isa.sve &&
+                data.f == ActivationLayerInfo::ActivationFunction::LOGISTIC;
+     },
+     REGISTER_FP16_SVE(arm_compute::cpu::sve_fp16_activation_lut)},
     {"sve_fp16_activation",
      [](const ActivationDataTypeISASelectorData &data)
      {
@@ -279,6 +286,9 @@ void CpuActivationKernel::configure(const ITensorInfo *src, ITensorInfo *dst, Ac
     _name       = std::string("CpuActivationKernel").append("/").append(uk->name);
 
 #ifdef __aarch64__
+    // Initialise lut_manager
+    LUTManager &lut_manager = LUTManager::get_instance();
+
     if ((src->data_type() == DataType::QASYMM8 || src->data_type() == DataType::QASYMM8_SIGNED) &&
         activation_info.activation() != ActivationFunction::RELU)
     {
@@ -288,6 +298,13 @@ void CpuActivationKernel::configure(const ITensorInfo *src, ITensorInfo *dst, Ac
                  activation_info.a(), activation_info.b());
         activation_info.setLookupTable256(tmp_lut);
     }
+
+    if (src->data_type() == DataType::F16 &&
+        activation_info.activation() == ActivationLayerInfo::ActivationFunction::LOGISTIC)
+    {
+        const LUTInfo info = {activation_info.activation(), src->data_type(), src->quantization_info()};
+        activation_info.setLookupTable65536((lut_manager.get_lut_table(info)));
+    }
 #endif // __aarch64__
     _act_info = activation_info;
 
diff --git a/src/cpu/kernels/CpuActivationKernel.h b/src/cpu/kernels/CpuActivationKernel.h
index 4bad9fb3e8..c1487499d6 100644
--- a/src/cpu/kernels/CpuActivationKernel.h
+++ b/src/cpu/kernels/CpuActivationKernel.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2023 Arm Limited.
+ * Copyright (c) 2017-2024 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -21,12 +21,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-#ifndef ARM_COMPUTE_CPU_ACTIVATION_KERNEL_H
-#define ARM_COMPUTE_CPU_ACTIVATION_KERNEL_H
+#ifndef ACL_SRC_CPU_KERNELS_CPUACTIVATIONKERNEL_H
+#define ACL_SRC_CPU_KERNELS_CPUACTIVATIONKERNEL_H
 
 #include "arm_compute/function_info/ActivationLayerInfo.h"
 
 #include "src/core/common/Macros.h"
+#include "src/core/helpers/LUTManager.h"
 #include "src/cpu/ICpuKernel.h"
 
 namespace arm_compute
@@ -103,4 +104,4 @@ private:
 } // namespace kernels
 } // namespace cpu
 } // namespace arm_compute
-#endif /* ARM_COMPUTE_CPU_ACTIVATION_KERNEL_H */
+#endif // ACL_SRC_CPU_KERNELS_CPUACTIVATIONKERNEL_H
diff --git a/src/cpu/kernels/activation/generic/sve/fp16.cpp b/src/cpu/kernels/activation/generic/sve/fp16.cpp
index 97399e01e0..19d9126556 100644
--- a/src/cpu/kernels/activation/generic/sve/fp16.cpp
+++ b/src/cpu/kernels/activation/generic/sve/fp16.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2023 Arm Limited.
+ * Copyright (c) 2020-2024 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -30,6 +30,7 @@
 #include "arm_compute/function_info/ActivationLayerInfo.h"
 
 #include "src/core/NEON/SVEMath.h"
+#include "src/cpu/kernels/lut/list.h"
 
 #include <arm_sve.h>
 #include <cmath>
@@ -141,6 +142,32 @@ void sve_fp16_activation(const ITensor *src, ITensor *dst, const ActivationLayer
         },
         input, output);
 }
+
+void sve_fp16_activation_lut(const ITensor             *src,
+                             ITensor                   *dst,
+                             const ActivationLayerInfo &act_info,
+                             const Window              &window)
+{
+    ARM_COMPUTE_ERROR_ON(src->info()->data_type() != DataType::F16);
+    const auto window_start_x = window.x().start();
+    const auto window_end_x   = window.x().end();
+    const auto size           = window_end_x - window_start_x;
+    Window     win_collapsed  = window.collapse_if_possible(window, Window::DimZ);
+    win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1));
+
+    Iterator input(src, win_collapsed);
+    Iterator output(dst, win_collapsed);
+    execute_window_loop(
+        win_collapsed,
+        [&](const Coordinates &)
+        {
+            const auto input_ptr  = reinterpret_cast<const uint16_t *>(input.ptr());
+            auto       output_ptr = reinterpret_cast<uint16_t *>(output.ptr());
+            lut_u16_sve(reinterpret_cast<const uint16_t *>(act_info.lut_fp16().data()), 1U /* num_strings (UNUSED) */,
+                        size, input_ptr + window_start_x, output_ptr + window_start_x);
+        },
+        input, output);
+}
 } // namespace cpu
 } // namespace arm_compute
 #endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */
diff --git a/src/cpu/kernels/activation/list.h b/src/cpu/kernels/activation/list.h
index 6550ddfeca..8c24adc3fe 100644
--- a/src/cpu/kernels/activation/list.h
+++ b/src/cpu/kernels/activation/list.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2023 Arm Limited.
+ * Copyright (c) 2020-2024 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -21,8 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-#ifndef SRC_CORE_NEON_KERNELS_ACTIVATION_LIST_H
-#define SRC_CORE_NEON_KERNELS_ACTIVATION_LIST_H
+#ifndef ACL_SRC_CPU_KERNELS_ACTIVATION_LIST_H
+#define ACL_SRC_CPU_KERNELS_ACTIVATION_LIST_H
 
 namespace arm_compute
 {
@@ -42,6 +42,7 @@ DECLARE_ACTIVATION_KERNEL(sve2_qasymm8_signed_activation);
 DECLARE_ACTIVATION_KERNEL(neon_qsymm16_activation);
 DECLARE_ACTIVATION_KERNEL(sve2_qsymm16_activation);
 DECLARE_ACTIVATION_KERNEL(sve_fp16_activation);
+DECLARE_ACTIVATION_KERNEL(sve_fp16_activation_lut);
 DECLARE_ACTIVATION_KERNEL(sve_fp32_activation);
 DECLARE_ACTIVATION_KERNEL(neon_fp16_activation);
 DECLARE_ACTIVATION_KERNEL(neon_fp32_activation);
@@ -50,4 +51,4 @@ DECLARE_ACTIVATION_KERNEL(neon_fp32_activation);
 } // namespace cpu
 } // namespace arm_compute
 
-#endif /* SRC_CORE_NEON_KERNELS_ACTIVATION_LIST_H */
+#endif // ACL_SRC_CPU_KERNELS_ACTIVATION_LIST_H
diff --git a/src/cpu/kernels/lut/generic/sve/u16.cpp b/src/cpu/kernels/lut/generic/sve/u16.cpp
new file mode 100644
index 0000000000..75b8dcaae2
--- /dev/null
+++ b/src/cpu/kernels/lut/generic/sve/u16.cpp
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2024 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "arm_compute/core/Error.h"
+
+#include "src/cpu/kernels/lut/list.h"
+
+#ifdef __aarch64__
+#ifdef ARM_COMPUTE_ENABLE_SVE
+
+#include <arm_sve.h>
+
+namespace arm_compute
+{
+namespace cpu
+{
+void lut_u16_sve(const uint16_t *table, size_t num_strings, size_t size, const uint16_t *input, uint16_t *output)
+{
+    int64_t cnth  = svcnth();
+    int64_t tail  = size & (4 * cnth - 1);
+    int64_t count = size - tail;
+    int64_t pos   = 0;
+    ARM_COMPUTE_UNUSED(num_strings);
+    __asm __volatile("cbz %[count], 2f\n"
+                     "mov z31.s, #0\n"
+                     "cnth x7, ALL, MUL #4\n"
+                     "cntb x8, ALL, MUL #4\n"
+                     "ptrue p0.b\n"
+                     "1:"
+                     "ld1h z0.h, p0/z, [%[input]]\n"
+                     "ld1h z1.h, p0/z, [%[input], #1, MUL VL]\n"
+                     "ld1h z2.h, p0/z, [%[input], #2, MUL VL]\n"
+                     "ld1h z3.h, p0/z, [%[input], #3, MUL VL]\n"
+                     "add %[input], %[input], x8\n"
+
+                     "zip1 z8.h, z0.h, z31.h\n"
+                     "ld1h z8.s, p0/z, [%[table], z8.s, UXTW #1]\n"
+                     "zip2 z0.h, z0.h, z31.h\n"
+                     "ld1h z0.s, p0/z, [%[table], z0.s, UXTW #1]\n"
+                     "uzp1 z0.h, z8.h, z0.h\n"
+                     "st1h z0.h, p0, [%[output]]\n"
+
+                     "zip1 z10.h, z1.h, z31.h\n"
+                     "ld1h z10.s, p0/z, [%[table], z10.s, UXTW #1]\n"
+                     "zip2 z1.h, z1.h, z31.h\n"
+                     "ld1h z1.s, p0/z, [%[table], z1.s, UXTW #1]\n"
+                     "uzp1 z1.h, z10.h, z1.h\n"
+                     "st1h z1.h, p0, [%[output], #1, MUL VL]\n"
+
+                     "zip1 z12.h, z2.h, z31.h\n"
+                     "ld1h z12.s, p0/z, [%[table], z12.s, UXTW #1]\n"
+                     "zip2 z2.h, z2.h, z31.h\n"
+                     "ld1h z2.s, p0/z, [%[table], z2.s, UXTW #1]\n"
+                     "uzp1 z2.h, z12.h, z2.h\n"
+                     "st1h z2.h, p0, [%[output], #2, MUL VL]\n"
+
+                     "zip1 z14.h, z3.h, z31.h\n"
+                     "ld1h z14.s, p0/z, [%[table], z14.s, UXTW #1]\n"
+                     "zip2 z3.h, z3.h, z31.h\n"
+                     "ld1h z3.s, p0/z, [%[table], z3.s, UXTW #1]\n"
+                     "uzp1 z3.h, z14.h, z3.h\n"
+                     "st1h z3.h, p0, [%[output], #3, MUL VL]\n"
+
+                     "add %[pos], %[pos], x7\n"
+                     "add %[output], %[output], x8\n"
+                     "cmp %[pos], %[count]\n"
+                     "blt 1b\n"
+                     "2:\n"
+                     : [count] "+r"(count), [input] "+r"(input), [output] "+r"(output), [pos] "+r"(pos)
+                     : [table] "r"(table)
+                     : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12",
+                       "z14", "z31", "p0", "p1", "z2", "z3", "z4", "x7", "x8");
+    for (int i = 0; i < tail; i++)
+    {
+        output[i] = table[input[i]];
+    }
+}
+
+} // namespace cpu
+} // namespace arm_compute
+
+#endif // ARM_COMPUTE_ENABLE_SVE
+#endif // __aarch64__
diff --git a/src/cpu/kernels/lut/list.h b/src/cpu/kernels/lut/list.h
index da90346267..9acfe97728 100644
--- a/src/cpu/kernels/lut/list.h
+++ b/src/cpu/kernels/lut/list.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023 Arm Limited.
+ * Copyright (c) 2023-2024 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -22,8 +22,8 @@
  * SOFTWARE.
  */
 
-#ifndef SRC_CORE_NEON_KERNELS_LUT_LIST_H
-#define SRC_CORE_NEON_KERNELS_LUT_LIST_H
+#ifndef ACL_SRC_CPU_KERNELS_LUT_LIST_H
+#define ACL_SRC_CPU_KERNELS_LUT_LIST_H
 
 #include <cstddef>
 #include <cstdint>
@@ -34,17 +34,27 @@ namespace cpu
 {
 
 #ifdef __aarch64__
-#define DECLARE_LUT_KERNEL(func_name)                                                                           \
+#define DECLARE_LUT_U8_KERNEL(func_name)                                                                        \
     void func_name(const uint8_t *table, size_t num_strings, size_t string_length, const uint8_t *const *input, \
                    uint8_t *const *output)
 
-DECLARE_LUT_KERNEL(lut_u8_neon);
-DECLARE_LUT_KERNEL(lut_u8_sve2);
+DECLARE_LUT_U8_KERNEL(lut_u8_neon);
+DECLARE_LUT_U8_KERNEL(lut_u8_sve2);
+
+#undef DECLARE_LUT_U8_KERNEL
+
+#define DECLARE_LUT_U16_KERNEL(func_name)                                                                  \
+    void func_name(const uint16_t *table, size_t num_strings, size_t string_length, const uint16_t *input, \
+                   uint16_t *output)
+
+DECLARE_LUT_U16_KERNEL(lut_u16_neon);
+DECLARE_LUT_U16_KERNEL(lut_u16_sve);
+
+#undef DECLARE_LUT_U16_KERNEL
 
-#undef DECLARE_LUT_KERNEL
 #endif // __aarch64__
 
 } // namespace cpu
 } // namespace arm_compute
 
-#endif // SRC_CORE_NEON_KERNELS_LUT_LIST_H
+#endif // ACL_SRC_CPU_KERNELS_LUT_LIST_H