From 579c0498e161215be1a36080b0b454e5198a992a Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Wed, 12 Jul 2017 16:12:12 +0100 Subject: COMPMID-417: Add Leaky RELU support for both NEON/CL. -Adds parametrizable leaky relu (x>0) ? x : a*x. Change-Id: Ief19a435b5832a30b56f4aaaf55125787addee94 Reviewed-on: http://mpd-gerrit.cambridge.arm.com/80575 Reviewed-by: Anthony Barbier Tested-by: Kaizen --- src/core/NEON/kernels/NEActivationLayerKernel.cpp | 72 +++++++++++++++-------- 1 file changed, 49 insertions(+), 23 deletions(-) (limited to 'src/core/NEON/kernels/NEActivationLayerKernel.cpp') diff --git a/src/core/NEON/kernels/NEActivationLayerKernel.cpp b/src/core/NEON/kernels/NEActivationLayerKernel.cpp index f530413453..70b7057fcd 100644 --- a/src/core/NEON/kernels/NEActivationLayerKernel.cpp +++ b/src/core/NEON/kernels/NEActivationLayerKernel.cpp @@ -73,6 +73,7 @@ void NEActivationLayerKernel::configure(ITensor *input, ITensor *output, Activat { ActivationFunction::LOGISTIC, &NEActivationLayerKernel::activation }, { ActivationFunction::RELU, &NEActivationLayerKernel::activation }, { ActivationFunction::BOUNDED_RELU, &NEActivationLayerKernel::activation }, + { ActivationFunction::LEAKY_RELU, &NEActivationLayerKernel::activation }, { ActivationFunction::SOFT_RELU, &NEActivationLayerKernel::activation }, { ActivationFunction::SQRT, &NEActivationLayerKernel::activation }, { ActivationFunction::SQUARE, &NEActivationLayerKernel::activation }, @@ -86,6 +87,7 @@ void NEActivationLayerKernel::configure(ITensor *input, ITensor *output, Activat { ActivationFunction::LOGISTIC, &NEActivationLayerKernel::activation }, { ActivationFunction::RELU, &NEActivationLayerKernel::activation }, { ActivationFunction::BOUNDED_RELU, &NEActivationLayerKernel::activation }, + { ActivationFunction::LEAKY_RELU, &NEActivationLayerKernel::activation }, { ActivationFunction::SOFT_RELU, &NEActivationLayerKernel::activation }, { ActivationFunction::SQRT, &NEActivationLayerKernel::activation }, { ActivationFunction::SQUARE, &NEActivationLayerKernel::activation }, @@ -99,6 +101,7 @@ void NEActivationLayerKernel::configure(ITensor *input, ITensor *output, Activat { ActivationFunction::LOGISTIC, &NEActivationLayerKernel::activation }, { ActivationFunction::RELU, &NEActivationLayerKernel::activation }, { ActivationFunction::BOUNDED_RELU, &NEActivationLayerKernel::activation }, + { ActivationFunction::LEAKY_RELU, &NEActivationLayerKernel::activation }, { ActivationFunction::SOFT_RELU, &NEActivationLayerKernel::activation }, { ActivationFunction::SQRT, &NEActivationLayerKernel::activation }, { ActivationFunction::SQUARE, &NEActivationLayerKernel::activation }, @@ -177,17 +180,6 @@ typename std::enable_if::value, void>::type NEActivationL } }; break; - case ActivationFunction::BOUNDED_RELU: - tmp = - { - { - vminq_f32(a, vmaxq_f32(CONST_0, in.val[0])), - vminq_f32(a, vmaxq_f32(CONST_0, in.val[1])), - vminq_f32(a, vmaxq_f32(CONST_0, in.val[2])), - vminq_f32(a, vmaxq_f32(CONST_0, in.val[3])), - } - }; - break; case ActivationFunction::LINEAR: tmp = { @@ -221,6 +213,28 @@ typename std::enable_if::value, void>::type NEActivationL } }; break; + case ActivationFunction::BOUNDED_RELU: + tmp = + { + { + vminq_f32(a, vmaxq_f32(CONST_0, in.val[0])), + vminq_f32(a, vmaxq_f32(CONST_0, in.val[1])), + vminq_f32(a, vmaxq_f32(CONST_0, in.val[2])), + vminq_f32(a, vmaxq_f32(CONST_0, in.val[3])), + } + }; + break; + case ActivationFunction::LEAKY_RELU: + tmp = + { + { + vbslq_f32(vcgtq_f32(in.val[0], CONST_0), in.val[0], vmulq_f32(a, in.val[0])), + vbslq_f32(vcgtq_f32(in.val[1], CONST_0), in.val[1], vmulq_f32(a, in.val[1])), + vbslq_f32(vcgtq_f32(in.val[2], CONST_0), in.val[2], vmulq_f32(a, in.val[2])), + vbslq_f32(vcgtq_f32(in.val[3], CONST_0), in.val[3], vmulq_f32(a, in.val[3])), + } + }; + break; case ActivationFunction::SOFT_RELU: tmp = { @@ -299,9 +313,6 @@ typename std::enable_if::value, void>::type NEActivation case ActivationFunction::ABS: tmp = vqabsq_qs8(in); break; - case ActivationFunction::BOUNDED_RELU: - tmp = vminq_qs8(a, vmaxq_qs8(CONST_0, in)); - break; case ActivationFunction::LINEAR: tmp = vqmlaq_qs8(b, a, in, fixed_point_position); break; @@ -311,6 +322,12 @@ typename std::enable_if::value, void>::type NEActivation case ActivationFunction::RELU: tmp = vmaxq_qs8(CONST_0, in); break; + case ActivationFunction::BOUNDED_RELU: + tmp = vminq_qs8(a, vmaxq_qs8(CONST_0, in)); + break; + case ActivationFunction::LEAKY_RELU: + tmp = vbslq_s8(vcgtq_s8(in, CONST_0), in, vmulq_qs8(a, in, fixed_point_position)); + break; case ActivationFunction::SOFT_RELU: tmp = vlogq_qs8(vqaddq_qs8(CONST_1, vqexpq_qs8(in, fixed_point_position)), fixed_point_position); break; @@ -363,15 +380,6 @@ typename std::enable_if::value, void>::type NEActivatio } }; break; - case ActivationFunction::BOUNDED_RELU: - tmp = - { - { - vminq_qs16(a, vmaxq_qs16(CONST_0, in.val[0])), - vminq_qs16(a, vmaxq_qs16(CONST_0, in.val[1])), - } - }; - break; case ActivationFunction::LINEAR: tmp = { @@ -399,6 +407,24 @@ typename std::enable_if::value, void>::type NEActivatio } }; break; + case ActivationFunction::BOUNDED_RELU: + tmp = + { + { + vminq_qs16(a, vmaxq_qs16(CONST_0, in.val[0])), + vminq_qs16(a, vmaxq_qs16(CONST_0, in.val[1])), + } + }; + break; + case ActivationFunction::LEAKY_RELU: + tmp = + { + { + vbslq_s16(vcgtq_s16(in.val[0], CONST_0), in.val[0], vmulq_qs16(a, in.val[0], fixed_point_position)), + vbslq_s16(vcgtq_s16(in.val[1], CONST_0), in.val[1], vmulq_qs16(a, in.val[1], fixed_point_position)), + } + }; + break; case ActivationFunction::SOFT_RELU: tmp = { -- cgit v1.2.1