From d1dc09c95602ec1506bb4934aed1792752b5ffcf Mon Sep 17 00:00:00 2001 From: Teresa Charlin Date: Thu, 4 Mar 2021 15:24:45 +0000 Subject: Port CpuTranspose to new API Partially Resolves: COMPMID-4277 (2/2) Signed-off-by: Teresa Charlin Change-Id: Id8ee520081fe905cb796d4376864fa84ac384caa Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/c/VisualCompute/ComputeLibrary/+/303714 Tested-by: bsgcomp Reviewed-by: Sang-Hoon Park Reviewed-by: Georgios Pinitas Comments-Addressed: bsgcomp Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/5217 Tested-by: Arm Jenkins Reviewed-by: Manuel Bottini Comments-Addressed: Arm Jenkins --- Android.bp | 3 +- .../runtime/NEON/functions/NEFullyConnectedLayer.h | 14 +- arm_compute/runtime/NEON/functions/NEPermute.h | 4 +- arm_compute/runtime/NEON/functions/NETranspose.h | 36 +- docs/00_introduction.dox | 4 +- src/core/NEON/NEKernels.h | 1 - src/core/NEON/kernels/NETransposeKernel.cpp | 517 --------------------- src/core/NEON/kernels/NETransposeKernel.h | 90 ---- src/core/cpu/kernels/CpuPermuteKernel.cpp | 2 +- src/core/cpu/kernels/CpuTransposeKernel.cpp | 510 ++++++++++++++++++++ src/core/cpu/kernels/CpuTransposeKernel.h | 64 +++ .../NEON/functions/NEFullyConnectedLayer.cpp | 37 +- src/runtime/NEON/functions/NEPermute.cpp | 4 - src/runtime/NEON/functions/NETranspose.cpp | 43 +- src/runtime/cpu/operators/CpuPermute.h | 2 +- src/runtime/cpu/operators/CpuTranspose.cpp | 44 ++ src/runtime/cpu/operators/CpuTranspose.h | 56 +++ 17 files changed, 784 insertions(+), 647 deletions(-) delete mode 100644 src/core/NEON/kernels/NETransposeKernel.cpp delete mode 100644 src/core/NEON/kernels/NETransposeKernel.h create mode 100644 src/core/cpu/kernels/CpuTransposeKernel.cpp create mode 100644 src/core/cpu/kernels/CpuTransposeKernel.h create mode 100644 src/runtime/cpu/operators/CpuTranspose.cpp create mode 100644 src/runtime/cpu/operators/CpuTranspose.h diff --git a/Android.bp b/Android.bp index 6dce0e40bc..94c952c808 100644 --- a/Android.bp +++ b/Android.bp @@ -238,7 +238,6 @@ cc_library_static { "src/core/NEON/kernels/NEStackLayerKernel.cpp", "src/core/NEON/kernels/NEStridedSliceKernel.cpp", "src/core/NEON/kernels/NETileKernel.cpp", - "src/core/NEON/kernels/NETransposeKernel.cpp", "src/core/NEON/kernels/NEWeightsReshapeKernel.cpp", "src/core/NEON/kernels/NEWinogradConvolutionLayerKernel.cpp", "src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst/generic.cpp", @@ -332,6 +331,7 @@ cc_library_static { "src/core/cpu/kernels/CpuScaleKernel.cpp", "src/core/cpu/kernels/CpuSoftmaxKernel.cpp", "src/core/cpu/kernels/CpuSubKernel.cpp", + "src/core/cpu/kernels/CpuTransposeKernel.cpp", "src/core/cpu/kernels/activation/NEON/fp16.cpp", "src/core/cpu/kernels/activation/NEON/fp32.cpp", "src/core/cpu/kernels/activation/NEON/qasymm8.cpp", @@ -670,6 +670,7 @@ cc_library_static { "src/runtime/cpu/operators/CpuScale.cpp", "src/runtime/cpu/operators/CpuSoftmax.cpp", "src/runtime/cpu/operators/CpuSub.cpp", + "src/runtime/cpu/operators/CpuTranspose.cpp", "src/runtime/gpu/cl/operators/ClActivation.cpp", "src/runtime/gpu/cl/operators/ClAdd.cpp", "src/runtime/gpu/cl/operators/ClConcatenate.cpp", diff --git a/arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h b/arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h index ffea02670f..1b3f36d866 100644 --- a/arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h +++ b/arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h @@ -25,7 +25,6 @@ #define ARM_COMPUTE_NEFULLYCONNECTEDLAYER_H #include "arm_compute/runtime/IFunction.h" -#include "arm_compute/runtime/NEON/INESimpleFunctionNoBorder.h" #include "arm_compute/runtime/MemoryGroup.h" #include "arm_compute/runtime/NEON/functions/NEConvertFullyConnectedWeights.h" @@ -40,11 +39,11 @@ namespace arm_compute * * @note The fully connected layer accepts "weights" tensors only with 2 dimensions. */ -class NEFullyConnectedLayerReshapeWeights : public INESimpleFunctionNoBorder +class NEFullyConnectedLayerReshapeWeights : public IFunction { public: /** Constructor */ - NEFullyConnectedLayerReshapeWeights() = default; + NEFullyConnectedLayerReshapeWeights(); /** Prevent instances of this class from being copied (As this class contains pointers) */ NEFullyConnectedLayerReshapeWeights(const NEFullyConnectedLayerReshapeWeights &) = delete; /** Prevent instances of this class from being copied (As this class contains pointers) */ @@ -54,7 +53,7 @@ public: /** Prevent instances of this class from being moved (As this class contains non movable objects) */ NEFullyConnectedLayerReshapeWeights &operator=(NEFullyConnectedLayerReshapeWeights &&) = delete; /** Default destructor */ - ~NEFullyConnectedLayerReshapeWeights() = default; + ~NEFullyConnectedLayerReshapeWeights(); /** Set the input and output tensors. * * @param[in] input Weights tensor. The weights must be 2 dimensional. Data types supported: QASYMM8/QASYMM8_SIGNED/F16/F32. @@ -69,6 +68,13 @@ public: * @return a status */ static Status validate(const ITensorInfo *input, const ITensorInfo *output); + + // Inherited methods overridden + void run() override; + +private: + struct Impl; + std::unique_ptr _impl; }; namespace weights_transformations diff --git a/arm_compute/runtime/NEON/functions/NEPermute.h b/arm_compute/runtime/NEON/functions/NEPermute.h index cf7e25213b..fb95e45bdb 100644 --- a/arm_compute/runtime/NEON/functions/NEPermute.h +++ b/arm_compute/runtime/NEON/functions/NEPermute.h @@ -47,11 +47,11 @@ public: /** Prevent instances of this class from being copied (As this class contains pointers) */ NEPermute(const NEPermute &) = delete; /** Default move constructor */ - NEPermute(NEPermute &&); + NEPermute(NEPermute &&) = default; /** Prevent instances of this class from being copied (As this class contains pointers) */ NEPermute &operator=(const NEPermute &) = delete; /** Default move assignment operator */ - NEPermute &operator=(NEPermute &&); + NEPermute &operator=(NEPermute &&) = default; /** Configure the permute Neon kernel * * @note Arbitrary permutation vectors are supported with rank not greater than 4 diff --git a/arm_compute/runtime/NEON/functions/NETranspose.h b/arm_compute/runtime/NEON/functions/NETranspose.h index fac1d406fb..78916f67b7 100644 --- a/arm_compute/runtime/NEON/functions/NETranspose.h +++ b/arm_compute/runtime/NEON/functions/NETranspose.h @@ -24,22 +24,34 @@ #ifndef ARM_COMPUTE_NETRANSPOSE_H #define ARM_COMPUTE_NETRANSPOSE_H +#include "arm_compute/runtime/IFunction.h" + #include "arm_compute/core/Types.h" -#include "arm_compute/runtime/NEON/INESimpleFunctionNoBorder.h" + +#include namespace arm_compute { +// Forward declarations class ITensor; class ITensorInfo; -/** Basic function to transpose a matrix on Neon. This function calls the following Neon kernel: - * - * -# @ref NETransposeKernel - * - */ -class NETranspose : public INESimpleFunctionNoBorder +/** Basic function to run @ref cpu::kernels::CpuTransposeKernel */ +class NETranspose : public IFunction { public: + /** Default Constructor */ + NETranspose(); + /** Default Destructor */ + ~NETranspose(); + /** Prevent instances of this class from being copied (As this class contains pointers) */ + NETranspose(const NETranspose &) = delete; + /** Default move constructor */ + NETranspose(NETranspose &&) = default; + /** Prevent instances of this class from being copied (As this class contains pointers) */ + NETranspose &operator=(const NETranspose &) = delete; + /** Default move assignment operator */ + NETranspose &operator=(NETranspose &&) = default; /** Initialise the kernel's inputs and output * * @param[in] input Input tensor. Data types supported: All @@ -54,7 +66,13 @@ public: * @return a status */ static Status validate(const ITensorInfo *input, const ITensorInfo *output); + + // Inherited methods overridden + void run() override; + +private: + struct Impl; + std::unique_ptr _impl; }; } // namespace arm_compute - -#endif /* ARM_COMPUTE_NETRANSPOSE_H */ +#endif /* ARM_COMPUTE_NETRANSPOSE_H */ \ No newline at end of file diff --git a/docs/00_introduction.dox b/docs/00_introduction.dox index d877236c66..af840ec2b8 100644 --- a/docs/00_introduction.dox +++ b/docs/00_introduction.dox @@ -552,7 +552,7 @@ v20.08 Public major release - NEHeightConcatenateLayerKernel - NEThresholdKernel - NEBatchConcatenateLayerKernel - - @ref NETransposeKernel + - NETransposeKernel - @ref NEBatchNormalizationLayerKernel - NEArithmeticSubtractionKernel - @ref NEBoundingBoxTransformKernel @@ -1379,7 +1379,7 @@ v17.03.1 First Major public release of the sources - CLGEMMLowpMatrixMultiplyKernel / CLGEMMLowp - New Neon kernels / functions: - @ref NENormalizationLayerKernel / @ref NENormalizationLayer - - @ref NETransposeKernel / @ref NETranspose + - NETransposeKernel / @ref NETranspose - NELogits1DMaxKernel, NELogits1DShiftExpSumKernel, NELogits1DNormKernel / @ref NESoftmaxLayer - @ref NEIm2ColKernel, @ref NECol2ImKernel, NEConvolutionLayerWeightsReshapeKernel / @ref NEConvolutionLayer - NEGEMMMatrixAccumulateBiasesKernel / @ref NEFullyConnectedLayer diff --git a/src/core/NEON/NEKernels.h b/src/core/NEON/NEKernels.h index 973c0f2d31..53e02261f1 100644 --- a/src/core/NEON/NEKernels.h +++ b/src/core/NEON/NEKernels.h @@ -88,7 +88,6 @@ #include "src/core/NEON/kernels/NEStackLayerKernel.h" #include "src/core/NEON/kernels/NEStridedSliceKernel.h" #include "src/core/NEON/kernels/NETileKernel.h" -#include "src/core/NEON/kernels/NETransposeKernel.h" #include "src/core/NEON/kernels/NEWeightsReshapeKernel.h" #include "src/core/NEON/kernels/NEWinogradConvolutionLayerKernel.h" diff --git a/src/core/NEON/kernels/NETransposeKernel.cpp b/src/core/NEON/kernels/NETransposeKernel.cpp deleted file mode 100644 index cd4ae52873..0000000000 --- a/src/core/NEON/kernels/NETransposeKernel.cpp +++ /dev/null @@ -1,517 +0,0 @@ -/* - * Copyright (c) 2017-2021 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#include "src/core/NEON/kernels/NETransposeKernel.h" - -#include "arm_compute/core/Error.h" -#include "arm_compute/core/Helpers.h" -#include "arm_compute/core/ITensor.h" -#include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" -#include "arm_compute/core/Validate.h" -#include "src/core/AccessWindowStatic.h" -#include "src/core/AccessWindowTranspose.h" -#include "src/core/helpers/AutoConfiguration.h" -#include "src/core/helpers/WindowHelpers.h" - -#include - -using namespace arm_compute; - -namespace arm_compute -{ -class Coordinates; -} // namespace arm_compute - -namespace -{ -TensorShape transposed_tensor_shape(const TensorShape &in) -{ - TensorShape output_shape{ in }; - const size_t w_out = in[1]; - const size_t h_out = in[0]; - output_shape.set(0, w_out); - output_shape.set(1, h_out); - - return output_shape; -} - -Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output) -{ - ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input); - //Note: ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(input) is not needed here as this kernel doesn't use Neon FP16 instructions. - ARM_COMPUTE_RETURN_ERROR_ON(input->data_type() == DataType::UNKNOWN); - - if(output->total_size() != 0) - { - const TensorInfo tensor_info = input->clone()->set_tensor_shape(transposed_tensor_shape(input->tensor_shape())); - - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(output, &tensor_info); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_QUANTIZATION_INFO(input, output); - } - - return Status{}; -} -unsigned int num_elems_processed(size_t element_size) -{ - switch(element_size) - { - case 1: - return 8; - case 2: - case 4: - return 4; - default: - break; - } - - ARM_COMPUTE_ERROR("Element size not supported"); -} - -void transpose_8bit_elements(const ITensor *in, ITensor *out, const Window &window) -{ - const int window_step_x = 8; - const int window_step_y = 8; - const int window_start_x = window.x().start(); - const int window_end_x = window.x().end(); - const int window_start_y = window.y().start(); - const int window_end_y = std::min(window.y().end(), static_cast(in->info()->dimension(1))); - const int window_end_y_multiple_of = ((window_end_y - window_start_y) / window_step_y) * window_step_y; - const size_t input_stride_in_bytes = in->info()->strides_in_bytes()[1]; - const size_t output_stride_in_bytes = out->info()->strides_in_bytes()[1]; - - // Check if we need a left-over loop for the y dimension - bool left_over_loop_y = (((window_end_y - window_start_y) % window_step_y) != 0); - - Window window_in(window); - window_in.set(Window::DimX, Window::Dimension(0, 1, 1)); - if(left_over_loop_y) - { - // Check if window_end_y_multiple_of is greater than window_start_y - if(window_end_y_multiple_of > window_start_y) - { - window_in.set(Window::DimY, Window::Dimension(window_start_y, window_end_y_multiple_of, window_step_y)); - } - else - { - window_in.set(Window::DimY, Window::Dimension(0, 0, 1)); - } - } - - Window window_out(window); - window_out.set(Window::DimX, Window::Dimension(0, 0, 0)); - window_out.set(Window::DimY, Window::Dimension(0, 0, 0)); - - Iterator output(out, window_out); - - // Run the Neon path if and only if the input is not a row-vector - if(in->info()->dimension(1) != 1) - { - Iterator input(in, window_in); - execute_window_loop(window_in, [&](const Coordinates & id) - { - // Compute 8x8 elements per iteration - int x = window_start_x; - for(; x <= (window_end_x - window_step_x); x += window_step_x) - { - const uint8x8_t row0 = vld1_u8(reinterpret_cast(input.ptr() + x + 0 * input_stride_in_bytes)); - const uint8x8_t row1 = vld1_u8(reinterpret_cast(input.ptr() + x + 1 * input_stride_in_bytes)); - const uint8x8_t row2 = vld1_u8(reinterpret_cast(input.ptr() + x + 2 * input_stride_in_bytes)); - const uint8x8_t row3 = vld1_u8(reinterpret_cast(input.ptr() + x + 3 * input_stride_in_bytes)); - const uint8x8_t row4 = vld1_u8(reinterpret_cast(input.ptr() + x + 4 * input_stride_in_bytes)); - const uint8x8_t row5 = vld1_u8(reinterpret_cast(input.ptr() + x + 5 * input_stride_in_bytes)); - const uint8x8_t row6 = vld1_u8(reinterpret_cast(input.ptr() + x + 6 * input_stride_in_bytes)); - const uint8x8_t row7 = vld1_u8(reinterpret_cast(input.ptr() + x + 7 * input_stride_in_bytes)); - - // Transpose 2x2 - const uint8x8x2_t k0_u8 = vtrn_u8(row0, row1); - const uint8x8x2_t k1_u8 = vtrn_u8(row2, row3); - const uint8x8x2_t k2_u8 = vtrn_u8(row4, row5); - const uint8x8x2_t k3_u8 = vtrn_u8(row6, row7); - - // Transpose 4x4 - const uint16x4x2_t k0_u16 = vtrn_u16(vreinterpret_u16_u8(k0_u8.val[0]), vreinterpret_u16_u8(k1_u8.val[0])); - const uint16x4x2_t k1_u16 = vtrn_u16(vreinterpret_u16_u8(k0_u8.val[1]), vreinterpret_u16_u8(k1_u8.val[1])); - const uint16x4x2_t k2_u16 = vtrn_u16(vreinterpret_u16_u8(k2_u8.val[0]), vreinterpret_u16_u8(k3_u8.val[0])); - const uint16x4x2_t k3_u16 = vtrn_u16(vreinterpret_u16_u8(k2_u8.val[1]), vreinterpret_u16_u8(k3_u8.val[1])); - - // Transpose 8x8 - const uint32x2x2_t k0_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[0]), vreinterpret_u32_u16(k2_u16.val[0])); - const uint32x2x2_t k1_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[1]), vreinterpret_u32_u16(k2_u16.val[1])); - const uint32x2x2_t k2_u32 = vtrn_u32(vreinterpret_u32_u16(k1_u16.val[0]), vreinterpret_u32_u16(k3_u16.val[0])); - const uint32x2x2_t k3_u32 = vtrn_u32(vreinterpret_u32_u16(k1_u16.val[1]), vreinterpret_u32_u16(k3_u16.val[1])); - - // Compute destination address - const size_t dst_offset_in_bytes = id.y() * sizeof(uint8_t) + x * output_stride_in_bytes; - - vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 0 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k0_u32.val[0]))); - vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 1 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k2_u32.val[0]))); - vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 2 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k1_u32.val[0]))); - vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 3 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k3_u32.val[0]))); - vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 4 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k0_u32.val[1]))); - vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 5 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k2_u32.val[1]))); - vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 6 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k1_u32.val[1]))); - vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 7 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k3_u32.val[1]))); - } - - // Compute left-over elements along the x dimension (1x8) - for(; x < window_end_x; ++x) - { - const uint8_t val0 = *(input.ptr() + x + 0 * input_stride_in_bytes); - const uint8_t val1 = *(input.ptr() + x + 1 * input_stride_in_bytes); - const uint8_t val2 = *(input.ptr() + x + 2 * input_stride_in_bytes); - const uint8_t val3 = *(input.ptr() + x + 3 * input_stride_in_bytes); - const uint8_t val4 = *(input.ptr() + x + 4 * input_stride_in_bytes); - const uint8_t val5 = *(input.ptr() + x + 5 * input_stride_in_bytes); - const uint8_t val6 = *(input.ptr() + x + 6 * input_stride_in_bytes); - const uint8_t val7 = *(input.ptr() + x + 7 * input_stride_in_bytes); - - uint8x8_t result = vdup_n_u8(0); - result = vset_lane_u8(val0, result, 0); - result = vset_lane_u8(val1, result, 1); - result = vset_lane_u8(val2, result, 2); - result = vset_lane_u8(val3, result, 3); - result = vset_lane_u8(val4, result, 4); - result = vset_lane_u8(val5, result, 5); - result = vset_lane_u8(val6, result, 6); - result = vset_lane_u8(val7, result, 7); - - // Compute destination address - const size_t dst_offset_in_bytes = id.y() * sizeof(uint8_t) + x * output_stride_in_bytes; - - vst1_u8(output.ptr() + dst_offset_in_bytes, result); - } - }, - input, output); - } - - if(left_over_loop_y) - { - window_in.set(Window::DimX, Window::Dimension(window.x().start(), window.x().end(), 1)); - window_in.set(Window::DimY, Window::Dimension(window_end_y_multiple_of, window_end_y, 1)); - - Iterator input(in, window_in); - Iterator output(out, window_out); - - // Compute left-over elements along the y dimension (1x1) - execute_window_loop(window_in, [&](const Coordinates & id) - { - const uint8_t val0 = *input.ptr(); - - // Compute destination address - const size_t dst_offset_in_bytes = id.y() * sizeof(uint8_t) + id.x() * output_stride_in_bytes; - - *(output.ptr() + dst_offset_in_bytes) = val0; - }, - input, output); - } -} - -void transpose_16bit_elements(const ITensor *in, ITensor *out, const Window &window) -{ - const int window_step_x = 4; - const int window_step_y = 4; - const int window_start_x = window.x().start(); - const int window_end_x = window.x().end(); - const int window_start_y = window.y().start(); - const int window_end_y = std::min(window.y().end(), static_cast(in->info()->dimension(1))); - const int window_end_y_multiple_of = ((window_end_y - window_start_y) / window_step_y) * window_step_y; - const size_t input_stride_in_bytes = in->info()->strides_in_bytes()[1]; - const size_t output_stride_in_bytes = out->info()->strides_in_bytes()[1]; - - // Check if we need a left-over loop for the y dimension - bool left_over_loop_y = (((window_end_y - window_start_y) % window_step_y) != 0); - - Window window_in(window); - window_in.set(Window::DimX, Window::Dimension(0, 1, 1)); - if(left_over_loop_y) - { - // Check if window_end_y_multiple_of is greater than window_start_y - if(window_end_y_multiple_of > window_start_y) - { - window_in.set(Window::DimY, Window::Dimension(window_start_y, window_end_y_multiple_of, window_step_y)); - } - else - { - window_in.set(Window::DimY, Window::Dimension(0, 0, 1)); - } - } - - Window window_out(window); - window_out.set(Window::DimX, Window::Dimension(0, 0, 0)); - window_out.set(Window::DimY, Window::Dimension(0, 0, 0)); - - Iterator output(out, window_out); - - // Run the Neon path if and only if the input is not a row-vector - if(in->info()->dimension(1) != 1) - { - Iterator input(in, window_in); - execute_window_loop(window_in, [&](const Coordinates & id) - { - // Compute 4x4 elements per iteration - int x = window_start_x; - for(; x <= (window_end_x - window_step_x); x += window_step_x) - { - const uint16x4_t row0 = vld1_u16(reinterpret_cast(input.ptr() + 0 * input_stride_in_bytes) + x); - const uint16x4_t row1 = vld1_u16(reinterpret_cast(input.ptr() + 1 * input_stride_in_bytes) + x); - const uint16x4_t row2 = vld1_u16(reinterpret_cast(input.ptr() + 2 * input_stride_in_bytes) + x); - const uint16x4_t row3 = vld1_u16(reinterpret_cast(input.ptr() + 3 * input_stride_in_bytes) + x); - - // Transpose 2x2 - const uint16x4x2_t k0_u16 = vtrn_u16(row0, row1); - const uint16x4x2_t k1_u16 = vtrn_u16(row2, row3); - - // Transpose 4x4 - const uint32x2x2_t k0_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[0]), vreinterpret_u32_u16(k1_u16.val[0])); - const uint32x2x2_t k1_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[1]), vreinterpret_u32_u16(k1_u16.val[1])); - - // Compute destination address - const size_t dst_offset_in_bytes = id.y() * sizeof(uint16_t) + x * output_stride_in_bytes; - - vst1_u16(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 0 * output_stride_in_bytes), vreinterpret_u16_u32(k0_u32.val[0])); - vst1_u16(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 1 * output_stride_in_bytes), vreinterpret_u16_u32(k1_u32.val[0])); - vst1_u16(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 2 * output_stride_in_bytes), vreinterpret_u16_u32(k0_u32.val[1])); - vst1_u16(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 3 * output_stride_in_bytes), vreinterpret_u16_u32(k1_u32.val[1])); - } - - // Compute left-over elements (1x4) - for(; x < window_end_x; ++x) - { - const uint16_t val0 = *(reinterpret_cast(input.ptr() + 0 * input_stride_in_bytes) + x); - const uint16_t val1 = *(reinterpret_cast(input.ptr() + 1 * input_stride_in_bytes) + x); - const uint16_t val2 = *(reinterpret_cast(input.ptr() + 2 * input_stride_in_bytes) + x); - const uint16_t val3 = *(reinterpret_cast(input.ptr() + 3 * input_stride_in_bytes) + x); - - uint16x4_t result = vdup_n_u16(0); - result = vset_lane_u16(val0, result, 0); - result = vset_lane_u16(val1, result, 1); - result = vset_lane_u16(val2, result, 2); - result = vset_lane_u16(val3, result, 3); - - // Compute destination address - const size_t dst_offset_in_bytes = id.y() * sizeof(uint16_t) + x * output_stride_in_bytes; - - vst1_u16(reinterpret_cast(output.ptr() + dst_offset_in_bytes), result); - } - }, - input, output); - } - - if(left_over_loop_y) - { - window_in.set(Window::DimX, Window::Dimension(window.x().start(), window.x().end(), 1)); - window_in.set(Window::DimY, Window::Dimension(window_end_y_multiple_of, window_end_y, 1)); - - Iterator input(in, window_in); - Iterator output(out, window_out); - - // Compute left-over elements along the y dimension (1x1) - execute_window_loop(window_in, [&](const Coordinates & id) - { - const uint16_t val0 = *(reinterpret_cast(input.ptr())); - - // Compute destination address - const size_t dst_offset_in_bytes = id.y() * sizeof(uint16_t) + id.x() * output_stride_in_bytes; - - *(reinterpret_cast(output.ptr() + dst_offset_in_bytes)) = val0; - }, - input, output); - } -} - -void transpose_32bit_elements(const ITensor *in, ITensor *out, const Window &window) -{ - const int window_step_x = 4; - const int window_step_y = 4; - const int window_start_x = window.x().start(); - const int window_end_x = window.x().end(); - const int window_start_y = window.y().start(); - const int window_end_y = std::min(window.y().end(), static_cast(in->info()->dimension(1))); - const int window_end_y_multiple_of = ((window_end_y - window_start_y) / window_step_y) * window_step_y; - const size_t input_stride_in_bytes = in->info()->strides_in_bytes()[1]; - const size_t output_stride_in_bytes = out->info()->strides_in_bytes()[1]; - - // Check if we need a left-over loop for the y dimension - bool left_over_loop_y = (((window_end_y - window_start_y) % window_step_y) != 0); - - Window window_in(window); - window_in.set(Window::DimX, Window::Dimension(0, 1, 1)); - if(left_over_loop_y) - { - // Check if window_end_y_multiple_of is greater than window_start_y - if(window_end_y_multiple_of > window_start_y) - { - window_in.set(Window::DimY, Window::Dimension(window_start_y, window_end_y_multiple_of, window_step_y)); - } - else - { - window_in.set(Window::DimY, Window::Dimension(0, 0, 1)); - } - } - - Window window_out(window); - window_out.set(Window::DimX, Window::Dimension(0, 0, 0)); - window_out.set(Window::DimY, Window::Dimension(0, 0, 0)); - - Iterator output(out, window_out); - - // Run the Neon path if and only if the input is not a row-vector - if(in->info()->dimension(1) != 1) - { - Iterator input(in, window_in); - execute_window_loop(window_in, [&](const Coordinates & id) - { - // Compute 4x4 elements per iteration - int x = window_start_x; - for(; x <= (window_end_x - window_step_x); x += window_step_x) - { - const uint32x4_t row0 = vld1q_u32(reinterpret_cast(input.ptr() + 0 * input_stride_in_bytes) + x); - const uint32x4_t row1 = vld1q_u32(reinterpret_cast(input.ptr() + 1 * input_stride_in_bytes) + x); - const uint32x4_t row2 = vld1q_u32(reinterpret_cast(input.ptr() + 2 * input_stride_in_bytes) + x); - const uint32x4_t row3 = vld1q_u32(reinterpret_cast(input.ptr() + 3 * input_stride_in_bytes) + x); - - // Transpose 2x2 - const uint32x2x2_t k0_u32 = vtrn_u32(vget_low_u32(row0), vget_low_u32(row1)); - const uint32x2x2_t k1_u32 = vtrn_u32(vget_high_u32(row2), vget_high_u32(row3)); - const uint32x2x2_t k2_u32 = vtrn_u32(vget_high_u32(row0), vget_high_u32(row1)); - const uint32x2x2_t k3_u32 = vtrn_u32(vget_low_u32(row2), vget_low_u32(row3)); - - // Compute destination address - const size_t dst_offset_in_bytes = id.y() * sizeof(uint32_t) + x * output_stride_in_bytes; - - // Swap block 01 with block 10 and store - vst1q_u32(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 0 * output_stride_in_bytes), vcombine_u32(k0_u32.val[0], k3_u32.val[0])); - vst1q_u32(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 1 * output_stride_in_bytes), vcombine_u32(k0_u32.val[1], k3_u32.val[1])); - vst1q_u32(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 2 * output_stride_in_bytes), vcombine_u32(k2_u32.val[0], k1_u32.val[0])); - vst1q_u32(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 3 * output_stride_in_bytes), vcombine_u32(k2_u32.val[1], k1_u32.val[1])); - } - - // Compute left-over elements (1x4) - for(; x < window_end_x; ++x) - { - const uint32_t val0 = *(reinterpret_cast(input.ptr() + 0 * input_stride_in_bytes) + x); - const uint32_t val1 = *(reinterpret_cast(input.ptr() + 1 * input_stride_in_bytes) + x); - const uint32_t val2 = *(reinterpret_cast(input.ptr() + 2 * input_stride_in_bytes) + x); - const uint32_t val3 = *(reinterpret_cast(input.ptr() + 3 * input_stride_in_bytes) + x); - - uint32x4_t result = vdupq_n_u32(0); - result = vsetq_lane_u32(val0, result, 0); - result = vsetq_lane_u32(val1, result, 1); - result = vsetq_lane_u32(val2, result, 2); - result = vsetq_lane_u32(val3, result, 3); - - // Compute destination address - const size_t dst_offset_in_bytes = id.y() * sizeof(uint32_t) + x * output_stride_in_bytes; - - vst1q_u32(reinterpret_cast(output.ptr() + dst_offset_in_bytes), result); - } - }, - input, output); - } - - if(left_over_loop_y) - { - window_in.set(Window::DimX, Window::Dimension(window.x().start(), window.x().end(), 1)); - window_in.set(Window::DimY, Window::Dimension(window_end_y_multiple_of, window_end_y, 1)); - - Iterator input(in, window_in); - Iterator output(out, window_out); - - // Compute left-over elements along the y dimension (1x1) - execute_window_loop(window_in, [&](const Coordinates & id) - { - const uint32_t val0 = *(reinterpret_cast(input.ptr())); - - // Compute destination address - const size_t dst_offset_in_bytes = id.y() * sizeof(uint32_t) + id.x() * output_stride_in_bytes; - - *(reinterpret_cast(output.ptr() + dst_offset_in_bytes)) = val0; - }, - input, output); - } -} -} // namespace - -Status NETransposeKernel::validate(const ITensorInfo *input, const ITensorInfo *output) -{ - ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); - ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, output)); - return Status{}; -} - -NETransposeKernel::NETransposeKernel() - : _func(nullptr), _input(nullptr), _output(nullptr) -{ -} - -void NETransposeKernel::configure(const ITensor *input, ITensor *output) -{ - ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); - - // Output tensor auto inizialitation if not yet initialized - auto_init_if_empty(*output->info(), input->info()->clone()->set_tensor_shape(transposed_tensor_shape(input->info()->tensor_shape()))); - - ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), output->info())); - - _input = input; - _output = output; - - switch(input->info()->element_size()) - { - case 1: - _func = &transpose_8bit_elements; - break; - case 2: - _func = &transpose_16bit_elements; - break; - case 4: - _func = &transpose_32bit_elements; - break; - default: - ARM_COMPUTE_ERROR("Element size not supported"); - break; - } - - // Note: This kernel performs 16 elements per iteration. - // However, since we use a left-over for loop on both dimensions (X and Y), we cannot have any read or write out of memory - // For this reason num_elems_processed_per_iteration_x is set to 1 - const unsigned int num_elems_processed_per_iteration_x = 1; - const unsigned int num_elems_processed_per_iteration_y = num_elems_processed(input->info()->element_size()); - - // Configure kernel window - Window win = calculate_max_window(*input->info(), Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y)); - - INEKernel::configure(win); -} - -void NETransposeKernel::run(const Window &window, const ThreadInfo &info) -{ - ARM_COMPUTE_UNUSED(info); - ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); - ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INEKernel::window(), window); - ARM_COMPUTE_ERROR_ON(_func == nullptr); - - (*_func)(_input, _output, window); -} diff --git a/src/core/NEON/kernels/NETransposeKernel.h b/src/core/NEON/kernels/NETransposeKernel.h deleted file mode 100644 index 88ece547e1..0000000000 --- a/src/core/NEON/kernels/NETransposeKernel.h +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2017-2021 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#ifndef ARM_COMPUTE_NETRANSPOSEKERNEL_H -#define ARM_COMPUTE_NETRANSPOSEKERNEL_H - -#include "src/core/NEON/INEKernel.h" - -namespace arm_compute -{ -class ITensor; - -/** Neon kernel which transposes the elements of a matrix. - * - * [width, height, batch] -> [height, width, batch] - * - */ -class NETransposeKernel : public INEKernel -{ -public: - const char *name() const override - { - return "NETransposeKernel"; - } - /** Default constructor */ - NETransposeKernel(); - /** Prevent instances of this class from being copied (As this class contains pointers) */ - NETransposeKernel(const NETransposeKernel &) = delete; - /** Prevent instances of this class from being copied (As this class contains pointers) */ - NETransposeKernel &operator=(const NETransposeKernel &) = delete; - /** Allow instances of this class to be moved */ - NETransposeKernel(NETransposeKernel &&) = default; - /** Allow instances of this class to be moved */ - NETransposeKernel &operator=(NETransposeKernel &&) = default; - /** Default destructor */ - ~NETransposeKernel() = default; - - /** Initialise the kernel's input and output. - * - * @param[in] input Input tensor. Data types supported: All - * @param[out] output Output tensor. Data type supported: Same as @p input - */ - void configure(const ITensor *input, ITensor *output); - /** Static function to check if given info will lead to a valid configuration of @ref NETransposeKernel - * - * @param[in] input Input tensor. Data types supported: All - * @param[in] output Output tensor. Data type supported: Same as @p input - * - * @return a status - */ - static Status validate(const ITensorInfo *input, const ITensorInfo *output); - - // Inherited methods overridden: - void run(const Window &window, const ThreadInfo &info) override; - -private: - /** Common signature for all the transpose functions - * - * @param[in] input An input tensor. Data types supported: All - * @param[out] output The output tensor. Data type supported: same as @p input - * @param[in] window Region on which to execute the kernel. - */ - using TransposeFunction = void(const ITensor *input, ITensor *output, const Window &window); - /** Transpose function to use for the particular tensor types passed to configure() */ - TransposeFunction *_func; - const ITensor *_input; - ITensor *_output; -}; -} // namespace arm_compute -#endif /* ARM_COMPUTE_NETRANSPOSEKERNEL_H */ diff --git a/src/core/cpu/kernels/CpuPermuteKernel.cpp b/src/core/cpu/kernels/CpuPermuteKernel.cpp index 7fd38a3ee7..270d6e222e 100644 --- a/src/core/cpu/kernels/CpuPermuteKernel.cpp +++ b/src/core/cpu/kernels/CpuPermuteKernel.cpp @@ -255,7 +255,7 @@ void CpuPermuteKernel::configure(const ITensorInfo *src, ITensorInfo *dst, const // Configure kernel window Window win = calculate_max_window(*src, Steps()); - // The NEPermute doesn't need padding so update_window_and_padding() can be skipped + // This kernel doesn't need padding so update_window_and_padding() can be skipped ICpuKernel::configure(win); } diff --git a/src/core/cpu/kernels/CpuTransposeKernel.cpp b/src/core/cpu/kernels/CpuTransposeKernel.cpp new file mode 100644 index 0000000000..ed08aa1aa0 --- /dev/null +++ b/src/core/cpu/kernels/CpuTransposeKernel.cpp @@ -0,0 +1,510 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "src/core/cpu/kernels/CpuTransposeKernel.h" + +#include "arm_compute/core/Error.h" +#include "arm_compute/core/Helpers.h" +#include "arm_compute/core/ITensor.h" +#include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Types.h" +#include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "src/core/helpers/AutoConfiguration.h" +#include "src/core/helpers/WindowHelpers.h" + +#include + +namespace arm_compute +{ +namespace cpu +{ +namespace kernels +{ +namespace +{ +unsigned int num_elems_processed(size_t element_size) +{ + switch(element_size) + { + case 1: + return 8; + case 2: + case 4: + return 4; + default: + break; + } + + ARM_COMPUTE_ERROR("Element size not supported"); +} + +void transpose_8bit_elements(const ITensor *in, ITensor *out, const Window &window) +{ + const int window_step_x = 8; + const int window_step_y = 8; + const int window_start_x = window.x().start(); + const int window_end_x = window.x().end(); + const int window_start_y = window.y().start(); + const int window_end_y = std::min(window.y().end(), static_cast(in->info()->dimension(1))); + const int window_end_y_multiple_of = ((window_end_y - window_start_y) / window_step_y) * window_step_y; + const size_t input_stride_in_bytes = in->info()->strides_in_bytes()[1]; + const size_t output_stride_in_bytes = out->info()->strides_in_bytes()[1]; + + // Check if we need a left-over loop for the y dimension + bool left_over_loop_y = (((window_end_y - window_start_y) % window_step_y) != 0); + + Window window_in(window); + window_in.set(Window::DimX, Window::Dimension(0, 1, 1)); + if(left_over_loop_y) + { + // Check if window_end_y_multiple_of is greater than window_start_y + if(window_end_y_multiple_of > window_start_y) + { + window_in.set(Window::DimY, Window::Dimension(window_start_y, window_end_y_multiple_of, window_step_y)); + } + else + { + window_in.set(Window::DimY, Window::Dimension(0, 0, 1)); + } + } + + Window window_out(window); + window_out.set(Window::DimX, Window::Dimension(0, 0, 0)); + window_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + + Iterator output(out, window_out); + + // Run the Neon path if and only if the input is not a row-vector + if(in->info()->dimension(1) != 1) + { + Iterator input(in, window_in); + execute_window_loop(window_in, [&](const Coordinates & id) + { + // Compute 8x8 elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) + { + const uint8x8_t row0 = vld1_u8(reinterpret_cast(input.ptr() + x + 0 * input_stride_in_bytes)); + const uint8x8_t row1 = vld1_u8(reinterpret_cast(input.ptr() + x + 1 * input_stride_in_bytes)); + const uint8x8_t row2 = vld1_u8(reinterpret_cast(input.ptr() + x + 2 * input_stride_in_bytes)); + const uint8x8_t row3 = vld1_u8(reinterpret_cast(input.ptr() + x + 3 * input_stride_in_bytes)); + const uint8x8_t row4 = vld1_u8(reinterpret_cast(input.ptr() + x + 4 * input_stride_in_bytes)); + const uint8x8_t row5 = vld1_u8(reinterpret_cast(input.ptr() + x + 5 * input_stride_in_bytes)); + const uint8x8_t row6 = vld1_u8(reinterpret_cast(input.ptr() + x + 6 * input_stride_in_bytes)); + const uint8x8_t row7 = vld1_u8(reinterpret_cast(input.ptr() + x + 7 * input_stride_in_bytes)); + + // Transpose 2x2 + const uint8x8x2_t k0_u8 = vtrn_u8(row0, row1); + const uint8x8x2_t k1_u8 = vtrn_u8(row2, row3); + const uint8x8x2_t k2_u8 = vtrn_u8(row4, row5); + const uint8x8x2_t k3_u8 = vtrn_u8(row6, row7); + + // Transpose 4x4 + const uint16x4x2_t k0_u16 = vtrn_u16(vreinterpret_u16_u8(k0_u8.val[0]), vreinterpret_u16_u8(k1_u8.val[0])); + const uint16x4x2_t k1_u16 = vtrn_u16(vreinterpret_u16_u8(k0_u8.val[1]), vreinterpret_u16_u8(k1_u8.val[1])); + const uint16x4x2_t k2_u16 = vtrn_u16(vreinterpret_u16_u8(k2_u8.val[0]), vreinterpret_u16_u8(k3_u8.val[0])); + const uint16x4x2_t k3_u16 = vtrn_u16(vreinterpret_u16_u8(k2_u8.val[1]), vreinterpret_u16_u8(k3_u8.val[1])); + + // Transpose 8x8 + const uint32x2x2_t k0_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[0]), vreinterpret_u32_u16(k2_u16.val[0])); + const uint32x2x2_t k1_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[1]), vreinterpret_u32_u16(k2_u16.val[1])); + const uint32x2x2_t k2_u32 = vtrn_u32(vreinterpret_u32_u16(k1_u16.val[0]), vreinterpret_u32_u16(k3_u16.val[0])); + const uint32x2x2_t k3_u32 = vtrn_u32(vreinterpret_u32_u16(k1_u16.val[1]), vreinterpret_u32_u16(k3_u16.val[1])); + + // Compute destination address + const size_t dst_offset_in_bytes = id.y() * sizeof(uint8_t) + x * output_stride_in_bytes; + + vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 0 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k0_u32.val[0]))); + vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 1 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k2_u32.val[0]))); + vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 2 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k1_u32.val[0]))); + vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 3 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k3_u32.val[0]))); + vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 4 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k0_u32.val[1]))); + vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 5 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k2_u32.val[1]))); + vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 6 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k1_u32.val[1]))); + vst1_u8(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 7 * output_stride_in_bytes), vreinterpret_u8_u16(vreinterpret_u16_u32(k3_u32.val[1]))); + } + + // Compute left-over elements along the x dimension (1x8) + for(; x < window_end_x; ++x) + { + const uint8_t val0 = *(input.ptr() + x + 0 * input_stride_in_bytes); + const uint8_t val1 = *(input.ptr() + x + 1 * input_stride_in_bytes); + const uint8_t val2 = *(input.ptr() + x + 2 * input_stride_in_bytes); + const uint8_t val3 = *(input.ptr() + x + 3 * input_stride_in_bytes); + const uint8_t val4 = *(input.ptr() + x + 4 * input_stride_in_bytes); + const uint8_t val5 = *(input.ptr() + x + 5 * input_stride_in_bytes); + const uint8_t val6 = *(input.ptr() + x + 6 * input_stride_in_bytes); + const uint8_t val7 = *(input.ptr() + x + 7 * input_stride_in_bytes); + + uint8x8_t result = vdup_n_u8(0); + result = vset_lane_u8(val0, result, 0); + result = vset_lane_u8(val1, result, 1); + result = vset_lane_u8(val2, result, 2); + result = vset_lane_u8(val3, result, 3); + result = vset_lane_u8(val4, result, 4); + result = vset_lane_u8(val5, result, 5); + result = vset_lane_u8(val6, result, 6); + result = vset_lane_u8(val7, result, 7); + + // Compute destination address + const size_t dst_offset_in_bytes = id.y() * sizeof(uint8_t) + x * output_stride_in_bytes; + + vst1_u8(output.ptr() + dst_offset_in_bytes, result); + } + }, + input, output); + } + + if(left_over_loop_y) + { + window_in.set(Window::DimX, Window::Dimension(window.x().start(), window.x().end(), 1)); + window_in.set(Window::DimY, Window::Dimension(window_end_y_multiple_of, window_end_y, 1)); + + Iterator input(in, window_in); + Iterator output(out, window_out); + + // Compute left-over elements along the y dimension (1x1) + execute_window_loop(window_in, [&](const Coordinates & id) + { + const uint8_t val0 = *input.ptr(); + + // Compute destination address + const size_t dst_offset_in_bytes = id.y() * sizeof(uint8_t) + id.x() * output_stride_in_bytes; + + *(output.ptr() + dst_offset_in_bytes) = val0; + }, + input, output); + } +} + +void transpose_16bit_elements(const ITensor *in, ITensor *out, const Window &window) +{ + const int window_step_x = 4; + const int window_step_y = 4; + const int window_start_x = window.x().start(); + const int window_end_x = window.x().end(); + const int window_start_y = window.y().start(); + const int window_end_y = std::min(window.y().end(), static_cast(in->info()->dimension(1))); + const int window_end_y_multiple_of = ((window_end_y - window_start_y) / window_step_y) * window_step_y; + const size_t input_stride_in_bytes = in->info()->strides_in_bytes()[1]; + const size_t output_stride_in_bytes = out->info()->strides_in_bytes()[1]; + + // Check if we need a left-over loop for the y dimension + bool left_over_loop_y = (((window_end_y - window_start_y) % window_step_y) != 0); + + Window window_in(window); + window_in.set(Window::DimX, Window::Dimension(0, 1, 1)); + if(left_over_loop_y) + { + // Check if window_end_y_multiple_of is greater than window_start_y + if(window_end_y_multiple_of > window_start_y) + { + window_in.set(Window::DimY, Window::Dimension(window_start_y, window_end_y_multiple_of, window_step_y)); + } + else + { + window_in.set(Window::DimY, Window::Dimension(0, 0, 1)); + } + } + + Window window_out(window); + window_out.set(Window::DimX, Window::Dimension(0, 0, 0)); + window_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + + Iterator output(out, window_out); + + // Run the Neon path if and only if the input is not a row-vector + if(in->info()->dimension(1) != 1) + { + Iterator input(in, window_in); + execute_window_loop(window_in, [&](const Coordinates & id) + { + // Compute 4x4 elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) + { + const uint16x4_t row0 = vld1_u16(reinterpret_cast(input.ptr() + 0 * input_stride_in_bytes) + x); + const uint16x4_t row1 = vld1_u16(reinterpret_cast(input.ptr() + 1 * input_stride_in_bytes) + x); + const uint16x4_t row2 = vld1_u16(reinterpret_cast(input.ptr() + 2 * input_stride_in_bytes) + x); + const uint16x4_t row3 = vld1_u16(reinterpret_cast(input.ptr() + 3 * input_stride_in_bytes) + x); + + // Transpose 2x2 + const uint16x4x2_t k0_u16 = vtrn_u16(row0, row1); + const uint16x4x2_t k1_u16 = vtrn_u16(row2, row3); + + // Transpose 4x4 + const uint32x2x2_t k0_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[0]), vreinterpret_u32_u16(k1_u16.val[0])); + const uint32x2x2_t k1_u32 = vtrn_u32(vreinterpret_u32_u16(k0_u16.val[1]), vreinterpret_u32_u16(k1_u16.val[1])); + + // Compute destination address + const size_t dst_offset_in_bytes = id.y() * sizeof(uint16_t) + x * output_stride_in_bytes; + + vst1_u16(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 0 * output_stride_in_bytes), vreinterpret_u16_u32(k0_u32.val[0])); + vst1_u16(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 1 * output_stride_in_bytes), vreinterpret_u16_u32(k1_u32.val[0])); + vst1_u16(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 2 * output_stride_in_bytes), vreinterpret_u16_u32(k0_u32.val[1])); + vst1_u16(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 3 * output_stride_in_bytes), vreinterpret_u16_u32(k1_u32.val[1])); + } + + // Compute left-over elements (1x4) + for(; x < window_end_x; ++x) + { + const uint16_t val0 = *(reinterpret_cast(input.ptr() + 0 * input_stride_in_bytes) + x); + const uint16_t val1 = *(reinterpret_cast(input.ptr() + 1 * input_stride_in_bytes) + x); + const uint16_t val2 = *(reinterpret_cast(input.ptr() + 2 * input_stride_in_bytes) + x); + const uint16_t val3 = *(reinterpret_cast(input.ptr() + 3 * input_stride_in_bytes) + x); + + uint16x4_t result = vdup_n_u16(0); + result = vset_lane_u16(val0, result, 0); + result = vset_lane_u16(val1, result, 1); + result = vset_lane_u16(val2, result, 2); + result = vset_lane_u16(val3, result, 3); + + // Compute destination address + const size_t dst_offset_in_bytes = id.y() * sizeof(uint16_t) + x * output_stride_in_bytes; + + vst1_u16(reinterpret_cast(output.ptr() + dst_offset_in_bytes), result); + } + }, + input, output); + } + + if(left_over_loop_y) + { + window_in.set(Window::DimX, Window::Dimension(window.x().start(), window.x().end(), 1)); + window_in.set(Window::DimY, Window::Dimension(window_end_y_multiple_of, window_end_y, 1)); + + Iterator input(in, window_in); + Iterator output(out, window_out); + + // Compute left-over elements along the y dimension (1x1) + execute_window_loop(window_in, [&](const Coordinates & id) + { + const uint16_t val0 = *(reinterpret_cast(input.ptr())); + + // Compute destination address + const size_t dst_offset_in_bytes = id.y() * sizeof(uint16_t) + id.x() * output_stride_in_bytes; + + *(reinterpret_cast(output.ptr() + dst_offset_in_bytes)) = val0; + }, + input, output); + } +} + +void transpose_32bit_elements(const ITensor *in, ITensor *out, const Window &window) +{ + const int window_step_x = 4; + const int window_step_y = 4; + const int window_start_x = window.x().start(); + const int window_end_x = window.x().end(); + const int window_start_y = window.y().start(); + const int window_end_y = std::min(window.y().end(), static_cast(in->info()->dimension(1))); + const int window_end_y_multiple_of = ((window_end_y - window_start_y) / window_step_y) * window_step_y; + const size_t input_stride_in_bytes = in->info()->strides_in_bytes()[1]; + const size_t output_stride_in_bytes = out->info()->strides_in_bytes()[1]; + + // Check if we need a left-over loop for the y dimension + bool left_over_loop_y = (((window_end_y - window_start_y) % window_step_y) != 0); + + Window window_in(window); + window_in.set(Window::DimX, Window::Dimension(0, 1, 1)); + if(left_over_loop_y) + { + // Check if window_end_y_multiple_of is greater than window_start_y + if(window_end_y_multiple_of > window_start_y) + { + window_in.set(Window::DimY, Window::Dimension(window_start_y, window_end_y_multiple_of, window_step_y)); + } + else + { + window_in.set(Window::DimY, Window::Dimension(0, 0, 1)); + } + } + + Window window_out(window); + window_out.set(Window::DimX, Window::Dimension(0, 0, 0)); + window_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + + Iterator output(out, window_out); + + // Run the Neon path if and only if the input is not a row-vector + if(in->info()->dimension(1) != 1) + { + Iterator input(in, window_in); + execute_window_loop(window_in, [&](const Coordinates & id) + { + // Compute 4x4 elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) + { + const uint32x4_t row0 = vld1q_u32(reinterpret_cast(input.ptr() + 0 * input_stride_in_bytes) + x); + const uint32x4_t row1 = vld1q_u32(reinterpret_cast(input.ptr() + 1 * input_stride_in_bytes) + x); + const uint32x4_t row2 = vld1q_u32(reinterpret_cast(input.ptr() + 2 * input_stride_in_bytes) + x); + const uint32x4_t row3 = vld1q_u32(reinterpret_cast(input.ptr() + 3 * input_stride_in_bytes) + x); + + // Transpose 2x2 + const uint32x2x2_t k0_u32 = vtrn_u32(vget_low_u32(row0), vget_low_u32(row1)); + const uint32x2x2_t k1_u32 = vtrn_u32(vget_high_u32(row2), vget_high_u32(row3)); + const uint32x2x2_t k2_u32 = vtrn_u32(vget_high_u32(row0), vget_high_u32(row1)); + const uint32x2x2_t k3_u32 = vtrn_u32(vget_low_u32(row2), vget_low_u32(row3)); + + // Compute destination address + const size_t dst_offset_in_bytes = id.y() * sizeof(uint32_t) + x * output_stride_in_bytes; + + // Swap block 01 with block 10 and store + vst1q_u32(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 0 * output_stride_in_bytes), vcombine_u32(k0_u32.val[0], k3_u32.val[0])); + vst1q_u32(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 1 * output_stride_in_bytes), vcombine_u32(k0_u32.val[1], k3_u32.val[1])); + vst1q_u32(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 2 * output_stride_in_bytes), vcombine_u32(k2_u32.val[0], k1_u32.val[0])); + vst1q_u32(reinterpret_cast(output.ptr() + dst_offset_in_bytes + 3 * output_stride_in_bytes), vcombine_u32(k2_u32.val[1], k1_u32.val[1])); + } + + // Compute left-over elements (1x4) + for(; x < window_end_x; ++x) + { + const uint32_t val0 = *(reinterpret_cast(input.ptr() + 0 * input_stride_in_bytes) + x); + const uint32_t val1 = *(reinterpret_cast(input.ptr() + 1 * input_stride_in_bytes) + x); + const uint32_t val2 = *(reinterpret_cast(input.ptr() + 2 * input_stride_in_bytes) + x); + const uint32_t val3 = *(reinterpret_cast(input.ptr() + 3 * input_stride_in_bytes) + x); + + uint32x4_t result = vdupq_n_u32(0); + result = vsetq_lane_u32(val0, result, 0); + result = vsetq_lane_u32(val1, result, 1); + result = vsetq_lane_u32(val2, result, 2); + result = vsetq_lane_u32(val3, result, 3); + + // Compute destination address + const size_t dst_offset_in_bytes = id.y() * sizeof(uint32_t) + x * output_stride_in_bytes; + + vst1q_u32(reinterpret_cast(output.ptr() + dst_offset_in_bytes), result); + } + }, + input, output); + } + + if(left_over_loop_y) + { + window_in.set(Window::DimX, Window::Dimension(window.x().start(), window.x().end(), 1)); + window_in.set(Window::DimY, Window::Dimension(window_end_y_multiple_of, window_end_y, 1)); + + Iterator input(in, window_in); + Iterator output(out, window_out); + + // Compute left-over elements along the y dimension (1x1) + execute_window_loop(window_in, [&](const Coordinates & id) + { + const uint32_t val0 = *(reinterpret_cast(input.ptr())); + + // Compute destination address + const size_t dst_offset_in_bytes = id.y() * sizeof(uint32_t) + id.x() * output_stride_in_bytes; + + *(reinterpret_cast(output.ptr() + dst_offset_in_bytes)) = val0; + }, + input, output); + } +} +} // namespace + +void CpuTransposeKernel::configure(const ITensorInfo *src, ITensorInfo *dst) +{ + ARM_COMPUTE_ERROR_ON_NULLPTR(src, dst); + + // Destination auto inizialitation if not yet initialized + const TensorShape dst_shape = misc::shape_calculator::compute_transposed_shape(*src); + auto_init_if_empty(*dst, src->clone()->set_tensor_shape(dst_shape)); + + // Perform validation step + ARM_COMPUTE_ERROR_THROW_ON(validate(src, dst)); + + // Note: This kernel performs 16 elements per iteration. + // However, since we use a left-over for loop on both dimensions (X and Y), we cannot have any read or write out of memory + // For this reason num_elems_processed_per_iteration_x is set to 1 + const unsigned int num_elems_processed_per_iteration_x = 1; + const unsigned int num_elems_processed_per_iteration_y = num_elems_processed(src->element_size()); + + // Configure kernel window + Window win = calculate_max_window(*src, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y)); + + // The CpuTranspose doesn't need padding so update_window_and_padding() can be skipped + Coordinates coord; + coord.set_num_dimensions(dst->num_dimensions()); + dst->set_valid_region(ValidRegion(coord, dst->tensor_shape())); + + ICpuKernel::configure(win); +} + +Status CpuTransposeKernel::validate(const ITensorInfo *src, const ITensorInfo *dst) +{ + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(src); + //Note: ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(input) is not needed here as this kernel doesn't use Neon FP16 instructions. + ARM_COMPUTE_RETURN_ERROR_ON(src->data_type() == DataType::UNKNOWN); + + // Error if input is not 8 bit, 16bit or 32bit + ARM_COMPUTE_RETURN_ERROR_ON_MSG(src->element_size() != 1 && src->element_size() != 2 && src->element_size() != 4, + "Element size not supported"); + + // Validate configured destination + if(dst->total_size() != 0) + { + const TensorShape dst_shape = misc::shape_calculator::compute_transposed_shape(*src); + + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(dst->tensor_shape(), dst_shape); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_QUANTIZATION_INFO(src, dst); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(src, dst); + } + + return Status{}; +} + +void CpuTransposeKernel::run_op(ITensorPack &tensors, const Window &window, const ThreadInfo &info) +{ + ARM_COMPUTE_UNUSED(info); + ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); + ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICpuKernel::window(), window); + + const auto src = tensors.get_const_tensor(TensorType::ACL_SRC); + auto dst = tensors.get_tensor(TensorType::ACL_DST); + + switch(src->info()->element_size()) + { + case 1: + transpose_8bit_elements(src, dst, window); + break; + case 2: + transpose_16bit_elements(src, dst, window); + break; + case 4: + transpose_32bit_elements(src, dst, window); + break; + default: + ARM_COMPUTE_ERROR("Element size not supported"); + break; + } +} + +const char *CpuTransposeKernel::name() const +{ + return "CpuTransposeKernel"; +} +} // namespace kernels +} // namespace cpu +} // namespace arm_compute diff --git a/src/core/cpu/kernels/CpuTransposeKernel.h b/src/core/cpu/kernels/CpuTransposeKernel.h new file mode 100644 index 0000000000..f09f427be8 --- /dev/null +++ b/src/core/cpu/kernels/CpuTransposeKernel.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_CPU_TRANSPOSE_KERNEL_H +#define ARM_COMPUTE_CPU_TRANSPOSE_KERNEL_H + +#include "src/core/common/Macros.h" +#include "src/core/cpu/ICpuKernel.h" + +namespace arm_compute +{ +namespace cpu +{ +namespace kernels +{ +/** Kernel which transposes the elements of a matrix */ +class CpuTransposeKernel : public ICpuKernel +{ +public: + CpuTransposeKernel() = default; + ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(CpuTransposeKernel); + /** Configure kernel for a given list of arguments + * + * @param[in] src Srouce tensor to permute. Data types supported: All + * @param[out] dst Destination tensor. Data types supported: Same as @p src + */ + void configure(const ITensorInfo *src, ITensorInfo *dst); + /** Static function to check if given info will lead to a valid configuration of @ref CpuTransposeKernel + * + * @param[in] src Source tensor to permute. Data types supported: All + * @param[in] dst Destination tensor. Data types supported: Same as @p src + * + * @return a status + */ + static Status validate(const ITensorInfo *src, const ITensorInfo *dst); + + // Inherited methods overridden: + void run_op(ITensorPack &tensors, const Window &window, const ThreadInfo &info) override; + const char *name() const override; +}; +} // namespace kernels +} // namespace cpu +} // namespace arm_compute +#endif /* ARM_COMPUTE_CPU_TRANSPOSE_KERNEL_H */ diff --git a/src/runtime/NEON/functions/NEFullyConnectedLayer.cpp b/src/runtime/NEON/functions/NEFullyConnectedLayer.cpp index ec782fc163..0a5318ac30 100644 --- a/src/runtime/NEON/functions/NEFullyConnectedLayer.cpp +++ b/src/runtime/NEON/functions/NEFullyConnectedLayer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #include "arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h" #include "arm_compute/core/Helpers.h" +#include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/Size2D.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" @@ -39,9 +40,8 @@ #include "src/core/NEON/kernels/NEGEMMMatrixAdditionKernel.h" #include "src/core/NEON/kernels/NEGEMMMatrixMultiplyKernel.h" #include "src/core/NEON/kernels/NEGEMMTranspose1xWKernel.h" -#include "src/core/NEON/kernels/NETransposeKernel.h" +#include "src/core/cpu/kernels/CpuTransposeKernel.h" -#include #include namespace arm_compute @@ -142,16 +142,39 @@ Status validate_mm(const ITensorInfo *input, const ITensorInfo *weights, const I } } // namespace +struct NEFullyConnectedLayerReshapeWeights::Impl +{ + const ITensor *src{ nullptr }; + ITensor *dst{ nullptr }; + std::unique_ptr op{ nullptr }; +}; + +NEFullyConnectedLayerReshapeWeights::NEFullyConnectedLayerReshapeWeights() + : _impl(std::make_unique()) +{ +} + +NEFullyConnectedLayerReshapeWeights::~NEFullyConnectedLayerReshapeWeights() = default; + void NEFullyConnectedLayerReshapeWeights::configure(const ITensor *input, ITensor *output) { - auto k = std::make_unique(); - k->configure(input, output); - _kernel = std::move(k); + _impl->op = std::make_unique(); + _impl->op->configure(input->info(), output->info()); + _impl->src = input; + _impl->dst = output; } Status NEFullyConnectedLayerReshapeWeights::validate(const ITensorInfo *input, const ITensorInfo *output) { - return NETransposeKernel::validate(input, output); + return cpu::kernels::CpuTransposeKernel::validate(input, output); +} + +void NEFullyConnectedLayerReshapeWeights::run() +{ + ITensorPack pack{}; + pack.add_tensor(TensorType::ACL_SRC, _impl->src); + pack.add_tensor(TensorType::ACL_DST, _impl->dst); + NEScheduler::get().schedule_op(_impl->op.get(), Window::DimY, _impl->op->window(), pack); } NEFullyConnectedLayer::~NEFullyConnectedLayer() = default; diff --git a/src/runtime/NEON/functions/NEPermute.cpp b/src/runtime/NEON/functions/NEPermute.cpp index 257c1a2e44..f707fad757 100644 --- a/src/runtime/NEON/functions/NEPermute.cpp +++ b/src/runtime/NEON/functions/NEPermute.cpp @@ -40,10 +40,6 @@ NEPermute::NEPermute() { } -NEPermute::NEPermute(NEPermute &&) = default; - -NEPermute &NEPermute::operator=(NEPermute &&) = default; - NEPermute::~NEPermute() = default; void NEPermute::configure(const ITensor *input, ITensor *output, const PermutationVector &perm) diff --git a/src/runtime/NEON/functions/NETranspose.cpp b/src/runtime/NEON/functions/NETranspose.cpp index aaa52e36b9..3b3023f3b3 100644 --- a/src/runtime/NEON/functions/NETranspose.cpp +++ b/src/runtime/NEON/functions/NETranspose.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,21 +23,48 @@ */ #include "arm_compute/runtime/NEON/functions/NETranspose.h" -#include "src/core/NEON/kernels/NETransposeKernel.h" - -#include +#include "arm_compute/core/Validate.h" +#include "src/runtime/cpu/operators/CpuTranspose.h" namespace arm_compute { +struct NETranspose::Impl +{ + const ITensor *src{ nullptr }; + ITensor *dst{ nullptr }; + std::unique_ptr op{ nullptr }; +}; + +NETranspose::NETranspose() + : _impl(std::make_unique()) +{ +} + +NETranspose::~NETranspose() = default; + void NETranspose::configure(const ITensor *input, ITensor *output) { - auto k = std::make_unique(); - k->configure(input, output); - _kernel = std::move(k); + ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); + + _impl->src = input; + _impl->dst = output; + _impl->op = std::make_unique(); + _impl->op->configure(input->info(), output->info()); } Status NETranspose::validate(const ITensorInfo *input, const ITensorInfo *output) { - return NETransposeKernel::validate(input, output); + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, output); + ARM_COMPUTE_RETURN_ON_ERROR(cpu::CpuTranspose::validate(input, output)); + return Status{}; +} + +void NETranspose::run() +{ + ITensorPack pack; + pack.add_tensor(TensorType::ACL_SRC, _impl->src); + pack.add_tensor(TensorType::ACL_DST, _impl->dst); + _impl->op->run(pack); } + } // namespace arm_compute diff --git a/src/runtime/cpu/operators/CpuPermute.h b/src/runtime/cpu/operators/CpuPermute.h index 12f0cf611d..2b30d7fbd8 100644 --- a/src/runtime/cpu/operators/CpuPermute.h +++ b/src/runtime/cpu/operators/CpuPermute.h @@ -59,4 +59,4 @@ public: }; } // namespace cpu } // namespace arm_compute -#endif /* ARM_COMPUTE_CPU_RESHAPE_H */ +#endif /* ARM_COMPUTE_CPU_PERMUTE_H */ diff --git a/src/runtime/cpu/operators/CpuTranspose.cpp b/src/runtime/cpu/operators/CpuTranspose.cpp new file mode 100644 index 0000000000..51eeb90b8b --- /dev/null +++ b/src/runtime/cpu/operators/CpuTranspose.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018-2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "src/runtime/cpu/operators/CpuTranspose.h" + +#include "src/core/cpu/kernels/CpuTransposeKernel.h" + +namespace arm_compute +{ +namespace cpu +{ +void CpuTranspose::configure(const ITensorInfo *src, ITensorInfo *dst) +{ + auto k = std::make_unique(); + k->configure(src, dst); + _kernel = std::move(k); +} + +Status CpuTranspose::validate(const ITensorInfo *src, const ITensorInfo *dst) +{ + return kernels::CpuTransposeKernel::validate(src, dst); +} +} // namesapce cpu +} // namespace arm_compute diff --git a/src/runtime/cpu/operators/CpuTranspose.h b/src/runtime/cpu/operators/CpuTranspose.h new file mode 100644 index 0000000000..c0232ddab2 --- /dev/null +++ b/src/runtime/cpu/operators/CpuTranspose.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_CPU_TRANSPOSE_H +#define ARM_COMPUTE_CPU_TRANSPOSE_H + +#include "src/runtime/cpu/ICpuOperator.h" + +namespace arm_compute +{ +namespace cpu +{ +/** Basic function to run @ref kernels::CpuTransposeKernel */ +class CpuTranspose : public ICpuOperator +{ +public: + /** Constructor */ + CpuTranspose() = default; + /** Configure operator for a given list of arguments + * + * @param[in] src Source tensor to permute. Data types supported: All + * @param[out] dst Destintation tensor. Data types supported: Same as @p src + */ + void configure(const ITensorInfo *src, ITensorInfo *dst); + /** Static function to check if given info will lead to a valid configuration of @ref CpuTranspose + * + * @param[in] src Source tensor to permute. Data types supported: All + * @param[in] dst Destination tensor. Data types supported: Same as @p dst + * + * @return a status + */ + static Status validate(const ITensorInfo *src, const ITensorInfo *dst); +}; +} // namespace cpu +} // namespace arm_compute +#endif /* ARM_COMPUTE_CPU_TRANSPOSE_H */ -- cgit v1.2.1