diff options
-rw-r--r-- | Android.bp | 4 | ||||
-rw-r--r-- | filelist.json | 8 | ||||
-rw-r--r-- | src/core/NEON/kernels/NECropKernel.cpp | 215 | ||||
-rw-r--r-- | src/core/NEON/kernels/NECropKernel.h | 4 | ||||
-rw-r--r-- | src/cpu/kernels/crop/generic/neon/crop_helper.h | 86 | ||||
-rw-r--r-- | src/cpu/kernels/crop/generic/neon/fp16.cpp | 40 | ||||
-rw-r--r-- | src/cpu/kernels/crop/generic/neon/fp32.cpp | 38 | ||||
-rw-r--r-- | src/cpu/kernels/crop/generic/neon/impl.cpp | 127 | ||||
-rw-r--r-- | src/cpu/kernels/crop/generic/neon/impl.h | 41 | ||||
-rw-r--r-- | src/cpu/kernels/crop/generic/neon/integer.cpp | 74 | ||||
-rw-r--r-- | src/cpu/kernels/crop/generic/neon/list.h | 54 |
11 files changed, 545 insertions, 146 deletions
diff --git a/Android.bp b/Android.bp index 5654b840fe..4d5f2bdf68 100644 --- a/Android.bp +++ b/Android.bp @@ -453,6 +453,10 @@ cc_library_static { "src/cpu/kernels/add/generic/sve2/qasymm8.cpp", "src/cpu/kernels/add/generic/sve2/qasymm8_signed.cpp", "src/cpu/kernels/add/generic/sve2/qsymm16.cpp", + "src/cpu/kernels/crop/generic/neon/fp16.cpp", + "src/cpu/kernels/crop/generic/neon/fp32.cpp", + "src/cpu/kernels/crop/generic/neon/impl.cpp", + "src/cpu/kernels/crop/generic/neon/integer.cpp", "src/cpu/kernels/elementwise/sve/elementwise.cpp", "src/cpu/kernels/elementwise/sve/elementwise_unary.cpp", "src/cpu/kernels/floor/neon/fp16.cpp", diff --git a/filelist.json b/filelist.json index d7847480a4..040ff51333 100644 --- a/filelist.json +++ b/filelist.json @@ -1066,7 +1066,13 @@ "common": [ "src/core/NEON/kernels/NECropKernel.cpp", "src/runtime/NEON/functions/NECropResize.cpp" - ] + ], + "neon": { + "common": [ "src/cpu/kernels/crop/generic/neon/impl.cpp" ], + "fp32": [ "src/cpu/kernels/crop/generic/neon/fp32.cpp" ], + "fp16": [ "src/cpu/kernels/crop/generic/neon/fp16.cpp" ], + "integer": [ "src/cpu/kernels/crop/generic/neon/integer.cpp" ] + } } }, "Deconv2d": { diff --git a/src/core/NEON/kernels/NECropKernel.cpp b/src/core/NEON/kernels/NECropKernel.cpp index fabbd64305..729402116d 100644 --- a/src/core/NEON/kernels/NECropKernel.cpp +++ b/src/core/NEON/kernels/NECropKernel.cpp @@ -31,136 +31,92 @@ #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "src/core/CPP/Validate.h" #include "src/core/NEON/wrapper/wrapper.h" +#include "src/core/common/Registrars.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" #include "src/core/utils/helpers/bit_ops.h" +#include "src/cpu/kernels/crop/generic/neon/list.h" namespace arm_compute { namespace { -template <typename T> -inline float32x4_t load_as_f32(T *ptr) +struct CropSelectorData { - ARM_COMPUTE_UNUSED(ptr); - ARM_COMPUTE_ERROR("Type not supported."); -} - -template <> -inline float32x4_t load_as_f32(float *ptr) -{ - return wrapper::vloadq(ptr); -} - -template <> -inline float32x4_t load_as_f32(int32_t *ptr) -{ - return vcvtq_f32_s32(wrapper::vloadq(ptr)); -} + DataType dt; +}; -template <> -inline float32x4_t load_as_f32(uint32_t *ptr) -{ - return vcvtq_f32_u32(wrapper::vloadq(ptr)); -} +using CropSelectorPtr = std::add_pointer<bool(const CropSelectorData &data)>::type; +using CropUKernelPtr = std::add_pointer<void(const ITensor *, const ITensor *, float *, Coordinates, int32_t, int32_t, int32_t, bool, bool)>::type; -template <> -inline float32x4_t load_as_f32(int16_t *ptr) +struct CropUKernel { - return vcvtq_f32_s32(vmovl_s16(wrapper::vload(ptr))); -} + const char *name; + const CropSelectorPtr is_selected; + CropUKernelPtr ukernel; +}; -template <> -inline float32x4_t load_as_f32(uint16_t *ptr) +static const CropUKernel available_kernels[] = { - return vcvtq_f32_u32(vmovl_u16(wrapper::vload(ptr))); -} - -template <> -inline float32x4_t load_as_f32(uint8_t *ptr) -{ - return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(wrapper::vload(ptr))))); -} - -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -template <> -inline float32x4_t load_as_f32(float16_t *ptr) -{ - return vcvt_f32_f16(wrapper::vload(ptr)); -} -#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ - -template <typename T> -inline void in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, - int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) -{ - // Reverse elements if width flipped. - if(is_width_flipped) { - // Collapse first dimension if possible. - if(input_has_single_channel) - { - int32_t x = output_width_start; - Coordinates negative_offset(input_offset); - negative_offset.set(1, negative_offset[1] - window_step_x + 1); - for(; x <= output_width_limit - window_step_x; x += window_step_x, negative_offset[1] -= window_step_x) - { - auto in = load_as_f32(reinterpret_cast<T *>(input->ptr_to_element(negative_offset))); - - in = wrapper::vrev64(in); - in = wrapper::vcombine(wrapper::vgethigh(in), wrapper::vgetlow(in)); + "fp16_neon_crop", + [](const CropSelectorData & data) { return data.dt == DataType::F16; }, + REGISTER_FP16_NEON(arm_compute::cpu::fp16_in_bounds_crop_window) + }, + { + "f32_neon_crop", + [](const CropSelectorData & data) { return data.dt == DataType::F32; }, + REGISTER_FP32_NEON(arm_compute::cpu::fp32_in_bounds_crop_window) + }, + { + "u8_neon_crop", + [](const CropSelectorData & data) { return data.dt == DataType::U8; }, + REGISTER_INTEGER_NEON(arm_compute::cpu::u8_in_bounds_crop_window) + }, + { + "u16_neon_crop", + [](const CropSelectorData & data) { return data.dt == DataType::U16; }, + REGISTER_INTEGER_NEON(arm_compute::cpu::u16_in_bounds_crop_window) + }, + { + "u32_neon_crop", + [](const CropSelectorData & data) { return data.dt == DataType::U32; }, + REGISTER_INTEGER_NEON(arm_compute::cpu::u32_in_bounds_crop_window) + }, + { + "s8_neon_crop", + [](const CropSelectorData & data) { return data.dt == DataType::S8; }, + REGISTER_INTEGER_NEON(arm_compute::cpu::s8_in_bounds_crop_window) + }, + { + "s16_neon_crop", + [](const CropSelectorData & data) { return data.dt == DataType::S16; }, + REGISTER_INTEGER_NEON(arm_compute::cpu::s16_in_bounds_crop_window) + }, + { + "s32_neon_crop", + [](const CropSelectorData & data) { return data.dt == DataType::S32; }, + REGISTER_INTEGER_NEON(arm_compute::cpu::s32_in_bounds_crop_window) + }, +}; - wrapper::vstore(output_ptr + x, in); - } - input_offset[1] = negative_offset[1] + window_step_x - 1; - for(; x < output_width_limit; ++x, --input_offset[1]) - { - *(output_ptr + x) = static_cast<float>(*reinterpret_cast<T *>(input->ptr_to_element(input_offset))); - } - } - else - { - for(int32_t x = output_width_start; x < output_width_limit; ++x, --input_offset[1]) - { - input_offset.set(0, 0); - int32_t c = 0; - for(; c <= static_cast<int32_t>(input->info()->dimension(0)) - window_step_x; c += window_step_x, input_offset[0] += window_step_x) - { - auto in = load_as_f32(reinterpret_cast<T *>(input->ptr_to_element(input_offset))); - wrapper::vstore(output_ptr + x * output->info()->dimension(0) + c, in); - } - for(; c < static_cast<int32_t>(input->info()->dimension(0)); ++c, ++input_offset[0]) - { - *(output_ptr + x * output->info()->dimension(0) + c) = static_cast<float>(*reinterpret_cast<T *>(input->ptr_to_element(input_offset))); - } - } - } - } - else +/** Micro-kernel selector + * + * @param[in] data Selection data passed to help pick the appropriate micro-kernel + * + * @return A matching micro-kernel else nullptr + */ +const CropUKernel *get_implementation(const CropSelectorData &data) +{ + for(const auto &uk : available_kernels) { - // Use memcpy if the elements don't need converting to float. - if(std::is_same<T, float>::value) + if(uk.is_selected(data)) { - memcpy(static_cast<void *>(output_ptr + output_width_start * output->info()->dimension(0)), - reinterpret_cast<const void *>(input->ptr_to_element(input_offset)), - (output_width_limit - output_width_start) * output->info()->dimension(0) * output->info()->element_size()); - } - else - { - int32_t x = 0; - int32_t limit = (output_width_limit - output_width_start) * static_cast<int32_t>(output->info()->dimension(0)); - float *output_start_ptr = output_ptr + output_width_start * output->info()->dimension(0); - for(; x <= limit - window_step_x; x += window_step_x, input_offset[0] += window_step_x) - { - auto in = load_as_f32(reinterpret_cast<T *>(input->ptr_to_element(input_offset))); - wrapper::vstore(output_start_ptr + x, in); - } - for(; x < limit; ++x, ++input_offset[0]) - { - *(output_start_ptr + x) = static_cast<float>(*reinterpret_cast<T *>(input->ptr_to_element(input_offset))); - } + return &uk; } } + + return nullptr; } inline void out_of_bounds_crop_window(const ITensor *output, float *output_ptr, float extrapolation_value, @@ -234,8 +190,7 @@ inline void execute_window(const ITensor *input, const ITensor *output, Coordina } // namespace NECropKernel::NECropKernel() - : _input(nullptr), _crop_boxes(nullptr), _box_ind(nullptr), _output(nullptr), _start(), _end(), _crop_box_ind(0), _extrapolation_value(0), _rows_out_of_bounds(), _cols_out_of_bounds(), - _in_bounds_crop_function(nullptr) + : _input(nullptr), _crop_boxes(nullptr), _box_ind(nullptr), _output(nullptr), _start(), _end(), _crop_box_ind(0), _extrapolation_value(0), _rows_out_of_bounds(), _cols_out_of_bounds() { } @@ -250,40 +205,14 @@ void NECropKernel::configure(const ITensor *input, const ITensor *crop_boxes, co _output = output; _crop_box_ind = crop_box_ind; _extrapolation_value = extrapolation_value; - - switch(input->info()->data_type()) - { - case DataType::F32: - _in_bounds_crop_function = &in_bounds_crop_window<float>; - break; -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - case DataType::F16: - _in_bounds_crop_function = &in_bounds_crop_window<float16_t>; - break; -#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ - case DataType::U32: - _in_bounds_crop_function = &in_bounds_crop_window<uint32_t>; - break; - case DataType::S32: - _in_bounds_crop_function = &in_bounds_crop_window<int32_t>; - break; - case DataType::U16: - _in_bounds_crop_function = &in_bounds_crop_window<uint16_t>; - break; - case DataType::S16: - _in_bounds_crop_function = &in_bounds_crop_window<int16_t>; - break; - case DataType::U8: - _in_bounds_crop_function = &in_bounds_crop_window<uint8_t>; - break; - default: - ARM_COMPUTE_ERROR("Datatype not supported"); - } } Status NECropKernel::validate(const ITensorInfo *input, const ITensorInfo *crop_boxes, const ITensorInfo *box_ind, const ITensorInfo *output, uint32_t crop_box_ind, float extrapolation_value) { ARM_COMPUTE_UNUSED(extrapolation_value); + const auto *uk = get_implementation(CropSelectorData{ input->data_type() }); + ARM_COMPUTE_RETURN_ERROR_ON(uk == nullptr || uk->ukernel == nullptr); + ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(input); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::U8, DataType::U16, DataType::S16, DataType::F16, DataType::U32, DataType::S32, DataType::F32); ARM_COMPUTE_RETURN_ERROR_ON_DATA_LAYOUT_NOT_IN(input, DataLayout::NHWC); @@ -369,10 +298,12 @@ void NECropKernel::run(const Window &window, const ThreadInfo &info) ARM_COMPUTE_ERROR_ON(_input->info()->has_padding()); ARM_COMPUTE_ERROR_ON(_output->info()->has_padding()); + const auto *uk = get_implementation(CropSelectorData{ _input->info()->data_type() }); + uint32_t batch_index = *(reinterpret_cast<int32_t *>(_box_ind->ptr_to_element(Coordinates(_crop_box_ind)))); Coordinates input_offset(0, _end[0] < _start[0] ? _start[0] - _cols_out_of_bounds[0] : _start[0] + _cols_out_of_bounds[0], _end[1] < _start[1] ? _start[1] - _rows_out_of_bounds[0] : _start[1] + _rows_out_of_bounds[0], batch_index); - execute_window(_input, _output, input_offset, _extrapolation_value, _rows_out_of_bounds, _cols_out_of_bounds, _in_bounds_crop_function, _end[1] < _start[1], + execute_window(_input, _output, input_offset, _extrapolation_value, _rows_out_of_bounds, _cols_out_of_bounds, uk->ukernel, _end[1] < _start[1], _cols_out_of_bounds[0] + _cols_out_of_bounds[1] < _output->info()->dimension(1), _cols_out_of_bounds[0] > 0, _cols_out_of_bounds[1] > 0, _start[0] <= _end[0], _end[0] < _start[0]); } diff --git a/src/core/NEON/kernels/NECropKernel.h b/src/core/NEON/kernels/NECropKernel.h index 742215e22b..6c989c1d2c 100644 --- a/src/core/NEON/kernels/NECropKernel.h +++ b/src/core/NEON/kernels/NECropKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020 Arm Limited. + * Copyright (c) 2019-2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -107,8 +107,6 @@ private: std::array<uint32_t, 2> _rows_out_of_bounds; /** The number of columns out of bounds at the start and end of output. */ std::array<uint32_t, 2> _cols_out_of_bounds; - - NECropKernel::InBoundsCropFunction *_in_bounds_crop_function; }; } // namespace arm_compute #endif /*ARM_COMPUTE_NEON_CROP_KERNEL_H */ diff --git a/src/cpu/kernels/crop/generic/neon/crop_helper.h b/src/cpu/kernels/crop/generic/neon/crop_helper.h new file mode 100644 index 0000000000..1fe8e11e98 --- /dev/null +++ b/src/cpu/kernels/crop/generic/neon/crop_helper.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef SRC_CORE_NEON_KERNELS_CROP_CROP_HELPER_H +#define SRC_CORE_NEON_KERNELS_CROP_CROP_HELPER_H + +#include "src/core/NEON/wrapper/wrapper.h" + +namespace arm_compute +{ +namespace cpu +{ +template <typename T> +inline float32x4_t load_as_f32(T *ptr) +{ + ARM_COMPUTE_UNUSED(ptr); + ARM_COMPUTE_ERROR("Type not supported."); +} + +#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) +template <> +inline float32x4_t load_as_f32(float16_t *ptr) +{ + return vcvt_f32_f16(wrapper::vload(ptr)); +} +#endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */ + +template <> +inline float32x4_t load_as_f32(float *ptr) +{ + return wrapper::vloadq(ptr); +} + +template <> +inline float32x4_t load_as_f32(int32_t *ptr) +{ + return vcvtq_f32_s32(wrapper::vloadq(ptr)); +} + +template <> +inline float32x4_t load_as_f32(uint32_t *ptr) +{ + return vcvtq_f32_u32(wrapper::vloadq(ptr)); +} + +template <> +inline float32x4_t load_as_f32(int16_t *ptr) +{ + return vcvtq_f32_s32(vmovl_s16(wrapper::vload(ptr))); +} + +template <> +inline float32x4_t load_as_f32(uint16_t *ptr) +{ + return vcvtq_f32_u32(vmovl_u16(wrapper::vload(ptr))); +} + +template <> +inline float32x4_t load_as_f32(uint8_t *ptr) +{ + return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(wrapper::vload(ptr))))); +} +} +} // namespace arm_compute + +#endif //SRC_CORE_NEON_KERNELS_CROP_CROP_HELPER_H
\ No newline at end of file diff --git a/src/cpu/kernels/crop/generic/neon/fp16.cpp b/src/cpu/kernels/crop/generic/neon/fp16.cpp new file mode 100644 index 0000000000..218ebba191 --- /dev/null +++ b/src/cpu/kernels/crop/generic/neon/fp16.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) + +#include "src/cpu/kernels/crop/generic/neon/impl.h" + +namespace arm_compute +{ +namespace cpu +{ +void fp16_in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) +{ + return in_bounds_crop_window<float16_t>(input, output, output_ptr, input_offset, + window_step_x, output_width_start, output_width_limit, input_has_single_channel, is_width_flipped); +} +} +} // namespace arm_compute +#endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */ diff --git a/src/cpu/kernels/crop/generic/neon/fp32.cpp b/src/cpu/kernels/crop/generic/neon/fp32.cpp new file mode 100644 index 0000000000..16d0218fce --- /dev/null +++ b/src/cpu/kernels/crop/generic/neon/fp32.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "src/cpu/kernels/crop/generic/neon/impl.h" + +namespace arm_compute +{ +namespace cpu +{ +void fp32_in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) +{ + return in_bounds_crop_window<float32_t>(input, output, output_ptr, input_offset, + window_step_x, output_width_start, output_width_limit, input_has_single_channel, is_width_flipped); +} +} +} // namespace arm_compute diff --git a/src/cpu/kernels/crop/generic/neon/impl.cpp b/src/cpu/kernels/crop/generic/neon/impl.cpp new file mode 100644 index 0000000000..95ab804940 --- /dev/null +++ b/src/cpu/kernels/crop/generic/neon/impl.cpp @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2018-2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "src/cpu/kernels/crop/generic/neon/impl.h" +#include "arm_compute/core/Helpers.h" +#include "arm_compute/core/TensorInfo.h" +#include "src/core/NEON/wrapper/wrapper.h" +#include "src/core/common/Registrars.h" +#include "src/cpu/kernels/crop/generic/neon/crop_helper.h" + +namespace arm_compute +{ +namespace cpu +{ +template <typename T> +void in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) +{ + // Reverse elements if width flipped. + if(is_width_flipped) + { + // Collapse first dimension if possible. + if(input_has_single_channel) + { + int32_t x = output_width_start; + Coordinates negative_offset(input_offset); + negative_offset.set(1, negative_offset[1] - window_step_x + 1); + for(; x <= output_width_limit - window_step_x; x += window_step_x, negative_offset[1] -= window_step_x) + { + auto in = load_as_f32(reinterpret_cast<T *>(input->ptr_to_element(negative_offset))); + + in = wrapper::vrev64(in); + in = wrapper::vcombine(wrapper::vgethigh(in), wrapper::vgetlow(in)); + + wrapper::vstore(output_ptr + x, in); + } + input_offset[1] = negative_offset[1] + window_step_x - 1; + for(; x < output_width_limit; ++x, --input_offset[1]) + { + *(output_ptr + x) = static_cast<float>(*reinterpret_cast<T *>(input->ptr_to_element(input_offset))); + } + } + else + { + for(int32_t x = output_width_start; x < output_width_limit; ++x, --input_offset[1]) + { + input_offset.set(0, 0); + int32_t c = 0; + for(; c <= static_cast<int32_t>(input->info()->dimension(0)) - window_step_x; c += window_step_x, input_offset[0] += window_step_x) + { + auto in = load_as_f32(reinterpret_cast<T *>(input->ptr_to_element(input_offset))); + wrapper::vstore(output_ptr + x * output->info()->dimension(0) + c, in); + } + for(; c < static_cast<int32_t>(input->info()->dimension(0)); ++c, ++input_offset[0]) + { + *(output_ptr + x * output->info()->dimension(0) + c) = static_cast<float>(*reinterpret_cast<T *>(input->ptr_to_element(input_offset))); + } + } + } + } + else + { + // Use memcpy if the elements don't need converting to float. + if(std::is_same<T, float>::value) + { + memcpy(static_cast<void *>(output_ptr + output_width_start * output->info()->dimension(0)), + reinterpret_cast<const void *>(input->ptr_to_element(input_offset)), + (output_width_limit - output_width_start) * output->info()->dimension(0) * output->info()->element_size()); + } + else + { + int32_t x = 0; + int32_t limit = (output_width_limit - output_width_start) * static_cast<int32_t>(output->info()->dimension(0)); + float *output_start_ptr = output_ptr + output_width_start * output->info()->dimension(0); + for(; x <= limit - window_step_x; x += window_step_x, input_offset[0] += window_step_x) + { + auto in = load_as_f32(reinterpret_cast<T *>(input->ptr_to_element(input_offset))); + wrapper::vstore(output_start_ptr + x, in); + } + for(; x < limit; ++x, ++input_offset[0]) + { + *(output_start_ptr + x) = static_cast<float>(*reinterpret_cast<T *>(input->ptr_to_element(input_offset))); + } + } + } +} + +#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) +template void in_bounds_crop_window<float16_t>(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped); +#endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */ +template void in_bounds_crop_window<float32_t>(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped); +template void in_bounds_crop_window<uint8_t>(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped); +template void in_bounds_crop_window<uint16_t>(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped); +template void in_bounds_crop_window<uint32_t>(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped); +template void in_bounds_crop_window<int8_t>(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped); +template void in_bounds_crop_window<int16_t>(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped); +template void in_bounds_crop_window<int32_t>(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped); +} +} // namespace arm_compute diff --git a/src/cpu/kernels/crop/generic/neon/impl.h b/src/cpu/kernels/crop/generic/neon/impl.h new file mode 100644 index 0000000000..50f889705a --- /dev/null +++ b/src/cpu/kernels/crop/generic/neon/impl.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef SRC_CORE_NEON_KERNELS_CROP_IMPL_H +#define SRC_CORE_NEON_KERNELS_CROP_IMPL_H + +#include "arm_compute/core/Helpers.h" +#include "arm_compute/core/TensorInfo.h" +#include "src/core/NEON/wrapper/wrapper.h" +#include "src/core/common/Registrars.h" + +namespace arm_compute +{ +namespace cpu +{ +template <typename T> +void in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped); +} // namespace cpu +} // namespace arm_compute +#endif //SRC_CORE_NEON_KERNELS_CROP_IMPL_H diff --git a/src/cpu/kernels/crop/generic/neon/integer.cpp b/src/cpu/kernels/crop/generic/neon/integer.cpp new file mode 100644 index 0000000000..7dbf0a7f5e --- /dev/null +++ b/src/cpu/kernels/crop/generic/neon/integer.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "src/cpu/kernels/crop/generic/neon/impl.h" +#include "src/cpu/kernels/crop/generic/neon/list.h" + +namespace arm_compute +{ +namespace cpu +{ +void u8_in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) +{ + return in_bounds_crop_window<uint8_t>(input, output, output_ptr, input_offset, + window_step_x, output_width_start, output_width_limit, input_has_single_channel, is_width_flipped); +} + +void u16_in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) +{ + return in_bounds_crop_window<uint16_t>(input, output, output_ptr, input_offset, + window_step_x, output_width_start, output_width_limit, input_has_single_channel, is_width_flipped); +} + +void u32_in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) +{ + return in_bounds_crop_window<uint32_t>(input, output, output_ptr, input_offset, + window_step_x, output_width_start, output_width_limit, input_has_single_channel, is_width_flipped); +} + +void s8_in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) +{ + return in_bounds_crop_window<int8_t>(input, output, output_ptr, input_offset, + window_step_x, output_width_start, output_width_limit, input_has_single_channel, is_width_flipped); +} + +void s16_in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) +{ + return in_bounds_crop_window<int16_t>(input, output, output_ptr, input_offset, + window_step_x, output_width_start, output_width_limit, input_has_single_channel, is_width_flipped); +} + +void s32_in_bounds_crop_window(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) +{ + return in_bounds_crop_window<int32_t>(input, output, output_ptr, input_offset, + window_step_x, output_width_start, output_width_limit, input_has_single_channel, is_width_flipped); +} +} +} // namespace arm_compute diff --git a/src/cpu/kernels/crop/generic/neon/list.h b/src/cpu/kernels/crop/generic/neon/list.h new file mode 100644 index 0000000000..f33049304d --- /dev/null +++ b/src/cpu/kernels/crop/generic/neon/list.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef SRC_CORE_NEON_KERNELS_CROP_LIST_H +#define SRC_CORE_NEON_KERNELS_CROP_LIST_H + +#include "arm_compute/core/Helpers.h" +#include "arm_compute/core/TensorInfo.h" +#include "src/core/NEON/wrapper/wrapper.h" +#include "src/core/common/Registrars.h" +#include "src/cpu/kernels/crop/generic/neon/impl.h" + +namespace arm_compute +{ +namespace cpu +{ +#define DECLARE_CROP_KERNEL(func_name) \ + void func_name(const ITensor *input, const ITensor *output, float *output_ptr, Coordinates input_offset, \ + int32_t window_step_x, int32_t output_width_start, int32_t output_width_limit, bool input_has_single_channel, bool is_width_flipped) + +DECLARE_CROP_KERNEL(fp16_in_bounds_crop_window); +DECLARE_CROP_KERNEL(fp32_in_bounds_crop_window); +DECLARE_CROP_KERNEL(s8_in_bounds_crop_window); +DECLARE_CROP_KERNEL(s16_in_bounds_crop_window); +DECLARE_CROP_KERNEL(s32_in_bounds_crop_window); +DECLARE_CROP_KERNEL(u8_in_bounds_crop_window); +DECLARE_CROP_KERNEL(u16_in_bounds_crop_window); +DECLARE_CROP_KERNEL(u32_in_bounds_crop_window); + +#undef DECLARE_CROP_KERNEL + +} // namespace cpu +} // namespace arm_compute +#endif //SRC_CORE_NEON_KERNELS_CROP_LIST_H |