diff options
Diffstat (limited to 'src/core')
-rw-r--r-- | src/core/CL/CLHelpers.cpp | 13 | ||||
-rw-r--r-- | src/core/CL/cl_kernels/concatenate.cl | 26 | ||||
-rw-r--r-- | src/core/CL/cl_kernels/helpers.h | 6 | ||||
-rw-r--r-- | src/core/CL/kernels/CLDepthConcatenateKernel.cpp | 63 | ||||
-rw-r--r-- | src/core/CL/kernels/CLFillBorderKernel.cpp | 9 | ||||
-rw-r--r-- | src/core/NEON/kernels/NEDepthConcatenateKernel.cpp | 108 |
6 files changed, 172 insertions, 53 deletions
diff --git a/src/core/CL/CLHelpers.cpp b/src/core/CL/CLHelpers.cpp index dd87e778d7..1073b39ca7 100644 --- a/src/core/CL/CLHelpers.cpp +++ b/src/core/CL/CLHelpers.cpp @@ -100,6 +100,19 @@ std::string get_cl_type_from_data_type(const DataType &dt) } } +std::string get_underlying_cl_type_from_data_type(const DataType &dt) +{ + switch(dt) + { + case DataType::QS8: + return "char"; + case DataType::QS16: + return "short"; + default: + return get_cl_type_from_data_type(dt); + } +} + const std::string &string_from_target(GPUTarget target) { static std::map<GPUTarget, const std::string> gpu_target_map = diff --git a/src/core/CL/cl_kernels/concatenate.cl b/src/core/CL/cl_kernels/concatenate.cl index 00f5189508..a92ab5bdad 100644 --- a/src/core/CL/cl_kernels/concatenate.cl +++ b/src/core/CL/cl_kernels/concatenate.cl @@ -25,29 +25,35 @@ /** This kernel concatenates the input tensor into the output tensor along the third dimension * - * @param[in] src_ptr Pointer to the source tensor. Supported data types: F32 + * @param[in] src_ptr Pointer to the source tensor. Supported data types: QS8, QS16, F16, F32 * @param[in] src_stride_x Stride of the source tensor in X dimension (in bytes) * @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes) * @param[in] src_stride_y Stride of the source tensor in Y dimension (in bytes) * @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes) + * @param[in] src_stride_z Stride of the source tensor in Z dimension (in bytes) + * @param[in] src_step_z src_stride_z * number of elements along Z processed per workitem(in bytes) * @param[in] src_offset_first_element_in_bytes The offset of the first element in the source tensor - * @param[out] dst_ptr Pointer to the destination tensor. Supported data types: F32 + * @param[out] dst_ptr Pointer to the destination tensor. Supported data types: same as @p src_ptr * @param[in] dst_stride_x Stride of the destination tensor in X dimension (in bytes) * @param[in] dst_step_x dst_stride_x * number of elements along X processed per workitem(in bytes) * @param[in] dst_stride_y Stride of the destination tensor in Y dimension (in bytes) * @param[in] dst_step_y dst_stride_y * number of elements along Y processed per workitem(in bytes) + * @param[in] dst_stride_z Stride of the source tensor in Z dimension (in bytes) + * @param[in] dst_step_z dst_stride_z * number of elements along Z processed per workitem(in bytes) * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the destination tensor - * @param[in] offset The offset to the first valid element of the output tensor in bytes + * @param[in] offsets The offsets to the first valid element of the output tensor in bytes */ __kernel void concatenate_depth( - IMAGE_DECLARATION(src), - IMAGE_DECLARATION(dst), - unsigned int offset) + TENSOR3D_DECLARATION(src), + TENSOR3D_DECLARATION(dst), + int3 offsets) { - Image src = CONVERT_TO_IMAGE_STRUCT(src); - Image dst = CONVERT_TO_IMAGE_STRUCT(dst); + Tensor3D src = CONVERT_TO_TENSOR3D_STRUCT(src); + Tensor3D dst = CONVERT_TO_TENSOR3D_STRUCT(dst); - float4 source_values = vload4(0, (__global float *)src.ptr); + VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE) + source_values = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)tensor3D_offset(&src, -offsets.x, -offsets.y, 0)); - vstore4(source_values, 0, (__global float *)(dst.ptr + offset)); + VSTORE(VEC_SIZE) + (source_values, 0, (__global DATA_TYPE *)(dst.ptr + offsets.z)); } diff --git a/src/core/CL/cl_kernels/helpers.h b/src/core/CL/cl_kernels/helpers.h index 29a43f769b..0b6d92dfd0 100644 --- a/src/core/CL/cl_kernels/helpers.h +++ b/src/core/CL/cl_kernels/helpers.h @@ -30,6 +30,12 @@ #define CLAMP(x, min_val, max_val) min(max(x, min_val), max_val) +#define VLOAD_STR(size) vload##size +#define VLOAD(size) VLOAD_STR(size) + +#define VSTORE_STR(size) vstore##size +#define VSTORE(size) VSTORE_STR(size) + #define VEC_DATA_TYPE_STR(type, size) type##size #define VEC_DATA_TYPE(type, size) VEC_DATA_TYPE_STR(type, size) diff --git a/src/core/CL/kernels/CLDepthConcatenateKernel.cpp b/src/core/CL/kernels/CLDepthConcatenateKernel.cpp index 73f1ba15df..6a699ae710 100644 --- a/src/core/CL/kernels/CLDepthConcatenateKernel.cpp +++ b/src/core/CL/kernels/CLDepthConcatenateKernel.cpp @@ -35,6 +35,10 @@ #include "arm_compute/core/Validate.h" #include "arm_compute/core/Window.h" +#include "support/ToolchainSupport.h" + +#include <map> + using namespace arm_compute; CLDepthConcatenateKernel::CLDepthConcatenateKernel() @@ -49,12 +53,22 @@ BorderSize CLDepthConcatenateKernel::border_size() const void CLDepthConcatenateKernel::configure(const ICLTensor *input, unsigned int depth_offset, ICLTensor *output) { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32); - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::F32); + static std::map<int, std::pair<std::string, int>> configs_map = + { + { 1, { "uchar", 16 } }, + { 2, { "ushort", 8 } }, + { 4, { "uint", 4 } }, + { 8, { "ulong", 2 } }, + }; + + ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32); + ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); + ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT_POSITION(input, output); ARM_COMPUTE_ERROR_ON(input->info()->dimension(2) + depth_offset > output->info()->dimension(2)); ARM_COMPUTE_ERROR_ON(input->info()->dimension(0) > output->info()->dimension(0)); ARM_COMPUTE_ERROR_ON(input->info()->dimension(1) > output->info()->dimension(1)); ARM_COMPUTE_ERROR_ON_MISMATCHING_SHAPES(3, input, output); + ARM_COMPUTE_ERROR_ON(configs_map.find(input->info()->element_size()) == configs_map.end()); // The gaps between the two lowest dimensions of input and output need to be divisible by 2 // Otherwise it is not clear how the padding should be added onto the input tensor @@ -64,33 +78,44 @@ void CLDepthConcatenateKernel::configure(const ICLTensor *input, unsigned int de _input = input; _output = output; + // Add build options + auto config = configs_map.find(static_cast<int>(input->info()->element_size())); + std::set<std::string> build_opts; + build_opts.emplace(("-DDATA_TYPE=" + config->second.first)); + build_opts.emplace(("-DVEC_SIZE=" + support::cpp11::to_string(config->second.second))); + // Create kernel - _kernel = static_cast<cl::Kernel>(CLKernelLibrary::get().create_kernel("concatenate_depth")); + _kernel = static_cast<cl::Kernel>(CLKernelLibrary::get().create_kernel("concatenate_depth", build_opts)); // Configure kernel window _left_right = (output->info()->dimension(0) - input->info()->dimension(0)) / 2; _top_bottom = (output->info()->dimension(1) - input->info()->dimension(1)) / 2; - const unsigned int offset_to_first_elements_in_bytes = depth_offset * output->info()->strides_in_bytes()[2] + _left_right * output->info()->strides_in_bytes()[0] + _top_bottom * - output->info()->strides_in_bytes()[1]; + const int offset_to_first_elements_in_bytes = depth_offset * output->info()->strides_in_bytes()[2]; - const unsigned int num_elems_processed_per_iteration = 4; - const unsigned int num_elems_read_per_iteration = 4; + const unsigned int num_elems_processed_per_iteration = 16 / input->info()->element_size(); + const unsigned int num_elems_read_per_iteration = 16 / input->info()->element_size(); const unsigned int num_rows_read_per_iteration = 1; // The window needs to be based on input as we copy all the depths of input - Window win = calculate_max_enlarged_window(*input->info(), Steps(num_elems_processed_per_iteration), border_size()); + Window win = calculate_max_window(*output->info(), Steps(num_elems_processed_per_iteration)); + win.set(Window::DimZ, Window::Dimension(0, input->info()->tensor_shape().z(), 1)); + AccessWindowRectangle input_access(input->info(), -_left_right, -_top_bottom, num_elems_read_per_iteration, num_rows_read_per_iteration); AccessWindowHorizontal output_access(output->info(), 0, num_elems_processed_per_iteration); - - update_window_and_padding(win, - AccessWindowRectangle(input->info(), -_left_right, -_top_bottom, num_elems_read_per_iteration, num_rows_read_per_iteration), - output_access); - + update_window_and_padding(win, input_access, output_access); output_access.set_valid_region(win, ValidRegion(Coordinates(0, 0), output->info()->tensor_shape())); - unsigned int idx = 2 * num_arguments_per_2D_tensor(); // Skip the input and output parameters - _kernel.setArg<unsigned int>(idx, offset_to_first_elements_in_bytes); + unsigned int idx = 2 * num_arguments_per_3D_tensor(); // Skip the input and output parameters + const cl_int3 offsets = + { + { + static_cast<cl_int>(_left_right), + static_cast<cl_int>(_top_bottom), + static_cast<cl_int>(offset_to_first_elements_in_bytes), + } + }; + _kernel.setArg<cl_int3>(idx, offsets); ICLKernel::configure(win); } @@ -100,14 +125,14 @@ void CLDepthConcatenateKernel::run(const Window &window, cl::CommandQueue &queue ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window); - Window slice = window.first_slice_window_2D(); + Window slice = window.first_slice_window_3D(); do { unsigned int idx = 0; - add_2D_tensor_argument(idx, _input, slice); - add_2D_tensor_argument(idx, _output, slice); + add_3D_tensor_argument(idx, _input, slice); + add_3D_tensor_argument(idx, _output, slice); enqueue(queue, *this, slice); } - while(window.slide_window_slice_2D(slice)); + while(window.slide_window_slice_3D(slice)); } diff --git a/src/core/CL/kernels/CLFillBorderKernel.cpp b/src/core/CL/kernels/CLFillBorderKernel.cpp index 2c751a489c..7667491710 100644 --- a/src/core/CL/kernels/CLFillBorderKernel.cpp +++ b/src/core/CL/kernels/CLFillBorderKernel.cpp @@ -76,7 +76,7 @@ void CLFillBorderKernel::configure(ICLTensor *tensor, BorderSize border_size, Bo // Define select type required by replicate border > 1 const DataType dt = tensor->info()->data_type(); - std::string select_type = get_cl_type_from_data_type(dt); + std::string select_type = get_underlying_cl_type_from_data_type(dt); if(is_data_type_float(dt)) { select_type = (DataType::F32 == dt) ? "int" : "short"; @@ -84,7 +84,7 @@ void CLFillBorderKernel::configure(ICLTensor *tensor, BorderSize border_size, Bo // Define build options std::set<std::string> build_opts; - build_opts.emplace(("-DDATA_TYPE=" + get_cl_type_from_data_type(dt))); + build_opts.emplace(("-DDATA_TYPE=" + get_underlying_cl_type_from_data_type(dt))); build_opts.emplace(("-DSELECT_TYPE=" + select_type)); build_opts.emplace(("-DBORDER_SIZE_TOP=" + support::cpp11::to_string(border_size.top))); build_opts.emplace(("-DBORDER_SIZE_BOTTOM=" + support::cpp11::to_string(border_size.bottom))); @@ -119,9 +119,14 @@ void CLFillBorderKernel::configure(ICLTensor *tensor, BorderSize border_size, Bo case DataType::U8: set_constant_border<uint8_t>(idx, constant_border_value); break; + case DataType::QS8: + case DataType::S8: + set_constant_border<int8_t>(idx, constant_border_value); + break; case DataType::U16: set_constant_border<uint16_t>(idx, constant_border_value); break; + case DataType::QS16: case DataType::S16: set_constant_border<int16_t>(idx, constant_border_value); break; diff --git a/src/core/NEON/kernels/NEDepthConcatenateKernel.cpp b/src/core/NEON/kernels/NEDepthConcatenateKernel.cpp index 902490ec38..d58e4e0aa5 100644 --- a/src/core/NEON/kernels/NEDepthConcatenateKernel.cpp +++ b/src/core/NEON/kernels/NEDepthConcatenateKernel.cpp @@ -27,17 +27,76 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/IAccessWindow.h" #include "arm_compute/core/ITensor.h" +#include "arm_compute/core/NEON/NEFixedPoint.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/Window.h" #include <arm_neon.h> +#include <cstdint> using namespace arm_compute; +namespace +{ +// Overloads of 128-bit vector loads +uint8x16_t loadq(const uint8_t *ptr) +{ + return vld1q_u8(ptr); +} +uint16x8_t loadq(const uint16_t *ptr) +{ + return vld1q_u16(ptr); +} +uint32x4_t loadq(const uint32_t *ptr) +{ + return vld1q_u32(ptr); +} +// Overloads of 128-bit vector stores +void storeq(uint8_t *ptr, uint8x16_t val) +{ + return vst1q_u8(ptr, val); +} +void storeq(uint16_t *ptr, uint16x8_t val) +{ + return vst1q_u16(ptr, val); +} +void storeq(uint32_t *ptr, uint32x4_t val) +{ + return vst1q_u32(ptr, val); +} + +template <typename T> +void depth_concat(const ITensor *in, ITensor *out, std::pair<int, int> start_xy, int depth_offset, const Window &window) +{ + const int start_x = start_xy.first; + const int start_y = start_xy.second; + + // Offset input + const int input_offset_to_first_elements_in_bytes = in->info()->offset_first_element_in_bytes() - start_x * in->info()->strides_in_bytes()[0] - start_y * in->info()->strides_in_bytes()[1]; + uint8_t *input_ptr = in->buffer() + input_offset_to_first_elements_in_bytes; + + // Offset output + const unsigned int output_offset_to_first_elements_in_bytes = out->info()->offset_first_element_in_bytes() + depth_offset * out->info()->strides_in_bytes()[2]; + uint8_t *output_ptr = out->buffer() + output_offset_to_first_elements_in_bytes; + + Iterator input(in, window); + Iterator output(out, window); + + execute_window_loop(window, [&](const Coordinates & id) + { + const auto in_ptr = reinterpret_cast<const T *>(input_ptr + input.offset()); + const auto out_ptr = reinterpret_cast<T *>(output_ptr + output.offset()); + + storeq(out_ptr, loadq(in_ptr)); + }, + input, output); +} +} // namespace + NEDepthConcatenateKernel::NEDepthConcatenateKernel() - : _input(nullptr), _output(nullptr), _top_bottom(0), _left_right(0), _depth_offset(0) + : _func(nullptr), _input(nullptr), _output(nullptr), _top_bottom(0), _left_right(0), _depth_offset(0) { } @@ -48,8 +107,9 @@ BorderSize NEDepthConcatenateKernel::border_size() const void NEDepthConcatenateKernel::configure(const ITensor *input, unsigned int depth_offset, ITensor *output) { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32); - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::F32); + ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::F32); + ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); + ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT_POSITION(input, output); ARM_COMPUTE_ERROR_ON(input->info()->dimension(2) + depth_offset > output->info()->dimension(2)); ARM_COMPUTE_ERROR_ON(input->info()->dimension(0) > output->info()->dimension(0)); ARM_COMPUTE_ERROR_ON(input->info()->dimension(1) > output->info()->dimension(1)); @@ -60,18 +120,36 @@ void NEDepthConcatenateKernel::configure(const ITensor *input, unsigned int dept ARM_COMPUTE_ERROR_ON((output->info()->dimension(0) - input->info()->dimension(0)) % 2); ARM_COMPUTE_ERROR_ON((output->info()->dimension(1) - input->info()->dimension(1)) % 2); + _func = nullptr; _input = input; _output = output; _depth_offset = depth_offset; _left_right = (output->info()->dimension(0) - input->info()->dimension(0)) / 2; _top_bottom = (output->info()->dimension(1) - input->info()->dimension(1)) / 2; - const unsigned int num_elems_processed_per_iteration = 4; - const unsigned int num_elems_read_per_iteration = 4; + switch(input->info()->data_type()) + { + case DataType::QS8: + _func = &depth_concat<uint8_t>; + break; + case DataType::QS16: + case DataType::F16: + _func = &depth_concat<uint16_t>; + break; + case DataType::F32: + _func = &depth_concat<uint32_t>; + break; + default: + ARM_COMPUTE_ERROR("Unsupported data type."); + } + + const unsigned int num_elems_processed_per_iteration = 16 / input->info()->element_size(); + const unsigned int num_elems_read_per_iteration = 16 / input->info()->element_size(); const unsigned int num_rows_read_per_iteration = 1; // The window needs to be based on input as we copy all the depths of input - Window win = calculate_max_enlarged_window(*input->info(), Steps(num_elems_processed_per_iteration), border_size()); + Window win = calculate_max_window(*output->info(), Steps(num_elems_processed_per_iteration)); + win.set(Window::DimZ, Window::Dimension(0, input->info()->tensor_shape().z(), 1)); AccessWindowRectangle input_access(input->info(), -_left_right, -_top_bottom, num_elems_read_per_iteration, num_rows_read_per_iteration); AccessWindowHorizontal output_access(output->info(), 0, num_elems_processed_per_iteration); @@ -85,21 +163,7 @@ void NEDepthConcatenateKernel::run(const Window &window) { ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INEKernel::window(), window); + ARM_COMPUTE_ERROR_ON(_func == nullptr); - // Offset output - const unsigned int offset_to_first_elements_in_bytes = _output->info()->offset_first_element_in_bytes() + _left_right * _output->info()->strides_in_bytes()[0] + _top_bottom * - _output->info()->strides_in_bytes()[1] + _depth_offset * _output->info()->strides_in_bytes()[2]; - uint8_t *output_ptr = _output->buffer() + offset_to_first_elements_in_bytes; - - Iterator input(_input, window); - Iterator output(_output, window); - - execute_window_loop(window, [&](const Coordinates & id) - { - const auto in_ptr = reinterpret_cast<const float *>(input.ptr()); - const auto out_ptr = reinterpret_cast<float *>(output_ptr + output.offset()); - - vst1q_f32(out_ptr, vld1q_f32(in_ptr)); - }, - input, output); + (*_func)(_input, _output, std::make_pair(_left_right, _top_bottom), _depth_offset, window); } |