From 6a14adb2faefd2f4638647595dfdef12e29f8a01 Mon Sep 17 00:00:00 2001 From: Pablo Tello Date: Tue, 5 Mar 2019 17:33:08 +0000 Subject: COMPMID-1932: Implement CLHeightConcatenateLayer. Change-Id: I244ff2352c3c9620148dab6475aee62b218b96ef Signed-off-by: Pablo Tello Reviewed-on: https://review.mlplatform.org/c/871 Tested-by: Arm Jenkins Comments-Addressed: Arm Jenkins Reviewed-by: Gian Marco Iodice --- src/core/CL/CLKernelLibrary.cpp | 1 + src/core/CL/cl_kernels/concatenate.cl | 54 ++++++- .../CL/kernels/CLHeightConcatenateLayerKernel.cpp | 160 +++++++++++++++++++++ src/runtime/CL/functions/CLConcatenateLayer.cpp | 105 +++++++++++++- src/runtime/NEON/functions/NEConcatenateLayer.cpp | 2 +- 5 files changed, 313 insertions(+), 9 deletions(-) create mode 100644 src/core/CL/kernels/CLHeightConcatenateLayerKernel.cpp (limited to 'src') diff --git a/src/core/CL/CLKernelLibrary.cpp b/src/core/CL/CLKernelLibrary.cpp index 5af8a09723..be457a74db 100644 --- a/src/core/CL/CLKernelLibrary.cpp +++ b/src/core/CL/CLKernelLibrary.cpp @@ -190,6 +190,7 @@ const std::map CLKernelLibrary::_kernel_program_map = { "compare_lessequal_quantized", "comparisons.cl" }, { "concatenate_depth", "concatenate.cl" }, { "concatenate_width", "concatenate.cl" }, + { "concatenate_height", "concatenate.cl" }, { "concatenate_width_x2", "concatenate.cl" }, { "concatenate_width_x4", "concatenate.cl" }, { "convolution_rectangle", "convolution_rectangle.cl" }, diff --git a/src/core/CL/cl_kernels/concatenate.cl b/src/core/CL/cl_kernels/concatenate.cl index c374769423..99f4659960 100644 --- a/src/core/CL/cl_kernels/concatenate.cl +++ b/src/core/CL/cl_kernels/concatenate.cl @@ -330,6 +330,59 @@ __kernel void concatenate_width( #endif /* defined(WIDTH_OFFSET) && defined(DEPTH) */ +#if defined(HEIGHT_OFFSET) && defined(DEPTH) && defined(VEC_SIZE) +/** This kernel concatenates the input tensor into the output tensor along the second dimension + * + * @note The data type has to be passed at compile time using -DDATA_TYPE. i.e. -DDATA_TYPE=float + * @note Vector size has to be passed at compile time using -DVEC_SIZE. i.e. -DVEC_SIZE=16 + * @note Vector sizes supported are 2,4,8 and 16. + * @note The offset for the second spatial dimension has to be passed at compile time using -DHEIGHT_OFFSET. i.e. -DHEIGHT_OFFSET=128 + * @note Tensor depth should be given as a preprocessor argument using -DDEPTH=size. e.g. -DDEPTH=16 + * + * @param[in] src_ptr Pointer to the source tensor. Supported data types: U8/S8/QASYMM8/U16/S16/F16/U32/F32 + * @param[in] src_stride_x Stride of the source tensor in X dimension (in bytes) + * @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes) + * @param[in] src_stride_y Stride of the source tensor in Y dimension (in bytes) + * @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes) + * @param[in] src_stride_z Stride of the source tensor in Z dimension (in bytes) + * @param[in] src_step_z src_stride_z * number of elements along Z processed per workitem(in bytes) + * @param[in] src_stride_w Stride of the first source tensor in Z dimension (in bytes) + * @param[in] src_step_w src_stride_z * number of elements along Z processed per workitem(in bytes) + * @param[in] src_offset_first_element_in_bytes The offset of the first element in the source tensor + * @param[out] dst_ptr Pointer to the destination tensor. Supported data types: same as @p src_ptr + * @param[in] dst_stride_x Stride of the destination tensor in X dimension (in bytes) + * @param[in] dst_step_x dst_stride_x * number of elements along X processed per workitem(in bytes) + * @param[in] dst_stride_y Stride of the destination tensor in Y dimension (in bytes) + * @param[in] dst_step_y dst_stride_y * number of elements along Y processed per workitem(in bytes) + * @param[in] dst_stride_z Stride of the source tensor in Z dimension (in bytes) + * @param[in] dst_step_z dst_stride_z * number of elements along Z processed per workitem(in bytes) + * @param[in] dst_stride_w Stride of the destination tensor in Z dimension (in bytes) + * @param[in] dst_step_w output_stride_z * number of elements along Z processed per workitem(in bytes) + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the destination tensor + */ + +__kernel void concatenate_height( + TENSOR4D_DECLARATION(src), + TENSOR4D_DECLARATION(dst)) +{ + Tensor4D src = CONVERT_TO_TENSOR4D_STRUCT(src, DEPTH); + Tensor4D dst = CONVERT_TO_TENSOR4D_STRUCT(dst, DEPTH); + + VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE) + source_values = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)src.ptr); + +#if defined(OFFSET_IN1) && defined(OFFSET_OUT) && defined(SCALE_IN1) && defined(SCALE_OUT) + const VEC_UCHAR out = requantize(source_values, OFFSET_IN1, OFFSET_OUT, SCALE_IN1, SCALE_OUT); + VSTORE(VEC_SIZE) + (out, 0, (__global DATA_TYPE *)(dst.ptr + HEIGHT_OFFSET * dst_stride_y)); +#else /* defined(OFFSET_IN1) && defined(OFFSET_OUT) && defined(SCALE_IN1) && defined(SCALE_OUT) */ + VSTORE(VEC_SIZE) + (source_values, 0, (__global DATA_TYPE *)(dst.ptr + HEIGHT_OFFSET * dst_stride_y)); +#endif /* defined(OFFSET_IN1) && defined(OFFSET_OUT) && defined(SCALE_IN1) && defined(SCALE_OUT) */ +} + +#endif /* defined(HEIGHT_OFFSET) && defined(DEPTH) */ + /** This kernel concatenates the input tensor into the output tensor along the third dimension * * @note The data type has to be passed at compile time using -DDATA_TYPE. i.e. -DDATA_TYPE=float @@ -370,6 +423,5 @@ __kernel void concatenate_depth( VSTORE(VEC_SIZE) (source_values, 0, (__global DATA_TYPE *)(dst.ptr + offsets.z)); - } #endif /* defined(DATA_TYPE) && defined(VEC_SIZE) */ diff --git a/src/core/CL/kernels/CLHeightConcatenateLayerKernel.cpp b/src/core/CL/kernels/CLHeightConcatenateLayerKernel.cpp new file mode 100644 index 0000000000..e3f2a96281 --- /dev/null +++ b/src/core/CL/kernels/CLHeightConcatenateLayerKernel.cpp @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/core/CL/kernels/CLHeightConcatenateLayerKernel.h" + +#include "arm_compute/core/CL/CLHelpers.h" +#include "arm_compute/core/CL/CLKernelLibrary.h" +#include "arm_compute/core/CL/CLValidate.h" +#include "arm_compute/core/CL/ICLTensor.h" +#include "arm_compute/core/CL/OpenCL.h" +#include "arm_compute/core/Error.h" +#include "arm_compute/core/Helpers.h" +#include "arm_compute/core/IAccessWindow.h" +#include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/Window.h" +#include "arm_compute/core/utils/misc/ShapeCalculator.h" + +#include "support/ToolchainSupport.h" + +#include + +using namespace arm_compute; +namespace +{ +std::pair validate_and_configure_window(ITensorInfo *input, unsigned int height_offset, ITensorInfo *output, unsigned int &num_elems_processed_per_iteration) +{ + num_elems_processed_per_iteration = 4; + // The window needs to be based on input as we copy all the heights of input + Window win = calculate_max_window(*input, Steps(num_elems_processed_per_iteration)); + AccessWindowHorizontal input_access(input, 0, num_elems_processed_per_iteration); + AccessWindowHorizontal output_access(output, height_offset, num_elems_processed_per_iteration); + bool window_changed = update_window_and_padding(win, input_access, output_access); + + Window win_collapsed = win.collapse(win, Window::DimZ); + + Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{}; + return std::make_pair(err, win_collapsed); +} +Status validate_arguments(const ITensorInfo *input, unsigned int height_offset, const ITensorInfo *output) +{ + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, output); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::U8, DataType::S8, DataType::QASYMM8, DataType::U16, DataType::S16, DataType::F16, DataType::U32, + DataType::F32); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); + ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(Window::DimY) + height_offset > output->dimension(Window::DimY)); + + ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(0) != output->dimension(0)); + for(size_t i = 2; i < Coordinates::num_max_dimensions; ++i) + { + ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(i) != output->dimension(i)); + } + ARM_COMPUTE_RETURN_ERROR_ON(input->num_dimensions() > 4); + + return Status{}; +} +} // namespace + +CLHeightConcatenateLayerKernel::CLHeightConcatenateLayerKernel() + : _input(nullptr), _output(nullptr), _height_offset(0), _num_elems_processed_per_iteration() +{ +} + +Status CLHeightConcatenateLayerKernel::validate(const ITensorInfo *input, unsigned int height_offset, const ITensorInfo *output) +{ + unsigned int num_elems_processed_per_iteration; + ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, height_offset, output)); + ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(input->clone().get(), height_offset, output->clone().get(), num_elems_processed_per_iteration).first); + return Status{}; +} + +void CLHeightConcatenateLayerKernel::configure(const ICLTensor *input, unsigned int height_offset, ICLTensor *output) +{ + ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), height_offset, output->info())); + + _input = input; + _output = output; + _height_offset = height_offset; + + auto win_config = validate_and_configure_window(input->info(), height_offset, output->info(), _num_elems_processed_per_iteration); + + // Add build options + CLBuildOptions build_opts; + + switch(input->info()->element_size()) + { + case 1: + { + build_opts.add_option("-DDATA_TYPE=uchar"); + break; + } + case 2: + { + build_opts.add_option("-DDATA_TYPE=short"); + break; + } + case 4: + { + build_opts.add_option("-DDATA_TYPE=int"); + break; + } + default: + { + ARM_COMPUTE_ERROR("Unsupported input data type."); + break; + } + } + + build_opts.add_option("-DVEC_SIZE=" + support::cpp11::to_string(_num_elems_processed_per_iteration)); + build_opts.add_option("-DHEIGHT_OFFSET=" + support::cpp11::to_string(_height_offset)); + build_opts.add_option("-DDEPTH=" + support::cpp11::to_string(input->info()->dimension(2))); + + if(is_data_type_quantized_asymmetric(input->info()->data_type()) && input->info()->quantization_info() != output->info()->quantization_info()) + { + build_opts.add_option("-DOFFSET_IN1=" + float_to_string_with_full_precision(input->info()->quantization_info().offset)); + build_opts.add_option("-DOFFSET_OUT=" + float_to_string_with_full_precision(output->info()->quantization_info().offset)); + build_opts.add_option("-DSCALE_IN1=" + float_to_string_with_full_precision(input->info()->quantization_info().scale)); + build_opts.add_option("-DSCALE_OUT=" + float_to_string_with_full_precision(output->info()->quantization_info().scale)); + } + + // Create kernel + _kernel = static_cast(CLKernelLibrary::get().create_kernel("concatenate_height", build_opts.options())); + // Configure kernel window + + ARM_COMPUTE_ERROR_THROW_ON(std::get<0>(win_config)); + + ICLKernel::configure_internal(std::get<1>(win_config)); +} + +void CLHeightConcatenateLayerKernel::run(const Window &window, cl::CommandQueue &queue) +{ + ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); + ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window); + + unsigned int idx = 0; + add_4D_tensor_argument(idx, _input, window); + add_4D_tensor_argument(idx, _output, window); + enqueue(queue, *this, window); +} diff --git a/src/runtime/CL/functions/CLConcatenateLayer.cpp b/src/runtime/CL/functions/CLConcatenateLayer.cpp index 018c674c83..13164fdff9 100644 --- a/src/runtime/CL/functions/CLConcatenateLayer.cpp +++ b/src/runtime/CL/functions/CLConcatenateLayer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 ARM Limited. + * Copyright (c) 2018-2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,9 @@ */ #include "arm_compute/runtime/CL/functions/CLConcatenateLayer.h" +#include "arm_compute/core/CL/kernels/CLHeightConcatenateLayerKernel.h" +#include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/runtime/CL/CLScheduler.h" #include "arm_compute/runtime/CL/functions/CLDepthConcatenateLayer.h" #include "arm_compute/runtime/CL/functions/CLWidthConcatenateLayer.h" @@ -35,15 +38,74 @@ namespace arm_compute { CLConcatenateLayer::CLConcatenateLayer() - : _concat_function(nullptr) + : _concat_function(nullptr), + _hconcat_kernels(), + _num_inputs(0), + _axis(Window::DimX) { } +Status CLConcatenateLayer::validate_h_concatenate(const std::vector &inputs_vector, const ITensorInfo *output) // NOLINT +{ + const unsigned int num_inputs = inputs_vector.size(); + + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(output); + ARM_COMPUTE_RETURN_ERROR_ON(num_inputs < 2); + + // Output auto inizialitation if not yet initialized + TensorInfo tmp_output_info = *output->clone(); + const TensorShape output_shape = arm_compute::misc::shape_calculator::calculate_concatenate_shape(inputs_vector, Window::DimY); + auto_init_if_empty(tmp_output_info, output_shape, 1, inputs_vector[0]->data_type()); + + unsigned int height_offset = 0; + // Validate generic case of WidthConcatenate kernel + for(const auto &input : inputs_vector) + { + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input); + ARM_COMPUTE_RETURN_ON_ERROR(CLHeightConcatenateLayerKernel::validate(input, height_offset, &tmp_output_info)); + height_offset += input->dimension(Window::DimY); + } + + return Status{}; +} + +void CLConcatenateLayer::configure_h_concatenate(std::vector inputs_vector, ICLTensor *output) // NOLINT +{ + _num_inputs = inputs_vector.size(); + + std::vector inputs_vector_info(inputs_vector.size()); + std::transform(inputs_vector.begin(), inputs_vector.end(), inputs_vector_info.begin(), [](ICLTensor * t) + { + ARM_COMPUTE_ERROR_ON_NULLPTR(t); + return t->info(); + }); + + const TensorShape output_shape = arm_compute::misc::shape_calculator::calculate_concatenate_shape(inputs_vector, Window::DimY); + + // Output auto inizialitation if not yet initialized + auto_init_if_empty(*output->info(), output_shape, 1, inputs_vector[0]->info()->data_type()); + + ARM_COMPUTE_ERROR_THROW_ON(CLConcatenateLayer::validate_h_concatenate(inputs_vector_info, output->info())); + + // Configure generic case WidthConcatenate kernels + _hconcat_kernels = arm_compute::support::cpp14::make_unique(_num_inputs); + + unsigned int height_offset = 0; + unsigned int i = 0; + std::transform(inputs_vector.begin(), inputs_vector.end(), inputs_vector.begin(), [&](ICLTensor * t) + { + auto &kernel = _hconcat_kernels[i++]; + kernel.configure(t, height_offset, output); + height_offset += t->info()->dimension(Window::DimY); + return t; + }); +} + void CLConcatenateLayer::configure(const std::vector &inputs_vector, ICLTensor *output, DataLayoutDimension axis) { ARM_COMPUTE_ERROR_ON(output == nullptr); - - switch(get_data_layout_dimension_index(output->info()->data_layout(), axis)) + _axis = get_data_layout_dimension_index(output->info()->data_layout(), axis); + switch(_axis) { case 0: { @@ -52,6 +114,11 @@ void CLConcatenateLayer::configure(const std::vector &inputs_vector _concat_function = std::move(func); break; } + case 1: + { + configure_h_concatenate(inputs_vector, output); + break; + } case 2: { auto func = support::cpp14::make_unique(); @@ -60,7 +127,7 @@ void CLConcatenateLayer::configure(const std::vector &inputs_vector break; } default: - ARM_COMPUTE_ERROR("Concatenation is supported across width and depth only!"); + ARM_COMPUTE_ERROR("Concatenation is supported across width, height and depth only!"); } } @@ -73,6 +140,9 @@ Status CLConcatenateLayer::validate(const std::vector &inputs_vec case 0: ARM_COMPUTE_RETURN_ON_ERROR(CLWidthConcatenateLayer::validate(inputs_vector, output)); break; + case 1: + ARM_COMPUTE_RETURN_ON_ERROR(CLConcatenateLayer::validate_h_concatenate(inputs_vector, output)); + break; case 2: ARM_COMPUTE_RETURN_ON_ERROR(CLDepthConcatenateLayer::validate(inputs_vector, output)); break; @@ -84,7 +154,28 @@ Status CLConcatenateLayer::validate(const std::vector &inputs_vec void CLConcatenateLayer::run() { - ARM_COMPUTE_ERROR_ON(_concat_function == nullptr); - _concat_function->run(); + switch(_axis) + { + case 0: + case 2: + { + ARM_COMPUTE_ERROR_ON(_concat_function == nullptr); + _concat_function->run(); + break; + } + case 1: + { + for(unsigned int i = 0; i < _num_inputs; ++i) + { + CLScheduler::get().enqueue(_hconcat_kernels[i], true); + } + break; + } + default: + { + ARM_COMPUTE_ERROR("Axis not supported"); + break; + } + } } } // namespace arm_compute diff --git a/src/runtime/NEON/functions/NEConcatenateLayer.cpp b/src/runtime/NEON/functions/NEConcatenateLayer.cpp index f764a126a0..1897915d33 100644 --- a/src/runtime/NEON/functions/NEConcatenateLayer.cpp +++ b/src/runtime/NEON/functions/NEConcatenateLayer.cpp @@ -119,7 +119,7 @@ void NEConcatenateLayer::configure(const std::vector &inputs_vector, break; } default: - ARM_COMPUTE_ERROR("Concatenation is supported across width and depth only!"); + ARM_COMPUTE_ERROR("Concatenation is supported across width, height and depth only!"); } } -- cgit v1.2.1