From de5a1cc7e5c929b19fb1d3ed7d0d8783b9ac6860 Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Fri, 2 Feb 2018 12:52:07 +0000 Subject: COMPMID-856: CL Depthwise Convolution QASYMM8 support Change-Id: Ic6097e7cf160e8b829fb521b7b99d9a57d9799d3 Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/118774 Tested-by: Jenkins Reviewed-by: Anthony Barbier --- src/core/CL/kernels/CLDepthwiseIm2ColKernel.cpp | 42 ++-- .../CL/kernels/CLDepthwiseVectorToTensorKernel.cpp | 14 +- .../CL/kernels/CLDepthwiseWeightsReshapeKernel.cpp | 5 +- .../CLDirectConvolutionOutputStageKernel.cpp | 217 +++++++++++++++++++++ .../kernels/CLGEMMMatrixVectorMultiplyKernel.cpp | 28 ++- 5 files changed, 269 insertions(+), 37 deletions(-) create mode 100644 src/core/CL/kernels/CLDirectConvolutionOutputStageKernel.cpp (limited to 'src/core/CL/kernels') diff --git a/src/core/CL/kernels/CLDepthwiseIm2ColKernel.cpp b/src/core/CL/kernels/CLDepthwiseIm2ColKernel.cpp index ad9ac0ecd6..8467b39910 100644 --- a/src/core/CL/kernels/CLDepthwiseIm2ColKernel.cpp +++ b/src/core/CL/kernels/CLDepthwiseIm2ColKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -44,9 +44,10 @@ CLDepthwiseIm2ColKernel::CLDepthwiseIm2ColKernel() void CLDepthwiseIm2ColKernel::configure(const ICLTensor *input, ICLTensor *output, const Size2D &kernel_dims, const PadStrideInfo &conv_info, bool has_bias) { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32); + ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::F16, DataType::F32); ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, output); + ARM_COMPUTE_ERROR_ON(is_data_type_quantized_asymmetric(input->info()->data_type()) && has_bias); ARM_COMPUTE_ERROR_ON(input->info()->dimension(2) != output->info()->dimension(2)); ARM_COMPUTE_ERROR_ON(output->info()->dimension(0) != (kernel_dims.width * kernel_dims.height + ((has_bias) ? 1 : 0))); @@ -54,24 +55,25 @@ void CLDepthwiseIm2ColKernel::configure(const ICLTensor *input, ICLTensor *outpu _output = output; // Create kernel - std::set build_opts; - - build_opts.emplace("-DDATA_TYPE=" + get_cl_type_from_data_type(input->info()->data_type())); - build_opts.emplace("-DSTRIDE_X=" + support::cpp11::to_string(conv_info.stride().first)); - build_opts.emplace("-DSTRIDE_Y=" + support::cpp11::to_string(conv_info.stride().second)); - build_opts.emplace("-DPAD_LEFT=" + support::cpp11::to_string(conv_info.pad_left())); - build_opts.emplace("-DPAD_TOP=" + support::cpp11::to_string(conv_info.pad_top())); - build_opts.emplace("-DPAD_RIGHT=" + support::cpp11::to_string(conv_info.pad_right())); - build_opts.emplace("-DPAD_BOTTOM=" + support::cpp11::to_string(conv_info.pad_bottom())); - build_opts.emplace("-DSRC_WIDTH=" + support::cpp11::to_string(input->info()->dimension(0))); - build_opts.emplace("-DSRC_HEIGHT=" + support::cpp11::to_string(input->info()->dimension(1))); - build_opts.emplace("-DKERNEL_WIDTH=" + support::cpp11::to_string(kernel_dims.width)); - build_opts.emplace("-DKERNEL_HEIGHT=" + support::cpp11::to_string(kernel_dims.height)); - if(has_bias) - { - build_opts.emplace("-DHAS_BIAS"); - } - _kernel = static_cast(CLKernelLibrary::get().create_kernel("depthwise_im2col", build_opts)); + CLBuildOptions build_opts; + + build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(input->info()->data_type())); + build_opts.add_option("-DSTRIDE_X=" + support::cpp11::to_string(conv_info.stride().first)); + build_opts.add_option("-DSTRIDE_Y=" + support::cpp11::to_string(conv_info.stride().second)); + build_opts.add_option("-DPAD_LEFT=" + support::cpp11::to_string(conv_info.pad_left())); + build_opts.add_option("-DPAD_TOP=" + support::cpp11::to_string(conv_info.pad_top())); + build_opts.add_option("-DPAD_RIGHT=" + support::cpp11::to_string(conv_info.pad_right())); + build_opts.add_option("-DPAD_BOTTOM=" + support::cpp11::to_string(conv_info.pad_bottom())); + build_opts.add_option("-DSRC_WIDTH=" + support::cpp11::to_string(input->info()->dimension(0))); + build_opts.add_option("-DSRC_HEIGHT=" + support::cpp11::to_string(input->info()->dimension(1))); + build_opts.add_option("-DKERNEL_WIDTH=" + support::cpp11::to_string(kernel_dims.width)); + build_opts.add_option("-DKERNEL_HEIGHT=" + support::cpp11::to_string(kernel_dims.height)); + build_opts.add_option_if(has_bias, "-DHAS_BIAS"); + build_opts.add_option_if_else(is_data_type_quantized(input->info()->data_type()), + "-DPAD_VALUE=" + support::cpp11::to_string(input->info()->quantization_info().offset), + "-DPAD_VALUE=0"); + + _kernel = static_cast(CLKernelLibrary::get().create_kernel("depthwise_im2col", build_opts.options())); // Configure the local work size for Bifrost with a value obtained // via exhaustive autotuning for the MobileNets tensor shapes. diff --git a/src/core/CL/kernels/CLDepthwiseVectorToTensorKernel.cpp b/src/core/CL/kernels/CLDepthwiseVectorToTensorKernel.cpp index dc47bb0adc..ae35bf64aa 100644 --- a/src/core/CL/kernels/CLDepthwiseVectorToTensorKernel.cpp +++ b/src/core/CL/kernels/CLDepthwiseVectorToTensorKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -41,7 +41,7 @@ CLDepthwiseVectorToTensorKernel::CLDepthwiseVectorToTensorKernel() void CLDepthwiseVectorToTensorKernel::configure(const ICLTensor *input, ICLTensor *output, size_t conv_w, size_t conv_h) { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32); + ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::S32, DataType::F16, DataType::F32); ARM_COMPUTE_ERROR_ON_NULLPTR(output); TensorShape output_shape = input->info()->tensor_shape(); @@ -60,12 +60,12 @@ void CLDepthwiseVectorToTensorKernel::configure(const ICLTensor *input, ICLTenso _output = output; // Create kernel - std::set build_opts; - build_opts.emplace("-DDATA_TYPE=" + get_cl_type_from_data_type(input->info()->data_type())); - build_opts.emplace("-DCONV_WIDTH=" + support::cpp11::to_string(conv_w)); - build_opts.emplace("-DCONV_HEIGHT=" + support::cpp11::to_string(conv_h)); + CLBuildOptions build_opts; + build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(input->info()->data_type())); + build_opts.add_option("-DCONV_WIDTH=" + support::cpp11::to_string(conv_w)); + build_opts.add_option("-DCONV_HEIGHT=" + support::cpp11::to_string(conv_h)); - _kernel = static_cast(CLKernelLibrary::get().create_kernel("depthwise_vector_to_tensor", build_opts)); + _kernel = static_cast(CLKernelLibrary::get().create_kernel("depthwise_vector_to_tensor", build_opts.options())); // Configure kernel window Window win = calculate_max_window(*input->info(), Steps()); diff --git a/src/core/CL/kernels/CLDepthwiseWeightsReshapeKernel.cpp b/src/core/CL/kernels/CLDepthwiseWeightsReshapeKernel.cpp index 81dd6b42cc..26da96f9ba 100644 --- a/src/core/CL/kernels/CLDepthwiseWeightsReshapeKernel.cpp +++ b/src/core/CL/kernels/CLDepthwiseWeightsReshapeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -41,9 +41,10 @@ CLDepthwiseWeightsReshapeKernel::CLDepthwiseWeightsReshapeKernel() void CLDepthwiseWeightsReshapeKernel::configure(const ICLTensor *input, ICLTensor *output, const ICLTensor *biases) { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32); + ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::F16, DataType::F32); ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input, output); + ARM_COMPUTE_ERROR_ON(is_data_type_quantized_asymmetric(input->info()->data_type()) && (biases != nullptr)); ARM_COMPUTE_ERROR_ON(input->info()->dimension(2) != output->info()->dimension(1)); ARM_COMPUTE_ERROR_ON(output->info()->dimension(0) != (input->info()->dimension(0) * input->info()->dimension(1) + ((biases != nullptr) ? 1 : 0))); diff --git a/src/core/CL/kernels/CLDirectConvolutionOutputStageKernel.cpp b/src/core/CL/kernels/CLDirectConvolutionOutputStageKernel.cpp new file mode 100644 index 0000000000..cbc281b6ac --- /dev/null +++ b/src/core/CL/kernels/CLDirectConvolutionOutputStageKernel.cpp @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/core/CL/kernels/CLDirectConvolutionLayerOutputStageKernel.h" + +#include "arm_compute/core/AccessWindowStatic.h" +#include "arm_compute/core/CL/ICLTensor.h" +#include "arm_compute/core/Error.h" +#include "arm_compute/core/Helpers.h" +#include "arm_compute/core/Types.h" +#include "arm_compute/core/Validate.h" +#include "arm_compute/core/Window.h" + +#include +#include + +using namespace arm_compute; + +namespace +{ +Status validate_arguments(const ITensorInfo *input, const ITensorInfo *bias, const ITensorInfo *output) +{ + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::S32, DataType::F16, + DataType::F32); + + if(bias != nullptr) + { + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(bias, 1, DataType::S32, DataType::F16, DataType::F32); + + if(is_data_type_quantized_asymmetric(input->data_type())) + { + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(bias, 1, DataType::S32); + } + else + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, bias); + } + + ARM_COMPUTE_RETURN_ERROR_ON(bias->num_dimensions() > 1); + } + else + { + ARM_COMPUTE_RETURN_ERROR_ON_MSG(is_data_type_float(input->data_type()), + "Calling output stage kernel with floating point arguments"); + } + + // Checks performed on output + if(input->data_type() == DataType::S32) + { + // Quantized configuration checks + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::QASYMM8); + } + else + { + // In case of out-of-place computation (supported for non-quantized configurations) + if((output != nullptr) && (output->total_size() != 0)) + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); + } + } + + return Status{}; +} + +std::pair validate_and_configure_window(ITensorInfo *input, ITensorInfo *bias, ITensorInfo *output) +{ + bool window_changed = false; + unsigned int num_elems_processed_per_iteration = 16 / element_size_from_data_type(input->data_type()); + + // Update processed elements when input is S32 (comes from quantization input) + if(input->data_type() == DataType::S32) + { + num_elems_processed_per_iteration = 16; + } + + // Configure kernel window + Window win = calculate_max_window(*input, Steps(num_elems_processed_per_iteration)); + AccessWindowHorizontal input_access(input, 0, num_elems_processed_per_iteration); + + if(output != nullptr && (output->total_size() != 0)) + { + AccessWindowHorizontal output_access(output, 0, num_elems_processed_per_iteration); + + if(bias == nullptr) + { + window_changed = update_window_and_padding(win, input_access, output_access); + } + else + { + AccessWindowStatic bias_access(bias, 0, 0, bias->dimension(0), bias->dimension(1)); + window_changed = update_window_and_padding(win, input_access, output_access, bias_access); + } + + output_access.set_valid_region(win, ValidRegion(Coordinates(), output->tensor_shape())); + } + else + { + if(bias == nullptr) + { + window_changed = update_window_and_padding(win, input_access); + } + else + { + AccessWindowStatic bias_access(bias, 0, 0, bias->dimension(0), bias->dimension(1)); + window_changed = update_window_and_padding(win, input_access, bias_access); + } + + input_access.set_valid_region(win, ValidRegion(Coordinates(), input->tensor_shape())); + } + + Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{}; + return std::make_pair(err, win); +} +} // namespace + +CLDirectConvolutionLayerOutputStageKernel::CLDirectConvolutionLayerOutputStageKernel() + : _input(nullptr), _bias(nullptr), _output(nullptr), _result_fixedpoint_multiplier(0), _result_shift(0), _result_offset_after_shift(0) +{ +} + +void CLDirectConvolutionLayerOutputStageKernel::configure(ICLTensor *input, const ICLTensor *bias, ICLTensor *output, + int result_fixedpoint_multiplier, int result_shift, int result_offset_after_shift) +{ + ARM_COMPUTE_ERROR_ON_NULLPTR(input); + + // Auto-initialize output output if required + if(output != nullptr) + { + // Work out expected output data type + const DataType output_dt = (input->info()->data_type() == DataType::S32) ? DataType::QASYMM8 : input->info()->data_type(); + // Output tensor auto initialization if not yet initialized + auto_init_if_empty(*output->info(), input->info()->clone()->set_data_type(output_dt)); + } + + // Perform validation step + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), (bias == nullptr) ? nullptr : bias->info(), (output == nullptr) ? nullptr : output->info())); + + _bias = bias; + _input = input; + _output = output; + _result_fixedpoint_multiplier = result_fixedpoint_multiplier; + _result_shift = result_shift; + _result_offset_after_shift = result_offset_after_shift; + + // Create kernel + CLBuildOptions build_opts; + build_opts.add_option_if(bias != nullptr, "-DHAS_BIAS"); + _kernel = static_cast(CLKernelLibrary::get().create_kernel("output_stage_quantized", build_opts.options())); + + // Set static kernel arguments + int idx = 2 * num_arguments_per_3D_tensor() + ((bias != nullptr) ? num_arguments_per_1D_tensor() : 0); + _kernel.setArg(idx++, _result_offset_after_shift); + _kernel.setArg(idx++, _result_fixedpoint_multiplier); + _kernel.setArg(idx++, _result_shift); + + // Configure kernel window + auto win_config = validate_and_configure_window(input->info(), (bias == nullptr) ? nullptr : bias->info(), (output == nullptr) ? nullptr : output->info()); + ARM_COMPUTE_ERROR_THROW_ON(win_config.first); + ICLKernel::configure(win_config.second); +} + +Status CLDirectConvolutionLayerOutputStageKernel::validate(const ITensorInfo *input, const ITensorInfo *bias, const ITensorInfo *output) +{ + ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, bias, output)); + ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(input->clone().get(), bias->clone().get(), output == nullptr ? nullptr : output->clone().get()).first); + + return Status{}; +} + +void CLDirectConvolutionLayerOutputStageKernel::run(const Window &window, cl::CommandQueue &queue) +{ + ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); + ARM_COMPUTE_ERROR_ON_MISMATCHING_WINDOWS(ICLKernel::window(), window); + + Window slice = window.first_slice_window_3D(); + + // Set bias vector + if(_bias != nullptr) + { + unsigned int idx1 = 2 * num_arguments_per_3D_tensor(); + Window slice_biases; + slice_biases.use_tensor_dimensions(_bias->info()->tensor_shape()); + add_1D_tensor_argument(idx1, _bias, slice_biases); + } + + // Run kernel + do + { + // Set arguments + unsigned int idx = 0; + add_3D_tensor_argument(idx, _input, slice); + add_3D_tensor_argument(idx, _output, slice); + enqueue(queue, *this, slice, _lws_hint); + } + while(window.slide_window_slice_3D(slice)); +} diff --git a/src/core/CL/kernels/CLGEMMMatrixVectorMultiplyKernel.cpp b/src/core/CL/kernels/CLGEMMMatrixVectorMultiplyKernel.cpp index 951bc144aa..cc483dc44e 100644 --- a/src/core/CL/kernels/CLGEMMMatrixVectorMultiplyKernel.cpp +++ b/src/core/CL/kernels/CLGEMMMatrixVectorMultiplyKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -45,23 +45,35 @@ BorderSize CLGEMMMatrixVectorMultiplyKernel::border_size() const void CLGEMMMatrixVectorMultiplyKernel::configure(const ICLTensor *input0, const ICLTensor *input1, ICLTensor *output) { - ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input0, 1, DataType::F16, DataType::F32); - ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input0, input1, output); + ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input0, 1, DataType::QASYMM8, DataType::F16, DataType::F32); + ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input0, input1); ARM_COMPUTE_ERROR_ON_MISMATCHING_FIXED_POINT(input0, input1, output); + ARM_COMPUTE_ERROR_ON(is_data_type_quantized_asymmetric(input0->info()->data_type()) && (output->info()->data_type() != DataType::S32)); ARM_COMPUTE_ERROR_ON(input0->info()->dimension(2) != input1->info()->dimension(1)); _input0 = input0; _input1 = input1; _output = output; + // Check if is a quantized operation + bool is_quantized = is_data_type_quantized_asymmetric(_input0->info()->data_type()); + // Create kernel - std::set build_opts; + CLBuildOptions build_opts; + build_opts.add_option_if(!is_quantized, "-DDATA_TYPE=" + get_cl_type_from_data_type(input0->info()->data_type())); + build_opts.add_option("-DSRC_WIDTH=" + support::cpp11::to_string(input0->info()->dimension(0))); + build_opts.add_option("-DSRC_HEIGHT=" + support::cpp11::to_string(input0->info()->dimension(1))); - build_opts.emplace("-DDATA_TYPE=" + get_cl_type_from_data_type(input0->info()->data_type())); - build_opts.emplace("-DSRC_WIDTH=" + support::cpp11::to_string(input0->info()->dimension(0))); - build_opts.emplace("-DSRC_HEIGHT=" + support::cpp11::to_string(input0->info()->dimension(1))); + std::string kernel_name = is_quantized ? std::string("gemm_mv_quantized") : std::string("gemm_mv"); + _kernel = static_cast(CLKernelLibrary::get().create_kernel(kernel_name, build_opts.options())); - _kernel = static_cast(CLKernelLibrary::get().create_kernel("gemm_mv", build_opts)); + // Add static arguments + if(is_quantized) + { + unsigned int idx = num_arguments_per_3D_tensor() + num_arguments_per_2D_tensor() + num_arguments_per_1D_tensor(); + _kernel.setArg(idx++, -_input0->info()->quantization_info().offset); + _kernel.setArg(idx++, -_input1->info()->quantization_info().offset); + } // Configure the local work size for Bifrost with a value obtained // via exhaustive autotuning for the MobileNets tensor shapes. -- cgit v1.2.1