diff options
Diffstat (limited to 'src/core/NEON/kernels')
-rw-r--r-- | src/core/NEON/kernels/NEDirectConvolutionLayerOutputStageKernel.cpp (renamed from src/core/NEON/kernels/NEDirectConvolutionLayerBiasAccumulateKernel.cpp) | 133 |
1 files changed, 97 insertions, 36 deletions
diff --git a/src/core/NEON/kernels/NEDirectConvolutionLayerBiasAccumulateKernel.cpp b/src/core/NEON/kernels/NEDirectConvolutionLayerOutputStageKernel.cpp index 65b7087d7e..40abdb1672 100644 --- a/src/core/NEON/kernels/NEDirectConvolutionLayerBiasAccumulateKernel.cpp +++ b/src/core/NEON/kernels/NEDirectConvolutionLayerOutputStageKernel.cpp @@ -21,7 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include "arm_compute/core/NEON/kernels/NEDirectConvolutionLayerBiasAccumulateKernel.h" +#include "arm_compute/core/NEON/kernels/NEDirectConvolutionLayerOutputStageKernel.h" #include "arm_compute/core/AccessWindowStatic.h" #include "arm_compute/core/Error.h" @@ -42,32 +42,49 @@ namespace { Status validate_arguments(const ITensorInfo *input, const ITensorInfo *bias, const ITensorInfo *output) { - ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, bias); + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::QS32, DataType::F32); - ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(bias, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::QS32, DataType::F32); - if(is_data_type_quantized(input->data_type())) + + if(bias != nullptr) { - ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->data_type() == DataType::QS8 && bias->data_type() != DataType::QS8, "Wrong data type for bias"); - ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->data_type() == DataType::QS16 && bias->data_type() != DataType::QS8, "Wrong data type for bias"); - ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->data_type() == DataType::QS32 && bias->data_type() != DataType::QS16, "Wrong data type for bias"); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(bias, 1, DataType::QS8, DataType::QS16, DataType::F16, DataType::QS32, DataType::F32); + + if(is_data_type_quantized(input->data_type())) + { + ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->data_type() == DataType::QS8 && bias->data_type() != DataType::QS8, "Wrong data type for bias"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->data_type() == DataType::QS16 && bias->data_type() != DataType::QS8, "Wrong data type for bias"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->data_type() == DataType::QS32 && bias->data_type() != DataType::QS16, "Wrong data type for bias"); + } + else + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, bias); + } + + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT_POSITION(input, bias); + ARM_COMPUTE_RETURN_ERROR_ON(bias->num_dimensions() > 1); } else { - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, bias); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(!is_data_type_quantized(input->data_type()), "Calling output stage kernel with floating point arguments"); } - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT_POSITION(input, bias); - // Checks performed when output is configured if((output != nullptr) && (output->total_size() != 0)) { ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::QS8, DataType::QS16, DataType::F32); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(bias, output); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT_POSITION(bias, output); + if(is_data_type_quantized(input->data_type())) + { + ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->data_type() == DataType::QS8 && output->data_type() != DataType::QS8, "Wrong data type for output"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->data_type() == DataType::QS16 && output->data_type() != DataType::QS8, "Wrong data type for output"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(input->data_type() == DataType::QS32 && output->data_type() != DataType::QS16, "Wrong data type for output"); + } + else + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); + } + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_FIXED_POINT_POSITION(input, output); } - ARM_COMPUTE_RETURN_ERROR_ON(bias->num_dimensions() > 1); - return Status{}; } @@ -79,16 +96,35 @@ std::pair<Status, Window> validate_and_configure_window(ITensorInfo *input, ITen // Configure kernel window Window win = calculate_max_window(*input, Steps(num_elems_processed_per_iteration)); AccessWindowHorizontal input_access(input, 0, num_elems_processed_per_iteration); - AccessWindowStatic bias_access(bias, 0, 0, bias->dimension(0), bias->dimension(1)); + if(output != nullptr && (output->total_size() != 0)) { AccessWindowHorizontal output_access(output, 0, num_elems_processed_per_iteration); - window_changed = update_window_and_padding(win, input_access, output_access, bias_access); + + if(bias == nullptr) + { + window_changed = update_window_and_padding(win, input_access, output_access); + } + else + { + AccessWindowStatic bias_access(bias, 0, 0, bias->dimension(0), bias->dimension(1)); + window_changed = update_window_and_padding(win, input_access, output_access, bias_access); + } + output_access.set_valid_region(win, ValidRegion(Coordinates(), output->tensor_shape())); } else { - window_changed = update_window_and_padding(win, input_access, bias_access); + if(bias == nullptr) + { + window_changed = update_window_and_padding(win, input_access); + } + else + { + AccessWindowStatic bias_access(bias, 0, 0, bias->dimension(0), bias->dimension(1)); + window_changed = update_window_and_padding(win, input_access, bias_access); + } + input_access.set_valid_region(win, ValidRegion(Coordinates(), input->tensor_shape())); } @@ -199,8 +235,8 @@ inline float16x8_t internal_vqaddq(const float16x8_t &x, const float16x8_t &y) } #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ -template <typename T1, typename T2, bool in_place> -void accumulate_bias(ITensor *input, const ITensor *bias, const Window window, ITensor *output) +template <typename T1, typename T2, bool in_place, bool has_bias> +void output_stage(ITensor *input, const ITensor *bias, const Window window, ITensor *output) { Iterator in(input, window); @@ -210,10 +246,17 @@ void accumulate_bias(ITensor *input, const ITensor *bias, const Window window, I { // Get bias and pointer to input const auto in_ptr = reinterpret_cast<T1 *>(in.ptr()); - const auto vb = internal_vdupq_n(static_cast<T1>(*reinterpret_cast<const T2 *>(bias->ptr_to_element(Coordinates(id.z()))))); // Accumulate bias - internal_vst1q(in_ptr, internal_vqaddq(internal_vld1q(in_ptr), vb)); + if(has_bias) + { + const auto vb = internal_vdupq_n(static_cast<T1>(*reinterpret_cast<const T2 *>(bias->ptr_to_element(Coordinates(id.z()))))); + internal_vst1q(in_ptr, internal_vqaddq(internal_vld1q(in_ptr), vb)); + } + else + { + internal_vst1q(in_ptr, internal_vld1q(in_ptr)); + } }, in); } @@ -225,24 +268,31 @@ void accumulate_bias(ITensor *input, const ITensor *bias, const Window window, I // Get bias and pointer to input const auto in_ptr = reinterpret_cast<const T1 *>(in.ptr()); const auto out_ptr = reinterpret_cast<T2 *>(out.ptr()); - const auto vb = internal_vdupq_n(static_cast<T1>(*reinterpret_cast<const T2 *>(bias->ptr_to_element(Coordinates(id.z()))))); // Accumulate bias - internal_vst1q(out_ptr, internal_vqaddq(internal_vld1q(in_ptr), vb)); + if(has_bias) + { + const auto vb = internal_vdupq_n(static_cast<T1>(*reinterpret_cast<const T2 *>(bias->ptr_to_element(Coordinates(id.z()))))); + internal_vst1q(out_ptr, internal_vqaddq(internal_vld1q(in_ptr), vb)); + } + else + { + internal_vst1q(out_ptr, internal_vld1q(in_ptr)); + } }, in, out); } } } // namespace -NEDirectConvolutionLayerBiasAccumulateKernel::NEDirectConvolutionLayerBiasAccumulateKernel() +NEDirectConvolutionLayerOutputStageKernel::NEDirectConvolutionLayerOutputStageKernel() : _func(nullptr), _input(nullptr), _bias(nullptr), _output(nullptr) { } -void NEDirectConvolutionLayerBiasAccumulateKernel::configure(ITensor *input, const ITensor *bias, ITensor *output) +void NEDirectConvolutionLayerOutputStageKernel::configure(ITensor *input, const ITensor *bias, ITensor *output) { - ARM_COMPUTE_ERROR_ON_NULLPTR(input, bias); + ARM_COMPUTE_ERROR_ON_NULLPTR(input); // Auto-initialize output output if required if(output != nullptr) @@ -252,7 +302,7 @@ void NEDirectConvolutionLayerBiasAccumulateKernel::configure(ITensor *input, con } // Perform validation step - ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), bias->info(), (output == nullptr) ? nullptr : output->info())); + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), (bias == nullptr) ? nullptr : bias->info(), (output == nullptr) ? nullptr : output->info())); _func = nullptr; _bias = bias; @@ -260,7 +310,7 @@ void NEDirectConvolutionLayerBiasAccumulateKernel::configure(ITensor *input, con _output = output; // Configure kernel window - auto win_config = validate_and_configure_window(input->info(), bias->info(), (output == nullptr) ? nullptr : output->info()); + auto win_config = validate_and_configure_window(input->info(), (bias == nullptr) ? nullptr : bias->info(), (output == nullptr) ? nullptr : output->info()); ARM_COMPUTE_ERROR_THROW_ON(win_config.first); INEKernel::configure(win_config.second); @@ -269,14 +319,25 @@ void NEDirectConvolutionLayerBiasAccumulateKernel::configure(ITensor *input, con { case DataType::QS8: { - _func = (output == nullptr) ? &accumulate_bias<qint8_t, qint8_t, true> : &accumulate_bias<qint8_t, qint8_t, false>; + if(bias == nullptr) + { + _func = (output == nullptr) ? &output_stage<qint8_t, qint8_t, true, false> : &output_stage<qint8_t, qint8_t, false, false>; + } + else + { + _func = (output == nullptr) ? &output_stage<qint8_t, qint8_t, true, true> : &output_stage<qint8_t, qint8_t, false, true>; + } break; } case DataType::QS16: { - if(bias->info()->data_type() == DataType::QS8) + if(bias != nullptr && bias->info()->data_type() == DataType::QS8) + { + _func = (output == nullptr) ? &output_stage<qint16_t, qint8_t, true, true> : &output_stage<qint16_t, qint8_t, false, true>; + } + else if(bias == nullptr) { - _func = (output == nullptr) ? &accumulate_bias<qint16_t, qint8_t, true> : &accumulate_bias<qint16_t, qint8_t, false>; + _func = (output == nullptr) ? &output_stage<qint16_t, qint8_t, true, false> : &output_stage<qint16_t, qint8_t, false, false>; } else { @@ -286,19 +347,19 @@ void NEDirectConvolutionLayerBiasAccumulateKernel::configure(ITensor *input, con } case DataType::QS32: { - _func = (output == nullptr) ? &accumulate_bias<qint32_t, qint16_t, true> : &accumulate_bias<qint32_t, qint16_t, false>; + _func = (output == nullptr) ? &output_stage<qint32_t, qint16_t, true, true> : &output_stage<qint32_t, qint16_t, false, true>; break; } #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC case DataType::F16: { - _func = (output == nullptr) ? &accumulate_bias<float16_t, float16_t, true> : &accumulate_bias<float16_t, float16_t, false>; + _func = (output == nullptr) ? &output_stage<float16_t, float16_t, true, true> : &output_stage<float16_t, float16_t, false, true>; break; } #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ case DataType::F32: { - _func = (output == nullptr) ? &accumulate_bias<float, float, true> : &accumulate_bias<float, float, false>; + _func = (output == nullptr) ? &output_stage<float, float, true, true> : &output_stage<float, float, false, true>; break; } default: @@ -309,7 +370,7 @@ void NEDirectConvolutionLayerBiasAccumulateKernel::configure(ITensor *input, con } } -Status NEDirectConvolutionLayerBiasAccumulateKernel::validate(const ITensorInfo *input, const ITensorInfo *bias, const ITensorInfo *output) +Status NEDirectConvolutionLayerOutputStageKernel::validate(const ITensorInfo *input, const ITensorInfo *bias, const ITensorInfo *output) { ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, bias, output)); ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(input->clone().get(), bias->clone().get(), output == nullptr ? nullptr : output->clone().get()).first); @@ -317,7 +378,7 @@ Status NEDirectConvolutionLayerBiasAccumulateKernel::validate(const ITensorInfo return Status{}; } -void NEDirectConvolutionLayerBiasAccumulateKernel::run(const Window &window, const ThreadInfo &info) +void NEDirectConvolutionLayerOutputStageKernel::run(const Window &window, const ThreadInfo &info) { ARM_COMPUTE_UNUSED(info); ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); |