/* * Copyright (c) 2021 Arm Limited. * * SPDX-License-Identifier: MIT * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to * deal in the Software without restriction, including without limitation the * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #pragma once #include "src/core/NEON/kernels/arm_gemm/utils.hpp" #ifdef CYCLE_PROFILING #include "profiler.hpp" #endif #include namespace arm_conv { namespace depthwise { struct IDepthwiseDepthfirstStrategy { virtual arm_gemm::VLType get_vl_type() const = 0; virtual unsigned int get_input_rows() const = 0; virtual unsigned int get_input_cols() const = 0; virtual unsigned int get_output_rows() const = 0; virtual unsigned int get_output_cols() const = 0; virtual unsigned int get_kernel_rows() const = 0; virtual unsigned int get_kernel_cols() const = 0; virtual unsigned int get_stride_rows() const = 0; virtual unsigned int get_stride_cols() const = 0; virtual void indirect_kernel( const void *const *const input_ptrs, void *const *const output_ptrs, const void *params, unsigned int n_channels, const void *activation_min, const void *activation_max ) const = 0; virtual void direct_kernel( const unsigned int n_tile_rows, const unsigned int n_tile_cols, const void *inptr, int64_t ld_input_row, int64_t ld_input_col, void *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const void *activation_min, const void *activation_max ) const = 0; virtual ~IDepthwiseDepthfirstStrategy() {} }; template class DepthwiseDepthfirst : public DepthwiseCommon { const std::unique_ptr m_strat; size_t sizeof_inptr_array(void) const { return sizeof(TInput *) * m_strat->get_input_rows() * m_strat->get_input_cols(); } size_t sizeof_input_buffer(unsigned int n_input_channels) const { return sizeof(TInput) * n_input_channels; } size_t sizeof_outptr_array(void) const { return sizeof(TInput *) * m_strat->get_output_rows() * m_strat->get_output_cols(); } size_t sizeof_output_buffer(unsigned int n_output_channels) const { return sizeof(TOutput) * n_output_channels; } public: DepthwiseDepthfirst( IDepthwiseDepthfirstStrategy *const strat, const DepthwiseArgs &args ) : DepthwiseCommon(args), m_strat(strat) { } DepthwiseDepthfirst(DepthwiseDepthfirst &) = delete; DepthwiseDepthfirst &operator=(DepthwiseDepthfirst &) = delete; size_t get_storage_size(void) const override { // TODO What if we insert extra padding? Biases are a different size to the inputs, ... const unsigned int vl = arm_gemm::utils::get_vector_length(m_strat->get_vl_type()); const auto rounded_channels = arm_gemm::roundup(this->m_args.input_channels, vl); return (1 + this->m_args.kernel_rows * this->m_args.kernel_cols) * rounded_channels * sizeof(TWeight); } void pack_parameters(void *_buffer, const void *_biases, const void *_weights, size_t ld_weight_col, size_t ld_weight_row) override { // TODO What if the kernel needs a different packing function? // Cast the pointers uint8_t *buffer = static_cast(_buffer); const TAccum *biases = static_cast(_biases); const TWeight *const weights = static_cast(_weights); const unsigned int vl = arm_gemm::utils::get_vector_length(m_strat->get_vl_type()); ld_weight_col = (ld_weight_col == 0) ? this->m_args.input_channels : ld_weight_col; ld_weight_row = (ld_weight_row == 0) ? this->m_args.kernel_cols * ld_weight_col : ld_weight_row; for (unsigned int n = 0; n < this->m_args.input_channels; n += vl) { const unsigned int todo = std::min(vl, this->m_args.input_channels - n); // Copy across the correct amount of bias (or 0) for (unsigned int i = 0; i < todo; i++) { reinterpret_cast(buffer)[i] = (biases == nullptr) ? 0 : biases[n + i]; } buffer += vl * sizeof(TAccum); // Copy each of the weights in turn auto weights_row = weights + n; for (unsigned int i = 0; i < this->m_args.kernel_rows; i++) { auto weights_col = weights_row; for (unsigned int j = 0; j < this->m_args.kernel_cols; j++) { for (unsigned int m = 0; m < todo; m++) { reinterpret_cast(buffer)[m] = weights_col[m]; } buffer += vl * sizeof(TWeight); weights_col += ld_weight_col; } weights_row += ld_weight_row; } } } size_t get_working_size(const unsigned int n_threads, const unsigned int n_channels) const override { const unsigned int n_output_channels = n_channels * this->m_args.channel_multiplier; return n_threads * (sizeof_inptr_array() + sizeof_outptr_array() + sizeof_output_buffer(n_output_channels) + sizeof_input_buffer(n_channels)); } using DepthwiseCommon::execute; void execute( const unsigned int batches, const unsigned int input_height, const unsigned int input_width, const unsigned int input_channels, const PaddingValues &padding, const void *const _input, const size_t ld_input_col, const size_t ld_input_row, const size_t ld_input_batch, const void *const parameters, const unsigned int output_height, const unsigned int output_width, void *const _output, const size_t ld_output_col, const size_t ld_output_row, const size_t ld_output_batch, void *const _working_space, const unsigned int thread_id, const unsigned int n_threads ) const override { #ifdef CYCLE_PROFILING arm_gemm::profiler prof; #endif // Compute activation values TAccum activation_min, activation_max; std::tie(activation_min, activation_max) = get_default_activation_values(); switch (this->m_args.activation.type) { case arm_gemm::Activation::Type::BoundedReLU: activation_max = static_cast(this->m_args.activation.param1); // Fall through case arm_gemm::Activation::Type::ReLU: activation_min = static_cast(0); break; default: break; } // Determine what portion of the work to do. const unsigned int n_rows_per_thread = arm_gemm::iceildiv(output_height, n_threads); const int start_out_height = std::min(thread_id * n_rows_per_thread, output_height); const int end_out_height = std::min(start_out_height + n_rows_per_thread, output_height); // Cast input and output pointers into the right types const TInput *const inptr = static_cast(_input); TOutput *const outptr = static_cast(_output); // Allocate portions of the working space uint8_t *working_space = static_cast(_working_space) + get_working_size(thread_id, input_channels); const void **const inptr_array = reinterpret_cast(working_space); working_space += sizeof_inptr_array(); void **const outptr_array = reinterpret_cast(working_space); working_space += sizeof_outptr_array(); TOutput *const output_buffer = reinterpret_cast(working_space); working_space += sizeof_output_buffer(input_channels * this->m_args.channel_multiplier); TInput *const input_buffer = reinterpret_cast(working_space); // Initialise the input buffer for (unsigned int c = 0; c < input_channels; c++) { input_buffer[c] = static_cast(0); } // For each output tile, construct the requisite set of pointers and call // into the kernel. for (unsigned int batch = 0; batch < batches; batch++) { // Get batch pointers const auto inptr_batch = inptr + batch * ld_input_batch; const auto outptr_batch = outptr + batch * ld_output_batch; for (int start_out_i = start_out_height; start_out_i < end_out_height; start_out_i += static_cast(m_strat->get_output_rows())) { const int end_out_i = start_out_i + m_strat->get_output_rows(); const int start_in_i = start_out_i * m_strat->get_stride_rows() - padding.top; const int end_in_i = start_in_i + m_strat->get_input_rows(); // Compute top/bottom padding const auto pad_top = static_cast(-std::min(start_in_i, 0)); const auto pad_bottom = static_cast(-std::min(static_cast(input_height) - end_in_i, 0)); const unsigned int valid_output_rows = std::min( end_out_i - start_out_i, static_cast(output_height) - start_out_i ); // Fill the input pointer array with padding values for (auto index = 0u; index < m_strat->get_input_rows() * m_strat->get_input_cols(); index++) { inptr_array[index] = input_buffer; } for (int start_out_j = 0; start_out_j < static_cast(output_width);) { const int start_in_j = start_out_j * m_strat->get_stride_cols() - this->m_args.padding.left; int pad_left = std::min(0, start_in_j); // Compute how many output tiles we can compute with the direct kernel. int n_direct_tiles = 0; if (!pad_top && !pad_bottom && !pad_left) { // Determine the maximum number of tiles we could handle. n_direct_tiles = (output_width - start_out_j) / m_strat->get_output_cols(); // Continue to reduce this number as required to avoid reading // padding on the right edge. int end_in_j = start_in_j + n_direct_tiles * m_strat->get_input_cols(); int pad_right = std::max(0, end_in_j - static_cast(input_width)); while (pad_right && n_direct_tiles) { n_direct_tiles--; end_in_j -= m_strat->get_input_cols(); pad_right = std::max(0, end_in_j - static_cast(input_width)); } } // Use the unpadded kernel if we can, otherwise use the padded one. if (n_direct_tiles) { auto inptr = inptr_batch + start_in_i*ld_input_row + start_in_j*ld_input_col; auto outptr = outptr_batch + start_out_i*ld_output_row + start_out_j*ld_output_col; start_out_j += n_direct_tiles*m_strat->get_output_cols(); #ifdef CYCLE_PROFILING auto p = prof.ScopedProfiler(PROFILE_KERNEL, 0); #endif m_strat->direct_kernel(1, n_direct_tiles, inptr, ld_input_row, ld_input_col, outptr, ld_output_row, ld_output_col, parameters, this->m_args.input_channels, &activation_min, &activation_max); continue; } const int end_out_j = start_out_j + m_strat->get_output_cols(); const int end_in_j = start_in_j + m_strat->get_input_cols(); const auto pad_right = static_cast(-std::min(static_cast(input_width) - end_in_j, 0)); const unsigned int valid_output_cols = std::min( end_out_j - start_out_j, static_cast(output_width) - start_out_j ); pad_left *= -1; // Construct the input pointer array - fill the array with pointers to // the input buffer and then fill in the required values. for (auto i = pad_top; i < m_strat->get_input_rows() - pad_bottom; i++) { // Can skip over the left padding because we will have either the // same or less than the previous tile. unsigned int j = pad_left; const TInput *colptr = inptr_batch + (start_in_i + i) * ld_input_row + (start_in_j + j) * ld_input_col; const void **ptrs = inptr_array + i * m_strat->get_input_cols() + j; for (; j < m_strat->get_input_cols() - pad_right; j++) { *(ptrs++) = colptr; colptr += ld_input_col; } for (; j < m_strat->get_input_cols(); j++) { *(ptrs++) = input_buffer; } } // Construct the output pointer array. void **outptr_pos = outptr_array; for (auto i = 0u; i < valid_output_rows; i++) { unsigned int j = 0u; TOutput *colptr = outptr_batch + (start_out_i + i) * ld_output_row + start_out_j * ld_output_col; for (; j < valid_output_cols; j++) { *(outptr_pos++) = colptr; colptr += ld_output_col; } for (; j < m_strat->get_output_cols(); j++) { *(outptr_pos++) = output_buffer; } } for (auto i = valid_output_rows; i < m_strat->get_output_rows(); i++) { for (auto j = 0u; j < m_strat->get_output_cols(); j++) { *(outptr_pos++) = output_buffer; } } start_out_j += m_strat->get_output_cols(); #ifdef CYCLE_PROFILING // TODO Work number auto p = prof.ScopedProfiler(PROFILE_KERNEL, (unsigned long)(0)); #endif m_strat->indirect_kernel(inptr_array, outptr_array, parameters, this->m_args.input_channels, &activation_min, &activation_max); } } } } }; } // namespace depthwise } // namespace arm_conv