diff options
Diffstat (limited to 'src/core/NEON/kernels/convolution/winograd')
11 files changed, 33 insertions, 227 deletions
diff --git a/src/core/NEON/kernels/convolution/winograd/input_transform.hpp b/src/core/NEON/kernels/convolution/winograd/input_transform.hpp index 113b7ea928..265551288d 100644 --- a/src/core/NEON/kernels/convolution/winograd/input_transform.hpp +++ b/src/core/NEON/kernels/convolution/winograd/input_transform.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,9 +24,7 @@ #pragma once -#include "arm_compute/core/Error.h" - -#include "src/core/NEON/kernels/assembly/winograd.hpp" +#include "winograd.hpp" #include "src/core/NEON/kernels/arm_conv/addressing.hpp" #include <algorithm> @@ -191,10 +189,9 @@ class TransformDirect : public TransformBase<TIn, TOut> TOut *outptr, size_t ld_out_matrix, unsigned int pad_top, unsigned int valid_rows, unsigned int pad_left, unsigned int valid_cols, - void *working_space + void * ) const override { - ARM_COMPUTE_UNUSED(working_space); const auto end_i = this->get_input_rows() - pad_top; const auto pad_bottom = end_i < valid_rows ? 0 : end_i - valid_rows; const auto end_j = this->get_input_cols() - pad_left; diff --git a/src/core/NEON/kernels/convolution/winograd/input_transforms/arm_fp32_1x8.cpp b/src/core/NEON/kernels/convolution/winograd/input_transforms/arm_fp32_1x8.cpp index 44f8752a0c..ae589f9772 100644 --- a/src/core/NEON/kernels/convolution/winograd/input_transforms/arm_fp32_1x8.cpp +++ b/src/core/NEON/kernels/convolution/winograd/input_transforms/arm_fp32_1x8.cpp @@ -30,12 +30,12 @@ namespace winograd { namespace input_transform { void arm_fp32_1x8( - unsigned int n_channels, - const float * input_base, + const unsigned int n_channels, + const float *const input_base, size_t, // We don't need to stride over rows - size_t input_col_stride, + const size_t input_col_stride, float *outptr, - size_t matrix_stride + const size_t matrix_stride ) { constexpr int inner_tile_cols = 8; diff --git a/src/core/NEON/kernels/convolution/winograd/output_transform.hpp b/src/core/NEON/kernels/convolution/winograd/output_transform.hpp index 5148495608..971cc99cd2 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transform.hpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transform.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#include "src/core/NEON/kernels/assembly/winograd.hpp" +#include "winograd.hpp" #include "src/core/NEON/kernels/arm_conv/addressing.hpp" diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp index bbf0ce58b4..cce3745c77 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp @@ -33,13 +33,13 @@ namespace output_transform { void arm_fp32_1x2_1x7( unsigned int n_channels, const float* inptr, - size_t matrix_stride, + const size_t matrix_stride, const float* bptr, float *outptr, size_t, // No need to stride across rows - size_t output_col_stride, - float output_min, - float output_max + const size_t output_col_stride, + const float output_min, + const float output_max ) { constexpr auto inner_tile_cols = 8u, output_tile_cols = 2u; diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp index feb2a5a2c1..10f25a4aab 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp @@ -33,13 +33,13 @@ namespace output_transform { void arm_fp32_1x4_1x5( unsigned int n_channels, const float* inptr, - size_t matrix_stride, + const size_t matrix_stride, const float* bptr, float *outptr, size_t, // No need to stride across rows - size_t output_col_stride, - float output_min, - float output_max + const size_t output_col_stride, + const float output_min, + const float output_max ) { constexpr auto inner_tile_cols = 8u, output_tile_cols = 4u; diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp index ffe60e700d..c45ff8cf2c 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp @@ -34,13 +34,13 @@ namespace output_transform { void arm_fp32_1x6_1x3( unsigned int n_channels, const float* inptr, - size_t matrix_stride, + const size_t matrix_stride, const float* bptr, float *outptr, size_t, // No need to stride across rows - size_t output_col_stride, - float output_min, - float output_max + const size_t output_col_stride, + const float output_min, + const float output_max ) { constexpr unsigned int inner_tile_cols = 8, output_tile_cols = 6; diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp index 54749e6f28..55ed24cd74 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) #include <cstddef> @@ -888,5 +888,4 @@ void sme_fp32_mopa_4x4_3x3( } // namespace winograd } // namespace arm_conv -#endif //defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) - +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms_fp32.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms_fp32.cpp index a221aee5d8..0a7030324e 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms_fp32.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms_fp32.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,9 +30,9 @@ namespace winograd { namespace output_transform { #if defined(__aarch64__) -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) void sme_fp32_mopa_4x4_3x3(unsigned int, const float *, size_t, const float *, float *, size_t, size_t, float, float); -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) #endif // defined(__aarch64__) void arm_fp32_4x4_3x3(unsigned int, const float *, size_t, const float *, float *, size_t, size_t, float, float); void arm_fp32_2x2_3x3(unsigned int, const float *, size_t, const float *, float *, size_t, size_t, float, float); @@ -49,9 +49,9 @@ void arm_fp32_1x2_1x7(unsigned int, const float *, size_t, const float *, float static const TransformImplementation<float> transforms_fp32[] = { #if defined(__aarch64__) -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) { IMPL(4, 4, 3, 3, sme_fp32_mopa_4x4_3x3, Unpadded), MethodConstraints::RequiresSME }, -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) #endif // defined(__aarch64__) { IMPL(4, 4, 3, 3, arm_fp32_4x4_3x3, Unpadded), MethodConstraints::LargerShape }, { IMPL(2, 2, 3, 3, arm_fp32_2x2_3x3, Unpadded) }, diff --git a/src/core/NEON/kernels/convolution/winograd/padding.cpp b/src/core/NEON/kernels/convolution/winograd/padding.cpp deleted file mode 100644 index aca8448658..0000000000 --- a/src/core/NEON/kernels/convolution/winograd/padding.cpp +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#include <cstring> -#include <cstdint> - -#include "padding.hpp" - -namespace padding -{ -template <typename T> -void copy_and_pad_tile( - unsigned int tile_rows, - unsigned int tile_cols, - unsigned int n_channels, - const T *inptr, - unsigned int in_row_stride, - unsigned int in_col_stride, - T* outptr, - unsigned int out_row_stride, - unsigned int out_col_stride, - unsigned int pad_top, - unsigned int pad_left, - unsigned int pad_bottom, - unsigned int pad_right, - T pad_value -) -{ - for (unsigned int out_i = 0; out_i < tile_rows; out_i++) - { - for (unsigned int out_j = 0; out_j < tile_cols; out_j++) - { - T* const output = outptr + out_i*out_row_stride + out_j*out_col_stride; - - if (out_i < pad_top || tile_rows - pad_bottom <= out_i || - out_j < pad_left || tile_cols - pad_right <= out_j) - { - for (unsigned int n = 0; n < n_channels; n++) - { - output[n] = pad_value; - } - } - else - { - const auto in_i = out_i - pad_top, in_j = out_j - pad_left; - const T* const input = inptr + in_i*in_row_stride + in_j*in_col_stride; - std::memcpy(output, input, n_channels * sizeof(T)); - } - } - } -} - -template void copy_and_pad_tile( - unsigned int, unsigned int, unsigned int, - const uint8_t *, unsigned int, unsigned int, - uint8_t *, unsigned int, unsigned int, - unsigned int, unsigned int, unsigned int, unsigned int, uint8_t -); - -template void copy_and_pad_tile( - unsigned int, unsigned int, unsigned int, - const float *, unsigned int, unsigned int, - float *, unsigned int, unsigned int, - unsigned int, unsigned int, unsigned int, unsigned int, float -); - -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -template void copy_and_pad_tile( - unsigned int, unsigned int, unsigned int, - const __fp16 *, unsigned int, unsigned int, - __fp16 *, unsigned int, unsigned int, - unsigned int, unsigned int, unsigned int, unsigned int, __fp16 -); -#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - -template <unsigned int TileRows, unsigned int TileCols> -void CopyCropped<TileRows, TileCols>::execute( - const size_t size, - const void * const inptr, - const size_t in_row_stride, - const size_t in_col_stride, - void * const outptr, - const size_t out_row_stride, - const size_t out_col_stride, - const unsigned int pad_top, - const unsigned int pad_left, - const unsigned int pad_bottom, - const unsigned int pad_right -) -{ - for (unsigned int out_i = 0, in_i = pad_top; in_i < TileRows - pad_bottom; out_i++, in_i++) - { - for (unsigned int out_j = 0, in_j = pad_left; in_j < TileCols - pad_right; out_j++, in_j++) - { - std::memcpy( - static_cast<uint8_t *>(outptr) + out_i*out_row_stride + out_j*out_col_stride, - static_cast<const uint8_t *>(inptr) + in_i*in_row_stride + in_j*in_col_stride, - size - ); - } - } -} - -template class CopyCropped<2, 2>; -template class CopyCropped<3, 3>; -template class CopyCropped<4, 4>; - -template <typename T> -void crop_and_copy_tile( - unsigned int tile_rows, - unsigned int tile_cols, - unsigned int n_channels, - const T *inptr, - unsigned int in_row_stride, - unsigned int in_col_stride, - T *outptr, - unsigned int out_row_stride, - unsigned int out_col_stride, - unsigned int crop_top, - unsigned int crop_left, - unsigned int crop_bottom, - unsigned int crop_right -) -{ - for (unsigned int out_i = 0, in_i = crop_top; in_i < tile_rows - crop_bottom; out_i++, in_i++) - { - for (unsigned int out_j = 0, in_j = crop_left; in_j < tile_cols - crop_right; out_j++, in_j++) - { - std::memcpy( - outptr + out_i*out_row_stride + out_j*out_col_stride, - inptr + in_i*in_row_stride + in_j*in_col_stride, - sizeof(T) * n_channels - ); - } - } -} - -template void crop_and_copy_tile( - unsigned int tile_rows, - unsigned int tile_cols, - unsigned int n_channels, - const float *inptr, - unsigned int in_row_stride, - unsigned int in_col_stride, - float *outptr, - unsigned int out_row_stride, - unsigned int out_col_stride, - unsigned int crop_top, - unsigned int crop_left, - unsigned int crop_bottom, - unsigned int crop_right -); - -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -template void crop_and_copy_tile( - unsigned int tile_rows, - unsigned int tile_cols, - unsigned int n_channels, - const __fp16 *inptr, - unsigned int in_row_stride, - unsigned int in_col_stride, - __fp16 *outptr, - unsigned int out_row_stride, - unsigned int out_col_stride, - unsigned int crop_top, - unsigned int crop_left, - unsigned int crop_bottom, - unsigned int crop_right -); -#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -} // namespace padding diff --git a/src/core/NEON/kernels/convolution/winograd/weight_transform.hpp b/src/core/NEON/kernels/convolution/winograd/weight_transform.hpp index db0f53df1b..5569bc1b89 100644 --- a/src/core/NEON/kernels/convolution/winograd/weight_transform.hpp +++ b/src/core/NEON/kernels/convolution/winograd/weight_transform.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#include "src/core/NEON/kernels/assembly/winograd.hpp" +#include "winograd.hpp" #include <algorithm> #include <functional> diff --git a/src/core/NEON/kernels/convolution/winograd/winograd_implementations.hpp b/src/core/NEON/kernels/convolution/winograd/winograd_implementations.hpp index 510f69baaa..af0dd04298 100644 --- a/src/core/NEON/kernels/convolution/winograd/winograd_implementations.hpp +++ b/src/core/NEON/kernels/convolution/winograd/winograd_implementations.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#include "src/core/NEON/kernels/assembly/winograd.hpp" +#include "winograd.hpp" #include <memory> #include <string> @@ -314,6 +314,7 @@ bool get_implementation( false, // Indirect input {}, // No activation max_threads, + false, // Not fixed format fast_mode, gemm_cfg )); |