From 79ffadebd8dff7eaecbcfa3a28106736f240f1c5 Mon Sep 17 00:00:00 2001 From: Pablo Tello Date: Fri, 4 May 2018 11:45:13 +0100 Subject: COMPMID-1112: Enabled multithreading transforms in Winograd. Updated RSH code as well. Change-Id: I9452ff5c7f0ff0cd60b8c223cdd71077288eb0c1 Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/130177 Tested-by: Jenkins Reviewed-by: Georgios Pinitas Reviewed-by: Anthony Barbier --- .../kernels/convolution/winograd/winograd_gemm.cpp | 301 +-------------------- 1 file changed, 3 insertions(+), 298 deletions(-) (limited to 'src/core/NEON/kernels/convolution/winograd') diff --git a/src/core/NEON/kernels/convolution/winograd/winograd_gemm.cpp b/src/core/NEON/kernels/convolution/winograd/winograd_gemm.cpp index a0ecaea4d4..a5d43024a4 100644 --- a/src/core/NEON/kernels/convolution/winograd/winograd_gemm.cpp +++ b/src/core/NEON/kernels/convolution/winograd/winograd_gemm.cpp @@ -21,11 +21,9 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include #include "arm_compute/core/NEON/kernels/convolution/winograd/winograd_gemm.hpp" #include "arm_compute/core/NEON/kernels/convolution/winograd/batched_blocked_gemm.hpp" - -#include - using namespace winograd; /** Get the output shape of a convolution. */ @@ -39,8 +37,8 @@ Tensor4DShape WinogradGEMM::Convolution::get_output { return Tensor4DShape { in_shape.n_batches, - (padding == PADDING_SAME) ? in_shape.n_rows : in_shape.n_rows - (kernel_rows - 1), - (padding == PADDING_SAME) ? in_shape.n_cols : in_shape.n_cols - (kernel_cols - 1), + (padding == PADDING_SAME) ? in_shape.n_rows : in_shape.n_rows - (kernel_rows - 1), + (padding == PADDING_SAME) ? in_shape.n_cols : in_shape.n_cols - (kernel_cols - 1), kernel_shape.n_output_channels, in_shape.ordering }; @@ -223,299 +221,6 @@ int WinogradGEMM:: } -/** Create a new Winograd operator. */ -template -template -WinogradGEMM::Convolution::Convolution( - const KernelShape &kernel_shape, - const Tensor4DShape &input_shape, - const PaddingType padding, - void *kernel_storage -) : kernel_shape(kernel_shape), // Store the kernel shape - kernel_matrix_row_stride(roundup(kernel_shape.n_output_channels, N_BLOCK)), - manage_kernel_storage(kernel_storage == NULL), - _kernel_storage(manage_kernel_storage ? - ALLOCATE(get_kernel_storage_size(kernel_shape)) : - kernel_storage), - input_shape(input_shape), - padding(padding), - output_shape(get_output_shape(kernel_shape, input_shape, padding)), - tile_rows(iceildiv(output_shape.n_rows, output_tile_rows)), - tile_cols(iceildiv(output_shape.n_cols, output_tile_cols)), - M(input_shape.n_batches * tile_rows * tile_cols), - K(kernel_shape.n_input_channels), - N(kernel_shape.n_output_channels) -{ - // Create pointers to the kernel matrices - const int kernel_matrix_size_bytes = get_kernel_matrix_size(kernel_shape); - int8_t* const ks_bytes = reinterpret_cast(_kernel_storage); - for (int i = 0; i < N_GEMMS; i++) { - kernel_matrices[i] = reinterpret_cast( - ks_bytes + i*kernel_matrix_size_bytes); - } -} - - -/** Create a new Winograd operator and initialise the weights. */ -template -template -WinogradGEMM::Convolution::Convolution( - const KernelShape &kernel_shape, - const Tensor4DShape &input_shape, - const PaddingType padding, - const TIn* const kernel, - void *kernel_storage, - void *transform_working_space -) : Convolution(kernel_shape, input_shape, padding, kernel_storage) -{ - transform_weights(kernel, transform_working_space); -} - - -/** Clean up a convolution engine. */ -template -template -WinogradGEMM:: -Convolution::~Convolution() -{ - // If we were responsible for managing kernel storage ensure that it is - // freed. - if (manage_kernel_storage) - { - free(_kernel_storage); - } -} - - -/** Transform weights into the Winograd domain and store them for later use/reuse. */ -template -template -template -void WinogradGEMM:: -Convolution::transform_weights( - const TIn* const kernel, - void *transform_working_space -) -{ - // Allocate working space if it is required - bool allocated_working_space = false; - if (transform_working_space == NULL && // If no memory has been provided - get_kernel_transform_working_size(kernel_shape) != 0) // And we need the space - { - allocated_working_space = true; - transform_working_space = ALLOCATE( - get_kernel_transform_working_size(kernel_shape) - ); - } - - // The transformation methods only work on weights laid out in HWIO form, if - // the weights are not in this form then we need to re-order them. - const TIn *kernel_hwio = kernel; - if (kernel_shape.ordering != HWIO) - { - kernel_hwio = reinterpret_cast(transform_working_space); - - // Re-order the weights from OIHW to HWIO - reorder::ofm_ifm_h_w_to_h_w_ifm_ofm( - kernel, const_cast(kernel_hwio), - kernel_shape.n_output_channels, - kernel_shape.n_input_channels, - kernel_shape.n_rows, - kernel_shape.n_cols - ); - } - - const int kernel_matrix_size_bytes = get_kernel_matrix_size(kernel_shape); - WeightsTransformT weights_transform( - kernel_hwio, kernel_matrices[0], - kernel_matrix_size_bytes / sizeof(TIn), - kernel_matrix_row_stride, - kernel_shape.n_output_channels, - kernel_shape.n_input_channels - ); - - // Transform the weights into the Winograd domain - weights_transform.run(0, weights_transform.get_window()); - - // Free memory if we allocated it - if (allocated_working_space) - { - free(transform_working_space); - } -} - - -/** Perform a convolution. */ -template -template -void WinogradGEMM:: -Convolution::execute( - TOut* const output, - const TIn* const input, - const TOut* const biases, - void *working_space, - const int n_threads -) -{ - const auto padding_type = padding; - const auto input_shape = this->input_shape; - - // Allocate working space if none has been provided - const bool manage_working_space = (working_space == NULL); - if (manage_working_space) - { - const size_t ws_size = get_working_space_size( - kernel_shape, input_shape, padding_type - ); - working_space = ALLOCATE(ws_size * sizeof(int8_t)); - memset(working_space, 0x00, ws_size); - } - int8_t* const ws_bytes = reinterpret_cast(working_space); - - // Split the working space into that required for 16 input matrices and - // output matrices. - TIn *input_matrices[N_GEMMS]; - TOut *output_matrices[N_GEMMS]; - const int in_matrix_stride_bytes = get_input_matrix_size(kernel_shape, input_shape, padding_type); - const int out_matrix_stride_bytes = get_output_matrix_size(kernel_shape, input_shape, padding_type); - - for (int i = 0; i < N_GEMMS; i++) - { - input_matrices[i] = reinterpret_cast( - ws_bytes + i*in_matrix_stride_bytes); - output_matrices[i] = reinterpret_cast( - ws_bytes + N_GEMMS*in_matrix_stride_bytes + i*out_matrix_stride_bytes); - } - - // If we need to re-order the input and output tensors then the final chunk - // of the working space can be used for this purpose. - // TODO - Overlay the input reorder on top of the output matrices - // - Overlay the output reorder on top of the input matrices - // Reorder the input input form if it was not provided in this ordering. - const TIn* input_nhwc = input; - if (input_shape.ordering == NCHW) - { - input_nhwc = reinterpret_cast( - ws_bytes + N_GEMMS*(in_matrix_stride_bytes + out_matrix_stride_bytes) - ); - - reorder::nchw_to_nhwc( - input, const_cast(input_nhwc), - input_shape.n_batches, - input_shape.n_channels, - input_shape.n_rows, - input_shape.n_cols - ); - } - - // Compute shape for the GEMM - const auto output_shape = this->output_shape; - int M = this->M; - int K = this->K; - int N = this->N; - - const int in_matrix_row_stride = K; - const int out_matrix_row_stride = kernel_matrix_row_stride; - - InputTransform input_transform( - input_nhwc, - input_shape.n_batches, - input_shape.n_rows, - input_shape.n_cols, - input_shape.n_channels, - padding_type, - input_matrices[0], - in_matrix_stride_bytes / sizeof(TIn), - in_matrix_row_stride - ); - - // Transform the input into the Winograd domain - input_transform.run(0, input_transform.get_window()); - - // Perform the GEMMs - const int kernel_matrix_stride_bytes = get_kernel_matrix_size(kernel_shape); - BatchedBlockedGemm gemms( - N_GEMMS, M, K, N, - in_matrix_stride_bytes / sizeof(TIn), - in_matrix_row_stride, - kernel_matrix_stride_bytes / sizeof(TIn), - kernel_matrix_row_stride, - out_matrix_stride_bytes / sizeof(TOut), - out_matrix_row_stride, - input_matrices[0], - kernel_matrices[0], - output_matrices[0] - ); - for (unsigned int i = 0; i < gemms.get_window(); i++) - { - gemms.run(i, i+1); - } - - // If the output tensor needs to be in NCHW form then store the NHWC output - // tensor in temporary storage and then reorder. If the output tensor needs - // to be in NHWC then just write straight to the output tensor. - TOut *output_nhwc = output; - if (input_shape.ordering == NCHW) - { - output_nhwc = reinterpret_cast( - ws_bytes + N_GEMMS*(in_matrix_stride_bytes + out_matrix_stride_bytes) - ); - } - - // Transform the output tensor from the Winograd domain to the spatial - // domain. - OutputTransform output_transform( - output_matrices[0], - out_matrix_stride_bytes / sizeof(TOut), - out_matrix_row_stride, - biases, - output_nhwc, - output_shape.n_batches, - output_shape.n_rows, - output_shape.n_cols, - output_shape.n_channels - ); - output_transform.run(0, output_transform.get_window()); - - // Reorder the output tensor if it is required to be in NCHW form. - if (input_shape.ordering == NCHW) - { - reorder::nhwc_to_nchw( - output_nhwc, output, - output_shape.n_batches, - output_shape.n_rows, - output_shape.n_cols, - output_shape.n_channels - ); - } - - // Free working space if we were responsible for allocating it - if (manage_working_space) - { - free(working_space); - } -} - - -/** Perform a convolution. */ -template -template -void WinogradGEMM:: -Convolution::execute( - TOut* const output, - const TIn* const input, - const TOut* const biases, - const int n_threads -) -{ - execute(output, input, biases, NULL, n_threads); -} - - // Instantiate required implementations template class WinogradGEMM<2, 2, 3, 3>::Convolution; template class WinogradGEMM<4, 4, 3, 3>::Convolution; -- cgit v1.2.1