From 716b1be5f92f5d025ed0ee15775b903728a8e73b Mon Sep 17 00:00:00 2001 From: Gian Marco Iodice Date: Wed, 10 Feb 2021 17:33:27 +0000 Subject: Update OpenCL GEMM benchmark script - Update cl_gemm_benchmark.sh script. Now it also possible to split the benchmark among different platforms - Update documentation Resolves COMPMID-3848 Change-Id: Ifbf71cee5ff83aead76e20dcbf7dc5ce41526594 Signed-off-by: Gian Marco Iodice Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/5047 Tested-by: Arm Jenkins Reviewed-by: Giorgio Arena --- docs/00_introduction.dox | 4 +- examples/gemm_tuner/README.md | 30 +- examples/gemm_tuner/benchmark_gemm_examples.sh | 499 --------------------- examples/gemm_tuner/cl_gemm_benchmark.sh | 585 +++++++++++++++++++++++++ 4 files changed, 608 insertions(+), 510 deletions(-) delete mode 100644 examples/gemm_tuner/benchmark_gemm_examples.sh create mode 100755 examples/gemm_tuner/cl_gemm_benchmark.sh diff --git a/docs/00_introduction.dox b/docs/00_introduction.dox index 3dc86fe059..564115ce62 100644 --- a/docs/00_introduction.dox +++ b/docs/00_introduction.dox @@ -1,5 +1,5 @@ /// -/// Copyright (c) 2017-2020 Arm Limited. +/// Copyright (c) 2017-2021 Arm Limited. /// /// SPDX-License-Identifier: MIT /// @@ -125,6 +125,8 @@ v21.02 Public major release - NEUpsampleLayerKernel / CLUpsampleLayerKernel - Extend OpenCL tuner with workgroup batch size support - Experimental extension for the OpenCL tuner to tune the batches of work groups distribute to compute units + - Add functionality to load the OpenCL GEMM heuristics at runtime + - The GEMM heuristic file (MLGO) can be used to update the default GEMM heuristics available for OpenCL v20.11 Public major release - Various bug fixes. diff --git a/examples/gemm_tuner/README.md b/examples/gemm_tuner/README.md index 73bddc9239..aae803eabb 100644 --- a/examples/gemm_tuner/README.md +++ b/examples/gemm_tuner/README.md @@ -9,7 +9,7 @@ The details of these strategies can be found in the documentations of the corres **CLGEMMMatrixMultiplyReshapedOnlyRHSKernel**. The Tuner consists of 2 scripts and 3 binaries: -* benchmark_gemm_examples.sh and GemmTuner.py under examples/gemm_tuner, and +* cl_gemm_benchmark and GemmTuner.py under examples/gemm_tuner, and * benchmark_cl_gemm_native, benchmark_cl_gemm_reshaped_rhs_only and benchmark_cl_gemm_reshaped under build/tests/gemm_tuner (you'll need to build the library first) @@ -42,7 +42,7 @@ what kernel and subsequently what configurations for that kernels are the most p ### Step2: Push relevant files to the target device All the files that need to be present on the target device are: -* benchmark script: \/examples/gemm_tuner/benchmark_gemm_examples.sh +* benchmark script: \/examples/gemm_tuner/cl_gemm_benchmark * shapes and configs csv files: gemm_shapes.csv, gemm_configs_native.csv, gemm_configs_reshaped_only_rhs.csv, gemm_configs_reshaped.csv * Example benchmark binaries: \/build/tests/gemm_tuner/benchmark_cl_gemm* @@ -51,15 +51,25 @@ With these files on device, we can collect benchmark data using the script. Assu to a folder called *gemm_tuner*. While logged onto our device: ``` # Native -./benchmark_gemm_examples.sh -s native -e ./gemm_tuner -g ./gemm_shapes.csv -c ./gemm_configs_native.csv -o results/native +./cl_gemm_benchmark -s native -e ./gemm_tuner -g ./gemm_shapes.csv -c ./gemm_configs_native.csv -o results/native # Reshaped Only RHS -./benchmark_gemm_examples.sh -s reshaped_rhs_only -e ./gemm_tuner -g ./gemm_shapes.csv -c ./gemm_configs_reshaped_only_rhs.csv -o results/reshaped_only_rhs +./cl_gemm_benchmark -s reshaped_rhs_only -e ./gemm_tuner -g ./gemm_shapes.csv -c ./gemm_configs_reshaped_only_rhs.csv -o results/reshaped_only_rhs # Reshaped -./benchmark_gemm_examples.sh -s reshaped -e ./gemm_tuner -g ./gemm_shapes.csv -c ./gemm_configs_reshaped.csv -o results/reshaped +./cl_gemm_benchmark -s reshaped -e ./gemm_tuner -g ./gemm_shapes.csv -c ./gemm_configs_reshaped.csv -o results/reshaped ``` You can repeat the 3 commands above to have a bit redundancy in your benchmark data (as you can imagine, measurement is noisy), but you may need to change the output folder for each repeat +It is also possible to split the benchmark phase among different platforms using the **-i** and **-n** options to specificy the starting experiment and the number of benchmark to run. + +# Reshaped benchmark on 3 different platforms +## Platform 1 +./cl_gemm_benchmark -s reshaped -e ./gemm_tuner -g ./gemm_shapes.csv -c ./gemm_configs_reshaped.csv -o results/reshaped -i 0 -n 8 +## Platform 2 +./cl_gemm_benchmark -s reshaped -e ./gemm_tuner -g ./gemm_shapes.csv -c ./gemm_configs_reshaped.csv -o results/reshaped -i 8 -n 8 +## Platform 3 +./cl_gemm_benchmark -s reshaped -e ./gemm_tuner -g ./gemm_shapes.csv -c ./gemm_configs_reshaped.csv -o results/reshaped -i 16 -n 8 + ### Step4: Generate the heuristics 1. After benchmarking, we pull the benchmark data, the *results* folder, from the target device to our host machine 2. We use the GemmTuner.py script to give us the heuristics @@ -77,7 +87,7 @@ passing a lower value to *-t \* to the GemmTuner.py script. * Android or Linux OS * Bash shell * Built Compute Library with benchmark examples binaries - * benchmark_gemm_examples.sh script + * cl_gemm_benchmark script * gemm shape file A csv file containing the **GEMMParam search list**. This is the list of GEMMParams/gemm shapes that we're @@ -202,13 +212,13 @@ passing a lower value to *-t \* to the GemmTuner.py script. ## Usage The usage of the 2 scripts: -1. benchmark_gemm_examples.sh +1. cl_gemm_benchmark - Run the shell script (**benchmark_gemm_examples.sh**) on your **target device**. Note that all the built benchmark + Run the shell script (**cl_gemm_benchmark**) on your **target device**. Note that all the built benchmark examples: build/tests/gemm_tuner/benchmark_cl_gemm*, have to be present on your target device prior to running. The benchmark results will be saved to json files in an output directory. ``` - Usage: benchmark_gemm_examples.sh [-h] -s \ -e \ -g \ + Usage: cl_gemm_benchmark [-h] -s \ -e \ -g \ -c \ [-d \] [-o \] Options: @@ -265,4 +275,4 @@ The usage of the 2 scripts: milliseconds. Recommended value: <= 0.1 ms -D, --debug Enable script debugging output - ``` \ No newline at end of file + ``` diff --git a/examples/gemm_tuner/benchmark_gemm_examples.sh b/examples/gemm_tuner/benchmark_gemm_examples.sh deleted file mode 100644 index 8789db91c7..0000000000 --- a/examples/gemm_tuner/benchmark_gemm_examples.sh +++ /dev/null @@ -1,499 +0,0 @@ -# Copyright (c) 2019-2020 Arm Limited. -# -# SPDX-License-Identifier: MIT -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -#!/bin/sh - -# Global: Global variables and global settings {{{ -# Treat unset variables as an error when substituting -set -u - -CMD=$( basename $0 ) - -# All supported strategy options -ALL_STRATEGY_OPTIONS=("native" "reshaped_rhs_only" "reshaped") - -# Names of example binary for each strategy -EXAMPLE_BIN_NATIVE="benchmark_cl_gemm_native" -EXAMPLE_BIN_RESHAPED_RHS_ONLY="benchmark_cl_gemm_reshaped_rhs_only" -EXAMPLE_BIN_RESHAPED="benchmark_cl_gemm_reshaped" -EXAMPLE_BIN_RESHAPED_RHS_ONLY_LOWP="benchmark_cl_gemmlowp_reshaped_rhs_only_fused_output_stage_fixedpoint" -EXAMPLE_BIN_RESHAPED_LOWP="benchmark_cl_gemmlowp_reshaped" - -# Default data type -DEFAULT_DATA_TYPE="F32" - -# Default output directory -DEFAULT_OUT_DIR="out" - -# Number of iterations for each benchmark run -NUM_ITERATION=5 -# Global }}} - -# Functions {{{ -####################################### -# Print gemm shape file help message -# Globals: -# None -# Arguments: -# None -# Returns: -# None -####################################### -function help_gemm_shape_file() { - cat >&2 << EOF -Gemm shape file: - Gemm shape file is a headerless csv file with fields separated by commas - - A gemm shape is a list of 4 positive integers describing the shapes of the two matrices (LHS and RHS) - with: - M - Number of lhs matrix rows - N - Number of rhs matrix columns - K - Number of lhs matrix columns/rhs matrix rows - B - Batch size - - An example gemm shape file looks like: - 100,100,30,1 - 100,100,30,3 - ... - -EOF -} - -####################################### -# Print gemm config file for native help message -# Globals: -# None -# Arguments: -# None -# Returns: -# None -####################################### -function help_gemm_config_file_native() { - cat >&2 << EOF -Gemm config file (Strategy native): - Gemm config file is a headerless csv file with fields separated by commas - - A gemm config is a list of 3 positive integers , with: - m0 - Number of rows processed by the matrix multiplication - n0 - Number of columns processed by the matrix multiplication - k0 - Number of partial accumulations performed by the matrix multiplication - - Only the following configurations of M0, N0 and K0 are currently supported: - M0 = 1, 2, 3, 4, 5, 6, 7, 8 - N0 = 2, 3, 4, 8, 16 - K0 = 2, 3, 4, 8, 16 - - An example gemm config file looks like: - 1,4,4 - 2,3,8 - ... - -EOF -} - -####################################### -# Print gemm config file for reshaped_rhs_only help message -# Globals: -# None -# Arguments: -# None -# Returns: -# None -####################################### -function help_gemm_config_file_reshaped_rhs_only() { - cat >&2 << EOF -Gemm config file (Strategy reshaped_rhs_only): - Gemm config file is a headerless csv file with fields separated by commas. - - Note also comments and extraneous empty lines are not permitted. - - A gemm config is a list of 4 positive integers and 3 boolean values: - m0 - Number of rows processed by the matrix multiplication - n0 - Number of columns processed by the matrix multiplication - k0 - Number of partial accumulations performed by the matrix multiplication - h0 - Number of horizontal blocks of size (k0xn0) stored on the same output row - interleave_rhs - Interleave rhs matrix (1) / Do not interleave rhs matrix (0) - transpose_rhs - Transpose rhs matrix (1) / Do not transpose rhs matrix (0) - export_to_cl_image_rhs - (Not supported for quantized types) Export rhs matrix to cl_image (1) / Do not export rhs matrix to cl_image (0). Can only be true - with certain combinations of the GEMMParams and other configs. Please refer to CLGEMMReshapeRHSMatrixKernel - for more details - - Only the following configurations of M0, N0 and K0 are currently supported: - M0 = 1, 2, 3, 4, 5, 6, 7, 8 - N0 = 2, 3, 4, 8, 16 - K0 = 2, 3, 4, 8, 16 - H0 >= 1 - - An example gemm config file looks like: - 4,4,4,1,1,1,0 - 4,4,4,3,1,0,1 - ... - -EOF -} - -####################################### -# Print gemm config file for reshaped help message -# Globals: -# None -# Arguments: -# None -# Returns: -# None -####################################### -function help_gemm_config_file_reshaped() { - cat >&2 << EOF -Gemm config file (Strategy reshaped): - Gemm config file is a headerless csv file with fields separated by commas - - A gemm config is a list of 5 positive integers and 4 boolean values: - m0 - Number of rows processed by the matrix multiplication - n0 - Number of columns processed by the matrix multiplication - k0 - Number of partial accumulations performed by the matrix multiplication - v0 - Number of vertical blocks of size (m0xk0) stored on the same output row - h0 - Number of horizontal blocks of size (k0xn0) stored on the same output row - interleave_lhs - Interleave lhs matrix (1) / Do not interleave lhs matrix (0) - interleave_rhs - Interleave rhs matrix (1) / Do not interleave rhs matrix (0) - transpose_rhs - Transpose rhs matrix but not lhs matrix (1) / Do not transpose rhs matrix but do transpose lhs matrix (0) - export_to_cl_image_rhs - (Not supported for quantized types) Export rhs matrix to cl_image (1) / Do not export rhs matrix to cl_image (0). Can only be true - with certain combinations of the GEMMParams and other configs. Please refer to CLGEMMReshapeRHSMatrixKernel - for more details - - If rhs matrix is transposed only the following configurations are currently supported: - M0 = 2, 3, 4, 5, 6, 7, 8 - N0 = 2, 3, 4, 8, 16 - K0 = 2, 3, 4, 8, 16 - V0 >= 1 - H0 >= 1 - - If lhs matrix is transposed only the following configurations are currently supported: - M0 = 2, 3, 4, 8 - N0 = 2, 3, 4, 8, 16 - K0 = 2, 3, 4, 8, 16 - V0 >= 1 - H0 >= 1 - - An example gemm config file looks like: - 4,4,4,1,3,1,1,1,0 - 4,4,4,3,3,1,1,0,1 - ... - -EOF -} - -####################################### -# Print usage of this program and exit with Error -# Globals: -# Assumes all globals are required -# Arguments: -# None -# Returns: -# Error(1) -####################################### -function usage() { - cat >&2 << EOF -Run gemm examples of a selected strategy, over provided tunable configurationsa and gemm shapes. -Save the benchmark results to json files in an output directory. - -Usage: ${CMD} [-h] -s -e -g -c [-d ] [-o ] - -Options: - -h - Print help messages. If a strategy is specified with -s , then only display messages relevant to that - strategy. Otherwise if no strategy is specified, display messages for all available strategies. - - -s - Strategy option. - Options: ${ALL_STRATEGY_OPTIONS[@]}. - - -e - Path to directory that holds all example binaries - - -g - Path to gemm shape csv file - - -c - Path to gemm config csv file - - -d - Data type option with which to run benchmark examples - Default: ${DEFAULT_DATA_TYPE} - Supported options: - Strategy : Data Types - Native : F32 - Reshaped : F16, F32, QASYMM8 - Reshaped RHS Only : F16, F32, QASYMM8 - - -o - Path to output directory that holds output json files - Default: ${DEFAULT_OUT_DIR} - -EOF -# Print help messages about gemm shapes and various gemm configs -$HELP && help_gemm_shape_file -$HELP && ( [ "${STRATEGY_OPTION}" == "" ] || [ "${STRATEGY_OPTION}" == "native" ] ) && help_gemm_config_file_native -$HELP && ( [ "${STRATEGY_OPTION}" == "" ] || [ "${STRATEGY_OPTION}" == "reshaped_rhs_only" ] ) && help_gemm_config_file_reshaped_rhs_only -$HELP && ( [ "${STRATEGY_OPTION}" == "" ] || [ "${STRATEGY_OPTION}" == "reshaped" ] ) && help_gemm_config_file_reshaped -exit 1 -} - -####################################### -# Print error message and exit with Error. -# Globals: -# None -# Arguments: -# $1 - Error message -# Returns: -# None -####################################### -function error_msg() { - echo "Error: $1" 1>&2 - exit 1 -} - -####################################### -# Convert string to lower-case -# Globals: -# None -# Arguments: -# target - String -# Returns: -# (stdout) - String in lowercase -####################################### -function to_lower() { - local target=$1 - echo "$target" | tr '[:upper:]' '[:lower:]' -} - -####################################### -# Test if the argument is an integer -# Globals: -# None -# Arguments: -# in - Input -# Returns: -# true/false -####################################### -function is_integer() { - local in=$1 - [ "$in" -eq "$in" ] 2> /dev/null -} - -####################################### -# Test if a string is in an array of strings -# Globals: -# None -# Arguments: -# target - String to test -# array - Array of strings to search -# Returns: -# true/false -####################################### -function arr_contains() { - local target=$1 - shift - local array - array=("$@") - for s in "${array[@]}" - do - [ "$s" == "${target}" ] && return - done - false -} - -####################################### -# Run a single example with all tunable gemm configurations on all gemm parameters -# Globals: -# OUT_DIR -# OUT_EXTENSION -# EXAMPLE_BIN_DIR -# NUM_ITERATION -# GEMM_CONFIGS_FILE -# GEMM_SHAPES_FILE -# Arguments: -# example_bin Name of the example binary to run -# Returns: -# None -####################################### -function run() { - local example_bin=$1 - echo "Running all configs for ${example_bin}" 1>&2 - local example_args - local expr_count=1 - # Total number of experiment runs scheduled for this session - local total_num_experiment - local num_params - local num_configs - local match_expression_shape="^([^,]*,){3}[^,]*$" - local match_expression_config="^(\s*[0-9]+\s*,)+\s*[0-9]\s*$" - # Don't count empty lines and lines starting with # (comments) - num_params=$( grep -E "$match_expression_shape" "${GEMM_SHAPES_FILE}" | wc -l | cut -d " " -f 1) - num_configs=$( grep -E "$match_expression_config" "${GEMM_CONFIGS_FILE}" | wc -l | cut -d " " -f 1) - (( total_num_experiment=${num_params} * ${num_configs} )) - # Time elapsed since the beginning in seconds - local time_elapsed_s - # Time estimated to finish in seconds - local time_est_s - echo "Running a total number of ${total_num_experiment} experiments" 1>&2 - - while read gemm_shape - do - while read gemm_config - do - # Ignore empty lines and lines starting with # (comments) - if echo "$gemm_shape" | grep -Eq "$match_expression_shape" && echo "$gemm_config" | grep -Eq "$match_expression_config";then - echo "Running..." 1>&2 - example_args="${gemm_shape},${gemm_config},--type=${DATA_TYPE}" - # Run experiment - ${EXAMPLE_BIN_DIR}/${example_bin} --example_args=${example_args} --iterations=${NUM_ITERATION} --json-file=${OUT_DIR}/${expr_count}.${OUT_EXTENSION} --instruments=OPENCL_TIMER_MS - # Print progress - print_progress ${expr_count} ${total_num_experiment} - # Print time statistics - time_elapsed_s=$SECONDS - echo "Time elapsed since beginning: $(( $time_elapsed_s / 60 ))m $(( $time_elapsed_s % 60 ))s" 1>&2 - (( time_est_s=(${total_num_experiment} - ${expr_count}) * ${time_elapsed_s} / ${expr_count} )) - echo "Time estimated to finish: $(( $time_est_s / 60 ))m $(( $time_est_s % 60 ))s" 1>&2 - (( expr_count++ )) - echo "Done." 1>&2 - fi - done < "${GEMM_CONFIGS_FILE}" - done < "${GEMM_SHAPES_FILE}" - echo "Finished running all configs for ${example_bin}" 1>&2 - echo "All results saved to ${OUT_DIR}" 1>&2 -} - -####################################### -# Print the progress of the current session -# Globals: -# None -# Arguments: -# current Current number of items -# total Total number of items -# Returns: -# None -####################################### -function print_progress() { - local current - local total - current=$1 - total=$2 - # Width of progress bar - local width - width=20 - (( current_width= $width * current / total )) - echo -n -e "Progress [" 1>&2 - for i in $(seq 1 ${width}); do - if [[ $i -le ${current_width} ]]; then - echo -n "#" 1>&2 - else - echo -n " " 1>&2 - fi - done - echo "] $current / $total Experiments" 1>&2 -} - -# Functions }}} - -# Main: Main script {{{ -# Path to directory containing all benchmark examples binaries -EXAMPLE_BIN_DIR="" -# Path to gemm shapes file -GEMM_SHAPES_FILE="" -# Path to gemm configs file -GEMM_CONFIGS_FILE="" -STRATEGY_OPTION="" -# Data type to use -DATA_TYPE=${DEFAULT_DATA_TYPE} -# Path to output directory -OUT_DIR=${DEFAULT_OUT_DIR} -# Output benchmark result file extension -OUT_EXTENSION="gemmtuner_benchmark" -# Toggle help -HELP=false - -# Obtain options -while getopts "hs:e:g:c:d:o:" opt; do - case "$opt" in - h) HELP=true ;; - s) STRATEGY_OPTION=$(to_lower "${OPTARG}");; - e) EXAMPLE_BIN_DIR="${OPTARG}";; - g) GEMM_SHAPES_FILE="${OPTARG}";; - c) GEMM_CONFIGS_FILE="${OPTARG}";; - d) DATA_TYPE=$(to_lower "${OPTARG}");; - o) OUT_DIR="${OPTARG}";; - esac -done -shift $((OPTIND - 1)) - -# Lazily print usage (after arguments have been parsed) -$HELP && - usage - -# Parse and validate options -# Verify all compulsory arguments are passed in -( [ ! -z "${STRATEGY_OPTION}" ] && [ ! -z "${EXAMPLE_BIN_DIR}" ] && [ ! -z "${GEMM_SHAPES_FILE}" ] && [ ! -z "${GEMM_CONFIGS_FILE}" ] ) || - usage - -# Verify example binaries directory exists -[ -d "${EXAMPLE_BIN_DIR}" ] || - error_msg "${EXAMPLE_BIN_DIR} does not exist." - -# Verify all benchmark example binaries exist -[ -f "${EXAMPLE_BIN_DIR}/${EXAMPLE_BIN_RESHAPED_RHS_ONLY}" ] || - error_msg "Cannot find ${EXAMPLE_BIN_RESHAPED_RHS_ONLY} at ${EXAMPLE_BIN_DIR}" - -# Verify Gemm shapes file exists -[ -f "${GEMM_SHAPES_FILE}" ] || - error_msg "Cannot find gemm shapes file ${GEMM_SHAPES_FILE}" - -# Verify Gemm configs file exists -[ -f "${GEMM_CONFIGS_FILE}" ] || - error_msg "Cannot find gemm configs file ${GEMM_CONFIGS_FILE}" - -# Verify strategy option is valid -arr_contains "${STRATEGY_OPTION}" "${ALL_STRATEGY_OPTIONS[@]}" || - error_msg "Does not support strategy ${STRATEGY_OPTION}" - -# Make sure existing benchmark outputs are not overwritten -[ ! -d "${OUT_DIR}" ] || - error_msg "Output directory ${OUT_DIR} already exists!" - -# Make output directory -echo "Making output directory ${OUT_DIR}" 1>&2 -mkdir -p ${OUT_DIR} || error_msg "Failed to make output directory ${OUT_DIR}" -date +%s > ${OUT_DIR}/start_time_unix_seconds - -# Run selected strategy with all configurations -# Restart the built-in timer -if [ "$DATA_TYPE" == "qasymm8" ]; then - SECONDS=0 - [ "${STRATEGY_OPTION}" == "reshaped_rhs_only" ] && run $EXAMPLE_BIN_RESHAPED_RHS_ONLY_LOWP - [ "${STRATEGY_OPTION}" == "reshaped" ] && run $EXAMPLE_BIN_RESHAPED_LOWP - date +%s > ${OUT_DIR}/end_time_unix_seconds -else - SECONDS=0 - [ "${STRATEGY_OPTION}" == "native" ] && run $EXAMPLE_BIN_NATIVE - [ "${STRATEGY_OPTION}" == "reshaped_rhs_only" ] && run $EXAMPLE_BIN_RESHAPED_RHS_ONLY - [ "${STRATEGY_OPTION}" == "reshaped" ] && run $EXAMPLE_BIN_RESHAPED - date +%s > ${OUT_DIR}/end_time_unix_seconds -fi -# Main: Main script }}} diff --git a/examples/gemm_tuner/cl_gemm_benchmark.sh b/examples/gemm_tuner/cl_gemm_benchmark.sh new file mode 100755 index 0000000000..a49c2e154b --- /dev/null +++ b/examples/gemm_tuner/cl_gemm_benchmark.sh @@ -0,0 +1,585 @@ +# Copyright (c) 2019-2021 Arm Limited. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +#!/bin/sh + +# Global: Global variables and global settings {{{ +# Treat unset variables as an error when substituting +set -u + +CMD=$( basename $0 ) + +# All supported strategy options +ALL_STRATEGY_OPTIONS=("native" "reshaped_rhs_only" "reshaped") + +# All supported data type options +ALL_DATA_TYPE_OPTIONS=("f32" "f16" "qasymm8") + +# Names of example binary for each strategy +EXAMPLE_BIN_NATIVE="benchmark_cl_gemm_native" +EXAMPLE_BIN_RESHAPED_RHS_ONLY="benchmark_cl_gemm_reshaped_rhs_only" +EXAMPLE_BIN_RESHAPED="benchmark_cl_gemm_reshaped" +EXAMPLE_BIN_RESHAPED_RHS_ONLY_LOWP="benchmark_cl_gemmlowp_reshaped_rhs_only_fused_output_stage_fixedpoint" +EXAMPLE_BIN_RESHAPED_LOWP="benchmark_cl_gemmlowp_reshaped" + +# Default data type +DEFAULT_DATA_TYPE="f32" + +# Default output directory +DEFAULT_OUT_DIR="out" + +# Default ID of the first experiment +DEFAULT_ID_EXPERIMENT_START=0 + +# Default total number of experiments +DEFAULT_NUM_EXPERIMENTS="all" + +# Default output file extension +DEFAULT_OUT_EXTENSION="gemm_benchmark" + +# Number of iterations for each benchmark run +NUM_ITERATION=5 +# Global }}} + +# Functions {{{ +####################################### +# Print gemm shape file help message +# Globals: +# None +# Arguments: +# None +# Returns: +# None +####################################### +function help_gemm_shape_file() { + cat >&2 << EOF +Gemm shape file: + Gemm shape file is a csv file with fields separated by commas. The optional header and comments are ignored by the parser. + + A gemm shape is a list of 4 positive integers describing the shapes of the two matrices (LHS and RHS) + with: + M - Number of lhs matrix rows + N - Number of rhs matrix columns + K - Number of lhs matrix columns/rhs matrix rows + B - Batch size + + An example gemm shape file looks like: + 100,100,30,1 + 100,100,30,3 + ... + +EOF +} + +####################################### +# Print gemm config file for native help message +# Globals: +# None +# Arguments: +# None +# Returns: +# None +####################################### +function help_gemm_config_file_native() { + cat >&2 << EOF +Gemm config file (Strategy native): + Gemm config file is a csv file with fields separated by commas. The optional header and comments are ignored by the parser. + + A gemm config is a list of 3 positive integers , with: + m0 - Number of rows processed by the matrix multiplication + n0 - Number of columns processed by the matrix multiplication + k0 - Number of partial accumulations performed by the matrix multiplication + + Only the following configurations of M0, N0 and K0 are currently supported: + M0 = 1, 2, 3, 4, 5, 6, 7, 8 + N0 = 2, 3, 4, 8, 16 + K0 = 2, 3, 4, 8, 16 + + An example gemm config file looks like: + 1,4,4 + 2,3,8 + ... + +EOF +} + +####################################### +# Print gemm config file for reshaped_rhs_only help message +# Globals: +# None +# Arguments: +# None +# Returns: +# None +####################################### +function help_gemm_config_file_reshaped_rhs_only() { + cat >&2 << EOF +Gemm config file (Strategy reshaped_rhs_only): + Gemm config file is a csv file with fields separated by commas. The optional header and comments are ignored by the parser. + + A gemm config is a list of 4 positive integers and 3 boolean values: + m0 - Number of rows processed by the matrix multiplication + n0 - Number of columns processed by the matrix multiplication + k0 - Number of partial accumulations performed by the matrix multiplication + h0 - Number of horizontal blocks of size (k0xn0) stored on the same output row + interleave_rhs - Interleave rhs matrix (1) / Do not interleave rhs matrix (0) + transpose_rhs - Transpose rhs matrix (1) / Do not transpose rhs matrix (0) + export_to_cl_image_rhs - (Not supported for quantized types) Export rhs matrix to cl_image (1) / Do not export rhs matrix to cl_image (0). Can only be true + with certain combinations of the GEMMParams and other configs. Please refer to CLGEMMReshapeRHSMatrixKernel + for more details + + Only the following configurations of M0, N0 and K0 are currently supported: + M0 = 1, 2, 3, 4, 5, 6, 7, 8 + N0 = 2, 3, 4, 8, 16 + K0 = 2, 3, 4, 8, 16 + H0 >= 1 + + An example gemm config file looks like: + 4,4,4,1,1,1,0 + 4,4,4,3,1,0,1 + ... + +EOF +} + +####################################### +# Print gemm config file for reshaped help message +# Globals: +# None +# Arguments: +# None +# Returns: +# None +####################################### +function help_gemm_config_file_reshaped() { + cat >&2 << EOF +Gemm config file (Strategy reshaped): + Gemm config file is a csv file with fields separated by commas. The header and comments are ignored by the parser. + + A gemm config is a list of 5 positive integers and 4 boolean values: + m0 - Number of rows processed by the matrix multiplication + n0 - Number of columns processed by the matrix multiplication + k0 - Number of partial accumulations performed by the matrix multiplication + v0 - Number of vertical blocks of size (m0xk0) stored on the same output row + h0 - Number of horizontal blocks of size (k0xn0) stored on the same output row + interleave_lhs - Interleave lhs matrix (1) / Do not interleave lhs matrix (0) + interleave_rhs - Interleave rhs matrix (1) / Do not interleave rhs matrix (0) + transpose_rhs - Transpose rhs matrix but not lhs matrix (1) / Do not transpose rhs matrix but do transpose lhs matrix (0) + export_to_cl_image_rhs - (Not supported for quantized types) Export rhs matrix to cl_image (1) / Do not export rhs matrix to cl_image (0). Can only be true + with certain combinations of the GEMMParams and other configs. Please refer to CLGEMMReshapeRHSMatrixKernel + for more details + + If rhs matrix is transposed only the following configurations are currently supported: + M0 = 2, 3, 4, 5, 6, 7, 8 + N0 = 2, 3, 4, 8, 16 + K0 = 2, 3, 4, 8, 16 + V0 >= 1 + H0 >= 1 + + If lhs matrix is transposed only the following configurations are currently supported: + M0 = 2, 3, 4, 8 + N0 = 2, 3, 4, 8, 16 + K0 = 2, 3, 4, 8, 16 + V0 >= 1 + H0 >= 1 + + An example gemm config file looks like: + 4,4,4,1,3,1,1,1,0 + 4,4,4,3,3,1,1,0,1 + ... + +EOF +} + +####################################### +# Print usage of this program and exit with Error +# Globals: +# Assumes all globals are required +# Arguments: +# None +# Returns: +# Error(1) +####################################### +function usage() { + cat >&2 << EOF +Run gemm examples of a selected strategy, over provided tunable configurationsa and gemm shapes. +Save the benchmark results to json files in an output directory. + +Usage: ${CMD} [-h] -s -e -g -c [-o ] [-d ] [-i ] [-n ] [-t ] + +Options: + -h + Print help messages. If a strategy is specified with -s , then only display messages relevant to that + strategy. Otherwise if no strategy is specified, display messages for all available strategies. + + -s + Strategy option. + Options: ${ALL_STRATEGY_OPTIONS[@]}. + + -e + Path to directory that holds all example binaries + + -g + Path to gemm shape csv file + + -c + Path to gemm config csv file + + -d + Data type option with which to run benchmark examples + Default: ${DEFAULT_DATA_TYPE} + Supported options: + Strategy : Data Types + Native : f32 + Reshaped : f32, f16, qasymm8 + Reshaped RHS Only : f32, f16, qasymm8 + + -o + Path to output directory that holds output json files + Default: ${DEFAULT_OUT_DIR} + + -i + ID of the first experiment. + Default: ${DEFAULT_ID_EXPERIMENT_START} + + -n + Total number of experiments to execute in this session. [1-all] + Default: ${DEFAULT_NUM_EXPERIMENTS} + + -t + Output file extension. + Default: ${DEFAULT_OUT_EXTENSION} + +EOF +# Print help messages about gemm shapes and various gemm configs +$HELP && help_gemm_shape_file +$HELP && ( [ "${STRATEGY_OPTION}" == "" ] || [ "${STRATEGY_OPTION}" == "native" ] ) && help_gemm_config_file_native +$HELP && ( [ "${STRATEGY_OPTION}" == "" ] || [ "${STRATEGY_OPTION}" == "reshaped_rhs_only" ] ) && help_gemm_config_file_reshaped_rhs_only +$HELP && ( [ "${STRATEGY_OPTION}" == "" ] || [ "${STRATEGY_OPTION}" == "reshaped" ] ) && help_gemm_config_file_reshaped +exit 1 +} + +####################################### +# Print error message and exit with Error. +# Globals: +# None +# Arguments: +# $1 - Error message +# Returns: +# None +####################################### +function error_msg() { + echo "Error: $1" 1>&2 + exit 1 +} + +####################################### +# Convert string to lower-case +# Globals: +# None +# Arguments: +# target - String +# Returns: +# (stdout) - String in lowercase +####################################### +function to_lower() { + local target=$1 + echo "$target" | tr '[:upper:]' '[:lower:]' +} + +####################################### +# Test if the argument is an integer +# Globals: +# None +# Arguments: +# in - Input +# Returns: +# true/false +####################################### +function is_integer() { + local in=$1 + [ "$in" -eq "$in" ] 2> /dev/null +} + +####################################### +# Test if a string is in an array of strings +# Globals: +# None +# Arguments: +# target - String to test +# array - Array of strings to search +# Returns: +# true/false +####################################### +function arr_contains() { + local target=$1 + shift + local array + array=("$@") + for s in "${array[@]}" + do + [ "$s" == "${target}" ] && return + done + false +} + +####################################### +# Run a single example with all tunable gemm configurations on all gemm parameters +# Globals: +# OUT_DIR +# OUT_EXTENSION +# EXAMPLE_BIN_DIR +# NUM_ITERATION +# GEMM_CONFIGS_FILE +# GEMM_SHAPES_FILE +# STRATEGY_OPTION +# DATA_TYPE +# OUT_DIR +# ID_EXPERIMENT_START +# NUM_EXPERIMENTS + +# Arguments: +# example_bin Name of the example binary to run +# Returns: +# None +####################################### +function run() { + local example_bin=$1 + echo "Running experiments for ${example_bin}" 1>&2 + local example_args + local json_filename + local expr_count=0 + # Total number of experiments available + local num_experiments_total + # Total number of experiment runs scheduled for this session + local num_experiments_session + local id_experiment_start + local id_experiment_end + local array_shapes + local array_configs + local array_shapes_len + local array_configs_len + local array_shapes_idx + local array_configs_idx + local match_expression_shape="^([^,]*,){3}[^,]*$" + local match_expression_config="^(\s*[0-9]+\s*,)+\s*[0-9]\s*$" + local shapes_list_cmd="grep -E "$match_expression_shape" "${GEMM_SHAPES_FILE}"" + local configs_list_cmd="grep -E "$match_expression_config" "${GEMM_CONFIGS_FILE}"" + + # Create array from CSV file + array_shapes=($( $shapes_list_cmd )) + array_configs=($( $configs_list_cmd )) + + # Get array length + array_shapes_len=${#array_shapes[@]} + array_configs_len=${#array_configs[@]} + + # Get the total number of experiments available + (( num_experiments_total=${array_shapes_len} * ${array_configs_len} )) + + # Get the number of experiments to execute in this session + if [ ${NUM_EXPERIMENTS} == ${DEFAULT_NUM_EXPERIMENTS} ] + then + (( num_experiments_session=${array_shapes_len} * ${array_configs_len} )) + else + num_experiments_session=$NUM_EXPERIMENTS + fi + + # Id experiment start + id_experiment_start=${ID_EXPERIMENT_START} + + # Id experiment end + (( id_experiment_end=(${num_experiments_session} + ${id_experiment_start} - 1) )) + + # Check if the id experiment end is grater than or equal to the total number of experiments available. + # If the condition is satisfied, clamp the id experiment end + if [ "$id_experiment_end" -ge "$num_experiments_total" ] + then + echo "Clamping idx experiment end" 1>&2 + (( id_experiment_end=${num_experiments_total} - 1 )) + (( num_experiments_session=${id_experiment_start} + ${id_experiment_end} + 1 )) + fi + + # Time elapsed since the beginning in seconds + local time_elapsed_s + # Time estimated to finish in seconds + local time_est_s + echo "Running a total number of ${num_experiments_session} experiments" 1>&2 + echo "Experiment idx start/end [${id_experiment_start}, ${id_experiment_end}]" 1>&2 + + # Run experiments + for i in $(seq $id_experiment_start $id_experiment_end); + do + (( array_shapes_idx=${i} / ${array_configs_len} )) + (( array_configs_idx=${i} % ${array_configs_len} )) + + gemm_shape=${array_shapes[$array_shapes_idx]} + gemm_config=${array_configs[$array_configs_idx]} + + echo "Running shape[$array_shapes_idx]=$gemm_shape with config[$array_configs_idx]=$gemm_config" 1>&2 + + example_args="${gemm_shape},${gemm_config},--type=${DATA_TYPE}" + json_filename="${STRATEGY_OPTION}_${gemm_shape}_${gemm_config}_${DATA_TYPE}" + # Replace "," with "_" + json_filename=${json_filename//,/_} + + # Run experiment + ${EXAMPLE_BIN_DIR}/${example_bin} --example_args=${example_args} --iterations=${NUM_ITERATION} --json-file=${OUT_DIR}/${json_filename}.${OUT_EXTENSION} --instruments=OPENCL_TIMER_MS + # Print progress + (( expr_count++ )) + print_progress ${expr_count} ${num_experiments_session} + # Print time statistics + time_elapsed_s=$SECONDS + echo "Time elapsed since beginning: $(( $time_elapsed_s / 60 ))m $(( $time_elapsed_s % 60 ))s" 1>&2 + (( time_est_s=(${num_experiments_session} - ${expr_count}) * ${time_elapsed_s} / ${expr_count} )) + echo "Time estimated to finish: $(( $time_est_s / 60 ))m $(( $time_est_s % 60 ))s" 1>&2 + echo "Done." 1>&2 + done + + echo "Finished running all configs for ${example_bin}" 1>&2 + echo "All results saved to ${OUT_DIR}" 1>&2 +} + +####################################### +# Print the progress of the current session +# Globals: +# None +# Arguments: +# current Current number of items +# total Total number of items +# Returns: +# None +####################################### +function print_progress() { + local current + local total + current=$1 + total=$2 + # Width of progress bar + local width + width=20 + (( current_width= $width * current / total )) + echo -n -e "Progress [" 1>&2 + for i in $(seq 1 ${width}); do + if [[ $i -le ${current_width} ]]; then + echo -n "#" 1>&2 + else + echo -n " " 1>&2 + fi + done + echo "] $current / $total Experiments" 1>&2 +} + +# Functions }}} + +# Main: Main script {{{ +# Path to directory containing all benchmark examples binaries +EXAMPLE_BIN_DIR="" +# Path to gemm shapes file +GEMM_SHAPES_FILE="" +# Path to gemm configs file +GEMM_CONFIGS_FILE="" +# Strategy option +STRATEGY_OPTION="" +# Data type to use +DATA_TYPE=${DEFAULT_DATA_TYPE} +# Path to output directory +OUT_DIR=${DEFAULT_OUT_DIR} +# ID of the first experiment +ID_EXPERIMENT_START=${DEFAULT_ID_EXPERIMENT_START} +# Total number of experiments to execute in this session +NUM_EXPERIMENTS=${DEFAULT_NUM_EXPERIMENTS} +# Output benchmark result file extension +OUT_EXTENSION=${DEFAULT_OUT_EXTENSION} +# Toggle help +HELP=false + +# Obtain options +while getopts "hs:e:g:c:d:o:i:n:t:" opt; do + case "$opt" in + h) HELP=true ;; + s) STRATEGY_OPTION=$(to_lower "${OPTARG}");; + e) EXAMPLE_BIN_DIR="${OPTARG}";; + g) GEMM_SHAPES_FILE="${OPTARG}";; + c) GEMM_CONFIGS_FILE="${OPTARG}";; + d) DATA_TYPE=$(to_lower "${OPTARG}");; + o) OUT_DIR="${OPTARG}";; + i) ID_EXPERIMENT_START="${OPTARG}";; + n) NUM_EXPERIMENTS="${OPTARG}";; + t) OUT_EXTENSION="${OPTARG}";; + esac +done +shift $((OPTIND - 1)) + +# Lazily print usage (after arguments have been parsed) +$HELP && + usage + +# Parse and validate options +# Verify all compulsory arguments are passed in +( [ ! -z "${STRATEGY_OPTION}" ] && [ ! -z "${EXAMPLE_BIN_DIR}" ] && [ ! -z "${GEMM_SHAPES_FILE}" ] && [ ! -z "${GEMM_CONFIGS_FILE}" ] ) || + usage + +# Verify example binaries directory exists +[ -d "${EXAMPLE_BIN_DIR}" ] || + error_msg "${EXAMPLE_BIN_DIR} does not exist." + +# Verify all benchmark example binaries exist +[ -f "${EXAMPLE_BIN_DIR}/${EXAMPLE_BIN_RESHAPED_RHS_ONLY}" ] || + error_msg "Cannot find ${EXAMPLE_BIN_RESHAPED_RHS_ONLY} at ${EXAMPLE_BIN_DIR}" + +# Verify Gemm shapes file exists +[ -f "${GEMM_SHAPES_FILE}" ] || + error_msg "Cannot find gemm shapes file ${GEMM_SHAPES_FILE}" + +# Verify Gemm configs file exists +[ -f "${GEMM_CONFIGS_FILE}" ] || + error_msg "Cannot find gemm configs file ${GEMM_CONFIGS_FILE}" + +# Verify strategy option is valid +arr_contains "${STRATEGY_OPTION}" "${ALL_STRATEGY_OPTIONS[@]}" || + error_msg "Does not support strategy ${STRATEGY_OPTION}" + +# Verify data type option is valid +arr_contains "${DATA_TYPE}" "${ALL_DATA_TYPE_OPTIONS[@]}" || + error_msg "Does not support data type ${DATA_TYPE}" + +# Make sure existing benchmark outputs are not overwritten +[ ! -d "${OUT_DIR}" ] || + error_msg "Output directory ${OUT_DIR} already exists!" + +# Make output directory +echo "Making output directory ${OUT_DIR}" 1>&2 +mkdir -p ${OUT_DIR} || error_msg "Failed to make output directory ${OUT_DIR}" + +# Run selected strategy with all configurations +# Restart the built-in timer +SECONDS=0 +if [ "$DATA_TYPE" == "qasymm8" ]; then + [ "${STRATEGY_OPTION}" == "reshaped_rhs_only" ] && run $EXAMPLE_BIN_RESHAPED_RHS_ONLY_LOWP + [ "${STRATEGY_OPTION}" == "reshaped" ] && run $EXAMPLE_BIN_RESHAPED_LOWP +else + [ "${STRATEGY_OPTION}" == "native" ] && run $EXAMPLE_BIN_NATIVE + [ "${STRATEGY_OPTION}" == "reshaped_rhs_only" ] && run $EXAMPLE_BIN_RESHAPED_RHS_ONLY + [ "${STRATEGY_OPTION}" == "reshaped" ] && run $EXAMPLE_BIN_RESHAPED +fi +# Main: Main script }}} -- cgit v1.2.1