From 74a3cf5755b801cf258177e8e55b4cda64a0c351 Mon Sep 17 00:00:00 2001 From: Narumol Prangnawarat Date: Fri, 29 Jan 2021 15:38:54 +0000 Subject: IVGCVSW-5619 Enable OptimizerOptions for the python external delegate * Add reduce-fp32-to-fp16, reduce-fp32-to-bf16, debug-data, memory-import options to external delegate * Simplify DelegateOptions * Add test mock models * Unit tests * Configure lfs to manage tflite files Signed-off-by: Narumol Prangnawarat Change-Id: I1e4db468862ba03d4cb031347bc307cf940b3cb1 --- delegate/include/DelegateOptions.hpp | 11 +-- delegate/python/test/test_data/conv2d.tflite | 3 + .../python/test/test_data/fallback_model.tflite | 3 + delegate/python/test/test_data/fp32_model.tflite | 3 + delegate/python/test/test_external_delegate.py | 106 ++++++++++++++++++++- delegate/python/test/utils.py | 31 +++++- delegate/src/DelegateOptions.cpp | 4 - delegate/src/armnn_delegate.cpp | 19 +++- delegate/src/armnn_external_delegate.cpp | 38 ++++++++ delegate/src/test/DelegateOptionsTest.cpp | 12 +-- 10 files changed, 201 insertions(+), 29 deletions(-) create mode 100644 delegate/python/test/test_data/conv2d.tflite create mode 100644 delegate/python/test/test_data/fallback_model.tflite create mode 100644 delegate/python/test/test_data/fp32_model.tflite (limited to 'delegate') diff --git a/delegate/include/DelegateOptions.hpp b/delegate/include/DelegateOptions.hpp index 82de07607e..ace0859f7e 100644 --- a/delegate/include/DelegateOptions.hpp +++ b/delegate/include/DelegateOptions.hpp @@ -29,13 +29,11 @@ public: DelegateOptions(armnn::Compute computeDevice, const armnn::OptimizerOptions& optimizerOptions, - const armnn::INetworkProperties& networkProperties = armnn::INetworkProperties(), const armnn::Optional& logSeverityLevel = armnn::EmptyOptional(), const armnn::Optional& func = armnn::EmptyOptional()); DelegateOptions(const std::vector& backends, const armnn::OptimizerOptions& optimizerOptions, - const armnn::INetworkProperties& networkProperties = armnn::INetworkProperties(), const armnn::Optional& logSeverityLevel = armnn::EmptyOptional(), const armnn::Optional& func = armnn::EmptyOptional()); @@ -59,11 +57,11 @@ public: const armnn::OptimizerOptions& GetOptimizerOptions() const { return m_OptimizerOptions; } + void SetOptimizerOptions(const armnn::OptimizerOptions& optimizerOptions) { m_OptimizerOptions = optimizerOptions; } + const armnn::Optional& GetDebugCallbackFunction() const { return m_DebugCallbackFunc; } - const armnn::INetworkProperties& GetNetworkProperties() const { return m_NetworkProperties; }; - private: /// Which backend to run Delegate on. /// Examples of possible values are: CpuRef, CpuAcc, GpuAcc. @@ -96,17 +94,12 @@ private: /// bool m_Debug; /// Reduce Fp32 data to Bf16 for faster processing /// bool m_ReduceFp32ToBf16; - /// Infer output size when not available - /// ShapeInferenceMethod m_shapeInferenceMethod; /// Enable Import /// bool m_ImportEnabled; /// Enable Model Options /// ModelOptions m_ModelOptions; armnn::OptimizerOptions m_OptimizerOptions; - /// Network properties to enable memory import - armnn::INetworkProperties m_NetworkProperties; - /// Severity level for logging within ArmNN that will be used on creation of the delegate armnn::Optional m_LoggingSeverity; diff --git a/delegate/python/test/test_data/conv2d.tflite b/delegate/python/test/test_data/conv2d.tflite new file mode 100644 index 0000000000..3350d6408a --- /dev/null +++ b/delegate/python/test/test_data/conv2d.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cafc0de9b9f36afe76feff0bc1e5e4dd7ab4201da359b9faca236ba24cbcdd60 +size 728 diff --git a/delegate/python/test/test_data/fallback_model.tflite b/delegate/python/test/test_data/fallback_model.tflite new file mode 100644 index 0000000000..b86ce6d556 --- /dev/null +++ b/delegate/python/test/test_data/fallback_model.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44adf470f17f1d635131c5b59b0c888871b4b793dbeaf9d91946375fd63a5ef3 +size 704 diff --git a/delegate/python/test/test_data/fp32_model.tflite b/delegate/python/test/test_data/fp32_model.tflite new file mode 100644 index 0000000000..15a0cabc22 --- /dev/null +++ b/delegate/python/test/test_data/fp32_model.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:549eeeeb3ee3984cf1636917f8b6c80914b6ce3b7553220c05858fa3d438371b +size 688 diff --git a/delegate/python/test/test_external_delegate.py b/delegate/python/test/test_external_delegate.py index 995de8cd70..93d373d0a1 100644 --- a/delegate/python/test/test_external_delegate.py +++ b/delegate/python/test/test_external_delegate.py @@ -1,11 +1,11 @@ # Copyright © 2020 Arm Ltd and Contributors. All rights reserved. # SPDX-License-Identifier: MIT +import numpy as np import pytest import tflite_runtime.interpreter as tflite import os -from utils import run_mock_model - +from utils import run_mock_model, run_inference, compare_outputs def test_external_delegate_unknown_options(delegate_dir): print(delegate_dir) @@ -14,7 +14,6 @@ def test_external_delegate_unknown_options(delegate_dir): delegate_dir, options={"wrong": "wrong"}) - def test_external_delegate_options_multiple_backends(delegate_dir): tflite.load_delegate( delegate_dir, @@ -63,3 +62,104 @@ def test_external_delegate_options_wrong_logging_level(delegate_dir): tflite.load_delegate( delegate_dir, options={"logging-severity": "wrong"}) + +def test_external_delegate_options_debug(capfd, delegate_dir, test_data_folder): + # create armnn delegate with debug option + armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuRef', 'debug-data': '1'}) + + model_file_name = 'fp32_model.tflite' + + tensor_shape = [1, 2, 2, 1] + + input0 = np.array([1, 2, 3, 4], dtype=np.float32).reshape(tensor_shape) + input1 = np.array([2, 2, 3, 4], dtype=np.float32).reshape(tensor_shape) + inputs = [input0, input0, input1] + expected_output = np.array([1, 2, 2, 2], dtype=np.float32).reshape(tensor_shape) + + # run the inference + armnn_outputs = run_inference(test_data_folder, model_file_name, inputs, [armnn_delegate]) + + # check results + compare_outputs(armnn_outputs, [expected_output]) + + captured = capfd.readouterr() + assert 'layerGuid' in captured.out + + +def test_external_delegate_options_fp32_to_fp16(capfd, delegate_dir, test_data_folder): + # create armnn delegate with reduce-fp32-to-fp16 option + armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuRef', + 'debug-data': '1', + 'reduce-fp32-to-fp16': '1'}) + + model_file_name = 'fp32_model.tflite' + + tensor_shape = [1, 2, 2, 1] + + input0 = np.array([1, 2, 3, 4], dtype=np.float32).reshape(tensor_shape) + input1 = np.array([2, 2, 3, 4], dtype=np.float32).reshape(tensor_shape) + inputs = [input0, input0, input1] + expected_output = np.array([1, 2, 2, 2], dtype=np.float32).reshape(tensor_shape) + + # run the inference + armnn_outputs = run_inference(test_data_folder, model_file_name, inputs, [armnn_delegate]) + + # check results + compare_outputs(armnn_outputs, [expected_output]) + + captured = capfd.readouterr() + assert 'convert_fp32_to_fp16' in captured.out + assert 'convert_fp16_to_fp32' in captured.out + +def test_external_delegate_options_fp32_to_bf16(capfd, delegate_dir, test_data_folder): + # create armnn delegate with reduce-fp32-to-bf16 option + armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuRef', + 'debug-data': '1', + 'reduce-fp32-to-bf16': '1'}) + + model_file_name = 'conv2d.tflite' + + inputShape = [ 1, 5, 5, 1 ] + outputShape = [ 1, 3, 3, 1 ] + + inputValues = [ 1, 5, 2, 3, 5, + 8, 7, 3, 6, 3, + 3, 3, 9, 1, 9, + 4, 1, 8, 1, 3, + 6, 8, 1, 9, 2 ] + + expectedResult = [ 28, 38, 29, + 96, 104, 53, + 31, 55, 24 ] + + input = np.array(inputValues, dtype=np.float32).reshape(inputShape) + expected_output = np.array(expectedResult, dtype=np.float32).reshape(outputShape) + + # run the inference + armnn_outputs = run_inference(test_data_folder, model_file_name, [input], [armnn_delegate]) + + # check results + compare_outputs(armnn_outputs, [expected_output]) + + captured = capfd.readouterr() + assert 'convert_fp32_to_bf16' in captured.out + +def test_external_delegate_options_memory_import(delegate_dir, test_data_folder): + # create armnn delegate with memory-import option + armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuAcc,CpuRef', + 'memory-import': '1'}) + + model_file_name = 'fallback_model.tflite' + + tensor_shape = [1, 2, 2, 1] + + input0 = np.array([1, 2, 3, 4], dtype=np.uint8).reshape(tensor_shape) + input1 = np.array([2, 2, 3, 4], dtype=np.uint8).reshape(tensor_shape) + inputs = [input0, input0, input1] + expected_output = np.array([1, 2, 2, 2], dtype=np.uint8).reshape(tensor_shape) + + # run the inference + armnn_outputs = run_inference(test_data_folder, model_file_name, inputs, [armnn_delegate]) + + # check results + compare_outputs(armnn_outputs, [expected_output]) \ No newline at end of file diff --git a/delegate/python/test/utils.py b/delegate/python/test/utils.py index 3adc24fe35..f3761ec8a1 100644 --- a/delegate/python/test/utils.py +++ b/delegate/python/test/utils.py @@ -21,4 +21,33 @@ def run_mock_model(delegate, test_data_folder): input_data = np.array(np.random.random_sample(input_shape), dtype=np.uint8) interpreter.set_tensor(input_details[0]['index'], input_data) - interpreter.invoke() \ No newline at end of file + interpreter.invoke() + +def run_inference(test_data_folder, model_filename, inputs, delegates=None): + model_path = os.path.join(test_data_folder, model_filename) + interpreter = tflite.Interpreter(model_path=model_path, + experimental_delegates=delegates) + interpreter.allocate_tensors() + + # Get input and output tensors. + input_details = interpreter.get_input_details() + output_details = interpreter.get_output_details() + + # Set inputs to tensors. + for i in range(len(inputs)): + interpreter.set_tensor(input_details[i]['index'], inputs[i]) + + interpreter.invoke() + + results = [] + for output in output_details: + results.append(interpreter.get_tensor(output['index'])) + + return results + +def compare_outputs(outputs, expected_outputs): + assert len(outputs) == len(expected_outputs), 'Incorrect number of outputs' + for i in range(len(expected_outputs)): + assert outputs[i].shape == expected_outputs[i].shape, 'Incorrect output shape on output#{}'.format(i) + assert outputs[i].dtype == expected_outputs[i].dtype, 'Incorrect output data type on output#{}'.format(i) + assert outputs[i].all() == expected_outputs[i].all(), 'Incorrect output value on output#{}'.format(i) \ No newline at end of file diff --git a/delegate/src/DelegateOptions.cpp b/delegate/src/DelegateOptions.cpp index 400bf78766..d4d906a83a 100644 --- a/delegate/src/DelegateOptions.cpp +++ b/delegate/src/DelegateOptions.cpp @@ -24,13 +24,11 @@ DelegateOptions::DelegateOptions(const std::vector& backends, DelegateOptions::DelegateOptions(armnn::Compute computeDevice, const armnn::OptimizerOptions& optimizerOptions, - const armnn::INetworkProperties& networkProperties, const armnn::Optional& logSeverityLevel, const armnn::Optional& func) : m_Backends({computeDevice}), m_BackendOptions({}), m_OptimizerOptions(optimizerOptions), - m_NetworkProperties(networkProperties), m_LoggingSeverity(logSeverityLevel), m_DebugCallbackFunc(func) { @@ -38,13 +36,11 @@ DelegateOptions::DelegateOptions(armnn::Compute computeDevice, DelegateOptions::DelegateOptions(const std::vector& backends, const armnn::OptimizerOptions& optimizerOptions, - const armnn::INetworkProperties& networkProperties, const armnn::Optional& logSeverityLevel, const armnn::Optional& func) : m_Backends(backends), m_BackendOptions({}), m_OptimizerOptions(optimizerOptions), - m_NetworkProperties(networkProperties), m_LoggingSeverity(logSeverityLevel), m_DebugCallbackFunc(func) { diff --git a/delegate/src/armnn_delegate.cpp b/delegate/src/armnn_delegate.cpp index 6f72d864b9..639e514a78 100644 --- a/delegate/src/armnn_delegate.cpp +++ b/delegate/src/armnn_delegate.cpp @@ -353,10 +353,21 @@ ArmnnSubgraph* ArmnnSubgraph::Create(TfLiteContext* tfLiteContext, { // Load graph into runtime std::string errorMessage; - auto loadingStatus = delegate->m_Runtime->LoadNetwork(networkId, - std::move(optNet), - errorMessage, - delegate->m_Options.GetNetworkProperties()); + armnn::Status loadingStatus; + if (delegate->m_Options.GetOptimizerOptions().m_ImportEnabled) + { + armnn::INetworkProperties networkProperties(true, true); + loadingStatus = delegate->m_Runtime->LoadNetwork(networkId, + std::move(optNet), + errorMessage, + networkProperties); + } + else + { + loadingStatus = delegate->m_Runtime->LoadNetwork(networkId, + std::move(optNet), + errorMessage); + } if (loadingStatus != armnn::Status::Success) { // Optimize failed diff --git a/delegate/src/armnn_external_delegate.cpp b/delegate/src/armnn_external_delegate.cpp index 53b17256af..4dba07d3b8 100644 --- a/delegate/src/armnn_external_delegate.cpp +++ b/delegate/src/armnn_external_delegate.cpp @@ -62,6 +62,22 @@ std::vector gpu_options {"gpu-tuning-level", * Possible values: ["true"/"false"] \n * Description: Enables GPU kernel profiling * + * Option key: "reduce-fp32-to-fp16" \n + * Possible values: ["true"/"false"] \n + * Description: Reduce Fp32 data to Fp16 for faster processing + * + * Option key: "reduce-fp32-to-bf16" \n + * Possible values: ["true"/"false"] \n + * Description: Reduce Fp32 data to Bf16 for faster processing + * + * Option key: "debug-data" \n + * Possible values: ["true"/"false"] \n + * Description: Add debug data for easier troubleshooting + * + * Option key: "memory-import" \n + * Possible values: ["true"/"false"] \n + * Description: Enable memory import + * * * @param[in] option_keys Delegate option names * @param[in] options_values Delegate option values @@ -81,6 +97,7 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys, { // (Initializes with CpuRef backend) armnnDelegate::DelegateOptions options = armnnDelegate::TfLiteArmnnDelegateOptionsDefault(); + armnn::OptimizerOptions optimizerOptions; for (size_t i = 0; i < num_options; ++i) { // Process backends @@ -118,11 +135,32 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys, armnn::BackendOptions option("GpuAcc", {{"KernelProfilingEnabled", (*options_values[i] != '0')}}); options.AddBackendOption(option); } + // Process reduce-fp32-to-fp16 option + else if (std::string(options_keys[i]) == std::string("reduce-fp32-to-fp16")) + { + optimizerOptions.m_ReduceFp32ToFp16 = *options_values[i] != '0'; + } + // Process reduce-fp32-to-bf16 option + else if (std::string(options_keys[i]) == std::string("reduce-fp32-to-bf16")) + { + optimizerOptions.m_ReduceFp32ToBf16 = *options_values[i] != '0'; + } + // Process debug-data + else if (std::string(options_keys[i]) == std::string("debug-data")) + { + optimizerOptions.m_Debug = *options_values[i] != '0'; + } + // Process memory-import + else if (std::string(options_keys[i]) == std::string("memory-import")) + { + optimizerOptions.m_ImportEnabled = *options_values[i] != '0'; + } else { throw armnn::Exception("Unknown option for the ArmNN Delegate given: " + std::string(options_keys[i])); } } + options.SetOptimizerOptions(optimizerOptions); delegate = TfLiteArmnnDelegateCreate(options); } catch (const std::exception& ex) diff --git a/delegate/src/test/DelegateOptionsTest.cpp b/delegate/src/test/DelegateOptionsTest.cpp index c623781301..23510c7777 100644 --- a/delegate/src/test/DelegateOptionsTest.cpp +++ b/delegate/src/test/DelegateOptionsTest.cpp @@ -25,8 +25,7 @@ TEST_CASE ("ArmnnDelegateOptimizerOptionsReduceFp32ToFp16") // Enable ReduceFp32ToFp16 armnn::OptimizerOptions optimizerOptions(true, true, false, false); - armnn::INetworkProperties networkProperties; - armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions, networkProperties); + armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions); DelegateOptionTest(::tflite::TensorType_FLOAT32, backends, @@ -56,8 +55,7 @@ TEST_CASE ("ArmnnDelegateOptimizerOptionsDebug") // Enable Debug armnn::OptimizerOptions optimizerOptions(false, true, false, false); - armnn::INetworkProperties networkProperties; - armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions, networkProperties); + armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions); DelegateOptionTest(::tflite::TensorType_FLOAT32, backends, @@ -98,7 +96,6 @@ TEST_CASE ("ArmnnDelegateOptimizerOptionsDebugFunction") armnn::INetworkProperties networkProperties; armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions, - networkProperties, armnn::EmptyOptional(), armnn::Optional(mockCallback)); @@ -136,11 +133,10 @@ TEST_CASE ("ArmnnDelegateOptimizerOptionsImport") std::vector tensorShape { 1, 2, 2, 1 }; std::vector inputData = { 1, 2, 3, 4 }; std::vector divData = { 2, 2, 3, 4 }; - std::vector expectedResult = { 1, 2, 2, 2}; + std::vector expectedResult = { 1, 2, 2, 2 }; armnn::OptimizerOptions optimizerOptions(false, false, false, true); - armnn::INetworkProperties networkProperties(true, true); - armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions, networkProperties); + armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions); DelegateOptionTest(::tflite::TensorType_UINT8, backends, -- cgit v1.2.1