IVGCVSW-5619 Enable OptimizerOptions for the python external delegate

 * Add reduce-fp32-to-fp16, reduce-fp32-to-bf16, debug-data, memory-import
options to external delegate
 * Simplify DelegateOptions
 * Add test mock models
 * Unit tests
 * Configure lfs to manage tflite files

Signed-off-by: Narumol Prangnawarat <narumol.prangnawarat@arm.com>
Change-Id: I1e4db468862ba03d4cb031347bc307cf940b3cb1

11 files changed, 202 insertions, 29 deletions

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000000..7d63010835
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+*.tflite filter=lfs diff=lfs merge=lfs -text
diff --git a/delegate/include/DelegateOptions.hpp b/delegate/include/DelegateOptions.hpp
index 82de07607e..ace0859f7e 100644
--- a/delegate/include/DelegateOptions.hpp
+++ b/delegate/include/DelegateOptions.hpp
@@ -29,13 +29,11 @@ public:
 
     DelegateOptions(armnn::Compute computeDevice,
                     const armnn::OptimizerOptions& optimizerOptions,
-                    const armnn::INetworkProperties& networkProperties = armnn::INetworkProperties(),
                     const armnn::Optional<armnn::LogSeverity>& logSeverityLevel = armnn::EmptyOptional(),
                     const armnn::Optional<armnn::DebugCallbackFunction>& func = armnn::EmptyOptional());
 
     DelegateOptions(const std::vector<armnn::BackendId>& backends,
                     const armnn::OptimizerOptions& optimizerOptions,
-                    const armnn::INetworkProperties& networkProperties = armnn::INetworkProperties(),
                     const armnn::Optional<armnn::LogSeverity>& logSeverityLevel = armnn::EmptyOptional(),
                     const armnn::Optional<armnn::DebugCallbackFunction>& func = armnn::EmptyOptional());
 
@@ -59,11 +57,11 @@ public:
 
     const armnn::OptimizerOptions& GetOptimizerOptions() const { return m_OptimizerOptions; }
 
+    void SetOptimizerOptions(const armnn::OptimizerOptions& optimizerOptions) { m_OptimizerOptions = optimizerOptions; }
+
     const armnn::Optional<armnn::DebugCallbackFunction>& GetDebugCallbackFunction() const
         { return m_DebugCallbackFunc; }
 
-    const armnn::INetworkProperties& GetNetworkProperties() const { return m_NetworkProperties; };
-
 private:
     /// Which backend to run Delegate on.
     /// Examples of possible values are: CpuRef, CpuAcc, GpuAcc.
@@ -96,17 +94,12 @@ private:
     /// bool m_Debug;
     /// Reduce Fp32 data to Bf16 for faster processing
     /// bool m_ReduceFp32ToBf16;
-    /// Infer output size when not available
-    /// ShapeInferenceMethod m_shapeInferenceMethod;
     /// Enable Import
     /// bool m_ImportEnabled;
     /// Enable Model Options
     /// ModelOptions m_ModelOptions;
     armnn::OptimizerOptions m_OptimizerOptions;
 
-    /// Network properties to enable memory import
-    armnn::INetworkProperties m_NetworkProperties;
-
     /// Severity level for logging within ArmNN that will be used on creation of the delegate
     armnn::Optional<armnn::LogSeverity> m_LoggingSeverity;
 
diff --git a/delegate/python/test/test_data/conv2d.tflite b/delegate/python/test/test_data/conv2d.tflite
new file mode 100644
index 0000000000..3350d6408a
--- /dev/null
+++ b/delegate/python/test/test_data/conv2d.tflite
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cafc0de9b9f36afe76feff0bc1e5e4dd7ab4201da359b9faca236ba24cbcdd60
+size 728
diff --git a/delegate/python/test/test_data/fallback_model.tflite b/delegate/python/test/test_data/fallback_model.tflite
new file mode 100644
index 0000000000..b86ce6d556
--- /dev/null
+++ b/delegate/python/test/test_data/fallback_model.tflite
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44adf470f17f1d635131c5b59b0c888871b4b793dbeaf9d91946375fd63a5ef3
+size 704
diff --git a/delegate/python/test/test_data/fp32_model.tflite b/delegate/python/test/test_data/fp32_model.tflite
new file mode 100644
index 0000000000..15a0cabc22
--- /dev/null
+++ b/delegate/python/test/test_data/fp32_model.tflite
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:549eeeeb3ee3984cf1636917f8b6c80914b6ce3b7553220c05858fa3d438371b
+size 688
diff --git a/delegate/python/test/test_external_delegate.py b/delegate/python/test/test_external_delegate.py
index 995de8cd70..93d373d0a1 100644
--- a/delegate/python/test/test_external_delegate.py
+++ b/delegate/python/test/test_external_delegate.py
@@ -1,11 +1,11 @@
 # Copyright © 2020 Arm Ltd and Contributors. All rights reserved.
 # SPDX-License-Identifier: MIT
 
+import numpy as np
 import pytest
 import tflite_runtime.interpreter as tflite
 import os
-from utils import run_mock_model
-
+from utils import run_mock_model, run_inference, compare_outputs
 
 def test_external_delegate_unknown_options(delegate_dir):
     print(delegate_dir)
@@ -14,7 +14,6 @@ def test_external_delegate_unknown_options(delegate_dir):
             delegate_dir,
             options={"wrong": "wrong"})
 
-
 def test_external_delegate_options_multiple_backends(delegate_dir):
     tflite.load_delegate(
         delegate_dir,
@@ -63,3 +62,104 @@ def test_external_delegate_options_wrong_logging_level(delegate_dir):
         tflite.load_delegate(
             delegate_dir,
             options={"logging-severity": "wrong"})
+
+def test_external_delegate_options_debug(capfd, delegate_dir, test_data_folder):
+    # create armnn delegate with debug option
+    armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuRef', 'debug-data': '1'})
+
+    model_file_name = 'fp32_model.tflite'
+
+    tensor_shape = [1, 2, 2, 1]
+
+    input0 = np.array([1, 2, 3, 4], dtype=np.float32).reshape(tensor_shape)
+    input1 = np.array([2, 2, 3, 4], dtype=np.float32).reshape(tensor_shape)
+    inputs = [input0, input0, input1]
+    expected_output = np.array([1, 2, 2, 2], dtype=np.float32).reshape(tensor_shape)
+
+    # run the inference
+    armnn_outputs = run_inference(test_data_folder, model_file_name, inputs, [armnn_delegate])
+
+    # check results
+    compare_outputs(armnn_outputs, [expected_output])
+
+    captured = capfd.readouterr()
+    assert 'layerGuid' in captured.out
+
+
+def test_external_delegate_options_fp32_to_fp16(capfd, delegate_dir, test_data_folder):
+    # create armnn delegate with reduce-fp32-to-fp16 option
+    armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuRef',
+                                                                   'debug-data': '1',
+                                                                   'reduce-fp32-to-fp16': '1'})
+
+    model_file_name = 'fp32_model.tflite'
+
+    tensor_shape = [1, 2, 2, 1]
+
+    input0 = np.array([1, 2, 3, 4], dtype=np.float32).reshape(tensor_shape)
+    input1 = np.array([2, 2, 3, 4], dtype=np.float32).reshape(tensor_shape)
+    inputs = [input0, input0, input1]
+    expected_output = np.array([1, 2, 2, 2], dtype=np.float32).reshape(tensor_shape)
+
+    # run the inference
+    armnn_outputs = run_inference(test_data_folder, model_file_name, inputs, [armnn_delegate])
+
+    # check results
+    compare_outputs(armnn_outputs, [expected_output])
+
+    captured = capfd.readouterr()
+    assert 'convert_fp32_to_fp16' in captured.out
+    assert 'convert_fp16_to_fp32' in captured.out
+
+def test_external_delegate_options_fp32_to_bf16(capfd, delegate_dir, test_data_folder):
+    # create armnn delegate with reduce-fp32-to-bf16 option
+    armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuRef',
+                                                                   'debug-data': '1',
+                                                                   'reduce-fp32-to-bf16': '1'})
+
+    model_file_name = 'conv2d.tflite'
+
+    inputShape = [ 1, 5, 5, 1 ]
+    outputShape = [ 1, 3, 3, 1 ]
+
+    inputValues = [ 1, 5, 2, 3, 5,
+                    8, 7, 3, 6, 3,
+                    3, 3, 9, 1, 9,
+                    4, 1, 8, 1, 3,
+                    6, 8, 1, 9, 2 ]
+
+    expectedResult = [ 28, 38, 29,
+                       96, 104, 53,
+                       31, 55, 24 ]
+
+    input = np.array(inputValues, dtype=np.float32).reshape(inputShape)
+    expected_output = np.array(expectedResult, dtype=np.float32).reshape(outputShape)
+
+    # run the inference
+    armnn_outputs = run_inference(test_data_folder, model_file_name, [input], [armnn_delegate])
+
+    # check results
+    compare_outputs(armnn_outputs, [expected_output])
+
+    captured = capfd.readouterr()
+    assert 'convert_fp32_to_bf16' in captured.out
+
+def test_external_delegate_options_memory_import(delegate_dir, test_data_folder):
+    # create armnn delegate with memory-import option
+    armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuAcc,CpuRef',
+                                                                   'memory-import': '1'})
+
+    model_file_name = 'fallback_model.tflite'
+
+    tensor_shape = [1, 2, 2, 1]
+
+    input0 = np.array([1, 2, 3, 4], dtype=np.uint8).reshape(tensor_shape)
+    input1 = np.array([2, 2, 3, 4], dtype=np.uint8).reshape(tensor_shape)
+    inputs = [input0, input0, input1]
+    expected_output = np.array([1, 2, 2, 2], dtype=np.uint8).reshape(tensor_shape)
+
+    # run the inference
+    armnn_outputs = run_inference(test_data_folder, model_file_name, inputs, [armnn_delegate])
+
+    # check results
+    compare_outputs(armnn_outputs, [expected_output])
\ No newline at end of file
diff --git a/delegate/python/test/utils.py b/delegate/python/test/utils.py
index 3adc24fe35..f3761ec8a1 100644
--- a/delegate/python/test/utils.py
+++ b/delegate/python/test/utils.py
@@ -21,4 +21,33 @@ def run_mock_model(delegate, test_data_folder):
     input_data = np.array(np.random.random_sample(input_shape), dtype=np.uint8)
     interpreter.set_tensor(input_details[0]['index'], input_data)
 
-    interpreter.invoke()
\ No newline at end of file
+    interpreter.invoke()
+
+def run_inference(test_data_folder, model_filename, inputs, delegates=None):
+    model_path = os.path.join(test_data_folder, model_filename)
+    interpreter = tflite.Interpreter(model_path=model_path,
+                                     experimental_delegates=delegates)
+    interpreter.allocate_tensors()
+
+    # Get input and output tensors.
+    input_details = interpreter.get_input_details()
+    output_details = interpreter.get_output_details()
+
+    # Set inputs to tensors.
+    for i in range(len(inputs)):
+        interpreter.set_tensor(input_details[i]['index'], inputs[i])
+
+    interpreter.invoke()
+
+    results = []
+    for output in output_details:
+        results.append(interpreter.get_tensor(output['index']))
+
+    return results
+
+def compare_outputs(outputs, expected_outputs):
+    assert len(outputs) == len(expected_outputs), 'Incorrect number of outputs'
+    for i in range(len(expected_outputs)):
+        assert outputs[i].shape == expected_outputs[i].shape, 'Incorrect output shape on output#{}'.format(i)
+        assert outputs[i].dtype == expected_outputs[i].dtype, 'Incorrect output data type on output#{}'.format(i)
+        assert outputs[i].all() == expected_outputs[i].all(), 'Incorrect output value on output#{}'.format(i)
\ No newline at end of file
diff --git a/delegate/src/DelegateOptions.cpp b/delegate/src/DelegateOptions.cpp
index 400bf78766..d4d906a83a 100644
--- a/delegate/src/DelegateOptions.cpp
+++ b/delegate/src/DelegateOptions.cpp
@@ -24,13 +24,11 @@ DelegateOptions::DelegateOptions(const std::vector<armnn::BackendId>& backends,
 
 DelegateOptions::DelegateOptions(armnn::Compute computeDevice,
                                  const armnn::OptimizerOptions& optimizerOptions,
-                                 const armnn::INetworkProperties& networkProperties,
                                  const armnn::Optional<armnn::LogSeverity>& logSeverityLevel,
                                  const armnn::Optional<armnn::DebugCallbackFunction>& func)
     : m_Backends({computeDevice}),
       m_BackendOptions({}),
       m_OptimizerOptions(optimizerOptions),
-      m_NetworkProperties(networkProperties),
       m_LoggingSeverity(logSeverityLevel),
       m_DebugCallbackFunc(func)
 {
@@ -38,13 +36,11 @@ DelegateOptions::DelegateOptions(armnn::Compute computeDevice,
 
 DelegateOptions::DelegateOptions(const std::vector<armnn::BackendId>& backends,
                                  const armnn::OptimizerOptions& optimizerOptions,
-                                 const armnn::INetworkProperties& networkProperties,
                                  const armnn::Optional<armnn::LogSeverity>& logSeverityLevel,
                                  const armnn::Optional<armnn::DebugCallbackFunction>& func)
     : m_Backends(backends),
       m_BackendOptions({}),
       m_OptimizerOptions(optimizerOptions),
-      m_NetworkProperties(networkProperties),
       m_LoggingSeverity(logSeverityLevel),
       m_DebugCallbackFunc(func)
 {
diff --git a/delegate/src/armnn_delegate.cpp b/delegate/src/armnn_delegate.cpp
index 6f72d864b9..639e514a78 100644
--- a/delegate/src/armnn_delegate.cpp
+++ b/delegate/src/armnn_delegate.cpp
@@ -353,10 +353,21 @@ ArmnnSubgraph* ArmnnSubgraph::Create(TfLiteContext* tfLiteContext,
     {
         // Load graph into runtime
         std::string errorMessage;
-        auto loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
-                                                              std::move(optNet),
-                                                              errorMessage,
-                                                              delegate->m_Options.GetNetworkProperties());
+        armnn::Status loadingStatus;
+        if (delegate->m_Options.GetOptimizerOptions().m_ImportEnabled)
+        {
+            armnn::INetworkProperties networkProperties(true, true);
+            loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
+                                                             std::move(optNet),
+                                                             errorMessage,
+                                                             networkProperties);
+        }
+        else
+        {
+            loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
+                                                             std::move(optNet),
+                                                             errorMessage);
+        }
         if (loadingStatus != armnn::Status::Success)
         {
             // Optimize failed
diff --git a/delegate/src/armnn_external_delegate.cpp b/delegate/src/armnn_external_delegate.cpp
index 53b17256af..4dba07d3b8 100644
--- a/delegate/src/armnn_external_delegate.cpp
+++ b/delegate/src/armnn_external_delegate.cpp
@@ -62,6 +62,22 @@ std::vector<std::string> gpu_options {"gpu-tuning-level",
  *    Possible values: ["true"/"false"] \n
  *    Description: Enables GPU kernel profiling
  *
+ *    Option key: "reduce-fp32-to-fp16" \n
+ *    Possible values: ["true"/"false"] \n
+ *    Description: Reduce Fp32 data to Fp16 for faster processing
+ *
+ *    Option key: "reduce-fp32-to-bf16" \n
+ *    Possible values: ["true"/"false"] \n
+ *    Description: Reduce Fp32 data to Bf16 for faster processing
+ *
+ *    Option key: "debug-data" \n
+ *    Possible values: ["true"/"false"] \n
+ *    Description: Add debug data for easier troubleshooting
+ *
+ *    Option key: "memory-import" \n
+ *    Possible values: ["true"/"false"] \n
+ *    Description: Enable memory import
+ *
  *
  * @param[in]     option_keys     Delegate option names
  * @param[in]     options_values  Delegate option values
@@ -81,6 +97,7 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
     {
         // (Initializes with CpuRef backend)
         armnnDelegate::DelegateOptions options = armnnDelegate::TfLiteArmnnDelegateOptionsDefault();
+        armnn::OptimizerOptions optimizerOptions;
         for (size_t i = 0; i < num_options; ++i)
         {
             // Process backends
@@ -118,11 +135,32 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
                 armnn::BackendOptions option("GpuAcc", {{"KernelProfilingEnabled", (*options_values[i] != '0')}});
                 options.AddBackendOption(option);
             }
+            // Process reduce-fp32-to-fp16 option
+            else if (std::string(options_keys[i]) == std::string("reduce-fp32-to-fp16"))
+            {
+               optimizerOptions.m_ReduceFp32ToFp16 = *options_values[i] != '0';
+            }
+            // Process reduce-fp32-to-bf16 option
+            else if (std::string(options_keys[i]) == std::string("reduce-fp32-to-bf16"))
+            {
+               optimizerOptions.m_ReduceFp32ToBf16 = *options_values[i] != '0';
+            }
+            // Process debug-data
+            else if (std::string(options_keys[i]) == std::string("debug-data"))
+            {
+               optimizerOptions.m_Debug = *options_values[i] != '0';
+            }
+            // Process memory-import
+            else if (std::string(options_keys[i]) == std::string("memory-import"))
+            {
+               optimizerOptions.m_ImportEnabled = *options_values[i] != '0';
+            }
             else
             {
                 throw armnn::Exception("Unknown option for the ArmNN Delegate given: " + std::string(options_keys[i]));
             }
         }
+        options.SetOptimizerOptions(optimizerOptions);
         delegate = TfLiteArmnnDelegateCreate(options);
     }
     catch (const std::exception& ex)
diff --git a/delegate/src/test/DelegateOptionsTest.cpp b/delegate/src/test/DelegateOptionsTest.cpp
index c623781301..23510c7777 100644
--- a/delegate/src/test/DelegateOptionsTest.cpp
+++ b/delegate/src/test/DelegateOptionsTest.cpp
@@ -25,8 +25,7 @@ TEST_CASE ("ArmnnDelegateOptimizerOptionsReduceFp32ToFp16")
 
         // Enable ReduceFp32ToFp16
         armnn::OptimizerOptions optimizerOptions(true, true, false, false);
-        armnn::INetworkProperties networkProperties;
-        armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions, networkProperties);
+        armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions);
 
         DelegateOptionTest<float>(::tflite::TensorType_FLOAT32,
                                   backends,
@@ -56,8 +55,7 @@ TEST_CASE ("ArmnnDelegateOptimizerOptionsDebug")
 
         // Enable Debug
         armnn::OptimizerOptions optimizerOptions(false, true, false, false);
-        armnn::INetworkProperties networkProperties;
-        armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions, networkProperties);
+        armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions);
 
         DelegateOptionTest<float>(::tflite::TensorType_FLOAT32,
                                   backends,
@@ -98,7 +96,6 @@ TEST_CASE ("ArmnnDelegateOptimizerOptionsDebugFunction")
     armnn::INetworkProperties networkProperties;
     armnnDelegate::DelegateOptions delegateOptions(backends,
                                                    optimizerOptions,
-                                                   networkProperties,
                                                    armnn::EmptyOptional(),
                                                    armnn::Optional<armnn::DebugCallbackFunction>(mockCallback));
 
@@ -136,11 +133,10 @@ TEST_CASE ("ArmnnDelegateOptimizerOptionsImport")
     std::vector<int32_t> tensorShape { 1, 2, 2, 1 };
     std::vector<uint8_t> inputData = { 1, 2, 3, 4 };
     std::vector<uint8_t> divData = { 2, 2, 3, 4 };
-    std::vector<uint8_t> expectedResult = { 1, 2, 2, 2};
+    std::vector<uint8_t> expectedResult = { 1, 2, 2, 2 };
 
     armnn::OptimizerOptions optimizerOptions(false, false, false, true);
-    armnn::INetworkProperties networkProperties(true, true);
-    armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions, networkProperties);
+    armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions);
 
     DelegateOptionTest<uint8_t>(::tflite::TensorType_UINT8,
                                 backends,