IVGCVSW-5619 Enable OptimizerOptions for the python external delegate

 * Add reduce-fp32-to-fp16, reduce-fp32-to-bf16, debug-data, memory-import
options to external delegate
 * Simplify DelegateOptions
 * Add test mock models
 * Unit tests
 * Configure lfs to manage tflite files

Signed-off-by: Narumol Prangnawarat <narumol.prangnawarat@arm.com>
Change-Id: I1e4db468862ba03d4cb031347bc307cf940b3cb1

5 files changed, 142 insertions, 4 deletions

diff --git a/delegate/python/test/test_data/conv2d.tflite b/delegate/python/test/test_data/conv2d.tflite
new file mode 100644
index 0000000000..3350d6408a
--- /dev/null
+++ b/delegate/python/test/test_data/conv2d.tflite
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cafc0de9b9f36afe76feff0bc1e5e4dd7ab4201da359b9faca236ba24cbcdd60
+size 728
diff --git a/delegate/python/test/test_data/fallback_model.tflite b/delegate/python/test/test_data/fallback_model.tflite
new file mode 100644
index 0000000000..b86ce6d556
--- /dev/null
+++ b/delegate/python/test/test_data/fallback_model.tflite
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44adf470f17f1d635131c5b59b0c888871b4b793dbeaf9d91946375fd63a5ef3
+size 704
diff --git a/delegate/python/test/test_data/fp32_model.tflite b/delegate/python/test/test_data/fp32_model.tflite
new file mode 100644
index 0000000000..15a0cabc22
--- /dev/null
+++ b/delegate/python/test/test_data/fp32_model.tflite
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:549eeeeb3ee3984cf1636917f8b6c80914b6ce3b7553220c05858fa3d438371b
+size 688
diff --git a/delegate/python/test/test_external_delegate.py b/delegate/python/test/test_external_delegate.py
index 995de8cd70..93d373d0a1 100644
--- a/delegate/python/test/test_external_delegate.py
+++ b/delegate/python/test/test_external_delegate.py
@@ -1,11 +1,11 @@
 # Copyright © 2020 Arm Ltd and Contributors. All rights reserved.
 # SPDX-License-Identifier: MIT
 
+import numpy as np
 import pytest
 import tflite_runtime.interpreter as tflite
 import os
-from utils import run_mock_model
-
+from utils import run_mock_model, run_inference, compare_outputs
 
 def test_external_delegate_unknown_options(delegate_dir):
     print(delegate_dir)
@@ -14,7 +14,6 @@ def test_external_delegate_unknown_options(delegate_dir):
             delegate_dir,
             options={"wrong": "wrong"})
 
-
 def test_external_delegate_options_multiple_backends(delegate_dir):
     tflite.load_delegate(
         delegate_dir,
@@ -63,3 +62,104 @@ def test_external_delegate_options_wrong_logging_level(delegate_dir):
         tflite.load_delegate(
             delegate_dir,
             options={"logging-severity": "wrong"})
+
+def test_external_delegate_options_debug(capfd, delegate_dir, test_data_folder):
+    # create armnn delegate with debug option
+    armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuRef', 'debug-data': '1'})
+
+    model_file_name = 'fp32_model.tflite'
+
+    tensor_shape = [1, 2, 2, 1]
+
+    input0 = np.array([1, 2, 3, 4], dtype=np.float32).reshape(tensor_shape)
+    input1 = np.array([2, 2, 3, 4], dtype=np.float32).reshape(tensor_shape)
+    inputs = [input0, input0, input1]
+    expected_output = np.array([1, 2, 2, 2], dtype=np.float32).reshape(tensor_shape)
+
+    # run the inference
+    armnn_outputs = run_inference(test_data_folder, model_file_name, inputs, [armnn_delegate])
+
+    # check results
+    compare_outputs(armnn_outputs, [expected_output])
+
+    captured = capfd.readouterr()
+    assert 'layerGuid' in captured.out
+
+
+def test_external_delegate_options_fp32_to_fp16(capfd, delegate_dir, test_data_folder):
+    # create armnn delegate with reduce-fp32-to-fp16 option
+    armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuRef',
+                                                                   'debug-data': '1',
+                                                                   'reduce-fp32-to-fp16': '1'})
+
+    model_file_name = 'fp32_model.tflite'
+
+    tensor_shape = [1, 2, 2, 1]
+
+    input0 = np.array([1, 2, 3, 4], dtype=np.float32).reshape(tensor_shape)
+    input1 = np.array([2, 2, 3, 4], dtype=np.float32).reshape(tensor_shape)
+    inputs = [input0, input0, input1]
+    expected_output = np.array([1, 2, 2, 2], dtype=np.float32).reshape(tensor_shape)
+
+    # run the inference
+    armnn_outputs = run_inference(test_data_folder, model_file_name, inputs, [armnn_delegate])
+
+    # check results
+    compare_outputs(armnn_outputs, [expected_output])
+
+    captured = capfd.readouterr()
+    assert 'convert_fp32_to_fp16' in captured.out
+    assert 'convert_fp16_to_fp32' in captured.out
+
+def test_external_delegate_options_fp32_to_bf16(capfd, delegate_dir, test_data_folder):
+    # create armnn delegate with reduce-fp32-to-bf16 option
+    armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuRef',
+                                                                   'debug-data': '1',
+                                                                   'reduce-fp32-to-bf16': '1'})
+
+    model_file_name = 'conv2d.tflite'
+
+    inputShape = [ 1, 5, 5, 1 ]
+    outputShape = [ 1, 3, 3, 1 ]
+
+    inputValues = [ 1, 5, 2, 3, 5,
+                    8, 7, 3, 6, 3,
+                    3, 3, 9, 1, 9,
+                    4, 1, 8, 1, 3,
+                    6, 8, 1, 9, 2 ]
+
+    expectedResult = [ 28, 38, 29,
+                       96, 104, 53,
+                       31, 55, 24 ]
+
+    input = np.array(inputValues, dtype=np.float32).reshape(inputShape)
+    expected_output = np.array(expectedResult, dtype=np.float32).reshape(outputShape)
+
+    # run the inference
+    armnn_outputs = run_inference(test_data_folder, model_file_name, [input], [armnn_delegate])
+
+    # check results
+    compare_outputs(armnn_outputs, [expected_output])
+
+    captured = capfd.readouterr()
+    assert 'convert_fp32_to_bf16' in captured.out
+
+def test_external_delegate_options_memory_import(delegate_dir, test_data_folder):
+    # create armnn delegate with memory-import option
+    armnn_delegate = tflite.load_delegate(delegate_dir, options = {'backends': 'CpuAcc,CpuRef',
+                                                                   'memory-import': '1'})
+
+    model_file_name = 'fallback_model.tflite'
+
+    tensor_shape = [1, 2, 2, 1]
+
+    input0 = np.array([1, 2, 3, 4], dtype=np.uint8).reshape(tensor_shape)
+    input1 = np.array([2, 2, 3, 4], dtype=np.uint8).reshape(tensor_shape)
+    inputs = [input0, input0, input1]
+    expected_output = np.array([1, 2, 2, 2], dtype=np.uint8).reshape(tensor_shape)
+
+    # run the inference
+    armnn_outputs = run_inference(test_data_folder, model_file_name, inputs, [armnn_delegate])
+
+    # check results
+    compare_outputs(armnn_outputs, [expected_output])
\ No newline at end of file
diff --git a/delegate/python/test/utils.py b/delegate/python/test/utils.py
index 3adc24fe35..f3761ec8a1 100644
--- a/delegate/python/test/utils.py
+++ b/delegate/python/test/utils.py
@@ -21,4 +21,33 @@ def run_mock_model(delegate, test_data_folder):
     input_data = np.array(np.random.random_sample(input_shape), dtype=np.uint8)
     interpreter.set_tensor(input_details[0]['index'], input_data)
 
-    interpreter.invoke()
\ No newline at end of file
+    interpreter.invoke()
+
+def run_inference(test_data_folder, model_filename, inputs, delegates=None):
+    model_path = os.path.join(test_data_folder, model_filename)
+    interpreter = tflite.Interpreter(model_path=model_path,
+                                     experimental_delegates=delegates)
+    interpreter.allocate_tensors()
+
+    # Get input and output tensors.
+    input_details = interpreter.get_input_details()
+    output_details = interpreter.get_output_details()
+
+    # Set inputs to tensors.
+    for i in range(len(inputs)):
+        interpreter.set_tensor(input_details[i]['index'], inputs[i])
+
+    interpreter.invoke()
+
+    results = []
+    for output in output_details:
+        results.append(interpreter.get_tensor(output['index']))
+
+    return results
+
+def compare_outputs(outputs, expected_outputs):
+    assert len(outputs) == len(expected_outputs), 'Incorrect number of outputs'
+    for i in range(len(expected_outputs)):
+        assert outputs[i].shape == expected_outputs[i].shape, 'Incorrect output shape on output#{}'.format(i)
+        assert outputs[i].dtype == expected_outputs[i].dtype, 'Incorrect output data type on output#{}'.format(i)
+        assert outputs[i].all() == expected_outputs[i].all(), 'Incorrect output value on output#{}'.format(i)
\ No newline at end of file