From f3e90a423a8ccf3e067c5c6d4e67db09149bae82 Mon Sep 17 00:00:00 2001 From: Sadik Armagan Date: Tue, 16 Aug 2022 12:17:24 +0100 Subject: IVGCVSW-6603 'Add a no fallback mode to the TfLite Delegate' * Added disable-tflite-runtime-fallback option to armnn_delegate * Updated armnn_delegate version Signed-off-by: Sadik Armagan Change-Id: I449b16404d3ffe98e6dac52a43e7c25225addd73 --- delegate/include/DelegateOptions.hpp | 18 ++++ delegate/include/Version.hpp | 4 +- delegate/src/DelegateOptions.cpp | 6 ++ delegate/src/armnn_delegate.cpp | 9 ++ delegate/src/test/DelegateOptionsTest.cpp | 74 +++++++++++++- delegate/src/test/DelegateOptionsTestHelper.hpp | 130 ++++++++++++++++++++++++ 6 files changed, 238 insertions(+), 3 deletions(-) diff --git a/delegate/include/DelegateOptions.hpp b/delegate/include/DelegateOptions.hpp index d789ea7285..2b0107e834 100644 --- a/delegate/include/DelegateOptions.hpp +++ b/delegate/include/DelegateOptions.hpp @@ -174,6 +174,12 @@ public: * This is an Experimental parameter that is incompatible with "infer-output-shape". \n * This parameter may be removed in a later update. * + * Option key: "disable-tflite-runtime-fallback" \n + * Possible values: ["true"/"false"] \n + * Description: Disable TfLite Runtime fallback in the Arm NN TfLite delegate. + * An exception will be thrown if unsupported operators are encountered. + * This option is only for testing purposes. + * * @param[in] option_keys Delegate option names * @param[in] options_values Delegate option values * @param[in] num_options Number of delegate options @@ -262,6 +268,15 @@ public: return m_RuntimeOptions; } + void DisableTfLiteRuntimeFallback(bool fallbackState) + { + m_DisableTfLiteRuntimeFallback = fallbackState; + } + bool TfLiteRuntimeFallbackDisabled() + { + return m_DisableTfLiteRuntimeFallback; + } + private: /// Which backend to run Delegate on. /// Examples of possible values are: CpuRef, CpuAcc, GpuAcc. @@ -295,6 +310,9 @@ private: /// If not empty then the optimized model will be serialized to a file with this file name in "dot" format. std::string m_SerializeToDot = ""; + + /// Option to disable TfLite Runtime fallback for unsupported operators. + bool m_DisableTfLiteRuntimeFallback = false; }; } // namespace armnnDelegate diff --git a/delegate/include/Version.hpp b/delegate/include/Version.hpp index c14857e320..36d8fb54af 100644 --- a/delegate/include/Version.hpp +++ b/delegate/include/Version.hpp @@ -13,8 +13,8 @@ namespace armnnDelegate #define STRINGIFY_MACRO(s) #s // ArmNN Delegate version components -#define DELEGATE_MAJOR_VERSION 26 -#define DELEGATE_MINOR_VERSION 1 +#define DELEGATE_MAJOR_VERSION 27 +#define DELEGATE_MINOR_VERSION 0 #define DELEGATE_PATCH_VERSION 0 /// DELEGATE_VERSION: "X.Y.Z" diff --git a/delegate/src/DelegateOptions.cpp b/delegate/src/DelegateOptions.cpp index f3e13c90c6..a55a579333 100644 --- a/delegate/src/DelegateOptions.cpp +++ b/delegate/src/DelegateOptions.cpp @@ -242,6 +242,12 @@ DelegateOptions::DelegateOptions(char const* const* options_keys, { this->SetSerializeToDot(options_values[i]); } + + // Process disable-tflite-runtime-fallback + else if (std::string(options_keys[i]) == std::string("disable-tflite-runtime-fallback")) + { + this->DisableTfLiteRuntimeFallback(armnn::stringUtils::StringToBool(options_values[i])); + } else { throw armnn::Exception("Unknown option for the ArmNN Delegate given: " + std::string(options_keys[i])); diff --git a/delegate/src/armnn_delegate.cpp b/delegate/src/armnn_delegate.cpp index eac3862c18..c041dd1714 100644 --- a/delegate/src/armnn_delegate.cpp +++ b/delegate/src/armnn_delegate.cpp @@ -222,6 +222,15 @@ TfLiteIntArray* Delegate::IdentifyOperatorsToDelegate(TfLiteContext* tfLiteConte *it); } + if (!unsupportedOperators.empty() && m_Options.TfLiteRuntimeFallbackDisabled()) + { + std::stringstream exMessage; + exMessage << "TfLiteArmnnDelegate: There are unsupported operators in the model. "; + exMessage << "Not falling back to TfLite Runtime as fallback is disabled. "; + exMessage << "This should only be disabled under test conditions."; + throw armnn::Exception(exMessage.str()); + } + std::sort(&nodesToDelegate->data[0], &nodesToDelegate->data[nodesToDelegate->size]); return nodesToDelegate; } diff --git a/delegate/src/test/DelegateOptionsTest.cpp b/delegate/src/test/DelegateOptionsTest.cpp index c9f1530968..50d3f78563 100644 --- a/delegate/src/test/DelegateOptionsTest.cpp +++ b/delegate/src/test/DelegateOptionsTest.cpp @@ -150,6 +150,79 @@ TEST_CASE ("ArmnnDelegateOptimizerOptionsImport") delegateOptions); } +TEST_CASE ("ArmnnDelegateStringParsingOptionDisableTfLiteRuntimeFallback") +{ + std::stringstream stringStream; + std::vector keys { "backends", "debug-data", "disable-tflite-runtime-fallback"}; + std::vector values { "CpuRef", "1", "1"}; + + std::vector backends = { armnn::Compute::CpuRef }; + std::vector tensorShape { 1, 2, 2, 1 }; + std::vector inputData = { 0.1f, -2.1f, 3.0f, -4.6f }; + std::vector expectedResult = { 1.0f, -2.0f, 3.0f, -4.0f }; + + // Create options_keys and options_values char array + size_t num_options = keys.size(); + std::unique_ptr options_keys = + std::unique_ptr(new const char*[num_options + 1]); + std::unique_ptr options_values = + std::unique_ptr(new const char*[num_options + 1]); + for (size_t i=0; i(::tflite::TensorType_FLOAT32, + backends, + tensorShape, + inputData, + expectedResult, + delegateOptions); + CHECK(stringStream.str().find("TfLiteArmnnDelegate: There are unsupported operators in the model") + != std::string::npos); +} + +TEST_CASE ("ArmnnDelegateStringParsingOptionEnableTfLiteRuntimeFallback") +{ + std::stringstream stringStream; + std::vector keys { "backends", "debug-data", "disable-tflite-runtime-fallback"}; + std::vector values { "CpuRef", "1", "0"}; + + std::vector backends = { armnn::Compute::CpuRef }; + std::vector tensorShape { 1, 2, 2, 1 }; + std::vector inputData = { 0.1f, -2.1f, 3.0f, -4.6f }; + std::vector expectedResult = { 1.0f, -2.0f, 3.0f, -4.0f }; + + // Create options_keys and options_values char array + size_t num_options = keys.size(); + std::unique_ptr options_keys = + std::unique_ptr(new const char*[num_options + 1]); + std::unique_ptr options_values = + std::unique_ptr(new const char*[num_options + 1]); + for (size_t i=0; i(::tflite::TensorType_FLOAT32, + backends, + tensorShape, + inputData, + expectedResult, + delegateOptions); + + CHECK(stringStream.str().find("TfLiteArmnnDelegate: There are unsupported operators in the model") + == std::string::npos); +} + } TEST_SUITE("DelegateOptions_CpuAccTests") @@ -307,7 +380,6 @@ TEST_CASE ("ArmnnDelegateStringParsingOptionReduceFp32ToFp16") } } - } } // namespace armnnDelegate diff --git a/delegate/src/test/DelegateOptionsTestHelper.hpp b/delegate/src/test/DelegateOptionsTestHelper.hpp index 6e0cc3154c..87bf0d6c3d 100644 --- a/delegate/src/test/DelegateOptionsTestHelper.hpp +++ b/delegate/src/test/DelegateOptionsTestHelper.hpp @@ -148,6 +148,77 @@ std::vector CreateAddDivTfLiteModel(tflite::TensorType tensorType, flatBufferBuilder.GetBufferPointer() + flatBufferBuilder.GetSize()); } +std::vector CreateCeilTfLiteModel(tflite::TensorType tensorType, + const std::vector & tensorShape, + float quantScale = 1.0f, + int quantOffset = 0) +{ + using namespace tflite; + flatbuffers::FlatBufferBuilder flatBufferBuilder; + + std::vector> buffers; + buffers.push_back(CreateBuffer(flatBufferBuilder, flatBufferBuilder.CreateVector({}))); + + auto quantizationParameters = + CreateQuantizationParameters(flatBufferBuilder, + 0, + 0, + flatBufferBuilder.CreateVector({quantScale}), + flatBufferBuilder.CreateVector({quantOffset})); + + std::array, 2> tensors; + tensors[0] = CreateTensor(flatBufferBuilder, + flatBufferBuilder.CreateVector(tensorShape.data(), + tensorShape.size()), + tensorType, + 0, + flatBufferBuilder.CreateString("input"), + quantizationParameters); + tensors[1] = CreateTensor(flatBufferBuilder, + flatBufferBuilder.CreateVector(tensorShape.data(), + tensorShape.size()), + tensorType, + 0, + flatBufferBuilder.CreateString("output"), + quantizationParameters); + + const std::vector operatorInputs({0}); + const std::vector operatorOutputs({1}); + + flatbuffers::Offset ceilOperator = + CreateOperator(flatBufferBuilder, + 0, + flatBufferBuilder.CreateVector(operatorInputs.data(), operatorInputs.size()), + flatBufferBuilder.CreateVector(operatorOutputs.data(), operatorOutputs.size()), + BuiltinOptions_NONE); + + flatbuffers::Offset modelDescription = + flatBufferBuilder.CreateString("ArmnnDelegate: CEIL Operator Model"); + flatbuffers::Offset operatorCode = + CreateOperatorCode(flatBufferBuilder, tflite::BuiltinOperator_CEIL); + + const std::vector subgraphInputs({0}); + const std::vector subgraphOutputs({1}); + flatbuffers::Offset subgraph = + CreateSubGraph(flatBufferBuilder, + flatBufferBuilder.CreateVector(tensors.data(), tensors.size()), + flatBufferBuilder.CreateVector(subgraphInputs.data(), subgraphInputs.size()), + flatBufferBuilder.CreateVector(subgraphOutputs.data(), subgraphOutputs.size()), + flatBufferBuilder.CreateVector(&ceilOperator, 1)); + + flatbuffers::Offset flatbufferModel = + CreateModel(flatBufferBuilder, + TFLITE_SCHEMA_VERSION, + flatBufferBuilder.CreateVector(&operatorCode, 1), + flatBufferBuilder.CreateVector(&subgraph, 1), + modelDescription, + flatBufferBuilder.CreateVector(buffers.data(), buffers.size())); + + flatBufferBuilder.Finish(flatbufferModel); + return std::vector(flatBufferBuilder.GetBufferPointer(), + flatBufferBuilder.GetBufferPointer() + flatBufferBuilder.GetSize()); +} + void ReduceFp32ToBf16TestImpl() { using namespace tflite; @@ -295,4 +366,63 @@ void DelegateOptionTest(tflite::TensorType tensorType, armnnDelegateInterpreter.reset(nullptr); } +template +void DelegateOptionNoFallbackTest(tflite::TensorType tensorType, + const std::vector& backends, + std::vector& tensorShape, + std::vector& inputValues, + std::vector& expectedOutputValues, + const armnnDelegate::DelegateOptions& delegateOptions, + float quantScale = 1.0f, + int quantOffset = 0) +{ + using namespace tflite; + std::vector modelBuffer = CreateCeilTfLiteModel(tensorType, + tensorShape, + quantScale, + quantOffset); + + const Model* tfLiteModel = GetModel(modelBuffer.data()); + // Create TfLite Interpreters + std::unique_ptr armnnDelegateInterpreter; + CHECK(InterpreterBuilder(tfLiteModel, ::tflite::ops::builtin::BuiltinOpResolver()) + (&armnnDelegateInterpreter) == kTfLiteOk); + CHECK(armnnDelegateInterpreter != nullptr); + CHECK(armnnDelegateInterpreter->AllocateTensors() == kTfLiteOk); + + std::unique_ptr tfLiteInterpreter; + CHECK(InterpreterBuilder(tfLiteModel, ::tflite::ops::builtin::BuiltinOpResolver()) + (&tfLiteInterpreter) == kTfLiteOk); + CHECK(tfLiteInterpreter != nullptr); + CHECK(tfLiteInterpreter->AllocateTensors() == kTfLiteOk); + + // Create the ArmNN Delegate + std::unique_ptr + theArmnnDelegate(armnnDelegate::TfLiteArmnnDelegateCreate(delegateOptions), + armnnDelegate::TfLiteArmnnDelegateDelete); + CHECK(theArmnnDelegate != nullptr); + // Modify armnnDelegateInterpreter to use armnnDelegate + try + { + armnnDelegateInterpreter->ModifyGraphWithDelegate(theArmnnDelegate.get()); + } + catch (const armnn::Exception& e) + { + // Forward the exception message to std::cout + std::cout << e.what() << std::endl; + } + + // Set input data + armnnDelegate::FillInput(tfLiteInterpreter, 0, inputValues); + armnnDelegate::FillInput(armnnDelegateInterpreter, 0, inputValues); + + // Run EnqueueWorkload + CHECK(tfLiteInterpreter->Invoke() == kTfLiteOk); + CHECK(armnnDelegateInterpreter->Invoke() == kTfLiteOk); + + armnnDelegate::CompareOutputData(tfLiteInterpreter, armnnDelegateInterpreter, tensorShape, expectedOutputValues); + + armnnDelegateInterpreter.reset(nullptr); +} + } // anonymous namespace \ No newline at end of file -- cgit v1.2.1