diff options
author | Sadik Armagan <sadik.armagan@arm.com> | 2022-08-16 12:17:24 +0100 |
---|---|---|
committer | Nikhil Raj <nikhil.raj@arm.com> | 2022-08-29 10:12:44 +0100 |
commit | ca565c1b04b767abfc13a59146680663a3ea4008 (patch) | |
tree | 58b260768fe8cb690f740ece44fca9c415620d7f | |
parent | 514d16b434102a4a7807548745af301baab13b6b (diff) | |
download | armnn-ca565c1b04b767abfc13a59146680663a3ea4008.tar.gz |
IVGCVSW-6603 'Add a no fallback mode to the TfLite Delegate'
* Added disable-tflite-runtime-fallback option to armnn_delegate
* Updated armnn_delegate version
Signed-off-by: Sadik Armagan <sadik.armagan@arm.com>
Change-Id: I449b16404d3ffe98e6dac52a43e7c25225addd73
-rw-r--r-- | delegate/include/DelegateOptions.hpp | 18 | ||||
-rw-r--r-- | delegate/include/Version.hpp | 4 | ||||
-rw-r--r-- | delegate/src/DelegateOptions.cpp | 6 | ||||
-rw-r--r-- | delegate/src/armnn_delegate.cpp | 9 | ||||
-rw-r--r-- | delegate/src/test/DelegateOptionsTest.cpp | 74 | ||||
-rw-r--r-- | delegate/src/test/DelegateOptionsTestHelper.hpp | 130 |
6 files changed, 238 insertions, 3 deletions
diff --git a/delegate/include/DelegateOptions.hpp b/delegate/include/DelegateOptions.hpp index d789ea7285..2b0107e834 100644 --- a/delegate/include/DelegateOptions.hpp +++ b/delegate/include/DelegateOptions.hpp @@ -174,6 +174,12 @@ public: * This is an Experimental parameter that is incompatible with "infer-output-shape". \n * This parameter may be removed in a later update. * + * Option key: "disable-tflite-runtime-fallback" \n + * Possible values: ["true"/"false"] \n + * Description: Disable TfLite Runtime fallback in the Arm NN TfLite delegate. + * An exception will be thrown if unsupported operators are encountered. + * This option is only for testing purposes. + * * @param[in] option_keys Delegate option names * @param[in] options_values Delegate option values * @param[in] num_options Number of delegate options @@ -262,6 +268,15 @@ public: return m_RuntimeOptions; } + void DisableTfLiteRuntimeFallback(bool fallbackState) + { + m_DisableTfLiteRuntimeFallback = fallbackState; + } + bool TfLiteRuntimeFallbackDisabled() + { + return m_DisableTfLiteRuntimeFallback; + } + private: /// Which backend to run Delegate on. /// Examples of possible values are: CpuRef, CpuAcc, GpuAcc. @@ -295,6 +310,9 @@ private: /// If not empty then the optimized model will be serialized to a file with this file name in "dot" format. std::string m_SerializeToDot = ""; + + /// Option to disable TfLite Runtime fallback for unsupported operators. + bool m_DisableTfLiteRuntimeFallback = false; }; } // namespace armnnDelegate diff --git a/delegate/include/Version.hpp b/delegate/include/Version.hpp index c14857e320..36d8fb54af 100644 --- a/delegate/include/Version.hpp +++ b/delegate/include/Version.hpp @@ -13,8 +13,8 @@ namespace armnnDelegate #define STRINGIFY_MACRO(s) #s // ArmNN Delegate version components -#define DELEGATE_MAJOR_VERSION 26 -#define DELEGATE_MINOR_VERSION 1 +#define DELEGATE_MAJOR_VERSION 27 +#define DELEGATE_MINOR_VERSION 0 #define DELEGATE_PATCH_VERSION 0 /// DELEGATE_VERSION: "X.Y.Z" diff --git a/delegate/src/DelegateOptions.cpp b/delegate/src/DelegateOptions.cpp index f3e13c90c6..a55a579333 100644 --- a/delegate/src/DelegateOptions.cpp +++ b/delegate/src/DelegateOptions.cpp @@ -242,6 +242,12 @@ DelegateOptions::DelegateOptions(char const* const* options_keys, { this->SetSerializeToDot(options_values[i]); } + + // Process disable-tflite-runtime-fallback + else if (std::string(options_keys[i]) == std::string("disable-tflite-runtime-fallback")) + { + this->DisableTfLiteRuntimeFallback(armnn::stringUtils::StringToBool(options_values[i])); + } else { throw armnn::Exception("Unknown option for the ArmNN Delegate given: " + std::string(options_keys[i])); diff --git a/delegate/src/armnn_delegate.cpp b/delegate/src/armnn_delegate.cpp index eac3862c18..c041dd1714 100644 --- a/delegate/src/armnn_delegate.cpp +++ b/delegate/src/armnn_delegate.cpp @@ -222,6 +222,15 @@ TfLiteIntArray* Delegate::IdentifyOperatorsToDelegate(TfLiteContext* tfLiteConte *it); } + if (!unsupportedOperators.empty() && m_Options.TfLiteRuntimeFallbackDisabled()) + { + std::stringstream exMessage; + exMessage << "TfLiteArmnnDelegate: There are unsupported operators in the model. "; + exMessage << "Not falling back to TfLite Runtime as fallback is disabled. "; + exMessage << "This should only be disabled under test conditions."; + throw armnn::Exception(exMessage.str()); + } + std::sort(&nodesToDelegate->data[0], &nodesToDelegate->data[nodesToDelegate->size]); return nodesToDelegate; } diff --git a/delegate/src/test/DelegateOptionsTest.cpp b/delegate/src/test/DelegateOptionsTest.cpp index c9f1530968..50d3f78563 100644 --- a/delegate/src/test/DelegateOptionsTest.cpp +++ b/delegate/src/test/DelegateOptionsTest.cpp @@ -150,6 +150,79 @@ TEST_CASE ("ArmnnDelegateOptimizerOptionsImport") delegateOptions); } +TEST_CASE ("ArmnnDelegateStringParsingOptionDisableTfLiteRuntimeFallback") +{ + std::stringstream stringStream; + std::vector<std::string> keys { "backends", "debug-data", "disable-tflite-runtime-fallback"}; + std::vector<std::string> values { "CpuRef", "1", "1"}; + + std::vector<armnn::BackendId> backends = { armnn::Compute::CpuRef }; + std::vector<int32_t> tensorShape { 1, 2, 2, 1 }; + std::vector<float> inputData = { 0.1f, -2.1f, 3.0f, -4.6f }; + std::vector<float> expectedResult = { 1.0f, -2.0f, 3.0f, -4.0f }; + + // Create options_keys and options_values char array + size_t num_options = keys.size(); + std::unique_ptr<const char*> options_keys = + std::unique_ptr<const char*>(new const char*[num_options + 1]); + std::unique_ptr<const char*> options_values = + std::unique_ptr<const char*>(new const char*[num_options + 1]); + for (size_t i=0; i<num_options; ++i) + { + options_keys.get()[i] = keys[i].c_str(); + options_values.get()[i] = values[i].c_str(); + } + + StreamRedirector redirect(std::cout, stringStream.rdbuf()); + + armnnDelegate::DelegateOptions delegateOptions(options_keys.get(), options_values.get(), num_options, nullptr); + DelegateOptionNoFallbackTest<float>(::tflite::TensorType_FLOAT32, + backends, + tensorShape, + inputData, + expectedResult, + delegateOptions); + CHECK(stringStream.str().find("TfLiteArmnnDelegate: There are unsupported operators in the model") + != std::string::npos); +} + +TEST_CASE ("ArmnnDelegateStringParsingOptionEnableTfLiteRuntimeFallback") +{ + std::stringstream stringStream; + std::vector<std::string> keys { "backends", "debug-data", "disable-tflite-runtime-fallback"}; + std::vector<std::string> values { "CpuRef", "1", "0"}; + + std::vector<armnn::BackendId> backends = { armnn::Compute::CpuRef }; + std::vector<int32_t> tensorShape { 1, 2, 2, 1 }; + std::vector<float> inputData = { 0.1f, -2.1f, 3.0f, -4.6f }; + std::vector<float> expectedResult = { 1.0f, -2.0f, 3.0f, -4.0f }; + + // Create options_keys and options_values char array + size_t num_options = keys.size(); + std::unique_ptr<const char*> options_keys = + std::unique_ptr<const char*>(new const char*[num_options + 1]); + std::unique_ptr<const char*> options_values = + std::unique_ptr<const char*>(new const char*[num_options + 1]); + for (size_t i=0; i<num_options; ++i) + { + options_keys.get()[i] = keys[i].c_str(); + options_values.get()[i] = values[i].c_str(); + } + + StreamRedirector redirect(std::cout, stringStream.rdbuf()); + + armnnDelegate::DelegateOptions delegateOptions(options_keys.get(), options_values.get(), num_options, nullptr); + DelegateOptionNoFallbackTest<float>(::tflite::TensorType_FLOAT32, + backends, + tensorShape, + inputData, + expectedResult, + delegateOptions); + + CHECK(stringStream.str().find("TfLiteArmnnDelegate: There are unsupported operators in the model") + == std::string::npos); +} + } TEST_SUITE("DelegateOptions_CpuAccTests") @@ -307,7 +380,6 @@ TEST_CASE ("ArmnnDelegateStringParsingOptionReduceFp32ToFp16") } } - } } // namespace armnnDelegate diff --git a/delegate/src/test/DelegateOptionsTestHelper.hpp b/delegate/src/test/DelegateOptionsTestHelper.hpp index 6e0cc3154c..87bf0d6c3d 100644 --- a/delegate/src/test/DelegateOptionsTestHelper.hpp +++ b/delegate/src/test/DelegateOptionsTestHelper.hpp @@ -148,6 +148,77 @@ std::vector<char> CreateAddDivTfLiteModel(tflite::TensorType tensorType, flatBufferBuilder.GetBufferPointer() + flatBufferBuilder.GetSize()); } +std::vector<char> CreateCeilTfLiteModel(tflite::TensorType tensorType, + const std::vector <int32_t>& tensorShape, + float quantScale = 1.0f, + int quantOffset = 0) +{ + using namespace tflite; + flatbuffers::FlatBufferBuilder flatBufferBuilder; + + std::vector<flatbuffers::Offset<tflite::Buffer>> buffers; + buffers.push_back(CreateBuffer(flatBufferBuilder, flatBufferBuilder.CreateVector({}))); + + auto quantizationParameters = + CreateQuantizationParameters(flatBufferBuilder, + 0, + 0, + flatBufferBuilder.CreateVector<float>({quantScale}), + flatBufferBuilder.CreateVector<int64_t>({quantOffset})); + + std::array<flatbuffers::Offset<Tensor>, 2> tensors; + tensors[0] = CreateTensor(flatBufferBuilder, + flatBufferBuilder.CreateVector<int32_t>(tensorShape.data(), + tensorShape.size()), + tensorType, + 0, + flatBufferBuilder.CreateString("input"), + quantizationParameters); + tensors[1] = CreateTensor(flatBufferBuilder, + flatBufferBuilder.CreateVector<int32_t>(tensorShape.data(), + tensorShape.size()), + tensorType, + 0, + flatBufferBuilder.CreateString("output"), + quantizationParameters); + + const std::vector<int32_t> operatorInputs({0}); + const std::vector<int32_t> operatorOutputs({1}); + + flatbuffers::Offset<Operator> ceilOperator = + CreateOperator(flatBufferBuilder, + 0, + flatBufferBuilder.CreateVector<int32_t>(operatorInputs.data(), operatorInputs.size()), + flatBufferBuilder.CreateVector<int32_t>(operatorOutputs.data(), operatorOutputs.size()), + BuiltinOptions_NONE); + + flatbuffers::Offset<flatbuffers::String> modelDescription = + flatBufferBuilder.CreateString("ArmnnDelegate: CEIL Operator Model"); + flatbuffers::Offset<OperatorCode> operatorCode = + CreateOperatorCode(flatBufferBuilder, tflite::BuiltinOperator_CEIL); + + const std::vector<int32_t> subgraphInputs({0}); + const std::vector<int32_t> subgraphOutputs({1}); + flatbuffers::Offset<SubGraph> subgraph = + CreateSubGraph(flatBufferBuilder, + flatBufferBuilder.CreateVector(tensors.data(), tensors.size()), + flatBufferBuilder.CreateVector<int32_t>(subgraphInputs.data(), subgraphInputs.size()), + flatBufferBuilder.CreateVector<int32_t>(subgraphOutputs.data(), subgraphOutputs.size()), + flatBufferBuilder.CreateVector(&ceilOperator, 1)); + + flatbuffers::Offset<Model> flatbufferModel = + CreateModel(flatBufferBuilder, + TFLITE_SCHEMA_VERSION, + flatBufferBuilder.CreateVector(&operatorCode, 1), + flatBufferBuilder.CreateVector(&subgraph, 1), + modelDescription, + flatBufferBuilder.CreateVector(buffers.data(), buffers.size())); + + flatBufferBuilder.Finish(flatbufferModel); + return std::vector<char>(flatBufferBuilder.GetBufferPointer(), + flatBufferBuilder.GetBufferPointer() + flatBufferBuilder.GetSize()); +} + void ReduceFp32ToBf16TestImpl() { using namespace tflite; @@ -295,4 +366,63 @@ void DelegateOptionTest(tflite::TensorType tensorType, armnnDelegateInterpreter.reset(nullptr); } +template <typename T> +void DelegateOptionNoFallbackTest(tflite::TensorType tensorType, + const std::vector<armnn::BackendId>& backends, + std::vector<int32_t>& tensorShape, + std::vector<T>& inputValues, + std::vector<T>& expectedOutputValues, + const armnnDelegate::DelegateOptions& delegateOptions, + float quantScale = 1.0f, + int quantOffset = 0) +{ + using namespace tflite; + std::vector<char> modelBuffer = CreateCeilTfLiteModel(tensorType, + tensorShape, + quantScale, + quantOffset); + + const Model* tfLiteModel = GetModel(modelBuffer.data()); + // Create TfLite Interpreters + std::unique_ptr<Interpreter> armnnDelegateInterpreter; + CHECK(InterpreterBuilder(tfLiteModel, ::tflite::ops::builtin::BuiltinOpResolver()) + (&armnnDelegateInterpreter) == kTfLiteOk); + CHECK(armnnDelegateInterpreter != nullptr); + CHECK(armnnDelegateInterpreter->AllocateTensors() == kTfLiteOk); + + std::unique_ptr<Interpreter> tfLiteInterpreter; + CHECK(InterpreterBuilder(tfLiteModel, ::tflite::ops::builtin::BuiltinOpResolver()) + (&tfLiteInterpreter) == kTfLiteOk); + CHECK(tfLiteInterpreter != nullptr); + CHECK(tfLiteInterpreter->AllocateTensors() == kTfLiteOk); + + // Create the ArmNN Delegate + std::unique_ptr<TfLiteDelegate, decltype(&armnnDelegate::TfLiteArmnnDelegateDelete)> + theArmnnDelegate(armnnDelegate::TfLiteArmnnDelegateCreate(delegateOptions), + armnnDelegate::TfLiteArmnnDelegateDelete); + CHECK(theArmnnDelegate != nullptr); + // Modify armnnDelegateInterpreter to use armnnDelegate + try + { + armnnDelegateInterpreter->ModifyGraphWithDelegate(theArmnnDelegate.get()); + } + catch (const armnn::Exception& e) + { + // Forward the exception message to std::cout + std::cout << e.what() << std::endl; + } + + // Set input data + armnnDelegate::FillInput(tfLiteInterpreter, 0, inputValues); + armnnDelegate::FillInput(armnnDelegateInterpreter, 0, inputValues); + + // Run EnqueueWorkload + CHECK(tfLiteInterpreter->Invoke() == kTfLiteOk); + CHECK(armnnDelegateInterpreter->Invoke() == kTfLiteOk); + + armnnDelegate::CompareOutputData<T>(tfLiteInterpreter, armnnDelegateInterpreter, tensorShape, expectedOutputValues); + + armnnDelegateInterpreter.reset(nullptr); +} + } // anonymous namespace
\ No newline at end of file |