From 4189cc5ca4bb12e02c5e7f86ec6079f76d845b59 Mon Sep 17 00:00:00 2001 From: Sadik Armagan Date: Wed, 11 Nov 2020 18:01:48 +0000 Subject: IVGCVSW-5504 'TfLiteDelegate: Introduce FP16 and BackendOptions' * Added BackendOptions creations of armnn_delegate * Included armnn/third-party the armnn_delegate unit tests * Updated the CreateConstTensor function Signed-off-by: Sadik Armagan Change-Id: I8e2099a465766b905bff701413307e5850b68e42 --- delegate/CMakeLists.txt | 22 +++++--- delegate/include/DelegateOptions.hpp | 26 ++++++++- delegate/src/DelegateOptions.cpp | 10 ++-- delegate/src/DelegateUtils.hpp | 94 ++++++--------------------------- delegate/src/FullyConnected.hpp | 6 +-- delegate/src/armnn_delegate.cpp | 7 ++- delegate/src/test/ArmnnDelegateTest.cpp | 13 ++++- 7 files changed, 80 insertions(+), 98 deletions(-) diff --git a/delegate/CMakeLists.txt b/delegate/CMakeLists.txt index 05ec851bf2..2ee00f3887 100644 --- a/delegate/CMakeLists.txt +++ b/delegate/CMakeLists.txt @@ -86,8 +86,10 @@ target_include_directories(armnnDelegate PRIVATE ${Flatbuffers_INCLUDE_DIR}) -set(armnnDelegate_unittest_sources) -list(APPEND armnnDelegate_unittest_sources +option(BUILD_UNIT_TESTS "Build unit tests" ON) +if(BUILD_UNIT_TESTS) + set(armnnDelegate_unittest_sources) + list(APPEND armnnDelegate_unittest_sources src/test/ArmnnDelegateTest.cpp src/test/ComparisonTest.cpp src/test/ComparisonTestHelper.hpp @@ -102,19 +104,23 @@ list(APPEND armnnDelegate_unittest_sources src/test/QuantizationTest.cpp src/test/QuantizationTestHelper.hpp) -add_executable(DelegateUnitTests ${armnnDelegate_unittest_sources}) -target_include_directories(DelegateUnitTests PRIVATE third-party) + add_executable(DelegateUnitTests ${armnnDelegate_unittest_sources}) + target_include_directories(DelegateUnitTests PRIVATE third-party) -target_link_libraries(DelegateUnitTests armnnDelegate) -target_link_libraries(DelegateUnitTests Armnn::armnnUtils) + # Add half library from armnn third-party libraries + target_include_directories(DelegateUnitTests PRIVATE ${ARMNN_SOURCE_DIR}/third-party) -target_include_directories(DelegateUnitTests + target_link_libraries(DelegateUnitTests armnnDelegate) + target_link_libraries(DelegateUnitTests Armnn::armnnUtils) + + target_include_directories(DelegateUnitTests PRIVATE ${TfLite_INCLUDE_DIR}) -target_include_directories(DelegateUnitTests + target_include_directories(DelegateUnitTests PRIVATE ${Flatbuffers_INCLUDE_DIR}) +endif() #################################################### ## Export targets diff --git a/delegate/include/DelegateOptions.hpp b/delegate/include/DelegateOptions.hpp index 0c8173d15f..daf20150be 100644 --- a/delegate/include/DelegateOptions.hpp +++ b/delegate/include/DelegateOptions.hpp @@ -17,19 +17,41 @@ namespace armnnDelegate class DelegateOptions { public: - DelegateOptions(armnn::Compute computeDevice); + DelegateOptions(armnn::Compute computeDevice, const std::vector& backendOptions = {}); - DelegateOptions(const std::vector& backends); + DelegateOptions(const std::vector& backends, + const std::vector& backendOptions = {}); const std::vector& GetBackends() const { return m_Backends; } void SetBackends(const std::vector& backends) { m_Backends = backends; } + const std::vector& GetBackendOptions() const { return m_BackendOptions; } + private: /// Which backend to run Delegate on. /// Examples of possible values are: CpuRef, CpuAcc, GpuAcc. /// CpuRef as default. std::vector m_Backends = { armnn::Compute::CpuRef }; + + /// Pass backend specific options to Delegate + /// + /// For example, tuning can be enabled on GpuAcc like below + /// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + /// m_BackendOptions.emplace_back( + /// BackendOptions{"GpuAcc", + /// { + /// {"TuningLevel", 2}, + /// {"TuningFile", filename} + /// } + /// }); + /// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + /// The following backend options are available: + /// GpuAcc: + /// "TuningLevel" : int [0..3] (0=UseOnly(default) | 1=RapidTuning | 2=NormalTuning | 3=ExhaustiveTuning) + /// "TuningFile" : string [filenameString] + /// "KernelProfilingEnabled" : bool [true | false] + std::vector m_BackendOptions; }; } // namespace armnnDelegate diff --git a/delegate/src/DelegateOptions.cpp b/delegate/src/DelegateOptions.cpp index e931fa4be4..af78685fa6 100644 --- a/delegate/src/DelegateOptions.cpp +++ b/delegate/src/DelegateOptions.cpp @@ -8,13 +8,15 @@ namespace armnnDelegate { -DelegateOptions::DelegateOptions(armnn::Compute computeDevice) - : m_Backends({computeDevice}) +DelegateOptions::DelegateOptions(armnn::Compute computeDevice, + const std::vector& backendOptions) + : m_Backends({computeDevice}), m_BackendOptions(backendOptions) { } -DelegateOptions::DelegateOptions(const std::vector& backends) - : m_Backends(backends) +DelegateOptions::DelegateOptions(const std::vector& backends, + const std::vector& backendOptions) + : m_Backends(backends), m_BackendOptions(backendOptions) { } diff --git a/delegate/src/DelegateUtils.hpp b/delegate/src/DelegateUtils.hpp index fb3f998283..71222276b4 100644 --- a/delegate/src/DelegateUtils.hpp +++ b/delegate/src/DelegateUtils.hpp @@ -397,91 +397,29 @@ armnn::TensorInfo GetTensorInfoForTfLiteTensor(const TfLiteTensor& tfLiteTensor) return ret; } -struct DataHolder +armnn::ConstTensor CreateConstTensor(const TfLiteTensor* tfLiteTensor, + armnn::TensorInfo& tensorInfo, + armnn::Optional permutationVector) { -public: - DataHolder() - : m_Fp32Data(nullptr), m_Uint8Data(nullptr), - m_Int8Data(nullptr), m_Int16Data(nullptr), m_Int32Data(nullptr) {} - - DataHolder(std::unique_ptr&& data) - : m_Fp32Data(std::move(data)), m_Uint8Data(nullptr), - m_Int8Data(nullptr), m_Int16Data(nullptr), m_Int32Data(nullptr) {} - - DataHolder(std::unique_ptr&& data) - : m_Fp32Data(nullptr), m_Uint8Data(std::move(data)), - m_Int8Data(nullptr), m_Int16Data(nullptr), m_Int32Data(nullptr) {} - - DataHolder(std::unique_ptr&& data) - : m_Fp32Data(nullptr), m_Uint8Data(nullptr), - m_Int8Data(std::move(data)), m_Int16Data(nullptr), m_Int32Data(nullptr) {} - - DataHolder(std::unique_ptr&& data) - : m_Fp32Data(nullptr), m_Uint8Data(nullptr), - m_Int8Data(nullptr), m_Int16Data(std::move(data)), m_Int32Data(nullptr) {} - - DataHolder(std::unique_ptr&& data) - : m_Fp32Data(nullptr), m_Uint8Data(nullptr), - m_Int8Data(nullptr), m_Int16Data(nullptr), m_Int32Data(std::move(data)) {} - -private: - std::unique_ptr m_Fp32Data; - std::unique_ptr m_Uint8Data; - std::unique_ptr m_Int8Data; - std::unique_ptr m_Int16Data; - std::unique_ptr m_Int32Data; -}; - -template -std::pair CreateConstTensorImpl( - const TfLiteTensor* tensor, - armnn::TensorInfo& tensorInfo, - armnn::Optional permutationVector) -{ - std::unique_ptr data(new T[tensorInfo.GetNumElements()]); + if (tfLiteTensor->allocation_type != kTfLiteMmapRo) + { + throw armnn::Exception("TfLiteArmnnDelegate: Not constant allocation type: " + tfLiteTensor->allocation_type); + } + if (permutationVector.has_value() && permutationVector.value().GetSize() > 0) { - tensorInfo = armnnUtils::Permuted(tensorInfo, permutationVector.value()); - armnnUtils::Permute(tensorInfo.GetShape(), + std::vector swizzledData; + swizzledData.resize(tensorInfo.GetNumBytes()); + armnnUtils::Permute(armnnUtils::Permuted(tensorInfo.GetShape(), permutationVector.value()), permutationVector.value(), - reinterpret_cast(tensor->data.raw), data.get(), sizeof(T)); + tfLiteTensor->data.data, + swizzledData.data(), + armnn::GetDataTypeSize(tensorInfo.GetDataType())); + return armnn::ConstTensor(armnnUtils::Permuted(tensorInfo, permutationVector.value()), swizzledData.data()); } else { - ::memcpy(data.get(), tensor->data.raw, tensorInfo.GetNumBytes()); - } - - auto constData = std::make_pair(armnn::ConstTensor(tensorInfo, data.get()), std::move(data)); - - DataHolder storedData(std::move(constData.second)); - return std::make_pair(constData.first, std::move(storedData)); -} - -std::pair CreateConstTensor( - const TfLiteTensor* tfLiteTensor, - armnn::TensorInfo& tensorInfo, - armnn::Optional permutationVector) -{ - switch (tensorInfo.GetDataType()) - { - case armnn::DataType::Float32: - return CreateConstTensorImpl(tfLiteTensor, tensorInfo, permutationVector); - case armnn::DataType::QAsymmU8: - return CreateConstTensorImpl(tfLiteTensor, tensorInfo, permutationVector); - case armnn::DataType::QSymmS8: - return CreateConstTensorImpl(tfLiteTensor, tensorInfo, permutationVector); - case armnn::DataType::QAsymmS8: - return CreateConstTensorImpl(tfLiteTensor, tensorInfo, permutationVector); - case armnn::DataType::QSymmS16: - return CreateConstTensorImpl(tfLiteTensor, tensorInfo, permutationVector); - case armnn::DataType::Signed32: - return CreateConstTensorImpl(tfLiteTensor, tensorInfo, permutationVector); - default: - { - throw armnn::Exception( - "TfLiteArmnnDelegate: Unsupported data type when creating const tensor: " - + std::string(armnn::GetDataTypeName(tensorInfo.GetDataType()))); - } + return armnn::ConstTensor(tensorInfo, tfLiteTensor->data.data); } } diff --git a/delegate/src/FullyConnected.hpp b/delegate/src/FullyConnected.hpp index f35f4c92b0..48bf06f94a 100644 --- a/delegate/src/FullyConnected.hpp +++ b/delegate/src/FullyConnected.hpp @@ -165,13 +165,13 @@ TfLiteStatus VisitFullyConnectedOperator(DelegateData& delegateData, biasTensorInfo, armnn::Optional()); layer = delegateData.m_Network->AddFullyConnectedLayer(descriptor, - weightsTensor.first, - armnn::Optional(biasTensor.first)); + weightsTensor, + armnn::Optional(biasTensor)); } else { layer = delegateData.m_Network->AddFullyConnectedLayer(descriptor, - weightsTensor.first, + weightsTensor, armnn::EmptyOptional()); } ARMNN_ASSERT(layer != nullptr); diff --git a/delegate/src/armnn_delegate.cpp b/delegate/src/armnn_delegate.cpp index 69bd4f7350..3380c84d0b 100644 --- a/delegate/src/armnn_delegate.cpp +++ b/delegate/src/armnn_delegate.cpp @@ -118,10 +118,15 @@ Delegate::Delegate(armnnDelegate::DelegateOptions options) { // Create ArmNN Runtime armnn::IRuntime::CreationOptions runtimeOptions; + + auto backendOptions = m_Options.GetBackendOptions(); + if (!backendOptions.empty()) + { + runtimeOptions.m_BackendOptions = backendOptions; + } m_Runtime = armnn::IRuntime::Create(runtimeOptions); std::vector backends; - if (m_Runtime) { const armnn::BackendIdSet supportedDevices = m_Runtime->GetDeviceSpec().GetSupportedBackends(); diff --git a/delegate/src/test/ArmnnDelegateTest.cpp b/delegate/src/test/ArmnnDelegateTest.cpp index 7cec70b022..050fe45ca0 100644 --- a/delegate/src/test/ArmnnDelegateTest.cpp +++ b/delegate/src/test/ArmnnDelegateTest.cpp @@ -35,9 +35,18 @@ TEST_CASE ("ArmnnDelegate Registered") const TfLiteRegistration* opRegister = opResolver.FindOp(BuiltinOperator_ADD, 1); tfLiteInterpreter->AddNodeWithParameters({0, 1}, {2}, "", 0, nullptr, opRegister); - // create the Armnn Delegate + // Create the Armnn Delegate std::vector backends = { armnn::Compute::CpuRef }; - armnnDelegate::DelegateOptions delegateOptions(backends); + std::vector backendOptions; + backendOptions.emplace_back( + armnn::BackendOptions{ "BackendName", + { + { "Option1", 42 }, + { "Option2", true } + }} + ); + + armnnDelegate::DelegateOptions delegateOptions(backends, backendOptions); std::unique_ptr theArmnnDelegate(armnnDelegate::TfLiteArmnnDelegateCreate(delegateOptions), armnnDelegate::TfLiteArmnnDelegateDelete); -- cgit v1.2.1