From 5d03e31aaf4d82e9f9cdc03c41d2328bbb2a0dee Mon Sep 17 00:00:00 2001 From: Sadik Armagan Date: Tue, 17 Nov 2020 16:43:56 +0000 Subject: IVGCVSW-5377 Add ArmNN TfLite delegate to ExecuteNetwork * Added package manger to turn internal calls to find_package into a no-op * Changed delegate cmake so it can now be built within armnn Change-Id: I2a7ecb9a3c1ca05474cd1dccd91498f6f6c0b32e Signed-off-by: Finn Williams Signed-off-by: Sadik Armagan --- CMakeLists.txt | 22 +++ cmake/GlobalConfig.cmake | 4 + delegate/CMakeLists.txt | 2 + .../cmake/Modules/ArmnnDelegateConfig.cmake.in | 2 + delegate/include/armnn_delegate.hpp | 4 +- delegate/src/Transpose.hpp | 2 +- tests/CMakeLists.txt | 11 +- tests/ExecuteNetwork/ExecuteNetwork.cpp | 195 +++++++++++++++++++++ tests/ExecuteNetwork/ExecuteNetworkParams.cpp | 1 + tests/ExecuteNetwork/ExecuteNetworkParams.hpp | 1 + .../ExecuteNetworkProgramOptions.cpp | 4 + 11 files changed, 240 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 30b03dce04..cee3c2a020 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,16 @@ foreach(cmake_file ${additional_cmake_files}) include(${cmake_file}) endforeach() +cmake_policy(SET CMP0057 NEW) + +set(as_subproject Armnn) + +macro(find_package) + if(NOT ${ARGV0} IN_LIST as_subproject) + _find_package(${ARGV}) + endif() +endmacro() + if (DYNAMIC_BACKEND_PATHS) # It's expected to have the format: DYNAMIC_BACKEND_PATHS="PATH_1:PATH_2...:PATH_N" add_definitions('-DDYNAMIC_BACKEND_PATHS="${DYNAMIC_BACKEND_PATHS}"') @@ -29,6 +39,15 @@ add_subdirectory(src/armnnTfLiteParser) add_subdirectory(src/armnnSerializer) add_subdirectory(src/armnnDeserializer) + +if (BUILD_ARMNN_TFLITE_DELEGATE) + + list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/delegate/cmake/Modules) + add_subdirectory(delegate) + + add_definitions(-DARMNN_TF_LITE_DELEGATE) +endif() + if (BUILD_TESTS) add_subdirectory(tests) endif() @@ -1164,6 +1183,9 @@ export( NAMESPACE Armnn:: ) +add_library(Armnn::Armnn ALIAS armnn) +add_library(Armnn::armnnUtils ALIAS armnnUtils) + #################################################### ## Build Python bindings if (BUILD_PYTHON_WHL OR BUILD_PYTHON_SRC) diff --git a/cmake/GlobalConfig.cmake b/cmake/GlobalConfig.cmake index 360f06860e..843ad6b639 100644 --- a/cmake/GlobalConfig.cmake +++ b/cmake/GlobalConfig.cmake @@ -35,6 +35,7 @@ option(BUILD_PYTHON_WHL "Build Python wheel package" OFF) option(BUILD_PYTHON_SRC "Build Python source package" OFF) option(BUILD_STATIC_PIPE_LIBS "Build Static PIPE libraries" OFF) option(BUILD_PIPE_ONLY "Build the PIPE libraries only" OFF) +option(BUILD_ARMNN_TFLITE_DELEGATE "Build the Arm NN TfLite delegate" OFF) include(SelectLibraryConfigurations) @@ -210,6 +211,9 @@ if(BUILD_ONNX_PARSER) include_directories(SYSTEM "${ONNX_GENERATED_SOURCES}") endif() +if(BUILD_ARMNN_TFLITE_DELEGATE) + add_definitions(-DARMNN_TFLITE_DELEGATE) +endif() # Flatbuffers support for TF Lite and Armnn Serializer if(BUILD_TF_LITE_PARSER OR BUILD_ARMNN_SERIALIZER) # verify we have a valid flatbuffers include path diff --git a/delegate/CMakeLists.txt b/delegate/CMakeLists.txt index 595784f37a..d53af88105 100644 --- a/delegate/CMakeLists.txt +++ b/delegate/CMakeLists.txt @@ -195,5 +195,7 @@ export( EXPORT armnn-delegate-targets FILE ${CMAKE_CURRENT_BINARY_DIR}/ArmnnDelegateTargets.cmake NAMESPACE ArmnnDelegate::) +add_library(ArmnnDelegate::ArmnnDelegate ALIAS armnnDelegate) + #################################################### diff --git a/delegate/cmake/Modules/ArmnnDelegateConfig.cmake.in b/delegate/cmake/Modules/ArmnnDelegateConfig.cmake.in index 12d1161afa..c403068db8 100644 --- a/delegate/cmake/Modules/ArmnnDelegateConfig.cmake.in +++ b/delegate/cmake/Modules/ArmnnDelegateConfig.cmake.in @@ -9,6 +9,8 @@ MESSAGE(STATUS "Found ArmnnDelegate: ${ARMNN_DELEGATE_CONFIG_FILE}") include(CMakeFindDependencyMacro) +find_dependency(Armnn REQUIRED CONFIG) + list(APPEND CMAKE_MODULE_PATH ${ARMNN_DELEGATE_CMAKE_DIR}) if(NOT TARGET ArmnnDelegate::ArmnnDelegate) diff --git a/delegate/include/armnn_delegate.hpp b/delegate/include/armnn_delegate.hpp index 6f18185d7b..adf264aabb 100644 --- a/delegate/include/armnn_delegate.hpp +++ b/delegate/include/armnn_delegate.hpp @@ -3,8 +3,7 @@ // SPDX-License-Identifier: MIT // -#ifndef ARMNN_TFLITE_DELEGATE -#define ARMNN_TFLITE_DELEGATE +#pragma once #include "DelegateOptions.hpp" @@ -114,5 +113,4 @@ private: } // armnnDelegate namespace -#endif // ARMNN_TFLITE_DELEGATE diff --git a/delegate/src/Transpose.hpp b/delegate/src/Transpose.hpp index dd79bd3bee..f3c348a1c2 100644 --- a/delegate/src/Transpose.hpp +++ b/delegate/src/Transpose.hpp @@ -64,7 +64,7 @@ TfLiteStatus VisitTransposeOperator(DelegateData& delegateData, auto* permTensorDataPtr = tflite::GetTensorData(&tfLiteInputTensor1); unsigned int numEl = tfLiteInputTensor1.dims->data[0]; - ARMNN_ASSERT( numEl <= armnn::MaxNumOfTensorDimensions); + ARMNN_ASSERT( numEl <= static_cast(armnn::MaxNumOfTensorDimensions)); ARMNN_ASSERT( tfLiteInputTensor1.dims->size == 1); // ensure only single dimension to the permutation tensor armnn::TransposeDescriptor descriptor(armnn::PermutationVector( diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9d3b026687..edea34dfee 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -264,12 +264,15 @@ if (BUILD_ARMNN_SERIALIZER OR BUILD_CAFFE_PARSER OR BUILD_TF_PARSER OR BUILD_TF_ target_link_libraries(ExecuteNetwork armnnTfLiteParser) endif() if (BUILD_ONNX_PARSER) - target_link_libraries(ExecuteNetwork armnnOnnxParser) + target_link_libraries(ExecuteNetwork armnnOnnxParser) + endif() + if (BUILD_ARMNN_TFLITE_DELEGATE) + target_link_libraries(ExecuteNetwork ArmnnDelegate::ArmnnDelegate) endif() - target_link_libraries(ExecuteNetwork armnn) - target_link_libraries(ExecuteNetwork ${CMAKE_THREAD_LIBS_INIT}) - addDllCopyCommands(ExecuteNetwork) + + target_link_libraries(ExecuteNetwork ${CMAKE_THREAD_LIBS_INIT}) + addDllCopyCommands(ExecuteNetwork) endif() if(BUILD_ACCURACY_TOOL) diff --git a/tests/ExecuteNetwork/ExecuteNetwork.cpp b/tests/ExecuteNetwork/ExecuteNetwork.cpp index c17eabd837..fa84a6ee4f 100644 --- a/tests/ExecuteNetwork/ExecuteNetwork.cpp +++ b/tests/ExecuteNetwork/ExecuteNetwork.cpp @@ -25,9 +25,194 @@ #if defined(ARMNN_ONNX_PARSER) #include "armnnOnnxParser/IOnnxParser.hpp" #endif +#if defined(ARMNN_TFLITE_DELEGATE) +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#endif #include +#if defined(ARMNN_TFLITE_DELEGATE) +int TfLiteDelegateMainImpl(const ExecuteNetworkParams& params, + const std::shared_ptr& runtime = nullptr) +{ + using namespace tflite; + + std::unique_ptr model = tflite::FlatBufferModel::BuildFromFile(params.m_ModelPath.c_str()); + + auto tfLiteInterpreter = std::make_unique(); + tflite::ops::builtin::BuiltinOpResolver resolver; + + tflite::InterpreterBuilder builder(*model, resolver); + builder(&tfLiteInterpreter); + tfLiteInterpreter->AllocateTensors(); + + // Create the Armnn Delegate + armnnDelegate::DelegateOptions delegateOptions(params.m_ComputeDevices); + std::unique_ptr + theArmnnDelegate(armnnDelegate::TfLiteArmnnDelegateCreate(delegateOptions), + armnnDelegate::TfLiteArmnnDelegateDelete); + // Register armnn_delegate to TfLiteInterpreter + int status = tfLiteInterpreter->ModifyGraphWithDelegate(std::move(theArmnnDelegate)); + + std::vector inputBindings; + for (const std::string& inputName: params.m_InputNames) + { + inputBindings.push_back(inputName); + } + + armnn::Optional dataFile = params.m_GenerateTensorData + ? armnn::EmptyOptional() + : armnn::MakeOptional(params.m_InputTensorDataFilePaths[0]); + + const size_t numInputs = inputBindings.size(); + + for(unsigned int inputIndex = 0; inputIndex < numInputs; ++inputIndex) + { + int input = tfLiteInterpreter->inputs()[inputIndex]; + if (params.m_InputTypes[inputIndex].compare("float") == 0) + { + auto inputData = tfLiteInterpreter->typed_tensor(input); + TContainer tensorData; + PopulateTensorWithData(tensorData, + params.m_InputTensorShapes[inputIndex]->GetNumElements(), + params.m_InputTypes[inputIndex], + armnn::EmptyOptional(), + dataFile); + inputData = reinterpret_cast(&tensorData); + armnn::IgnoreUnused(inputData); + } + else if (params.m_InputTypes[inputIndex].compare("int") == 0) + { + auto inputData = tfLiteInterpreter->typed_tensor(input); + TContainer tensorData; + PopulateTensorWithData(tensorData, + params.m_InputTensorShapes[inputIndex]->GetNumElements(), + params.m_InputTypes[inputIndex], + armnn::EmptyOptional(), + dataFile); + inputData = reinterpret_cast(&tensorData); + armnn::IgnoreUnused(inputData); + } + else if (params.m_InputTypes[inputIndex].compare("qasymm8") == 0) + { + auto inputData = tfLiteInterpreter->typed_tensor(input); + TContainer tensorData; + PopulateTensorWithData(tensorData, + params.m_InputTensorShapes[inputIndex]->GetNumElements(), + params.m_InputTypes[inputIndex], + armnn::EmptyOptional(), + dataFile); + inputData = reinterpret_cast(&tensorData); + armnn::IgnoreUnused(inputData); + } + else + { + ARMNN_LOG(fatal) << "Unsupported input tensor data type \"" << params.m_InputTypes[inputIndex] << "\". "; + return EXIT_FAILURE; + } + } + + for (size_t x = 0; x < params.m_Iterations; x++) + { + // Run the inference + tfLiteInterpreter->Invoke(); + + // Print out the output + for (unsigned int outputIndex = 0; outputIndex < params.m_OutputNames.size(); ++outputIndex) + { + std::cout << "Printing out the output" << std::endl; + auto tfLiteDelegateOutputId = tfLiteInterpreter->outputs()[outputIndex]; + TfLiteIntArray *outputDims = tfLiteInterpreter->tensor(tfLiteDelegateOutputId)->dims; + + int outputSize = 1; + for (unsigned int dim = 0; dim < static_cast(outputDims->size); ++dim) + { + outputSize *= outputDims->data[dim]; + } + + std::cout << params.m_OutputNames[outputIndex] << ": "; + if (params.m_OutputTypes[outputIndex].compare("float") == 0) + { + auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor(tfLiteDelegateOutputId); + + if(tfLiteDelageOutputData == NULL) + { + ARMNN_LOG(fatal) << "Output tensor is null, output type: " + "\"" << params.m_OutputTypes[outputIndex] << "\" may be incorrect."; + return EXIT_FAILURE; + } + + for (int i = 0; i < outputSize; ++i) + { + std::cout << tfLiteDelageOutputData[i] << ", "; + if (i % 60 == 0) + { + std::cout << std::endl; + } + } + } + else if (params.m_OutputTypes[outputIndex].compare("int") == 0) + { + auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor(tfLiteDelegateOutputId); + + if(tfLiteDelageOutputData == NULL) + { + ARMNN_LOG(fatal) << "Output tensor is null, output type: " + "\"" << params.m_OutputTypes[outputIndex] << "\" may be incorrect."; + return EXIT_FAILURE; + } + + for (int i = 0; i < outputSize; ++i) + { + std::cout << tfLiteDelageOutputData[i] << ", "; + if (i % 60 == 0) + { + std::cout << std::endl; + } + } + } + else if (params.m_OutputTypes[outputIndex].compare("qasymm8") == 0) + { + auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor(tfLiteDelegateOutputId); + if(tfLiteDelageOutputData == NULL) + { + ARMNN_LOG(fatal) << "Output tensor is null, output type: " + "\"" << params.m_OutputTypes[outputIndex] << "\" may be incorrect."; + return EXIT_FAILURE; + } + + for (int i = 0; i < outputSize; ++i) + { + std::cout << unsigned(tfLiteDelageOutputData[i]) << ", "; + if (i % 60 == 0) + { + std::cout << std::endl; + } + } + } + else + { + ARMNN_LOG(fatal) << "Output tensor is null, output type: " + "\"" << params.m_OutputTypes[outputIndex] << + "\" may be incorrect. Output type can be specified with -z argument"; + return EXIT_FAILURE; + } + std::cout << std::endl; + } + } + + return status; +} +#endif template int MainImpl(const ExecuteNetworkParams& params, const std::shared_ptr& runtime = nullptr) @@ -242,6 +427,16 @@ int main(int argc, const char* argv[]) } else if(modelFormat.find("tflite") != std::string::npos) { + + if (ProgramOptions.m_ExNetParams.m_EnableDelegate) + { + #if defined(ARMNN_TF_LITE_DELEGATE) + return TfLiteDelegateMainImpl(ProgramOptions.m_ExNetParams, runtime); + #else + ARMNN_LOG(fatal) << "Not built with Tensorflow-Lite parser support."; + return EXIT_FAILURE; + #endif + } #if defined(ARMNN_TF_LITE_PARSER) return MainImpl(ProgramOptions.m_ExNetParams, runtime); #else diff --git a/tests/ExecuteNetwork/ExecuteNetworkParams.cpp b/tests/ExecuteNetwork/ExecuteNetworkParams.cpp index a3a7f6a753..890ab2a658 100644 --- a/tests/ExecuteNetwork/ExecuteNetworkParams.cpp +++ b/tests/ExecuteNetwork/ExecuteNetworkParams.cpp @@ -74,6 +74,7 @@ void CheckModelFormat(const std::string& modelFormat) "format supported for tflite files", modelFormat)); } +#elif defined(ARMNN_TFLITE_DELEGATE) #else throw armnn::InvalidArgumentException("Can't run model in tflite format without a " "built with Tensorflow Lite parser support."); diff --git a/tests/ExecuteNetwork/ExecuteNetworkParams.hpp b/tests/ExecuteNetwork/ExecuteNetworkParams.hpp index 5490230ede..8f176c2fd6 100644 --- a/tests/ExecuteNetwork/ExecuteNetworkParams.hpp +++ b/tests/ExecuteNetwork/ExecuteNetworkParams.hpp @@ -24,6 +24,7 @@ struct ExecuteNetworkParams bool m_EnableProfiling; bool m_GenerateTensorData; bool m_InferOutputShape = false; + bool m_EnableDelegate = false; std::vector m_InputNames; std::vector m_InputTensorDataFilePaths; std::vector m_InputTensorShapes; diff --git a/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp b/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp index e37c4eb42e..b499289f61 100644 --- a/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp +++ b/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp @@ -168,6 +168,10 @@ ProgramOptions::ProgramOptions() : m_CxxOptions{"ExecuteNetwork", "tensorflow-text.", cxxopts::value()) + ("D,armnn-tflite-delegate", + "enable Arm NN TfLite delegate", + cxxopts::value(m_ExNetParams.m_EnableDelegate)->default_value("false")->implicit_value("true")) + ("m,model-path", "Path to model file, e.g. .armnn, .caffemodel, .prototxt, .tflite, .onnx", cxxopts::value(m_ExNetParams.m_ModelPath)) -- cgit v1.2.1