diff options
author | telsoa01 <telmo.soares@arm.com> | 2018-08-31 09:22:23 +0100 |
---|---|---|
committer | telsoa01 <telmo.soares@arm.com> | 2018-08-31 09:22:23 +0100 |
commit | c577f2c6a3b4ddb6ba87a882723c53a248afbeba (patch) | |
tree | bd7d4c148df27f8be6649d313efb24f536b7cf34 /tests/InferenceModel.hpp | |
parent | 4c7098bfeab1ffe1cdc77f6c15548d3e73274746 (diff) | |
download | armnn-c577f2c6a3b4ddb6ba87a882723c53a248afbeba.tar.gz |
Release 18.08
Diffstat (limited to 'tests/InferenceModel.hpp')
-rw-r--r-- | tests/InferenceModel.hpp | 270 |
1 files changed, 213 insertions, 57 deletions
diff --git a/tests/InferenceModel.hpp b/tests/InferenceModel.hpp index f5f00378ca..dfd21bbed1 100644 --- a/tests/InferenceModel.hpp +++ b/tests/InferenceModel.hpp @@ -4,7 +4,15 @@ // #pragma once #include "armnn/ArmNN.hpp" -#include "HeapProfiling.hpp" + +#if defined(ARMNN_TF_LITE_PARSER) +#include "armnnTfLiteParser/ITfLiteParser.hpp" +#endif + +#include <HeapProfiling.hpp> +#if defined(ARMNN_ONNX_PARSER) +#include "armnnOnnxParser/IOnnxParser.hpp" +#endif #include <boost/exception/exception.hpp> #include <boost/exception/diagnostic_information.hpp> @@ -16,9 +24,148 @@ #include <map> #include <string> #include <fstream> +#include <type_traits> + +namespace InferenceModelInternal +{ +// This needs to go when the armnnCaffeParser, armnnTfParser and armnnTfLiteParser +// definitions of BindingPointInfo gets consolidated. +using BindingPointInfo = std::pair<armnn::LayerBindingId, armnn::TensorInfo>; + +using QuantizationParams = std::pair<float,int32_t>; + +struct Params +{ + std::string m_ModelPath; + std::string m_InputBinding; + std::string m_OutputBinding; + const armnn::TensorShape* m_InputTensorShape; + std::vector<armnn::Compute> m_ComputeDevice; + bool m_EnableProfiling; + size_t m_SubgraphId; + bool m_IsModelBinary; + bool m_VisualizePostOptimizationModel; + bool m_EnableFp16TurboMode; + + Params() + : m_InputTensorShape(nullptr) + , m_ComputeDevice{armnn::Compute::CpuRef} + , m_EnableProfiling(false) + , m_SubgraphId(0) + , m_IsModelBinary(true) + , m_VisualizePostOptimizationModel(false) + , m_EnableFp16TurboMode(false) + {} +}; + +} // namespace InferenceModelInternal + +template <typename IParser> +struct CreateNetworkImpl +{ +public: + using Params = InferenceModelInternal::Params; + using BindingPointInfo = InferenceModelInternal::BindingPointInfo; + + static armnn::INetworkPtr Create(const Params& params, + BindingPointInfo& inputBindings, + BindingPointInfo& outputBindings) + { + const std::string& modelPath = params.m_ModelPath; + + // Create a network from a file on disk + auto parser(IParser::Create()); + + std::map<std::string, armnn::TensorShape> inputShapes; + if (params.m_InputTensorShape) + { + inputShapes[params.m_InputBinding] = *params.m_InputTensorShape; + } + std::vector<std::string> requestedOutputs{ params.m_OutputBinding }; + armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}}; + + { + ARMNN_SCOPED_HEAP_PROFILING("Parsing"); + // Handle text and binary input differently by calling the corresponding parser function + network = (params.m_IsModelBinary ? + parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) : + parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs)); + } + + inputBindings = parser->GetNetworkInputBindingInfo(params.m_InputBinding); + outputBindings = parser->GetNetworkOutputBindingInfo(params.m_OutputBinding); + return network; + } +}; + +#if defined(ARMNN_TF_LITE_PARSER) +template <> +struct CreateNetworkImpl<armnnTfLiteParser::ITfLiteParser> +{ +public: + using IParser = armnnTfLiteParser::ITfLiteParser; + using Params = InferenceModelInternal::Params; + using BindingPointInfo = InferenceModelInternal::BindingPointInfo; + + static armnn::INetworkPtr Create(const Params& params, + BindingPointInfo& inputBindings, + BindingPointInfo& outputBindings) + { + const std::string& modelPath = params.m_ModelPath; + + // Create a network from a file on disk + auto parser(IParser::Create()); + + armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}}; + + { + ARMNN_SCOPED_HEAP_PROFILING("Parsing"); + network = parser->CreateNetworkFromBinaryFile(modelPath.c_str()); + } + + inputBindings = parser->GetNetworkInputBindingInfo(params.m_SubgraphId, params.m_InputBinding); + outputBindings = parser->GetNetworkOutputBindingInfo(params.m_SubgraphId, params.m_OutputBinding); + return network; + } +}; +#endif + +#if defined(ARMNN_ONNX_PARSER) +template <> +struct CreateNetworkImpl<armnnOnnxParser::IOnnxParser> +{ +public: + using IParser = armnnOnnxParser::IOnnxParser; + using Params = InferenceModelInternal::Params; + using BindingPointInfo = InferenceModelInternal::BindingPointInfo; + + static armnn::INetworkPtr Create(const Params& params, + BindingPointInfo& inputBindings, + BindingPointInfo& outputBindings) + { + const std::string& modelPath = params.m_ModelPath; + + // Create a network from a file on disk + auto parser(IParser::Create()); + + armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}}; + + { + ARMNN_SCOPED_HEAP_PROFILING("Parsing"); + network = (params.m_IsModelBinary ? + parser->CreateNetworkFromBinaryFile(modelPath.c_str()) : + parser->CreateNetworkFromTextFile(modelPath.c_str())); + } + + inputBindings = parser->GetNetworkInputBindingInfo(params.m_InputBinding); + outputBindings = parser->GetNetworkOutputBindingInfo(params.m_OutputBinding); + return network; + } +}; +#endif template<typename TContainer> -inline armnn::InputTensors MakeInputTensors(const std::pair<armnn::LayerBindingId, armnn::TensorInfo>& input, +inline armnn::InputTensors MakeInputTensors(const InferenceModelInternal::BindingPointInfo& input, const TContainer& inputTensorData) { if (inputTensorData.size() != input.second.GetNumElements()) @@ -30,7 +177,7 @@ inline armnn::InputTensors MakeInputTensors(const std::pair<armnn::LayerBindingI } catch (const boost::exception& e) { // Coverity fix: it should not be possible to get here but boost::str and boost::format can both - // throw uncaught exceptions - convert them to armnn exceptions and rethrow + // throw uncaught exceptions, convert them to armnn exceptions and rethrow. throw armnn::Exception(diagnostic_information(e)); } } @@ -38,7 +185,7 @@ inline armnn::InputTensors MakeInputTensors(const std::pair<armnn::LayerBindingI } template<typename TContainer> -inline armnn::OutputTensors MakeOutputTensors(const std::pair<armnn::LayerBindingId, armnn::TensorInfo>& output, +inline armnn::OutputTensors MakeOutputTensors(const InferenceModelInternal::BindingPointInfo& output, TContainer& outputTensorData) { if (outputTensorData.size() != output.second.GetNumElements()) @@ -48,17 +195,21 @@ inline armnn::OutputTensors MakeOutputTensors(const std::pair<armnn::LayerBindin return { { output.first, armnn::Tensor(output.second, outputTensorData.data()) } }; } + + template <typename IParser, typename TDataType> class InferenceModel { public: using DataType = TDataType; + using Params = InferenceModelInternal::Params; struct CommandLineOptions { std::string m_ModelDir; - armnn::Compute m_ComputeDevice; + std::vector<armnn::Compute> m_ComputeDevice; bool m_VisualizePostOptimizationModel; + bool m_EnableFp16TurboMode; }; static void AddCommandLineOptions(boost::program_options::options_description& desc, CommandLineOptions& options) @@ -67,66 +218,47 @@ public: desc.add_options() ("model-dir,m", po::value<std::string>(&options.m_ModelDir)->required(), - "Path to directory containing model files (.caffemodel/.prototxt)") - ("compute,c", po::value<armnn::Compute>(&options.m_ComputeDevice)->default_value(armnn::Compute::CpuAcc), + "Path to directory containing model files (.caffemodel/.prototxt/.tflite)") + ("compute,c", po::value<std::vector<armnn::Compute>>(&options.m_ComputeDevice)->default_value + ({armnn::Compute::CpuAcc, armnn::Compute::CpuRef}), "Which device to run layers on by default. Possible choices: CpuAcc, CpuRef, GpuAcc") ("visualize-optimized-model,v", po::value<bool>(&options.m_VisualizePostOptimizationModel)->default_value(false), "Produce a dot file useful for visualizing the graph post optimization." - "The file will have the same name as the model with the .dot extention."); + "The file will have the same name as the model with the .dot extention.") + ("fp16-turbo-mode", po::value<bool>(&options.m_EnableFp16TurboMode)->default_value(false), + "If this option is enabled FP32 layers, weights and biases will be converted " + "to FP16 where the backend supports it."); } - struct Params + InferenceModel(const Params& params, const std::shared_ptr<armnn::IRuntime>& runtime = nullptr) + : m_EnableProfiling(params.m_EnableProfiling) { - std::string m_ModelPath; - std::string m_InputBinding; - std::string m_OutputBinding; - const armnn::TensorShape* m_InputTensorShape; - armnn::Compute m_ComputeDevice; - bool m_IsModelBinary; - bool m_VisualizePostOptimizationModel; - - Params() - : m_InputTensorShape(nullptr) - , m_ComputeDevice(armnn::Compute::CpuRef) - , m_IsModelBinary(true) - , m_VisualizePostOptimizationModel(false) + if (runtime) { + m_Runtime = runtime; } - }; - - - InferenceModel(const Params& params) - : m_Runtime(armnn::IRuntime::Create(params.m_ComputeDevice)) - { - const std::string& modelPath = params.m_ModelPath; - - // Create a network from a file on disk - auto parser(IParser::Create()); - - std::map<std::string, armnn::TensorShape> inputShapes; - if (params.m_InputTensorShape) + else { - inputShapes[params.m_InputBinding] = *params.m_InputTensorShape; + armnn::IRuntime::CreationOptions options; + m_Runtime = std::move(armnn::IRuntime::Create(options)); } - std::vector<std::string> requestedOutputs{ params.m_OutputBinding }; - armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}}; - { - ARMNN_SCOPED_HEAP_PROFILING("Parsing"); - // Handle text and binary input differently by calling the corresponding parser function - network = (params.m_IsModelBinary ? - parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) : - parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs)); - } - - m_InputBindingInfo = parser->GetNetworkInputBindingInfo(params.m_InputBinding); - m_OutputBindingInfo = parser->GetNetworkOutputBindingInfo(params.m_OutputBinding); + armnn::INetworkPtr network = CreateNetworkImpl<IParser>::Create(params, m_InputBindingInfo, + m_OutputBindingInfo); armnn::IOptimizedNetworkPtr optNet{nullptr, [](armnn::IOptimizedNetwork *){}}; { ARMNN_SCOPED_HEAP_PROFILING("Optimizing"); - optNet = armnn::Optimize(*network, m_Runtime->GetDeviceSpec()); + + armnn::OptimizerOptions options; + options.m_ReduceFp32ToFp16 = params.m_EnableFp16TurboMode; + + optNet = armnn::Optimize(*network, params.m_ComputeDevice, m_Runtime->GetDeviceSpec(), options); + if (!optNet) + { + throw armnn::Exception("Optimize returned nullptr"); + } } if (params.m_VisualizePostOptimizationModel) @@ -157,16 +289,46 @@ public: void Run(const std::vector<TDataType>& input, std::vector<TDataType>& output) { BOOST_ASSERT(output.size() == GetOutputSize()); + + std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier); + if (profiler) + { + profiler->EnableProfiling(m_EnableProfiling); + } + armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier, - MakeInputTensors(input), - MakeOutputTensors(output)); + MakeInputTensors(input), + MakeOutputTensors(output)); if (ret == armnn::Status::Failure) { throw armnn::Exception("IRuntime::EnqueueWorkload failed"); } } + const InferenceModelInternal::BindingPointInfo & GetInputBindingInfo() const + { + return m_InputBindingInfo; + } + + const InferenceModelInternal::BindingPointInfo & GetOutputBindingInfo() const + { + return m_OutputBindingInfo; + } + + InferenceModelInternal::QuantizationParams GetQuantizationParams() const + { + return std::make_pair(m_OutputBindingInfo.second.GetQuantizationScale(), + m_OutputBindingInfo.second.GetQuantizationOffset()); + } + private: + armnn::NetworkId m_NetworkIdentifier; + std::shared_ptr<armnn::IRuntime> m_Runtime; + + InferenceModelInternal::BindingPointInfo m_InputBindingInfo; + InferenceModelInternal::BindingPointInfo m_OutputBindingInfo; + bool m_EnableProfiling; + template<typename TContainer> armnn::InputTensors MakeInputTensors(const TContainer& inputTensorData) { @@ -178,10 +340,4 @@ private: { return ::MakeOutputTensors(m_OutputBindingInfo, outputTensorData); } - - armnn::NetworkId m_NetworkIdentifier; - armnn::IRuntimePtr m_Runtime; - - std::pair<armnn::LayerBindingId, armnn::TensorInfo> m_InputBindingInfo; - std::pair<armnn::LayerBindingId, armnn::TensorInfo> m_OutputBindingInfo; }; |