ArmNN
 20.11
InferenceModel< IParser, TDataType > Class Template Reference

#include <InferenceModel.hpp>

Classes

struct  CommandLineOptions
 

Public Types

using DataType = TDataType
 
using Params = InferenceModelInternal::Params
 
using QuantizationParams = InferenceModelInternal::QuantizationParams
 
using TContainer = mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > >
 

Public Member Functions

 InferenceModel (const Params &params, bool enableProfiling, const std::string &dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
 
void CheckInputIndexIsValid (unsigned int inputIndex) const
 
void CheckOutputIndexIsValid (unsigned int outputIndex) const
 
unsigned int GetInputSize (unsigned int inputIndex=0u) const
 
unsigned int GetOutputSize (unsigned int outputIndex=0u) const
 
std::chrono::duration< double, std::milli > Run (const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
 
const armnn::BindingPointInfoGetInputBindingInfo (unsigned int inputIndex=0u) const
 
const std::vector< armnn::BindingPointInfo > & GetInputBindingInfos () const
 
const armnn::BindingPointInfoGetOutputBindingInfo (unsigned int outputIndex=0u) const
 
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos () const
 
QuantizationParams GetQuantizationParams (unsigned int outputIndex=0u) const
 
QuantizationParams GetInputQuantizationParams (unsigned int inputIndex=0u) const
 
std::vector< QuantizationParamsGetAllQuantizationParams () const
 

Static Public Member Functions

static void AddCommandLineOptions (cxxopts::Options &options, CommandLineOptions &cLineOptions, std::vector< std::string > &required)
 

Detailed Description

template<typename IParser, typename TDataType>
class InferenceModel< IParser, TDataType >

Definition at line 323 of file InferenceModel.hpp.

Member Typedef Documentation

◆ DataType

using DataType = TDataType

Definition at line 326 of file InferenceModel.hpp.

◆ Params

Definition at line 327 of file InferenceModel.hpp.

◆ QuantizationParams

◆ TContainer

using TContainer = mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char> >

Definition at line 329 of file InferenceModel.hpp.

Constructor & Destructor Documentation

◆ InferenceModel()

InferenceModel ( const Params params,
bool  enableProfiling,
const std::string &  dynamicBackendsPath,
const std::shared_ptr< armnn::IRuntime > &  runtime = nullptr 
)
inline

Definition at line 387 of file InferenceModel.hpp.

References ARMNN_LOG, ARMNN_SCOPED_HEAP_PROFILING, IRuntime::Create(), CreateNetworkImpl< IParser >::Create(), armnn::Failure, armnn::GetTimeDuration(), armnn::GetTimeNow(), Params::m_ComputeDevices, OptimizerOptions::m_Debug, IRuntime::CreationOptions::m_DynamicBackendsPath, Params::m_DynamicBackendsPath, Params::m_EnableBf16TurboMode, Params::m_EnableFastMath, Params::m_EnableFp16TurboMode, IRuntime::CreationOptions::m_EnableGpuProfiling, Params::m_InputBindings, OptimizerOptions::m_ModelOptions, Params::m_ModelPath, Params::m_OutputBindings, Params::m_PrintIntermediateLayers, OptimizerOptions::m_ReduceFp32ToBf16, OptimizerOptions::m_ReduceFp32ToFp16, Params::m_VisualizePostOptimizationModel, and armnn::Optimize().

391  : m_EnableProfiling(enableProfiling)
392  , m_DynamicBackendsPath(dynamicBackendsPath)
393  {
394  if (runtime)
395  {
396  m_Runtime = runtime;
397  }
398  else
399  {
401  options.m_EnableGpuProfiling = m_EnableProfiling;
402  options.m_DynamicBackendsPath = m_DynamicBackendsPath;
403  m_Runtime = std::move(armnn::IRuntime::Create(options));
404  }
405 
406  std::string invalidBackends;
407  if (!CheckRequestedBackendsAreValid(params.m_ComputeDevices, armnn::Optional<std::string&>(invalidBackends)))
408  {
409  throw armnn::Exception("Some backend IDs are invalid: " + invalidBackends);
410  }
411 
412  const auto parsing_start_time = armnn::GetTimeNow();
413  armnn::INetworkPtr network = CreateNetworkImpl<IParser>::Create(params, m_InputBindings, m_OutputBindings);
414 
415  ARMNN_LOG(info) << "Network parsing time: " << std::setprecision(2)
416  << std::fixed << armnn::GetTimeDuration(parsing_start_time).count() << " ms\n";
417 
419  {
420  ARMNN_SCOPED_HEAP_PROFILING("Optimizing");
421 
422  armnn::OptimizerOptions options;
423  options.m_ReduceFp32ToFp16 = params.m_EnableFp16TurboMode;
424  options.m_ReduceFp32ToBf16 = params.m_EnableBf16TurboMode;
425  options.m_Debug = params.m_PrintIntermediateLayers;
426 
427  armnn::BackendOptions gpuAcc("GpuAcc",
428  {
429  { "FastMathEnabled", params.m_EnableFastMath }
430  });
431  armnn::BackendOptions cpuAcc("CpuAcc",
432  {
433  { "FastMathEnabled", params.m_EnableFastMath }
434  });
435  options.m_ModelOptions.push_back(gpuAcc);
436  options.m_ModelOptions.push_back(cpuAcc);
437 
438  const auto optimization_start_time = armnn::GetTimeNow();
439  optNet = armnn::Optimize(*network, params.m_ComputeDevices, m_Runtime->GetDeviceSpec(), options);
440 
441  ARMNN_LOG(info) << "Optimization time: " << std::setprecision(2)
442  << std::fixed << armnn::GetTimeDuration(optimization_start_time).count() << " ms\n";
443 
444  if (!optNet)
445  {
446  throw armnn::Exception("Optimize returned nullptr");
447  }
448  }
449 
450  if (params.m_VisualizePostOptimizationModel)
451  {
452  fs::path filename = params.m_ModelPath;
453  filename.replace_extension("dot");
454  std::fstream file(filename.c_str(), std::ios_base::out);
455  optNet->SerializeToDot(file);
456  }
457 
458  armnn::Status ret;
459  {
460  ARMNN_SCOPED_HEAP_PROFILING("LoadNetwork");
461  ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
462  }
463 
464  if (ret == armnn::Status::Failure)
465  {
466  throw armnn::Exception("IRuntime::LoadNetwork failed");
467  }
468  }
ModelOptions m_ModelOptions
Definition: INetwork.hpp:674
static IRuntimePtr Create(const CreationOptions &options)
Definition: Runtime.cpp:32
std::chrono::duration< double, std::milli > GetTimeDuration(std::chrono::high_resolution_clock::time_point start_time)
Definition: Timer.hpp:19
#define ARMNN_LOG(severity)
Definition: Logging.hpp:163
std::chrono::high_resolution_clock::time_point GetTimeNow()
Definition: Timer.hpp:14
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
Definition: Network.cpp:1011
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
Status
enumeration
Definition: Types.hpp:26
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
Definition: INetwork.hpp:600
Struct for the users to pass backend specific options.
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
Definition: IRuntime.hpp:59
bool m_EnableGpuProfiling
Setting this flag will allow the user to obtain GPU profiling information from the runtime...
Definition: IRuntime.hpp:55
static armnn::INetworkPtr Create(const Params &params, std::vector< armnn::BindingPointInfo > &inputBindings, std::vector< armnn::BindingPointInfo > &outputBindings)
Base class for all ArmNN exceptions so that users can filter to just those.
Definition: Exceptions.hpp:46
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
Definition: INetwork.hpp:101

Member Function Documentation

◆ AddCommandLineOptions()

static void AddCommandLineOptions ( cxxopts::Options &  options,
CommandLineOptions cLineOptions,
std::vector< std::string > &  required 
)
inlinestatic

Definition at line 349 of file InferenceModel.hpp.

References armnn::BackendRegistryInstance(), BackendRegistry::GetBackendIdsAsString(), InferenceModel< IParser, TDataType >::CommandLineOptions::m_ComputeDevices, InferenceModel< IParser, TDataType >::CommandLineOptions::m_DynamicBackendsPath, InferenceModel< IParser, TDataType >::CommandLineOptions::m_EnableBf16TurboMode, InferenceModel< IParser, TDataType >::CommandLineOptions::m_EnableFp16TurboMode, InferenceModel< IParser, TDataType >::CommandLineOptions::m_Labels, InferenceModel< IParser, TDataType >::CommandLineOptions::m_ModelDir, and InferenceModel< IParser, TDataType >::CommandLineOptions::m_VisualizePostOptimizationModel.

Referenced by ClassifierTestCaseProvider< TDatabase, InferenceModel >::AddCommandLineOptions().

351  {
352  const std::vector<std::string> defaultComputes = { "CpuAcc", "CpuRef" };
353 
354  const std::string backendsMessage = "Which device to run layers on by default. Possible choices: "
356 
357  options
358  .allow_unrecognised_options()
359  .add_options()
360  ("m,model-dir", "Path to directory containing model files (.caffemodel/.prototxt/.tflite)",
361  cxxopts::value<std::string>(cLineOptions.m_ModelDir))
362  ("c,compute", backendsMessage.c_str(),
363  cxxopts::value<std::vector<std::string>>(cLineOptions.m_ComputeDevices)->default_value("CpuRef"))
364  ("b,dynamic-backends-path",
365  "Path where to load any available dynamic backend from. "
366  "If left empty (the default), dynamic backends will not be used.",
367  cxxopts::value(cLineOptions.m_DynamicBackendsPath))
368  ("l,labels",
369  "Text file containing one image filename - correct label pair per line, "
370  "used to test the accuracy of the network.", cxxopts::value<std::string>(cLineOptions.m_Labels))
371  ("v,visualize-optimized-model",
372  "Produce a dot file useful for visualizing the graph post optimization."
373  "The file will have the same name as the model with the .dot extention.",
374  cxxopts::value<bool>(cLineOptions.m_VisualizePostOptimizationModel)->default_value("false"))
375  ("fp16-turbo-mode",
376  "If this option is enabled FP32 layers, weights and biases will be converted "
377  "to FP16 where the backend supports it.",
378  cxxopts::value<bool>(cLineOptions.m_EnableFp16TurboMode)->default_value("false"))
379  ("bf16-turbo-mode",
380  "If this option is enabled FP32 layers, weights and biases will be converted "
381  "to BF16 where the backend supports it.",
382  cxxopts::value<bool>(cLineOptions.m_EnableBf16TurboMode)->default_value("false"));
383 
384  required.emplace_back("model-dir");
385  }
BackendRegistry & BackendRegistryInstance()
std::string GetBackendIdsAsString() const

◆ CheckInputIndexIsValid()

void CheckInputIndexIsValid ( unsigned int  inputIndex) const
inline

Definition at line 470 of file InferenceModel.hpp.

References Params::m_InputBindings.

471  {
472  if (m_InputBindings.size() < inputIndex + 1)
473  {
474  throw armnn::Exception(fmt::format("Input index out of range: {}", inputIndex));
475  }
476  }
Base class for all ArmNN exceptions so that users can filter to just those.
Definition: Exceptions.hpp:46

◆ CheckOutputIndexIsValid()

void CheckOutputIndexIsValid ( unsigned int  outputIndex) const
inline

Definition at line 478 of file InferenceModel.hpp.

References Params::m_OutputBindings.

479  {
480  if (m_OutputBindings.size() < outputIndex + 1)
481  {
482  throw armnn::Exception(fmt::format("Output index out of range: {}", outputIndex));
483  }
484  }
Base class for all ArmNN exceptions so that users can filter to just those.
Definition: Exceptions.hpp:46

◆ GetAllQuantizationParams()

std::vector<QuantizationParams> GetAllQuantizationParams ( ) const
inline

Definition at line 587 of file InferenceModel.hpp.

References Params::m_DynamicBackendsPath, Params::m_InputBindings, Params::m_OutputBindings, armnnUtils::MakeInputTensors(), MakeInputTensors(), armnnUtils::MakeOutputTensors(), and MakeOutputTensors().

588  {
589  std::vector<QuantizationParams> quantizationParams;
590  for (unsigned int i = 0u; i < m_OutputBindings.size(); i++)
591  {
592  quantizationParams.push_back(GetQuantizationParams(i));
593  }
594  return quantizationParams;
595  }
QuantizationParams GetQuantizationParams(unsigned int outputIndex=0u) const

◆ GetInputBindingInfo()

const armnn::BindingPointInfo& GetInputBindingInfo ( unsigned int  inputIndex = 0u) const
inline

Definition at line 551 of file InferenceModel.hpp.

References Params::m_InputBindings.

Referenced by main().

552  {
553  CheckInputIndexIsValid(inputIndex);
554  return m_InputBindings[inputIndex];
555  }
void CheckInputIndexIsValid(unsigned int inputIndex) const

◆ GetInputBindingInfos()

const std::vector<armnn::BindingPointInfo>& GetInputBindingInfos ( ) const
inline

Definition at line 557 of file InferenceModel.hpp.

References Params::m_InputBindings.

558  {
559  return m_InputBindings;
560  }

◆ GetInputQuantizationParams()

QuantizationParams GetInputQuantizationParams ( unsigned int  inputIndex = 0u) const
inline

Definition at line 580 of file InferenceModel.hpp.

References Params::m_InputBindings.

Referenced by MainImpl().

581  {
582  CheckInputIndexIsValid(inputIndex);
583  return std::make_pair(m_InputBindings[inputIndex].second.GetQuantizationScale(),
584  m_InputBindings[inputIndex].second.GetQuantizationOffset());
585  }
void CheckInputIndexIsValid(unsigned int inputIndex) const

◆ GetInputSize()

unsigned int GetInputSize ( unsigned int  inputIndex = 0u) const
inline

Definition at line 486 of file InferenceModel.hpp.

References Params::m_InputBindings.

Referenced by MainImpl().

487  {
488  CheckInputIndexIsValid(inputIndex);
489  return m_InputBindings[inputIndex].second.GetNumElements();
490  }
void CheckInputIndexIsValid(unsigned int inputIndex) const

◆ GetOutputBindingInfo()

const armnn::BindingPointInfo& GetOutputBindingInfo ( unsigned int  outputIndex = 0u) const
inline

Definition at line 562 of file InferenceModel.hpp.

References Params::m_OutputBindings.

563  {
564  CheckOutputIndexIsValid(outputIndex);
565  return m_OutputBindings[outputIndex];
566  }
void CheckOutputIndexIsValid(unsigned int outputIndex) const

◆ GetOutputBindingInfos()

const std::vector<armnn::BindingPointInfo>& GetOutputBindingInfos ( ) const
inline

Definition at line 568 of file InferenceModel.hpp.

References Params::m_OutputBindings.

Referenced by MainImpl().

569  {
570  return m_OutputBindings;
571  }

◆ GetOutputSize()

unsigned int GetOutputSize ( unsigned int  outputIndex = 0u) const
inline

Definition at line 492 of file InferenceModel.hpp.

References Params::m_OutputBindings.

Referenced by main(), and MainImpl().

493  {
494  CheckOutputIndexIsValid(outputIndex);
495  return m_OutputBindings[outputIndex].second.GetNumElements();
496  }
void CheckOutputIndexIsValid(unsigned int outputIndex) const

◆ GetQuantizationParams()

QuantizationParams GetQuantizationParams ( unsigned int  outputIndex = 0u) const
inline

Definition at line 573 of file InferenceModel.hpp.

References Params::m_OutputBindings.

574  {
575  CheckOutputIndexIsValid(outputIndex);
576  return std::make_pair(m_OutputBindings[outputIndex].second.GetQuantizationScale(),
577  m_OutputBindings[outputIndex].second.GetQuantizationOffset());
578  }
void CheckOutputIndexIsValid(unsigned int outputIndex) const

◆ Run()

std::chrono::duration<double, std::milli> Run ( const std::vector< TContainer > &  inputContainers,
std::vector< TContainer > &  outputContainers 
)
inline

Definition at line 498 of file InferenceModel.hpp.

References armnn::Failure, armnn::GetTimeDuration(), armnn::GetTimeNow(), MakeInputTensors(), MakeOutputTensors(), and armnn::numeric_cast().

Referenced by MainImpl().

501  {
502  for (unsigned int i = 0; i < outputContainers.size(); ++i)
503  {
504  const unsigned int expectedOutputDataSize = GetOutputSize(i);
505 
506  mapbox::util::apply_visitor([expectedOutputDataSize, i](auto&& value)
507  {
508  const unsigned int actualOutputDataSize = armnn::numeric_cast<unsigned int>(value.size());
509  if (actualOutputDataSize < expectedOutputDataSize)
510  {
511  unsigned int outputIndex = i;
512  throw armnn::Exception(
513  fmt::format("Not enough data for output #{0}: expected "
514  "{1} elements, got {2}", outputIndex, expectedOutputDataSize, actualOutputDataSize));
515  }
516  },
517  outputContainers[i]);
518  }
519 
520  std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
521  if (profiler)
522  {
523  profiler->EnableProfiling(m_EnableProfiling);
524  }
525 
526  // Start timer to record inference time in EnqueueWorkload (in milliseconds)
527  const auto start_time = armnn::GetTimeNow();
528 
529  armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
530  MakeInputTensors(inputContainers),
531  MakeOutputTensors(outputContainers));
532 
533  const auto duration = armnn::GetTimeDuration(start_time);
534 
535  // if profiling is enabled print out the results
536  if (profiler && profiler->IsProfilingEnabled())
537  {
538  profiler->Print(std::cout);
539  }
540 
541  if (ret == armnn::Status::Failure)
542  {
543  throw armnn::Exception("IRuntime::EnqueueWorkload failed");
544  }
545  else
546  {
547  return duration;
548  }
549  }
std::chrono::duration< double, std::milli > GetTimeDuration(std::chrono::high_resolution_clock::time_point start_time)
Definition: Timer.hpp:19
std::chrono::high_resolution_clock::time_point GetTimeNow()
Definition: Timer.hpp:14
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
Status
enumeration
Definition: Types.hpp:26
Base class for all ArmNN exceptions so that users can filter to just those.
Definition: Exceptions.hpp:46
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
Definition: NumericCast.hpp:35

The documentation for this class was generated from the following file: