56 std::string modelPath;
57 std::string modelFormat;
58 std::vector<std::string> inputNames;
59 std::vector<std::string> outputNames;
61 std::string modelOutputLabelsPath;
62 std::string validationLabelPath;
63 std::string inputLayout;
64 std::vector<armnn::BackendId> computeDevice;
65 std::string validationRange;
66 std::string excludelistPath;
68 const std::string backendsMessage =
"Which device to run layers on by default. Possible choices: " 73 cxxopts::Options options(
"ModeAccuracyTool-Armnn",
"Options");
76 (
"h,help",
"Display help messages")
78 "Path to armnn format model file",
79 cxxopts::value<std::string>(modelPath))
81 "The model format. Supported values: tflite",
82 cxxopts::value<std::string>(modelFormat))
84 "Identifier of the input tensors in the network separated by comma with no space.",
85 cxxopts::value<std::vector<std::string>>(inputNames))
87 "Identifier of the output tensors in the network separated by comma with no space.",
88 cxxopts::value<std::vector<std::string>>(outputNames))
90 "Path to directory containing the ImageNet test data",
91 cxxopts::value<std::string>(dataDir))
92 (
"p,model-output-labels",
93 "Path to model output labels file.",
94 cxxopts::value<std::string>(modelOutputLabelsPath))
95 (
"v,validation-labels-path",
96 "Path to ImageNet Validation Label file",
97 cxxopts::value<std::string>(validationLabelPath))
99 "Data layout. Supported value: NHWC, NCHW. Default: NHWC",
100 cxxopts::value<std::string>(inputLayout)->default_value(
"NHWC"))
102 backendsMessage.c_str(),
103 cxxopts::value<std::vector<armnn::BackendId>>(computeDevice)->default_value(
"CpuAcc,CpuRef"))
104 (
"r,validation-range",
105 "The range of the images to be evaluated. Specified in the form <begin index>:<end index>." 106 "The index starts at 1 and the range is inclusive." 107 "By default the evaluation will be performed on all images.",
108 cxxopts::value<std::string>(validationRange)->default_value(
"1:0"))
109 (
"e,excludelist-path",
110 "Path to a excludelist file where each line denotes the index of an image to be " 111 "excluded from evaluation.",
112 cxxopts::value<std::string>(excludelistPath)->default_value(
""));
115 "Path to a blacklist file where each line denotes the index of an image to be " 116 "excluded from evaluation. This command will be deprecated in favor of: --excludelist-path ",
117 cxxopts::value<std::string>(excludelistPath)->default_value(
""));
119 auto result = options.parse(argc, argv);
121 if (result.count(
"help") > 0)
123 std::cout << options.help() << std::endl;
128 std::string mandatorySingleParameters[] = {
"model-path",
"model-format",
"input-name",
"output-name",
129 "data-dir",
"model-output-labels",
"validation-labels-path" };
130 for (
auto param : mandatorySingleParameters)
132 if (result.count(param) != 1)
134 std::cerr <<
"Parameter \'--" << param <<
"\' is required but missing." << std::endl;
139 catch (
const cxxopts::OptionException& e)
141 std::cerr << e.what() << std::endl << std::endl;
144 catch (
const std::exception& e)
147 std::cerr <<
"Fatal internal error: " << e.what() << std::endl;
152 std::string invalidBackends;
155 ARMNN_LOG(fatal) <<
"The list of preferred devices contains invalid backend IDs: " 164 std::ifstream file(modelPath);
168 auto armnnparser(IParser::Create());
177 optimizedNet =
armnn::Optimize(*network, computeDevice, runtime->GetDeviceSpec());
181 std::stringstream message;
182 message <<
"armnn::Exception (" << e.
what() <<
") caught from optimize.";
189 status = runtime->LoadNetwork(networkId, std::move(optimizedNet));
192 ARMNN_LOG(fatal) <<
"armnn::IRuntime: Failed to load network";
200 std::vector<BindingPointInfo> inputBindings;
201 for(
auto& input: inputNames)
204 inputBindingInfo = armnnparser->GetNetworkInputBindingInfo(0, input);
206 std::pair<armnn::LayerBindingId, armnn::TensorInfo>
208 inputBindings.push_back(m_InputBindingInfo);
211 std::vector<BindingPointInfo> outputBindings;
212 for(
auto& output: outputNames)
215 outputBindingInfo = armnnparser->GetNetworkOutputBindingInfo(0, output);
217 std::pair<armnn::LayerBindingId, armnn::TensorInfo>
219 outputBindings.push_back(m_OutputBindingInfo);
223 if (modelOutputLabelsPath.empty() || !fs::exists(modelOutputLabelsPath) ||
224 !fs::is_regular_file(modelOutputLabelsPath))
226 ARMNN_LOG(fatal) <<
"Invalid model output labels path at " << modelOutputLabelsPath;
228 const std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels =
233 size_t imageBegIndex;
234 size_t imageEndIndex;
235 if (imageIndexStrs.size() != 2)
237 ARMNN_LOG(fatal) <<
"Invalid validation range specification: Invalid format " << validationRange;
242 imageBegIndex = std::stoul(imageIndexStrs[0]);
243 imageEndIndex = std::stoul(imageIndexStrs[1]);
245 catch (
const std::exception& e)
247 ARMNN_LOG(fatal) <<
"Invalid validation range specification: " << validationRange;
252 if (!excludelistPath.empty() &&
253 !(fs::exists(excludelistPath) && fs::is_regular_file(excludelistPath)))
255 ARMNN_LOG(fatal) <<
"Invalid path to excludelist file at " << excludelistPath;
259 fs::path pathToDataDir(dataDir);
261 validationLabelPath, pathToDataDir.string(), imageBegIndex, imageEndIndex, excludelistPath);
273 std::begin(inputNames),
274 std::end(inputNames));
276 std::begin(outputNames),
277 std::end(outputNames));
288 if (inputLayout ==
"NCHW")
292 else if (inputLayout ==
"NHWC")
298 ARMNN_LOG(fatal) <<
"Invalid Data layout: " << inputLayout;
301 const unsigned int inputTensorWidth =
303 const unsigned int inputTensorHeight =
306 const unsigned int outputNumElements = model.GetOutputSize();
308 if (modelOutputLabels.size() != outputNumElements)
310 ARMNN_LOG(fatal) <<
"Number of output elements: " << outputNumElements
311 <<
" , mismatches the number of output labels: " << modelOutputLabels.size();
315 const unsigned int batchSize = 1;
318 if (modelFormat ==
"tflite")
324 ARMNN_LOG(fatal) <<
"Unsupported frontend: " << modelFormat;
328 for (
const auto& imageEntry : imageNameToLabel)
330 const std::string imageName = imageEntry.first;
331 std::cout <<
"Processing image: " << imageName <<
"\n";
333 vector<armnnUtils::TContainer> inputDataContainers;
334 vector<armnnUtils::TContainer> outputDataContainers;
336 auto imagePath = pathToDataDir / fs::path(imageName);
337 switch (inputTensorDataType)
340 inputDataContainers.push_back(
342 inputTensorWidth, inputTensorHeight,
345 inputTensorDataLayout));
346 outputDataContainers = { vector<int>(outputNumElements) };
349 inputDataContainers.push_back(
351 inputTensorWidth, inputTensorHeight,
354 inputTensorDataLayout));
355 outputDataContainers = { vector<uint8_t>(outputNumElements) };
359 inputDataContainers.push_back(
361 inputTensorWidth, inputTensorHeight,
364 inputTensorDataLayout));
365 outputDataContainers = { vector<float>(outputNumElements) };
369 status = runtime->EnqueueWorkload(networkId,
375 ARMNN_LOG(fatal) <<
"armnn::IRuntime: Failed to enqueue workload for image: " << imageName;
386 for(
unsigned int i = 1; i <= 5; ++i)
388 std::cout <<
"Top " << i <<
" Accuracy: " << checker.GetAccuracy(i) <<
"%" <<
"\n";
391 ARMNN_LOG(info) <<
"Accuracy Tool ran successfully!";
399 std::cerr <<
"Armnn Error: " << e.
what() << std::endl;
402 catch (
const std::exception& e)
405 std::cerr <<
"WARNING: ModelAccuracyTool-Armnn: An error has occurred when running the " 406 "Accuracy Tool: " << e.what() << std::endl;
static IRuntimePtr Create(const CreationOptions &options)
const TensorShape & GetShape() const
void ConfigureLogging(bool printToStandardOutput, bool printToDebugOutput, LogSeverity severity)
Configures the logging behaviour of the ARMNN library.
NormalizationParameters GetNormalizationParameters(const SupportedFrontend &modelFormat, const armnn::DataType &outputType)
Get normalization parameters.
std::unique_ptr< IRuntime, void(*)(IRuntime *runtime)> IRuntimePtr
virtual const char * what() const noexcept override
#define ARMNN_LOG(severity)
BackendRegistry & BackendRegistryInstance()
std::vector< uint8_t > PrepareImageTensor< uint8_t >(const std::string &imagePath, unsigned int newWidth, unsigned int newHeight, const NormalizationParameters &normParams, unsigned int batchSize, const armnn::DataLayout &outputLayout)
armnn::BindingPointInfo BindingPointInfo
std::string GetBackendIdsAsString() const
std::vector< std::string > m_InputBindings
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< TContainer > &inputDataContainers)
std::vector< std::string > SplitBy(const std::string &originalString, const std::string &delimiter, bool includeEmptyToken)
Split a string into tokens by a delimiter.
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
#define ARMNN_ASSERT_MSG(COND, MSG)
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
DataType GetDataType() const
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
std::vector< int > PrepareImageTensor< int >(const std::string &imagePath, unsigned int newWidth, unsigned int newHeight, const NormalizationParameters &normParams, unsigned int batchSize, const armnn::DataLayout &outputLayout)
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, std::vector< TContainer > &outputDataContainers)
#define ARMNN_DEPRECATED_MSG_REMOVAL_DATE(message, removed_in_release)
std::pair< armnn::LayerBindingId, armnn::TensorInfo > BindingPointInfo
armnn::TensorInfo m_TensorInfo
Base class for all ArmNN exceptions so that users can filter to just those.
mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char >, std::vector< int8_t > > TContainer
bool ValidateDirectory(std::string &dir)
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
armnn::LayerBindingId m_BindingId
std::vector< float > PrepareImageTensor< float >(const std::string &imagePath, unsigned int newWidth, unsigned int newHeight, const NormalizationParameters &normParams, unsigned int batchSize, const armnn::DataLayout &outputLayout)