6 #include "../ImageTensorGenerator/ImageTensorGenerator.hpp" 7 #include "../InferenceTest.hpp" 12 #include <cxxopts/cxxopts.hpp> 33 const string& imageDirectoryPath,
36 const string& blacklistPath =
"");
45 std::vector<armnnUtils::LabelCategoryNames>
LoadModelOutputLabels(
const std::string& modelOutputLabelsPath);
47 int main(
int argc,
char* argv[])
54 std::string modelPath;
55 std::string modelFormat;
56 std::vector<std::string> inputNames;
57 std::vector<std::string> outputNames;
59 std::string modelOutputLabelsPath;
60 std::string validationLabelPath;
61 std::string inputLayout;
62 std::vector<armnn::BackendId> computeDevice;
63 std::string validationRange;
64 std::string blacklistPath;
66 const std::string backendsMessage =
"Which device to run layers on by default. Possible choices: " 71 cxxopts::Options options(
"ModeAccuracyTool-Armnn",
"Options");
74 (
"h,help",
"Display help messages")
76 "Path to armnn format model file",
77 cxxopts::value<std::string>(modelPath))
79 "The model format. Supported values: tflite",
80 cxxopts::value<std::string>(modelFormat))
82 "Identifier of the input tensors in the network separated by comma with no space.",
83 cxxopts::value<std::vector<std::string>>(inputNames))
85 "Identifier of the output tensors in the network separated by comma with no space.",
86 cxxopts::value<std::vector<std::string>>(outputNames))
88 "Path to directory containing the ImageNet test data",
89 cxxopts::value<std::string>(dataDir))
90 (
"p,model-output-labels",
91 "Path to model output labels file.",
92 cxxopts::value<std::string>(modelOutputLabelsPath))
93 (
"v,validation-labels-path",
94 "Path to ImageNet Validation Label file",
95 cxxopts::value<std::string>(validationLabelPath))
97 "Data layout. Supported value: NHWC, NCHW. Default: NHWC",
98 cxxopts::value<std::string>(inputLayout)->default_value(
"NHWC"))
100 backendsMessage.c_str(),
101 cxxopts::value<std::vector<armnn::BackendId>>(computeDevice)->default_value(
"CpuAcc,CpuRef"))
102 (
"r,validation-range",
103 "The range of the images to be evaluated. Specified in the form <begin index>:<end index>." 104 "The index starts at 1 and the range is inclusive." 105 "By default the evaluation will be performed on all images.",
106 cxxopts::value<std::string>(validationRange)->default_value(
"1:0"))
108 "Path to a blacklist file where each line denotes the index of an image to be " 109 "excluded from evaluation.",
110 cxxopts::value<std::string>(blacklistPath)->default_value(
""));
112 auto result = options.parse(argc, argv);
114 if (result.count(
"help") > 0)
116 std::cout << options.help() << std::endl;
121 std::string mandatorySingleParameters[] = {
"model-path",
"model-format",
"input-name",
"output-name",
122 "data-dir",
"model-output-labels",
"validation-labels-path" };
123 for (
auto param : mandatorySingleParameters)
125 if (result.count(param) != 1)
127 std::cerr <<
"Parameter \'--" << param <<
"\' is required but missing." << std::endl;
132 catch (
const cxxopts::OptionException& e)
134 std::cerr << e.what() << std::endl << std::endl;
137 catch (
const std::exception& e)
140 std::cerr <<
"Fatal internal error: " << e.what() << std::endl;
145 std::string invalidBackends;
148 ARMNN_LOG(fatal) <<
"The list of preferred devices contains invalid backend IDs: " 157 std::ifstream file(modelPath);
161 auto armnnparser(IParser::Create());
170 optimizedNet =
armnn::Optimize(*network, computeDevice, runtime->GetDeviceSpec());
174 std::stringstream message;
175 message <<
"armnn::Exception (" << e.
what() <<
") caught from optimize.";
182 status = runtime->LoadNetwork(networkId, std::move(optimizedNet));
185 ARMNN_LOG(fatal) <<
"armnn::IRuntime: Failed to load network";
193 std::vector<BindingPointInfo> inputBindings;
194 for(
auto& input: inputNames)
197 inputBindingInfo = armnnparser->GetNetworkInputBindingInfo(0, input);
199 std::pair<armnn::LayerBindingId, armnn::TensorInfo>
201 inputBindings.push_back(m_InputBindingInfo);
204 std::vector<BindingPointInfo> outputBindings;
205 for(
auto& output: outputNames)
208 outputBindingInfo = armnnparser->GetNetworkOutputBindingInfo(0, output);
210 std::pair<armnn::LayerBindingId, armnn::TensorInfo>
212 outputBindings.push_back(m_OutputBindingInfo);
216 if (modelOutputLabelsPath.empty() || !fs::exists(modelOutputLabelsPath) ||
217 !fs::is_regular_file(modelOutputLabelsPath))
219 ARMNN_LOG(fatal) <<
"Invalid model output labels path at " << modelOutputLabelsPath;
221 const std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels =
226 size_t imageBegIndex;
227 size_t imageEndIndex;
228 if (imageIndexStrs.size() != 2)
230 ARMNN_LOG(fatal) <<
"Invalid validation range specification: Invalid format " << validationRange;
235 imageBegIndex = std::stoul(imageIndexStrs[0]);
236 imageEndIndex = std::stoul(imageIndexStrs[1]);
238 catch (
const std::exception& e)
240 ARMNN_LOG(fatal) <<
"Invalid validation range specification: " << validationRange;
245 if (!blacklistPath.empty() &&
246 !(fs::exists(blacklistPath) && fs::is_regular_file(blacklistPath)))
248 ARMNN_LOG(fatal) <<
"Invalid path to blacklist file at " << blacklistPath;
252 fs::path pathToDataDir(dataDir);
254 validationLabelPath, pathToDataDir.string(), imageBegIndex, imageEndIndex, blacklistPath);
256 using TContainer = mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<uint8_t>>;
267 std::begin(inputNames),
268 std::end(inputNames));
270 std::begin(outputNames),
271 std::end(outputNames));
282 if (inputLayout ==
"NCHW")
286 else if (inputLayout ==
"NHWC")
292 ARMNN_LOG(fatal) <<
"Invalid Data layout: " << inputLayout;
295 const unsigned int inputTensorWidth =
297 const unsigned int inputTensorHeight =
300 const unsigned int outputNumElements = model.
GetOutputSize();
302 if (modelOutputLabels.size() != outputNumElements)
304 ARMNN_LOG(fatal) <<
"Number of output elements: " << outputNumElements
305 <<
" , mismatches the number of output labels: " << modelOutputLabels.size();
309 const unsigned int batchSize = 1;
312 if (modelFormat ==
"tflite")
318 ARMNN_LOG(fatal) <<
"Unsupported frontend: " << modelFormat;
322 for (
const auto& imageEntry : imageNameToLabel)
324 const std::string imageName = imageEntry.first;
325 std::cout <<
"Processing image: " << imageName <<
"\n";
327 vector<TContainer> inputDataContainers;
328 vector<TContainer> outputDataContainers;
330 auto imagePath = pathToDataDir / fs::path(imageName);
331 switch (inputTensorDataType)
334 inputDataContainers.push_back(
336 inputTensorWidth, inputTensorHeight,
339 inputTensorDataLayout));
340 outputDataContainers = { vector<int>(outputNumElements) };
343 inputDataContainers.push_back(
345 inputTensorWidth, inputTensorHeight,
348 inputTensorDataLayout));
349 outputDataContainers = { vector<uint8_t>(outputNumElements) };
353 inputDataContainers.push_back(
355 inputTensorWidth, inputTensorHeight,
358 inputTensorDataLayout));
359 outputDataContainers = { vector<float>(outputNumElements) };
363 status = runtime->EnqueueWorkload(networkId,
369 ARMNN_LOG(fatal) <<
"armnn::IRuntime: Failed to enqueue workload for image: " << imageName;
372 checker.AddImageResult<
TContainer>(imageName, outputDataContainers);
380 for(
unsigned int i = 1; i <= 5; ++i)
382 std::cout <<
"Top " << i <<
" Accuracy: " << checker.GetAccuracy(i) <<
"%" <<
"\n";
385 ARMNN_LOG(info) <<
"Accuracy Tool ran successfully!";
393 std::cerr <<
"Armnn Error: " << e.
what() << std::endl;
396 catch (
const std::exception& e)
399 std::cerr <<
"WARNING: ModelAccuracyTool-Armnn: An error has occurred when running the " 400 "Accuracy Tool: " << e.what() << std::endl;
406 const string& imageDirectoryPath,
409 const string& blacklistPath)
412 std::vector<std::string> imageFilenames;
413 for (
const auto& imageEntry : fs::directory_iterator(fs::path(imageDirectoryPath)))
415 fs::path imagePath = imageEntry.path();
418 std::string imageExtension = imagePath.extension().string();
419 std::transform(imageExtension.begin(), imageExtension.end(), imageExtension.begin(), ::toupper);
421 if (fs::is_regular_file(imagePath) && (imageExtension ==
".JPEG" || imageExtension ==
".PNG"))
423 imageFilenames.push_back(imagePath.filename().string());
426 if (imageFilenames.empty())
428 throw armnn::Exception(
"No image file (JPEG, PNG) found at " + imageDirectoryPath);
432 std::sort(imageFilenames.begin(), imageFilenames.end());
434 std::cout << imageFilenames.size() <<
" images found at " << imageDirectoryPath << std::endl;
437 if (begIndex < 1 || endIndex > imageFilenames.size())
441 endIndex = endIndex == 0 ? imageFilenames.size() : endIndex;
442 if (begIndex > endIndex)
448 std::vector<unsigned int> blacklist;
449 if (!blacklistPath.empty())
451 std::ifstream blacklistFile(blacklistPath);
453 while (blacklistFile >> index)
455 blacklist.push_back(index);
460 std::string classification;
461 map<std::string, std::string> imageNameToLabel;
462 ifstream infile(validationLabelPath);
463 size_t imageIndex = begIndex;
464 size_t blacklistIndexCount = 0;
465 while (std::getline(infile, classification))
467 if (imageIndex > endIndex)
472 if (blacklistIndexCount < blacklist.size() && imageIndex == blacklist[blacklistIndexCount])
475 ++blacklistIndexCount;
478 imageNameToLabel.insert(std::pair<std::string, std::string>(imageFilenames[imageIndex - 1], classification));
481 std::cout << blacklistIndexCount <<
" images blacklisted" << std::endl;
482 std::cout << imageIndex - begIndex - blacklistIndexCount <<
" images to be loaded" << std::endl;
483 return imageNameToLabel;
488 std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels;
489 ifstream modelOutputLablesFile(modelOutputLabelsPath);
491 while (std::getline(modelOutputLablesFile, line))
495 std::transform(predictionCategoryNames.begin(), predictionCategoryNames.end(), predictionCategoryNames.begin(),
497 modelOutputLabels.push_back(predictionCategoryNames);
499 return modelOutputLabels;
static IRuntimePtr Create(const CreationOptions &options)
const TensorShape & GetShape() const
void ConfigureLogging(bool printToStandardOutput, bool printToDebugOutput, LogSeverity severity)
Configures the logging behaviour of the ARMNN library.
NormalizationParameters GetNormalizationParameters(const SupportedFrontend &modelFormat, const armnn::DataType &outputType)
Get normalization parameters.
std::unique_ptr< IRuntime, void(*)(IRuntime *runtime)> IRuntimePtr
virtual const char * what() const noexcept override
#define ARMNN_LOG(severity)
BackendRegistry & BackendRegistryInstance()
const armnn::BindingPointInfo & GetInputBindingInfo(unsigned int inputIndex=0u) const
std::vector< uint8_t > PrepareImageTensor< uint8_t >(const std::string &imagePath, unsigned int newWidth, unsigned int newHeight, const NormalizationParameters &normParams, unsigned int batchSize, const armnn::DataLayout &outputLayout)
armnn::BindingPointInfo BindingPointInfo
std::string GetBackendIdsAsString() const
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
std::vector< std::string > m_InputBindings
std::string Strip(const std::string &originalString, const std::string &characterSet)
Remove any preceding and trailing character specified in the characterSet.
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< TContainer > &inputDataContainers)
std::vector< std::string > SplitBy(const std::string &originalString, const std::string &delimiter, bool includeEmptyToken)
Split a string into tokens by a delimiter.
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
#define ARMNN_ASSERT_MSG(COND, MSG)
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
DataType GetDataType() const
mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char >, std::vector< int8_t > > TContainer
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
std::vector< int > PrepareImageTensor< int >(const std::string &imagePath, unsigned int newWidth, unsigned int newHeight, const NormalizationParameters &normParams, unsigned int batchSize, const armnn::DataLayout &outputLayout)
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, std::vector< TContainer > &outputDataContainers)
std::pair< armnn::LayerBindingId, armnn::TensorInfo > BindingPointInfo
armnn::TensorInfo m_TensorInfo
Base class for all ArmNN exceptions so that users can filter to just those.
bool ValidateDirectory(std::string &dir)
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
armnn::LayerBindingId m_BindingId
std::vector< std::string > LabelCategoryNames
std::vector< float > PrepareImageTensor< float >(const std::string &imagePath, unsigned int newWidth, unsigned int newHeight, const NormalizationParameters &normParams, unsigned int batchSize, const armnn::DataLayout &outputLayout)