From 898a324d4e5c09e53bbc5925d70577b2f45f753d Mon Sep 17 00:00:00 2001 From: SiCong Li Date: Mon, 24 Jun 2019 16:03:33 +0100 Subject: MLCE-103 Add necessary enhancements to ModelAccuracyTool * Evaluate model accuracy using category names instead of numerical labels. * Add blacklist support * Add range selection support Signed-off-by: SiCong Li Change-Id: I7b1d2d298cfcaa56a27a028147169404b73580bb --- src/armnn/test/ModelAccuracyCheckerTest.cpp | 58 ++++-- src/armnnUtils/ModelAccuracyChecker.cpp | 62 +++++- src/armnnUtils/ModelAccuracyChecker.hpp | 93 ++++++--- .../ModelAccuracyTool-Armnn.cpp | 208 ++++++++++++++++++--- 4 files changed, 337 insertions(+), 84 deletions(-) diff --git a/src/armnn/test/ModelAccuracyCheckerTest.cpp b/src/armnn/test/ModelAccuracyCheckerTest.cpp index f3a6c9d81d..aa1fba212c 100644 --- a/src/armnn/test/ModelAccuracyCheckerTest.cpp +++ b/src/armnn/test/ModelAccuracyCheckerTest.cpp @@ -7,32 +7,50 @@ #include #include -#include -#include -#include #include +#include #include #include +#include +#include using namespace armnnUtils; -struct TestHelper { - const std::map GetValidationLabelSet() +struct TestHelper +{ + const std::map GetValidationLabelSet() { - std::map validationLabelSet; - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000001", 2)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000002", 9)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000003", 1)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000004", 6)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000005", 5)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000006", 0)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000007", 8)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000008", 4)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000009", 3)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000009", 7)); + std::map validationLabelSet; + validationLabelSet.insert(std::make_pair("val_01.JPEG", "goldfinch")); + validationLabelSet.insert(std::make_pair("val_02.JPEG", "magpie")); + validationLabelSet.insert(std::make_pair("val_03.JPEG", "brambling")); + validationLabelSet.insert(std::make_pair("val_04.JPEG", "robin")); + validationLabelSet.insert(std::make_pair("val_05.JPEG", "indigo bird")); + validationLabelSet.insert(std::make_pair("val_06.JPEG", "ostrich")); + validationLabelSet.insert(std::make_pair("val_07.JPEG", "jay")); + validationLabelSet.insert(std::make_pair("val_08.JPEG", "snowbird")); + validationLabelSet.insert(std::make_pair("val_09.JPEG", "house finch")); + validationLabelSet.insert(std::make_pair("val_09.JPEG", "bulbul")); return validationLabelSet; } + const std::vector GetModelOutputLabels() + { + const std::vector modelOutputLabels = + { + {"ostrich", "Struthio camelus"}, + {"brambling", "Fringilla montifringilla"}, + {"goldfinch", "Carduelis carduelis"}, + {"house finch", "linnet", "Carpodacus mexicanus"}, + {"junco", "snowbird"}, + {"indigo bunting", "indigo finch", "indigo bird", "Passerina cyanea"}, + {"robin", "American robin", "Turdus migratorius"}, + {"bulbul"}, + {"jay"}, + {"magpie"} + }; + return modelOutputLabels; + } }; BOOST_AUTO_TEST_SUITE(ModelAccuracyCheckerTest) @@ -41,7 +59,7 @@ using TContainer = boost::variant, std::vector, std::vec BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper) { - ModelAccuracyChecker checker(GetValidationLabelSet()); + ModelAccuracyChecker checker(GetValidationLabelSet(), GetModelOutputLabels()); // Add image 1 and check accuracy std::vector inferenceOutputVector1 = {0.05f, 0.10f, 0.70f, 0.15f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; @@ -49,7 +67,7 @@ BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper) std::vector outputTensor1; outputTensor1.push_back(inference1Container); - std::string imageName = "ILSVRC2012_val_00000001.JPEG"; + std::string imageName = "val_01.JPEG"; checker.AddImageResult(imageName, outputTensor1); // Top 1 Accuracy @@ -62,7 +80,7 @@ BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper) std::vector outputTensor2; outputTensor2.push_back(inference2Container); - imageName = "ILSVRC2012_val_00000002.JPEG"; + imageName = "val_02.JPEG"; checker.AddImageResult(imageName, outputTensor2); // Top 1 Accuracy @@ -79,7 +97,7 @@ BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper) std::vector outputTensor3; outputTensor3.push_back(inference3Container); - imageName = "ILSVRC2012_val_00000003.JPEG"; + imageName = "val_03.JPEG"; checker.AddImageResult(imageName, outputTensor3); // Top 1 Accuracy diff --git a/src/armnnUtils/ModelAccuracyChecker.cpp b/src/armnnUtils/ModelAccuracyChecker.cpp index bee5ca2365..81942dc2be 100644 --- a/src/armnnUtils/ModelAccuracyChecker.cpp +++ b/src/armnnUtils/ModelAccuracyChecker.cpp @@ -3,22 +3,27 @@ // SPDX-License-Identifier: MIT // -#include -#include -#include #include "ModelAccuracyChecker.hpp" +#include +#include +#include +#include namespace armnnUtils { -armnnUtils::ModelAccuracyChecker::ModelAccuracyChecker(const std::map& validationLabels) - : m_GroundTruthLabelSet(validationLabels){} +armnnUtils::ModelAccuracyChecker::ModelAccuracyChecker(const std::map& validationLabels, + const std::vector& modelOutputLabels) + : m_GroundTruthLabelSet(validationLabels) + , m_ModelOutputLabels(modelOutputLabels) +{} float ModelAccuracyChecker::GetAccuracy(unsigned int k) { - if(k > 10) { - BOOST_LOG_TRIVIAL(info) << "Accuracy Tool only supports a maximum of Top 10 Accuracy. " - "Printing Top 10 Accuracy result!"; + if (k > 10) + { + BOOST_LOG_TRIVIAL(warning) << "Accuracy Tool only supports a maximum of Top 10 Accuracy. " + "Printing Top 10 Accuracy result!"; k = 10; } unsigned int total = 0; @@ -28,4 +33,43 @@ float ModelAccuracyChecker::GetAccuracy(unsigned int k) } return static_cast(total * 100) / static_cast(m_ImagesProcessed); } -} \ No newline at end of file + +// Split a string into tokens by a delimiter +std::vector + SplitBy(const std::string& originalString, const std::string& delimiter, bool includeEmptyToken) +{ + std::vector tokens; + size_t cur = 0; + size_t next = 0; + while ((next = originalString.find(delimiter, cur)) != std::string::npos) + { + // Skip empty tokens, unless explicitly stated to include them. + if (next - cur > 0 || includeEmptyToken) + { + tokens.push_back(originalString.substr(cur, next - cur)); + } + cur = next + delimiter.size(); + } + // Get the remaining token + // Skip empty tokens, unless explicitly stated to include them. + if (originalString.size() - cur > 0 || includeEmptyToken) + { + tokens.push_back(originalString.substr(cur, originalString.size() - cur)); + } + return tokens; +} + +// Remove any preceding and trailing character specified in the characterSet. +std::string Strip(const std::string& originalString, const std::string& characterSet) +{ + BOOST_ASSERT(!characterSet.empty()); + const std::size_t firstFound = originalString.find_first_not_of(characterSet); + const std::size_t lastFound = originalString.find_last_not_of(characterSet); + // Return empty if the originalString is empty or the originalString contains only to-be-striped characters + if (firstFound == std::string::npos || lastFound == std::string::npos) + { + return ""; + } + return originalString.substr(firstFound, lastFound + 1 - firstFound); +} +} // namespace armnnUtils \ No newline at end of file diff --git a/src/armnnUtils/ModelAccuracyChecker.hpp b/src/armnnUtils/ModelAccuracyChecker.hpp index cdd2af0ac5..c4dd4f1b05 100644 --- a/src/armnnUtils/ModelAccuracyChecker.hpp +++ b/src/armnnUtils/ModelAccuracyChecker.hpp @@ -5,39 +5,81 @@ #pragma once +#include +#include +#include +#include #include -#include +#include +#include #include +#include #include -#include -#include -#include -#include -#include namespace armnnUtils { using namespace armnn; +// Category names associated with a label +using LabelCategoryNames = std::vector; + +/** Split a string into tokens by a delimiter + * + * @param[in] originalString Original string to be split + * @param[in] delimiter Delimiter used to split \p originalString + * @param[in] includeEmptyToekn If true, include empty tokens in the result + * @return A vector of tokens split from \p originalString by \delimiter + */ +std::vector + SplitBy(const std::string& originalString, const std::string& delimiter = " ", bool includeEmptyToken = false); + +/** Remove any preceding and trailing character specified in the characterSet. + * + * @param[in] originalString Original string to be stripped + * @param[in] characterSet Set of characters to be stripped from \p originalString + * @return A string stripped of all characters specified in \p characterSet from \p originalString + */ +std::string Strip(const std::string& originalString, const std::string& characterSet = " "); + class ModelAccuracyChecker { public: - ModelAccuracyChecker(const std::map& validationLabelSet); - + /** Constructor for a model top k accuracy checker + * + * @param[in] validationLabelSet Mapping from names of images to be validated, to category names of their + corresponding ground-truth labels. + * @param[in] modelOutputLabels Mapping from output nodes to the category names of their corresponding labels + Note that an output node can have multiple category names. + */ + ModelAccuracyChecker(const std::map& validationLabelSet, + const std::vector& modelOutputLabels); + + /** Get Top K accuracy + * + * @param[in] k The number of top predictions to use for validating the ground-truth label. For example, if \p k is + 3, then a prediction is considered correct as long as the ground-truth appears in the top 3 + predictions. + * @return The accuracy, according to the top \p k th predictions. + */ float GetAccuracy(unsigned int k); - template + /** Record the prediction result of an image + * + * @param[in] imageName Name of the image. + * @param[in] outputTensor Output tensor of the network running \p imageName. + */ + template void AddImageResult(const std::string& imageName, std::vector outputTensor) { // Increment the total number of images processed ++m_ImagesProcessed; std::map confidenceMap; - auto & output = outputTensor[0]; + auto& output = outputTensor[0]; // Create a map of all predictions - boost::apply_visitor([&](auto && value) + boost::apply_visitor([&confidenceMap](auto && value) { int index = 0; for (const auto & o : value) @@ -64,8 +106,7 @@ public: std::set, Comparator> setOfPredictions( confidenceMap.begin(), confidenceMap.end(), compFunctor); - std::string trimmedName = GetTrimmedImageName(imageName); - int value = m_GroundTruthLabelSet.find(trimmedName)->second; + const std::string correctLabel = m_GroundTruthLabelSet.at(imageName); unsigned int index = 1; for (std::pair element : setOfPredictions) @@ -74,7 +115,10 @@ public: { break; } - if (element.first == value) + // Check if the ground truth label value is included in the topi prediction. + // Note that a prediction can have multiple prediction labels. + const LabelCategoryNames predictionLabels = m_ModelOutputLabels[static_cast(element.first)]; + if (std::find(predictionLabels.begin(), predictionLabels.end(), correctLabel) != predictionLabels.end()) { ++m_TopK[index]; break; @@ -83,24 +127,11 @@ public: } } - std::string GetTrimmedImageName(const std::string& imageName) const - { - std::string trimmedName; - size_t lastindex = imageName.find_last_of("."); - if(lastindex != std::string::npos) - { - trimmedName = imageName.substr(0, lastindex); - } else - { - trimmedName = imageName; - } - return trimmedName; - } - private: - const std::map m_GroundTruthLabelSet; - std::vector m_TopK = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - unsigned int m_ImagesProcessed = 0; + const std::map m_GroundTruthLabelSet; + const std::vector m_ModelOutputLabels; + std::vector m_TopK = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + unsigned int m_ImagesProcessed = 0; }; } //namespace armnnUtils diff --git a/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp b/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp index 85241e889c..23e2f432a5 100644 --- a/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp +++ b/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp @@ -8,15 +8,43 @@ #include "ModelAccuracyChecker.hpp" #include "armnnDeserializer/IDeserializer.hpp" +#include #include #include #include - #include using namespace armnn::test; -map LoadValidationLabels(const string & validationLabelPath); +/** Load image names and ground-truth labels from the image directory and the ground truth label file + * + * @pre \p validationLabelPath exists and is valid regular file + * @pre \p imageDirectoryPath exists and is valid directory + * @pre labels in validation file correspond to images which are in lexicographical order with the image name + * @pre image index starts at 1 + * @pre \p begIndex and \p endIndex are end-inclusive + * + * @param[in] validationLabelPath Path to validation label file + * @param[in] imageDirectoryPath Path to directory containing validation images + * @param[in] begIndex Begin index of images to be loaded. Inclusive + * @param[in] endIndex End index of images to be loaded. Inclusive + * @param[in] blacklistPath Path to blacklist file + * @return A map mapping image file names to their corresponding ground-truth labels + */ +map LoadValidationImageFilenamesAndLabels(const string& validationLabelPath, + const string& imageDirectoryPath, + size_t begIndex = 0, + size_t endIndex = 0, + const string& blacklistPath = ""); + +/** Load model output labels from file + * + * @pre \p modelOutputLabelsPath exists and is a regular file + * + * @param[in] modelOutputLabelsPath path to model output labels file + * @return A vector of labels, which in turn is described by a list of category names + */ +std::vector LoadModelOutputLabels(const std::string& modelOutputLabelsPath); int main(int argc, char* argv[]) { @@ -38,7 +66,10 @@ int main(int argc, char* argv[]) std::string inputName; std::string inputLayout; std::string outputName; + std::string modelOutputLabelsPath; std::string validationLabelPath; + std::string validationRange; + std::string blacklistPath; const std::string backendsMessage = "Which device to run layers on by default. Possible choices: " + armnn::BackendRegistryInstance().GetBackendIdsAsString(); @@ -58,12 +89,21 @@ int main(int argc, char* argv[]) "Identifier of the output tensors in the network separated by comma.") ("data-dir,d", po::value(&dataDir)->required(), "Path to directory containing the ImageNet test data") + ("model-output-labels,p", po::value(&modelOutputLabelsPath)->required(), + "Path to model output labels file.") ("validation-labels-path,v", po::value(&validationLabelPath)->required(), "Path to ImageNet Validation Label file") ("data-layout,l", po::value(&inputLayout)->default_value("NHWC"), "Data layout. Supported value: NHWC, NCHW. Default: NHCW") ("compute,c", po::value>(&computeDevice)->default_value(defaultBackends), - backendsMessage.c_str()); + backendsMessage.c_str()) + ("validation-range,r", po::value(&validationRange)->default_value("1:0"), + "The range of the images to be evaluated. Specified in the form :." + "The index starts at 1 and the range is inclusive." + "By default the evaluation will be performed on all images.") + ("blacklist-path,b", po::value(&blacklistPath)->default_value(""), + "Path to a blacklist file where each line denotes the index of an image to be " + "excluded from evaluation."); } catch (const std::exception& e) { @@ -156,9 +196,47 @@ int main(int argc, char* argv[]) m_OutputBindingInfo(outputBindingInfo.m_BindingId, outputBindingInfo.m_TensorInfo); std::vector outputBindings = { m_OutputBindingInfo }; + // Load model output labels + if (modelOutputLabelsPath.empty() || !boost::filesystem::exists(modelOutputLabelsPath) || + !boost::filesystem::is_regular_file(modelOutputLabelsPath)) + { + BOOST_LOG_TRIVIAL(fatal) << "Invalid model output labels path at " << modelOutputLabelsPath; + } + const std::vector modelOutputLabels = + LoadModelOutputLabels(modelOutputLabelsPath); + + // Parse begin and end image indices + std::vector imageIndexStrs = armnnUtils::SplitBy(validationRange, ":"); + size_t imageBegIndex; + size_t imageEndIndex; + if (imageIndexStrs.size() != 2) + { + BOOST_LOG_TRIVIAL(fatal) << "Invalid validation range specification: Invalid format " << validationRange; + return 1; + } + try + { + imageBegIndex = std::stoul(imageIndexStrs[0]); + imageEndIndex = std::stoul(imageIndexStrs[1]); + } + catch (const std::exception& e) + { + BOOST_LOG_TRIVIAL(fatal) << "Invalid validation range specification: " << validationRange; + return 1; + } + + // Validate blacklist file if it's specified + if (!blacklistPath.empty() && + !(boost::filesystem::exists(blacklistPath) && boost::filesystem::is_regular_file(blacklistPath))) + { + BOOST_LOG_TRIVIAL(fatal) << "Invalid path to blacklist file at " << blacklistPath; + return 1; + } + path pathToDataDir(dataDir); - map validationLabels = LoadValidationLabels(validationLabelPath); - armnnUtils::ModelAccuracyChecker checker(validationLabels); + const map imageNameToLabel = LoadValidationImageFilenamesAndLabels( + validationLabelPath, pathToDataDir.string(), imageBegIndex, imageEndIndex, blacklistPath); + armnnUtils::ModelAccuracyChecker checker(imageNameToLabel, modelOutputLabels); using TContainer = boost::variant, std::vector, std::vector>; if (ValidateDirectory(dataDir)) @@ -196,6 +274,13 @@ int main(int argc, char* argv[]) inputTensorDataLayout == armnn::DataLayout::NCHW ? inputTensorShape[2] : inputTensorShape[1]; // Get output tensor info const unsigned int outputNumElements = model.GetOutputSize(); + // Check output tensor shape is valid + if (modelOutputLabels.size() != outputNumElements) + { + BOOST_LOG_TRIVIAL(fatal) << "Number of output elements: " << outputNumElements + << " , mismatches the number of output labels: " << modelOutputLabels.size(); + return 1; + } const unsigned int batchSize = 1; // Get normalisation parameters @@ -218,19 +303,20 @@ int main(int argc, char* argv[]) return 1; } const NormalizationParameters& normParams = GetNormalizationParameters(modelFrontend, inputTensorDataType); - for (auto& imageEntry : boost::make_iterator_range(directory_iterator(pathToDataDir), {})) + for (const auto& imageEntry : imageNameToLabel) { - cout << "Processing image: " << imageEntry << "\n"; + const std::string imageName = imageEntry.first; + std::cout << "Processing image: " << imageName << "\n"; vector inputDataContainers; vector outputDataContainers; - const string& imagePath = imageEntry.path().string(); + auto imagePath = pathToDataDir / boost::filesystem::path(imageName); switch (inputTensorDataType) { case armnn::DataType::Signed32: inputDataContainers.push_back( - PrepareImageTensor(imagePath, + PrepareImageTensor(imagePath.string(), inputTensorWidth, inputTensorHeight, normParams, batchSize, @@ -239,7 +325,7 @@ int main(int argc, char* argv[]) break; case armnn::DataType::QuantisedAsymm8: inputDataContainers.push_back( - PrepareImageTensor(imagePath, + PrepareImageTensor(imagePath.string(), inputTensorWidth, inputTensorHeight, normParams, batchSize, @@ -249,7 +335,7 @@ int main(int argc, char* argv[]) case armnn::DataType::Float32: default: inputDataContainers.push_back( - PrepareImageTensor(imagePath, + PrepareImageTensor(imagePath.string(), inputTensorWidth, inputTensorHeight, normParams, batchSize, @@ -264,10 +350,9 @@ int main(int argc, char* argv[]) if (status == armnn::Status::Failure) { - BOOST_LOG_TRIVIAL(fatal) << "armnn::IRuntime: Failed to enqueue workload for image: " << imageEntry; + BOOST_LOG_TRIVIAL(fatal) << "armnn::IRuntime: Failed to enqueue workload for image: " << imageName; } - const std::string imageName = imageEntry.path().filename().string(); checker.AddImageResult(imageName, outputDataContainers); } } @@ -301,21 +386,96 @@ int main(int argc, char* argv[]) } } -map LoadValidationLabels(const string & validationLabelPath) +map LoadValidationImageFilenamesAndLabels(const string& validationLabelPath, + const string& imageDirectoryPath, + size_t begIndex, + size_t endIndex, + const string& blacklistPath) { - std::string imageName; - int classification; - map validationLabel; + // Populate imageFilenames with names of all .JPEG, .PNG images + std::vector imageFilenames; + for (const auto& imageEntry : + boost::make_iterator_range(boost::filesystem::directory_iterator(boost::filesystem::path(imageDirectoryPath)))) + { + boost::filesystem::path imagePath = imageEntry.path(); + std::string imageExtension = boost::to_upper_copy(imagePath.extension().string()); + if (boost::filesystem::is_regular_file(imagePath) && (imageExtension == ".JPEG" || imageExtension == ".PNG")) + { + imageFilenames.push_back(imagePath.filename().string()); + } + } + if (imageFilenames.empty()) + { + throw armnn::Exception("No image file (JPEG, PNG) found at " + imageDirectoryPath); + } + + // Sort the image filenames lexicographically + std::sort(imageFilenames.begin(), imageFilenames.end()); + + std::cout << imageFilenames.size() << " images found at " << imageDirectoryPath << std::endl; + + // Get default end index + if (begIndex < 1 || endIndex > imageFilenames.size()) + { + throw armnn::Exception("Invalid image index range"); + } + endIndex = endIndex == 0 ? imageFilenames.size() : endIndex; + if (begIndex > endIndex) + { + throw armnn::Exception("Invalid image index range"); + } + + // Load blacklist if there is one + std::vector blacklist; + if (!blacklistPath.empty()) + { + std::ifstream blacklistFile(blacklistPath); + unsigned int index; + while (blacklistFile >> index) + { + blacklist.push_back(index); + } + } + + // Load ground truth labels and pair them with corresponding image names + std::string classification; + map imageNameToLabel; ifstream infile(validationLabelPath); - while (infile >> imageName >> classification) + size_t imageIndex = begIndex; + size_t blacklistIndexCount = 0; + while (std::getline(infile, classification)) { - std::string trimmedName; - size_t lastindex = imageName.find_last_of("."); - if(lastindex != std::string::npos) + if (imageIndex > endIndex) { - trimmedName = imageName.substr(0, lastindex); + break; } - validationLabel.insert(pair(trimmedName, classification)); + // If current imageIndex is included in blacklist, skip the current image + if (blacklistIndexCount < blacklist.size() && imageIndex == blacklist[blacklistIndexCount]) + { + ++imageIndex; + ++blacklistIndexCount; + continue; + } + imageNameToLabel.insert(std::pair(imageFilenames[imageIndex - 1], classification)); + ++imageIndex; } - return validationLabel; + std::cout << blacklistIndexCount << " images blacklisted" << std::endl; + std::cout << imageIndex - begIndex - blacklistIndexCount << " images to be loaded" << std::endl; + return imageNameToLabel; } + +std::vector LoadModelOutputLabels(const std::string& modelOutputLabelsPath) +{ + std::vector modelOutputLabels; + ifstream modelOutputLablesFile(modelOutputLabelsPath); + std::string line; + while (std::getline(modelOutputLablesFile, line)) + { + armnnUtils::LabelCategoryNames tokens = armnnUtils::SplitBy(line, ":"); + armnnUtils::LabelCategoryNames predictionCategoryNames = armnnUtils::SplitBy(tokens.back(), ","); + std::transform(predictionCategoryNames.begin(), predictionCategoryNames.end(), predictionCategoryNames.begin(), + [](const std::string& category) { return armnnUtils::Strip(category); }); + modelOutputLabels.push_back(predictionCategoryNames); + } + return modelOutputLabels; +} \ No newline at end of file -- cgit v1.2.1