diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/armnn/test/ModelAccuracyCheckerTest.cpp | 58 | ||||
-rw-r--r-- | src/armnnUtils/ModelAccuracyChecker.cpp | 62 | ||||
-rw-r--r-- | src/armnnUtils/ModelAccuracyChecker.hpp | 93 |
3 files changed, 153 insertions, 60 deletions
diff --git a/src/armnn/test/ModelAccuracyCheckerTest.cpp b/src/armnn/test/ModelAccuracyCheckerTest.cpp index f3a6c9d81d..aa1fba212c 100644 --- a/src/armnn/test/ModelAccuracyCheckerTest.cpp +++ b/src/armnn/test/ModelAccuracyCheckerTest.cpp @@ -7,32 +7,50 @@ #include <boost/algorithm/string.hpp> #include <boost/test/unit_test.hpp> -#include <iostream> -#include <string> -#include <boost/log/core/core.hpp> #include <boost/filesystem.hpp> +#include <boost/log/core/core.hpp> #include <boost/optional.hpp> #include <boost/variant.hpp> +#include <iostream> +#include <string> using namespace armnnUtils; -struct TestHelper { - const std::map<std::string, int> GetValidationLabelSet() +struct TestHelper +{ + const std::map<std::string, std::string> GetValidationLabelSet() { - std::map<std::string, int> validationLabelSet; - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000001", 2)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000002", 9)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000003", 1)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000004", 6)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000005", 5)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000006", 0)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000007", 8)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000008", 4)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000009", 3)); - validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000009", 7)); + std::map<std::string, std::string> validationLabelSet; + validationLabelSet.insert(std::make_pair("val_01.JPEG", "goldfinch")); + validationLabelSet.insert(std::make_pair("val_02.JPEG", "magpie")); + validationLabelSet.insert(std::make_pair("val_03.JPEG", "brambling")); + validationLabelSet.insert(std::make_pair("val_04.JPEG", "robin")); + validationLabelSet.insert(std::make_pair("val_05.JPEG", "indigo bird")); + validationLabelSet.insert(std::make_pair("val_06.JPEG", "ostrich")); + validationLabelSet.insert(std::make_pair("val_07.JPEG", "jay")); + validationLabelSet.insert(std::make_pair("val_08.JPEG", "snowbird")); + validationLabelSet.insert(std::make_pair("val_09.JPEG", "house finch")); + validationLabelSet.insert(std::make_pair("val_09.JPEG", "bulbul")); return validationLabelSet; } + const std::vector<armnnUtils::LabelCategoryNames> GetModelOutputLabels() + { + const std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels = + { + {"ostrich", "Struthio camelus"}, + {"brambling", "Fringilla montifringilla"}, + {"goldfinch", "Carduelis carduelis"}, + {"house finch", "linnet", "Carpodacus mexicanus"}, + {"junco", "snowbird"}, + {"indigo bunting", "indigo finch", "indigo bird", "Passerina cyanea"}, + {"robin", "American robin", "Turdus migratorius"}, + {"bulbul"}, + {"jay"}, + {"magpie"} + }; + return modelOutputLabels; + } }; BOOST_AUTO_TEST_SUITE(ModelAccuracyCheckerTest) @@ -41,7 +59,7 @@ using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vec BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper) { - ModelAccuracyChecker checker(GetValidationLabelSet()); + ModelAccuracyChecker checker(GetValidationLabelSet(), GetModelOutputLabels()); // Add image 1 and check accuracy std::vector<float> inferenceOutputVector1 = {0.05f, 0.10f, 0.70f, 0.15f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; @@ -49,7 +67,7 @@ BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper) std::vector<TContainer> outputTensor1; outputTensor1.push_back(inference1Container); - std::string imageName = "ILSVRC2012_val_00000001.JPEG"; + std::string imageName = "val_01.JPEG"; checker.AddImageResult<TContainer>(imageName, outputTensor1); // Top 1 Accuracy @@ -62,7 +80,7 @@ BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper) std::vector<TContainer> outputTensor2; outputTensor2.push_back(inference2Container); - imageName = "ILSVRC2012_val_00000002.JPEG"; + imageName = "val_02.JPEG"; checker.AddImageResult<TContainer>(imageName, outputTensor2); // Top 1 Accuracy @@ -79,7 +97,7 @@ BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper) std::vector<TContainer> outputTensor3; outputTensor3.push_back(inference3Container); - imageName = "ILSVRC2012_val_00000003.JPEG"; + imageName = "val_03.JPEG"; checker.AddImageResult<TContainer>(imageName, outputTensor3); // Top 1 Accuracy diff --git a/src/armnnUtils/ModelAccuracyChecker.cpp b/src/armnnUtils/ModelAccuracyChecker.cpp index bee5ca2365..81942dc2be 100644 --- a/src/armnnUtils/ModelAccuracyChecker.cpp +++ b/src/armnnUtils/ModelAccuracyChecker.cpp @@ -3,22 +3,27 @@ // SPDX-License-Identifier: MIT // -#include <vector> -#include <map> -#include <boost/log/trivial.hpp> #include "ModelAccuracyChecker.hpp" +#include <boost/filesystem.hpp> +#include <boost/log/trivial.hpp> +#include <map> +#include <vector> namespace armnnUtils { -armnnUtils::ModelAccuracyChecker::ModelAccuracyChecker(const std::map<std::string, int>& validationLabels) - : m_GroundTruthLabelSet(validationLabels){} +armnnUtils::ModelAccuracyChecker::ModelAccuracyChecker(const std::map<std::string, std::string>& validationLabels, + const std::vector<LabelCategoryNames>& modelOutputLabels) + : m_GroundTruthLabelSet(validationLabels) + , m_ModelOutputLabels(modelOutputLabels) +{} float ModelAccuracyChecker::GetAccuracy(unsigned int k) { - if(k > 10) { - BOOST_LOG_TRIVIAL(info) << "Accuracy Tool only supports a maximum of Top 10 Accuracy. " - "Printing Top 10 Accuracy result!"; + if (k > 10) + { + BOOST_LOG_TRIVIAL(warning) << "Accuracy Tool only supports a maximum of Top 10 Accuracy. " + "Printing Top 10 Accuracy result!"; k = 10; } unsigned int total = 0; @@ -28,4 +33,43 @@ float ModelAccuracyChecker::GetAccuracy(unsigned int k) } return static_cast<float>(total * 100) / static_cast<float>(m_ImagesProcessed); } -}
\ No newline at end of file + +// Split a string into tokens by a delimiter +std::vector<std::string> + SplitBy(const std::string& originalString, const std::string& delimiter, bool includeEmptyToken) +{ + std::vector<std::string> tokens; + size_t cur = 0; + size_t next = 0; + while ((next = originalString.find(delimiter, cur)) != std::string::npos) + { + // Skip empty tokens, unless explicitly stated to include them. + if (next - cur > 0 || includeEmptyToken) + { + tokens.push_back(originalString.substr(cur, next - cur)); + } + cur = next + delimiter.size(); + } + // Get the remaining token + // Skip empty tokens, unless explicitly stated to include them. + if (originalString.size() - cur > 0 || includeEmptyToken) + { + tokens.push_back(originalString.substr(cur, originalString.size() - cur)); + } + return tokens; +} + +// Remove any preceding and trailing character specified in the characterSet. +std::string Strip(const std::string& originalString, const std::string& characterSet) +{ + BOOST_ASSERT(!characterSet.empty()); + const std::size_t firstFound = originalString.find_first_not_of(characterSet); + const std::size_t lastFound = originalString.find_last_not_of(characterSet); + // Return empty if the originalString is empty or the originalString contains only to-be-striped characters + if (firstFound == std::string::npos || lastFound == std::string::npos) + { + return ""; + } + return originalString.substr(firstFound, lastFound + 1 - firstFound); +} +} // namespace armnnUtils
\ No newline at end of file diff --git a/src/armnnUtils/ModelAccuracyChecker.hpp b/src/armnnUtils/ModelAccuracyChecker.hpp index cdd2af0ac5..c4dd4f1b05 100644 --- a/src/armnnUtils/ModelAccuracyChecker.hpp +++ b/src/armnnUtils/ModelAccuracyChecker.hpp @@ -5,39 +5,81 @@ #pragma once +#include <algorithm> +#include <armnn/Types.hpp> +#include <boost/assert.hpp> +#include <boost/variant/apply_visitor.hpp> #include <cstddef> -#include <string> +#include <functional> +#include <iostream> #include <map> +#include <string> #include <vector> -#include <boost/variant/apply_visitor.hpp> -#include <iostream> -#include <armnn/Types.hpp> -#include <functional> -#include <algorithm> namespace armnnUtils { using namespace armnn; +// Category names associated with a label +using LabelCategoryNames = std::vector<std::string>; + +/** Split a string into tokens by a delimiter + * + * @param[in] originalString Original string to be split + * @param[in] delimiter Delimiter used to split \p originalString + * @param[in] includeEmptyToekn If true, include empty tokens in the result + * @return A vector of tokens split from \p originalString by \delimiter + */ +std::vector<std::string> + SplitBy(const std::string& originalString, const std::string& delimiter = " ", bool includeEmptyToken = false); + +/** Remove any preceding and trailing character specified in the characterSet. + * + * @param[in] originalString Original string to be stripped + * @param[in] characterSet Set of characters to be stripped from \p originalString + * @return A string stripped of all characters specified in \p characterSet from \p originalString + */ +std::string Strip(const std::string& originalString, const std::string& characterSet = " "); + class ModelAccuracyChecker { public: - ModelAccuracyChecker(const std::map<std::string, int>& validationLabelSet); - + /** Constructor for a model top k accuracy checker + * + * @param[in] validationLabelSet Mapping from names of images to be validated, to category names of their + corresponding ground-truth labels. + * @param[in] modelOutputLabels Mapping from output nodes to the category names of their corresponding labels + Note that an output node can have multiple category names. + */ + ModelAccuracyChecker(const std::map<std::string, std::string>& validationLabelSet, + const std::vector<LabelCategoryNames>& modelOutputLabels); + + /** Get Top K accuracy + * + * @param[in] k The number of top predictions to use for validating the ground-truth label. For example, if \p k is + 3, then a prediction is considered correct as long as the ground-truth appears in the top 3 + predictions. + * @return The accuracy, according to the top \p k th predictions. + */ float GetAccuracy(unsigned int k); - template<typename TContainer> + /** Record the prediction result of an image + * + * @param[in] imageName Name of the image. + * @param[in] outputTensor Output tensor of the network running \p imageName. + */ + template <typename TContainer> void AddImageResult(const std::string& imageName, std::vector<TContainer> outputTensor) { // Increment the total number of images processed ++m_ImagesProcessed; std::map<int, float> confidenceMap; - auto & output = outputTensor[0]; + auto& output = outputTensor[0]; // Create a map of all predictions - boost::apply_visitor([&](auto && value) + boost::apply_visitor([&confidenceMap](auto && value) { int index = 0; for (const auto & o : value) @@ -64,8 +106,7 @@ public: std::set<std::pair<int, float>, Comparator> setOfPredictions( confidenceMap.begin(), confidenceMap.end(), compFunctor); - std::string trimmedName = GetTrimmedImageName(imageName); - int value = m_GroundTruthLabelSet.find(trimmedName)->second; + const std::string correctLabel = m_GroundTruthLabelSet.at(imageName); unsigned int index = 1; for (std::pair<int, float> element : setOfPredictions) @@ -74,7 +115,10 @@ public: { break; } - if (element.first == value) + // Check if the ground truth label value is included in the topi prediction. + // Note that a prediction can have multiple prediction labels. + const LabelCategoryNames predictionLabels = m_ModelOutputLabels[static_cast<size_t>(element.first)]; + if (std::find(predictionLabels.begin(), predictionLabels.end(), correctLabel) != predictionLabels.end()) { ++m_TopK[index]; break; @@ -83,24 +127,11 @@ public: } } - std::string GetTrimmedImageName(const std::string& imageName) const - { - std::string trimmedName; - size_t lastindex = imageName.find_last_of("."); - if(lastindex != std::string::npos) - { - trimmedName = imageName.substr(0, lastindex); - } else - { - trimmedName = imageName; - } - return trimmedName; - } - private: - const std::map<std::string, int> m_GroundTruthLabelSet; - std::vector<unsigned int> m_TopK = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - unsigned int m_ImagesProcessed = 0; + const std::map<std::string, std::string> m_GroundTruthLabelSet; + const std::vector<LabelCategoryNames> m_ModelOutputLabels; + std::vector<unsigned int> m_TopK = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + unsigned int m_ImagesProcessed = 0; }; } //namespace armnnUtils |