diff options
author | Éanna Ó Catháin <eanna.ocathain@arm.com> | 2019-05-08 14:00:45 +0100 |
---|---|---|
committer | Eanna O Cathain Arm <eanna.ocathain@arm.com> | 2019-05-08 15:23:27 +0000 |
commit | a4247d5a50502811a6956dffd990c0254622b7e1 (patch) | |
tree | a2e8742695673bc8e958cce316e6ddeafcc59642 /src | |
parent | c2fe5fb3a070ce2c7daebf63d0def3d57cec09d3 (diff) | |
download | armnn-a4247d5a50502811a6956dffd990c0254622b7e1.tar.gz |
IVGCVSW-2900 Adding the Accuracy Checker Tool and tests
Change-Id: I4ac325e45f2236b8e0757d21046f117024ce3979
Signed-off-by: Éanna Ó Catháin <eanna.ocathain@arm.com>
Diffstat (limited to 'src')
-rw-r--r-- | src/armnn/test/ModelAccuracyCheckerTest.cpp | 98 | ||||
-rw-r--r-- | src/armnnUtils/ModelAccuracyChecker.cpp | 31 | ||||
-rw-r--r-- | src/armnnUtils/ModelAccuracyChecker.hpp | 103 |
3 files changed, 232 insertions, 0 deletions
diff --git a/src/armnn/test/ModelAccuracyCheckerTest.cpp b/src/armnn/test/ModelAccuracyCheckerTest.cpp new file mode 100644 index 0000000000..f3a6c9d81d --- /dev/null +++ b/src/armnn/test/ModelAccuracyCheckerTest.cpp @@ -0,0 +1,98 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// +#include "ModelAccuracyChecker.hpp" + +#include <boost/algorithm/string.hpp> +#include <boost/test/unit_test.hpp> + +#include <iostream> +#include <string> +#include <boost/log/core/core.hpp> +#include <boost/filesystem.hpp> +#include <boost/optional.hpp> +#include <boost/variant.hpp> + +using namespace armnnUtils; + +struct TestHelper { + const std::map<std::string, int> GetValidationLabelSet() + { + std::map<std::string, int> validationLabelSet; + validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000001", 2)); + validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000002", 9)); + validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000003", 1)); + validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000004", 6)); + validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000005", 5)); + validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000006", 0)); + validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000007", 8)); + validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000008", 4)); + validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000009", 3)); + validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000009", 7)); + + return validationLabelSet; + } +}; + +BOOST_AUTO_TEST_SUITE(ModelAccuracyCheckerTest) + +using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>; + +BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper) +{ + ModelAccuracyChecker checker(GetValidationLabelSet()); + + // Add image 1 and check accuracy + std::vector<float> inferenceOutputVector1 = {0.05f, 0.10f, 0.70f, 0.15f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + TContainer inference1Container(inferenceOutputVector1); + std::vector<TContainer> outputTensor1; + outputTensor1.push_back(inference1Container); + + std::string imageName = "ILSVRC2012_val_00000001.JPEG"; + checker.AddImageResult<TContainer>(imageName, outputTensor1); + + // Top 1 Accuracy + float totalAccuracy = checker.GetAccuracy(1); + BOOST_CHECK(totalAccuracy == 100.0f); + + // Add image 2 and check accuracy + std::vector<float> inferenceOutputVector2 = {0.10f, 0.0f, 0.0f, 0.0f, 0.05f, 0.70f, 0.0f, 0.0f, 0.0f, 0.15f}; + TContainer inference2Container(inferenceOutputVector2); + std::vector<TContainer> outputTensor2; + outputTensor2.push_back(inference2Container); + + imageName = "ILSVRC2012_val_00000002.JPEG"; + checker.AddImageResult<TContainer>(imageName, outputTensor2); + + // Top 1 Accuracy + totalAccuracy = checker.GetAccuracy(1); + BOOST_CHECK(totalAccuracy == 50.0f); + + // Top 2 Accuracy + totalAccuracy = checker.GetAccuracy(2); + BOOST_CHECK(totalAccuracy == 100.0f); + + // Add image 3 and check accuracy + std::vector<float> inferenceOutputVector3 = {0.0f, 0.10f, 0.0f, 0.0f, 0.05f, 0.70f, 0.0f, 0.0f, 0.0f, 0.15f}; + TContainer inference3Container(inferenceOutputVector3); + std::vector<TContainer> outputTensor3; + outputTensor3.push_back(inference3Container); + + imageName = "ILSVRC2012_val_00000003.JPEG"; + checker.AddImageResult<TContainer>(imageName, outputTensor3); + + // Top 1 Accuracy + totalAccuracy = checker.GetAccuracy(1); + BOOST_CHECK(totalAccuracy == 33.3333321f); + + // Top 2 Accuracy + totalAccuracy = checker.GetAccuracy(2); + BOOST_CHECK(totalAccuracy == 66.6666641f); + + // Top 3 Accuracy + totalAccuracy = checker.GetAccuracy(3); + BOOST_CHECK(totalAccuracy == 100.0f); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/armnnUtils/ModelAccuracyChecker.cpp b/src/armnnUtils/ModelAccuracyChecker.cpp new file mode 100644 index 0000000000..bee5ca2365 --- /dev/null +++ b/src/armnnUtils/ModelAccuracyChecker.cpp @@ -0,0 +1,31 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include <vector> +#include <map> +#include <boost/log/trivial.hpp> +#include "ModelAccuracyChecker.hpp" + +namespace armnnUtils +{ + +armnnUtils::ModelAccuracyChecker::ModelAccuracyChecker(const std::map<std::string, int>& validationLabels) + : m_GroundTruthLabelSet(validationLabels){} + +float ModelAccuracyChecker::GetAccuracy(unsigned int k) +{ + if(k > 10) { + BOOST_LOG_TRIVIAL(info) << "Accuracy Tool only supports a maximum of Top 10 Accuracy. " + "Printing Top 10 Accuracy result!"; + k = 10; + } + unsigned int total = 0; + for (unsigned int i = k; i > 0; --i) + { + total += m_TopK[i]; + } + return static_cast<float>(total * 100) / static_cast<float>(m_ImagesProcessed); +} +}
\ No newline at end of file diff --git a/src/armnnUtils/ModelAccuracyChecker.hpp b/src/armnnUtils/ModelAccuracyChecker.hpp new file mode 100644 index 0000000000..abf994b5e1 --- /dev/null +++ b/src/armnnUtils/ModelAccuracyChecker.hpp @@ -0,0 +1,103 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include <cstddef> +#include <string> +#include <map> +#include <vector> +#include <boost/variant/apply_visitor.hpp> +#include <iostream> +#include <armnn/Types.hpp> +#include <functional> +#include <algorithm> + +namespace armnnUtils +{ + +using namespace armnn; + +class ModelAccuracyChecker +{ +public: + ModelAccuracyChecker(const std::map<std::string, int>& validationLabelSet); + + float GetAccuracy(unsigned int k); + + template<typename TContainer> + void AddImageResult(const std::string& imageName, std::vector<TContainer> outputTensor) + { + // Increment the total number of images processed + ++m_ImagesProcessed; + + std::map<int, float> confidenceMap; + auto & output = outputTensor[0]; + + // Create a map of all predictions + boost::apply_visitor([&](auto && value) + { + int index = 0; + for (const auto & o : value) + { + if (o > 0) + { + confidenceMap.insert(std::pair<int, float>(index, static_cast<float>(o))); + } + ++index; + } + }, + output); + + // Create a comparator for sorting the map in order of highest probability + typedef std::function<bool(std::pair<int, float>, std::pair<int, float>)> Comparator; + + Comparator compFunctor = + [](std::pair<int, float> element1, std::pair<int, float> element2) + { + return element1.second > element2.second; + }; + + // Do the sorting and store in an ordered set + std::set<std::pair<int, float>, Comparator> setOfPredictions( + confidenceMap.begin(), confidenceMap.end(), compFunctor); + + std::string trimmedName = GetTrimmedImageName(imageName); + int value = m_GroundTruthLabelSet.find(trimmedName)->second; + + unsigned int index = 1; + for (std::pair<int, float> element : setOfPredictions) + { + if(element.first == value) + { + ++m_TopK[index]; + } else + { + ++index; + } + } + } + + std::string GetTrimmedImageName(const std::string& imageName) const + { + std::string trimmedName; + size_t lastindex = imageName.find_last_of("."); + if(lastindex != std::string::npos) + { + trimmedName = imageName.substr(0, lastindex); + } else + { + trimmedName = imageName; + } + return trimmedName; + } + +private: + const std::map<std::string, int> m_GroundTruthLabelSet; + std::vector<unsigned int> m_TopK = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + unsigned int m_ImagesProcessed = 0; +}; +} //namespace armnnUtils + |