diff options
author | Éanna Ó Catháin <eanna.ocathain@arm.com> | 2019-05-08 14:00:45 +0100 |
---|---|---|
committer | Eanna O Cathain Arm <eanna.ocathain@arm.com> | 2019-05-08 15:23:27 +0000 |
commit | a4247d5a50502811a6956dffd990c0254622b7e1 (patch) | |
tree | a2e8742695673bc8e958cce316e6ddeafcc59642 /src/armnnUtils | |
parent | c2fe5fb3a070ce2c7daebf63d0def3d57cec09d3 (diff) | |
download | armnn-a4247d5a50502811a6956dffd990c0254622b7e1.tar.gz |
IVGCVSW-2900 Adding the Accuracy Checker Tool and tests
Change-Id: I4ac325e45f2236b8e0757d21046f117024ce3979
Signed-off-by: Éanna Ó Catháin <eanna.ocathain@arm.com>
Diffstat (limited to 'src/armnnUtils')
-rw-r--r-- | src/armnnUtils/ModelAccuracyChecker.cpp | 31 | ||||
-rw-r--r-- | src/armnnUtils/ModelAccuracyChecker.hpp | 103 |
2 files changed, 134 insertions, 0 deletions
diff --git a/src/armnnUtils/ModelAccuracyChecker.cpp b/src/armnnUtils/ModelAccuracyChecker.cpp new file mode 100644 index 0000000000..bee5ca2365 --- /dev/null +++ b/src/armnnUtils/ModelAccuracyChecker.cpp @@ -0,0 +1,31 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include <vector> +#include <map> +#include <boost/log/trivial.hpp> +#include "ModelAccuracyChecker.hpp" + +namespace armnnUtils +{ + +armnnUtils::ModelAccuracyChecker::ModelAccuracyChecker(const std::map<std::string, int>& validationLabels) + : m_GroundTruthLabelSet(validationLabels){} + +float ModelAccuracyChecker::GetAccuracy(unsigned int k) +{ + if(k > 10) { + BOOST_LOG_TRIVIAL(info) << "Accuracy Tool only supports a maximum of Top 10 Accuracy. " + "Printing Top 10 Accuracy result!"; + k = 10; + } + unsigned int total = 0; + for (unsigned int i = k; i > 0; --i) + { + total += m_TopK[i]; + } + return static_cast<float>(total * 100) / static_cast<float>(m_ImagesProcessed); +} +}
\ No newline at end of file diff --git a/src/armnnUtils/ModelAccuracyChecker.hpp b/src/armnnUtils/ModelAccuracyChecker.hpp new file mode 100644 index 0000000000..abf994b5e1 --- /dev/null +++ b/src/armnnUtils/ModelAccuracyChecker.hpp @@ -0,0 +1,103 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include <cstddef> +#include <string> +#include <map> +#include <vector> +#include <boost/variant/apply_visitor.hpp> +#include <iostream> +#include <armnn/Types.hpp> +#include <functional> +#include <algorithm> + +namespace armnnUtils +{ + +using namespace armnn; + +class ModelAccuracyChecker +{ +public: + ModelAccuracyChecker(const std::map<std::string, int>& validationLabelSet); + + float GetAccuracy(unsigned int k); + + template<typename TContainer> + void AddImageResult(const std::string& imageName, std::vector<TContainer> outputTensor) + { + // Increment the total number of images processed + ++m_ImagesProcessed; + + std::map<int, float> confidenceMap; + auto & output = outputTensor[0]; + + // Create a map of all predictions + boost::apply_visitor([&](auto && value) + { + int index = 0; + for (const auto & o : value) + { + if (o > 0) + { + confidenceMap.insert(std::pair<int, float>(index, static_cast<float>(o))); + } + ++index; + } + }, + output); + + // Create a comparator for sorting the map in order of highest probability + typedef std::function<bool(std::pair<int, float>, std::pair<int, float>)> Comparator; + + Comparator compFunctor = + [](std::pair<int, float> element1, std::pair<int, float> element2) + { + return element1.second > element2.second; + }; + + // Do the sorting and store in an ordered set + std::set<std::pair<int, float>, Comparator> setOfPredictions( + confidenceMap.begin(), confidenceMap.end(), compFunctor); + + std::string trimmedName = GetTrimmedImageName(imageName); + int value = m_GroundTruthLabelSet.find(trimmedName)->second; + + unsigned int index = 1; + for (std::pair<int, float> element : setOfPredictions) + { + if(element.first == value) + { + ++m_TopK[index]; + } else + { + ++index; + } + } + } + + std::string GetTrimmedImageName(const std::string& imageName) const + { + std::string trimmedName; + size_t lastindex = imageName.find_last_of("."); + if(lastindex != std::string::npos) + { + trimmedName = imageName.substr(0, lastindex); + } else + { + trimmedName = imageName; + } + return trimmedName; + } + +private: + const std::map<std::string, int> m_GroundTruthLabelSet; + std::vector<unsigned int> m_TopK = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + unsigned int m_ImagesProcessed = 0; +}; +} //namespace armnnUtils + |