ModelAccuracyChecker.hpp

Go to the documentation of this file.

//
// Copyright © 2017 Arm Ltd. All rights reserved.
// SPDX-License-Identifier: MIT
//

#pragma once

#include <algorithm>
#include <armnn/Types.hpp>
#include <boost/assert.hpp>
#include <boost/variant/apply_visitor.hpp>
#include <cstddef>
#include <functional>
#include <iostream>
#include <map>
#include <string>
#include <vector>

namespace armnnUtils
{

using namespace armnn;

// Category names associated with a label
using LabelCategoryNames = std::vector<std::string>;

std::vector<std::string>
    SplitBy(const std::string& originalString, const std::string& delimiter = " ", bool includeEmptyToken = false);

std::string Strip(const std::string& originalString, const std::string& characterSet = " ");

class ModelAccuracyChecker
{
public:
    ModelAccuracyChecker(const std::map<std::string, std::string>& validationLabelSet,
                         const std::vector<LabelCategoryNames>& modelOutputLabels);

    float GetAccuracy(unsigned int k);

    template <typename TContainer>
    void AddImageResult(const std::string& imageName, std::vector<TContainer> outputTensor)
    {
        // Increment the total number of images processed
        ++m_ImagesProcessed;

        std::map<int, float> confidenceMap;
        auto& output = outputTensor[0];

        // Create a map of all predictions
        boost::apply_visitor([&confidenceMap](auto && value)
                             {
                                 int index = 0;
                                 for (const auto & o : value)
                                 {
                                     if (o > 0)
                                     {
                                         confidenceMap.insert(std::pair<int, float>(index, static_cast<float>(o)));
                                     }
                                     ++index;
                                 }
                             },
                             output);

        // Create a comparator for sorting the map in order of highest probability
        typedef std::function<bool(std::pair<int, float>, std::pair<int, float>)> Comparator;

        Comparator compFunctor =
            [](std::pair<int, float> element1, std::pair<int, float> element2)
            {
                return element1.second > element2.second;
            };

        // Do the sorting and store in an ordered set
        std::set<std::pair<int, float>, Comparator> setOfPredictions(
            confidenceMap.begin(), confidenceMap.end(), compFunctor);

        const std::string correctLabel = m_GroundTruthLabelSet.at(imageName);

        unsigned int index = 1;
        for (std::pair<int, float> element : setOfPredictions)
        {
            if (index >= m_TopK.size())
            {
                break;
            }
            // Check if the ground truth label value is included in the topi prediction.
            // Note that a prediction can have multiple prediction labels.
            const LabelCategoryNames predictionLabels = m_ModelOutputLabels[static_cast<size_t>(element.first)];
            if (std::find(predictionLabels.begin(), predictionLabels.end(), correctLabel) != predictionLabels.end())
            {
                ++m_TopK[index];
                break;
            }
            ++index;
        }
    }

private:
    const std::map<std::string, std::string> m_GroundTruthLabelSet;
    const std::vector<LabelCategoryNames> m_ModelOutputLabels;
    std::vector<unsigned int> m_TopK = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
    unsigned int m_ImagesProcessed   = 0;
};
} //namespace armnnUtils