MLCE-103 Add necessary enhancements to ModelAccuracyTool

* Evaluate model accuracy using category names instead of numerical
labels.
* Add blacklist support
* Add range selection support

Signed-off-by: SiCong Li <sicong.li@arm.com>
Change-Id: I7b1d2d298cfcaa56a27a028147169404b73580bb

1 files changed, 184 insertions, 24 deletions

diff --git a/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp b/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp
index 85241e889c..23e2f432a5 100644
--- a/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp
+++ b/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp
@@ -8,15 +8,43 @@
 #include "ModelAccuracyChecker.hpp"
 #include "armnnDeserializer/IDeserializer.hpp"
 
+#include <boost/algorithm/string.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/program_options/variables_map.hpp>
 #include <boost/range/iterator_range.hpp>
-
 #include <map>
 
 using namespace armnn::test;
 
-map<std::string, int> LoadValidationLabels(const string & validationLabelPath);
+/** Load image names and ground-truth labels from the image directory and the ground truth label file
+ *
+ * @pre \p validationLabelPath exists and is valid regular file
+ * @pre \p imageDirectoryPath exists and is valid directory
+ * @pre labels in validation file correspond to images which are in lexicographical order with the image name
+ * @pre image index starts at 1
+ * @pre \p begIndex and \p endIndex are end-inclusive
+ *
+ * @param[in] validationLabelPath Path to validation label file
+ * @param[in] imageDirectoryPath  Path to directory containing validation images
+ * @param[in] begIndex            Begin index of images to be loaded. Inclusive
+ * @param[in] endIndex            End index of images to be loaded. Inclusive
+ * @param[in] blacklistPath       Path to blacklist file
+ * @return A map mapping image file names to their corresponding ground-truth labels
+ */
+map<std::string, std::string> LoadValidationImageFilenamesAndLabels(const string& validationLabelPath,
+                                                                    const string& imageDirectoryPath,
+                                                                    size_t begIndex             = 0,
+                                                                    size_t endIndex             = 0,
+                                                                    const string& blacklistPath = "");
+
+/** Load model output labels from file
+ * 
+ * @pre \p modelOutputLabelsPath exists and is a regular file
+ *
+ * @param[in] modelOutputLabelsPath path to model output labels file
+ * @return A vector of labels, which in turn is described by a list of category names
+ */
+std::vector<armnnUtils::LabelCategoryNames> LoadModelOutputLabels(const std::string& modelOutputLabelsPath);
 
 int main(int argc, char* argv[])
 {
@@ -38,7 +66,10 @@ int main(int argc, char* argv[])
         std::string inputName;
         std::string inputLayout;
         std::string outputName;
+        std::string modelOutputLabelsPath;
         std::string validationLabelPath;
+        std::string validationRange;
+        std::string blacklistPath;
 
         const std::string backendsMessage = "Which device to run layers on by default. Possible choices: "
                                             + armnn::BackendRegistryInstance().GetBackendIdsAsString();
@@ -58,12 +89,21 @@ int main(int argc, char* argv[])
                  "Identifier of the output tensors in the network separated by comma.")
                 ("data-dir,d", po::value<std::string>(&dataDir)->required(),
                  "Path to directory containing the ImageNet test data")
+                ("model-output-labels,p", po::value<std::string>(&modelOutputLabelsPath)->required(),
+                 "Path to model output labels file.")
                 ("validation-labels-path,v", po::value<std::string>(&validationLabelPath)->required(),
                  "Path to ImageNet Validation Label file")
                 ("data-layout,l", po::value<std::string>(&inputLayout)->default_value("NHWC"),
                  "Data layout. Supported value: NHWC, NCHW. Default: NHCW")
                 ("compute,c", po::value<std::vector<armnn::BackendId>>(&computeDevice)->default_value(defaultBackends),
-                 backendsMessage.c_str());
+                 backendsMessage.c_str())
+                ("validation-range,r", po::value<std::string>(&validationRange)->default_value("1:0"),
+                 "The range of the images to be evaluated. Specified in the form <begin index>:<end index>."
+                 "The index starts at 1 and the range is inclusive."
+                 "By default the evaluation will be performed on all images.")
+                ("blacklist-path,b", po::value<std::string>(&blacklistPath)->default_value(""),
+                 "Path to a blacklist file where each line denotes the index of an image to be "
+                 "excluded from evaluation.");
         }
         catch (const std::exception& e)
         {
@@ -156,9 +196,47 @@ int main(int argc, char* argv[])
             m_OutputBindingInfo(outputBindingInfo.m_BindingId, outputBindingInfo.m_TensorInfo);
         std::vector<BindingPointInfo> outputBindings = { m_OutputBindingInfo };
 
+        // Load model output labels
+        if (modelOutputLabelsPath.empty() || !boost::filesystem::exists(modelOutputLabelsPath) ||
+            !boost::filesystem::is_regular_file(modelOutputLabelsPath))
+        {
+            BOOST_LOG_TRIVIAL(fatal) << "Invalid model output labels path at " << modelOutputLabelsPath;
+        }
+        const std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels =
+            LoadModelOutputLabels(modelOutputLabelsPath);
+
+        // Parse begin and end image indices
+        std::vector<std::string> imageIndexStrs = armnnUtils::SplitBy(validationRange, ":");
+        size_t imageBegIndex;
+        size_t imageEndIndex;
+        if (imageIndexStrs.size() != 2)
+        {
+            BOOST_LOG_TRIVIAL(fatal) << "Invalid validation range specification: Invalid format " << validationRange;
+            return 1;
+        }
+        try
+        {
+            imageBegIndex = std::stoul(imageIndexStrs[0]);
+            imageEndIndex = std::stoul(imageIndexStrs[1]);
+        }
+        catch (const std::exception& e)
+        {
+            BOOST_LOG_TRIVIAL(fatal) << "Invalid validation range specification: " << validationRange;
+            return 1;
+        }
+
+        // Validate  blacklist file if it's specified
+        if (!blacklistPath.empty() &&
+            !(boost::filesystem::exists(blacklistPath) && boost::filesystem::is_regular_file(blacklistPath)))
+        {
+            BOOST_LOG_TRIVIAL(fatal) << "Invalid path to blacklist file at " << blacklistPath;
+            return 1;
+        }
+
         path pathToDataDir(dataDir);
-        map<string, int> validationLabels = LoadValidationLabels(validationLabelPath);
-        armnnUtils::ModelAccuracyChecker checker(validationLabels);
+        const map<std::string, std::string> imageNameToLabel = LoadValidationImageFilenamesAndLabels(
+            validationLabelPath, pathToDataDir.string(), imageBegIndex, imageEndIndex, blacklistPath);
+        armnnUtils::ModelAccuracyChecker checker(imageNameToLabel, modelOutputLabels);
         using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<uint8_t>>;
 
         if (ValidateDirectory(dataDir))
@@ -196,6 +274,13 @@ int main(int argc, char* argv[])
                 inputTensorDataLayout == armnn::DataLayout::NCHW ? inputTensorShape[2] : inputTensorShape[1];
             // Get output tensor info
             const unsigned int outputNumElements = model.GetOutputSize();
+            // Check output tensor shape is valid
+            if (modelOutputLabels.size() != outputNumElements)
+            {
+                BOOST_LOG_TRIVIAL(fatal) << "Number of output elements: " << outputNumElements
+                                         << " , mismatches the number of output labels: " << modelOutputLabels.size();
+                return 1;
+            }
 
             const unsigned int batchSize = 1;
             // Get normalisation parameters
@@ -218,19 +303,20 @@ int main(int argc, char* argv[])
                 return 1;
             }
             const NormalizationParameters& normParams = GetNormalizationParameters(modelFrontend, inputTensorDataType);
-            for (auto& imageEntry : boost::make_iterator_range(directory_iterator(pathToDataDir), {}))
+            for (const auto& imageEntry : imageNameToLabel)
             {
-                cout << "Processing image: " << imageEntry << "\n";
+                const std::string imageName = imageEntry.first;
+                std::cout << "Processing image: " << imageName << "\n";
 
                 vector<TContainer> inputDataContainers;
                 vector<TContainer> outputDataContainers;
 
-                const string& imagePath = imageEntry.path().string();
+                auto imagePath = pathToDataDir / boost::filesystem::path(imageName);
                 switch (inputTensorDataType)
                 {
                     case armnn::DataType::Signed32:
                         inputDataContainers.push_back(
-                            PrepareImageTensor<int>(imagePath,
+                            PrepareImageTensor<int>(imagePath.string(),
                             inputTensorWidth, inputTensorHeight,
                             normParams,
                             batchSize,
@@ -239,7 +325,7 @@ int main(int argc, char* argv[])
                         break;
                     case armnn::DataType::QuantisedAsymm8:
                         inputDataContainers.push_back(
-                            PrepareImageTensor<uint8_t>(imagePath,
+                            PrepareImageTensor<uint8_t>(imagePath.string(),
                             inputTensorWidth, inputTensorHeight,
                             normParams,
                             batchSize,
@@ -249,7 +335,7 @@ int main(int argc, char* argv[])
                     case armnn::DataType::Float32:
                     default:
                         inputDataContainers.push_back(
-                            PrepareImageTensor<float>(imagePath,
+                            PrepareImageTensor<float>(imagePath.string(),
                             inputTensorWidth, inputTensorHeight,
                             normParams,
                             batchSize,
@@ -264,10 +350,9 @@ int main(int argc, char* argv[])
 
                 if (status == armnn::Status::Failure)
                 {
-                    BOOST_LOG_TRIVIAL(fatal) << "armnn::IRuntime: Failed to enqueue workload for image: " << imageEntry;
+                    BOOST_LOG_TRIVIAL(fatal) << "armnn::IRuntime: Failed to enqueue workload for image: " << imageName;
                 }
 
-                const std::string imageName = imageEntry.path().filename().string();
                 checker.AddImageResult<TContainer>(imageName, outputDataContainers);
             }
         }
@@ -301,21 +386,96 @@ int main(int argc, char* argv[])
     }
 }
 
-map<std::string, int> LoadValidationLabels(const string & validationLabelPath)
+map<std::string, std::string> LoadValidationImageFilenamesAndLabels(const string& validationLabelPath,
+                                                                    const string& imageDirectoryPath,
+                                                                    size_t begIndex,
+                                                                    size_t endIndex,
+                                                                    const string& blacklistPath)
 {
-    std::string imageName;
-    int classification;
-    map<std::string, int> validationLabel;
+    // Populate imageFilenames with names of all .JPEG, .PNG images
+    std::vector<std::string> imageFilenames;
+    for (const auto& imageEntry :
+         boost::make_iterator_range(boost::filesystem::directory_iterator(boost::filesystem::path(imageDirectoryPath))))
+    {
+        boost::filesystem::path imagePath = imageEntry.path();
+        std::string imageExtension        = boost::to_upper_copy<std::string>(imagePath.extension().string());
+        if (boost::filesystem::is_regular_file(imagePath) && (imageExtension == ".JPEG" || imageExtension == ".PNG"))
+        {
+            imageFilenames.push_back(imagePath.filename().string());
+        }
+    }
+    if (imageFilenames.empty())
+    {
+        throw armnn::Exception("No image file (JPEG, PNG) found at " + imageDirectoryPath);
+    }
+
+    // Sort the image filenames lexicographically
+    std::sort(imageFilenames.begin(), imageFilenames.end());
+
+    std::cout << imageFilenames.size() << " images found at " << imageDirectoryPath << std::endl;
+
+    // Get default end index
+    if (begIndex < 1 || endIndex > imageFilenames.size())
+    {
+        throw armnn::Exception("Invalid image index range");
+    }
+    endIndex = endIndex == 0 ? imageFilenames.size() : endIndex;
+    if (begIndex > endIndex)
+    {
+        throw armnn::Exception("Invalid image index range");
+    }
+
+    // Load blacklist if there is one
+    std::vector<unsigned int> blacklist;
+    if (!blacklistPath.empty())
+    {
+        std::ifstream blacklistFile(blacklistPath);
+        unsigned int index;
+        while (blacklistFile >> index)
+        {
+            blacklist.push_back(index);
+        }
+    }
+
+    // Load ground truth labels and pair them with corresponding image names
+    std::string classification;
+    map<std::string, std::string> imageNameToLabel;
     ifstream infile(validationLabelPath);
-    while (infile >> imageName >> classification)
+    size_t imageIndex          = begIndex;
+    size_t blacklistIndexCount = 0;
+    while (std::getline(infile, classification))
     {
-        std::string trimmedName;
-        size_t lastindex = imageName.find_last_of(".");
-        if(lastindex != std::string::npos)
+        if (imageIndex > endIndex)
         {
-            trimmedName = imageName.substr(0, lastindex);
+            break;
         }
-        validationLabel.insert(pair<string, int>(trimmedName, classification));
+        // If current imageIndex is included in blacklist, skip the current image
+        if (blacklistIndexCount < blacklist.size() && imageIndex == blacklist[blacklistIndexCount])
+        {
+            ++imageIndex;
+            ++blacklistIndexCount;
+            continue;
+        }
+        imageNameToLabel.insert(std::pair<std::string, std::string>(imageFilenames[imageIndex - 1], classification));
+        ++imageIndex;
     }
-    return validationLabel;
+    std::cout << blacklistIndexCount << " images blacklisted" << std::endl;
+    std::cout << imageIndex - begIndex - blacklistIndexCount << " images to be loaded" << std::endl;
+    return imageNameToLabel;
 }
+
+std::vector<armnnUtils::LabelCategoryNames> LoadModelOutputLabels(const std::string& modelOutputLabelsPath)
+{
+    std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels;
+    ifstream modelOutputLablesFile(modelOutputLabelsPath);
+    std::string line;
+    while (std::getline(modelOutputLablesFile, line))
+    {
+        armnnUtils::LabelCategoryNames tokens                  = armnnUtils::SplitBy(line, ":");
+        armnnUtils::LabelCategoryNames predictionCategoryNames = armnnUtils::SplitBy(tokens.back(), ",");
+        std::transform(predictionCategoryNames.begin(), predictionCategoryNames.end(), predictionCategoryNames.begin(),
+                       [](const std::string& category) { return armnnUtils::Strip(category); });
+        modelOutputLabels.push_back(predictionCategoryNames);
+    }
+    return modelOutputLabels;
+}
\ No newline at end of file