MLCE-103 Add necessary enhancements to ModelAccuracyTool

* Evaluate model accuracy using category names instead of numerical
labels.
* Add blacklist support
* Add range selection support

Signed-off-by: SiCong Li <sicong.li@arm.com>
Change-Id: I7b1d2d298cfcaa56a27a028147169404b73580bb

4 files changed, 337 insertions, 84 deletions

diff --git a/src/armnn/test/ModelAccuracyCheckerTest.cpp b/src/armnn/test/ModelAccuracyCheckerTest.cpp
index f3a6c9d81d..aa1fba212c 100644
--- a/src/armnn/test/ModelAccuracyCheckerTest.cpp
+++ b/src/armnn/test/ModelAccuracyCheckerTest.cpp
@@ -7,32 +7,50 @@
 #include <boost/algorithm/string.hpp>
 #include <boost/test/unit_test.hpp>
 
-#include <iostream>
-#include <string>
-#include <boost/log/core/core.hpp>
 #include <boost/filesystem.hpp>
+#include <boost/log/core/core.hpp>
 #include <boost/optional.hpp>
 #include <boost/variant.hpp>
+#include <iostream>
+#include <string>
 
 using namespace armnnUtils;
 
-struct TestHelper {
-    const std::map<std::string, int> GetValidationLabelSet()
+struct TestHelper
+{
+    const std::map<std::string, std::string> GetValidationLabelSet()
     {
-        std::map<std::string, int> validationLabelSet;
-        validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000001", 2));
-        validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000002", 9));
-        validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000003", 1));
-        validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000004", 6));
-        validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000005", 5));
-        validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000006", 0));
-        validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000007", 8));
-        validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000008", 4));
-        validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000009", 3));
-        validationLabelSet.insert( std::make_pair("ILSVRC2012_val_00000009", 7));
+        std::map<std::string, std::string> validationLabelSet;
+        validationLabelSet.insert(std::make_pair("val_01.JPEG", "goldfinch"));
+        validationLabelSet.insert(std::make_pair("val_02.JPEG", "magpie"));
+        validationLabelSet.insert(std::make_pair("val_03.JPEG", "brambling"));
+        validationLabelSet.insert(std::make_pair("val_04.JPEG", "robin"));
+        validationLabelSet.insert(std::make_pair("val_05.JPEG", "indigo bird"));
+        validationLabelSet.insert(std::make_pair("val_06.JPEG", "ostrich"));
+        validationLabelSet.insert(std::make_pair("val_07.JPEG", "jay"));
+        validationLabelSet.insert(std::make_pair("val_08.JPEG", "snowbird"));
+        validationLabelSet.insert(std::make_pair("val_09.JPEG", "house finch"));
+        validationLabelSet.insert(std::make_pair("val_09.JPEG", "bulbul"));
 
         return validationLabelSet;
     }
+    const std::vector<armnnUtils::LabelCategoryNames> GetModelOutputLabels()
+    {
+        const std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels =
+        {
+            {"ostrich", "Struthio camelus"},
+            {"brambling", "Fringilla montifringilla"},
+            {"goldfinch", "Carduelis carduelis"},
+            {"house finch", "linnet", "Carpodacus mexicanus"},
+            {"junco", "snowbird"},
+            {"indigo bunting", "indigo finch", "indigo bird", "Passerina cyanea"},
+            {"robin", "American robin", "Turdus migratorius"},
+            {"bulbul"},
+            {"jay"},
+            {"magpie"}
+        };
+        return modelOutputLabels;
+    }
 };
 
 BOOST_AUTO_TEST_SUITE(ModelAccuracyCheckerTest)
@@ -41,7 +59,7 @@ using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vec
 
 BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper)
 {
-    ModelAccuracyChecker checker(GetValidationLabelSet());
+    ModelAccuracyChecker checker(GetValidationLabelSet(), GetModelOutputLabels());
 
     // Add image 1 and check accuracy
     std::vector<float> inferenceOutputVector1 = {0.05f, 0.10f, 0.70f, 0.15f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
@@ -49,7 +67,7 @@ BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper)
     std::vector<TContainer> outputTensor1;
     outputTensor1.push_back(inference1Container);
 
-    std::string imageName = "ILSVRC2012_val_00000001.JPEG";
+    std::string imageName = "val_01.JPEG";
     checker.AddImageResult<TContainer>(imageName, outputTensor1);
 
     // Top 1 Accuracy
@@ -62,7 +80,7 @@ BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper)
     std::vector<TContainer> outputTensor2;
     outputTensor2.push_back(inference2Container);
 
-    imageName = "ILSVRC2012_val_00000002.JPEG";
+    imageName = "val_02.JPEG";
     checker.AddImageResult<TContainer>(imageName, outputTensor2);
 
     // Top 1 Accuracy
@@ -79,7 +97,7 @@ BOOST_FIXTURE_TEST_CASE(TestFloat32OutputTensorAccuracy, TestHelper)
     std::vector<TContainer> outputTensor3;
     outputTensor3.push_back(inference3Container);
 
-    imageName = "ILSVRC2012_val_00000003.JPEG";
+    imageName = "val_03.JPEG";
     checker.AddImageResult<TContainer>(imageName, outputTensor3);
 
     // Top 1 Accuracy
diff --git a/src/armnnUtils/ModelAccuracyChecker.cpp b/src/armnnUtils/ModelAccuracyChecker.cpp
index bee5ca2365..81942dc2be 100644
--- a/src/armnnUtils/ModelAccuracyChecker.cpp
+++ b/src/armnnUtils/ModelAccuracyChecker.cpp
@@ -3,22 +3,27 @@
 // SPDX-License-Identifier: MIT
 //
 
-#include <vector>
-#include <map>
-#include <boost/log/trivial.hpp>
 #include "ModelAccuracyChecker.hpp"
+#include <boost/filesystem.hpp>
+#include <boost/log/trivial.hpp>
+#include <map>
+#include <vector>
 
 namespace armnnUtils
 {
 
-armnnUtils::ModelAccuracyChecker::ModelAccuracyChecker(const std::map<std::string, int>& validationLabels)
-    : m_GroundTruthLabelSet(validationLabels){}
+armnnUtils::ModelAccuracyChecker::ModelAccuracyChecker(const std::map<std::string, std::string>& validationLabels,
+                                                       const std::vector<LabelCategoryNames>& modelOutputLabels)
+    : m_GroundTruthLabelSet(validationLabels)
+    , m_ModelOutputLabels(modelOutputLabels)
+{}
 
 float ModelAccuracyChecker::GetAccuracy(unsigned int k)
 {
-    if(k > 10) {
-        BOOST_LOG_TRIVIAL(info) << "Accuracy Tool only supports a maximum of Top 10 Accuracy. "
-                                   "Printing Top 10 Accuracy result!";
+    if (k > 10)
+    {
+        BOOST_LOG_TRIVIAL(warning) << "Accuracy Tool only supports a maximum of Top 10 Accuracy. "
+                                      "Printing Top 10 Accuracy result!";
         k = 10;
     }
     unsigned int total = 0;
@@ -28,4 +33,43 @@ float ModelAccuracyChecker::GetAccuracy(unsigned int k)
     }
     return static_cast<float>(total * 100) / static_cast<float>(m_ImagesProcessed);
 }
-}
\ No newline at end of file
+
+// Split a string into tokens by a delimiter
+std::vector<std::string>
+    SplitBy(const std::string& originalString, const std::string& delimiter, bool includeEmptyToken)
+{
+    std::vector<std::string> tokens;
+    size_t cur  = 0;
+    size_t next = 0;
+    while ((next = originalString.find(delimiter, cur)) != std::string::npos)
+    {
+        // Skip empty tokens, unless explicitly stated to include them.
+        if (next - cur > 0 || includeEmptyToken)
+        {
+            tokens.push_back(originalString.substr(cur, next - cur));
+        }
+        cur = next + delimiter.size();
+    }
+    // Get the remaining token
+    // Skip empty tokens, unless explicitly stated to include them.
+    if (originalString.size() - cur > 0 || includeEmptyToken)
+    {
+        tokens.push_back(originalString.substr(cur, originalString.size() - cur));
+    }
+    return tokens;
+}
+
+// Remove any preceding and trailing character specified in the characterSet.
+std::string Strip(const std::string& originalString, const std::string& characterSet)
+{
+    BOOST_ASSERT(!characterSet.empty());
+    const std::size_t firstFound = originalString.find_first_not_of(characterSet);
+    const std::size_t lastFound  = originalString.find_last_not_of(characterSet);
+    // Return empty if the originalString is empty or the originalString contains only to-be-striped characters
+    if (firstFound == std::string::npos || lastFound == std::string::npos)
+    {
+        return "";
+    }
+    return originalString.substr(firstFound, lastFound + 1 - firstFound);
+}
+}    // namespace armnnUtils
\ No newline at end of file
diff --git a/src/armnnUtils/ModelAccuracyChecker.hpp b/src/armnnUtils/ModelAccuracyChecker.hpp
index cdd2af0ac5..c4dd4f1b05 100644
--- a/src/armnnUtils/ModelAccuracyChecker.hpp
+++ b/src/armnnUtils/ModelAccuracyChecker.hpp
@@ -5,39 +5,81 @@
 
 #pragma once
 
+#include <algorithm>
+#include <armnn/Types.hpp>
+#include <boost/assert.hpp>
+#include <boost/variant/apply_visitor.hpp>
 #include <cstddef>
-#include <string>
+#include <functional>
+#include <iostream>
 #include <map>
+#include <string>
 #include <vector>
-#include <boost/variant/apply_visitor.hpp>
-#include <iostream>
-#include <armnn/Types.hpp>
-#include <functional>
-#include <algorithm>
 
 namespace armnnUtils
 {
 
 using namespace armnn;
 
+// Category names associated with a label
+using LabelCategoryNames = std::vector<std::string>;
+
+/** Split a string into tokens by a delimiter
+ *
+ * @param[in] originalString    Original string to be split
+ * @param[in] delimiter         Delimiter used to split \p originalString
+ * @param[in] includeEmptyToekn If true, include empty tokens in the result
+ * @return A vector of tokens split from \p originalString by \delimiter
+ */
+std::vector<std::string>
+    SplitBy(const std::string& originalString, const std::string& delimiter = " ", bool includeEmptyToken = false);
+
+/** Remove any preceding and trailing character specified in the characterSet.
+ *
+ * @param[in] originalString    Original string to be stripped
+ * @param[in] characterSet      Set of characters to be stripped from \p originalString
+ * @return A string stripped of all characters specified in \p characterSet from \p originalString
+ */
+std::string Strip(const std::string& originalString, const std::string& characterSet = " ");
+
 class ModelAccuracyChecker
 {
 public:
-    ModelAccuracyChecker(const std::map<std::string, int>& validationLabelSet);
-
+    /** Constructor for a model top k accuracy checker
+     *
+     * @param[in] validationLabelSet Mapping from names of images to be validated, to category names of their
+                                     corresponding ground-truth labels.
+     * @param[in] modelOutputLabels  Mapping from output nodes to the category names of their corresponding labels
+                                     Note that an output node can have multiple category names.
+     */
+    ModelAccuracyChecker(const std::map<std::string, std::string>& validationLabelSet,
+                         const std::vector<LabelCategoryNames>& modelOutputLabels);
+
+    /** Get Top K accuracy
+     *
+     * @param[in] k The number of top predictions to use for validating the ground-truth label. For example, if \p k is
+                    3, then a prediction is considered correct as long as the ground-truth appears in the top 3
+                    predictions.
+     * @return  The accuracy, according to the top \p k th predictions.
+     */
     float GetAccuracy(unsigned int k);
 
-    template<typename TContainer>
+    /** Record the prediction result of an image
+     *
+     * @param[in] imageName     Name of the image.
+     * @param[in] outputTensor  Output tensor of the network running \p imageName.
+     */
+    template <typename TContainer>
     void AddImageResult(const std::string& imageName, std::vector<TContainer> outputTensor)
     {
         // Increment the total number of images processed
         ++m_ImagesProcessed;
 
         std::map<int, float> confidenceMap;
-        auto & output = outputTensor[0];
+        auto& output = outputTensor[0];
 
         // Create a map of all predictions
-        boost::apply_visitor([&](auto && value)
+        boost::apply_visitor([&confidenceMap](auto && value)
                              {
                                  int index = 0;
                                  for (const auto & o : value)
@@ -64,8 +106,7 @@ public:
         std::set<std::pair<int, float>, Comparator> setOfPredictions(
             confidenceMap.begin(), confidenceMap.end(), compFunctor);
 
-        std::string trimmedName = GetTrimmedImageName(imageName);
-        int value = m_GroundTruthLabelSet.find(trimmedName)->second;
+        const std::string correctLabel = m_GroundTruthLabelSet.at(imageName);
 
         unsigned int index = 1;
         for (std::pair<int, float> element : setOfPredictions)
@@ -74,7 +115,10 @@ public:
             {
                 break;
             }
-            if (element.first == value)
+            // Check if the ground truth label value is included in the topi prediction.
+            // Note that a prediction can have multiple prediction labels.
+            const LabelCategoryNames predictionLabels = m_ModelOutputLabels[static_cast<size_t>(element.first)];
+            if (std::find(predictionLabels.begin(), predictionLabels.end(), correctLabel) != predictionLabels.end())
             {
                 ++m_TopK[index];
                 break;
@@ -83,24 +127,11 @@ public:
         }
     }
 
-    std::string GetTrimmedImageName(const std::string& imageName) const
-    {
-        std::string trimmedName;
-        size_t lastindex = imageName.find_last_of(".");
-        if(lastindex != std::string::npos)
-        {
-            trimmedName = imageName.substr(0, lastindex);
-        } else
-        {
-            trimmedName = imageName;
-        }
-        return trimmedName;
-    }
-
 private:
-    const std::map<std::string, int> m_GroundTruthLabelSet;
-    std::vector<unsigned int> m_TopK = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-    unsigned int m_ImagesProcessed = 0;
+    const std::map<std::string, std::string> m_GroundTruthLabelSet;
+    const std::vector<LabelCategoryNames> m_ModelOutputLabels;
+    std::vector<unsigned int> m_TopK = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+    unsigned int m_ImagesProcessed   = 0;
 };
 } //namespace armnnUtils
 
diff --git a/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp b/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp
index 85241e889c..23e2f432a5 100644
--- a/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp
+++ b/tests/ModelAccuracyTool-Armnn/ModelAccuracyTool-Armnn.cpp
@@ -8,15 +8,43 @@
 #include "ModelAccuracyChecker.hpp"
 #include "armnnDeserializer/IDeserializer.hpp"
 
+#include <boost/algorithm/string.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/program_options/variables_map.hpp>
 #include <boost/range/iterator_range.hpp>
-
 #include <map>
 
 using namespace armnn::test;
 
-map<std::string, int> LoadValidationLabels(const string & validationLabelPath);
+/** Load image names and ground-truth labels from the image directory and the ground truth label file
+ *
+ * @pre \p validationLabelPath exists and is valid regular file
+ * @pre \p imageDirectoryPath exists and is valid directory
+ * @pre labels in validation file correspond to images which are in lexicographical order with the image name
+ * @pre image index starts at 1
+ * @pre \p begIndex and \p endIndex are end-inclusive
+ *
+ * @param[in] validationLabelPath Path to validation label file
+ * @param[in] imageDirectoryPath  Path to directory containing validation images
+ * @param[in] begIndex            Begin index of images to be loaded. Inclusive
+ * @param[in] endIndex            End index of images to be loaded. Inclusive
+ * @param[in] blacklistPath       Path to blacklist file
+ * @return A map mapping image file names to their corresponding ground-truth labels
+ */
+map<std::string, std::string> LoadValidationImageFilenamesAndLabels(const string& validationLabelPath,
+                                                                    const string& imageDirectoryPath,
+                                                                    size_t begIndex             = 0,
+                                                                    size_t endIndex             = 0,
+                                                                    const string& blacklistPath = "");
+
+/** Load model output labels from file
+ * 
+ * @pre \p modelOutputLabelsPath exists and is a regular file
+ *
+ * @param[in] modelOutputLabelsPath path to model output labels file
+ * @return A vector of labels, which in turn is described by a list of category names
+ */
+std::vector<armnnUtils::LabelCategoryNames> LoadModelOutputLabels(const std::string& modelOutputLabelsPath);
 
 int main(int argc, char* argv[])
 {
@@ -38,7 +66,10 @@ int main(int argc, char* argv[])
         std::string inputName;
         std::string inputLayout;
         std::string outputName;
+        std::string modelOutputLabelsPath;
         std::string validationLabelPath;
+        std::string validationRange;
+        std::string blacklistPath;
 
         const std::string backendsMessage = "Which device to run layers on by default. Possible choices: "
                                             + armnn::BackendRegistryInstance().GetBackendIdsAsString();
@@ -58,12 +89,21 @@ int main(int argc, char* argv[])
                  "Identifier of the output tensors in the network separated by comma.")
                 ("data-dir,d", po::value<std::string>(&dataDir)->required(),
                  "Path to directory containing the ImageNet test data")
+                ("model-output-labels,p", po::value<std::string>(&modelOutputLabelsPath)->required(),
+                 "Path to model output labels file.")
                 ("validation-labels-path,v", po::value<std::string>(&validationLabelPath)->required(),
                  "Path to ImageNet Validation Label file")
                 ("data-layout,l", po::value<std::string>(&inputLayout)->default_value("NHWC"),
                  "Data layout. Supported value: NHWC, NCHW. Default: NHCW")
                 ("compute,c", po::value<std::vector<armnn::BackendId>>(&computeDevice)->default_value(defaultBackends),
-                 backendsMessage.c_str());
+                 backendsMessage.c_str())
+                ("validation-range,r", po::value<std::string>(&validationRange)->default_value("1:0"),
+                 "The range of the images to be evaluated. Specified in the form <begin index>:<end index>."
+                 "The index starts at 1 and the range is inclusive."
+                 "By default the evaluation will be performed on all images.")
+                ("blacklist-path,b", po::value<std::string>(&blacklistPath)->default_value(""),
+                 "Path to a blacklist file where each line denotes the index of an image to be "
+                 "excluded from evaluation.");
         }
         catch (const std::exception& e)
         {
@@ -156,9 +196,47 @@ int main(int argc, char* argv[])
             m_OutputBindingInfo(outputBindingInfo.m_BindingId, outputBindingInfo.m_TensorInfo);
         std::vector<BindingPointInfo> outputBindings = { m_OutputBindingInfo };
 
+        // Load model output labels
+        if (modelOutputLabelsPath.empty() || !boost::filesystem::exists(modelOutputLabelsPath) ||
+            !boost::filesystem::is_regular_file(modelOutputLabelsPath))
+        {
+            BOOST_LOG_TRIVIAL(fatal) << "Invalid model output labels path at " << modelOutputLabelsPath;
+        }
+        const std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels =
+            LoadModelOutputLabels(modelOutputLabelsPath);
+
+        // Parse begin and end image indices
+        std::vector<std::string> imageIndexStrs = armnnUtils::SplitBy(validationRange, ":");
+        size_t imageBegIndex;
+        size_t imageEndIndex;
+        if (imageIndexStrs.size() != 2)
+        {
+            BOOST_LOG_TRIVIAL(fatal) << "Invalid validation range specification: Invalid format " << validationRange;
+            return 1;
+        }
+        try
+        {
+            imageBegIndex = std::stoul(imageIndexStrs[0]);
+            imageEndIndex = std::stoul(imageIndexStrs[1]);
+        }
+        catch (const std::exception& e)
+        {
+            BOOST_LOG_TRIVIAL(fatal) << "Invalid validation range specification: " << validationRange;
+            return 1;
+        }
+
+        // Validate  blacklist file if it's specified
+        if (!blacklistPath.empty() &&
+            !(boost::filesystem::exists(blacklistPath) && boost::filesystem::is_regular_file(blacklistPath)))
+        {
+            BOOST_LOG_TRIVIAL(fatal) << "Invalid path to blacklist file at " << blacklistPath;
+            return 1;
+        }
+
         path pathToDataDir(dataDir);
-        map<string, int> validationLabels = LoadValidationLabels(validationLabelPath);
-        armnnUtils::ModelAccuracyChecker checker(validationLabels);
+        const map<std::string, std::string> imageNameToLabel = LoadValidationImageFilenamesAndLabels(
+            validationLabelPath, pathToDataDir.string(), imageBegIndex, imageEndIndex, blacklistPath);
+        armnnUtils::ModelAccuracyChecker checker(imageNameToLabel, modelOutputLabels);
         using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<uint8_t>>;
 
         if (ValidateDirectory(dataDir))
@@ -196,6 +274,13 @@ int main(int argc, char* argv[])
                 inputTensorDataLayout == armnn::DataLayout::NCHW ? inputTensorShape[2] : inputTensorShape[1];
             // Get output tensor info
             const unsigned int outputNumElements = model.GetOutputSize();
+            // Check output tensor shape is valid
+            if (modelOutputLabels.size() != outputNumElements)
+            {
+                BOOST_LOG_TRIVIAL(fatal) << "Number of output elements: " << outputNumElements
+                                         << " , mismatches the number of output labels: " << modelOutputLabels.size();
+                return 1;
+            }
 
             const unsigned int batchSize = 1;
             // Get normalisation parameters
@@ -218,19 +303,20 @@ int main(int argc, char* argv[])
                 return 1;
             }
             const NormalizationParameters& normParams = GetNormalizationParameters(modelFrontend, inputTensorDataType);
-            for (auto& imageEntry : boost::make_iterator_range(directory_iterator(pathToDataDir), {}))
+            for (const auto& imageEntry : imageNameToLabel)
             {
-                cout << "Processing image: " << imageEntry << "\n";
+                const std::string imageName = imageEntry.first;
+                std::cout << "Processing image: " << imageName << "\n";
 
                 vector<TContainer> inputDataContainers;
                 vector<TContainer> outputDataContainers;
 
-                const string& imagePath = imageEntry.path().string();
+                auto imagePath = pathToDataDir / boost::filesystem::path(imageName);
                 switch (inputTensorDataType)
                 {
                     case armnn::DataType::Signed32:
                         inputDataContainers.push_back(
-                            PrepareImageTensor<int>(imagePath,
+                            PrepareImageTensor<int>(imagePath.string(),
                             inputTensorWidth, inputTensorHeight,
                             normParams,
                             batchSize,
@@ -239,7 +325,7 @@ int main(int argc, char* argv[])
                         break;
                     case armnn::DataType::QuantisedAsymm8:
                         inputDataContainers.push_back(
-                            PrepareImageTensor<uint8_t>(imagePath,
+                            PrepareImageTensor<uint8_t>(imagePath.string(),
                             inputTensorWidth, inputTensorHeight,
                             normParams,
                             batchSize,
@@ -249,7 +335,7 @@ int main(int argc, char* argv[])
                     case armnn::DataType::Float32:
                     default:
                         inputDataContainers.push_back(
-                            PrepareImageTensor<float>(imagePath,
+                            PrepareImageTensor<float>(imagePath.string(),
                             inputTensorWidth, inputTensorHeight,
                             normParams,
                             batchSize,
@@ -264,10 +350,9 @@ int main(int argc, char* argv[])
 
                 if (status == armnn::Status::Failure)
                 {
-                    BOOST_LOG_TRIVIAL(fatal) << "armnn::IRuntime: Failed to enqueue workload for image: " << imageEntry;
+                    BOOST_LOG_TRIVIAL(fatal) << "armnn::IRuntime: Failed to enqueue workload for image: " << imageName;
                 }
 
-                const std::string imageName = imageEntry.path().filename().string();
                 checker.AddImageResult<TContainer>(imageName, outputDataContainers);
             }
         }
@@ -301,21 +386,96 @@ int main(int argc, char* argv[])
     }
 }
 
-map<std::string, int> LoadValidationLabels(const string & validationLabelPath)
+map<std::string, std::string> LoadValidationImageFilenamesAndLabels(const string& validationLabelPath,
+                                                                    const string& imageDirectoryPath,
+                                                                    size_t begIndex,
+                                                                    size_t endIndex,
+                                                                    const string& blacklistPath)
 {
-    std::string imageName;
-    int classification;
-    map<std::string, int> validationLabel;
+    // Populate imageFilenames with names of all .JPEG, .PNG images
+    std::vector<std::string> imageFilenames;
+    for (const auto& imageEntry :
+         boost::make_iterator_range(boost::filesystem::directory_iterator(boost::filesystem::path(imageDirectoryPath))))
+    {
+        boost::filesystem::path imagePath = imageEntry.path();
+        std::string imageExtension        = boost::to_upper_copy<std::string>(imagePath.extension().string());
+        if (boost::filesystem::is_regular_file(imagePath) && (imageExtension == ".JPEG" || imageExtension == ".PNG"))
+        {
+            imageFilenames.push_back(imagePath.filename().string());
+        }
+    }
+    if (imageFilenames.empty())
+    {
+        throw armnn::Exception("No image file (JPEG, PNG) found at " + imageDirectoryPath);
+    }
+
+    // Sort the image filenames lexicographically
+    std::sort(imageFilenames.begin(), imageFilenames.end());
+
+    std::cout << imageFilenames.size() << " images found at " << imageDirectoryPath << std::endl;
+
+    // Get default end index
+    if (begIndex < 1 || endIndex > imageFilenames.size())
+    {
+        throw armnn::Exception("Invalid image index range");
+    }
+    endIndex = endIndex == 0 ? imageFilenames.size() : endIndex;
+    if (begIndex > endIndex)
+    {
+        throw armnn::Exception("Invalid image index range");
+    }
+
+    // Load blacklist if there is one
+    std::vector<unsigned int> blacklist;
+    if (!blacklistPath.empty())
+    {
+        std::ifstream blacklistFile(blacklistPath);
+        unsigned int index;
+        while (blacklistFile >> index)
+        {
+            blacklist.push_back(index);
+        }
+    }
+
+    // Load ground truth labels and pair them with corresponding image names
+    std::string classification;
+    map<std::string, std::string> imageNameToLabel;
     ifstream infile(validationLabelPath);
-    while (infile >> imageName >> classification)
+    size_t imageIndex          = begIndex;
+    size_t blacklistIndexCount = 0;
+    while (std::getline(infile, classification))
     {
-        std::string trimmedName;
-        size_t lastindex = imageName.find_last_of(".");
-        if(lastindex != std::string::npos)
+        if (imageIndex > endIndex)
         {
-            trimmedName = imageName.substr(0, lastindex);
+            break;
         }
-        validationLabel.insert(pair<string, int>(trimmedName, classification));
+        // If current imageIndex is included in blacklist, skip the current image
+        if (blacklistIndexCount < blacklist.size() && imageIndex == blacklist[blacklistIndexCount])
+        {
+            ++imageIndex;
+            ++blacklistIndexCount;
+            continue;
+        }
+        imageNameToLabel.insert(std::pair<std::string, std::string>(imageFilenames[imageIndex - 1], classification));
+        ++imageIndex;
     }
-    return validationLabel;
+    std::cout << blacklistIndexCount << " images blacklisted" << std::endl;
+    std::cout << imageIndex - begIndex - blacklistIndexCount << " images to be loaded" << std::endl;
+    return imageNameToLabel;
 }
+
+std::vector<armnnUtils::LabelCategoryNames> LoadModelOutputLabels(const std::string& modelOutputLabelsPath)
+{
+    std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels;
+    ifstream modelOutputLablesFile(modelOutputLabelsPath);
+    std::string line;
+    while (std::getline(modelOutputLablesFile, line))
+    {
+        armnnUtils::LabelCategoryNames tokens                  = armnnUtils::SplitBy(line, ":");
+        armnnUtils::LabelCategoryNames predictionCategoryNames = armnnUtils::SplitBy(tokens.back(), ",");
+        std::transform(predictionCategoryNames.begin(), predictionCategoryNames.end(), predictionCategoryNames.begin(),
+                       [](const std::string& category) { return armnnUtils::Strip(category); });
+        modelOutputLabels.push_back(predictionCategoryNames);
+    }
+    return modelOutputLabels;
+}
\ No newline at end of file