IVGCVSW-2834 Add dynamic quantization via datasets

* Add QuantizationDataSet class for quantization data parsed from CSV file
* Add QuantizationInput for retrieving quantization data for each layer ID
* Add unit tests for command line processor and QuantizationDataSet

Change-Id: Iaf0a747b5f25a59a766ac04f7158e8cb7909d179
Signed-off-by: Nina Drozd <nina.drozd@arm.com>

1 files changed, 165 insertions, 0 deletions

diff --git a/src/armnnQuantizer/QuantizationDataSet.cpp b/src/armnnQuantizer/QuantizationDataSet.cpp
new file mode 100644
index 0000000000..d225883854
--- /dev/null
+++ b/src/armnnQuantizer/QuantizationDataSet.cpp
@@ -0,0 +1,165 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include "QuantizationDataSet.hpp"
+#include "CsvReader.hpp"
+
+#define BOOST_FILESYSTEM_NO_DEPRECATED
+
+#include <boost/filesystem/operations.hpp>
+#include <boost/filesystem/path.hpp>
+
+namespace armnnQuantizer
+{
+
+QuantizationDataSet::QuantizationDataSet()
+{
+}
+
+QuantizationDataSet::QuantizationDataSet(const std::string csvFilePath):
+    m_QuantizationInputs(),
+    m_CsvFilePath(csvFilePath)
+{
+    ParseCsvFile();
+}
+
+void AddInputData(unsigned int passId,
+                  armnn::LayerBindingId bindingId,
+                  const std::string& inputFilePath,
+                  std::map<unsigned int, QuantizationInput>& passIdToQuantizationInput)
+{
+    auto iterator = passIdToQuantizationInput.find(passId);
+    if (iterator == passIdToQuantizationInput.end())
+    {
+        QuantizationInput input(passId, bindingId, inputFilePath);
+        passIdToQuantizationInput.emplace(passId, input);
+    }
+    else
+    {
+        auto existingQuantizationInput = iterator->second;
+        existingQuantizationInput.AddEntry(bindingId, inputFilePath);
+    }
+}
+
+QuantizationDataSet::~QuantizationDataSet()
+{
+}
+
+void InputLayerVisitor::VisitInputLayer(const armnn::IConnectableLayer* layer,
+                                        armnn::LayerBindingId id,
+                                        const char* name)
+{
+    m_TensorInfos.emplace(id, layer->GetInputSlot(0).GetConnection()->GetTensorInfo());
+}
+
+armnn::TensorInfo InputLayerVisitor::GetTensorInfo(armnn::LayerBindingId layerBindingId)
+{
+    auto iterator = m_TensorInfos.find(layerBindingId);
+    if (iterator != m_TensorInfos.end())
+    {
+        return m_TensorInfos.at(layerBindingId);
+    }
+    else
+    {
+        throw armnn::Exception("Could not retrieve tensor info for binding ID " + std::to_string(layerBindingId));
+    }
+}
+
+
+unsigned int GetPassIdFromCsvRow(std::vector<armnnUtils::CsvRow> csvRows, unsigned int rowIndex)
+{
+    unsigned int passId;
+    try
+    {
+        passId = static_cast<unsigned int>(std::stoi(csvRows[rowIndex].values[0]));
+    }
+    catch (std::invalid_argument)
+    {
+        throw armnn::ParseException("Pass ID [" + csvRows[rowIndex].values[0] + "]" +
+                                    " is not correct format on CSV row " + std::to_string(rowIndex));
+    }
+    return passId;
+}
+
+armnn::LayerBindingId GetBindingIdFromCsvRow(std::vector<armnnUtils::CsvRow> csvRows, unsigned int rowIndex)
+{
+    armnn::LayerBindingId bindingId;
+    try
+    {
+        bindingId = std::stoi(csvRows[rowIndex].values[1]);
+    }
+    catch (std::invalid_argument)
+    {
+        throw armnn::ParseException("Binding ID [" + csvRows[rowIndex].values[0] + "]" +
+                                    " is not correct format on CSV row " + std::to_string(rowIndex));
+    }
+    return bindingId;
+}
+
+std::string GetFileNameFromCsvRow(std::vector<armnnUtils::CsvRow> csvRows, unsigned int rowIndex)
+{
+    std::string fileName = csvRows[rowIndex].values[2];
+
+    if (!boost::filesystem::exists(fileName))
+    {
+        throw armnn::ParseException("File [ " + fileName + "] provided on CSV row " + std::to_string(rowIndex) +
+                                    " does not exist.");
+    }
+
+    if (fileName.empty())
+    {
+        throw armnn::ParseException("Filename cannot be empty on CSV row " + std::to_string(rowIndex));
+    }
+    return fileName;
+}
+
+
+void QuantizationDataSet::ParseCsvFile()
+{
+    std::map<unsigned int, QuantizationInput> passIdToQuantizationInput;
+    armnnUtils::CsvReader reader;
+
+    if (m_CsvFilePath == "")
+    {
+        throw armnn::Exception("CSV file not specified.");
+    }
+
+    // Parse CSV file and extract data
+    std::vector<armnnUtils::CsvRow> csvRows = reader.ParseFile(m_CsvFilePath);
+    if (csvRows.empty())
+    {
+        throw armnn::Exception("CSV file [" + m_CsvFilePath + "] is empty.");
+    }
+
+    for (unsigned int i = 0; i < csvRows.size(); ++i)
+    {
+        if (csvRows[i].values.size() != 3)
+        {
+            throw armnn::Exception("CSV file [" + m_CsvFilePath + "] does not have correct number of entries " +
+                                   "on line " + std::to_string(i) + ". Expected 3 entries " +
+                                   "but was " + std::to_string(csvRows[i].values.size()));
+        }
+
+        unsigned int passId = GetPassIdFromCsvRow(csvRows, i);
+        armnn::LayerBindingId bindingId = GetBindingIdFromCsvRow(csvRows, i);
+        std::string rawFileName = GetFileNameFromCsvRow(csvRows, i);
+
+        AddInputData(passId, bindingId, rawFileName, passIdToQuantizationInput);
+    }
+
+    if (passIdToQuantizationInput.empty())
+    {
+        throw armnn::Exception("Could not parse CSV file.");
+    }
+
+    // Once all entries in CSV file are parsed successfully and QuantizationInput map is populated, populate
+    // QuantizationInputs iterator for easier access and clear the map
+    for (auto itr = passIdToQuantizationInput.begin(); itr != passIdToQuantizationInput.end(); ++itr)
+    {
+        m_QuantizationInputs.emplace_back(itr->second);
+    }
+}
+
+}