IVGCVSW-2564 Add support for multiple input and output bindings in InferenceModel

Change-Id: I64d724367d42dca4b768b6c6e42acda714985950

1 files changed, 234 insertions, 107 deletions

diff --git a/tests/InferenceModel.hpp b/tests/InferenceModel.hpp
index 5fefd05619..1c89238d32 100644
--- a/tests/InferenceModel.hpp
+++ b/tests/InferenceModel.hpp
@@ -27,6 +27,7 @@
 #include <fstream>
 #include <map>
 #include <string>
+#include <vector>
 #include <type_traits>
 
 namespace
@@ -73,20 +74,19 @@ using QuantizationParams = std::pair<float,int32_t>;
 
 struct Params
 {
-    std::string m_ModelPath;
-    std::string m_InputBinding;
-    std::string m_OutputBinding;
-    const armnn::TensorShape* m_InputTensorShape;
-    std::vector<armnn::BackendId> m_ComputeDevice;
-    bool m_EnableProfiling;
-    size_t m_SubgraphId;
-    bool m_IsModelBinary;
-    bool m_VisualizePostOptimizationModel;
-    bool m_EnableFp16TurboMode;
+    std::string                     m_ModelPath;
+    std::vector<std::string>        m_InputBindings;
+    std::vector<armnn::TensorShape> m_InputShapes;
+    std::vector<std::string>        m_OutputBindings;
+    std::vector<armnn::BackendId>   m_ComputeDevice;
+    bool                            m_EnableProfiling;
+    size_t                          m_SubgraphId;
+    bool                            m_IsModelBinary;
+    bool                            m_VisualizePostOptimizationModel;
+    bool                            m_EnableFp16TurboMode;
 
     Params()
-        : m_InputTensorShape(nullptr)
-        , m_ComputeDevice{armnn::Compute::CpuRef}
+        : m_ComputeDevice{armnn::Compute::CpuRef}
         , m_EnableProfiling(false)
         , m_SubgraphId(0)
         , m_IsModelBinary(true)
@@ -105,33 +105,54 @@ public:
     using BindingPointInfo = InferenceModelInternal::BindingPointInfo;
 
     static armnn::INetworkPtr Create(const Params& params,
-                                     BindingPointInfo& inputBindings,
-                                     BindingPointInfo& outputBindings)
+                                     std::vector<BindingPointInfo>& inputBindings,
+                                     std::vector<BindingPointInfo>& outputBindings)
     {
-      const std::string& modelPath = params.m_ModelPath;
-
-      // Create a network from a file on disk
-      auto parser(IParser::Create());
-
-      std::map<std::string, armnn::TensorShape> inputShapes;
-      if (params.m_InputTensorShape)
-      {
-          inputShapes[params.m_InputBinding] = *params.m_InputTensorShape;
-      }
-      std::vector<std::string> requestedOutputs{ params.m_OutputBinding };
-      armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
-
-      {
-          ARMNN_SCOPED_HEAP_PROFILING("Parsing");
-          // Handle text and binary input differently by calling the corresponding parser function
-          network = (params.m_IsModelBinary ?
-              parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) :
-              parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs));
-      }
-
-      inputBindings  = parser->GetNetworkInputBindingInfo(params.m_InputBinding);
-      outputBindings = parser->GetNetworkOutputBindingInfo(params.m_OutputBinding);
-      return network;
+        const std::string& modelPath = params.m_ModelPath;
+
+        // Create a network from a file on disk
+        auto parser(IParser::Create());
+
+        std::map<std::string, armnn::TensorShape> inputShapes;
+        if (!params.m_InputShapes.empty())
+        {
+            const size_t numInputShapes   = params.m_InputShapes.size();
+            const size_t numInputBindings = params.m_InputBindings.size();
+            if (numInputShapes < numInputBindings)
+            {
+                throw armnn::Exception(boost::str(boost::format(
+                    "Not every input has its tensor shape specified: expected=%1%, got=%2%")
+                    % numInputBindings % numInputShapes));
+            }
+
+            for (size_t i = 0; i < numInputShapes; i++)
+            {
+                inputShapes[params.m_InputBindings[i]] = params.m_InputShapes[i];
+            }
+        }
+
+        std::vector<std::string> requestedOutputs = params.m_OutputBindings;
+        armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
+
+        {
+            ARMNN_SCOPED_HEAP_PROFILING("Parsing");
+            // Handle text and binary input differently by calling the corresponding parser function
+            network = (params.m_IsModelBinary ?
+                parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) :
+                parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs));
+        }
+
+        for (const std::string& inputLayerName : params.m_InputBindings)
+        {
+            inputBindings.push_back(parser->GetNetworkInputBindingInfo(inputLayerName));
+        }
+
+        for (const std::string& outputLayerName : params.m_OutputBindings)
+        {
+            outputBindings.push_back(parser->GetNetworkOutputBindingInfo(outputLayerName));
+        }
+
+        return network;
     }
 };
 
@@ -145,24 +166,36 @@ public:
     using BindingPointInfo = InferenceModelInternal::BindingPointInfo;
 
     static armnn::INetworkPtr Create(const Params& params,
-                                     BindingPointInfo& inputBindings,
-                                     BindingPointInfo& outputBindings)
+                                     std::vector<BindingPointInfo>& inputBindings,
+                                     std::vector<BindingPointInfo>& outputBindings)
     {
-      const std::string& modelPath = params.m_ModelPath;
+        const std::string& modelPath = params.m_ModelPath;
 
-      // Create a network from a file on disk
-      auto parser(IParser::Create());
+        // Create a network from a file on disk
+        auto parser(IParser::Create());
 
-      armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
+        armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
 
-      {
-          ARMNN_SCOPED_HEAP_PROFILING("Parsing");
-          network = parser->CreateNetworkFromBinaryFile(modelPath.c_str());
-      }
+        {
+            ARMNN_SCOPED_HEAP_PROFILING("Parsing");
+            network = parser->CreateNetworkFromBinaryFile(modelPath.c_str());
+        }
+
+        for (const std::string& inputLayerName : params.m_InputBindings)
+        {
+            BindingPointInfo inputBinding =
+                parser->GetNetworkInputBindingInfo(params.m_SubgraphId, inputLayerName);
+            inputBindings.push_back(inputBinding);
+        }
 
-      inputBindings  = parser->GetNetworkInputBindingInfo(params.m_SubgraphId, params.m_InputBinding);
-      outputBindings = parser->GetNetworkOutputBindingInfo(params.m_SubgraphId, params.m_OutputBinding);
-      return network;
+        for (const std::string& outputLayerName : params.m_OutputBindings)
+        {
+            BindingPointInfo outputBinding =
+                parser->GetNetworkOutputBindingInfo(params.m_SubgraphId, outputLayerName);
+            outputBindings.push_back(outputBinding);
+        }
+
+        return network;
     }
 };
 #endif
@@ -177,67 +210,111 @@ public:
     using BindingPointInfo = InferenceModelInternal::BindingPointInfo;
 
     static armnn::INetworkPtr Create(const Params& params,
-                                     BindingPointInfo& inputBindings,
-                                     BindingPointInfo& outputBindings)
+                                     std::vector<BindingPointInfo>& inputBindings,
+                                     std::vector<BindingPointInfo>& outputBindings)
     {
-      const std::string& modelPath = params.m_ModelPath;
+        const std::string& modelPath = params.m_ModelPath;
+
+        // Create a network from a file on disk
+        auto parser(IParser::Create());
+
+        armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
 
-      // Create a network from a file on disk
-      auto parser(IParser::Create());
+        {
+            ARMNN_SCOPED_HEAP_PROFILING("Parsing");
+            network = (params.m_IsModelBinary ?
+                parser->CreateNetworkFromBinaryFile(modelPath.c_str()) :
+                parser->CreateNetworkFromTextFile(modelPath.c_str()));
+        }
 
-      armnn::INetworkPtr network{nullptr, [](armnn::INetwork *){}};
+        for (const std::string& inputLayerName : params.m_InputBindings)
+        {
+            BindingPointInfo inputBinding = parser->GetNetworkInputBindingInfo(inputLayerName);
+            inputBindings.push_back(inputBinding);
+        }
 
-      {
-          ARMNN_SCOPED_HEAP_PROFILING("Parsing");
-          network = (params.m_IsModelBinary ?
-              parser->CreateNetworkFromBinaryFile(modelPath.c_str()) :
-              parser->CreateNetworkFromTextFile(modelPath.c_str()));
-      }
+        for (const std::string& outputLayerName : params.m_OutputBindings)
+        {
+            BindingPointInfo outputBinding = parser->GetNetworkOutputBindingInfo(outputLayerName);
+            outputBindings.push_back(outputBinding);
+        }
 
-      inputBindings  = parser->GetNetworkInputBindingInfo(params.m_InputBinding);
-      outputBindings = parser->GetNetworkOutputBindingInfo(params.m_OutputBinding);
-      return network;
+        return network;
     }
 };
 #endif
 
 template<typename TContainer>
-inline armnn::InputTensors MakeInputTensors(const InferenceModelInternal::BindingPointInfo& input,
-    const TContainer& inputTensorData)
+inline armnn::InputTensors MakeInputTensors(
+    const std::vector<InferenceModelInternal::BindingPointInfo>& inputBindings,
+    const std::vector<TContainer>& inputDataContainers)
 {
-    if (inputTensorData.size() != input.second.GetNumElements())
+    armnn::InputTensors inputTensors;
+
+    const size_t numInputs = inputBindings.size();
+    if (numInputs != inputDataContainers.size())
     {
-        try
-        {
-            throw armnn::Exception(boost::str(boost::format("Input tensor has incorrect size. Expected %1% elements "
-                "but got %2%.") % input.second.GetNumElements() % inputTensorData.size()));
-        } catch (const boost::exception& e)
+        throw armnn::Exception(boost::str(boost::format("Number of inputs does not match number of "
+            "tensor data containers: %1% != %2%") % numInputs % inputDataContainers.size()));
+    }
+
+    for (size_t i = 0; i < numInputs; i++)
+    {
+        const InferenceModelInternal::BindingPointInfo& inputBinding = inputBindings[i];
+        const TContainer& inputData = inputDataContainers[i];
+
+        if (inputData.size() != inputBinding.second.GetNumElements())
         {
-            // Coverity fix: it should not be possible to get here but boost::str and boost::format can both
-            // throw uncaught exceptions, convert them to armnn exceptions and rethrow.
-            throw armnn::Exception(diagnostic_information(e));
+            throw armnn::Exception("Input tensor has incorrect size");
         }
+
+        armnn::ConstTensor inputTensor(inputBinding.second, inputData.data());
+        inputTensors.push_back(std::make_pair(inputBinding.first, inputTensor));
     }
-    return { { input.first, armnn::ConstTensor(input.second, inputTensorData.data()) } };
+
+    return inputTensors;
 }
 
 template<typename TContainer>
-inline armnn::OutputTensors MakeOutputTensors(const InferenceModelInternal::BindingPointInfo& output,
-    TContainer& outputTensorData)
+inline armnn::OutputTensors MakeOutputTensors(
+    const std::vector<InferenceModelInternal::BindingPointInfo>& outputBindings,
+    std::vector<TContainer>& outputDataContainers)
 {
-    if (outputTensorData.size() != output.second.GetNumElements())
+    armnn::OutputTensors outputTensors;
+
+    const size_t numOutputs = outputBindings.size();
+    if (numOutputs != outputDataContainers.size())
     {
-        throw armnn::Exception("Output tensor has incorrect size");
+        throw armnn::Exception(boost::str(boost::format("Number of outputs does not match number of "
+            "tensor data containers: %1% != %2%") % numOutputs % outputDataContainers.size()));
+    }
+
+    for (size_t i = 0; i < numOutputs; i++)
+    {
+        const InferenceModelInternal::BindingPointInfo& outputBinding = outputBindings[i];
+        TContainer& outputData = outputDataContainers[i];
+
+        if (outputData.size() != outputBinding.second.GetNumElements())
+        {
+            throw armnn::Exception("Output tensor has incorrect size");
+        }
+
+        armnn::Tensor outputTensor(outputBinding.second, outputData.data());
+        outputTensors.push_back(std::make_pair(outputBinding.first, outputTensor));
     }
-    return { { output.first, armnn::Tensor(output.second, outputTensorData.data()) } };
+
+    return outputTensors;
 }
 
 template <typename IParser, typename TDataType>
 class InferenceModel
 {
 public:
-    using DataType = TDataType;
-    using Params = InferenceModelInternal::Params;
+    using DataType           = TDataType;
+    using Params             = InferenceModelInternal::Params;
+    using BindingPointInfo   = InferenceModelInternal::BindingPointInfo;
+    using QuantizationParams = InferenceModelInternal::QuantizationParams;
+    using TContainer         = std::vector<TDataType>;
 
     struct CommandLineOptions
     {
@@ -290,8 +367,8 @@ public:
             throw armnn::Exception("Some backend IDs are invalid: " + invalidBackends);
         }
 
-        armnn::INetworkPtr network = CreateNetworkImpl<IParser>::Create(params, m_InputBindingInfo,
-           m_OutputBindingInfo);
+        armnn::INetworkPtr network =
+            CreateNetworkImpl<IParser>::Create(params, m_InputBindings, m_OutputBindings);
 
         armnn::IOptimizedNetworkPtr optNet{nullptr, [](armnn::IOptimizedNetwork *){}};
         {
@@ -327,14 +404,41 @@ public:
         }
     }
 
-    unsigned int GetOutputSize() const
+    void CheckInputIndexIsValid(unsigned int inputIndex) const
+    {
+        if (m_InputBindings.size() < inputIndex + 1)
+        {
+            throw armnn::Exception(boost::str(boost::format("Input index out of range: %1%") % inputIndex));
+        }
+    }
+
+    void CheckOutputIndexIsValid(unsigned int outputIndex) const
+    {
+        if (m_OutputBindings.size() < outputIndex + 1)
+        {
+            throw armnn::Exception(boost::str(boost::format("Output index out of range: %1%") % outputIndex));
+        }
+    }
+
+    unsigned int GetOutputSize(unsigned int outputIndex = 0u) const
     {
-        return m_OutputBindingInfo.second.GetNumElements();
+        CheckOutputIndexIsValid(outputIndex);
+        return m_OutputBindings[outputIndex].second.GetNumElements();
     }
 
-    void Run(const std::vector<TDataType>& input, std::vector<TDataType>& output)
+    void Run(const std::vector<TContainer>& inputContainers, std::vector<TContainer>& outputContainers)
     {
-        BOOST_ASSERT(output.size() == GetOutputSize());
+        for (unsigned int i = 0; i < outputContainers.size(); i++)
+        {
+            const unsigned int expectedOutputDataSize = GetOutputSize(i);
+            const unsigned int actualOutputDataSize   = boost::numeric_cast<unsigned int>(outputContainers[i].size());
+            if (actualOutputDataSize < expectedOutputDataSize)
+            {
+                unsigned int outputIndex = boost::numeric_cast<unsigned int>(i);
+                throw armnn::Exception(boost::str(boost::format("Not enough data for output #%1%: expected "
+                    "%2% elements, got %3%") % outputIndex % expectedOutputDataSize % actualOutputDataSize));
+            }
+        }
 
         std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
         if (profiler)
@@ -343,8 +447,8 @@ public:
         }
 
         armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
-                                                       MakeInputTensors(input),
-                                                       MakeOutputTensors(output));
+                                                       MakeInputTensors(inputContainers),
+                                                       MakeOutputTensors(outputContainers));
 
         // if profiling is enabled print out the results
         if (profiler && profiler->IsProfilingEnabled())
@@ -358,39 +462,62 @@ public:
         }
     }
 
-    const InferenceModelInternal::BindingPointInfo & GetInputBindingInfo() const
+    const BindingPointInfo& GetInputBindingInfo(unsigned int inputIndex = 0u) const
+    {
+        CheckInputIndexIsValid(inputIndex);
+        return m_InputBindings[inputIndex];
+    }
+
+    const std::vector<BindingPointInfo>& GetInputBindingInfos() const
+    {
+        return m_InputBindings;
+    }
+
+    const BindingPointInfo& GetOutputBindingInfo(unsigned int outputIndex = 0u) const
+    {
+        CheckOutputIndexIsValid(outputIndex);
+        return m_OutputBindings[outputIndex];
+    }
+
+    const std::vector<BindingPointInfo>& GetOutputBindingInfos() const
     {
-        return m_InputBindingInfo;
+        return m_OutputBindings;
     }
 
-    const InferenceModelInternal::BindingPointInfo & GetOutputBindingInfo() const
+    QuantizationParams GetQuantizationParams(unsigned int outputIndex = 0u) const
     {
-        return m_OutputBindingInfo;
+        CheckOutputIndexIsValid(outputIndex);
+        return std::make_pair(m_OutputBindings[outputIndex].second.GetQuantizationScale(),
+                              m_OutputBindings[outputIndex].second.GetQuantizationOffset());
     }
 
-    InferenceModelInternal::QuantizationParams GetQuantizationParams() const
+    std::vector<QuantizationParams> GetAllQuantizationParams() const
     {
-        return std::make_pair(m_OutputBindingInfo.second.GetQuantizationScale(),
-                              m_OutputBindingInfo.second.GetQuantizationOffset());
+        std::vector<QuantizationParams> quantizationParams;
+        for (unsigned int i = 0u; i < m_OutputBindings.size(); i++)
+        {
+            quantizationParams.push_back(GetQuantizationParams(i));
+        }
+        return quantizationParams;
     }
 
 private:
     armnn::NetworkId m_NetworkIdentifier;
     std::shared_ptr<armnn::IRuntime> m_Runtime;
 
-    InferenceModelInternal::BindingPointInfo m_InputBindingInfo;
-    InferenceModelInternal::BindingPointInfo m_OutputBindingInfo;
+    std::vector<InferenceModelInternal::BindingPointInfo> m_InputBindings;
+    std::vector<InferenceModelInternal::BindingPointInfo> m_OutputBindings;
     bool m_EnableProfiling;
 
     template<typename TContainer>
-    armnn::InputTensors MakeInputTensors(const TContainer& inputTensorData)
+    armnn::InputTensors MakeInputTensors(const std::vector<TContainer>& inputDataContainers)
     {
-        return ::MakeInputTensors(m_InputBindingInfo, inputTensorData);
+        return ::MakeInputTensors(m_InputBindings, inputDataContainers);
     }
 
     template<typename TContainer>
-    armnn::OutputTensors MakeOutputTensors(TContainer& outputTensorData)
+    armnn::OutputTensors MakeOutputTensors(std::vector<TContainer>& outputDataContainers)
     {
-        return ::MakeOutputTensors(m_OutputBindingInfo, outputTensorData);
+        return ::MakeOutputTensors(m_OutputBindings, outputDataContainers);
     }
-};
+};
\ No newline at end of file