IVGCVSW-4314 Per-layer backend hint API

Change-Id: I6ddcffe792e39b17fcdb8af7f13f4a689ef8019d
Signed-off-by: Derek Lamberti <derek.lamberti@arm.com>

6 files changed, 395 insertions, 108 deletions

diff --git a/src/armnn/BackendSettings.hpp b/src/armnn/BackendSettings.hpp
index e1344ab650..211af8b539 100644
--- a/src/armnn/BackendSettings.hpp
+++ b/src/armnn/BackendSettings.hpp
@@ -6,6 +6,9 @@
 #pragma once
 
 #include <armnn/BackendId.hpp>
+
+#include "DeviceSpec.hpp"
+
 #include <vector>
 
 namespace armnn
diff --git a/src/armnn/Layer.cpp b/src/armnn/Layer.cpp
index dee47f2cba..1f63d6ed3c 100644
--- a/src/armnn/Layer.cpp
+++ b/src/armnn/Layer.cpp
@@ -193,6 +193,7 @@ Layer::Layer(unsigned int numInputSlots,
 , m_LayerName(name ? name : "")
 , m_Type(type)
 , m_BackendId()
+, m_BackendHint(EmptyOptional())
 , m_Guid(profiling::ProfilingService::Instance().NextGuid())
 {
     boost::ignore_unused(layout);
diff --git a/src/armnn/Layer.hpp b/src/armnn/Layer.hpp
index c62b67ac0b..5ad38f0b9e 100644
--- a/src/armnn/Layer.hpp
+++ b/src/armnn/Layer.hpp
@@ -320,6 +320,13 @@ public:
     const std::list<std::string>& GetRelatedLayerNames() { return m_RelatedLayerNames; }
 
     virtual void Reparent(Graph& dest, std::list<Layer*>::const_iterator iterator) = 0;
+
+    void BackendSelectionHint(Optional<BackendId> backend) final
+    {
+        m_BackendHint = backend;
+    }
+    Optional<BackendId> GetBackendHint() const { return m_BackendHint; }
+
 protected:
     // Graph needs access to the virtual destructor.
     friend class Graph;
@@ -371,6 +378,7 @@ private:
 
     const LayerType m_Type;
     BackendId m_BackendId;
+    Optional<BackendId> m_BackendHint;
 
     /// Used for sorting.
     mutable LayerPriority m_Priority = 0;
diff --git a/src/armnn/Network.cpp b/src/armnn/Network.cpp
index b405a77829..55bf51af00 100644
--- a/src/armnn/Network.cpp
+++ b/src/armnn/Network.cpp
@@ -95,6 +95,21 @@ void ReportWarning(const std::string& warningMessage,
     }
 }
 
+OptimizationResult ReturnWithError(OptimizationResult res,
+                                   const Layer* layer,
+                                   const BackendSettings& backendSettings,
+                                   Optional<std::vector<std::string>&> errMessages)
+{
+    std::stringstream failureMsg;
+    failureMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType())
+               << " is not supported on any preferred backend " << backendSettings.m_PreferredBackends;
+    ReportError(failureMsg.str(), errMessages);
+
+    res.m_Error = true;
+    return res;
+}
+
+
 bool CheckScaleSetOnQuantizedType(Layer* layer, Optional<std::vector<std::string>&> errMessages)
 {
     bool noErrors = true;
@@ -130,6 +145,126 @@ bool CheckScaleSetOnQuantizedType(Layer* layer, Optional<std::vector<std::string
     return noErrors;
 }
 
+OptimizationResult AttemptBackendAssignment(BackendSettings& backendSettings,
+                                            Graph& graph,
+                                            Layer* layer,
+                                            BackendId backend,
+                                            DataType dataTypeIn,
+                                            DataType dataTypeOut,
+                                            const std::vector<BackendId>& availablePreferredBackends,
+                                            std::string& reasonIfUnsupported,
+                                            Optional<std::vector<std::string>&> errMessages)
+{
+    OptimizationResult result;
+
+    // Helper lambda to compose meaningful error message before returning with error
+    auto ReturnError = [&](const Layer* layer)
+        {
+            return ReturnWithError(result, layer, backendSettings, errMessages);
+        };
+
+    // need to set the compute device on the layer
+    // before we can check if it is supported
+    layer->SetBackendId(backend);
+    if (!IWorkloadFactory::IsLayerSupported(*layer, EmptyOptional(), reasonIfUnsupported))
+    {
+        if (dataTypeIn == DataType::Float16 || dataTypeOut == DataType::Float16)
+        {
+            if (IWorkloadFactory::IsLayerSupported(*layer, DataType::Float32, reasonIfUnsupported)
+                && layer->GetType() != LayerType::ConvertFp32ToFp16
+                && layer->GetType() != LayerType::ConvertFp16ToFp32)
+            {
+                // Insert FP16 -> FP32 conversion layer before current layer
+                std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers;
+                if (dataTypeIn == DataType::Float16)
+                {
+                    convertFp16ToFp32Layers =
+                        InsertConvertFp16ToFp32LayersBefore(graph, *layer);
+                }
+
+                // Insert FP32 -> FP16 conversion layer after current layer
+                std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers;
+                if (dataTypeOut == DataType::Float16)
+                {
+                    convertFp32ToFp16Layers =
+                        InsertConvertFp32ToFp16LayersAfter(graph, *layer);
+                }
+
+                // Assign a supported backend to the newly introduced conversion layers
+                auto AssignFirstSupportedBackend = [&](Layer* layer, BackendId preferredBackend)
+                    {
+                        bool supportedBackendFound = false;
+                        std::string reasonIfUnsupported;
+
+                        // Try preferred backend first
+                        layer->SetBackendId(preferredBackend);
+                        if (IWorkloadFactory::IsLayerSupported(*layer,
+                                                               EmptyOptional(),
+                                                               reasonIfUnsupported))
+                        {
+                            supportedBackendFound = true;
+                        }
+                        else
+                        {
+                            for (const auto& backend : availablePreferredBackends)
+                            {
+                                // Skip preferred backend (we already determined that it is not supported)
+                                if (backend == preferredBackend)
+                                {
+                                    continue;
+                                }
+
+                                layer->SetBackendId(backend);
+                                if (IWorkloadFactory::IsLayerSupported(*layer,
+                                                                       EmptyOptional(),
+                                                                       reasonIfUnsupported))
+                                {
+                                    supportedBackendFound = true;
+                                    break;
+                                }
+                            }
+                        }
+
+                        return supportedBackendFound;
+                    };
+
+                for (ConvertFp16ToFp32Layer* convertLayer : convertFp16ToFp32Layers)
+                {
+                    if (!AssignFirstSupportedBackend(convertLayer, backend))
+                    {
+                        return ReturnError(convertLayer);
+                    }
+                }
+
+                for (ConvertFp32ToFp16Layer* convertLayer : convertFp32ToFp16Layers)
+                {
+                    if (!AssignFirstSupportedBackend(convertLayer, backend))
+                    {
+                        return ReturnError(convertLayer);
+                    }
+                }
+
+                return result;
+            }
+        }
+        std::stringstream warningMsg;
+        warningMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType())
+                   << " is not supported on requested backend " << layer->GetBackendId().Get()
+                   << " for input data type " << GetDataTypeName(dataTypeIn)
+                   << " and output data type " << GetDataTypeName(dataTypeOut)
+                   << " (reason: " << reasonIfUnsupported
+                   << "), falling back to the next backend.";
+        ReportWarning(warningMsg.str(), errMessages);
+
+        return OptimizationResult(true, false);
+    }
+    else
+    {
+        return result;
+    }
+}
+
+
 OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr,
                                   BackendSettings& backendSettings,
                                   Graph::Iterator& firstLayer,
@@ -139,16 +274,11 @@ OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr,
     OptimizationResult result;
 
     // Helper lambda to compose meaningful error message before returning with error
-    auto ReturnWithError = [&](const Layer* layer)
-    {
-        std::stringstream failureMsg;
-        failureMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType())
-                   << " is not supported on any preferred backend " << backendSettings.m_PreferredBackends;
-        ReportError(failureMsg.str(), errMessages);
+    auto ReturnError = [&](const Layer* layer)
+        {
+            return ReturnWithError(result, layer, backendSettings, errMessages);
+        };
 
-        result.m_Error = true;
-        return result;
-    };
 
     auto availablePreferredBackends = backendSettings.GetAvailablePreferredBackends();
     if (availablePreferredBackends.empty())
@@ -179,107 +309,59 @@ OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr,
             result.m_Error = true;
         }
 
-        for (const auto& backend : availablePreferredBackends)
+        // First try assign layer to hint backend
+        if (layer->GetBackendHint().has_value() &&
+            backendSettings.IsBackendSupported(layer->GetBackendHint().value()) &&
+            AttemptBackendAssignment(backendSettings,
+                                     optNetObjPtr->GetGraph(),
+                                     layer,
+                                     layer->GetBackendHint().value(),
+                                     dataTypeIn,
+                                     dataTypeOut,
+                                     availablePreferredBackends,
+                                     reasonIfUnsupported,
+                                     errMessages).IsOk())
+        {
+            found = true;
+            backendSettings.m_SelectedBackends.insert(layer->GetBackendHint().value());
+        }
+        else
         {
-            // need to set the compute device on the layer
-            // before we can check if it is supported
-            layer->SetBackendId(backend);
-            if (!IWorkloadFactory::IsLayerSupported(*layer, EmptyOptional(), reasonIfUnsupported))
+            // Try assign layer to prefered list of backends
+            for (const auto& backend : availablePreferredBackends)
             {
-                if (dataTypeIn == DataType::Float16 || dataTypeOut == DataType::Float16)
+                if (layer->GetBackendHint().has_value() &&
+                    layer->GetBackendHint().value() == backend)
                 {
-                    if (IWorkloadFactory::IsLayerSupported(*layer, DataType::Float32, reasonIfUnsupported)
-                        && layer->GetType() != LayerType::ConvertFp32ToFp16
-                        && layer->GetType() != LayerType::ConvertFp16ToFp32)
-                    {
-                        // Insert FP16 -> FP32 conversion layer before current layer
-                        std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers;
-                        if (dataTypeIn == DataType::Float16)
-                        {
-                            convertFp16ToFp32Layers =
-                                InsertConvertFp16ToFp32LayersBefore(optNetObjPtr->GetGraph(), *layer);
-                        }
-
-                        // Insert FP32 -> FP16 conversion layer after current layer
-                        std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers;
-                        if (dataTypeOut == DataType::Float16)
-                        {
-                            convertFp32ToFp16Layers =
-                                InsertConvertFp32ToFp16LayersAfter(optNetObjPtr->GetGraph(), *layer);
-                        }
-
-                        // Assign a supported backend to the newly introduced conversion layers
-                        auto AssignFirstSupportedBackend = [&](Layer* layer, BackendId preferredBackend)
-                        {
-                            bool supportedBackendFound = false;
-                            std::string reasonIfUnsupported;
-
-                            // Try preferred backend first
-                            layer->SetBackendId(preferredBackend);
-                            if (IWorkloadFactory::IsLayerSupported(*layer,
-                                                                   EmptyOptional(),
-                                                                   reasonIfUnsupported))
-                            {
-                                supportedBackendFound = true;
-                            }
-                            else
-                            {
-                                for (const auto& backend : availablePreferredBackends)
-                                {
-                                    // Skip preferred backend (we already determined that it is not supported)
-                                    if (backend == preferredBackend)
-                                    {
-                                        continue;
-                                    }
-
-                                    layer->SetBackendId(backend);
-                                    if (IWorkloadFactory::IsLayerSupported(*layer,
-                                                                           EmptyOptional(),
-                                                                           reasonIfUnsupported))
-                                    {
-                                        supportedBackendFound = true;
-                                        break;
-                                    }
-                                }
-                            }
-
-                            return supportedBackendFound;
-                        };
-
-                        for (ConvertFp16ToFp32Layer* convertLayer : convertFp16ToFp32Layers)
-                        {
-                            if (!AssignFirstSupportedBackend(convertLayer, backend))
-                            {
-                                return ReturnWithError(convertLayer);
-                            }
-                        }
-
-                        for (ConvertFp32ToFp16Layer* convertLayer : convertFp32ToFp16Layers)
-                        {
-                            if (!AssignFirstSupportedBackend(convertLayer, backend))
-                            {
-                                return ReturnWithError(convertLayer);
-                            }
-                        }
+                    continue; //Don't re-test the backend hint
+                }
 
-                        found = true;
-                        break;
-                    }
+                OptimizationResult res = AttemptBackendAssignment(backendSettings,
+                                                                  optNetObjPtr->GetGraph(),
+                                                                  layer,
+                                                                  backend,
+                                                                  dataTypeIn,
+                                                                  dataTypeOut,
+                                                                  availablePreferredBackends,
+                                                                  reasonIfUnsupported,
+                                                                  errMessages);
+
+                if (res.IsOk())
+                {
+                    found = true;
+                    backendSettings.m_SelectedBackends.insert(backend);
+                    break;
+                }
+                else if (res.IsError())
+                {
+                   return res;  // Cannot continue.
+                   // Note: we don't need to log the error as it would already
+                   // be logged in AttemptBackendAssignment().
+                }
+                else
+                {
+                    BOOST_ASSERT_MSG(res.IsWarningOnly(), "OptimizationResult in unexpected state.");
                 }
-                std::stringstream warningMsg;
-                warningMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType())
-                           << " is not supported on requested backend " << layer->GetBackendId().Get()
-                           << " for input data type " << GetDataTypeName(dataTypeIn)
-                           << " and output data type " << GetDataTypeName(dataTypeOut)
-                           << " (reason: " << reasonIfUnsupported
-                           << "), falling back to the next backend.";
-                ReportWarning(warningMsg.str(), errMessages);
-            }
-            else
-            {
-                found = true;
-                backendSettings.m_SelectedBackends.insert(backend);
-                break;
             }
         }
 
@@ -301,7 +383,7 @@ OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr,
             }
             else
             {
-                return ReturnWithError(layer);
+                return ReturnError(layer);
             }
         }
     }
diff --git a/src/armnn/Network.hpp b/src/armnn/Network.hpp
index 5da681306c..089b46c9ca 100644
--- a/src/armnn/Network.hpp
+++ b/src/armnn/Network.hpp
@@ -17,6 +17,7 @@
 #include <map>
 #include <memory>
 
+#include "Graph.hpp"
 #include "Layer.hpp"
 
 namespace armnn
@@ -286,10 +287,19 @@ struct OptimizationResult
     bool m_Warning;
     bool m_Error;
 
+    OptimizationResult(bool warning, bool error)
+        : m_Warning(warning)
+        , m_Error(error)
+    {}
+
     OptimizationResult()
-        : m_Warning(false)
-        , m_Error(false)
+        : OptimizationResult(false, false)
     {}
+
+    bool IsOk() const { return !m_Warning && !m_Error; }
+    bool IsWarningOnly() const { return m_Warning && !m_Error; }
+    bool IsError() const { return m_Error; }
+
 };
 
 using BackendsMap = std::map<BackendId, std::unique_ptr<class IBackendInternal>>;
@@ -302,4 +312,10 @@ OptimizationResult SelectTensorHandleStrategy(Graph& optGraph,
                                               TensorHandleFactoryRegistry& registry,
                                               Optional<std::vector<std::string>&> errMessages);
 
+OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr,
+                                  BackendSettings& backendSettings,
+                                  Graph::Iterator& firstLayer,
+                                  Graph::Iterator& lastLayer,
+                                  Optional<std::vector<std::string>&> errMessages);
+
 } // namespace armnn
diff --git a/src/armnn/test/OptimizerTests.cpp b/src/armnn/test/OptimizerTests.cpp
index 7ceb104cb5..0ca4fc4764 100644
--- a/src/armnn/test/OptimizerTests.cpp
+++ b/src/armnn/test/OptimizerTests.cpp
@@ -5,13 +5,20 @@
 
 #include "TestUtils.hpp"
 
+#include <BackendSettings.hpp>
 #include <Graph.hpp>
+#include <Network.hpp>
 #include <Optimizer.hpp>
 
+#include <armnn/BackendRegistry.hpp>
+#include <armnn/INetwork.hpp>
+#include <armnn/LayerVisitorBase.hpp>
 
 #include <armnnUtils/FloatingPointConverter.hpp>
 
 #include <backendsCommon/CpuTensorHandle.hpp>
+#include <backendsCommon/IBackendInternal.hpp>
+#include <backendsCommon/LayerSupportBase.hpp>
 
 #include <boost/test/unit_test.hpp>
 
@@ -616,4 +623,174 @@ BOOST_AUTO_TEST_CASE(FoldPadLayerIntoConvolution2dLayer)
         &IsLayerOfType<armnn::OutputLayer>));
 }
 
+
+
+
+class MockLayerSupport : public LayerSupportBase {
+public:
+    bool IsInputSupported(const TensorInfo& /*input*/,
+                          Optional<std::string&> /*reasonIfUnsupported = EmptyOptional()*/) const override
+    {
+        return true;
+    }
+
+    bool IsOutputSupported(const TensorInfo& /*input*/,
+                           Optional<std::string&> /*reasonIfUnsupported = EmptyOptional()*/) const override
+    {
+        return true;
+    }
+
+    bool IsActivationSupported(const TensorInfo& /*input0*/,
+                               const TensorInfo& /*output*/,
+                               const ActivationDescriptor& /*descriptor*/,
+                               Optional<std::string&> /*reasonIfUnsupported = EmptyOptional()*/) const override
+    {
+        return true;
+    }
+};
+
+template<typename NamePolicy>
+class MockBackend : public IBackendInternal
+{
+public:
+    MockBackend()  = default;
+    ~MockBackend() = default;
+
+    static const BackendId& GetIdStatic() { return NamePolicy::GetIdStatic(); }
+    const BackendId& GetId() const override { return GetIdStatic(); }
+
+    IBackendInternal::IMemoryManagerUniquePtr CreateMemoryManager() const override { return nullptr; };
+
+    IBackendInternal::IWorkloadFactoryPtr CreateWorkloadFactory(
+        const IBackendInternal::IMemoryManagerSharedPtr&) const override { return nullptr; }
+
+    IBackendInternal::IBackendContextPtr CreateBackendContext(const IRuntime::CreationOptions&) const override
+    {
+        return nullptr;
+    }
+
+    IBackendInternal::Optimizations GetOptimizations() const override { return {}; }
+    IBackendInternal::ILayerSupportSharedPtr GetLayerSupport() const override
+    {
+        return std::make_shared<MockLayerSupport>();
+    }
+
+    OptimizationViews OptimizeSubgraphView(const SubgraphView&) const override
+    {
+        return {};
+    };
+};
+
+
+BOOST_AUTO_TEST_CASE(BackendHintTest)
+{
+    class TestBackendAssignment : public LayerVisitorBase<VisitorNoThrowPolicy>
+    {
+    public:
+        void VisitInputLayer(const IConnectableLayer* layer,
+                             LayerBindingId id,
+                             const char* name = nullptr) override
+        {
+            boost::ignore_unused(id, name);
+            auto inputLayer = boost::polymorphic_downcast<const InputLayer*>(layer);
+            BOOST_TEST((inputLayer->GetBackendId() == "MockBackend"));
+        }
+
+        void VisitOutputLayer(const IConnectableLayer* layer,
+                              LayerBindingId id,
+                              const char* name = nullptr) override
+        {
+            boost::ignore_unused(id, name);
+            auto outputLayer = boost::polymorphic_downcast<const OutputLayer*>(layer);
+            BOOST_TEST((outputLayer->GetBackendId() == "MockBackend"));
+        }
+
+        void VisitActivationLayer(const IConnectableLayer* layer,
+                                  const ActivationDescriptor& activationDescriptor,
+                                  const char* name = nullptr) override
+        {
+            boost::ignore_unused(activationDescriptor, name);
+            auto activation = boost::polymorphic_downcast<const ActivationLayer*>(layer);
+            BOOST_TEST((activation->GetBackendId() == "CustomBackend"));
+        }
+    };
+
+    struct CustomPolicy
+    {
+        static const BackendId& GetIdStatic()
+        {
+            static BackendId id="CustomBackend";
+            return id;
+        }
+    };
+
+    struct MockPolicy
+    {
+        static const BackendId& GetIdStatic()
+        {
+            static BackendId id="MockBackend";
+            return id;
+        }
+    };
+
+    auto& backendRegistry = BackendRegistryInstance();
+
+    backendRegistry.Register("MockBackend", [](){
+            return std::make_unique<MockBackend<MockPolicy>>();
+        });
+
+    backendRegistry.Register("CustomBackend", [](){
+            return std::make_unique<MockBackend<CustomPolicy>>();
+        });
+
+    // Define the network
+    auto network = INetwork::Create();
+    ActivationDescriptor desc;
+    desc.m_Function = ActivationFunction::Linear;
+
+    std::unique_ptr<Graph> graph = std::make_unique<Graph>();
+    auto input = graph->AddLayer<InputLayer>(0, "input");
+    auto act = graph->AddLayer<ActivationLayer>(desc, "activation");
+    auto output = graph->AddLayer<OutputLayer>(0, "output");
+
+    BackendId customBackendId("CustomBackend");
+    act->BackendSelectionHint(customBackendId);
+
+    input->GetOutputSlot(0).Connect(act->GetInputSlot(0));
+    act->GetOutputSlot(0).Connect(output->GetInputSlot(0));
+
+
+    auto optNet = IOptimizedNetworkPtr(new OptimizedNetwork(std::move(graph)), &IOptimizedNetwork::Destroy);
+
+    OptimizedNetwork* optNetObjPtr = boost::polymorphic_downcast<OptimizedNetwork*>(optNet.get());
+
+    // Get the optimized graph
+    Graph& optGraph = optNetObjPtr->GetGraph();
+
+
+    std::vector<BackendId> prefs{"MockBackend", "CustomBackend"};
+
+    BackendIdSet availableBackends = {"CustomBackend", "MockBackend"};
+    DeviceSpec spec(availableBackends);
+
+    BackendSettings backendSettings(prefs, spec);
+
+    // Assign an available backend to each layer
+    Graph::Iterator firstLayer = optGraph.begin();
+    Graph::Iterator lastLayer  = optGraph.end();
+    OptimizationResult res = AssignBackends(optNetObjPtr,
+                                            backendSettings,
+                                            firstLayer,
+                                            lastLayer,
+                                            EmptyOptional());
+
+    BOOST_TEST(res.IsOk());
+
+    TestBackendAssignment visitor;
+    for (auto it =firstLayer; it != lastLayer; ++it)
+    {
+        (*it)->Accept(visitor);
+    }
+}
+
 BOOST_AUTO_TEST_SUITE_END()