IVGCVSW-2458 Refactor the Optimize function (Network.cpp) so that

subgraphs are optimized by the backends

 * Added a new method OptimizeSubGraph to the backend interface
 * Refactored the Optimize function so that the backend-specific
   optimization is performed by the backend itself (through the new
   OptimizeSubGraph interface method)
 * Added a new ApplyBackendOptimizations function to apply the new
   changes
 * Added some new convenient constructors to the SubGraph class
 * Added AddLayer method and a pointer to the parent graph to the
   SubGraph class
 * Updated the sub-graph unit tests to match the changes
 * Added SelectSubGraphs and ReplaceSubGraphConnections overloads
   that work with sub-graphs
 * Removed unused code and minor refactoring where necessary

Change-Id: I46181794c6a9e3b10558944f804e06a8f693a6d0

1 files changed, 128 insertions, 75 deletions

diff --git a/src/armnn/Network.cpp b/src/armnn/Network.cpp
index 662a9ccd3c..7897a81d1e 100644
--- a/src/armnn/Network.cpp
+++ b/src/armnn/Network.cpp
@@ -295,70 +295,112 @@ OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr,
     return result;
 }
 
-OptimizationResult InsertPreCompiledLayers(OptimizedNetwork* optNetObjPtr,
-                                           const IBackendInternalUniquePtr& backendObjPtr,
-                                           BackendSettings& backendSettings,
-                                           Optional<std::vector<std::string>&> errMessages)
+OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr,
+                                  BackendSettings& backendSettings,
+                                  SubGraph& subGraph,
+                                  Optional<std::vector<std::string>&> errMessages)
 {
-    BOOST_ASSERT(backendObjPtr);
+    Graph::Iterator firstLayer = subGraph.begin();
+    Graph::Iterator lastLayer  = subGraph.end();
+    return AssignBackends(optNetObjPtr,
+                          backendSettings,
+                          firstLayer,
+                          lastLayer,
+                          errMessages);
+}
+
+OptimizationResult ApplyBackendOptimizations(OptimizedNetwork* optNetObjPtr,
+                                             BackendSettings& backendSettings,
+                                             Optional<std::vector<std::string>&> errMessages)
+{
+    BOOST_ASSERT(optNetObjPtr);
 
     OptimizationResult result;
 
-    // Select sub-graphs based on backend
-    SubGraphSelector::SubGraphs subGraphs =
-            SubGraphSelector::SelectSubGraphs(optNetObjPtr->GetGraph(),
-                                              // select layers assigned to requested backend
-                                              [&](const Layer& layer)
-                                              {
-                                                  return layer.GetType() != LayerType::Input &&
-                                                         layer.GetType() != LayerType::Output &&
-                                                         layer.GetBackendId() == backendObjPtr->GetId();
-                                              });
-
-    if (subGraphs.empty())
-    {
-        // No sub-graphs found -> return with no error
-        return result;
-    }
+    // Get the optimized graph
+    Graph& optGraph = optNetObjPtr->GetGraph();
 
-    // Convert sub-graphs and substitute them with pre-compiled layers
-    unsigned int index = 0u;
-    for (auto& subGraph : subGraphs)
+    // Get the entire graph as a sub-graph
+    SubGraph mainSubGraph(optGraph);
+
+    // Run backend specific optimizations
+    auto const& backendRegistry = BackendRegistryInstance();
+    for (auto&& selectedBackend : backendSettings.m_SelectedBackends)
     {
-        // Create a pre-compiled layer
-        PreCompiledLayer* preCompiledLayer = CreatePreCompiledLayer(optNetObjPtr->GetGraph(),
-                                                                    *subGraph,
-                                                                    index++,
-                                                                    backendObjPtr);
-        if (preCompiledLayer)
+        auto backendFactory = backendRegistry.GetFactory(selectedBackend);
+        auto backendObjPtr  = backendFactory();
+        BOOST_ASSERT(backendObjPtr);
+
+        // Select sub-graphs based on backend
+        SubGraphSelector::SubGraphs subGraphs =
+                SubGraphSelector::SelectSubGraphs(mainSubGraph,
+                                                  // Select layers assigned to the requested backend
+                                                  [&backendObjPtr](const Layer& layer)
+                                                  {
+                                                      return layer.GetType() != LayerType::Input &&
+                                                             layer.GetType() != LayerType::Output &&
+                                                             layer.GetBackendId() == backendObjPtr->GetId();
+                                                  });
+        if (subGraphs.empty())
         {
-            // Substitute sub-graph with pre-compiled layer in graph
-            optNetObjPtr->GetGraph().SubstituteSubGraph(std::move(subGraph), preCompiledLayer);
+            // No sub-graphs found, try with next selected backend
+            continue;
         }
-        else
+
+        // Try to optimize each sub-graph
+        for (auto& subGraph : subGraphs)
         {
-            // Failed to create pre-compiled layer from sub-graph ->
-            // re-assign sub-graph layers to other available backends
-            std::stringstream warningMsg;
-            warningMsg << "Sub-graph #" << index << " failed to compile on "
-                       << backendObjPtr->GetId() << ". Re-assigning backends to "
-                       << subGraph->GetLayers().size() << " layers inside sub-graph";
-            ReportWarning(warningMsg.str(), errMessages);
-
-            backendSettings.m_IgnoredBackends = { backendObjPtr->GetId() };
-
-            Graph::Iterator firstLayer = subGraph->begin();
-            Graph::Iterator lastLayer  = subGraph->end();
-            OptimizationResult reassignmentResult = AssignBackends(optNetObjPtr,
-                                                                   backendSettings,
-                                                                   firstLayer,
-                                                                   lastLayer,
-                                                                   errMessages);
-
-            if (reassignmentResult.m_Error)
+            // Try to optimize the current sub-graph
+            bool optimizationAttempted = false;
+            SubGraph::SubGraphPtr optSubGraph = backendObjPtr->OptimizeSubGraph(*subGraph, optimizationAttempted);
+
+            // Check if the optimization has been attempted
+            if (!optimizationAttempted)
             {
-                result.m_Error = true;
-                return result;
+                // No optimization attempted, keep the current sub-graph as it is and move to the next one
+                continue;
+            }
+
+            // Optimization attempted, check the resulting optimized sub-graph
+            if (optSubGraph)
+            {
+                // Sub-graph optimized, substitute the sub-graph with the new optimized one in the main optimized graph
+                optGraph.SubstituteSubGraph(std::move(subGraph), *optSubGraph);
+
+                // Assign the current backend to the optimized sub-graph
+                std::for_each(optSubGraph->begin(), optSubGraph->end(), [&selectedBackend](Layer* l)
+                {
+                    BOOST_ASSERT(l);
+                    l->SetBackendId(selectedBackend);
+                });
+
+                // Recreate the sub-graph representing the entire graph
+                mainSubGraph.Update(optGraph);
+            }
+            else
+            {
+                // An error occurred: the optimization was attempted but not performed, try different backends
+                std::stringstream warningMsg;
+                warningMsg << "Sub-graph failed to get optimized on " << backendObjPtr->GetId() << ". "
+                           << "Re-assigning backends to " << subGraph->GetLayers().size() << " layers inside sub-graph";
+                ReportWarning(warningMsg.str(), errMessages);
+
+                // Failed to optimize the given sub-graph, re-assign the sub-graph layers to other available backends
+                if (!backendObjPtr->GetId().IsCpuRef())
+                {
+                    // Add the current backend to the list of backends to ignore
+                    backendSettings.m_IgnoredBackends.insert(backendObjPtr->GetId());
+                }
+                OptimizationResult reassignmentResult = AssignBackends(optNetObjPtr,
+                                                                       backendSettings,
+                                                                       *subGraph,
+                                                                       errMessages);
+                if (reassignmentResult.m_Error)
+                {
+                    // Failed to re-assign one of the remaining backends to each layer of the sub-graph
+                    result.m_Error = true;
+                    return result;
+                }
             }
         }
     }
@@ -384,22 +426,25 @@ IOptimizedNetworkPtr Optimize(const INetwork& inNetwork,
 
     OptimizedNetwork* optNetObjPtr = boost::polymorphic_downcast<OptimizedNetwork*>(optNet.get());
 
+    // Get the optimized graph
+    Graph& optGraph = optNetObjPtr->GetGraph();
+
     // Perform optimisation passes
     using namespace optimizations;
-    Optimizer::Pass(optNetObjPtr->GetGraph(), MakeOptimizations(SquashEqualPermuteSiblings(),
-                                                                SquashEqualReshapeSiblings(),
-                                                                OptimizeInversePermutes(),
-                                                                MovePermuteUp(),
-                                                                PermuteAsReshape(),
-                                                                OptimizeConsecutiveReshapes()));
+    Optimizer::Pass(optGraph, MakeOptimizations(SquashEqualPermuteSiblings(),
+                                                SquashEqualReshapeSiblings(),
+                                                OptimizeInversePermutes(),
+                                                MovePermuteUp(),
+                                                PermuteAsReshape(),
+                                                OptimizeConsecutiveReshapes()));
 
-    // Infer the tensor infos for all output slots. Throws an exception on failure.
-    optNetObjPtr->GetGraph().InferTensorInfos();
+    // Infer the tensor infos for all output slots. Throws an exception on failure
+    optGraph.InferTensorInfos();
 
     // If Fp32 to Fp16 optimization is set convert Fp32 network to Fp16
     if (options.m_ReduceFp32ToFp16)
     {
-        Optimizer::Pass(optNetObjPtr->GetGraph(), MakeOptimizations(Fp32NetworkToFp16Converter()));
+        Optimizer::Pass(optGraph, MakeOptimizations(Fp32NetworkToFp16Converter()));
     }
 
     // Initialize backend settings
@@ -414,8 +459,8 @@ IOptimizedNetworkPtr Optimize(const INetwork& inNetwork,
     }
 
     // Assign an available backend to each layer
-    Graph::Iterator firstLayer = optNetObjPtr->GetGraph().begin();
-    Graph::Iterator lastLayer  = optNetObjPtr->GetGraph().end();
+    Graph::Iterator firstLayer = optGraph.begin();
+    Graph::Iterator lastLayer  = optGraph.end();
     OptimizationResult assigBackendsResult = AssignBackends(optNetObjPtr,
                                                             backendSettings,
                                                             firstLayer,
@@ -427,22 +472,31 @@ IOptimizedNetworkPtr Optimize(const INetwork& inNetwork,
         return IOptimizedNetworkPtr(nullptr, &IOptimizedNetwork::Destroy);
     }
 
-    Optimizer::Pass(optNetObjPtr->GetGraph(), MakeOptimizations(OptimizeInverseConversionsFp16(),
-                                                                OptimizeInverseConversionsFp32()));
+    Optimizer::Pass(optGraph, MakeOptimizations(OptimizeInverseConversionsFp16(),
+                                                OptimizeInverseConversionsFp32()));
+
+    // Apply the backend-specific optimizations
+    OptimizationResult backendOptimizationResult = ApplyBackendOptimizations(optNetObjPtr,
+                                                                             backendSettings,
+                                                                             errMessages);
+    if (backendOptimizationResult.m_Error)
+    {
+        // Failed to apply the backend-specific optimizations
+        return IOptimizedNetworkPtr(nullptr, &IOptimizedNetwork::Destroy);
+    }
 
-    // If the debug flag is set, then insert a DebugLayer after each layer.
-    // NOTE: This optimization can only happen strictly after the PreCompiled layers have
-    //       already been inserted
+    // If the debug flag is set, then insert a DebugLayer after each layer
+    // Doing this after applying the backend optimizations as they might have changed some layers
     if (options.m_Debug)
     {
-        Optimizer::Pass(optNetObjPtr->GetGraph(), MakeOptimizations(InsertDebugLayer()));
+        Optimizer::Pass(optGraph, MakeOptimizations(InsertDebugLayer()));
     }
 
-    optNetObjPtr->GetGraph().AddCopyLayers();
+    optGraph.AddCopyLayers();
 
     // Convert constants
-    Optimizer::Pass(optNetObjPtr->GetGraph(), MakeOptimizations(ConvertConstantsFloatToHalf()));
-    Optimizer::Pass(optNetObjPtr->GetGraph(), MakeOptimizations(ConvertConstantsHalfToFloat()));
+    Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsFloatToHalf()));
+    Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsHalfToFloat()));
 
     // Run backend specific optimizations
     for (auto&& chosenBackend : backendSettings.m_SelectedBackends)
@@ -461,7 +515,6 @@ IOptimizedNetworkPtr Optimize(const INetwork& inNetwork,
     return optNet;
 }
 
-
 Network::Network()
 : m_Graph(std::make_unique<Graph>())
 {