diff options
author | Derek Lamberti <derek.lamberti@arm.com> | 2020-01-03 16:53:38 +0000 |
---|---|---|
committer | Derek Lamberti <derek.lamberti@arm.com> | 2020-03-03 14:56:00 +0000 |
commit | 4a9e24bfc51eec7e593470091fb7e6e435ae3991 (patch) | |
tree | a9be306fe10d6cd7d07fb06817c224e5e5eb8ada /src/armnn | |
parent | 9077473e0db8548506360a5196a1514f785332bb (diff) | |
download | armnn-4a9e24bfc51eec7e593470091fb7e6e435ae3991.tar.gz |
IVGCVSW-4314 Per-layer backend hint API
Change-Id: I6ddcffe792e39b17fcdb8af7f13f4a689ef8019d
Signed-off-by: Derek Lamberti <derek.lamberti@arm.com>
Diffstat (limited to 'src/armnn')
-rw-r--r-- | src/armnn/BackendSettings.hpp | 3 | ||||
-rw-r--r-- | src/armnn/Layer.cpp | 1 | ||||
-rw-r--r-- | src/armnn/Layer.hpp | 8 | ||||
-rw-r--r-- | src/armnn/Network.cpp | 294 | ||||
-rw-r--r-- | src/armnn/Network.hpp | 20 | ||||
-rw-r--r-- | src/armnn/test/OptimizerTests.cpp | 177 |
6 files changed, 395 insertions, 108 deletions
diff --git a/src/armnn/BackendSettings.hpp b/src/armnn/BackendSettings.hpp index e1344ab650..211af8b539 100644 --- a/src/armnn/BackendSettings.hpp +++ b/src/armnn/BackendSettings.hpp @@ -6,6 +6,9 @@ #pragma once #include <armnn/BackendId.hpp> + +#include "DeviceSpec.hpp" + #include <vector> namespace armnn diff --git a/src/armnn/Layer.cpp b/src/armnn/Layer.cpp index dee47f2cba..1f63d6ed3c 100644 --- a/src/armnn/Layer.cpp +++ b/src/armnn/Layer.cpp @@ -193,6 +193,7 @@ Layer::Layer(unsigned int numInputSlots, , m_LayerName(name ? name : "") , m_Type(type) , m_BackendId() +, m_BackendHint(EmptyOptional()) , m_Guid(profiling::ProfilingService::Instance().NextGuid()) { boost::ignore_unused(layout); diff --git a/src/armnn/Layer.hpp b/src/armnn/Layer.hpp index c62b67ac0b..5ad38f0b9e 100644 --- a/src/armnn/Layer.hpp +++ b/src/armnn/Layer.hpp @@ -320,6 +320,13 @@ public: const std::list<std::string>& GetRelatedLayerNames() { return m_RelatedLayerNames; } virtual void Reparent(Graph& dest, std::list<Layer*>::const_iterator iterator) = 0; + + void BackendSelectionHint(Optional<BackendId> backend) final + { + m_BackendHint = backend; + } + Optional<BackendId> GetBackendHint() const { return m_BackendHint; } + protected: // Graph needs access to the virtual destructor. friend class Graph; @@ -371,6 +378,7 @@ private: const LayerType m_Type; BackendId m_BackendId; + Optional<BackendId> m_BackendHint; /// Used for sorting. mutable LayerPriority m_Priority = 0; diff --git a/src/armnn/Network.cpp b/src/armnn/Network.cpp index b405a77829..55bf51af00 100644 --- a/src/armnn/Network.cpp +++ b/src/armnn/Network.cpp @@ -95,6 +95,21 @@ void ReportWarning(const std::string& warningMessage, } } +OptimizationResult ReturnWithError(OptimizationResult res, + const Layer* layer, + const BackendSettings& backendSettings, + Optional<std::vector<std::string>&> errMessages) +{ + std::stringstream failureMsg; + failureMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType()) + << " is not supported on any preferred backend " << backendSettings.m_PreferredBackends; + ReportError(failureMsg.str(), errMessages); + + res.m_Error = true; + return res; +} + + bool CheckScaleSetOnQuantizedType(Layer* layer, Optional<std::vector<std::string>&> errMessages) { bool noErrors = true; @@ -130,6 +145,126 @@ bool CheckScaleSetOnQuantizedType(Layer* layer, Optional<std::vector<std::string return noErrors; } +OptimizationResult AttemptBackendAssignment(BackendSettings& backendSettings, + Graph& graph, + Layer* layer, + BackendId backend, + DataType dataTypeIn, + DataType dataTypeOut, + const std::vector<BackendId>& availablePreferredBackends, + std::string& reasonIfUnsupported, + Optional<std::vector<std::string>&> errMessages) +{ + OptimizationResult result; + + // Helper lambda to compose meaningful error message before returning with error + auto ReturnError = [&](const Layer* layer) + { + return ReturnWithError(result, layer, backendSettings, errMessages); + }; + + // need to set the compute device on the layer + // before we can check if it is supported + layer->SetBackendId(backend); + if (!IWorkloadFactory::IsLayerSupported(*layer, EmptyOptional(), reasonIfUnsupported)) + { + if (dataTypeIn == DataType::Float16 || dataTypeOut == DataType::Float16) + { + if (IWorkloadFactory::IsLayerSupported(*layer, DataType::Float32, reasonIfUnsupported) + && layer->GetType() != LayerType::ConvertFp32ToFp16 + && layer->GetType() != LayerType::ConvertFp16ToFp32) + { + // Insert FP16 -> FP32 conversion layer before current layer + std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers; + if (dataTypeIn == DataType::Float16) + { + convertFp16ToFp32Layers = + InsertConvertFp16ToFp32LayersBefore(graph, *layer); + } + + // Insert FP32 -> FP16 conversion layer after current layer + std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers; + if (dataTypeOut == DataType::Float16) + { + convertFp32ToFp16Layers = + InsertConvertFp32ToFp16LayersAfter(graph, *layer); + } + + // Assign a supported backend to the newly introduced conversion layers + auto AssignFirstSupportedBackend = [&](Layer* layer, BackendId preferredBackend) + { + bool supportedBackendFound = false; + std::string reasonIfUnsupported; + + // Try preferred backend first + layer->SetBackendId(preferredBackend); + if (IWorkloadFactory::IsLayerSupported(*layer, + EmptyOptional(), + reasonIfUnsupported)) + { + supportedBackendFound = true; + } + else + { + for (const auto& backend : availablePreferredBackends) + { + // Skip preferred backend (we already determined that it is not supported) + if (backend == preferredBackend) + { + continue; + } + + layer->SetBackendId(backend); + if (IWorkloadFactory::IsLayerSupported(*layer, + EmptyOptional(), + reasonIfUnsupported)) + { + supportedBackendFound = true; + break; + } + } + } + + return supportedBackendFound; + }; + + for (ConvertFp16ToFp32Layer* convertLayer : convertFp16ToFp32Layers) + { + if (!AssignFirstSupportedBackend(convertLayer, backend)) + { + return ReturnError(convertLayer); + } + } + + for (ConvertFp32ToFp16Layer* convertLayer : convertFp32ToFp16Layers) + { + if (!AssignFirstSupportedBackend(convertLayer, backend)) + { + return ReturnError(convertLayer); + } + } + + return result; + } + } + std::stringstream warningMsg; + warningMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType()) + << " is not supported on requested backend " << layer->GetBackendId().Get() + << " for input data type " << GetDataTypeName(dataTypeIn) + << " and output data type " << GetDataTypeName(dataTypeOut) + << " (reason: " << reasonIfUnsupported + << "), falling back to the next backend."; + ReportWarning(warningMsg.str(), errMessages); + + return OptimizationResult(true, false); + } + else + { + return result; + } +} + + OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr, BackendSettings& backendSettings, Graph::Iterator& firstLayer, @@ -139,16 +274,11 @@ OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr, OptimizationResult result; // Helper lambda to compose meaningful error message before returning with error - auto ReturnWithError = [&](const Layer* layer) - { - std::stringstream failureMsg; - failureMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType()) - << " is not supported on any preferred backend " << backendSettings.m_PreferredBackends; - ReportError(failureMsg.str(), errMessages); + auto ReturnError = [&](const Layer* layer) + { + return ReturnWithError(result, layer, backendSettings, errMessages); + }; - result.m_Error = true; - return result; - }; auto availablePreferredBackends = backendSettings.GetAvailablePreferredBackends(); if (availablePreferredBackends.empty()) @@ -179,107 +309,59 @@ OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr, result.m_Error = true; } - for (const auto& backend : availablePreferredBackends) + // First try assign layer to hint backend + if (layer->GetBackendHint().has_value() && + backendSettings.IsBackendSupported(layer->GetBackendHint().value()) && + AttemptBackendAssignment(backendSettings, + optNetObjPtr->GetGraph(), + layer, + layer->GetBackendHint().value(), + dataTypeIn, + dataTypeOut, + availablePreferredBackends, + reasonIfUnsupported, + errMessages).IsOk()) + { + found = true; + backendSettings.m_SelectedBackends.insert(layer->GetBackendHint().value()); + } + else { - // need to set the compute device on the layer - // before we can check if it is supported - layer->SetBackendId(backend); - if (!IWorkloadFactory::IsLayerSupported(*layer, EmptyOptional(), reasonIfUnsupported)) + // Try assign layer to prefered list of backends + for (const auto& backend : availablePreferredBackends) { - if (dataTypeIn == DataType::Float16 || dataTypeOut == DataType::Float16) + if (layer->GetBackendHint().has_value() && + layer->GetBackendHint().value() == backend) { - if (IWorkloadFactory::IsLayerSupported(*layer, DataType::Float32, reasonIfUnsupported) - && layer->GetType() != LayerType::ConvertFp32ToFp16 - && layer->GetType() != LayerType::ConvertFp16ToFp32) - { - // Insert FP16 -> FP32 conversion layer before current layer - std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers; - if (dataTypeIn == DataType::Float16) - { - convertFp16ToFp32Layers = - InsertConvertFp16ToFp32LayersBefore(optNetObjPtr->GetGraph(), *layer); - } - - // Insert FP32 -> FP16 conversion layer after current layer - std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers; - if (dataTypeOut == DataType::Float16) - { - convertFp32ToFp16Layers = - InsertConvertFp32ToFp16LayersAfter(optNetObjPtr->GetGraph(), *layer); - } - - // Assign a supported backend to the newly introduced conversion layers - auto AssignFirstSupportedBackend = [&](Layer* layer, BackendId preferredBackend) - { - bool supportedBackendFound = false; - std::string reasonIfUnsupported; - - // Try preferred backend first - layer->SetBackendId(preferredBackend); - if (IWorkloadFactory::IsLayerSupported(*layer, - EmptyOptional(), - reasonIfUnsupported)) - { - supportedBackendFound = true; - } - else - { - for (const auto& backend : availablePreferredBackends) - { - // Skip preferred backend (we already determined that it is not supported) - if (backend == preferredBackend) - { - continue; - } - - layer->SetBackendId(backend); - if (IWorkloadFactory::IsLayerSupported(*layer, - EmptyOptional(), - reasonIfUnsupported)) - { - supportedBackendFound = true; - break; - } - } - } - - return supportedBackendFound; - }; - - for (ConvertFp16ToFp32Layer* convertLayer : convertFp16ToFp32Layers) - { - if (!AssignFirstSupportedBackend(convertLayer, backend)) - { - return ReturnWithError(convertLayer); - } - } - - for (ConvertFp32ToFp16Layer* convertLayer : convertFp32ToFp16Layers) - { - if (!AssignFirstSupportedBackend(convertLayer, backend)) - { - return ReturnWithError(convertLayer); - } - } + continue; //Don't re-test the backend hint + } - found = true; - break; - } + OptimizationResult res = AttemptBackendAssignment(backendSettings, + optNetObjPtr->GetGraph(), + layer, + backend, + dataTypeIn, + dataTypeOut, + availablePreferredBackends, + reasonIfUnsupported, + errMessages); + + if (res.IsOk()) + { + found = true; + backendSettings.m_SelectedBackends.insert(backend); + break; + } + else if (res.IsError()) + { + return res; // Cannot continue. + // Note: we don't need to log the error as it would already + // be logged in AttemptBackendAssignment(). + } + else + { + BOOST_ASSERT_MSG(res.IsWarningOnly(), "OptimizationResult in unexpected state."); } - std::stringstream warningMsg; - warningMsg << "Layer of type " << GetLayerTypeAsCString(layer->GetType()) - << " is not supported on requested backend " << layer->GetBackendId().Get() - << " for input data type " << GetDataTypeName(dataTypeIn) - << " and output data type " << GetDataTypeName(dataTypeOut) - << " (reason: " << reasonIfUnsupported - << "), falling back to the next backend."; - ReportWarning(warningMsg.str(), errMessages); - } - else - { - found = true; - backendSettings.m_SelectedBackends.insert(backend); - break; } } @@ -301,7 +383,7 @@ OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr, } else { - return ReturnWithError(layer); + return ReturnError(layer); } } } diff --git a/src/armnn/Network.hpp b/src/armnn/Network.hpp index 5da681306c..089b46c9ca 100644 --- a/src/armnn/Network.hpp +++ b/src/armnn/Network.hpp @@ -17,6 +17,7 @@ #include <map> #include <memory> +#include "Graph.hpp" #include "Layer.hpp" namespace armnn @@ -286,10 +287,19 @@ struct OptimizationResult bool m_Warning; bool m_Error; + OptimizationResult(bool warning, bool error) + : m_Warning(warning) + , m_Error(error) + {} + OptimizationResult() - : m_Warning(false) - , m_Error(false) + : OptimizationResult(false, false) {} + + bool IsOk() const { return !m_Warning && !m_Error; } + bool IsWarningOnly() const { return m_Warning && !m_Error; } + bool IsError() const { return m_Error; } + }; using BackendsMap = std::map<BackendId, std::unique_ptr<class IBackendInternal>>; @@ -302,4 +312,10 @@ OptimizationResult SelectTensorHandleStrategy(Graph& optGraph, TensorHandleFactoryRegistry& registry, Optional<std::vector<std::string>&> errMessages); +OptimizationResult AssignBackends(OptimizedNetwork* optNetObjPtr, + BackendSettings& backendSettings, + Graph::Iterator& firstLayer, + Graph::Iterator& lastLayer, + Optional<std::vector<std::string>&> errMessages); + } // namespace armnn diff --git a/src/armnn/test/OptimizerTests.cpp b/src/armnn/test/OptimizerTests.cpp index 7ceb104cb5..0ca4fc4764 100644 --- a/src/armnn/test/OptimizerTests.cpp +++ b/src/armnn/test/OptimizerTests.cpp @@ -5,13 +5,20 @@ #include "TestUtils.hpp" +#include <BackendSettings.hpp> #include <Graph.hpp> +#include <Network.hpp> #include <Optimizer.hpp> +#include <armnn/BackendRegistry.hpp> +#include <armnn/INetwork.hpp> +#include <armnn/LayerVisitorBase.hpp> #include <armnnUtils/FloatingPointConverter.hpp> #include <backendsCommon/CpuTensorHandle.hpp> +#include <backendsCommon/IBackendInternal.hpp> +#include <backendsCommon/LayerSupportBase.hpp> #include <boost/test/unit_test.hpp> @@ -616,4 +623,174 @@ BOOST_AUTO_TEST_CASE(FoldPadLayerIntoConvolution2dLayer) &IsLayerOfType<armnn::OutputLayer>)); } + + + +class MockLayerSupport : public LayerSupportBase { +public: + bool IsInputSupported(const TensorInfo& /*input*/, + Optional<std::string&> /*reasonIfUnsupported = EmptyOptional()*/) const override + { + return true; + } + + bool IsOutputSupported(const TensorInfo& /*input*/, + Optional<std::string&> /*reasonIfUnsupported = EmptyOptional()*/) const override + { + return true; + } + + bool IsActivationSupported(const TensorInfo& /*input0*/, + const TensorInfo& /*output*/, + const ActivationDescriptor& /*descriptor*/, + Optional<std::string&> /*reasonIfUnsupported = EmptyOptional()*/) const override + { + return true; + } +}; + +template<typename NamePolicy> +class MockBackend : public IBackendInternal +{ +public: + MockBackend() = default; + ~MockBackend() = default; + + static const BackendId& GetIdStatic() { return NamePolicy::GetIdStatic(); } + const BackendId& GetId() const override { return GetIdStatic(); } + + IBackendInternal::IMemoryManagerUniquePtr CreateMemoryManager() const override { return nullptr; }; + + IBackendInternal::IWorkloadFactoryPtr CreateWorkloadFactory( + const IBackendInternal::IMemoryManagerSharedPtr&) const override { return nullptr; } + + IBackendInternal::IBackendContextPtr CreateBackendContext(const IRuntime::CreationOptions&) const override + { + return nullptr; + } + + IBackendInternal::Optimizations GetOptimizations() const override { return {}; } + IBackendInternal::ILayerSupportSharedPtr GetLayerSupport() const override + { + return std::make_shared<MockLayerSupport>(); + } + + OptimizationViews OptimizeSubgraphView(const SubgraphView&) const override + { + return {}; + }; +}; + + +BOOST_AUTO_TEST_CASE(BackendHintTest) +{ + class TestBackendAssignment : public LayerVisitorBase<VisitorNoThrowPolicy> + { + public: + void VisitInputLayer(const IConnectableLayer* layer, + LayerBindingId id, + const char* name = nullptr) override + { + boost::ignore_unused(id, name); + auto inputLayer = boost::polymorphic_downcast<const InputLayer*>(layer); + BOOST_TEST((inputLayer->GetBackendId() == "MockBackend")); + } + + void VisitOutputLayer(const IConnectableLayer* layer, + LayerBindingId id, + const char* name = nullptr) override + { + boost::ignore_unused(id, name); + auto outputLayer = boost::polymorphic_downcast<const OutputLayer*>(layer); + BOOST_TEST((outputLayer->GetBackendId() == "MockBackend")); + } + + void VisitActivationLayer(const IConnectableLayer* layer, + const ActivationDescriptor& activationDescriptor, + const char* name = nullptr) override + { + boost::ignore_unused(activationDescriptor, name); + auto activation = boost::polymorphic_downcast<const ActivationLayer*>(layer); + BOOST_TEST((activation->GetBackendId() == "CustomBackend")); + } + }; + + struct CustomPolicy + { + static const BackendId& GetIdStatic() + { + static BackendId id="CustomBackend"; + return id; + } + }; + + struct MockPolicy + { + static const BackendId& GetIdStatic() + { + static BackendId id="MockBackend"; + return id; + } + }; + + auto& backendRegistry = BackendRegistryInstance(); + + backendRegistry.Register("MockBackend", [](){ + return std::make_unique<MockBackend<MockPolicy>>(); + }); + + backendRegistry.Register("CustomBackend", [](){ + return std::make_unique<MockBackend<CustomPolicy>>(); + }); + + // Define the network + auto network = INetwork::Create(); + ActivationDescriptor desc; + desc.m_Function = ActivationFunction::Linear; + + std::unique_ptr<Graph> graph = std::make_unique<Graph>(); + auto input = graph->AddLayer<InputLayer>(0, "input"); + auto act = graph->AddLayer<ActivationLayer>(desc, "activation"); + auto output = graph->AddLayer<OutputLayer>(0, "output"); + + BackendId customBackendId("CustomBackend"); + act->BackendSelectionHint(customBackendId); + + input->GetOutputSlot(0).Connect(act->GetInputSlot(0)); + act->GetOutputSlot(0).Connect(output->GetInputSlot(0)); + + + auto optNet = IOptimizedNetworkPtr(new OptimizedNetwork(std::move(graph)), &IOptimizedNetwork::Destroy); + + OptimizedNetwork* optNetObjPtr = boost::polymorphic_downcast<OptimizedNetwork*>(optNet.get()); + + // Get the optimized graph + Graph& optGraph = optNetObjPtr->GetGraph(); + + + std::vector<BackendId> prefs{"MockBackend", "CustomBackend"}; + + BackendIdSet availableBackends = {"CustomBackend", "MockBackend"}; + DeviceSpec spec(availableBackends); + + BackendSettings backendSettings(prefs, spec); + + // Assign an available backend to each layer + Graph::Iterator firstLayer = optGraph.begin(); + Graph::Iterator lastLayer = optGraph.end(); + OptimizationResult res = AssignBackends(optNetObjPtr, + backendSettings, + firstLayer, + lastLayer, + EmptyOptional()); + + BOOST_TEST(res.IsOk()); + + TestBackendAssignment visitor; + for (auto it =firstLayer; it != lastLayer; ++it) + { + (*it)->Accept(visitor); + } +} + BOOST_AUTO_TEST_SUITE_END() |