38 #include <boost/format.hpp> 39 #include <boost/numeric/conversion/converter_policies.hpp> 40 #include <boost/cast.hpp> 57 delete PolymorphicDowncast<Network*>(network);
62 delete PolymorphicDowncast<OptimizedNetwork*>(network);
73 return m_Graph->SerializeToDot(stream);
77 Optional<std::vector<std::string>&> errorMessages)
79 std::stringstream fullErrorMessage;
80 fullErrorMessage <<
"ERROR: " << errorMessage;
84 errorMessages.value().push_back(fullErrorMessage.str());
89 Optional<std::vector<std::string>&> warningMessages)
91 std::stringstream fullWarningMessage;
92 fullWarningMessage <<
"WARNING: " << warningMessage;
96 warningMessages.value().push_back(fullWarningMessage.str());
103 Optional<std::vector<std::string>&> errMessages)
105 std::stringstream failureMsg;
117 bool noErrors =
true;
119 for (
unsigned int i = 0; i < numOutputs; i++) {
125 std::stringstream ss;
127 <<
" (" << layer->
GetNameStr() <<
") is of type" 128 <<
" Quantized 8 bit but its scale parameter has not been set";
136 std::stringstream ss;
137 ss <<
"Quantization parameters for Softmax layer (Scale: " <<
139 ") are incorrect and have been updated to Scale: 0.00390625 and Offset: 0";
150 template <
typename LayerT>
153 LayerT* layer = PolymorphicDowncast<LayerT*>(l);
164 layer->m_Weight->template GetTensor<armnn::BFloat16>(), info.
GetNumElements(), newValues.data());
180 const std::vector<BackendId>& availablePreferredBackends,
181 std::string& reasonIfUnsupported,
182 Optional<std::vector<std::string>&> errMessages)
187 auto ReturnError = [&](
const Layer* layer)
204 std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers;
207 convertFp16ToFp32Layers =
212 std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers;
215 convertFp32ToFp16Layers =
220 auto AssignFirstSupportedBackend = [&](
Layer* layer,
BackendId preferredBackend)
222 bool supportedBackendFound =
false;
223 std::string reasonIfUnsupported;
229 reasonIfUnsupported))
231 supportedBackendFound =
true;
235 for (
const auto& backend : availablePreferredBackends)
238 if (backend == preferredBackend)
246 reasonIfUnsupported))
248 supportedBackendFound =
true;
254 return supportedBackendFound;
259 if (!AssignFirstSupportedBackend(convertLayer, backend))
261 return ReturnError(convertLayer);
267 if (!AssignFirstSupportedBackend(convertLayer, backend))
269 return ReturnError(convertLayer);
283 std::vector<ConvertBf16ToFp32Layer*> convertBf16ToFp32Layers;
286 convertBf16ToFp32Layers =
290 ConvertBf16ToFp32Weight<Convolution2dLayer>(layer);
294 ConvertBf16ToFp32Weight<FullyConnectedLayer>(layer);
299 std::vector<ConvertFp32ToBf16Layer*> convertFp32ToBf16Layers;
302 convertFp32ToBf16Layers =
307 auto AssignFirstSupportedBackend = [&](
Layer* layer,
BackendId preferredBackend)
309 bool supportedBackendFound =
false;
310 std::string reasonIfUnsupported;
316 reasonIfUnsupported))
318 supportedBackendFound =
true;
322 for (
const auto& backend : availablePreferredBackends)
325 if (backend == preferredBackend)
333 reasonIfUnsupported))
335 supportedBackendFound =
true;
341 return supportedBackendFound;
346 if (!AssignFirstSupportedBackend(convertLayer, backend))
348 return ReturnError(convertLayer);
354 if (!AssignFirstSupportedBackend(convertLayer, backend))
356 return ReturnError(convertLayer);
364 std::stringstream warningMsg;
366 <<
" is not supported on requested backend " << layer->
GetBackendId().
Get()
369 <<
" (reason: " << reasonIfUnsupported
370 <<
"), falling back to the next backend.";
386 Optional<std::vector<std::string>&> errMessages)
391 auto ReturnError = [&](
const Layer* layer)
398 if (availablePreferredBackends.empty())
400 std::stringstream failureMsg;
401 failureMsg <<
"No preferred backends are available";
408 for (
auto it = firstLayer; it != lastLayer; ++it)
413 layer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo().GetDataType();
415 layer->GetOutputSlot(0).GetTensorInfo().GetDataType();
417 std::string reasonIfUnsupported;
427 if (layer->GetBackendHint().has_value() &&
432 layer->GetBackendHint().value(),
435 availablePreferredBackends,
445 for (
const auto& backend : availablePreferredBackends)
447 if (layer->GetBackendHint().has_value() &&
448 layer->GetBackendHint().value() == backend)
459 availablePreferredBackends,
495 layer->SetBackendId(cpuBackendId);
500 return ReturnError(layer);
511 Optional<std::vector<std::string>&> errMessages)
529 auto backendFactory = backendRegistry.GetFactory(selectedBackend);
530 auto backendObjPtr = backendFactory();
533 backendObjPtr->RegisterTensorHandleFactories(handleFactoryRegistry);
535 backends[backendObjPtr->GetId()] = std::move(backendObjPtr);
544 Optional<std::vector<std::string>&> errMessages)
556 auto backendObjPtr = backends.find(selectedBackend)->second.get();
563 [&backendObjPtr](
const Layer& layer)
569 if (subgraphs.empty())
576 for (
auto& subgraph : subgraphs)
579 OptimizationViews optimizationViews = backendObjPtr->OptimizeSubgraphView(*subgraph);
586 SubgraphView& replacementSubgraph = substitution.m_ReplacementSubgraph;
587 SubgraphView& substitutableSubgraph = substitution.m_SubstitutableSubgraph;
591 std::for_each(replacementSubgraph.
begin(), replacementSubgraph.
end(), [&selectedBackend](
Layer* l)
594 l->SetBackendId(selectedBackend);
600 std::stringstream warningMsg;
601 warningMsg <<
"Some sub-graph(s) failed to optimized on " << backendObjPtr->GetId() <<
" backend.";
606 if (!backendObjPtr->GetId().IsCpuRef())
609 settingsCopy.m_IgnoredBackends.insert(backendObjPtr->GetId());
616 std::stringstream subgraphMsg;
617 subgraphMsg <<
"Re-assigning backends to " << failedSubgraph.GetLayers().size()
618 <<
" layers inside sub-graph " << count++;
625 if (reassignmentResult.m_Error)
648 if (srcFactory && dstFactory &&
673 if (frmBackend == backends.end() ||
674 !frmBackend->second->SupportsTensorAllocatorAPI())
681 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
687 const Layer& connectedLayer = connection->GetOwningLayer();
689 auto toBackend = backends.find(connectedLayer.
GetBackendId());
690 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
692 if (!toBackend->second.get()->SupportsTensorAllocatorAPI())
698 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
699 for (
auto&& dst : dstPrefs)
712 auto it = factoryScores.find(dst);
713 if (it == factoryScores.end())
716 factoryScores[dst] = 0;
725 factoryScores[dst]++;
728 if (factoryScores[dst] > topScore)
730 topScore = factoryScores[dst];
758 if (frmBackend == backends.end() ||
759 !frmBackend->second->SupportsTensorAllocatorAPI())
765 bool requiresMapUnmap =
false;
768 const Layer& connectedLayer = connection->GetOwningLayer();
771 requiresMapUnmap =
true;
779 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
780 for (
auto&& pref : srcPrefs)
782 if (requiresMapUnmap)
792 auto it = factoryScores.find(pref);
793 if (it == factoryScores.end())
796 factoryScores[pref] = 0;
803 const Layer& connectedLayer = connection->GetOwningLayer();
805 auto toBackend = backends.find(connectedLayer.
GetBackendId());
806 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
808 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
809 for (
auto&& src : srcPrefs)
811 if (factoryScores.find(src) == factoryScores.end())
816 for (
auto&& dst : dstPrefs)
821 factoryScores[src]++;
829 int minScore = std::numeric_limits<int>::max();
830 for (
auto it : factoryScores)
832 minScore = std::min(minScore, it.second);
836 std::vector<ITensorHandleFactory::FactoryId> optimalFactories;
837 for (
auto it : factoryScores)
839 if (it.second == minScore)
841 optimalFactories.push_back(it.first);
846 for (
auto&& srcPref : srcPrefs)
848 for (
auto&& comp : optimalFactories)
863 const Layer& connectedLayer,
866 auto toBackend = backends.find(connectedLayer.
GetBackendId());
867 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
869 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
892 for (
auto&& pref : dstPrefs)
894 if (pref == srcFactoryId)
904 for (
auto&& pref : dstPrefs)
923 for (
auto&& pref : dstPrefs)
940 Optional<std::vector<std::string>&> errMessages)
944 optGraph.
ForEachLayer([&backends, ®istry, &result, &errMessages](
Layer* layer)
975 unsigned int connectionIdx = 0;
978 const Layer& connectedLayer = connection->GetOwningLayer();
987 errMessages.value().emplace_back(
"Could not find valid strategy required for compatibility" 988 " between backends.");
1004 const std::vector<BackendId>& backendPreferences,
1007 Optional<std::vector<std::string>&> messages)
1009 if (backendPreferences.empty())
1019 const Network& network = *PolymorphicDowncast<const Network*>(&inNetwork);
1020 std::unique_ptr<Graph> graph = std::make_unique<Graph>(network.
GetGraph());
1024 OptimizedNetwork* optNetObjPtr = PolymorphicDowncast<OptimizedNetwork*>(optNet.get());
1030 using namespace optimizations;
1046 optGraph.InferTensorInfos();
1067 std::stringstream failureMsg;
1068 failureMsg <<
"None of the preferred backends " << backendPreferences
1086 if (assignBackendsResult.
m_Error)
1100 if (backendOptimizationResult.
m_Error)
1116 tensorHandleFactoryRegistry,
1125 optGraph.AddCompatibilityLayers(backends, tensorHandleFactoryRegistry);
1135 auto backendPtr = factoryFun();
1139 auto backendSpecificOptimizations = backendPtr->GetOptimizations();
1142 if (!backendSpecificOptimizations.empty())
1152 : m_Graph(
std::make_unique<
Graph>())
1168 return m_Graph->AddLayer<
InputLayer>(id, name);
1180 return m_Graph->AddLayer<
ComparisonLayer>(comparisonDescriptor, name);
1199 const auto layer = m_Graph->AddLayer<
FullyConnectedLayer>(fullyConnectedDescriptor, name);
1201 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1203 if (fullyConnectedDescriptor.m_BiasEnabled)
1205 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1216 return AddFullyConnectedLayerImpl(fullyConnectedDescriptor, weights, biases, name);
1224 return AddFullyConnectedLayerImpl(fullyConnectedDescriptor, weights, biases, name);
1233 return AddFullyConnectedLayerImpl(fullyConnectedDescriptor, weights, optionalBiases, name);
1239 return m_Graph->AddLayer<
ConcatLayer>(concatDescriptor, name);
1252 const auto layer = m_Graph->AddLayer<
Convolution2dLayer>(convolution2dDescriptor, name);
1254 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1256 if (convolution2dDescriptor.m_BiasEnabled)
1258 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1269 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1277 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1286 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, optionalBiases, name);
1302 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1304 if (convolution2dDescriptor.m_BiasEnabled)
1306 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1324 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1333 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1343 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, optionalBiases, name);
1359 return m_Graph->AddLayer<
PermuteLayer>(permuteDescriptor, name);
1365 return m_Graph->AddLayer<
Pooling2dLayer>(pooling2dDescriptor, name);
1371 return m_Graph->AddLayer<
ActivationLayer>(activationDescriptor, name);
1377 return m_Graph->AddLayer<
ArgMinMaxLayer>(argMinMaxDescriptor, name);
1381 normalizationDescriptor,
1389 return m_Graph->AddLayer<
SliceLayer>(sliceDescriptor, name);
1395 return m_Graph->AddLayer<
SoftmaxLayer>(softmaxDescriptor, name);
1401 return m_Graph->AddLayer<
SplitterLayer>(splitterDescriptor, name);
1449 layer->
m_Mean = std::make_unique<ScopedCpuTensorHandle>(mean);
1450 layer->m_Variance = std::make_unique<ScopedCpuTensorHandle>(variance);
1451 layer->m_Beta = std::make_unique<ScopedCpuTensorHandle>(beta);
1452 layer->m_Gamma = std::make_unique<ScopedCpuTensorHandle>(gamma);
1466 return m_Graph->AddLayer<
ResizeLayer>(resizeDescriptor, name);
1470 resizeDescriptor,
const char* name)
1472 return m_Graph->AddLayer<
ResizeLayer>(resizeDescriptor, name);
1497 layer->
m_LayerOutput = std::make_unique<ScopedCpuTensorHandle>(input);
1505 return m_Graph->AddLayer<
ReshapeLayer>(reshapeDescriptor, name);
1529 const auto layer = m_Graph->AddLayer<
LstmLayer>(descriptor, name);
1534 layer->m_BasicParameters.m_InputToCellWeights =
1536 layer->m_BasicParameters.m_InputToOutputWeights =
1538 layer->m_BasicParameters.m_RecurrentToForgetWeights =
1540 layer->m_BasicParameters.m_RecurrentToCellWeights =
1542 layer->m_BasicParameters.m_RecurrentToOutputWeights =
1544 layer->m_BasicParameters.m_ForgetGateBias =
1546 layer->m_BasicParameters.m_CellBias =
1547 std::make_unique<ScopedCpuTensorHandle>(*(params.
m_CellBias));
1548 layer->m_BasicParameters.m_OutputGateBias =
1552 if(!descriptor.m_CifgEnabled)
1557 "when CIFG is disabled.");
1562 "AddLstmLayer: Recurrent To Input Weights cannot be NULL " 1563 "when CIFG is disabled.");
1568 "when CIFG is disabled.");
1570 layer->m_CifgParameters.m_InputToInputWeights =
1572 layer->m_CifgParameters.m_RecurrentToInputWeights =
1574 layer->m_CifgParameters.m_InputGateBias =
1579 if(descriptor.m_ProjectionEnabled)
1584 "when projection is enabled.");
1586 layer->m_ProjectionParameters.m_ProjectionWeights =
1590 layer->m_ProjectionParameters.m_ProjectionBias =
1596 if(descriptor.m_PeepholeEnabled)
1598 if(!descriptor.m_CifgEnabled)
1603 "when Peephole is enabled and CIFG disabled.");
1606 layer->m_PeepholeParameters.m_CellToInputWeights =
1613 "when Peephole is enabled.");
1618 "when Peephole is enabled.");
1621 layer->m_PeepholeParameters.m_CellToForgetWeights =
1623 layer->m_PeepholeParameters.m_CellToOutputWeights =
1628 if(descriptor.m_LayerNormEnabled)
1630 if(!descriptor.m_CifgEnabled)
1635 "when layer normalization is enabled and CIFG disabled.");
1637 layer->m_LayerNormParameters.m_InputLayerNormWeights =
1644 "when layer normalization is enabled.");
1649 "when layer normalization is enabled.");
1654 "when layer normalization is enabled.");
1656 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
1658 layer->m_LayerNormParameters.m_CellLayerNormWeights =
1660 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
1678 return m_Graph->AddLayer<
MeanLayer>(meanDescriptor,name);
1683 return m_Graph->AddLayer<
PadLayer>(padDescriptor,name);
1749 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1751 if (descriptor.m_BiasEnabled)
1753 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1762 return m_Graph->AddLayer<
TransposeLayer>(transposeDescriptor, name);
1768 return m_Graph->AddLayer<
StackLayer>(stackDescriptor, name);
1786 layer->m_QuantizedLstmParameters.m_InputToForgetWeights =
1788 layer->m_QuantizedLstmParameters.m_InputToCellWeights =
1790 layer->m_QuantizedLstmParameters.m_InputToOutputWeights =
1794 layer->m_QuantizedLstmParameters.m_RecurrentToInputWeights =
1796 layer->m_QuantizedLstmParameters.m_RecurrentToForgetWeights =
1798 layer->m_QuantizedLstmParameters.m_RecurrentToCellWeights =
1800 layer->m_QuantizedLstmParameters.m_RecurrentToOutputWeights =
1804 layer->m_QuantizedLstmParameters.m_InputGateBias =
1806 layer->m_QuantizedLstmParameters.m_ForgetGateBias =
1808 layer->m_QuantizedLstmParameters.m_CellBias =
1809 std::make_unique<ScopedCpuTensorHandle>(params.
GetCellBias());
1810 layer->m_QuantizedLstmParameters.m_OutputGateBias =
1820 const auto layer = m_Graph->AddLayer<
QLstmLayer>(descriptor, name);
1825 layer->m_BasicParameters.m_InputToCellWeights =
1827 layer->m_BasicParameters.m_InputToOutputWeights =
1829 layer->m_BasicParameters.m_RecurrentToForgetWeights =
1831 layer->m_BasicParameters.m_RecurrentToCellWeights =
1833 layer->m_BasicParameters.m_RecurrentToOutputWeights =
1835 layer->m_BasicParameters.m_ForgetGateBias =
1837 layer->m_BasicParameters.m_CellBias =
1838 std::make_unique<ScopedCpuTensorHandle>(*(params.
m_CellBias));
1839 layer->m_BasicParameters.m_OutputGateBias =
1843 if(!descriptor.m_CifgEnabled)
1853 "AddQLstmLayer: Recurrent To Input Weights cannot be NULL");
1861 layer->m_CifgParameters.m_InputToInputWeights =
1863 layer->m_CifgParameters.m_RecurrentToInputWeights =
1865 layer->m_CifgParameters.m_InputGateBias =
1870 if(descriptor.m_ProjectionEnabled)
1877 layer->m_ProjectionParameters.m_ProjectionWeights =
1883 layer->m_ProjectionParameters.m_ProjectionBias =
1890 if(descriptor.m_PeepholeEnabled)
1902 if(!descriptor.m_CifgEnabled)
1909 layer->m_PeepholeParameters.m_CellToInputWeights =
1913 layer->m_PeepholeParameters.m_CellToForgetWeights =
1915 layer->m_PeepholeParameters.m_CellToOutputWeights =
1920 if(descriptor.m_LayerNormEnabled)
1937 if(!descriptor.m_CifgEnabled)
1944 layer->m_LayerNormParameters.m_InputLayerNormWeights =
1948 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
1950 layer->m_LayerNormParameters.m_CellLayerNormWeights =
1952 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
1962 layer->Accept(visitor);
A layer that the constant data can be bound to.
OptimizeForConnection< Layer, PermuteLayer, SquashEqualSiblingsImpl< PermuteLayer > > SquashEqualPermuteSiblings
void ReportError(const std::string &errorMessage, Optional< std::vector< std::string > &> errorMessages)
bool m_BiasEnabled
Enable/disable bias.
IConnectableLayer * AddPooling2dLayer(const Pooling2dDescriptor &pooling2dDescriptor, const char *name=nullptr) override
Adds a pooling layer to the network.
This layer represents a minimum operation.
static const FactoryId DeferredFactoryId
Use the workload factory to create the tensor handle.
This layer represents a split operation.
LstmBasicParameters m_BasicParameters
const std::vector< InputSlot * > & GetConnections() const
FactoryFunction GetFactory(const BackendId &id) const
This layer represents a batch normalization operation.
static void Destroy(INetwork *network)
A ViewsDescriptor for the SplitterLayer.
Interface for a layer that is connectable to other layers via InputSlots and OutputSlots.
OptimizeForConnection< PermuteLayer, PermuteLayer, OptimizeInversePermutesImpl< PermuteLayer > > OptimizeInversePermutes
virtual bool SupportsMapUnmap() const final
std::vector< ConvertFp32ToFp16Layer * > InsertConvertFp32ToFp16LayersAfter(Graph &graph, Layer &layer)
bool m_BiasEnabled
Enable/disable bias.
IConnectableLayer * AddOutputLayer(LayerBindingId id, const char *name=nullptr) override
Adds an output layer to the network.
void SetEdgeStrategy(unsigned int connectionIndex, EdgeStrategy strategy)
QuantizedLstmParameters m_QuantizedLstmParameters
This layer represents a 2D transpose convolution operation.
No strategy has been defined. Used internally to verify integrity of optimizations.
std::vector< ConvertFp16ToFp32Layer * > InsertConvertFp16ToFp32LayersBefore(Graph &graph, Layer &layer, bool expectCorrectInputType)
IConnectableLayer * AddL2NormalizationLayer(const L2NormalizationDescriptor &desc, const char *name=nullptr) override
Adds an L2 normalization layer to the network.
A TransposeConvolution2dDescriptor for the TransposeConvolution2dLayer.
const TensorShape & GetShape() const
CPU Execution: Reference C++ kernels.
Optimizer::Optimizations MakeOptimizations(Args &&... args)
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store Weight values.
IConnectableLayer * AddResizeBilinearLayer(const ResizeBilinearDescriptor &resizeDesc, const char *name=nullptr) override
Adds a resize bilinear layer to the network.
IConnectableLayer * AddMeanLayer(const MeanDescriptor &meanDescriptor, const char *name=nullptr) override
Add a Mean layer to the network.
A ReshapeDescriptor for the ReshapeLayer.
ITensorHandleFactory::FactoryId CalculateSlotOption(BackendsMap &backends, OutputSlot &outputSlot, TensorHandleFactoryRegistry ®istry)
OptimizeForConnection< TransposeLayer, TransposeLayer, OptimizeInversePermutesImpl< TransposeLayer > > OptimizeInverseTransposes
OptimizeForConnection< TransposeLayer, BatchToSpaceNdLayer, PermuteAndBatchToSpaceAsDepthToSpaceImpl< TransposeLayer > > TransposeAndBatchToSpaceAsDepthToSpace
#define ARMNN_NO_DEPRECATE_WARN_BEGIN
ITensorHandleFactory::FactoryId CalculateSlotOptionForOutput(BackendsMap &backends, OutputSlot &slot, TensorHandleFactoryRegistry ®istry)
A ComparisonDescriptor for the ComparisonLayer.
This layer represents a depthwise convolution 2d operation.
static void ConvertBFloat16ToFloat32(const void *srcBFloat16Buffer, size_t numElements, float *dstFloat32Buffer)
bool RequiresCopy(ITensorHandleFactory::FactoryId src, ITensorHandleFactory::FactoryId dst, TensorHandleFactoryRegistry ®istry)
uint32_t m_TargetWidth
Target width value.
DataLayout m_DataLayout
The data layout to be used (NCHW, NHWC).
IConnectableLayer * AddLogSoftmaxLayer(const LogSoftmaxDescriptor &logSoftmaxDescriptor, const char *name=nullptr) override
Adds a log softmax layer to the network.
A Convolution2dDescriptor for the Convolution2dLayer.
Layer & GetOwningLayer() const
Source backends tensor data can be exported to destination backend tensor without copy...
IConnectableLayer * AddDepthwiseConvolution2dLayer(const DepthwiseConvolution2dDescriptor &convolution2dDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a 2D depthwise convolution layer to the network.
This layer converts data type Float 16 to Float 32.
bool m_BiasEnabled
Enable/disable bias.
const Graph & GetGraph() const
IConnectableLayer * AddSwitchLayer(const char *name=nullptr) override
Adds a switch layer to the network.
ResizeMethod m_Method
The Interpolation method to use (Bilinear, NearestNeighbor).
IConnectableLayer * AddFloorLayer(const char *name=nullptr) override
Adds a floor layer to the network.
static void Pass(Graph &graph, const Optimizations &optimizations)
This layer represents a SpaceToDepth operation.
IConnectableLayer * AddInputLayer(LayerBindingId id, const char *name=nullptr) override
Adds an input layer to the network.
This layer represents a reshape operation.
std::unique_ptr< ScopedCpuTensorHandle > m_InputToInputWeights
A unique pointer to represent 2D weights tensor with dimensions [outputSize, inputSize] (QAsymm8)...
#define ARMNN_LOG(severity)
std::unique_ptr< ScopedCpuTensorHandle > m_InputToForgetWeights
A unique pointer to represent 2D weights tensor with dimensions [num_units, inputSize] (QSymmS8)...
Main network class which provides the interface for building up a neural network. ...
This layer represents an activation operation with the specified activation function.
BackendRegistry & BackendRegistryInstance()
This layer converts data type BFloat16 to Float32.
LayerT * ConvertBf16ToFp32Weight(Layer *l)
void Accept(ILayerVisitor &visitor) const override
IConnectableLayer * AddMinimumLayer(const char *name=nullptr) override
Add a Minimum layer to the network.
This layer represents an unknown operation in the input graph.
OptimizeForConnection< Layer, ReshapeLayer, SquashEqualSiblingsImpl< ReshapeLayer > > SquashEqualReshapeSiblings
This layer represents a detection postprocess operator.
BackendIdSet m_SupportedBackends
OptimizeForConnection< Layer, TransposeLayer, MoveTransposeUpImpl > MoveTransposeUp
OptimizationResult ReturnWithError(OptimizationResult res, const Layer *layer, const BackendSettings &backendSettings, Optional< std::vector< std::string > &> errMessages)
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store weight values.
Copyright (c) 2020 ARM Limited.
IConnectableLayer * AddQLstmLayer(const QLstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr) override
Add a QLstm layer to the network.
This layer represents a pad operation.
This layer represents a LSTM operation.
void IgnoreUnused(Ts &&...)
OptimizeForConnection< PadLayer, Convolution2dLayer, FoldPadIntoConvolution2dImpl > FoldPadIntoConvolution2d
void SetBackendId(const BackendId &id)
bool IsBackendSupported(const BackendId &backend) const
LayerList::const_iterator Iterator
IConnectableLayer * AddFullyConnectedLayer(const FullyConnectedDescriptor &fullyConnectedDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a fully connected layer to the network.
A SpaceToDepthDescriptor for the SpaceToDepthLayer.
This layer represents a permutation operation.
OptimizationResult SelectTensorHandleStrategy(Graph &optGraph, BackendsMap &backends, TensorHandleFactoryRegistry ®istry, Optional< std::vector< std::string > &> errMessages)
unsigned int GetNumOutputSlots() const override
Returns the number of connectable output slots.
This layer represents a SpaceToBatchNd operation.
IConnectableLayer * AddRsqrtLayer(const char *name=nullptr) override
Add Reciprocal of square root layer to the network.
A BatchToSpaceNdDescriptor for the BatchToSpaceNdLayer.
OptimizeForType< Layer, AddDebugImpl > InsertDebugLayer
OptimizeForConnection< ReshapeLayer, ReshapeLayer, OptimizeConsecutiveReshapesImpl > OptimizeConsecutiveReshapes
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
This layer represents a elementwiseUnary operation.
constexpr const char * GetDataTypeName(DataType dataType)
A ResizeDescriptor for the ResizeLayer.
IConnectableLayer * AddEqualLayer(const char *name=nullptr) override
Add a Equal layer to the network.
IConnectableLayer * AddConvolution2dLayer(const Convolution2dDescriptor &convolution2dDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a 2D convolution layer to the network.
A StackDescriptor for the StackLayer.
IConnectableLayer * AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor &descriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a 2D transpose convolution layer to the network.
Destination backend can work directly with tensors on source backend.
virtual std::vector< ITensorHandleFactory::FactoryId > GetHandleFactoryPreferences() const
(Optional) Returns a vector of supported TensorHandleFactory ids in preference order.
OptimizeForConnection< ConvertFp16ToFp32Layer, ConvertFp32ToFp16Layer, OptimizeInverseConversionsImpl > OptimizeInverseConversionsFp16
The SubgraphView class represents a subgraph of a Graph.
OptimizationResult ApplyBackendOptimizations(OptimizedNetwork *optNetObjPtr, BackendSettings &backendSettings, BackendsMap &backends, Optional< std::vector< std::string > &> errMessages)
A PadDescriptor for the PadLayer.
This layer represents an instance normalization operation.
EdgeStrategy CalculateEdgeStrategy(BackendsMap &backends, ITensorHandleFactory::FactoryId srcFactoryId, const Layer &layer, const Layer &connectedLayer, TensorHandleFactoryRegistry ®istry)
OptimizeForConnection< PermuteLayer, BatchToSpaceNdLayer, PermuteAndBatchToSpaceAsDepthToSpaceImpl< PermuteLayer > > PermuteAndBatchToSpaceAsDepthToSpace
OptimizeForConnection< Layer, PermuteLayer, MovePermuteUpImpl > MovePermuteUp
std::unique_ptr< ScopedCpuTensorHandle > m_LayerOutput
IConnectableLayer * AddReshapeLayer(const ReshapeDescriptor &reshapeDescriptor, const char *name=nullptr) override
Adds a reshape layer to the network.
IConnectableLayer * AddTransposeLayer(const TransposeDescriptor &transposeDescriptor, const char *name=nullptr) override
Adds a transpose layer to the network.
A layer user-provided data can be bound to (e.g. inputs, outputs).
void ForEachLayer(Func func) const
Status PrintGraph() override
IConnectableLayer * AddMergeLayer(const char *name=nullptr) override
Adds a merge layer to the network.
This layer dequantizes the input tensor.
ConvertConstants< Float32ToFloat16, IsFloat16Layer > ConvertConstantsFloatToHalf
OptimizeForType< TransposeLayer, TransposeAsReshapeImpl > TransposeAsReshape
IConnectableLayer * AddMaximumLayer(const char *name=nullptr) override
Add a Maximum layer to the network.
This layer represents a Gather operator.
std::unique_ptr< ScopedCpuTensorHandle > m_Anchors
A unique pointer to store Anchor values.
This layer represents a fully connected operation.
An LstmDescriptor for the LstmLayer.
#define ARMNN_NO_DEPRECATE_WARN_END
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
#define ARMNN_ASSERT_MSG(COND, MSG)
This layer represents a QuantizedLstm operation.
char const * GetLayerTypeAsCString(LayerType type)
This layer represents a log softmax operation.
std::unique_ptr< ScopedCpuTensorHandle > m_Mean
A unique pointer to store Mean values.
A L2NormalizationDescriptor for the L2NormalizationLayer.
int32_t GetQuantizationOffset() const
An ArgMinMaxDescriptor for ArgMinMaxLayer.
float GetQuantizationScale() const
DataType GetDataType() const
An OriginsDescriptor for the ConcatLayer.
bool has_value() const noexcept
A FullyConnectedDescriptor for the FullyConnectedLayer.
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store Weight values.
bool m_BiasEnabled
Enable/disable bias.
This layer represents a stack operation.
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
IConnectableLayer * AddConcatLayer(const ConcatDescriptor &concatDescriptor, const char *name=nullptr) override
Adds a concatenation layer to the network.
const Subgraphs & GetFailedSubgraphs() const
This layer represents a merge operation.
This layer represents a softmax operation.
const std::string & GetNameStr() const
uint32_t m_TargetWidth
Target width value.
std::vector< ConvertBf16ToFp32Layer * > InsertConvertBf16ToFp32LayersBefore(Graph &graph, Layer &layer, bool expectCorrectInputType)
This layer represents a BatchToSpaceNd operation.
std::vector< SubgraphViewPtr > Subgraphs
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
IConnectableLayer * AddPadLayer(const PadDescriptor &padDescriptor, const char *name=nullptr) override
Adds a fully pad layer to the network.
IConnectableLayer * AddStandInLayer(const StandInDescriptor &descriptor, const char *name=nullptr) override
Add a stand-in layer for a type unknown to the Arm NN framework.
void SetQuantizationScale(float scale)
This layer represents a ArgMinMax operation.
#define ARMNN_ASSERT(COND)
A StandInDescriptor for the StandIn layer.
A QLstmDescriptor for the QLstmLayer.
IConnectableLayer * AddActivationLayer(const ActivationDescriptor &activationDescriptor, const char *name=nullptr) override
Adds an activation layer to the network.
BackendIdVector GetAvailablePreferredBackends() const
IConnectableLayer * AddSubtractionLayer(const char *name=nullptr) override
Adds a subtraction layer to the network.
IConnectableLayer * AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor &desc, const char *name=nullptr) override
Adds an instance normalization layer to the network.
Device specific knowledge to be passed to the optimizer.
static bool IsLayerSupported(const BackendId &backendId, const IConnectableLayer &layer, Optional< DataType > dataType, std::string &outReasonIfUnsupported)
IConnectableLayer * AddDepthToSpaceLayer(const DepthToSpaceDescriptor &depthToSpaceDescriptor, const char *name=nullptr) override
Adds a depth to space layer to the network.
bool Validate(const SubgraphView &originalSubgraph) const
An ActivationDescriptor for the ActivationLayer.
std::vector< ConvertFp32ToBf16Layer * > InsertConvertFp32ToBf16LayersAfter(Graph &graph, Layer &layer)
const BackendId & GetBackendId() const
uint32_t m_TargetHeight
Target height value.
IConnectableLayer * AddStackLayer(const StackDescriptor &stackDescriptor, const char *name=nullptr) override
Adds a stack layer to the network.
This layer represents a floor operation.
uint32_t m_TargetHeight
Target height value.
A SliceDescriptor for the SliceLayer.
This layer represents a normalization operation.
virtual MemorySourceFlags GetExportFlags() const
This layer represents a pooling 2d operation.
This layer converts data type Float 32 to Float 16.
OptimizedNetwork(std::unique_ptr< Graph > graph)
This layer represents a transpose operation.
IConnectableLayer * AddSplitterLayer(const ViewsDescriptor &splitterDescriptor, const char *name=nullptr) override
Adds a splitter layer to the network.
This layer represents an addition operation.
QLstmBasicParameters m_BasicParameters
void SubstituteSubgraph(SubgraphView &subgraph, IConnectableLayer *substituteLayer)
Substitutes the given sub-graph with either a new layer or a new sub-graph.
Status PrintGraph() override
IConnectableLayer * AddSpaceToDepthLayer(const SpaceToDepthDescriptor &spaceToDepthDescriptor, const char *name=nullptr) override
Adds a space to depth layer to the network.
bool CheckScaleSetOnQuantizedType(Layer *layer, Optional< std::vector< std::string > &> errMessages)
Private implementation of INetwork.
void SetTensorHandleFactory(const ITensorHandleFactory::FactoryId &id)
A SpaceToBatchNdDescriptor for the SpaceToBatchNdLayer.
IConnectableLayer * AddPreluLayer(const char *name=nullptr) override
Adds a PReLU layer to the network.
OptimizeForType< PermuteLayer, PermuteAsReshapeImpl > PermuteAsReshape
bool IsWarningOnly() const
OptimizeForConnection< Layer, TransposeLayer, SquashEqualSiblingsImpl< TransposeLayer > > SquashEqualTransposeSiblings
static INetwork * CreateRaw()
This layer represents a QLstm operation.
IConnectableLayer * AddQuantizeLayer(const char *name=nullptr) override
Add a quantize layer to the network.
const Substitutions & GetSubstitutions() const
IConnectableLayer * AddArgMinMaxLayer(const ArgMinMaxDescriptor &desc, const char *name=nullptr) override
Adds an ArgMinMax layer to the network.
BackendIdVector m_PreferredBackends
OptimizationResult AssignBackends(OptimizedNetwork *optNetObjPtr, BackendSettings &backendSettings, Graph::Iterator &firstLayer, Graph::Iterator &lastLayer, Optional< std::vector< std::string > &> errMessages)
This layer represents a subtraction operation.
This layer calculates both true and false outputs for input.
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
ConvertConstants< Float16ToFloat32, IsFloat32Layer > ConvertConstantsHalfToFloat
A ElementwiseUnaryDescriptor for the ElementwiseUnaryLayer.
IConnectableLayer * AddBatchNormalizationLayer(const BatchNormalizationDescriptor &desc, const ConstTensor &mean, const ConstTensor &variance, const ConstTensor &beta, const ConstTensor &gamma, const char *name=nullptr) override
Adds a batch normalization layer to the network.
static Subgraphs SelectSubgraphs(Graph &graph, const LayerSelectorFunction &selector)
Selects subgraphs from a graph based on the selector function and the algorithm.
IConnectableLayer * AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor &batchToSpaceNdDescriptor, const char *name=nullptr) override
Adds a batch to space ND layer to the network.
This layer represents a L2 normalization operation.
BackendsMap CreateSupportedBackends(TensorHandleFactoryRegistry &handleFactoryRegistry, BackendSettings &backendSettings)
OptimizeForConnection< ConvertFp32ToFp16Layer, ConvertFp16ToFp32Layer, OptimizeInverseConversionsImpl > OptimizeInverseConversionsFp32
ITensorHandleFactory::FactoryId CalculateSlotOptionForInput(BackendsMap &backends, OutputSlot &slot, TensorHandleFactoryRegistry ®istry)
const std::string & Get() const
Status SerializeToDot(std::ostream &stream) const override
IConnectableLayer * AddAdditionLayer(const char *name=nullptr) override
Adds an addition layer to the network.
BackendIdSet m_SelectedBackends
IConnectableLayer * AddDequantizeLayer(const char *name=nullptr) override
Adds a Dequantize layer to the network.
IConnectableLayer * AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor &spaceToBatchNdDescriptor, const char *name=nullptr) override
Adds a space to batch layer to the network.
OptimizationResult AttemptBackendAssignment(BackendSettings &backendSettings, Graph &graph, Layer *layer, BackendId backend, DataType dataTypeIn, DataType dataTypeOut, const std::vector< BackendId > &availablePreferredBackends, std::string &reasonIfUnsupported, Optional< std::vector< std::string > &> errMessages)
ITensorHandleFactory * GetFactory(ITensorHandleFactory::FactoryId id) const
Find a TensorHandleFactory by Id Returns nullptr if not found.
void SetTensorInfo(const TensorInfo &tensorInfo) override
IConnectableLayer * AddAbsLayer(const char *name=nullptr) override
Add absolute layer to the network.
A MeanDescriptor for the MeanLayer.
This layer represents a division operation.
IConnectableLayer * AddSliceLayer(const SliceDescriptor &sliceDescriptor, const char *name=nullptr) override
Adds a slice layer to the network.
This layer represents a strided slice operation.
LayerType GetType() const
IConnectableLayer * AddGreaterLayer(const char *name=nullptr) override
Add a Greater layer to the network.
This layer represents a maximum operation.
const OutputSlot & GetOutputSlot(unsigned int index=0) const override
Get the const output slot handle by slot index.
IConnectableLayer * AddMergerLayer(const MergerDescriptor &mergerDescriptor, const char *name=nullptr) override
Adds a concat layer to the network.
OptimizeForType< Layer, ConvertFp32NetworkToFp16Impl > Fp32NetworkToFp16Converter
IConnectableLayer * AddResizeLayer(const ResizeDescriptor &resizeDescriptor, const char *name=nullptr) override
Adds a resize layer to the network.
IConnectableLayer * AddConstantLayer(const ConstTensor &input, const char *name=nullptr) override
Adds a layer with no inputs and a single output, which always corresponds to the passed in constant t...
A TransposeDescriptor for the TransposeLayer.
A StridedSliceDescriptor for the StridedSliceLayer.
IConnectableLayer * AddMultiplicationLayer(const char *name=nullptr) override
Adds a multiplication layer to the network.
IConnectableLayer * AddComparisonLayer(const ComparisonDescriptor &comparisonDescriptor, const char *name=nullptr) override
Add a Comparison layer to the network.
void ReportWarning(const std::string &warningMessage, Optional< std::vector< std::string > &> warningMessages)
This layer represents a convolution 2d operation.
This layer converts data type Float32 to BFloat16.
bool CheckFlag(MemorySourceFlags flags, MemorySource source)
void SetQuantizationOffset(int32_t offset)
This layer represents a mean operation.
This layer represents a comparison operation.
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
IConnectableLayer * AddNormalizationLayer(const NormalizationDescriptor &normalizationDescriptor, const char *name=nullptr) override
Adds a normalization layer to the network.
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store Weight values.
A Pooling2dDescriptor for the Pooling2dLayer.
This layer dequantizes the input tensor.
armnn::Runtime::CreationOptions::ExternalProfilingOptions options
A NormalizationDescriptor for the NormalizationLayer.
IConnectableLayer * AddPermuteLayer(const PermuteDescriptor &permuteDescriptor, const char *name=nullptr) override
Adds a permute layer to the network.
DataLayout m_DataLayout
The data layout to be used (NCHW, NHWC).
An InstanceNormalizationDescriptor for InstanceNormalizationLayer.
This layer represents a multiplication operation.
IConnectableLayer * AddGatherLayer(const char *name=nullptr) override
Add Gather layer to the network.
IConnectableLayer * AddSoftmaxLayer(const SoftmaxDescriptor &softmaxDescriptor, const char *name=nullptr) override
Adds a softmax layer to the network.
IConnectableLayer * AddLstmLayer(const LstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr) override
Add a Lstm layer to the network.
A ResizeBilinearDescriptor for the ResizeBilinearLayer.
IConnectableLayer * AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor &elementwiseUnaryDescriptor, const char *name=nullptr) override
Add an ElementwiseUnary layer to the network.
const TensorInfo & GetTensorInfo() const override
IConnectableLayer * AddDivisionLayer(const char *name=nullptr) override
Adds a division layer to the network.
static void Destroy(IOptimizedNetwork *network)
virtual MemorySourceFlags GetImportFlags() const
std::unique_ptr< ScopedCpuTensorHandle > m_InputToForgetWeights
A unique pointer to represent 2D weights tensor with dimensions [input_size, num_units].
OptimizeForType< Layer, ConvertFp32NetworkToBf16Impl > Fp32NetworkToBf16Converter
IConnectableLayer * AddStridedSliceLayer(const StridedSliceDescriptor &stridedSliceDescriptor, const char *name=nullptr) override
Adds a strided slice layer to the network.
A SoftmaxDescriptor for the SoftmaxLayer.
bool IsCpuRefUsed() const
IConnectableLayer * AddQuantizedLstmLayer(const QuantizedLstmInputParams ¶ms, const char *name=nullptr) override
Add a QuantizedLstm layer to the network.
static INetworkPtr Create()
static const FactoryId LegacyFactoryId
IConnectableLayer * AddDetectionPostProcessLayer(const DetectionPostProcessDescriptor &descriptor, const ConstTensor &anchors, const char *name=nullptr) override
Adds a Detection PostProcess layer to the network.
A DepthwiseConvolution2dDescriptor for the DepthwiseConvolution2dLayer.
This layer represents a DepthToSpace operation.
A BatchNormalizationDescriptor for the BatchNormalizationLayer.
unsigned int GetNumElements() const
std::map< BackendId, std::unique_ptr< class IBackendInternal > > BackendsMap
This layer represents a resize operation.
A PermuteDescriptor for the PermuteLayer.
std::vector< float > anchors({ 0.5f, 0.5f, 1.0f, 1.0f, 0.5f, 0.5f, 1.0f, 1.0f, 0.5f, 0.5f, 1.0f, 1.0f, 0.5f, 10.5f, 1.0f, 1.0f, 0.5f, 10.5f, 1.0f, 1.0f, 0.5f, 100.5f, 1.0f, 1.0f })