38 #include <boost/format.hpp> 39 #include <boost/numeric/conversion/converter_policies.hpp> 40 #include <boost/cast.hpp> 47 return new Network(networkOptions);
57 delete PolymorphicDowncast<Network*>(network);
62 delete PolymorphicDowncast<OptimizedNetwork*>(network);
73 return m_Graph->SerializeToDot(stream);
77 Optional<std::vector<std::string>&> errorMessages)
79 std::stringstream fullErrorMessage;
80 fullErrorMessage <<
"ERROR: " << errorMessage;
84 errorMessages.value().push_back(fullErrorMessage.str());
89 Optional<std::vector<std::string>&> warningMessages)
91 std::stringstream fullWarningMessage;
92 fullWarningMessage <<
"WARNING: " << warningMessage;
96 warningMessages.value().push_back(fullWarningMessage.str());
103 Optional<std::vector<std::string>&> errMessages)
105 std::stringstream failureMsg;
117 bool noErrors =
true;
119 for (
unsigned int i = 0; i < numOutputs; i++) {
125 std::stringstream ss;
127 <<
" (" << layer->
GetNameStr() <<
") is of type" 128 <<
" Quantized 8 bit but its scale parameter has not been set";
136 std::stringstream ss;
137 ss <<
"Quantization parameters for Softmax layer (Scale: " <<
139 ") are incorrect and have been updated to Scale: 0.00390625 and Offset: 0";
150 template <
typename LayerT>
153 LayerT* layer = PolymorphicDowncast<LayerT*>(l);
164 layer->m_Weight->template GetTensor<armnn::BFloat16>(), info.
GetNumElements(), newValues.data());
180 const std::vector<BackendId>& availablePreferredBackends,
181 std::string& reasonIfUnsupported,
182 Optional<std::vector<std::string>&> errMessages)
187 auto ReturnError = [&](
const Layer* layer)
204 std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers;
207 convertFp16ToFp32Layers =
212 std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers;
215 convertFp32ToFp16Layers =
220 auto AssignFirstSupportedBackend = [&](
Layer* layer,
BackendId preferredBackend)
222 bool supportedBackendFound =
false;
223 std::string reasonIfUnsupported;
229 reasonIfUnsupported))
231 supportedBackendFound =
true;
235 for (
const auto& backend : availablePreferredBackends)
238 if (backend == preferredBackend)
246 reasonIfUnsupported))
248 supportedBackendFound =
true;
254 return supportedBackendFound;
259 if (!AssignFirstSupportedBackend(convertLayer, backend))
261 return ReturnError(convertLayer);
267 if (!AssignFirstSupportedBackend(convertLayer, backend))
269 return ReturnError(convertLayer);
283 std::vector<ConvertBf16ToFp32Layer*> convertBf16ToFp32Layers;
286 convertBf16ToFp32Layers =
290 ConvertBf16ToFp32Weight<Convolution2dLayer>(layer);
294 ConvertBf16ToFp32Weight<FullyConnectedLayer>(layer);
299 std::vector<ConvertFp32ToBf16Layer*> convertFp32ToBf16Layers;
302 convertFp32ToBf16Layers =
307 auto AssignFirstSupportedBackend = [&](
Layer* layer,
BackendId preferredBackend)
309 bool supportedBackendFound =
false;
310 std::string reasonIfUnsupported;
316 reasonIfUnsupported))
318 supportedBackendFound =
true;
322 for (
const auto& backend : availablePreferredBackends)
325 if (backend == preferredBackend)
333 reasonIfUnsupported))
335 supportedBackendFound =
true;
341 return supportedBackendFound;
346 if (!AssignFirstSupportedBackend(convertLayer, backend))
348 return ReturnError(convertLayer);
354 if (!AssignFirstSupportedBackend(convertLayer, backend))
356 return ReturnError(convertLayer);
364 std::stringstream warningMsg;
366 <<
" is not supported on requested backend " << layer->
GetBackendId().
Get()
369 <<
" (reason: " << reasonIfUnsupported
370 <<
"), falling back to the next backend.";
386 Optional<std::vector<std::string>&> errMessages)
391 auto ReturnError = [&](
const Layer* layer)
398 if (availablePreferredBackends.empty())
400 std::stringstream failureMsg;
401 failureMsg <<
"No preferred backends are available";
408 for (
auto it = firstLayer; it != lastLayer; ++it)
413 layer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo().GetDataType();
415 layer->GetOutputSlot(0).GetTensorInfo().GetDataType();
417 std::string reasonIfUnsupported;
427 if (layer->GetBackendHint().has_value() &&
432 layer->GetBackendHint().value(),
435 availablePreferredBackends,
445 for (
const auto& backend : availablePreferredBackends)
447 if (layer->GetBackendHint().has_value() &&
448 layer->GetBackendHint().value() == backend)
459 availablePreferredBackends,
495 layer->SetBackendId(cpuBackendId);
500 return ReturnError(layer);
511 Optional<std::vector<std::string>&> errMessages)
529 auto backendFactory = backendRegistry.GetFactory(selectedBackend);
530 auto backendObjPtr = backendFactory();
533 backendObjPtr->RegisterTensorHandleFactories(handleFactoryRegistry);
535 backends[backendObjPtr->GetId()] = std::move(backendObjPtr);
544 Optional<std::vector<std::string>&> errMessages)
556 auto backendObjPtr = backends.find(selectedBackend)->second.get();
563 [&backendObjPtr](
const Layer& layer)
569 if (subgraphs.empty())
576 for (
auto& subgraph : subgraphs)
579 OptimizationViews optimizationViews = backendObjPtr->OptimizeSubgraphView(*subgraph);
586 SubgraphView& replacementSubgraph = substitution.m_ReplacementSubgraph;
587 SubgraphView& substitutableSubgraph = substitution.m_SubstitutableSubgraph;
591 std::for_each(replacementSubgraph.
begin(), replacementSubgraph.
end(), [&selectedBackend](
Layer* l)
594 l->SetBackendId(selectedBackend);
600 std::stringstream warningMsg;
601 warningMsg <<
"Some sub-graph(s) failed to optimized on " << backendObjPtr->GetId() <<
" backend.";
606 if (!backendObjPtr->GetId().IsCpuRef())
609 settingsCopy.m_IgnoredBackends.insert(backendObjPtr->GetId());
616 std::stringstream subgraphMsg;
617 subgraphMsg <<
"Re-assigning backends to " << failedSubgraph.GetLayers().size()
618 <<
" layers inside sub-graph " << count++;
625 if (reassignmentResult.m_Error)
648 if (srcFactory && dstFactory &&
673 if (frmBackend == backends.end() ||
674 !frmBackend->second->SupportsTensorAllocatorAPI())
681 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
687 const Layer& connectedLayer = connection->GetOwningLayer();
689 auto toBackend = backends.find(connectedLayer.
GetBackendId());
690 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
692 if (!toBackend->second.get()->SupportsTensorAllocatorAPI())
698 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
699 for (
auto&& dst : dstPrefs)
712 auto it = factoryScores.find(dst);
713 if (it == factoryScores.end())
716 factoryScores[dst] = 0;
725 factoryScores[dst]++;
728 if (factoryScores[dst] > topScore)
730 topScore = factoryScores[dst];
758 if (frmBackend == backends.end() ||
759 !frmBackend->second->SupportsTensorAllocatorAPI())
765 bool requiresMapUnmap =
false;
768 const Layer& connectedLayer = connection->GetOwningLayer();
771 requiresMapUnmap =
true;
779 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
780 for (
auto&& pref : srcPrefs)
782 if (requiresMapUnmap)
792 auto it = factoryScores.find(pref);
793 if (it == factoryScores.end())
796 factoryScores[pref] = 0;
803 const Layer& connectedLayer = connection->GetOwningLayer();
805 auto toBackend = backends.find(connectedLayer.
GetBackendId());
806 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
808 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
809 for (
auto&& src : srcPrefs)
811 if (factoryScores.find(src) == factoryScores.end())
816 for (
auto&& dst : dstPrefs)
821 factoryScores[src]++;
829 int minScore = std::numeric_limits<int>::max();
830 for (
auto it : factoryScores)
832 minScore = std::min(minScore, it.second);
836 std::vector<ITensorHandleFactory::FactoryId> optimalFactories;
837 for (
auto it : factoryScores)
839 if (it.second == minScore)
841 optimalFactories.push_back(it.first);
846 for (
auto&& srcPref : srcPrefs)
848 for (
auto&& comp : optimalFactories)
863 const Layer& connectedLayer,
867 auto toBackend = backends.find(connectedLayer.
GetBackendId());
868 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
870 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
893 for (
auto&& pref : dstPrefs)
895 if (pref == srcFactoryId)
905 for (
auto&& pref : dstPrefs)
921 if (srcCapability.empty() && dstCapability.empty())
932 for (
auto&& pref : dstPrefs)
950 Optional<std::vector<std::string>&> errMessages)
954 optGraph.
ForEachLayer([&backends, ®istry, &result, &errMessages, importEnabled](
Layer* layer)
985 unsigned int connectionIdx = 0;
988 const Layer& connectedLayer = connection->GetOwningLayer();
991 registry, importEnabled);
998 errMessages.value().emplace_back(
"Could not find valid strategy required for compatibility" 999 " between backends.");
1015 const std::vector<BackendId>& backendPreferences,
1018 Optional<std::vector<std::string>&> messages)
1020 if (backendPreferences.empty())
1030 const Network& network = *PolymorphicDowncast<const Network*>(&inNetwork);
1031 std::unique_ptr<Graph> graph = std::make_unique<Graph>(network.
GetGraph());
1035 OptimizedNetwork* optNetObjPtr = PolymorphicDowncast<OptimizedNetwork*>(optNet.get());
1041 using namespace optimizations;
1057 optGraph.InferTensorInfos();
1078 std::stringstream failureMsg;
1079 failureMsg <<
"None of the preferred backends " << backendPreferences
1097 if (assignBackendsResult.
m_Error)
1111 if (backendOptimizationResult.
m_Error)
1127 tensorHandleFactoryRegistry,
1137 optGraph.AddCompatibilityLayers(backends, tensorHandleFactoryRegistry);
1147 auto backendPtr = factoryFun();
1151 auto backendSpecificOptimizations = backendPtr->GetOptimizations();
1154 if (!backendSpecificOptimizations.empty())
1162 bool Network::GetShapeInferenceMethod()
1164 if (m_NetworkOptions.size() > 0 && m_NetworkOptions[0].GetBackendId().Get() ==
"ShapeInferenceMethod")
1166 return m_NetworkOptions[0].GetOption(0).GetValue().AsBool();
1172 : m_NetworkOptions(networkOptions),
1173 m_Graph(
std::make_unique<
Graph>(GetShapeInferenceMethod()))
1188 return m_Graph->AddLayer<
InputLayer>(id, name);
1200 return m_Graph->AddLayer<
ComparisonLayer>(comparisonDescriptor, name);
1212 return m_Graph->AddLayer<
FillLayer>(fillDescriptor, name);
1225 const auto layer = m_Graph->AddLayer<
FullyConnectedLayer>(fullyConnectedDescriptor, name);
1227 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1229 if (fullyConnectedDescriptor.m_BiasEnabled)
1231 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1242 return AddFullyConnectedLayerImpl(fullyConnectedDescriptor, weights, biases, name);
1250 return AddFullyConnectedLayerImpl(fullyConnectedDescriptor, weights, biases, name);
1259 return AddFullyConnectedLayerImpl(fullyConnectedDescriptor, weights, optionalBiases, name);
1265 return m_Graph->AddLayer<
ConcatLayer>(concatDescriptor, name);
1278 const auto layer = m_Graph->AddLayer<
Convolution2dLayer>(convolution2dDescriptor, name);
1280 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1282 if (convolution2dDescriptor.m_BiasEnabled)
1284 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1295 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1303 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1312 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, optionalBiases, name);
1328 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1330 if (convolution2dDescriptor.m_BiasEnabled)
1332 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1350 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1359 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1369 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, optionalBiases, name);
1385 return m_Graph->AddLayer<
PermuteLayer>(permuteDescriptor, name);
1391 return m_Graph->AddLayer<
Pooling2dLayer>(pooling2dDescriptor, name);
1397 return m_Graph->AddLayer<
ActivationLayer>(activationDescriptor, name);
1403 return m_Graph->AddLayer<
ArgMinMaxLayer>(argMinMaxDescriptor, name);
1407 normalizationDescriptor,
1415 return m_Graph->AddLayer<
SliceLayer>(sliceDescriptor, name);
1421 return m_Graph->AddLayer<
SoftmaxLayer>(softmaxDescriptor, name);
1427 return m_Graph->AddLayer<
SplitterLayer>(splitterDescriptor, name);
1475 layer->
m_Mean = std::make_unique<ScopedCpuTensorHandle>(mean);
1476 layer->m_Variance = std::make_unique<ScopedCpuTensorHandle>(variance);
1477 layer->m_Beta = std::make_unique<ScopedCpuTensorHandle>(beta);
1478 layer->m_Gamma = std::make_unique<ScopedCpuTensorHandle>(gamma);
1485 return m_Graph->AddLayer<
RankLayer>(name);
1499 return m_Graph->AddLayer<
ResizeLayer>(resizeDescriptor, name);
1503 resizeDescriptor,
const char* name)
1505 return m_Graph->AddLayer<
ResizeLayer>(resizeDescriptor, name);
1530 layer->
m_LayerOutput = std::make_unique<ScopedCpuTensorHandle>(input);
1538 return m_Graph->AddLayer<
ReshapeLayer>(reshapeDescriptor, name);
1562 const auto layer = m_Graph->AddLayer<
LstmLayer>(descriptor, name);
1567 layer->m_BasicParameters.m_InputToCellWeights =
1569 layer->m_BasicParameters.m_InputToOutputWeights =
1571 layer->m_BasicParameters.m_RecurrentToForgetWeights =
1573 layer->m_BasicParameters.m_RecurrentToCellWeights =
1575 layer->m_BasicParameters.m_RecurrentToOutputWeights =
1577 layer->m_BasicParameters.m_ForgetGateBias =
1579 layer->m_BasicParameters.m_CellBias =
1580 std::make_unique<ScopedCpuTensorHandle>(*(params.
m_CellBias));
1581 layer->m_BasicParameters.m_OutputGateBias =
1585 if(!descriptor.m_CifgEnabled)
1590 "when CIFG is disabled.");
1595 "AddLstmLayer: Recurrent To Input Weights cannot be NULL " 1596 "when CIFG is disabled.");
1601 "when CIFG is disabled.");
1603 layer->m_CifgParameters.m_InputToInputWeights =
1605 layer->m_CifgParameters.m_RecurrentToInputWeights =
1607 layer->m_CifgParameters.m_InputGateBias =
1612 if(descriptor.m_ProjectionEnabled)
1617 "when projection is enabled.");
1619 layer->m_ProjectionParameters.m_ProjectionWeights =
1623 layer->m_ProjectionParameters.m_ProjectionBias =
1629 if(descriptor.m_PeepholeEnabled)
1631 if(!descriptor.m_CifgEnabled)
1636 "when Peephole is enabled and CIFG disabled.");
1639 layer->m_PeepholeParameters.m_CellToInputWeights =
1646 "when Peephole is enabled.");
1651 "when Peephole is enabled.");
1654 layer->m_PeepholeParameters.m_CellToForgetWeights =
1656 layer->m_PeepholeParameters.m_CellToOutputWeights =
1661 if(descriptor.m_LayerNormEnabled)
1663 if(!descriptor.m_CifgEnabled)
1668 "when layer normalization is enabled and CIFG disabled.");
1670 layer->m_LayerNormParameters.m_InputLayerNormWeights =
1677 "when layer normalization is enabled.");
1682 "when layer normalization is enabled.");
1687 "when layer normalization is enabled.");
1689 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
1691 layer->m_LayerNormParameters.m_CellLayerNormWeights =
1693 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
1711 return m_Graph->AddLayer<
MeanLayer>(meanDescriptor,name);
1716 return m_Graph->AddLayer<
PadLayer>(padDescriptor,name);
1759 return m_Graph->AddLayer<
GatherLayer>(gatherDescriptor, name);
1789 layer->
m_Weight = std::make_unique<ScopedCpuTensorHandle>(weights);
1791 if (descriptor.m_BiasEnabled)
1793 layer->m_Bias = std::make_unique<ScopedCpuTensorHandle>(biases.
value());
1802 return m_Graph->AddLayer<
TransposeLayer>(transposeDescriptor, name);
1808 return m_Graph->AddLayer<
StackLayer>(stackDescriptor, name);
1826 layer->m_QuantizedLstmParameters.m_InputToForgetWeights =
1828 layer->m_QuantizedLstmParameters.m_InputToCellWeights =
1830 layer->m_QuantizedLstmParameters.m_InputToOutputWeights =
1834 layer->m_QuantizedLstmParameters.m_RecurrentToInputWeights =
1836 layer->m_QuantizedLstmParameters.m_RecurrentToForgetWeights =
1838 layer->m_QuantizedLstmParameters.m_RecurrentToCellWeights =
1840 layer->m_QuantizedLstmParameters.m_RecurrentToOutputWeights =
1844 layer->m_QuantizedLstmParameters.m_InputGateBias =
1846 layer->m_QuantizedLstmParameters.m_ForgetGateBias =
1848 layer->m_QuantizedLstmParameters.m_CellBias =
1849 std::make_unique<ScopedCpuTensorHandle>(params.
GetCellBias());
1850 layer->m_QuantizedLstmParameters.m_OutputGateBias =
1860 const auto layer = m_Graph->AddLayer<
QLstmLayer>(descriptor, name);
1865 layer->m_BasicParameters.m_InputToCellWeights =
1867 layer->m_BasicParameters.m_InputToOutputWeights =
1869 layer->m_BasicParameters.m_RecurrentToForgetWeights =
1871 layer->m_BasicParameters.m_RecurrentToCellWeights =
1873 layer->m_BasicParameters.m_RecurrentToOutputWeights =
1875 layer->m_BasicParameters.m_ForgetGateBias =
1877 layer->m_BasicParameters.m_CellBias =
1878 std::make_unique<ScopedCpuTensorHandle>(*(params.
m_CellBias));
1879 layer->m_BasicParameters.m_OutputGateBias =
1883 if(!descriptor.m_CifgEnabled)
1893 "AddQLstmLayer: Recurrent To Input Weights cannot be NULL");
1901 layer->m_CifgParameters.m_InputToInputWeights =
1903 layer->m_CifgParameters.m_RecurrentToInputWeights =
1905 layer->m_CifgParameters.m_InputGateBias =
1910 if(descriptor.m_ProjectionEnabled)
1917 layer->m_ProjectionParameters.m_ProjectionWeights =
1923 layer->m_ProjectionParameters.m_ProjectionBias =
1930 if(descriptor.m_PeepholeEnabled)
1942 if(!descriptor.m_CifgEnabled)
1949 layer->m_PeepholeParameters.m_CellToInputWeights =
1953 layer->m_PeepholeParameters.m_CellToForgetWeights =
1955 layer->m_PeepholeParameters.m_CellToOutputWeights =
1960 if(descriptor.m_LayerNormEnabled)
1977 if(!descriptor.m_CifgEnabled)
1984 layer->m_LayerNormParameters.m_InputLayerNormWeights =
1988 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
1990 layer->m_LayerNormParameters.m_CellLayerNormWeights =
1992 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
2002 layer->Accept(visitor);
A layer that the constant data can be bound to.
OptimizeForConnection< Layer, PermuteLayer, SquashEqualSiblingsImpl< PermuteLayer > > SquashEqualPermuteSiblings
void ReportError(const std::string &errorMessage, Optional< std::vector< std::string > &> errorMessages)
bool m_BiasEnabled
Enable/disable bias.
bool m_HalfPixelCenters
Half Pixel Centers.
IConnectableLayer * AddPooling2dLayer(const Pooling2dDescriptor &pooling2dDescriptor, const char *name=nullptr) override
Adds a pooling layer to the network.
bool m_AlignCorners
Aligned corners.
This layer represents a minimum operation.
static const FactoryId DeferredFactoryId
Use the workload factory to create the tensor handle.
This layer represents a split operation.
LstmBasicParameters m_BasicParameters
const std::vector< InputSlot * > & GetConnections() const
FactoryFunction GetFactory(const BackendId &id) const
This layer represents a batch normalization operation.
static void Destroy(INetwork *network)
A ViewsDescriptor for the SplitterLayer.
Interface for a layer that is connectable to other layers via InputSlots and OutputSlots.
OptimizeForConnection< PermuteLayer, PermuteLayer, OptimizeInversePermutesImpl< PermuteLayer > > OptimizeInversePermutes
virtual bool SupportsMapUnmap() const final
std::vector< ConvertFp32ToFp16Layer * > InsertConvertFp32ToFp16LayersAfter(Graph &graph, Layer &layer)
bool m_BiasEnabled
Enable/disable bias.
IConnectableLayer * AddOutputLayer(LayerBindingId id, const char *name=nullptr) override
Adds an output layer to the network.
IConnectableLayer * AddRankLayer(const char *name=nullptr) override
Adds a rank layer to the network.
void SetEdgeStrategy(unsigned int connectionIndex, EdgeStrategy strategy)
QuantizedLstmParameters m_QuantizedLstmParameters
This layer represents a 2D transpose convolution operation.
No strategy has been defined. Used internally to verify integrity of optimizations.
std::vector< ConvertFp16ToFp32Layer * > InsertConvertFp16ToFp32LayersBefore(Graph &graph, Layer &layer, bool expectCorrectInputType)
IConnectableLayer * AddL2NormalizationLayer(const L2NormalizationDescriptor &desc, const char *name=nullptr) override
Adds an L2 normalization layer to the network.
A TransposeConvolution2dDescriptor for the TransposeConvolution2dLayer.
const TensorShape & GetShape() const
CPU Execution: Reference C++ kernels.
Optimizer::Optimizations MakeOptimizations(Args &&... args)
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store Weight values.
IConnectableLayer * AddResizeBilinearLayer(const ResizeBilinearDescriptor &resizeDesc, const char *name=nullptr) override
Adds a resize bilinear layer to the network.
IConnectableLayer * AddFillLayer(const FillDescriptor &fillDescriptor, const char *name=nullptr) override
Add an Fill layer to the network.
IConnectableLayer * AddMeanLayer(const MeanDescriptor &meanDescriptor, const char *name=nullptr) override
Add a Mean layer to the network.
A ReshapeDescriptor for the ReshapeLayer.
ITensorHandleFactory::FactoryId CalculateSlotOption(BackendsMap &backends, OutputSlot &outputSlot, TensorHandleFactoryRegistry ®istry)
OptimizeForConnection< TransposeLayer, TransposeLayer, OptimizeInversePermutesImpl< TransposeLayer > > OptimizeInverseTransposes
OptimizeForConnection< TransposeLayer, BatchToSpaceNdLayer, PermuteAndBatchToSpaceAsDepthToSpaceImpl< TransposeLayer > > TransposeAndBatchToSpaceAsDepthToSpace
#define ARMNN_NO_DEPRECATE_WARN_BEGIN
ITensorHandleFactory::FactoryId CalculateSlotOptionForOutput(BackendsMap &backends, OutputSlot &slot, TensorHandleFactoryRegistry ®istry)
A ComparisonDescriptor for the ComparisonLayer.
This layer represents a depthwise convolution 2d operation.
static void ConvertBFloat16ToFloat32(const void *srcBFloat16Buffer, size_t numElements, float *dstFloat32Buffer)
bool RequiresCopy(ITensorHandleFactory::FactoryId src, ITensorHandleFactory::FactoryId dst, TensorHandleFactoryRegistry ®istry)
uint32_t m_TargetWidth
Target width value.
DataLayout m_DataLayout
The data layout to be used (NCHW, NHWC).
IConnectableLayer * AddLogSoftmaxLayer(const LogSoftmaxDescriptor &logSoftmaxDescriptor, const char *name=nullptr) override
Adds a log softmax layer to the network.
A Convolution2dDescriptor for the Convolution2dLayer.
Layer & GetOwningLayer() const
Source backends tensor data can be exported to destination backend tensor without copy...
IConnectableLayer * AddDepthwiseConvolution2dLayer(const DepthwiseConvolution2dDescriptor &convolution2dDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a 2D depthwise convolution layer to the network.
This layer converts data type Float 16 to Float 32.
bool m_BiasEnabled
Enable/disable bias.
const Graph & GetGraph() const
IConnectableLayer * AddSwitchLayer(const char *name=nullptr) override
Adds a switch layer to the network.
ResizeMethod m_Method
The Interpolation method to use (Bilinear, NearestNeighbor).
IConnectableLayer * AddFloorLayer(const char *name=nullptr) override
Adds a floor layer to the network.
static void Pass(Graph &graph, const Optimizations &optimizations)
This layer represents a SpaceToDepth operation.
IConnectableLayer * AddInputLayer(LayerBindingId id, const char *name=nullptr) override
Adds an input layer to the network.
This layer represents a reshape operation.
std::unique_ptr< ScopedCpuTensorHandle > m_InputToInputWeights
A unique pointer to represent 2D weights tensor with dimensions [outputSize, inputSize] (QAsymm8)...
#define ARMNN_LOG(severity)
std::unique_ptr< ScopedCpuTensorHandle > m_InputToForgetWeights
A unique pointer to represent 2D weights tensor with dimensions [num_units, inputSize] (QSymmS8)...
Main network class which provides the interface for building up a neural network. ...
This layer represents an activation operation with the specified activation function.
BackendRegistry & BackendRegistryInstance()
This layer converts data type BFloat16 to Float32.
LayerT * ConvertBf16ToFp32Weight(Layer *l)
std::vector< BackendOptions > NetworkOptions
void Accept(ILayerVisitor &visitor) const override
IConnectableLayer * AddMinimumLayer(const char *name=nullptr) override
Add a Minimum layer to the network.
This layer represents an unknown operation in the input graph.
OptimizeForConnection< Layer, ReshapeLayer, SquashEqualSiblingsImpl< ReshapeLayer > > SquashEqualReshapeSiblings
This layer represents a detection postprocess operator.
BackendIdSet m_SupportedBackends
OptimizeForConnection< Layer, TransposeLayer, MoveTransposeUpImpl > MoveTransposeUp
OptimizationResult ReturnWithError(OptimizationResult res, const Layer *layer, const BackendSettings &backendSettings, Optional< std::vector< std::string > &> errMessages)
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store weight values.
Copyright (c) 2020 ARM Limited.
IConnectableLayer * AddQLstmLayer(const QLstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr) override
Add a QLstm layer to the network.
This layer represents a pad operation.
This layer represents a LSTM operation.
void IgnoreUnused(Ts &&...)
OptimizeForConnection< PadLayer, Convolution2dLayer, FoldPadIntoConvolution2dImpl > FoldPadIntoConvolution2d
void SetBackendId(const BackendId &id)
bool IsBackendSupported(const BackendId &backend) const
LayerList::const_iterator Iterator
IConnectableLayer * AddFullyConnectedLayer(const FullyConnectedDescriptor &fullyConnectedDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a fully connected layer to the network.
A SpaceToDepthDescriptor for the SpaceToDepthLayer.
This layer represents a permutation operation.
unsigned int GetNumOutputSlots() const override
Returns the number of connectable output slots.
This layer represents a SpaceToBatchNd operation.
IConnectableLayer * AddRsqrtLayer(const char *name=nullptr) override
Add Reciprocal of square root layer to the network.
A BatchToSpaceNdDescriptor for the BatchToSpaceNdLayer.
OptimizeForType< Layer, AddDebugImpl > InsertDebugLayer
OptimizeForConnection< ReshapeLayer, ReshapeLayer, OptimizeConsecutiveReshapesImpl > OptimizeConsecutiveReshapes
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
This layer represents a elementwiseUnary operation.
constexpr const char * GetDataTypeName(DataType dataType)
A ResizeDescriptor for the ResizeLayer.
IConnectableLayer * AddEqualLayer(const char *name=nullptr) override
Add a Equal layer to the network.
IConnectableLayer * AddConvolution2dLayer(const Convolution2dDescriptor &convolution2dDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a 2D convolution layer to the network.
A StackDescriptor for the StackLayer.
IConnectableLayer * AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor &descriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr) override
Adds a 2D transpose convolution layer to the network.
Destination backend can work directly with tensors on source backend.
virtual std::vector< ITensorHandleFactory::FactoryId > GetHandleFactoryPreferences() const
(Optional) Returns a vector of supported TensorHandleFactory ids in preference order.
OptimizeForConnection< ConvertFp16ToFp32Layer, ConvertFp32ToFp16Layer, OptimizeInverseConversionsImpl > OptimizeInverseConversionsFp16
The SubgraphView class represents a subgraph of a Graph.
OptimizationResult ApplyBackendOptimizations(OptimizedNetwork *optNetObjPtr, BackendSettings &backendSettings, BackendsMap &backends, Optional< std::vector< std::string > &> errMessages)
A PadDescriptor for the PadLayer.
This layer represents an instance normalization operation.
OptimizeForConnection< PermuteLayer, BatchToSpaceNdLayer, PermuteAndBatchToSpaceAsDepthToSpaceImpl< PermuteLayer > > PermuteAndBatchToSpaceAsDepthToSpace
OptimizeForConnection< Layer, PermuteLayer, MovePermuteUpImpl > MovePermuteUp
std::unique_ptr< ScopedCpuTensorHandle > m_LayerOutput
IConnectableLayer * AddReshapeLayer(const ReshapeDescriptor &reshapeDescriptor, const char *name=nullptr) override
Adds a reshape layer to the network.
IConnectableLayer * AddTransposeLayer(const TransposeDescriptor &transposeDescriptor, const char *name=nullptr) override
Adds a transpose layer to the network.
A layer user-provided data can be bound to (e.g. inputs, outputs).
void ForEachLayer(Func func) const
virtual std::vector< Capability > GetCapabilities(const IConnectableLayer *layer, const IConnectableLayer *connectedLayer, CapabilityClass capabilityClass)
Status PrintGraph() override
IConnectableLayer * AddMergeLayer(const char *name=nullptr) override
Adds a merge layer to the network.
This layer dequantizes the input tensor.
ConvertConstants< Float32ToFloat16, IsFloat16Layer > ConvertConstantsFloatToHalf
OptimizeForType< TransposeLayer, TransposeAsReshapeImpl > TransposeAsReshape
IConnectableLayer * AddMaximumLayer(const char *name=nullptr) override
Add a Maximum layer to the network.
This layer represents a Gather operator.
std::unique_ptr< ScopedCpuTensorHandle > m_Anchors
A unique pointer to store Anchor values.
This layer represents a fully connected operation.
An LstmDescriptor for the LstmLayer.
#define ARMNN_NO_DEPRECATE_WARN_END
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
#define ARMNN_ASSERT_MSG(COND, MSG)
This layer represents a QuantizedLstm operation.
char const * GetLayerTypeAsCString(LayerType type)
This layer represents a log softmax operation.
std::unique_ptr< ScopedCpuTensorHandle > m_Mean
A unique pointer to store Mean values.
A L2NormalizationDescriptor for the L2NormalizationLayer.
int32_t GetQuantizationOffset() const
An ArgMinMaxDescriptor for ArgMinMaxLayer.
float GetQuantizationScale() const
DataType GetDataType() const
An OriginsDescriptor for the ConcatLayer.
bool has_value() const noexcept
A FullyConnectedDescriptor for the FullyConnectedLayer.
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store Weight values.
bool m_BiasEnabled
Enable/disable bias.
This layer represents a stack operation.
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
IConnectableLayer * AddConcatLayer(const ConcatDescriptor &concatDescriptor, const char *name=nullptr) override
Adds a concatenation layer to the network.
const Subgraphs & GetFailedSubgraphs() const
This layer represents a merge operation.
This layer represents a softmax operation.
const std::string & GetNameStr() const
uint32_t m_TargetWidth
Target width value.
std::vector< ConvertBf16ToFp32Layer * > InsertConvertBf16ToFp32LayersBefore(Graph &graph, Layer &layer, bool expectCorrectInputType)
A GatherDescriptor for the GatherLayer.
This layer represents a BatchToSpaceNd operation.
std::vector< SubgraphViewPtr > Subgraphs
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
bool m_HalfPixelCenters
Half Pixel Centers.
IConnectableLayer * AddPadLayer(const PadDescriptor &padDescriptor, const char *name=nullptr) override
Adds a fully pad layer to the network.
IConnectableLayer * AddStandInLayer(const StandInDescriptor &descriptor, const char *name=nullptr) override
Add a stand-in layer for a type unknown to the Arm NN framework.
void SetQuantizationScale(float scale)
This layer represents a ArgMinMax operation.
#define ARMNN_ASSERT(COND)
A StandInDescriptor for the StandIn layer.
A QLstmDescriptor for the QLstmLayer.
IConnectableLayer * AddActivationLayer(const ActivationDescriptor &activationDescriptor, const char *name=nullptr) override
Adds an activation layer to the network.
BackendIdVector GetAvailablePreferredBackends() const
IConnectableLayer * AddSubtractionLayer(const char *name=nullptr) override
Adds a subtraction layer to the network.
IConnectableLayer * AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor &desc, const char *name=nullptr) override
Adds an instance normalization layer to the network.
Device specific knowledge to be passed to the optimizer.
static bool IsLayerSupported(const BackendId &backendId, const IConnectableLayer &layer, Optional< DataType > dataType, std::string &outReasonIfUnsupported)
IConnectableLayer * AddDepthToSpaceLayer(const DepthToSpaceDescriptor &depthToSpaceDescriptor, const char *name=nullptr) override
Adds a depth to space layer to the network.
bool Validate(const SubgraphView &originalSubgraph) const
An ActivationDescriptor for the ActivationLayer.
std::vector< ConvertFp32ToBf16Layer * > InsertConvertFp32ToBf16LayersAfter(Graph &graph, Layer &layer)
const BackendId & GetBackendId() const
uint32_t m_TargetHeight
Target height value.
IConnectableLayer * AddStackLayer(const StackDescriptor &stackDescriptor, const char *name=nullptr) override
Adds a stack layer to the network.
This layer represents a floor operation.
uint32_t m_TargetHeight
Target height value.
A SliceDescriptor for the SliceLayer.
This layer represents a normalization operation.
virtual MemorySourceFlags GetExportFlags() const
This layer represents a pooling 2d operation.
This layer converts data type Float 32 to Float 16.
OptimizedNetwork(std::unique_ptr< Graph > graph)
This layer represents a transpose operation.
IConnectableLayer * AddSplitterLayer(const ViewsDescriptor &splitterDescriptor, const char *name=nullptr) override
Adds a splitter layer to the network.
This layer represents an addition operation.
QLstmBasicParameters m_BasicParameters
void SubstituteSubgraph(SubgraphView &subgraph, IConnectableLayer *substituteLayer)
Substitutes the given sub-graph with either a new layer or a new sub-graph.
Status PrintGraph() override
IConnectableLayer * AddSpaceToDepthLayer(const SpaceToDepthDescriptor &spaceToDepthDescriptor, const char *name=nullptr) override
Adds a space to depth layer to the network.
bool CheckScaleSetOnQuantizedType(Layer *layer, Optional< std::vector< std::string > &> errMessages)
Private implementation of INetwork.
void SetTensorHandleFactory(const ITensorHandleFactory::FactoryId &id)
A SpaceToBatchNdDescriptor for the SpaceToBatchNdLayer.
IConnectableLayer * AddPreluLayer(const char *name=nullptr) override
Adds a PReLU layer to the network.
OptimizeForType< PermuteLayer, PermuteAsReshapeImpl > PermuteAsReshape
bool IsWarningOnly() const
OptimizeForConnection< Layer, TransposeLayer, SquashEqualSiblingsImpl< TransposeLayer > > SquashEqualTransposeSiblings
This layer represents a QLstm operation.
IConnectableLayer * AddQuantizeLayer(const char *name=nullptr) override
Add a quantize layer to the network.
const Substitutions & GetSubstitutions() const
IConnectableLayer * AddArgMinMaxLayer(const ArgMinMaxDescriptor &desc, const char *name=nullptr) override
Adds an ArgMinMax layer to the network.
BackendIdVector m_PreferredBackends
OptimizationResult AssignBackends(OptimizedNetwork *optNetObjPtr, BackendSettings &backendSettings, Graph::Iterator &firstLayer, Graph::Iterator &lastLayer, Optional< std::vector< std::string > &> errMessages)
This layer represents a subtraction operation.
This layer calculates both true and false outputs for input.
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
ConvertConstants< Float16ToFloat32, IsFloat32Layer > ConvertConstantsHalfToFloat
bool m_AlignCorners
Aligned corners.
A ElementwiseUnaryDescriptor for the ElementwiseUnaryLayer.
IConnectableLayer * AddBatchNormalizationLayer(const BatchNormalizationDescriptor &desc, const ConstTensor &mean, const ConstTensor &variance, const ConstTensor &beta, const ConstTensor &gamma, const char *name=nullptr) override
Adds a batch normalization layer to the network.
static Subgraphs SelectSubgraphs(Graph &graph, const LayerSelectorFunction &selector)
Selects subgraphs from a graph based on the selector function and the algorithm.
IConnectableLayer * AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor &batchToSpaceNdDescriptor, const char *name=nullptr) override
Adds a batch to space ND layer to the network.
This layer represents a L2 normalization operation.
BackendsMap CreateSupportedBackends(TensorHandleFactoryRegistry &handleFactoryRegistry, BackendSettings &backendSettings)
OptimizeForConnection< ConvertFp32ToFp16Layer, ConvertFp16ToFp32Layer, OptimizeInverseConversionsImpl > OptimizeInverseConversionsFp32
ITensorHandleFactory::FactoryId CalculateSlotOptionForInput(BackendsMap &backends, OutputSlot &slot, TensorHandleFactoryRegistry ®istry)
const std::string & Get() const
Status SerializeToDot(std::ostream &stream) const override
IConnectableLayer * AddAdditionLayer(const char *name=nullptr) override
Adds an addition layer to the network.
BackendIdSet m_SelectedBackends
IConnectableLayer * AddDequantizeLayer(const char *name=nullptr) override
Adds a Dequantize layer to the network.
IConnectableLayer * AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor &spaceToBatchNdDescriptor, const char *name=nullptr) override
Adds a space to batch layer to the network.
OptimizationResult AttemptBackendAssignment(BackendSettings &backendSettings, Graph &graph, Layer *layer, BackendId backend, DataType dataTypeIn, DataType dataTypeOut, const std::vector< BackendId > &availablePreferredBackends, std::string &reasonIfUnsupported, Optional< std::vector< std::string > &> errMessages)
ITensorHandleFactory * GetFactory(ITensorHandleFactory::FactoryId id) const
Find a TensorHandleFactory by Id Returns nullptr if not found.
void SetTensorInfo(const TensorInfo &tensorInfo) override
IConnectableLayer * AddAbsLayer(const char *name=nullptr) override
Add absolute layer to the network.
A MeanDescriptor for the MeanLayer.
This layer represents a division operation.
IConnectableLayer * AddSliceLayer(const SliceDescriptor &sliceDescriptor, const char *name=nullptr) override
Adds a slice layer to the network.
This layer represents a strided slice operation.
LayerType GetType() const
IConnectableLayer * AddGreaterLayer(const char *name=nullptr) override
Add a Greater layer to the network.
This layer represents a maximum operation.
const OutputSlot & GetOutputSlot(unsigned int index=0) const override
Get the const output slot handle by slot index.
IConnectableLayer * AddMergerLayer(const MergerDescriptor &mergerDescriptor, const char *name=nullptr) override
Adds a concat layer to the network.
OptimizeForType< Layer, ConvertFp32NetworkToFp16Impl > Fp32NetworkToFp16Converter
IConnectableLayer * AddResizeLayer(const ResizeDescriptor &resizeDescriptor, const char *name=nullptr) override
Adds a resize layer to the network.
IConnectableLayer * AddConstantLayer(const ConstTensor &input, const char *name=nullptr) override
Adds a layer with no inputs and a single output, which always corresponds to the passed in constant t...
A TransposeDescriptor for the TransposeLayer.
A StridedSliceDescriptor for the StridedSliceLayer.
IConnectableLayer * AddMultiplicationLayer(const char *name=nullptr) override
Adds a multiplication layer to the network.
OptimizationResult SelectTensorHandleStrategy(Graph &optGraph, BackendsMap &backends, TensorHandleFactoryRegistry ®istry, bool importEnabled, Optional< std::vector< std::string > &> errMessages)
IConnectableLayer * AddComparisonLayer(const ComparisonDescriptor &comparisonDescriptor, const char *name=nullptr) override
Add a Comparison layer to the network.
void ReportWarning(const std::string &warningMessage, Optional< std::vector< std::string > &> warningMessages)
This layer represents a convolution 2d operation.
This layer converts data type Float32 to BFloat16.
bool CheckFlag(MemorySourceFlags flags, MemorySource source)
void SetQuantizationOffset(int32_t offset)
static INetwork * CreateRaw(NetworkOptions networkOptions={})
This layer represents a mean operation.
This layer represents a comparison operation.
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
IConnectableLayer * AddNormalizationLayer(const NormalizationDescriptor &normalizationDescriptor, const char *name=nullptr) override
Adds a normalization layer to the network.
std::unique_ptr< ScopedCpuTensorHandle > m_Weight
A unique pointer to store Weight values.
A Pooling2dDescriptor for the Pooling2dLayer.
This layer dequantizes the input tensor.
A NormalizationDescriptor for the NormalizationLayer.
IConnectableLayer * AddPermuteLayer(const PermuteDescriptor &permuteDescriptor, const char *name=nullptr) override
Adds a permute layer to the network.
DataLayout m_DataLayout
The data layout to be used (NCHW, NHWC).
An InstanceNormalizationDescriptor for InstanceNormalizationLayer.
This layer represents a multiplication operation.
IConnectableLayer * AddGatherLayer(const char *name=nullptr) override
Add Gather layer to the network.
IConnectableLayer * AddSoftmaxLayer(const SoftmaxDescriptor &softmaxDescriptor, const char *name=nullptr) override
Adds a softmax layer to the network.
IConnectableLayer * AddLstmLayer(const LstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr) override
Add a Lstm layer to the network.
A ResizeBilinearDescriptor for the ResizeBilinearLayer.
IConnectableLayer * AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor &elementwiseUnaryDescriptor, const char *name=nullptr) override
Add an ElementwiseUnary layer to the network.
const TensorInfo & GetTensorInfo() const override
static INetworkPtr Create(NetworkOptions networkOptions={})
IConnectableLayer * AddDivisionLayer(const char *name=nullptr) override
Adds a division layer to the network.
EdgeStrategy CalculateEdgeStrategy(BackendsMap &backends, ITensorHandleFactory::FactoryId srcFactoryId, const Layer &layer, const Layer &connectedLayer, TensorHandleFactoryRegistry ®istry, bool importEnabled)
static void Destroy(IOptimizedNetwork *network)
virtual MemorySourceFlags GetImportFlags() const
std::unique_ptr< ScopedCpuTensorHandle > m_InputToForgetWeights
A unique pointer to represent 2D weights tensor with dimensions [input_size, num_units].
OptimizeForType< Layer, ConvertFp32NetworkToBf16Impl > Fp32NetworkToBf16Converter
IConnectableLayer * AddStridedSliceLayer(const StridedSliceDescriptor &stridedSliceDescriptor, const char *name=nullptr) override
Adds a strided slice layer to the network.
A SoftmaxDescriptor for the SoftmaxLayer.
bool IsCpuRefUsed() const
IConnectableLayer * AddQuantizedLstmLayer(const QuantizedLstmInputParams ¶ms, const char *name=nullptr) override
Add a QuantizedLstm layer to the network.
static const FactoryId LegacyFactoryId
This layer represents a fill operation.
IConnectableLayer * AddDetectionPostProcessLayer(const DetectionPostProcessDescriptor &descriptor, const ConstTensor &anchors, const char *name=nullptr) override
Adds a Detection PostProcess layer to the network.
A DepthwiseConvolution2dDescriptor for the DepthwiseConvolution2dLayer.
A FillDescriptor for the FillLayer.
This layer represents a DepthToSpace operation.
A BatchNormalizationDescriptor for the BatchNormalizationLayer.
unsigned int GetNumElements() const
Network(NetworkOptions networkOptions={})
std::map< BackendId, std::unique_ptr< class IBackendInternal > > BackendsMap
This layer represents a resize operation.
A PermuteDescriptor for the PermuteLayer.
std::vector< float > anchors({ 0.5f, 0.5f, 1.0f, 1.0f, 0.5f, 0.5f, 1.0f, 1.0f, 0.5f, 0.5f, 1.0f, 1.0f, 0.5f, 10.5f, 1.0f, 1.0f, 0.5f, 10.5f, 1.0f, 1.0f, 0.5f, 100.5f, 1.0f, 1.0f })