23.02
|
Go to the documentation of this file.
31 #include <client/include/IProfilingService.hpp>
33 #include <common/include/ProfilingGuid.hpp>
35 #include <fmt/format.h>
76 return pNetworkImpl->AddComparisonLayer(comparisonDescriptor, name);
83 return pNetworkImpl->AddConcatLayer(concatDescriptor, name);
90 return pNetworkImpl->AddConvolution2dLayer(convolution2dDescriptor, name);
96 return pNetworkImpl->AddConvolution3dLayer(convolution3dDescriptor, name);
103 return pNetworkImpl->AddDepthToSpaceLayer(depthToSpaceDescriptor, name);
111 return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, name);
126 return pNetworkImpl->AddDetectionPostProcessLayer(descriptor, anchors, name);
133 return pNetworkImpl->AddElementwiseUnaryLayer(elementwiseUnaryDescriptor, name);
140 return pNetworkImpl->AddFillLayer(fillDescriptor, name);
146 return pNetworkImpl->AddFullyConnectedLayer(fullyConnectedDescriptor, name);
152 return pNetworkImpl->AddPermuteLayer(permuteDescriptor, name);
158 return pNetworkImpl->AddBatchToSpaceNdLayer(batchToSpaceNdDescriptor, name);
164 return pNetworkImpl->AddPooling2dLayer(pooling2dDescriptor, name);
170 return pNetworkImpl->AddPooling3dLayer(pooling3dDescriptor, name);
178 return pNetworkImpl->AddPrecompiledLayer(preCompiledDescriptor, std::move(compiledBlobPtr), backend, name);
184 return pNetworkImpl->AddActivationLayer(activationDescriptor, name);
190 return pNetworkImpl->AddNormalizationLayer(normalizationDescriptor, name);
195 return pNetworkImpl->AddSliceLayer(sliceDescriptor, name);
200 return pNetworkImpl->AddSoftmaxLayer(softmaxDescriptor, name);
206 return pNetworkImpl->AddSplitterLayer(splitterDescriptor, name);
231 return pNetworkImpl->AddBatchNormalizationLayer(desc, mean, variance, beta, gamma, name);
242 return pNetworkImpl->AddResizeLayer(resizeDescriptor, name);
248 return pNetworkImpl->AddReduceLayer(reduceDescriptor, name);
254 return pNetworkImpl->AddInstanceNormalizationLayer(desc, name);
260 return pNetworkImpl->AddL2NormalizationLayer(desc, name);
266 return pNetworkImpl->AddLogSoftmaxLayer(logSoftmaxDescriptor, name);
278 return pNetworkImpl->AddReshapeLayer(reshapeDescriptor, name);
284 return pNetworkImpl->AddSpaceToBatchNdLayer(spaceToBatchNdDescriptor, name);
290 return pNetworkImpl->AddSpaceToDepthLayer(spaceToDepthDescriptor, name);
306 return pNetworkImpl->AddLstmLayer(descriptor, params, name);
326 return pNetworkImpl->AddMeanLayer(meanDescriptor, name);
343 return pNetworkImpl->AddStridedSliceLayer(stridedSliceDescriptor, name);
377 return pNetworkImpl->AddTransposeConvolution2dLayer(descriptor, weights, biases, name);
383 return pNetworkImpl->AddTransposeLayer(transposeDescriptor, name);
406 return pNetworkImpl->AddQuantizedLstmLayer(params, name);
413 return pNetworkImpl->AddQLstmLayer(descriptor, params, name);
419 return pNetworkImpl->AddLogicalBinaryLayer(descriptor, name);
427 return pNetworkImpl->AddUnidirectionalSequenceLstmLayer(descriptor, params, name);
433 return pNetworkImpl->AddChannelShuffleLayer(descriptor, name);
439 return pNetworkImpl->AddBatchMatMulLayer(descriptor, name);
449 return new INetwork(networkOptions);
463 : pOptimizedNetworkImpl(new
OptimizedNetworkImpl(*other.pOptimizedNetworkImpl.get(), modelOptions)) {}
469 : pOptimizedNetworkImpl(
std::move(impl)) {}
519 return m_Graph->SerializeToDot(stream);
524 return m_Graph->GetNumInputs();
529 return m_Graph->GetNumOutputs();
533 Optional<std::vector<std::string>&> errorMessages)
535 std::stringstream fullErrorMessage;
536 fullErrorMessage <<
"ERROR: " << errorMessage;
540 errorMessages.value().push_back(fullErrorMessage.str());
545 Optional<std::vector<std::string>&> warningMessages)
547 std::stringstream fullWarningMessage;
548 fullWarningMessage <<
"WARNING: " << warningMessage;
552 warningMessages.value().push_back(fullWarningMessage.str());
559 Optional<std::vector<std::string>&> errMessages)
561 std::stringstream failureMsg;
573 bool noErrors =
true;
575 for (
unsigned int i = 0; i < numOutputs; i++) {
580 if (0.f ==
info.GetQuantizationScale())
583 std::stringstream ss;
585 <<
" (" << layer->
GetNameStr() <<
") is of type"
586 <<
" Quantized 8 bit but its scale parameter has not been set";
590 if ((
info.GetQuantizationScale() != (1.0f / 256.0f) ||
591 info.GetQuantizationOffset() != 0) &&
594 std::stringstream ss;
595 ss <<
"Quantization parameters for Softmax layer (Scale: " <<
596 info.GetQuantizationScale() <<
" and Offset: " <<
info.GetQuantizationOffset() <<
597 ") are incorrect and have been updated to Scale: 0.00390625 and Offset: 0";
599 info.SetQuantizationScale((1.0f /256.0f));
600 info.SetQuantizationOffset(0);
614 const std::vector<BackendId>& availablePreferredBackends,
615 std::string& reasonIfUnsupported,
616 Optional<std::vector<std::string>&> errMessages)
621 auto ReturnError = [&](
const Layer* layer)
634 std::string checkStr =
"This CPU architecture does not support F16 data type, you need v8.2 or above";
635 if (!isLayerSupported ||
636 reasonIfUnsupported.find(checkStr) != std::string::npos)
644 auto ConstantLayerFromFp16ToFp32 = [](
Layer& layer)
648 ConstantLayer* constantLayer = PolymorphicDowncast<ConstantLayer*>(&layer);
654 std::vector<float> newValues(
info.GetNumElements());
658 info.GetNumElements(),
671 bool checkType =
false;
675 auto connectedOutputSlot = inputSlot.GetConnectedOutputSlot();
678 if (connectedOutputSlot->GetNumConnections() == 1)
681 ConstantLayerFromFp16ToFp32(connectedOutputSlot->GetOwningLayer());
687 std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers;
690 convertFp16ToFp32Layers =
695 std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers;
698 convertFp32ToFp16Layers =
703 auto AssignFirstSupportedBackend = [&](
Layer* layer,
BackendId preferredBackend)
705 bool supportedBackendFound =
false;
706 std::string reasonIfUnsupported;
712 reasonIfUnsupported))
714 supportedBackendFound =
true;
718 for (
const auto& backend : availablePreferredBackends)
721 if (backend == preferredBackend)
729 reasonIfUnsupported))
731 supportedBackendFound =
true;
737 return supportedBackendFound;
742 if (!AssignFirstSupportedBackend(convertLayer, backend))
744 return ReturnError(convertLayer);
750 if (!AssignFirstSupportedBackend(convertLayer, backend))
752 return ReturnError(convertLayer);
760 std::stringstream warningMsg;
762 <<
" is not supported on requested backend " << layer->
GetBackendId().
Get()
765 <<
" (reason: " << reasonIfUnsupported
766 <<
"), falling back to the next backend.";
783 return {dataTypeIn, dataTypeOut};
790 Optional<std::vector<std::string>&> errMessages,
793 std::vector<BackendId>& availablePreferredBackends)
795 auto ReturnError = [&](
const Layer* layer)
800 auto layer = PolymorphicDowncast<Layer*>(it);
809 std::string reasonIfUnsupported;
819 if (layer->GetBackendHint().has_value() &&
824 layer->GetBackendHint().value(),
827 availablePreferredBackends,
837 for (
const auto& backend : availablePreferredBackends)
839 if (layer->GetBackendHint().has_value() &&
840 layer->GetBackendHint().value() == backend)
851 availablePreferredBackends,
887 layer->SetBackendId(cpuBackendId);
892 result = ReturnError(layer);
902 Optional<std::vector<std::string>&> errMessages)
908 if (availablePreferredBackends.empty())
910 std::stringstream failureMsg;
911 failureMsg <<
"No preferred backends are available";
918 for (
auto it = firstLayer; it != lastLayer; ++it)
920 auto layer = PolymorphicDowncast<Layer*>(*it);
928 bool isFloat16 =
false;
929 for (
auto type : inOutDataType)
938 if (layer->GetBackendId() ==
"Unknown" || isFloat16)
945 availablePreferredBackends);
949 for (
auto it = firstLayer; it != lastLayer; ++it)
951 auto layer = PolymorphicDowncast<Layer*>(*it);
955 BackendId connectedBackendId = layer->GetOutputSlot(0).GetConnection(0)->GetOwningLayer().GetBackendId();
956 layer->SetBackendId(connectedBackendId);
967 Optional<std::vector<std::string>&> errMessages)
973 if (availablePreferredBackends.empty())
975 std::stringstream failureMsg;
976 failureMsg <<
"No preferred backends are available";
983 for (
auto it = firstLayer; it != lastLayer; ++it)
990 availablePreferredBackends);
993 for (
auto it = firstLayer; it != lastLayer; ++it)
995 auto layer = PolymorphicDowncast<Layer*>(*it);
999 BackendId connectedBackendId = layer->GetOutputSlot(0).GetConnection(0)->GetOwningLayer().GetBackendId();
1000 layer->SetBackendId(connectedBackendId);
1010 Optional<std::vector<std::string>&> errMessages)
1028 auto backendFactory = backendRegistry.GetFactory(selectedBackend);
1029 auto backendObjPtr = backendFactory();
1032 backendObjPtr->RegisterTensorHandleFactories(handleFactoryRegistry);
1034 backends[backendObjPtr->GetId()] = std::move(backendObjPtr);
1044 Optional<std::vector<std::string>&> errMessages)
1056 auto backendObjPtr = backends.find(selectedBackend)->second.get();
1069 [&backendObjPtr](
const Layer& layer)
1076 if (subgraphs.empty())
1083 for (
auto& subgraph : subgraphs)
1087 OptimizationViews optimizationViews = backendObjPtr->OptimizeSubgraphView(*subgraph, modelOptions);
1094 SubgraphView& replacementSubgraph = substitution.m_ReplacementSubgraph;
1095 SubgraphView& substitutableSubgraph = substitution.m_SubstitutableSubgraph;
1100 std::for_each(subgraphLayers.begin(), subgraphLayers.end(), [&selectedBackend](
IConnectableLayer* l)
1103 PolymorphicDowncast<Layer*>(l)->SetBackendId(selectedBackend);
1109 std::stringstream warningMsg;
1110 warningMsg <<
"Some sub-graph(s) failed to optimized on " << backendObjPtr->GetId() <<
" backend.";
1115 if (!backendObjPtr->GetId().IsCpuRef())
1125 std::stringstream subgraphMsg;
1126 subgraphMsg <<
"Re-assigning backends to " << failedSubgraph.GetIConnectableLayers().size()
1127 <<
" layers inside sub-graph " << count++;
1134 if (reassignmentResult.
m_Error)
1157 if (srcFactory && dstFactory &&
1183 if (frmBackend == backends.end() ||
1184 !frmBackend->second->SupportsTensorAllocatorAPI())
1191 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
1198 const Layer& connectedLayer = connection->GetOwningLayer();
1200 auto toBackend = backends.find(connectedLayer.
GetBackendId());
1201 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
1203 if (!toBackend->second.get()->SupportsTensorAllocatorAPI())
1209 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1210 for (
auto&& dst : dstPrefs)
1224 auto it = factoryScores.find(dst);
1225 if (it == factoryScores.end())
1228 factoryScores[dst] = 0;
1237 factoryScores[dst]++;
1240 if (factoryScores[dst] > topScore)
1242 topScore = factoryScores[dst];
1271 if (frmBackend == backends.end() ||
1272 !frmBackend->second->SupportsTensorAllocatorAPI())
1277 bool outputConnection =
false;
1280 const Layer& connectedLayer = connection->GetOwningLayer();
1283 outputConnection =
true;
1291 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
1292 for (
auto&& pref : srcPrefs)
1297 if (outputConnection)
1300 bool fallbackConnection =
false;
1303 if (inputSlot.GetConnectedOutputSlot()->GetOwningLayer().GetBackendId() != layer.
GetBackendId())
1305 fallbackConnection =
true;
1308 if (fallbackConnection)
1312 if (!factoryCap.empty())
1322 if (!outputConnection)
1326 if (!factoryCap.empty())
1345 auto it = factoryScores.find(pref);
1346 if (it == factoryScores.end())
1349 factoryScores[pref] = 0;
1356 const Layer& connectedLayer = connection->GetOwningLayer();
1358 auto toBackend = backends.find(connectedLayer.
GetBackendId());
1359 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
1361 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1362 for (
auto&& src : srcPrefs)
1364 if (factoryScores.find(src) == factoryScores.end())
1369 for (
auto&& dst : dstPrefs)
1374 factoryScores[src]++;
1382 int minScore = std::numeric_limits<int>::max();
1383 for (
auto it : factoryScores)
1385 minScore = std::min(minScore, it.second);
1389 std::vector<ITensorHandleFactory::FactoryId> optimalFactories;
1390 for (
auto it : factoryScores)
1392 if (it.second == minScore)
1394 optimalFactories.push_back(it.first);
1399 for (
auto&& srcPref : srcPrefs)
1401 for (
auto&& comp : optimalFactories)
1403 if (comp == srcPref)
1416 const Layer& connectedLayer,
1420 auto toBackend = backends.find(connectedLayer.
GetBackendId());
1421 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
1423 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1446 for (
auto&& pref : dstPrefs)
1448 if (pref == srcFactoryId)
1458 for (
auto&& pref : dstPrefs)
1477 if (srcCapability.empty() && dstCapability.empty() && srcFallback.empty() && dstFallback.empty())
1488 for (
auto&& pref : dstPrefs)
1507 Optional<std::vector<std::string>&> errMessages)
1512 optGraph.
ForEachLayer([&backends, ®istry, &result, &errMessages, importEnabled, exportEnabled](
Layer* layer)
1543 unsigned int connectionIdx = 0;
1546 const Layer& connectedLayer = connection->GetOwningLayer();
1549 registry, importEnabled);
1556 errMessages.value().emplace_back(
"Could not find valid strategy required for compatibility"
1557 " between backends.");
1573 const std::vector<BackendId>& backendPreferences,
1576 Optional<std::vector<std::string>&> messages)
1586 if (backendPreferences.empty())
1594 "Please use the FastMathEnabled backend option for CpuAcc or GpuAcc.");
1605 std::unique_ptr<Graph> graph = std::make_unique<Graph>(inGraph);
1613 optimizedOptions.push_back(importExport);
1630 using namespace optimizations;
1680 std::stringstream failureMsg;
1681 failureMsg <<
"None of the preferred backends " << backendPreferences
1699 if (assignBackendsResult.
m_Error)
1714 if (backendOptimizationResult.
m_Error)
1740 ARMNN_LOG(
info) <<
"Intermediate tensors will be written to: " << result;
1753 tensorHandleFactoryRegistry,
1774 const std::vector<BackendId>& backendPreferences,
1777 Optional<std::vector<std::string>&> messages)
1786 bool NetworkImpl::GetShapeInferenceMethod()
1788 bool shapeInferenceMethod =
false;
1790 ParseOptions(m_NetworkOptions,
"ShapeInferenceMethod", [&](std::string name,
const BackendOptions::Var& value)
1792 if (name ==
"InferAndValidate")
1794 shapeInferenceMethod |= value.AsBool();
1797 return shapeInferenceMethod;
1800 bool NetworkImpl::GetAllowExpandedDims()
1802 bool allowExpandedDims =
false;
1804 ParseOptions(m_NetworkOptions,
"AllowExpandedDims", [&](std::string name,
const BackendOptions::Var& value)
1806 if (name ==
"AllowExpandedDims")
1808 allowExpandedDims |= value.AsBool();
1811 return allowExpandedDims;
1815 : m_NetworkOptions(networkOptions),
1816 m_Graph(
std::make_unique<
Graph>(GetShapeInferenceMethod(), GetAllowExpandedDims()))
1831 return m_Graph->AddLayer<
InputLayer>(id, name);
1842 return m_Graph->AddLayer<
CastLayer>(name);
1853 return m_Graph->AddLayer<
ComparisonLayer>(comparisonDescriptor, name);
1865 return m_Graph->AddLayer<
FillLayer>(fillDescriptor, name);
1877 return m_Graph->AddLayer<
ConcatLayer>(concatDescriptor, name);
1920 layer->
m_Anchors = std::make_shared<ScopedTensorHandle>(anchors);
1928 return m_Graph->AddLayer<
PermuteLayer>(permuteDescriptor, name);
1934 return m_Graph->AddLayer<
Pooling2dLayer>(pooling2dDescriptor, name);
1940 return m_Graph->AddLayer<
Pooling3dLayer>(pooling3dDescriptor, name);
1946 return m_Graph->AddLayer<
ActivationLayer>(activationDescriptor, name);
1952 return m_Graph->AddLayer<
ArgMinMaxLayer>(argMinMaxDescriptor, name);
1956 normalizationDescriptor,
1964 return m_Graph->AddLayer<
SliceLayer>(sliceDescriptor, name);
1970 return m_Graph->AddLayer<
SoftmaxLayer>(softmaxDescriptor, name);
1976 return m_Graph->AddLayer<
SplitterLayer>(splitterDescriptor, name);
2013 layer->
m_Mean = std::make_shared<ScopedTensorHandle>(mean);
2014 layer->m_Variance = std::make_shared<ScopedTensorHandle>(variance);
2015 layer->m_Beta = std::make_shared<ScopedTensorHandle>(beta);
2016 layer->m_Gamma = std::make_shared<ScopedTensorHandle>(gamma);
2023 return m_Graph->AddLayer<
RankLayer>(name);
2029 return m_Graph->AddLayer<
ReduceLayer>(reduceDescriptor, name);
2034 return m_Graph->AddLayer<
ResizeLayer>(resizeDescriptor, name);
2064 layer->
m_LayerOutput = std::make_shared<ScopedTensorHandle>(input);
2072 return m_Graph->AddLayer<
ReshapeLayer>(reshapeDescriptor, name);
2096 const auto layer = m_Graph->AddLayer<
LstmLayer>(descriptor, name);
2101 layer->m_BasicParameters.m_InputToCellWeights =
2103 layer->m_BasicParameters.m_InputToOutputWeights =
2105 layer->m_BasicParameters.m_RecurrentToForgetWeights =
2107 layer->m_BasicParameters.m_RecurrentToCellWeights =
2109 layer->m_BasicParameters.m_RecurrentToOutputWeights =
2111 layer->m_BasicParameters.m_ForgetGateBias =
2113 layer->m_BasicParameters.m_CellBias =
2114 std::make_shared<ScopedTensorHandle>(*(params.
m_CellBias));
2115 layer->m_BasicParameters.m_OutputGateBias =
2119 if(!descriptor.m_CifgEnabled)
2124 "when CIFG is disabled.");
2129 "AddLstmLayer: Recurrent To Input Weights cannot be NULL "
2130 "when CIFG is disabled.");
2135 "when CIFG is disabled.");
2137 layer->m_CifgParameters.m_InputToInputWeights =
2139 layer->m_CifgParameters.m_RecurrentToInputWeights =
2141 layer->m_CifgParameters.m_InputGateBias =
2146 if(descriptor.m_ProjectionEnabled)
2151 "when projection is enabled.");
2153 layer->m_ProjectionParameters.m_ProjectionWeights =
2157 layer->m_ProjectionParameters.m_ProjectionBias =
2163 if(descriptor.m_PeepholeEnabled)
2165 if(!descriptor.m_CifgEnabled)
2170 "when Peephole is enabled and CIFG disabled.");
2173 layer->m_PeepholeParameters.m_CellToInputWeights =
2180 "when Peephole is enabled.");
2185 "when Peephole is enabled.");
2188 layer->m_PeepholeParameters.m_CellToForgetWeights =
2190 layer->m_PeepholeParameters.m_CellToOutputWeights =
2195 if(descriptor.m_LayerNormEnabled)
2197 if(!descriptor.m_CifgEnabled)
2202 "when layer normalization is enabled and CIFG disabled.");
2204 layer->m_LayerNormParameters.m_InputLayerNormWeights =
2211 "when layer normalization is enabled.");
2216 "when layer normalization is enabled.");
2221 "when layer normalization is enabled.");
2223 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
2225 layer->m_LayerNormParameters.m_CellLayerNormWeights =
2227 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
2245 return m_Graph->AddLayer<
MeanLayer>(meanDescriptor,name);
2250 return m_Graph->AddLayer<
PadLayer>(padDescriptor,name);
2272 return m_Graph->AddLayer<
GatherLayer>(gatherDescriptor, name);
2307 layer->
m_Weight = std::make_shared<ScopedTensorHandle>(weights);
2309 if (descriptor.m_BiasEnabled)
2311 layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.
value());
2320 return m_Graph->AddLayer<
TransposeLayer>(transposeDescriptor, name);
2326 return m_Graph->AddLayer<
StackLayer>(stackDescriptor, name);
2344 layer->m_QuantizedLstmParameters.m_InputToForgetWeights =
2346 layer->m_QuantizedLstmParameters.m_InputToCellWeights =
2348 layer->m_QuantizedLstmParameters.m_InputToOutputWeights =
2352 layer->m_QuantizedLstmParameters.m_RecurrentToInputWeights =
2354 layer->m_QuantizedLstmParameters.m_RecurrentToForgetWeights =
2356 layer->m_QuantizedLstmParameters.m_RecurrentToCellWeights =
2358 layer->m_QuantizedLstmParameters.m_RecurrentToOutputWeights =
2362 layer->m_QuantizedLstmParameters.m_InputGateBias =
2364 layer->m_QuantizedLstmParameters.m_ForgetGateBias =
2366 layer->m_QuantizedLstmParameters.m_CellBias =
2367 std::make_shared<ScopedTensorHandle>(params.
GetCellBias());
2368 layer->m_QuantizedLstmParameters.m_OutputGateBias =
2378 const auto layer = m_Graph->AddLayer<
QLstmLayer>(descriptor, name);
2383 layer->m_BasicParameters.m_InputToCellWeights =
2385 layer->m_BasicParameters.m_InputToOutputWeights =
2387 layer->m_BasicParameters.m_RecurrentToForgetWeights =
2389 layer->m_BasicParameters.m_RecurrentToCellWeights =
2391 layer->m_BasicParameters.m_RecurrentToOutputWeights =
2393 layer->m_BasicParameters.m_ForgetGateBias =
2395 layer->m_BasicParameters.m_CellBias =
2396 std::make_shared<ScopedTensorHandle>(*(params.
m_CellBias));
2397 layer->m_BasicParameters.m_OutputGateBias =
2401 if(!descriptor.m_CifgEnabled)
2411 "AddQLstmLayer: Recurrent To Input Weights cannot be NULL");
2419 layer->m_CifgParameters.m_InputToInputWeights =
2421 layer->m_CifgParameters.m_RecurrentToInputWeights =
2423 layer->m_CifgParameters.m_InputGateBias =
2428 if(descriptor.m_ProjectionEnabled)
2435 layer->m_ProjectionParameters.m_ProjectionWeights =
2441 layer->m_ProjectionParameters.m_ProjectionBias =
2448 if(descriptor.m_PeepholeEnabled)
2460 if(!descriptor.m_CifgEnabled)
2467 layer->m_PeepholeParameters.m_CellToInputWeights =
2471 layer->m_PeepholeParameters.m_CellToForgetWeights =
2473 layer->m_PeepholeParameters.m_CellToOutputWeights =
2478 if(descriptor.m_LayerNormEnabled)
2495 if(!descriptor.m_CifgEnabled)
2502 layer->m_LayerNormParameters.m_InputLayerNormWeights =
2506 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
2508 layer->m_LayerNormParameters.m_CellLayerNormWeights =
2510 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
2532 layer->m_BasicParameters.m_InputToCellWeights =
2534 layer->m_BasicParameters.m_InputToOutputWeights =
2536 layer->m_BasicParameters.m_RecurrentToForgetWeights =
2538 layer->m_BasicParameters.m_RecurrentToCellWeights =
2540 layer->m_BasicParameters.m_RecurrentToOutputWeights =
2542 layer->m_BasicParameters.m_ForgetGateBias =
2544 layer->m_BasicParameters.m_CellBias =
2545 std::make_shared<ScopedTensorHandle>(*(params.
m_CellBias));
2546 layer->m_BasicParameters.m_OutputGateBias =
2550 if(!descriptor.m_CifgEnabled)
2555 "when CIFG is disabled.");
2560 "AddUnidirectionalSequenceLstmLayer: Recurrent To Input Weights cannot be NULL "
2561 "when CIFG is disabled.");
2566 "when CIFG is disabled.");
2568 layer->m_CifgParameters.m_InputToInputWeights =
2570 layer->m_CifgParameters.m_RecurrentToInputWeights =
2572 layer->m_CifgParameters.m_InputGateBias =
2577 if(descriptor.m_ProjectionEnabled)
2582 "when projection is enabled.");
2584 layer->m_ProjectionParameters.m_ProjectionWeights =
2588 layer->m_ProjectionParameters.m_ProjectionBias =
2594 if(descriptor.m_PeepholeEnabled)
2596 if(!descriptor.m_CifgEnabled)
2601 "cannot be NULL when Peephole is enabled and CIFG disabled.");
2604 layer->m_PeepholeParameters.m_CellToInputWeights =
2611 "when Peephole is enabled.");
2616 "when Peephole is enabled.");
2619 layer->m_PeepholeParameters.m_CellToForgetWeights =
2621 layer->m_PeepholeParameters.m_CellToOutputWeights =
2626 if(descriptor.m_LayerNormEnabled)
2628 if(!descriptor.m_CifgEnabled)
2633 "cannot be NULL when layer normalization is enabled and CIFG disabled.");
2635 layer->m_LayerNormParameters.m_InputLayerNormWeights =
2642 "cannot be NULL when layer normalization is enabled.");
2647 "cannot be NULL when layer normalization is enabled.");
2652 "cannot be NULL when layer normalization is enabled.");
2654 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
2656 layer->m_LayerNormParameters.m_CellLayerNormWeights =
2658 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
2682 layer = m_Graph->AddLayer<
PreCompiledLayer>(preCompiledDescriptor,
"pre-compiled");
2706 layer->ExecuteStrategy(strategy);
2711 : m_Graph(new
Graph(*other.m_Graph.get()))
2712 , m_Guid(
arm::pipe::IProfilingService::GetNextGuid())
2713 , m_ModelOptions(modelOptions)
2718 : m_Graph(
std::move(graph)), m_Guid(
arm::pipe::IProfilingService::GetNextGuid())
2723 : m_Graph(
std::move(graph)), m_Guid(
arm::pipe::IProfilingService::GetNextGuid()), m_ModelOptions(modelOptions)
2740 layer->ExecuteStrategy(strategy);
OptimizeForType< Layer, AddDebugToFileImpl > InsertDebugToFileLayer
IConnectableLayer * AddConvolution2dLayer(const Convolution2dDescriptor &convolution2dDescriptor, const char *name=nullptr)
Adds a 2D convolution layer to the network.
OptimizeForConnection< ConvertFp16ToFp32Layer, ConvertFp32ToFp16Layer, OptimizeInverseConversionsImpl > OptimizeInverseConversionsFp16
std::shared_ptr< ConstTensorHandle > m_Mean
A unique pointer to store Mean values.
OptimizationResult SelectTensorHandleStrategy(Graph &optGraph, BackendsMap &backends, TensorHandleFactoryRegistry ®istry, bool importEnabled, bool exportEnabled, Optional< std::vector< std::string > & > errMessages)
IConnectableLayer * AddMergeLayer(const char *name=nullptr)
IConnectableLayer * AddDepthwiseConvolution2dLayer(const DepthwiseConvolution2dDescriptor &convolution2dDescriptor, const char *name=nullptr)
Adds a 2D depthwise convolution layer to the network.
IConnectableLayer * AddPrecompiledLayer(const PreCompiledDescriptor &preCompiledDescriptor, CompiledBlobPtr compiledBlobPtr, const Optional< BackendId > &backend, const char *name=nullptr)
const std::shared_ptr< IProfiler > & GetProfiler() const
IConnectableLayer * AddResizeLayer(const ResizeDescriptor &resizeDescriptor, const char *name=nullptr)
IConnectableLayer * AddQuantizedLstmLayer(const QuantizedLstmInputParams ¶ms, const char *name=nullptr)
IConnectableLayer * AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor &elementwiseUnaryDescriptor, const char *name=nullptr)
void SetTensorHandleFactory(const ITensorHandleFactory::FactoryId &id)
const std::shared_ptr< IProfiler > & GetProfiler() const
LayerList::const_iterator Iterator
@ DirectCompatibility
No strategy has been defined. Used internally to verify integrity of optimizations.
This layer represents a ArgMinMax operation.
BackendRegistry & BackendRegistryInstance()
IConnectableLayer * AddDepthToSpaceLayer(const DepthToSpaceDescriptor &depthToSpaceDescriptor, const char *name=nullptr)
OptimizedNetworkImpl(const OptimizedNetworkImpl &other, const ModelOptions &modelOptions)
IConnectableLayer * AddNormalizationLayer(const NormalizationDescriptor &normalizationDescriptor, const char *name=nullptr)
OptimizeForExclusiveConnection< PadLayer, Convolution2dLayer, pad_fold::FoldPadIntoConvolution2dImpl > FoldPadIntoConvolution2d
unsigned int GetNumOutputSlots() const override
Returns the number of connectable output slots.
IConnectableLayer * AddFloorLayer(const char *name=nullptr)
Adds a floor layer to the network.
IConnectableLayer * AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor &batchToSpaceNdDescriptor, const char *name=nullptr)
A layer that the constant data can be bound to.
A GatherDescriptor for the GatherLayer.
A NormalizationDescriptor for the NormalizationLayer.
void SetDataType(DataType type)
static const FactoryId DeferredFactoryId
Use the workload factory to create the tensor handle.
A TransposeDescriptor for the TransposeLayer.
const std::string & Get() const
std::vector< SubgraphView::SubgraphViewPtr > Subgraphs
IConnectableLayer * AddConstantLayer(const ConstTensor &input, const char *name=nullptr)
Adds a layer with no inputs and a single output, which always corresponds to the passed in constant t...
OptimizeForConnection< ConstantLayer, PermuteLayer, ConvertConstPermuteLayersToConstLayers > FusePermuteIntoConstLayer
std::vector< ConvertFp32ToFp16Layer * > InsertConvertFp32ToFp16LayersAfter(Graph &graph, Layer &layer)
void ExecuteStrategy(IStrategy &strategy) const
IConnectableLayer * AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor &descriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr)
Adds a 2D transpose convolution layer to the network.
This layer represents a batch normalization operation.
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
A ElementwiseUnaryDescriptor for the ElementwiseUnaryLayer.
IConnectableLayer * AddPooling2dLayer(const Pooling2dDescriptor &pooling2dDescriptor, const char *name=nullptr)
A PadDescriptor for the PadLayer.
A SoftmaxDescriptor for the SoftmaxLayer.
static Subgraphs SelectSubgraphs(Graph &graph, const LayerSelectorFunction &selector)
Selects subgraphs from a graph based on the selector function and the algorithm.
IConnectableLayer * AddConvolution2dLayer(const Convolution2dDescriptor &convolution2dDescriptor, const char *name=nullptr)
std::unique_ptr< NetworkImpl > pNetworkImpl
OptimizeForConnection< Layer, TransposeLayer, MoveTransposeUpImpl > MoveTransposeUp
BackendIdVector m_PreferredBackends
A StackDescriptor for the StackLayer.
IConnectableLayer * AddPooling3dLayer(const Pooling3dDescriptor &pooling3dDescriptor, const char *name=nullptr)
Adds a 3D pooling layer to the network.
This layer represents a fill operation.
A SliceDescriptor for the SliceLayer.
IConnectableLayer * AddDivisionLayer(const char *name=nullptr)
Adds a division layer to the network.
Status SerializeToDot(std::ostream &stream) const
BackendIdVector GetAvailablePreferredBackends() const
bool m_ExportEnabled
Enable Export.
bool m_ReduceFp32ToFp16
Reduces all Fp32 operators in the model to Fp16 for faster processing.
static INetwork * CreateRaw(const NetworkOptions &networkOptions={})
static void Pass(Graph &graph, const Optimizations &optimizations)
OptimizationResult ReturnWithError(OptimizationResult res, const Layer *layer, const BackendSettings &backendSettings, Optional< std::vector< std::string > & > errMessages)
std::string CreateDirectory(std::string sPath)
Returns full path to temporary folder.
IConnectableLayer * AddQLstmLayer(const QLstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
Add a QLstm layer to the network.
IConnectableLayer * AddLogSoftmaxLayer(const LogSoftmaxDescriptor &logSoftmaxDescriptor, const char *name=nullptr)
Adds a log softmax layer to the network.
std::map< BackendId, std::unique_ptr< class IBackendInternal > > BackendsMap
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
static const FactoryId LegacyFactoryId
An ActivationDescriptor for the ActivationLayer.
@ ValidateOnly
Validate all output shapes.
IConnectableLayer * AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor &desc, const char *name=nullptr)
OptimizeForConnection< ConvertFp32ToFp16Layer, ConvertFp16ToFp32Layer, OptimizeInverseConversionsImpl > OptimizeInverseConversionsFp32
virtual bool SupportsMapUnmap() const
Interface for a layer that is connectable to other layers via InputSlots and OutputSlots.
IConnectableLayer * AddRankLayer(const char *name=nullptr)
An LstmDescriptor for the LstmLayer.
This layer represents a split operation.
This layer represents a SpaceToBatchNd operation.
A FullyConnectedDescriptor for the FullyConnectedLayer.
void SetTensorInfo(const TensorInfo &tensorInfo) override
ModelOptions m_ModelOptions
Enable Model Options.
ITensorHandleFactory::FactoryId CalculateSlotOptionForOutput(BackendsMap &backends, OutputSlot &slot, TensorHandleFactoryRegistry ®istry)
virtual MemorySourceFlags GetImportFlags() const
IConnectableLayer * AddMeanLayer(const MeanDescriptor &meanDescriptor, const char *name=nullptr)
void ParseOptions(const std::vector< BackendOptions > &options, BackendId backend, F f)
OptimizationResult AttemptBackendAssignment(BackendSettings &backendSettings, Graph &graph, Layer *layer, BackendId backend, DataType dataTypeIn, DataType dataTypeOut, const std::vector< BackendId > &availablePreferredBackends, std::string &reasonIfUnsupported, Optional< std::vector< std::string > & > errMessages)
void SetBackendId(const BackendId &id) override
Set the backend of the IConnectableLayer.
IConnectableLayer * AddGatherNdLayer(const char *name=nullptr)
IConnectableLayerIterator beginIConnectable()
OptimizationResult ApplyBackendOptimizations(OptimizedNetworkImpl *optNetObjPtr, BackendSettings &backendSettings, BackendsMap &backends, const ModelOptions &modelOptions, Optional< std::vector< std::string > & > errMessages)
IConnectableLayer * AddDetectionPostProcessLayer(const DetectionPostProcessDescriptor &descriptor, const ConstTensor &anchors, const char *name=nullptr)
Adds a Detection PostProcess layer to the network.
IConnectableLayer * AddSubtractionLayer(const char *name=nullptr)
Adds a subtraction layer to the network.
BackendIdSet m_SelectedBackends
A BatchMatMulDescriptor for the BatchMatMul operator.
This layer represents a QLstm operation.
@ CpuAcc
CPU Execution: NEON: ArmCompute.
This layer represents a transpose operation.
@ GpuAcc
GPU Execution: OpenCL: ArmCompute.
A ResizeDescriptor for the ResizeLayer.
IConnectableLayer * AddSliceLayer(const SliceDescriptor &sliceDescriptor, const char *name=nullptr)
Adds a slice layer to the network.
IConnectableLayer * AddMinimumLayer(const char *name=nullptr)
A StridedSliceDescriptor for the StridedSliceLayer.
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
OptimizeForConnection< TransposeLayer, TransposeLayer, OptimizeInversePermutesImpl< TransposeLayer > > OptimizeInverseTransposes
This layer represents a merge operation.
A Pooling3dDescriptor for the Pooling3dLayer.
A ReduceDescriptor for the REDUCE operators.
bool m_BiasEnabled
Enable/disable bias.
Struct for the users to pass backend specific options.
IConnectableLayers::iterator IConnectableLayerIterator
This layer represents a multiplication operation.
This layer represents a GatherNd operator.
std::vector< BackendOptions > ModelOptions
OptimizeForConnection< Layer, ReshapeLayer, SquashEqualSiblingsImpl< ReshapeLayer > > SquashEqualReshapeSiblings
A ComparisonDescriptor for the ComparisonLayer.
void IgnoreUnused(Ts &&...)
IConnectableLayer * AddGatherLayer(const GatherDescriptor &gatherDescriptor, const char *name=nullptr)
IConnectableLayer * AddBatchMatMulLayer(const BatchMatMulDescriptor &desc, const char *name=nullptr)
A StandInDescriptor for the StandIn layer.
QuantizedLstmParameters m_QuantizedLstmParameters
std::list< IConnectableLayer * > IConnectableLayers
This layer represents a LSTM operation.
IConnectableLayer * AddQuantizeLayer(const char *name=nullptr)
Add a quantize layer to the network.
IConnectableLayer * AddMinimumLayer(const char *name=nullptr)
Add a Minimum layer to the network.
virtual const char * what() const noexcept override
IConnectableLayer * AddMultiplicationLayer(const char *name=nullptr)
A ViewsDescriptor for the SplitterLayer.
IConnectableLayer * AddAdditionLayer(const char *name=nullptr)
IConnectableLayer * AddLogSoftmaxLayer(const LogSoftmaxDescriptor &logSoftmaxDescriptor, const char *name=nullptr)
IConnectableLayer * AddChannelShuffleLayer(const ChannelShuffleDescriptor &channelShuffleDescriptor, const char *name=nullptr)
Iterator begin()
Returns iterator pointing to the beginning of the list. Lowercase for range-based for loops.
IConnectableLayer * AddStridedSliceLayer(const StridedSliceDescriptor &stridedSliceDescriptor, const char *name=nullptr)
OptimizeForType< Layer, AddBroadcastReshapeLayerImpl > AddBroadcastReshapeLayer
A PreCompiledDescriptor for the PreCompiledLayer.
virtual Status SerializeToDot(std::ostream &stream) const
BackendsMap CreateSupportedBackends(TensorHandleFactoryRegistry &handleFactoryRegistry, BackendSettings &backendSettings)
void VerifyConstantLayerSetTensorInfo() const
For each ConstantLayer in Graph, ensures TensorInfo is set on all output slots.
IConnectableLayer * AddPooling2dLayer(const Pooling2dDescriptor &pooling2dDescriptor, const char *name=nullptr)
Adds a 2D pooling layer to the network.
This layer represents an activation operation with the specified activation function.
This layer dequantizes the input tensor.
Optimizer::Optimizations MakeOptimizations(Args &&... args)
This layer represents a permutation operation.
OptimizeForConnection< Layer, PermuteLayer, MovePermuteUpImpl > MovePermuteUp
This layer represents a Logical Binary operation.
This layer represents a reshape operation.
const IConnectableLayers & GetIConnectableLayers() const
bool RequiresCopy(ITensorHandleFactory::FactoryId src, ITensorHandleFactory::FactoryId dst, TensorHandleFactoryRegistry ®istry)
IConnectableLayer * AddBatchMatMulLayer(const BatchMatMulDescriptor &descriptor, const char *name=nullptr)
Add a BatchMatMul layer to the network.
#define ARMNN_LOG(severity)
size_t GetNumOutputs() const
IConnectableLayer * AddDepthwiseConvolution2dLayer(const DepthwiseConvolution2dDescriptor &convolution2dDescriptor, const char *name=nullptr)
This layer represents a log softmax operation.
IConnectableLayer * AddActivationLayer(const ActivationDescriptor &activationDescriptor, const char *name=nullptr)
IConnectableLayer * AddReduceLayer(const ReduceDescriptor &reduceDescriptor, const char *name=nullptr)
Adds a reduce layer to the network.
This layer represents a detection postprocess operator.
IConnectableLayer * AddConstantLayer(const ConstTensor &input, const char *name=nullptr)
IConnectableLayer * AddDequantizeLayer(const char *name=nullptr)
Adds a Dequantize layer to the network.
static void Destroy(INetwork *network)
IConnectableLayer * AddConvertFp32ToFp16Layer(const char *name=nullptr)
Copyright (c) 2021 ARM Limited and Contributors.
virtual std::vector< Capability > GetCapabilities(const IConnectableLayer *layer, const IConnectableLayer *connectedLayer, CapabilityClass capabilityClass)
IConnectableLayer * AddMaximumLayer(const char *name=nullptr)
Add a Maximum layer to the network.
IConnectableLayerIterator endIConnectable()
IConnectableLayer * AddStackLayer(const StackDescriptor &descriptor, const char *name=nullptr)
Adds a stack layer to the network.
ConvertConstants< Float32ToFloat16, IsFloat16Layer > ConvertConstantsFloatToHalf
INetwork(NetworkOptions networkOptions={})
void ReportWarning(const std::string &warningMessage, Optional< std::vector< std::string > & > warningMessages)
const TensorInfo & GetTensorInfo() const override
A BatchToSpaceNdDescriptor for the BatchToSpaceNdLayer.
IConnectableLayer * AddPrecompiledLayer(const PreCompiledDescriptor &preCompiledDescriptor, CompiledBlobPtr compiledBlobPtr, const Optional< BackendId > &backend, const char *name=nullptr)
Adds a Precompiled layer to the network.
This layer represents a 2D transpose convolution operation.
A SpaceToDepthDescriptor for the SpaceToDepthLayer.
IConnectableLayer * AddMultiplicationLayer(const char *name=nullptr)
Adds a multiplication layer to the network.
OptimizeForConnection< Layer, TransposeLayer, SquashEqualSiblingsImpl< TransposeLayer > > SquashEqualTransposeSiblings
IConnectableLayer * AddDivisionLayer(const char *name=nullptr)
std::shared_ptr< ConstTensorHandle > m_Weight
A unique pointer to store weight values.
This layer calculates both true and false outputs for input.
void SubstituteSubgraph(SubgraphView &subgraph, IConnectableLayer *substituteLayer)
Substitutes the given sub-graph with either a new layer or a new sub-graph.
std::vector< BackendOptions > NetworkOptions
This layer represents a Gather operator.
IConnectableLayer * AddNormalizationLayer(const NormalizationDescriptor &normalizationDescriptor, const char *name=nullptr)
Adds a normalization layer to the network.
std::vector< DataType > GetLayerInOutDatatype(const Layer *layer)
IConnectableLayer * AddOutputLayer(LayerBindingId id, const char *name=nullptr)
bool m_ProfilingEnabled
Enable profiling dump of the optimizer phase.
IConnectableLayer * AddSpaceToDepthLayer(const SpaceToDepthDescriptor &spaceToDepthDescriptor, const char *name=nullptr)
const std::string & GetNameStr() const
This layer dequantizes the input tensor.
IConnectableLayer * AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor &spaceToBatchNdDescriptor, const char *name=nullptr)
Adds a space to batch layer to the network.
Iterator end()
Returns iterator pointing to the end of the list. Lowercase for range-based for loops.
void SetEdgeStrategy(unsigned int connectionIndex, EdgeStrategy strategy)
A FillDescriptor for the FillLayer.
IConnectableLayer * AddConvolution3dLayer(const Convolution3dDescriptor &convolution3dDescriptor, const char *name=nullptr)
Adds a 3D convolution layer to the network.
LayerType
When adding a new layer, adapt also the LastLayer enum value in the enum class LayerType below.
IConnectableLayer * AddConvertFp16ToFp32Layer(const char *name=nullptr)
unsigned int GetNumInputSlots() const override
Returns the number of connectable input slots.
bool Validate(const SubgraphView &originalSubgraph) const
std::shared_ptr< ConstTensorHandle > m_Anchors
A unique pointer to store Anchor values.
IConnectableLayer * AddL2NormalizationLayer(const L2NormalizationDescriptor &desc, const char *name=nullptr)
Adds an L2 normalization layer to the network.
IConnectableLayer * AddSubtractionLayer(const char *name=nullptr)
IConnectableLayer * AddDetectionPostProcessLayer(const DetectionPostProcessDescriptor &descriptor, const ConstTensor &anchors, const char *name=nullptr)
const OutputSlot & GetOutputSlot(unsigned int index=0) const override
Get the const output slot handle by slot index.
bool IsCpuRefUsed() const
virtual std::vector< ITensorHandleFactory::FactoryId > GetHandleFactoryPreferences() const
(Optional) Returns a vector of supported TensorHandleFactory ids in preference order.
IConnectableLayer * AddFloorLayer(const char *name=nullptr)
IConnectableLayer * AddTransposeLayer(const TransposeDescriptor &transposeDescriptor, const char *name=nullptr)
Adds a transpose layer to the network.
IConnectableLayer * AddStandInLayer(const StandInDescriptor &descriptor, const char *name=nullptr)
Add a stand-in layer for a type unknown to the Arm NN framework.
This layer represents a subtraction operation.
IConnectableLayer * AddAdditionLayer(const char *name=nullptr)
Adds an addition layer to the network.
This layer represents a normalization operation.
IConnectableLayer * AddPadLayer(const PadDescriptor &padDescriptor, const char *name=nullptr)
Adds a fully pad layer to the network.
A DepthwiseConvolution2dDescriptor for the DepthwiseConvolution2dLayer.
A MeanDescriptor for the MeanLayer.
std::shared_ptr< ConstTensorHandle > m_InputToInputWeights
A unique pointer to represent 2D weights tensor with dimensions [outputSize, inputSize] (QAsymm8).
IConnectableLayer * AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor &desc, const char *name=nullptr)
Adds an instance normalization layer to the network.
IConnectableLayer * AddPermuteLayer(const PermuteDescriptor &permuteDescriptor, const char *name=nullptr)
IConnectableLayer * AddPooling3dLayer(const Pooling3dDescriptor &pooling3dDescriptor, const char *name=nullptr)
bool m_DebugToFile
Pass debug data to separate output files for easier troubleshooting.
This layer represents a strided slice operation.
const std::vector< InputSlot * > & GetConnections() const
Device specific knowledge to be passed to the optimizer.
arm::pipe::ProfilingGuid GetGuid() const
LayerType GetType() const override
Returns the armnn::LayerType of this layer.
static void Destroy(IOptimizedNetwork *network)
IConnectableLayer * AddInputLayer(LayerBindingId id, const char *name=nullptr)
IConnectableLayer * AddMergeLayer(const char *name=nullptr)
Adds a merge layer to the network.
std::unique_ptr< void, CompiledBlobDeleter > CompiledBlobPtr
IConnectableLayer * AddConcatLayer(const ConcatDescriptor &concatDescriptor, const char *name=nullptr)
void ReportError(const std::string &errorMessage, Optional< std::vector< std::string > & > errorMessages)
std::unique_ptr< OptimizedNetworkImpl > pOptimizedNetworkImpl
#define ARMNN_SCOPED_PROFILING_EVENT(backendId, name)
The SubgraphView class represents a subgraph of a Graph.
A L2NormalizationDescriptor for the L2NormalizationLayer.
IConnectableLayer * AddPermuteLayer(const PermuteDescriptor &permuteDescriptor, const char *name=nullptr)
Adds a permute layer to the network.
static bool IsLayerSupported(const BackendId &backendId, const IConnectableLayer &layer, Optional< DataType > dataType, std::string &outReasonIfUnsupported)
LstmBasicParameters m_BasicParameters
This layer represents a QuantizedLstm operation.
A ChannelShuffleDescriptor for the ChannelShuffle operator.
A Convolution3dDescriptor for the Convolution3dLayer.
This layer represents a BatchToSpaceNd operation.
const std::string ToString() const
OptimizeForType< Layer, ConvertFp32NetworkToFp16Impl > Fp32NetworkToFp16Converter
static ProfilerManager & GetInstance()
A Convolution2dDescriptor for the Convolution2dLayer.
IConnectableLayer * AddComparisonLayer(const ComparisonDescriptor &comparisonDescriptor, const char *name=nullptr)
Add a Comparison layer to the network.
IConnectableLayer * AddStandInLayer(const StandInDescriptor &descriptor, const char *name=nullptr)
const BackendId & GetBackendId() const
void AddCompatibilityLayers(std::map< BackendId, std::unique_ptr< class IBackendInternal >> &backends, TensorHandleFactoryRegistry ®istry)
Modifies the graph in-place, removing edges connecting layers using different compute devices,...
IConnectableLayer * AddQLstmLayer(const QLstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
IOptimizedNetwork(const IOptimizedNetwork &other, const ModelOptions &modelOptions)
Creates a copy of the IOptimizedNetwork.
This layer represents an instance normalization operation.
IConnectableLayer * AddCastLayer(const char *name=nullptr)
Adds a cast layer to the network.
A BatchNormalizationDescriptor for the BatchNormalizationLayer.
This layer represents a cast operation.
OptimizeForType< Layer, AddDebugImpl > InsertDebugLayer
A QLstmDescriptor for the QLstmLayer.
QLstmBasicParameters m_BasicParameters
BackendIdSet m_IgnoredBackends
bool has_value() const noexcept
This layer represents a softmax operation.
OptimizeForExclusiveConnection< DepthwiseConvolution2dLayer, BatchNormalizationLayer, FuseBatchNorm< DepthwiseConvolution2dLayer, armnn::DataType::Float16 > > FuseBatchNormIntoDepthwiseConvolution2DFloat16
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
bool m_ReduceFp32ToBf16
@Note This feature has been replaced by enabling Fast Math in compute library backend options.
OptimizeForType< TransposeLayer, TransposeAsReshapeImpl > TransposeAsReshape
void SetPreCompiledObject(PreCompiledObjectPtr preCompiledObject)
void ExecuteStrategy(IStrategy &strategy) const
ShapeInferenceMethod m_shapeInferenceMethod
Infer output size when not available.
This layer represents a reduction operation.
IConnectableLayer * AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor &descriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr)
IConnectableLayer * AddFillLayer(const FillDescriptor &fillDescriptor, const char *name=nullptr)
Layer & GetOwningLayer() const
IConnectableLayer * AddActivationLayer(const ActivationDescriptor &activationDescriptor, const char *name=nullptr)
Adds an activation layer to the network.
IConnectableLayer * AddGatherNdLayer(const char *name=nullptr)
Add GatherNd layer to the network.
ITensorHandleFactory * GetFactory(ITensorHandleFactory::FactoryId id) const
Find a TensorHandleFactory by Id Returns nullptr if not found.
This layer represents a fully connected operation.
IConnectableLayer * AddPreluLayer(const char *name=nullptr)
const std::vector< InputSlot > & GetInputSlots() const
std::shared_ptr< ConstTensorHandle > m_LayerOutput
#define ARMNN_ASSERT_MSG(COND, MSG)
This layer represents a pad operation.
virtual Status PrintGraph()
size_t GetNumInputs() const
IConnectableLayer * AddChannelShuffleLayer(const ChannelShuffleDescriptor &descriptor, const char *name=nullptr)
Add a ChannelShuffle layer to the network.
IConnectableLayer * AddGatherLayer(const GatherDescriptor &descriptor, const char *name=nullptr)
Add Gather layer to the network.
This layer represents a pooling 3d operation.
const Subgraphs & GetFailedSubgraphs() const
This layer represents a pooling 2d operation.
IConnectableLayer * AddComparisonLayer(const ComparisonDescriptor &comparisonDescriptor, const char *name=nullptr)
const InputSlot & GetInputSlot(unsigned int index) const override
Get a const input slot handle by slot index.
static INetworkPtr Create(const NetworkOptions &networkOptions={})
IConnectableLayer * AddCastLayer(const char *name=nullptr)
IConnectableLayer * AddArgMinMaxLayer(const ArgMinMaxDescriptor &desc, const char *name=nullptr)
IConnectableLayer * AddDepthToSpaceLayer(const DepthToSpaceDescriptor &depthToSpaceDescriptor, const char *name=nullptr)
Adds a depth to space layer to the network.
An OriginsDescriptor for the ConcatLayer.
IConnectableLayer * AddDequantizeLayer(const char *name=nullptr)
A ReshapeDescriptor for the ReshapeLayer.
void ForEachLayer(Func func) const
IConnectableLayer * AddL2NormalizationLayer(const L2NormalizationDescriptor &desc, const char *name=nullptr)
EdgeStrategy CalculateEdgeStrategy(BackendsMap &backends, ITensorHandleFactory::FactoryId srcFactoryId, const Layer &layer, const Layer &connectedLayer, TensorHandleFactoryRegistry ®istry, bool importEnabled)
IConnectableLayer * AddPreluLayer(const char *name=nullptr)
Adds a PReLU layer to the network.
OptimizeForExclusiveConnection< PadLayer, DepthwiseConvolution2dLayer, pad_fold::FoldPadIntoDepthwiseConvolution2dImpl > FoldPadIntoDepthwiseConvolution2d
IConnectableLayer * AddConvolution3dLayer(const Convolution3dDescriptor &convolution3dDescriptor, const char *name=nullptr)
This layer represents a depthwise convolution 2d operation.
A PermuteDescriptor for the PermuteLayer.
A TransposeConvolution2dDescriptor for the TransposeConvolution2dLayer.
IConnectableLayer * AddSpaceToDepthLayer(const SpaceToDepthDescriptor &spaceToDepthDescriptor, const char *name=nullptr)
Adds a space to depth layer to the network.
void AssignBackendsIConnectable(OptimizedNetworkImpl *optNetObjPtr, IConnectableLayer *it, Optional< std::vector< std::string > & > errMessages, OptimizationResult &result, BackendSettings &backendSettings, std::vector< BackendId > &availablePreferredBackends)
IConnectableLayer * AddSwitchLayer(const char *name=nullptr)
Adds a switch layer to the network.
const Graph & GetGraph() const
IConnectableLayer * AddSwitchLayer(const char *name=nullptr)
OptimizeForConnection< PermuteLayer, BatchToSpaceNdLayer, PermuteAndBatchToSpaceAsDepthToSpaceImpl< PermuteLayer > > PermuteAndBatchToSpaceAsDepthToSpace
Optional< BackendId > GetBackendHint() const
This layer represents a L2 normalization operation.
#define ARMNN_ASSERT(COND)
IConnectableLayer * AddMeanLayer(const MeanDescriptor &meanDescriptor, const char *name=nullptr)
Add a Mean layer to the network.
BackendIdSet m_SupportedBackends
IConnectableLayer * AddLogicalBinaryLayer(const LogicalBinaryDescriptor &logicalBinaryDescriptor, const char *name=nullptr)
void ExecuteStrategy(IStrategy &strategy) const
IConnectableLayer * AddShapeLayer(const char *name=nullptr)
Adds a shape layer to the network.
A Pooling2dDescriptor for the Pooling2dLayer.
A LogicalBinaryDescriptor for the LogicalBinaryLayer.
IConnectableLayer * AddResizeLayer(const ResizeDescriptor &resizeDescriptor, const char *name=nullptr)
Adds a resize layer to the network.
This layer represents an addition operation.
ITensorHandleFactory::FactoryId CalculateSlotOption(BackendsMap &backends, OutputSlot &outputSlot, TensorHandleFactoryRegistry ®istry, bool exportEnabled)
IConnectableLayer * AddUnidirectionalSequenceLstmLayer(const UnidirectionalSequenceLstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
OptimizeForConnection< ConstantLayer, DequantizeLayer, ConvertConstDequantisationLayersToConstLayersImpl > ConvertConstDequantisationLayersToConstLayers
IConnectableLayer * AddConcatLayer(const ConcatDescriptor &concatDescriptor, const char *name=nullptr)
Adds a concatenation layer to the network.
OptimizeForConnection< PermuteLayer, PermuteLayer, OptimizeInversePermutesImpl< PermuteLayer > > OptimizeInversePermutes
IConnectableLayer * AddFullyConnectedLayer(const FullyConnectedDescriptor &fullyConnectedDescriptor, const char *name=nullptr)
IConnectableLayer * AddReshapeLayer(const ReshapeDescriptor &reshapeDescriptor, const char *name=nullptr)
IConnectableLayer * AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor &batchToSpaceNdDescriptor, const char *name=nullptr)
Adds a batch to space ND layer to the network.
Main network class which provides the interface for building up a neural network.
OptimizeForExclusiveConnection< Convolution2dLayer, BatchNormalizationLayer, FuseBatchNorm< Convolution2dLayer, armnn::DataType::Float16 > > FuseBatchNormIntoConvolution2DFloat16
std::shared_ptr< ConstTensorHandle > m_InputToForgetWeights
A unique pointer to represent 2D weights tensor with dimensions [input_size, num_units].
OptimizeForExclusiveConnection< PadLayer, Pooling2dLayer, pad_fold::FoldPadIntoPooling2dImpl > FoldPadIntoPooling2d
void ExecuteStrategy(IStrategy &strategy) const
IConnectableLayer * AddFillLayer(const FillDescriptor &fillDescriptor, const char *name=nullptr)
Add an Fill layer to the network.
LstmBasicParameters m_BasicParameters
This layer represents a mean operation.
IConnectableLayer * AddQuantizedLstmLayer(const QuantizedLstmInputParams ¶ms, const char *name=nullptr)
Add a QuantizedLstm layer to the network.
bool IsWarningOnly() const
This layer converts data type Float 16 to Float 32.
This layer represents an unknown operation in the input graph.
IConnectableLayer * AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor &elementwiseUnaryDescriptor, const char *name=nullptr)
Add an ElementwiseUnary layer to the network.
IConnectableLayer * AddStridedSliceLayer(const StridedSliceDescriptor &stridedSliceDescriptor, const char *name=nullptr)
Adds a strided slice layer to the network.
This layer represents a floor operation.
IConnectableLayer * AddRankLayer(const char *name=nullptr)
Adds a rank layer to the network.
OptimizeForExclusiveConnection< Convolution2dLayer, BatchNormalizationLayer, FuseBatchNorm< Convolution2dLayer, armnn::DataType::Float32 > > FuseBatchNormIntoConvolution2DFloat32
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
An ArgMinMaxDescriptor for ArgMinMaxLayer.
bool IsBackendSupported(const BackendId &backend) const
const Substitutions & GetSubstitutions() const
Private implementation of INetwork.
This layer represents a DepthToSpace operation.
This layer represents a resize operation.
This layer represents a LSTM operation.
IConnectableLayer * AddTransposeLayer(const TransposeDescriptor &transposeDescriptor, const char *name=nullptr)
IConnectableLayer * AddUnidirectionalSequenceLstmLayer(const UnidirectionalSequenceLstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
Add a UnidirectionalSequenceLstm layer to the network.
bool CheckScaleSetOnQuantizedType(Layer *layer, Optional< std::vector< std::string > & > errMessages)
OptimizeForConnection< Layer, PermuteLayer, SquashEqualSiblingsImpl< PermuteLayer > > SquashEqualPermuteSiblings
std::shared_ptr< ConstTensorHandle > m_InputToForgetWeights
A unique pointer to represent 2D weights tensor with dimensions [num_units, inputSize] (QSymmS8).
std::vector< ConvertFp16ToFp32Layer * > InsertConvertFp16ToFp32LayersBefore(Graph &graph, Layer &layer, bool expectCorrectInputType)
OptimizeForExclusiveConnection< DepthwiseConvolution2dLayer, BatchNormalizationLayer, FuseBatchNorm< DepthwiseConvolution2dLayer, armnn::DataType::Float32 > > FuseBatchNormIntoDepthwiseConvolution2DFloat32
IConnectableLayer * AddLstmLayer(const LstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
This layer represents a division operation.
This layer represents a maximum operation.
This layer represents a minimum operation.
@ ExportToTarget
Destination backend can work directly with tensors on source backend.
This layer represents a convolution 2d operation.
This layer represents a convolution 3d operation.
ConvertConstants< Float16ToFloat32, IsFloat32Layer > ConvertConstantsHalfToFloat
This layer converts data type Float 32 to Float 16.
IConnectableLayer * AddMaximumLayer(const char *name=nullptr)
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > & > messages=EmptyOptional())
Create an optimized version of the network.
IConnectableLayer * AddShapeLayer(const char *name=nullptr)
OptimizationResult AssignBackends(OptimizedNetworkImpl *optNetObjPtr, BackendSettings &backendSettings, Graph::Iterator &firstLayer, Graph::Iterator &lastLayer, Optional< std::vector< std::string > & > errMessages)
virtual MemorySourceFlags GetExportFlags() const
This layer represents a comparison operation.
IConnectableLayer * AddBatchNormalizationLayer(const BatchNormalizationDescriptor &desc, const ConstTensor &mean, const ConstTensor &variance, const ConstTensor &beta, const ConstTensor &gamma, const char *name=nullptr)
This layer represents a SpaceToDepth operation.
IConnectableLayer * AddStackLayer(const StackDescriptor &stackDescriptor, const char *name=nullptr)
constexpr const char * GetDataTypeName(DataType dataType)
IConnectableLayer * AddSoftmaxLayer(const SoftmaxDescriptor &softmaxDescriptor, const char *name=nullptr)
static void ConvertFloat16To32(const void *srcFloat16Buffer, size_t numElements, float *dstFloat32Buffer)
virtual ~OptimizedNetworkImpl()
An InstanceNormalizationDescriptor for InstanceNormalizationLayer.
IConnectableLayer * AddBatchNormalizationLayer(const BatchNormalizationDescriptor &desc, const ConstTensor &mean, const ConstTensor &variance, const ConstTensor &beta, const ConstTensor &gamma, const char *name=nullptr)
Adds a batch normalization layer to the network.
OptimizeForType< PermuteLayer, PermuteAsReshapeImpl > PermuteAsReshape
IConnectableLayer * AddSliceLayer(const SliceDescriptor &sliceDescriptor, const char *name=nullptr)
IConnectableLayer * AddInputLayer(LayerBindingId id, const char *name=nullptr)
Adds an input layer to the network.
IConnectableLayer * AddQuantizeLayer(const char *name=nullptr)
IConnectableLayer * AddArgMinMaxLayer(const ArgMinMaxDescriptor &desc, const char *name=nullptr)
Adds an ArgMinMax layer to the network.
virtual size_t GetNumInputs() const
This layer represents a elementwiseUnary operation.
@ CpuRef
CPU Execution: Reference C++ kernels.
ArmNN performs an optimization on each model/network before it gets loaded for execution.
A layer user-provided data can be bound to (e.g. inputs, outputs).
IConnectableLayer * AddLogicalBinaryLayer(const LogicalBinaryDescriptor &descriptor, const char *name=nullptr)
Adds a Logical Binary layer to the network.
@ CopyToTarget
Source backends tensor data can be exported to destination backend tensor without copy.
DataType GetDataType() const
OptimizeForConnection< ReshapeLayer, ReshapeLayer, OptimizeConsecutiveReshapesImpl > OptimizeConsecutiveReshapes
IConnectableLayer * AddLstmLayer(const LstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
Add a Lstm layer to the network.
IConnectableLayer * AddSplitterLayer(const ViewsDescriptor &splitterDescriptor, const char *name=nullptr)
IConnectableLayer * AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor &spaceToBatchNdDescriptor, const char *name=nullptr)
OptimizeForConnection< TransposeLayer, BatchToSpaceNdLayer, PermuteAndBatchToSpaceAsDepthToSpaceImpl< TransposeLayer > > TransposeAndBatchToSpaceAsDepthToSpace
IConnectableLayer * AddSplitterLayer(const ViewsDescriptor &splitterDescriptor, const char *name=nullptr)
Adds a splitter layer to the network.
virtual size_t GetNumOutputs() const
bool m_Debug
Add debug data for easier troubleshooting.
void RegisterProfiler(IProfiler *profiler)
IConnectableLayer * AddReduceLayer(const ReduceDescriptor &reduceDescriptor, const char *name=nullptr)
IConnectableLayer * AddSoftmaxLayer(const SoftmaxDescriptor &softmaxDescriptor, const char *name=nullptr)
Adds a softmax layer to the network.
const char * GetLayerTypeAsCString(LayerType type)
IConnectableLayer * AddOutputLayer(LayerBindingId id, const char *name=nullptr)
Adds an output layer to the network.
IConnectableLayer * AddPadLayer(const PadDescriptor &padDescriptor, const char *name=nullptr)
IConnectableLayer * AddFullyConnectedLayer(const FullyConnectedDescriptor &fullyConnectedDescriptor, const char *name=nullptr)
Adds a fully connected layer to the network.
ITensorHandleFactory::FactoryId CalculateSlotOptionForInput(BackendsMap &backends, OutputSlot &slot, TensorHandleFactoryRegistry ®istry, bool importEnabled)
This layer represents a stack operation.
@ InferAndValidate
Infer missing output shapes and validate all output shapes.
IConnectableLayer * AddReshapeLayer(const ReshapeDescriptor &reshapeDescriptor, const char *name=nullptr)
Adds a reshape layer to the network.
NetworkImpl(const NetworkOptions &networkOptions={})
A SpaceToBatchNdDescriptor for the SpaceToBatchNdLayer.
bool m_ImportEnabled
Enable Import.