15 #include <fmt/format.h>
17 #include <google/protobuf/text_format.h>
18 #include <google/protobuf/io/zero_copy_stream_impl.h>
24 using namespace armnn;
29 IOnnxParser::IOnnxParser() : pOnnxParserImpl(new OnnxParserImpl()) {}
31 IOnnxParser::~IOnnxParser() =
default;
53 armnn::INetworkPtr IOnnxParser::CreateNetworkFromBinary(
const std::vector<uint8_t>& binaryContent)
55 return pOnnxParserImpl->CreateNetworkFromBinary(binaryContent);
58 armnn::INetworkPtr IOnnxParser::CreateNetworkFromBinary(
const std::vector<uint8_t>& binaryContent,
59 const std::map<std::string, armnn::TensorShape>& inputShapes)
61 return pOnnxParserImpl->CreateNetworkFromBinary(binaryContent, inputShapes);
66 return pOnnxParserImpl->CreateNetworkFromTextFile(graphFile);
71 return pOnnxParserImpl->CreateNetworkFromString(protoText);
75 const char* graphFile,
76 const std::map<std::string, armnn::TensorShape>& inputShapes)
78 return pOnnxParserImpl->CreateNetworkFromBinaryFile(graphFile, inputShapes);
82 const std::map<std::string, armnn::TensorShape>& inputShapes)
84 return pOnnxParserImpl->CreateNetworkFromTextFile(graphFile, inputShapes);
88 const std::map<std::string, armnn::TensorShape>& inputShapes)
90 return pOnnxParserImpl->CreateNetworkFromString(protoText, inputShapes);
93 BindingPointInfo IOnnxParser::GetNetworkInputBindingInfo(
const std::string& name)
const
95 return pOnnxParserImpl->GetNetworkInputBindingInfo(name);
98 BindingPointInfo IOnnxParser::GetNetworkOutputBindingInfo(
const std::string& name)
const
100 return pOnnxParserImpl->GetNetworkOutputBindingInfo(name);
105 void CheckValidDataType(std::initializer_list<onnx::TensorProto::DataType> validInputTypes,
107 const char* validExpr,
108 std::string nodeName,
109 std::string tensorName,
112 bool isValid = std::any_of(validInputTypes.begin(),
113 validInputTypes.end(),
118 fmt::format(
"Datatype {} is not valid for tensor '{}' of node '{}', not in {{{}}}. {}",
119 onnx::TensorProto::DataType_Name(actualValue),
127 #define CHECK_VALID_DATATYPE(NODE, TENSOR, ACTUAL, ...) \
128 CheckValidDataType({__VA_ARGS__}, ACTUAL, #__VA_ARGS__, NODE, TENSOR, CHECK_LOCATION())
130 using StrTypeListPair = std::pair<const char*, std::initializer_list<onnx::TensorProto::DataType>>;
131 #define STR_LIST(...) StrTypeListPair(#__VA_ARGS__, {__VA_ARGS__})
133 template <
typename Callable>
134 void ReadMandatoryNodeAttributeImpl(
const onnx::NodeProto& node,
135 const std::string& attribName,
136 onnx::AttributeProto::AttributeType expectedType,
139 auto attribs = node.attribute();
141 while (attriNum < node.attribute_size())
143 if (attribs.Get(attriNum).name() == attribName)
145 if (attribs.Get(attriNum).type() == expectedType)
147 callable(attribs.Get(attriNum));
151 throw ParseException(fmt::format(
"Attribute {} of node {} expected to have {} as "
152 "onnx::AttributeProto::AttributeType, but found {} instead {}",
155 onnx::AttributeProto::AttributeType_Name(expectedType),
156 onnx::AttributeProto::AttributeType_Name(attribs.Get(attriNum).type()),
163 if (attriNum == node.attribute_size())
165 throw ParseException(fmt::format(
"Could not find required attribute {} in node {} {}",
170 template <
typename Callable>
171 void ReadOptionalNodeAttributeImpl(
const onnx::NodeProto& node,
172 const std::string& attribName,
173 onnx::AttributeProto::AttributeType expectedType,
176 auto attribs = node.attribute();
177 for (
int attriNum = 0; attriNum < node.attribute_size(); ++attriNum)
179 if (attribs.Get(attriNum).name() == attribName)
181 if (attribs.Get(attriNum).type() == expectedType)
183 callable(attribs.Get(attriNum));
188 fmt::format(
"Attribute {} of node {} expected to have {} as onnx::AttributeProto::AttributeType, "
189 "but found {} instead {}",
192 onnx::AttributeProto::AttributeType_Name(expectedType),
193 onnx::AttributeProto::AttributeType_Name(attribs.Get(attriNum).type()),
200 int ReadMandatoryNodeIntAttribute(
const onnx::NodeProto& node,
201 const std::string& name)
204 ReadMandatoryNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
205 [&attribValue](
const onnx::AttributeProto& attrValue)
212 int64_t ReadOptionalNodeInt64Attribute(
const onnx::NodeProto& node,
213 const std::string& name,
214 const int64_t defaultValue = 0)
216 int64_t attribValue = defaultValue;
217 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
218 [&attribValue](
const onnx::AttributeProto& attrValue)
220 attribValue = attrValue.i();
225 std::vector<uint32_t> ReadMandatoryNodeUint32ListAttribute(
const onnx::NodeProto& node,
226 const std::string& name)
228 std::vector<uint32_t> attriList;
229 ReadMandatoryNodeAttributeImpl(node, name, onnx::AttributeProto::INTS,
230 [&attriList](
const onnx::AttributeProto& attrValue)
232 for (
int attriNum = 0; attriNum < attrValue.ints_size(); ++attriNum)
240 uint32_t ReadOptionalNodeUint32Attribute(
const onnx::NodeProto& node,
241 const std::string& name,
242 const uint32_t defaultVal = 0u)
244 uint32_t attribValue = defaultVal;
245 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
246 [&attribValue](
const onnx::AttributeProto& attrValue)
253 std::vector<uint32_t> ReadOptionalNodeUint32ListAttribute(
const onnx::NodeProto& node,
254 const std::string& name)
256 std::vector<uint32_t> attriList;
257 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INTS,
258 [&attriList](
const onnx::AttributeProto& attrValue)
260 for (
int attriNum = 0; attriNum < attrValue.ints_size(); ++attriNum)
269 float ReadOptionalNodeFloatAttribute(
const onnx::NodeProto& node,
270 const std::string& name,
271 const float defaultValue = 0.0f)
273 float attribValue = defaultValue;
274 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::FLOAT,
275 [&attribValue](
const onnx::AttributeProto& attrValue)
277 attribValue = attrValue.f();
282 std::string ReadOptionalNodeStringAttribute(
const onnx::NodeProto& node,
const std::string& name)
284 std::string attribValue =
"";
285 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::STRING,
286 [&attribValue](
const onnx::AttributeProto& attrValue)
288 attribValue = attrValue.s();
298 case onnx::TensorProto::FLOAT:
300 type = DataType::Float32;
303 case onnx::TensorProto::INT32:
304 case onnx::TensorProto::INT64:
306 type = DataType::Signed32;
312 fmt::format(
"'{}' is not a currently supported datatype for tensor {}."
313 " Supported dataTypes are FLOAT, INT32 and INT64. {}",
327 if(std::find(shape.begin(), shape.end(), 0) != shape.end())
337 const onnx::TensorShapeProto onnxShape =
info.type().tensor_type().shape();
338 std::vector<unsigned int> shapeDims;
339 for (
int i = 0; i < onnxShape.dim_size(); ++i)
349 std::vector<unsigned int> shapeDims;
351 for (
auto dim: tensor.dims())
356 return ToTensorInfo(tensor.name(), shapeDims, tensor.data_type());
359 std::string TensorInfoAsString(
const TensorInfo& info,
360 const std::string& name,
364 std::stringstream ss;
365 ss <<
"tensor '" << name <<
"' contains "
366 << onnx::TensorProto::DataType_Name(type)
367 <<
" and has shape [";
371 ss << shape[i] <<
", ";
377 void CalcPadding(uint32_t inputSize,
381 uint32_t* paddingFront,
382 uint32_t* paddingBack,
385 uint32_t outputSize = (inputSize + stride - 1) / stride;
386 uint32_t dilatedSize = filterSize + (dilation - 1) * (filterSize - 1);
387 uint32_t temp = (outputSize - 1) * stride + dilatedSize;
388 *paddingFront = (temp - inputSize) / 2;
389 *paddingBack = *paddingFront;
390 if((temp - inputSize) % 2 == 1)
405 const std::string& outName,
406 DataType dataType = DataType::Float32)
408 std::vector<int> targetDims;
414 targetDims.push_back(
static_cast<int>(inShape[
static_cast<uint
>(i)]));
418 targetDims.push_back(val);
422 std::vector<unsigned int> outDims(targetDims.begin(), targetDims.end());
423 const auto stretchDim = std::find(targetDims.begin(), targetDims.end(), -1);
424 if (stretchDim != targetDims.end())
426 if (std::find(std::next(stretchDim), targetDims.end(), -1) != targetDims.end())
428 std::stringstream ss;
430 for(uint i = 0; i < targetDims.size() - 1; ++i)
432 ss << targetDims[i] <<
", ";
434 ss << targetDims[targetDims.size() - 1] <<
" ]";
437 fmt::format(
"Error during creation of reshaped tensor '{}'. At most one component of shape can be "
438 " -1 and here, shape is {} {}",
444 auto targetNumElements = armnn::numeric_cast<unsigned int>(std::accumulate(targetDims.begin(), targetDims.end(),
445 -1, std::multiplies<int32_t>()));
446 auto stretchIndex =
static_cast<size_t>(std::distance(targetDims.begin(), stretchDim));
447 outDims[stretchIndex] = inShape.
GetNumElements() / targetNumElements;
455 const std::map<std::string, OnnxParserImpl::OperationParsingFunction> OnnxParserImpl::m_ParserFunctions = {
456 {
"BatchNormalization", &OnnxParserImpl::ParseBatchNormalization},
457 {
"GlobalAveragePool", &OnnxParserImpl::ParseGlobalAveragePool},
458 {
"AveragePool", &OnnxParserImpl::ParseAveragePool },
459 {
"Clip", &OnnxParserImpl::ParseClip },
460 {
"Constant", &OnnxParserImpl::ParseConstant },
461 {
"MaxPool", &OnnxParserImpl::ParseMaxPool },
462 {
"Reshape", &OnnxParserImpl::ParseReshape },
463 {
"Sigmoid", &OnnxParserImpl::ParseSigmoid },
464 {
"Tanh", &OnnxParserImpl::ParseTanh },
465 {
"Relu", &OnnxParserImpl::ParseRelu },
466 {
"LeakyRelu", &OnnxParserImpl::ParseLeakyRelu },
467 {
"Conv", &OnnxParserImpl::ParseConv },
468 {
"Add", &OnnxParserImpl::ParseAdd },
469 {
"Flatten", &OnnxParserImpl::ParseFlatten },
470 {
"Shape", &OnnxParserImpl::ParseShape },
471 {
"Gather", &OnnxParserImpl::ParseGather },
472 {
"Unsqueeze", &OnnxParserImpl::ParseUnsqueeze },
473 {
"Concat", &OnnxParserImpl::ParseConcat },
474 {
"Gemm", &OnnxParserImpl::ParseGemm }
477 template<
typename TypePair,
typename Location>
478 void OnnxParserImpl::ValidateInputs(
const onnx::NodeProto& node,
479 TypePair validInputs,
480 const Location& location)
482 for(
auto input : node.input())
484 CheckValidDataType(validInputs.second,
485 m_TensorsInfo[input].m_dtype,
493 #define VALID_INPUTS(NODE, VALID_INPUTS) \
494 OnnxParserImpl::ValidateInputs(NODE, \
498 std::vector<TensorInfo> OnnxParserImpl::ComputeOutputInfo(std::vector<std::string> outNames,
500 std::vector<TensorShape> inputShapes,
504 bool needCompute = std::any_of(outNames.begin(),
506 [
this](std::string name)
508 return (m_TensorsInfo.count(name) == 0 ||
509 m_TensorsInfo[name].m_info == nullptr ||
510 m_TensorsInfo[name].m_info->GetShape().GetDimensionality() ==
511 Dimensionality::NotSpecified);
513 std::vector<TensorInfo> outInfo;
515 std::vector<TensorShape> inferredShapes;
516 DataType armnnType = DataType::Float32;
521 case onnx::TensorProto::FLOAT: {
522 armnnType = DataType::Float32;
525 case onnx::TensorProto::INT32:
526 case onnx::TensorProto::INT64: {
527 armnnType = DataType::Signed32;
532 fmt::format(
"'{}' is not a currently supported datatype for {}."
533 " Supported dataTypes are FLOAT, INT32 and INT64. {}",
540 for (uint i = 0; i < outNames.size(); ++i)
544 m_TensorsInfo[outNames[i]] = OnnxTensor();
545 m_TensorsInfo[outNames[i]].m_info = std::make_unique<TensorInfo>(
547 m_TensorsInfo[outNames[i]].m_dtype = dataType;
549 outInfo.push_back(*m_TensorsInfo[outNames[i]].m_info);
554 OnnxParserImpl::OnnxParserImpl()
555 : m_Network(nullptr, nullptr)
559 void OnnxParserImpl::ResetParser()
563 m_InputInfos.clear();
564 m_OutputInfos.clear();
567 void OnnxParserImpl::Cleanup()
569 m_TensorConnections.clear();
570 m_TensorsInfo.clear();
571 m_OutputsMap.clear();
572 m_OutputsFusedAndUsed.clear();
573 m_InputShapes.clear();
577 std::pair<armnn::ConstTensor, std::unique_ptr<T[]>>
582 ARMNN_ASSERT_MSG(bufferPtr !=
nullptr, fmt::format(
"Buffer for permutation is null").c_str());
590 reinterpret_cast<const T*
>(bufferPtr), data.get(),
sizeof(T));
594 ::memcpy(data.get(), bufferPtr, tensorInfo.
GetNumBytes());
597 return std::make_pair(
ConstTensor(tensorInfo, data.get()), std::move(data));
600 std::pair<ConstTensor, std::unique_ptr<float[]>>
601 OnnxParserImpl::CreateConstTensor(
const std::string name,
604 TensorInfo tensorInfo = *m_TensorsInfo[name].m_info;
605 onnx::TensorProto onnxTensor = *m_TensorsInfo[name].m_tensor;
617 throw ParseException(fmt::format(
"No tensor data found for Const tensor '{}' {}",
622 auto srcData = onnxTensor.float_data().data();
624 if (!onnxTensor.has_raw_data())
626 if(tensorInfo.
GetNumElements() !=
static_cast<uint
>(onnxTensor.float_data_size()))
629 fmt::format(
"The number of data provided ({}) does not match the tensor '{}' number of "
631 onnxTensor.float_data_size(),
636 return CreateConstTensorImpl<float>(srcData, tensorInfo, permutationVector);
640 return CreateConstTensorImpl<float>(
reinterpret_cast<const float*
>(onnxTensor.raw_data().c_str()),
646 std::pair<ConstTensor, std::unique_ptr<int32_t[]>>
647 OnnxParserImpl::CreateInt64ConstTensor(
const std::string name,
650 TensorInfo tensorInfo = *m_TensorsInfo[name].m_info;
651 onnx::TensorProto onnxTensor = *m_TensorsInfo[name].m_tensor;
661 if (numElements == 0)
663 throw ParseException(fmt::format(
"No tensor data found for Const tensor '{}' {}",
669 if (!onnxTensor.has_raw_data())
671 auto srcData = onnxTensor.int64_data().data();
672 if(numElements !=
static_cast<uint
>(onnxTensor.int64_data_size()))
675 fmt::format(
"The number of data provided ({}) does not match the tensor '{}' number of "
677 onnxTensor.int64_data_size(),
683 std::vector<int32_t> int32Data;
684 for(uint i = 0; i < numElements; i++)
687 int32Data.push_back(int32Value);
690 return CreateConstTensorImpl<int32_t>(int32Data.data(), tensorInfo, permutationVector);
694 auto srcData =
reinterpret_cast<const int64_t*
>(onnxTensor.raw_data().c_str());
695 std::vector<int32_t> int32Data;
696 for(uint i = 0; i < numElements; i++)
699 int32Data.push_back(int32Value);
701 return CreateConstTensorImpl<int32_t>(int32Data.data(), tensorInfo, permutationVector);
707 FILE* fd = fopen(graphFile,
"r");
715 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
716 using google::protobuf::io::FileInputStream;
717 std::unique_ptr<FileInputStream> input = std::make_unique<FileInputStream>(fileno(fd));
718 bool success = google::protobuf::TextFormat::Parse(input.get(), modelProto.get());
723 std::stringstream
error;
724 error <<
"Failed to parse graph file";
734 return CreateNetworkFromModel(*modelProto);
738 const std::map<std::string, armnn::TensorShape>& inputShapes)
741 m_InputShapes = inputShapes;
743 return CreateNetworkFromModel(*modelProto);
750 return CreateNetworkFromModel(*modelProto);
754 const std::map<std::string, armnn::TensorShape>& inputShapes)
757 m_InputShapes = inputShapes;
759 return CreateNetworkFromModel(*modelProto);
764 if (binaryContent.size() == 0)
769 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
771 google::protobuf::io::CodedInputStream codedStream(binaryContent.data(),
static_cast<int>(binaryContent.size()));
772 codedStream.SetTotalBytesLimit(INT_MAX);
773 bool success = modelProto.get()->ParseFromCodedStream(&codedStream);
777 std::stringstream
error;
778 error <<
"Failed to parse graph";
786 FILE* fd = fopen(graphFile,
"rb");
794 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
796 google::protobuf::io::FileInputStream inStream(fileno(fd));
797 google::protobuf::io::CodedInputStream codedStream(&inStream);
798 codedStream.SetTotalBytesLimit(INT_MAX);
799 bool success = modelProto.get()->ParseFromCodedStream(&codedStream);
804 std::stringstream
error;
805 error <<
"Failed to parse graph file";
816 return CreateNetworkFromModel(*modelProto);
820 const std::map<std::string, armnn::TensorShape>& inputShapes)
823 m_InputShapes = inputShapes;
825 return CreateNetworkFromModel(*modelProto);
836 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
837 bool success = google::protobuf::TextFormat::ParseFromString(protoText, modelProto.get());
840 std::stringstream
error;
841 error <<
"Failed to parse graph file";
851 return CreateNetworkFromModel(*modelProto);
855 const std::map<std::string, armnn::TensorShape>& inputShapes)
858 m_InputShapes = inputShapes;
860 return CreateNetworkFromModel(*modelProto);
863 INetworkPtr OnnxParserImpl::CreateNetworkFromModel(onnx::ModelProto& model)
865 m_Network = INetwork::Create();
868 m_Graph = std::make_unique<onnx::GraphProto>(*model.mutable_graph());
877 return std::move(m_Network);
880 void OnnxParserImpl::LoadGraph()
885 SetupInfo(m_Graph->mutable_output());
886 SetupInfo(m_Graph->mutable_input());
887 SetupInfo(m_Graph->mutable_value_info());
889 for (
auto tensor : m_Graph->initializer())
891 m_TensorsInfo[tensor.name()].m_tensor = std::make_unique<const onnx::TensorProto>(tensor);
892 m_TensorsInfo[tensor.name()].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(tensor));
893 m_TensorsInfo[tensor.name()].m_dtype =
901 DetectFullyConnected();
904 for(
size_t nodeIndex = 0; nodeIndex < static_cast<size_t>(m_Graph->node_size()); nodeIndex++)
906 auto node = m_Graph->node(
static_cast<int>(nodeIndex));
907 const std::string& operation = node.op_type();
910 if (operation ==
"MatMul" )
912 if(m_OutputsFusedAndUsed[nodeIndex].inputForNodes != m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.size())
915 AddFullyConnected(node);
918 else if (!(m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.empty()) && operation ==
"Add")
920 int matmulIndex =
static_cast<int> (m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes[0]);
921 AddFullyConnected(m_Graph->node(matmulIndex), &node);
923 else if (m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.empty())
925 auto it = m_ParserFunctions.find(operation);
926 if (it != m_ParserFunctions.end())
928 auto func = it->second;
933 throw ParseException(fmt::format(
"Unsupported operation {} for node '{}' {}",
942 for (
const auto& tensorCon : m_TensorConnections)
944 if (tensorCon.second.outputSlot !=
nullptr)
946 for (
size_t inputSlotIdx = 0; inputSlotIdx < tensorCon.second.inputSlots.size(); ++inputSlotIdx)
948 tensorCon.second.outputSlot->Connect(*(tensorCon.second.inputSlots[inputSlotIdx]));
954 for(
int outputIndex = 0; outputIndex < m_Graph->output_size(); ++outputIndex)
956 auto output = m_Graph->output(outputIndex);
957 m_OutputInfos[output.name()] = *m_TensorsInfo[output.name()].m_info;
961 void OnnxParserImpl::SetupInfo(
const google::protobuf::RepeatedPtrField<onnx::ValueInfoProto >* list)
963 for (
auto tensor : *list)
965 m_TensorsInfo[tensor.name()] = OnnxTensor();
966 m_TensorsInfo[tensor.name()].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(tensor));
967 m_TensorsInfo[tensor.name()].m_dtype =
972 void OnnxParserImpl::DetectFullyConnected()
974 m_OutputsFusedAndUsed = std::vector<UsageSummary> (
static_cast<size_t>(m_Graph->node_size()), UsageSummary());
975 auto matmulAndConstant = [&](
const std::string& constInput,
976 const std::string& matmulInput,
979 auto matmulIt = m_OutputsMap.find(matmulInput);
980 if(matmulIt != m_OutputsMap.end() && matmulIt->second.first->op_type() ==
"MatMul"
981 && m_TensorsInfo[constInput].isConstant())
983 nodeIndex = matmulIt->second.second;
989 for(
int nodeIndex = 0; nodeIndex < m_Graph->node_size(); nodeIndex++)
991 const onnx::NodeProto* node = &m_Graph->node(nodeIndex);
992 for (
const std::string& output : node->output())
994 m_OutputsMap[output] = std::make_pair(node, nodeIndex);
997 for (
const std::string& input : node->input())
999 auto matmulIt = m_OutputsMap.find(input);
1000 if(matmulIt != m_OutputsMap.end()){
1001 ++m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIt->second.second)].inputForNodes;
1005 if (node->op_type() ==
"Add")
1007 int matmulIndex = 0;
1008 if (matmulAndConstant(node->input(0), node->input(1), matmulIndex) ||
1009 matmulAndConstant(node->input(1), node->input(0), matmulIndex))
1012 m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIndex)].fusedWithNodes
1013 .push_back(
static_cast<size_t>(nodeIndex));
1015 m_OutputsFusedAndUsed[
static_cast<size_t>(nodeIndex)].fusedWithNodes
1016 .push_back(
static_cast<size_t>(matmulIndex));
1021 for (
auto output: m_Graph->output()) {
1022 auto matmulIt = m_OutputsMap.find(output.name());
1023 if(matmulIt != m_OutputsMap.end()){
1024 ++m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIt->second.second)].inputForNodes;
1029 template<
typename Location>
1030 void OnnxParserImpl::GetInputAndParam(
const onnx::NodeProto& node,
1031 std::string* inputName,
1032 std::string* constName,
1033 const Location& location)
1036 if (m_TensorsInfo[node.input(0)].isConstant())
1040 else if (m_TensorsInfo[node.input(1)].isConstant())
1046 throw ParseException(fmt::format(
"One of the input tensors ('{}' or '{}') should be constant in node '{}' {}",
1050 location.AsString()));
1054 *constName = node.input(cstIndex);
1058 *inputName = node.input(!cstIndex);
1062 template<
typename Location>
1063 void OnnxParserImpl::To1DTensor(
const std::string& name,
const Location& location)
1065 TensorShape shape = m_TensorsInfo[name].m_info->GetShape();
1066 std::vector<uint32_t> newShape;
1072 fmt::format(
"Only tensors with shape [1, ..., 1, X] can be converted to 1D and {} {}",
1073 TensorInfoAsString(*m_TensorsInfo[name].m_info, name, m_TensorsInfo[name].m_dtype),
1074 location.AsString()));
1079 m_TensorsInfo[name].m_info->SetShape(
TensorShape(
static_cast<unsigned int>(newShape.size()), newShape.data()));
1082 void OnnxParserImpl::AddConvLayerWithDepthwiseConv(
const onnx::NodeProto& node,
const Convolution2dDescriptor& convDesc)
1096 std::string permuteStr =
"permute_" + node.input(1);
1097 std::vector<std::string> tensorIndexes= {node.input(0), permuteStr};
1099 auto weightTensor = CreateConstTensor(node.input(1));
1100 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(weightTensor.first);
1115 if (node.input_size() == 3)
1117 if(!m_TensorsInfo[node.input(2)].isConstant())
1119 throw ParseException(fmt::format(
"Bias '{}' should be constant in Conv layer '{}' {}",
1126 auto biasTensor = CreateConstTensor(node.input(2));
1127 tensorIndexes.emplace_back(node.input(2));
1136 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1137 { m_TensorsInfo[node.input(0)].m_info->GetShape(),
1144 RegisterInputSlots(layer, tensorIndexes);
1147 RegisterOutputSlots(layer, {node.output(0)});
1150 void OnnxParserImpl::AddFullyConnected(
const onnx::NodeProto& matmulNode,
const onnx::NodeProto* addNode)
1153 std::string inputName;
1154 std::string weightName;
1155 std::string biasName;
1156 std::string outputName;
1161 GetInputAndParam(matmulNode, &inputName, &weightName,
CHECK_LOCATION());
1163 TensorInfo inputInfo = *m_TensorsInfo[inputName].m_info;
1164 TensorInfo weightInfo = *m_TensorsInfo[weightName].m_info;
1167 std::vector<std::string> inputNames;
1180 GetInputAndParam(*addNode,
nullptr, &biasName,
CHECK_LOCATION());
1184 biasInfo = *m_TensorsInfo[biasName].m_info;
1189 fmt::format(
"Shape of weights '{}' and bias of following Add node '{}' do not match : {}"
1190 " and {} ( /!\\ bias should be a 1D tensor) {}",
1193 TensorInfoAsString(*m_TensorsInfo[weightName].m_info, weightName,
1194 m_TensorsInfo[weightName].m_dtype),
1195 TensorInfoAsString(*m_TensorsInfo[biasName].m_info, biasName,
1196 m_TensorsInfo[biasName].m_dtype ),
1200 inputNames = { inputName, weightName, biasName };
1201 outputName = addNode->output(0);
1205 inputNames = { inputName, weightName };
1206 outputName = matmulNode.output(0);
1210 layer = m_Network->AddFullyConnectedLayer(desc, matmulNode.name().c_str());
1219 std::vector<unsigned int> reshapedDimensions(2);
1220 reshapedDimensions[1] = weightInfo.
GetShape()[0];
1221 reshapedDimensions[0] = inputInfo.
GetNumElements() / reshapedDimensions[1];
1226 fmt::format(
"Failed to deduce input tensor shape from filter size {} {}",
1227 reshapedDimensions[1],
1233 inputInfo = reshapedTensorInfo;
1238 std::string reshapeLayerName = fmt::format(
"Reshape_for:{}", layer->
GetName());
1239 IConnectableLayer* reshapeLayer = m_Network->AddReshapeLayer(reshapeDescriptor, reshapeLayerName.c_str());
1244 RegisterInputSlots(reshapeLayer, {inputName});
1245 inputNames[0] = reshapeLayerName;
1248 auto outputInfo = ComputeOutputInfo({ outputName },
1254 RegisterInputSlots(layer, inputNames);
1257 if(m_TensorsInfo[weightName].isConstant())
1259 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(CreateConstTensor(weightName).first);
1266 if(desc.
m_BiasEnabled && m_TensorsInfo[biasName].isConstant())
1268 IConnectableLayer* biasLayer = m_Network->AddConstantLayer(CreateConstTensor(biasName).first);
1275 if (outputInfo[0].GetNumDimensions() > 2)
1278 std::vector<unsigned int> reshapedDimensions(2);
1279 reshapedDimensions[1] = weightInfo.
GetShape()[1];
1280 reshapedDimensions[0] = outputInfo[0].
GetNumElements() / reshapedDimensions[1];
1282 if (outputInfo[0].GetNumElements() % reshapedDimensions[1] != 0)
1285 fmt::format(
"Failed to deduce output tensor shape from filter size {} {}",
1286 reshapedDimensions[1],
1297 std::string reshapeLayerName = fmt::format(
"ExpandDims_for:{}", layer->
GetName());
1298 IConnectableLayer* reshapeLayer = m_Network->AddReshapeLayer(desc, reshapeLayerName.c_str());
1303 RegisterInputSlots(reshapeLayer, {layer->
GetName()});
1304 layer = reshapeLayer;
1307 RegisterOutputSlots(layer, { outputName });
1310 void OnnxParserImpl::AddPoolingLayer(
const onnx::NodeProto& node,
Pooling2dDescriptor& desc)
1318 std::vector<uint32_t> kernel_shape = ReadMandatoryNodeUint32ListAttribute(node,
"kernel_shape");
1319 std::vector<uint32_t> strides = ReadOptionalNodeUint32ListAttribute(node,
"strides");
1320 std::vector<uint32_t> pads = ReadOptionalNodeUint32ListAttribute(node,
"pads");
1341 std::string paddingString = ReadOptionalNodeStringAttribute(node,
"auto_pad");
1342 if(paddingString !=
"VALID" && paddingString !=
"" && paddingString !=
"NOTSET")
1345 if( paddingString ==
"SAME_LOWER")
1349 else if (paddingString ==
"SAME_UPPER")
1355 throw ParseException(fmt::format(
"Invalid auto_pad attribute for node {}. "
1356 "Only SAME_UPPER, SAME_LOWER or VALID supported and found {} {}",
1361 auto inputInfo = *m_TensorsInfo[node.input(0)].m_info;
1362 uint32_t inputHeight = inputInfo.
GetShape()[2];
1363 uint32_t inputWidth = inputInfo.
GetShape()[3];
1364 CalcPadding(inputHeight,
1371 CalcPadding(inputWidth,
1388 IConnectableLayer* layer = m_Network->AddPooling2dLayer(desc, node.name().c_str());
1391 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1396 RegisterInputSlots(layer, {node.input(0)});
1399 RegisterOutputSlots(layer, {node.output(0)});
1402 std::pair<std::string, std::string> OnnxParserImpl::AddPrepareBroadcast(
const std::string& input0,
1403 const std::string& input1)
1405 std::pair<std::string, std::string> inputs = std::make_pair(input0, input1);
1407 TensorShape input0Shape = m_TensorsInfo[input0].m_info->GetShape();
1408 TensorShape input1Shape = m_TensorsInfo[input1].m_info->GetShape();
1412 auto outputName = fmt::format(
"reshape_output_{}", input1);
1413 PrependForBroadcast(outputName, input1, input0);
1414 inputs.second = outputName;
1418 auto outputName = fmt::format(
"reshape_output_{}", input0);
1419 PrependForBroadcast(outputName, input0, input1);
1420 inputs.first = outputName;
1425 void OnnxParserImpl::CreateConstantLayer(
const std::string& tensorName,
const std::string& layerName)
1427 auto armnnTensor = CreateConstTensor(tensorName);
1428 IConnectableLayer* layer = m_Network->AddConstantLayer(armnnTensor.first, layerName.c_str());
1430 RegisterOutputSlots(layer, {tensorName});
1433 void OnnxParserImpl::CreateInt64ConstantLayer(
const std::string& tensorName,
const std::string& layerName)
1435 auto armnnTensor = CreateInt64ConstTensor(tensorName);
1436 IConnectableLayer* layer = m_Network->AddConstantLayer(armnnTensor.first, layerName.c_str());
1438 RegisterOutputSlots(layer, {tensorName});
1441 void OnnxParserImpl::CreateReshapeLayer(
const std::string& inputName,
1442 const std::string& outputName,
1443 const std::string& layerName)
1445 const TensorInfo outputTensorInfo = *m_TensorsInfo[outputName].m_info;
1449 IConnectableLayer* layer = m_Network->AddReshapeLayer(reshapeDesc, layerName.c_str());
1455 RegisterInputSlots(layer, {inputName});
1458 RegisterOutputSlots(layer, {outputName});
1471 if (func == ActivationFunction::BoundedReLu)
1473 if (node.input_size() == 1 && node.attribute_size() > 0)
1475 desc.
m_A = ReadOptionalNodeFloatAttribute(node,
"max", std::numeric_limits<float>::max());
1476 desc.
m_B = ReadOptionalNodeFloatAttribute(node,
"min", std::numeric_limits<float>::lowest());
1480 desc.
m_A = node.input(2).empty() ? std::numeric_limits<float>::max() :
std::stof(node.input(2));
1481 desc.
m_B = node.input(1).empty() ? std::numeric_limits<float>::lowest() :
std::stof(node.input(1));
1485 IConnectableLayer*
const layer = m_Network->AddActivationLayer(desc, node.name().c_str());
1488 auto outputInfo = ComputeOutputInfo({ node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1493 RegisterInputSlots(layer, {node.input(0)});
1496 RegisterOutputSlots(layer, {node.output(0)});
1499 void OnnxParserImpl::ParseClip(
const onnx::NodeProto& node)
1501 ParseActivation(node, ActivationFunction::BoundedReLu);
1504 void OnnxParserImpl::ParseSigmoid(
const onnx::NodeProto& node)
1506 ParseActivation(node, ActivationFunction::Sigmoid);
1509 void OnnxParserImpl::ParseTanh(
const onnx::NodeProto& node)
1511 ParseActivation(node, ActivationFunction::TanH);
1514 void OnnxParserImpl::ParseRelu(
const onnx::NodeProto& node)
1516 ParseActivation(node, ActivationFunction::ReLu);
1519 void OnnxParserImpl::ParseLeakyRelu(
const onnx::NodeProto& node)
1521 ParseActivation(node, ActivationFunction::LeakyReLu);
1524 void OnnxParserImpl::ParseAdd(
const onnx::NodeProto& node)
1534 auto inputs = AddPrepareBroadcast(node.input(0), node.input(1));
1535 auto input0 = *m_TensorsInfo[inputs.first].m_info;
1536 auto input1 = *m_TensorsInfo[inputs.second].m_info;
1537 ARMNN_ASSERT(input0.GetNumDimensions() == input1.GetNumDimensions());
1539 unsigned int numDims = input0.GetNumDimensions();
1540 for (
unsigned int i = 0; i < numDims; i++)
1542 unsigned int dim0 = input0.GetShape()[i];
1543 unsigned int dim1 = input1.GetShape()[i];
1544 if (dim0 != dim1 && dim0 != 1 && dim1 != 1)
1547 fmt::format(
"Broadcast is only supported for scalar or 1D tensors in Add node '{}'. "
1548 "Input dimensions should either match or one should be of size 1 and here, "
1551 TensorInfoAsString(*m_TensorsInfo[inputs.first].m_info, inputs.first,
1552 m_TensorsInfo[inputs.first].m_dtype),
1553 TensorInfoAsString(*m_TensorsInfo[inputs.second].m_info, inputs.second,
1554 m_TensorsInfo[inputs.second].m_dtype),
1560 IConnectableLayer* layer = m_Network->AddElementwiseBinaryLayer(BinaryOperation::Add, node.name().c_str());
1563 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1564 { m_TensorsInfo[inputs.first].m_info->GetShape(),
1565 m_TensorsInfo[inputs.second].m_info->GetShape() });
1569 if(m_TensorsInfo[inputs.first].isConstant()) {
1570 CreateConstantLayer(inputs.first, fmt::format(
"Add:constant_of_{}", node.input(0)));
1572 if(m_TensorsInfo[inputs.second].isConstant()) {
1573 CreateConstantLayer(inputs.second, fmt::format(
"Add:constant_of_{}", node.input(1)));
1575 RegisterInputSlots(layer, {inputs.first, inputs.second});
1578 RegisterOutputSlots(layer, {node.output(0)});
1581 void OnnxParserImpl::ParseAveragePool(
const onnx::NodeProto& node)
1586 uint32_t count_include_pad = 0;
1587 count_include_pad = ReadOptionalNodeUint32Attribute(node,
"count_include_pad");
1588 if(count_include_pad) {
1591 AddPoolingLayer(node, desc);
1594 void OnnxParserImpl::ParseBatchNormalization(
const onnx::NodeProto& node)
1602 for(
int ind = 1; ind < node.input_size(); ++ind)
1604 auto tensor = node.input(ind);
1605 if(! m_TensorsInfo[tensor].isConstant())
1608 fmt::format(
"Input tensor '{}' should be constant in BatchNormalization node '{}' {}",
1615 float epsilon = ReadOptionalNodeFloatAttribute(node,
"epsilon", 1e-5f);
1617 desc.
m_Eps = epsilon;
1619 auto scaleTensor = CreateConstTensor(node.input(1));
1620 auto biasTensor = CreateConstTensor(node.input(2));
1621 auto meanTensor = CreateConstTensor(node.input(3));
1622 auto varTensor = CreateConstTensor(node.input(4));
1629 node.name().c_str());
1632 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1635 RegisterInputSlots(layer, {node.input(0)});
1638 RegisterOutputSlots(layer, {node.output(0)});
1641 void OnnxParserImpl::ParseConcat(
const onnx::NodeProto& node)
1645 uint32_t numConcatView =
static_cast<uint32_t
>(node.input_size());
1646 uint32_t inputRank = m_TensorsInfo[node.input(0)].m_info->GetNumDimensions();
1648 int axisInt = ReadMandatoryNodeIntAttribute(node,
"axis");
1650 unsigned int concatDimInput =
static_cast<unsigned int>(
1651 (
static_cast<int>(inputRank) + axisInt) %
static_cast<int>(inputRank));
1654 concatDescriptor.SetConcatAxis(concatDimInput);
1656 unsigned int mergeDimOrigin = 0;
1658 std::vector<TensorShape> inputShapes;
1659 std::vector<std::string> tensorIds;
1661 for (
unsigned int viewIndex = 0; viewIndex < numConcatView; ++viewIndex)
1663 std::string nodeName = node.input(
static_cast<int>(viewIndex));
1664 auto inputTensorInfo = *m_TensorsInfo[nodeName].m_info;
1665 inputShapes.push_back(inputTensorInfo.GetShape());
1666 tensorIds.push_back(nodeName);
1670 inputTensorInfo, concatDescriptor, concatDimInput, viewIndex, mergeDimOrigin);
1673 IConnectableLayer* layer = m_Network->AddConcatLayer(concatDescriptor, node.name().c_str());
1676 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, inputShapes,
1677 m_TensorsInfo[node.input(0)].m_dtype);
1682 RegisterInputSlots(layer, tensorIds);
1685 RegisterOutputSlots(layer, { node.output(0) });
1688 void OnnxParserImpl::ParseConstant(
const onnx::NodeProto& node)
1691 if (!node.attribute(0).has_t())
1693 throw ParseException(fmt::format(
"Value not found for Constant node '{}' {}",
1697 const onnx::TensorProto& onnxTensor = node.attribute(0).t();
1700 m_TensorsInfo[node.output(0)].m_tensor = std::make_unique<const onnx::TensorProto>(onnxTensor);
1701 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(onnxTensor));
1704 if (m_TensorsInfo[node.output(0)].m_dtype == onnx::TensorProto_DataType_FLOAT)
1706 CreateConstantLayer(node.output(0), node.name());
1708 else if (m_TensorsInfo[node.output(0)].m_dtype == onnx::TensorProto_DataType_INT64)
1710 CreateInt64ConstantLayer(node.output(0), node.name());
1714 throw ParseException(fmt::format(
"Data type not support for Constant node '{}' {}",
1720 void OnnxParserImpl::ParseConv(
const onnx::NodeProto& node)
1727 if(m_TensorsInfo[node.input(0)].m_info->GetNumDimensions() != 4)
1730 fmt::format(
"ArmNN only supports 2D convolution and Conv layer '{}' input {} {}",
1732 TensorInfoAsString(*m_TensorsInfo[node.input(0)].m_info, node.input(0),
1733 m_TensorsInfo[node.input(0)].m_dtype),
1737 if(!m_TensorsInfo[node.input(1)].isConstant())
1740 fmt::format(
"Weights '{}' should be constant in Conv layer '{}' {}",
1746 auto inputInfo = *m_TensorsInfo[node.input(0)].m_info;
1751 std::vector<uint32_t> strides = ReadOptionalNodeUint32ListAttribute(node,
"strides");
1763 std::vector<uint32_t> dilations = ReadOptionalNodeUint32ListAttribute(node,
"dilations");
1764 if(!dilations.empty())
1770 std::vector<uint32_t> pads = ReadOptionalNodeUint32ListAttribute(node,
"pads");
1775 std::string paddingString = ReadOptionalNodeStringAttribute(node,
"auto_pad");
1776 if(paddingString !=
"VALID" && paddingString !=
"" && paddingString !=
"NOTSET")
1779 if( paddingString ==
"SAME_LOWER")
1783 else if (paddingString ==
"SAME_UPPER")
1790 fmt::format(
"Invalid auto_pad attribute for node {}. Only SAME_UPPER, SAME_LOWER or VALID "
1791 "supported and found {} {}",
1796 uint32_t inputHeight = inputInfo.
GetShape()[2];
1797 uint32_t inputWidth = inputInfo.
GetShape()[3];
1799 uint32_t weightHeight;
1800 uint32_t weightWidth;
1801 std::vector<uint32_t> kernel_shape = ReadOptionalNodeUint32ListAttribute(node,
"kernel_shape");
1802 if (kernel_shape.empty())
1804 const TensorInfo weightTensorInfo = *m_TensorsInfo[node.input(1)].m_info;
1805 weightHeight = weightTensorInfo.
GetShape()[2];
1806 weightWidth = weightTensorInfo.
GetShape()[3];
1810 weightHeight = kernel_shape[0];
1811 weightWidth = kernel_shape[1];
1813 CalcPadding(inputHeight,
1820 CalcPadding(inputWidth,
1837 uint32_t group = ReadOptionalNodeUint32Attribute(node,
"group", 1);
1840 if (group > inputInfo.
GetShape()[1])
1843 fmt::format(
"Error parsing Convolution node: {}. "
1844 "The 'group'={} parameter cannot be larger than the "
1845 "channel of the input shape={} (in NCHW format). {}",
1851 else if (group == inputInfo.
GetShape()[1])
1855 AddConvLayerWithDepthwiseConv(node, desc);
1860 throw ParseException(fmt::format(
"Error parsing Convolution node: {}. "
1861 "The 'group'={} parameter should be 1 or be equal to the "
1862 "channel of the input shape={} (in NCHW format). {}",
1872 std::vector<std::string> tensorIndexes= {node.input(0), node.input(1)};
1874 auto weightTensor = CreateConstTensor(node.input(1));
1876 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(weightTensor.first);
1880 if (node.input_size() == 3)
1882 if(!m_TensorsInfo[node.input(2)].isConstant())
1884 throw ParseException(fmt::format(
"Bias '{}' should be constant in Conv layer '{}' {}",
1890 auto biasTensor = CreateConstTensor(node.input(2));
1896 tensorIndexes.emplace_back(node.input(2));
1901 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1902 { m_TensorsInfo[node.input(0)].m_info->GetShape(),
1903 m_TensorsInfo[node.input(1)].m_info->GetShape() });
1908 RegisterInputSlots(layer, tensorIndexes);
1911 RegisterOutputSlots(layer, {node.output(0)});
1914 void OnnxParserImpl::ParseFlatten(
const onnx::NodeProto& node)
1920 m_TensorsInfo[node.input(0)].m_dtype,
1921 onnx::TensorProto::FLOAT);
1923 int64_t axis = ReadOptionalNodeInt64Attribute(node,
"axis", 1);
1924 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
1935 throw ParseException(fmt::format(
"Axis '{}' invalid. Tensor has '{}' dimensions in FlattenLayer '{}'",
1945 for (i = 0; i < axis; i++){
1946 dimension1 *= inputShape[i];
1951 dimension2 *= inputShape[i];
1956 auto outInfo = ComputeReshapeInfo(outputShape, inputShape, node.output(0));
1957 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
1958 CreateReshapeLayer(node.input(0), node.output(0), node.name());
1961 void OnnxParserImpl::ParseGather(
const onnx::NodeProto& node)
1967 gatherDescriptor.
m_Axis =
static_cast<int>(ReadOptionalNodeInt64Attribute(node,
"axis", 0));
1969 IConnectableLayer* layer = m_Network->AddGatherLayer(gatherDescriptor, node.name().c_str());
1972 const TensorShape& inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
1973 const TensorShape& indicesShape = m_TensorsInfo[node.input(1)].m_info->GetShape();
1974 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, { inputShape, indicesShape },
1975 m_TensorsInfo[node.input(0)].m_dtype);
1979 RegisterInputSlots(layer, { node.input(0), node.input(1) });
1982 RegisterOutputSlots(layer, { node.output(0) });
1985 void OnnxParserImpl::ParseGemm(
const onnx::NodeProto& node)
1990 int transA =
static_cast<int>(ReadOptionalNodeUint32Attribute(node,
"transA", 0));
1991 int transB =
static_cast<int>(ReadOptionalNodeUint32Attribute(node,
"transB", 0));
1992 float alpha = ReadOptionalNodeFloatAttribute(node,
"alpha", 1.0);
1993 float beta = ReadOptionalNodeFloatAttribute(node,
"beta", 1.0);
1994 bool biasEnabled = node.input_size() == 3;
1996 TensorShape input0Shape = m_TensorsInfo[node.input(0)].m_info->GetShape();
1997 TensorShape input1Shape = m_TensorsInfo[node.input(1)].m_info->GetShape();
2007 layer = m_Network->AddFullyConnectedLayer(fullyConnectedDescriptor, node.name().c_str());
2013 std::string transAName =
"transpose_" + node.input(0);
2016 IConnectableLayer* transALayer = m_Network->AddTransposeLayer(transposeADescriptor, transAName.c_str());
2018 auto transAInfo = ComputeOutputInfo({ transAName }, transALayer, { input0Shape });
2022 RegisterInputSlot(transALayer, node.input(0), 0);
2023 input0Shape = transAInfo[0].GetShape();
2027 RegisterInputSlot(layer, node.input(0), 0);
2031 if(m_TensorsInfo[node.input(1)].isConstant())
2033 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(CreateConstTensor(node.input(1)).first);
2034 TensorInfo weightInfo = *m_TensorsInfo[node.input(1)].m_info;
2041 std::string activationName =
"activation_" + node.input(1);
2043 activationDescriptor.
m_A = alpha;
2044 activationDescriptor.
m_Function = ActivationFunction::Linear;
2045 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2048 auto actInfo = ComputeOutputInfo({ activationName }, actLayer, { weightInfo.
GetShape() });
2052 input1Shape = actInfo[0].GetShape();
2057 input1Shape = weightInfo.
GetShape();
2065 std::string activationName =
"activation_" + node.input(1);
2067 activationDescriptor.
m_A = alpha;
2068 activationDescriptor.
m_Function = ActivationFunction::Linear;
2069 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2072 auto actInfo = ComputeOutputInfo({ activationName }, actLayer, { input1Shape });
2075 RegisterInputSlot(actLayer, node.input(1), 0);
2076 input1Shape = actInfo[0].GetShape();
2080 RegisterInputSlot(layer, node.input(1), 1);
2084 if(biasEnabled && m_TensorsInfo[node.input(2)].isConstant())
2087 IConnectableLayer* biasLayer = m_Network->AddConstantLayer(CreateConstTensor(node.input(2)).first);
2088 TensorInfo biasInfo = *m_TensorsInfo[node.input(2)].m_info;
2095 std::string activationName =
"activation_" + node.input(2);
2097 activationDescriptor.
m_A = beta;
2098 activationDescriptor.
m_Function = ActivationFunction::Linear;
2099 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2102 auto actInfo = ComputeOutputInfo({ activationName }, actLayer, { biasInfo.
GetShape() });
2112 else if (biasEnabled)
2115 if (m_TensorsInfo[node.input(2)].m_info->GetNumDimensions() != 1)
2117 throw ParseException(fmt::format(
"The parser supports constant or non-constant with 1 dimension for "
2118 "Input C of Gemm. Input '{}' in '{}' is not supported '{}'",
2126 std::string activationName =
"activation_" + node.input(2);
2128 activationDescriptor.
m_A = beta;
2129 activationDescriptor.
m_Function = ActivationFunction::Linear;
2130 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2133 auto actInfo = ComputeOutputInfo({ activationName },
2135 { m_TensorsInfo[node.input(2)].m_info->GetShape() });
2138 RegisterInputSlot(actLayer, node.input(2), 0);
2142 RegisterInputSlot(layer, node.input(2), 2);
2147 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
2148 { input0Shape, input1Shape });
2151 RegisterOutputSlots(layer, {node.output(0)});
2154 void OnnxParserImpl::ParseGlobalAveragePool(
const onnx::NodeProto& node)
2160 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2164 IConnectableLayer* layer = m_Network->AddPooling2dLayer(desc, node.name().c_str());
2167 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {inputShape});
2172 RegisterInputSlots(layer, {node.input(0)});
2175 RegisterOutputSlots(layer, {node.output(0)});
2178 void OnnxParserImpl::ParseMaxPool(
const onnx::NodeProto& node)
2183 AddPoolingLayer(node, desc);
2186 void OnnxParserImpl::ParseShape(
const onnx::NodeProto& node)
2194 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2195 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {inputShape}, onnx::TensorProto::INT64);
2199 RegisterInputSlots(layer, {node.input(0)});
2202 RegisterOutputSlots(layer, {node.output(0)});
2205 void OnnxParserImpl::ParseReshape(
const onnx::NodeProto& node)
2211 m_TensorsInfo[node.input(0)].m_dtype,
2212 onnx::TensorProto::FLOAT);
2214 m_TensorsInfo[node.input(1)].m_dtype,
2215 onnx::TensorProto::INT64);
2217 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2219 std::vector<unsigned int> targetShape;
2220 if(m_TensorsInfo[node.input(1)].isConstant())
2222 unsigned int dims =
static_cast<unsigned int>(m_TensorsInfo[node.input(1)].m_tensor->int64_data_size());
2223 targetShape.reserve(dims);
2225 for(uint i = 0; i < dims; i++)
2227 int val =
CHECKED_INT32(m_TensorsInfo[node.input(1)].m_tensor->int64_data(
static_cast<int>(i)));
2228 targetShape[i]=
static_cast<unsigned int>(val);
2234 unsigned int dims = m_TensorsInfo[node.input(1)].m_info->GetNumDimensions();
2235 TensorShape shapes = m_TensorsInfo[node.input(1)].m_info->GetShape();
2236 if (dims != 1 || shapes[0] > 2)
2238 throw ParseException(fmt::format(
"Invalid input shape '{}' in Reshape layer '{}' {}",
2244 unsigned int numInputElements = m_TensorsInfo[node.input(0)].m_info->GetNumElements();
2247 targetShape = { numInputElements };
2249 else if (shapes[0] == 2)
2251 targetShape = { inputShape[0] , numInputElements / inputShape[0] };
2255 if(m_TensorsInfo[node.input(0)].isConstant())
2258 if(m_TensorsInfo.count(node.output(0)) == 0)
2260 m_TensorsInfo[node.output(0)] = OnnxTensor();
2262 m_TensorsInfo[node.output(0)].m_tensor =
2263 std::make_unique<onnx::TensorProto>(*m_TensorsInfo[node.input(0)].m_tensor);
2267 if(m_TensorsInfo.count(node.output(0)) == 0 || m_TensorsInfo[node.output(0)].m_info ==
nullptr)
2269 auto outInfo = ComputeReshapeInfo(
2270 TensorShape(
static_cast<unsigned int>(targetShape.size()), targetShape.data()),
2271 inputShape, node.output(0));
2272 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
2275 CreateReshapeLayer(node.input(0), node.output(0), node.name());
2279 void OnnxParserImpl::ParseUnsqueeze(
const onnx::NodeProto& node)
2284 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2285 std::vector<uint32_t> dims;
2286 if (node.input_size() == 1 && node.attribute_size() > 0)
2288 dims = ReadMandatoryNodeUint32ListAttribute(node,
"axes");
2293 m_TensorsInfo[node.input(1)].m_dtype,
2294 onnx::TensorProto::INT64);
2296 auto int64Axes = m_TensorsInfo[node.input(1)].m_tensor->int64_data().data();
2297 uint numDim = armnn::numeric_cast<uint>(m_TensorsInfo[node.input(1)].m_tensor->int64_data_size());
2299 for(uint i = 0; i < numDim; i++)
2302 dims.push_back(uint32Value);
2307 std::sort(dims.begin(), dims.end());
2309 std::vector<unsigned int> targetShape;
2315 targetShape.push_back(inputShape[i]);
2319 for(uint i = 0; i < dims.size(); i++)
2321 targetShape.insert(targetShape.begin() + armnn::numeric_cast<int>(dims[i]), 1);
2324 auto outInfo = ComputeReshapeInfo(
TensorShape(
static_cast<unsigned int>(targetShape.size()), targetShape.data()),
2325 inputShape, node.output(0), m_TensorsInfo[node.input(0)].m_info->GetDataType());
2326 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
2327 m_TensorsInfo[node.output(0)].m_dtype = m_TensorsInfo[node.input(0)].m_dtype;
2329 CreateReshapeLayer(node.input(0), node.output(0), node.name());
2332 void OnnxParserImpl::PrependForBroadcast(
const std::string& outputName,
2333 const std::string& input0,
2334 const std::string& input1)
2339 TensorShape input0Shape = m_TensorsInfo[input0].m_info->GetShape();
2340 TensorShape input1Shape = m_TensorsInfo[input1].m_info->GetShape();
2343 std::vector<uint32_t> newShape;
2346 newShape.push_back(1);
2351 newShape.push_back(input0Shape[dim]);
2353 outputTensorInfo.
SetShape(
TensorShape(
static_cast<unsigned int>(newShape.size()), newShape.data()));
2356 m_TensorsInfo[outputName] = OnnxTensor();
2357 m_TensorsInfo[outputName].m_info = std::make_unique<TensorInfo>(outputTensorInfo);
2360 if( ! m_TensorsInfo[input0].isConstant())
2362 CreateReshapeLayer(input0, outputName, fmt::format(
"Add:reshapeOf{}", input0));
2366 m_TensorsInfo[outputName].m_tensor = std::make_unique<onnx::TensorProto>(*m_TensorsInfo[input0].m_tensor);
2371 void OnnxParserImpl::SetupInputLayers()
2374 for(
int inputIndex = 0; inputIndex < m_Graph->input_size(); ++inputIndex)
2376 auto input = m_Graph->input(inputIndex);
2377 if (!m_TensorsInfo[input.name()].isConstant())
2381 TensorInfo tensorInfo = *m_TensorsInfo[input.name()].m_info;
2384 if (m_InputShapes.find(input.name()) == m_InputShapes.end())
2386 throw ParseException(fmt::format(
"The parser does not support dynamic tensor, "
2387 "please specify input shape for {}. {}",
2393 tensorInfo.
SetShape(m_InputShapes[input.name()]);
2394 m_TensorsInfo[input.name()].m_info = std::make_unique<TensorInfo>(tensorInfo);
2400 m_InputInfos[input.name()] = tensorInfo;
2402 RegisterOutputSlots(layer,{ input.name() });
2407 void OnnxParserImpl::SetupOutputLayers()
2409 if(m_Graph->output_size() == 0)
2414 for(
int outputIndex = 0; outputIndex < m_Graph->output_size(); ++outputIndex)
2418 m_Graph->output(outputIndex).name().c_str());
2420 RegisterInputSlots(layer, { m_Graph->output(outputIndex).name() });
2425 const std::string& tensorId,
2426 unsigned int slotIndex)
2430 auto it = m_TensorConnections.find(tensorId);
2432 if (it == m_TensorConnections.end())
2435 m_TensorConnections[tensorId] = TensorSlots();
2437 m_TensorConnections[tensorId].inputSlots.push_back(slot);
2440 void OnnxParserImpl::RegisterInputSlots(
IConnectableLayer* layer,
const std::vector<std::string>& tensorIds)
2446 fmt::format(
"The number of tensor inputs ({}) does not match the number expected ({}) {}",
2452 for (
unsigned int slotIndex = 0; slotIndex < layer->
GetNumInputSlots(); ++slotIndex)
2454 std::string tensorId = tensorIds[slotIndex];
2457 auto it = m_TensorConnections.find(tensorId);
2459 if (it == m_TensorConnections.end())
2462 m_TensorConnections[tensorId] = TensorSlots();
2464 m_TensorConnections[tensorId].inputSlots.push_back(slot);
2468 void OnnxParserImpl::RegisterOutputSlots(
IConnectableLayer* layer,
const std::vector<std::string>& tensorIds)
2474 fmt::format(
"The number of tensor outputs ({}) does not match the number expected ({}) {} ",
2480 for (
unsigned int slotIndex = 0; slotIndex < layer->
GetNumOutputSlots(); ++slotIndex)
2482 std::string tensorId = tensorIds[slotIndex];
2485 auto it = m_TensorConnections.find(tensorId);
2487 if (it == m_TensorConnections.end())
2490 m_TensorConnections[tensorId] = TensorSlots();
2493 TensorSlots& tensorSlots = m_TensorConnections[tensorId];
2496 if (tensorSlots.outputSlot !=
nullptr)
2498 throw ParseException(fmt::format(
"Another layer has already registered itself as the producer of "
2503 tensorSlots.outputSlot = slot;
2510 for(
int i = 0; i < m_Graph->input_size(); ++i)
2512 auto input = m_Graph->input(i);
2513 if(input.name() == name)
2515 auto it = m_InputInfos.find(name);
2517 if (it != m_InputInfos.end())
2529 for(
int i = 0; i < m_Graph->output_size(); ++i)
2531 auto output = m_Graph->output(i);
2532 if(output.name() == name)
2534 auto it = m_OutputInfos.find(name);
2536 if (it != m_OutputInfos.end())
2548 if(model ==
nullptr) {
2553 std::vector<std::string> inputNames;
2554 std::map<std::string, bool> isConstant;
2555 for(
auto tensor : model->graph().initializer())
2557 isConstant[tensor.name()] =
true;
2559 for(
auto input : model->graph().input())
2561 auto it = isConstant.find(input.name());
2562 if(it == isConstant.end())
2564 inputNames.push_back(input.name());
2572 if(model ==
nullptr) {
2577 std::vector<std::string> outputNames;
2578 for(
auto output : model->graph().output())
2580 outputNames.push_back(output.name());