15 #include <fmt/format.h>
17 #include <google/protobuf/text_format.h>
18 #include <google/protobuf/io/zero_copy_stream_impl.h>
24 using namespace armnn;
29 IOnnxParser::IOnnxParser() : pOnnxParserImpl(new OnnxParserImpl()) {}
31 IOnnxParser::~IOnnxParser() =
default;
53 armnn::INetworkPtr IOnnxParser::CreateNetworkFromBinary(
const std::vector<uint8_t>& binaryContent)
55 return pOnnxParserImpl->CreateNetworkFromBinary(binaryContent);
58 armnn::INetworkPtr IOnnxParser::CreateNetworkFromBinary(
const std::vector<uint8_t>& binaryContent,
59 const std::map<std::string, armnn::TensorShape>& inputShapes)
61 return pOnnxParserImpl->CreateNetworkFromBinary(binaryContent, inputShapes);
66 return pOnnxParserImpl->CreateNetworkFromTextFile(graphFile);
71 return pOnnxParserImpl->CreateNetworkFromString(protoText);
75 const char* graphFile,
76 const std::map<std::string, armnn::TensorShape>& inputShapes)
78 return pOnnxParserImpl->CreateNetworkFromBinaryFile(graphFile, inputShapes);
82 const std::map<std::string, armnn::TensorShape>& inputShapes)
84 return pOnnxParserImpl->CreateNetworkFromTextFile(graphFile, inputShapes);
88 const std::map<std::string, armnn::TensorShape>& inputShapes)
90 return pOnnxParserImpl->CreateNetworkFromString(protoText, inputShapes);
93 BindingPointInfo IOnnxParser::GetNetworkInputBindingInfo(
const std::string& name)
const
95 return pOnnxParserImpl->GetNetworkInputBindingInfo(name);
98 BindingPointInfo IOnnxParser::GetNetworkOutputBindingInfo(
const std::string& name)
const
100 return pOnnxParserImpl->GetNetworkOutputBindingInfo(name);
105 void CheckValidDataType(std::initializer_list<onnx::TensorProto::DataType> validInputTypes,
107 const char* validExpr,
108 std::string nodeName,
109 std::string tensorName,
112 bool isValid = std::any_of(validInputTypes.begin(),
113 validInputTypes.end(),
118 fmt::format(
"Datatype {} is not valid for tensor '{}' of node '{}', not in {{{}}}. {}",
119 onnx::TensorProto::DataType_Name(actualValue),
127 #define CHECK_VALID_DATATYPE(NODE, TENSOR, ACTUAL, ...) \
128 CheckValidDataType({__VA_ARGS__}, ACTUAL, #__VA_ARGS__, NODE, TENSOR, CHECK_LOCATION())
130 using StrTypeListPair = std::pair<const char*, std::initializer_list<onnx::TensorProto::DataType>>;
131 #define STR_LIST(...) StrTypeListPair(#__VA_ARGS__, {__VA_ARGS__})
133 template <
typename Callable>
134 void ReadMandatoryNodeAttributeImpl(
const onnx::NodeProto& node,
135 const std::string& attribName,
136 onnx::AttributeProto::AttributeType expectedType,
139 auto attribs = node.attribute();
141 while (attriNum < node.attribute_size())
143 if (attribs.Get(attriNum).name() == attribName)
145 if (attribs.Get(attriNum).type() == expectedType)
147 callable(attribs.Get(attriNum));
151 throw ParseException(fmt::format(
"Attribute {} of node {} expected to have {} as "
152 "onnx::AttributeProto::AttributeType, but found {} instead {}",
155 onnx::AttributeProto::AttributeType_Name(expectedType),
156 onnx::AttributeProto::AttributeType_Name(attribs.Get(attriNum).type()),
163 if (attriNum == node.attribute_size())
165 throw ParseException(fmt::format(
"Could not find required attribute {} in node {} {}",
170 template <
typename Callable>
171 void ReadOptionalNodeAttributeImpl(
const onnx::NodeProto& node,
172 const std::string& attribName,
173 onnx::AttributeProto::AttributeType expectedType,
176 auto attribs = node.attribute();
177 for (
int attriNum = 0; attriNum < node.attribute_size(); ++attriNum)
179 if (attribs.Get(attriNum).name() == attribName)
181 if (attribs.Get(attriNum).type() == expectedType)
183 callable(attribs.Get(attriNum));
188 fmt::format(
"Attribute {} of node {} expected to have {} as onnx::AttributeProto::AttributeType, "
189 "but found {} instead {}",
192 onnx::AttributeProto::AttributeType_Name(expectedType),
193 onnx::AttributeProto::AttributeType_Name(attribs.Get(attriNum).type()),
200 int ReadMandatoryNodeIntAttribute(
const onnx::NodeProto& node,
201 const std::string& name)
204 ReadMandatoryNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
205 [&attribValue](
const onnx::AttributeProto& attrValue)
212 int64_t ReadOptionalNodeInt64Attribute(
const onnx::NodeProto& node,
213 const std::string& name,
214 const int64_t defaultValue = 0)
216 int64_t attribValue = defaultValue;
217 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
218 [&attribValue](
const onnx::AttributeProto& attrValue)
220 attribValue = attrValue.i();
225 std::vector<uint32_t> ReadMandatoryNodeUint32ListAttribute(
const onnx::NodeProto& node,
226 const std::string& name)
228 std::vector<uint32_t> attriList;
229 ReadMandatoryNodeAttributeImpl(node, name, onnx::AttributeProto::INTS,
230 [&attriList](
const onnx::AttributeProto& attrValue)
232 for (
int attriNum = 0; attriNum < attrValue.ints_size(); ++attriNum)
240 uint32_t ReadOptionalNodeUint32Attribute(
const onnx::NodeProto& node,
241 const std::string& name,
242 const uint32_t defaultVal = 0u)
244 uint32_t attribValue = defaultVal;
245 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
246 [&attribValue](
const onnx::AttributeProto& attrValue)
253 std::vector<uint32_t> ReadOptionalNodeUint32ListAttribute(
const onnx::NodeProto& node,
254 const std::string& name)
256 std::vector<uint32_t> attriList;
257 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INTS,
258 [&attriList](
const onnx::AttributeProto& attrValue)
260 for (
int attriNum = 0; attriNum < attrValue.ints_size(); ++attriNum)
269 float ReadOptionalNodeFloatAttribute(
const onnx::NodeProto& node,
270 const std::string& name,
271 const float defaultValue = 0.0f)
273 float attribValue = defaultValue;
274 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::FLOAT,
275 [&attribValue](
const onnx::AttributeProto& attrValue)
277 attribValue = attrValue.f();
282 std::string ReadOptionalNodeStringAttribute(
const onnx::NodeProto& node,
const std::string& name)
284 std::string attribValue =
"";
285 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::STRING,
286 [&attribValue](
const onnx::AttributeProto& attrValue)
288 attribValue = attrValue.s();
298 case onnx::TensorProto::FLOAT:
300 type = DataType::Float32;
303 case onnx::TensorProto::INT32:
304 case onnx::TensorProto::INT64:
306 type = DataType::Signed32;
312 fmt::format(
"'{}' is not a currently supported datatype for tensor {}."
313 " Supported dataTypes are FLOAT, INT32 and INT64. {}",
327 if(std::find(shape.begin(), shape.end(), 0) != shape.end())
337 const onnx::TensorShapeProto onnxShape =
info.type().tensor_type().shape();
338 std::vector<unsigned int> shapeDims;
339 for (
int i = 0; i < onnxShape.dim_size(); ++i)
349 std::vector<unsigned int> shapeDims;
351 for (
auto dim: tensor.dims())
356 return ToTensorInfo(tensor.name(), shapeDims, tensor.data_type());
359 std::string TensorInfoAsString(
const TensorInfo& info,
360 const std::string& name,
364 std::stringstream ss;
365 ss <<
"tensor '" << name <<
"' contains "
366 << onnx::TensorProto::DataType_Name(type)
367 <<
" and has shape [";
371 ss << shape[i] <<
", ";
377 void CalcPadding(uint32_t inputSize,
381 uint32_t* paddingFront,
382 uint32_t* paddingBack,
385 uint32_t outputSize = (inputSize + stride - 1) / stride;
386 uint32_t dilatedSize = filterSize + (dilation - 1) * (filterSize - 1);
387 uint32_t temp = (outputSize - 1) * stride + dilatedSize;
388 *paddingFront = (temp - inputSize) / 2;
389 *paddingBack = *paddingFront;
390 if((temp - inputSize) % 2 == 1)
405 const std::string& outName,
406 DataType dataType = DataType::Float32)
408 std::vector<int> targetDims;
414 targetDims.push_back(
static_cast<int>(inShape[
static_cast<uint
>(i)]));
418 targetDims.push_back(val);
422 std::vector<unsigned int> outDims(targetDims.begin(), targetDims.end());
423 const auto stretchDim = std::find(targetDims.begin(), targetDims.end(), -1);
424 if (stretchDim != targetDims.end())
426 if (std::find(std::next(stretchDim), targetDims.end(), -1) != targetDims.end())
428 std::stringstream ss;
430 for(uint i = 0; i < targetDims.size() - 1; ++i)
432 ss << targetDims[i] <<
", ";
434 ss << targetDims[targetDims.size() - 1] <<
" ]";
437 fmt::format(
"Error during creation of reshaped tensor '{}'. At most one component of shape can be "
438 " -1 and here, shape is {} {}",
444 auto targetNumElements = armnn::numeric_cast<unsigned int>(std::accumulate(targetDims.begin(), targetDims.end(),
445 -1, std::multiplies<int32_t>()));
446 auto stretchIndex =
static_cast<size_t>(std::distance(targetDims.begin(), stretchDim));
447 outDims[stretchIndex] = inShape.
GetNumElements() / targetNumElements;
455 const std::map<std::string, OnnxParserImpl::OperationParsingFunction> OnnxParserImpl::m_ParserFunctions = {
456 {
"BatchNormalization", &OnnxParserImpl::ParseBatchNormalization},
457 {
"GlobalAveragePool", &OnnxParserImpl::ParseGlobalAveragePool},
458 {
"AveragePool", &OnnxParserImpl::ParseAveragePool },
459 {
"Clip", &OnnxParserImpl::ParseClip },
460 {
"Constant", &OnnxParserImpl::ParseConstant },
461 {
"MaxPool", &OnnxParserImpl::ParseMaxPool },
462 {
"Reshape", &OnnxParserImpl::ParseReshape },
463 {
"Sigmoid", &OnnxParserImpl::ParseSigmoid },
464 {
"Tanh", &OnnxParserImpl::ParseTanh },
465 {
"Relu", &OnnxParserImpl::ParseRelu },
466 {
"LeakyRelu", &OnnxParserImpl::ParseLeakyRelu },
467 {
"Conv", &OnnxParserImpl::ParseConv },
468 {
"Add", &OnnxParserImpl::ParseAdd },
469 {
"Flatten", &OnnxParserImpl::ParseFlatten },
470 {
"Shape", &OnnxParserImpl::ParseShape },
471 {
"Gather", &OnnxParserImpl::ParseGather },
472 {
"Unsqueeze", &OnnxParserImpl::ParseUnsqueeze },
473 {
"Concat", &OnnxParserImpl::ParseConcat },
474 {
"Gemm", &OnnxParserImpl::ParseGemm }
477 template<
typename TypePair,
typename Location>
478 void OnnxParserImpl::ValidateInputs(
const onnx::NodeProto& node,
479 TypePair validInputs,
480 const Location& location)
482 for(
auto input : node.input())
484 CheckValidDataType(validInputs.second,
485 m_TensorsInfo[input].m_dtype,
493 #define VALID_INPUTS(NODE, VALID_INPUTS) \
494 OnnxParserImpl::ValidateInputs(NODE, \
498 std::vector<TensorInfo> OnnxParserImpl::ComputeOutputInfo(std::vector<std::string> outNames,
500 std::vector<TensorShape> inputShapes,
503 if (outNames.empty())
508 bool needCompute = std::any_of(outNames.begin(),
510 [
this](std::string name)
512 return (m_TensorsInfo.count(name) == 0 ||
513 m_TensorsInfo[name].m_info == nullptr ||
514 m_TensorsInfo[name].m_info->GetShape().GetDimensionality() ==
515 Dimensionality::NotSpecified);
517 std::vector<TensorInfo> outInfo;
519 std::vector<TensorShape> inferredShapes;
520 DataType armnnType = DataType::Float32;
523 if (inferredShapes.size() != outNames.size())
529 case onnx::TensorProto::FLOAT: {
530 armnnType = DataType::Float32;
533 case onnx::TensorProto::INT32:
534 case onnx::TensorProto::INT64: {
535 armnnType = DataType::Signed32;
540 fmt::format(
"'{}' is not a currently supported datatype for {}."
541 " Supported dataTypes are FLOAT, INT32 and INT64. {}",
548 for (uint i = 0; i < outNames.size(); ++i)
552 m_TensorsInfo[outNames[i]] = OnnxTensor();
553 m_TensorsInfo[outNames[i]].m_info = std::make_unique<TensorInfo>(
555 m_TensorsInfo[outNames[i]].m_dtype = dataType;
557 outInfo.push_back(*m_TensorsInfo[outNames[i]].m_info);
562 OnnxParserImpl::OnnxParserImpl()
563 : m_Network(nullptr, nullptr)
567 void OnnxParserImpl::ResetParser()
571 m_InputInfos.clear();
572 m_OutputInfos.clear();
575 void OnnxParserImpl::Cleanup()
577 m_TensorConnections.clear();
578 m_TensorsInfo.clear();
579 m_OutputsMap.clear();
580 m_OutputsFusedAndUsed.clear();
581 m_InputShapes.clear();
585 std::pair<armnn::ConstTensor, std::unique_ptr<T[]>>
590 if (bufferPtr ==
nullptr)
601 reinterpret_cast<const T*
>(bufferPtr), data.get(),
sizeof(T));
605 ::memcpy(data.get(), bufferPtr, tensorInfo.
GetNumBytes());
608 return std::make_pair(
ConstTensor(tensorInfo, data.get()), std::move(data));
611 std::pair<ConstTensor, std::unique_ptr<float[]>>
612 OnnxParserImpl::CreateConstTensor(
const std::string name,
615 TensorInfo tensorInfo = *m_TensorsInfo[name].m_info;
616 onnx::TensorProto onnxTensor = *m_TensorsInfo[name].m_tensor;
628 throw ParseException(fmt::format(
"No tensor data found for Const tensor '{}' {}",
633 auto srcData = onnxTensor.float_data().data();
635 if (!onnxTensor.has_raw_data())
637 if(tensorInfo.
GetNumElements() !=
static_cast<uint
>(onnxTensor.float_data_size()))
640 fmt::format(
"The number of data provided ({}) does not match the tensor '{}' number of "
642 onnxTensor.float_data_size(),
647 return CreateConstTensorImpl<float>(srcData, tensorInfo, permutationVector);
651 return CreateConstTensorImpl<float>(
reinterpret_cast<const float*
>(onnxTensor.raw_data().c_str()),
657 std::pair<ConstTensor, std::unique_ptr<int32_t[]>>
658 OnnxParserImpl::CreateInt64ConstTensor(
const std::string name,
661 TensorInfo tensorInfo = *m_TensorsInfo[name].m_info;
662 onnx::TensorProto onnxTensor = *m_TensorsInfo[name].m_tensor;
672 if (numElements == 0)
674 throw ParseException(fmt::format(
"No tensor data found for Const tensor '{}' {}",
680 if (!onnxTensor.has_raw_data())
682 auto srcData = onnxTensor.int64_data().data();
683 if(numElements !=
static_cast<uint
>(onnxTensor.int64_data_size()))
686 fmt::format(
"The number of data provided ({}) does not match the tensor '{}' number of "
688 onnxTensor.int64_data_size(),
694 std::vector<int32_t> int32Data;
695 for(uint i = 0; i < numElements; i++)
698 int32Data.push_back(int32Value);
701 return CreateConstTensorImpl<int32_t>(int32Data.data(), tensorInfo, permutationVector);
705 auto srcData =
reinterpret_cast<const int64_t*
>(onnxTensor.raw_data().c_str());
706 std::vector<int32_t> int32Data;
707 for(uint i = 0; i < numElements; i++)
710 int32Data.push_back(int32Value);
712 return CreateConstTensorImpl<int32_t>(int32Data.data(), tensorInfo, permutationVector);
718 FILE* fd = fopen(graphFile,
"r");
726 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
727 using google::protobuf::io::FileInputStream;
728 std::unique_ptr<FileInputStream> input = std::make_unique<FileInputStream>(fileno(fd));
729 bool success = google::protobuf::TextFormat::Parse(input.get(), modelProto.get());
734 std::stringstream
error;
735 error <<
"Failed to parse graph file";
745 return CreateNetworkFromModel(*modelProto);
749 const std::map<std::string, armnn::TensorShape>& inputShapes)
752 m_InputShapes = inputShapes;
754 return CreateNetworkFromModel(*modelProto);
761 return CreateNetworkFromModel(*modelProto);
765 const std::map<std::string, armnn::TensorShape>& inputShapes)
768 m_InputShapes = inputShapes;
770 return CreateNetworkFromModel(*modelProto);
775 if (binaryContent.size() == 0)
780 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
782 google::protobuf::io::CodedInputStream codedStream(binaryContent.data(),
static_cast<int>(binaryContent.size()));
783 codedStream.SetTotalBytesLimit(INT_MAX);
784 bool success = modelProto.get()->ParseFromCodedStream(&codedStream);
788 std::stringstream
error;
789 error <<
"Failed to parse graph";
797 FILE* fd = fopen(graphFile,
"rb");
805 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
807 google::protobuf::io::FileInputStream inStream(fileno(fd));
808 google::protobuf::io::CodedInputStream codedStream(&inStream);
809 codedStream.SetTotalBytesLimit(INT_MAX);
810 bool success = modelProto.get()->ParseFromCodedStream(&codedStream);
815 std::stringstream
error;
816 error <<
"Failed to parse graph file";
827 return CreateNetworkFromModel(*modelProto);
831 const std::map<std::string, armnn::TensorShape>& inputShapes)
834 m_InputShapes = inputShapes;
836 return CreateNetworkFromModel(*modelProto);
847 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
848 bool success = google::protobuf::TextFormat::ParseFromString(protoText, modelProto.get());
851 std::stringstream
error;
852 error <<
"Failed to parse graph file";
862 return CreateNetworkFromModel(*modelProto);
866 const std::map<std::string, armnn::TensorShape>& inputShapes)
869 m_InputShapes = inputShapes;
871 return CreateNetworkFromModel(*modelProto);
874 INetworkPtr OnnxParserImpl::CreateNetworkFromModel(onnx::ModelProto& model)
876 m_Network = INetwork::Create();
879 m_Graph = std::make_unique<onnx::GraphProto>(*model.mutable_graph());
888 return std::move(m_Network);
891 void OnnxParserImpl::LoadGraph()
893 if (m_Graph.get() ==
nullptr)
899 SetupInfo(m_Graph->mutable_output());
900 SetupInfo(m_Graph->mutable_input());
901 SetupInfo(m_Graph->mutable_value_info());
903 for (
auto tensor : m_Graph->initializer())
905 m_TensorsInfo[tensor.name()].m_tensor = std::make_unique<const onnx::TensorProto>(tensor);
906 m_TensorsInfo[tensor.name()].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(tensor));
907 m_TensorsInfo[tensor.name()].m_dtype =
915 DetectFullyConnected();
918 for(
size_t nodeIndex = 0; nodeIndex < static_cast<size_t>(m_Graph->node_size()); nodeIndex++)
920 auto node = m_Graph->node(
static_cast<int>(nodeIndex));
921 const std::string& operation = node.op_type();
924 if (operation ==
"MatMul" )
926 if(m_OutputsFusedAndUsed[nodeIndex].inputForNodes != m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.size())
929 AddFullyConnected(node);
932 else if (!(m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.empty()) && operation ==
"Add")
934 int matmulIndex =
static_cast<int> (m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes[0]);
935 AddFullyConnected(m_Graph->node(matmulIndex), &node);
937 else if (m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.empty())
939 auto it = m_ParserFunctions.find(operation);
940 if (it != m_ParserFunctions.end())
942 auto func = it->second;
947 throw ParseException(fmt::format(
"Unsupported operation {} for node '{}' {}",
956 for (
const auto& tensorCon : m_TensorConnections)
958 if (tensorCon.second.outputSlot !=
nullptr)
960 for (
size_t inputSlotIdx = 0; inputSlotIdx < tensorCon.second.inputSlots.size(); ++inputSlotIdx)
962 tensorCon.second.outputSlot->Connect(*(tensorCon.second.inputSlots[inputSlotIdx]));
968 for(
int outputIndex = 0; outputIndex < m_Graph->output_size(); ++outputIndex)
970 auto output = m_Graph->output(outputIndex);
971 m_OutputInfos[output.name()] = *m_TensorsInfo[output.name()].m_info;
975 void OnnxParserImpl::SetupInfo(
const google::protobuf::RepeatedPtrField<onnx::ValueInfoProto >* list)
977 for (
auto tensor : *list)
979 m_TensorsInfo[tensor.name()] = OnnxTensor();
980 m_TensorsInfo[tensor.name()].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(tensor));
981 m_TensorsInfo[tensor.name()].m_dtype =
986 void OnnxParserImpl::DetectFullyConnected()
988 m_OutputsFusedAndUsed = std::vector<UsageSummary> (
static_cast<size_t>(m_Graph->node_size()), UsageSummary());
989 auto matmulAndConstant = [&](
const std::string& constInput,
990 const std::string& matmulInput,
993 auto matmulIt = m_OutputsMap.find(matmulInput);
994 if(matmulIt != m_OutputsMap.end() && matmulIt->second.first->op_type() ==
"MatMul"
995 && m_TensorsInfo[constInput].isConstant())
997 nodeIndex = matmulIt->second.second;
1003 for(
int nodeIndex = 0; nodeIndex < m_Graph->node_size(); nodeIndex++)
1005 const onnx::NodeProto* node = &m_Graph->node(nodeIndex);
1006 for (
const std::string& output : node->output())
1008 m_OutputsMap[output] = std::make_pair(node, nodeIndex);
1011 for (
const std::string& input : node->input())
1013 auto matmulIt = m_OutputsMap.find(input);
1014 if(matmulIt != m_OutputsMap.end()){
1015 ++m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIt->second.second)].inputForNodes;
1019 if (node->op_type() ==
"Add")
1021 int matmulIndex = 0;
1022 if (matmulAndConstant(node->input(0), node->input(1), matmulIndex) ||
1023 matmulAndConstant(node->input(1), node->input(0), matmulIndex))
1026 m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIndex)].fusedWithNodes
1027 .push_back(
static_cast<size_t>(nodeIndex));
1029 m_OutputsFusedAndUsed[
static_cast<size_t>(nodeIndex)].fusedWithNodes
1030 .push_back(
static_cast<size_t>(matmulIndex));
1035 for (
auto output: m_Graph->output()) {
1036 auto matmulIt = m_OutputsMap.find(output.name());
1037 if(matmulIt != m_OutputsMap.end()){
1038 ++m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIt->second.second)].inputForNodes;
1043 template<
typename Location>
1044 void OnnxParserImpl::GetInputAndParam(
const onnx::NodeProto& node,
1045 std::string* inputName,
1046 std::string* constName,
1047 const Location& location)
1050 if (m_TensorsInfo[node.input(0)].isConstant())
1054 else if (m_TensorsInfo[node.input(1)].isConstant())
1060 throw ParseException(fmt::format(
"One of the input tensors ('{}' or '{}') should be constant in node '{}' {}",
1064 location.AsString()));
1068 *constName = node.input(cstIndex);
1072 *inputName = node.input(!cstIndex);
1076 template<
typename Location>
1077 void OnnxParserImpl::To1DTensor(
const std::string& name,
const Location& location)
1079 TensorShape shape = m_TensorsInfo[name].m_info->GetShape();
1080 std::vector<uint32_t> newShape;
1086 fmt::format(
"Only tensors with shape [1, ..., 1, X] can be converted to 1D and {} {}",
1087 TensorInfoAsString(*m_TensorsInfo[name].m_info, name, m_TensorsInfo[name].m_dtype),
1088 location.AsString()));
1093 m_TensorsInfo[name].m_info->SetShape(
TensorShape(
static_cast<unsigned int>(newShape.size()), newShape.data()));
1096 void OnnxParserImpl::AddConvLayerWithDepthwiseConv(
const onnx::NodeProto& node,
const Convolution2dDescriptor& convDesc)
1110 std::string permuteStr =
"permute_" + node.input(1);
1111 std::vector<std::string> tensorIndexes= {node.input(0), permuteStr};
1113 auto weightTensor = CreateConstTensor(node.input(1));
1114 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(weightTensor.first);
1129 if (node.input_size() == 3)
1131 if(!m_TensorsInfo[node.input(2)].isConstant())
1133 throw ParseException(fmt::format(
"Bias '{}' should be constant in Conv layer '{}' {}",
1140 auto biasTensor = CreateConstTensor(node.input(2));
1141 tensorIndexes.emplace_back(node.input(2));
1153 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1154 { m_TensorsInfo[node.input(0)].m_info->GetShape(),
1161 RegisterInputSlots(layer, tensorIndexes);
1164 RegisterOutputSlots(layer, {node.output(0)});
1167 void OnnxParserImpl::AddFullyConnected(
const onnx::NodeProto& matmulNode,
const onnx::NodeProto* addNode)
1170 std::string inputName;
1171 std::string weightName;
1172 std::string biasName;
1173 std::string outputName;
1178 GetInputAndParam(matmulNode, &inputName, &weightName,
CHECK_LOCATION());
1180 TensorInfo inputInfo = *m_TensorsInfo[inputName].m_info;
1181 TensorInfo weightInfo = *m_TensorsInfo[weightName].m_info;
1184 std::vector<std::string> inputNames;
1197 GetInputAndParam(*addNode,
nullptr, &biasName,
CHECK_LOCATION());
1201 biasInfo = *m_TensorsInfo[biasName].m_info;
1206 fmt::format(
"Shape of weights '{}' and bias of following Add node '{}' do not match : {}"
1207 " and {} ( /!\\ bias should be a 1D tensor) {}",
1210 TensorInfoAsString(*m_TensorsInfo[weightName].m_info, weightName,
1211 m_TensorsInfo[weightName].m_dtype),
1212 TensorInfoAsString(*m_TensorsInfo[biasName].m_info, biasName,
1213 m_TensorsInfo[biasName].m_dtype ),
1217 inputNames = { inputName, weightName, biasName };
1218 outputName = addNode->output(0);
1222 inputNames = { inputName, weightName };
1223 outputName = matmulNode.output(0);
1227 layer = m_Network->AddFullyConnectedLayer(desc, matmulNode.name().c_str());
1240 std::vector<unsigned int> reshapedDimensions(2);
1241 reshapedDimensions[1] = weightInfo.
GetShape()[0];
1242 reshapedDimensions[0] = inputInfo.
GetNumElements() / reshapedDimensions[1];
1247 fmt::format(
"Failed to deduce input tensor shape from filter size {} {}",
1248 reshapedDimensions[1],
1254 inputInfo = reshapedTensorInfo;
1259 std::string reshapeLayerName = fmt::format(
"Reshape_for:{}", layer->
GetName());
1260 IConnectableLayer* reshapeLayer = m_Network->AddReshapeLayer(reshapeDescriptor, reshapeLayerName.c_str());
1265 RegisterInputSlots(reshapeLayer, {inputName});
1266 inputNames[0] = reshapeLayerName;
1269 auto outputInfo = ComputeOutputInfo({ outputName },
1275 RegisterInputSlots(layer, inputNames);
1278 if(m_TensorsInfo[weightName].isConstant())
1280 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(CreateConstTensor(weightName).first);
1287 if(desc.
m_BiasEnabled && m_TensorsInfo[biasName].isConstant())
1289 IConnectableLayer* biasLayer = m_Network->AddConstantLayer(CreateConstTensor(biasName).first);
1296 if (outputInfo[0].GetNumDimensions() > 2)
1299 std::vector<unsigned int> reshapedDimensions(2);
1300 reshapedDimensions[1] = weightInfo.
GetShape()[1];
1301 reshapedDimensions[0] = outputInfo[0].
GetNumElements() / reshapedDimensions[1];
1303 if (outputInfo[0].GetNumElements() % reshapedDimensions[1] != 0)
1306 fmt::format(
"Failed to deduce output tensor shape from filter size {} {}",
1307 reshapedDimensions[1],
1318 std::string reshapeLayerName = fmt::format(
"ExpandDims_for:{}", layer->
GetName());
1319 IConnectableLayer* reshapeLayer = m_Network->AddReshapeLayer(desc, reshapeLayerName.c_str());
1324 RegisterInputSlots(reshapeLayer, {layer->
GetName()});
1325 layer = reshapeLayer;
1328 RegisterOutputSlots(layer, { outputName });
1331 void OnnxParserImpl::AddPoolingLayer(
const onnx::NodeProto& node,
Pooling2dDescriptor& desc)
1339 std::vector<uint32_t> kernel_shape = ReadMandatoryNodeUint32ListAttribute(node,
"kernel_shape");
1340 std::vector<uint32_t> strides = ReadOptionalNodeUint32ListAttribute(node,
"strides");
1341 std::vector<uint32_t> pads = ReadOptionalNodeUint32ListAttribute(node,
"pads");
1362 std::string paddingString = ReadOptionalNodeStringAttribute(node,
"auto_pad");
1363 if(paddingString !=
"VALID" && paddingString !=
"" && paddingString !=
"NOTSET")
1366 if( paddingString ==
"SAME_LOWER")
1370 else if (paddingString ==
"SAME_UPPER")
1376 throw ParseException(fmt::format(
"Invalid auto_pad attribute for node {}. "
1377 "Only SAME_UPPER, SAME_LOWER or VALID supported and found {} {}",
1382 auto inputInfo = *m_TensorsInfo[node.input(0)].m_info;
1383 uint32_t inputHeight = inputInfo.
GetShape()[2];
1384 uint32_t inputWidth = inputInfo.
GetShape()[3];
1385 CalcPadding(inputHeight,
1392 CalcPadding(inputWidth,
1409 IConnectableLayer* layer = m_Network->AddPooling2dLayer(desc, node.name().c_str());
1416 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1421 RegisterInputSlots(layer, {node.input(0)});
1424 RegisterOutputSlots(layer, {node.output(0)});
1427 std::pair<std::string, std::string> OnnxParserImpl::AddPrepareBroadcast(
const std::string& input0,
1428 const std::string& input1)
1430 std::pair<std::string, std::string> inputs = std::make_pair(input0, input1);
1432 TensorShape input0Shape = m_TensorsInfo[input0].m_info->GetShape();
1433 TensorShape input1Shape = m_TensorsInfo[input1].m_info->GetShape();
1437 auto outputName = fmt::format(
"reshape_output_{}", input1);
1438 PrependForBroadcast(outputName, input1, input0);
1439 inputs.second = outputName;
1443 auto outputName = fmt::format(
"reshape_output_{}", input0);
1444 PrependForBroadcast(outputName, input0, input1);
1445 inputs.first = outputName;
1450 void OnnxParserImpl::CreateConstantLayer(
const std::string& tensorName,
const std::string& layerName)
1452 auto armnnTensor = CreateConstTensor(tensorName);
1453 IConnectableLayer* layer = m_Network->AddConstantLayer(armnnTensor.first, layerName.c_str());
1455 RegisterOutputSlots(layer, {tensorName});
1458 void OnnxParserImpl::CreateInt64ConstantLayer(
const std::string& tensorName,
const std::string& layerName)
1460 auto armnnTensor = CreateInt64ConstTensor(tensorName);
1461 IConnectableLayer* layer = m_Network->AddConstantLayer(armnnTensor.first, layerName.c_str());
1463 RegisterOutputSlots(layer, {tensorName});
1466 void OnnxParserImpl::CreateReshapeLayer(
const std::string& inputName,
1467 const std::string& outputName,
1468 const std::string& layerName)
1470 const TensorInfo outputTensorInfo = *m_TensorsInfo[outputName].m_info;
1474 IConnectableLayer* layer = m_Network->AddReshapeLayer(reshapeDesc, layerName.c_str());
1485 RegisterInputSlots(layer, {inputName});
1488 RegisterOutputSlots(layer, {outputName});
1501 if (func == ActivationFunction::BoundedReLu)
1503 if (node.input_size() == 1 && node.attribute_size() > 0)
1505 desc.
m_A = ReadOptionalNodeFloatAttribute(node,
"max", std::numeric_limits<float>::max());
1506 desc.
m_B = ReadOptionalNodeFloatAttribute(node,
"min", std::numeric_limits<float>::lowest());
1510 desc.
m_A = node.input(2).empty() ? std::numeric_limits<float>::max() :
std::stof(node.input(2));
1511 desc.
m_B = node.input(1).empty() ? std::numeric_limits<float>::lowest() :
std::stof(node.input(1));
1515 IConnectableLayer*
const layer = m_Network->AddActivationLayer(desc, node.name().c_str());
1522 auto outputInfo = ComputeOutputInfo({ node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1527 RegisterInputSlots(layer, {node.input(0)});
1530 RegisterOutputSlots(layer, {node.output(0)});
1533 void OnnxParserImpl::ParseClip(
const onnx::NodeProto& node)
1535 ParseActivation(node, ActivationFunction::BoundedReLu);
1538 void OnnxParserImpl::ParseSigmoid(
const onnx::NodeProto& node)
1540 ParseActivation(node, ActivationFunction::Sigmoid);
1543 void OnnxParserImpl::ParseTanh(
const onnx::NodeProto& node)
1545 ParseActivation(node, ActivationFunction::TanH);
1548 void OnnxParserImpl::ParseRelu(
const onnx::NodeProto& node)
1550 ParseActivation(node, ActivationFunction::ReLu);
1553 void OnnxParserImpl::ParseLeakyRelu(
const onnx::NodeProto& node)
1555 ParseActivation(node, ActivationFunction::LeakyReLu);
1558 void OnnxParserImpl::ParseAdd(
const onnx::NodeProto& node)
1568 auto inputs = AddPrepareBroadcast(node.input(0), node.input(1));
1569 auto input0 = *m_TensorsInfo[inputs.first].m_info;
1570 auto input1 = *m_TensorsInfo[inputs.second].m_info;
1571 if (input0.GetNumDimensions() != input1.GetNumDimensions())
1578 unsigned int numDims = input0.GetNumDimensions();
1579 for (
unsigned int i = 0; i < numDims; i++)
1581 unsigned int dim0 = input0.GetShape()[i];
1582 unsigned int dim1 = input1.GetShape()[i];
1583 if (dim0 != dim1 && dim0 != 1 && dim1 != 1)
1586 fmt::format(
"Broadcast is only supported for scalar or 1D tensors in Add node '{}'. "
1587 "Input dimensions should either match or one should be of size 1 and here, "
1590 TensorInfoAsString(*m_TensorsInfo[inputs.first].m_info, inputs.first,
1591 m_TensorsInfo[inputs.first].m_dtype),
1592 TensorInfoAsString(*m_TensorsInfo[inputs.second].m_info, inputs.second,
1593 m_TensorsInfo[inputs.second].m_dtype),
1599 IConnectableLayer* layer = m_Network->AddElementwiseBinaryLayer(BinaryOperation::Add, node.name().c_str());
1606 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1607 { m_TensorsInfo[inputs.first].m_info->GetShape(),
1608 m_TensorsInfo[inputs.second].m_info->GetShape() });
1612 if(m_TensorsInfo[inputs.first].isConstant()) {
1613 CreateConstantLayer(inputs.first, fmt::format(
"Add:constant_of_{}", node.input(0)));
1615 if(m_TensorsInfo[inputs.second].isConstant()) {
1616 CreateConstantLayer(inputs.second, fmt::format(
"Add:constant_of_{}", node.input(1)));
1618 RegisterInputSlots(layer, {inputs.first, inputs.second});
1621 RegisterOutputSlots(layer, {node.output(0)});
1624 void OnnxParserImpl::ParseAveragePool(
const onnx::NodeProto& node)
1629 uint32_t count_include_pad = 0;
1630 count_include_pad = ReadOptionalNodeUint32Attribute(node,
"count_include_pad");
1631 if(count_include_pad) {
1634 AddPoolingLayer(node, desc);
1637 void OnnxParserImpl::ParseBatchNormalization(
const onnx::NodeProto& node)
1645 for(
int ind = 1; ind < node.input_size(); ++ind)
1647 auto tensor = node.input(ind);
1648 if(! m_TensorsInfo[tensor].isConstant())
1651 fmt::format(
"Input tensor '{}' should be constant in BatchNormalization node '{}' {}",
1658 float epsilon = ReadOptionalNodeFloatAttribute(node,
"epsilon", 1e-5f);
1660 desc.
m_Eps = epsilon;
1662 auto scaleTensor = CreateConstTensor(node.input(1));
1663 auto biasTensor = CreateConstTensor(node.input(2));
1664 auto meanTensor = CreateConstTensor(node.input(3));
1665 auto varTensor = CreateConstTensor(node.input(4));
1672 node.name().c_str());
1679 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1682 RegisterInputSlots(layer, {node.input(0)});
1685 RegisterOutputSlots(layer, {node.output(0)});
1688 void OnnxParserImpl::ParseConcat(
const onnx::NodeProto& node)
1692 uint32_t numConcatView =
static_cast<uint32_t
>(node.input_size());
1693 uint32_t inputRank = m_TensorsInfo[node.input(0)].m_info->GetNumDimensions();
1695 int axisInt = ReadMandatoryNodeIntAttribute(node,
"axis");
1697 unsigned int concatDimInput =
static_cast<unsigned int>(
1698 (
static_cast<int>(inputRank) + axisInt) %
static_cast<int>(inputRank));
1701 concatDescriptor.SetConcatAxis(concatDimInput);
1703 unsigned int mergeDimOrigin = 0;
1705 std::vector<TensorShape> inputShapes;
1706 std::vector<std::string> tensorIds;
1708 for (
unsigned int viewIndex = 0; viewIndex < numConcatView; ++viewIndex)
1710 std::string nodeName = node.input(
static_cast<int>(viewIndex));
1711 auto inputTensorInfo = *m_TensorsInfo[nodeName].m_info;
1712 inputShapes.push_back(inputTensorInfo.GetShape());
1713 tensorIds.push_back(nodeName);
1717 inputTensorInfo, concatDescriptor, concatDimInput, viewIndex, mergeDimOrigin);
1720 IConnectableLayer* layer = m_Network->AddConcatLayer(concatDescriptor, node.name().c_str());
1727 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, inputShapes,
1728 m_TensorsInfo[node.input(0)].m_dtype);
1733 RegisterInputSlots(layer, tensorIds);
1736 RegisterOutputSlots(layer, { node.output(0) });
1739 void OnnxParserImpl::ParseConstant(
const onnx::NodeProto& node)
1742 if (!node.attribute(0).has_t())
1744 throw ParseException(fmt::format(
"Value not found for Constant node '{}' {}",
1748 const onnx::TensorProto& onnxTensor = node.attribute(0).t();
1751 m_TensorsInfo[node.output(0)].m_tensor = std::make_unique<const onnx::TensorProto>(onnxTensor);
1752 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(onnxTensor));
1755 if (m_TensorsInfo[node.output(0)].m_dtype == onnx::TensorProto_DataType_FLOAT)
1757 CreateConstantLayer(node.output(0), node.name());
1759 else if (m_TensorsInfo[node.output(0)].m_dtype == onnx::TensorProto_DataType_INT64)
1761 CreateInt64ConstantLayer(node.output(0), node.name());
1765 throw ParseException(fmt::format(
"Data type not support for Constant node '{}' {}",
1771 void OnnxParserImpl::ParseConv(
const onnx::NodeProto& node)
1778 if(m_TensorsInfo[node.input(0)].m_info->GetNumDimensions() != 4)
1781 fmt::format(
"ArmNN only supports 2D convolution and Conv layer '{}' input {} {}",
1783 TensorInfoAsString(*m_TensorsInfo[node.input(0)].m_info, node.input(0),
1784 m_TensorsInfo[node.input(0)].m_dtype),
1788 if(!m_TensorsInfo[node.input(1)].isConstant())
1791 fmt::format(
"Weights '{}' should be constant in Conv layer '{}' {}",
1797 auto inputInfo = *m_TensorsInfo[node.input(0)].m_info;
1802 std::vector<uint32_t> strides = ReadOptionalNodeUint32ListAttribute(node,
"strides");
1814 std::vector<uint32_t> dilations = ReadOptionalNodeUint32ListAttribute(node,
"dilations");
1815 if(!dilations.empty())
1821 std::vector<uint32_t> pads = ReadOptionalNodeUint32ListAttribute(node,
"pads");
1826 std::string paddingString = ReadOptionalNodeStringAttribute(node,
"auto_pad");
1827 if(paddingString !=
"VALID" && paddingString !=
"" && paddingString !=
"NOTSET")
1830 if( paddingString ==
"SAME_LOWER")
1834 else if (paddingString ==
"SAME_UPPER")
1841 fmt::format(
"Invalid auto_pad attribute for node {}. Only SAME_UPPER, SAME_LOWER or VALID "
1842 "supported and found {} {}",
1847 uint32_t inputHeight = inputInfo.
GetShape()[2];
1848 uint32_t inputWidth = inputInfo.
GetShape()[3];
1850 uint32_t weightHeight;
1851 uint32_t weightWidth;
1852 std::vector<uint32_t> kernel_shape = ReadOptionalNodeUint32ListAttribute(node,
"kernel_shape");
1853 if (kernel_shape.empty())
1855 const TensorInfo weightTensorInfo = *m_TensorsInfo[node.input(1)].m_info;
1856 weightHeight = weightTensorInfo.
GetShape()[2];
1857 weightWidth = weightTensorInfo.
GetShape()[3];
1861 weightHeight = kernel_shape[0];
1862 weightWidth = kernel_shape[1];
1864 CalcPadding(inputHeight,
1871 CalcPadding(inputWidth,
1888 uint32_t group = ReadOptionalNodeUint32Attribute(node,
"group", 1);
1891 if (group > inputInfo.
GetShape()[1])
1894 fmt::format(
"Error parsing Convolution node: {}. "
1895 "The 'group'={} parameter cannot be larger than the "
1896 "channel of the input shape={} (in NCHW format). {}",
1902 else if (group == inputInfo.
GetShape()[1])
1906 AddConvLayerWithDepthwiseConv(node, desc);
1911 throw ParseException(fmt::format(
"Error parsing Convolution node: {}. "
1912 "The 'group'={} parameter should be 1 or be equal to the "
1913 "channel of the input shape={} (in NCHW format). {}",
1923 std::vector<std::string> tensorIndexes= {node.input(0), node.input(1)};
1925 auto weightTensor = CreateConstTensor(node.input(1));
1927 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(weightTensor.first);
1931 if (node.input_size() == 3)
1933 if(!m_TensorsInfo[node.input(2)].isConstant())
1935 throw ParseException(fmt::format(
"Bias '{}' should be constant in Conv layer '{}' {}",
1941 auto biasTensor = CreateConstTensor(node.input(2));
1947 tensorIndexes.emplace_back(node.input(2));
1955 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1956 { m_TensorsInfo[node.input(0)].m_info->GetShape(),
1957 m_TensorsInfo[node.input(1)].m_info->GetShape() });
1962 RegisterInputSlots(layer, tensorIndexes);
1965 RegisterOutputSlots(layer, {node.output(0)});
1968 void OnnxParserImpl::ParseFlatten(
const onnx::NodeProto& node)
1974 m_TensorsInfo[node.input(0)].m_dtype,
1975 onnx::TensorProto::FLOAT);
1977 int64_t axis = ReadOptionalNodeInt64Attribute(node,
"axis", 1);
1978 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
1989 throw ParseException(fmt::format(
"Axis '{}' invalid. Tensor has '{}' dimensions in FlattenLayer '{}'",
1999 for (i = 0; i < axis; i++){
2000 dimension1 *= inputShape[i];
2005 dimension2 *= inputShape[i];
2010 auto outInfo = ComputeReshapeInfo(outputShape, inputShape, node.output(0));
2011 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
2012 CreateReshapeLayer(node.input(0), node.output(0), node.name());
2015 void OnnxParserImpl::ParseGather(
const onnx::NodeProto& node)
2021 gatherDescriptor.
m_Axis =
static_cast<int>(ReadOptionalNodeInt64Attribute(node,
"axis", 0));
2023 IConnectableLayer* layer = m_Network->AddGatherLayer(gatherDescriptor, node.name().c_str());
2030 const TensorShape& inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2031 const TensorShape& indicesShape = m_TensorsInfo[node.input(1)].m_info->GetShape();
2032 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, { inputShape, indicesShape },
2033 m_TensorsInfo[node.input(0)].m_dtype);
2037 RegisterInputSlots(layer, { node.input(0), node.input(1) });
2040 RegisterOutputSlots(layer, { node.output(0) });
2043 void OnnxParserImpl::ParseGemm(
const onnx::NodeProto& node)
2048 int transA =
static_cast<int>(ReadOptionalNodeUint32Attribute(node,
"transA", 0));
2049 int transB =
static_cast<int>(ReadOptionalNodeUint32Attribute(node,
"transB", 0));
2050 float alpha = ReadOptionalNodeFloatAttribute(node,
"alpha", 1.0);
2051 float beta = ReadOptionalNodeFloatAttribute(node,
"beta", 1.0);
2052 bool biasEnabled = node.input_size() == 3;
2054 TensorShape input0Shape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2055 TensorShape input1Shape = m_TensorsInfo[node.input(1)].m_info->GetShape();
2065 layer = m_Network->AddFullyConnectedLayer(fullyConnectedDescriptor, node.name().c_str());
2075 std::string transAName =
"transpose_" + node.input(0);
2078 IConnectableLayer* transALayer = m_Network->AddTransposeLayer(transposeADescriptor, transAName.c_str());
2085 auto transAInfo = ComputeOutputInfo({ transAName }, transALayer, { input0Shape });
2089 RegisterInputSlot(transALayer, node.input(0), 0);
2090 input0Shape = transAInfo[0].GetShape();
2094 RegisterInputSlot(layer, node.input(0), 0);
2098 if(m_TensorsInfo[node.input(1)].isConstant())
2100 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(CreateConstTensor(node.input(1)).first);
2101 TensorInfo weightInfo = *m_TensorsInfo[node.input(1)].m_info;
2108 std::string activationName =
"activation_" + node.input(1);
2110 activationDescriptor.
m_A = alpha;
2111 activationDescriptor.
m_Function = ActivationFunction::Linear;
2112 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2119 auto actInfo = ComputeOutputInfo({ activationName }, actLayer, { weightInfo.
GetShape() });
2123 input1Shape = actInfo[0].GetShape();
2128 input1Shape = weightInfo.
GetShape();
2136 std::string activationName =
"activation_" + node.input(1);
2138 activationDescriptor.
m_A = alpha;
2139 activationDescriptor.
m_Function = ActivationFunction::Linear;
2140 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2147 auto actInfo = ComputeOutputInfo({ activationName }, actLayer, { input1Shape });
2150 RegisterInputSlot(actLayer, node.input(1), 0);
2151 input1Shape = actInfo[0].GetShape();
2155 RegisterInputSlot(layer, node.input(1), 1);
2159 if(biasEnabled && m_TensorsInfo[node.input(2)].isConstant())
2162 IConnectableLayer* biasLayer = m_Network->AddConstantLayer(CreateConstTensor(node.input(2)).first);
2163 TensorInfo biasInfo = *m_TensorsInfo[node.input(2)].m_info;
2170 std::string activationName =
"activation_" + node.input(2);
2172 activationDescriptor.
m_A = beta;
2173 activationDescriptor.
m_Function = ActivationFunction::Linear;
2174 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2181 auto actInfo = ComputeOutputInfo({ activationName }, actLayer, { biasInfo.
GetShape() });
2191 else if (biasEnabled)
2194 if (m_TensorsInfo[node.input(2)].m_info->GetNumDimensions() != 1)
2196 throw ParseException(fmt::format(
"The parser supports constant or non-constant with 1 dimension for "
2197 "Input C of Gemm. Input '{}' in '{}' is not supported '{}'",
2205 std::string activationName =
"activation_" + node.input(2);
2207 activationDescriptor.
m_A = beta;
2208 activationDescriptor.
m_Function = ActivationFunction::Linear;
2209 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2216 auto actInfo = ComputeOutputInfo({ activationName },
2218 { m_TensorsInfo[node.input(2)].m_info->GetShape() });
2221 RegisterInputSlot(actLayer, node.input(2), 0);
2225 RegisterInputSlot(layer, node.input(2), 2);
2230 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
2231 { input0Shape, input1Shape });
2234 RegisterOutputSlots(layer, {node.output(0)});
2237 void OnnxParserImpl::ParseGlobalAveragePool(
const onnx::NodeProto& node)
2243 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2247 IConnectableLayer* layer = m_Network->AddPooling2dLayer(desc, node.name().c_str());
2254 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {inputShape});
2259 RegisterInputSlots(layer, {node.input(0)});
2262 RegisterOutputSlots(layer, {node.output(0)});
2265 void OnnxParserImpl::ParseMaxPool(
const onnx::NodeProto& node)
2270 AddPoolingLayer(node, desc);
2273 void OnnxParserImpl::ParseShape(
const onnx::NodeProto& node)
2285 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2286 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {inputShape}, onnx::TensorProto::INT64);
2290 RegisterInputSlots(layer, {node.input(0)});
2293 RegisterOutputSlots(layer, {node.output(0)});
2296 void OnnxParserImpl::ParseReshape(
const onnx::NodeProto& node)
2302 m_TensorsInfo[node.input(0)].m_dtype,
2303 onnx::TensorProto::FLOAT);
2305 m_TensorsInfo[node.input(1)].m_dtype,
2306 onnx::TensorProto::INT64);
2308 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2310 std::vector<unsigned int> targetShape;
2311 if(m_TensorsInfo[node.input(1)].isConstant())
2313 unsigned int dims =
static_cast<unsigned int>(m_TensorsInfo[node.input(1)].m_tensor->int64_data_size());
2314 targetShape.reserve(dims);
2316 for(uint i = 0; i < dims; i++)
2318 int val =
CHECKED_INT32(m_TensorsInfo[node.input(1)].m_tensor->int64_data(
static_cast<int>(i)));
2319 targetShape[i]=
static_cast<unsigned int>(val);
2325 unsigned int dims = m_TensorsInfo[node.input(1)].m_info->GetNumDimensions();
2326 TensorShape shapes = m_TensorsInfo[node.input(1)].m_info->GetShape();
2327 if (dims != 1 || shapes[0] > 2)
2329 throw ParseException(fmt::format(
"Invalid input shape '{}' in Reshape layer '{}' {}",
2335 unsigned int numInputElements = m_TensorsInfo[node.input(0)].m_info->GetNumElements();
2338 targetShape = { numInputElements };
2340 else if (shapes[0] == 2)
2342 targetShape = { inputShape[0] , numInputElements / inputShape[0] };
2346 if(m_TensorsInfo[node.input(0)].isConstant())
2349 if(m_TensorsInfo.count(node.output(0)) == 0)
2351 m_TensorsInfo[node.output(0)] = OnnxTensor();
2353 m_TensorsInfo[node.output(0)].m_tensor =
2354 std::make_unique<onnx::TensorProto>(*m_TensorsInfo[node.input(0)].m_tensor);
2358 if(m_TensorsInfo.count(node.output(0)) == 0 || m_TensorsInfo[node.output(0)].m_info ==
nullptr)
2360 auto outInfo = ComputeReshapeInfo(
2361 TensorShape(
static_cast<unsigned int>(targetShape.size()), targetShape.data()),
2362 inputShape, node.output(0));
2363 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
2366 CreateReshapeLayer(node.input(0), node.output(0), node.name());
2370 void OnnxParserImpl::ParseUnsqueeze(
const onnx::NodeProto& node)
2375 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2376 std::vector<uint32_t> dims;
2377 if (node.input_size() == 1 && node.attribute_size() > 0)
2379 dims = ReadMandatoryNodeUint32ListAttribute(node,
"axes");
2384 m_TensorsInfo[node.input(1)].m_dtype,
2385 onnx::TensorProto::INT64);
2387 auto int64Axes = m_TensorsInfo[node.input(1)].m_tensor->int64_data().data();
2388 uint numDim = armnn::numeric_cast<uint>(m_TensorsInfo[node.input(1)].m_tensor->int64_data_size());
2390 for(uint i = 0; i < numDim; i++)
2393 dims.push_back(uint32Value);
2398 std::sort(dims.begin(), dims.end());
2400 std::vector<unsigned int> targetShape;
2406 targetShape.push_back(inputShape[i]);
2410 for(uint i = 0; i < dims.size(); i++)
2412 targetShape.insert(targetShape.begin() + armnn::numeric_cast<int>(dims[i]), 1);
2415 auto outInfo = ComputeReshapeInfo(
TensorShape(
static_cast<unsigned int>(targetShape.size()), targetShape.data()),
2416 inputShape, node.output(0), m_TensorsInfo[node.input(0)].m_info->GetDataType());
2417 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
2418 m_TensorsInfo[node.output(0)].m_dtype = m_TensorsInfo[node.input(0)].m_dtype;
2420 CreateReshapeLayer(node.input(0), node.output(0), node.name());
2423 void OnnxParserImpl::PrependForBroadcast(
const std::string& outputName,
2424 const std::string& input0,
2425 const std::string& input1)
2430 TensorShape input0Shape = m_TensorsInfo[input0].m_info->GetShape();
2431 TensorShape input1Shape = m_TensorsInfo[input1].m_info->GetShape();
2434 std::vector<uint32_t> newShape;
2437 newShape.push_back(1);
2442 newShape.push_back(input0Shape[dim]);
2444 outputTensorInfo.
SetShape(
TensorShape(
static_cast<unsigned int>(newShape.size()), newShape.data()));
2447 m_TensorsInfo[outputName] = OnnxTensor();
2448 m_TensorsInfo[outputName].m_info = std::make_unique<TensorInfo>(outputTensorInfo);
2451 if( ! m_TensorsInfo[input0].isConstant())
2453 CreateReshapeLayer(input0, outputName, fmt::format(
"Add:reshapeOf{}", input0));
2457 m_TensorsInfo[outputName].m_tensor = std::make_unique<onnx::TensorProto>(*m_TensorsInfo[input0].m_tensor);
2462 void OnnxParserImpl::SetupInputLayers()
2465 for(
int inputIndex = 0; inputIndex < m_Graph->input_size(); ++inputIndex)
2467 auto input = m_Graph->input(inputIndex);
2468 if (!m_TensorsInfo[input.name()].isConstant())
2472 TensorInfo tensorInfo = *m_TensorsInfo[input.name()].m_info;
2475 if (m_InputShapes.find(input.name()) == m_InputShapes.end())
2477 throw ParseException(fmt::format(
"The parser does not support dynamic tensor, "
2478 "please specify input shape for {}. {}",
2484 tensorInfo.
SetShape(m_InputShapes[input.name()]);
2485 m_TensorsInfo[input.name()].m_info = std::make_unique<TensorInfo>(tensorInfo);
2491 m_InputInfos[input.name()] = tensorInfo;
2493 RegisterOutputSlots(layer,{ input.name() });
2498 void OnnxParserImpl::SetupOutputLayers()
2500 if(m_Graph->output_size() == 0)
2505 for(
int outputIndex = 0; outputIndex < m_Graph->output_size(); ++outputIndex)
2509 m_Graph->output(outputIndex).name().c_str());
2511 RegisterInputSlots(layer, { m_Graph->output(outputIndex).name() });
2516 const std::string& tensorId,
2517 unsigned int slotIndex)
2521 auto it = m_TensorConnections.find(tensorId);
2523 if (it == m_TensorConnections.end())
2526 m_TensorConnections[tensorId] = TensorSlots();
2528 m_TensorConnections[tensorId].inputSlots.push_back(slot);
2531 void OnnxParserImpl::RegisterInputSlots(
IConnectableLayer* layer,
const std::vector<std::string>& tensorIds)
2541 fmt::format(
"The number of tensor inputs ({}) does not match the number expected ({}) {}",
2547 for (
unsigned int slotIndex = 0; slotIndex < layer->
GetNumInputSlots(); ++slotIndex)
2549 std::string tensorId = tensorIds[slotIndex];
2552 auto it = m_TensorConnections.find(tensorId);
2554 if (it == m_TensorConnections.end())
2557 m_TensorConnections[tensorId] = TensorSlots();
2559 m_TensorConnections[tensorId].inputSlots.push_back(slot);
2563 void OnnxParserImpl::RegisterOutputSlots(
IConnectableLayer* layer,
const std::vector<std::string>& tensorIds)
2573 fmt::format(
"The number of tensor outputs ({}) does not match the number expected ({}) {} ",
2579 for (
unsigned int slotIndex = 0; slotIndex < layer->
GetNumOutputSlots(); ++slotIndex)
2581 std::string tensorId = tensorIds[slotIndex];
2584 auto it = m_TensorConnections.find(tensorId);
2586 if (it == m_TensorConnections.end())
2589 m_TensorConnections[tensorId] = TensorSlots();
2592 TensorSlots& tensorSlots = m_TensorConnections[tensorId];
2595 if (tensorSlots.outputSlot !=
nullptr)
2597 throw ParseException(fmt::format(
"Another layer has already registered itself as the producer of "
2602 tensorSlots.outputSlot = slot;
2609 for(
int i = 0; i < m_Graph->input_size(); ++i)
2611 auto input = m_Graph->input(i);
2612 if(input.name() == name)
2614 auto it = m_InputInfos.find(name);
2616 if (it != m_InputInfos.end())
2628 for(
int i = 0; i < m_Graph->output_size(); ++i)
2630 auto output = m_Graph->output(i);
2631 if(output.name() == name)
2633 auto it = m_OutputInfos.find(name);
2635 if (it != m_OutputInfos.end())
2647 if(model ==
nullptr) {
2652 std::vector<std::string> inputNames;
2653 std::map<std::string, bool> isConstant;
2654 for(
auto tensor : model->graph().initializer())
2656 isConstant[tensor.name()] =
true;
2658 for(
auto input : model->graph().input())
2660 auto it = isConstant.find(input.name());
2661 if(it == isConstant.end())
2663 inputNames.push_back(input.name());
2671 if(model ==
nullptr) {
2676 std::vector<std::string> outputNames;
2677 for(
auto output : model->graph().output())
2679 outputNames.push_back(output.name());