15 #include <fmt/format.h>
17 #include <google/protobuf/text_format.h>
18 #include <google/protobuf/io/zero_copy_stream_impl.h>
24 using namespace armnn;
29 IOnnxParser::IOnnxParser() : pOnnxParserImpl(new OnnxParserImpl()) {}
31 IOnnxParser::~IOnnxParser() =
default;
53 armnn::INetworkPtr IOnnxParser::CreateNetworkFromBinary(
const std::vector<uint8_t>& binaryContent)
55 return pOnnxParserImpl->CreateNetworkFromBinary(binaryContent);
58 armnn::INetworkPtr IOnnxParser::CreateNetworkFromBinary(
const std::vector<uint8_t>& binaryContent,
59 const std::map<std::string, armnn::TensorShape>& inputShapes)
61 return pOnnxParserImpl->CreateNetworkFromBinary(binaryContent, inputShapes);
66 return pOnnxParserImpl->CreateNetworkFromTextFile(graphFile);
71 return pOnnxParserImpl->CreateNetworkFromString(protoText);
75 const char* graphFile,
76 const std::map<std::string, armnn::TensorShape>& inputShapes)
78 return pOnnxParserImpl->CreateNetworkFromBinaryFile(graphFile, inputShapes);
82 const std::map<std::string, armnn::TensorShape>& inputShapes)
84 return pOnnxParserImpl->CreateNetworkFromTextFile(graphFile, inputShapes);
88 const std::map<std::string, armnn::TensorShape>& inputShapes)
90 return pOnnxParserImpl->CreateNetworkFromString(protoText, inputShapes);
93 BindingPointInfo IOnnxParser::GetNetworkInputBindingInfo(
const std::string& name)
const
95 return pOnnxParserImpl->GetNetworkInputBindingInfo(name);
98 BindingPointInfo IOnnxParser::GetNetworkOutputBindingInfo(
const std::string& name)
const
100 return pOnnxParserImpl->GetNetworkOutputBindingInfo(name);
105 void CheckValidDataType(std::initializer_list<onnx::TensorProto::DataType> validInputTypes,
107 const char* validExpr,
108 std::string nodeName,
109 std::string tensorName,
112 bool isValid = std::any_of(validInputTypes.begin(),
113 validInputTypes.end(),
118 fmt::format(
"Datatype {} is not valid for tensor '{}' of node '{}', not in {{{}}}. {}",
119 onnx::TensorProto::DataType_Name(actualValue),
127 #define CHECK_VALID_DATATYPE(NODE, TENSOR, ACTUAL, ...) \
128 CheckValidDataType({__VA_ARGS__}, ACTUAL, #__VA_ARGS__, NODE, TENSOR, CHECK_LOCATION())
130 using StrTypeListPair = std::pair<const char*, std::initializer_list<onnx::TensorProto::DataType>>;
131 #define STR_LIST(...) StrTypeListPair(#__VA_ARGS__, {__VA_ARGS__})
133 template <
typename Callable>
134 void ReadMandatoryNodeAttributeImpl(
const onnx::NodeProto& node,
135 const std::string& attribName,
136 onnx::AttributeProto::AttributeType expectedType,
139 auto attribs = node.attribute();
141 while (attriNum < node.attribute_size())
143 if (attribs.Get(attriNum).name() == attribName)
145 if (attribs.Get(attriNum).type() == expectedType)
147 callable(attribs.Get(attriNum));
151 throw ParseException(fmt::format(
"Attribute {} of node {} expected to have {} as "
152 "onnx::AttributeProto::AttributeType, but found {} instead {}",
155 onnx::AttributeProto::AttributeType_Name(expectedType),
156 onnx::AttributeProto::AttributeType_Name(attribs.Get(attriNum).type()),
163 if (attriNum == node.attribute_size())
165 throw ParseException(fmt::format(
"Could not find required attribute {} in node {} {}",
170 template <
typename Callable>
171 void ReadOptionalNodeAttributeImpl(
const onnx::NodeProto& node,
172 const std::string& attribName,
173 onnx::AttributeProto::AttributeType expectedType,
176 auto attribs = node.attribute();
177 for (
int attriNum = 0; attriNum < node.attribute_size(); ++attriNum)
179 if (attribs.Get(attriNum).name() == attribName)
181 if (attribs.Get(attriNum).type() == expectedType)
183 callable(attribs.Get(attriNum));
188 fmt::format(
"Attribute {} of node {} expected to have {} as onnx::AttributeProto::AttributeType, "
189 "but found {} instead {}",
192 onnx::AttributeProto::AttributeType_Name(expectedType),
193 onnx::AttributeProto::AttributeType_Name(attribs.Get(attriNum).type()),
200 int ReadMandatoryNodeIntAttribute(
const onnx::NodeProto& node,
201 const std::string& name)
204 ReadMandatoryNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
205 [&attribValue](
const onnx::AttributeProto& attrValue)
212 int64_t ReadOptionalNodeInt64Attribute(
const onnx::NodeProto& node,
213 const std::string& name,
214 const int64_t defaultValue = 0)
216 int64_t attribValue = defaultValue;
217 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
218 [&attribValue](
const onnx::AttributeProto& attrValue)
220 attribValue = attrValue.i();
225 std::vector<uint32_t> ReadMandatoryNodeUint32ListAttribute(
const onnx::NodeProto& node,
226 const std::string& name)
228 std::vector<uint32_t> attriList;
229 ReadMandatoryNodeAttributeImpl(node, name, onnx::AttributeProto::INTS,
230 [&attriList](
const onnx::AttributeProto& attrValue)
232 for (
int attriNum = 0; attriNum < attrValue.ints_size(); ++attriNum)
240 uint32_t ReadOptionalNodeUint32Attribute(
const onnx::NodeProto& node,
241 const std::string& name,
242 const uint32_t defaultVal = 0u)
244 uint32_t attribValue = defaultVal;
245 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
246 [&attribValue](
const onnx::AttributeProto& attrValue)
253 std::vector<uint32_t> ReadOptionalNodeUint32ListAttribute(
const onnx::NodeProto& node,
254 const std::string& name)
256 std::vector<uint32_t> attriList;
257 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INTS,
258 [&attriList](
const onnx::AttributeProto& attrValue)
260 for (
int attriNum = 0; attriNum < attrValue.ints_size(); ++attriNum)
269 float ReadOptionalNodeFloatAttribute(
const onnx::NodeProto& node,
270 const std::string& name,
271 const float defaultValue = 0.0f)
273 float attribValue = defaultValue;
274 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::FLOAT,
275 [&attribValue](
const onnx::AttributeProto& attrValue)
277 attribValue = attrValue.f();
282 std::string ReadOptionalNodeStringAttribute(
const onnx::NodeProto& node,
const std::string& name)
284 std::string attribValue =
"";
285 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::STRING,
286 [&attribValue](
const onnx::AttributeProto& attrValue)
288 attribValue = attrValue.s();
298 case onnx::TensorProto::FLOAT:
300 type = DataType::Float32;
303 case onnx::TensorProto::INT32:
304 case onnx::TensorProto::INT64:
306 type = DataType::Signed32;
312 fmt::format(
"'{}' is not a currently supported datatype for tensor {}."
313 " Supported dataTypes are FLOAT, INT32 and INT64. {}",
327 if(std::find(shape.begin(), shape.end(), 0) != shape.end())
337 const onnx::TensorShapeProto onnxShape =
info.type().tensor_type().shape();
338 std::vector<unsigned int> shapeDims;
339 for (
int i = 0; i < onnxShape.dim_size(); ++i)
349 std::vector<unsigned int> shapeDims;
351 for (
auto dim: tensor.dims())
356 return ToTensorInfo(tensor.name(), shapeDims, tensor.data_type());
359 std::string TensorInfoAsString(
const TensorInfo& info,
360 const std::string& name,
364 std::stringstream ss;
365 ss <<
"tensor '" << name <<
"' contains "
366 << onnx::TensorProto::DataType_Name(type)
367 <<
" and has shape [";
371 ss << shape[i] <<
", ";
377 void CalcPadding(uint32_t inputSize,
381 uint32_t* paddingFront,
382 uint32_t* paddingBack,
385 uint32_t outputSize = (inputSize + stride - 1) / stride;
386 uint32_t dilatedSize = filterSize + (dilation - 1) * (filterSize - 1);
387 uint32_t temp = (outputSize - 1) * stride + dilatedSize;
388 *paddingFront = (temp - inputSize) / 2;
389 *paddingBack = *paddingFront;
390 if((temp - inputSize) % 2 == 1)
405 const std::string& outName,
406 DataType dataType = DataType::Float32)
408 std::vector<int> targetDims;
414 targetDims.push_back(
static_cast<int>(inShape[
static_cast<uint
>(i)]));
418 targetDims.push_back(val);
422 std::vector<unsigned int> outDims(targetDims.begin(), targetDims.end());
423 const auto stretchDim = std::find(targetDims.begin(), targetDims.end(), -1);
424 if (stretchDim != targetDims.end())
426 if (std::find(std::next(stretchDim), targetDims.end(), -1) != targetDims.end())
428 std::stringstream ss;
430 for(uint i = 0; i < targetDims.size() - 1; ++i)
432 ss << targetDims[i] <<
", ";
434 ss << targetDims[targetDims.size() - 1] <<
" ]";
437 fmt::format(
"Error during creation of reshaped tensor '{}'. At most one component of shape can be "
438 " -1 and here, shape is {} {}",
444 auto targetNumElements = armnn::numeric_cast<unsigned int>(std::accumulate(targetDims.begin(), targetDims.end(),
445 -1, std::multiplies<int32_t>()));
446 auto stretchIndex =
static_cast<size_t>(std::distance(targetDims.begin(), stretchDim));
447 outDims[stretchIndex] = inShape.
GetNumElements() / targetNumElements;
455 const std::map<std::string, OnnxParserImpl::OperationParsingFunction> OnnxParserImpl::m_ParserFunctions = {
456 {
"BatchNormalization", &OnnxParserImpl::ParseBatchNormalization},
457 {
"GlobalAveragePool", &OnnxParserImpl::ParseGlobalAveragePool},
458 {
"AveragePool", &OnnxParserImpl::ParseAveragePool },
459 {
"Clip", &OnnxParserImpl::ParseClip },
460 {
"Constant", &OnnxParserImpl::ParseConstant },
461 {
"MaxPool", &OnnxParserImpl::ParseMaxPool },
462 {
"Reshape", &OnnxParserImpl::ParseReshape },
463 {
"Sigmoid", &OnnxParserImpl::ParseSigmoid },
464 {
"Tanh", &OnnxParserImpl::ParseTanh },
465 {
"Relu", &OnnxParserImpl::ParseRelu },
466 {
"LeakyRelu", &OnnxParserImpl::ParseLeakyRelu },
467 {
"Conv", &OnnxParserImpl::ParseConv },
468 {
"Add", &OnnxParserImpl::ParseAdd },
469 {
"Flatten", &OnnxParserImpl::ParseFlatten },
470 {
"Shape", &OnnxParserImpl::ParseShape },
471 {
"Gather", &OnnxParserImpl::ParseGather },
472 {
"Unsqueeze", &OnnxParserImpl::ParseUnsqueeze },
473 {
"Concat", &OnnxParserImpl::ParseConcat },
474 {
"Gemm", &OnnxParserImpl::ParseGemm }
477 template<
typename TypePair,
typename Location>
478 void OnnxParserImpl::ValidateInputs(
const onnx::NodeProto& node,
479 TypePair validInputs,
480 const Location& location)
482 for(
auto input : node.input())
484 CheckValidDataType(validInputs.second,
485 m_TensorsInfo[input].m_dtype,
493 #define VALID_INPUTS(NODE, VALID_INPUTS) \
494 OnnxParserImpl::ValidateInputs(NODE, \
498 std::vector<TensorInfo> OnnxParserImpl::ComputeOutputInfo(std::vector<std::string> outNames,
500 std::vector<TensorShape> inputShapes,
504 bool needCompute = std::any_of(outNames.begin(),
506 [
this](std::string name)
508 return (m_TensorsInfo.count(name) == 0 || m_TensorsInfo[name].m_info == nullptr
509 || m_TensorsInfo[name].m_info->GetShape().GetDimensionality() ==
510 Dimensionality::NotSpecified);
512 std::vector<TensorInfo> outInfo;
514 std::vector<TensorShape> inferredShapes;
515 DataType armnnType = DataType::Float32;
520 case onnx::TensorProto::FLOAT: {
521 armnnType = DataType::Float32;
524 case onnx::TensorProto::INT32:
525 case onnx::TensorProto::INT64: {
526 armnnType = DataType::Signed32;
531 fmt::format(
"'{}' is not a currently supported datatype for {}."
532 " Supported dataTypes are FLOAT, INT32 and INT64. {}",
539 for (uint i = 0; i < outNames.size(); ++i)
543 m_TensorsInfo[outNames[i]] = OnnxTensor();
544 m_TensorsInfo[outNames[i]].m_info = std::make_unique<TensorInfo>(
546 m_TensorsInfo[outNames[i]].m_dtype = dataType;
548 outInfo.push_back(*m_TensorsInfo[outNames[i]].m_info);
553 OnnxParserImpl::OnnxParserImpl()
554 : m_Network(nullptr, nullptr)
558 void OnnxParserImpl::ResetParser()
562 m_InputInfos.clear();
563 m_OutputInfos.clear();
566 void OnnxParserImpl::Cleanup()
568 m_TensorConnections.clear();
569 m_TensorsInfo.clear();
570 m_OutputsMap.clear();
571 m_OutputsFusedAndUsed.clear();
572 m_InputShapes.clear();
576 std::pair<armnn::ConstTensor, std::unique_ptr<T[]>>
581 ARMNN_ASSERT_MSG(bufferPtr !=
nullptr, fmt::format(
"Buffer for permutation is null").c_str());
589 reinterpret_cast<const T*
>(bufferPtr), data.get(),
sizeof(T));
593 ::memcpy(data.get(), bufferPtr, tensorInfo.
GetNumBytes());
596 return std::make_pair(
ConstTensor(tensorInfo, data.get()), std::move(data));
599 std::pair<ConstTensor, std::unique_ptr<float[]>>
600 OnnxParserImpl::CreateConstTensor(
const std::string name,
603 TensorInfo tensorInfo = *m_TensorsInfo[name].m_info;
604 onnx::TensorProto onnxTensor = *m_TensorsInfo[name].m_tensor;
616 throw ParseException(fmt::format(
"No tensor data found for Const tensor '{}' {}",
621 auto srcData = onnxTensor.float_data().data();
623 if (!onnxTensor.has_raw_data())
625 if(tensorInfo.
GetNumElements() !=
static_cast<uint
>(onnxTensor.float_data_size()))
628 fmt::format(
"The number of data provided ({}) does not match the tensor '{}' number of "
630 onnxTensor.float_data_size(),
635 return CreateConstTensorImpl<float>(srcData, tensorInfo, permutationVector);
639 return CreateConstTensorImpl<float>(
reinterpret_cast<const float*
>(onnxTensor.raw_data().c_str()),
645 std::pair<ConstTensor, std::unique_ptr<int32_t[]>>
646 OnnxParserImpl::CreateInt64ConstTensor(
const std::string name,
649 TensorInfo tensorInfo = *m_TensorsInfo[name].m_info;
650 onnx::TensorProto onnxTensor = *m_TensorsInfo[name].m_tensor;
660 if (numElements == 0)
662 throw ParseException(fmt::format(
"No tensor data found for Const tensor '{}' {}",
668 if (!onnxTensor.has_raw_data())
670 auto srcData = onnxTensor.int64_data().data();
671 if(numElements !=
static_cast<uint
>(onnxTensor.int64_data_size()))
674 fmt::format(
"The number of data provided ({}) does not match the tensor '{}' number of "
676 onnxTensor.int64_data_size(),
682 std::vector<int32_t> int32Data;
683 for(uint i = 0; i < numElements; i++)
686 int32Data.push_back(int32Value);
689 return CreateConstTensorImpl<int32_t>(int32Data.data(), tensorInfo, permutationVector);
693 auto srcData =
reinterpret_cast<const int64_t*
>(onnxTensor.raw_data().c_str());
694 std::vector<int32_t> int32Data;
695 for(uint i = 0; i < numElements; i++)
698 int32Data.push_back(int32Value);
700 return CreateConstTensorImpl<int32_t>(int32Data.data(), tensorInfo, permutationVector);
706 FILE* fd = fopen(graphFile,
"r");
714 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
715 using google::protobuf::io::FileInputStream;
716 std::unique_ptr<FileInputStream> input = std::make_unique<FileInputStream>(fileno(fd));
717 bool success = google::protobuf::TextFormat::Parse(input.get(), modelProto.get());
722 std::stringstream
error;
723 error <<
"Failed to parse graph file";
733 return CreateNetworkFromModel(*modelProto);
737 const std::map<std::string, armnn::TensorShape>& inputShapes)
740 m_InputShapes = inputShapes;
742 return CreateNetworkFromModel(*modelProto);
749 return CreateNetworkFromModel(*modelProto);
753 const std::map<std::string, armnn::TensorShape>& inputShapes)
756 m_InputShapes = inputShapes;
758 return CreateNetworkFromModel(*modelProto);
763 if (binaryContent.size() == 0)
768 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
770 google::protobuf::io::CodedInputStream codedStream(binaryContent.data(),
static_cast<int>(binaryContent.size()));
771 codedStream.SetTotalBytesLimit(INT_MAX);
772 bool success = modelProto.get()->ParseFromCodedStream(&codedStream);
776 std::stringstream
error;
777 error <<
"Failed to parse graph";
785 FILE* fd = fopen(graphFile,
"rb");
793 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
795 google::protobuf::io::FileInputStream inStream(fileno(fd));
796 google::protobuf::io::CodedInputStream codedStream(&inStream);
797 codedStream.SetTotalBytesLimit(INT_MAX);
798 bool success = modelProto.get()->ParseFromCodedStream(&codedStream);
803 std::stringstream
error;
804 error <<
"Failed to parse graph file";
815 return CreateNetworkFromModel(*modelProto);
819 const std::map<std::string, armnn::TensorShape>& inputShapes)
822 m_InputShapes = inputShapes;
824 return CreateNetworkFromModel(*modelProto);
835 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
836 bool success = google::protobuf::TextFormat::ParseFromString(protoText, modelProto.get());
839 std::stringstream
error;
840 error <<
"Failed to parse graph file";
850 return CreateNetworkFromModel(*modelProto);
854 const std::map<std::string, armnn::TensorShape>& inputShapes)
857 m_InputShapes = inputShapes;
859 return CreateNetworkFromModel(*modelProto);
862 INetworkPtr OnnxParserImpl::CreateNetworkFromModel(onnx::ModelProto& model)
864 m_Network = INetwork::Create();
867 m_Graph = std::make_unique<onnx::GraphProto>(*model.mutable_graph());
876 return std::move(m_Network);
879 void OnnxParserImpl::LoadGraph()
884 SetupInfo(m_Graph->mutable_output());
885 SetupInfo(m_Graph->mutable_input());
886 SetupInfo(m_Graph->mutable_value_info());
888 for (
auto tensor : m_Graph->initializer())
890 m_TensorsInfo[tensor.name()].m_tensor = std::make_unique<const onnx::TensorProto>(tensor);
891 m_TensorsInfo[tensor.name()].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(tensor));
892 m_TensorsInfo[tensor.name()].m_dtype =
900 DetectFullyConnected();
903 for(
size_t nodeIndex = 0; nodeIndex < static_cast<size_t>(m_Graph->node_size()); nodeIndex++)
905 auto node = m_Graph->node(
static_cast<int>(nodeIndex));
906 const std::string& operation = node.op_type();
909 if (operation ==
"MatMul" )
911 if(m_OutputsFusedAndUsed[nodeIndex].inputForNodes != m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.size())
914 AddFullyConnected(node);
917 else if (!(m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.empty()) && operation ==
"Add")
919 int matmulIndex =
static_cast<int> (m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes[0]);
920 AddFullyConnected(m_Graph->node(matmulIndex), &node);
922 else if (m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.empty())
924 auto it = m_ParserFunctions.find(operation);
925 if (it != m_ParserFunctions.end())
927 auto func = it->second;
932 throw ParseException(fmt::format(
"Unsupported operation {} for node '{}' {}",
941 for (
const auto& tensorCon : m_TensorConnections)
943 if (tensorCon.second.outputSlot !=
nullptr)
945 for (
size_t inputSlotIdx = 0; inputSlotIdx < tensorCon.second.inputSlots.size(); ++inputSlotIdx)
947 tensorCon.second.outputSlot->Connect(*(tensorCon.second.inputSlots[inputSlotIdx]));
953 for(
int outputIndex = 0; outputIndex < m_Graph->output_size(); ++outputIndex)
955 auto output = m_Graph->output(outputIndex);
956 m_OutputInfos[output.name()] = *m_TensorsInfo[output.name()].m_info;
960 void OnnxParserImpl::SetupInfo(
const google::protobuf::RepeatedPtrField<onnx::ValueInfoProto >* list)
962 for (
auto tensor : *list)
964 m_TensorsInfo[tensor.name()] = OnnxTensor();
965 m_TensorsInfo[tensor.name()].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(tensor));
966 m_TensorsInfo[tensor.name()].m_dtype =
971 void OnnxParserImpl::DetectFullyConnected()
973 m_OutputsFusedAndUsed = std::vector<UsageSummary> (
static_cast<size_t>(m_Graph->node_size()), UsageSummary());
974 auto matmulAndConstant = [&](
const std::string& constInput,
975 const std::string& matmulInput,
978 auto matmulIt = m_OutputsMap.find(matmulInput);
979 if(matmulIt != m_OutputsMap.end() && matmulIt->second.first->op_type() ==
"MatMul"
980 && m_TensorsInfo[constInput].isConstant())
982 nodeIndex = matmulIt->second.second;
988 for(
int nodeIndex = 0; nodeIndex < m_Graph->node_size(); nodeIndex++)
990 const onnx::NodeProto* node = &m_Graph->node(nodeIndex);
991 for (
const std::string& output : node->output())
993 m_OutputsMap[output] = std::make_pair(node, nodeIndex);
996 for (
const std::string& input : node->input())
998 auto matmulIt = m_OutputsMap.find(input);
999 if(matmulIt != m_OutputsMap.end()){
1000 ++m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIt->second.second)].inputForNodes;
1004 if (node->op_type() ==
"Add")
1006 int matmulIndex = 0;
1007 if (matmulAndConstant(node->input(0), node->input(1), matmulIndex) ||
1008 matmulAndConstant(node->input(1), node->input(0), matmulIndex))
1011 m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIndex)].fusedWithNodes
1012 .push_back(
static_cast<size_t>(nodeIndex));
1014 m_OutputsFusedAndUsed[
static_cast<size_t>(nodeIndex)].fusedWithNodes
1015 .push_back(
static_cast<size_t>(matmulIndex));
1020 for (
auto output: m_Graph->output()) {
1021 auto matmulIt = m_OutputsMap.find(output.name());
1022 if(matmulIt != m_OutputsMap.end()){
1023 ++m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIt->second.second)].inputForNodes;
1028 template<
typename Location>
1029 void OnnxParserImpl::GetInputAndParam(
const onnx::NodeProto& node,
1030 std::string* inputName,
1031 std::string* constName,
1032 const Location& location)
1035 if (m_TensorsInfo[node.input(0)].isConstant())
1039 else if (m_TensorsInfo[node.input(1)].isConstant())
1045 throw ParseException(fmt::format(
"One of the input tensors ('{}' or '{}') should be constant in node '{}' {}",
1049 location.AsString()));
1053 *constName = node.input(cstIndex);
1057 *inputName = node.input(!cstIndex);
1061 template<
typename Location>
1062 void OnnxParserImpl::To1DTensor(
const std::string& name,
const Location& location)
1064 TensorShape shape = m_TensorsInfo[name].m_info->GetShape();
1065 std::vector<uint32_t> newShape;
1071 fmt::format(
"Only tensors with shape [1, ..., 1, X] can be converted to 1D and {} {}",
1072 TensorInfoAsString(*m_TensorsInfo[name].m_info, name, m_TensorsInfo[name].m_dtype),
1073 location.AsString()));
1078 m_TensorsInfo[name].m_info->SetShape(
TensorShape(
static_cast<unsigned int>(newShape.size()), newShape.data()));
1081 void OnnxParserImpl::AddConvLayerWithDepthwiseConv(
const onnx::NodeProto& node,
const Convolution2dDescriptor& convDesc)
1095 std::string permuteStr =
"permute_" + node.input(1);
1096 std::vector<std::string> tensorIndexes= {node.input(0), permuteStr};
1098 auto weightTensor = CreateConstTensor(node.input(1));
1099 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(weightTensor.first);
1114 if (node.input_size() == 3)
1116 if(!m_TensorsInfo[node.input(2)].isConstant())
1118 throw ParseException(fmt::format(
"Bias '{}' should be constant in Conv layer '{}' {}",
1125 auto biasTensor = CreateConstTensor(node.input(2));
1126 tensorIndexes.emplace_back(node.input(2));
1135 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1136 { m_TensorsInfo[node.input(0)].m_info->GetShape(),
1143 RegisterInputSlots(layer, tensorIndexes);
1146 RegisterOutputSlots(layer, {node.output(0)});
1149 void OnnxParserImpl::AddFullyConnected(
const onnx::NodeProto& matmulNode,
const onnx::NodeProto* addNode)
1153 std::string weightName;
1154 std::string inputName;
1159 GetInputAndParam(matmulNode, &inputName, &weightName,
CHECK_LOCATION());
1168 std::string biasName;
1173 GetInputAndParam(*addNode,
nullptr, &biasName,
CHECK_LOCATION());
1177 TensorInfo weightInfo = *m_TensorsInfo[weightName].m_info;
1178 TensorInfo biasInfo = *m_TensorsInfo[biasName].m_info;
1183 fmt::format(
"Shape of weights '{}' and bias of following Add node '{}' do not match : {}"
1184 " and {} ( /!\\ bias should be a 1D tensor) {}",
1187 TensorInfoAsString(*m_TensorsInfo[weightName].m_info, weightName,
1188 m_TensorsInfo[weightName].m_dtype),
1189 TensorInfoAsString(*m_TensorsInfo[biasName].m_info, biasName,
1190 m_TensorsInfo[biasName].m_dtype ),
1195 layer = m_Network->AddFullyConnectedLayer(desc, matmulNode.name().c_str());
1198 auto outputInfo = ComputeOutputInfo({addNode->output(0)}, layer,
1199 {m_TensorsInfo[inputName].m_info->GetShape(),
1200 m_TensorsInfo[weightName].m_info->GetShape()});
1204 if(m_TensorsInfo[weightName].isConstant())
1206 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(CreateConstTensor(weightName).first);
1213 if(m_TensorsInfo[biasName].isConstant())
1215 IConnectableLayer* biasLayer = m_Network->AddConstantLayer(CreateConstTensor(biasName).first);
1222 RegisterInputSlots(layer, {inputName, weightName, biasName});
1223 RegisterOutputSlots(layer, {addNode->output(0)});
1227 layer = m_Network->AddFullyConnectedLayer(desc, matmulNode.name().c_str());
1230 auto outputInfo = ComputeOutputInfo({matmulNode.output(0)}, layer,
1231 {m_TensorsInfo[inputName].m_info->GetShape(),
1232 m_TensorsInfo[weightName].m_info->GetShape()});
1236 if(m_TensorsInfo[weightName].isConstant())
1238 TensorInfo weightInfo = *m_TensorsInfo[weightName].m_info;
1239 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(CreateConstTensor(weightName).first);
1246 RegisterInputSlots(layer, {inputName, weightName});
1247 RegisterOutputSlots(layer, {matmulNode.output(0)});
1251 void OnnxParserImpl::AddPoolingLayer(
const onnx::NodeProto& node,
Pooling2dDescriptor& desc)
1259 std::vector<uint32_t> kernel_shape = ReadMandatoryNodeUint32ListAttribute(node,
"kernel_shape");
1260 std::vector<uint32_t> strides = ReadOptionalNodeUint32ListAttribute(node,
"strides");
1261 std::vector<uint32_t> pads = ReadOptionalNodeUint32ListAttribute(node,
"pads");
1282 std::string paddingString = ReadOptionalNodeStringAttribute(node,
"auto_pad");
1283 if(paddingString !=
"VALID" && paddingString !=
"" && paddingString !=
"NOTSET")
1286 if( paddingString ==
"SAME_LOWER")
1290 else if (paddingString ==
"SAME_UPPER")
1296 throw ParseException(fmt::format(
"Invalid auto_pad attribute for node {}. "
1297 "Only SAME_UPPER, SAME_LOWER or VALID supported and found {} {}",
1302 auto inputInfo = *m_TensorsInfo[node.input(0)].m_info;
1303 uint32_t inputHeight = inputInfo.GetShape()[2];
1304 uint32_t inputWidth = inputInfo.GetShape()[3];
1305 CalcPadding(inputHeight,
1312 CalcPadding(inputWidth,
1329 IConnectableLayer* layer = m_Network->AddPooling2dLayer(desc, node.name().c_str());
1332 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1337 RegisterInputSlots(layer, {node.input(0)});
1340 RegisterOutputSlots(layer, {node.output(0)});
1343 std::pair<std::string, std::string> OnnxParserImpl::AddPrepareBroadcast(
const std::string& input0,
1344 const std::string& input1)
1346 std::pair<std::string, std::string> inputs = std::make_pair(input0, input1);
1348 TensorShape input0Shape = m_TensorsInfo[input0].m_info->GetShape();
1349 TensorShape input1Shape = m_TensorsInfo[input1].m_info->GetShape();
1353 auto outputName = fmt::format(
"reshape_output_{}", input1);
1354 PrependForBroadcast(outputName, input1, input0);
1355 inputs.second = outputName;
1359 auto outputName = fmt::format(
"reshape_output_{}", input0);
1360 PrependForBroadcast(outputName, input0, input1);
1361 inputs.first = outputName;
1366 void OnnxParserImpl::CreateConstantLayer(
const std::string& tensorName,
const std::string& layerName)
1368 auto armnnTensor = CreateConstTensor(tensorName);
1369 IConnectableLayer* layer = m_Network->AddConstantLayer(armnnTensor.first, layerName.c_str());
1371 RegisterOutputSlots(layer, {tensorName});
1374 void OnnxParserImpl::CreateInt64ConstantLayer(
const std::string& tensorName,
const std::string& layerName)
1376 auto armnnTensor = CreateInt64ConstTensor(tensorName);
1377 IConnectableLayer* layer = m_Network->AddConstantLayer(armnnTensor.first, layerName.c_str());
1379 RegisterOutputSlots(layer, {tensorName});
1382 void OnnxParserImpl::CreateReshapeLayer(
const std::string& inputName,
1383 const std::string& outputName,
1384 const std::string& layerName)
1386 const TensorInfo outputTensorInfo = *m_TensorsInfo[outputName].m_info;
1390 IConnectableLayer* layer = m_Network->AddReshapeLayer(reshapeDesc, layerName.c_str());
1396 RegisterInputSlots(layer, {inputName});
1399 RegisterOutputSlots(layer, {outputName});
1412 if (func == ActivationFunction::BoundedReLu)
1414 if (node.input_size() == 1 && node.attribute_size() > 0)
1416 desc.
m_A = ReadOptionalNodeFloatAttribute(node,
"max", std::numeric_limits<float>::max());
1417 desc.
m_B = ReadOptionalNodeFloatAttribute(node,
"min", std::numeric_limits<float>::lowest());
1421 desc.
m_A = node.input(2).empty() ? std::numeric_limits<float>::max() :
std::stof(node.input(2));
1422 desc.
m_B = node.input(1).empty() ? std::numeric_limits<float>::lowest() :
std::stof(node.input(1));
1426 IConnectableLayer*
const layer = m_Network->AddActivationLayer(desc, node.name().c_str());
1429 auto outputInfo = ComputeOutputInfo({ node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1434 RegisterInputSlots(layer, {node.input(0)});
1437 RegisterOutputSlots(layer, {node.output(0)});
1440 void OnnxParserImpl::ParseClip(
const onnx::NodeProto& node)
1442 ParseActivation(node, ActivationFunction::BoundedReLu);
1445 void OnnxParserImpl::ParseSigmoid(
const onnx::NodeProto& node)
1447 ParseActivation(node, ActivationFunction::Sigmoid);
1450 void OnnxParserImpl::ParseTanh(
const onnx::NodeProto& node)
1452 ParseActivation(node, ActivationFunction::TanH);
1455 void OnnxParserImpl::ParseRelu(
const onnx::NodeProto& node)
1457 ParseActivation(node, ActivationFunction::ReLu);
1460 void OnnxParserImpl::ParseLeakyRelu(
const onnx::NodeProto& node)
1462 ParseActivation(node, ActivationFunction::LeakyReLu);
1465 void OnnxParserImpl::ParseAdd(
const onnx::NodeProto& node)
1476 auto inputs = AddPrepareBroadcast(node.input(0), node.input(1));
1477 auto input0 = *m_TensorsInfo[inputs.first].m_info;
1478 auto input1 = *m_TensorsInfo[inputs.second].m_info;
1479 ARMNN_ASSERT(input0.GetNumDimensions() == input1.GetNumDimensions());
1481 unsigned int numDims = input0.GetNumDimensions();
1482 for (
unsigned int i = 0; i < numDims; i++)
1484 unsigned int dim0 = input0.GetShape()[i];
1485 unsigned int dim1 = input1.GetShape()[i];
1486 if (dim0 != dim1 && dim0 != 1 && dim1 != 1)
1489 fmt::format(
"Broadcast is only supported for scalar or 1D tensors in Add node '{}'. "
1490 "Input dimensions should either match or one should be of size 1 and here, "
1493 TensorInfoAsString(*m_TensorsInfo[inputs.first].m_info, inputs.first,
1494 m_TensorsInfo[inputs.first].m_dtype),
1495 TensorInfoAsString(*m_TensorsInfo[inputs.second].m_info, inputs.second,
1496 m_TensorsInfo[inputs.second].m_dtype),
1505 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1506 { m_TensorsInfo[inputs.first].m_info->GetShape(),
1507 m_TensorsInfo[inputs.second].m_info->GetShape() });
1511 if(m_TensorsInfo[inputs.first].isConstant()) {
1512 CreateConstantLayer(inputs.first, fmt::format(
"Add:constant_of_{}", node.input(0)));
1514 if(m_TensorsInfo[inputs.second].isConstant()) {
1515 CreateConstantLayer(inputs.second, fmt::format(
"Add:constant_of_{}", node.input(1)));
1517 RegisterInputSlots(layer, {inputs.first, inputs.second});
1520 RegisterOutputSlots(layer, {node.output(0)});
1523 void OnnxParserImpl::ParseAveragePool(
const onnx::NodeProto& node)
1528 uint32_t count_include_pad = 0;
1529 count_include_pad = ReadOptionalNodeUint32Attribute(node,
"count_include_pad");
1530 if(count_include_pad) {
1533 AddPoolingLayer(node, desc);
1536 void OnnxParserImpl::ParseBatchNormalization(
const onnx::NodeProto& node)
1544 for(
int ind = 1; ind < node.input_size(); ++ind)
1546 auto tensor = node.input(ind);
1547 if(! m_TensorsInfo[tensor].isConstant())
1550 fmt::format(
"Input tensor '{}' should be constant in BatchNormalization node '{}' {}",
1557 float epsilon = ReadOptionalNodeFloatAttribute(node,
"epsilon", 1e-5f);
1559 desc.
m_Eps = epsilon;
1561 auto scaleTensor = CreateConstTensor(node.input(1));
1562 auto biasTensor = CreateConstTensor(node.input(2));
1563 auto meanTensor = CreateConstTensor(node.input(3));
1564 auto varTensor = CreateConstTensor(node.input(4));
1571 node.name().c_str());
1574 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1577 RegisterInputSlots(layer, {node.input(0)});
1580 RegisterOutputSlots(layer, {node.output(0)});
1583 void OnnxParserImpl::ParseConcat(
const onnx::NodeProto& node)
1587 uint32_t numConcatView =
static_cast<uint32_t
>(node.input_size());
1588 uint32_t inputRank = m_TensorsInfo[node.input(0)].m_info->GetNumDimensions();
1590 int axisInt = ReadMandatoryNodeIntAttribute(node,
"axis");
1592 unsigned int concatDimInput =
static_cast<unsigned int>(
1593 (
static_cast<int>(inputRank) + axisInt) %
static_cast<int>(inputRank));
1596 concatDescriptor.SetConcatAxis(concatDimInput);
1598 unsigned int mergeDimOrigin = 0;
1600 std::vector<TensorShape> inputShapes;
1601 std::vector<std::string> tensorIds;
1603 for (
unsigned int viewIndex = 0; viewIndex < numConcatView; ++viewIndex)
1605 std::string nodeName = node.input(
static_cast<int>(viewIndex));
1606 auto inputTensorInfo = *m_TensorsInfo[nodeName].m_info;
1607 inputShapes.push_back(inputTensorInfo.GetShape());
1608 tensorIds.push_back(nodeName);
1612 inputTensorInfo, concatDescriptor, concatDimInput, viewIndex, mergeDimOrigin);
1615 IConnectableLayer* layer = m_Network->AddConcatLayer(concatDescriptor, node.name().c_str());
1618 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, inputShapes,
1619 m_TensorsInfo[node.input(0)].m_dtype);
1624 RegisterInputSlots(layer, tensorIds);
1627 RegisterOutputSlots(layer, { node.output(0) });
1630 void OnnxParserImpl::ParseConstant(
const onnx::NodeProto& node)
1633 if (!node.attribute(0).has_t())
1635 throw ParseException(fmt::format(
"Value not found for Constant node '{}' {}",
1639 const onnx::TensorProto& onnxTensor = node.attribute(0).t();
1642 m_TensorsInfo[node.output(0)].m_tensor = std::make_unique<const onnx::TensorProto>(onnxTensor);
1643 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(onnxTensor));
1646 if (m_TensorsInfo[node.output(0)].m_dtype == onnx::TensorProto_DataType_FLOAT)
1648 CreateConstantLayer(node.output(0), node.name());
1650 else if (m_TensorsInfo[node.output(0)].m_dtype == onnx::TensorProto_DataType_INT64)
1652 CreateInt64ConstantLayer(node.output(0), node.name());
1656 throw ParseException(fmt::format(
"Data type not support for Constant node '{}' {}",
1662 void OnnxParserImpl::ParseConv(
const onnx::NodeProto& node)
1669 if(m_TensorsInfo[node.input(0)].m_info->GetNumDimensions() != 4)
1672 fmt::format(
"ArmNN only supports 2D convolution and Conv layer '{}' input {} {}",
1674 TensorInfoAsString(*m_TensorsInfo[node.input(0)].m_info, node.input(0),
1675 m_TensorsInfo[node.input(0)].m_dtype),
1679 if(!m_TensorsInfo[node.input(1)].isConstant())
1682 fmt::format(
"Weights '{}' should be constant in Conv layer '{}' {}",
1688 auto inputInfo = *m_TensorsInfo[node.input(0)].m_info;
1693 std::vector<uint32_t> strides = ReadOptionalNodeUint32ListAttribute(node,
"strides");
1705 std::vector<uint32_t> dilations = ReadOptionalNodeUint32ListAttribute(node,
"dilations");
1706 if(!dilations.empty())
1712 std::vector<uint32_t> pads = ReadOptionalNodeUint32ListAttribute(node,
"pads");
1717 std::string paddingString = ReadOptionalNodeStringAttribute(node,
"auto_pad");
1718 if(paddingString !=
"VALID" && paddingString !=
"" && paddingString !=
"NOTSET")
1721 if( paddingString ==
"SAME_LOWER")
1725 else if (paddingString ==
"SAME_UPPER")
1732 fmt::format(
"Invalid auto_pad attribute for node {}. Only SAME_UPPER, SAME_LOWER or VALID "
1733 "supported and found {} {}",
1738 uint32_t inputHeight = inputInfo.GetShape()[2];
1739 uint32_t inputWidth = inputInfo.GetShape()[3];
1741 uint32_t weightHeight;
1742 uint32_t weightWidth;
1743 std::vector<uint32_t> kernel_shape = ReadOptionalNodeUint32ListAttribute(node,
"kernel_shape");
1744 if (kernel_shape.empty())
1746 const TensorInfo weightTensorInfo = *m_TensorsInfo[node.input(1)].m_info;
1747 weightHeight = weightTensorInfo.
GetShape()[2];
1748 weightWidth = weightTensorInfo.
GetShape()[3];
1752 weightHeight = kernel_shape[0];
1753 weightWidth = kernel_shape[1];
1755 CalcPadding(inputHeight,
1762 CalcPadding(inputWidth,
1779 uint32_t group = ReadOptionalNodeUint32Attribute(node,
"group", 1);
1782 if (group > inputInfo.GetShape()[1])
1785 fmt::format(
"Error parsing Convolution node: {}. "
1786 "The 'group'={} parameter cannot be larger than the "
1787 "channel of the input shape={} (in NCHW format). {}",
1790 inputInfo.GetShape()[1],
1793 else if (group == inputInfo.GetShape()[1])
1797 AddConvLayerWithDepthwiseConv(node, desc);
1804 throw ParseException(fmt::format(
"Error parsing Convolution node: {}. "
1805 "The 'group'={} parameter should be 1 or be equal to the "
1806 "channel of the input shape={} (in NCHW format). {}",
1809 inputInfo.GetShape()[1],
1816 std::vector<std::string> tensorIndexes= {node.input(0), node.input(1)};
1818 auto weightTensor = CreateConstTensor(node.input(1));
1820 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(weightTensor.first);
1824 if (node.input_size() == 3)
1826 if(!m_TensorsInfo[node.input(2)].isConstant())
1828 throw ParseException(fmt::format(
"Bias '{}' should be constant in Conv layer '{}' {}",
1834 auto biasTensor = CreateConstTensor(node.input(2));
1840 tensorIndexes.emplace_back(node.input(2));
1845 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1846 { m_TensorsInfo[node.input(0)].m_info->GetShape(),
1847 m_TensorsInfo[node.input(1)].m_info->GetShape() });
1852 RegisterInputSlots(layer, tensorIndexes);
1855 RegisterOutputSlots(layer, {node.output(0)});
1858 void OnnxParserImpl::ParseFlatten(
const onnx::NodeProto& node)
1864 m_TensorsInfo[node.input(0)].m_dtype,
1865 onnx::TensorProto::FLOAT);
1867 int64_t axis = ReadOptionalNodeInt64Attribute(node,
"axis", 1);
1868 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
1879 throw ParseException(fmt::format(
"Axis '{}' invalid. Tensor has '{}' dimensions in FlattenLayer '{}'",
1889 for (i = 0; i < axis; i++){
1890 dimension1 *= inputShape[i];
1895 dimension2 *= inputShape[i];
1900 auto outInfo = ComputeReshapeInfo(outputShape, inputShape, node.output(0));
1901 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
1902 CreateReshapeLayer(node.input(0), node.output(0), node.name());
1905 void OnnxParserImpl::ParseGather(
const onnx::NodeProto& node)
1911 gatherDescriptor.
m_Axis =
static_cast<int>(ReadOptionalNodeInt64Attribute(node,
"axis", 0));
1913 IConnectableLayer* layer = m_Network->AddGatherLayer(gatherDescriptor, node.name().c_str());
1916 const TensorShape& inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
1917 const TensorShape& indicesShape = m_TensorsInfo[node.input(1)].m_info->GetShape();
1918 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, { inputShape, indicesShape },
1919 m_TensorsInfo[node.input(0)].m_dtype);
1923 RegisterInputSlots(layer, { node.input(0), node.input(1) });
1926 RegisterOutputSlots(layer, { node.output(0) });
1929 void OnnxParserImpl::ParseGemm(
const onnx::NodeProto& node)
1934 int transA =
static_cast<int>(ReadOptionalNodeUint32Attribute(node,
"transA", 0));
1935 int transB =
static_cast<int>(ReadOptionalNodeUint32Attribute(node,
"transB", 0));
1936 float alpha = ReadOptionalNodeFloatAttribute(node,
"alpha", 1.0);
1937 float beta = ReadOptionalNodeFloatAttribute(node,
"beta", 1.0);
1938 bool biasEnabled = node.input_size() == 3;
1940 TensorShape input0Shape = m_TensorsInfo[node.input(0)].m_info->GetShape();
1941 TensorShape input1Shape = m_TensorsInfo[node.input(1)].m_info->GetShape();
1951 layer = m_Network->AddFullyConnectedLayer(fullyConnectedDescriptor, node.name().c_str());
1957 std::string transAName =
"transpose_" + node.input(0);
1960 IConnectableLayer* transALayer = m_Network->AddTransposeLayer(transposeADescriptor, transAName.c_str());
1962 auto transAInfo = ComputeOutputInfo({ transAName }, transALayer, { input0Shape });
1966 RegisterInputSlot(transALayer, node.input(0), 0);
1967 input0Shape = transAInfo[0].GetShape();
1971 RegisterInputSlot(layer, node.input(0), 0);
1975 if(m_TensorsInfo[node.input(1)].isConstant())
1977 IConnectableLayer* weightsLayer = m_Network->AddConstantLayer(CreateConstTensor(node.input(1)).first);
1978 TensorInfo weightInfo = *m_TensorsInfo[node.input(1)].m_info;
1985 std::string activationName =
"activation_" + node.input(1);
1987 activationDescriptor.
m_A = alpha;
1988 activationDescriptor.
m_Function = ActivationFunction::Linear;
1989 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
1992 auto actInfo = ComputeOutputInfo({ activationName }, actLayer, { weightInfo.
GetShape() });
1996 input1Shape = actInfo[0].GetShape();
2001 input1Shape = weightInfo.
GetShape();
2009 std::string activationName =
"activation_" + node.input(1);
2011 activationDescriptor.
m_A = alpha;
2012 activationDescriptor.
m_Function = ActivationFunction::Linear;
2013 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2016 auto actInfo = ComputeOutputInfo({ activationName }, actLayer, { input1Shape });
2019 RegisterInputSlot(actLayer, node.input(1), 0);
2020 input1Shape = actInfo[0].GetShape();
2024 RegisterInputSlot(layer, node.input(1), 1);
2028 if(biasEnabled && m_TensorsInfo[node.input(2)].isConstant())
2031 IConnectableLayer* biasLayer = m_Network->AddConstantLayer(CreateConstTensor(node.input(2)).first);
2032 TensorInfo biasInfo = *m_TensorsInfo[node.input(2)].m_info;
2039 std::string activationName =
"activation_" + node.input(2);
2041 activationDescriptor.
m_A = beta;
2042 activationDescriptor.
m_Function = ActivationFunction::Linear;
2043 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2046 auto actInfo = ComputeOutputInfo({ activationName }, actLayer, { biasInfo.
GetShape() });
2056 else if (biasEnabled)
2059 if (m_TensorsInfo[node.input(2)].m_info->GetNumDimensions() != 1)
2061 throw ParseException(fmt::format(
"The parser supports constant or non-constant with 1 dimension for "
2062 "Input C of Gemm. Input '{}' in '{}' is not supported '{}'",
2070 std::string activationName =
"activation_" + node.input(2);
2072 activationDescriptor.
m_A = beta;
2073 activationDescriptor.
m_Function = ActivationFunction::Linear;
2074 IConnectableLayer* actLayer = m_Network->AddActivationLayer(activationDescriptor, activationName.c_str());
2077 auto actInfo = ComputeOutputInfo({ activationName },
2079 { m_TensorsInfo[node.input(2)].m_info->GetShape() });
2082 RegisterInputSlot(actLayer, node.input(2), 0);
2086 RegisterInputSlot(layer, node.input(2), 2);
2091 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
2092 { input0Shape, input1Shape });
2095 RegisterOutputSlots(layer, {node.output(0)});
2098 void OnnxParserImpl::ParseGlobalAveragePool(
const onnx::NodeProto& node)
2104 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2108 IConnectableLayer* layer = m_Network->AddPooling2dLayer(desc, node.name().c_str());
2111 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {inputShape});
2116 RegisterInputSlots(layer, {node.input(0)});
2119 RegisterOutputSlots(layer, {node.output(0)});
2122 void OnnxParserImpl::ParseMaxPool(
const onnx::NodeProto& node)
2127 AddPoolingLayer(node, desc);
2130 void OnnxParserImpl::ParseShape(
const onnx::NodeProto& node)
2138 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2139 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {inputShape}, onnx::TensorProto::INT64);
2143 RegisterInputSlots(layer, {node.input(0)});
2146 RegisterOutputSlots(layer, {node.output(0)});
2149 void OnnxParserImpl::ParseReshape(
const onnx::NodeProto& node)
2155 m_TensorsInfo[node.input(0)].m_dtype,
2156 onnx::TensorProto::FLOAT);
2158 m_TensorsInfo[node.input(1)].m_dtype,
2159 onnx::TensorProto::INT64);
2161 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2163 std::vector<unsigned int> targetShape;
2164 if(m_TensorsInfo[node.input(1)].isConstant())
2166 unsigned int dims =
static_cast<unsigned int>(m_TensorsInfo[node.input(1)].m_tensor->int64_data_size());
2167 targetShape.reserve(dims);
2169 for(uint i = 0; i < dims; i++)
2171 int val =
CHECKED_INT32(m_TensorsInfo[node.input(1)].m_tensor->int64_data(
static_cast<int>(i)));
2172 targetShape[i]=
static_cast<unsigned int>(val);
2178 unsigned int dims = m_TensorsInfo[node.input(1)].m_info->GetNumDimensions();
2179 TensorShape shapes = m_TensorsInfo[node.input(1)].m_info->GetShape();
2180 if (dims != 1 || shapes[0] > 2)
2182 throw ParseException(fmt::format(
"Invalid input shape '{}' in Reshape layer '{}' {}",
2188 unsigned int numInputElements = m_TensorsInfo[node.input(0)].m_info->GetNumElements();
2191 targetShape = { numInputElements };
2193 else if (shapes[0] == 2)
2195 targetShape = { inputShape[0] , numInputElements / inputShape[0] };
2199 if(m_TensorsInfo[node.input(0)].isConstant())
2202 if(m_TensorsInfo.count(node.output(0)) == 0)
2204 m_TensorsInfo[node.output(0)] = OnnxTensor();
2206 m_TensorsInfo[node.output(0)].m_tensor =
2207 std::make_unique<onnx::TensorProto>(*m_TensorsInfo[node.input(0)].m_tensor);
2211 if(m_TensorsInfo.count(node.output(0)) == 0 || m_TensorsInfo[node.output(0)].m_info ==
nullptr)
2213 auto outInfo = ComputeReshapeInfo(
2214 TensorShape(
static_cast<unsigned int>(targetShape.size()), targetShape.data()),
2215 inputShape, node.output(0));
2216 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
2219 CreateReshapeLayer(node.input(0), node.output(0), node.name());
2223 void OnnxParserImpl::ParseUnsqueeze(
const onnx::NodeProto& node)
2228 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
2229 std::vector<uint32_t> dims;
2230 if (node.input_size() == 1 && node.attribute_size() > 0)
2232 dims = ReadMandatoryNodeUint32ListAttribute(node,
"axes");
2237 m_TensorsInfo[node.input(1)].m_dtype,
2238 onnx::TensorProto::INT64);
2240 auto int64Axes = m_TensorsInfo[node.input(1)].m_tensor->int64_data().data();
2241 uint numDim = armnn::numeric_cast<uint>(m_TensorsInfo[node.input(1)].m_tensor->int64_data_size());
2243 for(uint i = 0; i < numDim; i++)
2246 dims.push_back(uint32Value);
2251 std::sort(dims.begin(), dims.end());
2253 std::vector<unsigned int> targetShape;
2259 targetShape.push_back(inputShape[i]);
2263 for(uint i = 0; i < dims.size(); i++)
2265 targetShape.insert(targetShape.begin() + armnn::numeric_cast<int>(dims[i]), 1);
2268 auto outInfo = ComputeReshapeInfo(
TensorShape(
static_cast<unsigned int>(targetShape.size()), targetShape.data()),
2269 inputShape, node.output(0), m_TensorsInfo[node.input(0)].m_info->GetDataType());
2270 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
2271 m_TensorsInfo[node.output(0)].m_dtype = m_TensorsInfo[node.input(0)].m_dtype;
2273 CreateReshapeLayer(node.input(0), node.output(0), node.name());
2276 void OnnxParserImpl::PrependForBroadcast(
const std::string& outputName,
2277 const std::string& input0,
2278 const std::string& input1)
2283 TensorShape input0Shape = m_TensorsInfo[input0].m_info->GetShape();
2284 TensorShape input1Shape = m_TensorsInfo[input1].m_info->GetShape();
2287 std::vector<uint32_t> newShape;
2290 newShape.push_back(1);
2295 newShape.push_back(input0Shape[dim]);
2297 outputTensorInfo.
SetShape(
TensorShape(
static_cast<unsigned int>(newShape.size()), newShape.data()));
2300 m_TensorsInfo[outputName] = OnnxTensor();
2301 m_TensorsInfo[outputName].m_info = std::make_unique<TensorInfo>(outputTensorInfo);
2304 if( ! m_TensorsInfo[input0].isConstant())
2306 CreateReshapeLayer(input0, outputName, fmt::format(
"Add:reshapeOf{}", input0));
2310 m_TensorsInfo[outputName].m_tensor = std::make_unique<onnx::TensorProto>(*m_TensorsInfo[input0].m_tensor);
2315 void OnnxParserImpl::SetupInputLayers()
2318 for(
int inputIndex = 0; inputIndex < m_Graph->input_size(); ++inputIndex)
2320 auto input = m_Graph->input(inputIndex);
2321 if (!m_TensorsInfo[input.name()].isConstant())
2325 TensorInfo tensorInfo = *m_TensorsInfo[input.name()].m_info;
2328 if (m_InputShapes.find(input.name()) == m_InputShapes.end())
2330 throw ParseException(fmt::format(
"The parser does not support dynamic tensor, "
2331 "please specify input shape for {}. {}",
2337 tensorInfo.
SetShape(m_InputShapes[input.name()]);
2338 m_TensorsInfo[input.name()].m_info = std::make_unique<TensorInfo>(tensorInfo);
2344 m_InputInfos[input.name()] = tensorInfo;
2346 RegisterOutputSlots(layer,{ input.name() });
2351 void OnnxParserImpl::SetupOutputLayers()
2353 if(m_Graph->output_size() == 0)
2358 for(
int outputIndex = 0; outputIndex < m_Graph->output_size(); ++outputIndex)
2362 m_Graph->output(outputIndex).name().c_str());
2364 RegisterInputSlots(layer, { m_Graph->output(outputIndex).name() });
2369 const std::string& tensorId,
2370 unsigned int slotIndex)
2374 auto it = m_TensorConnections.find(tensorId);
2376 if (it == m_TensorConnections.end())
2379 m_TensorConnections[tensorId] = TensorSlots();
2381 m_TensorConnections[tensorId].inputSlots.push_back(slot);
2384 void OnnxParserImpl::RegisterInputSlots(
IConnectableLayer* layer,
const std::vector<std::string>& tensorIds)
2390 fmt::format(
"The number of tensor inputs ({}) does not match the number expected ({}) {}",
2396 for (
unsigned int slotIndex = 0; slotIndex < layer->
GetNumInputSlots(); ++slotIndex)
2398 std::string tensorId = tensorIds[slotIndex];
2401 auto it = m_TensorConnections.find(tensorId);
2403 if (it == m_TensorConnections.end())
2406 m_TensorConnections[tensorId] = TensorSlots();
2408 m_TensorConnections[tensorId].inputSlots.push_back(slot);
2412 void OnnxParserImpl::RegisterOutputSlots(
IConnectableLayer* layer,
const std::vector<std::string>& tensorIds)
2418 fmt::format(
"The number of tensor outputs ({}) does not match the number expected ({}) {} ",
2424 for (
unsigned int slotIndex = 0; slotIndex < layer->
GetNumOutputSlots(); ++slotIndex)
2426 std::string tensorId = tensorIds[slotIndex];
2429 auto it = m_TensorConnections.find(tensorId);
2431 if (it == m_TensorConnections.end())
2434 m_TensorConnections[tensorId] = TensorSlots();
2437 TensorSlots& tensorSlots = m_TensorConnections[tensorId];
2440 if (tensorSlots.outputSlot !=
nullptr)
2442 throw ParseException(fmt::format(
"Another layer has already registered itself as the producer of "
2447 tensorSlots.outputSlot = slot;
2454 for(
int i = 0; i < m_Graph->input_size(); ++i)
2456 auto input = m_Graph->input(i);
2457 if(input.name() == name)
2459 auto it = m_InputInfos.find(name);
2461 if (it != m_InputInfos.end())
2473 for(
int i = 0; i < m_Graph->output_size(); ++i)
2475 auto output = m_Graph->output(i);
2476 if(output.name() == name)
2478 auto it = m_OutputInfos.find(name);
2480 if (it != m_OutputInfos.end())
2492 if(model ==
nullptr) {
2497 std::vector<std::string> inputNames;
2498 std::map<std::string, bool> isConstant;
2499 for(
auto tensor : model->graph().initializer())
2501 isConstant[tensor.name()] =
true;
2503 for(
auto input : model->graph().input())
2505 auto it = isConstant.find(input.name());
2506 if(it == isConstant.end())
2508 inputNames.push_back(input.name());
2516 if(model ==
nullptr) {
2521 std::vector<std::string> outputNames;
2522 for(
auto output : model->graph().output())
2524 outputNames.push_back(output.name());