12 #include <fmt/format.h> 14 #include <google/protobuf/text_format.h> 15 #include <google/protobuf/io/zero_copy_stream_impl.h> 19 using namespace armnn;
25 void CheckValidDataType(std::initializer_list<onnx::TensorProto::DataType> validInputTypes,
27 const char* validExpr,
29 std::string tensorName,
32 bool isValid = std::any_of(validInputTypes.begin(),
33 validInputTypes.end(),
38 fmt::format(
"Datatype {} is not valid for tensor '{}' of node '{}', not in {{{}}}. {}",
39 onnx::TensorProto::DataType_Name(actualValue),
47 #define CHECK_VALID_DATATYPE(NODE, TENSOR, ACTUAL, ...) \ 48 CheckValidDataType({__VA_ARGS__}, ACTUAL, #__VA_ARGS__, NODE, TENSOR, CHECK_LOCATION()) 50 using StrTypeListPair = std::pair<const char*, std::initializer_list<onnx::TensorProto::DataType>>;
51 #define STR_LIST(...) StrTypeListPair(#__VA_ARGS__, {__VA_ARGS__}) 53 template <
typename Callable>
54 void ReadMandatoryNodeAttributeImpl(
const onnx::NodeProto& node,
55 const std::string& attribName,
56 onnx::AttributeProto::AttributeType expectedType,
59 auto attribs = node.attribute();
61 while (attriNum < node.attribute_size())
63 if (attribs.Get(attriNum).name() == attribName)
65 if (attribs.Get(attriNum).type() == expectedType)
67 callable(attribs.Get(attriNum));
71 throw ParseException(fmt::format(
"Attribute {} of node {} expected to have {} as " 72 "onnx::AttributeProto::AttributeType, but found {} instead {}",
75 onnx::AttributeProto::AttributeType_Name(expectedType),
76 onnx::AttributeProto::AttributeType_Name(attribs.Get(attriNum).type()),
83 if (attriNum == node.attribute_size())
85 throw ParseException(fmt::format(
"Could not find required attribute {} in node {} {}",
90 template <
typename Callable>
91 void ReadOptionalNodeAttributeImpl(
const onnx::NodeProto& node,
92 const std::string& attribName,
93 onnx::AttributeProto::AttributeType expectedType,
96 auto attribs = node.attribute();
97 for (
int attriNum = 0; attriNum < node.attribute_size(); ++attriNum)
99 if (attribs.Get(attriNum).name() == attribName)
101 if (attribs.Get(attriNum).type() == expectedType)
103 callable(attribs.Get(attriNum));
108 fmt::format(
"Attribute {} of node {} expected to have {} as onnx::AttributeProto::AttributeType, " 109 "but found {} instead {}",
112 onnx::AttributeProto::AttributeType_Name(expectedType),
113 onnx::AttributeProto::AttributeType_Name(attribs.Get(attriNum).type()),
120 int64_t ReadOptionalNodeInt64Attribute(
const onnx::NodeProto& node,
121 const std::string& name,
122 const int64_t defaultValue = 0)
124 int64_t attribValue = defaultValue;
125 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
126 [&attribValue](
const onnx::AttributeProto& attrValue)
128 attribValue = attrValue.i();
133 std::vector<uint32_t> ReadMandatoryNodeUint32ListAttribute(
const onnx::NodeProto& node,
134 const std::string& name)
136 std::vector<uint32_t> attriList;
137 ReadMandatoryNodeAttributeImpl(node, name, onnx::AttributeProto::INTS,
138 [&attriList](
const onnx::AttributeProto& attrValue)
140 for (
int attriNum = 0; attriNum < attrValue.ints_size(); ++attriNum)
148 uint32_t ReadOptionalNodeUint32Attribute(
const onnx::NodeProto& node,
149 const std::string& name,
150 const uint32_t defaultVal = 0u)
152 uint32_t attribValue = defaultVal;
153 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INT,
154 [&attribValue](
const onnx::AttributeProto& attrValue)
161 std::vector<uint32_t> ReadOptionalNodeUint32ListAttribute(
const onnx::NodeProto& node,
162 const std::string& name)
164 std::vector<uint32_t> attriList;
165 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::INTS,
166 [&attriList](
const onnx::AttributeProto& attrValue)
168 for (
int attriNum = 0; attriNum < attrValue.ints_size(); ++attriNum)
177 float ReadOptionalNodeFloatAttribute(
const onnx::NodeProto& node,
178 const std::string& name,
179 const float defaultValue = 0.0f)
181 float attribValue = defaultValue;
182 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::FLOAT,
183 [&attribValue](
const onnx::AttributeProto& attrValue)
185 attribValue = attrValue.f();
190 std::string ReadOptionalNodeStringAttribute(
const onnx::NodeProto& node,
const std::string& name)
192 std::string attribValue =
"";
193 ReadOptionalNodeAttributeImpl(node, name, onnx::AttributeProto::STRING,
194 [&attribValue](
const onnx::AttributeProto& attrValue)
196 attribValue = attrValue.s();
206 case onnx::TensorProto::FLOAT:
211 case onnx::TensorProto::INT32:
212 case onnx::TensorProto::INT64:
220 fmt::format(
"'{}' is not a currently supported datatype for tensor {}." 221 " Supported dataTypes are FLOAT, INT32 and INT64. {}",
222 onnx::TensorProto::DataType_Name(static_cast<onnx::TensorProto::DataType>(data_type)),
239 const onnx::TensorShapeProto onnxShape = info.type().tensor_type().shape();
240 std::vector<unsigned int> shapeDims;
241 for (
int i = 0; i < onnxShape.dim_size(); ++i)
246 if (shapeDims.empty())
248 shapeDims.push_back(1);
251 return ToTensorInfo(info.name(), shapeDims, info.type().tensor_type().elem_type());
256 std::vector<unsigned int> shapeDims;
258 for (
auto dim: tensor.dims())
263 if (shapeDims.empty())
265 shapeDims.push_back(1);
268 return ToTensorInfo(tensor.name(), shapeDims, tensor.data_type());
271 std::string TensorInfoAsString(
const TensorInfo& info,
272 const std::string& name,
276 std::stringstream ss;
277 ss <<
"tensor '" << name <<
"' contains " 278 << onnx::TensorProto::DataType_Name(type)
279 <<
" and has shape [";
283 ss << shape[i] <<
", ";
289 void CalcPadding(uint32_t inputSize, uint32_t filterSize, uint32_t stride, uint32_t* paddingFront,
290 uint32_t* paddingBack,
bool isUpper)
292 uint32_t outputSize = (inputSize + stride - 1) / stride;
293 uint32_t temp = (outputSize - 1) * stride + filterSize;
294 *paddingFront = (temp - inputSize) / 2;
295 *paddingBack = *paddingFront;
296 if((temp - inputSize) % 2 == 1)
311 const std::string& outName)
313 std::vector<int> targetDims;
319 targetDims.push_back(static_cast<int>(inShape[static_cast<uint>(i)]));
323 targetDims.push_back(val);
327 std::vector<unsigned int> outDims(targetDims.begin(), targetDims.end());
328 const auto stretchDim = std::find(targetDims.begin(), targetDims.end(), -1);
329 if (stretchDim != targetDims.end())
331 if (std::find(std::next(stretchDim), targetDims.end(), -1) != targetDims.end())
333 std::stringstream ss;
335 for(uint i = 0; i < targetDims.size() - 1; ++i)
337 ss << targetDims[i] <<
", ";
339 ss << targetDims[targetDims.size() - 1] <<
" ]";
342 fmt::format(
"Error during creation of reshaped tensor '{}'. At most one component of shape can be " 343 " -1 and here, shape is {} {}",
349 auto targetNumElements =
armnn::numeric_cast<
unsigned int>(std::accumulate(targetDims.begin(), targetDims.end(),
350 -1, std::multiplies<int32_t>()));
351 auto stretchIndex =
static_cast<size_t>(std::distance(targetDims.begin(), stretchDim));
352 outDims[stretchIndex] = inShape.
GetNumElements() / targetNumElements;
360 const std::map<std::string, OnnxParser::OperationParsingFunction> OnnxParser::m_ParserFunctions = {
361 {
"BatchNormalization", &OnnxParser::ParseBatchNormalization},
362 {
"GlobalAveragePool", &OnnxParser::ParseGlobalAveragePool},
363 {
"AveragePool", &OnnxParser::ParseAveragePool },
364 {
"Clip", &OnnxParser::ParseClip },
365 {
"Constant", &OnnxParser::ParseConstant },
366 {
"MaxPool", &OnnxParser::ParseMaxPool },
367 {
"Reshape", &OnnxParser::ParseReshape },
368 {
"Sigmoid", &OnnxParser::ParseSigmoid },
369 {
"Tanh", &OnnxParser::ParseTanh },
370 {
"Relu", &OnnxParser::ParseRelu },
371 {
"LeakyRelu", &OnnxParser::ParseLeakyRelu },
372 {
"Conv", &OnnxParser::ParseConv },
373 {
"Add", &OnnxParser::ParseAdd },
374 {
"Flatten", &OnnxParser::ParseFlatten},
377 template<
typename TypePair,
typename Location>
378 void OnnxParser::ValidateInputs(
const onnx::NodeProto& node,
379 TypePair validInputs,
380 const Location& location)
382 for(
auto input : node.input())
384 CheckValidDataType(validInputs.second,
385 m_TensorsInfo[input].m_dtype,
393 #define VALID_INPUTS(NODE, VALID_INPUTS) \ 394 OnnxParser::ValidateInputs(NODE, \ 398 std::vector<TensorInfo> OnnxParser::ComputeOutputInfo(std::vector<std::string> outNames,
400 std::vector<TensorShape> inputShapes)
403 bool needCompute = std::any_of(outNames.begin(),
405 [
this](std::string name)
407 return (m_TensorsInfo.count(name) == 0 || m_TensorsInfo[name].m_info ==
nullptr);
409 std::vector<TensorInfo> outInfo;
411 std::vector<TensorShape> inferredShapes;
417 for (uint i = 0; i < outNames.size(); ++i)
421 m_TensorsInfo[outNames[i]] = OnnxTensor();
422 m_TensorsInfo[outNames[i]].m_info = std::make_unique<TensorInfo>(
425 outInfo.push_back(*m_TensorsInfo[outNames[i]].m_info);
445 OnnxParser::OnnxParser()
446 : m_Network(nullptr, nullptr)
450 void OnnxParser::ResetParser()
456 void OnnxParser::Cleanup()
458 m_TensorConnections.clear();
459 m_TensorsInfo.clear();
460 m_OutputsMap.clear();
461 m_OutputsFusedAndUsed.clear();
464 std::pair<ConstTensor, std::unique_ptr<float[]>> OnnxParser::CreateConstTensor(
const std::string name)
466 const TensorInfo tensorInfo = *m_TensorsInfo[name].m_info;
467 onnx::TensorProto onnxTensor = *m_TensorsInfo[name].m_tensor;
469 auto srcData = onnxTensor.float_data().data();
470 std::unique_ptr<float[]> tensorData(
new float[tensorInfo.
GetNumElements()]);
471 const size_t tensorSizeInBytes = tensorInfo.
GetNumBytes();
473 if (!onnxTensor.has_raw_data())
475 if(tensorInfo.
GetNumElements() !=
static_cast<uint
>(onnxTensor.float_data_size()))
478 fmt::format(
"The number of data provided ({}) does not match the tensor '{}' number of " 480 onnxTensor.float_data_size(),
485 ::memcpy(tensorData.get(), srcData, tensorSizeInBytes);
489 ::memcpy(tensorData.get(), onnxTensor.raw_data().c_str(), tensorSizeInBytes);
495 throw ParseException(fmt::format(
"No tensor data found for Const tensor '{}' {}",
499 return std::make_pair(
ConstTensor(tensorInfo, tensorData.get()), std::move(tensorData));
504 FILE* fd = fopen(graphFile,
"r");
512 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
513 using google::protobuf::io::FileInputStream;
514 std::unique_ptr<FileInputStream> input = std::make_unique<FileInputStream>(fileno(fd));
515 bool success = google::protobuf::TextFormat::Parse(input.get(), modelProto.get());
520 std::stringstream
error;
521 error <<
"Failed to parse graph file";
531 return CreateNetworkFromModel(*modelProto);
537 FILE* fd = fopen(graphFile,
"rb");
545 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
547 google::protobuf::io::FileInputStream inStream(fileno(fd));
548 google::protobuf::io::CodedInputStream codedStream(&inStream);
549 codedStream.SetTotalBytesLimit(INT_MAX);
550 bool success = modelProto.get()->ParseFromCodedStream(&codedStream);
555 std::stringstream
error;
556 error <<
"Failed to parse graph file";
567 return CreateNetworkFromModel(*modelProto);
578 ModelPtr modelProto = std::make_unique<onnx::ModelProto>();
579 bool success = google::protobuf::TextFormat::ParseFromString(protoText, modelProto.get());
582 std::stringstream
error;
583 error <<
"Failed to parse graph file";
593 return CreateNetworkFromModel(*modelProto);
596 INetworkPtr OnnxParser::CreateNetworkFromModel(onnx::ModelProto& model)
598 m_Network = INetwork::Create();
601 m_Graph = std::make_unique<onnx::GraphProto>(*model.mutable_graph());
610 return std::move(m_Network);
613 void OnnxParser::LoadGraph()
618 SetupInfo(m_Graph->mutable_output());
619 SetupInfo(m_Graph->mutable_input());
620 SetupInfo(m_Graph->mutable_value_info());
622 for (
auto tensor : m_Graph->initializer())
624 m_TensorsInfo[tensor.name()].m_tensor = std::make_unique<const onnx::TensorProto>(tensor);
625 m_TensorsInfo[tensor.name()].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(tensor));
626 m_TensorsInfo[tensor.name()].m_dtype =
634 DetectFullyConnected();
637 for(
size_t nodeIndex = 0; nodeIndex < static_cast<size_t>(m_Graph->node_size()); nodeIndex++)
639 auto node = m_Graph->node(static_cast<int>(nodeIndex));
640 const std::string& operation = node.op_type();
643 if (operation ==
"MatMul" )
645 if(m_OutputsFusedAndUsed[nodeIndex].inputForNodes != m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.size())
648 AddFullyConnected(node);
651 else if (!(m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.empty()) && operation ==
"Add")
653 int matmulIndex =
static_cast<int> (m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes[0]);
654 AddFullyConnected(m_Graph->node(matmulIndex), &node);
656 else if (m_OutputsFusedAndUsed[nodeIndex].fusedWithNodes.empty())
658 auto it = m_ParserFunctions.find(operation);
659 if (it != m_ParserFunctions.end())
661 auto func = it->second;
666 throw ParseException(fmt::format(
"Unsupported operation {} for node '{}' {}",
675 for (
const auto& tensorCon : m_TensorConnections)
677 if (tensorCon.second.outputSlot !=
nullptr)
679 for (
size_t inputSlotIdx = 0; inputSlotIdx < tensorCon.second.inputSlots.size(); ++inputSlotIdx)
681 tensorCon.second.outputSlot->Connect(*(tensorCon.second.inputSlots[inputSlotIdx]));
687 void OnnxParser::SetupInfo(
const google::protobuf::RepeatedPtrField<onnx::ValueInfoProto >* list)
689 for (
auto tensor : *list)
691 m_TensorsInfo[tensor.name()] = OnnxTensor();
692 m_TensorsInfo[tensor.name()].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(tensor));
693 m_TensorsInfo[tensor.name()].m_dtype =
698 void OnnxParser::DetectFullyConnected()
700 m_OutputsFusedAndUsed = std::vector<UsageSummary> (
static_cast<size_t>(m_Graph->node_size()), UsageSummary());
701 auto matmulAndConstant = [&](
const std::string& constInput,
702 const std::string& matmulInput,
705 auto matmulIt = m_OutputsMap.find(matmulInput);
706 if(matmulIt != m_OutputsMap.end() && matmulIt->second.first->op_type() ==
"MatMul" 707 && m_TensorsInfo[constInput].isConstant())
709 nodeIndex = matmulIt->second.second;
715 for(
int nodeIndex = 0; nodeIndex < m_Graph->node_size(); nodeIndex++)
717 const onnx::NodeProto* node = &m_Graph->node(nodeIndex);
718 for (
const std::string& output : node->output())
720 m_OutputsMap[output] = std::make_pair(node, nodeIndex);
723 for (
const std::string& input : node->input())
725 auto matmulIt = m_OutputsMap.find(input);
726 if(matmulIt != m_OutputsMap.end()){
727 ++m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIt->second.second)].inputForNodes;
731 if (node->op_type() ==
"Add")
734 if (matmulAndConstant(node->input(0), node->input(1), matmulIndex) ||
735 matmulAndConstant(node->input(1), node->input(0), matmulIndex))
738 m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIndex)].fusedWithNodes
739 .push_back(static_cast<size_t>(nodeIndex));
741 m_OutputsFusedAndUsed[
static_cast<size_t>(nodeIndex)].fusedWithNodes
742 .push_back(static_cast<size_t>(matmulIndex));
747 for (
auto output: m_Graph->output()) {
748 auto matmulIt = m_OutputsMap.find(output.name());
749 if(matmulIt != m_OutputsMap.end()){
750 ++m_OutputsFusedAndUsed[
static_cast<size_t>(matmulIt->second.second)].inputForNodes;
755 template<
typename Location>
756 void OnnxParser::GetInputAndParam(
const onnx::NodeProto& node,
757 std::string* inputName,
758 std::string* constName,
759 const Location& location)
762 if (m_TensorsInfo[node.input(0)].isConstant())
766 else if (m_TensorsInfo[node.input(1)].isConstant())
772 throw ParseException(fmt::format(
"One of the input tensors ('{}' or '{}') should be constant in node '{}' {}",
776 location.AsString()));
780 *constName = node.input(cstIndex);
784 *inputName = node.input(!cstIndex);
788 template<
typename Location>
789 void OnnxParser::To1DTensor(
const std::string& name,
const Location& location)
791 TensorShape shape = m_TensorsInfo[name].m_info->GetShape();
792 std::vector<uint32_t> newShape;
798 fmt::format(
"Only tensors with shape [1, ..., 1, X] can be converted to 1D and {} {}",
799 TensorInfoAsString(*m_TensorsInfo[name].m_info, name, m_TensorsInfo[name].m_dtype),
800 location.AsString()));
805 m_TensorsInfo[name].m_info->SetShape(
TensorShape(static_cast<unsigned int>(newShape.size()), newShape.data()));
808 void OnnxParser::AddConvLayerWithDepthwiseConv(
const onnx::NodeProto& node,
const Convolution2dDescriptor& convDesc)
822 auto weightTensor = CreateConstTensor(node.input(1));
823 TensorShape& weightShape = weightTensor.first.GetShape();
824 weightShape[1] = weightShape[0];
826 m_TensorsInfo[node.input(1)].m_info->SetShape(weightShape);
828 if (node.input_size() == 3)
830 if(!m_TensorsInfo[node.input(2)].isConstant())
832 throw ParseException(fmt::format(
"Bias '{}' should be constant in Conv layer '{}' {}",
837 desc.m_BiasEnabled =
true;
838 auto biasTensor = CreateConstTensor(node.input(2));
839 layer = m_Network->AddDepthwiseConvolution2dLayer(desc,
842 node.name().c_str());
846 layer = m_Network->AddDepthwiseConvolution2dLayer(desc,
849 node.name().c_str());
853 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
854 { m_TensorsInfo[node.input(0)].m_info->GetShape(),
855 m_TensorsInfo[node.input(1)].m_info->GetShape() });
861 RegisterInputSlots(layer, {node.input(0)});
864 RegisterOutputSlots(layer, {node.output(0)});
867 void OnnxParser::AddFullyConnected(
const onnx::NodeProto& matmulNode,
const onnx::NodeProto* addNode)
871 std::string weightName;
872 std::string inputName;
877 GetInputAndParam(matmulNode, &inputName, &weightName,
CHECK_LOCATION());
886 std::string biasName;
895 TensorInfo weightInfo = *m_TensorsInfo[weightName].m_info;
896 TensorInfo biasInfo = *m_TensorsInfo[biasName].m_info;
901 fmt::format(
"Shape of weights '{}' and bias of following Add node '{}' do not match : {}" 902 " and {} ( /!\\ bias should be a 1D tensor) {}",
905 TensorInfoAsString(*m_TensorsInfo[weightName].m_info, weightName,
906 m_TensorsInfo[weightName].m_dtype),
907 TensorInfoAsString(*m_TensorsInfo[biasName].m_info, biasName,
908 m_TensorsInfo[biasName].m_dtype ),
911 layer = m_Network->AddFullyConnectedLayer(desc,
912 CreateConstTensor(weightName).first,
914 matmulNode.name().c_str());
917 auto outputInfo = ComputeOutputInfo({addNode->output(0)}, layer,
918 {m_TensorsInfo[inputName].m_info->GetShape(),
919 m_TensorsInfo[weightName].m_info->GetShape()});
923 RegisterInputSlots(layer, {inputName});
924 RegisterOutputSlots(layer, {addNode->output(0)});
928 layer = m_Network->AddFullyConnectedLayer(desc,
929 CreateConstTensor(weightName).first,
931 matmulNode.name().c_str());
934 auto outputInfo = ComputeOutputInfo({matmulNode.output(0)}, layer,
935 {m_TensorsInfo[inputName].m_info->GetShape(),
936 m_TensorsInfo[weightName].m_info->GetShape()});
939 RegisterInputSlots(layer, {inputName});
940 RegisterOutputSlots(layer, {matmulNode.output(0)});
952 std::vector<uint32_t> kernel_shape = ReadMandatoryNodeUint32ListAttribute(node,
"kernel_shape");
953 std::vector<uint32_t> strides = ReadOptionalNodeUint32ListAttribute(node,
"strides");
954 std::vector<uint32_t> pads = ReadOptionalNodeUint32ListAttribute(node,
"pads");
975 std::string paddingString = ReadOptionalNodeStringAttribute(node,
"auto_pad");
976 if(paddingString !=
"VALID" && paddingString !=
"" && paddingString !=
"NOTSET")
979 if( paddingString ==
"SAME_LOWER")
983 else if (paddingString ==
"SAME_UPPER")
989 throw ParseException(fmt::format(
"Invalid auto_pad attribute for node {}. " 990 "Only SAME_UPPER, SAME_LOWER or VALID supported and found {} {}",
995 auto inputInfo = *m_TensorsInfo[node.input(0)].m_info;
996 uint32_t inputHeight = inputInfo.GetShape()[2];
997 uint32_t inputWidth = inputInfo.GetShape()[3];
1010 IConnectableLayer* layer = m_Network->AddPooling2dLayer(desc, node.name().c_str());
1013 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1018 RegisterInputSlots(layer, {node.input(0)});
1021 RegisterOutputSlots(layer, {node.output(0)});
1024 std::pair<std::string, std::string> OnnxParser::AddPrepareBroadcast(
const std::string& input0,
1025 const std::string& input1)
1027 std::pair<std::string, std::string> inputs = std::make_pair(input0, input1);
1029 TensorShape input0Shape = m_TensorsInfo[input0].m_info->GetShape();
1030 TensorShape input1Shape = m_TensorsInfo[input1].m_info->GetShape();
1034 auto outputName = fmt::format(
"reshape_output_{}", input1);
1035 PrependForBroadcast(outputName, input1, input0);
1036 inputs.second = outputName;
1040 auto outputName = fmt::format(
"reshape_output_{}", input0);
1041 PrependForBroadcast(outputName, input0, input1);
1042 inputs.first = outputName;
1047 void OnnxParser::CreateConstantLayer(
const std::string& tensorName,
const std::string& layerName)
1049 auto armnnTensor = CreateConstTensor(tensorName);
1051 IConnectableLayer* layer = m_Network->AddConstantLayer(armnnTensor.first, layerName.c_str());
1053 RegisterOutputSlots(layer, {tensorName});
1056 void OnnxParser::CreateReshapeLayer(
const std::string& inputName,
1057 const std::string& outputName,
1058 const std::string& layerName)
1060 const TensorInfo outputTensorInfo = *m_TensorsInfo[outputName].m_info;
1064 IConnectableLayer* layer = m_Network->AddReshapeLayer(reshapeDesc, layerName.c_str());
1070 RegisterInputSlots(layer, {inputName});
1073 RegisterOutputSlots(layer, {outputName});
1086 if (func == ActivationFunction::BoundedReLu)
1088 desc.
m_A = node.input(2).empty() ? std::numeric_limits<float>::max() : std::stof(node.input(2));
1089 desc.
m_B = node.input(1).empty() ? std::numeric_limits<float>::lowest() : std::stof(node.input(1));
1092 IConnectableLayer*
const layer = m_Network->AddActivationLayer(desc, node.name().c_str());
1095 auto outputInfo = ComputeOutputInfo({ node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1096 layer->GetOutputSlot(0).SetTensorInfo(outputInfo[0]);
1100 RegisterInputSlots(layer, {node.input(0)});
1103 RegisterOutputSlots(layer, {node.output(0)});
1106 void OnnxParser::ParseClip(
const onnx::NodeProto& node)
1108 ParseActivation(node, ActivationFunction::BoundedReLu);
1111 void OnnxParser::ParseSigmoid(
const onnx::NodeProto& node)
1113 ParseActivation(node, ActivationFunction::Sigmoid);
1116 void OnnxParser::ParseTanh(
const onnx::NodeProto& node)
1118 ParseActivation(node, ActivationFunction::TanH);
1121 void OnnxParser::ParseRelu(
const onnx::NodeProto& node)
1123 ParseActivation(node, ActivationFunction::ReLu);
1126 void OnnxParser::ParseLeakyRelu(
const onnx::NodeProto& node)
1128 ParseActivation(node, ActivationFunction::LeakyReLu);
1131 void OnnxParser::ParseAdd(
const onnx::NodeProto& node)
1142 auto inputs = AddPrepareBroadcast(node.input(0), node.input(1));
1143 auto input0 = *m_TensorsInfo[inputs.first].m_info;
1144 auto input1 = *m_TensorsInfo[inputs.second].m_info;
1145 ARMNN_ASSERT(input0.GetNumDimensions() == input1.GetNumDimensions());
1147 unsigned int numDims = input0.GetNumDimensions();
1148 for (
unsigned int i = 0; i < numDims; i++)
1150 unsigned int dim0 = input0.GetShape()[i];
1151 unsigned int dim1 = input1.GetShape()[i];
1152 if (dim0 != dim1 && dim0 != 1 && dim1 != 1)
1155 fmt::format(
"Broadcast is only supported for scalar or 1D tensors in Add node '{}'. " 1156 "Input dimensions should either match or one should be of size 1 and here, " 1159 TensorInfoAsString(*m_TensorsInfo[inputs.first].m_info, inputs.first,
1160 m_TensorsInfo[inputs.first].m_dtype),
1161 TensorInfoAsString(*m_TensorsInfo[inputs.second].m_info, inputs.second,
1162 m_TensorsInfo[inputs.second].m_dtype),
1171 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1172 { m_TensorsInfo[inputs.first].m_info->GetShape(),
1173 m_TensorsInfo[inputs.second].m_info->GetShape() });
1177 if(m_TensorsInfo[inputs.first].isConstant()) {
1178 CreateConstantLayer(inputs.first, fmt::format(
"Add:constant_of_{}", node.input(0)));
1180 if(m_TensorsInfo[inputs.second].isConstant()) {
1181 CreateConstantLayer(inputs.second, fmt::format(
"Add:constant_of_{}", node.input(1)));
1183 RegisterInputSlots(layer, {inputs.first, inputs.second});
1186 RegisterOutputSlots(layer, {node.output(0)});
1189 void OnnxParser::ParseAveragePool(
const onnx::NodeProto& node)
1194 uint32_t count_include_pad = 0;
1195 count_include_pad = ReadOptionalNodeUint32Attribute(node,
"count_include_pad");
1196 if(count_include_pad) {
1199 AddPoolingLayer(node, desc);
1202 void OnnxParser::ParseBatchNormalization(
const onnx::NodeProto& node)
1210 for(
int ind = 1; ind < node.input_size(); ++ind)
1212 auto tensor = node.input(ind);
1213 if(! m_TensorsInfo[tensor].isConstant())
1216 fmt::format(
"Input tensor '{}' should be constant in BatchNormalization node '{}' {}",
1223 float epsilon = ReadOptionalNodeFloatAttribute(node,
"epsilon", 1e-5f);
1225 desc.
m_Eps = epsilon;
1227 auto scaleTensor = CreateConstTensor(node.input(1));
1228 auto biasTensor = CreateConstTensor(node.input(2));
1229 auto meanTensor = CreateConstTensor(node.input(3));
1230 auto varTensor = CreateConstTensor(node.input(4));
1237 node.name().c_str());
1240 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {m_TensorsInfo[node.input(0)].m_info->GetShape()});
1241 layer->GetOutputSlot(0).SetTensorInfo(outputInfo[0]);
1243 RegisterInputSlots(layer, {node.input(0)});
1246 RegisterOutputSlots(layer, {node.output(0)});
1249 void OnnxParser::ParseConstant(
const onnx::NodeProto& node)
1252 if (!node.attribute(0).has_t())
1254 throw ParseException(fmt::format(
"Value not found for Constant node '{}' {}",
1258 const onnx::TensorProto& onnxTensor = node.attribute(0).t();
1265 m_TensorsInfo[node.output(0)].m_tensor = std::make_unique<const onnx::TensorProto>(onnxTensor);
1266 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(
ToTensorInfo(onnxTensor));
1269 CreateConstantLayer(node.output(0), node.name());
1272 void OnnxParser::ParseConv(
const onnx::NodeProto& node)
1279 if(m_TensorsInfo[node.input(0)].m_info->GetNumDimensions() != 4)
1282 fmt::format(
"ArmNN only supports 2D convolution and Conv layer '{}' input {} {}",
1284 TensorInfoAsString(*m_TensorsInfo[node.input(0)].m_info, node.input(0),
1285 m_TensorsInfo[node.input(0)].m_dtype),
1289 if(!m_TensorsInfo[node.input(1)].isConstant())
1292 fmt::format(
"Weights '{}' should be constant in Conv layer '{}' {}",
1298 auto inputInfo = *m_TensorsInfo[node.input(0)].m_info;
1300 std::vector<uint32_t> dilations = ReadOptionalNodeUint32ListAttribute(node,
"dilations");
1301 if (!dilations.empty())
1303 std::stringstream ss;
1305 for (
auto dilation : dilations)
1307 ss << dilation <<
", ";
1312 fmt::format(
"ArmNN only supports Convolution layers with dilations [1,1], and node '{}' " 1313 "has dilatation {} {}",
1322 std::vector<uint32_t> strides = ReadOptionalNodeUint32ListAttribute(node,
"strides");
1334 std::vector<uint32_t> pads = ReadOptionalNodeUint32ListAttribute(node,
"pads");
1339 std::string paddingString = ReadOptionalNodeStringAttribute(node,
"auto_pad");
1340 if(paddingString !=
"VALID" && paddingString !=
"" && paddingString !=
"NOTSET")
1343 if( paddingString ==
"SAME_LOWER")
1347 else if (paddingString ==
"SAME_UPPER")
1354 fmt::format(
"Invalid auto_pad attribute for node {}. Only SAME_UPPER, SAME_LOWER or VALID " 1355 "supported and found {} {}",
1360 uint32_t inputHeight = inputInfo.GetShape()[2];
1361 uint32_t inputWidth = inputInfo.GetShape()[3];
1363 uint32_t weightHeight;
1364 uint32_t weightWidth;
1365 std::vector<uint32_t> kernel_shape = ReadOptionalNodeUint32ListAttribute(node,
"kernel_shape");
1366 if (kernel_shape.empty())
1368 const TensorInfo weightTensorInfo = *m_TensorsInfo[node.input(1)].m_info;
1369 weightHeight = weightTensorInfo.
GetShape()[2];
1370 weightWidth = weightTensorInfo.
GetShape()[3];
1374 weightHeight = kernel_shape[0];
1375 weightWidth = kernel_shape[1];
1389 uint32_t group = ReadOptionalNodeUint32Attribute(node,
"group", 1);
1392 if (group > inputInfo.GetShape()[1])
1395 fmt::format(
"Error parsing Convolution node: {}. " 1396 "The 'group'={} parameter cannot be larger than the " 1397 "channel of the input shape={} (in NCHW format). {}",
1400 inputInfo.GetShape()[1],
1403 else if (group == inputInfo.GetShape()[1])
1407 AddConvLayerWithDepthwiseConv(node, desc);
1414 throw ParseException(fmt::format(
"Error parsing Convolution node: {}. " 1415 "The 'group'={} parameter should be 1 or be equal to the " 1416 "channel of the input shape={} (in NCHW format). {}",
1419 inputInfo.GetShape()[1],
1425 auto weightTensor = CreateConstTensor(node.input(1));
1427 if (node.input_size() == 3)
1429 if(!m_TensorsInfo[node.input(2)].isConstant())
1431 throw ParseException(fmt::format(
"Bias '{}' should be constant in Conv layer '{}' {}",
1437 auto biasTensor = CreateConstTensor(node.input(2));
1438 layer = m_Network->AddConvolution2dLayer(desc,
1441 node.name().c_str());
1445 layer = m_Network->AddConvolution2dLayer(desc,
1448 node.name().c_str());
1452 auto outputInfo = ComputeOutputInfo({ node.output(0) }, layer,
1453 { m_TensorsInfo[node.input(0)].m_info->GetShape(),
1454 m_TensorsInfo[node.input(1)].m_info->GetShape() });
1459 RegisterInputSlots(layer, {node.input(0)});
1462 RegisterOutputSlots(layer, {node.output(0)});
1465 void OnnxParser::ParseFlatten(
const onnx::NodeProto& node)
1471 m_TensorsInfo[node.input(0)].m_dtype,
1472 onnx::TensorProto::FLOAT);
1474 int64_t axis = ReadOptionalNodeInt64Attribute(node,
"axis", 1);
1475 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
1486 throw ParseException(fmt::format(
"Axis '{}' invalid. Tensor has '{}' dimensions in FlattenLayer '{}'",
1496 for (i = 0; i < axis; i++){
1497 dimension1 *= inputShape[i];
1502 dimension2 *= inputShape[i];
1507 auto outInfo = ComputeReshapeInfo(outputShape, inputShape, node.output(0));
1508 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
1509 CreateReshapeLayer(node.input(0), node.output(0), node.name());
1512 void OnnxParser::ParseGlobalAveragePool(
const onnx::NodeProto& node)
1518 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
1522 IConnectableLayer* layer = m_Network->AddPooling2dLayer(desc, node.name().c_str());
1525 auto outputInfo = ComputeOutputInfo({node.output(0)}, layer, {inputShape});
1530 RegisterInputSlots(layer, {node.input(0)});
1533 RegisterOutputSlots(layer, {node.output(0)});
1536 void OnnxParser::ParseMaxPool(
const onnx::NodeProto& node)
1541 AddPoolingLayer(node, desc);
1544 void OnnxParser::ParseReshape(
const onnx::NodeProto& node)
1550 m_TensorsInfo[node.input(0)].m_dtype,
1551 onnx::TensorProto::FLOAT);
1553 m_TensorsInfo[node.input(1)].m_dtype,
1554 onnx::TensorProto::INT64);
1556 if(!m_TensorsInfo[node.input(1)].isConstant())
1558 throw ParseException(fmt::format(
"Shape '{}' should be constant in Reshape layer '{}' {}",
1564 if(m_TensorsInfo[node.input(0)].isConstant())
1567 if(m_TensorsInfo.count(node.output(0)) == 0)
1569 m_TensorsInfo[node.output(0)] = OnnxTensor();
1571 m_TensorsInfo[node.output(0)].m_tensor =
1572 std::make_unique<onnx::TensorProto>(*m_TensorsInfo[node.input(0)].m_tensor);
1576 TensorShape inputShape = m_TensorsInfo[node.input(0)].m_info->GetShape();
1578 if(m_TensorsInfo.count(node.output(0)) == 0 || m_TensorsInfo[node.output(0)].m_info ==
nullptr)
1580 uint64_t dims =
static_cast<uint64_t
>(m_TensorsInfo[node.input(1)].m_tensor->int64_data_size());
1581 TensorShape targetShape{
static_cast<unsigned int>(dims), 1};
1583 for(uint i = 0; i < dims; i++)
1585 int val =
CHECKED_INT32(m_TensorsInfo[node.input(1)].m_tensor->int64_data(static_cast<int>(i)));
1586 targetShape[i]=
static_cast<unsigned int>(val);
1589 auto outInfo = ComputeReshapeInfo(targetShape, inputShape, node.output(0));
1590 m_TensorsInfo[node.output(0)].m_info = std::make_unique<TensorInfo>(outInfo);
1593 CreateReshapeLayer(node.input(0), node.output(0), node.name());
1597 void OnnxParser::PrependForBroadcast(
const std::string& outputName,
1598 const std::string& input0,
1599 const std::string& input1)
1604 TensorShape input0Shape = m_TensorsInfo[input0].m_info->GetShape();
1605 TensorShape input1Shape = m_TensorsInfo[input1].m_info->GetShape();
1608 std::vector<uint32_t> newShape;
1611 newShape.push_back(1);
1616 newShape.push_back(input0Shape[dim]);
1618 outputTensorInfo.
SetShape(
TensorShape(static_cast<unsigned int>(newShape.size()), newShape.data()));
1621 m_TensorsInfo[outputName] = OnnxTensor();
1622 m_TensorsInfo[outputName].m_info = std::make_unique<TensorInfo>(outputTensorInfo);
1625 if( ! m_TensorsInfo[input0].isConstant())
1627 CreateReshapeLayer(input0, outputName, fmt::format(
"Add:reshapeOf{}", input0));
1631 m_TensorsInfo[outputName].m_tensor = std::make_unique<onnx::TensorProto>(*m_TensorsInfo[input0].m_tensor);
1636 void OnnxParser::SetupInputLayers()
1639 for(
int inputIndex = 0; inputIndex < m_Graph->input_size(); ++inputIndex)
1641 auto input = m_Graph->input(inputIndex);
1642 if (! m_TensorsInfo[input.name()].isConstant())
1645 m_Network->AddInputLayer(static_cast<armnn::LayerBindingId>(inputIndex), input.name().c_str());
1649 RegisterOutputSlots(layer,{ input.name() });
1654 void OnnxParser::SetupOutputLayers()
1656 if(m_Graph->output_size() == 0)
1661 for(
int outputIndex = 0; outputIndex < m_Graph->output_size(); ++outputIndex)
1664 m_Network->AddOutputLayer(static_cast<armnn::LayerBindingId>(outputIndex),
1665 m_Graph->output(outputIndex).name().c_str());
1667 RegisterInputSlots(layer, { m_Graph->output(outputIndex).name() });
1671 void OnnxParser::RegisterInputSlots(
IConnectableLayer* layer,
const std::vector<std::string>& tensorIds)
1677 fmt::format(
"The number of tensor inputs ({}) does not match the number expected ({}) {}",
1682 for (
unsigned int slotIndex = 0; slotIndex < layer->
GetNumInputSlots(); ++slotIndex)
1684 std::string tensorId = tensorIds[slotIndex];
1687 auto it = m_TensorConnections.find(tensorId);
1689 if (it == m_TensorConnections.end())
1692 m_TensorConnections[tensorId] = TensorSlots();
1694 m_TensorConnections[tensorId].inputSlots.push_back(slot);
1698 void OnnxParser::RegisterOutputSlots(
IConnectableLayer* layer,
const std::vector<std::string>& tensorIds)
1704 fmt::format(
"The number of tensor outputs ({}) does not match the number expected ({}) {} ",
1710 for (
unsigned int slotIndex = 0; slotIndex < layer->
GetNumOutputSlots(); ++slotIndex)
1712 std::string tensorId = tensorIds[slotIndex];
1715 auto it = m_TensorConnections.find(tensorId);
1717 if (it == m_TensorConnections.end())
1720 m_TensorConnections[tensorId] = TensorSlots();
1723 TensorSlots& tensorSlots = m_TensorConnections[tensorId];
1726 if (tensorSlots.outputSlot !=
nullptr)
1728 throw ParseException(fmt::format(
"Another layer has already registered itself as the producer of " 1733 tensorSlots.outputSlot = slot;
1739 for(
int i = 0; i < m_Graph->input_size(); ++i)
1741 auto input = m_Graph->input(i);
1742 if(input.name() == name)
1744 return std::make_pair(static_cast<armnn::LayerBindingId>(i),
ToTensorInfo(input));
1753 for(
int i = 0; i < m_Graph->output_size(); ++i)
1755 auto output = m_Graph->output(i);
1756 if(output.name() == name)
1758 return std::make_pair(static_cast<armnn::LayerBindingId>(i),
ToTensorInfo(output));
1767 if(model ==
nullptr) {
1772 std::vector<std::string> inputNames;
1773 std::map<std::string, bool> isConstant;
1774 for(
auto tensor : model->graph().initializer())
1776 isConstant[tensor.name()] =
true;
1778 for(
auto input : model->graph().input())
1780 auto it = isConstant.find(input.name());
1781 if(it == isConstant.end())
1783 inputNames.push_back(input.name());
1791 if(model ==
nullptr) {
1796 std::vector<std::string> outputNames;
1797 for(
auto output : model->graph().output())
1799 outputNames.push_back(output.name());
unsigned int GetNumElements() const
Function that calculates the tensor elements by multiplying all dimension size which are Specified...
uint32_t m_PadBottom
Padding bottom value in the height dimension.
bool m_BiasEnabled
Enable/disable bias.
virtual unsigned int GetNumOutputSlots() const =0
Returns the number of connectable output slots.
Interface for a layer that is connectable to other layers via InputSlots and OutputSlots.
uint32_t m_PadBottom
Padding bottom value in the height dimension.
virtual unsigned int GetNumInputSlots() const =0
Returns the number of connectable input slots.
const TensorShape & GetShape() const
uint32_t m_PadLeft
Padding left value in the width dimension.
std::string AsString() const
A ReshapeDescriptor for the ReshapeLayer.
virtual BindingPointInfo GetNetworkOutputBindingInfo(const std::string &name) const override
Retrieve binding info (layer id and tensor info) for the network output identified by the given layer...
uint32_t m_PoolWidth
Pooling width value.
virtual armnn::INetworkPtr CreateNetworkFromTextFile(const char *graphFile) override
Create the network from a protobuf text file on disk.
A Convolution2dDescriptor for the Convolution2dLayer.
uint32_t m_PadLeft
Padding left value in the width dimension.
unsigned int GetNumBytes() const
float m_Eps
Value to add to the variance. Used to avoid dividing by zero.
PaddingMethod m_PaddingMethod
The padding method to be used. (Exclude, IgnoreValue).
uint32_t m_PadTop
Padding top value in the height dimension.
void CalcPadding(uint32_t input, uint32_t kernel, uint32_t stride, uint32_t &outPadHead, uint32_t &outPadTail, bool samePadding)
uint32_t m_PadRight
Padding right value in the width dimension.
#define VALID_INPUTS(NODE, VALID_INPUTS)
Copyright (c) 2020 ARM Limited.
static ModelPtr LoadModelFromBinaryFile(const char *fileName)
uint32_t m_StrideX
Stride value when proceeding through input for the width dimension.
virtual void SetTensorInfo(const TensorInfo &tensorInfo)=0
void SetShape(const TensorShape &newShape)
static ModelPtr LoadModelFromTextFile(const char *fileName)
virtual armnn::INetworkPtr CreateNetworkFromBinaryFile(const char *graphFile) override
Create the network from a protobuf binary file on disk.
TensorShape m_TargetShape
Target shape value.
static std::vector< std::string > GetInputs(ModelPtr &model)
Retrieve inputs names.
uint32_t m_PoolHeight
Pooling height value.
uint32_t m_PadTop
Padding top value in the height dimension.
static std::vector< std::string > GetOutputs(ModelPtr &model)
Retrieve outputs names.
armnn::TensorInfo ToTensorInfo(Deserializer::TensorRawPtr tensorPtr)
uint32_t m_StrideX
Stride value when proceeding through input for the width dimension.
std::unique_ptr< onnx::ModelProto > ModelPtr
virtual BindingPointInfo GetNetworkInputBindingInfo(const std::string &name) const override
Retrieve binding info (layer id and tensor info) for the network input identified by the given layer ...
uint32_t m_PadRight
Padding right value in the width dimension.
An output connection slot for a layer.
A FullyConnectedDescriptor for the FullyConnectedLayer.
bool m_BiasEnabled
Enable/disable bias.
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
#define CHECK_VALID_SIZE(ACTUAL,...)
#define CHECKED_NON_NEGATIVE(VALUE)
#define ARMNN_ASSERT(COND)
An ActivationDescriptor for the ActivationLayer.
uint32_t m_StrideY
Stride value when proceeding through input for the height dimension.
float m_A
Alpha upper bound value used by the activation functions. (BoundedReLu, Linear, TanH, Elu).
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
PoolingAlgorithm m_PoolType
The pooling algorithm to use (Max. Average, L2).
unsigned int GetNumDimensions() const
Function that returns the tensor rank.
OutputShapeRounding m_OutputShapeRounding
The rounding method for the output shape. (Floor, Ceiling).
virtual const IInputSlot & GetInputSlot(unsigned int index) const =0
Get a const input slot handle by slot index.
virtual armnn::INetworkPtr CreateNetworkFromString(const std::string &protoText) override
Create the network directly from protobuf text in a string. Useful for debugging/testing.
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
virtual const IOutputSlot & GetOutputSlot(unsigned int index) const =0
Get the const output slot handle by slot index.
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
static ModelPtr LoadModelFromString(const std::string &inputString)
#define CHECK_VALID_DATATYPE(NODE, TENSOR, ACTUAL,...)
armnn::BindingPointInfo BindingPointInfo
virtual std::vector< TensorShape > InferOutputShapes(const std::vector< TensorShape > &inputShapes) const =0
Infer the shape of the output(s) based on the provided input shape(s)
#define CHECKED_INT32(VALUE)
A Pooling2dDescriptor for the Pooling2dLayer.
std::unique_ptr< IOnnxParser, void(*)(IOnnxParser *parser)> IOnnxParserPtr
float m_B
Beta lower bound value used by the activation functions. (BoundedReLu, Linear, TanH).
ActivationFunction m_Function
The activation function to use (Sigmoid, TanH, Linear, ReLu, BoundedReLu, SoftReLu, LeakyReLu, Abs, Sqrt, Square, Elu).
uint32_t m_StrideY
Stride value when proceeding through input for the height dimension.
A DepthwiseConvolution2dDescriptor for the DepthwiseConvolution2dLayer.
A BatchNormalizationDescriptor for the BatchNormalizationLayer.
uint32_t m_PadLeft
Padding left value in the width dimension.
unsigned int GetNumElements() const