// // Copyright © 2023 Arm Ltd and Contributors. All rights reserved. // SPDX-License-Identifier: MIT // #include #include #include #include #include #include namespace armnnOpaqueDelegate { TfLiteStatus VisitConv2dOperator(DelegateData& delegateData, TfLiteOpaqueContext* tfLiteContext, TfLiteOpaqueNode* tfLiteNode, int nodeIndex, int32_t operatorCode) { auto numInputs = TfLiteOpaqueNodeNumberOfInputs(tfLiteNode); if (numInputs < 2) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Minimum number of inputs (%d != %d) in node #%d", 2, numInputs, nodeIndex); return kTfLiteError; } TF_LITE_ENSURE_STATUS(ValidateNumOutputs(tfLiteContext, tfLiteNode, 1, nodeIndex)); // Gather input indices and use to get input tensor. const int* inputTensors; if (TfLiteOpaqueNodeInputs(tfLiteNode, &inputTensors, &numInputs) != kTfLiteOk) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Unable to gather input tensor indices from node #%d: ", nodeIndex); return kTfLiteError; } const TfLiteOpaqueTensor* tfLiteInputTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[0]); if (!IsValid(tfLiteContext, tfLiteInputTensor, operatorCode, nodeIndex)) { return kTfLiteError; } // Use input indices to get filter tensor. const TfLiteOpaqueTensor* tfLiteFilterTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[1]); if (!IsValid(tfLiteContext, tfLiteFilterTensor, operatorCode, nodeIndex)) { return kTfLiteError; } // Gather output indices and use to get output tensors. int numOutputs = 0; const int* outputTensors; if (TfLiteOpaqueNodeOutputs(tfLiteNode, &outputTensors, &numOutputs) != kTfLiteOk) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Unable to gather output tensor indices from node #%d: ", nodeIndex); return kTfLiteError; } const TfLiteOpaqueTensor* tfLiteOutputTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, outputTensors[0]); if (!IsValid(tfLiteContext, tfLiteOutputTensor, operatorCode, nodeIndex)) { return kTfLiteError; } const armnn::TensorInfo& inputTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteInputTensor); const armnn::TensorInfo& filterTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteFilterTensor); const armnn::TensorInfo& outputTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteOutputTensor, true); auto* tfLiteNodeParameters = reinterpret_cast(TfLiteOpaqueNodeGetBuiltinData(tfLiteNode)); TfLiteFusedActivation activationType = kTfLiteActNone; if (tfLiteNodeParameters) { activationType = tfLiteNodeParameters->activation; TfLiteStatus activationStatus = ValidateFusedActivationOperator(delegateData, tfLiteContext, outputTensorInfo, outputTensorInfo, activationType); if(activationStatus != kTfLiteOk) { return kTfLiteError; } } armnn::TensorInfo biasTensorInfo; const TfLiteOpaqueTensor* tfLiteBiasTensor = nullptr; bool biasEnabled = IsOptionalOperandPresent(tfLiteNode, 2); if(biasEnabled) { // Use input indices to get bias tensor. tfLiteBiasTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[2]); if (!IsValid(tfLiteContext, tfLiteBiasTensor, operatorCode, nodeIndex)) { return kTfLiteError; } biasTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteBiasTensor); } else { biasTensorInfo = armnn::TensorInfo(armnn::TensorShape({1}), GetDataType(tfLiteInputTensor)); } armnn::Optional optionalBiasInfo(biasTensorInfo); armnn::Convolution2dDescriptor descriptor; descriptor.m_BiasEnabled = biasEnabled; descriptor.m_StrideX = NonNegative(tfLiteNodeParameters->stride_width, nodeIndex); descriptor.m_StrideY = NonNegative(tfLiteNodeParameters->stride_height, nodeIndex); descriptor.m_DataLayout = armnn::DataLayout::NHWC; descriptor.m_DilationX = NonNegative(tfLiteNodeParameters->dilation_width_factor, nodeIndex); descriptor.m_DilationY = NonNegative(tfLiteNodeParameters->dilation_height_factor, nodeIndex); // TfLite uses NHWC tensors const unsigned int inputHeight = inputTensorInfo.GetShape()[1]; const unsigned int inputWidth = inputTensorInfo.GetShape()[2]; const unsigned int filterHeight = filterTensorInfo.GetShape()[1]; const unsigned int filterWidth = filterTensorInfo.GetShape()[2]; // Calculate padding CalcPadding(inputHeight, filterHeight, descriptor.m_StrideY, descriptor.m_DilationY, descriptor.m_PadTop, descriptor.m_PadBottom, tfLiteNodeParameters->padding); CalcPadding(inputWidth, filterWidth, descriptor.m_StrideX, descriptor.m_DilationX, descriptor.m_PadLeft, descriptor.m_PadRight, tfLiteNodeParameters->padding); armnn::BackendId setBackend; if (!delegateData.m_Network) { bool isSupported = false; FORWARD_LAYER_OPAQUE_SUPPORT_FUNC("CONV2D", tfLiteContext, IsConvolution2dSupported, delegateData.m_Backends, isSupported, setBackend, inputTensorInfo, outputTensorInfo, descriptor, filterTensorInfo, optionalBiasInfo); return isSupported ? kTfLiteOk : kTfLiteError; } // Set up filter and biases auto layerName = GetName(armnn::LayerType::Convolution2d, nodeIndex); armnn::IConnectableLayer* layer = delegateData.m_Network->AddConvolution2dLayer(descriptor, layerName.c_str()); layer->SetBackendId(setBackend); if(filterTensorInfo.IsConstant()) { auto filter = CreateConstTensor(tfLiteFilterTensor, filterTensorInfo); auto filterName = GetName(armnn::LayerType::Constant, nodeIndex, "Filter"); armnn::IConnectableLayer* weightsLayer = delegateData.m_Network->AddConstantLayer(filter, filterName.c_str()); weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1u)); weightsLayer->GetOutputSlot(0).SetTensorInfo(filterTensorInfo); } if (biasEnabled) { if (biasTensorInfo.IsConstant()) { auto biasTensor = CreateConstTensor(tfLiteBiasTensor, biasTensorInfo); auto biasName = GetName(armnn::LayerType::Constant, nodeIndex, "Bias"); armnn::IConnectableLayer* biasLayer = delegateData.m_Network->AddConstantLayer(biasTensor, biasName.c_str()); ARMNN_ASSERT(biasLayer != nullptr); biasLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(2u)); biasLayer->GetOutputSlot(0).SetTensorInfo(biasTensorInfo); } } // The data input can also be constant, so we must check that this is also allocated to an input slot if (inputTensorInfo.IsConstant()) { auto input = CreateConstTensor(tfLiteInputTensor, inputTensorInfo); auto inputName = GetName(armnn::LayerType::Constant, nodeIndex, "Input"); armnn::IConnectableLayer* inputLayer = delegateData.m_Network->AddConstantLayer(input, inputName.c_str()); inputLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(0u)); inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo); } ARMNN_ASSERT(layer != nullptr); armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(0); outputSlot.SetTensorInfo(outputTensorInfo); if (Connect(layer, tfLiteContext, tfLiteNode, delegateData) != kTfLiteOk) { return kTfLiteError; } if (!tfLiteNodeParameters) { // No Activation return kTfLiteOk; } // Check and Create activation return FusedActivation(tfLiteContext, tfLiteNode, activationType, layer, 0, delegateData, nodeIndex); } TfLiteStatus VisitDepthwiseConv2dOperator(DelegateData& delegateData, TfLiteOpaqueContext* tfLiteContext, TfLiteOpaqueNode* tfLiteNode, int nodeIndex, int32_t operatorCode) { auto numInputs = TfLiteOpaqueNodeNumberOfInputs(tfLiteNode); if (numInputs < 2) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Minimum number of inputs (%d != %d) in node #%d", 2, numInputs, nodeIndex); return kTfLiteError; } TF_LITE_ENSURE_STATUS(ValidateNumOutputs(tfLiteContext, tfLiteNode, 1, nodeIndex)); // Gather input indices and use to get input tensor. const int* inputTensors; if (TfLiteOpaqueNodeInputs(tfLiteNode, &inputTensors, &numInputs) != kTfLiteOk) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Unable to gather input tensor indices from node #%d: ", nodeIndex); return kTfLiteError; } const TfLiteOpaqueTensor* tfLiteInputTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[0]); if (!IsValid(tfLiteContext, tfLiteInputTensor, operatorCode, nodeIndex)) { return kTfLiteError; } // Use input indices to get filter tensor. const TfLiteOpaqueTensor* tfLiteFilterTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[1]); if (!IsValid(tfLiteContext, tfLiteFilterTensor, operatorCode, nodeIndex)) { return kTfLiteError; } // Gather output indices and use to get output tensors. int numOutputs = 0; const int* outputTensors; if (TfLiteOpaqueNodeOutputs(tfLiteNode, &outputTensors, &numOutputs) != kTfLiteOk) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Unable to gather output tensor indices from node #%d: ", nodeIndex); return kTfLiteError; } const TfLiteOpaqueTensor* tfLiteOutputTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, outputTensors[0]); if (!IsValid(tfLiteContext, tfLiteOutputTensor, operatorCode, nodeIndex)) { return kTfLiteError; } const armnn::TensorInfo& inputTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteInputTensor); const armnn::TensorInfo& filterTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteFilterTensor); const armnn::TensorInfo& outputTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteOutputTensor, true); auto* tfLiteNodeParameters = reinterpret_cast(TfLiteOpaqueNodeGetBuiltinData(tfLiteNode)); TfLiteFusedActivation activationType = kTfLiteActNone; if (tfLiteNodeParameters) { activationType = tfLiteNodeParameters->activation; TfLiteStatus activationStatus = ValidateFusedActivationOperator(delegateData, tfLiteContext, outputTensorInfo, outputTensorInfo, activationType); if(activationStatus != kTfLiteOk) { return kTfLiteError; } } armnn::TensorInfo biasTensorInfo; const TfLiteOpaqueTensor* tfLiteBiasTensor = nullptr; bool biasEnabled = IsOptionalOperandPresent(tfLiteNode, 2); if(biasEnabled) { // Use input indices to get bias tensor. tfLiteBiasTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[2]); if (!IsValid(tfLiteContext, tfLiteBiasTensor, operatorCode, nodeIndex)) { return kTfLiteError; } biasTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteBiasTensor); } else { biasTensorInfo = armnn::TensorInfo(armnn::TensorShape({1}), GetDataType(tfLiteInputTensor)); } armnn::DepthwiseConvolution2dDescriptor descriptor; descriptor.m_BiasEnabled = biasEnabled; descriptor.m_StrideX = NonNegative(tfLiteNodeParameters->stride_width, nodeIndex); descriptor.m_StrideY = NonNegative(tfLiteNodeParameters->stride_height, nodeIndex); descriptor.m_DataLayout = armnn::DataLayout::NHWC; descriptor.m_DilationX = NonNegative(tfLiteNodeParameters->dilation_width_factor, nodeIndex); descriptor.m_DilationY = NonNegative(tfLiteNodeParameters->dilation_height_factor, nodeIndex); // Assuming input is NHWC unsigned int inputHeight = inputTensorInfo.GetShape()[1]; unsigned int inputWidth = inputTensorInfo.GetShape()[2]; // TensorflowLite weights come in the format [1, H, W, I * M] unsigned int filterHeight = filterTensorInfo.GetShape()[1]; unsigned int filterWidth = filterTensorInfo.GetShape()[2]; // Calculate padding CalcPadding(inputHeight, filterHeight, descriptor.m_StrideY, descriptor.m_DilationY, descriptor.m_PadTop, descriptor.m_PadBottom, tfLiteNodeParameters->padding); CalcPadding(inputWidth, filterWidth, descriptor.m_StrideX, descriptor.m_DilationX, descriptor.m_PadLeft, descriptor.m_PadRight, tfLiteNodeParameters->padding); armnn::BackendId setBackend; if (!delegateData.m_Network) { bool isSupported = false; FORWARD_LAYER_OPAQUE_SUPPORT_FUNC("DEPTHWISE_CONV2D", tfLiteContext, IsDepthwiseConvolutionSupported, delegateData.m_Backends, isSupported, setBackend, inputTensorInfo, outputTensorInfo, descriptor, filterTensorInfo, armnn::Optional(biasTensorInfo)); return isSupported ? kTfLiteOk : kTfLiteError; } auto layerName = GetName(armnn::LayerType::DepthwiseConvolution2d, nodeIndex); armnn::IConnectableLayer* layer = delegateData.m_Network->AddDepthwiseConvolution2dLayer(descriptor, layerName.c_str()); layer->SetBackendId(setBackend); if(filterTensorInfo.IsConstant()) { // For depthwise the weights layout is the same as for tflite [1, H, W, I*M]. No permutation required. auto filter = CreateConstTensor(tfLiteFilterTensor, filterTensorInfo); auto filterName = GetName(armnn::LayerType::Constant, nodeIndex, "Filter"); armnn::IConnectableLayer* weightsLayer = delegateData.m_Network->AddConstantLayer(filter, filterName.c_str()); weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1u)); weightsLayer->GetOutputSlot(0).SetTensorInfo(filterTensorInfo); } if (biasEnabled) { if(biasTensorInfo.IsConstant()) { auto biasTensor = CreateConstTensor(tfLiteBiasTensor, biasTensorInfo); auto biasName = GetName(armnn::LayerType::Constant, nodeIndex, "Bias"); armnn::IConnectableLayer* biasLayer = delegateData.m_Network->AddConstantLayer(biasTensor, biasName.c_str()); ARMNN_ASSERT(biasLayer != nullptr); biasLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(2u)); biasLayer->GetOutputSlot(0).SetTensorInfo(biasTensorInfo); } } // The data input can also be constant, so we must check that this is also allocated to an input slot if(inputTensorInfo.IsConstant()) { auto input = CreateConstTensor(tfLiteInputTensor, inputTensorInfo); auto inputName = GetName(armnn::LayerType::Constant, nodeIndex, "Input"); armnn::IConnectableLayer* inputLayer = delegateData.m_Network->AddConstantLayer(input, inputName.c_str()); inputLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(0u)); inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo); } ARMNN_ASSERT(layer != nullptr); armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(0); outputSlot.SetTensorInfo(outputTensorInfo); if(Connect(layer, tfLiteContext, tfLiteNode, delegateData) != kTfLiteOk) { return kTfLiteError; } if (!tfLiteNodeParameters) { // No Activation return kTfLiteOk; } // Check and create activation return FusedActivation(tfLiteContext, tfLiteNode, activationType, layer, 0, delegateData, nodeIndex); } TfLiteStatus VisitConv3dOperator(DelegateData& delegateData, TfLiteOpaqueContext* tfLiteContext, TfLiteOpaqueNode* tfLiteNode, int nodeIndex, int32_t operatorCode) { auto numInputs = TfLiteOpaqueNodeNumberOfInputs(tfLiteNode); if (numInputs < 2) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Minimum number of inputs (%d != %d) in node #%d", 2, numInputs, nodeIndex); return kTfLiteError; } TF_LITE_ENSURE_STATUS(ValidateNumOutputs(tfLiteContext, tfLiteNode, 1, nodeIndex)); armnn::Convolution3dDescriptor descriptor; auto* params = reinterpret_cast(TfLiteOpaqueNodeGetBuiltinData(tfLiteNode)); bool biasEnabled = IsOptionalOperandPresent(tfLiteNode, 2); descriptor.m_BiasEnabled = biasEnabled; descriptor.m_DataLayout = armnn::DataLayout::NDHWC; descriptor.m_StrideX = NonNegative(params->stride_width, nodeIndex); descriptor.m_StrideY = NonNegative(params->stride_height, nodeIndex); descriptor.m_StrideZ = NonNegative(params->stride_depth, nodeIndex); descriptor.m_DilationX = NonNegative(params->dilation_width_factor, nodeIndex); descriptor.m_DilationY = NonNegative(params->dilation_height_factor, nodeIndex); descriptor.m_DilationZ = NonNegative(params->dilation_depth_factor, nodeIndex); // Gather input indices and use to get input tensor. const int* inputTensors; if (TfLiteOpaqueNodeInputs(tfLiteNode, &inputTensors, &numInputs) != kTfLiteOk) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Unable to gather input tensor indices from node #%d: ", nodeIndex); return kTfLiteError; } const TfLiteOpaqueTensor* tfLiteInputTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[0]); if (!IsValid(tfLiteContext, tfLiteInputTensor, operatorCode, nodeIndex)) { return kTfLiteError; } // Use input indices to get filter tensor. const TfLiteOpaqueTensor* tfLiteFilterTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[1]); if (!IsValid(tfLiteContext, tfLiteFilterTensor, operatorCode, nodeIndex)) { return kTfLiteError; } // Gather output indices and use to get output tensors. int numOutputs = 0; const int* outputTensors; if (TfLiteOpaqueNodeOutputs(tfLiteNode, &outputTensors, &numOutputs) != kTfLiteOk) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Unable to gather output tensor indices from node #%d: ", nodeIndex); return kTfLiteError; } const TfLiteOpaqueTensor* tfLiteOutputTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, outputTensors[0]); if (!IsValid(tfLiteContext, tfLiteOutputTensor, operatorCode, nodeIndex)) { return kTfLiteError; } const armnn::TensorInfo& inputTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteInputTensor); const armnn::TensorInfo& outputTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteOutputTensor, true); auto* tfLiteNodeParameters = reinterpret_cast(TfLiteOpaqueNodeGetBuiltinData(tfLiteNode)); TfLiteFusedActivation activationType=kTfLiteActNone; if (tfLiteNodeParameters) { activationType = tfLiteNodeParameters->activation; TfLiteStatus activationStatus = ValidateFusedActivationOperator(delegateData, tfLiteContext, outputTensorInfo, outputTensorInfo, activationType); if(activationStatus != kTfLiteOk) { return kTfLiteError; } } const armnn::TensorInfo& filterTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteFilterTensor); armnn::TensorInfo biasTensorInfo; const TfLiteOpaqueTensor* tfLiteBiasTensor = nullptr; if (biasEnabled) { // Use input indices to get bias tensor. tfLiteBiasTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[2]); if (!IsValid(tfLiteContext, tfLiteBiasTensor, operatorCode, nodeIndex)) { return kTfLiteError; } biasTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteBiasTensor); } else { biasTensorInfo = armnn::TensorInfo(armnn::TensorShape({1}), GetDataType(tfLiteInputTensor)); } armnn::Optional optionalBiasInfo(biasTensorInfo); // TfLite uses NDHWC tensors const unsigned int inputDepth = inputTensorInfo.GetShape()[1]; const unsigned int inputHeight = inputTensorInfo.GetShape()[2]; const unsigned int inputWidth = inputTensorInfo.GetShape()[3]; // Assuming the filter is DHWIO : Depth, Height, Width, OutputChannels, InputChannels const unsigned int filterDepth = filterTensorInfo.GetShape()[0]; const unsigned int filterHeight = filterTensorInfo.GetShape()[1]; const unsigned int filterWidth = filterTensorInfo.GetShape()[2]; // Calculate padding CalcPadding(inputDepth, filterDepth, descriptor.m_StrideZ, descriptor.m_DilationZ, descriptor.m_PadFront, descriptor.m_PadBack, params->padding); CalcPadding(inputHeight, filterHeight, descriptor.m_StrideY, descriptor.m_DilationY, descriptor.m_PadTop, descriptor.m_PadBottom, params->padding); CalcPadding(inputWidth, filterWidth, descriptor.m_StrideX, descriptor.m_DilationX, descriptor.m_PadLeft, descriptor.m_PadRight, params->padding); // If the m_Network is a nullptr, this signals that a prerequisite TfLite callback is required to clarify the // support for the operator // If supported, VisitConvolutionOperator will be called again to add the layer to the network as seen below. armnn::BackendId setBackend; if (!delegateData.m_Network) { bool isSupported = false; FORWARD_LAYER_OPAQUE_SUPPORT_FUNC("CONV3D", tfLiteContext, IsConvolution3dSupported, delegateData.m_Backends, isSupported, setBackend, inputTensorInfo, outputTensorInfo, descriptor, filterTensorInfo, optionalBiasInfo); return isSupported ? kTfLiteOk : kTfLiteError; } auto layerName = GetName(armnn::LayerType::Convolution3d, nodeIndex); armnn::IConnectableLayer* layer = delegateData.m_Network->AddConvolution3dLayer(descriptor, layerName.c_str()); layer->SetBackendId(setBackend); ARMNN_ASSERT(layer != nullptr); // Add a constant layer for weights and biases if inputs are constant, // which are connected to the Convolution3d layer as inputs. if (filterTensorInfo.IsConstant()) { auto filter = CreateConstTensor(tfLiteFilterTensor, filterTensorInfo); auto filterName = GetName(armnn::LayerType::Constant, nodeIndex, "Filter"); armnn::IConnectableLayer* weightsLayer = delegateData.m_Network->AddConstantLayer(filter, filterName.c_str()); ARMNN_ASSERT(weightsLayer != nullptr); weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1u)); weightsLayer->GetOutputSlot(0).SetTensorInfo(filterTensorInfo); } if (biasEnabled) { if (biasTensorInfo.IsConstant()) { auto biasTensor = CreateConstTensor(tfLiteBiasTensor, biasTensorInfo); auto biasName = GetName(armnn::LayerType::Constant, nodeIndex, "Bias"); armnn::IConnectableLayer* biasLayer = delegateData.m_Network->AddConstantLayer(biasTensor, biasName.c_str()); ARMNN_ASSERT(biasLayer != nullptr); biasLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(2u)); biasLayer->GetOutputSlot(0).SetTensorInfo(biasTensorInfo); } } // The data input can also be constant, so we must check that this is also allocated to an input slot if (inputTensorInfo.IsConstant()) { auto input = CreateConstTensor(tfLiteInputTensor, inputTensorInfo); auto inputName = GetName(armnn::LayerType::Constant, nodeIndex, "Input"); armnn::IConnectableLayer* inputLayer = delegateData.m_Network->AddConstantLayer(input, inputName.c_str()); inputLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(0u)); inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo); } armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(0); outputSlot.SetTensorInfo(outputTensorInfo); if (Connect(layer, tfLiteContext, tfLiteNode, delegateData) != kTfLiteOk) { return kTfLiteError; } if (!tfLiteNodeParameters) { // No Activation return kTfLiteOk; } // Check and create activation return FusedActivation(tfLiteContext, tfLiteNode, activationType, layer, 0, delegateData, nodeIndex); } TfLiteStatus VisitTransposeConv2dOperator(DelegateData& delegateData, TfLiteOpaqueContext* tfLiteContext, TfLiteOpaqueNode* tfLiteNode, int nodeIndex, int32_t operatorCode) { TF_LITE_ENSURE_STATUS(ValidateNumInputs(tfLiteContext, tfLiteNode, 3, nodeIndex)); TF_LITE_ENSURE_STATUS(ValidateNumOutputs(tfLiteContext, tfLiteNode, 1, nodeIndex)); armnn::TransposeConvolution2dDescriptor descriptor; auto* parameters = reinterpret_cast(TfLiteOpaqueNodeGetBuiltinData(tfLiteNode)); descriptor.m_BiasEnabled = false; descriptor.m_StrideX = NonNegative(parameters->stride_width, nodeIndex); descriptor.m_StrideY = NonNegative(parameters->stride_height, nodeIndex); descriptor.m_DataLayout = armnn::DataLayout::NHWC; auto numInputs = TfLiteOpaqueNodeNumberOfInputs(tfLiteNode); // Gather input indices and use to get input tensor. const int* inputTensors; if (TfLiteOpaqueNodeInputs(tfLiteNode, &inputTensors, &numInputs) != kTfLiteOk) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Unable to gather input tensor indices from node #%d: ", nodeIndex); return kTfLiteError; } const TfLiteOpaqueTensor* tfLiteOutputShapeTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[0]); if (!IsValid(tfLiteContext, tfLiteOutputShapeTensor, operatorCode, nodeIndex)) { return kTfLiteError; } const TfLiteOpaqueTensor* tfLiteInputTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[2]); if (!IsValid(tfLiteContext, tfLiteInputTensor, operatorCode, nodeIndex)) { return kTfLiteError; } const TfLiteOpaqueTensor* tfLiteFilterTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, inputTensors[1]); if (!IsValid(tfLiteContext, tfLiteFilterTensor, operatorCode, nodeIndex)) { return kTfLiteError; } // Gather output indices and use to get output tensors. int numOutputs = 0; const int* outputTensors; if (TfLiteOpaqueNodeOutputs(tfLiteNode, &outputTensors, &numOutputs) != kTfLiteOk) { TF_LITE_OPAQUE_MAYBE_KERNEL_LOG( tfLiteContext, "TfLiteArmnnOpaqueDelegate: Unable to gather output tensor indices from node #%d: ", nodeIndex); return kTfLiteError; } const TfLiteOpaqueTensor* tfLiteOutputTensor = TfLiteOpaqueContextGetOpaqueTensor(tfLiteContext, outputTensors[0]); if (!IsValid(tfLiteContext, tfLiteOutputTensor, operatorCode, nodeIndex)) { return kTfLiteError; } const armnn::TensorInfo& inputTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteInputTensor); const armnn::TensorInfo& outputTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteOutputTensor, true); const armnn::TensorInfo& filterTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteFilterTensor); // TfLite uses NHWC tensors const unsigned int inputHeight = inputTensorInfo.GetShape()[1]; const unsigned int inputWidth = inputTensorInfo.GetShape()[2]; const unsigned int filterHeight = filterTensorInfo.GetShape()[1]; const unsigned int filterWidth = filterTensorInfo.GetShape()[2]; // This block determines the output shape of the transpose convolution. // If the output shape tensor is a constant, we can access the data at load time and set the shape of the layer. // If this is not constant, we do not have access to the shape data, so we have to use infer output shape. if (IsConstantTensor(tfLiteOutputShapeTensor)) { const armnn::TensorInfo outputShapeTensorInfo = GetTensorInfoForTfLiteOpaqueTensor(tfLiteOutputShapeTensor); std::vector outputShape(outputShapeTensorInfo.GetNumElements()); if (outputShapeTensorInfo.GetDataType() == armnn::DataType::Signed32) { for(unsigned int i=0; i < outputShapeTensorInfo.GetNumElements(); ++i) { outputShape[i] = static_cast(TfLiteOpaqueTensorData(tfLiteOutputShapeTensor))[i]; } } if (outputShapeTensorInfo.GetDataType() == armnn::DataType::QAsymmU8) { for(unsigned int i=0; i < outputShapeTensorInfo.GetNumElements(); ++i) { outputShape[i] = static_cast(TfLiteOpaqueTensorData(tfLiteOutputShapeTensor))[i]; } } // Change from signed to unsigned int to store in TransposeConvolution2dDescriptor. for (int dimension : outputShape) { descriptor.m_OutputShape.push_back(static_cast(dimension)); } descriptor.m_OutputShapeEnabled = true; // TfLite uses NHWC tensors const unsigned int outputHeight = descriptor.m_OutputShape[1]; const unsigned int outputWidth = descriptor.m_OutputShape[2]; CalcPadding(inputHeight, filterHeight, descriptor.m_StrideY, 1, // DilationY descriptor.m_PadTop, descriptor.m_PadBottom, parameters->padding, outputHeight); CalcPadding(inputWidth, filterWidth, descriptor.m_StrideX, 1, // DilationX descriptor.m_PadLeft, descriptor.m_PadRight, parameters->padding, outputWidth); } else { CalcPadding(inputHeight, filterHeight, descriptor.m_StrideY, 1, // DilationY descriptor.m_PadTop, descriptor.m_PadBottom, parameters->padding); CalcPadding(inputWidth, filterWidth, descriptor.m_StrideX, 1, // DilationX descriptor.m_PadLeft, descriptor.m_PadRight, parameters->padding); } // Set up filter auto filterTensor = CreateConstTensor(tfLiteFilterTensor, filterTensorInfo); armnn::BackendId setBackend; if (!delegateData.m_Network) { bool isSupported = false; FORWARD_LAYER_OPAQUE_SUPPORT_FUNC("TRANSPOSE_CONV2D", tfLiteContext, IsTransposeConvolution2dSupported, delegateData.m_Backends, isSupported, setBackend, inputTensorInfo, outputTensorInfo, descriptor, filterTensorInfo, armnn::EmptyOptional()); return isSupported ? kTfLiteOk : kTfLiteError; } auto layerName = GetName(armnn::LayerType::TransposeConvolution2d, nodeIndex); armnn::IConnectableLayer* layer = delegateData.m_Network->AddTransposeConvolution2dLayer(descriptor, filterTensor, armnn::EmptyOptional(), layerName.c_str()); layer->SetBackendId(setBackend); ARMNN_ASSERT(layer != nullptr); // The data input can be constant, so we must check that this is allocated to an input slot if(inputTensorInfo.IsConstant()) { auto input = CreateConstTensor(tfLiteInputTensor, inputTensorInfo); auto inputName = GetName(armnn::LayerType::Constant, nodeIndex, "Input"); armnn::IConnectableLayer *inputLayer = delegateData.m_Network->AddConstantLayer(input, inputName.c_str()); inputLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(0u)); inputLayer->GetOutputSlot(0).SetTensorInfo(inputTensorInfo); } armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(0); outputSlot.SetTensorInfo(outputTensorInfo); // Connect if (delegateData.m_OutputSlotForNode[static_cast(inputTensors[2])] != nullptr) { delegateData.m_OutputSlotForNode[static_cast(inputTensors[2])]-> Connect(layer->GetInputSlot(0)); } if (Connect(layer, tfLiteContext, tfLiteNode, delegateData) != kTfLiteOk) { return kTfLiteError; } return kTfLiteOk; } TfLiteStatus VisitConvolutionOperator(DelegateData& delegateData, TfLiteOpaqueContext* tfLiteContext, TfLiteOpaqueNode* tfLiteNode, int nodeIndex, int32_t operatorCode) { switch(operatorCode) { case kTfLiteBuiltinConv2d: return VisitConv2dOperator(delegateData, tfLiteContext, tfLiteNode, nodeIndex, operatorCode); case kTfLiteBuiltinConv3d: return VisitConv3dOperator(delegateData, tfLiteContext, tfLiteNode, nodeIndex, operatorCode); case kTfLiteBuiltinDepthwiseConv2d: return VisitDepthwiseConv2dOperator(delegateData, tfLiteContext, tfLiteNode, nodeIndex, operatorCode); case kTfLiteBuiltinTransposeConv: return VisitTransposeConv2dOperator(delegateData, tfLiteContext, tfLiteNode, nodeIndex, operatorCode); default: return kTfLiteError; } } }