From dbb0c0ca0c8425886ee3a2095e0ced07099134f9 Mon Sep 17 00:00:00 2001 From: Sadik Armagan Date: Thu, 21 Feb 2019 09:01:41 +0000 Subject: IVGCVSW-2639 Add Serializer & Deserializer for Fully Connected * Added FullyConnectedLayer to Serializer Schema Schema.fbs * Added FullyConnected serialization and deserialization support * Added FullyConnected serialization and deserialization unit tests Change-Id: I8ef14f9728158f849fa4d1a8d05a1a4170cd5b41 Signed-off-by: Sadik Armagan Signed-off-by: Aron Virginas-Tar --- CMakeLists.txt | 1 + src/armnnDeserializer/Deserializer.cpp | 47 +++++++ src/armnnDeserializer/Deserializer.hpp | 1 + .../test/DeserializeFullyConnected.cpp | 140 +++++++++++++++++++++ src/armnnSerializer/Schema.fbs | 16 ++- src/armnnSerializer/Serializer.cpp | 39 +++++- src/armnnSerializer/Serializer.hpp | 6 + src/armnnSerializer/test/SerializerTests.cpp | 41 ++++++ 8 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 src/armnnDeserializer/test/DeserializeFullyConnected.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 147db7175c..813ac2c318 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -590,6 +590,7 @@ if(BUILD_UNIT_TESTS) src/armnnDeserializer/test/DeserializeActivation.cpp src/armnnDeserializer/test/DeserializeAdd.cpp src/armnnDeserializer/test/DeserializeConvolution2d.cpp + src/armnnDeserializer/test/DeserializeFullyConnected.cpp src/armnnDeserializer/test/DeserializeMultiplication.cpp src/armnnDeserializer/test/DeserializePermute.cpp src/armnnDeserializer/test/DeserializePooling2d.cpp diff --git a/src/armnnDeserializer/Deserializer.cpp b/src/armnnDeserializer/Deserializer.cpp index ead4fc5453..9ec7835021 100644 --- a/src/armnnDeserializer/Deserializer.cpp +++ b/src/armnnDeserializer/Deserializer.cpp @@ -174,6 +174,7 @@ m_ParserFunctions(Layer_MAX+1, &Deserializer::ParseUnsupportedLayer) m_ParserFunctions[Layer_AdditionLayer] = &Deserializer::ParseAdd; m_ParserFunctions[Layer_Convolution2dLayer] = &Deserializer::ParseConvolution2d; m_ParserFunctions[Layer_DepthwiseConvolution2dLayer] = &Deserializer::ParseDepthwiseConvolution2d; + m_ParserFunctions[Layer_FullyConnectedLayer] = &Deserializer::ParseFullyConnected; m_ParserFunctions[Layer_MultiplicationLayer] = &Deserializer::ParseMultiplication; m_ParserFunctions[Layer_PermuteLayer] = &Deserializer::ParsePermute; m_ParserFunctions[Layer_Pooling2dLayer] = &Deserializer::ParsePooling2d; @@ -195,6 +196,8 @@ Deserializer::LayerBaseRawPtr Deserializer::GetBaseLayer(const GraphPtr& graphPt return graphPtr->layers()->Get(layerIndex)->layer_as_Convolution2dLayer()->base(); case Layer::Layer_DepthwiseConvolution2dLayer: return graphPtr->layers()->Get(layerIndex)->layer_as_DepthwiseConvolution2dLayer()->base(); + case Layer::Layer_FullyConnectedLayer: + return graphPtr->layers()->Get(layerIndex)->layer_as_FullyConnectedLayer()->base(); case Layer::Layer_InputLayer: return graphPtr->layers()->Get(layerIndex)->layer_as_InputLayer()->base()->base(); case Layer::Layer_MultiplicationLayer: @@ -834,6 +837,50 @@ void Deserializer::ParseMultiplication(unsigned int layerIndex) RegisterOutputSlots(layerIndex, layer); } +void Deserializer::ParseFullyConnected(unsigned int layerIndex) +{ + CHECK_LAYERS(m_Graph, 0, layerIndex); + auto inputs = GetInputs(m_Graph, layerIndex); + CHECK_LOCATION(); + CHECK_VALID_SIZE(inputs.size(), 1); + + auto outputs = GetOutputs(m_Graph, layerIndex); + CHECK_VALID_SIZE(outputs.size(), 1); + + auto layerName = boost::str(boost::format("FullyConnected:%1%") % layerIndex); + + auto flatBufferLayer = m_Graph->layers()->Get(layerIndex)->layer_as_FullyConnectedLayer(); + auto flatBufferDescriptor = flatBufferLayer->descriptor(); + + armnn::FullyConnectedDescriptor fullyConnectedDescriptor; + fullyConnectedDescriptor.m_BiasEnabled = flatBufferDescriptor->biasEnabled(); + fullyConnectedDescriptor.m_TransposeWeightMatrix = flatBufferDescriptor->transposeWeightsMatrix(); + + armnn::ConstTensor weightsTensor = ToConstTensor(flatBufferLayer->weights()); + + armnn::IConnectableLayer* layer; + if (flatBufferDescriptor->biasEnabled()) + { + armnn::ConstTensor biasTensorData = ToConstTensor(flatBufferLayer->biases()); + layer = m_Network->AddFullyConnectedLayer(fullyConnectedDescriptor, + weightsTensor, + biasTensorData, + layerName.c_str()); + } + else + { + layer = m_Network->AddFullyConnectedLayer(fullyConnectedDescriptor, + weightsTensor, + layerName.c_str()); + } + + armnn::TensorInfo outputTensorInfo = ToTensorInfo(outputs[0]); + layer->GetOutputSlot(0).SetTensorInfo(outputTensorInfo); + + RegisterInputSlots(layerIndex, layer); + RegisterOutputSlots(layerIndex, layer); +} + void Deserializer::ParsePermute(unsigned int layerIndex) { CHECK_LAYERS(m_Graph, 0, layerIndex); diff --git a/src/armnnDeserializer/Deserializer.hpp b/src/armnnDeserializer/Deserializer.hpp index 6af1afb776..25c651a50b 100644 --- a/src/armnnDeserializer/Deserializer.hpp +++ b/src/armnnDeserializer/Deserializer.hpp @@ -72,6 +72,7 @@ private: void ParseAdd(unsigned int layerIndex); void ParseConvolution2d(unsigned int layerIndex); void ParseDepthwiseConvolution2d(unsigned int layerIndex); + void ParseFullyConnected(unsigned int layerIndex); void ParseMultiplication(unsigned int layerIndex); void ParsePermute(unsigned int layerIndex); void ParsePooling2d(unsigned int layerIndex); diff --git a/src/armnnDeserializer/test/DeserializeFullyConnected.cpp b/src/armnnDeserializer/test/DeserializeFullyConnected.cpp new file mode 100644 index 0000000000..77d0acc782 --- /dev/null +++ b/src/armnnDeserializer/test/DeserializeFullyConnected.cpp @@ -0,0 +1,140 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include +#include "ParserFlatbuffersSerializeFixture.hpp" +#include "../Deserializer.hpp" + +#include +#include + +BOOST_AUTO_TEST_SUITE(DeserializeParser) + +struct FullyConnectedFixture : public ParserFlatbuffersSerializeFixture +{ + explicit FullyConnectedFixture(const std::string & inputShape1, + const std::string & outputShape, + const std::string & weightsShape, + const std::string & dataType) + { + m_JsonString = R"( + { + inputIds: [0], + outputIds: [2], + layers: [{ + layer_type: "InputLayer", + layer: { + base: { + layerBindingId: 0, + base: { + index: 0, + layerName: "InputLayer", + layerType: "Input", + inputSlots: [{ + index: 0, + connection: {sourceLayerIndex:0, outputSlotIndex:0 }, + }], + outputSlots: [{ + index: 0, + tensorInfo: { + dimensions: )" + inputShape1 + R"(, + dataType: )" + dataType + R"(, + quantizationScale: 1.0, + quantizationOffset: 0 + }, + }] + }, + } + }, + }, + { + layer_type: "FullyConnectedLayer", + layer : { + base: { + index:1, + layerName: "FullyConnectedLayer", + layerType: "FullyConnected", + inputSlots: [{ + index: 0, + connection: {sourceLayerIndex:0, outputSlotIndex:0 }, + }], + outputSlots: [{ + index: 0, + tensorInfo: { + dimensions: )" + outputShape + R"(, + dataType: )" + dataType + R"(, + quantizationScale: 2.0, + quantizationOffset: 0 + }, + }], + }, + descriptor: { + biasEnabled: false, + transposeWeightsMatrix: true + }, + weights: { + info: { + dimensions: )" + weightsShape + R"(, + dataType: )" + dataType + R"(, + quantizationScale: 1.0, + quantizationOffset: 0 + }, + data_type: ByteData, + data: { + data: [ + 2, 3, 4, 5 + ], + } + } + }, + }, + { + layer_type: "OutputLayer", + layer: { + base:{ + layerBindingId: 0, + base: { + index: 2, + layerName: "OutputLayer", + layerType: "Output", + inputSlots: [{ + index: 0, + connection: {sourceLayerIndex:1, outputSlotIndex:0 }, + }], + outputSlots: [ { + index: 0, + tensorInfo: { + dimensions: )" + outputShape + R"(, + dataType: )" + dataType + R"( + }, + }], + } + }}, + }] + } + )"; + Setup(); + } +}; + +struct FullyConnectedWithNoBiasFixture : FullyConnectedFixture +{ + FullyConnectedWithNoBiasFixture() + : FullyConnectedFixture("[ 1, 4, 1, 1 ]", // inputShape + "[ 1, 1 ]", // outputShape + "[ 1, 4 ]", // filterShape + "QuantisedAsymm8") // filterData + {} +}; + +BOOST_FIXTURE_TEST_CASE(FullyConnectedWithNoBias, FullyConnectedWithNoBiasFixture) +{ + RunTest<2, armnn::DataType::QuantisedAsymm8>( + 0, + {{"InputLayer", { 10, 20, 30, 40 }}}, + {{"OutputLayer", { 400/2 }}}); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/armnnSerializer/Schema.fbs b/src/armnnSerializer/Schema.fbs index 94ca23b0cd..dc14069798 100644 --- a/src/armnnSerializer/Schema.fbs +++ b/src/armnnSerializer/Schema.fbs @@ -91,7 +91,8 @@ enum LayerType : uint { Convolution2d = 7, DepthwiseConvolution2d = 8, Activation = 9, - Permute = 10 + Permute = 10, + FullyConnected = 11 } // Base layer table to be used as part of other layers @@ -142,6 +143,18 @@ table Convolution2dDescriptor { dataLayout:DataLayout = NCHW; } +table FullyConnectedLayer { + base:LayerBase; + descriptor:FullyConnectedDescriptor; + weights:ConstTensor; + biases:ConstTensor; +} + +table FullyConnectedDescriptor { + biasEnabled:bool = false; + transposeWeightsMatrix:bool = false; +} + table InputLayer { base:BindableLayerBase; } @@ -240,6 +253,7 @@ union Layer { AdditionLayer, Convolution2dLayer, DepthwiseConvolution2dLayer, + FullyConnectedLayer, InputLayer, MultiplicationLayer, OutputLayer, diff --git a/src/armnnSerializer/Serializer.cpp b/src/armnnSerializer/Serializer.cpp index e1d22ec406..b4afd37b99 100644 --- a/src/armnnSerializer/Serializer.cpp +++ b/src/armnnSerializer/Serializer.cpp @@ -324,7 +324,44 @@ void SerializerVisitor::VisitPooling2dLayer(const armnn::IConnectableLayer* laye CreateAnyLayer(fbPooling2dLayer.o, serializer::Layer::Layer_Pooling2dLayer); } -fb::Offset SerializerVisitor::CreateLayerBase(const armnn::IConnectableLayer* layer, +// Build FlatBuffer for FullyConnected Layer +void SerializerVisitor::VisitFullyConnectedLayer(const armnn::IConnectableLayer* layer, + const armnn::FullyConnectedDescriptor& fullyConnectedDescriptor, + const armnn::ConstTensor& weights, + const armnn::Optional& biases, + const char* name) +{ + // Create FlatBuffer BaseLayer + auto flatBufferBaseLayer = CreateLayerBase(layer, serializer::LayerType::LayerType_FullyConnected); + + // Create FlatBuffer FullyConnectedDescriptor + auto flatBufferDescriptor = + serializer::CreateFullyConnectedDescriptor(m_flatBufferBuilder, + fullyConnectedDescriptor.m_BiasEnabled, + fullyConnectedDescriptor.m_TransposeWeightMatrix); + + // Create FlatBuffer weights data + auto flatBufferWeights = CreateConstTensorInfo(weights); + + // Create FlatBuffer bias data + flatbuffers::Offset flatBufferBiases; + if (fullyConnectedDescriptor.m_BiasEnabled) + { + flatBufferBiases = CreateConstTensorInfo(biases.value()); + } + + // Create FlatBuffer FullyConnectedLayer + auto flatBufferLayer = serializer::CreateFullyConnectedLayer(m_flatBufferBuilder, + flatBufferBaseLayer, + flatBufferDescriptor, + flatBufferWeights, + flatBufferBiases); + + // Add created FullyConnectedLayer to the FlatBufferLayers + CreateAnyLayer(flatBufferLayer.o, serializer::Layer::Layer_FullyConnectedLayer); +} + +fb::Offset SerializerVisitor::CreateLayerBase(const IConnectableLayer* layer, const serializer::LayerType layerType) { std::vector> inputSlots = CreateInputSlots(layer); diff --git a/src/armnnSerializer/Serializer.hpp b/src/armnnSerializer/Serializer.hpp index 329b005624..0a62732ef2 100644 --- a/src/armnnSerializer/Serializer.hpp +++ b/src/armnnSerializer/Serializer.hpp @@ -61,6 +61,12 @@ public: const armnn::Optional& biases, const char* name = nullptr) override; + void VisitFullyConnectedLayer(const armnn::IConnectableLayer* layer, + const armnn::FullyConnectedDescriptor& fullyConnectedDescriptor, + const armnn::ConstTensor& weights, + const armnn::Optional& biases, + const char* name = nullptr) override; + void VisitInputLayer(const armnn::IConnectableLayer* layer, armnn::LayerBindingId id, const char* name = nullptr) override; diff --git a/src/armnnSerializer/test/SerializerTests.cpp b/src/armnnSerializer/test/SerializerTests.cpp index 822f9c7e00..ede24baf9e 100644 --- a/src/armnnSerializer/test/SerializerTests.cpp +++ b/src/armnnSerializer/test/SerializerTests.cpp @@ -404,4 +404,45 @@ BOOST_AUTO_TEST_CASE(SerializeDeserializePermute) outputTensorInfo.GetShape()); } +BOOST_AUTO_TEST_CASE(SerializeDeserializeFullyConnected) +{ + armnn::TensorInfo inputInfo ({ 2, 5, 1, 1 }, armnn::DataType::Float32); + armnn::TensorInfo outputInfo({ 2, 3 }, armnn::DataType::Float32); + + armnn::TensorInfo weightsInfo({ 5, 3 }, armnn::DataType::Float32); + armnn::TensorInfo biasesInfo ({ 3 }, armnn::DataType::Float32); + + armnn::FullyConnectedDescriptor descriptor; + descriptor.m_BiasEnabled = true; + descriptor.m_TransposeWeightMatrix = false; + + std::vector weightsData = GenerateRandomData(weightsInfo.GetNumElements()); + std::vector biasesData = GenerateRandomData(biasesInfo.GetNumElements()); + + armnn::ConstTensor weights(weightsInfo, weightsData); + armnn::ConstTensor biases(biasesInfo, biasesData); + + armnn::INetworkPtr network = armnn::INetwork::Create(); + armnn::IConnectableLayer* const inputLayer = network->AddInputLayer(0, "input"); + armnn::IConnectableLayer* const fullyConnectedLayer = network->AddFullyConnectedLayer(descriptor, + weights, + biases, + "fully_connected"); + armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(0, "output"); + + inputLayer->GetOutputSlot(0).Connect(fullyConnectedLayer->GetInputSlot(0)); + inputLayer->GetOutputSlot(0).SetTensorInfo(inputInfo); + + fullyConnectedLayer->GetOutputSlot(0).Connect(outputLayer->GetInputSlot(0)); + fullyConnectedLayer->GetOutputSlot(0).SetTensorInfo(outputInfo); + + armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network)); + BOOST_CHECK(deserializedNetwork); + + CheckDeserializedNetworkAgainstOriginal(*network, + *deserializedNetwork, + inputInfo.GetShape(), + outputInfo.GetShape()); +} + BOOST_AUTO_TEST_SUITE_END() -- cgit v1.2.1