IVGCVSW-2686 Add Serializer and Deserializer for DetectionPostProcess

Change-Id: Ife48db5fdb005ebca0a6f21862b0ce971ccf58b7
Signed-off-by: Nattapat Chaimanowong <nattapat.chaimanowong@arm.com>

8 files changed, 190 insertions, 2 deletions

diff --git a/src/armnnDeserializer/Deserializer.cpp b/src/armnnDeserializer/Deserializer.cpp
index ba12c37ed8..fc84462abc 100644
--- a/src/armnnDeserializer/Deserializer.cpp
+++ b/src/armnnDeserializer/Deserializer.cpp
@@ -193,6 +193,7 @@ m_ParserFunctions(Layer_MAX+1, &Deserializer::ParseUnsupportedLayer)
     m_ParserFunctions[Layer_ConstantLayer]               = &Deserializer::ParseConstant;
     m_ParserFunctions[Layer_Convolution2dLayer]          = &Deserializer::ParseConvolution2d;
     m_ParserFunctions[Layer_DepthwiseConvolution2dLayer] = &Deserializer::ParseDepthwiseConvolution2d;
+    m_ParserFunctions[Layer_DetectionPostProcessLayer]   = &Deserializer::ParseDetectionPostProcess;
     m_ParserFunctions[Layer_DivisionLayer]               = &Deserializer::ParseDivision;
     m_ParserFunctions[Layer_EqualLayer]                  = &Deserializer::ParseEqual;
     m_ParserFunctions[Layer_FullyConnectedLayer]         = &Deserializer::ParseFullyConnected;
@@ -239,6 +240,8 @@ Deserializer::LayerBaseRawPtr Deserializer::GetBaseLayer(const GraphPtr& graphPt
             return graphPtr->layers()->Get(layerIndex)->layer_as_Convolution2dLayer()->base();
         case Layer::Layer_DepthwiseConvolution2dLayer:
             return graphPtr->layers()->Get(layerIndex)->layer_as_DepthwiseConvolution2dLayer()->base();
+        case Layer::Layer_DetectionPostProcessLayer:
+            return graphPtr->layers()->Get(layerIndex)->layer_as_DetectionPostProcessLayer()->base();
         case Layer::Layer_DivisionLayer:
             return graphPtr->layers()->Get(layerIndex)->layer_as_DivisionLayer()->base();
         case Layer::Layer_EqualLayer:
@@ -1021,6 +1024,48 @@ void Deserializer::ParseDepthwiseConvolution2d(GraphPtr graph, unsigned int laye
     RegisterOutputSlots(graph, layerIndex, layer);
 }
 
+void Deserializer::ParseDetectionPostProcess(GraphPtr graph, unsigned int layerIndex)
+{
+    CHECK_LAYERS(graph, 0, layerIndex);
+    auto inputs = GetInputs(graph, layerIndex);
+    CHECK_LOCATION();
+    CHECK_VALID_SIZE(inputs.size(), 2);
+
+    auto outputs = GetOutputs(graph, layerIndex);
+    CHECK_VALID_SIZE(outputs.size(), 4);
+
+    auto flatBufferLayer = graph->layers()->Get(layerIndex)->layer_as_DetectionPostProcessLayer();
+    auto layerName = GetLayerName(graph, layerIndex);
+    auto flatBufferDescriptor = flatBufferLayer->descriptor();
+
+    armnn::DetectionPostProcessDescriptor descriptor;
+    descriptor.m_MaxDetections = flatBufferDescriptor->maxDetections();
+    descriptor.m_MaxClassesPerDetection = flatBufferDescriptor->maxClassesPerDetection();
+    descriptor.m_DetectionsPerClass = flatBufferDescriptor->detectionsPerClass();
+    descriptor.m_NmsScoreThreshold = flatBufferDescriptor->nmsScoreThreshold();
+    descriptor.m_NmsIouThreshold = flatBufferDescriptor->nmsIouThreshold();
+    descriptor.m_NumClasses = flatBufferDescriptor->numClasses();
+    descriptor.m_UseRegularNms = flatBufferDescriptor->useRegularNms();
+    descriptor.m_ScaleX = flatBufferDescriptor->scaleX();
+    descriptor.m_ScaleY = flatBufferDescriptor->scaleY();
+    descriptor.m_ScaleW = flatBufferDescriptor->scaleW();
+    descriptor.m_ScaleH = flatBufferDescriptor->scaleH();
+
+    armnn::ConstTensor anchors = ToConstTensor(flatBufferLayer->anchors());
+
+    IConnectableLayer* layer = m_Network->AddDetectionPostProcessLayer(descriptor,
+                                                                       anchors,
+                                                                       layerName.c_str());
+
+    for (unsigned int i = 0; i < 4; i++)
+    {
+        layer->GetOutputSlot(i).SetTensorInfo(ToTensorInfo(outputs[i]));
+    }
+
+    RegisterInputSlots(graph, layerIndex, layer);
+    RegisterOutputSlots(graph, layerIndex, layer);
+}
+
 void Deserializer::ParseDivision(GraphPtr graph, unsigned int layerIndex)
 {
     CHECK_LAYERS(graph, 0, layerIndex);
diff --git a/src/armnnDeserializer/Deserializer.hpp b/src/armnnDeserializer/Deserializer.hpp
index 67e6e849e8..effc7ae144 100644
--- a/src/armnnDeserializer/Deserializer.hpp
+++ b/src/armnnDeserializer/Deserializer.hpp
@@ -80,6 +80,7 @@ private:
     void ParseConstant(GraphPtr graph, unsigned int layerIndex);
     void ParseConvolution2d(GraphPtr graph, unsigned int layerIndex);
     void ParseDepthwiseConvolution2d(GraphPtr graph, unsigned int layerIndex);
+    void ParseDetectionPostProcess(GraphPtr graph, unsigned int layerIndex);
     void ParseDivision(GraphPtr graph, unsigned int layerIndex);
     void ParseEqual(GraphPtr graph, unsigned int layerIndex);
     void ParseFloor(GraphPtr graph, unsigned int layerIndex);
diff --git a/src/armnnDeserializer/DeserializerSupport.md b/src/armnnDeserializer/DeserializerSupport.md
index 398489b5b0..48b8c88103 100644
--- a/src/armnnDeserializer/DeserializerSupport.md
+++ b/src/armnnDeserializer/DeserializerSupport.md
@@ -13,6 +13,7 @@ The Arm NN SDK Deserialize parser currently supports the following layers:
 * Constant
 * Convolution2d
 * DepthwiseConvolution2d
+* DetectionPostProcess
 * Division
 * Equal
 * Floor
diff --git a/src/armnnSerializer/ArmnnSchema.fbs b/src/armnnSerializer/ArmnnSchema.fbs
index 40ee7a5e45..00c1a4502b 100644
--- a/src/armnnSerializer/ArmnnSchema.fbs
+++ b/src/armnnSerializer/ArmnnSchema.fbs
@@ -113,7 +113,8 @@ enum LayerType : uint {
     Mean = 29,
     Merger = 30,
     L2Normalization = 31,
-    Splitter = 32
+    Splitter = 32,
+    DetectionPostProcess = 33
 }
 
 // Base layer table to be used as part of other layers
@@ -453,6 +454,26 @@ table SplitterLayer {
    descriptor:ViewsDescriptor;
 }
 
+table DetectionPostProcessLayer {
+    base:LayerBase;
+    descriptor:DetectionPostProcessDescriptor;
+    anchors:ConstTensor;
+}
+
+table DetectionPostProcessDescriptor {
+    maxDetections:uint;
+    maxClassesPerDetection:uint;
+    detectionsPerClass:uint;
+    nmsScoreThreshold:float;
+    nmsIouThreshold:float;
+    numClasses:uint;
+    useRegularNms:bool;
+    scaleX:float;
+    scaleY:float;
+    scaleW:float;
+    scaleH:float;
+}
+
 union Layer {
     ActivationLayer,
     AdditionLayer,
@@ -486,7 +507,8 @@ union Layer {
     MeanLayer,
     MergerLayer,
     L2NormalizationLayer,
-    SplitterLayer
+    SplitterLayer,
+    DetectionPostProcessLayer
 }
 
 table AnyLayer {
diff --git a/src/armnnSerializer/Serializer.cpp b/src/armnnSerializer/Serializer.cpp
index 3774c250e1..0340f56d17 100644
--- a/src/armnnSerializer/Serializer.cpp
+++ b/src/armnnSerializer/Serializer.cpp
@@ -289,6 +289,35 @@ void SerializerVisitor::VisitDepthwiseConvolution2dLayer(const armnn::IConnectab
     CreateAnyLayer(flatBufferLayer.o, serializer::Layer::Layer_DepthwiseConvolution2dLayer);
 }
 
+void SerializerVisitor::VisitDetectionPostProcessLayer(const armnn::IConnectableLayer* layer,
+                                                       const armnn::DetectionPostProcessDescriptor& descriptor,
+                                                       const armnn::ConstTensor& anchors,
+                                                       const char* name)
+{
+    auto fbBaseLayer  = CreateLayerBase(layer, serializer::LayerType::LayerType_DetectionPostProcess);
+    auto fbDescriptor = CreateDetectionPostProcessDescriptor(m_flatBufferBuilder,
+                                                             descriptor.m_MaxDetections,
+                                                             descriptor.m_MaxClassesPerDetection,
+                                                             descriptor.m_DetectionsPerClass,
+                                                             descriptor.m_NmsScoreThreshold,
+                                                             descriptor.m_NmsIouThreshold,
+                                                             descriptor.m_NumClasses,
+                                                             descriptor.m_UseRegularNms,
+                                                             descriptor.m_ScaleX,
+                                                             descriptor.m_ScaleY,
+                                                             descriptor.m_ScaleW,
+                                                             descriptor.m_ScaleH);
+
+    flatbuffers::Offset<serializer::ConstTensor> fbAnchorsConstTensorInfo = CreateConstTensorInfo(anchors);
+
+    auto flatBufferLayer = CreateDetectionPostProcessLayer(m_flatBufferBuilder,
+                                                           fbBaseLayer,
+                                                           fbDescriptor,
+                                                           fbAnchorsConstTensorInfo);
+
+    CreateAnyLayer(flatBufferLayer.o, serializer::Layer::Layer_DetectionPostProcessLayer);
+}
+
 void SerializerVisitor::VisitDivisionLayer(const armnn::IConnectableLayer* layer, const char* name)
 {
     auto fbDivisionBaseLayer = CreateLayerBase(layer, serializer::LayerType::LayerType_Division);
diff --git a/src/armnnSerializer/Serializer.hpp b/src/armnnSerializer/Serializer.hpp
index cb05792863..71066d2699 100644
--- a/src/armnnSerializer/Serializer.hpp
+++ b/src/armnnSerializer/Serializer.hpp
@@ -77,6 +77,11 @@ public:
                                           const armnn::Optional<armnn::ConstTensor>& biases,
                                           const char* name = nullptr) override;
 
+    void VisitDetectionPostProcessLayer(const armnn::IConnectableLayer* layer,
+                                        const armnn::DetectionPostProcessDescriptor& descriptor,
+                                        const armnn::ConstTensor& anchors,
+                                        const char* name = nullptr) override;
+
     void VisitDivisionLayer(const armnn::IConnectableLayer* layer,
                             const char* name = nullptr) override;
 
diff --git a/src/armnnSerializer/SerializerSupport.md b/src/armnnSerializer/SerializerSupport.md
index 81d2faa8fe..4e127b3f9f 100644
--- a/src/armnnSerializer/SerializerSupport.md
+++ b/src/armnnSerializer/SerializerSupport.md
@@ -13,6 +13,7 @@ The Arm NN SDK Serializer currently supports the following layers:
 * Constant
 * Convolution2d
 * DepthwiseConvolution2d
+* DetectionPostProcess
 * Division
 * Equal
 * Floor
diff --git a/src/armnnSerializer/test/SerializerTests.cpp b/src/armnnSerializer/test/SerializerTests.cpp
index 41f5d14ce3..d0586c988c 100644
--- a/src/armnnSerializer/test/SerializerTests.cpp
+++ b/src/armnnSerializer/test/SerializerTests.cpp
@@ -1595,4 +1595,88 @@ BOOST_AUTO_TEST_CASE(SerializeDeserializeSplitter)
                                                    {0, 1, 2});
 }
 
+BOOST_AUTO_TEST_CASE(SerializeDeserializeDetectionPostProcess)
+{
+    class VerifyDetectionPostProcessName : public armnn::LayerVisitorBase<armnn::VisitorNoThrowPolicy>
+    {
+    public:
+        void VisitDetectionPostProcessLayer(const armnn::IConnectableLayer* layer,
+                                            const armnn::DetectionPostProcessDescriptor& descriptor,
+                                            const armnn::ConstTensor& anchors,
+                                            const char* name) override
+        {
+            BOOST_TEST(name == "DetectionPostProcessLayer");
+        }
+    };
+
+    const armnn::TensorInfo inputInfos[] = {
+        armnn::TensorInfo({ 1, 6, 4 }, armnn::DataType::Float32),
+        armnn::TensorInfo({ 1, 6, 3}, armnn::DataType::Float32)
+    };
+
+    const armnn::TensorInfo outputInfos[] = {
+        armnn::TensorInfo({ 1, 3, 4 }, armnn::DataType::Float32),
+        armnn::TensorInfo({ 1, 3 }, armnn::DataType::Float32),
+        armnn::TensorInfo({ 1, 3 }, armnn::DataType::Float32),
+        armnn::TensorInfo({ 1 }, armnn::DataType::Float32)
+    };
+
+    armnn::DetectionPostProcessDescriptor desc;
+    desc.m_UseRegularNms = true;
+    desc.m_MaxDetections = 3;
+    desc.m_MaxClassesPerDetection = 1;
+    desc.m_DetectionsPerClass =1;
+    desc.m_NmsScoreThreshold = 0.0;
+    desc.m_NmsIouThreshold = 0.5;
+    desc.m_NumClasses = 2;
+    desc.m_ScaleY = 10.0;
+    desc.m_ScaleX = 10.0;
+    desc.m_ScaleH = 5.0;
+    desc.m_ScaleW = 5.0;
+
+    const armnn::TensorInfo anchorsInfo({ 6, 4 }, armnn::DataType::Float32);
+    const std::vector<float> anchorsData({
+        0.5f, 0.5f, 1.0f, 1.0f,
+        0.5f, 0.5f, 1.0f, 1.0f,
+        0.5f, 0.5f, 1.0f, 1.0f,
+        0.5f, 10.5f, 1.0f, 1.0f,
+        0.5f, 10.5f, 1.0f, 1.0f,
+        0.5f, 100.5f, 1.0f, 1.0f
+    });
+    armnn::ConstTensor anchors(anchorsInfo, anchorsData);
+
+    armnn::INetworkPtr network = armnn::INetwork::Create();
+
+    armnn::IConnectableLayer* const detectionLayer =
+    network->AddDetectionPostProcessLayer(desc, anchors, "DetectionPostProcessLayer");
+
+    for (unsigned int i = 0; i < 2; i++)
+    {
+        armnn::IConnectableLayer* const inputLayer = network->AddInputLayer(static_cast<int>(i));
+        inputLayer->GetOutputSlot(0).Connect(detectionLayer->GetInputSlot(i));
+        inputLayer->GetOutputSlot(0).SetTensorInfo(inputInfos[i]);
+    }
+
+    for (unsigned int i = 0; i < 4; i++)
+    {
+        armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(static_cast<int>(i));
+        detectionLayer->GetOutputSlot(i).Connect(outputLayer->GetInputSlot(0));
+        detectionLayer->GetOutputSlot(i).SetTensorInfo(outputInfos[i]);
+    }
+
+    armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(SerializeNetwork(*network));
+    BOOST_CHECK(deserializedNetwork);
+
+    VerifyDetectionPostProcessName nameChecker;
+    deserializedNetwork->Accept(nameChecker);
+
+    CheckDeserializedNetworkAgainstOriginal<float>(
+        *network,
+        *deserializedNetwork,
+        {inputInfos[0].GetShape(), inputInfos[1].GetShape()},
+        {outputInfos[0].GetShape(), outputInfos[1].GetShape(), outputInfos[2].GetShape(), outputInfos[3].GetShape()},
+        {0, 1},
+        {0, 1, 2, 3});
+}
+
 BOOST_AUTO_TEST_SUITE_END()