IVGCVSW-3291 Add L2Normalization epsilon value to serialization

Signed-off-by: Ferran Balaguer <ferran.balaguer@arm.com>
Change-Id: Icfff3fb596a03c126a42b1d0c254a68e498df734

6 files changed, 231 insertions, 26 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f07295f5b2..976f8dee2e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -623,6 +623,7 @@ if(BUILD_UNIT_TESTS)
             src/armnnDeserializer/test/DeserializeFullyConnected.cpp
             src/armnnDeserializer/test/DeserializeGather.cpp
             src/armnnDeserializer/test/DeserializeGreater.cpp
+            src/armnnDeserializer/test/DeserializeL2Normalization.cpp
             src/armnnDeserializer/test/DeserializeMean.cpp
             src/armnnDeserializer/test/DeserializeMultiplication.cpp
             src/armnnDeserializer/test/DeserializeNormalization.cpp
diff --git a/src/armnnDeserializer/Deserializer.cpp b/src/armnnDeserializer/Deserializer.cpp
index b23ed97b4c..90ca6d3f9a 100644
--- a/src/armnnDeserializer/Deserializer.cpp
+++ b/src/armnnDeserializer/Deserializer.cpp
@@ -1187,6 +1187,7 @@ void Deserializer::ParseL2Normalization(GraphPtr graph, unsigned int layerIndex)
     auto layerName = GetLayerName(graph, layerIndex);
     armnn::L2NormalizationDescriptor descriptor;
     descriptor.m_DataLayout = ToDataLayout(flatBufferDescriptor->dataLayout());
+    descriptor.m_Eps = flatBufferDescriptor->eps();
 
     IConnectableLayer* layer = m_Network->AddL2NormalizationLayer(descriptor, layerName.c_str());
     layer->GetOutputSlot(0).SetTensorInfo(outputInfo);
diff --git a/src/armnnDeserializer/test/DeserializeL2Normalization.cpp b/src/armnnDeserializer/test/DeserializeL2Normalization.cpp
new file mode 100644
index 0000000000..d8604a5cd5
--- /dev/null
+++ b/src/armnnDeserializer/test/DeserializeL2Normalization.cpp
@@ -0,0 +1,142 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include <boost/test/unit_test.hpp>
+#include "ParserFlatbuffersSerializeFixture.hpp"
+#include "../Deserializer.hpp"
+
+#include <string>
+#include <iostream>
+
+BOOST_AUTO_TEST_SUITE(Deserializer)
+
+struct L2NormalizationFixture : public ParserFlatbuffersSerializeFixture
+{
+    explicit L2NormalizationFixture(const std::string &inputShape,
+                                    const std::string &outputShape,
+                                    const std::string &dataType,
+                                    const std::string &dataLayout,
+                                    const std::string epsilon)
+    {
+        m_JsonString = R"(
+    {
+        inputIds: [0],
+        outputIds: [2],
+        layers: [
+           {
+            layer_type: "InputLayer",
+            layer: {
+                base: {
+                    layerBindingId: 0,
+                    base: {
+                        index: 0,
+                        layerName: "InputLayer",
+                        layerType: "Input",
+                        inputSlots: [{
+                            index: 0,
+                            connection: {sourceLayerIndex:0, outputSlotIndex:0 },
+                            }],
+                        outputSlots: [{
+                            index: 0,
+                            tensorInfo: {
+                                dimensions: )" + inputShape + R"(,
+                                dataType: ")" + dataType + R"(",
+                                quantizationScale: 0.5,
+                                quantizationOffset: 0
+                                },
+                            }]
+                        },
+                    }
+                },
+            },
+        {
+        layer_type: "L2NormalizationLayer",
+        layer : {
+            base: {
+                index:1,
+                layerName: "L2NormalizationLayer",
+                layerType: "L2Normalization",
+                inputSlots: [{
+                        index: 0,
+                        connection: {sourceLayerIndex:0, outputSlotIndex:0 },
+                   }],
+                outputSlots: [{
+                    index: 0,
+                    tensorInfo: {
+                        dimensions: )" + outputShape + R"(,
+                        dataType: ")" + dataType + R"("
+                    },
+                    }],
+                },
+            descriptor: {
+                dataLayout: ")" + dataLayout + R"(",
+                eps: )" + epsilon + R"(
+                },
+            },
+        },
+        {
+        layer_type: "OutputLayer",
+        layer: {
+            base:{
+                layerBindingId: 0,
+                base: {
+                    index: 2,
+                    layerName: "OutputLayer",
+                    layerType: "Output",
+                    inputSlots: [{
+                        index: 0,
+                        connection: {sourceLayerIndex:1, outputSlotIndex:0 },
+                    }],
+                    outputSlots: [ {
+                        index: 0,
+                        tensorInfo: {
+                            dimensions: )" + outputShape + R"(,
+                            dataType: ")" + dataType + R"("
+                        },
+                    }],
+                }
+            }},
+        }]
+    }
+)";
+        Setup();
+    }
+};
+
+struct L2NormFixture : L2NormalizationFixture
+{
+    // Using a non standard epsilon value of 1e-8
+    L2NormFixture():L2NormalizationFixture("[ 1, 3, 1, 1 ]",
+                                           "[ 1, 3, 1, 1 ]",
+                                           "Float32",
+                                           "NCHW",
+                                           "0.00000001"){}
+};
+
+BOOST_FIXTURE_TEST_CASE(L2NormalizationFloat32, L2NormFixture)
+{
+    // 1 / sqrt(1^2 + 2^2 + 3^2)
+    const float approxInvL2Norm = 0.267261f;
+
+    RunTest<4, armnn::DataType::Float32>(0,
+                                         {{"InputLayer", { 1.0f, 2.0f, 3.0f }}},
+                                         {{"OutputLayer",{ 1.0f * approxInvL2Norm,
+                                                           2.0f * approxInvL2Norm,
+                                                           3.0f * approxInvL2Norm }}});
+}
+
+BOOST_FIXTURE_TEST_CASE(L2NormalizationEpsilonLimitFloat32, L2NormFixture)
+{
+    // 1 / sqrt(1e-8)
+    const float approxInvL2Norm = 10000;
+
+    RunTest<4, armnn::DataType::Float32>(0,
+                                         {{"InputLayer", { 0.00000001f, 0.00000002f, 0.00000003f }}},
+                                         {{"OutputLayer",{ 0.00000001f * approxInvL2Norm,
+                                                           0.00000002f * approxInvL2Norm,
+                                                           0.00000003f * approxInvL2Norm }}});
+}
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/armnnSerializer/ArmnnSchema.fbs b/src/armnnSerializer/ArmnnSchema.fbs
index db5672f948..794789390a 100644
--- a/src/armnnSerializer/ArmnnSchema.fbs
+++ b/src/armnnSerializer/ArmnnSchema.fbs
@@ -224,6 +224,7 @@ table L2NormalizationLayer {
 
 table L2NormalizationDescriptor {
     dataLayout:DataLayout = NCHW;
+    eps:float = 1e-12;
 }
 
 table MinimumLayer {
diff --git a/src/armnnSerializer/Serializer.cpp b/src/armnnSerializer/Serializer.cpp
index dabe977179..efadbb38ca 100644
--- a/src/armnnSerializer/Serializer.cpp
+++ b/src/armnnSerializer/Serializer.cpp
@@ -380,9 +380,11 @@ void SerializerVisitor::VisitL2NormalizationLayer(const armnn::IConnectableLayer
 
     // Create the FlatBuffer L2Normalization Descriptor
     auto fbDescriptor = serializer::CreateL2NormalizationDescriptor(
-            m_flatBufferBuilder, GetFlatBufferDataLayout(l2NormalizationDescriptor.m_DataLayout));
+            m_flatBufferBuilder,
+            GetFlatBufferDataLayout(l2NormalizationDescriptor.m_DataLayout),
+            l2NormalizationDescriptor.m_Eps);
 
-    // Create Flatuffer layer
+    // Create FlatBuffer layer
     auto fbLayer = serializer::CreateL2NormalizationLayer(m_flatBufferBuilder, fbBaseLayer, fbDescriptor);
 
     CreateAnyLayer(fbLayer.o, serializer::Layer::Layer_L2NormalizationLayer);
diff --git a/src/armnnSerializer/test/SerializerTests.cpp b/src/armnnSerializer/test/SerializerTests.cpp
index 812a4780f4..ddebd1435c 100644
--- a/src/armnnSerializer/test/SerializerTests.cpp
+++ b/src/armnnSerializer/test/SerializerTests.cpp
@@ -1046,39 +1046,41 @@ BOOST_AUTO_TEST_CASE(SerializeGreater)
     deserializedNetwork->Accept(verifier);
 }
 
-BOOST_AUTO_TEST_CASE(SerializeL2Normalization)
+class L2NormalizationLayerVerifier : public LayerVerifierBase
 {
-    class L2NormalizationLayerVerifier : public LayerVerifierBase
-    {
-    public:
-        L2NormalizationLayerVerifier(const std::string& layerName,
-                                     const std::vector<armnn::TensorInfo>& inputInfos,
-                                     const std::vector<armnn::TensorInfo>& outputInfos,
-                                     const armnn::L2NormalizationDescriptor& descriptor)
-        : LayerVerifierBase(layerName, inputInfos, outputInfos)
-        , m_Descriptor(descriptor) {}
+public:
+    L2NormalizationLayerVerifier(const std::string& layerName,
+                                 const std::vector<armnn::TensorInfo>& inputInfos,
+                                 const std::vector<armnn::TensorInfo>& outputInfos,
+                                 const armnn::L2NormalizationDescriptor& descriptor)
+            : LayerVerifierBase(layerName, inputInfos, outputInfos)
+            , m_Descriptor(descriptor) {}
 
-        void VisitL2NormalizationLayer(const armnn::IConnectableLayer* layer,
-                                       const armnn::L2NormalizationDescriptor& descriptor,
-                                       const char* name) override
-        {
-            VerifyNameAndConnections(layer, name);
-            VerifyDescriptor(descriptor);
-        }
-    private:
-        void VerifyDescriptor(const armnn::L2NormalizationDescriptor& descriptor)
-        {
-            BOOST_TEST(GetDataLayoutName(descriptor.m_DataLayout) == GetDataLayoutName(m_Descriptor.m_DataLayout));
-        }
+    void VisitL2NormalizationLayer(const armnn::IConnectableLayer* layer,
+                                   const armnn::L2NormalizationDescriptor& descriptor,
+                                   const char* name) override
+    {
+        VerifyNameAndConnections(layer, name);
+        VerifyDescriptor(descriptor);
+    }
+private:
+    void VerifyDescriptor(const armnn::L2NormalizationDescriptor& descriptor)
+    {
+        BOOST_TEST(descriptor.m_Eps == m_Descriptor.m_Eps);
+        BOOST_TEST(GetDataLayoutName(descriptor.m_DataLayout) == GetDataLayoutName(m_Descriptor.m_DataLayout));
+    }
 
-        armnn::L2NormalizationDescriptor m_Descriptor;
-    };
+    armnn::L2NormalizationDescriptor m_Descriptor;
+};
 
+BOOST_AUTO_TEST_CASE(SerializeL2Normalization)
+{
     const std::string l2NormLayerName("l2Normalization");
     const armnn::TensorInfo info({1, 2, 1, 5}, armnn::DataType::Float32);
 
     armnn::L2NormalizationDescriptor desc;
     desc.m_DataLayout = armnn::DataLayout::NCHW;
+    desc.m_Eps = 0.0001f;
 
     armnn::INetworkPtr network = armnn::INetwork::Create();
     armnn::IConnectableLayer* const inputLayer0 = network->AddInputLayer(0);
@@ -1098,6 +1100,62 @@ BOOST_AUTO_TEST_CASE(SerializeL2Normalization)
     deserializedNetwork->Accept(verifier);
 }
 
+BOOST_AUTO_TEST_CASE(EnsureL2NormalizationBackwardCompatibility)
+{
+    // The hex array below is a flat buffer containing a simple network with one input
+    // a L2Normalization layer and an output layer with dimensions as per the tensor infos below.
+    //
+    // This test verifies that we can still read back these old style
+    // models without the normalization epsilon value.
+    unsigned int size = 508;
+    const unsigned char l2NormalizationModel[] = {
+            0x10,0x00,0x00,0x00,0x00,0x00,0x0A,0x00,0x10,0x00,0x04,0x00,0x08,0x00,0x0C,0x00,0x0A,0x00,0x00,0x00,
+            0x0C,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x1C,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x3C,0x01,0x00,0x00,
+            0x74,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,
+            0x02,0x00,0x00,0x00,0xE8,0xFE,0xFF,0xFF,0x00,0x00,0x00,0x0B,0x04,0x00,0x00,0x00,0xD6,0xFE,0xFF,0xFF,
+            0x0C,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x08,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,
+            0x9E,0xFF,0xFF,0xFF,0x02,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x10,0x00,0x00,0x00,
+            0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,
+            0x00,0x00,0x00,0x00,0x4C,0xFF,0xFF,0xFF,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x44,0xFF,0xFF,0xFF,
+            0x00,0x00,0x00,0x20,0x0C,0x00,0x00,0x00,0x08,0x00,0x0C,0x00,0x04,0x00,0x08,0x00,0x08,0x00,0x00,0x00,
+            0x20,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x04,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x0E,0x00,
+            0x18,0x00,0x04,0x00,0x08,0x00,0x0C,0x00,0x10,0x00,0x14,0x00,0x0E,0x00,0x00,0x00,0x01,0x00,0x00,0x00,
+            0x10,0x00,0x00,0x00,0x1F,0x00,0x00,0x00,0x1C,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x0F,0x00,0x00,0x00,
+            0x6C,0x32,0x4E,0x6F,0x72,0x6D,0x61,0x6C,0x69,0x7A,0x61,0x74,0x69,0x6F,0x6E,0x00,0x01,0x00,0x00,0x00,
+            0x48,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0C,0x00,0x00,0x00,0x08,0x00,0x08,0x00,0x00,0x00,0x04,0x00,
+            0x08,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x52,0xFF,0xFF,0xFF,0x00,0x00,0x00,0x01,0x08,0x00,0x00,0x00,
+            0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x01,0x00,0x00,0x00,
+            0x05,0x00,0x00,0x00,0x08,0x00,0x0C,0x00,0x00,0x00,0x04,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+            0x00,0x00,0x00,0x00,0x08,0x00,0x0C,0x00,0x07,0x00,0x08,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x09,
+            0x04,0x00,0x00,0x00,0xF6,0xFF,0xFF,0xFF,0x0C,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x0A,0x00,0x04,0x00,
+            0x06,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x00,0x00,0x0E,0x00,0x14,0x00,0x00,0x00,0x04,0x00,0x08,0x00,
+            0x0C,0x00,0x10,0x00,0x0E,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x10,0x00,0x00,0x00,
+            0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,
+            0x0C,0x00,0x00,0x00,0x08,0x00,0x0A,0x00,0x00,0x00,0x04,0x00,0x08,0x00,0x00,0x00,0x10,0x00,0x00,0x00,
+            0x00,0x00,0x0A,0x00,0x10,0x00,0x08,0x00,0x07,0x00,0x0C,0x00,0x0A,0x00,0x00,0x00,0x00,0x00,0x00,0x01,
+            0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,
+            0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0 };
+
+    std::stringstream ss;
+    for (unsigned int i = 0; i < size; ++i)
+    {
+        ss << l2NormalizationModel[i];
+    }
+    std::string l2NormalizationLayerNetwork = ss.str();
+    armnn::INetworkPtr deserializedNetwork = DeserializeNetwork(l2NormalizationLayerNetwork);
+    BOOST_CHECK(deserializedNetwork);
+    const std::string layerName("l2Normalization");
+    const armnn::TensorInfo inputInfo = armnn::TensorInfo({1, 2, 1, 5}, armnn::DataType::Float32);
+
+    armnn::L2NormalizationDescriptor desc;
+    desc.m_DataLayout = armnn::DataLayout::NCHW;
+    // Since this variable does not exist in the l2NormalizationModel[] dump, the default value will be loaded.
+    desc.m_Eps = 1e-12f;
+
+    L2NormalizationLayerVerifier verifier(layerName, {inputInfo}, {inputInfo}, desc);
+    deserializedNetwork->Accept(verifier);
+}
+
 BOOST_AUTO_TEST_CASE(SerializeMaximum)
 {
     class MaximumLayerVerifier : public LayerVerifierBase