IVGCVSW-4549 Add front end for new QLSTM layer

* Added new layer QLstm (Android R HAL 1.3)
* Made necessary updates to APIs
* Added unit tests
* This layer is functionally equivalent to the
  original unquantized LSTM layer with some
  additonal quantization features added. Due
  to this, original LstmParams are used for
  this layer.

Signed-off-by: James Conroy <james.conroy@arm.com>
Change-Id: I5b7f2d2fb6e17e81573b41a31bc55f49ae79608f

6 files changed, 250 insertions, 0 deletions

diff --git a/src/backends/backendsCommon/LayerSupportBase.cpp b/src/backends/backendsCommon/LayerSupportBase.cpp
index c3c8421267..c55f51d315 100644
--- a/src/backends/backendsCommon/LayerSupportBase.cpp
+++ b/src/backends/backendsCommon/LayerSupportBase.cpp
@@ -447,6 +447,19 @@ bool LayerSupportBase::IsQuantizeSupported(const armnn::TensorInfo& /*input*/,
     return DefaultLayerSupport(__func__, __FILE__, __LINE__, reasonIfUnsupported);
 }
 
+bool LayerSupportBase::IsQLstmSupported(const TensorInfo& /*input*/,
+                                        const TensorInfo& /*previousOutputIn*/,
+                                        const TensorInfo& /*previousCellStateIn*/,
+                                        const TensorInfo& /*outputStateOut*/,
+                                        const TensorInfo& /*cellStateOut*/,
+                                        const TensorInfo& /*output*/,
+                                        const QLstmDescriptor& /*descriptor*/,
+                                        const LstmInputParamsInfo& /*paramsInfo*/,
+                                        Optional<std::string&> reasonIfUnsupported) const
+{
+    return DefaultLayerSupport(__func__, __FILE__, __LINE__, reasonIfUnsupported);
+}
+
 bool LayerSupportBase::IsQuantizedLstmSupported(const TensorInfo& /*input*/,
                                                 const TensorInfo& /*previousCellStateIn*/,
                                                 const TensorInfo& /*previousOutputIn*/,
diff --git a/src/backends/backendsCommon/LayerSupportBase.hpp b/src/backends/backendsCommon/LayerSupportBase.hpp
index 063983357e..fcc3326601 100644
--- a/src/backends/backendsCommon/LayerSupportBase.hpp
+++ b/src/backends/backendsCommon/LayerSupportBase.hpp
@@ -270,6 +270,16 @@ public:
                              const TensorInfo& output,
                              Optional<std::string&> reasonIfUnsupported = EmptyOptional()) const override;
 
+    bool IsQLstmSupported(const TensorInfo& input,
+                          const TensorInfo& previousOutputIn,
+                          const TensorInfo& previousCellStateIn,
+                          const TensorInfo& outputStateOut,
+                          const TensorInfo& cellStateOut,
+                          const TensorInfo& output,
+                          const QLstmDescriptor& descriptor,
+                          const LstmInputParamsInfo& paramsInfo,
+                          Optional<std::string&> reasonIfUnsupported = EmptyOptional()) const override;
+
     bool IsQuantizedLstmSupported(const TensorInfo& input,
                                   const TensorInfo& previousCellStateIn,
                                   const TensorInfo& previousOutputIn,
diff --git a/src/backends/backendsCommon/WorkloadData.hpp b/src/backends/backendsCommon/WorkloadData.hpp
index 85bda5469a..448de6a1ee 100644
--- a/src/backends/backendsCommon/WorkloadData.hpp
+++ b/src/backends/backendsCommon/WorkloadData.hpp
@@ -519,6 +519,58 @@ struct TransposeQueueDescriptor : QueueDescriptorWithParameters<TransposeDescrip
     void Validate(const WorkloadInfo& workloadInfo) const;
 };
 
+struct QLstmQueueDescriptor : QueueDescriptorWithParameters<QLstmDescriptor>
+{
+    QLstmQueueDescriptor()
+            : m_InputToInputWeights(nullptr)
+            , m_InputToForgetWeights(nullptr)
+            , m_InputToCellWeights(nullptr)
+            , m_InputToOutputWeights(nullptr)
+            , m_RecurrentToInputWeights(nullptr)
+            , m_RecurrentToForgetWeights(nullptr)
+            , m_RecurrentToCellWeights(nullptr)
+            , m_RecurrentToOutputWeights(nullptr)
+            , m_CellToInputWeights(nullptr)
+            , m_CellToForgetWeights(nullptr)
+            , m_CellToOutputWeights(nullptr)
+            , m_InputGateBias(nullptr)
+            , m_ForgetGateBias(nullptr)
+            , m_CellBias(nullptr)
+            , m_OutputGateBias(nullptr)
+            , m_ProjectionWeights(nullptr)
+            , m_ProjectionBias(nullptr)
+            , m_InputLayerNormWeights(nullptr)
+            , m_ForgetLayerNormWeights(nullptr)
+            , m_CellLayerNormWeights(nullptr)
+            , m_OutputLayerNormWeights(nullptr)
+    {
+    }
+
+    const ConstCpuTensorHandle* m_InputToInputWeights;
+    const ConstCpuTensorHandle* m_InputToForgetWeights;
+    const ConstCpuTensorHandle* m_InputToCellWeights;
+    const ConstCpuTensorHandle* m_InputToOutputWeights;
+    const ConstCpuTensorHandle* m_RecurrentToInputWeights;
+    const ConstCpuTensorHandle* m_RecurrentToForgetWeights;
+    const ConstCpuTensorHandle* m_RecurrentToCellWeights;
+    const ConstCpuTensorHandle* m_RecurrentToOutputWeights;
+    const ConstCpuTensorHandle* m_CellToInputWeights;
+    const ConstCpuTensorHandle* m_CellToForgetWeights;
+    const ConstCpuTensorHandle* m_CellToOutputWeights;
+    const ConstCpuTensorHandle* m_InputGateBias;
+    const ConstCpuTensorHandle* m_ForgetGateBias;
+    const ConstCpuTensorHandle* m_CellBias;
+    const ConstCpuTensorHandle* m_OutputGateBias;
+    const ConstCpuTensorHandle* m_ProjectionWeights;
+    const ConstCpuTensorHandle* m_ProjectionBias;
+    const ConstCpuTensorHandle* m_InputLayerNormWeights;
+    const ConstCpuTensorHandle* m_ForgetLayerNormWeights;
+    const ConstCpuTensorHandle* m_CellLayerNormWeights;
+    const ConstCpuTensorHandle* m_OutputLayerNormWeights;
+
+    void Validate(const WorkloadInfo& workloadInfo) const;
+};
+
 struct QuantizedLstmQueueDescriptor : QueueDescriptor
 {
     QuantizedLstmQueueDescriptor()
diff --git a/src/backends/backendsCommon/WorkloadFactory.cpp b/src/backends/backendsCommon/WorkloadFactory.cpp
index 5854bece3c..40ab798ba2 100644
--- a/src/backends/backendsCommon/WorkloadFactory.cpp
+++ b/src/backends/backendsCommon/WorkloadFactory.cpp
@@ -749,6 +749,94 @@ bool IWorkloadFactory::IsLayerSupported(const BackendId& backendId,
             result = layerSupportObject->IsQuantizeSupported(input, output, reason);
             break;
         }
+        case LayerType::QLstm:
+        {
+            auto cLayer = boost::polymorphic_downcast<const QLstmLayer*>(&layer);
+            const QLstmDescriptor& descriptor = cLayer->GetParameters();
+
+            // Inputs
+            const TensorInfo& input = layer.GetInputSlot(0).GetConnection()->GetTensorInfo();
+            const TensorInfo& previousOutputIn = layer.GetInputSlot(1).GetConnection()->GetTensorInfo();
+            const TensorInfo& previousCellStateIn = layer.GetInputSlot(2).GetConnection()->GetTensorInfo();
+
+            // Outputs
+            const TensorInfo& outputStateOut = layer.GetOutputSlot(0).GetTensorInfo();
+            const TensorInfo& cellStateOut = layer.GetOutputSlot(1).GetTensorInfo();
+            const TensorInfo& output = layer.GetOutputSlot(2).GetTensorInfo();
+
+            // Lstm parameters
+            LstmInputParamsInfo paramsInfo;
+
+            // Basic parameters
+            paramsInfo.m_InputToForgetWeights = &cLayer->m_BasicParameters.m_InputToForgetWeights->GetTensorInfo();
+            paramsInfo.m_InputToCellWeights   = &cLayer->m_BasicParameters.m_InputToCellWeights->GetTensorInfo();
+            paramsInfo.m_InputToOutputWeights = &cLayer->m_BasicParameters.m_InputToOutputWeights->GetTensorInfo();
+
+            paramsInfo.m_RecurrentToForgetWeights =
+                    &cLayer->m_BasicParameters.m_RecurrentToForgetWeights->GetTensorInfo();
+            paramsInfo.m_RecurrentToCellWeights   =
+                    &cLayer->m_BasicParameters.m_RecurrentToCellWeights->GetTensorInfo();
+            paramsInfo.m_RecurrentToOutputWeights =
+                    &cLayer->m_BasicParameters.m_RecurrentToOutputWeights->GetTensorInfo();
+
+            paramsInfo.m_ForgetGateBias = &cLayer->m_BasicParameters.m_ForgetGateBias->GetTensorInfo();
+            paramsInfo.m_CellBias       = &cLayer->m_BasicParameters.m_CellBias->GetTensorInfo();
+            paramsInfo.m_OutputGateBias = &cLayer->m_BasicParameters.m_OutputGateBias->GetTensorInfo();
+
+            if(!descriptor.m_CifgEnabled)
+            {
+                paramsInfo.m_InputToInputWeights = &cLayer->m_CifgParameters.m_InputToInputWeights->GetTensorInfo();
+                paramsInfo.m_RecurrentToInputWeights =
+                        &cLayer->m_CifgParameters.m_RecurrentToInputWeights->GetTensorInfo();
+                paramsInfo.m_InputGateBias = &cLayer->m_CifgParameters.m_InputGateBias->GetTensorInfo();
+            }
+
+            if(descriptor.m_ProjectionEnabled)
+            {
+                paramsInfo.m_ProjectionWeights = &cLayer->m_ProjectionParameters.m_ProjectionWeights->GetTensorInfo();
+                paramsInfo.m_ProjectionBias = &cLayer->m_ProjectionParameters.m_ProjectionBias->GetTensorInfo();
+            }
+
+            if(descriptor.m_PeepholeEnabled)
+            {
+                if (!descriptor.m_CifgEnabled)
+                {
+                    paramsInfo.m_CellToInputWeights =
+                            &cLayer->m_PeepholeParameters.m_CellToInputWeights->GetTensorInfo();
+                }
+
+                paramsInfo.m_CellToForgetWeights =
+                        &cLayer->m_PeepholeParameters.m_CellToForgetWeights->GetTensorInfo();
+                paramsInfo.m_CellToOutputWeights = &cLayer->m_PeepholeParameters.m_CellToOutputWeights->GetTensorInfo();
+            }
+
+            if(descriptor.m_LayerNormEnabled)
+            {
+                if (!descriptor.m_CifgEnabled)
+                {
+                    paramsInfo.m_InputLayerNormWeights =
+                            &cLayer->m_LayerNormParameters.m_InputLayerNormWeights->GetTensorInfo();
+                }
+
+                paramsInfo.m_ForgetLayerNormWeights =
+                        &cLayer->m_LayerNormParameters.m_ForgetLayerNormWeights->GetTensorInfo();
+                paramsInfo.m_CellLayerNormWeights =
+                        &cLayer->m_LayerNormParameters.m_CellLayerNormWeights->GetTensorInfo();
+                paramsInfo.m_OutputLayerNormWeights =
+                        &cLayer->m_LayerNormParameters.m_OutputLayerNormWeights->GetTensorInfo();
+            }
+
+            result = layerSupportObject->IsQLstmSupported(input,
+                                                          previousOutputIn,
+                                                          previousCellStateIn,
+                                                          outputStateOut,
+                                                          cellStateOut,
+                                                          output,
+                                                          descriptor,
+                                                          paramsInfo,
+                                                          reason);
+            break;
+        }
         case LayerType::QuantizedLstm:
         {
             auto cLayer = boost::polymorphic_downcast<const QuantizedLstmLayer*>(&layer);
@@ -1387,6 +1475,12 @@ std::unique_ptr<IWorkload> IWorkloadFactory::CreateQuantize(const QuantizeQueueD
     return std::unique_ptr<IWorkload>();
 }
 
+std::unique_ptr<IWorkload> IWorkloadFactory::CreateQLstm(const QLstmQueueDescriptor& /*descriptor*/,
+                                                         const WorkloadInfo& /*info*/) const
+{
+    return std::unique_ptr<IWorkload>();
+}
+
 std::unique_ptr<IWorkload> IWorkloadFactory::CreateQuantizedLstm(const QuantizedLstmQueueDescriptor& /*descriptor*/,
                                                                  const WorkloadInfo& /*info*/) const
 {
diff --git a/src/backends/backendsCommon/WorkloadFactory.hpp b/src/backends/backendsCommon/WorkloadFactory.hpp
index 0fc7ab9bce..98a6c36a2b 100644
--- a/src/backends/backendsCommon/WorkloadFactory.hpp
+++ b/src/backends/backendsCommon/WorkloadFactory.hpp
@@ -197,6 +197,9 @@ public:
     virtual std::unique_ptr<IWorkload> CreateQuantize(const QuantizeQueueDescriptor& descriptor,
                                                       const WorkloadInfo& Info) const;
 
+    virtual std::unique_ptr<IWorkload> CreateQLstm(const QLstmQueueDescriptor& descriptor,
+                                                   const WorkloadInfo& info) const;
+
     virtual std::unique_ptr<IWorkload> CreateQuantizedLstm(const QuantizedLstmQueueDescriptor& descriptor,
                                                            const WorkloadInfo& info) const;
 
diff --git a/src/backends/backendsCommon/test/IsLayerSupportedTestImpl.hpp b/src/backends/backendsCommon/test/IsLayerSupportedTestImpl.hpp
index d646847cbc..7534c8a7c9 100644
--- a/src/backends/backendsCommon/test/IsLayerSupportedTestImpl.hpp
+++ b/src/backends/backendsCommon/test/IsLayerSupportedTestImpl.hpp
@@ -308,6 +308,82 @@ struct DummyLayer<armnn::LstmLayer>
 {
 };
 
+template <typename QLstmLayerType>
+struct DummyQLstmLayer
+{
+    DummyQLstmLayer()
+    {
+        typename QLstmLayerType::DescriptorType desc;
+        desc.m_CifgEnabled = false;
+        desc.m_PeepholeEnabled = true;
+        desc.m_ProjectionEnabled = true;
+        desc.m_LayerNormEnabled = true;
+
+        m_Layer = dummyGraph.AddLayer<QLstmLayerType>(armnn::QLstmDescriptor(), "qLstm");
+
+        // Basic params
+        m_Layer->m_BasicParameters.m_InputToForgetWeights     = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS8));
+        m_Layer->m_BasicParameters.m_InputToCellWeights       = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS8));
+        m_Layer->m_BasicParameters.m_InputToOutputWeights     = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS8));
+
+        m_Layer->m_BasicParameters.m_RecurrentToForgetWeights = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS8));
+        m_Layer->m_BasicParameters.m_RecurrentToCellWeights   = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS8));
+        m_Layer->m_BasicParameters.m_RecurrentToOutputWeights = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS8));
+
+        m_Layer->m_BasicParameters.m_ForgetGateBias           = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::Signed32));
+        m_Layer->m_BasicParameters.m_CellBias                 = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::Signed32));
+        m_Layer->m_BasicParameters.m_OutputGateBias           = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::Signed32));
+
+        // CIFG optional params
+        m_Layer->m_CifgParameters.m_InputToInputWeights     = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS8));
+        m_Layer->m_CifgParameters.m_RecurrentToInputWeights = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS8));
+        m_Layer->m_CifgParameters.m_InputGateBias           = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::Signed32));
+
+        // Projection optional params
+        m_Layer->m_ProjectionParameters.m_ProjectionWeights = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS8));
+        m_Layer->m_ProjectionParameters.m_ProjectionBias    = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::Signed32));
+
+        // Peephole optional params
+        m_Layer->m_PeepholeParameters.m_CellToInputWeights  = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS16));
+        m_Layer->m_PeepholeParameters.m_CellToForgetWeights = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS16));
+        m_Layer->m_PeepholeParameters.m_CellToOutputWeights = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS16));
+
+        // Layer normalization optional params
+        m_Layer->m_LayerNormParameters.m_InputLayerNormWeights = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS16));
+        m_Layer->m_LayerNormParameters.m_ForgetLayerNormWeights = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS16));
+        m_Layer->m_LayerNormParameters.m_CellLayerNormWeights = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS16));
+        m_Layer->m_LayerNormParameters.m_OutputLayerNormWeights = std::make_unique<armnn::ScopedCpuTensorHandle>(
+                armnn::TensorInfo(armnn::TensorShape({1,1,1,1}), armnn::DataType::QSymmS16));
+    }
+
+    ~DummyQLstmLayer()
+    {
+        dummyGraph.EraseLayer(m_Layer);
+    }
+
+    armnn::QLstmLayer* m_Layer;
+};
+
 template<>
 struct DummyLayer<armnn::QuantizedLstmLayer, void>
 {
@@ -513,6 +589,8 @@ DECLARE_LAYER_POLICY_2_PARAM(PreCompiled)
 
 DECLARE_LAYER_POLICY_1_PARAM(Prelu)
 
+DECLARE_LAYER_POLICY_2_PARAM(QLstm)
+
 DECLARE_LAYER_POLICY_1_PARAM(QuantizedLstm)
 
 DECLARE_LAYER_POLICY_1_PARAM(Division)