IVGCVSW-3597 Add complete test suite for the dynamic backends

 * Added Runtime tests with dynamic backends
 * All other unit tests have already implemented in previous stories

Change-Id: I0167b3b45dd00c1956514e2df051e4445727f014
Signed-off-by: Matteo Martincigh <matteo.martincigh@arm.com>

3 files changed, 147 insertions, 2 deletions

diff --git a/src/armnn/Runtime.cpp b/src/armnn/Runtime.cpp
index ddfa6b4ffd..6b91863deb 100644
--- a/src/armnn/Runtime.cpp
+++ b/src/armnn/Runtime.cpp
@@ -143,7 +143,9 @@ Runtime::Runtime(const CreationOptions& options)
     for (const auto& id : BackendRegistryInstance().GetBackendIds())
     {
         // Store backend contexts for the supported ones
-        if (m_DeviceSpec.GetSupportedBackends().count(id) > 0)
+        const BackendIdSet& supportedBackends = m_DeviceSpec.GetSupportedBackends();
+        auto it = supportedBackends.find(id);
+        if (it != supportedBackends.end())
         {
             auto factoryFun = BackendRegistryInstance().GetFactory(id);
             auto backend = factoryFun();
diff --git a/src/backends/backendsCommon/test/DynamicBackendTests.cpp b/src/backends/backendsCommon/test/DynamicBackendTests.cpp
index 21ce2dbde4..f13c961eaf 100644
--- a/src/backends/backendsCommon/test/DynamicBackendTests.cpp
+++ b/src/backends/backendsCommon/test/DynamicBackendTests.cpp
@@ -55,4 +55,10 @@ ARMNN_SIMPLE_TEST_CASE(RegisterMultipleDynamicBackends, RegisterMultipleDynamicB
 ARMNN_SIMPLE_TEST_CASE(RegisterMultipleInvalidDynamicBackends, RegisterMultipleInvalidDynamicBackendsTestImpl);
 ARMNN_SIMPLE_TEST_CASE(RegisterMixedDynamicBackends, RegisterMixedDynamicBackendsTestImpl);
 
+ARMNN_SIMPLE_TEST_CASE(RuntimeEmpty, RuntimeEmptyTestImpl);
+ARMNN_SIMPLE_TEST_CASE(RuntimeDynamicBackends, RuntimeDynamicBackendsTestImpl);
+ARMNN_SIMPLE_TEST_CASE(RuntimeDuplicateDynamicBackends, RuntimeDuplicateDynamicBackendsTestImpl);
+ARMNN_SIMPLE_TEST_CASE(RuntimeInvalidDynamicBackends, RuntimeInvalidDynamicBackendsTestImpl);
+ARMNN_SIMPLE_TEST_CASE(RuntimeInvalidOverridePath, RuntimeInvalidOverridePathTestImpl);
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/backends/backendsCommon/test/DynamicBackendTests.hpp b/src/backends/backendsCommon/test/DynamicBackendTests.hpp
index ae922bc26d..3a44f506d6 100644
--- a/src/backends/backendsCommon/test/DynamicBackendTests.hpp
+++ b/src/backends/backendsCommon/test/DynamicBackendTests.hpp
@@ -8,6 +8,8 @@
 #include <backendsCommon/DynamicBackend.hpp>
 #include <backendsCommon/DynamicBackendUtils.hpp>
 
+#include <Runtime.hpp>
+
 #include <string>
 #include <memory>
 #include <string>
@@ -52,7 +54,7 @@ static std::string g_TestDynamicBackendsSubDir7  = "backendsTestPath7/";
 static std::string g_TestDynamicBackendsSubDir8  = "backendsTestPath8/";
 static std::string g_TestDynamicBackendsSubDir9  = "backendsTestPath9/";
 
-// Wrapper class used for testing
+// DynamicBackendUtils wrapper class used for testing (allows to directly invoke the protected methods)
 class TestDynamicBackendUtils : public armnn::DynamicBackendUtils
 {
 public:
@@ -74,6 +76,24 @@ public:
     }
 };
 
+// BackendRegistry wrapper class used for testing (swaps the underlying factory storage)
+class TestBackendRegistry : public armnn::BackendRegistry
+{
+public:
+    TestBackendRegistry() : armnn::BackendRegistry()
+    {
+        Swap(armnn::BackendRegistryInstance(), m_TempStorage);
+    }
+
+    ~TestBackendRegistry()
+    {
+        Swap(armnn::BackendRegistryInstance(), m_TempStorage);
+    }
+
+private:
+    FactoryStorage m_TempStorage;
+};
+
 std::string GetTestDirectoryBasePath()
 {
     using namespace boost::filesystem;
@@ -1131,3 +1151,120 @@ void RegisterMixedDynamicBackendsTestImpl()
         BOOST_TEST((dynamicBackend->GetId() == expectedRegisteredbackendId));
     }
 }
+
+void RuntimeEmptyTestImpl()
+{
+    using namespace armnn;
+
+    // Swapping the backend registry storage for testing
+    TestBackendRegistry testBackendRegistry;
+
+    IRuntime::CreationOptions creationOptions;
+    IRuntimePtr runtime = IRuntime::Create(creationOptions);
+
+    const BackendRegistry& backendRegistry = BackendRegistryInstance();
+    BOOST_TEST(backendRegistry.Size() == 0);
+}
+
+void RuntimeDynamicBackendsTestImpl()
+{
+    using namespace armnn;
+    using namespace boost::filesystem;
+
+    // Swapping the backend registry storage for testing
+    TestBackendRegistry testBackendRegistry;
+
+    // This directory contains valid and invalid backends
+    std::string testDynamicBackendsSubDir5 = GetTestSubDirectory(g_TestDynamicBackendsSubDir5);
+    BOOST_CHECK(exists(testDynamicBackendsSubDir5));
+
+    // Using the path override in CreationOptions to load some test dynamic backends
+    IRuntime::CreationOptions creationOptions;
+    creationOptions.m_DynamicBackendsPath = testDynamicBackendsSubDir5;
+    IRuntimePtr runtime = IRuntime::Create(creationOptions);
+
+    std::vector<BackendId> expectedRegisteredbackendIds
+    {
+        "TestValid2",
+        "TestValid3"
+    };
+
+    const BackendRegistry& backendRegistry = BackendRegistryInstance();
+    BOOST_TEST(backendRegistry.Size() == expectedRegisteredbackendIds.size());
+
+    BackendIdSet backendIds = backendRegistry.GetBackendIds();
+    for (const BackendId& expectedRegisteredbackendId : expectedRegisteredbackendIds)
+    {
+        BOOST_TEST((backendIds.find(expectedRegisteredbackendId) != backendIds.end()));
+    }
+}
+
+void RuntimeDuplicateDynamicBackendsTestImpl()
+{
+    using namespace armnn;
+    using namespace boost::filesystem;
+
+    // Swapping the backend registry storage for testing
+    TestBackendRegistry testBackendRegistry;
+
+    // This directory contains valid, invalid and duplicate backends
+    std::string testDynamicBackendsSubDir6 = GetTestSubDirectory(g_TestDynamicBackendsSubDir6);
+    BOOST_CHECK(exists(testDynamicBackendsSubDir6));
+
+    // Using the path override in CreationOptions to load some test dynamic backends
+    IRuntime::CreationOptions creationOptions;
+    creationOptions.m_DynamicBackendsPath = testDynamicBackendsSubDir6;
+    IRuntimePtr runtime = IRuntime::Create(creationOptions);
+
+    std::vector<BackendId> expectedRegisteredbackendIds
+    {
+        "TestValid2",
+        "TestValid5"
+    };
+
+    const BackendRegistry& backendRegistry = BackendRegistryInstance();
+    BOOST_TEST(backendRegistry.Size() == expectedRegisteredbackendIds.size());
+
+    BackendIdSet backendIds = backendRegistry.GetBackendIds();
+    for (const BackendId& expectedRegisteredbackendId : expectedRegisteredbackendIds)
+    {
+        BOOST_TEST((backendIds.find(expectedRegisteredbackendId) != backendIds.end()));
+    }
+}
+
+void RuntimeInvalidDynamicBackendsTestImpl()
+{
+    using namespace armnn;
+    using namespace boost::filesystem;
+
+    // Swapping the backend registry storage for testing
+    TestBackendRegistry testBackendRegistry;
+
+    // This directory contains only invalid backends
+    std::string testDynamicBackendsSubDir9 = GetTestSubDirectory(g_TestDynamicBackendsSubDir9);
+    BOOST_CHECK(exists(testDynamicBackendsSubDir9));
+
+    // Using the path override in CreationOptions to load some test dynamic backends
+    IRuntime::CreationOptions creationOptions;
+    creationOptions.m_DynamicBackendsPath = testDynamicBackendsSubDir9;
+    IRuntimePtr runtime = IRuntime::Create(creationOptions);
+
+    const BackendRegistry& backendRegistry = BackendRegistryInstance();
+    BOOST_TEST(backendRegistry.Size() == 0);
+}
+
+void RuntimeInvalidOverridePathTestImpl()
+{
+    using namespace armnn;
+
+    // Swapping the backend registry storage for testing
+    TestBackendRegistry testBackendRegistry;
+
+    // Using the path override in CreationOptions to load some test dynamic backends
+    IRuntime::CreationOptions creationOptions;
+    creationOptions.m_DynamicBackendsPath = "InvalidPath";
+    IRuntimePtr runtime = IRuntime::Create(creationOptions);
+
+    const BackendRegistry& backendRegistry = BackendRegistryInstance();
+    BOOST_TEST(backendRegistry.Size() == 0);
+}