IVGCVSW-3656 Make the reference backend optional

 * Made the build of the reference backend depend on a new ARMCOMPUTEREF
   macro
 * Made the relevant targets dependent on the ref backend
 * Moved Cl and Neon static registry initializers to separate files
 * Wrapped some of the unit tests into proper ifdefs where necessary

Change-Id: I7f2c42699682630233a4c4b6aed2f005083de189
Signed-off-by: Matteo Martincigh <matteo.martincigh@arm.com>

29 files changed, 320 insertions, 142 deletions

diff --git a/Android.mk b/Android.mk
index e590ba5988..4b34999bbc 100644
--- a/Android.mk
+++ b/Android.mk
@@ -49,11 +49,13 @@ ARMNN_BACKEND_SOURCES :=
 #
 $(foreach mkPath,$(ARMNN_BACKEND_COMMON_MAKEFILE_DIRS),\
         $(eval include $(LOCAL_PATH)/$(mkPath)/common.mk)\
-        $(eval ARMNN_BACKEND_SOURCES := $(ARMNN_BACKEND_SOURCES) $(patsubst %,$(mkPath)/%,$(COMMON_SOURCES))))
+        $(eval ARMNN_BACKEND_SOURCES := $(ARMNN_BACKEND_SOURCES)\
+        $(patsubst %,$(mkPath)/%,$(COMMON_SOURCES))))
 
 $(foreach mkPath,$(ARMNN_BACKEND_MAKEFILE_DIRS),\
         $(eval include $(LOCAL_PATH)/$(mkPath)/backend.mk)\
-        $(eval ARMNN_BACKEND_SOURCES := $(ARMNN_BACKEND_SOURCES) $(patsubst %,$(mkPath)/%,$(BACKEND_SOURCES))))
+        $(eval ARMNN_BACKEND_SOURCES := $(ARMNN_BACKEND_SOURCES)\
+        $(patsubst %,$(mkPath)/%,$(BACKEND_SOURCES))))
 
 # Mark source files as dependent on Android.mk and backend makefiles
 LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk \
@@ -76,6 +78,29 @@ LOCAL_C_INCLUDES := \
 
 LOCAL_SRC_FILES := \
         $(ARMNN_BACKEND_SOURCES) \
+        src/armnn/BackendHelper.cpp \
+        src/armnn/Descriptors.cpp \
+        src/armnn/Exceptions.cpp \
+        src/armnn/Graph.cpp \
+        src/armnn/InternalTypes.cpp \
+        src/armnn/JsonPrinter.cpp \
+        src/armnn/Layer.cpp \
+        src/armnn/LayerSupport.cpp \
+        src/armnn/LoadedNetwork.cpp \
+        src/armnn/Network.cpp \
+        src/armnn/NetworkUtils.cpp \
+        src/armnn/Observable.cpp \
+        src/armnn/Optimizer.cpp \
+        src/armnn/ProfilingEvent.cpp \
+        src/armnn/Profiling.cpp \
+        src/armnn/Runtime.cpp \
+        src/armnn/SerializeLayerParameters.cpp \
+        src/armnn/SubgraphView.cpp \
+        src/armnn/SubgraphViewSelector.cpp \
+        src/armnn/Tensor.cpp \
+        src/armnn/TypesUtils.cpp \
+        src/armnn/Utils.cpp \
+        src/armnn/WallClockTimer.cpp \
         src/armnnUtils/CsvReader.cpp \
         src/armnnUtils/DataLayoutIndexed.cpp \
         src/armnnUtils/DotSerializer.cpp \
@@ -138,30 +163,7 @@ LOCAL_SRC_FILES := \
         src/armnn/layers/StridedSliceLayer.cpp \
         src/armnn/layers/SubtractionLayer.cpp \
         src/armnn/layers/SwitchLayer.cpp \
-        src/armnn/layers/TransposeConvolution2dLayer.cpp \
-        src/armnn/BackendHelper.cpp \
-        src/armnn/Descriptors.cpp \
-        src/armnn/Exceptions.cpp \
-        src/armnn/Graph.cpp \
-        src/armnn/Optimizer.cpp \
-        src/armnn/Runtime.cpp \
-        src/armnn/SerializeLayerParameters.cpp \
-        src/armnn/SubgraphView.cpp \
-        src/armnn/SubgraphViewSelector.cpp \
-        src/armnn/InternalTypes.cpp \
-        src/armnn/Layer.cpp \
-        src/armnn/LoadedNetwork.cpp \
-        src/armnn/Network.cpp \
-        src/armnn/NetworkUtils.cpp \
-        src/armnn/WallClockTimer.cpp \
-        src/armnn/ProfilingEvent.cpp \
-        src/armnn/Profiling.cpp \
-        src/armnn/JsonPrinter.cpp \
-        src/armnn/Tensor.cpp \
-        src/armnn/TypesUtils.cpp \
-        src/armnn/Utils.cpp \
-        src/armnn/LayerSupport.cpp \
-        src/armnn/Observable.cpp
+        src/armnn/layers/TransposeConvolution2dLayer.cpp
 
 LOCAL_STATIC_LIBRARIES := \
         armnn-arm_compute \
@@ -188,6 +190,11 @@ ifeq ($(ARMNN_COMPUTE_NEON_ENABLED),1)
 LOCAL_CFLAGS += \
         -DARMCOMPUTENEON_ENABLED
 endif # ARMNN_COMPUTE_NEON_ENABLED == 1
+# The variable to enable/disable the REFERENCE backend (ARMNN_COMPUTE_REF_ENABLED) is declared in android-nn-driver/Android.mk
+ifeq ($(ARMNN_COMPUTE_REF_ENABLED),1)
+LOCAL_CFLAGS += \
+        -DARMCOMPUTEREF_ENABLED
+endif # ARMNN_COMPUTE_REF_ENABLED == 1
 
 ifeq ($(Q_OR_LATER),1)
 LOCAL_CFLAGS += \
@@ -256,6 +263,11 @@ ifeq ($(ARMNN_COMPUTE_NEON_ENABLED),1)
 LOCAL_CFLAGS += \
         -DARMCOMPUTENEON_ENABLED
 endif # ARMNN_COMPUTE_NEON_ENABLED == 1
+# The variable to enable/disable the REFERENCE backend (ARMNN_COMPUTE_REF_ENABLED) is declared in android-nn-driver/Android.mk
+ifeq ($(ARMNN_COMPUTE_REF_ENABLED),1)
+LOCAL_CFLAGS += \
+        -DARMCOMPUTEREF_ENABLED
+endif # ARMNN_COMPUTE_REF_ENABLED == 1
 
 ifeq ($(Q_OR_LATER),1)
 LOCAL_CFLAGS += \
@@ -264,22 +276,27 @@ endif # PLATFORM_VERSION == Q or later
 
 LOCAL_SRC_FILES := \
         $(ARMNN_BACKEND_TEST_SOURCES) \
-        src/armnn/test/UnitTests.cpp \
         src/armnn/test/EndToEndTest.cpp \
-        src/armnn/test/UtilsTests.cpp \
         src/armnn/test/GraphTests.cpp \
         src/armnn/test/GraphUtils.cpp \
         src/armnn/test/InferOutputTests.cpp \
-        src/armnn/test/RuntimeTests.cpp \
-        src/armnn/test/SubgraphViewTests.cpp \
-        src/armnn/test/TensorTest.cpp \
-        src/armnn/test/NetworkTests.cpp \
         src/armnn/test/InstrumentTests.cpp \
-        src/armnn/test/ProfilingEventTest.cpp \
+        src/armnn/test/NetworkTests.cpp \
         src/armnn/test/ObservableTest.cpp \
         src/armnn/test/OptionalTest.cpp \
+        src/armnn/test/ProfilingEventTest.cpp \
+        src/armnn/test/SubgraphViewTests.cpp \
+        src/armnn/test/TensorHandleStrategyTest.cpp \
+        src/armnn/test/TensorTest.cpp \
         src/armnn/test/TestUtils.cpp \
-        src/armnn/test/TensorHandleStrategyTest.cpp
+        src/armnn/test/UnitTests.cpp \
+        src/armnn/test/UtilsTests.cpp
+
+ifeq ($(ARMNN_COMPUTE_REF_ENABLED),1)
+LOCAL_SRC_FILES += \
+        src/armnn/test/DebugCallbackTest.cpp \
+        src/armnn/test/RuntimeTests.cpp
+endif
 
 LOCAL_STATIC_LIBRARIES := \
         libneuralnetworks_common \
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9c186e3e6e..2236e2469e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -144,7 +144,7 @@ if(BUILD_TF_PARSER)
     target_link_libraries(armnnTfParser ${PROTOBUF_LIBRARIES})
 endif()
 
-if(BUILD_ARMNN_QUANTIZER)
+if(BUILD_ARMNN_QUANTIZER AND ARMCOMPUTEREF)
     if(NOT BUILD_ARMNN_SERIALIZER)
         message(ERROR, "In order to build the ArmNN Quantization Tool you must set BUILD_ARMNN_SERIALIZER = YES")
     endif()
@@ -486,7 +486,6 @@ if(BUILD_UNIT_TESTS)
         src/armnn/test/ConstTensorLayerVisitor.cpp
         src/armnn/test/CreateWorkload.hpp
         src/armnn/test/CsvReaderTest.cpp
-        src/armnn/test/DebugCallbackTest.cpp
         src/armnn/test/EndToEndTest.cpp
         src/armnn/test/ExecutionFrameTest.cpp
         src/armnn/test/FloatingPointConverterTest.cpp
@@ -504,9 +503,6 @@ if(BUILD_UNIT_TESTS)
         src/armnn/test/OptionalTest.cpp
         src/armnn/test/ProfilerTests.cpp
         src/armnn/test/ProfilingEventTest.cpp
-        src/armnn/test/QuantizerTest.cpp
-        src/armnn/test/RuntimeTests.cpp
-        src/armnn/test/RuntimeTests.hpp
         src/armnn/test/SubgraphViewTests.cpp
         src/armnn/test/TensorHandleStrategyTest.cpp
         src/armnn/test/TensorHelpers.hpp
@@ -529,7 +525,16 @@ if(BUILD_UNIT_TESTS)
         src/profiling/test/ProfilingTests.cpp
         )
 
-    if(BUILD_TF_PARSER)
+    if(ARMCOMPUTEREF)
+        list(APPEND unittest_sources
+            src/armnn/test/DebugCallbackTest.cpp
+            src/armnn/test/QuantizerTest.cpp
+            src/armnn/test/RuntimeTests.cpp
+            src/armnn/test/RuntimeTests.hpp
+            )
+    endif()
+
+    if(BUILD_TF_PARSER AND ARMCOMPUTEREF)
         list(APPEND unittest_sources
             src/armnnTfParser/test/Activations.cpp
             src/armnnTfParser/test/Addition.cpp
@@ -571,12 +576,12 @@ if(BUILD_UNIT_TESTS)
             src/armnnTfParser/test/Squeeze.cpp
             src/armnnTfParser/test/Sub.cpp
             )
+
     endif()
 
-    if(BUILD_TF_LITE_PARSER)
+    if(BUILD_TF_LITE_PARSER AND ARMCOMPUTEREF)
         enable_language(ASM)
         list(APPEND unittest_sources
-             src/armnnTfLiteParser/test/ParserFlatbuffersFixture.hpp
              src/armnnTfLiteParser/test/Activations.cpp
              src/armnnTfLiteParser/test/Addition.cpp
              src/armnnTfLiteParser/test/AvgPool2D.cpp
@@ -615,9 +620,10 @@ if(BUILD_UNIT_TESTS)
              src/armnnTfLiteParser/test/Schema.s
              )
         set_source_files_properties(src/armnnTfLiteParser/test/Schema.s PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp")
+
     endif()
 
-    if(BUILD_CAFFE_PARSER)
+    if(BUILD_CAFFE_PARSER AND ARMCOMPUTEREF)
         list(APPEND unittest_sources
             src/armnnCaffeParser/test/TestAdd.cpp
             src/armnnCaffeParser/test/TestConcat.cpp
@@ -632,13 +638,31 @@ if(BUILD_UNIT_TESTS)
             )
     endif()
 
-    if(BUILD_ARMNN_QUANTIZER)
+    if(BUILD_ONNX_PARSER AND ARMCOMPUTEREF)
+        list(APPEND unittest_sources
+            src/armnnOnnxParser/test/Constructor.cpp
+            src/armnnOnnxParser/test/CreateNetwork.cpp
+            src/armnnOnnxParser/test/ProtoxtFixture.cpp
+            src/armnnOnnxParser/test/Const.cpp
+            src/armnnOnnxParser/test/Pooling.cpp
+            src/armnnOnnxParser/test/Reshape.cpp
+            src/armnnOnnxParser/test/Relu.cpp
+            src/armnnOnnxParser/test/Conv2D.cpp
+            src/armnnOnnxParser/test/Addition.cpp
+            src/armnnOnnxParser/test/FullyConnected.cpp
+            src/armnnOnnxParser/test/GetInputsOutputs.cpp
+            src/armnnOnnxParser/test/BatchNorm.cpp
+            src/armnnOnnxParser/test/DepthConv.cpp
+            )
+    endif()
+
+    if(BUILD_ARMNN_QUANTIZER AND ARMCOMPUTEREF)
         list(APPEND unittest_sources
              src/armnnQuantizer/test/QuantizationDataSetTests.cpp
              )
     endif()
 
-    if(BUILD_ARMNN_SERIALIZER)
+    if(BUILD_ARMNN_SERIALIZER AND ARMCOMPUTEREF)
         enable_language(ASM)
         list(APPEND unittest_sources
             src/armnnSerializer/test/ActivationSerializationTests.cpp
@@ -674,24 +698,6 @@ if(BUILD_UNIT_TESTS)
         set_source_files_properties(src/armnnDeserializer/test/SchemaSerialize.s PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp")
     endif()
 
-    if(BUILD_ONNX_PARSER)
-        list(APPEND unittest_sources
-            src/armnnOnnxParser/test/Constructor.cpp
-            src/armnnOnnxParser/test/CreateNetwork.cpp
-            src/armnnOnnxParser/test/ProtoxtFixture.cpp
-            src/armnnOnnxParser/test/Const.cpp
-            src/armnnOnnxParser/test/Pooling.cpp
-            src/armnnOnnxParser/test/Reshape.cpp
-            src/armnnOnnxParser/test/Relu.cpp
-            src/armnnOnnxParser/test/Conv2D.cpp
-            src/armnnOnnxParser/test/Addition.cpp
-            src/armnnOnnxParser/test/FullyConnected.cpp
-            src/armnnOnnxParser/test/GetInputsOutputs.cpp
-            src/armnnOnnxParser/test/BatchNorm.cpp
-            src/armnnOnnxParser/test/DepthConv.cpp
-            )
-    endif()
-
     foreach(lib ${armnnUnitTestLibraries})
         message("Adding object library dependency to UnitTests: ${lib}")
         list(APPEND unittest_sources $<TARGET_OBJECTS:${lib}>)
@@ -731,13 +737,13 @@ if(BUILD_UNIT_TESTS)
         target_link_libraries(UnitTests armnnCaffeParser)
     endif()
 
-    if(BUILD_ARMNN_SERIALIZER)
+    if(BUILD_ARMNN_SERIALIZER AND ARMCOMPUTEREF)
         target_include_directories(UnitTests SYSTEM PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/armnnSerializer)
         target_include_directories(UnitTests SYSTEM PRIVATE "${FLATBUFFERS_INCLUDE_PATH}")
         target_link_libraries(UnitTests armnnSerializer)
     endif()
 
-    if(BUILD_ARMNN_QUANTIZER)
+    if(BUILD_ARMNN_QUANTIZER AND ARMCOMPUTEREF)
         target_include_directories(UnitTests SYSTEM PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/armnnQuantizer)
         target_include_directories(UnitTests SYSTEM PRIVATE "${FLATBUFFERS_INCLUDE_PATH}")
         target_link_libraries(UnitTests armnnQuantizer armnnSerializer)
@@ -750,7 +756,7 @@ if(BUILD_UNIT_TESTS)
     addDllCopyCommands(UnitTests)
 endif()
 
-if (BUILD_ARMNN_SERIALIZER AND (BUILD_TF_PARSER OR BUILD_TF_LITE_PARSER OR BUILD_ONNX_PARSER OR BUILD_CAFFE_PARSER))
+if (BUILD_ARMNN_SERIALIZER AND (BUILD_TF_PARSER OR BUILD_TF_LITE_PARSER OR BUILD_ONNX_PARSER OR BUILD_CAFFE_PARSER) AND ARMCOMPUTEREF)
     set(ArmnnConverter_sources
         src/armnnConverter/ArmnnConverter.cpp)
 
diff --git a/cmake/GlobalConfig.cmake b/cmake/GlobalConfig.cmake
index 8dad57a297..0063cd19d5 100644
--- a/cmake/GlobalConfig.cmake
+++ b/cmake/GlobalConfig.cmake
@@ -6,6 +6,7 @@ option(BUILD_TESTS "Build test applications" OFF)
 option(BUILD_FOR_COVERAGE "Use no optimization and output .gcno and .gcda files" OFF)
 option(ARMCOMPUTENEON "Build with ARM Compute NEON support" OFF)
 option(ARMCOMPUTECL "Build with ARM Compute OpenCL support" OFF)
+option(ARMCOMPUTEREF "Build with ARM Compute reference support" OFF)
 option(PROFILING_BACKEND_STREAMLINE "Forward the armNN profiling events to DS-5/Streamline as annotations" OFF)
 # options used for heap profiling and leak checking
 option(HEAP_PROFILING "Build with heap profiling enabled" OFF)
@@ -289,6 +290,11 @@ if(ARMCOMPUTENEON OR ARMCOMPUTECL)
     include_directories(SYSTEM ${HALF_INCLUDE})
 endif()
 
+# ARM Compute reference backend
+if(ARMCOMPUTEREF)
+    add_definitions(-DARMCOMPUTEREF_ENABLED)
+endif()
+
 # Streamline annotate
 if(PROFILING_BACKEND_STREAMLINE)
     include_directories("${GATOR_ROOT}/annotate")
diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt
index 5716054eb1..c12842d06f 100644
--- a/samples/CMakeLists.txt
+++ b/samples/CMakeLists.txt
@@ -1,4 +1,4 @@
-if(BUILD_SAMPLE_APP)
+if(BUILD_SAMPLE_APP AND ARMCOMPUTEREF)
     add_executable(SimpleSample SimpleSample.cpp)
     target_link_libraries(SimpleSample armnn ${CMAKE_THREAD_LIBS_INIT})
 endif()
diff --git a/src/armnn/test/ProfilerTests.cpp b/src/armnn/test/ProfilerTests.cpp
index 90f6d8b133..9290e69c96 100644
--- a/src/armnn/test/ProfilerTests.cpp
+++ b/src/armnn/test/ProfilerTests.cpp
@@ -158,6 +158,10 @@ BOOST_AUTO_TEST_CASE(ProfilingMacros)
     profiler->EnableProfiling(false);
 }
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// This test unit needs the reference backend, it's not available if the reference backend is not built
+
 BOOST_AUTO_TEST_CASE(RuntimeLoadNetwork)
 {
     // Get a reference to the profiler manager.
@@ -185,6 +189,8 @@ BOOST_AUTO_TEST_CASE(RuntimeLoadNetwork)
     BOOST_TEST(!profilerManager.GetProfiler());
 }
 
+#endif
+
 BOOST_AUTO_TEST_CASE(WriteEventResults)
 {
     // Get a reference to the profiler manager.
diff --git a/src/backends/backendsCommon/common.mk b/src/backends/backendsCommon/common.mk
index 0ac7cfeada..d7fe9b159f 100644
--- a/src/backends/backendsCommon/common.mk
+++ b/src/backends/backendsCommon/common.mk
@@ -33,5 +33,9 @@ COMMON_TEST_SOURCES := \
     test/CommonTestUtils.cpp \
     test/JsonPrinterTestImpl.cpp \
     test/LayerTests.cpp \
-    test/TensorCopyUtils.cpp \
+    test/TensorCopyUtils.cpp
+
+ifeq ($(ARMNN_COMPUTE_REF_ENABLED),1)
+COMMON_TEST_SOURCES += \
     test/WorkloadDataValidation.cpp
+endif # ARMNN_COMPUTE_REF_ENABLED == 1
diff --git a/src/backends/backendsCommon/test/CMakeLists.txt b/src/backends/backendsCommon/test/CMakeLists.txt
index 684b27f2e1..d6f8a6f20b 100644
--- a/src/backends/backendsCommon/test/CMakeLists.txt
+++ b/src/backends/backendsCommon/test/CMakeLists.txt
@@ -37,7 +37,6 @@ list(APPEND armnnBackendsCommonUnitTests_sources
     MockBackend.cpp
     MockBackend.hpp
     MockBackendId.hpp
-    OptimizedNetworkTests.cpp
     OptimizeSubgraphViewTests.cpp
     OptimizationViewsTests.cpp
     PermuteTestImpl.hpp
@@ -56,11 +55,17 @@ list(APPEND armnnBackendsCommonUnitTests_sources
     StridedSliceTestImpl.hpp
     TensorCopyUtils.cpp
     TensorCopyUtils.hpp
-    WorkloadDataValidation.cpp
     WorkloadFactoryHelper.hpp
     WorkloadTestUtils.hpp
 )
 
+if (ARMCOMPUTEREF)
+    list(APPEND armnnBackendsCommonUnitTests_sources
+        OptimizedNetworkTests.cpp
+        WorkloadDataValidation.cpp
+        )
+endif()
+
 add_library(armnnBackendsCommonUnitTests OBJECT ${armnnBackendsCommonUnitTests_sources})
 target_include_directories(armnnBackendsCommonUnitTests PRIVATE ${PROJECT_SOURCE_DIR}/src/armnn)
 target_include_directories(armnnBackendsCommonUnitTests PRIVATE ${PROJECT_SOURCE_DIR}/src/armnnUtils)
diff --git a/src/backends/backendsCommon/test/DynamicBackendTests.cpp b/src/backends/backendsCommon/test/DynamicBackendTests.cpp
index e42a08adc7..17a99f8b35 100644
--- a/src/backends/backendsCommon/test/DynamicBackendTests.cpp
+++ b/src/backends/backendsCommon/test/DynamicBackendTests.cpp
@@ -61,6 +61,12 @@ ARMNN_SIMPLE_TEST_CASE(RuntimeDuplicateDynamicBackends, RuntimeDuplicateDynamicB
 ARMNN_SIMPLE_TEST_CASE(RuntimeInvalidDynamicBackends, RuntimeInvalidDynamicBackendsTestImpl);
 ARMNN_SIMPLE_TEST_CASE(RuntimeInvalidOverridePath, RuntimeInvalidOverridePathTestImpl);
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// This test unit needs the reference backend, it's not available if the reference backend is not built
+
 ARMNN_SIMPLE_TEST_CASE(CreateReferenceDynamicBackend, CreateReferenceDynamicBackendTestImpl);
 
+#endif
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/backends/backendsCommon/test/DynamicBackendTests.hpp b/src/backends/backendsCommon/test/DynamicBackendTests.hpp
index e225124e01..6c093fc952 100644
--- a/src/backends/backendsCommon/test/DynamicBackendTests.hpp
+++ b/src/backends/backendsCommon/test/DynamicBackendTests.hpp
@@ -1344,6 +1344,10 @@ void RuntimeInvalidOverridePathTestImpl()
     BOOST_TEST(supportedBackendIds.empty());
 }
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// This test unit needs the reference backend, it's not available if the reference backend is not built
+
 void CreateReferenceDynamicBackendTestImpl()
 {
     using namespace armnn;
@@ -1427,3 +1431,5 @@ void CreateReferenceDynamicBackendTestImpl()
     BOOST_TEST((workload != nullptr));
     BOOST_TEST(workload.get() == boost::polymorphic_downcast<RefConvolution2dWorkload*>(workload.get()));
 }
+
+#endif
diff --git a/src/backends/backendsCommon/test/LayerTests.cpp b/src/backends/backendsCommon/test/LayerTests.cpp
index 2201499b3a..561e526345 100644
--- a/src/backends/backendsCommon/test/LayerTests.cpp
+++ b/src/backends/backendsCommon/test/LayerTests.cpp
@@ -20,8 +20,6 @@
 #include <backendsCommon/IBackendInternal.hpp>
 #include <backendsCommon/WorkloadFactory.hpp>
 
-#include <reference/workloads/RefWorkloads.hpp>
-
 #include <algorithm>
 #include <boost/cast.hpp>
 
@@ -2088,6 +2086,10 @@ LayerTestResult<int16_t, 3> CopyViaSplitterInt16Test(
     return CopyViaSplitterTestImpl<armnn::DataType::QuantisedSymm16>(workloadFactory, memoryManager, 1.0f, 0);
 }
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// The LSTM test units are run only for the reference backend at the moment
+
 void LstmUtilsZeroVectorTest()
 {
     armnn::TensorInfo inputDesc({4}, armnn::DataType::Float32);
@@ -2234,6 +2236,7 @@ void LstmUtilsVectorBatchVectorAddTest()
             vecSize, batchSize, expectedOutput);
 }
 
+#endif
 
 LayerTestResult<float, 2> LstmLayerFloat32WithCifgWithPeepholeNoProjectionTest(
     armnn::IWorkloadFactory& workloadFactory,
diff --git a/src/backends/cl/CMakeLists.txt b/src/backends/cl/CMakeLists.txt
index 36db107a8a..ad4a15f2e8 100644
--- a/src/backends/cl/CMakeLists.txt
+++ b/src/backends/cl/CMakeLists.txt
@@ -14,6 +14,7 @@ if(ARMCOMPUTECL)
         ClContextControl.hpp
         ClLayerSupport.cpp
         ClLayerSupport.hpp
+        ClRegistryInitializer.cpp
         ClTensorHandle.hpp
         ClTensorHandleFactory.cpp
         ClTensorHandleFactory.hpp
diff --git a/src/backends/cl/ClBackend.cpp b/src/backends/cl/ClBackend.cpp
index 123d0639de..95ffbc48fd 100644
--- a/src/backends/cl/ClBackend.cpp
+++ b/src/backends/cl/ClBackend.cpp
@@ -25,21 +25,6 @@
 namespace armnn
 {
 
-namespace
-{
-
-static BackendRegistry::StaticRegistryInitializer g_RegisterHelper
-{
-    BackendRegistryInstance(),
-    ClBackend::GetIdStatic(),
-    []()
-    {
-        return IBackendInternalUniquePtr(new ClBackend);
-    }
-};
-
-}
-
 const BackendId& ClBackend::GetIdStatic()
 {
     static const BackendId s_Id{ClBackendId()};
diff --git a/src/backends/cl/ClRegistryInitializer.cpp b/src/backends/cl/ClRegistryInitializer.cpp
new file mode 100644
index 0000000000..4d6f5f4a05
--- /dev/null
+++ b/src/backends/cl/ClRegistryInitializer.cpp
@@ -0,0 +1,25 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include "ClBackend.hpp"
+
+#include <backendsCommon/BackendRegistry.hpp>
+
+namespace
+{
+
+using namespace armnn;
+
+static BackendRegistry::StaticRegistryInitializer g_RegisterHelper
+{
+    BackendRegistryInstance(),
+    ClBackend::GetIdStatic(),
+    []()
+    {
+        return IBackendInternalUniquePtr(new ClBackend);
+    }
+};
+
+} // Anonymous namespace
diff --git a/src/backends/cl/backend.mk b/src/backends/cl/backend.mk
index 98da871e70..104200470b 100644
--- a/src/backends/cl/backend.mk
+++ b/src/backends/cl/backend.mk
@@ -7,7 +7,7 @@
 # in the Android build and it is picked up by the Android.mk
 # file in the root of ArmNN
 
-# The variable to enable/disable the CL backend (ARMNN_COMPUTE_CL_ENABLED) is declared in android-nn-driver/Android.mk
+# The variable to enable/disable the CL backend (ARMNN_COMPUTE_CL_ENABLED is declared in android-nn-driver/Android.mk)
 ifeq ($(ARMNN_COMPUTE_CL_ENABLED),1)
 
 # ARMNN_COMPUTE_CL_ENABLED == 1
@@ -18,6 +18,7 @@ BACKEND_SOURCES := \
         ClBackendContext.cpp \
         ClContextControl.cpp \
         ClLayerSupport.cpp \
+        ClRegistryInitializer.cpp \
         ClTensorHandleFactory.cpp \
         ClWorkloadFactory.cpp \
         OpenClTimer.cpp \
@@ -74,7 +75,7 @@ endif
 # in the Android unit test build (armnn-tests) and it is picked
 # up by the Android.mk file in the root of ArmNN
 
-# The variable to enable/disable the CL backend (ARMNN_COMPUTE_CL_ENABLED) is declared in android-nn-driver/Android.mk
+# The variable to enable/disable the CL backend (ARMNN_COMPUTE_CL_ENABLED is declared in android-nn-driver/Android.mk)
 ifeq ($(ARMNN_COMPUTE_CL_ENABLED),1)
 
 # ARMNN_COMPUTE_CL_ENABLED == 1
@@ -86,11 +87,16 @@ BACKEND_TEST_SOURCES := \
         test/ClJsonPrinterTests.cpp \
         test/ClLayerSupportTests.cpp \
         test/ClLayerTests.cpp \
-        test/ClMemCopyTests.cpp \
         test/ClOptimizedNetworkTests.cpp \
         test/ClRuntimeTests.cpp \
         test/Fp16SupportTest.cpp \
         test/OpenClTimerTest.cpp
+
+ifeq ($(ARMNN_COMPUTE_REF_ENABLED),1)
+BACKEND_TEST_SOURCES += \
+        test/ClMemCopyTests.cpp
+endif # ARMNN_COMPUTE_REF_ENABLED == 1
+
 else
 
 # ARMNN_COMPUTE_CL_ENABLED == 0
diff --git a/src/backends/cl/test/CMakeLists.txt b/src/backends/cl/test/CMakeLists.txt
index 206cf5a9dd..4ec5051fa2 100644
--- a/src/backends/cl/test/CMakeLists.txt
+++ b/src/backends/cl/test/CMakeLists.txt
@@ -10,7 +10,6 @@ list(APPEND armnnClBackendUnitTests_sources
     ClJsonPrinterTests.cpp
     ClLayerSupportTests.cpp
     ClLayerTests.cpp
-    ClMemCopyTests.cpp
     ClOptimizedNetworkTests.cpp
     ClRuntimeTests.cpp
     ClWorkloadFactoryHelper.hpp
@@ -18,6 +17,12 @@ list(APPEND armnnClBackendUnitTests_sources
     OpenClTimerTest.cpp
 )
 
+if (ARMCOMPUTEREF)
+    list(APPEND armnnClBackendUnitTests_sources
+        ClMemCopyTests.cpp
+        )
+endif()
+
 add_library(armnnClBackendUnitTests OBJECT ${armnnClBackendUnitTests_sources})
 target_include_directories(armnnClBackendUnitTests PRIVATE ${PROJECT_SOURCE_DIR}/src/armnn)
 target_include_directories(armnnClBackendUnitTests PRIVATE ${PROJECT_SOURCE_DIR}/src/armnnUtils)
diff --git a/src/backends/cl/test/ClCreateWorkloadTests.cpp b/src/backends/cl/test/ClCreateWorkloadTests.cpp
index bb36504214..ff9d23a699 100644
--- a/src/backends/cl/test/ClCreateWorkloadTests.cpp
+++ b/src/backends/cl/test/ClCreateWorkloadTests.cpp
@@ -686,6 +686,10 @@ BOOST_AUTO_TEST_CASE(CreateSingleOutputMultipleInputs)
     BOOST_TEST(validDataPointers);
 }
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// This test unit needs the reference backend, it's not available if the reference backend is not built
+
 BOOST_AUTO_TEST_CASE(CreateMemCopyWorkloadsCl)
 {
     ClWorkloadFactory factory =
@@ -694,6 +698,8 @@ BOOST_AUTO_TEST_CASE(CreateMemCopyWorkloadsCl)
     CreateMemCopyWorkloads<IClTensorHandle>(factory);
 }
 
+#endif
+
 template <typename L2NormalizationWorkloadType, typename armnn::DataType DataType>
 static void ClL2NormalizationWorkloadTest(DataLayout dataLayout)
 {
diff --git a/src/backends/cl/test/ClLayerTests.cpp b/src/backends/cl/test/ClLayerTests.cpp
index 8a5435b83c..092aa61a1f 100644
--- a/src/backends/cl/test/ClLayerTests.cpp
+++ b/src/backends/cl/test/ClLayerTests.cpp
@@ -13,7 +13,6 @@
 #include <cl/ClLayerSupport.hpp>
 #include <cl/ClWorkloadFactory.hpp>
 #include <cl/workloads/ClWorkloadUtils.hpp>
-#include <reference/RefWorkloadFactory.hpp>
 #include <backendsCommon/test/ActivationFixture.hpp>
 #include <backendsCommon/test/LayerTests.hpp>
 #include <backendsCommon/test/PermuteTestImpl.hpp>
@@ -463,10 +462,6 @@ ARMNN_AUTO_TEST_CASE(Simple3dSoftmaxUint8, Simple3dSoftmaxUint8Test, 1.0f)
 ARMNN_AUTO_TEST_CASE(Simple4dSoftmax, Simple4dSoftmaxTest, 1.0f)
 ARMNN_AUTO_TEST_CASE(Simple4dSoftmaxUint8, Simple4dSoftmaxUint8Test, 1.0f)
 
-ARMNN_COMPARE_REF_AUTO_TEST_CASE(CompareSoftmaxBeta1WithReference, CompareSoftmaxTest, 1.0f)
-ARMNN_COMPARE_REF_AUTO_TEST_CASE(CompareSoftmaxBeta2WithReference, CompareSoftmaxTest, 2.0f)
-ARMNN_COMPARE_REF_AUTO_TEST_CASE(CompareSoftmaxUint8, CompareSoftmaxUint8Test, 1.0f)
-
 // Space To Batch Nd
 ARMNN_AUTO_TEST_CASE(SpaceToBatchNdSimpleFloat32, SpaceToBatchNdSimpleFloat32Test)
 ARMNN_AUTO_TEST_CASE(SpaceToBatchNdMultiChannelsFloat32, SpaceToBatchNdMultiChannelsFloat32Test)
@@ -773,9 +768,18 @@ ARMNN_AUTO_TEST_CASE(MultiChannelTransposeConvolution2dUint8Nhwc,
                      MultiChannelTransposeConvolution2dTest<DataType::QuantisedAsymm8, DataType::Signed32>,
                      DataLayout::NHWC)
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// The ARMNN_COMPARE_REF_AUTO_TEST_CASE and the ARMNN_COMPARE_REF_FIXTURE_TEST_CASE test units are not available
+// if the reference backend is not built
+
 // ============================================================================
 // COMPARE tests
 
+ARMNN_COMPARE_REF_AUTO_TEST_CASE(CompareSoftmaxBeta1WithReference, CompareSoftmaxTest, 1.0f)
+ARMNN_COMPARE_REF_AUTO_TEST_CASE(CompareSoftmaxBeta2WithReference, CompareSoftmaxTest, 2.0f)
+ARMNN_COMPARE_REF_AUTO_TEST_CASE(CompareSoftmaxUint8, CompareSoftmaxUint8Test, 1.0f)
+
 ARMNN_COMPARE_REF_AUTO_TEST_CASE(CompareConv2dWithReference, CompareConvolution2dTest)
 
 ARMNN_COMPARE_REF_AUTO_TEST_CASE(CompareDepthwiseConv2dWithReferenceFloat32,
@@ -851,4 +855,6 @@ ARMNN_COMPARE_REF_FIXTURE_TEST_CASE(CompareSqrtActivationWithReference, Positive
 ARMNN_COMPARE_REF_FIXTURE_TEST_CASE(CompareSquareActivationWithReference, ActivationFixture,
                                     CompareActivationTest, ActivationFunction::Square, 5u)
 
+#endif
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/backends/dynamic/reference/CMakeLists.txt b/src/backends/dynamic/reference/CMakeLists.txt
index e9a94af2fc..6c040ecc81 100644
--- a/src/backends/dynamic/reference/CMakeLists.txt
+++ b/src/backends/dynamic/reference/CMakeLists.txt
@@ -14,7 +14,8 @@ set(RefBackendPath ${PROJECT_SOURCE_DIR}/src/backends/reference)
 
 # Source files of the backend, taken directly from the source tree
 file(GLOB RefBackendBaseFiles ${RefBackendPath}/*.cpp)
-set(RefBackendFiles ${RefBackendBaseFiles})
+file(GLOB RefBackendWorloadFiles ${RefBackendPath}/workloads/*.cpp)
+set(RefBackendFiles ${RefBackendBaseFiles} ${RefBackendWorloadFiles})
 
 # Remove the file that contains the static backend registration
 list(REMOVE_ITEM RefBackendFiles ${RefBackendPath}/RefRegistryInitializer.cpp)
diff --git a/src/backends/neon/CMakeLists.txt b/src/backends/neon/CMakeLists.txt
index 7464a2e623..f99f1ac3e7 100644
--- a/src/backends/neon/CMakeLists.txt
+++ b/src/backends/neon/CMakeLists.txt
@@ -12,13 +12,14 @@ if(ARMCOMPUTENEON)
         NeonInterceptorScheduler.cpp
         NeonLayerSupport.cpp
         NeonLayerSupport.hpp
-        NeonWorkloadFactory.cpp
-        NeonWorkloadFactory.hpp
+        NeonRegistryInitializer.cpp
         NeonTensorHandle.hpp
         NeonTensorHandleFactory.cpp
         NeonTensorHandleFactory.hpp
         NeonTimer.hpp
         NeonTimer.cpp
+        NeonWorkloadFactory.cpp
+        NeonWorkloadFactory.hpp
     )
 
     add_subdirectory(workloads)
diff --git a/src/backends/neon/NeonBackend.cpp b/src/backends/neon/NeonBackend.cpp
index 5df231b5e3..60f8ba6dab 100644
--- a/src/backends/neon/NeonBackend.cpp
+++ b/src/backends/neon/NeonBackend.cpp
@@ -25,21 +25,6 @@
 namespace armnn
 {
 
-namespace
-{
-
-static BackendRegistry::StaticRegistryInitializer g_RegisterHelper
-{
-    BackendRegistryInstance(),
-    NeonBackend::GetIdStatic(),
-    []()
-    {
-        return IBackendInternalUniquePtr(new NeonBackend);
-    }
-};
-
-}
-
 const BackendId& NeonBackend::GetIdStatic()
 {
     static const BackendId s_Id{NeonBackendId()};
diff --git a/src/backends/neon/NeonRegistryInitializer.cpp b/src/backends/neon/NeonRegistryInitializer.cpp
new file mode 100644
index 0000000000..c74acae96b
--- /dev/null
+++ b/src/backends/neon/NeonRegistryInitializer.cpp
@@ -0,0 +1,25 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include "NeonBackend.hpp"
+
+#include <backendsCommon/BackendRegistry.hpp>
+
+namespace
+{
+
+using namespace armnn;
+
+static BackendRegistry::StaticRegistryInitializer g_RegisterHelper
+{
+    BackendRegistryInstance(),
+    NeonBackend::GetIdStatic(),
+    []()
+    {
+        return IBackendInternalUniquePtr(new NeonBackend);
+    }
+};
+
+} // Anonymous namespace
diff --git a/src/backends/neon/backend.mk b/src/backends/neon/backend.mk
index 9ceb843868..bf7e25d5f0 100644
--- a/src/backends/neon/backend.mk
+++ b/src/backends/neon/backend.mk
@@ -7,7 +7,7 @@
 # in the Android build and it is picked up by the Android.mk
 # file in the root of ArmNN
 
-# The variable to enable/disable the NEON backend (ARMNN_COMPUTE_NEON_ENABLED) is declared in android-nn-driver/Android.mk
+# The variable to enable/disable the NEON backend (ARMNN_COMPUTE_NEON_ENABLED is declared in android-nn-driver/Android.mk)
 ifeq ($(ARMNN_COMPUTE_NEON_ENABLED),1)
 
 # ARMNN_COMPUTE_NEON_ENABLED == 1
@@ -17,6 +17,7 @@ BACKEND_SOURCES := \
         NeonBackend.cpp \
         NeonInterceptorScheduler.cpp \
         NeonLayerSupport.cpp \
+        NeonRegistryInitializer.cpp \
         NeonTensorHandleFactory.cpp \
         NeonTimer.cpp \
         NeonWorkloadFactory.cpp \
@@ -71,7 +72,7 @@ endif
 # in the Android unit test build (armnn-tests) and it is picked
 # up by the Android.mk file in the root of ArmNN
 
-# The variable to enable/disable the NEON backend (ARMNN_COMPUTE_NEON_ENABLED) is declared in android-nn-driver/Android.mk
+# The variable to enable/disable the NEON backend (ARMNN_COMPUTE_NEON_ENABLED is declared in android-nn-driver/Android.mk)
 ifeq ($(ARMNN_COMPUTE_NEON_ENABLED),1)
 
 # ARMNN_COMPUTE_NEON_ENABLED == 1
@@ -83,11 +84,15 @@ BACKEND_TEST_SOURCES := \
         test/NeonJsonPrinterTests.cpp \
         test/NeonLayerSupportTests.cpp \
         test/NeonLayerTests.cpp \
-        test/NeonMemCopyTests.cpp \
         test/NeonOptimizedNetworkTests.cpp \
         test/NeonRuntimeTests.cpp \
         test/NeonTimerTest.cpp
 
+ifeq ($(ARMNN_COMPUTE_REF_ENABLED),1)
+BACKEND_TEST_SOURCES += \
+        test/NeonMemCopyTests.cpp
+endif # ARMNN_COMPUTE_REF_ENABLED == 1
+
 else
 
 # ARMNN_COMPUTE_NEON_ENABLED == 0
diff --git a/src/backends/neon/test/CMakeLists.txt b/src/backends/neon/test/CMakeLists.txt
index 19512f98eb..87929fba43 100644
--- a/src/backends/neon/test/CMakeLists.txt
+++ b/src/backends/neon/test/CMakeLists.txt
@@ -9,13 +9,18 @@ list(APPEND armnnNeonBackendUnitTests_sources
     NeonJsonPrinterTests.cpp
     NeonLayerSupportTests.cpp
     NeonLayerTests.cpp
-    NeonMemCopyTests.cpp
     NeonOptimizedNetworkTests.cpp
     NeonRuntimeTests.cpp
     NeonTimerTest.cpp
     NeonWorkloadFactoryHelper.hpp
 )
 
+if (ARMCOMPUTEREF)
+    list(APPEND armnnNeonBackendUnitTests_sources
+        NeonMemCopyTests.cpp
+        )
+endif()
+
 add_library(armnnNeonBackendUnitTests OBJECT ${armnnNeonBackendUnitTests_sources})
 target_include_directories(armnnNeonBackendUnitTests PRIVATE ${PROJECT_SOURCE_DIR}/src/armnn)
 target_include_directories(armnnNeonBackendUnitTests PRIVATE ${PROJECT_SOURCE_DIR}/src/armnnUtils)
diff --git a/src/backends/neon/test/NeonCreateWorkloadTests.cpp b/src/backends/neon/test/NeonCreateWorkloadTests.cpp
index 056bfb283f..6317ac2cc7 100644
--- a/src/backends/neon/test/NeonCreateWorkloadTests.cpp
+++ b/src/backends/neon/test/NeonCreateWorkloadTests.cpp
@@ -710,6 +710,10 @@ BOOST_AUTO_TEST_CASE(CreateSingleOutputMultipleInputs)
     BOOST_TEST(validDataPointers);
 }
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// This test unit needs the reference backend, it's not available if the reference backend is not built
+
 BOOST_AUTO_TEST_CASE(CreateMemCopyWorkloadsNeon)
 {
     NeonWorkloadFactory factory =
@@ -717,6 +721,8 @@ BOOST_AUTO_TEST_CASE(CreateMemCopyWorkloadsNeon)
     CreateMemCopyWorkloads<IAclTensorHandle>(factory);
 }
 
+#endif
+
 template <typename L2NormalizationWorkloadType, typename armnn::DataType DataType>
 static void NeonCreateL2NormalizationWorkloadTest(DataLayout dataLayout)
 {
diff --git a/src/backends/neon/test/NeonEndToEndTests.cpp b/src/backends/neon/test/NeonEndToEndTests.cpp
index 9f94151c1d..bee5b7e76c 100644
--- a/src/backends/neon/test/NeonEndToEndTests.cpp
+++ b/src/backends/neon/test/NeonEndToEndTests.cpp
@@ -25,6 +25,10 @@ BOOST_AUTO_TEST_CASE(ConstantUsage_Neon_Float32)
     BOOST_TEST(ConstantUsageFloat32Test(defaultBackends));
 }
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// This test unit needs the reference backend, it's not available if the reference backend is not built
+
 BOOST_AUTO_TEST_CASE(FallbackToCpuRef)
 {
     using namespace armnn;
@@ -59,6 +63,8 @@ BOOST_AUTO_TEST_CASE(FallbackToCpuRef)
     BOOST_TEST(runtime->LoadNetwork(netId, std::move(optNet)) == Status::Success);
 }
 
+#endif
+
 BOOST_AUTO_TEST_CASE(NeonGreaterSimpleEndToEndTest)
 {
     const std::vector<uint8_t> expectedOutput({ 0, 0, 0, 0,  1, 1, 1, 1,
diff --git a/src/backends/neon/test/NeonLayerTests.cpp b/src/backends/neon/test/NeonLayerTests.cpp
index e795174144..27f1dd24da 100644
--- a/src/backends/neon/test/NeonLayerTests.cpp
+++ b/src/backends/neon/test/NeonLayerTests.cpp
@@ -848,6 +848,11 @@ ARMNN_AUTO_TEST_CASE(MultiChannelTransposeConvolution2dUint8Nhwc,
                      MultiChannelTransposeConvolution2dTest<DataType::QuantisedAsymm8, DataType::Signed32>,
                      DataLayout::NHWC)
 
+#if defined(ARMCOMPUTEREF_ENABLED)
+
+// The ARMNN_COMPARE_REF_AUTO_TEST_CASE and the ARMNN_COMPARE_REF_FIXTURE_TEST_CASE test units are not available
+// if the reference backend is not built
+
 // ============================================================================
 // COMPARE tests
 
@@ -934,4 +939,7 @@ ARMNN_COMPARE_REF_FIXTURE_TEST_CASE(CompareSqrtActivationWithReference, Positive
 
 ARMNN_COMPARE_REF_FIXTURE_TEST_CASE(CompareSquareActivationWithReference, ActivationFixture,
                                     CompareActivationTest, ActivationFunction::Square, 5u)
+
+#endif
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/backends/reference/CMakeLists.txt b/src/backends/reference/CMakeLists.txt
index 6852ab0b74..ff253787a1 100644
--- a/src/backends/reference/CMakeLists.txt
+++ b/src/backends/reference/CMakeLists.txt
@@ -3,30 +3,39 @@
 # SPDX-License-Identifier: MIT
 #
 
-list(APPEND armnnRefBackend_sources
-    RefBackend.cpp
-    RefBackend.hpp
-    RefBackendId.hpp
-    RefTensorHandle.hpp
-    RefTensorHandle.cpp
-    RefLayerSupport.cpp
-    RefLayerSupport.hpp
-    RefMemoryManager.hpp
-    RefMemoryManager.cpp
-    RefRegistryInitializer.cpp
-    RefWorkloadFactory.cpp
-    RefWorkloadFactory.hpp
-    RefTensorHandleFactory.hpp
-    RefTensorHandleFactory.cpp
-)
+if(ARMCOMPUTEREF)
+    list(APPEND armnnRefBackend_sources
+        RefBackend.cpp
+        RefBackend.hpp
+        RefBackendId.hpp
+        RefTensorHandle.hpp
+        RefTensorHandle.cpp
+        RefLayerSupport.cpp
+        RefLayerSupport.hpp
+        RefMemoryManager.hpp
+        RefMemoryManager.cpp
+        RefRegistryInitializer.cpp
+        RefWorkloadFactory.cpp
+        RefWorkloadFactory.hpp
+        RefTensorHandleFactory.cpp
+        RefTensorHandleFactory.hpp
+    )
+
+    add_subdirectory(workloads)
+
+    if(BUILD_UNIT_TESTS)
+        add_subdirectory(test)
+    endif()
+
+else()
+    list(APPEND armnnRefBackend_sources
+        RefBackendId.hpp
+        RefLayerSupport.cpp
+        RefLayerSupport.hpp
+    )
+endif()
 
 add_library(armnnRefBackend OBJECT ${armnnRefBackend_sources})
 target_include_directories(armnnRefBackend PRIVATE ${PROJECT_SOURCE_DIR}/src/armnn)
 target_include_directories(armnnRefBackend PRIVATE ${PROJECT_SOURCE_DIR}/src/armnnUtils)
 target_include_directories(armnnRefBackend PRIVATE ${PROJECT_SOURCE_DIR}/src/backends)
-
-add_subdirectory(workloads)
-
-if(BUILD_UNIT_TESTS)
-    add_subdirectory(test)
-endif()
diff --git a/src/backends/reference/backend.cmake b/src/backends/reference/backend.cmake
index 5ae088a124..e88968de8b 100644
--- a/src/backends/reference/backend.cmake
+++ b/src/backends/reference/backend.cmake
@@ -4,5 +4,11 @@
 #
 
 add_subdirectory(${PROJECT_SOURCE_DIR}/src/backends/reference)
-list(APPEND armnnLibraries armnnRefBackend armnnRefBackendWorkloads)
-list(APPEND armnnUnitTestLibraries armnnRefBackendUnitTests)
+list(APPEND armnnLibraries armnnRefBackend)
+
+if(ARMCOMPUTEREF)
+    list(APPEND armnnLibraries armnnRefBackendWorkloads)
+    list(APPEND armnnUnitTestLibraries armnnRefBackendUnitTests)
+else()
+    message("Reference backend is disabled")
+endif()
diff --git a/src/backends/reference/backend.mk b/src/backends/reference/backend.mk
index a8df565947..13251b779a 100644
--- a/src/backends/reference/backend.mk
+++ b/src/backends/reference/backend.mk
@@ -7,6 +7,12 @@
 # in the Android build and it is picked up by the Android.mk
 # file in the root of ArmNN
 
+# The variable to enable/disable the reference backend (ARMNN_COMPUTE_REF_ENABLED is declared in android-nn-driver/Android.mk)
+ifeq ($(ARMNN_COMPUTE_REF_ENABLED),1)
+
+# ARMNN_COMPUTE_REF_ENABLED == 1
+# Include the source files for the reference backend
+
 BACKEND_SOURCES := \
         RefBackend.cpp \
         RefLayerSupport.cpp \
@@ -78,11 +84,25 @@ BACKEND_SOURCES := \
         workloads/Softmax.cpp \
         workloads/Splitter.cpp \
         workloads/TransposeConvolution2d.cpp
+else
+
+# ARMNN_COMPUTE_REF_ENABLED == 0
+# No source file will be compiled for the reference backend
+
+BACKEND_SOURCES :=
+
+endif
 
 # BACKEND_TEST_SOURCES contains the list of files to be included
 # in the Android unit test build (armnn-tests) and it is picked
 # up by the Android.mk file in the root of ArmNN
 
+# The variable to enable/disable the CL backend (ARMNN_COMPUTE_REF_ENABLED is declared in android-nn-driver/Android.mk)
+ifeq ($(ARMNN_COMPUTE_REF_ENABLED),1)
+
+# ARMNN_COMPUTE_REF_ENABLED == 1
+# Include the source files for the CL backend tests
+
 BACKEND_TEST_SOURCES := \
         test/RefCreateWorkloadTests.cpp \
         test/RefDetectionPostProcessTests.cpp \
@@ -93,3 +113,11 @@ BACKEND_TEST_SOURCES := \
         test/RefMemoryManagerTests.cpp \
         test/RefOptimizedNetworkTests.cpp \
         test/RefRuntimeTests.cpp
+else
+
+# ARMNN_COMPUTE_REF_ENABLED == 0
+# No source file will be compiled for the reference backend tests
+
+BACKEND_TEST_SOURCES :=
+
+endif