COMPMID-1081: Introduced test-wide instruments

Change-Id: I5831241f3fc503717cc51136453c2bf96d4b420b
Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/128484
Tested-by: Jenkins <bsgcomp@arm.com>
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>

14 files changed, 163 insertions, 103 deletions

diff --git a/SConscript b/SConscript
index 936fde0f1f..983570b7dc 100644
--- a/SConscript
+++ b/SConscript
@@ -153,9 +153,6 @@ if env["build"] == "embed_only":
 arm_compute_env.Append(LINKFLAGS=['-Wl,--no-undefined'])
 arm_compute_env.Append(CPPPATH =[Dir("./src/core/").path] )
 
-if env["os"] not in ["android", "bare_metal"]:
-    arm_compute_env.Append(LIBS = ['pthread'])
-
 arm_compute_env.Append(LIBS = ['dl'])
 
 core_files = Glob('src/core/*.cpp')
diff --git a/SConstruct b/SConstruct
index 626a302df6..23b9b2f7b4 100644
--- a/SConstruct
+++ b/SConstruct
@@ -196,8 +196,6 @@ if not GetOption("help"):
 if env['standalone']:
     env.Append(CXXFLAGS = ['-fPIC'])
     env.Append(LINKFLAGS = ['-static-libgcc','-static-libstdc++'])
-    if env['cppthreads']:
-        env.Append(LINKFLAGS = ['-lpthread'])
 
 if env['Werror']:
     env.Append(CXXFLAGS = ['-Werror'])
@@ -222,6 +220,9 @@ if env['gles_compute']:
         print("Cannot link OpenGLES statically, which is required for bare metal / standalone builds")
         Exit(1)
 
+if env["os"] not in ["android", "bare_metal"] and (env['opencl'] or env['cppthreads']):
+    env.Append(LIBS = ['pthread'])
+
 if env['opencl'] or env['gles_compute']:
     if env['embed_kernels']:
         env.Append(CPPDEFINES = ['EMBEDDED_KERNELS'])
diff --git a/arm_compute/runtime/CL/CLScheduler.h b/arm_compute/runtime/CL/CLScheduler.h
index 2fe3c2cf42..0d81d73182 100644
--- a/arm_compute/runtime/CL/CLScheduler.h
+++ b/arm_compute/runtime/CL/CLScheduler.h
@@ -216,6 +216,11 @@ public:
         }
     }
 
+    bool is_initialised() const
+    {
+        return _is_initialised;
+    }
+
 private:
     /** Flag to ensure symbols initialisation is happening before Scheduler creation */
     static std::once_flag _initialize_symbols;
diff --git a/examples/SConscript b/examples/SConscript
index 7f7df3e9aa..230305b5ea 100644
--- a/examples/SConscript
+++ b/examples/SConscript
@@ -52,7 +52,7 @@ for file in Glob("./graph_*.cpp"):
     prog = None
     arm_compute_graph_libs = arm_compute_libs
 
-    if env['os'] == 'android':
+    if env['os'] in ['android', 'bare_metal'] or env['standalone']:
         prog = examples_env.Program(example, ["{}.cpp".format(example), utils, graph_utils], LIBS = arm_compute_graph_libs, LINKFLAGS=examples_env["LINKFLAGS"]+['-Wl,--whole-archive',graph_dependency,'-Wl,--no-whole-archive'])
         Depends(prog, graph_dependency)
     else:
@@ -89,14 +89,7 @@ if env['neon']:
 if env['gles_compute']:
     for file in Glob("./gc_*.cpp"):
         example = os.path.basename(os.path.splitext(str(file))[0])
-        if env['os'] != 'android':
-            prog = examples_env.Program(example, ["{}.cpp".format(example), utils], CPPDEFINES=['ARM_COMPUTE_GC'], LIBS = arm_compute_libs)
-            Depends(prog, arm_compute_dependency)
-        else:
-            if env['arch'] != 'armv7a':
-                prog = examples_env.Program(example, ["{}.cpp".format(example), utils], CPPDEFINES=['ARM_COMPUTE_GC'], LIBS = arm_compute_libs)
-            else:
-                prog = examples_env.Program(example, ["{}.cpp".format(example), utils], CPPDEFINES=['ARM_COMPUTE_GC'], LIBS = arm_compute_libs)
-            Depends(prog, [arm_compute_dependency])
+        prog = examples_env.Program(example, ["{}.cpp".format(example), utils], CPPDEFINES=['ARM_COMPUTE_GC'], LIBS = arm_compute_libs)
+        Depends(prog, arm_compute_dependency)
         alias = examples_env.Alias(example, prog)
         Default(alias)
diff --git a/tests/SConscript b/tests/SConscript
index 20095e56fb..7aa4d9391f 100644
--- a/tests/SConscript
+++ b/tests/SConscript
@@ -158,7 +158,7 @@ if test_env['validation_tests']:
                 cl_examples += [ test_env.Program(example, [ test_env.Object(source=file, target=example) ] + files_validate_examples, LIBS = test_env["LIBS"]) ]
             arm_compute_validate_examples += cl_examples
             if test_env['opencl'] and test_env['neon']:
-                if env['os'] == 'android':
+                if env['os'] in ['android', 'bare_metal'] or env['standalone']:
                     Import('arm_compute_graph_a')
                     graph_dependency = [ arm_compute_graph_a]
                 else:
@@ -168,7 +168,7 @@ if test_env['validation_tests']:
                 graph_utils = test_env.Object(source="../utils/GraphUtils.cpp", target="GraphUtils")
                 for file in Glob("validate_examples/graph_*.cpp"):
                     example = "validate_" + os.path.basename(os.path.splitext(str(file))[0])
-                    if env['os'] == 'android':
+                    if env['os'] in ['android', 'bare_metal'] or env['standalone']:
                         prog = test_env.Program(example, [ test_env.Object(source=file, target=example), graph_utils]+ files_validate_examples, LIBS = test_env["LIBS"], LINKFLAGS=test_env["LINKFLAGS"]+['-Wl,--whole-archive',graph_dependency,'-Wl,--no-whole-archive'])
                         Depends(prog, graph_dependency)
                         arm_compute_validate_examples += [ prog ]
@@ -201,7 +201,7 @@ if test_env['benchmark_examples']:
         arm_compute_benchmark_examples += cl_examples
 
     # Graph examples
-    if env['os'] == 'android':
+    if env['os'] in ['android', 'bare_metal'] or env['standalone']:
         Import('arm_compute_graph_a')
         graph_dependency = [arm_compute_graph_a]
     else:
@@ -210,7 +210,7 @@ if test_env['benchmark_examples']:
     graph_utils = test_env.Object(source="../utils/GraphUtils.cpp", target="GraphUtils")
     for file in Glob("../examples/graph_*.cpp"):
         example = "benchmark_" + os.path.basename(os.path.splitext(str(file))[0])
-        if env['os'] == 'android':
+        if env['os'] in ['android', 'bare_metal'] or env['standalone']:
             prog = test_env.Program(example, [ test_env.Object(source=file, target=example), graph_utils]+ files_benchmark_examples, LIBS = test_env["LIBS"], LINKFLAGS=test_env["LINKFLAGS"]+['-Wl,--whole-archive',graph_dependency,'-Wl,--no-whole-archive'])
             Depends(prog, [graph_dependency])
             arm_compute_benchmark_examples += [ prog ]
diff --git a/tests/benchmark_examples/RunExample.cpp b/tests/benchmark_examples/RunExample.cpp
index 656a06a099..f6a9742048 100644
--- a/tests/benchmark_examples/RunExample.cpp
+++ b/tests/benchmark_examples/RunExample.cpp
@@ -50,11 +50,17 @@ namespace arm_compute
 {
 namespace utils
 {
-static Example *g_example = nullptr;
+static std::unique_ptr<Example> g_example      = nullptr;
+static std::vector<char *>      g_example_argv = {};
 class ExampleTest : public arm_compute::test::framework::TestCase
 {
 public:
     ExampleTest() = default;
+    void do_setup() override
+    {
+        ARM_COMPUTE_ERROR_ON_NULLPTR(g_example.get());
+        g_example->do_setup(g_example_argv.size(), &g_example_argv[0]);
+    }
     void do_run() override
     {
         g_example->do_run();
@@ -62,10 +68,11 @@ public:
     void do_teardown() override
     {
         g_example->do_teardown();
+        g_example = nullptr;
     }
 };
 
-int run_example(int argc, char **argv, Example &example)
+int run_example(int argc, char **argv, std::unique_ptr<Example> example)
 {
     framework::CommandLineParser parser;
     framework::CommonOptions     options(parser);
@@ -82,42 +89,17 @@ int run_example(int argc, char **argv, Example &example)
     }
 
     std::vector<std::unique_ptr<framework::Printer>> printers = options.create_printers();
-    g_example                                                 = &example;
-    std::vector<char *> example_argv                          = {};
-    example_argv.clear();
-    example_argv.emplace_back(argv[0]);
+    g_example                                                 = std::move(example);
+    g_example_argv.clear();
+    g_example_argv.emplace_back(argv[0]);
     for(auto &arg : example_args->value())
     {
-        example_argv.emplace_back(const_cast<char *>(arg.c_str())); // NOLINT
+        g_example_argv.emplace_back(const_cast<char *>(arg.c_str())); // NOLINT
     }
 
     // Set number of threads in Scheduler
     Scheduler::get().set_num_threads(options.threads->value());
 
-    // We need to do the setup here because framework.init() will need CL / GLES to be initialised
-    try
-    {
-        example.do_setup(example_argv.size(), &example_argv[0]);
-    }
-#ifdef ARM_COMPUTE_CL
-    catch(cl::Error &err)
-    {
-        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
-        std::cerr << std::endl
-                  << "ERROR " << err.what() << "(" << err.err() << ")" << std::endl;
-        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
-        return 1;
-    }
-#endif /* ARM_COMPUTE_CL */
-    catch(std::runtime_error &err)
-    {
-        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
-        std::cerr << std::endl
-                  << "ERROR " << err.what() << " " << (errno ? strerror(errno) : "") << std::endl;
-        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
-        return 1;
-    }
-
     if(options.log_level->value() > framework::LogLevel::NONE)
     {
         for(auto &p : printers)
@@ -134,6 +116,10 @@ int run_example(int argc, char **argv, Example &example)
 #ifdef ARM_COMPUTE_CL
             if(opencl_is_available())
             {
+                if(!CLScheduler::get().is_initialised())
+                {
+                    CLScheduler::get().default_init();
+                }
                 p->print_entry("CL_DEVICE_VERSION", CLKernelLibrary::get().get_device_version());
             }
             else
diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp
index 3091b6677b..238b5ab2c0 100644
--- a/tests/framework/Framework.cpp
+++ b/tests/framework/Framework.cpp
@@ -288,6 +288,8 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 
         try
         {
+            profiler.test_start();
+
             test_case->do_setup();
 
             for(int i = 0; i < _num_iterations; ++i)
@@ -312,6 +314,8 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 
             test_case->do_teardown();
 
+            profiler.test_stop();
+
             // Change status to success if no error has happend
             if(result.status == TestResult::Status::NOT_RUN)
             {
diff --git a/tests/framework/Profiler.cpp b/tests/framework/Profiler.cpp
index 646c66556c..69ea527a80 100644
--- a/tests/framework/Profiler.cpp
+++ b/tests/framework/Profiler.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2018 ARM Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -37,6 +37,14 @@ void Profiler::add(std::unique_ptr<Instrument> instrument)
     _instruments.emplace_back(std::move(instrument));
 }
 
+void Profiler::test_start()
+{
+    for(auto &instrument : _instruments)
+    {
+        instrument->test_start();
+    }
+}
+
 void Profiler::start()
 {
     for(auto &instrument : _instruments)
@@ -51,7 +59,6 @@ void Profiler::stop()
     {
         instrument->stop();
     }
-
     for(const auto &instrument : _instruments)
     {
         for(const auto &measurement : instrument->measurements())
@@ -61,6 +68,22 @@ void Profiler::stop()
     }
 }
 
+void Profiler::test_stop()
+{
+    for(auto &instrument : _instruments)
+    {
+        instrument->test_stop();
+    }
+
+    for(const auto &instrument : _instruments)
+    {
+        for(const auto &measurement : instrument->test_measurements())
+        {
+            _measurements[instrument->id() + "/" + measurement.first].push_back(measurement.second);
+        }
+    }
+}
+
 const Profiler::MeasurementsMap &Profiler::measurements() const
 {
     return _measurements;
diff --git a/tests/framework/Profiler.h b/tests/framework/Profiler.h
index 62a3dee92e..34c5224528 100644
--- a/tests/framework/Profiler.h
+++ b/tests/framework/Profiler.h
@@ -57,12 +57,30 @@ public:
      */
     void add(std::unique_ptr<Instrument> instrument);
 
-    /** Start all added instruments to measure performance. */
+    /** Call test_start() on all the added instruments
+     *
+     * Called before the test set up starts
+     */
+    void test_start();
+
+    /** Call start() on all the added instruments
+     *
+     * Called just before the run of the test starts
+     */
     void start();
 
-    /** Stop all added instruments. */
+    /** Call stop() on all the added instruments
+     *
+     * Called just after the run of the test ends
+    */
     void stop();
 
+    /** Call test_stop() on all the added instruments
+     *
+     * Called after the test teardown ended
+     */
+    void test_stop();
+
     /** Return measurements for all instruments.
      *
      * @return measurements for all instruments.
diff --git a/tests/framework/instruments/Instrument.h b/tests/framework/instruments/Instrument.h
index 0df53f4210..ae4644b200 100644
--- a/tests/framework/instruments/Instrument.h
+++ b/tests/framework/instruments/Instrument.h
@@ -74,20 +74,57 @@ public:
     /** Identifier for the instrument */
     virtual std::string id() const = 0;
 
-    /** Start measuring. */
-    virtual void start() = 0;
+    /** Start of the test
+     *
+     * Called before the test set up starts
+     */
+    virtual void test_start()
+    {
+    }
+
+    /** Start measuring.
+     *
+     * Called just before the run of the test starts
+     */
+    virtual void start()
+    {
+    }
 
-    /** Stop measuring. */
-    virtual void stop() = 0;
+    /** Stop measuring.
+     *
+     * Called just after the run of the test ends
+    */
+    virtual void stop()
+    {
+    }
 
+    /** End of the test
+     *
+     * Called after the test teardown ended
+     */
+    virtual void test_stop()
+    {
+    }
     /** Map of measurements */
     using MeasurementsMap = std::map<std::string, Measurement>;
 
-    /** Return the latest measurement.
+    /** Return the latest measurements.
+     *
+     * @return the latest measurements.
+     */
+    virtual MeasurementsMap measurements() const
+    {
+        return MeasurementsMap();
+    }
+
+    /** Return the latest test measurements.
      *
-     * @return the latest measurement.
+     * @return the latest test measurements.
      */
-    virtual MeasurementsMap measurements() const = 0;
+    virtual MeasurementsMap test_measurements() const
+    {
+        return MeasurementsMap();
+    }
 
 protected:
     std::string _unit{};
diff --git a/tests/validate_examples/RunExample.cpp b/tests/validate_examples/RunExample.cpp
index a7b18ce555..8b1c39b844 100644
--- a/tests/validate_examples/RunExample.cpp
+++ b/tests/validate_examples/RunExample.cpp
@@ -57,12 +57,19 @@ std::unique_ptr<AssetsLibrary> library;
 } // namespace test
 namespace utils
 {
-static ValidateExample *g_example = nullptr;
-template <bool          validate>
+static std::unique_ptr<ValidateExample> g_example      = nullptr;
+static std::vector<char *>              g_example_argv = {};
+
+template <bool validate>
 class ExampleTest : public arm_compute::test::framework::TestCase
 {
 public:
     ExampleTest() = default;
+    void do_setup() override
+    {
+        ARM_COMPUTE_ERROR_ON_NULLPTR(g_example.get());
+        g_example->do_setup(g_example_argv.size(), &g_example_argv[0]);
+    }
     void do_run() override
     {
         g_example->do_run();
@@ -74,10 +81,11 @@ public:
             g_example->do_validate();
         }
         g_example->do_teardown();
+        g_example = nullptr;
     }
 };
 
-int run_example(int argc, char **argv, ValidateExample &example)
+int run_example(int argc, char **argv, std::unique_ptr<ValidateExample> example)
 {
     framework::CommandLineParser parser;
     framework::CommonOptions     options(parser);
@@ -99,43 +107,18 @@ int run_example(int argc, char **argv, ValidateExample &example)
     }
 
     std::vector<std::unique_ptr<framework::Printer>> printers = options.create_printers();
-    g_example                                                 = &example;
-    std::vector<char *> example_argv                          = {};
-    example_argv.clear();
-    example_argv.emplace_back(argv[0]);
+    g_example                                                 = std::move(example);
+    g_example_argv.clear();
+    g_example_argv.emplace_back(argv[0]);
     for(auto &arg : example_args->value())
     {
-        example_argv.emplace_back(const_cast<char *>(arg.c_str())); // NOLINT
+        g_example_argv.emplace_back(const_cast<char *>(arg.c_str())); // NOLINT
     }
 
     // Set number of threads in Scheduler
     Scheduler::get().set_num_threads(options.threads->value());
     library = support::cpp14::make_unique<AssetsLibrary>("." /* Only using random values */, seed->value());
 
-    // We need to do the setup here because framework.init() will need CL / GLES to be initialised
-    try
-    {
-        example.do_setup(example_argv.size(), &example_argv[0]);
-    }
-#ifdef ARM_COMPUTE_CL
-    catch(cl::Error &err)
-    {
-        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
-        std::cerr << std::endl
-                  << "ERROR " << err.what() << "(" << err.err() << ")" << std::endl;
-        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
-        return 1;
-    }
-#endif /* ARM_COMPUTE_CL */
-    catch(std::runtime_error &err)
-    {
-        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
-        std::cerr << std::endl
-                  << "ERROR " << err.what() << " " << (errno ? strerror(errno) : "") << std::endl;
-        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
-        return 1;
-    }
-
     if(options.log_level->value() > framework::LogLevel::NONE)
     {
         for(auto &p : printers)
@@ -153,6 +136,10 @@ int run_example(int argc, char **argv, ValidateExample &example)
 #ifdef ARM_COMPUTE_CL
             if(opencl_is_available())
             {
+                if(!CLScheduler::get().is_initialised())
+                {
+                    CLScheduler::get().default_init();
+                }
                 p->print_entry("CL_DEVICE_VERSION", CLKernelLibrary::get().get_device_version());
             }
             else
@@ -162,7 +149,7 @@ int run_example(int argc, char **argv, ValidateExample &example)
 #endif /* ARM_COMPUTE_CL */
             p->print_entry("Iterations", support::cpp11::to_string(options.iterations->value()));
             p->print_entry("Threads", support::cpp11::to_string(options.threads->value()));
-            example.print_parameters(*p);
+            g_example->print_parameters(*p);
         }
     }
 
diff --git a/tests/validate_examples/ValidateExample.h b/tests/validate_examples/ValidateExample.h
index cedbe3d0d9..4dd552ab10 100644
--- a/tests/validate_examples/ValidateExample.h
+++ b/tests/validate_examples/ValidateExample.h
@@ -40,14 +40,24 @@ namespace utils
  *
  * All examples with a validation stage have to inherit from this class.
  */
-class ValidateExample : public Example
+class ValidateExample
 {
 public:
+    /** Setup the example.
+     *
+     * @param[in] argc Argument count.
+     * @param[in] argv Argument values.
+     */
+    virtual void do_setup(int argc, char **argv) {};
+    /** Run the example. */
+    virtual void do_run() {};
     /** Run reference implementation and validate against the target output
      */
     virtual void do_validate()
     {
     }
+    /** Teardown the example. */
+    virtual void do_teardown() {};
     /** Print the example parameters
      *
      * @param[in,out] printer Printer to use to print the parameters
@@ -65,7 +75,7 @@ public:
  * @param[in] argv    Command line arguments
  * @param[in] example Example to run
  */
-int run_example(int argc, char **argv, ValidateExample &example);
+int run_example(int argc, char **argv, std::unique_ptr<ValidateExample> example);
 
 } // namespace utils
 } // namespace arm_compute
diff --git a/utils/Utils.cpp b/utils/Utils.cpp
index 8a2d11814e..a5c6a95a2a 100644
--- a/utils/Utils.cpp
+++ b/utils/Utils.cpp
@@ -67,16 +67,16 @@ void discard_comments_and_spaces(std::ifstream &fs)
 } // namespace
 
 #ifndef BENCHMARK_EXAMPLES
-int run_example(int argc, char **argv, Example &example)
+int run_example(int argc, char **argv, std::unique_ptr<Example> example)
 {
     std::cout << "\n"
               << argv[0] << "\n\n";
 
     try
     {
-        example.do_setup(argc, argv);
-        example.do_run();
-        example.do_teardown();
+        example->do_setup(argc, argv);
+        example->do_run();
+        example->do_teardown();
 
         std::cout << "\nTest passed\n";
         return 0;
diff --git a/utils/Utils.h b/utils/Utils.h
index 73b6d4f832..6241562a28 100644
--- a/utils/Utils.h
+++ b/utils/Utils.h
@@ -83,13 +83,12 @@ public:
  * @param[in] argv    Command line arguments
  * @param[in] example Example to run
  */
-int run_example(int argc, char **argv, Example &example);
+int run_example(int argc, char **argv, std::unique_ptr<Example> example);
 
 template <typename T>
 int run_example(int argc, char **argv)
 {
-    T example;
-    return run_example(argc, argv, example);
+    return run_example(argc, argv, support::cpp14::make_unique<T>());
 }
 
 /** Draw a RGB rectangular window for the detected object