From 9fb0cac961f70d1937c5fa3eafeaee1385c89768 Mon Sep 17 00:00:00 2001 From: Anthony Barbier Date: Fri, 20 Apr 2018 15:46:21 +0100 Subject: COMPMID-1081: Introduced test-wide instruments Change-Id: I5831241f3fc503717cc51136453c2bf96d4b420b Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/128484 Tested-by: Jenkins Reviewed-by: Georgios Pinitas --- tests/SConscript | 8 ++--- tests/benchmark_examples/RunExample.cpp | 48 ++++++++++------------------ tests/framework/Framework.cpp | 4 +++ tests/framework/Profiler.cpp | 27 ++++++++++++++-- tests/framework/Profiler.h | 22 +++++++++++-- tests/framework/instruments/Instrument.h | 51 +++++++++++++++++++++++++---- tests/validate_examples/RunExample.cpp | 53 ++++++++++++------------------- tests/validate_examples/ValidateExample.h | 14 ++++++-- 8 files changed, 146 insertions(+), 81 deletions(-) (limited to 'tests') diff --git a/tests/SConscript b/tests/SConscript index 20095e56fb..7aa4d9391f 100644 --- a/tests/SConscript +++ b/tests/SConscript @@ -158,7 +158,7 @@ if test_env['validation_tests']: cl_examples += [ test_env.Program(example, [ test_env.Object(source=file, target=example) ] + files_validate_examples, LIBS = test_env["LIBS"]) ] arm_compute_validate_examples += cl_examples if test_env['opencl'] and test_env['neon']: - if env['os'] == 'android': + if env['os'] in ['android', 'bare_metal'] or env['standalone']: Import('arm_compute_graph_a') graph_dependency = [ arm_compute_graph_a] else: @@ -168,7 +168,7 @@ if test_env['validation_tests']: graph_utils = test_env.Object(source="../utils/GraphUtils.cpp", target="GraphUtils") for file in Glob("validate_examples/graph_*.cpp"): example = "validate_" + os.path.basename(os.path.splitext(str(file))[0]) - if env['os'] == 'android': + if env['os'] in ['android', 'bare_metal'] or env['standalone']: prog = test_env.Program(example, [ test_env.Object(source=file, target=example), graph_utils]+ files_validate_examples, LIBS = test_env["LIBS"], LINKFLAGS=test_env["LINKFLAGS"]+['-Wl,--whole-archive',graph_dependency,'-Wl,--no-whole-archive']) Depends(prog, graph_dependency) arm_compute_validate_examples += [ prog ] @@ -201,7 +201,7 @@ if test_env['benchmark_examples']: arm_compute_benchmark_examples += cl_examples # Graph examples - if env['os'] == 'android': + if env['os'] in ['android', 'bare_metal'] or env['standalone']: Import('arm_compute_graph_a') graph_dependency = [arm_compute_graph_a] else: @@ -210,7 +210,7 @@ if test_env['benchmark_examples']: graph_utils = test_env.Object(source="../utils/GraphUtils.cpp", target="GraphUtils") for file in Glob("../examples/graph_*.cpp"): example = "benchmark_" + os.path.basename(os.path.splitext(str(file))[0]) - if env['os'] == 'android': + if env['os'] in ['android', 'bare_metal'] or env['standalone']: prog = test_env.Program(example, [ test_env.Object(source=file, target=example), graph_utils]+ files_benchmark_examples, LIBS = test_env["LIBS"], LINKFLAGS=test_env["LINKFLAGS"]+['-Wl,--whole-archive',graph_dependency,'-Wl,--no-whole-archive']) Depends(prog, [graph_dependency]) arm_compute_benchmark_examples += [ prog ] diff --git a/tests/benchmark_examples/RunExample.cpp b/tests/benchmark_examples/RunExample.cpp index 656a06a099..f6a9742048 100644 --- a/tests/benchmark_examples/RunExample.cpp +++ b/tests/benchmark_examples/RunExample.cpp @@ -50,11 +50,17 @@ namespace arm_compute { namespace utils { -static Example *g_example = nullptr; +static std::unique_ptr g_example = nullptr; +static std::vector g_example_argv = {}; class ExampleTest : public arm_compute::test::framework::TestCase { public: ExampleTest() = default; + void do_setup() override + { + ARM_COMPUTE_ERROR_ON_NULLPTR(g_example.get()); + g_example->do_setup(g_example_argv.size(), &g_example_argv[0]); + } void do_run() override { g_example->do_run(); @@ -62,10 +68,11 @@ public: void do_teardown() override { g_example->do_teardown(); + g_example = nullptr; } }; -int run_example(int argc, char **argv, Example &example) +int run_example(int argc, char **argv, std::unique_ptr example) { framework::CommandLineParser parser; framework::CommonOptions options(parser); @@ -82,42 +89,17 @@ int run_example(int argc, char **argv, Example &example) } std::vector> printers = options.create_printers(); - g_example = &example; - std::vector example_argv = {}; - example_argv.clear(); - example_argv.emplace_back(argv[0]); + g_example = std::move(example); + g_example_argv.clear(); + g_example_argv.emplace_back(argv[0]); for(auto &arg : example_args->value()) { - example_argv.emplace_back(const_cast(arg.c_str())); // NOLINT + g_example_argv.emplace_back(const_cast(arg.c_str())); // NOLINT } // Set number of threads in Scheduler Scheduler::get().set_num_threads(options.threads->value()); - // We need to do the setup here because framework.init() will need CL / GLES to be initialised - try - { - example.do_setup(example_argv.size(), &example_argv[0]); - } -#ifdef ARM_COMPUTE_CL - catch(cl::Error &err) - { - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - std::cerr << std::endl - << "ERROR " << err.what() << "(" << err.err() << ")" << std::endl; - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - return 1; - } -#endif /* ARM_COMPUTE_CL */ - catch(std::runtime_error &err) - { - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - std::cerr << std::endl - << "ERROR " << err.what() << " " << (errno ? strerror(errno) : "") << std::endl; - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - return 1; - } - if(options.log_level->value() > framework::LogLevel::NONE) { for(auto &p : printers) @@ -134,6 +116,10 @@ int run_example(int argc, char **argv, Example &example) #ifdef ARM_COMPUTE_CL if(opencl_is_available()) { + if(!CLScheduler::get().is_initialised()) + { + CLScheduler::get().default_init(); + } p->print_entry("CL_DEVICE_VERSION", CLKernelLibrary::get().get_device_version()); } else diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp index 3091b6677b..238b5ab2c0 100644 --- a/tests/framework/Framework.cpp +++ b/tests/framework/Framework.cpp @@ -288,6 +288,8 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) try { + profiler.test_start(); + test_case->do_setup(); for(int i = 0; i < _num_iterations; ++i) @@ -312,6 +314,8 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) test_case->do_teardown(); + profiler.test_stop(); + // Change status to success if no error has happend if(result.status == TestResult::Status::NOT_RUN) { diff --git a/tests/framework/Profiler.cpp b/tests/framework/Profiler.cpp index 646c66556c..69ea527a80 100644 --- a/tests/framework/Profiler.cpp +++ b/tests/framework/Profiler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -37,6 +37,14 @@ void Profiler::add(std::unique_ptr instrument) _instruments.emplace_back(std::move(instrument)); } +void Profiler::test_start() +{ + for(auto &instrument : _instruments) + { + instrument->test_start(); + } +} + void Profiler::start() { for(auto &instrument : _instruments) @@ -51,7 +59,6 @@ void Profiler::stop() { instrument->stop(); } - for(const auto &instrument : _instruments) { for(const auto &measurement : instrument->measurements()) @@ -61,6 +68,22 @@ void Profiler::stop() } } +void Profiler::test_stop() +{ + for(auto &instrument : _instruments) + { + instrument->test_stop(); + } + + for(const auto &instrument : _instruments) + { + for(const auto &measurement : instrument->test_measurements()) + { + _measurements[instrument->id() + "/" + measurement.first].push_back(measurement.second); + } + } +} + const Profiler::MeasurementsMap &Profiler::measurements() const { return _measurements; diff --git a/tests/framework/Profiler.h b/tests/framework/Profiler.h index 62a3dee92e..34c5224528 100644 --- a/tests/framework/Profiler.h +++ b/tests/framework/Profiler.h @@ -57,12 +57,30 @@ public: */ void add(std::unique_ptr instrument); - /** Start all added instruments to measure performance. */ + /** Call test_start() on all the added instruments + * + * Called before the test set up starts + */ + void test_start(); + + /** Call start() on all the added instruments + * + * Called just before the run of the test starts + */ void start(); - /** Stop all added instruments. */ + /** Call stop() on all the added instruments + * + * Called just after the run of the test ends + */ void stop(); + /** Call test_stop() on all the added instruments + * + * Called after the test teardown ended + */ + void test_stop(); + /** Return measurements for all instruments. * * @return measurements for all instruments. diff --git a/tests/framework/instruments/Instrument.h b/tests/framework/instruments/Instrument.h index 0df53f4210..ae4644b200 100644 --- a/tests/framework/instruments/Instrument.h +++ b/tests/framework/instruments/Instrument.h @@ -74,20 +74,57 @@ public: /** Identifier for the instrument */ virtual std::string id() const = 0; - /** Start measuring. */ - virtual void start() = 0; + /** Start of the test + * + * Called before the test set up starts + */ + virtual void test_start() + { + } + + /** Start measuring. + * + * Called just before the run of the test starts + */ + virtual void start() + { + } - /** Stop measuring. */ - virtual void stop() = 0; + /** Stop measuring. + * + * Called just after the run of the test ends + */ + virtual void stop() + { + } + /** End of the test + * + * Called after the test teardown ended + */ + virtual void test_stop() + { + } /** Map of measurements */ using MeasurementsMap = std::map; - /** Return the latest measurement. + /** Return the latest measurements. + * + * @return the latest measurements. + */ + virtual MeasurementsMap measurements() const + { + return MeasurementsMap(); + } + + /** Return the latest test measurements. * - * @return the latest measurement. + * @return the latest test measurements. */ - virtual MeasurementsMap measurements() const = 0; + virtual MeasurementsMap test_measurements() const + { + return MeasurementsMap(); + } protected: std::string _unit{}; diff --git a/tests/validate_examples/RunExample.cpp b/tests/validate_examples/RunExample.cpp index a7b18ce555..8b1c39b844 100644 --- a/tests/validate_examples/RunExample.cpp +++ b/tests/validate_examples/RunExample.cpp @@ -57,12 +57,19 @@ std::unique_ptr library; } // namespace test namespace utils { -static ValidateExample *g_example = nullptr; -template +static std::unique_ptr g_example = nullptr; +static std::vector g_example_argv = {}; + +template class ExampleTest : public arm_compute::test::framework::TestCase { public: ExampleTest() = default; + void do_setup() override + { + ARM_COMPUTE_ERROR_ON_NULLPTR(g_example.get()); + g_example->do_setup(g_example_argv.size(), &g_example_argv[0]); + } void do_run() override { g_example->do_run(); @@ -74,10 +81,11 @@ public: g_example->do_validate(); } g_example->do_teardown(); + g_example = nullptr; } }; -int run_example(int argc, char **argv, ValidateExample &example) +int run_example(int argc, char **argv, std::unique_ptr example) { framework::CommandLineParser parser; framework::CommonOptions options(parser); @@ -99,43 +107,18 @@ int run_example(int argc, char **argv, ValidateExample &example) } std::vector> printers = options.create_printers(); - g_example = &example; - std::vector example_argv = {}; - example_argv.clear(); - example_argv.emplace_back(argv[0]); + g_example = std::move(example); + g_example_argv.clear(); + g_example_argv.emplace_back(argv[0]); for(auto &arg : example_args->value()) { - example_argv.emplace_back(const_cast(arg.c_str())); // NOLINT + g_example_argv.emplace_back(const_cast(arg.c_str())); // NOLINT } // Set number of threads in Scheduler Scheduler::get().set_num_threads(options.threads->value()); library = support::cpp14::make_unique("." /* Only using random values */, seed->value()); - // We need to do the setup here because framework.init() will need CL / GLES to be initialised - try - { - example.do_setup(example_argv.size(), &example_argv[0]); - } -#ifdef ARM_COMPUTE_CL - catch(cl::Error &err) - { - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - std::cerr << std::endl - << "ERROR " << err.what() << "(" << err.err() << ")" << std::endl; - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - return 1; - } -#endif /* ARM_COMPUTE_CL */ - catch(std::runtime_error &err) - { - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - std::cerr << std::endl - << "ERROR " << err.what() << " " << (errno ? strerror(errno) : "") << std::endl; - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - return 1; - } - if(options.log_level->value() > framework::LogLevel::NONE) { for(auto &p : printers) @@ -153,6 +136,10 @@ int run_example(int argc, char **argv, ValidateExample &example) #ifdef ARM_COMPUTE_CL if(opencl_is_available()) { + if(!CLScheduler::get().is_initialised()) + { + CLScheduler::get().default_init(); + } p->print_entry("CL_DEVICE_VERSION", CLKernelLibrary::get().get_device_version()); } else @@ -162,7 +149,7 @@ int run_example(int argc, char **argv, ValidateExample &example) #endif /* ARM_COMPUTE_CL */ p->print_entry("Iterations", support::cpp11::to_string(options.iterations->value())); p->print_entry("Threads", support::cpp11::to_string(options.threads->value())); - example.print_parameters(*p); + g_example->print_parameters(*p); } } diff --git a/tests/validate_examples/ValidateExample.h b/tests/validate_examples/ValidateExample.h index cedbe3d0d9..4dd552ab10 100644 --- a/tests/validate_examples/ValidateExample.h +++ b/tests/validate_examples/ValidateExample.h @@ -40,14 +40,24 @@ namespace utils * * All examples with a validation stage have to inherit from this class. */ -class ValidateExample : public Example +class ValidateExample { public: + /** Setup the example. + * + * @param[in] argc Argument count. + * @param[in] argv Argument values. + */ + virtual void do_setup(int argc, char **argv) {}; + /** Run the example. */ + virtual void do_run() {}; /** Run reference implementation and validate against the target output */ virtual void do_validate() { } + /** Teardown the example. */ + virtual void do_teardown() {}; /** Print the example parameters * * @param[in,out] printer Printer to use to print the parameters @@ -65,7 +75,7 @@ public: * @param[in] argv Command line arguments * @param[in] example Example to run */ -int run_example(int argc, char **argv, ValidateExample &example); +int run_example(int argc, char **argv, std::unique_ptr example); } // namespace utils } // namespace arm_compute -- cgit v1.2.1