From 9fb0cac961f70d1937c5fa3eafeaee1385c89768 Mon Sep 17 00:00:00 2001 From: Anthony Barbier Date: Fri, 20 Apr 2018 15:46:21 +0100 Subject: COMPMID-1081: Introduced test-wide instruments Change-Id: I5831241f3fc503717cc51136453c2bf96d4b420b Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/128484 Tested-by: Jenkins Reviewed-by: Georgios Pinitas --- SConscript | 3 -- SConstruct | 5 +-- arm_compute/runtime/CL/CLScheduler.h | 5 +++ examples/SConscript | 13 ++------ tests/SConscript | 8 ++--- tests/benchmark_examples/RunExample.cpp | 48 ++++++++++------------------ tests/framework/Framework.cpp | 4 +++ tests/framework/Profiler.cpp | 27 ++++++++++++++-- tests/framework/Profiler.h | 22 +++++++++++-- tests/framework/instruments/Instrument.h | 51 +++++++++++++++++++++++++---- tests/validate_examples/RunExample.cpp | 53 ++++++++++++------------------- tests/validate_examples/ValidateExample.h | 14 ++++++-- utils/Utils.cpp | 8 ++--- utils/Utils.h | 5 ++- 14 files changed, 163 insertions(+), 103 deletions(-) diff --git a/SConscript b/SConscript index 936fde0f1f..983570b7dc 100644 --- a/SConscript +++ b/SConscript @@ -153,9 +153,6 @@ if env["build"] == "embed_only": arm_compute_env.Append(LINKFLAGS=['-Wl,--no-undefined']) arm_compute_env.Append(CPPPATH =[Dir("./src/core/").path] ) -if env["os"] not in ["android", "bare_metal"]: - arm_compute_env.Append(LIBS = ['pthread']) - arm_compute_env.Append(LIBS = ['dl']) core_files = Glob('src/core/*.cpp') diff --git a/SConstruct b/SConstruct index 626a302df6..23b9b2f7b4 100644 --- a/SConstruct +++ b/SConstruct @@ -196,8 +196,6 @@ if not GetOption("help"): if env['standalone']: env.Append(CXXFLAGS = ['-fPIC']) env.Append(LINKFLAGS = ['-static-libgcc','-static-libstdc++']) - if env['cppthreads']: - env.Append(LINKFLAGS = ['-lpthread']) if env['Werror']: env.Append(CXXFLAGS = ['-Werror']) @@ -222,6 +220,9 @@ if env['gles_compute']: print("Cannot link OpenGLES statically, which is required for bare metal / standalone builds") Exit(1) +if env["os"] not in ["android", "bare_metal"] and (env['opencl'] or env['cppthreads']): + env.Append(LIBS = ['pthread']) + if env['opencl'] or env['gles_compute']: if env['embed_kernels']: env.Append(CPPDEFINES = ['EMBEDDED_KERNELS']) diff --git a/arm_compute/runtime/CL/CLScheduler.h b/arm_compute/runtime/CL/CLScheduler.h index 2fe3c2cf42..0d81d73182 100644 --- a/arm_compute/runtime/CL/CLScheduler.h +++ b/arm_compute/runtime/CL/CLScheduler.h @@ -216,6 +216,11 @@ public: } } + bool is_initialised() const + { + return _is_initialised; + } + private: /** Flag to ensure symbols initialisation is happening before Scheduler creation */ static std::once_flag _initialize_symbols; diff --git a/examples/SConscript b/examples/SConscript index 7f7df3e9aa..230305b5ea 100644 --- a/examples/SConscript +++ b/examples/SConscript @@ -52,7 +52,7 @@ for file in Glob("./graph_*.cpp"): prog = None arm_compute_graph_libs = arm_compute_libs - if env['os'] == 'android': + if env['os'] in ['android', 'bare_metal'] or env['standalone']: prog = examples_env.Program(example, ["{}.cpp".format(example), utils, graph_utils], LIBS = arm_compute_graph_libs, LINKFLAGS=examples_env["LINKFLAGS"]+['-Wl,--whole-archive',graph_dependency,'-Wl,--no-whole-archive']) Depends(prog, graph_dependency) else: @@ -89,14 +89,7 @@ if env['neon']: if env['gles_compute']: for file in Glob("./gc_*.cpp"): example = os.path.basename(os.path.splitext(str(file))[0]) - if env['os'] != 'android': - prog = examples_env.Program(example, ["{}.cpp".format(example), utils], CPPDEFINES=['ARM_COMPUTE_GC'], LIBS = arm_compute_libs) - Depends(prog, arm_compute_dependency) - else: - if env['arch'] != 'armv7a': - prog = examples_env.Program(example, ["{}.cpp".format(example), utils], CPPDEFINES=['ARM_COMPUTE_GC'], LIBS = arm_compute_libs) - else: - prog = examples_env.Program(example, ["{}.cpp".format(example), utils], CPPDEFINES=['ARM_COMPUTE_GC'], LIBS = arm_compute_libs) - Depends(prog, [arm_compute_dependency]) + prog = examples_env.Program(example, ["{}.cpp".format(example), utils], CPPDEFINES=['ARM_COMPUTE_GC'], LIBS = arm_compute_libs) + Depends(prog, arm_compute_dependency) alias = examples_env.Alias(example, prog) Default(alias) diff --git a/tests/SConscript b/tests/SConscript index 20095e56fb..7aa4d9391f 100644 --- a/tests/SConscript +++ b/tests/SConscript @@ -158,7 +158,7 @@ if test_env['validation_tests']: cl_examples += [ test_env.Program(example, [ test_env.Object(source=file, target=example) ] + files_validate_examples, LIBS = test_env["LIBS"]) ] arm_compute_validate_examples += cl_examples if test_env['opencl'] and test_env['neon']: - if env['os'] == 'android': + if env['os'] in ['android', 'bare_metal'] or env['standalone']: Import('arm_compute_graph_a') graph_dependency = [ arm_compute_graph_a] else: @@ -168,7 +168,7 @@ if test_env['validation_tests']: graph_utils = test_env.Object(source="../utils/GraphUtils.cpp", target="GraphUtils") for file in Glob("validate_examples/graph_*.cpp"): example = "validate_" + os.path.basename(os.path.splitext(str(file))[0]) - if env['os'] == 'android': + if env['os'] in ['android', 'bare_metal'] or env['standalone']: prog = test_env.Program(example, [ test_env.Object(source=file, target=example), graph_utils]+ files_validate_examples, LIBS = test_env["LIBS"], LINKFLAGS=test_env["LINKFLAGS"]+['-Wl,--whole-archive',graph_dependency,'-Wl,--no-whole-archive']) Depends(prog, graph_dependency) arm_compute_validate_examples += [ prog ] @@ -201,7 +201,7 @@ if test_env['benchmark_examples']: arm_compute_benchmark_examples += cl_examples # Graph examples - if env['os'] == 'android': + if env['os'] in ['android', 'bare_metal'] or env['standalone']: Import('arm_compute_graph_a') graph_dependency = [arm_compute_graph_a] else: @@ -210,7 +210,7 @@ if test_env['benchmark_examples']: graph_utils = test_env.Object(source="../utils/GraphUtils.cpp", target="GraphUtils") for file in Glob("../examples/graph_*.cpp"): example = "benchmark_" + os.path.basename(os.path.splitext(str(file))[0]) - if env['os'] == 'android': + if env['os'] in ['android', 'bare_metal'] or env['standalone']: prog = test_env.Program(example, [ test_env.Object(source=file, target=example), graph_utils]+ files_benchmark_examples, LIBS = test_env["LIBS"], LINKFLAGS=test_env["LINKFLAGS"]+['-Wl,--whole-archive',graph_dependency,'-Wl,--no-whole-archive']) Depends(prog, [graph_dependency]) arm_compute_benchmark_examples += [ prog ] diff --git a/tests/benchmark_examples/RunExample.cpp b/tests/benchmark_examples/RunExample.cpp index 656a06a099..f6a9742048 100644 --- a/tests/benchmark_examples/RunExample.cpp +++ b/tests/benchmark_examples/RunExample.cpp @@ -50,11 +50,17 @@ namespace arm_compute { namespace utils { -static Example *g_example = nullptr; +static std::unique_ptr g_example = nullptr; +static std::vector g_example_argv = {}; class ExampleTest : public arm_compute::test::framework::TestCase { public: ExampleTest() = default; + void do_setup() override + { + ARM_COMPUTE_ERROR_ON_NULLPTR(g_example.get()); + g_example->do_setup(g_example_argv.size(), &g_example_argv[0]); + } void do_run() override { g_example->do_run(); @@ -62,10 +68,11 @@ public: void do_teardown() override { g_example->do_teardown(); + g_example = nullptr; } }; -int run_example(int argc, char **argv, Example &example) +int run_example(int argc, char **argv, std::unique_ptr example) { framework::CommandLineParser parser; framework::CommonOptions options(parser); @@ -82,42 +89,17 @@ int run_example(int argc, char **argv, Example &example) } std::vector> printers = options.create_printers(); - g_example = &example; - std::vector example_argv = {}; - example_argv.clear(); - example_argv.emplace_back(argv[0]); + g_example = std::move(example); + g_example_argv.clear(); + g_example_argv.emplace_back(argv[0]); for(auto &arg : example_args->value()) { - example_argv.emplace_back(const_cast(arg.c_str())); // NOLINT + g_example_argv.emplace_back(const_cast(arg.c_str())); // NOLINT } // Set number of threads in Scheduler Scheduler::get().set_num_threads(options.threads->value()); - // We need to do the setup here because framework.init() will need CL / GLES to be initialised - try - { - example.do_setup(example_argv.size(), &example_argv[0]); - } -#ifdef ARM_COMPUTE_CL - catch(cl::Error &err) - { - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - std::cerr << std::endl - << "ERROR " << err.what() << "(" << err.err() << ")" << std::endl; - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - return 1; - } -#endif /* ARM_COMPUTE_CL */ - catch(std::runtime_error &err) - { - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - std::cerr << std::endl - << "ERROR " << err.what() << " " << (errno ? strerror(errno) : "") << std::endl; - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - return 1; - } - if(options.log_level->value() > framework::LogLevel::NONE) { for(auto &p : printers) @@ -134,6 +116,10 @@ int run_example(int argc, char **argv, Example &example) #ifdef ARM_COMPUTE_CL if(opencl_is_available()) { + if(!CLScheduler::get().is_initialised()) + { + CLScheduler::get().default_init(); + } p->print_entry("CL_DEVICE_VERSION", CLKernelLibrary::get().get_device_version()); } else diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp index 3091b6677b..238b5ab2c0 100644 --- a/tests/framework/Framework.cpp +++ b/tests/framework/Framework.cpp @@ -288,6 +288,8 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) try { + profiler.test_start(); + test_case->do_setup(); for(int i = 0; i < _num_iterations; ++i) @@ -312,6 +314,8 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) test_case->do_teardown(); + profiler.test_stop(); + // Change status to success if no error has happend if(result.status == TestResult::Status::NOT_RUN) { diff --git a/tests/framework/Profiler.cpp b/tests/framework/Profiler.cpp index 646c66556c..69ea527a80 100644 --- a/tests/framework/Profiler.cpp +++ b/tests/framework/Profiler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -37,6 +37,14 @@ void Profiler::add(std::unique_ptr instrument) _instruments.emplace_back(std::move(instrument)); } +void Profiler::test_start() +{ + for(auto &instrument : _instruments) + { + instrument->test_start(); + } +} + void Profiler::start() { for(auto &instrument : _instruments) @@ -51,7 +59,6 @@ void Profiler::stop() { instrument->stop(); } - for(const auto &instrument : _instruments) { for(const auto &measurement : instrument->measurements()) @@ -61,6 +68,22 @@ void Profiler::stop() } } +void Profiler::test_stop() +{ + for(auto &instrument : _instruments) + { + instrument->test_stop(); + } + + for(const auto &instrument : _instruments) + { + for(const auto &measurement : instrument->test_measurements()) + { + _measurements[instrument->id() + "/" + measurement.first].push_back(measurement.second); + } + } +} + const Profiler::MeasurementsMap &Profiler::measurements() const { return _measurements; diff --git a/tests/framework/Profiler.h b/tests/framework/Profiler.h index 62a3dee92e..34c5224528 100644 --- a/tests/framework/Profiler.h +++ b/tests/framework/Profiler.h @@ -57,12 +57,30 @@ public: */ void add(std::unique_ptr instrument); - /** Start all added instruments to measure performance. */ + /** Call test_start() on all the added instruments + * + * Called before the test set up starts + */ + void test_start(); + + /** Call start() on all the added instruments + * + * Called just before the run of the test starts + */ void start(); - /** Stop all added instruments. */ + /** Call stop() on all the added instruments + * + * Called just after the run of the test ends + */ void stop(); + /** Call test_stop() on all the added instruments + * + * Called after the test teardown ended + */ + void test_stop(); + /** Return measurements for all instruments. * * @return measurements for all instruments. diff --git a/tests/framework/instruments/Instrument.h b/tests/framework/instruments/Instrument.h index 0df53f4210..ae4644b200 100644 --- a/tests/framework/instruments/Instrument.h +++ b/tests/framework/instruments/Instrument.h @@ -74,20 +74,57 @@ public: /** Identifier for the instrument */ virtual std::string id() const = 0; - /** Start measuring. */ - virtual void start() = 0; + /** Start of the test + * + * Called before the test set up starts + */ + virtual void test_start() + { + } + + /** Start measuring. + * + * Called just before the run of the test starts + */ + virtual void start() + { + } - /** Stop measuring. */ - virtual void stop() = 0; + /** Stop measuring. + * + * Called just after the run of the test ends + */ + virtual void stop() + { + } + /** End of the test + * + * Called after the test teardown ended + */ + virtual void test_stop() + { + } /** Map of measurements */ using MeasurementsMap = std::map; - /** Return the latest measurement. + /** Return the latest measurements. + * + * @return the latest measurements. + */ + virtual MeasurementsMap measurements() const + { + return MeasurementsMap(); + } + + /** Return the latest test measurements. * - * @return the latest measurement. + * @return the latest test measurements. */ - virtual MeasurementsMap measurements() const = 0; + virtual MeasurementsMap test_measurements() const + { + return MeasurementsMap(); + } protected: std::string _unit{}; diff --git a/tests/validate_examples/RunExample.cpp b/tests/validate_examples/RunExample.cpp index a7b18ce555..8b1c39b844 100644 --- a/tests/validate_examples/RunExample.cpp +++ b/tests/validate_examples/RunExample.cpp @@ -57,12 +57,19 @@ std::unique_ptr library; } // namespace test namespace utils { -static ValidateExample *g_example = nullptr; -template +static std::unique_ptr g_example = nullptr; +static std::vector g_example_argv = {}; + +template class ExampleTest : public arm_compute::test::framework::TestCase { public: ExampleTest() = default; + void do_setup() override + { + ARM_COMPUTE_ERROR_ON_NULLPTR(g_example.get()); + g_example->do_setup(g_example_argv.size(), &g_example_argv[0]); + } void do_run() override { g_example->do_run(); @@ -74,10 +81,11 @@ public: g_example->do_validate(); } g_example->do_teardown(); + g_example = nullptr; } }; -int run_example(int argc, char **argv, ValidateExample &example) +int run_example(int argc, char **argv, std::unique_ptr example) { framework::CommandLineParser parser; framework::CommonOptions options(parser); @@ -99,43 +107,18 @@ int run_example(int argc, char **argv, ValidateExample &example) } std::vector> printers = options.create_printers(); - g_example = &example; - std::vector example_argv = {}; - example_argv.clear(); - example_argv.emplace_back(argv[0]); + g_example = std::move(example); + g_example_argv.clear(); + g_example_argv.emplace_back(argv[0]); for(auto &arg : example_args->value()) { - example_argv.emplace_back(const_cast(arg.c_str())); // NOLINT + g_example_argv.emplace_back(const_cast(arg.c_str())); // NOLINT } // Set number of threads in Scheduler Scheduler::get().set_num_threads(options.threads->value()); library = support::cpp14::make_unique("." /* Only using random values */, seed->value()); - // We need to do the setup here because framework.init() will need CL / GLES to be initialised - try - { - example.do_setup(example_argv.size(), &example_argv[0]); - } -#ifdef ARM_COMPUTE_CL - catch(cl::Error &err) - { - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - std::cerr << std::endl - << "ERROR " << err.what() << "(" << err.err() << ")" << std::endl; - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - return 1; - } -#endif /* ARM_COMPUTE_CL */ - catch(std::runtime_error &err) - { - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - std::cerr << std::endl - << "ERROR " << err.what() << " " << (errno ? strerror(errno) : "") << std::endl; - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; - return 1; - } - if(options.log_level->value() > framework::LogLevel::NONE) { for(auto &p : printers) @@ -153,6 +136,10 @@ int run_example(int argc, char **argv, ValidateExample &example) #ifdef ARM_COMPUTE_CL if(opencl_is_available()) { + if(!CLScheduler::get().is_initialised()) + { + CLScheduler::get().default_init(); + } p->print_entry("CL_DEVICE_VERSION", CLKernelLibrary::get().get_device_version()); } else @@ -162,7 +149,7 @@ int run_example(int argc, char **argv, ValidateExample &example) #endif /* ARM_COMPUTE_CL */ p->print_entry("Iterations", support::cpp11::to_string(options.iterations->value())); p->print_entry("Threads", support::cpp11::to_string(options.threads->value())); - example.print_parameters(*p); + g_example->print_parameters(*p); } } diff --git a/tests/validate_examples/ValidateExample.h b/tests/validate_examples/ValidateExample.h index cedbe3d0d9..4dd552ab10 100644 --- a/tests/validate_examples/ValidateExample.h +++ b/tests/validate_examples/ValidateExample.h @@ -40,14 +40,24 @@ namespace utils * * All examples with a validation stage have to inherit from this class. */ -class ValidateExample : public Example +class ValidateExample { public: + /** Setup the example. + * + * @param[in] argc Argument count. + * @param[in] argv Argument values. + */ + virtual void do_setup(int argc, char **argv) {}; + /** Run the example. */ + virtual void do_run() {}; /** Run reference implementation and validate against the target output */ virtual void do_validate() { } + /** Teardown the example. */ + virtual void do_teardown() {}; /** Print the example parameters * * @param[in,out] printer Printer to use to print the parameters @@ -65,7 +75,7 @@ public: * @param[in] argv Command line arguments * @param[in] example Example to run */ -int run_example(int argc, char **argv, ValidateExample &example); +int run_example(int argc, char **argv, std::unique_ptr example); } // namespace utils } // namespace arm_compute diff --git a/utils/Utils.cpp b/utils/Utils.cpp index 8a2d11814e..a5c6a95a2a 100644 --- a/utils/Utils.cpp +++ b/utils/Utils.cpp @@ -67,16 +67,16 @@ void discard_comments_and_spaces(std::ifstream &fs) } // namespace #ifndef BENCHMARK_EXAMPLES -int run_example(int argc, char **argv, Example &example) +int run_example(int argc, char **argv, std::unique_ptr example) { std::cout << "\n" << argv[0] << "\n\n"; try { - example.do_setup(argc, argv); - example.do_run(); - example.do_teardown(); + example->do_setup(argc, argv); + example->do_run(); + example->do_teardown(); std::cout << "\nTest passed\n"; return 0; diff --git a/utils/Utils.h b/utils/Utils.h index 73b6d4f832..6241562a28 100644 --- a/utils/Utils.h +++ b/utils/Utils.h @@ -83,13 +83,12 @@ public: * @param[in] argv Command line arguments * @param[in] example Example to run */ -int run_example(int argc, char **argv, Example &example); +int run_example(int argc, char **argv, std::unique_ptr example); template int run_example(int argc, char **argv) { - T example; - return run_example(argc, argv, example); + return run_example(argc, argv, support::cpp14::make_unique()); } /** Draw a RGB rectangular window for the detected object -- cgit v1.2.1