diff options
author | Anthony Barbier <anthony.barbier@arm.com> | 2018-01-05 10:59:12 +0000 |
---|---|---|
committer | Anthony Barbier <anthony.barbier@arm.com> | 2018-11-02 16:42:33 +0000 |
commit | 6db0ff5b4bb49f834c7caa532a7feab228df10f9 (patch) | |
tree | 36c32ab3e94a86752d6b73c4a3503b191431d586 /tests | |
parent | fde97fb84943d0328c2c532d117e9b875149f27e (diff) | |
download | ComputeLibrary-6db0ff5b4bb49f834c7caa532a7feab228df10f9.tar.gz |
COMPMID-771 Allow examples to be profiled
Change-Id: I180281e796e1670b9ad391d82d66ecde0119ef78
Note: this is for internal use only which is why I think the hackiness of RunExample.cpp is acceptable.
Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/115154
Tested-by: Jenkins <bsgcomp@arm.com>
Reviewed-by: Pablo Tello <pablo.tello@arm.com>
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>
Diffstat (limited to 'tests')
-rw-r--r-- | tests/SConscript | 50 | ||||
-rw-r--r-- | tests/benchmark_examples/RunExample.cpp | 157 | ||||
-rw-r--r-- | tests/framework/command_line/CommonOptions.cpp | 158 | ||||
-rw-r--r-- | tests/framework/command_line/CommonOptions.h | 88 | ||||
-rw-r--r-- | tests/main.cpp | 131 |
5 files changed, 464 insertions, 120 deletions
diff --git a/tests/SConscript b/tests/SConscript index e4c561d844..4261331d43 100644 --- a/tests/SConscript +++ b/tests/SConscript @@ -31,6 +31,7 @@ SConscript('./framework/SConscript', duplicate=0) variables = [ BoolVariable("validation_tests", "Build validation test programs", True), BoolVariable("benchmark_tests", "Build benchmark test programs", True), + BoolVariable("benchmark_examples", "Build benchmark examples programs", True), ("test_filter", "Pattern to specify the tests' filenames to be compiled", "*.cpp") ] @@ -50,6 +51,9 @@ vars.Update(test_env) Help(new_options.GenerateHelpText(test_env)) +Import("arm_compute_test_framework") +test_env.Append(LIBS = arm_compute_test_framework) + if env['os'] in ['android', 'bare_metal'] or env['standalone']: Import("arm_compute_a") Import("arm_compute_core_a") @@ -69,9 +73,6 @@ test_env.Append(LIBPATH = ["#3rdparty/%s/%s" % (env['os'], env['arch'])]) test_env.Append(LIBPATH = ["#build/%s" % env['build_dir']]) test_env.Append(LIBPATH = ["#build/%s/opencl-1.2-stubs" % env['build_dir']]) -Import("arm_compute_test_framework") -test_env.Append(LIBS = arm_compute_test_framework) - common_files = Glob('*.cpp') common_objects = [test_env.StaticObject(f) for f in common_files] @@ -162,3 +163,46 @@ if test_env['validation_tests']: Default(arm_compute_validation) Export('arm_compute_validation') + +if test_env['benchmark_examples']: + files_benchmark_examples = test_env.Object('benchmark_examples/RunExample.cpp') + arm_compute_benchmark_examples = [] + if test_env['neon']: + for file in Glob("../examples/neon_*.cpp"): + example = "benchmark_" + os.path.basename(os.path.splitext(str(file))[0]) + arm_compute_benchmark_examples += [ test_env.Program(example, [ test_env.Object(source=file, target=example) ] + files_benchmark_examples) ] + if test_env['opencl']: + cl_examples = [] + files = Glob("../examples/cl_*.cpp") + if test_env['neon']: + files += Glob("../examples/neoncl_*.cpp") + for file in files: + example = "benchmark_" + os.path.basename(os.path.splitext(str(file))[0]) + cl_examples += [ test_env.Program(example, [ test_env.Object(source=file, target=example) ] + files_benchmark_examples, CPPDEFINES=['ARM_COMPUTE_CL'], LIBS = test_env["LIBS"] + ["OpenCL"]) ] + Depends(cl_examples, opencl) + arm_compute_benchmark_examples += cl_examples + if test_env['opencl'] and test_env['neon']: + if env['os'] == 'android': + Import('arm_compute_graph_a') + graph_dependency = arm_compute_graph_a + else: + Import('arm_compute_graph_so') + graph_dependency = arm_compute_graph_so + + graph_utils = test_env.Object(source="../utils/GraphUtils.cpp", target="GraphUtils") + for file in Glob("../examples/graph_*.cpp"): + example = "benchmark_" + os.path.basename(os.path.splitext(str(file))[0]) + if env['os'] == 'android': + prog = test_env.Program(example, [ test_env.Object(source=file, target=example), graph_utils]+ files_benchmark_examples, LIBS = test_env["LIBS"] + ["OpenCL"], LINKFLAGS=test_env["LINKFLAGS"]+['-Wl,--whole-archive',graph_dependency,'-Wl,--no-whole-archive']) + Depends(prog, [graph_dependency, opencl]) + arm_compute_benchmark_examples += [ prog ] + else: + #-Wl,--allow-shlib-undefined: Ignore dependencies of dependencies + prog = test_env.Program(example, [ test_env.Object(source=file, target=example), graph_utils]+ files_benchmark_examples, LIBS = test_env["LIBS"] + ["arm_compute_graph"], LINKFLAGS=test_env["LINKFLAGS"]+['-Wl,--allow-shlib-undefined'] ) + Depends(prog, graph_dependency) + arm_compute_benchmark_examples += [ prog ] + Depends(arm_compute_benchmark_examples, arm_compute_test_framework) + Depends(arm_compute_benchmark_examples, arm_compute_lib) + Default(arm_compute_benchmark_examples) + Export('arm_compute_benchmark_examples') + diff --git a/tests/benchmark_examples/RunExample.cpp b/tests/benchmark_examples/RunExample.cpp new file mode 100644 index 0000000000..dea9d9b2e3 --- /dev/null +++ b/tests/benchmark_examples/RunExample.cpp @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "utils/Utils.h" +//FIXME / INTERNAL_ONLY: This file should not be released! + +#define BENCHMARK_EXAMPLES +#include "utils/Utils.cpp" + +#include "arm_compute/runtime/Scheduler.h" +#include "tests/framework/Framework.h" +#include "tests/framework/Macros.h" +#include "tests/framework/command_line/CommandLineParser.h" +#include "tests/framework/command_line/CommonOptions.h" +#include "tests/framework/instruments/Instruments.h" + +#ifdef ARM_COMPUTE_CL +#include "arm_compute/runtime/CL/CLScheduler.h" +#endif /* ARM_COMPUTE_CL */ +#ifdef ARM_COMPUTE_GC +#include "arm_compute/runtime/GLES_COMPUTE/GCScheduler.h" +#endif /* ARM_COMPUTE_GC */ + +using namespace arm_compute; +using namespace arm_compute::test; + +namespace arm_compute +{ +namespace utils +{ +static Example *g_example = nullptr; +class ExampleTest : public arm_compute::test::framework::TestCase +{ +public: + ExampleTest() = default; + void do_run() override + { + g_example->do_run(); + } + void do_teardown() override + { + g_example->do_teardown(); + } +}; + +int run_example(int argc, char **argv, Example &example) +{ + framework::CommandLineParser parser; + framework::CommonOptions options(parser); + auto example_args = parser.add_option<framework::ListOption<std::string>>("example_args"); + example_args->set_help("Arguments to pass to the example"); + framework::Framework &framework = framework::Framework::get(); + + parser.parse(argc, argv); + + if(options.help->is_set() && options.help->value()) + { + parser.print_help(argv[0]); + return 0; + } + + std::vector<std::unique_ptr<framework::Printer>> printers = options.create_printers(); + g_example = &example; + std::vector<char *> example_argv = {}; + example_argv.clear(); + example_argv.emplace_back(argv[0]); + for(auto &arg : example_args->value()) + { + example_argv.emplace_back(const_cast<char *>(arg.c_str())); // NOLINT + } + // We need to do the setup here because framework.init() will need CL / GLES to be initialised + try + { + example.do_setup(example_argv.size(), &example_argv[0]); + } +#ifdef ARM_COMPUTE_CL + catch(cl::Error &err) + { + std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; + std::cerr << std::endl + << "ERROR " << err.what() << "(" << err.err() << ")" << std::endl; + std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; + return 1; + } +#endif /* ARM_COMPUTE_CL */ + catch(std::runtime_error &err) + { + std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; + std::cerr << std::endl + << "ERROR " << err.what() << " " << (errno ? strerror(errno) : "") << std::endl; + std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; + return 1; + } + + Scheduler::get().set_num_threads(options.threads->value()); + + if(options.log_level->value() > framework::LogLevel::NONE) + { + for(auto &p : printers) + { + p->print_global_header(); + } + } + + if(options.log_level->value() >= framework::LogLevel::CONFIG) + { + for(auto &p : printers) + { + p->print_entry("Iterations", support::cpp11::to_string(options.iterations->value())); + p->print_entry("Threads", support::cpp11::to_string(options.threads->value())); + } + } + + framework.init(options.instruments->value(), options.iterations->value(), framework::DatasetMode::ALL, "", "", options.log_level->value()); + for(auto &p : printers) + { + framework.add_printer(p.get()); + } + framework.set_throw_errors(options.throw_errors->value()); + arm_compute::test::framework::detail::TestSuiteRegistrar suite{ "Examples" }; + framework.add_test_case<ExampleTest>(argv[0], framework::DatasetMode::ALL, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE); + + //func(argc, argv); + bool success = framework.run(); + if(options.log_level->value() > framework::LogLevel::NONE) + { + for(auto &p : printers) + { + p->print_global_footer(); + } + } + + return (success ? 0 : 1); +} + +} // namespace utils +} // namespace arm_compute diff --git a/tests/framework/command_line/CommonOptions.cpp b/tests/framework/command_line/CommonOptions.cpp new file mode 100644 index 0000000000..631981be3d --- /dev/null +++ b/tests/framework/command_line/CommonOptions.cpp @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "CommonOptions.h" + +#include "../Framework.h" +#include "../printers/Printers.h" +#include "CommandLineParser.h" + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +CommonOptions::CommonOptions(CommandLineParser &parser) + : help(parser.add_option<ToggleOption>("help")), + instruments(), + iterations(parser.add_option<SimpleOption<int>>("iterations", 1)), + threads(parser.add_option<SimpleOption<int>>("threads", 1)), + log_format(), + log_file(parser.add_option<SimpleOption<std::string>>("log-file")), + log_level(), + throw_errors(parser.add_option<ToggleOption>("throw-errors")), + color_output(parser.add_option<ToggleOption>("color-output", true)), + pretty_console(parser.add_option<ToggleOption>("pretty-console", false)), + json_file(parser.add_option<SimpleOption<std::string>>("json-file")), + pretty_file(parser.add_option<SimpleOption<std::string>>("pretty-file")), + log_streams() +{ + Framework &framework = Framework::get(); + std::set<InstrumentsDescription> allowed_instruments + { + std::pair<InstrumentType, ScaleFactor>(InstrumentType::ALL, ScaleFactor::NONE), + std::pair<InstrumentType, ScaleFactor>(InstrumentType::NONE, ScaleFactor::NONE), + }; + + for(const auto &type : framework.available_instruments()) + { + allowed_instruments.insert(type); + } + + std::set<LogFormat> supported_log_formats + { + LogFormat::NONE, + LogFormat::PRETTY, + LogFormat::JSON, + }; + + std::set<LogLevel> supported_log_levels + { + LogLevel::NONE, + LogLevel::CONFIG, + LogLevel::TESTS, + LogLevel::ERRORS, + LogLevel::DEBUG, + LogLevel::MEASUREMENTS, + LogLevel::ALL, + }; + + instruments = parser.add_option<EnumListOption<InstrumentsDescription>>("instruments", allowed_instruments, std::initializer_list<InstrumentsDescription> { std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::NONE) }); + log_format = parser.add_option<EnumOption<LogFormat>>("log-format", supported_log_formats, LogFormat::PRETTY); + log_level = parser.add_option<EnumOption<LogLevel>>("log-level", supported_log_levels, LogLevel::ALL); + + help->set_help("Show this help message"); + instruments->set_help("Set the profiling instruments to use"); + iterations->set_help("Number of iterations per test case"); + threads->set_help("Number of threads to use"); + log_format->set_help("Output format for measurements and failures (affects only log-file)"); + log_file->set_help("Write output to file instead of to the console (affected by log-format)"); + log_level->set_help("Verbosity of the output"); + throw_errors->set_help("Don't catch fatal errors (useful for debugging)"); + color_output->set_help("Produce colored output on the console"); + pretty_console->set_help("Produce pretty output on the console"); + json_file->set_help("Write output to a json file."); + pretty_file->set_help("Write output to a text file"); +} +std::vector<std::unique_ptr<Printer>> CommonOptions::create_printers() +{ + std::vector<std::unique_ptr<Printer>> printers; + + if(pretty_console->value() && (log_file->is_set() || log_format->value() != LogFormat::PRETTY)) + { + auto pretty_printer = support::cpp14::make_unique<PrettyPrinter>(); + pretty_printer->set_color_output(color_output->value()); + printers.push_back(std::move(pretty_printer)); + } + + std::unique_ptr<Printer> printer; + switch(log_format->value()) + { + case LogFormat::JSON: + printer = support::cpp14::make_unique<JSONPrinter>(); + break; + case LogFormat::NONE: + break; + case LogFormat::PRETTY: + default: + auto pretty_printer = support::cpp14::make_unique<PrettyPrinter>(); + // Don't use colours if we print to a file: + pretty_printer->set_color_output((!log_file->is_set()) && color_output->value()); + printer = std::move(pretty_printer); + break; + } + + if(log_file->is_set()) + { + log_streams.push_back(std::make_shared<std::ofstream>(log_file->value())); + if(printer != nullptr) + { + printer->set_stream(*log_streams.back().get()); + } + } + + if(printer != nullptr) + { + printers.push_back(std::move(printer)); + } + + if(json_file->is_set()) + { + printers.push_back(support::cpp14::make_unique<JSONPrinter>()); + log_streams.push_back(std::make_shared<std::ofstream>(json_file->value())); + printers.back()->set_stream(*log_streams.back().get()); + } + + if(pretty_file->is_set()) + { + printers.push_back(support::cpp14::make_unique<PrettyPrinter>()); + log_streams.push_back(std::make_shared<std::ofstream>(pretty_file->value())); + printers.back()->set_stream(*log_streams.back().get()); + } + + return printers; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/command_line/CommonOptions.h b/tests/framework/command_line/CommonOptions.h new file mode 100644 index 0000000000..2da2c99874 --- /dev/null +++ b/tests/framework/command_line/CommonOptions.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_COMMONOPTIONS +#define ARM_COMPUTE_TEST_COMMONOPTIONS + +#include "../instruments/Instruments.h" +#include "CommandLineOptions.h" +#include <memory> + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +class CommandLineParser; +class Printer; +enum class LogFormat; +enum class LogLevel; + +/** Common command line options used to configure the framework + * + * The options in this object get populated when "parse()" is called on the parser used to construct it. + * The expected workflow is: + * + * CommandLineParser parser; + * CommonOptions options( parser ); + * parser.parse(argc, argv); + * if(options.log_level->value() > LogLevel::NONE) --> Use the options values + */ +class CommonOptions +{ +public: + /** Constructor + * + * @param[in,out] parser A parser on which "parse()" hasn't been called yet. + */ + CommonOptions(CommandLineParser &parser); + CommonOptions(const CommonOptions &) = delete; + CommonOptions &operator=(const CommonOptions &) = delete; + /** Create the printers based on parsed command line options + * + * @pre "parse()" has been called on the parser used to construct this object + * + * @return List of printers + */ + std::vector<std::unique_ptr<Printer>> create_printers(); + + ToggleOption *help; + EnumListOption<InstrumentsDescription> *instruments; + SimpleOption<int> *iterations; + SimpleOption<int> *threads; + EnumOption<LogFormat> *log_format; + SimpleOption<std::string> *log_file; + EnumOption<LogLevel> *log_level; + ToggleOption *throw_errors; + ToggleOption *color_output; + ToggleOption *pretty_console; + SimpleOption<std::string> *json_file; + SimpleOption<std::string> *pretty_file; + std::vector<std::shared_ptr<std::ofstream>> log_streams; +}; + +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_COMMONOPTIONS */ diff --git a/tests/main.cpp b/tests/main.cpp index c62de66588..1797b21114 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017, 2018 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -30,6 +30,7 @@ #include "tests/framework/Profiler.h" #include "tests/framework/command_line/CommandLineOptions.h" #include "tests/framework/command_line/CommandLineParser.h" +#include "tests/framework/command_line/CommonOptions.h" #include "tests/framework/instruments/Instruments.h" #include "tests/framework/printers/Printers.h" @@ -73,17 +74,6 @@ int main(int argc, char **argv) framework::CommandLineParser parser; - std::set<framework::InstrumentsDescription> allowed_instruments - { - std::pair<framework::InstrumentType, framework::ScaleFactor>(framework::InstrumentType::ALL, framework::ScaleFactor::NONE), - std::pair<framework::InstrumentType, framework::ScaleFactor>(framework::InstrumentType::NONE, framework::ScaleFactor::NONE), - }; - - for(const auto &type : framework.available_instruments()) - { - allowed_instruments.insert(type); - } - std::set<framework::DatasetMode> allowed_modes { framework::DatasetMode::PRECOMMIT, @@ -91,52 +81,18 @@ int main(int argc, char **argv) framework::DatasetMode::ALL }; - std::set<framework::LogFormat> supported_log_formats - { - framework::LogFormat::NONE, - framework::LogFormat::PRETTY, - framework::LogFormat::JSON, - }; - - std::set<framework::LogLevel> supported_log_levels - { - framework::LogLevel::NONE, - framework::LogLevel::CONFIG, - framework::LogLevel::TESTS, - framework::LogLevel::ERRORS, - framework::LogLevel::DEBUG, - framework::LogLevel::MEASUREMENTS, - framework::LogLevel::ALL, - }; + framework::CommonOptions options(parser); - auto help = parser.add_option<framework::ToggleOption>("help"); - help->set_help("Show this help message"); auto dataset_mode = parser.add_option<framework::EnumOption<framework::DatasetMode>>("mode", allowed_modes, framework::DatasetMode::PRECOMMIT); dataset_mode->set_help("For managed datasets select which group to use"); - auto instruments = parser.add_option<framework::EnumListOption<framework::InstrumentsDescription>>("instruments", allowed_instruments, std::initializer_list<framework::InstrumentsDescription> { std::pair<framework::InstrumentType, framework::ScaleFactor>(framework::InstrumentType::WALL_CLOCK_TIMER, framework::ScaleFactor::NONE) }); - instruments->set_help("Set the profiling instruments to use"); - auto iterations = parser.add_option<framework::SimpleOption<int>>("iterations", 1); - iterations->set_help("Number of iterations per test case"); - auto threads = parser.add_option<framework::SimpleOption<int>>("threads", 1); - threads->set_help("Number of threads to use"); - auto log_format = parser.add_option<framework::EnumOption<framework::LogFormat>>("log-format", supported_log_formats, framework::LogFormat::PRETTY); - log_format->set_help("Output format for measurements and failures (affects only log-file)"); auto filter = parser.add_option<framework::SimpleOption<std::string>>("filter", ".*"); filter->set_help("Regular expression to select test cases"); auto filter_id = parser.add_option<framework::SimpleOption<std::string>>("filter-id"); filter_id->set_help("List of test ids. ... can be used to define a range."); - auto log_file = parser.add_option<framework::SimpleOption<std::string>>("log-file"); - log_file->set_help("Write output to file instead of to the console (affected by log-format)"); - auto log_level = parser.add_option<framework::EnumOption<framework::LogLevel>>("log-level", supported_log_levels, framework::LogLevel::ALL); - log_level->set_help("Verbosity of the output"); - auto throw_errors = parser.add_option<framework::ToggleOption>("throw-errors"); - throw_errors->set_help("Don't catch fatal errors (useful for debugging)"); auto stop_on_error = parser.add_option<framework::ToggleOption>("stop-on-error"); stop_on_error->set_help("Abort execution after the first failed test (useful for debugging)"); auto seed = parser.add_option<framework::SimpleOption<std::random_device::result_type>>("seed", std::random_device()()); seed->set_help("Global seed for random number generation"); - auto color_output = parser.add_option<framework::ToggleOption>("color-output", true); - color_output->set_help("Produce colored output on the console"); auto list_tests = parser.add_option<framework::ToggleOption>("list-tests", false); list_tests->set_help("List all test names"); auto test_instruments = parser.add_option<framework::ToggleOption>("test-instruments", false); @@ -145,81 +101,22 @@ int main(int argc, char **argv) error_on_missing_assets->set_help("Mark a test as failed instead of skipping it when assets are missing"); auto assets = parser.add_positional_option<framework::SimpleOption<std::string>>("assets"); assets->set_help("Path to the assets directory"); - auto pretty_console = parser.add_option<framework::ToggleOption>("pretty-console", false); - pretty_console->set_help("Produce pretty output on the console"); - auto json_file = parser.add_option<framework::SimpleOption<std::string>>("json-file"); - json_file->set_help("Write output to a json file."); - auto pretty_file = parser.add_option<framework::SimpleOption<std::string>>("pretty-file"); - pretty_file->set_help("Write output to a text file"); try { parser.parse(argc, argv); - if(help->is_set() && help->value()) + if(options.help->is_set() && options.help->value()) { parser.print_help(argv[0]); return 0; } - std::vector<std::unique_ptr<framework::Printer>> printers; - std::vector<std::shared_ptr<std::ofstream>> log_streams; - - if(pretty_console->value() && (log_file->is_set() || log_format->value() != framework::LogFormat::PRETTY)) - { - auto pretty_printer = support::cpp14::make_unique<framework::PrettyPrinter>(); - pretty_printer->set_color_output(color_output->value()); - printers.push_back(std::move(pretty_printer)); - } - - std::unique_ptr<framework::Printer> printer; - switch(log_format->value()) - { - case framework::LogFormat::JSON: - printer = support::cpp14::make_unique<framework::JSONPrinter>(); - break; - case framework::LogFormat::NONE: - break; - case framework::LogFormat::PRETTY: - default: - auto pretty_printer = support::cpp14::make_unique<framework::PrettyPrinter>(); - // Don't use colours if we print to a file: - pretty_printer->set_color_output((!log_file->is_set()) && color_output->value()); - printer = std::move(pretty_printer); - break; - } - - if(log_file->is_set()) - { - log_streams.push_back(std::make_shared<std::ofstream>(log_file->value())); - if(printer != nullptr) - { - printer->set_stream(*log_streams.back().get()); - } - } - - if(printer != nullptr) - { - printers.push_back(std::move(printer)); - } - - if(json_file->is_set()) - { - printers.push_back(support::cpp14::make_unique<framework::JSONPrinter>()); - log_streams.push_back(std::make_shared<std::ofstream>(json_file->value())); - printers.back()->set_stream(*log_streams.back().get()); - } - - if(pretty_file->is_set()) - { - printers.push_back(support::cpp14::make_unique<framework::PrettyPrinter>()); - log_streams.push_back(std::make_shared<std::ofstream>(pretty_file->value())); - printers.back()->set_stream(*log_streams.back().get()); - } + std::vector<std::unique_ptr<framework::Printer>> printers = options.create_printers(); - Scheduler::get().set_num_threads(threads->value()); + Scheduler::get().set_num_threads(options.threads->value()); - if(log_level->value() > framework::LogLevel::NONE) + if(options.log_level->value() > framework::LogLevel::NONE) { for(auto &p : printers) { @@ -227,13 +124,13 @@ int main(int argc, char **argv) } } - if(log_level->value() >= framework::LogLevel::CONFIG) + if(options.log_level->value() >= framework::LogLevel::CONFIG) { for(auto &p : printers) { p->print_entry("Seed", support::cpp11::to_string(seed->value())); - p->print_entry("Iterations", support::cpp11::to_string(iterations->value())); - p->print_entry("Threads", support::cpp11::to_string(threads->value())); + p->print_entry("Iterations", support::cpp11::to_string(options.iterations->value())); + p->print_entry("Threads", support::cpp11::to_string(options.threads->value())); { using support::cpp11::to_string; p->print_entry("Dataset mode", to_string(dataset_mode->value())); @@ -241,12 +138,12 @@ int main(int argc, char **argv) } } - framework.init(instruments->value(), iterations->value(), dataset_mode->value(), filter->value(), filter_id->value(), log_level->value()); + framework.init(options.instruments->value(), options.iterations->value(), dataset_mode->value(), filter->value(), filter_id->value(), options.log_level->value()); for(auto &p : printers) { framework.add_printer(p.get()); } - framework.set_throw_errors(throw_errors->value()); + framework.set_throw_errors(options.throw_errors->value()); framework.set_stop_on_error(stop_on_error->value()); framework.set_error_on_missing_assets(error_on_missing_assets->value()); @@ -285,7 +182,7 @@ int main(int argc, char **argv) success = framework.run(); - if(log_level->value() > framework::LogLevel::NONE) + if(options.log_level->value() > framework::LogLevel::NONE) { for(auto &p : printers) { @@ -299,7 +196,7 @@ int main(int argc, char **argv) { std::cerr << error.what() << "\n"; - if(throw_errors->value()) + if(options.throw_errors->value()) { throw; } |