COMPMID-771 Allow examples to be profiled

Change-Id: I180281e796e1670b9ad391d82d66ecde0119ef78
Note: this is for internal use only which is why I think the hackiness of RunExample.cpp is acceptable.
Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/115154
Tested-by: Jenkins <bsgcomp@arm.com>
Reviewed-by: Pablo Tello <pablo.tello@arm.com>
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>

5 files changed, 464 insertions, 120 deletions

diff --git a/tests/SConscript b/tests/SConscript
index e4c561d844..4261331d43 100644
--- a/tests/SConscript
+++ b/tests/SConscript
@@ -31,6 +31,7 @@ SConscript('./framework/SConscript', duplicate=0)
 variables = [
     BoolVariable("validation_tests", "Build validation test programs", True),
     BoolVariable("benchmark_tests", "Build benchmark test programs", True),
+    BoolVariable("benchmark_examples", "Build benchmark examples programs", True),
     ("test_filter", "Pattern to specify the tests' filenames to be compiled", "*.cpp")
 ]
 
@@ -50,6 +51,9 @@ vars.Update(test_env)
 
 Help(new_options.GenerateHelpText(test_env))
 
+Import("arm_compute_test_framework")
+test_env.Append(LIBS = arm_compute_test_framework)
+
 if env['os'] in ['android', 'bare_metal'] or env['standalone']:
     Import("arm_compute_a")
     Import("arm_compute_core_a")
@@ -69,9 +73,6 @@ test_env.Append(LIBPATH = ["#3rdparty/%s/%s" % (env['os'], env['arch'])])
 test_env.Append(LIBPATH = ["#build/%s" % env['build_dir']])
 test_env.Append(LIBPATH = ["#build/%s/opencl-1.2-stubs" % env['build_dir']])
 
-Import("arm_compute_test_framework")
-test_env.Append(LIBS = arm_compute_test_framework)
-
 common_files = Glob('*.cpp')
 common_objects = [test_env.StaticObject(f) for f in common_files]
 
@@ -162,3 +163,46 @@ if test_env['validation_tests']:
 
     Default(arm_compute_validation)
     Export('arm_compute_validation')
+
+if test_env['benchmark_examples']:
+    files_benchmark_examples = test_env.Object('benchmark_examples/RunExample.cpp')
+    arm_compute_benchmark_examples = []
+    if test_env['neon']:
+        for file in Glob("../examples/neon_*.cpp"):
+            example = "benchmark_" + os.path.basename(os.path.splitext(str(file))[0])
+            arm_compute_benchmark_examples += [ test_env.Program(example, [ test_env.Object(source=file, target=example) ] + files_benchmark_examples) ]
+    if test_env['opencl']:
+        cl_examples = []
+        files = Glob("../examples/cl_*.cpp")
+        if test_env['neon']:
+            files += Glob("../examples/neoncl_*.cpp")
+        for file in files:
+            example = "benchmark_" + os.path.basename(os.path.splitext(str(file))[0])
+            cl_examples += [ test_env.Program(example, [ test_env.Object(source=file, target=example) ] + files_benchmark_examples, CPPDEFINES=['ARM_COMPUTE_CL'], LIBS = test_env["LIBS"] + ["OpenCL"]) ]
+        Depends(cl_examples, opencl)
+        arm_compute_benchmark_examples += cl_examples
+        if test_env['opencl'] and test_env['neon']:
+            if env['os'] == 'android':
+                Import('arm_compute_graph_a')
+                graph_dependency = arm_compute_graph_a
+            else:
+                Import('arm_compute_graph_so')
+                graph_dependency = arm_compute_graph_so
+
+            graph_utils = test_env.Object(source="../utils/GraphUtils.cpp", target="GraphUtils")
+            for file in Glob("../examples/graph_*.cpp"):
+                example = "benchmark_" + os.path.basename(os.path.splitext(str(file))[0])
+                if env['os'] == 'android':
+                    prog = test_env.Program(example, [ test_env.Object(source=file, target=example), graph_utils]+ files_benchmark_examples, LIBS = test_env["LIBS"] + ["OpenCL"], LINKFLAGS=test_env["LINKFLAGS"]+['-Wl,--whole-archive',graph_dependency,'-Wl,--no-whole-archive'])
+                    Depends(prog, [graph_dependency, opencl])
+                    arm_compute_benchmark_examples += [ prog ]
+                else:
+                    #-Wl,--allow-shlib-undefined: Ignore dependencies of dependencies
+                    prog = test_env.Program(example, [ test_env.Object(source=file, target=example), graph_utils]+ files_benchmark_examples, LIBS = test_env["LIBS"] + ["arm_compute_graph"], LINKFLAGS=test_env["LINKFLAGS"]+['-Wl,--allow-shlib-undefined'] )
+                    Depends(prog, graph_dependency)
+                    arm_compute_benchmark_examples += [ prog ]
+    Depends(arm_compute_benchmark_examples, arm_compute_test_framework)
+    Depends(arm_compute_benchmark_examples, arm_compute_lib)
+    Default(arm_compute_benchmark_examples)
+    Export('arm_compute_benchmark_examples')
+
diff --git a/tests/benchmark_examples/RunExample.cpp b/tests/benchmark_examples/RunExample.cpp
new file mode 100644
index 0000000000..dea9d9b2e3
--- /dev/null
+++ b/tests/benchmark_examples/RunExample.cpp
@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) 2018 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "utils/Utils.h"
+//FIXME / INTERNAL_ONLY: This file should not be released!
+
+#define BENCHMARK_EXAMPLES
+#include "utils/Utils.cpp"
+
+#include "arm_compute/runtime/Scheduler.h"
+#include "tests/framework/Framework.h"
+#include "tests/framework/Macros.h"
+#include "tests/framework/command_line/CommandLineParser.h"
+#include "tests/framework/command_line/CommonOptions.h"
+#include "tests/framework/instruments/Instruments.h"
+
+#ifdef ARM_COMPUTE_CL
+#include "arm_compute/runtime/CL/CLScheduler.h"
+#endif /* ARM_COMPUTE_CL */
+#ifdef ARM_COMPUTE_GC
+#include "arm_compute/runtime/GLES_COMPUTE/GCScheduler.h"
+#endif /* ARM_COMPUTE_GC */
+
+using namespace arm_compute;
+using namespace arm_compute::test;
+
+namespace arm_compute
+{
+namespace utils
+{
+static Example *g_example = nullptr;
+class ExampleTest : public arm_compute::test::framework::TestCase
+{
+public:
+    ExampleTest() = default;
+    void do_run() override
+    {
+        g_example->do_run();
+    }
+    void do_teardown() override
+    {
+        g_example->do_teardown();
+    }
+};
+
+int run_example(int argc, char **argv, Example &example)
+{
+    framework::CommandLineParser parser;
+    framework::CommonOptions     options(parser);
+    auto                         example_args = parser.add_option<framework::ListOption<std::string>>("example_args");
+    example_args->set_help("Arguments to pass to the example");
+    framework::Framework &framework = framework::Framework::get();
+
+    parser.parse(argc, argv);
+
+    if(options.help->is_set() && options.help->value())
+    {
+        parser.print_help(argv[0]);
+        return 0;
+    }
+
+    std::vector<std::unique_ptr<framework::Printer>> printers = options.create_printers();
+    g_example                                                 = &example;
+    std::vector<char *> example_argv                          = {};
+    example_argv.clear();
+    example_argv.emplace_back(argv[0]);
+    for(auto &arg : example_args->value())
+    {
+        example_argv.emplace_back(const_cast<char *>(arg.c_str())); // NOLINT
+    }
+    // We need to do the setup here because framework.init() will need CL / GLES to be initialised
+    try
+    {
+        example.do_setup(example_argv.size(), &example_argv[0]);
+    }
+#ifdef ARM_COMPUTE_CL
+    catch(cl::Error &err)
+    {
+        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
+        std::cerr << std::endl
+                  << "ERROR " << err.what() << "(" << err.err() << ")" << std::endl;
+        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
+        return 1;
+    }
+#endif /* ARM_COMPUTE_CL */
+    catch(std::runtime_error &err)
+    {
+        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
+        std::cerr << std::endl
+                  << "ERROR " << err.what() << " " << (errno ? strerror(errno) : "") << std::endl;
+        std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl;
+        return 1;
+    }
+
+    Scheduler::get().set_num_threads(options.threads->value());
+
+    if(options.log_level->value() > framework::LogLevel::NONE)
+    {
+        for(auto &p : printers)
+        {
+            p->print_global_header();
+        }
+    }
+
+    if(options.log_level->value() >= framework::LogLevel::CONFIG)
+    {
+        for(auto &p : printers)
+        {
+            p->print_entry("Iterations", support::cpp11::to_string(options.iterations->value()));
+            p->print_entry("Threads", support::cpp11::to_string(options.threads->value()));
+        }
+    }
+
+    framework.init(options.instruments->value(), options.iterations->value(), framework::DatasetMode::ALL, "", "", options.log_level->value());
+    for(auto &p : printers)
+    {
+        framework.add_printer(p.get());
+    }
+    framework.set_throw_errors(options.throw_errors->value());
+    arm_compute::test::framework::detail::TestSuiteRegistrar suite{ "Examples" };
+    framework.add_test_case<ExampleTest>(argv[0], framework::DatasetMode::ALL, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE);
+
+    //func(argc, argv);
+    bool success = framework.run();
+    if(options.log_level->value() > framework::LogLevel::NONE)
+    {
+        for(auto &p : printers)
+        {
+            p->print_global_footer();
+        }
+    }
+
+    return (success ? 0 : 1);
+}
+
+} // namespace utils
+} // namespace arm_compute
diff --git a/tests/framework/command_line/CommonOptions.cpp b/tests/framework/command_line/CommonOptions.cpp
new file mode 100644
index 0000000000..631981be3d
--- /dev/null
+++ b/tests/framework/command_line/CommonOptions.cpp
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2018 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "CommonOptions.h"
+
+#include "../Framework.h"
+#include "../printers/Printers.h"
+#include "CommandLineParser.h"
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+CommonOptions::CommonOptions(CommandLineParser &parser)
+    : help(parser.add_option<ToggleOption>("help")),
+      instruments(),
+      iterations(parser.add_option<SimpleOption<int>>("iterations", 1)),
+      threads(parser.add_option<SimpleOption<int>>("threads", 1)),
+      log_format(),
+      log_file(parser.add_option<SimpleOption<std::string>>("log-file")),
+      log_level(),
+      throw_errors(parser.add_option<ToggleOption>("throw-errors")),
+      color_output(parser.add_option<ToggleOption>("color-output", true)),
+      pretty_console(parser.add_option<ToggleOption>("pretty-console", false)),
+      json_file(parser.add_option<SimpleOption<std::string>>("json-file")),
+      pretty_file(parser.add_option<SimpleOption<std::string>>("pretty-file")),
+      log_streams()
+{
+    Framework                       &framework = Framework::get();
+    std::set<InstrumentsDescription> allowed_instruments
+    {
+        std::pair<InstrumentType, ScaleFactor>(InstrumentType::ALL, ScaleFactor::NONE),
+        std::pair<InstrumentType, ScaleFactor>(InstrumentType::NONE, ScaleFactor::NONE),
+    };
+
+    for(const auto &type : framework.available_instruments())
+    {
+        allowed_instruments.insert(type);
+    }
+
+    std::set<LogFormat> supported_log_formats
+    {
+        LogFormat::NONE,
+        LogFormat::PRETTY,
+        LogFormat::JSON,
+    };
+
+    std::set<LogLevel> supported_log_levels
+    {
+        LogLevel::NONE,
+        LogLevel::CONFIG,
+        LogLevel::TESTS,
+        LogLevel::ERRORS,
+        LogLevel::DEBUG,
+        LogLevel::MEASUREMENTS,
+        LogLevel::ALL,
+    };
+
+    instruments = parser.add_option<EnumListOption<InstrumentsDescription>>("instruments", allowed_instruments, std::initializer_list<InstrumentsDescription> { std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::NONE) });
+    log_format  = parser.add_option<EnumOption<LogFormat>>("log-format", supported_log_formats, LogFormat::PRETTY);
+    log_level   = parser.add_option<EnumOption<LogLevel>>("log-level", supported_log_levels, LogLevel::ALL);
+
+    help->set_help("Show this help message");
+    instruments->set_help("Set the profiling instruments to use");
+    iterations->set_help("Number of iterations per test case");
+    threads->set_help("Number of threads to use");
+    log_format->set_help("Output format for measurements and failures (affects only log-file)");
+    log_file->set_help("Write output to file instead of to the console (affected by log-format)");
+    log_level->set_help("Verbosity of the output");
+    throw_errors->set_help("Don't catch fatal errors (useful for debugging)");
+    color_output->set_help("Produce colored output on the console");
+    pretty_console->set_help("Produce pretty output on the console");
+    json_file->set_help("Write output to a json file.");
+    pretty_file->set_help("Write output to a text file");
+}
+std::vector<std::unique_ptr<Printer>> CommonOptions::create_printers()
+{
+    std::vector<std::unique_ptr<Printer>> printers;
+
+    if(pretty_console->value() && (log_file->is_set() || log_format->value() != LogFormat::PRETTY))
+    {
+        auto pretty_printer = support::cpp14::make_unique<PrettyPrinter>();
+        pretty_printer->set_color_output(color_output->value());
+        printers.push_back(std::move(pretty_printer));
+    }
+
+    std::unique_ptr<Printer> printer;
+    switch(log_format->value())
+    {
+        case LogFormat::JSON:
+            printer = support::cpp14::make_unique<JSONPrinter>();
+            break;
+        case LogFormat::NONE:
+            break;
+        case LogFormat::PRETTY:
+        default:
+            auto pretty_printer = support::cpp14::make_unique<PrettyPrinter>();
+            // Don't use colours if we print to a file:
+            pretty_printer->set_color_output((!log_file->is_set()) && color_output->value());
+            printer = std::move(pretty_printer);
+            break;
+    }
+
+    if(log_file->is_set())
+    {
+        log_streams.push_back(std::make_shared<std::ofstream>(log_file->value()));
+        if(printer != nullptr)
+        {
+            printer->set_stream(*log_streams.back().get());
+        }
+    }
+
+    if(printer != nullptr)
+    {
+        printers.push_back(std::move(printer));
+    }
+
+    if(json_file->is_set())
+    {
+        printers.push_back(support::cpp14::make_unique<JSONPrinter>());
+        log_streams.push_back(std::make_shared<std::ofstream>(json_file->value()));
+        printers.back()->set_stream(*log_streams.back().get());
+    }
+
+    if(pretty_file->is_set())
+    {
+        printers.push_back(support::cpp14::make_unique<PrettyPrinter>());
+        log_streams.push_back(std::make_shared<std::ofstream>(pretty_file->value()));
+        printers.back()->set_stream(*log_streams.back().get());
+    }
+
+    return printers;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/command_line/CommonOptions.h b/tests/framework/command_line/CommonOptions.h
new file mode 100644
index 0000000000..2da2c99874
--- /dev/null
+++ b/tests/framework/command_line/CommonOptions.h
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2018 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_COMMONOPTIONS
+#define ARM_COMPUTE_TEST_COMMONOPTIONS
+
+#include "../instruments/Instruments.h"
+#include "CommandLineOptions.h"
+#include <memory>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+class CommandLineParser;
+class Printer;
+enum class LogFormat;
+enum class LogLevel;
+
+/** Common command line options used to configure the framework
+     *
+     * The options in this object get populated when "parse()" is called on the parser used to construct it.
+     * The expected workflow is:
+     *
+     * CommandLineParser parser;
+     * CommonOptions options( parser );
+     * parser.parse(argc, argv);
+     * if(options.log_level->value() > LogLevel::NONE) --> Use the options values
+     */
+class CommonOptions
+{
+public:
+    /** Constructor
+     *
+     * @param[in,out] parser A parser on which "parse()" hasn't been called yet.
+     */
+    CommonOptions(CommandLineParser &parser);
+    CommonOptions(const CommonOptions &) = delete;
+    CommonOptions &operator=(const CommonOptions &) = delete;
+    /** Create the printers based on parsed command line options
+     *
+     * @pre "parse()" has been called on the parser used to construct this object
+     *
+     * @return List of printers
+     */
+    std::vector<std::unique_ptr<Printer>> create_printers();
+
+    ToggleOption                               *help;
+    EnumListOption<InstrumentsDescription>     *instruments;
+    SimpleOption<int>                          *iterations;
+    SimpleOption<int>                          *threads;
+    EnumOption<LogFormat>                      *log_format;
+    SimpleOption<std::string>                  *log_file;
+    EnumOption<LogLevel>                       *log_level;
+    ToggleOption                               *throw_errors;
+    ToggleOption                               *color_output;
+    ToggleOption                               *pretty_console;
+    SimpleOption<std::string>                  *json_file;
+    SimpleOption<std::string>                  *pretty_file;
+    std::vector<std::shared_ptr<std::ofstream>> log_streams;
+};
+
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_COMMONOPTIONS */
diff --git a/tests/main.cpp b/tests/main.cpp
index c62de66588..1797b21114 100644
--- a/tests/main.cpp
+++ b/tests/main.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017, 2018 ARM Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -30,6 +30,7 @@
 #include "tests/framework/Profiler.h"
 #include "tests/framework/command_line/CommandLineOptions.h"
 #include "tests/framework/command_line/CommandLineParser.h"
+#include "tests/framework/command_line/CommonOptions.h"
 #include "tests/framework/instruments/Instruments.h"
 #include "tests/framework/printers/Printers.h"
 
@@ -73,17 +74,6 @@ int main(int argc, char **argv)
 
     framework::CommandLineParser parser;
 
-    std::set<framework::InstrumentsDescription> allowed_instruments
-    {
-        std::pair<framework::InstrumentType, framework::ScaleFactor>(framework::InstrumentType::ALL, framework::ScaleFactor::NONE),
-        std::pair<framework::InstrumentType, framework::ScaleFactor>(framework::InstrumentType::NONE, framework::ScaleFactor::NONE),
-    };
-
-    for(const auto &type : framework.available_instruments())
-    {
-        allowed_instruments.insert(type);
-    }
-
     std::set<framework::DatasetMode> allowed_modes
     {
         framework::DatasetMode::PRECOMMIT,
@@ -91,52 +81,18 @@ int main(int argc, char **argv)
         framework::DatasetMode::ALL
     };
 
-    std::set<framework::LogFormat> supported_log_formats
-    {
-        framework::LogFormat::NONE,
-        framework::LogFormat::PRETTY,
-        framework::LogFormat::JSON,
-    };
-
-    std::set<framework::LogLevel> supported_log_levels
-    {
-        framework::LogLevel::NONE,
-        framework::LogLevel::CONFIG,
-        framework::LogLevel::TESTS,
-        framework::LogLevel::ERRORS,
-        framework::LogLevel::DEBUG,
-        framework::LogLevel::MEASUREMENTS,
-        framework::LogLevel::ALL,
-    };
+    framework::CommonOptions options(parser);
 
-    auto help = parser.add_option<framework::ToggleOption>("help");
-    help->set_help("Show this help message");
     auto dataset_mode = parser.add_option<framework::EnumOption<framework::DatasetMode>>("mode", allowed_modes, framework::DatasetMode::PRECOMMIT);
     dataset_mode->set_help("For managed datasets select which group to use");
-    auto instruments = parser.add_option<framework::EnumListOption<framework::InstrumentsDescription>>("instruments", allowed_instruments, std::initializer_list<framework::InstrumentsDescription> { std::pair<framework::InstrumentType, framework::ScaleFactor>(framework::InstrumentType::WALL_CLOCK_TIMER, framework::ScaleFactor::NONE) });
-    instruments->set_help("Set the profiling instruments to use");
-    auto iterations = parser.add_option<framework::SimpleOption<int>>("iterations", 1);
-    iterations->set_help("Number of iterations per test case");
-    auto threads = parser.add_option<framework::SimpleOption<int>>("threads", 1);
-    threads->set_help("Number of threads to use");
-    auto log_format = parser.add_option<framework::EnumOption<framework::LogFormat>>("log-format", supported_log_formats, framework::LogFormat::PRETTY);
-    log_format->set_help("Output format for measurements and failures (affects only log-file)");
     auto filter = parser.add_option<framework::SimpleOption<std::string>>("filter", ".*");
     filter->set_help("Regular expression to select test cases");
     auto filter_id = parser.add_option<framework::SimpleOption<std::string>>("filter-id");
     filter_id->set_help("List of test ids. ... can be used to define a range.");
-    auto log_file = parser.add_option<framework::SimpleOption<std::string>>("log-file");
-    log_file->set_help("Write output to file instead of to the console (affected by log-format)");
-    auto log_level = parser.add_option<framework::EnumOption<framework::LogLevel>>("log-level", supported_log_levels, framework::LogLevel::ALL);
-    log_level->set_help("Verbosity of the output");
-    auto throw_errors = parser.add_option<framework::ToggleOption>("throw-errors");
-    throw_errors->set_help("Don't catch fatal errors (useful for debugging)");
     auto stop_on_error = parser.add_option<framework::ToggleOption>("stop-on-error");
     stop_on_error->set_help("Abort execution after the first failed test (useful for debugging)");
     auto seed = parser.add_option<framework::SimpleOption<std::random_device::result_type>>("seed", std::random_device()());
     seed->set_help("Global seed for random number generation");
-    auto color_output = parser.add_option<framework::ToggleOption>("color-output", true);
-    color_output->set_help("Produce colored output on the console");
     auto list_tests = parser.add_option<framework::ToggleOption>("list-tests", false);
     list_tests->set_help("List all test names");
     auto test_instruments = parser.add_option<framework::ToggleOption>("test-instruments", false);
@@ -145,81 +101,22 @@ int main(int argc, char **argv)
     error_on_missing_assets->set_help("Mark a test as failed instead of skipping it when assets are missing");
     auto assets = parser.add_positional_option<framework::SimpleOption<std::string>>("assets");
     assets->set_help("Path to the assets directory");
-    auto pretty_console = parser.add_option<framework::ToggleOption>("pretty-console", false);
-    pretty_console->set_help("Produce pretty output on the console");
-    auto json_file = parser.add_option<framework::SimpleOption<std::string>>("json-file");
-    json_file->set_help("Write output to a json file.");
-    auto pretty_file = parser.add_option<framework::SimpleOption<std::string>>("pretty-file");
-    pretty_file->set_help("Write output to a text file");
 
     try
     {
         parser.parse(argc, argv);
 
-        if(help->is_set() && help->value())
+        if(options.help->is_set() && options.help->value())
         {
             parser.print_help(argv[0]);
             return 0;
         }
 
-        std::vector<std::unique_ptr<framework::Printer>> printers;
-        std::vector<std::shared_ptr<std::ofstream>>      log_streams;
-
-        if(pretty_console->value() && (log_file->is_set() || log_format->value() != framework::LogFormat::PRETTY))
-        {
-            auto pretty_printer = support::cpp14::make_unique<framework::PrettyPrinter>();
-            pretty_printer->set_color_output(color_output->value());
-            printers.push_back(std::move(pretty_printer));
-        }
-
-        std::unique_ptr<framework::Printer> printer;
-        switch(log_format->value())
-        {
-            case framework::LogFormat::JSON:
-                printer = support::cpp14::make_unique<framework::JSONPrinter>();
-                break;
-            case framework::LogFormat::NONE:
-                break;
-            case framework::LogFormat::PRETTY:
-            default:
-                auto pretty_printer = support::cpp14::make_unique<framework::PrettyPrinter>();
-                // Don't use colours if we print to a file:
-                pretty_printer->set_color_output((!log_file->is_set()) && color_output->value());
-                printer = std::move(pretty_printer);
-                break;
-        }
-
-        if(log_file->is_set())
-        {
-            log_streams.push_back(std::make_shared<std::ofstream>(log_file->value()));
-            if(printer != nullptr)
-            {
-                printer->set_stream(*log_streams.back().get());
-            }
-        }
-
-        if(printer != nullptr)
-        {
-            printers.push_back(std::move(printer));
-        }
-
-        if(json_file->is_set())
-        {
-            printers.push_back(support::cpp14::make_unique<framework::JSONPrinter>());
-            log_streams.push_back(std::make_shared<std::ofstream>(json_file->value()));
-            printers.back()->set_stream(*log_streams.back().get());
-        }
-
-        if(pretty_file->is_set())
-        {
-            printers.push_back(support::cpp14::make_unique<framework::PrettyPrinter>());
-            log_streams.push_back(std::make_shared<std::ofstream>(pretty_file->value()));
-            printers.back()->set_stream(*log_streams.back().get());
-        }
+        std::vector<std::unique_ptr<framework::Printer>> printers = options.create_printers();
 
-        Scheduler::get().set_num_threads(threads->value());
+        Scheduler::get().set_num_threads(options.threads->value());
 
-        if(log_level->value() > framework::LogLevel::NONE)
+        if(options.log_level->value() > framework::LogLevel::NONE)
         {
             for(auto &p : printers)
             {
@@ -227,13 +124,13 @@ int main(int argc, char **argv)
             }
         }
 
-        if(log_level->value() >= framework::LogLevel::CONFIG)
+        if(options.log_level->value() >= framework::LogLevel::CONFIG)
         {
             for(auto &p : printers)
             {
                 p->print_entry("Seed", support::cpp11::to_string(seed->value()));
-                p->print_entry("Iterations", support::cpp11::to_string(iterations->value()));
-                p->print_entry("Threads", support::cpp11::to_string(threads->value()));
+                p->print_entry("Iterations", support::cpp11::to_string(options.iterations->value()));
+                p->print_entry("Threads", support::cpp11::to_string(options.threads->value()));
                 {
                     using support::cpp11::to_string;
                     p->print_entry("Dataset mode", to_string(dataset_mode->value()));
@@ -241,12 +138,12 @@ int main(int argc, char **argv)
             }
         }
 
-        framework.init(instruments->value(), iterations->value(), dataset_mode->value(), filter->value(), filter_id->value(), log_level->value());
+        framework.init(options.instruments->value(), options.iterations->value(), dataset_mode->value(), filter->value(), filter_id->value(), options.log_level->value());
         for(auto &p : printers)
         {
             framework.add_printer(p.get());
         }
-        framework.set_throw_errors(throw_errors->value());
+        framework.set_throw_errors(options.throw_errors->value());
         framework.set_stop_on_error(stop_on_error->value());
         framework.set_error_on_missing_assets(error_on_missing_assets->value());
 
@@ -285,7 +182,7 @@ int main(int argc, char **argv)
 
         success = framework.run();
 
-        if(log_level->value() > framework::LogLevel::NONE)
+        if(options.log_level->value() > framework::LogLevel::NONE)
         {
             for(auto &p : printers)
             {
@@ -299,7 +196,7 @@ int main(int argc, char **argv)
     {
         std::cerr << error.what() << "\n";
 
-        if(throw_errors->value())
+        if(options.throw_errors->value())
         {
             throw;
         }