From 59fd7a722e5bc7e85309d6200bc37a772721a719 Mon Sep 17 00:00:00 2001 From: Freddie Liardet Date: Thu, 17 Jun 2021 13:30:11 +0100 Subject: Add layer data to JSON output Add layer data information to SchedulerTimer JSON output. Resolves: COMPMID-4423 Change-Id: Ife78dee8afc0910cf47b135bc6809cc170ec4ed3 Signed-off-by: Freddie Liardet Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/5923 Reviewed-by: Georgios Pinitas Comments-Addressed: Georgios Pinitas Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins --- src/graph/DataLayerVisitor.cpp | 2 - tests/framework/Framework.cpp | 5 ++- tests/framework/Profiler.cpp | 9 ++++- tests/framework/Profiler.h | 9 ++++- tests/framework/TestResult.h | 3 +- tests/framework/instruments/Instrument.h | 11 +++++- tests/framework/instruments/Measurement.h | 10 ++--- tests/framework/instruments/SchedulerTimer.cpp | 52 +++++++++++++++++++++++--- tests/framework/instruments/SchedulerTimer.h | 5 ++- tests/framework/printers/JSONPrinter.cpp | 9 ++++- tests/framework/printers/JSONPrinter.h | 3 +- tests/framework/printers/PrettyPrinter.cpp | 8 +++- tests/framework/printers/PrettyPrinter.h | 3 +- tests/framework/printers/Printer.h | 8 +++- 14 files changed, 113 insertions(+), 24 deletions(-) diff --git a/src/graph/DataLayerVisitor.cpp b/src/graph/DataLayerVisitor.cpp index 81bb8e523f..3d5d9578ca 100644 --- a/src/graph/DataLayerVisitor.cpp +++ b/src/graph/DataLayerVisitor.cpp @@ -91,8 +91,6 @@ void add_convolution_layer_method(DataLayerVisitor::LayerData &layer_data, T &no template void add_generic_layer_data(DataLayerVisitor::LayerData &layer_data, T &node) { - // Add layer name - layer_data["layer_name"] = node.name(); // Loop over each input tensor for(size_t tensor_no = 0; tensor_no < node.num_inputs(); ++tensor_no) { diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp index 436aac0a34..e59c5a45e9 100644 --- a/tests/framework/Framework.cpp +++ b/tests/framework/Framework.cpp @@ -209,6 +209,7 @@ void Framework::log_test_end(const TestInfo &info) { func_on_all_printers([&](Printer * p) { + p->print_profiler_header(_test_results.at(info).header_data); p->print_measurements(_test_results.at(info).measurements); }); } @@ -532,6 +533,7 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) } } + result.header_data = profiler.header(); result.measurements = profiler.measurements(); set_test_result(info, result); @@ -630,6 +632,7 @@ void Framework::print_test_results(Printer &printer) const for(const auto &test : _test_results) { printer.print_test_header(test.first); + printer.print_profiler_header(test.second.header_data); printer.print_measurements(test.second.measurements); printer.print_test_footer(); } @@ -679,7 +682,7 @@ std::vector Framework::test_infos() const for(const auto &factory : _test_factories) { - TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() }; + const TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() }; if(_test_filter->is_selected(test_info)) { diff --git a/tests/framework/Profiler.cpp b/tests/framework/Profiler.cpp index b527eb4f09..a4a9beaa29 100644 --- a/tests/framework/Profiler.cpp +++ b/tests/framework/Profiler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018 Arm Limited. + * Copyright (c) 2017-2018,2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -81,6 +81,8 @@ void Profiler::test_stop() { _measurements[instrument->id() + "/" + measurement.first].push_back(measurement.second); } + + _header_data = instrument->instrument_header(); } } @@ -88,6 +90,11 @@ const Profiler::MeasurementsMap &Profiler::measurements() const { return _measurements; } + +const std::string &Profiler::header() const +{ + return _header_data; +} } // namespace framework } // namespace test } // namespace arm_compute diff --git a/tests/framework/Profiler.h b/tests/framework/Profiler.h index 588276fcd5..7df085c8f1 100644 --- a/tests/framework/Profiler.h +++ b/tests/framework/Profiler.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018 Arm Limited. + * Copyright (c) 2017-2018,2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -87,9 +87,16 @@ public: */ const MeasurementsMap &measurements() const; + /** Return JSON formatted header data. + * + * @returns JSON formmated string + */ + const std::string &header() const; + private: std::vector> _instruments{}; MeasurementsMap _measurements{}; + std::string _header_data{}; }; } // namespace framework } // namespace test diff --git a/tests/framework/TestResult.h b/tests/framework/TestResult.h index 10f10c1d86..18e54343dc 100644 --- a/tests/framework/TestResult.h +++ b/tests/framework/TestResult.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018 Arm Limited. + * Copyright (c) 2017-2018,2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -73,6 +73,7 @@ struct TestResult Status status{ Status::NOT_RUN }; /**< Execution status */ Profiler::MeasurementsMap measurements{}; /**< Profiling information */ + std::string header_data{}; /**< Test header data */ }; } // namespace framework } // namespace test diff --git a/tests/framework/instruments/Instrument.h b/tests/framework/instruments/Instrument.h index 3ea15825ad..1770a492ac 100644 --- a/tests/framework/instruments/Instrument.h +++ b/tests/framework/instruments/Instrument.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -117,6 +117,15 @@ public: return MeasurementsMap(); } + /** Return JSON formatted instrument header string. + * + * @return JSON formatted string + */ + virtual std::string instrument_header() const + { + return std::string{}; + } + /** Return the latest test measurements. * * @return the latest test measurements. diff --git a/tests/framework/instruments/Measurement.h b/tests/framework/instruments/Measurement.h index af272a9945..2ec68d424b 100644 --- a/tests/framework/instruments/Measurement.h +++ b/tests/framework/instruments/Measurement.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018 Arm Limited. + * Copyright (c) 2017-2018,2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -209,10 +209,10 @@ struct Measurement /** Stored value */ union - { - double floating_point; - long long int integer; - } v; + { + double floating_point; + long long int integer; + } v; bool is_floating_point; /**< Is the stored value floating point or integer ? */ }; diff --git a/tests/framework/instruments/SchedulerTimer.cpp b/tests/framework/instruments/SchedulerTimer.cpp index c31cd42d19..35f960d368 100644 --- a/tests/framework/instruments/SchedulerTimer.cpp +++ b/tests/framework/instruments/SchedulerTimer.cpp @@ -26,6 +26,7 @@ #include "Instruments.h" #include "WallClockTimer.h" #include "arm_compute/core/CPP/ICPPKernel.h" +#include "arm_compute/graph/DataLayerVisitor.h" #include "arm_compute/graph/INode.h" #include "support/Cast.h" @@ -53,8 +54,10 @@ class Interceptor final : public IScheduler { public: /** Default constructor. */ - Interceptor(std::list::kernel_info> &kernels, IScheduler &real_scheduler, ScaleFactor scale_factor) - : _kernels(kernels), _real_scheduler(real_scheduler), _timer(scale_factor), _prefix() + Interceptor(std::list::kernel_info> &kernels, + std::map &layers, IScheduler &real_scheduler, + ScaleFactor scale_factor) + : _kernels(kernels), _layer_data_map(layers), _real_scheduler(real_scheduler), _timer(scale_factor), _prefix() { } @@ -126,6 +129,7 @@ protected: private: std::list::kernel_info> &_kernels; + std::map &_layer_data_map; IScheduler &_real_scheduler; WallClock _timer; std::string _prefix; @@ -133,7 +137,8 @@ private: template SchedulerClock::SchedulerClock(ScaleFactor scale_factor) - : _kernels(), _real_scheduler(nullptr), _real_scheduler_type(), _real_graph_function(nullptr), _scale_factor(scale_factor), _interceptor(nullptr), _scheduler_users() + : _kernels(), _layer_data_map(), _real_scheduler(nullptr), _real_scheduler_type(), _real_graph_function(nullptr), + _scale_factor(scale_factor), _interceptor(nullptr), _scheduler_users() { if(instruments_info != nullptr) { @@ -156,6 +161,13 @@ void SchedulerClock::test_start() if(task.node != nullptr && !task.node->name().empty()) { scheduler->set_prefix(task.node->name() + "/"); + + if(_layer_data_map.find(task.node->name()) == _layer_data_map.end()) + { + arm_compute::graph::DataLayerVisitor dlv = {}; + task.node->accept(dlv); + _layer_data_map[task.node->name()] = dlv.layer_data(); + } } else { @@ -177,7 +189,7 @@ void SchedulerClock::test_start() if(_real_scheduler_type != Scheduler::Type::CUSTOM) { _real_scheduler = &Scheduler::get(); - _interceptor = std::make_shared>(_kernels, *_real_scheduler, _scale_factor); + _interceptor = std::make_shared>(_kernels, _layer_data_map, *_real_scheduler, _scale_factor); Scheduler::set(std::static_pointer_cast(_interceptor)); graph::TaskExecutor::get().execute_function = task_interceptor; @@ -188,7 +200,7 @@ void SchedulerClock::test_start() { if(user != nullptr && user->scheduler() != nullptr) { - user->intercept_scheduler(std::make_unique>(_kernels, *user->scheduler(), _scale_factor)); + user->intercept_scheduler(std::make_unique>(_kernels, _layer_data_map, *user->scheduler(), _scale_factor)); } }); } @@ -257,6 +269,36 @@ Instrument::MeasurementsMap SchedulerClock::measurements() co return measurements; } +template +std::string SchedulerClock::instrument_header() const +{ + std::string output{""}; + output += R"("layer_data" : {)"; + for(auto i_it = _layer_data_map.cbegin(), i_end = _layer_data_map.cend(); i_it != i_end; ++i_it) + { + output += "\"" + i_it->first + "\" : {"; + if(i_it->second.size() != 0) + { + // Print for each entry in layer + for(auto entry_it = i_it->second.cbegin(), entry_end = i_it->second.cend(); entry_it != entry_end; ++entry_it) + { + output += "\"" + entry_it->first + "\" : \"" + entry_it->second + "\""; + if(std::next(entry_it) != entry_end) + { + output += ","; + } + } + } + output += "}"; + if(std::next(i_it) != i_end) + { + output += ","; + } + } + output += "}"; + return output; +} + } // namespace framework } // namespace test } // namespace arm_compute diff --git a/tests/framework/instruments/SchedulerTimer.h b/tests/framework/instruments/SchedulerTimer.h index aa948d32a6..9cc0381a9a 100644 --- a/tests/framework/instruments/SchedulerTimer.h +++ b/tests/framework/instruments/SchedulerTimer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Arm Limited. + * Copyright (c) 2017-2019,2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -63,6 +63,7 @@ template class SchedulerClock : public Instrument { public: + using LayerData = std::map; /** Construct a Scheduler timer. * * @param[in] scale_factor Measurement scale factor. @@ -85,6 +86,7 @@ public: void start() override; void test_stop() override; Instrument::MeasurementsMap measurements() const override; + std::string instrument_header() const override; /** Kernel information */ struct kernel_info @@ -96,6 +98,7 @@ public: private: std::list _kernels; + std::map _layer_data_map; IScheduler *_real_scheduler; Scheduler::Type _real_scheduler_type; std::function _real_graph_function; diff --git a/tests/framework/printers/JSONPrinter.cpp b/tests/framework/printers/JSONPrinter.cpp index 0995ff3594..cf1a4c38cb 100644 --- a/tests/framework/printers/JSONPrinter.cpp +++ b/tests/framework/printers/JSONPrinter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Arm Limited. + * Copyright (c) 2017-2019,2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -169,10 +169,15 @@ void JSONPrinter::print_info(const std::string &info) _infos.push_back(info); } -void JSONPrinter::print_measurements(const Profiler::MeasurementsMap &measurements) +void JSONPrinter::print_profiler_header(const std::string &header_data) { print_separator(_first_test_entry); + *_stream << header_data; +} +void JSONPrinter::print_measurements(const Profiler::MeasurementsMap &measurements) +{ + print_separator(_first_test_entry); *_stream << R"("measurements" : {)"; for(auto i_it = measurements.cbegin(), i_end = measurements.cend(); i_it != i_end;) diff --git a/tests/framework/printers/JSONPrinter.h b/tests/framework/printers/JSONPrinter.h index ce587ade04..ad996708e7 100644 --- a/tests/framework/printers/JSONPrinter.h +++ b/tests/framework/printers/JSONPrinter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Arm Limited. + * Copyright (c) 2017,2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -51,6 +51,7 @@ public: void print_errors_footer() override; void print_error(const std::exception &error, bool expected) override; void print_info(const std::string &info) override; + void print_profiler_header(const std::string &header_data) override; void print_measurements(const Profiler::MeasurementsMap &measurements) override; void print_list_tests(const std::vector &infos) override; diff --git a/tests/framework/printers/PrettyPrinter.cpp b/tests/framework/printers/PrettyPrinter.cpp index aa06eb9b4e..529ff2c2d9 100644 --- a/tests/framework/printers/PrettyPrinter.cpp +++ b/tests/framework/printers/PrettyPrinter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Arm Limited. + * Copyright (c) 2017-2019,2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -116,6 +116,12 @@ void PrettyPrinter::print_list_tests(const std::vector &infos) *_stream << "[" << info.id << ", " << info.mode << ", " << info.status << "] " << info.name << "\n"; } } + +void PrettyPrinter::print_profiler_header(const std::string &header_data) +{ + ARM_COMPUTE_UNUSED(header_data); +} + void PrettyPrinter::print_measurements(const Profiler::MeasurementsMap &measurements) { for(const auto &instrument : measurements) diff --git a/tests/framework/printers/PrettyPrinter.h b/tests/framework/printers/PrettyPrinter.h index ded0da04d8..b9d5d3957c 100644 --- a/tests/framework/printers/PrettyPrinter.h +++ b/tests/framework/printers/PrettyPrinter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Arm Limited. + * Copyright (c) 2017,2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -55,6 +55,7 @@ public: void print_errors_footer() override; void print_error(const std::exception &error, bool expected) override; void print_info(const std::string &info) override; + void print_profiler_header(const std::string &header_data) override; void print_measurements(const Profiler::MeasurementsMap &measurements) override; void print_list_tests(const std::vector &infos) override; diff --git a/tests/framework/printers/Printer.h b/tests/framework/printers/Printer.h index 669b7f6a95..af0209788c 100644 --- a/tests/framework/printers/Printer.h +++ b/tests/framework/printers/Printer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018 Arm Limited. + * Copyright (c) 2017-2018,2021 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -125,6 +125,12 @@ public: */ virtual void print_info(const std::string &info) = 0; + /** Print header data. + * + * @param[in] header_data JSON formmated header data. + */ + virtual void print_profiler_header(const std::string &header_data) = 0; + /** Print measurements for a test. * * @param[in] measurements Measurements as collected by a @ref Profiler. -- cgit v1.2.1