From 836b27bd73d62795e82d0ce666d728c94c216067 Mon Sep 17 00:00:00 2001 From: Derek Lamberti Date: Wed, 20 Nov 2019 10:51:57 +0000 Subject: IVGCVSW-4157 Pass custom options directly to backends Change-Id: I98cfb913dbd00cb94bdb5dbe82753ca147f7f671 Signed-off-by: Derek Lamberti --- CMakeLists.txt | 1 + include/armnn/BackendId.hpp | 8 +- include/armnn/BackendOptions.hpp | 263 +++++++++++++++++++++++++++++++++++ include/armnn/IRuntime.hpp | 33 ++++- src/armnn/test/RuntimeTests.cpp | 51 ++++++- src/backends/cl/ClBackendContext.cpp | 186 ++++++++++++++++++++++++- src/backends/cl/ClBackendContext.hpp | 4 + src/backends/cl/ClContextControl.cpp | 43 ++---- src/backends/cl/ClContextControl.hpp | 9 +- 9 files changed, 546 insertions(+), 52 deletions(-) create mode 100644 include/armnn/BackendOptions.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f2099e06c..8f6d794c34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -210,6 +210,7 @@ list(APPEND armnn_sources include/armnn/ArmNN.hpp include/armnn/BackendHelper.hpp include/armnn/BackendId.hpp + include/armnn/BackendOptions.hpp include/armnn/BackendRegistry.hpp include/armnn/Conversion.hpp include/armnn/Deprecated.hpp diff --git a/include/armnn/BackendId.hpp b/include/armnn/BackendId.hpp index 00ece377d4..d7b54979b6 100644 --- a/include/armnn/BackendId.hpp +++ b/include/armnn/BackendId.hpp @@ -79,12 +79,18 @@ public: BackendId(const std::string& id) : m_Id{id} {} BackendId(const char* id) : m_Id{id} {} + + BackendId(const BackendId& other) = default; + BackendId(BackendId&& other) = default; + BackendId& operator=(const BackendId& other) = default; + BackendId& operator=(BackendId&& other) = default; + ~BackendId(){} + /// Deprecated function that will be removed together with /// the Compute enum BackendId(Compute compute) : m_Id{GetComputeDeviceAsCString(compute)} {} operator std::string() const { return m_Id; } - BackendId& operator=(const std::string& other) { m_Id = other; diff --git a/include/armnn/BackendOptions.hpp b/include/armnn/BackendOptions.hpp new file mode 100644 index 0000000000..a1b6b09cad --- /dev/null +++ b/include/armnn/BackendOptions.hpp @@ -0,0 +1,263 @@ +// +// Copyright © 2019 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include "BackendId.hpp" +#include + +namespace armnn +{ + + +/// Struct for the users to pass backend specific options +struct BackendOptions +{ +private: + template + struct CheckAllowed + { + static const bool value = std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value; + }; +public: + + // Very basic type safe variant + class Var + { + + public: + /// Constructors + explicit Var(int i) : m_Vals(i), m_Type(VarTypes::Integer) {}; + explicit Var(float f) : m_Vals(f), m_Type(VarTypes::Float) {}; + explicit Var(bool b) : m_Vals(b), m_Type(VarTypes::Boolean) {}; + explicit Var(const char* s) : m_Vals(s), m_Type(VarTypes::String) {}; + explicit Var(std::string s) : m_Vals(s), m_Type(VarTypes::String) {}; + + //Disallow implicit conversions from types not explicitly allowed below. + template + Var(DisallowedType) + { + static_assert(CheckAllowed::value, "Type is not allowed for Var."); + assert(false && "Unreachable code"); + } + + /// Copy Construct + Var(const Var& other) + : m_Type(other.m_Type) + { + switch(m_Type) + { + case VarTypes::String: + { + new (&m_Vals.s) std::string(other.m_Vals.s); + break; + } + default: + { + DoOp(other, [](auto& a, auto& b) + { + a = b; + }); + break; + } + } + } + + /// Copy operator + Var& operator=(const Var& other) + { + // Destroy existing string + if (m_Type == VarTypes::String) + { + Destruct(m_Vals.s); + } + + m_Type = other.m_Type; + switch(m_Type) + { + case VarTypes::String: + { + + new (&m_Vals.s) std::string(other.m_Vals.s); + break; + } + default: + { + DoOp(other, [](auto& a, auto& b) + { + a = b; + }); + break; + } + } + + return *this; + }; + + /// Type getters + bool IsBool() const { return m_Type == VarTypes::Boolean; } + bool IsInt() const { return m_Type == VarTypes::Integer; } + bool IsFloat() const { return m_Type == VarTypes::Float; } + bool IsString() const { return m_Type == VarTypes::String; } + + /// Value getters + bool AsBool() const { assert(IsBool()); return m_Vals.b; } + int AsInt() const { assert(IsInt()); return m_Vals.i; } + float AsFloat() const { assert(IsFloat()); return m_Vals.f; } + std::string AsString() const { assert(IsString()); return m_Vals.s; } + + /// Destructor + ~Var() + { + DoOp(*this, [this](auto& a, auto&) + { + Destruct(a); + }); + } + private: + template + void DoOp(const Var& other, Func func) + { + if (other.IsBool()) + { + func(m_Vals.b, other.m_Vals.b); + } + else if (other.IsInt()) + { + func(m_Vals.i, other.m_Vals.i); + } + else if (other.IsFloat()) + { + func(m_Vals.f, other.m_Vals.f); + } + else if (other.IsString()) + { + func(m_Vals.s, other.m_Vals.s); + } + } + + template + void Destruct(Destructable& d) + { + if (std::is_destructible::value) + { + d.~Destructable(); + } + } + + private: + /// Types which can be stored + enum class VarTypes + { + Boolean, + Integer, + Float, + String, + }; + + // Union of potential type values. + union Vals + { + int i; + float f; + bool b; + std::string s; + + Vals(){} + ~Vals(){} + + explicit Vals(int i) : i(i) {}; + explicit Vals(float f) : f(f) {}; + explicit Vals(bool b) : b(b) {}; + explicit Vals(const char* s) : s(std::string(s)) {} + explicit Vals(std::string s) : s(s) {} + }; + + Vals m_Vals; + VarTypes m_Type; + }; + + struct BackendOption + { + public: + BackendOption(std::string name, bool value) + : m_Name(name), m_Value(value) + {} + BackendOption(std::string name, int value) + : m_Name(name), m_Value(value) + {} + BackendOption(std::string name, float value) + : m_Name(name), m_Value(value) + {} + BackendOption(std::string name, std::string value) + : m_Name(name), m_Value(value) + {} + BackendOption(std::string name, const char* value) + : m_Name(name), m_Value(value) + {} + + template + BackendOption(std::string, DisallowedType) + : m_Value(0) + { + static_assert(CheckAllowed::value, "Type is not allowed for BackendOption."); + assert(false && "Unreachable code"); + } + + BackendOption(const BackendOption& other) = default; + BackendOption(BackendOption&& other) = default; + BackendOption& operator=(const BackendOption& other) = default; + BackendOption& operator=(BackendOption&& other) = default; + ~BackendOption() = default; + + std::string GetName() const { return m_Name; } + Var GetValue() const { return m_Value; } + + private: + std::string m_Name; ///< Name of the option + Var m_Value; ///< Value of the option. (Bool, int, Float, String) + }; + + explicit BackendOptions(BackendId backend) + : m_TargetBackend(backend) + {} + + BackendOptions(BackendId backend, std::initializer_list options) + : m_TargetBackend(backend) + , m_Options(options) + {} + + BackendOptions(const BackendOptions& other) = default; + BackendOptions(BackendOptions&& other) = default; + BackendOptions& operator=(const BackendOptions& other) = default; + BackendOptions& operator=(BackendOptions&& other) = default; + + void AddOption(BackendOption&& option) + { + m_Options.push_back(option); + } + + void AddOption(const BackendOption& option) + { + m_Options.push_back(option); + } + + const BackendId& GetBackendId() const noexcept { return m_TargetBackend; } + size_t GetOptionCount() const noexcept { return m_Options.size(); } + const BackendOption& GetOption(size_t idx) const { return m_Options[idx]; } + +private: + /// The id for the backend to which the options should be passed. + BackendId m_TargetBackend; + + /// The array of options to pass to the backend context + std::vector m_Options; +}; + +} //namespace armnn diff --git a/include/armnn/IRuntime.hpp b/include/armnn/IRuntime.hpp index 08db22e4bb..49c18113b3 100644 --- a/include/armnn/IRuntime.hpp +++ b/include/armnn/IRuntime.hpp @@ -4,7 +4,7 @@ // #pragma once - +#include "BackendOptions.hpp" #include "INetwork.hpp" #include "IProfiler.hpp" #include "Tensor.hpp" @@ -73,8 +73,35 @@ public: bool m_FileOnly; uint32_t m_CapturePeriod; }; - ExternalProfilingOptions m_ProfilingOptions; + + /// Pass backend specific options. + /// + /// For example, to enable GpuAcc tuning add the following + /// m_BackendOption.emplace_back( + /// BackendOptions{"GpuAcc", + /// { + /// {"TuningLevel", 2}, + /// {"TuningFile", filename} + /// } + /// }); + /// Execute representative workloads through the runtime to generate tuning data. + /// The tuning file is written once the runtime is destroyed + + /// To execute with the tuning data, start up with just the tuning file specified. + /// m_BackendOption.emplace_back( + /// BackendOptions{"GpuAcc", + /// { + /// {"TuningFile", filename} + /// } + /// }); + + /// The following backend options are available: + /// GpuAcc: + /// "TuningLevel" : int [0..3] (0=UseOnly(default) | 1=RapidTuning | 2=NormalTuning | 3=ExhaustiveTuning) + /// "TuningFile" : string [filenameString] + /// "KernelProfilingEnabled" : bool [true | false] + std::vector m_BackendOptions; }; static IRuntime* CreateRaw(const CreationOptions& options); @@ -134,6 +161,8 @@ protected: ~IRuntime() {} }; + +/// The following API is replaced by the backend options API. using IGpuAccTunedParametersPtr = std::shared_ptr; /// Manages a set of GpuAcc parameters which have been tuned for maximum performance. diff --git a/src/armnn/test/RuntimeTests.cpp b/src/armnn/test/RuntimeTests.cpp index 4fd847a577..642f334575 100644 --- a/src/armnn/test/RuntimeTests.cpp +++ b/src/armnn/test/RuntimeTests.cpp @@ -288,6 +288,55 @@ BOOST_AUTO_TEST_CASE(IVGCVSW_1929_QuantizedSoftmaxIssue) BOOST_TEST(!optNet); } +BOOST_AUTO_TEST_CASE(RuntimeBackendOptions) +{ + using namespace armnn; + + IRuntime::CreationOptions creationOptions; + auto& backendOptions = creationOptions.m_BackendOptions; + + + // Define Options on explicit construction + BackendOptions options1("FakeBackend1", + { + {"Option1", 1.3f}, + {"Option2", true} + }); + + // Add an option after construction + options1.AddOption({"Option3", "some_value"}); + + // Add the options to CreationOptions struct + backendOptions.push_back(options1); + + // Add more Options via inplace explicit construction + backendOptions.emplace_back( + BackendOptions{"FakeBackend1", + {{"Option4", 42}} + }); + + + // First group + BOOST_TEST(backendOptions[0].GetBackendId().Get() == "FakeBackend1"); + BOOST_TEST(backendOptions[0].GetOption(0).GetName() == "Option1"); + BOOST_TEST(backendOptions[0].GetOption(0).GetValue().IsFloat() == true); + BOOST_TEST(backendOptions[0].GetOption(0).GetValue().AsFloat() == 1.3f); + + BOOST_TEST(backendOptions[0].GetOption(1).GetName() == "Option2"); + BOOST_TEST(backendOptions[0].GetOption(1).GetValue().IsBool() == true); + BOOST_TEST(backendOptions[0].GetOption(1).GetValue().AsBool() == true); + + BOOST_TEST(backendOptions[0].GetOption(2).GetName() == "Option3"); + BOOST_TEST(backendOptions[0].GetOption(2).GetValue().IsString() == true); + BOOST_TEST(backendOptions[0].GetOption(2).GetValue().AsString() == "some_value"); + + // Second group + BOOST_TEST(backendOptions[1].GetBackendId().Get() == "FakeBackend1"); + BOOST_TEST(backendOptions[1].GetOption(0).GetName() == "Option4"); + BOOST_TEST(backendOptions[1].GetOption(0).GetValue().IsInt() == true); + BOOST_TEST(backendOptions[1].GetOption(0).GetValue().AsInt() == 42); +} + BOOST_AUTO_TEST_CASE(ProfilingDisable) { using namespace armnn; @@ -635,7 +684,7 @@ BOOST_AUTO_TEST_CASE(ProfilingEnableCpuRef) LabelsAndEventClasses::TYPE_GUID, readableData, offset); - + bufferManager.MarkRead(readableBuffer); // Creates structures for input & output. diff --git a/src/backends/cl/ClBackendContext.cpp b/src/backends/cl/ClBackendContext.cpp index a82391cce5..b435c29323 100644 --- a/src/backends/cl/ClBackendContext.cpp +++ b/src/backends/cl/ClBackendContext.cpp @@ -4,23 +4,25 @@ // #include "ClBackendContext.hpp" +#include "ClContextControl.hpp" #include -#include "ClContextControl.hpp" - #include #include #include +#include + +#include namespace armnn { struct ClBackendContext::ClContextControlWrapper { - ClContextControlWrapper(IGpuAccTunedParameters* clTunedParameters, + ClContextControlWrapper(arm_compute::CLTuner* tuner, bool profilingEnabled) - : m_ClContextControl(clTunedParameters, profilingEnabled) + : m_ClContextControl(tuner, profilingEnabled) {} bool Sync() @@ -56,13 +58,172 @@ struct ClBackendContext::ClContextControlWrapper ClContextControl m_ClContextControl; }; +std::string LowerString(std::string value) +{ + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char c){ return std::tolower(c); }); + + return value; +} + +enum class TuningLevel +{ + None, + Rapid, + Normal, + Exhaustive +}; + + +TuningLevel ParseTuningLevel(const BackendOptions::Var& value, TuningLevel defaultValue) +{ + if (value.IsInt()) + { + int v = value.IsInt(); + if (v > static_cast(TuningLevel::Exhaustive) || + v < static_cast(TuningLevel::None)) + { + ARMNN_LOG(warning) << "Invalid GpuAcc tuning level ("<< v << ") selected. " + "Using default(" << static_cast(defaultValue) << ")"; + } else + { + return static_cast(v); + } + } + return defaultValue; +} + +bool ParseBoolean(const BackendOptions::Var& value, bool defaultValue) +{ + if (value.IsBool()) + { + return value.AsBool(); + } + + return defaultValue; +} + +std::string ParseFile(const BackendOptions::Var& value, std::string defaultValue) +{ + if (value.IsString()) + { + return value.AsString(); + } + return defaultValue; +} + +template +void ParseOptions(const std::vector& options, BackendId backend, F f) +{ + for (auto optionsGroup : options) + { + if (optionsGroup.GetBackendId() == backend) + { + for (size_t i=0; i < optionsGroup.GetOptionCount(); i++) + { + const BackendOptions::BackendOption option = optionsGroup.GetOption(i); + f(option.GetName(), option.GetValue()); + } + } + } +} ClBackendContext::ClBackendContext(const IRuntime::CreationOptions& options) : IBackendContext(options) - , m_ClContextControlWrapper( - std::make_unique(options.m_GpuAccTunedParameters.get(), - options.m_EnableGpuProfiling)) { + bool kernelProfiling = options.m_EnableGpuProfiling; + const TuningLevel defaultTuningLevel = TuningLevel::None; + auto tuningLevel = defaultTuningLevel; + m_TuningFile = ""; + + + arm_compute::CLTuner* tuner = nullptr; + if (m_TuningFile.empty() == false) + { + bool useLegacyTunerAPI = options.m_GpuAccTunedParameters.get() != nullptr; + if (useLegacyTunerAPI) + { + auto clTunerParams = boost::polymorphic_downcast( + options.m_GpuAccTunedParameters.get()); + auto clTuner = &clTunerParams->m_Tuner; + + if (clTuner) + { + auto ConvertTuningLevel = [](IGpuAccTunedParameters::TuningLevel level) + { + switch(level) + { + case IGpuAccTunedParameters::TuningLevel::Rapid: + return arm_compute::CLTunerMode::RAPID; + case IGpuAccTunedParameters::TuningLevel::Normal: + return arm_compute::CLTunerMode::NORMAL; + case IGpuAccTunedParameters::TuningLevel::Exhaustive: + return arm_compute::CLTunerMode::EXHAUSTIVE; + default: + { + BOOST_ASSERT_MSG(false, "Tuning level not recognised."); + return arm_compute::CLTunerMode::NORMAL; + } + } + }; + + clTuner->set_tuner_mode(ConvertTuningLevel(clTunerParams->m_TuningLevel)); + clTuner->set_tune_new_kernels( + clTunerParams->m_Mode == armnn::IGpuAccTunedParameters::Mode::UpdateTunedParameters); + } + } + else //New backend options API + { + ParseOptions(options.m_BackendOptions, "GpuAcc", [&](std::string name, const BackendOptions::Var& value) + { + if (name == "KernelProfilingEnabled") + { + kernelProfiling |= ParseBoolean(value, false); + } else if (name == "TuningFile") + { + m_TuningFile = ParseFile(value, ""); + } else if (name == "TuningLevel") + { + tuningLevel = ParseTuningLevel(value, defaultTuningLevel); + } + }); + + // Create the tuner, in tuning mode initially. + m_Tuner = std::make_unique(true); + + switch (tuningLevel) + { + case TuningLevel::Rapid: + m_Tuner->set_tuner_mode(arm_compute::CLTunerMode::RAPID); + break; + case TuningLevel::Normal: + m_Tuner->set_tuner_mode(arm_compute::CLTunerMode::NORMAL); + break; + case TuningLevel::Exhaustive: + m_Tuner->set_tuner_mode(arm_compute::CLTunerMode::EXHAUSTIVE); + break; + case TuningLevel::None: + default: + m_Tuner->set_tune_new_kernels(false); // Turn of tuning. Set to "use" only mode. + break; + } + + try + { + m_Tuner->load_from_file(m_TuningFile.c_str()); + } catch (const std::exception& e) + { + ARMNN_LOG(warning) << "Could not load GpuAcc tuner data file."; + } + + tuner = m_Tuner.get(); + } + } + + m_ClContextControlWrapper = std::make_unique( + tuner, + kernelProfiling + ); } bool ClBackendContext::BeforeLoadNetwork(NetworkId) @@ -103,6 +264,17 @@ bool ClBackendContext::AfterUnloadNetwork(NetworkId networkId) ClBackendContext::~ClBackendContext() { + if (m_Tuner && !m_TuningFile.empty()) + { + try + { + m_Tuner->save_to_file(m_TuningFile.c_str()); + } + catch(const std::exception& e) + { + ARMNN_LOG(warning) << "Could not save GpuAcc tuner data to file " << m_TuningFile; + } + } } } // namespace armnn \ No newline at end of file diff --git a/src/backends/cl/ClBackendContext.hpp b/src/backends/cl/ClBackendContext.hpp index 8d4960214f..bcac0d245e 100644 --- a/src/backends/cl/ClBackendContext.hpp +++ b/src/backends/cl/ClBackendContext.hpp @@ -8,6 +8,8 @@ #include #include +#include + namespace armnn { @@ -31,6 +33,8 @@ private: std::unordered_set m_NetworkIds; + std::unique_ptr m_Tuner; + std::string m_TuningFile; }; } // namespace armnn \ No newline at end of file diff --git a/src/backends/cl/ClContextControl.cpp b/src/backends/cl/ClContextControl.cpp index cf5ae64c78..72c8e9fe45 100644 --- a/src/backends/cl/ClContextControl.cpp +++ b/src/backends/cl/ClContextControl.cpp @@ -27,9 +27,9 @@ class Device; namespace armnn { -ClContextControl::ClContextControl(IGpuAccTunedParameters* clTunedParameters, +ClContextControl::ClContextControl(arm_compute::CLTuner *tuner, bool profilingEnabled) - : m_clTunedParameters(boost::polymorphic_downcast(clTunedParameters)) + : m_Tuner(tuner) , m_ProfilingEnabled(profilingEnabled) { // Ignore m_ProfilingEnabled if unused to avoid compiling problems when ArmCompute is disabled. @@ -97,7 +97,7 @@ void ClContextControl::UnloadOpenClRuntime() DoLoadOpenClRuntime(false); } -void ClContextControl::DoLoadOpenClRuntime(bool useTunedParameters) +void ClContextControl::DoLoadOpenClRuntime(bool updateTunedParameters) { cl::Device device = cl::Device::getDefault(); cl::Context context; @@ -133,8 +133,8 @@ void ClContextControl::DoLoadOpenClRuntime(bool useTunedParameters) // NOTE: In this specific case profiling has to be enabled on the command queue // in order for the CLTuner to work. - bool profilingNeededForClTuner = useTunedParameters && m_clTunedParameters && - m_clTunedParameters->m_Mode == IGpuAccTunedParameters::Mode::UpdateTunedParameters; + bool profilingNeededForClTuner = updateTunedParameters && m_Tuner && + m_Tuner->tune_new_kernels(); if (m_ProfilingEnabled || profilingNeededForClTuner) { @@ -156,34 +156,7 @@ void ClContextControl::DoLoadOpenClRuntime(bool useTunedParameters) // Note the first argument (path to cl source code) will be ignored as they should be embedded in the armcompute. arm_compute::CLKernelLibrary::get().init(".", context, device); - - arm_compute::ICLTuner* tuner = nullptr; - if (useTunedParameters && m_clTunedParameters) - { - tuner = &m_clTunedParameters->m_Tuner; - auto clTuner = boost::polymorphic_downcast(tuner); - - auto ConvertTuningLevel = [](IGpuAccTunedParameters::TuningLevel level) - { - switch(level) - { - case IGpuAccTunedParameters::TuningLevel::Rapid: - return arm_compute::CLTunerMode::RAPID; - case IGpuAccTunedParameters::TuningLevel::Normal: - return arm_compute::CLTunerMode::NORMAL; - case IGpuAccTunedParameters::TuningLevel::Exhaustive: - return arm_compute::CLTunerMode::EXHAUSTIVE; - default: - { - BOOST_ASSERT_MSG(false, "Tuning level not recognised."); - return arm_compute::CLTunerMode::NORMAL; - } - } - }; - - clTuner->set_tuner_mode(ConvertTuningLevel(m_clTunedParameters->m_TuningLevel)); - } - arm_compute::CLScheduler::get().init(context, commandQueue, device, tuner); + arm_compute::CLScheduler::get().init(context, commandQueue, device, m_Tuner); } void ClContextControl::ClearClCache() @@ -225,7 +198,7 @@ void ClTunedParameters::Load(const char* filename) catch (const std::exception& e) { throw armnn::Exception(std::string("Failed to load tuned parameters file '") + filename + "': " + - e.what()); + e.what()); } } @@ -238,7 +211,7 @@ void ClTunedParameters::Save(const char* filename) const catch (const std::exception& e) { throw armnn::Exception(std::string("Failed to save tuned parameters file to '") + filename + "': " + - e.what()); + e.what()); } } diff --git a/src/backends/cl/ClContextControl.hpp b/src/backends/cl/ClContextControl.hpp index 8a5abf7bb2..fd27ced1f9 100644 --- a/src/backends/cl/ClContextControl.hpp +++ b/src/backends/cl/ClContextControl.hpp @@ -11,15 +11,12 @@ namespace armnn { -class IGpuAccTunedParameters; -class ClTunedParameters; - // ARM Compute OpenCL context control. class ClContextControl { public: - ClContextControl(IGpuAccTunedParameters* clTunedParameters = nullptr, + ClContextControl(arm_compute::CLTuner* = nullptr, bool profilingEnabled = false); virtual ~ClContextControl(); @@ -35,9 +32,9 @@ public: private: - void DoLoadOpenClRuntime(bool useTunedParameters); + void DoLoadOpenClRuntime(bool updateTunedParameters); - ClTunedParameters* m_clTunedParameters; + arm_compute::CLTuner* m_Tuner; bool m_ProfilingEnabled; }; -- cgit v1.2.1