diff options
-rw-r--r-- | delegate/include/DelegateOptions.hpp | 102 | ||||
-rw-r--r-- | delegate/src/DelegateOptions.cpp | 10 | ||||
-rw-r--r-- | delegate/src/armnn_delegate.cpp | 16 | ||||
-rw-r--r-- | delegate/src/armnn_external_delegate.cpp | 16 | ||||
-rw-r--r-- | include/armnn/INetwork.hpp | 14 | ||||
-rw-r--r-- | include/armnn/IRuntime.hpp | 8 | ||||
-rw-r--r-- | include/armnn/Types.hpp | 2 |
7 files changed, 81 insertions, 87 deletions
diff --git a/delegate/include/DelegateOptions.hpp b/delegate/include/DelegateOptions.hpp index b8847440ed..24a2e5c597 100644 --- a/delegate/include/DelegateOptions.hpp +++ b/delegate/include/DelegateOptions.hpp @@ -41,16 +41,34 @@ public: void SetBackends(const std::vector<armnn::BackendId>& backends) { m_Backends = backends; } - void SetDynamicBackendsPath(const std::string& dynamicBackendsPath) { m_DynamicBackendsPath = dynamicBackendsPath; } - const std::string& GetDynamicBackendsPath() const { return m_DynamicBackendsPath; } - - void SetGpuProfilingState(bool gpuProfilingState) { m_EnableGpuProfiling = gpuProfilingState; } - bool GetGpuProfilingState() { return m_EnableGpuProfiling; } - - const std::vector<armnn::BackendOptions>& GetBackendOptions() const { return m_BackendOptions; } + void SetDynamicBackendsPath(const std::string& dynamicBackendsPath) + { + m_RuntimeOptions.m_DynamicBackendsPath = dynamicBackendsPath; + } + const std::string& GetDynamicBackendsPath() const + { + return m_RuntimeOptions.m_DynamicBackendsPath; + } + + void SetGpuProfilingState(bool gpuProfilingState) + { + m_RuntimeOptions.m_EnableGpuProfiling = gpuProfilingState; + } + bool GetGpuProfilingState() + { + return m_RuntimeOptions.m_EnableGpuProfiling; + } + + const std::vector<armnn::BackendOptions>& GetBackendOptions() const + { + return m_RuntimeOptions.m_BackendOptions; + } /// Appends a backend option to the list of backend options - void AddBackendOption(const armnn::BackendOptions& option) { m_BackendOptions.push_back(option); } + void AddBackendOption(const armnn::BackendOptions& option) + { + m_RuntimeOptions.m_BackendOptions.push_back(option); + } /// Sets the severity level for logging within ArmNN that will be used on creation of the delegate void SetLoggingSeverity(const armnn::LogSeverity& level) { m_LoggingSeverity = level; } @@ -85,66 +103,32 @@ public: void SetSerializeToDot(const std::string& serializeToDotFile) { m_SerializeToDot = serializeToDotFile; } const std::string& GetSerializeToDot() const { return m_SerializeToDot; } + /// @Note: This might overwrite options that were set with other setter functions of DelegateOptions + void SetRuntimeOptions(const armnn::IRuntime::CreationOptions& runtimeOptions) + { + m_RuntimeOptions = runtimeOptions; + } + + const armnn::IRuntime::CreationOptions& GetRuntimeOptions() + { + return m_RuntimeOptions; + } + private: /// Which backend to run Delegate on. /// Examples of possible values are: CpuRef, CpuAcc, GpuAcc. /// CpuRef as default. std::vector<armnn::BackendId> m_Backends = { armnn::Compute::CpuRef }; - /// Pass backend specific options to Delegate - /// - /// For example, tuning can be enabled on GpuAcc like below - /// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - /// m_BackendOptions.emplace_back( - /// BackendOptions{"GpuAcc", - /// { - /// {"TuningLevel", 2}, - /// {"TuningFile", filename} - /// } - /// }); - /// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - /// The following backend options are available: - /// GpuAcc: - /// "TuningLevel" : int [0..3] (0=UseOnly(default) | 1=RapidTuning | 2=NormalTuning | 3=ExhaustiveTuning) - /// "TuningFile" : string [filenameString] - /// "KernelProfilingEnabled" : bool [true | false] - std::vector<armnn::BackendOptions> m_BackendOptions; - - /// Dynamic backend path. - /// This is the directory that will be searched for any dynamic backends. - std::string m_DynamicBackendsPath = ""; - - /// Enable Gpu Profiling. - bool m_EnableGpuProfiling = false; - - /// OptimizerOptions - /// Reduce Fp32 data to Fp16 for faster processing - /// bool m_ReduceFp32ToFp16; - /// Add debug data for easier troubleshooting - /// bool m_Debug; - /// Reduce Fp32 data to Bf16 for faster processing - /// bool m_ReduceFp32ToBf16; - /// Enable Import - /// bool m_ImportEnabled; - /// Enable Model Options - /// ModelOptions m_ModelOptions; + /// Creation options for the ArmNN runtime + /// Contains options for global settings that are valid for the whole lifetime of ArmNN + /// i.e. BackendOptions, DynamicBackendPath, ExternalProfilingOptions and more + armnn::IRuntime::CreationOptions m_RuntimeOptions; + + /// Options for the optimization step for the network armnn::OptimizerOptions m_OptimizerOptions; /// External profiling options. - /// Indicates whether external profiling is enabled or not. - /// bool m_EnableProfiling - /// Indicates whether external timeline profiling is enabled or not. - /// bool m_TimelineEnabled - /// Path to a file in which outgoing timeline profiling messages will be stored. - /// std::string m_OutgoingCaptureFile - /// Path to a file in which incoming timeline profiling messages will be stored. - /// std::string m_IncomingCaptureFile - /// Enable profiling output to file only. - /// bool m_FileOnly - /// The duration at which captured profiling messages will be flushed. - /// uint32_t m_CapturePeriod - /// The format of the file used for outputting profiling data. - /// std::string m_FileFormat armnn::IRuntime::CreationOptions::ExternalProfilingOptions m_ProfilingOptions; /// Internal profiling options. diff --git a/delegate/src/DelegateOptions.cpp b/delegate/src/DelegateOptions.cpp index d4d906a83a..c19697d6c6 100644 --- a/delegate/src/DelegateOptions.cpp +++ b/delegate/src/DelegateOptions.cpp @@ -11,15 +11,17 @@ namespace armnnDelegate DelegateOptions::DelegateOptions(armnn::Compute computeDevice, const std::vector<armnn::BackendOptions>& backendOptions, const armnn::Optional<armnn::LogSeverity> logSeverityLevel) - : m_Backends({computeDevice}), m_BackendOptions(backendOptions), m_LoggingSeverity(logSeverityLevel) + : m_Backends({computeDevice}), m_RuntimeOptions(), m_LoggingSeverity(logSeverityLevel) { + m_RuntimeOptions.m_BackendOptions = backendOptions; } DelegateOptions::DelegateOptions(const std::vector<armnn::BackendId>& backends, const std::vector<armnn::BackendOptions>& backendOptions, const armnn::Optional<armnn::LogSeverity> logSeverityLevel) - : m_Backends(backends), m_BackendOptions(backendOptions), m_LoggingSeverity(logSeverityLevel) + : m_Backends(backends), m_RuntimeOptions(), m_LoggingSeverity(logSeverityLevel) { + m_RuntimeOptions.m_BackendOptions = backendOptions; } DelegateOptions::DelegateOptions(armnn::Compute computeDevice, @@ -27,7 +29,7 @@ DelegateOptions::DelegateOptions(armnn::Compute computeDevice, const armnn::Optional<armnn::LogSeverity>& logSeverityLevel, const armnn::Optional<armnn::DebugCallbackFunction>& func) : m_Backends({computeDevice}), - m_BackendOptions({}), + m_RuntimeOptions(), m_OptimizerOptions(optimizerOptions), m_LoggingSeverity(logSeverityLevel), m_DebugCallbackFunc(func) @@ -39,7 +41,7 @@ DelegateOptions::DelegateOptions(const std::vector<armnn::BackendId>& backends, const armnn::Optional<armnn::LogSeverity>& logSeverityLevel, const armnn::Optional<armnn::DebugCallbackFunction>& func) : m_Backends(backends), - m_BackendOptions({}), + m_RuntimeOptions(), m_OptimizerOptions(optimizerOptions), m_LoggingSeverity(logSeverityLevel), m_DebugCallbackFunc(func) diff --git a/delegate/src/armnn_delegate.cpp b/delegate/src/armnn_delegate.cpp index ae25430e0d..966d17ca98 100644 --- a/delegate/src/armnn_delegate.cpp +++ b/delegate/src/armnn_delegate.cpp @@ -137,21 +137,7 @@ Delegate::Delegate(armnnDelegate::DelegateOptions options) } // Create ArmNN Runtime - armnn::IRuntime::CreationOptions runtimeOptions; - runtimeOptions.m_DynamicBackendsPath = m_Options.GetDynamicBackendsPath(); - runtimeOptions.m_EnableGpuProfiling = m_Options.GetGpuProfilingState(); - runtimeOptions.m_ProfilingOptions = m_Options.GetExternalProfilingParams(); - - auto backendOptions = m_Options.GetBackendOptions(); - if (!backendOptions.empty()) - { - runtimeOptions.m_BackendOptions = backendOptions; - } - else if (!m_Options.GetOptimizerOptions().m_ModelOptions.empty()) - { - runtimeOptions.m_BackendOptions = m_Options.GetOptimizerOptions().m_ModelOptions; - } - m_Runtime = armnn::IRuntime::Create(runtimeOptions); + m_Runtime = armnn::IRuntime::Create(options.GetRuntimeOptions()); std::vector<armnn::BackendId> backends; if (m_Runtime) diff --git a/delegate/src/armnn_external_delegate.cpp b/delegate/src/armnn_external_delegate.cpp index 4f583e80d4..5919a6c5ab 100644 --- a/delegate/src/armnn_external_delegate.cpp +++ b/delegate/src/armnn_external_delegate.cpp @@ -171,6 +171,8 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys, { // (Initializes with CpuRef backend) armnnDelegate::DelegateOptions options = armnnDelegate::TfLiteArmnnDelegateOptionsDefault(); + + armnn::IRuntime::CreationOptions runtimeOptions; armnn::OptimizerOptions optimizerOptions; bool internalProfilingState = false; armnn::ProfilingDetailsMethod internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents; @@ -194,7 +196,7 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys, // Process dynamic-backends-path else if (std::string(options_keys[i]) == std::string("dynamic-backends-path")) { - options.SetDynamicBackendsPath(std::string(options_values[i])); + runtimeOptions.m_DynamicBackendsPath = std::string(options_values[i]); } // Process logging level else if (std::string(options_keys[i]) == std::string("logging-severity")) @@ -205,26 +207,26 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys, else if (std::string(options_keys[i]) == std::string("gpu-tuning-level")) { armnn::BackendOptions option("GpuAcc", {{"TuningLevel", atoi(options_values[i])}}); - options.AddBackendOption(option); + runtimeOptions.m_BackendOptions.push_back(option); } else if (std::string(options_keys[i]) == std::string("gpu-mlgo-tuning-file")) { armnn::BackendOptions option("GpuAcc", {{"MLGOTuningFilePath", std::string(options_values[i])}}); - options.AddBackendOption(option); + optimizerOptions.m_ModelOptions.push_back(option); } else if (std::string(options_keys[i]) == std::string("gpu-tuning-file")) { armnn::BackendOptions option("GpuAcc", {{"TuningFile", std::string(options_values[i])}}); - options.AddBackendOption(option); + runtimeOptions.m_BackendOptions.push_back(option); } else if (std::string(options_keys[i]) == std::string("gpu-enable-profiling")) { - options.SetGpuProfilingState(*options_values[i] != '0'); + runtimeOptions.m_EnableGpuProfiling = (*options_values[i] != '0'); } else if (std::string(options_keys[i]) == std::string("gpu-kernel-profiling-enabled")) { armnn::BackendOptions option("GpuAcc", {{"KernelProfilingEnabled", (*options_values[i] != '0')}}); - options.AddBackendOption(option); + runtimeOptions.m_BackendOptions.push_back(option); } else if (std::string(options_keys[i]) == std::string("save-cached-network")) { @@ -340,6 +342,8 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys, throw armnn::Exception("Unknown option for the ArmNN Delegate given: " + std::string(options_keys[i])); } } + + options.SetRuntimeOptions(runtimeOptions); options.SetOptimizerOptions(optimizerOptions); options.SetInternalProfilingParams(internalProfilingState, internalProfilingDetail); options.SetExternalProfilingParams(extProfilingParams); diff --git a/include/armnn/INetwork.hpp b/include/armnn/INetwork.hpp index f85b29ee81..a79afead95 100644 --- a/include/armnn/INetwork.hpp +++ b/include/armnn/INetwork.hpp @@ -115,6 +115,8 @@ protected: }; +/// ArmNN performs an optimization on each model/network before it gets loaded for execution. OptimizerOptions provides +/// a set of features that allows the user to customize this optimization on a per model basis. struct OptimizerOptions { OptimizerOptions() @@ -160,13 +162,21 @@ struct OptimizerOptions } } - // Reduce Fp32 data to Fp16 for faster processing + /// Reduces all Fp32 operators in the model to Fp16 for faster processing. + /// @Note This feature works best if all operators of the model are in Fp32. ArmNN will add conversion layers + /// between layers that weren't in Fp32 in the first place or if the operator is not supported in Fp16. + /// The overhead of these conversions can lead to a slower overall performance if too many conversions are + /// required. bool m_ReduceFp32ToFp16; // Add debug data for easier troubleshooting bool m_Debug; - // Reduce Fp32 data to Bf16 for faster processing + /// Reduces all Fp32 operators in the model to Bf16 for faster processing. + /// @Note This feature works best if all operators of the model are in Fp32. ArmNN will add conversion layers + /// between layers that weren't in Fp32 in the first place or if the operator is not supported in Bf16. + /// The overhead of these conversions can lead to a slower overall performance if too many conversions are + /// required. bool m_ReduceFp32ToBf16; // Infer output size when not available diff --git a/include/armnn/IRuntime.hpp b/include/armnn/IRuntime.hpp index b89cfd72cd..93f8b0fd5b 100644 --- a/include/armnn/IRuntime.hpp +++ b/include/armnn/IRuntime.hpp @@ -119,6 +119,7 @@ public: /// Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive /// Only a single path is allowed for the override + /// It defines the path to search for any [dynamic backend libraries](src/dynamic/README.md). std::string m_DynamicBackendsPath; /// Setting this flag will allow the user to create the Runtime in protected mode. @@ -158,12 +159,19 @@ public: , m_LocalPacketHandlers() {} + /// Indicates whether external profiling is enabled or not. bool m_EnableProfiling; + /// Indicates whether external timeline profiling is enabled or not. bool m_TimelineEnabled; + /// Path to a file in which outgoing timeline profiling messages will be stored. std::string m_OutgoingCaptureFile; + /// Path to a file in which incoming timeline profiling messages will be stored. std::string m_IncomingCaptureFile; + /// Enable profiling output to file only. bool m_FileOnly; + /// The duration at which captured profiling messages will be flushed. uint32_t m_CapturePeriod; + /// The format of the file used for outputting profiling data. std::string m_FileFormat; std::vector<armnn::profiling::ILocalPacketHandlerSharedPtr> m_LocalPacketHandlers; }; diff --git a/include/armnn/Types.hpp b/include/armnn/Types.hpp index deaa0b3a50..f61a1fdca8 100644 --- a/include/armnn/Types.hpp +++ b/include/armnn/Types.hpp @@ -201,7 +201,7 @@ enum class OutputShapeRounding /// The ShapeInferenceMethod modify how the output shapes are treated. /// When ValidateOnly is selected, the output shapes are inferred from the input parameters of the layer /// and any mismatch is reported. -/// When InferAndValidate is selected 2 actions must be performed: (1)infer output shape from inputs and (2)validate the +/// When InferAndValidate is selected 2 actions are performed: (1)infer output shape from inputs and (2)validate the /// shapes as in ValidateOnly. This option has been added to work with tensors which rank or dimension sizes are not /// specified explicitly, however this information can be calculated from the inputs. /// |