Fix armnn_external_delegate option parsing

 * Fixed the mechanism in armnn_delegate.cpp which creates the runtime
   options by moving it to DelegateOptions. BackendOptions are used in
   the runtime options as well as in the optimizer options but need
   to be processed separately.
 * Added runtime options to DelegateOptions to make it easier to differ
   between backend options of the runtime and the optimization
 * Updated armnn_external_delegate to use runtime options
 * Added explanations to OptimizerOptions

Signed-off-by: Jan Eilers <jan.eilers@arm.com>
Change-Id: I34235474147def261821e4576dd41a83ad492258

7 files changed, 81 insertions, 87 deletions

diff --git a/delegate/include/DelegateOptions.hpp b/delegate/include/DelegateOptions.hpp
index b8847440ed..24a2e5c597 100644
--- a/delegate/include/DelegateOptions.hpp
+++ b/delegate/include/DelegateOptions.hpp
@@ -41,16 +41,34 @@ public:
 
     void SetBackends(const std::vector<armnn::BackendId>& backends) { m_Backends = backends; }
 
-    void SetDynamicBackendsPath(const std::string& dynamicBackendsPath) { m_DynamicBackendsPath = dynamicBackendsPath; }
-    const std::string& GetDynamicBackendsPath() const { return m_DynamicBackendsPath; }
-
-    void SetGpuProfilingState(bool gpuProfilingState) { m_EnableGpuProfiling = gpuProfilingState; }
-    bool GetGpuProfilingState() { return m_EnableGpuProfiling; }
-
-    const std::vector<armnn::BackendOptions>& GetBackendOptions() const { return m_BackendOptions; }
+    void SetDynamicBackendsPath(const std::string& dynamicBackendsPath)
+    {
+        m_RuntimeOptions.m_DynamicBackendsPath = dynamicBackendsPath;
+    }
+    const std::string& GetDynamicBackendsPath() const
+    {
+        return m_RuntimeOptions.m_DynamicBackendsPath;
+    }
+
+    void SetGpuProfilingState(bool gpuProfilingState)
+    {
+        m_RuntimeOptions.m_EnableGpuProfiling = gpuProfilingState;
+    }
+    bool GetGpuProfilingState()
+    {
+        return m_RuntimeOptions.m_EnableGpuProfiling;
+    }
+
+    const std::vector<armnn::BackendOptions>& GetBackendOptions() const
+    {
+        return m_RuntimeOptions.m_BackendOptions;
+    }
 
     /// Appends a backend option to the list of backend options
-    void AddBackendOption(const armnn::BackendOptions& option) { m_BackendOptions.push_back(option); }
+    void AddBackendOption(const armnn::BackendOptions& option)
+    {
+        m_RuntimeOptions.m_BackendOptions.push_back(option);
+    }
 
     /// Sets the severity level for logging within ArmNN that will be used on creation of the delegate
     void SetLoggingSeverity(const armnn::LogSeverity& level) { m_LoggingSeverity = level; }
@@ -85,66 +103,32 @@ public:
     void SetSerializeToDot(const std::string& serializeToDotFile) { m_SerializeToDot = serializeToDotFile; }
     const std::string& GetSerializeToDot() const { return m_SerializeToDot; }
 
+    /// @Note: This might overwrite options that were set with other setter functions of DelegateOptions
+    void SetRuntimeOptions(const armnn::IRuntime::CreationOptions& runtimeOptions)
+    {
+        m_RuntimeOptions = runtimeOptions;
+    }
+
+    const armnn::IRuntime::CreationOptions& GetRuntimeOptions()
+    {
+        return m_RuntimeOptions;
+    }
+
 private:
     /// Which backend to run Delegate on.
     /// Examples of possible values are: CpuRef, CpuAcc, GpuAcc.
     /// CpuRef as default.
     std::vector<armnn::BackendId> m_Backends = { armnn::Compute::CpuRef };
 
-    /// Pass backend specific options to Delegate
-    ///
-    /// For example, tuning can be enabled on GpuAcc like below
-    /// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    /// m_BackendOptions.emplace_back(
-    ///     BackendOptions{"GpuAcc",
-    ///       {
-    ///         {"TuningLevel", 2},
-    ///         {"TuningFile", filename}
-    ///       }
-    ///     });
-    /// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    /// The following backend options are available:
-    /// GpuAcc:
-    ///   "TuningLevel" : int [0..3] (0=UseOnly(default) | 1=RapidTuning | 2=NormalTuning | 3=ExhaustiveTuning)
-    ///   "TuningFile" : string [filenameString]
-    ///   "KernelProfilingEnabled" : bool [true | false]
-    std::vector<armnn::BackendOptions> m_BackendOptions;
-
-    /// Dynamic backend path.
-    /// This is the directory that will be searched for any dynamic backends.
-    std::string m_DynamicBackendsPath = "";
-
-    /// Enable Gpu Profiling.
-    bool m_EnableGpuProfiling = false;
-
-    /// OptimizerOptions
-    /// Reduce Fp32 data to Fp16 for faster processing
-    /// bool m_ReduceFp32ToFp16;
-    /// Add debug data for easier troubleshooting
-    /// bool m_Debug;
-    /// Reduce Fp32 data to Bf16 for faster processing
-    /// bool m_ReduceFp32ToBf16;
-    /// Enable Import
-    /// bool m_ImportEnabled;
-    /// Enable Model Options
-    /// ModelOptions m_ModelOptions;
+    /// Creation options for the ArmNN runtime
+    /// Contains options for global settings that are valid for the whole lifetime of ArmNN
+    /// i.e. BackendOptions, DynamicBackendPath, ExternalProfilingOptions and more
+    armnn::IRuntime::CreationOptions m_RuntimeOptions;
+
+    /// Options for the optimization step for the network
     armnn::OptimizerOptions m_OptimizerOptions;
 
     /// External profiling options.
-    /// Indicates whether external profiling is enabled or not.
-    /// bool m_EnableProfiling
-    /// Indicates whether external timeline profiling is enabled or not.
-    /// bool m_TimelineEnabled
-    /// Path to a file in which outgoing timeline profiling messages will be stored.
-    /// std::string m_OutgoingCaptureFile
-    /// Path to a file in which incoming timeline profiling messages will be stored.
-    /// std::string m_IncomingCaptureFile
-    /// Enable profiling output to file only.
-    /// bool m_FileOnly
-    /// The duration at which captured profiling messages will be flushed.
-    /// uint32_t m_CapturePeriod
-    /// The format of the file used for outputting profiling data.
-    /// std::string m_FileFormat
     armnn::IRuntime::CreationOptions::ExternalProfilingOptions m_ProfilingOptions;
 
     /// Internal profiling options.
diff --git a/delegate/src/DelegateOptions.cpp b/delegate/src/DelegateOptions.cpp
index d4d906a83a..c19697d6c6 100644
--- a/delegate/src/DelegateOptions.cpp
+++ b/delegate/src/DelegateOptions.cpp
@@ -11,15 +11,17 @@ namespace armnnDelegate
 DelegateOptions::DelegateOptions(armnn::Compute computeDevice,
                                  const std::vector<armnn::BackendOptions>& backendOptions,
                                  const armnn::Optional<armnn::LogSeverity> logSeverityLevel)
-    : m_Backends({computeDevice}), m_BackendOptions(backendOptions), m_LoggingSeverity(logSeverityLevel)
+    : m_Backends({computeDevice}), m_RuntimeOptions(), m_LoggingSeverity(logSeverityLevel)
 {
+    m_RuntimeOptions.m_BackendOptions = backendOptions;
 }
 
 DelegateOptions::DelegateOptions(const std::vector<armnn::BackendId>& backends,
                                  const std::vector<armnn::BackendOptions>& backendOptions,
                                  const armnn::Optional<armnn::LogSeverity> logSeverityLevel)
-    : m_Backends(backends), m_BackendOptions(backendOptions), m_LoggingSeverity(logSeverityLevel)
+    : m_Backends(backends), m_RuntimeOptions(), m_LoggingSeverity(logSeverityLevel)
 {
+    m_RuntimeOptions.m_BackendOptions = backendOptions;
 }
 
 DelegateOptions::DelegateOptions(armnn::Compute computeDevice,
@@ -27,7 +29,7 @@ DelegateOptions::DelegateOptions(armnn::Compute computeDevice,
                                  const armnn::Optional<armnn::LogSeverity>& logSeverityLevel,
                                  const armnn::Optional<armnn::DebugCallbackFunction>& func)
     : m_Backends({computeDevice}),
-      m_BackendOptions({}),
+      m_RuntimeOptions(),
       m_OptimizerOptions(optimizerOptions),
       m_LoggingSeverity(logSeverityLevel),
       m_DebugCallbackFunc(func)
@@ -39,7 +41,7 @@ DelegateOptions::DelegateOptions(const std::vector<armnn::BackendId>& backends,
                                  const armnn::Optional<armnn::LogSeverity>& logSeverityLevel,
                                  const armnn::Optional<armnn::DebugCallbackFunction>& func)
     : m_Backends(backends),
-      m_BackendOptions({}),
+      m_RuntimeOptions(),
       m_OptimizerOptions(optimizerOptions),
       m_LoggingSeverity(logSeverityLevel),
       m_DebugCallbackFunc(func)
diff --git a/delegate/src/armnn_delegate.cpp b/delegate/src/armnn_delegate.cpp
index ae25430e0d..966d17ca98 100644
--- a/delegate/src/armnn_delegate.cpp
+++ b/delegate/src/armnn_delegate.cpp
@@ -137,21 +137,7 @@ Delegate::Delegate(armnnDelegate::DelegateOptions options)
     }
 
     // Create ArmNN Runtime
-    armnn::IRuntime::CreationOptions runtimeOptions;
-    runtimeOptions.m_DynamicBackendsPath = m_Options.GetDynamicBackendsPath();
-    runtimeOptions.m_EnableGpuProfiling = m_Options.GetGpuProfilingState();
-    runtimeOptions.m_ProfilingOptions = m_Options.GetExternalProfilingParams();
-
-    auto backendOptions = m_Options.GetBackendOptions();
-    if (!backendOptions.empty())
-    {
-        runtimeOptions.m_BackendOptions = backendOptions;
-    }
-    else if (!m_Options.GetOptimizerOptions().m_ModelOptions.empty())
-    {
-        runtimeOptions.m_BackendOptions = m_Options.GetOptimizerOptions().m_ModelOptions;
-    }
-    m_Runtime = armnn::IRuntime::Create(runtimeOptions);
+    m_Runtime = armnn::IRuntime::Create(options.GetRuntimeOptions());
 
     std::vector<armnn::BackendId> backends;
     if (m_Runtime)
diff --git a/delegate/src/armnn_external_delegate.cpp b/delegate/src/armnn_external_delegate.cpp
index 4f583e80d4..5919a6c5ab 100644
--- a/delegate/src/armnn_external_delegate.cpp
+++ b/delegate/src/armnn_external_delegate.cpp
@@ -171,6 +171,8 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
     {
         // (Initializes with CpuRef backend)
         armnnDelegate::DelegateOptions options = armnnDelegate::TfLiteArmnnDelegateOptionsDefault();
+
+        armnn::IRuntime::CreationOptions runtimeOptions;
         armnn::OptimizerOptions optimizerOptions;
         bool internalProfilingState = false;
         armnn::ProfilingDetailsMethod internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents;
@@ -194,7 +196,7 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
             // Process dynamic-backends-path
             else if (std::string(options_keys[i]) == std::string("dynamic-backends-path"))
             {
-                options.SetDynamicBackendsPath(std::string(options_values[i]));
+                runtimeOptions.m_DynamicBackendsPath = std::string(options_values[i]);
             }
             // Process logging level
             else if (std::string(options_keys[i]) == std::string("logging-severity"))
@@ -205,26 +207,26 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
             else if (std::string(options_keys[i]) == std::string("gpu-tuning-level"))
             {
                 armnn::BackendOptions option("GpuAcc", {{"TuningLevel", atoi(options_values[i])}});
-                options.AddBackendOption(option);
+                runtimeOptions.m_BackendOptions.push_back(option);
             }
             else if (std::string(options_keys[i]) == std::string("gpu-mlgo-tuning-file"))
             {
                 armnn::BackendOptions option("GpuAcc", {{"MLGOTuningFilePath", std::string(options_values[i])}});
-                options.AddBackendOption(option);
+                optimizerOptions.m_ModelOptions.push_back(option);
             }
             else if (std::string(options_keys[i]) == std::string("gpu-tuning-file"))
             {
                 armnn::BackendOptions option("GpuAcc", {{"TuningFile", std::string(options_values[i])}});
-                options.AddBackendOption(option);
+                runtimeOptions.m_BackendOptions.push_back(option);
             }
             else if (std::string(options_keys[i]) == std::string("gpu-enable-profiling"))
             {
-                options.SetGpuProfilingState(*options_values[i] != '0');
+                runtimeOptions.m_EnableGpuProfiling = (*options_values[i] != '0');
             }
             else if (std::string(options_keys[i]) == std::string("gpu-kernel-profiling-enabled"))
             {
                 armnn::BackendOptions option("GpuAcc", {{"KernelProfilingEnabled", (*options_values[i] != '0')}});
-                options.AddBackendOption(option);
+                runtimeOptions.m_BackendOptions.push_back(option);
             }
             else if (std::string(options_keys[i]) == std::string("save-cached-network"))
             {
@@ -340,6 +342,8 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
                 throw armnn::Exception("Unknown option for the ArmNN Delegate given: " + std::string(options_keys[i]));
             }
         }
+
+        options.SetRuntimeOptions(runtimeOptions);
         options.SetOptimizerOptions(optimizerOptions);
         options.SetInternalProfilingParams(internalProfilingState, internalProfilingDetail);
         options.SetExternalProfilingParams(extProfilingParams);
diff --git a/include/armnn/INetwork.hpp b/include/armnn/INetwork.hpp
index f85b29ee81..a79afead95 100644
--- a/include/armnn/INetwork.hpp
+++ b/include/armnn/INetwork.hpp
@@ -115,6 +115,8 @@ protected:
 };
 
 
+/// ArmNN performs an optimization on each model/network before it gets loaded for execution. OptimizerOptions provides
+/// a set of features that allows the user to customize this optimization on a per model basis.
 struct OptimizerOptions
 {
     OptimizerOptions()
@@ -160,13 +162,21 @@ struct OptimizerOptions
         }
     }
 
-    // Reduce Fp32 data to Fp16 for faster processing
+    /// Reduces all Fp32 operators in the model to Fp16 for faster processing.
+    /// @Note This feature works best if all operators of the model are in Fp32. ArmNN will add conversion layers
+    ///       between layers that weren't in Fp32 in the first place or if the operator is not supported in Fp16.
+    ///       The overhead of these conversions can lead to a slower overall performance if too many conversions are
+    ///       required.
     bool m_ReduceFp32ToFp16;
 
     // Add debug data for easier troubleshooting
     bool m_Debug;
 
-    // Reduce Fp32 data to Bf16 for faster processing
+    /// Reduces all Fp32 operators in the model to Bf16 for faster processing.
+    /// @Note This feature works best if all operators of the model are in Fp32. ArmNN will add conversion layers
+    ///       between layers that weren't in Fp32 in the first place or if the operator is not supported in Bf16.
+    ///       The overhead of these conversions can lead to a slower overall performance if too many conversions are
+    ///       required.
     bool m_ReduceFp32ToBf16;
 
     // Infer output size when not available
diff --git a/include/armnn/IRuntime.hpp b/include/armnn/IRuntime.hpp
index b89cfd72cd..93f8b0fd5b 100644
--- a/include/armnn/IRuntime.hpp
+++ b/include/armnn/IRuntime.hpp
@@ -119,6 +119,7 @@ public:
 
         /// Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive
         /// Only a single path is allowed for the override
+        /// It defines the path to search for any [dynamic backend libraries](src/dynamic/README.md).
         std::string m_DynamicBackendsPath;
 
         /// Setting this flag will allow the user to create the Runtime in protected mode.
@@ -158,12 +159,19 @@ public:
                 , m_LocalPacketHandlers()
             {}
 
+            /// Indicates whether external profiling is enabled or not.
             bool        m_EnableProfiling;
+            /// Indicates whether external timeline profiling is enabled or not.
             bool        m_TimelineEnabled;
+            /// Path to a file in which outgoing timeline profiling messages will be stored.
             std::string m_OutgoingCaptureFile;
+            /// Path to a file in which incoming timeline profiling messages will be stored.
             std::string m_IncomingCaptureFile;
+            /// Enable profiling output to file only.
             bool        m_FileOnly;
+            /// The duration at which captured profiling messages will be flushed.
             uint32_t    m_CapturePeriod;
+            /// The format of the file used for outputting profiling data.
             std::string m_FileFormat;
             std::vector<armnn::profiling::ILocalPacketHandlerSharedPtr> m_LocalPacketHandlers;
         };
diff --git a/include/armnn/Types.hpp b/include/armnn/Types.hpp
index deaa0b3a50..f61a1fdca8 100644
--- a/include/armnn/Types.hpp
+++ b/include/armnn/Types.hpp
@@ -201,7 +201,7 @@ enum class OutputShapeRounding
 /// The ShapeInferenceMethod modify how the output shapes are treated.
 /// When ValidateOnly is selected, the output shapes are inferred from the input parameters of the layer
 /// and any mismatch is reported.
-/// When InferAndValidate is selected 2 actions must be performed: (1)infer output shape from inputs and (2)validate the
+/// When InferAndValidate is selected 2 actions are performed: (1)infer output shape from inputs and (2)validate the
 /// shapes as in ValidateOnly. This option has been added to work with tensors which rank or dimension sizes are not
 /// specified explicitly, however this information can be calculated from the inputs.
 ///