IVGCVSW-5752 Add missing runtime parameters to TfLite delegate.

* Adding Runtime parameter: dynamic-backends-path
* Add profiling parameters:
     gpu-enable-profiling,
     enable-internal-profiling, internal-profiling-detail,
     enable-external-profiling, timeline-profiling, outgoing-capture-file,
     incoming-capture-file, file-only-external-profiling,
     counter-capture-period, profiling-file-format
* Adding utility parameter "serialize-to-dot"

Signed-off-by: Colm Donelan <Colm.Donelan@arm.com>
Change-Id: Ibff4b9a85ff0f0da5d70e8aa0bb6cba96aaabbc3

4 files changed, 243 insertions, 14 deletions

diff --git a/delegate/include/DelegateOptions.hpp b/delegate/include/DelegateOptions.hpp
index ace0859f7e..b8847440ed 100644
--- a/delegate/include/DelegateOptions.hpp
+++ b/delegate/include/DelegateOptions.hpp
@@ -41,6 +41,12 @@ public:
 
     void SetBackends(const std::vector<armnn::BackendId>& backends) { m_Backends = backends; }
 
+    void SetDynamicBackendsPath(const std::string& dynamicBackendsPath) { m_DynamicBackendsPath = dynamicBackendsPath; }
+    const std::string& GetDynamicBackendsPath() const { return m_DynamicBackendsPath; }
+
+    void SetGpuProfilingState(bool gpuProfilingState) { m_EnableGpuProfiling = gpuProfilingState; }
+    bool GetGpuProfilingState() { return m_EnableGpuProfiling; }
+
     const std::vector<armnn::BackendOptions>& GetBackendOptions() const { return m_BackendOptions; }
 
     /// Appends a backend option to the list of backend options
@@ -62,6 +68,23 @@ public:
     const armnn::Optional<armnn::DebugCallbackFunction>& GetDebugCallbackFunction() const
         { return m_DebugCallbackFunc; }
 
+    void SetInternalProfilingParams(bool internalProfilingState,
+                                    const armnn::ProfilingDetailsMethod& internalProfilingDetail)
+        { m_InternalProfilingEnabled = internalProfilingState; m_InternalProfilingDetail = internalProfilingDetail; }
+
+    bool GetInternalProfilingState() const { return m_InternalProfilingEnabled; }
+    const armnn::ProfilingDetailsMethod& GetInternalProfilingDetail() const { return m_InternalProfilingDetail; }
+
+    void SetExternalProfilingParams(
+        const armnn::IRuntime::CreationOptions::ExternalProfilingOptions& externalProfilingParams)
+        { m_ProfilingOptions = externalProfilingParams; }
+
+    const armnn::IRuntime::CreationOptions::ExternalProfilingOptions& GetExternalProfilingParams() const
+        { return m_ProfilingOptions; }
+
+    void SetSerializeToDot(const std::string& serializeToDotFile) { m_SerializeToDot = serializeToDotFile; }
+    const std::string& GetSerializeToDot() const { return m_SerializeToDot; }
+
 private:
     /// Which backend to run Delegate on.
     /// Examples of possible values are: CpuRef, CpuAcc, GpuAcc.
@@ -87,6 +110,13 @@ private:
     ///   "KernelProfilingEnabled" : bool [true | false]
     std::vector<armnn::BackendOptions> m_BackendOptions;
 
+    /// Dynamic backend path.
+    /// This is the directory that will be searched for any dynamic backends.
+    std::string m_DynamicBackendsPath = "";
+
+    /// Enable Gpu Profiling.
+    bool m_EnableGpuProfiling = false;
+
     /// OptimizerOptions
     /// Reduce Fp32 data to Fp16 for faster processing
     /// bool m_ReduceFp32ToFp16;
@@ -100,13 +130,39 @@ private:
     /// ModelOptions m_ModelOptions;
     armnn::OptimizerOptions m_OptimizerOptions;
 
+    /// External profiling options.
+    /// Indicates whether external profiling is enabled or not.
+    /// bool m_EnableProfiling
+    /// Indicates whether external timeline profiling is enabled or not.
+    /// bool m_TimelineEnabled
+    /// Path to a file in which outgoing timeline profiling messages will be stored.
+    /// std::string m_OutgoingCaptureFile
+    /// Path to a file in which incoming timeline profiling messages will be stored.
+    /// std::string m_IncomingCaptureFile
+    /// Enable profiling output to file only.
+    /// bool m_FileOnly
+    /// The duration at which captured profiling messages will be flushed.
+    /// uint32_t m_CapturePeriod
+    /// The format of the file used for outputting profiling data.
+    /// std::string m_FileFormat
+    armnn::IRuntime::CreationOptions::ExternalProfilingOptions m_ProfilingOptions;
+
+    /// Internal profiling options.
+    /// Indicates whether internal profiling is enabled or not.
+    bool m_InternalProfilingEnabled = false;
+    /// Sets the level of detail output by the profiling. Options are DetailsWithEvents = 1 and DetailsOnly = 2
+    armnn::ProfilingDetailsMethod m_InternalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents;
+
     /// Severity level for logging within ArmNN that will be used on creation of the delegate
     armnn::Optional<armnn::LogSeverity> m_LoggingSeverity;
 
     /// A callback function to debug layers performing custom computations on intermediate tensors.
     /// If a function is not registered, and debug is enabled in OptimizerOptions,
     /// debug will print information of the intermediate tensors.
-     armnn::Optional<armnn::DebugCallbackFunction> m_DebugCallbackFunc;
+    armnn::Optional<armnn::DebugCallbackFunction> m_DebugCallbackFunc;
+
+    /// If not empty then the optimized model will be serialized to a file with this file name in "dot" format.
+    std::string m_SerializeToDot = "";
 };
 
 } // namespace armnnDelegate
diff --git a/delegate/src/armnn_delegate.cpp b/delegate/src/armnn_delegate.cpp
index f13bb5779c..2ede23c12b 100644
--- a/delegate/src/armnn_delegate.cpp
+++ b/delegate/src/armnn_delegate.cpp
@@ -39,6 +39,7 @@
 #include "UnidirectionalSequenceLstm.hpp"
 #include "Unpack.hpp"
 
+#include <armnnUtils/Filesystem.hpp>
 #include <flatbuffers/flatbuffers.h>
 #include <tensorflow/lite/context_util.h>
 
@@ -137,6 +138,9 @@ Delegate::Delegate(armnnDelegate::DelegateOptions options)
 
     // Create ArmNN Runtime
     armnn::IRuntime::CreationOptions runtimeOptions;
+    runtimeOptions.m_DynamicBackendsPath = m_Options.GetDynamicBackendsPath();
+    runtimeOptions.m_EnableGpuProfiling = m_Options.GetGpuProfilingState();
+    runtimeOptions.m_ProfilingOptions = m_Options.GetExternalProfilingParams();
 
     auto backendOptions = m_Options.GetBackendOptions();
     if (!backendOptions.empty())
@@ -363,27 +367,34 @@ ArmnnSubgraph* ArmnnSubgraph::Create(TfLiteContext* tfLiteContext,
         throw armnn::Exception("TfLiteArmnnDelegate: Unable to optimize the network!");
     }
 
+    // If set, we will serialize the optimized model into a dot file.
+    const std::string serializeToDotFile = delegate->m_Options.GetSerializeToDot();
+    if (!serializeToDotFile.empty())
+    {
+        fs::path filename = serializeToDotFile;
+        std::fstream file(filename.c_str(), std::ios_base::out);
+        optNet->SerializeToDot(file);
+    }
+
     try
     {
         // Load graph into runtime
         std::string errorMessage;
         armnn::Status loadingStatus;
+        armnn::MemorySource memorySource = armnn::MemorySource::Undefined;
         if (delegate->m_Options.GetOptimizerOptions().m_ImportEnabled)
         {
-            armnn::INetworkProperties networkProperties(false,
-                                                        armnn::MemorySource::Malloc,
-                                                        armnn::MemorySource::Malloc);
-            loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
-                                                             std::move(optNet),
-                                                             errorMessage,
-                                                             networkProperties);
-        }
-        else
-        {
-            loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
-                                                             std::move(optNet),
-                                                             errorMessage);
+            memorySource = armnn::MemorySource::Malloc;
         }
+        armnn::INetworkProperties networkProperties(false,
+                                                    memorySource,
+                                                    memorySource,
+                                                    delegate->m_Options.GetInternalProfilingState(),
+                                                    delegate->m_Options.GetInternalProfilingDetail());
+        loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
+                                                         std::move(optNet),
+                                                         errorMessage,
+                                                         networkProperties);
         if (loadingStatus != armnn::Status::Success)
         {
             // Optimize failed
diff --git a/delegate/src/armnn_external_delegate.cpp b/delegate/src/armnn_external_delegate.cpp
index 27eaf64f73..fb327de442 100644
--- a/delegate/src/armnn_external_delegate.cpp
+++ b/delegate/src/armnn_external_delegate.cpp
@@ -43,6 +43,10 @@ std::vector<std::string> gpu_options {"gpu-tuning-level",
  *                  back to next backend in list if previous doesn't
  *                  provide support for operation. e.g. "GpuAcc,CpuAcc"
  *
+ *    Option key: "dynamic-backends-path" \n
+ *    Possible values: [filenameString] \n
+ *    Descriptions: This is the directory that will be searched for any dynamic backends.
+ *
  *    Option key: "logging-severity" \n
  *    Possible values: ["trace"/"debug"/"info"/"warning"/"error"/"fatal"] \n
  *    Description: Sets the logging severity level for ArmNN. Logging
@@ -63,6 +67,10 @@ std::vector<std::string> gpu_options {"gpu-tuning-level",
  *    Possible values: [filenameString] \n
  *    Description: File name for the tuning file.
  *
+ *    Option key: "gpu-enable-profiling" \n
+ *    Possible values: ["true"/"false"] \n
+ *    Description: Enables GPU profiling
+ *
  *    Option key: "gpu-kernel-profiling-enabled" \n
  *    Possible values: ["true"/"false"] \n
  *    Description: Enables GPU kernel profiling
@@ -105,6 +113,45 @@ std::vector<std::string> gpu_options {"gpu-tuning-level",
  *    Possible values: ["true"/"false"] \n
  *    Description: Enable memory import
  *
+ *    Option key: "enable-internal-profiling" \n
+ *    Possible values: ["true"/"false"] \n
+ *    Description: Enable the internal profiling feature.
+ *
+ *    Option key: "internal-profiling-detail" \n
+ *    Possible values: [1/2] \n
+ *    Description: Set the detail on the internal profiling. 1 = DetailsWithEvents, 2 = DetailsOnly.
+ *
+ *    Option key: "enable-external-profiling" \n
+ *    Possible values: ["true"/"false"] \n
+ *    Description: Enable the external profiling feature.
+ *
+ *    Option key: "timeline-profiling" \n
+ *    Possible values: ["true"/"false"] \n
+ *    Description: Indicates whether external timeline profiling is enabled or not.
+ *
+ *    Option key: "outgoing-capture-file" \n
+ *    Possible values: [filenameString] \n
+ *    Description: Path to a file in which outgoing timeline profiling messages will be stored.
+ *
+ *    Option key: "incoming-capture-file" \n
+ *    Possible values: [filenameString] \n
+ *    Description: Path to a file in which incoming timeline profiling messages will be stored.
+ *
+ *    Option key: "file-only-external-profiling" \n
+ *    Possible values: ["true"/"false"] \n
+ *    Description: Enable profiling output to file only.
+ *
+ *    Option key: "counter-capture-period" \n
+ *    Possible values: Integer, Default is 10000u
+ *    Description: Value in microseconds of the profiling capture period. \n
+ *
+ *    Option key: "profiling-file-format" \n
+ *    Possible values: String of ["binary"] \n
+ *    Description: The format of the file used for outputting profiling data. Currently on "binary" is supported.
+ *
+ *    Option key: "serialize-to-dot" \n
+ *    Possible values: [filenameString] \n
+ *    Description: Serialize the optimized network to the file specified in "dot" format.
  *
  * @param[in]     option_keys     Delegate option names
  * @param[in]     options_values  Delegate option values
@@ -125,6 +172,9 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
         // (Initializes with CpuRef backend)
         armnnDelegate::DelegateOptions options = armnnDelegate::TfLiteArmnnDelegateOptionsDefault();
         armnn::OptimizerOptions optimizerOptions;
+        bool internalProfilingState = false;
+        armnn::ProfilingDetailsMethod internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents;
+        armnn::IRuntime::CreationOptions::ExternalProfilingOptions extProfilingParams;
         for (size_t i = 0; i < num_options; ++i)
         {
             // Process backends
@@ -141,6 +191,11 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
                 }
                 options.SetBackends(backends);
             }
+            // Process dynamic-backends-path
+            else if (std::string(options_keys[i]) == std::string("dynamic-backends-path"))
+            {
+                options.SetDynamicBackendsPath(std::string(options_values[i]));
+            }
             // Process logging level
             else if (std::string(options_keys[i]) == std::string("logging-severity"))
             {
@@ -162,6 +217,10 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
                 armnn::BackendOptions option("GpuAcc", {{"TuningFile", std::string(options_values[i])}});
                 options.AddBackendOption(option);
             }
+            else if (std::string(options_keys[i]) == std::string("gpu-enable-profiling"))
+            {
+                options.SetGpuProfilingState(*options_values[i] != '0');
+            }
             else if (std::string(options_keys[i]) == std::string("gpu-kernel-profiling-enabled"))
             {
                 armnn::BackendOptions option("GpuAcc", {{"KernelProfilingEnabled", (*options_values[i] != '0')}});
@@ -213,12 +272,76 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
             {
                optimizerOptions.m_ImportEnabled = *options_values[i] != '0';
             }
+            // Process enable-internal-profiling
+            else if (std::string(options_keys[i]) == std::string("enable-internal-profiling"))
+            {
+                internalProfilingState = *options_values[i] != '0';
+            }
+            // Process internal-profiling-detail
+            else if (std::string(options_keys[i]) == std::string("internal-profiling-detail"))
+            {
+                uint32_t detailLevel = static_cast<uint32_t>(std::stoul(options_values[i]));
+                switch (detailLevel)
+                {
+                    case 1:
+                        internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents;
+                        break;
+                    case 2:
+                        internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsOnly;
+                        break;
+                    default:
+                        internalProfilingDetail = armnn::ProfilingDetailsMethod::Undefined;
+                        break;
+                }
+            }
+            // Process enable-external-profiling
+            else if (std::string(options_keys[i]) == std::string("enable-external-profiling"))
+            {
+                extProfilingParams.m_EnableProfiling = *options_values[i] != '0';
+            }
+            // Process timeline-profiling
+            else if (std::string(options_keys[i]) == std::string("timeline-profiling"))
+            {
+                extProfilingParams.m_TimelineEnabled = *options_values[i] != '0';
+            }
+            // Process outgoing-capture-file
+            else if (std::string(options_keys[i]) == std::string("outgoing-capture-file"))
+            {
+                extProfilingParams.m_OutgoingCaptureFile = options_values[i];
+            }
+            // Process incoming-capture-file
+            else if (std::string(options_keys[i]) == std::string("incoming-capture-file"))
+            {
+                extProfilingParams.m_IncomingCaptureFile = options_values[i];
+            }
+            // Process file-only-external-profiling
+            else if (std::string(options_keys[i]) == std::string("file-only-external-profiling"))
+            {
+                extProfilingParams.m_FileOnly = *options_values[i] != '0';
+            }
+            // Process counter-capture-period
+            else if (std::string(options_keys[i]) == std::string("counter-capture-period"))
+            {
+                extProfilingParams.m_CapturePeriod = static_cast<uint32_t>(std::stoul(options_values[i]));
+            }
+            // Process profiling-file-format
+            else if (std::string(options_keys[i]) == std::string("profiling-file-format"))
+            {
+                extProfilingParams.m_FileFormat = options_values[i];
+            }
+            // Process serialize-to-dot
+            else if (std::string(options_keys[i]) == std::string("serialize-to-dot"))
+            {
+                options.SetSerializeToDot(options_values[i]);
+            }
             else
             {
                 throw armnn::Exception("Unknown option for the ArmNN Delegate given: " + std::string(options_keys[i]));
             }
         }
         options.SetOptimizerOptions(optimizerOptions);
+        options.SetInternalProfilingParams(internalProfilingState, internalProfilingDetail);
+        options.SetExternalProfilingParams(extProfilingParams);
         delegate = TfLiteArmnnDelegateCreate(options);
     }
     catch (const std::exception& ex)
diff --git a/delegate/src/test/DelegateOptionsTest.cpp b/delegate/src/test/DelegateOptionsTest.cpp
index c7231fb279..338772596b 100644
--- a/delegate/src/test/DelegateOptionsTest.cpp
+++ b/delegate/src/test/DelegateOptionsTest.cpp
@@ -5,6 +5,7 @@
 
 #include "DelegateOptionsTestHelper.hpp"
 #include <common/include/ProfilingGuid.hpp>
+#include <armnnUtils/Filesystem.hpp>
 
 namespace armnnDelegate
 {
@@ -185,6 +186,44 @@ TEST_CASE ("ArmnnDelegateModelOptions_CpuAcc_Test")
                               delegateOptions);
 }
 
+TEST_CASE ("ArmnnDelegateSerializeToDot")
+{
+    const fs::path filename(fs::temp_directory_path() / "ArmnnDelegateSerializeToDot.dot");
+    if ( fs::exists(filename) )
+    {
+        fs::remove(filename);
+    }
+    std::stringstream ss;
+    {
+        StreamRedirector redirect(std::cout, ss.rdbuf());
+
+        std::vector<armnn::BackendId> backends = { armnn::Compute::CpuRef };
+        std::vector<int32_t> tensorShape { 1, 2, 2, 1 };
+        std::vector<float> inputData = { 1, 2, 3, 4 };
+        std::vector<float> divData = { 2, 2, 3, 4 };
+        std::vector<float> expectedResult = { 1, 2, 2, 2 };
+
+        armnn::OptimizerOptions optimizerOptions(false, false, false, false);
+        armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions);
+        // Enable serialize to dot by specifying the target file name.
+        delegateOptions.SetSerializeToDot(filename);
+        DelegateOptionTest<float>(::tflite::TensorType_FLOAT32,
+                                  backends,
+                                  tensorShape,
+                                  inputData,
+                                  inputData,
+                                  divData,
+                                  expectedResult,
+                                  delegateOptions);
+    }
+    CHECK(fs::exists(filename));
+    // The file should have a size greater than 0 bytes.
+    CHECK(fs::file_size(filename) > 0);
+    // Clean up.
+    fs::remove(filename);
+}
+
+
 }
 
 } // namespace armnnDelegate