From 3e32a8700bf12d3b70d2824c12cdae907bde9360 Mon Sep 17 00:00:00 2001 From: Colm Donelan Date: Mon, 4 Oct 2021 22:55:37 +0100 Subject: IVGCVSW-5752 Add missing runtime parameters to TfLite delegate. * Adding Runtime parameter: dynamic-backends-path * Add profiling parameters: gpu-enable-profiling, enable-internal-profiling, internal-profiling-detail, enable-external-profiling, timeline-profiling, outgoing-capture-file, incoming-capture-file, file-only-external-profiling, counter-capture-period, profiling-file-format * Adding utility parameter "serialize-to-dot" Signed-off-by: Colm Donelan Change-Id: Ibff4b9a85ff0f0da5d70e8aa0bb6cba96aaabbc3 --- delegate/include/DelegateOptions.hpp | 58 +++++++++++++- delegate/src/armnn_delegate.cpp | 37 +++++---- delegate/src/armnn_external_delegate.cpp | 123 ++++++++++++++++++++++++++++++ delegate/src/test/DelegateOptionsTest.cpp | 39 ++++++++++ 4 files changed, 243 insertions(+), 14 deletions(-) diff --git a/delegate/include/DelegateOptions.hpp b/delegate/include/DelegateOptions.hpp index ace0859f7e..b8847440ed 100644 --- a/delegate/include/DelegateOptions.hpp +++ b/delegate/include/DelegateOptions.hpp @@ -41,6 +41,12 @@ public: void SetBackends(const std::vector& backends) { m_Backends = backends; } + void SetDynamicBackendsPath(const std::string& dynamicBackendsPath) { m_DynamicBackendsPath = dynamicBackendsPath; } + const std::string& GetDynamicBackendsPath() const { return m_DynamicBackendsPath; } + + void SetGpuProfilingState(bool gpuProfilingState) { m_EnableGpuProfiling = gpuProfilingState; } + bool GetGpuProfilingState() { return m_EnableGpuProfiling; } + const std::vector& GetBackendOptions() const { return m_BackendOptions; } /// Appends a backend option to the list of backend options @@ -62,6 +68,23 @@ public: const armnn::Optional& GetDebugCallbackFunction() const { return m_DebugCallbackFunc; } + void SetInternalProfilingParams(bool internalProfilingState, + const armnn::ProfilingDetailsMethod& internalProfilingDetail) + { m_InternalProfilingEnabled = internalProfilingState; m_InternalProfilingDetail = internalProfilingDetail; } + + bool GetInternalProfilingState() const { return m_InternalProfilingEnabled; } + const armnn::ProfilingDetailsMethod& GetInternalProfilingDetail() const { return m_InternalProfilingDetail; } + + void SetExternalProfilingParams( + const armnn::IRuntime::CreationOptions::ExternalProfilingOptions& externalProfilingParams) + { m_ProfilingOptions = externalProfilingParams; } + + const armnn::IRuntime::CreationOptions::ExternalProfilingOptions& GetExternalProfilingParams() const + { return m_ProfilingOptions; } + + void SetSerializeToDot(const std::string& serializeToDotFile) { m_SerializeToDot = serializeToDotFile; } + const std::string& GetSerializeToDot() const { return m_SerializeToDot; } + private: /// Which backend to run Delegate on. /// Examples of possible values are: CpuRef, CpuAcc, GpuAcc. @@ -87,6 +110,13 @@ private: /// "KernelProfilingEnabled" : bool [true | false] std::vector m_BackendOptions; + /// Dynamic backend path. + /// This is the directory that will be searched for any dynamic backends. + std::string m_DynamicBackendsPath = ""; + + /// Enable Gpu Profiling. + bool m_EnableGpuProfiling = false; + /// OptimizerOptions /// Reduce Fp32 data to Fp16 for faster processing /// bool m_ReduceFp32ToFp16; @@ -100,13 +130,39 @@ private: /// ModelOptions m_ModelOptions; armnn::OptimizerOptions m_OptimizerOptions; + /// External profiling options. + /// Indicates whether external profiling is enabled or not. + /// bool m_EnableProfiling + /// Indicates whether external timeline profiling is enabled or not. + /// bool m_TimelineEnabled + /// Path to a file in which outgoing timeline profiling messages will be stored. + /// std::string m_OutgoingCaptureFile + /// Path to a file in which incoming timeline profiling messages will be stored. + /// std::string m_IncomingCaptureFile + /// Enable profiling output to file only. + /// bool m_FileOnly + /// The duration at which captured profiling messages will be flushed. + /// uint32_t m_CapturePeriod + /// The format of the file used for outputting profiling data. + /// std::string m_FileFormat + armnn::IRuntime::CreationOptions::ExternalProfilingOptions m_ProfilingOptions; + + /// Internal profiling options. + /// Indicates whether internal profiling is enabled or not. + bool m_InternalProfilingEnabled = false; + /// Sets the level of detail output by the profiling. Options are DetailsWithEvents = 1 and DetailsOnly = 2 + armnn::ProfilingDetailsMethod m_InternalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents; + /// Severity level for logging within ArmNN that will be used on creation of the delegate armnn::Optional m_LoggingSeverity; /// A callback function to debug layers performing custom computations on intermediate tensors. /// If a function is not registered, and debug is enabled in OptimizerOptions, /// debug will print information of the intermediate tensors. - armnn::Optional m_DebugCallbackFunc; + armnn::Optional m_DebugCallbackFunc; + + /// If not empty then the optimized model will be serialized to a file with this file name in "dot" format. + std::string m_SerializeToDot = ""; }; } // namespace armnnDelegate diff --git a/delegate/src/armnn_delegate.cpp b/delegate/src/armnn_delegate.cpp index f13bb5779c..2ede23c12b 100644 --- a/delegate/src/armnn_delegate.cpp +++ b/delegate/src/armnn_delegate.cpp @@ -39,6 +39,7 @@ #include "UnidirectionalSequenceLstm.hpp" #include "Unpack.hpp" +#include #include #include @@ -137,6 +138,9 @@ Delegate::Delegate(armnnDelegate::DelegateOptions options) // Create ArmNN Runtime armnn::IRuntime::CreationOptions runtimeOptions; + runtimeOptions.m_DynamicBackendsPath = m_Options.GetDynamicBackendsPath(); + runtimeOptions.m_EnableGpuProfiling = m_Options.GetGpuProfilingState(); + runtimeOptions.m_ProfilingOptions = m_Options.GetExternalProfilingParams(); auto backendOptions = m_Options.GetBackendOptions(); if (!backendOptions.empty()) @@ -363,27 +367,34 @@ ArmnnSubgraph* ArmnnSubgraph::Create(TfLiteContext* tfLiteContext, throw armnn::Exception("TfLiteArmnnDelegate: Unable to optimize the network!"); } + // If set, we will serialize the optimized model into a dot file. + const std::string serializeToDotFile = delegate->m_Options.GetSerializeToDot(); + if (!serializeToDotFile.empty()) + { + fs::path filename = serializeToDotFile; + std::fstream file(filename.c_str(), std::ios_base::out); + optNet->SerializeToDot(file); + } + try { // Load graph into runtime std::string errorMessage; armnn::Status loadingStatus; + armnn::MemorySource memorySource = armnn::MemorySource::Undefined; if (delegate->m_Options.GetOptimizerOptions().m_ImportEnabled) { - armnn::INetworkProperties networkProperties(false, - armnn::MemorySource::Malloc, - armnn::MemorySource::Malloc); - loadingStatus = delegate->m_Runtime->LoadNetwork(networkId, - std::move(optNet), - errorMessage, - networkProperties); - } - else - { - loadingStatus = delegate->m_Runtime->LoadNetwork(networkId, - std::move(optNet), - errorMessage); + memorySource = armnn::MemorySource::Malloc; } + armnn::INetworkProperties networkProperties(false, + memorySource, + memorySource, + delegate->m_Options.GetInternalProfilingState(), + delegate->m_Options.GetInternalProfilingDetail()); + loadingStatus = delegate->m_Runtime->LoadNetwork(networkId, + std::move(optNet), + errorMessage, + networkProperties); if (loadingStatus != armnn::Status::Success) { // Optimize failed diff --git a/delegate/src/armnn_external_delegate.cpp b/delegate/src/armnn_external_delegate.cpp index 27eaf64f73..fb327de442 100644 --- a/delegate/src/armnn_external_delegate.cpp +++ b/delegate/src/armnn_external_delegate.cpp @@ -43,6 +43,10 @@ std::vector gpu_options {"gpu-tuning-level", * back to next backend in list if previous doesn't * provide support for operation. e.g. "GpuAcc,CpuAcc" * + * Option key: "dynamic-backends-path" \n + * Possible values: [filenameString] \n + * Descriptions: This is the directory that will be searched for any dynamic backends. + * * Option key: "logging-severity" \n * Possible values: ["trace"/"debug"/"info"/"warning"/"error"/"fatal"] \n * Description: Sets the logging severity level for ArmNN. Logging @@ -63,6 +67,10 @@ std::vector gpu_options {"gpu-tuning-level", * Possible values: [filenameString] \n * Description: File name for the tuning file. * + * Option key: "gpu-enable-profiling" \n + * Possible values: ["true"/"false"] \n + * Description: Enables GPU profiling + * * Option key: "gpu-kernel-profiling-enabled" \n * Possible values: ["true"/"false"] \n * Description: Enables GPU kernel profiling @@ -105,6 +113,45 @@ std::vector gpu_options {"gpu-tuning-level", * Possible values: ["true"/"false"] \n * Description: Enable memory import * + * Option key: "enable-internal-profiling" \n + * Possible values: ["true"/"false"] \n + * Description: Enable the internal profiling feature. + * + * Option key: "internal-profiling-detail" \n + * Possible values: [1/2] \n + * Description: Set the detail on the internal profiling. 1 = DetailsWithEvents, 2 = DetailsOnly. + * + * Option key: "enable-external-profiling" \n + * Possible values: ["true"/"false"] \n + * Description: Enable the external profiling feature. + * + * Option key: "timeline-profiling" \n + * Possible values: ["true"/"false"] \n + * Description: Indicates whether external timeline profiling is enabled or not. + * + * Option key: "outgoing-capture-file" \n + * Possible values: [filenameString] \n + * Description: Path to a file in which outgoing timeline profiling messages will be stored. + * + * Option key: "incoming-capture-file" \n + * Possible values: [filenameString] \n + * Description: Path to a file in which incoming timeline profiling messages will be stored. + * + * Option key: "file-only-external-profiling" \n + * Possible values: ["true"/"false"] \n + * Description: Enable profiling output to file only. + * + * Option key: "counter-capture-period" \n + * Possible values: Integer, Default is 10000u + * Description: Value in microseconds of the profiling capture period. \n + * + * Option key: "profiling-file-format" \n + * Possible values: String of ["binary"] \n + * Description: The format of the file used for outputting profiling data. Currently on "binary" is supported. + * + * Option key: "serialize-to-dot" \n + * Possible values: [filenameString] \n + * Description: Serialize the optimized network to the file specified in "dot" format. * * @param[in] option_keys Delegate option names * @param[in] options_values Delegate option values @@ -125,6 +172,9 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys, // (Initializes with CpuRef backend) armnnDelegate::DelegateOptions options = armnnDelegate::TfLiteArmnnDelegateOptionsDefault(); armnn::OptimizerOptions optimizerOptions; + bool internalProfilingState = false; + armnn::ProfilingDetailsMethod internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents; + armnn::IRuntime::CreationOptions::ExternalProfilingOptions extProfilingParams; for (size_t i = 0; i < num_options; ++i) { // Process backends @@ -141,6 +191,11 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys, } options.SetBackends(backends); } + // Process dynamic-backends-path + else if (std::string(options_keys[i]) == std::string("dynamic-backends-path")) + { + options.SetDynamicBackendsPath(std::string(options_values[i])); + } // Process logging level else if (std::string(options_keys[i]) == std::string("logging-severity")) { @@ -162,6 +217,10 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys, armnn::BackendOptions option("GpuAcc", {{"TuningFile", std::string(options_values[i])}}); options.AddBackendOption(option); } + else if (std::string(options_keys[i]) == std::string("gpu-enable-profiling")) + { + options.SetGpuProfilingState(*options_values[i] != '0'); + } else if (std::string(options_keys[i]) == std::string("gpu-kernel-profiling-enabled")) { armnn::BackendOptions option("GpuAcc", {{"KernelProfilingEnabled", (*options_values[i] != '0')}}); @@ -213,12 +272,76 @@ TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys, { optimizerOptions.m_ImportEnabled = *options_values[i] != '0'; } + // Process enable-internal-profiling + else if (std::string(options_keys[i]) == std::string("enable-internal-profiling")) + { + internalProfilingState = *options_values[i] != '0'; + } + // Process internal-profiling-detail + else if (std::string(options_keys[i]) == std::string("internal-profiling-detail")) + { + uint32_t detailLevel = static_cast(std::stoul(options_values[i])); + switch (detailLevel) + { + case 1: + internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsWithEvents; + break; + case 2: + internalProfilingDetail = armnn::ProfilingDetailsMethod::DetailsOnly; + break; + default: + internalProfilingDetail = armnn::ProfilingDetailsMethod::Undefined; + break; + } + } + // Process enable-external-profiling + else if (std::string(options_keys[i]) == std::string("enable-external-profiling")) + { + extProfilingParams.m_EnableProfiling = *options_values[i] != '0'; + } + // Process timeline-profiling + else if (std::string(options_keys[i]) == std::string("timeline-profiling")) + { + extProfilingParams.m_TimelineEnabled = *options_values[i] != '0'; + } + // Process outgoing-capture-file + else if (std::string(options_keys[i]) == std::string("outgoing-capture-file")) + { + extProfilingParams.m_OutgoingCaptureFile = options_values[i]; + } + // Process incoming-capture-file + else if (std::string(options_keys[i]) == std::string("incoming-capture-file")) + { + extProfilingParams.m_IncomingCaptureFile = options_values[i]; + } + // Process file-only-external-profiling + else if (std::string(options_keys[i]) == std::string("file-only-external-profiling")) + { + extProfilingParams.m_FileOnly = *options_values[i] != '0'; + } + // Process counter-capture-period + else if (std::string(options_keys[i]) == std::string("counter-capture-period")) + { + extProfilingParams.m_CapturePeriod = static_cast(std::stoul(options_values[i])); + } + // Process profiling-file-format + else if (std::string(options_keys[i]) == std::string("profiling-file-format")) + { + extProfilingParams.m_FileFormat = options_values[i]; + } + // Process serialize-to-dot + else if (std::string(options_keys[i]) == std::string("serialize-to-dot")) + { + options.SetSerializeToDot(options_values[i]); + } else { throw armnn::Exception("Unknown option for the ArmNN Delegate given: " + std::string(options_keys[i])); } } options.SetOptimizerOptions(optimizerOptions); + options.SetInternalProfilingParams(internalProfilingState, internalProfilingDetail); + options.SetExternalProfilingParams(extProfilingParams); delegate = TfLiteArmnnDelegateCreate(options); } catch (const std::exception& ex) diff --git a/delegate/src/test/DelegateOptionsTest.cpp b/delegate/src/test/DelegateOptionsTest.cpp index c7231fb279..338772596b 100644 --- a/delegate/src/test/DelegateOptionsTest.cpp +++ b/delegate/src/test/DelegateOptionsTest.cpp @@ -5,6 +5,7 @@ #include "DelegateOptionsTestHelper.hpp" #include +#include namespace armnnDelegate { @@ -185,6 +186,44 @@ TEST_CASE ("ArmnnDelegateModelOptions_CpuAcc_Test") delegateOptions); } +TEST_CASE ("ArmnnDelegateSerializeToDot") +{ + const fs::path filename(fs::temp_directory_path() / "ArmnnDelegateSerializeToDot.dot"); + if ( fs::exists(filename) ) + { + fs::remove(filename); + } + std::stringstream ss; + { + StreamRedirector redirect(std::cout, ss.rdbuf()); + + std::vector backends = { armnn::Compute::CpuRef }; + std::vector tensorShape { 1, 2, 2, 1 }; + std::vector inputData = { 1, 2, 3, 4 }; + std::vector divData = { 2, 2, 3, 4 }; + std::vector expectedResult = { 1, 2, 2, 2 }; + + armnn::OptimizerOptions optimizerOptions(false, false, false, false); + armnnDelegate::DelegateOptions delegateOptions(backends, optimizerOptions); + // Enable serialize to dot by specifying the target file name. + delegateOptions.SetSerializeToDot(filename); + DelegateOptionTest(::tflite::TensorType_FLOAT32, + backends, + tensorShape, + inputData, + inputData, + divData, + expectedResult, + delegateOptions); + } + CHECK(fs::exists(filename)); + // The file should have a size greater than 0 bytes. + CHECK(fs::file_size(filename) > 0); + // Clean up. + fs::remove(filename); +} + + } } // namespace armnnDelegate -- cgit v1.2.1