1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
//
// Copyright © 2020 Arm Ltd and Contributors. All rights reserved.
// SPDX-License-Identifier: MIT
//
#include "armnn_delegate.hpp"
#include <armnn/Logging.hpp>
#include <iostream>
#include <tensorflow/lite/minimal_logging.h>
namespace tflite
{
/**
* This file defines two symbols that need to be exported to use the TFLite external delegate provider. This is a plugin
* that can be used for fast integration of delegates into benchmark tests and other tools. It allows loading of
* a dynamic delegate library at runtime.
*
* The external delegate also has Tensorflow Lite Python bindings. Therefore the dynamic external delegate
* can be directly used with Tensorflow Lite Python APIs.
*
* See tensorflow/lite/delegates/external for details or visit the tensorflow guide
* [here](https://www.tensorflow.org/lite/performance/implementing_delegate#option_2_leverage_external_delegate)
*/
extern "C"
{
std::vector<std::string> gpu_options {"gpu-tuning-level",
"gpu-tuning-file",
"gpu-kernel-profiling-enabled"};
/**
* Create an ArmNN delegate plugin
*
* Available options:
*
* Option key: "backends" \n
* Possible values: ["EthosNPU"/"GpuAcc"/"CpuAcc"/"CpuRef"] \n
* Descriptions: A comma separated list without whitespaces of
* backends which should be used for execution. Falls
* back to next backend in list if previous doesn't
* provide support for operation. e.g. "GpuAcc,CpuAcc"
*
* Option key: "logging-severity" \n
* Possible values: ["trace"/"debug"/"info"/"warning"/"error"/"fatal"] \n
* Description: Sets the logging severity level for ArmNN. Logging
* is turned off if this option is not provided.
*
* Option key: "gpu-tuning-level" \n
* Possible values: ["0"/"1"/"2"/"3"] \n
* Description: 0=UseOnly(default), 1=RapidTuning, 2=NormalTuning,
* 3=ExhaustiveTuning. Requires option gpu-tuning-file.
* 1,2 and 3 will create a tuning-file, 0 will apply the
* tunings from an existing file
*
* Option key: "gpu-tuning-file" \n
* Possible values: [filenameString] \n
* Description: File name for the tuning file.
*
* Option key: "gpu-kernel-profiling-enabled" \n
* Possible values: ["true"/"false"] \n
* Description: Enables GPU kernel profiling
*
* Option key: "reduce-fp32-to-fp16" \n
* Possible values: ["true"/"false"] \n
* Description: Reduce Fp32 data to Fp16 for faster processing
*
* Option key: "reduce-fp32-to-bf16" \n
* Possible values: ["true"/"false"] \n
* Description: Reduce Fp32 data to Bf16 for faster processing
*
* Option key: "debug-data" \n
* Possible values: ["true"/"false"] \n
* Description: Add debug data for easier troubleshooting
*
* Option key: "memory-import" \n
* Possible values: ["true"/"false"] \n
* Description: Enable memory import
*
*
* @param[in] option_keys Delegate option names
* @param[in] options_values Delegate option values
* @param[in] num_options Number of delegate options
* @param[in,out] report_error Error callback function
*
* @return An ArmNN delegate if it succeeds else NULL
*/
TfLiteDelegate* tflite_plugin_create_delegate(char** options_keys,
char** options_values,
size_t num_options,
void (*report_error)(const char*))
{
// Returning null indicates an error during delegate creation so we initialize with that
TfLiteDelegate* delegate = nullptr;
try
{
// (Initializes with CpuRef backend)
armnnDelegate::DelegateOptions options = armnnDelegate::TfLiteArmnnDelegateOptionsDefault();
armnn::OptimizerOptions optimizerOptions;
for (size_t i = 0; i < num_options; ++i)
{
// Process backends
if (std::string(options_keys[i]) == std::string("backends"))
{
// The backend option is a comma separated string of backendIDs that needs to be split
std::vector<armnn::BackendId> backends;
char* pch;
pch = strtok(options_values[i],",");
while (pch != NULL)
{
backends.push_back(pch);
pch = strtok (NULL, ",");
}
options.SetBackends(backends);
}
// Process logging level
else if (std::string(options_keys[i]) == std::string("logging-severity"))
{
options.SetLoggingSeverity(options_values[i]);
}
// Process GPU backend options
else if (std::string(options_keys[i]) == std::string("gpu-tuning-level"))
{
armnn::BackendOptions option("GpuAcc", {{"TuningLevel", atoi(options_values[i])}});
options.AddBackendOption(option);
}
else if (std::string(options_keys[i]) == std::string("gpu-mlgo-tuning-file"))
{
armnn::BackendOptions option("GpuAcc", {{"MLGOTuningFilePath", std::string(options_values[i])}});
options.AddBackendOption(option);
}
else if (std::string(options_keys[i]) == std::string("gpu-tuning-file"))
{
armnn::BackendOptions option("GpuAcc", {{"TuningFile", std::string(options_values[i])}});
options.AddBackendOption(option);
}
else if (std::string(options_keys[i]) == std::string("gpu-kernel-profiling-enabled"))
{
armnn::BackendOptions option("GpuAcc", {{"KernelProfilingEnabled", (*options_values[i] != '0')}});
options.AddBackendOption(option);
}
// Process reduce-fp32-to-fp16 option
else if (std::string(options_keys[i]) == std::string("reduce-fp32-to-fp16"))
{
optimizerOptions.m_ReduceFp32ToFp16 = *options_values[i] != '0';
}
// Process reduce-fp32-to-bf16 option
else if (std::string(options_keys[i]) == std::string("reduce-fp32-to-bf16"))
{
optimizerOptions.m_ReduceFp32ToBf16 = *options_values[i] != '0';
}
// Process debug-data
else if (std::string(options_keys[i]) == std::string("debug-data"))
{
optimizerOptions.m_Debug = *options_values[i] != '0';
}
// Process memory-import
else if (std::string(options_keys[i]) == std::string("memory-import"))
{
optimizerOptions.m_ImportEnabled = *options_values[i] != '0';
}
else
{
throw armnn::Exception("Unknown option for the ArmNN Delegate given: " + std::string(options_keys[i]));
}
}
options.SetOptimizerOptions(optimizerOptions);
delegate = TfLiteArmnnDelegateCreate(options);
}
catch (const std::exception& ex)
{
if(report_error)
{
report_error(ex.what());
}
}
return delegate;
}
/** Destroy a given delegate plugin
*
* @param[in] delegate Delegate to destruct
*/
void tflite_plugin_destroy_delegate(TfLiteDelegate* delegate)
{
armnnDelegate::TfLiteArmnnDelegateDelete(delegate);
}
} // extern "C"
} // namespace tflite
|
