19 std::string testCasesFile;
21 std::string modelFormat;
22 std::string modelPath;
23 std::string inputNames;
24 std::string inputTensorShapes;
25 std::string inputTensorDataFilePaths;
26 std::string outputNames;
27 std::string inputTypes;
28 std::string outputTypes;
29 std::string dynamicBackendsPath;
30 std::string outputTensorFiles;
33 std::string outgoingCaptureFile;
34 std::string incomingCaptureFile;
35 uint32_t counterCapturePeriod;
36 std::string fileFormat;
38 size_t iterations = 1;
40 std::string tuningPath;
42 double thresholdTime = 0.0;
44 size_t subgraphId = 0;
46 const std::string backendsMessage =
"REQUIRED: Which device to run layers on by default. Possible choices: " 48 po::options_description desc(
"Options");
52 (
"help",
"Display usage information")
53 (
"compute,c", po::value<std::vector<std::string>>()->multitoken()->required(),
54 backendsMessage.c_str())
55 (
"test-cases,t", po::value(&testCasesFile),
"Path to a CSV file containing test cases to run. " 56 "If set, further parameters -- with the exception of compute device and concurrency -- will be ignored, " 57 "as they are expected to be defined in the file for each test in particular.")
58 (
"concurrent,n", po::bool_switch()->default_value(
false),
59 "Whether or not the test cases should be executed in parallel")
60 (
"model-format,f", po::value(&modelFormat)->required(),
61 "armnn-binary, caffe-binary, caffe-text, onnx-binary, onnx-text, tflite-binary, tensorflow-binary or " 63 (
"model-path,m", po::value(&modelPath)->required(),
"Path to model file, e.g. .armnn, .caffemodel, " 64 ".prototxt, .tflite, .onnx")
65 (
"dynamic-backends-path,b", po::value(&dynamicBackendsPath),
66 "Path where to load any available dynamic backend from. " 67 "If left empty (the default), dynamic backends will not be used.")
68 (
"input-name,i", po::value(&inputNames),
69 "Identifier of the input tensors in the network separated by comma.")
70 (
"subgraph-number,x", po::value<size_t>(&subgraphId)->default_value(0),
"Id of the subgraph to be executed." 72 (
"input-tensor-shape,s", po::value(&inputTensorShapes),
73 "The shape of the input tensors in the network as a flat array of integers separated by comma." 74 "Several shapes can be passed by separating them with a colon (:)." 75 "This parameter is optional, depending on the network.")
76 (
"input-tensor-data,d", po::value(&inputTensorDataFilePaths)->default_value(
""),
77 "Path to files containing the input data as a flat array separated by whitespace. " 78 "Several paths can be passed by separating them with a comma. If not specified, the network will be run " 79 "with dummy data (useful for profiling).")
80 (
"input-type,y",po::value(&inputTypes),
"The type of the input tensors in the network separated by comma. " 81 "If unset, defaults to \"float\" for all defined inputs. " 82 "Accepted values (float, int or qasymm8)")
83 (
"quantize-input,q",po::bool_switch()->default_value(
false),
84 "If this option is enabled, all float inputs will be quantized to qasymm8. " 85 "If unset, default to not quantized. " 86 "Accepted values (true or false)")
87 (
"output-type,z",po::value(&outputTypes),
88 "The type of the output tensors in the network separated by comma. " 89 "If unset, defaults to \"float\" for all defined outputs. " 90 "Accepted values (float, int or qasymm8).")
91 (
"dequantize-output,l",po::bool_switch()->default_value(
false),
92 "If this option is enabled, all quantized outputs will be dequantized to float. " 93 "If unset, default to not get dequantized. " 94 "Accepted values (true or false)")
95 (
"output-name,o", po::value(&outputNames),
96 "Identifier of the output tensors in the network separated by comma.")
97 (
"write-outputs-to-file,w", po::value(&outputTensorFiles),
98 "Comma-separated list of output file paths keyed with the binding-id of the output slot. " 99 "If left empty (the default), the output tensors will not be written to a file.")
100 (
"event-based-profiling,e", po::bool_switch()->default_value(
false),
101 "Enables built in profiler. If unset, defaults to off.")
102 (
"visualize-optimized-model,v", po::bool_switch()->default_value(
false),
103 "Enables built optimized model visualizer. If unset, defaults to off.")
104 (
"bf16-turbo-mode", po::bool_switch()->default_value(
false),
"If this option is enabled, FP32 layers, " 105 "weights and biases will be converted to BFloat16 where the backend supports it")
106 (
"fp16-turbo-mode,h", po::bool_switch()->default_value(
false),
"If this option is enabled, FP32 layers, " 107 "weights and biases will be converted to FP16 where the backend supports it")
108 (
"threshold-time,r", po::value<double>(&thresholdTime)->default_value(0.0),
109 "Threshold time is the maximum allowed time for inference measured in milliseconds. If the actual " 110 "inference time is greater than the threshold time, the test will fail. By default, no threshold " 112 (
"print-intermediate-layers,p", po::bool_switch()->default_value(
false),
113 "If this option is enabled, the output of every graph layer will be printed.")
114 (
"enable-external-profiling,a", po::bool_switch()->default_value(
false),
115 "If enabled external profiling will be switched on")
116 (
"timeline-profiling", po::bool_switch()->default_value(
false),
117 "If enabled timeline profiling will be switched on, requires external profiling")
118 (
"outgoing-capture-file,j", po::value(&outgoingCaptureFile),
119 "If specified the outgoing external profiling packets will be captured in this binary file")
120 (
"incoming-capture-file,k", po::value(&incomingCaptureFile),
121 "If specified the incoming external profiling packets will be captured in this binary file")
122 (
"file-only-external-profiling,g", po::bool_switch()->default_value(
false),
123 "If enabled then the 'file-only' test mode of external profiling will be enabled")
124 (
"counter-capture-period,u", po::value<uint32_t>(&counterCapturePeriod)->default_value(150u),
125 "If profiling is enabled in 'file-only' mode this is the capture period that will be used in the test")
126 (
"file-format", po::value(&fileFormat)->default_value(
"binary"),
127 "If profiling is enabled specifies the output file format")
128 (
"iterations", po::value<size_t>(&iterations)->default_value(1),
129 "Number of iterations to run the network for, default is set to 1")
130 (
"tuning-path", po::value(&tuningPath),
131 "Path to tuning file. Enables use of CL tuning")
132 (
"tuning-level", po::value<int>(&tuningLevel)->default_value(0),
133 "Sets the tuning level which enables a tuning run which will update/create a tuning file. " 134 "Available options are: 1 (Rapid), 2 (Normal), 3 (Exhaustive). " 135 "Requires tuning-path to be set, default is set to 0 (No tuning run)")
136 (
"parse-unsupported", po::bool_switch()->default_value(
false),
137 "Add unsupported operators as stand-in layers (where supported by parser)")
138 (
"infer-output-shape", po::bool_switch()->default_value(
false),
139 "Infers output tensor shape from input tensor shape and validate where applicable (where supported by " 142 catch (
const std::exception& e)
148 ARMNN_LOG(fatal) <<
"Fatal internal error: " << e.what();
153 po::variables_map vm;
156 po::store(po::parse_command_line(argc, argv, desc), vm);
158 if (CheckOption(vm,
"help") || argc <= 1)
160 std::cout <<
"Executes a neural network model using the provided input tensor. " << std::endl;
161 std::cout <<
"Prints the resulting output tensor." << std::endl;
162 std::cout << std::endl;
163 std::cout << desc << std::endl;
169 catch (
const po::error& e)
171 std::cerr << e.what() << std::endl << std::endl;
172 std::cerr << desc << std::endl;
177 bool concurrent = vm[
"concurrent"].as<
bool>();
178 bool enableProfiling = vm[
"event-based-profiling"].as<
bool>();
179 bool enableLayerDetails = vm[
"visualize-optimized-model"].as<
bool>();
180 bool enableBf16TurboMode = vm[
"bf16-turbo-mode"].as<
bool>();
181 bool enableFp16TurboMode = vm[
"fp16-turbo-mode"].as<
bool>();
182 bool quantizeInput = vm[
"quantize-input"].as<
bool>();
183 bool dequantizeOutput = vm[
"dequantize-output"].as<
bool>();
184 bool printIntermediate = vm[
"print-intermediate-layers"].as<
bool>();
185 bool enableExternalProfiling = vm[
"enable-external-profiling"].as<
bool>();
186 bool fileOnlyExternalProfiling = vm[
"file-only-external-profiling"].as<
bool>();
187 bool parseUnsupported = vm[
"parse-unsupported"].as<
bool>();
188 bool timelineEnabled = vm[
"timeline-profiling"].as<
bool>();
189 bool inferOutputShape = vm[
"infer-output-shape"].as<
bool>();
191 if (enableBf16TurboMode && enableFp16TurboMode)
193 ARMNN_LOG(fatal) <<
"BFloat16 and Float16 turbo mode cannot be enabled at the same time.";
209 if (timelineEnabled && !enableExternalProfiling)
211 ARMNN_LOG(fatal) <<
"Timeline profiling requires external profiling to be turned on";
216 if (CheckOption(vm,
"test-cases"))
219 if (!fs::exists(testCasesFile))
221 ARMNN_LOG(fatal) <<
"Given file \"" << testCasesFile <<
"\" does not exist";
227 std::vector<armnnUtils::CsvRow> testCases = reader.
ParseFile(testCasesFile);
230 if (testCases.empty())
232 ARMNN_LOG(fatal) <<
"Given file \"" << testCasesFile <<
"\" has no test cases";
238 const std::string executableName(
"ExecuteNetwork");
243 std::vector<std::future<int>> results;
244 results.reserve(testCases.size());
247 for (
auto& testCase : testCases)
249 testCase.values.insert(testCase.values.begin(), executableName);
250 results.push_back(std::async(std::launch::async,
RunCsvTest, std::cref(testCase), std::cref(runtime),
251 enableProfiling, enableFp16TurboMode, enableBf16TurboMode, thresholdTime,
252 printIntermediate, enableLayerDetails, parseUnsupported,
257 for (
auto& result : results)
259 if (result.get() != EXIT_SUCCESS)
268 for (
auto& testCase : testCases)
270 testCase.values.insert(testCase.values.begin(), executableName);
271 if (
RunCsvTest(testCase, runtime, enableProfiling,
272 enableFp16TurboMode, enableBf16TurboMode, thresholdTime, printIntermediate,
273 enableLayerDetails, parseUnsupported, inferOutputShape) != EXIT_SUCCESS)
285 const std::string computeOption(
"compute");
286 std::vector<std::string> computeDevicesAsStrings =
287 CheckOption(vm, computeOption.c_str()) ?
288 vm[computeOption].as<std::vector<std::string>>() :
289 std::vector<
std::string>();
290 std::vector<armnn::BackendId> computeDevices(computeDevicesAsStrings.begin(), computeDevicesAsStrings.end());
293 RemoveDuplicateDevices(computeDevices);
295 #if defined(ARMCOMPUTECL_ENABLED) 296 std::shared_ptr<armnn::IGpuAccTunedParameters> tuned_params;
298 if (tuningPath !=
"")
300 if (tuningLevel != 0)
302 RunCLTuning(tuningPath, tuningLevel, modelFormat, inputTensorShapes, computeDevices,
303 dynamicBackendsPath, modelPath, inputNames, inputTensorDataFilePaths, inputTypes, quantizeInput,
304 outputTypes, outputNames, outputTensorFiles, dequantizeOutput, enableProfiling,
305 enableFp16TurboMode, enableBf16TurboMode, thresholdTime, printIntermediate, subgraphId,
306 enableLayerDetails, parseUnsupported, inferOutputShape);
308 ARMNN_LOG(info) <<
"Using tuning params: " << tuningPath <<
"\n";
315 {
"TuningFile", tuningPath.c_str()},
316 {
"KernelProfilingEnabled", enableProfiling}
324 CheckOptionDependencies(vm);
326 catch (
const po::error& e)
328 std::cerr << e.what() << std::endl << std::endl;
329 std::cerr << desc << std::endl;
335 return RunTest(modelFormat, inputTensorShapes, computeDevices, dynamicBackendsPath, modelPath,
336 inputNames, inputTensorDataFilePaths, inputTypes, quantizeInput, outputTypes, outputNames,
337 outputTensorFiles, dequantizeOutput, enableProfiling, enableFp16TurboMode, enableBf16TurboMode,
338 thresholdTime, printIntermediate, subgraphId, enableLayerDetails, parseUnsupported, inferOutputShape,
339 iterations, runtime);
static IRuntimePtr Create(const CreationOptions &options)
int RunCsvTest(const armnnUtils::CsvRow &csvRow, const std::shared_ptr< armnn::IRuntime > &runtime, const bool enableProfiling, const bool enableFp16TurboMode, const bool enableBf16TurboMode, const double &thresholdTime, const bool printIntermediate, bool enableLayerDetails=false, bool parseUnuspported=false, bool inferOutputShape=false)
std::string m_OutgoingCaptureFile
int RunCLTuning(const std::string &tuningPath, const int tuningLevel, const std::string &modelFormat, const std::string &inputTensorShapes, const vector< armnn::BackendId > &computeDevices, const std::string &dynamicBackendsPath, const std::string &modelPath, const std::string &inputNames, const std::string &inputTensorDataFilePaths, const std::string &inputTypes, bool quantizeInput, const std::string &outputTypes, const std::string &outputNames, const std::string &outputTensorFiles, bool dequantizeOutput, bool enableProfiling, bool enableFp16TurboMode, bool enableBf16TurboMode, const double &thresholdTime, bool printIntermediate, const size_t subgraphId, bool enableLayerDetails=false, bool parseUnsupported=false, bool inferOutputShape=false)
void ConfigureLogging(bool printToStandardOutput, bool printToDebugOutput, LogSeverity severity)
Configures the logging behaviour of the ARMNN library.
int RunTest(const std::string &format, const std::string &inputTensorShapesStr, const vector< armnn::BackendId > &computeDevices, const std::string &dynamicBackendsPath, const std::string &path, const std::string &inputNames, const std::string &inputTensorDataFilePaths, const std::string &inputTypes, bool quantizeInput, const std::string &outputTypes, const std::string &outputNames, const std::string &outputTensorFiles, bool dequantizeOuput, bool enableProfiling, bool enableFp16TurboMode, bool enableBf16TurboMode, const double &thresholdTime, bool printIntermediate, const size_t subgraphId, bool enableLayerDetails=false, bool parseUnsupported=false, bool inferOutputShape=false, const size_t iterations=1, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
#define ARMNN_LOG(severity)
BackendRegistry & BackendRegistryInstance()
std::string GetBackendIdsAsString() const
std::string m_IncomingCaptureFile
std::vector< BackendOptions > m_BackendOptions
Pass backend specific options.
#define ARMNN_ASSERT_MSG(COND, MSG)
Struct for the users to pass backend specific options.
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
bool m_EnableGpuProfiling
Setting this flag will allow the user to obtain GPU profiling information from the runtime...
static std::vector< CsvRow > ParseFile(const std::string &csvFile)
ExternalProfilingOptions m_ProfilingOptions