+ + + + + + + + + + + + + + + + +

+Classes
class	ClassifierTestCase

class	ClassifierTestCaseProvider

class	IInferenceTestCase

class	IInferenceTestCaseProvider

class	InferenceModelTestCase

struct	InferenceTestOptions

class	TestFrameworkException

+ + + +

+Typedefs
using	TContainer = mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > >

+ + + +

+Enumerations
enum	TestCaseResult { Ok, +Failed, +Abort + }

+ + + + + + + + + + + + + + +

+Functions
bool	ParseCommandLine (int argc, char **argv, IInferenceTestCaseProvider &testCaseProvider, InferenceTestOptions &outParams)
	Parse the command line of an ArmNN (or referencetests) inference test program. More...

bool	ValidateDirectory (std::string &dir)

bool	InferenceTest (const InferenceTestOptions &params, const std::vector< unsigned int > &defaultTestCaseIds, IInferenceTestCaseProvider &testCaseProvider)

template<typename TConstructTestCaseProvider >
int	InferenceTestMain (int argc, char *argv[], const std::vector< unsigned int > &defaultTestCaseIds, TConstructTestCaseProvider constructTestCaseProvider)

template<typename TDatabase , typename TParser , typename TConstructDatabaseCallable >
int	ClassifierInferenceTestMain (int argc, char argv[], const char modelFilename, bool isModelBinary, const char inputBindingName, const char outputBindingName, const std::vector< unsigned int > &defaultTestCaseIds, TConstructDatabaseCallable constructDatabase, const armnn::TensorShape *inputTensorShape=nullptr)

+

Typedef Documentation

+ +

◆ TContainer

+ +

+

+ + + + +

using TContainer = mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char> >

+

+ +

Definition at line 29 of file InferenceTest.inl.

+ +

+

Enumeration Type Documentation

+ +

◆ TestCaseResult

+ +

+

+ + + + + +

+ + + + +

enum TestCaseResult

+

+strong

+

+ + + + +

Enumerator

Ok

The test completed without any errors.

+

Failed

The test failed (e.g.

+

the prediction didn't match the validation file). This will eventually fail the whole program but the remaining test cases will still be run.

+

Abort

The test failed with a fatal error. The remaining tests will not be run.

+

+ +

Definition at line 71 of file InferenceTest.hpp.

+

 {
     /// The test completed without any errors.
     Ok,
     /// The test failed (e.g. the prediction didn't match the validation file).
     /// This will eventually fail the whole program but the remaining test cases will still be run.
     Failed,
     /// The test failed with a fatal error. The remaining tests will not be run.
     Abort
 };

+
+
+

+

Function Documentation

+ +

◆ ClassifierInferenceTestMain()

+ +

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

int ClassifierInferenceTestMain	(	int	argc,
		char *	argv[],
		const char *	modelFilename,
		bool	isModelBinary,
		const char *	inputBindingName,
		const char *	outputBindingName,
		const std::vector< unsigned int > &	defaultTestCaseIds,
		TConstructDatabaseCallable	constructDatabase,
		const armnn::TensorShape *	inputTensorShape = `nullptr`
	)

+

+ +

Definition at line 353 of file InferenceTest.inl.

+ +

References ARMNN_ASSERT, InferenceTestMain(), Params::m_ModelPath, and ValidateDirectory().

+ +

Referenced by main().

+

 {
     ARMNN_ASSERT(modelFilename);
     ARMNN_ASSERT(inputBindingName);
     ARMNN_ASSERT(outputBindingName);
 
     return InferenceTestMain(argc, argv, defaultTestCaseIds,
         [=]
         ()
         {
             using InferenceModel = InferenceModel<TParser, typename TDatabase::DataType>;
             using TestCaseProvider = ClassifierTestCaseProvider<TDatabase, InferenceModel>;
 
             return make_unique<TestCaseProvider>(constructDatabase,
                 [&]
                 (const InferenceTestOptions &commonOptions,
                  typename InferenceModel::CommandLineOptions modelOptions)
                 {
                     if (!ValidateDirectory(modelOptions.m_ModelDir))
                     {
                         return std::unique_ptr<InferenceModel>();
                     }
 
                     typename InferenceModel::Params modelParams;
                     modelParams.m_ModelPath = modelOptions.m_ModelDir + modelFilename;
                     modelParams.m_InputBindings  = { inputBindingName };
                     modelParams.m_OutputBindings = { outputBindingName };
 
                     if (inputTensorShape)
                     {
                         modelParams.m_InputShapes.push_back(*inputTensorShape);
                     }
 
                     modelParams.m_IsModelBinary = isModelBinary;
                     modelParams.m_ComputeDevices = modelOptions.GetComputeDevicesAsBackendIds();
                     modelParams.m_VisualizePostOptimizationModel = modelOptions.m_VisualizePostOptimizationModel;
                     modelParams.m_EnableFp16TurboMode = modelOptions.m_EnableFp16TurboMode;
 
                     return std::make_unique<InferenceModel>(modelParams,
                                                             commonOptions.m_EnableProfiling,
                                                             commonOptions.m_DynamicBackendsPath);
             });
         });
 }

+
+
+
+
+
+
+
+
+

+

+ +

◆ InferenceTest()

+ +

+

+ + + + + + + + + + + + + + + + + + + + + + + + +

bool InferenceTest	(	const InferenceTestOptions &	params,
		const std::vector< unsigned int > &	defaultTestCaseIds,
		IInferenceTestCaseProvider &	testCaseProvider
	)

+

+ +

Definition at line 115 of file InferenceTest.cpp.

+ +

References ARMNN_ASSERT_MSG, ARMNN_LOG, armnn::error, IInferenceTestCaseProvider::GetTestCase(), armnn::info, InferenceTestOptions::m_EnableProfiling, InferenceTestOptions::m_InferenceTimesFile, InferenceTestOptions::m_IterationCount, IInferenceTestCaseProvider::OnInferenceTestFinished(), armnn::warning, and Exception::what().

+ +

Referenced by InferenceTestMain().

+

 {
 #if !defined (NDEBUG)
     if (params.m_IterationCount > 0) // If just running a few select images then don't bother to warn.
     {
         ARMNN_LOG(warning) << "Performance test running in DEBUG build - results may be inaccurate.";
     }
 #endif
 
     double totalTime = 0;
     unsigned int nbProcessed = 0;
     bool success = true;
 
     // Opens the file to write inference times too, if needed.
     ofstream inferenceTimesFile;
     const bool recordInferenceTimes = !params.m_InferenceTimesFile.empty();
     if (recordInferenceTimes)
     {
         inferenceTimesFile.open(params.m_InferenceTimesFile.c_str(), ios_base::trunc | ios_base::out);
         if (!inferenceTimesFile.good())
         {
             ARMNN_LOG(error) << "Failed to open inference times file for writing: "
                 << params.m_InferenceTimesFile;
             return false;
         }
     }
 
     // Create a profiler and register it for the current thread.
     std::unique_ptr<IProfiler> profiler = std::make_unique<IProfiler>();
     ProfilerManager::GetInstance().RegisterProfiler(profiler.get());
 
     // Enable profiling if requested.
     profiler->EnableProfiling(params.m_EnableProfiling);
 
     // Run a single test case to 'warm-up' the model. The first one can sometimes take up to 10x longer
     std::unique_ptr<IInferenceTestCase> warmupTestCase = testCaseProvider.GetTestCase(0);
     if (warmupTestCase == nullptr)
     {
         ARMNN_LOG(error) << "Failed to load test case";
         return false;
     }
 
     try
     {
         warmupTestCase->Run();
     }
     catch (const TestFrameworkException& testError)
     {
         ARMNN_LOG(error) << testError.what();
         return false;
     }
 
     const unsigned int nbTotalToProcess = params.m_IterationCount > 0 ? params.m_IterationCount
         : static_cast<unsigned int>(defaultTestCaseIds.size());
 
     for (; nbProcessed < nbTotalToProcess; nbProcessed++)
     {
         const unsigned int testCaseId = params.m_IterationCount > 0 ? nbProcessed : defaultTestCaseIds[nbProcessed];
         std::unique_ptr<IInferenceTestCase> testCase = testCaseProvider.GetTestCase(testCaseId);
 
         if (testCase == nullptr)
         {
             ARMNN_LOG(error) << "Failed to load test case";
             return false;
         }
 
         time_point<high_resolution_clock> predictStart;
         time_point<high_resolution_clock> predictEnd;
 
         TestCaseResult result = TestCaseResult::Ok;
 
         try
         {
             predictStart = high_resolution_clock::now();
 
             testCase->Run();
 
             predictEnd = high_resolution_clock::now();
 
             // duration<double> will convert the time difference into seconds as a double by default.
             double timeTakenS = duration<double>(predictEnd - predictStart).count();
             totalTime += timeTakenS;
 
             // Outputss inference times, if needed.
             if (recordInferenceTimes)
             {
                 inferenceTimesFile << testCaseId << " " << (timeTakenS * 1000.0) << std::endl;
             }
 
             result = testCase->ProcessResult(params);
 
         }
         catch (const TestFrameworkException& testError)
         {
             ARMNN_LOG(error) << testError.what();
             result = TestCaseResult::Abort;
         }
 
         switch (result)
         {
         case TestCaseResult::Ok:
             break;
         case TestCaseResult::Abort:
             return false;
         case TestCaseResult::Failed:
             // This test failed so we will fail the entire program eventually, but keep going for now.
             success = false;
             break;
         default:
             ARMNN_ASSERT_MSG(false, "Unexpected TestCaseResult");
             return false;
         }
     }
 
     const double averageTimePerTestCaseMs = totalTime / nbProcessed * 1000.0f;
 
     ARMNN_LOG(info) << std::fixed << std::setprecision(3) <<
         "Total time for " << nbProcessed << " test cases: " << totalTime << " seconds";
     ARMNN_LOG(info) << std::fixed << std::setprecision(3) <<
         "Average time per test case: " << averageTimePerTestCaseMs << " ms";
 
     // if profiling is enabled print out the results
     if (profiler && profiler->IsProfilingEnabled())
     {
         profiler->Print(std::cout);
     }
 
     if (!success)
     {
         ARMNN_LOG(error) << "One or more test cases failed";
         return false;
     }
 
     return testCaseProvider.OnInferenceTestFinished();
 }

+
+
+
+
+
+
+
+
+
+

+

+ +

◆ InferenceTestMain()

+ +

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

int InferenceTestMain	(	int	argc,
		char *	argv[],
		const std::vector< unsigned int > &	defaultTestCaseIds,
		TConstructTestCaseProvider	constructTestCaseProvider
	)

+

+ +

Definition at line 302 of file InferenceTest.inl.

+ +

References ARMNN_LOG, armnn::ConfigureLogging(), armnn::Debug, armnn::fatal, InferenceTest(), armnn::Info, ParseCommandLine(), and Exception::what().

+ +

Referenced by ClassifierInferenceTestMain(), and main().

+

 {
     // Configures logging for both the ARMNN library and this test program.
 #ifdef NDEBUG
     armnn::LogSeverity level = armnn::LogSeverity::Info;
 #else
     armnn::LogSeverity level = armnn::LogSeverity::Debug;
 #endif
     armnn::ConfigureLogging(true, true, level);
 
     try
     {
         std::unique_ptr<IInferenceTestCaseProvider> testCaseProvider = constructTestCaseProvider();
         if (!testCaseProvider)
         {
             return 1;
         }
 
         InferenceTestOptions inferenceTestOptions;
         if (!ParseCommandLine(argc, argv, *testCaseProvider, inferenceTestOptions))
         {
             return 1;
         }
 
         const bool success = InferenceTest(inferenceTestOptions, defaultTestCaseIds, *testCaseProvider);
         return success ? 0 : 1;
     }
     catch (armnn::Exception const& e)
     {
         ARMNN_LOG(fatal) << "Armnn Error: " << e.what();
         return 1;
     }
 }

+
+
+
+
+
+
+
+
+
+

+

+ +

◆ ParseCommandLine()

+ +

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

bool ParseCommandLine	(	int	argc,
		char **	argv,
		IInferenceTestCaseProvider &	testCaseProvider,
		InferenceTestOptions &	outParams
	)

+

+ +

Parse the command line of an ArmNN (or referencetests) inference test program.

+

Returns: false if any error occurred during options processing, otherwise true

+ +

Definition at line 28 of file InferenceTest.cpp.

+ +

References IInferenceTestCaseProvider::AddCommandLineOptions(), ARMNN_ASSERT_MSG, CheckRequiredOptions(), InferenceTestOptions::m_EnableProfiling, InferenceTestOptions::m_InferenceTimesFile, InferenceTestOptions::m_IterationCount, and IInferenceTestCaseProvider::ProcessCommandLineOptions().

+ +

Referenced by InferenceTestMain().

+

 {
     cxxopts::Options options("InferenceTest", "Inference iteration parameters");
 
     try
     {
         // Adds generic options needed for all inference tests.
         options
             .allow_unrecognised_options()
             .add_options()
                 ("h,help", "Display help messages")
                 ("i,iterations", "Sets the number of inferences to perform. If unset, will only be run once.",
                  cxxopts::value<unsigned int>(outParams.m_IterationCount)->default_value("0"))
                 ("inference-times-file",
                  "If non-empty, each individual inference time will be recorded and output to this file",
                  cxxopts::value<std::string>(outParams.m_InferenceTimesFile)->default_value(""))
                 ("e,event-based-profiling", "Enables built in profiler. If unset, defaults to off.",
                  cxxopts::value<bool>(outParams.m_EnableProfiling)->default_value("0"));
 
         std::vector<std::string> required; //to be passed as reference to derived inference tests
 
         // Adds options specific to the ITestCaseProvider.
         testCaseProvider.AddCommandLineOptions(options, required);
 
         auto result = options.parse(argc, argv);
 
         if (result.count("help"))
         {
             std::cout << options.help() << std::endl;
             return false;
         }
 
         CheckRequiredOptions(result, required);
 
     }
     catch (const cxxopts::OptionException& e)
     {
         std::cerr << e.what() << std::endl << options.help() << std::endl;
         return false;
     }
     catch (const std::exception& e)
     {
         // Coverity points out that default_value(...) can throw a bad_lexical_cast,
         // and that desc.add_options() can throw boost::io::too_few_args.
         // They really won't in any of these cases.
         ARMNN_ASSERT_MSG(false, "Caught unexpected exception");
         std::cerr << "Fatal internal error: " << e.what() << std::endl;
         return false;
     }
 
     if (!testCaseProvider.ProcessCommandLineOptions(outParams))
     {
         return false;
     }
 
     return true;
 }

+
+
+
+
+
+
+

+

+ +

◆ ValidateDirectory()

+ +

+

+ + + + + + + + +

bool ValidateDirectory ( std::string & dir )

+

+ +

Definition at line 87 of file InferenceTest.cpp.

+ +

Referenced by ClassifierInferenceTestMain(), main(), ClassifierTestCaseProvider< TDatabase, InferenceModel >::ProcessCommandLineOptions(), and YoloTestCaseProvider< Model >::ProcessCommandLineOptions().

+

 {
     if (dir.empty())
     {
         std::cerr << "No directory specified" << std::endl;
         return false;
     }
 
     if (dir[dir.length() - 1] != '/')
     {
         dir += "/";
     }
 
     if (!fs::exists(dir))
     {
         std::cerr << "Given directory " << dir << " does not exist" << std::endl;
         return false;
     }
 
     if (!fs::is_directory(dir))
     {
         std::cerr << "Given directory [" << dir << "] is not a directory" << std::endl;
         return false;
     }
 
     return true;
 }

+

+Classes

+Typedefs

+Enumerations

+Functions

Typedef Documentation

◆ TContainer

Enumeration Type Documentation

◆ TestCaseResult

Function Documentation

◆ ClassifierInferenceTestMain()

◆ InferenceTest()

◆ InferenceTestMain()

◆ ParseCommandLine()

◆ ValidateDirectory()