armnn::test Namespace Reference

Classes
class	ClassifierTestCase
class	ClassifierTestCaseProvider
class	IInferenceTestCase
class	IInferenceTestCaseProvider
class	InferenceModelTestCase
struct	InferenceTestOptions
class	TestFrameworkException

Typedefs
using	TContainer = mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > >

Enumerations
enum	TestCaseResult { Ok, Failed, Abort }

Functions
bool	ParseCommandLine (int argc, char **argv, IInferenceTestCaseProvider &testCaseProvider, InferenceTestOptions &outParams)
	Parse the command line of an ArmNN (or referencetests) inference test program. More...
bool	ValidateDirectory (std::string &dir)
bool	InferenceTest (const InferenceTestOptions &params, const std::vector< unsigned int > &defaultTestCaseIds, IInferenceTestCaseProvider &testCaseProvider)
template<typename TConstructTestCaseProvider >
int	InferenceTestMain (int argc, char *argv[], const std::vector< unsigned int > &defaultTestCaseIds, TConstructTestCaseProvider constructTestCaseProvider)
template<typename TDatabase , typename TParser , typename TConstructDatabaseCallable >
int	ClassifierInferenceTestMain (int argc, char *argv[], const char *modelFilename, bool isModelBinary, const char *inputBindingName, const char *outputBindingName, const std::vector< unsigned int > &defaultTestCaseIds, TConstructDatabaseCallable constructDatabase, const armnn::TensorShape *inputTensorShape=nullptr)

Typedef Documentation

TContainer

using TContainer = mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char> >

Definition at line 29 of file InferenceTest.inl.

Enumeration Type Documentation

TestCaseResult

enum TestCaseResult

strong

Enumerator
Ok	The test completed without any errors.
Failed	The test failed (e.g. the prediction didn't match the validation file). This will eventually fail the whole program but the remaining test cases will still be run.
Abort	The test failed with a fatal error. The remaining tests will not be run.

Definition at line 71 of file InferenceTest.hpp.

{
    /// The test completed without any errors.
    Ok,
    /// The test failed (e.g. the prediction didn't match the validation file).
    /// This will eventually fail the whole program but the remaining test cases will still be run.
    Failed,
    /// The test failed with a fatal error. The remaining tests will not be run.
    Abort
};

Function Documentation

ClassifierInferenceTestMain()

int ClassifierInferenceTestMain	(	int	argc,
		char *	argv[],
		const char *	modelFilename,
		bool	isModelBinary,
		const char *	inputBindingName,
		const char *	outputBindingName,
		const std::vector< unsigned int > &	defaultTestCaseIds,
		TConstructDatabaseCallable	constructDatabase,
		const armnn::TensorShape *	inputTensorShape = nullptr
	)

Definition at line 353 of file InferenceTest.inl.

References ARMNN_ASSERT, InferenceTestMain(), Params::m_ModelPath, and ValidateDirectory().

Referenced by main().

{
    ARMNN_ASSERT(modelFilename);
    ARMNN_ASSERT(inputBindingName);
    ARMNN_ASSERT(outputBindingName);

    return InferenceTestMain(argc, argv, defaultTestCaseIds,
        [=]
        ()
        {
            using InferenceModel = InferenceModel<TParser, typename TDatabase::DataType>;
            using TestCaseProvider = ClassifierTestCaseProvider<TDatabase, InferenceModel>;

            return make_unique<TestCaseProvider>(constructDatabase,
                [&]
                (const InferenceTestOptions &commonOptions,
                 typename InferenceModel::CommandLineOptions modelOptions)
                {
                    if (!ValidateDirectory(modelOptions.m_ModelDir))
                    {
                        return std::unique_ptr<InferenceModel>();
                    }

                    typename InferenceModel::Params modelParams;
                    modelParams.m_ModelPath = modelOptions.m_ModelDir + modelFilename;
                    modelParams.m_InputBindings  = { inputBindingName };
                    modelParams.m_OutputBindings = { outputBindingName };

                    if (inputTensorShape)
                    {
                        modelParams.m_InputShapes.push_back(*inputTensorShape);
                    }

                    modelParams.m_IsModelBinary = isModelBinary;
                    modelParams.m_ComputeDevices = modelOptions.GetComputeDevicesAsBackendIds();
                    modelParams.m_VisualizePostOptimizationModel = modelOptions.m_VisualizePostOptimizationModel;
                    modelParams.m_EnableFp16TurboMode = modelOptions.m_EnableFp16TurboMode;

                    return std::make_unique<InferenceModel>(modelParams,
                                                            commonOptions.m_EnableProfiling,
                                                            commonOptions.m_DynamicBackendsPath);
            });
        });
}

InferenceTest()

bool InferenceTest	(	const InferenceTestOptions &	params,
		const std::vector< unsigned int > &	defaultTestCaseIds,
		IInferenceTestCaseProvider &	testCaseProvider
	)

Definition at line 115 of file InferenceTest.cpp.

References ARMNN_ASSERT_MSG, ARMNN_LOG, armnn::error, IInferenceTestCaseProvider::GetTestCase(), armnn::info, InferenceTestOptions::m_EnableProfiling, InferenceTestOptions::m_InferenceTimesFile, InferenceTestOptions::m_IterationCount, IInferenceTestCaseProvider::OnInferenceTestFinished(), armnn::warning, and Exception::what().

Referenced by InferenceTestMain().

{
#if !defined (NDEBUG)
    if (params.m_IterationCount > 0) // If just running a few select images then don't bother to warn.
    {
        ARMNN_LOG(warning) << "Performance test running in DEBUG build - results may be inaccurate.";
    }
#endif

    double totalTime = 0;
    unsigned int nbProcessed = 0;
    bool success = true;

    // Opens the file to write inference times too, if needed.
    ofstream inferenceTimesFile;
    const bool recordInferenceTimes = !params.m_InferenceTimesFile.empty();
    if (recordInferenceTimes)
    {
        inferenceTimesFile.open(params.m_InferenceTimesFile.c_str(), ios_base::trunc | ios_base::out);
        if (!inferenceTimesFile.good())
        {
            ARMNN_LOG(error) << "Failed to open inference times file for writing: "
                << params.m_InferenceTimesFile;
            return false;
        }
    }

    // Create a profiler and register it for the current thread.
    std::unique_ptr<IProfiler> profiler = std::make_unique<IProfiler>();
    ProfilerManager::GetInstance().RegisterProfiler(profiler.get());

    // Enable profiling if requested.
    profiler->EnableProfiling(params.m_EnableProfiling);

    // Run a single test case to 'warm-up' the model. The first one can sometimes take up to 10x longer
    std::unique_ptr<IInferenceTestCase> warmupTestCase = testCaseProvider.GetTestCase(0);
    if (warmupTestCase == nullptr)
    {
        ARMNN_LOG(error) << "Failed to load test case";
        return false;
    }

    try
    {
        warmupTestCase->Run();
    }
    catch (const TestFrameworkException& testError)
    {
        ARMNN_LOG(error) << testError.what();
        return false;
    }

    const unsigned int nbTotalToProcess = params.m_IterationCount > 0 ? params.m_IterationCount
        : static_cast<unsigned int>(defaultTestCaseIds.size());

    for (; nbProcessed < nbTotalToProcess; nbProcessed++)
    {
        const unsigned int testCaseId = params.m_IterationCount > 0 ? nbProcessed : defaultTestCaseIds[nbProcessed];
        std::unique_ptr<IInferenceTestCase> testCase = testCaseProvider.GetTestCase(testCaseId);

        if (testCase == nullptr)
        {
            ARMNN_LOG(error) << "Failed to load test case";
            return false;
        }

        time_point<high_resolution_clock> predictStart;
        time_point<high_resolution_clock> predictEnd;

        TestCaseResult result = TestCaseResult::Ok;

        try
        {
            predictStart = high_resolution_clock::now();

            testCase->Run();

            predictEnd = high_resolution_clock::now();

            // duration<double> will convert the time difference into seconds as a double by default.
            double timeTakenS = duration<double>(predictEnd - predictStart).count();
            totalTime += timeTakenS;

            // Outputss inference times, if needed.
            if (recordInferenceTimes)
            {
                inferenceTimesFile << testCaseId << " " << (timeTakenS * 1000.0) << std::endl;
            }

            result = testCase->ProcessResult(params);

        }
        catch (const TestFrameworkException& testError)
        {
            ARMNN_LOG(error) << testError.what();
            result = TestCaseResult::Abort;
        }

        switch (result)
        {
        case TestCaseResult::Ok:
            break;
        case TestCaseResult::Abort:
            return false;
        case TestCaseResult::Failed:
            // This test failed so we will fail the entire program eventually, but keep going for now.
            success = false;
            break;
        default:
            ARMNN_ASSERT_MSG(false, "Unexpected TestCaseResult");
            return false;
        }
    }

    const double averageTimePerTestCaseMs = totalTime / nbProcessed * 1000.0f;

    ARMNN_LOG(info) << std::fixed << std::setprecision(3) <<
        "Total time for " << nbProcessed << " test cases: " << totalTime << " seconds";
    ARMNN_LOG(info) << std::fixed << std::setprecision(3) <<
        "Average time per test case: " << averageTimePerTestCaseMs << " ms";

    // if profiling is enabled print out the results
    if (profiler && profiler->IsProfilingEnabled())
    {
        profiler->Print(std::cout);
    }

    if (!success)
    {
        ARMNN_LOG(error) << "One or more test cases failed";
        return false;
    }

    return testCaseProvider.OnInferenceTestFinished();
}

InferenceTestMain()

int InferenceTestMain	(	int	argc,
		char *	argv[],
		const std::vector< unsigned int > &	defaultTestCaseIds,
		TConstructTestCaseProvider	constructTestCaseProvider
	)

Definition at line 302 of file InferenceTest.inl.

References ARMNN_LOG, armnn::ConfigureLogging(), armnn::Debug, armnn::fatal, InferenceTest(), armnn::Info, ParseCommandLine(), and Exception::what().

Referenced by ClassifierInferenceTestMain(), and main().

{
    // Configures logging for both the ARMNN library and this test program.
#ifdef NDEBUG
    armnn::LogSeverity level = armnn::LogSeverity::Info;
#else
    armnn::LogSeverity level = armnn::LogSeverity::Debug;
#endif
    armnn::ConfigureLogging(true, true, level);

    try
    {
        std::unique_ptr<IInferenceTestCaseProvider> testCaseProvider = constructTestCaseProvider();
        if (!testCaseProvider)
        {
            return 1;
        }

        InferenceTestOptions inferenceTestOptions;
        if (!ParseCommandLine(argc, argv, *testCaseProvider, inferenceTestOptions))
        {
            return 1;
        }

        const bool success = InferenceTest(inferenceTestOptions, defaultTestCaseIds, *testCaseProvider);
        return success ? 0 : 1;
    }
    catch (armnn::Exception const& e)
    {
        ARMNN_LOG(fatal) << "Armnn Error: " << e.what();
        return 1;
    }
}

ParseCommandLine()

bool ParseCommandLine	(	int	argc,
		char **	argv,
		IInferenceTestCaseProvider &	testCaseProvider,
		InferenceTestOptions &	outParams
	)

Parse the command line of an ArmNN (or referencetests) inference test program.

Returns

false if any error occurred during options processing, otherwise true

Definition at line 28 of file InferenceTest.cpp.

References IInferenceTestCaseProvider::AddCommandLineOptions(), ARMNN_ASSERT_MSG, CheckRequiredOptions(), InferenceTestOptions::m_EnableProfiling, InferenceTestOptions::m_InferenceTimesFile, InferenceTestOptions::m_IterationCount, and IInferenceTestCaseProvider::ProcessCommandLineOptions().

Referenced by InferenceTestMain().

{
    cxxopts::Options options("InferenceTest", "Inference iteration parameters");

    try
    {
        // Adds generic options needed for all inference tests.
        options
            .allow_unrecognised_options()
            .add_options()
                ("h,help", "Display help messages")
                ("i,iterations", "Sets the number of inferences to perform. If unset, will only be run once.",
                 cxxopts::value<unsigned int>(outParams.m_IterationCount)->default_value("0"))
                ("inference-times-file",
                 "If non-empty, each individual inference time will be recorded and output to this file",
                 cxxopts::value<std::string>(outParams.m_InferenceTimesFile)->default_value(""))
                ("e,event-based-profiling", "Enables built in profiler. If unset, defaults to off.",
                 cxxopts::value<bool>(outParams.m_EnableProfiling)->default_value("0"));

        std::vector<std::string> required; //to be passed as reference to derived inference tests

        // Adds options specific to the ITestCaseProvider.
        testCaseProvider.AddCommandLineOptions(options, required);

        auto result = options.parse(argc, argv);

        if (result.count("help"))
        {
            std::cout << options.help() << std::endl;
            return false;
        }

        CheckRequiredOptions(result, required);

    }
    catch (const cxxopts::OptionException& e)
    {
        std::cerr << e.what() << std::endl << options.help() << std::endl;
        return false;
    }
    catch (const std::exception& e)
    {
        // Coverity points out that default_value(...) can throw a bad_lexical_cast,
        // and that desc.add_options() can throw boost::io::too_few_args.
        // They really won't in any of these cases.
        ARMNN_ASSERT_MSG(false, "Caught unexpected exception");
        std::cerr << "Fatal internal error: " << e.what() << std::endl;
        return false;
    }

    if (!testCaseProvider.ProcessCommandLineOptions(outParams))
    {
        return false;
    }

    return true;
}

ValidateDirectory()

bool ValidateDirectory

(

std::string &

dir

)

Definition at line 87 of file InferenceTest.cpp.

Referenced by ClassifierInferenceTestMain(), main(), ClassifierTestCaseProvider< TDatabase, InferenceModel >::ProcessCommandLineOptions(), and YoloTestCaseProvider< Model >::ProcessCommandLineOptions().

{
    if (dir.empty())
    {
        std::cerr << "No directory specified" << std::endl;
        return false;
    }

    if (dir[dir.length() - 1] != '/')
    {
        dir += "/";
    }

    if (!fs::exists(dir))
    {
        std::cerr << "Given directory " << dir << " does not exist" << std::endl;
        return false;
    }

    if (!fs::is_directory(dir))
    {
        std::cerr << "Given directory [" << dir << "] is not a directory" << std::endl;
        return false;
    }

    return true;
}