ExecuteNetwork.cpp File Reference

#include "NetworkExecutionUtils/NetworkExecutionUtils.hpp"
#include "ExecuteNetworkProgramOptions.hpp"
#include <armnn/Logging.hpp>
#include <Filesystem.hpp>
#include <InferenceTest.hpp>
#include <future>

Go to the source code of this file.

Functions
template<typename TParser , typename TDataType >
int	MainImpl (const ExecuteNetworkParams &params, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
int	main (int argc, const char *argv[])

Function Documentation

main()

int main	(	int	argc,
		const char *	argv[]
	)

Definition at line 537 of file ExecuteNetwork.cpp.

References ARMNN_LOG, ExecuteNetworkParams::ArmNNTfLiteDelegate, ExecuteNetworkParams::ArmNNTfLiteParser, armnn::ConfigureLogging(), IRuntime::Create(), armnn::Debug, armnn::Info, ProgramOptions::m_ExNetParams, ExecuteNetworkParams::m_ModelFormat, ProgramOptions::m_RuntimeOptions, ExecuteNetworkParams::m_TfLiteExecutor, and ExecuteNetworkParams::TfliteInterpreter.

{
    // Configures logging for both the ARMNN library and this test program.
    #ifdef NDEBUG
    armnn::LogSeverity level = armnn::LogSeverity::Info;
    #else
    armnn::LogSeverity level = armnn::LogSeverity::Debug;
    #endif
    armnn::ConfigureLogging(true, true, level);


    // Get ExecuteNetwork parameters and runtime options from command line
    ProgramOptions ProgramOptions(argc, argv);

    // Create runtime
    std::shared_ptr<armnn::IRuntime> runtime(armnn::IRuntime::Create(ProgramOptions.m_RuntimeOptions));

    std::string modelFormat = ProgramOptions.m_ExNetParams.m_ModelFormat;

    // Forward to implementation based on the parser type
    if (modelFormat.find("armnn") != std::string::npos)
    {
    #if defined(ARMNN_SERIALIZER)
        return MainImpl<armnnDeserializer::IDeserializer, float>(ProgramOptions.m_ExNetParams, runtime);
    #else
        ARMNN_LOG(fatal) << "Not built with serialization support.";
        return EXIT_FAILURE;
    #endif
    }
    else if (modelFormat.find("onnx") != std::string::npos)
    {
    #if defined(ARMNN_ONNX_PARSER)
        return MainImpl<armnnOnnxParser::IOnnxParser, float>(ProgramOptions.m_ExNetParams, runtime);
    #else
        ARMNN_LOG(fatal) << "Not built with Onnx parser support.";
        return EXIT_FAILURE;
    #endif
    }
    else if(modelFormat.find("tflite") != std::string::npos)
    {
        if (ProgramOptions.m_ExNetParams.m_TfLiteExecutor == ExecuteNetworkParams::TfLiteExecutor::ArmNNTfLiteParser)
        {
            #if defined(ARMNN_TF_LITE_PARSER)
                        return MainImpl<armnnTfLiteParser::ITfLiteParser, float>(ProgramOptions.m_ExNetParams, runtime);
            #else
                        ARMNN_LOG(fatal) << "Not built with Tensorflow-Lite parser support.";
                        return EXIT_FAILURE;
            #endif
        }
        else if (ProgramOptions.m_ExNetParams.m_TfLiteExecutor ==
                    ExecuteNetworkParams::TfLiteExecutor::ArmNNTfLiteDelegate ||
                ProgramOptions.m_ExNetParams.m_TfLiteExecutor ==
                    ExecuteNetworkParams::TfLiteExecutor::TfliteInterpreter)
        {
        #if defined(ARMNN_TF_LITE_DELEGATE)
            return TfLiteDelegateMainImpl(ProgramOptions.m_ExNetParams, runtime);
        #else
            ARMNN_LOG(fatal) << "Not built with Arm NN Tensorflow-Lite delegate support.";
            return EXIT_FAILURE;
        #endif
        }
    }
    else
    {
        ARMNN_LOG(fatal) << "Unknown model format: '" << modelFormat
                         << "'. Please include 'tflite' or 'onnx'";
        return EXIT_FAILURE;
    }
}

MainImpl()

int MainImpl	(	const ExecuteNetworkParams &	params,
		const std::shared_ptr< armnn::IRuntime > &	runtime = nullptr
	)

Definition at line 276 of file ExecuteNetwork.cpp.

References ARMNN_LOG, InferenceModel< IParser, TDataType >::CreateWorkingMemHandle(), InferenceModel< IParser, TDataType >::GetInputQuantizationParams(), InferenceModel< IParser, TDataType >::GetInputSize(), InferenceModel< IParser, TDataType >::GetOutputBindingInfos(), InferenceModel< IParser, TDataType >::GetOutputSize(), Params::m_AsyncEnabled, ExecuteNetworkParams::m_CachedNetworkFilePath, Params::m_CachedNetworkFilePath, ExecuteNetworkParams::m_ComputeDevices, Params::m_ComputeDevices, ExecuteNetworkParams::m_Concurrent, ExecuteNetworkParams::m_DequantizeOutput, ExecuteNetworkParams::m_DynamicBackendsPath, Params::m_DynamicBackendsPath, ExecuteNetworkParams::m_EnableBf16TurboMode, Params::m_EnableBf16TurboMode, ExecuteNetworkParams::m_EnableFastMath, Params::m_EnableFastMath, ExecuteNetworkParams::m_EnableFp16TurboMode, Params::m_EnableFp16TurboMode, ExecuteNetworkParams::m_EnableLayerDetails, ExecuteNetworkParams::m_EnableProfiling, ExecuteNetworkParams::m_GenerateTensorData, ExecuteNetworkParams::m_InferOutputShape, Params::m_InferOutputShape, Params::m_InputBindings, ExecuteNetworkParams::m_InputNames, Params::m_InputShapes, ExecuteNetworkParams::m_InputTensorDataFilePaths, ExecuteNetworkParams::m_InputTensorShapes, ExecuteNetworkParams::m_InputTypes, ExecuteNetworkParams::m_IsModelBinary, Params::m_IsModelBinary, ExecuteNetworkParams::m_Iterations, ExecuteNetworkParams::m_MLGOTuningFilePath, Params::m_MLGOTuningFilePath, ExecuteNetworkParams::m_ModelPath, Params::m_ModelPath, ExecuteNetworkParams::m_NumberOfThreads, Params::m_NumberOfThreads, Params::m_OutputBindings, ExecuteNetworkParams::m_OutputNames, ExecuteNetworkParams::m_OutputTensorFiles, ExecuteNetworkParams::m_OutputTypes, ExecuteNetworkParams::m_ParseUnsupported, Params::m_ParseUnsupported, ExecuteNetworkParams::m_PrintIntermediate, Params::m_PrintIntermediateLayers, ExecuteNetworkParams::m_QuantizeInput, ExecuteNetworkParams::m_SaveCachedNetwork, Params::m_SaveCachedNetwork, ExecuteNetworkParams::m_SimultaneousIterations, ExecuteNetworkParams::m_SubgraphId, Params::m_SubgraphId, ExecuteNetworkParams::m_ThresholdTime, Params::m_VisualizePostOptimizationModel, PopulateTensorWithData(), InferenceModel< IParser, TDataType >::Run(), InferenceModel< IParser, TDataType >::RunAsync(), and Exception::what().

{
    using TContainer =
           mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>, std::vector<int8_t>>;

    std::vector<std::vector<TContainer>> inputs;
    std::vector<std::vector<TContainer>> outputs;

    try
    {
        // Creates an InferenceModel, which will parse the model and load it into an IRuntime.
        typename InferenceModel<TParser, TDataType>::Params inferenceModelParams;
        inferenceModelParams.m_ModelPath                      = params.m_ModelPath;
        inferenceModelParams.m_IsModelBinary                  = params.m_IsModelBinary;
        inferenceModelParams.m_ComputeDevices                 = params.m_ComputeDevices;
        inferenceModelParams.m_DynamicBackendsPath            = params.m_DynamicBackendsPath;
        inferenceModelParams.m_PrintIntermediateLayers        = params.m_PrintIntermediate;
        inferenceModelParams.m_VisualizePostOptimizationModel = params.m_EnableLayerDetails;
        inferenceModelParams.m_ParseUnsupported               = params.m_ParseUnsupported;
        inferenceModelParams.m_InferOutputShape               = params.m_InferOutputShape;
        inferenceModelParams.m_EnableFastMath                 = params.m_EnableFastMath;
        inferenceModelParams.m_SaveCachedNetwork              = params.m_SaveCachedNetwork;
        inferenceModelParams.m_CachedNetworkFilePath          = params.m_CachedNetworkFilePath;
        inferenceModelParams.m_NumberOfThreads                = params.m_NumberOfThreads;
        inferenceModelParams.m_MLGOTuningFilePath             = params.m_MLGOTuningFilePath;
        inferenceModelParams.m_AsyncEnabled                   = params.m_Concurrent;

        for(const std::string& inputName: params.m_InputNames)
        {
            inferenceModelParams.m_InputBindings.push_back(inputName);
        }

        for(unsigned int i = 0; i < params.m_InputTensorShapes.size(); ++i)
        {
            inferenceModelParams.m_InputShapes.push_back(*params.m_InputTensorShapes[i]);
        }

        for(const std::string& outputName: params.m_OutputNames)
        {
            inferenceModelParams.m_OutputBindings.push_back(outputName);
        }

        inferenceModelParams.m_SubgraphId          = params.m_SubgraphId;
        inferenceModelParams.m_EnableFp16TurboMode = params.m_EnableFp16TurboMode;
        inferenceModelParams.m_EnableBf16TurboMode = params.m_EnableBf16TurboMode;

        InferenceModel<TParser, TDataType> model(inferenceModelParams,
                                                 params.m_EnableProfiling,
                                                 params.m_DynamicBackendsPath,
                                                 runtime);

        const size_t numInputs = inferenceModelParams.m_InputBindings.size();

        armnn::Optional<QuantizationParams> qParams = params.m_QuantizeInput ?
                                                      armnn::MakeOptional<QuantizationParams>(
                                                          model.GetInputQuantizationParams()) :
                                                      armnn::EmptyOptional();

        for(unsigned int j = 0; j < params.m_SimultaneousIterations ; ++j)
        {
            std::vector<TContainer> inputDataContainers;
            for(unsigned int i = 0; i < numInputs; ++i)
            {
                armnn::Optional<std::string> dataFile = params.m_GenerateTensorData ?
                                                        armnn::EmptyOptional() :
                                                        armnn::MakeOptional<std::string>(
                                                            params.m_InputTensorDataFilePaths[(j * numInputs) + i]);

                unsigned int numElements = model.GetInputSize(i);
                if (params.m_InputTensorShapes.size() > i && params.m_InputTensorShapes[i])
                {
                    // If the user has provided a tensor shape for the current input,
                    // override numElements
                    numElements = params.m_InputTensorShapes[i]->GetNumElements();
                }

                TContainer tensorData;
                PopulateTensorWithData(tensorData,
                                       numElements,
                                       params.m_InputTypes[i],
                                       qParams,
                                       dataFile);

                inputDataContainers.push_back(tensorData);
            }
            inputs.push_back(inputDataContainers);
        }

        const size_t numOutputs = inferenceModelParams.m_OutputBindings.size();

        for (unsigned int j = 0; j < params.m_SimultaneousIterations; ++j)
        {
            std::vector <TContainer> outputDataContainers;
            for (unsigned int i = 0; i < numOutputs; ++i)
            {
                if (params.m_OutputTypes[i].compare("float") == 0)
                {
                    outputDataContainers.push_back(std::vector<float>(model.GetOutputSize(i)));
                } else if (params.m_OutputTypes[i].compare("int") == 0)
                {
                    outputDataContainers.push_back(std::vector<int>(model.GetOutputSize(i)));
                } else if (params.m_OutputTypes[i].compare("qasymm8") == 0)
                {
                    outputDataContainers.push_back(std::vector<uint8_t>(model.GetOutputSize(i)));
                } else if (params.m_OutputTypes[i].compare("qsymms8") == 0)
                {
                    outputDataContainers.push_back(std::vector<int8_t>(model.GetOutputSize(i)));
                } else
                {
                    ARMNN_LOG(fatal) << "Unsupported tensor data type \"" << params.m_OutputTypes[i] << "\". ";
                    return EXIT_FAILURE;
                }
            }
            outputs.push_back(outputDataContainers);
        }

        if (!params.m_Concurrent)
        {
            // Synchronous Execution
            for (size_t x = 0; x < params.m_Iterations; x++)
            {
                // model.Run returns the inference time elapsed in EnqueueWorkload (in milliseconds)
                auto inference_duration = model.Run(inputs[0], outputs[0]);

                if (params.m_GenerateTensorData)
                {
                    ARMNN_LOG(warning) << "The input data was generated, note that the output will not be useful";
                }

                // Print output tensors
                const auto& infosOut = model.GetOutputBindingInfos();
                for (size_t i = 0; i < numOutputs; i++)
                {
                    const armnn::TensorInfo& infoOut = infosOut[i].second;
                    auto outputTensorFile = params.m_OutputTensorFiles.empty() ? "" : params.m_OutputTensorFiles[i];

                    TensorPrinter printer(inferenceModelParams.m_OutputBindings[i],
                                          infoOut,
                                          outputTensorFile,
                                          params.m_DequantizeOutput);
                    mapbox::util::apply_visitor(printer, outputs[0][i]);
                }

                ARMNN_LOG(info) << "\nInference time: " << std::setprecision(2)
                                << std::fixed << inference_duration.count() << " ms\n";

                // If thresholdTime == 0.0 (default), then it hasn't been supplied at command line
                if (params.m_ThresholdTime != 0.0)
                {
                    ARMNN_LOG(info) << "Threshold time: " << std::setprecision(2)
                                    << std::fixed << params.m_ThresholdTime << " ms";
                    auto thresholdMinusInference = params.m_ThresholdTime - inference_duration.count();
                    ARMNN_LOG(info) << "Threshold time - Inference time: " << std::setprecision(2)
                                    << std::fixed << thresholdMinusInference << " ms" << "\n";

                    if (thresholdMinusInference < 0)
                    {
                        std::string errorMessage = "Elapsed inference time is greater than provided threshold time.";
                        ARMNN_LOG(fatal) << errorMessage;
                    }
                }
            }
        }
        else
        {
            try
            {
                ARMNN_LOG(info) << "Asynchronous Execution...  \n";
                std::vector<std::future<std::tuple<armnn::profiling::ProfilingGuid,
                std::chrono::duration<double, std::milli>>>> inferenceResults;
                inferenceResults.reserve(params.m_SimultaneousIterations);

                // Create WorkingMemHandles for each inference
                std::vector<std::unique_ptr<armnn::experimental::IWorkingMemHandle>> workingMemHandles;
                workingMemHandles.reserve(params.m_SimultaneousIterations);
                for (unsigned int i = 0; i < params.m_SimultaneousIterations; ++i)
                {
                    workingMemHandles.push_back(model.CreateWorkingMemHandle());
                }

                // Run each inference in its own thread
                for (unsigned int i = 0; i < params.m_SimultaneousIterations; ++i)
                {
                    armnn::experimental::IWorkingMemHandle& workingMemHandleRef = *workingMemHandles[i].get();
                    inferenceResults.push_back(std::async(
                        std::launch::async, [&model, &workingMemHandleRef, &inputs, &outputs, i]() {
                            return model.RunAsync(workingMemHandleRef, inputs[i], outputs[i]);
                        }
                        ));
                }

                // Check the results
                for (unsigned int j = 0; j < inferenceResults.size(); ++j)
                {
                    // Get the results
                    auto inferenceResult = inferenceResults[j].get();
                    auto inference_duration = std::get<1>(inferenceResult);
                    auto inferenceID = std::get<0>(inferenceResult);

                    if (params.m_GenerateTensorData)
                    {
                        ARMNN_LOG(warning) << "The input data was generated, note that the output will not be useful";
                    }

                    // Print output tensors
                    const auto& infosOut = model.GetOutputBindingInfos();
                    for (size_t i = 0; i < numOutputs; i++)
                    {
                        const armnn::TensorInfo& infoOut = infosOut[i].second;
                        auto outputTensorFile = params.m_OutputTensorFiles.empty()
                                                ? ""
                                                : params.m_OutputTensorFiles[(j * numOutputs) + i];

                        TensorPrinter printer(inferenceModelParams.m_OutputBindings[i],
                                              infoOut,
                                              outputTensorFile,
                                              params.m_DequantizeOutput);
                        mapbox::util::apply_visitor(printer, outputs[j][i]);
                    }

                    ARMNN_LOG(info) << "\nInference time: " << std::setprecision(2)
                                    << std::fixed << inference_duration.count() << " ms\n";

                    // If thresholdTime == 0.0 (default), then it hasn't been supplied at command line
                    if (params.m_ThresholdTime != 0.0)
                    {
                        ARMNN_LOG(info) << "Threshold time: " << std::setprecision(2)
                                        << std::fixed << params.m_ThresholdTime << " ms";
                        auto thresholdMinusInference = params.m_ThresholdTime - inference_duration.count();
                        ARMNN_LOG(info) << "Threshold time - Inference time: " << std::setprecision(2)
                                        << std::fixed << thresholdMinusInference << " ms" << "\n";

                        if (thresholdMinusInference < 0)
                        {
                            ARMNN_LOG(fatal) << "Elapsed inference time is greater than provided threshold time. \n";
                        }
                    }
                    ARMNN_LOG(info) << "Asynchronous Execution is finished for Inference ID: " << inferenceID << " \n";

                }
            }
            catch (const armnn::Exception& e)
            {
                ARMNN_LOG(fatal) << "Armnn Error: " << e.what();
                return EXIT_FAILURE;
            }

        }
    }
    catch (const armnn::Exception& e)
    {
        ARMNN_LOG(fatal) << "Armnn Error: " << e.what();
        return EXIT_FAILURE;
    }

    return EXIT_SUCCESS;
}