IOptimizedNetwork Class Reference

#include <INetwork.hpp>

Public Member Functions
Status	PrintGraph ()
Status	SerializeToDot (std::ostream &stream) const
arm::pipe::ProfilingGuid	GetGuid () const
size_t	GetNumInputs () const
size_t	GetNumOutputs () const
void	ExecuteStrategy (IStrategy &strategy) const
	IOptimizedNetwork (const IOptimizedNetwork &other, const ModelOptions &modelOptions)
	IOptimizedNetwork (std::unique_ptr< Graph > graph)
	IOptimizedNetwork (std::unique_ptr< OptimizedNetworkImpl > impl)
	~IOptimizedNetwork ()
const std::shared_ptr< IProfiler > &	GetProfiler () const

Static Public Member Functions
static void	Destroy (IOptimizedNetwork *network)

Protected Member Functions
	IOptimizedNetwork (std::unique_ptr< Graph > graph, const ModelOptions &modelOptions)

Protected Attributes
std::unique_ptr< OptimizedNetworkImpl >	pOptimizedNetworkImpl

Friends
class	LoadedNetwork
class	experimental::AsyncNetworkImpl
class	experimental::WorkingMemHandle
Graph &	GetGraphForTesting (IOptimizedNetwork *optNetPtr)
ModelOptions &	GetModelOptionsForTesting (IOptimizedNetwork *optNetPtr)
IOptimizedNetworkPtr	Optimize (const INetwork &inNetwork, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options, Optional< std::vector< std::string > &> messages)
	Create an optimized version of the network. More...
IOptimizedNetworkPtr	Optimize (const Graph &inGraph, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options, Optional< std::vector< std::string > &> messages)
	Create an optimized version of the network. More...

Detailed Description

Definition at line 770 of file INetwork.hpp.

Constructor & Destructor Documentation

IOptimizedNetwork() [1/4]

IOptimizedNetwork	(	const IOptimizedNetwork &	other,
		const ModelOptions &	modelOptions
	)

Definition at line 461 of file Network.cpp.

Referenced by armnn::Optimize().

    : pOptimizedNetworkImpl(new OptimizedNetworkImpl(*other.pOptimizedNetworkImpl.get(), modelOptions)) {}

IOptimizedNetwork() [2/4]

IOptimizedNetwork

(

std::unique_ptr< Graph >

graph

)

Definition at line 464 of file Network.cpp.

    : pOptimizedNetworkImpl(new OptimizedNetworkImpl(std::move(graph))) {}

IOptimizedNetwork() [3/4]

IOptimizedNetwork

(

std::unique_ptr< OptimizedNetworkImpl >

impl

)

Definition at line 467 of file Network.cpp.

    : pOptimizedNetworkImpl(std::move(impl)) {}

~IOptimizedNetwork()

~IOptimizedNetwork ( )

default

Referenced by IOptimizedNetwork::IOptimizedNetwork().

IOptimizedNetwork() [4/4]

IOptimizedNetwork ( std::unique_ptr< Graph >  graph, const ModelOptions &  modelOptions  )

protected

Definition at line 470 of file Network.cpp.

References IOptimizedNetwork::~IOptimizedNetwork().

    : pOptimizedNetworkImpl(new OptimizedNetworkImpl(std::move(graph), modelOptions)) {}

Member Function Documentation

Destroy()

void Destroy ( IOptimizedNetwork *  network )

static

Definition at line 475 of file Network.cpp.

Referenced by armnn::Optimize().

{
    delete network;
}

ExecuteStrategy()

void ExecuteStrategy

(

IStrategy &

strategy

)

const

Definition at line 2942 of file Network.cpp.

{
    pOptimizedNetworkImpl->ExecuteStrategy(strategy);
}

GetGuid()

arm::pipe::ProfilingGuid GetGuid

(

)

const

Definition at line 495 of file Network.cpp.

References IOptimizedNetwork::pOptimizedNetworkImpl.

{
    return pOptimizedNetworkImpl->GetGuid();
}

GetNumInputs()

size_t GetNumInputs

(

)

const

Definition at line 500 of file Network.cpp.

References IOptimizedNetwork::pOptimizedNetworkImpl.

{
    return pOptimizedNetworkImpl->GetNumInputs();
}

GetNumOutputs()

size_t GetNumOutputs

(

)

const

Definition at line 505 of file Network.cpp.

References IOptimizedNetwork::pOptimizedNetworkImpl.

{
    return pOptimizedNetworkImpl->GetNumOutputs();
}

GetProfiler()

const std::shared_ptr< IProfiler > & GetProfiler

(

)

const

Definition at line 490 of file Network.cpp.

References IOptimizedNetwork::pOptimizedNetworkImpl.

{
    return pOptimizedNetworkImpl->GetGraph().GetProfiler();
}

PrintGraph()

Status PrintGraph

(

)

Definition at line 480 of file Network.cpp.

References IOptimizedNetwork::pOptimizedNetworkImpl.

{
    return pOptimizedNetworkImpl->PrintGraph();
}

SerializeToDot()

Status SerializeToDot

(

std::ostream &

stream

)

const

Definition at line 485 of file Network.cpp.

References IOptimizedNetwork::pOptimizedNetworkImpl.

Referenced by armnn_driver::ExportNetworkGraphToDotFile().

{
    return pOptimizedNetworkImpl->SerializeToDot(stream);
}

Friends And Related Function Documentation

experimental::AsyncNetworkImpl

friend class experimental::AsyncNetworkImpl

friend

Definition at line 797 of file INetwork.hpp.

experimental::WorkingMemHandle

friend class experimental::WorkingMemHandle

friend

Definition at line 798 of file INetwork.hpp.

GetGraphForTesting

Graph& GetGraphForTesting ( IOptimizedNetwork *  optNetPtr )

friend

Definition at line 49 of file TestUtils.cpp.

{
    return optNet->pOptimizedNetworkImpl->GetGraph();
}

GetModelOptionsForTesting

ModelOptions& GetModelOptionsForTesting ( IOptimizedNetwork *  optNetPtr )

friend

Definition at line 54 of file TestUtils.cpp.

{
    return optNet->pOptimizedNetworkImpl->GetModelOptions();
}

LoadedNetwork

friend class LoadedNetwork

friend

Definition at line 795 of file INetwork.hpp.

Optimize [1/2]

IOptimizedNetworkPtr Optimize ( const INetwork &  inNetwork, const std::vector< BackendId > &  backendPreferences, const IDeviceSpec &  deviceSpec, const OptimizerOptions &  options = OptimizerOptions(), Optional< std::vector< std::string > &>  messages = EmptyOptional()  )

friend

Create an optimized version of the network.

Parameters

network	INetwork description of the network to be optimized.
backendPreferences	The choice of the backend ordered by user preferences.
deviceSpec	DeviceSpec object as queried from the runtime. See IRuntime::GetDeviceSpec()
messages	If there are failures or warnings a string describing same will be added to the vector
options	OptimizerOptions object with optimizer configuration options

Returns

An IOptimizedNetworkPtr interface to the optimized network, throws an exception derived from armnn::Exception if process fails.

Definition at line 1850 of file Network.cpp.

Referenced by armnn::Optimize().

{
    return Optimize(inNetwork.pNetworkImpl->GetGraph(),
                    backendPreferences,
                    deviceSpec,
                    options,
                    messages);
}

Optimize [2/2]

IOptimizedNetworkPtr Optimize ( const Graph &  inGraph, const std::vector< BackendId > &  backendPreferences, const IDeviceSpec &  deviceSpec, const OptimizerOptions &  options, Optional< std::vector< std::string > &>  messages = EmptyOptional()  )

friend

Create an optimized version of the network.

Parameters

inGraph	Graph to be optimized.
backendPreferences	The choice of the backend ordered by user preferences.
deviceSpec	DeviceSpec object as queried from the runtime. See IRuntime::GetDeviceSpec()
messages	If there are failures or warnings a string describing same will be added to the vector
options	OptimizerOptions object with optimizer configuration options

Returns

An IOptimizedNetworkPtr interface to the optimized network, throws an exception derived from armnn::Exception if process fails.

Definition at line 1653 of file Network.cpp.

{
    ARMNN_LOG(debug) << options.ToString();

    // Enable profiling
    auto profiler = inGraph.GetProfiler();
    ProfilerManager::GetInstance().RegisterProfiler(profiler.get());
    profiler->EnableProfiling(options.m_ProfilingEnabled);

    ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer");
    if (backendPreferences.empty())
    {
        throw InvalidArgumentException("Invoked Optimize with no backends specified");
    }

    if (options.m_ReduceFp32ToFp16 && options.m_ReduceFp32ToBf16)
    {
        throw InvalidArgumentException("BFloat16 and Float16 optimization cannot be enabled at the same time.");
    }

    // Ensure TensorInfo is set on all output slots of ConstantLayers in the graph
    inGraph.VerifyConstantLayerSetTensorInfo();

    std::unique_ptr<Graph> graph = std::make_unique<Graph>(inGraph);

    // We need to pass on the information about whether import and export is enabled to the LoadNetwork phase.
    // The mechanism to do that is to add model options to the optimized network.
    armnn::BackendOptions importExport("Global",
                                        {{"ImportEnabled", options.m_ImportEnabled},
                                         {"ExportEnabled", options.m_ExportEnabled}});
    ModelOptions optimizedOptions(options.m_ModelOptions);
    optimizedOptions.push_back(importExport);

    auto optNet = IOptimizedNetworkPtr(new IOptimizedNetwork(std::move(graph), optimizedOptions),
                                       &IOptimizedNetwork::Destroy);

    IOptimizedNetwork* optNetObjPtr = optNet.get();

    // Get the optimized graph
    Graph& optGraph = optNetObjPtr->pOptimizedNetworkImpl->GetGraph();

    if(options.m_shapeInferenceMethod == ShapeInferenceMethod::InferAndValidate)
    {
        // Infer the tensor infos for all output slots. Throws an exception on failure
        optGraph.InferTensorInfos();
    }

    // Perform AddBroadcastReshapeLayer optimisation
    using namespace optimizations;
    Optimizer::Pass(optGraph, MakeOptimizations(AddBroadcastReshapeLayer()));

    if(options.m_shapeInferenceMethod == ShapeInferenceMethod::ValidateOnly)
    {
        // Validate the tensor infos for all output slots. Throws an exception on failure
        optGraph.InferTensorInfos();
    }

    // Need to FusePermuteIntoConstantLayer before FoldPadIntoDepthwiseConvolution2d or
    // FuseBatchNormIntoDepthwiseConvolution2D optimizations are called.
    Optimizer::Pass(optGraph, MakeOptimizations(FusePermuteIntoConstLayer()));

    // Perform optimisation passes
    Optimizer::Pass(optGraph, MakeOptimizations(SquashEqualPermuteSiblings(),
                                                SquashEqualTransposeSiblings(),
                                                SquashEqualReshapeSiblings(),
                                                OptimizeInversePermutes(),
                                                OptimizeInverseTransposes(),
                                                MovePermuteUp(),
                                                MoveTransposeUp(),
                                                PermuteAsReshape(),
                                                TransposeAsReshape(),
                                                OptimizeConsecutiveReshapes(),
                                                FoldPadIntoConvolution2d(),
                                                FoldPadIntoDepthwiseConvolution2d(),
                                                FoldPadIntoPooling2d(),
                                                PermuteAndBatchToSpaceAsDepthToSpace(),
                                                TransposeAndBatchToSpaceAsDepthToSpace(),
                                                FuseBatchNormIntoConvolution2DFloat32(),
                                                FuseBatchNormIntoConvolution2DFloat16(),
                                                FuseBatchNormIntoDepthwiseConvolution2DFloat32(),
                                                FuseBatchNormIntoDepthwiseConvolution2DFloat16(),
                                                ConvertConstDequantisationLayersToConstLayers()));

    // If Fp32 to Fp16 optimization is set convert Fp32 network to Fp16
    if (options.m_ReduceFp32ToFp16)
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_ReduceFp32ToFp16");
        Optimizer::Pass(optGraph, MakeOptimizations(Fp32NetworkToFp16Converter()));
        Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsFloatToHalf()));
    }

    // If Fp32 to Bf16 optimization is set convert Fp32 network to Bf16
    // Convert input of Convolution2d and FullyConnected from Fp32 to Bf16
    // Only Constant weight of Convolution2d and FullyConnected are converted from Fp32 to Bf16
    // Constant and Fp32ToBf16 layers will also be fused so conversion is no longer needed at inference time
    if (options.m_ReduceFp32ToBf16)
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_ReduceFp32ToBf16");
        Optimizer::Pass(optGraph, MakeOptimizations(Fp32NetworkToBf16Converter()));
        Optimizer::Pass(optGraph, MakeOptimizations(FuseConversionLayersIntoConstLayers()));
    }

    // Initialize backend settings
    BackendSettings backendSettings(backendPreferences, deviceSpec);
    if (backendSettings.GetAvailablePreferredBackends().empty())
    {
        std::stringstream failureMsg;
        failureMsg << "None of the preferred backends " << backendPreferences
                   << " are supported. Current platform provides " << backendSettings.m_SupportedBackends;
        ReportError(failureMsg.str(), messages);
        throw InvalidArgumentException(failureMsg.str());
    }

    // Create a map to temporarily hold initialized backend objects
    TensorHandleFactoryRegistry tensorHandleFactoryRegistry;
    BackendsMap backends = CreateSupportedBackends(tensorHandleFactoryRegistry, backendSettings);

    // Assign an available backend to each layer
    Graph::Iterator firstLayer = optGraph.begin();
    Graph::Iterator lastLayer  = optGraph.end();
    OptimizationResult assignBackendsResult = AssignBackends(optNetObjPtr->pOptimizedNetworkImpl.get(),
                                                             backendSettings,
                                                             firstLayer,
                                                             lastLayer,
                                                             messages);
    if (assignBackendsResult.m_Error)
    {
        // Failed to assign a backend to each layer
        throw InvalidArgumentException("Failed to assign a backend to each layer");
    }

    Optimizer::Pass(optGraph, MakeOptimizations(OptimizeInverseConversionsFp16(),
                                                OptimizeInverseConversionsFp32()));

    // Apply the backend-specific optimizations
    OptimizationResult backendOptimizationResult = ApplyBackendOptimizations(optNetObjPtr->pOptimizedNetworkImpl.get(),
                                                                             backendSettings,
                                                                             backends,
                                                                             options.m_ModelOptions,
                                                                             messages);
    if (backendOptimizationResult.m_Error)
    {
        // Failed to apply the backend-specific optimizations
        throw InvalidArgumentException("Failed to apply the backend-specific optimizations");
    }

    // Convert constants
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_ConvertConstants");
        Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsFloatToHalf()));
        Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsHalfToFloat()));

        // Once the constants are converted we can now safely call RedirectMembersToConstantInputs
        Optimizer::Pass(optGraph, MakeOptimizations(RedirectMembersToConstantInputs()));
    }

    // This must occur after all topological changes to the graph and any redirection of variables
    // If the debug flag is set, then insert a DebugLayer after each layer
    // Doing this after applying the backend optimizations as they might have changed some layers
    if (options.m_Debug && !options.m_DebugToFile)
    {
        Optimizer::Pass(optGraph, MakeOptimizations(InsertDebugLayer()));
    }
    else if (options.m_DebugToFile)
    {
        // Setup the output file path
        armnnUtils::Filesystem::CreateDirectory("/ArmNNIntermediateLayerOutputs");
        Optimizer::Pass(optGraph, MakeOptimizations(InsertDebugToFileLayer()));
    }

    // Calculate the compatibility strategies for tensor handles
    OptimizationResult strategyResult = SelectTensorHandleStrategy(optGraph,
                                                                   backends,
                                                                   tensorHandleFactoryRegistry,
                                                                   options.m_ImportEnabled,
                                                                   options.m_ExportEnabled,
                                                                   messages);

    if (strategyResult.m_Error)
    {
        // Failed to apply the backend-specific optimizations
        return IOptimizedNetworkPtr(nullptr, &IOptimizedNetwork::Destroy);
    }

    // Based on the tensor handle strategy determined above, insert copy layers where required.
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_AddCompatibilityLayers");
        optGraph.AddCompatibilityLayers(backends, tensorHandleFactoryRegistry);
    }

    return optNet;
}

Member Data Documentation

pOptimizedNetworkImpl

std::unique_ptr<OptimizedNetworkImpl> pOptimizedNetworkImpl

protected

Definition at line 815 of file INetwork.hpp.

Referenced by armnn::GetGraphForTesting(), IOptimizedNetwork::GetGuid(), armnn::GetModelOptionsForTesting(), IOptimizedNetwork::GetNumInputs(), IOptimizedNetwork::GetNumOutputs(), IOptimizedNetwork::GetProfiler(), armnn::Optimize(), IOptimizedNetwork::PrintGraph(), and IOptimizedNetwork::SerializeToDot().

---

The documentation for this class was generated from the following files:

• include/armnn/INetwork.hpp
• src/armnn/Network.cpp