19 #include <fmt/format.h> 21 #include <unordered_map> 29 : m_LayersInOrder(other.m_LayersInOrder)
30 , m_Profiler(other.m_Profiler)
32 std::unordered_map<const Layer*, Layer*> otherToClonedMap;
34 for (
auto&& otherLayer : other.m_Layers)
37 otherToClonedMap.emplace(otherLayer, layer);
41 for (
auto&& otherLayer : other.m_Layers)
43 Layer*
const thisLayer = otherToClonedMap[otherLayer];
46 for (
auto&& otherOutputSlot : otherLayer->GetOutputSlots())
48 for (
auto&& otherInputSlot : otherOutputSlot.GetConnections())
50 const Layer& otherTgtLayer = otherInputSlot->GetOwningLayer();
51 Layer*
const thisTgtLayer = otherToClonedMap[&otherTgtLayer];
54 outputSlot->Connect(inputSlot);
56 outputSlot->SetTensorInfo(otherOutputSlot.GetTensorInfo());
74 auto numInputSlots = it->GetNumInputSlots();
75 auto numOutputSlots = it->GetNumOutputSlots();
78 <<
":" << it->GetBackendId().Get()
79 <<
" has " << numInputSlots <<
" input slots" 80 <<
" and " << numOutputSlots <<
" output slots.";
82 for (
auto i : it->GetInputSlots())
84 std::ostringstream message;
85 auto inputTensorShape = i.GetConnectedOutputSlot()->GetTensorInfo().GetShape();
86 unsigned int numDims = inputTensorShape.GetNumDimensions();
88 message <<
"The input slot has shape [ ";
89 for (
unsigned int dim=0; dim < numDims; dim++)
91 message << inputTensorShape[dim] <<
",";
97 for (
unsigned int i = 0; i < it->GetNumOutputSlots(); i++)
100 std::ostringstream message;
101 auto outputTensorShape = layer->
GetOutputSlots()[i].GetTensorInfo().GetShape();
102 unsigned int numDims = outputTensorShape.GetNumDimensions();
104 message <<
"The output slot has shape [ ";
105 for (
unsigned int dim=0; dim < numDims; dim++)
107 message << outputTensorShape[dim] <<
",";
122 DotGraph graph(stream,
"Optimized");
141 for (
auto&& layer : m_Layers)
148 layer->SerializeLayerParameters(extractParams);
152 for (
auto&& layer : m_Layers)
156 for (
unsigned int i=0;i<layer->GetNumInputSlots(); i++)
160 DotEdge edge(stream, fromId, toId);
165 std::stringstream ss;
186 std::unordered_set<const ITensorHandle*> preallocatedTensors;
187 std::unordered_map<const ITensorHandle*, unsigned int> handleReferenceCounts;
191 auto TraceSubTensorHandleAncestry = [](
ITensorHandle*
const subTensorHandle)
194 while (ancestor && ancestor->
GetParent())
204 return tensorHandle && preallocatedTensors.find(tensorHandle) != preallocatedTensors.end();
209 for (
auto&& layer : m_Layers)
213 for (
auto&& slot = layer->BeginOutputSlots(); slot != layer->EndOutputSlots(); ++slot)
215 ITensorHandle *tensorHandle = TraceSubTensorHandleAncestry(slot->GetOutputHandler().GetData());
217 if (tensorHandle && !IsPreallocated(tensorHandle))
220 preallocatedTensors.insert(tensorHandle);
227 for (
auto&& layer : m_Layers)
231 for (
auto&& slot = layer->BeginOutputSlots(); slot != layer->EndOutputSlots(); ++slot)
233 ITensorHandle *tensorHandle = TraceSubTensorHandleAncestry(slot->GetOutputHandler().GetData());
235 if (tensorHandle && !IsPreallocated(tensorHandle))
237 unsigned int numConnections = slot->GetNumConnections();
238 if (handleReferenceCounts.find(tensorHandle) == handleReferenceCounts.end())
240 handleReferenceCounts[tensorHandle] = numConnections;
242 if (handleReferenceCounts[tensorHandle] == 0u)
250 handleReferenceCounts[tensorHandle] += numConnections;
257 for (
auto&& slot = layer->BeginInputSlots(); slot != layer->EndInputSlots(); ++slot)
260 slot->GetConnectedOutputSlot()->GetOutputHandler().GetData());
262 if (tensorHandle && !IsPreallocated(tensorHandle))
264 --handleReferenceCounts[tensorHandle];
266 if (handleReferenceCounts[tensorHandle] == 0u)
270 handleReferenceCounts.erase(tensorHandle);
281 if (!m_LayersInOrder)
284 for (
auto&& it : m_Layers)
289 auto compareLayerPriority = [](
const LayerList::value_type& layerA,
const LayerList::value_type& layerB)
291 return layerA->GetPriority() < layerB->GetPriority();
294 m_Layers.sort(compareLayerPriority);
296 m_LayersInOrder =
true;
307 auto MayNeedCompatibilityLayer = [](
const Layer& layer)
316 auto IsCompatibilityStrategy = [](
EdgeStrategy strategy)
322 ForEachLayer([
this, &backends, ®istry, MayNeedCompatibilityLayer, IsCompatibilityStrategy](
Layer* srcLayer)
326 if (!MayNeedCompatibilityLayer(*srcLayer))
332 const std::vector<OutputSlot>& srcOutputSlots = srcLayer->
GetOutputSlots();
333 for (
unsigned int srcOutputIndex = 0; srcOutputIndex < srcOutputSlots.size(); srcOutputIndex++)
336 const std::vector<InputSlot*> srcConnections = srcOutputSlot.
GetConnections();
337 const std::vector<EdgeStrategy> srcEdgeStrategies = srcOutputSlot.
GetEdgeStrategies();
338 for (
unsigned int srcConnectionIndex = 0; srcConnectionIndex < srcConnections.size(); srcConnectionIndex++)
340 InputSlot* dstInputSlot = srcConnections[srcConnectionIndex];
343 EdgeStrategy strategy = srcEdgeStrategies[srcConnectionIndex];
345 "Undefined memory strategy found while adding copy layers for compatibility");
348 if (MayNeedCompatibilityLayer(dstLayer) &&
349 IsCompatibilityStrategy(strategy))
354 const std::string compLayerName = fmt::format(
"[ {} ({}) -> {} ({}) ]",
359 Layer* compLayer =
nullptr;
362 compLayer = InsertNewLayer<MemCopyLayer>(*dstInputSlot, compLayerName.c_str());
367 compLayer = InsertNewLayer<MemImportLayer>(*dstInputSlot, compLayerName.c_str());
372 OutputSlot& compOutputSlot = compLayer->GetOutputSlot(0);
373 auto backendIt = backends.find(dstLayer.
GetBackendId());
374 if (backendIt != backends.end() &&
376 backendIt->second->SupportsTensorAllocatorAPI())
378 auto backend = backendIt->second.get();
379 auto tensorHandleFactoryIds = backend->GetHandleFactoryPreferences();
382 for (
auto preference : tensorHandleFactoryIds)
384 auto factory = registry.
GetFactory(preference);
388 auto srcFactory = registry.
GetFactory(srcPref);
392 bool canExportImport =
393 (factory->GetImportFlags() & srcFactory->GetExportFlags()) != 0;
395 if (factory->SupportsMapUnmap() || canExportImport)
419 const std::vector<InputSlot*>& newSourceConnections = srcOutputSlot.
GetConnections();
420 auto newSrcConnectionIndex = std::distance(newSourceConnections.begin(),
421 std::find(newSourceConnections.begin(),
422 newSourceConnections.end(),
423 &compLayer->GetInputSlot(0)));
426 srcOutputSlot.
SetEdgeStrategy(armnn::numeric_cast<unsigned int>(newSrcConnectionIndex),
438 ReplaceSubgraphConnections(subgraph, substituteLayer);
439 EraseSubgraphLayers(subgraph);
447 if (std::find(std::begin(m_Layers), std::end(m_Layers), layer) == std::end(m_Layers))
449 layer->
Reparent(*
this, m_Layers.end());
450 m_LayersInOrder =
false;
454 ReplaceSubgraphConnections(subgraph, substituteSubgraph);
455 EraseSubgraphLayers(subgraph);
466 ReplaceSubgraphConnections(subgraph, substituteSubgraph);
474 std::for_each(substituteSubgraphLayers.begin(), substituteSubgraphLayers.end(), [&](
Layer* layer)
477 ARMNN_ASSERT_MSG(std::find(m_Layers.begin(), m_Layers.end(), layer) != m_Layers.end(),
478 "Substitute layer is not a member of graph");
484 unsigned int subgraphNumInputSlots =
armnn::numeric_cast<
unsigned int>(subgraphInputSlots.size());
485 unsigned int subgraphNumOutputSlots =
armnn::numeric_cast<
unsigned int>(subgraphOutputSlots.size());
490 ARMNN_ASSERT(subgraphNumInputSlots == substituteSubgraphInputSlots.size());
491 ARMNN_ASSERT(subgraphNumOutputSlots == substituteSubgraphOutputSlots.size());
496 for (
unsigned int inputSlotIdx = 0; inputSlotIdx < subgraphNumInputSlots; ++inputSlotIdx)
498 InputSlot* subgraphInputSlot = subgraphInputSlots.at(inputSlotIdx);
503 connectedOutputSlot->
Disconnect(*subgraphInputSlot);
505 IInputSlot* substituteInputSlot = substituteSubgraphInputSlots.at(inputSlotIdx);
507 connectedOutputSlot->
Connect(*substituteInputSlot);
511 for(
unsigned int outputSlotIdx = 0; outputSlotIdx < subgraphNumOutputSlots; ++outputSlotIdx)
513 OutputSlot* subgraphOutputSlot = subgraphOutputSlots.at(outputSlotIdx);
516 OutputSlot* substituteOutputSlot = substituteSubgraphOutputSlots.at(outputSlotIdx);
541 for (
auto&& output: layer->GetOutputSlots())
543 if (!output.IsTensorInfoSet())
545 std::ostringstream message;
546 message <<
"Output slot TensorInfo not set on " 562 for (
auto&& input : layer->GetInputSlots())
564 const IOutputSlot* source = input.GetConnectedOutputSlot();
569 ConstructErrorMessageForUnconnectedInputs(layer, input.GetSlotIndex());
574 std::ostringstream message;
575 message <<
"Output slot TensorInfo not set on " 578 << std::quoted(layer->GetName());
585 layer->ValidateTensorShapesFromInputs();
597 void Graph::ConstructErrorMessageForUnconnectedInputs(
Layer*
const layer,
598 unsigned int slotIndex)
600 std::ostringstream message;
601 bool noWeightsAndBias =
false;
610 if (biasSource == NULL)
612 message << layer->
GetName() <<
" layer weights and bias not set: ";
613 noWeightsAndBias =
true;
618 if (!noWeightsAndBias)
622 message << layer->
GetName() <<
" layer weights not set: ";
626 message << layer->
GetName() <<
" layer bias not set: ";
631 std::string slotString = noWeightsAndBias ?
"1 & 2" : std::to_string(slotIndex);
632 message <<
"Input slot(s) " 634 <<
" not connected to an output slot on " 637 << std::quoted(layer->
GetName());
Graph(bool shapeInferenceMethod=false)
const std::vector< InputSlot * > & GetConnections() const
Interface for a layer that is connectable to other layers via InputSlots and OutputSlots.
NodeContent & AddContent(const std::string &content)
void SetEdgeStrategy(unsigned int connectionIndex, EdgeStrategy strategy)
No strategy has been defined. Used internally to verify integrity of optimizations.
unsigned int GetNumInputSlots() const override
Returns the number of connectable input slots.
const TensorShape & GetShape() const
Status SerializeToDot(std::ostream &stream)
const std::vector< EdgeStrategy > & GetEdgeStrategies() const
virtual void Reparent(Graph &dest, std::list< Layer *>::const_iterator iterator)=0
Layer & GetOwningLayer() const
Source backends tensor data can be exported to destination backend tensor without copy...
void EraseLayer(Iterator pos)
Deletes the layer at the specified position.
std::vector< OutputSlot * > OutputSlots
DotAttributeSet & GetAttributeSet()
virtual void Allocate()=0
Indicate to the memory manager that this resource is no longer active.
virtual Layer * Clone(Graph &graph) const =0
Creates a dynamically-allocated copy of this layer.
#define ARMNN_LOG(severity)
virtual void Manage()=0
Indicate to the memory manager that this resource is active.
Copyright (c) 2021 ARM Limited and Contributors.
void IgnoreUnused(Ts &&...)
#define ARMNN_SCOPED_PROFILING_EVENT(backendId, name)
Destination backend can work directly with tensors on source backend.
The SubgraphView class represents a subgraph of a Graph.
const InputSlot & GetInputSlot(unsigned int index) const override
Get a const input slot handle by slot index.
void ForEachLayer(Func func) const
#define ARMNN_ASSERT_MSG(COND, MSG)
DotAttributeSet & AddAttribute(const std::string &name, const std::stringstream &value)
NodeContent & GetContents()
An output connection slot for a layer.
virtual void Disconnect(IInputSlot &slot)=0
virtual ITensorHandle * GetParent() const =0
Get the parent tensor if this is a subtensor.
Validate all output shapes.
LayerType GetType() const override
Returns the armnn::LayerType of this layer.
#define ARMNN_ASSERT(COND)
const BackendId & GetBackendId() const
const std::vector< OutputSlot > & GetOutputSlots() const
std::vector< InputSlot * > InputSlots
void VerifyConstantLayerSetTensorInfo() const
For each ConstantLayer in Graph, ensures TensorInfo is set on all output slots.
void SubstituteSubgraph(SubgraphView &subgraph, IConnectableLayer *substituteLayer)
Substitutes the given sub-graph with either a new layer or a new sub-graph.
const std::shared_ptr< IProfiler > & GetProfiler() const
void SetTensorHandleFactory(const ITensorHandleFactory::FactoryId &id)
const InputSlots & GetInputSlots() const
std::vector< OutputSlot >::iterator BeginOutputSlots()
const OutputSlots & GetOutputSlots() const
profiling::ProfilingGuid LayerGuid
Define LayerGuid type.
ITensorHandleFactory * GetFactory(ITensorHandleFactory::FactoryId id) const
Find a TensorHandleFactory by Id Returns nullptr if not found.
virtual bool IsTensorInfoSet() const =0
const Layers & GetLayers() const
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
const OutputSlot & GetOutputSlot(unsigned int index=0) const override
Get the const output slot handle by slot index.
DotAttributeSet & GetAttributeSet()
const char * GetName() const override
Returns the name of the layer.
ITensorHandleFactory::FactoryId GetTensorHandleFactoryId() const
Graph & TopologicalSort()
Sorts layers in topological order and return this.
virtual int Connect(IInputSlot &destination)=0
void ForEachLayer(Func func) const
std::list< Layer * > Layers
std::function< void(const std::string &name, const std::string &value)> ParameterStringifyFunction
Status AllocateDynamicBuffers()
Allocates memory for all tensors under output tensor handers of each layer.
const TensorInfo & GetTensorInfo() const override
const char * GetLayerTypeAsCString(LayerType type)
void AddCompatibilityLayers(std::map< BackendId, std::unique_ptr< class IBackendInternal >> &backends, TensorHandleFactoryRegistry ®istry)
Modifies the graph in-place, removing edges connecting layers using different compute devices...
static const FactoryId LegacyFactoryId
void MoveAllConnections(OutputSlot &destination)
Moves all connections to another OutputSlot.
LayerGuid GetGuid() const final
Returns the unique id of the layer.