18 #include <fmt/format.h> 20 #include <unordered_map> 28 : m_LayersInOrder(other.m_LayersInOrder)
29 , m_AllowExpandedDims(other.m_AllowExpandedDims)
30 , m_ShapeInferenceMethod(other.m_ShapeInferenceMethod)
31 , m_Profiler(other.m_Profiler)
33 std::unordered_map<const Layer*, Layer*> otherToClonedMap;
35 for (
auto&& otherLayer : other.m_Layers)
38 otherToClonedMap.emplace(otherLayer, layer);
42 for (
auto&& otherLayer : other.m_Layers)
44 Layer*
const thisLayer = otherToClonedMap[otherLayer];
47 for (
auto&& otherOutputSlot : otherLayer->GetOutputSlots())
49 for (
auto&& otherInputSlot : otherOutputSlot.GetConnections())
51 const Layer& otherTgtLayer = otherInputSlot->GetOwningLayer();
52 Layer*
const thisTgtLayer = otherToClonedMap[&otherTgtLayer];
55 outputSlot->Connect(inputSlot);
57 outputSlot->SetTensorInfo(otherOutputSlot.GetTensorInfo());
75 auto numInputSlots = it->GetNumInputSlots();
76 auto numOutputSlots = it->GetNumOutputSlots();
79 <<
":" << it->GetBackendId().Get()
80 <<
" has " << numInputSlots <<
" input slots" 81 <<
" and " << numOutputSlots <<
" output slots.";
83 for (
auto i : it->GetInputSlots())
85 std::ostringstream message;
86 auto inputTensorShape = i.GetConnectedOutputSlot()->GetTensorInfo().GetShape();
87 unsigned int numDims = inputTensorShape.GetNumDimensions();
89 message <<
"The input slot has shape [ ";
90 for (
unsigned int dim=0; dim < numDims; dim++)
92 message << inputTensorShape[dim] <<
",";
98 for (
unsigned int i = 0; i < it->GetNumOutputSlots(); i++)
101 std::ostringstream message;
102 auto outputTensorShape = layer->
GetOutputSlots()[i].GetTensorInfo().GetShape();
103 unsigned int numDims = outputTensorShape.GetNumDimensions();
105 message <<
"The output slot has shape [ ";
106 for (
unsigned int dim=0; dim < numDims; dim++)
108 message << outputTensorShape[dim] <<
",";
123 DotGraph graph(stream,
"Optimized");
142 for (
auto&& layer : m_Layers)
149 layer->SerializeLayerParameters(extractParams);
153 for (
auto&& layer : m_Layers)
157 for (
unsigned int i=0;i<layer->GetNumInputSlots(); i++)
161 DotEdge edge(stream, fromId, toId);
166 std::stringstream ss;
187 std::unordered_set<const ITensorHandle*> preallocatedTensors;
188 std::unordered_map<const ITensorHandle*, unsigned int> handleReferenceCounts;
192 auto TraceSubTensorHandleAncestry = [](
ITensorHandle*
const subTensorHandle)
195 while (ancestor && ancestor->
GetParent())
205 return tensorHandle && preallocatedTensors.find(tensorHandle) != preallocatedTensors.end();
210 for (
auto&& layer : m_Layers)
214 for (
auto&& slot = layer->BeginOutputSlots(); slot != layer->EndOutputSlots(); ++slot)
216 ITensorHandle *tensorHandle = TraceSubTensorHandleAncestry(slot->GetOutputHandler().GetData());
218 if (tensorHandle && !IsPreallocated(tensorHandle))
221 preallocatedTensors.insert(tensorHandle);
228 for (
auto&& layer : m_Layers)
232 for (
auto&& slot = layer->BeginOutputSlots(); slot != layer->EndOutputSlots(); ++slot)
234 ITensorHandle *tensorHandle = TraceSubTensorHandleAncestry(slot->GetOutputHandler().GetData());
236 if (tensorHandle && !IsPreallocated(tensorHandle))
238 unsigned int numConnections = slot->GetNumConnections();
239 if (handleReferenceCounts.find(tensorHandle) == handleReferenceCounts.end())
241 handleReferenceCounts[tensorHandle] = numConnections;
243 if (handleReferenceCounts[tensorHandle] == 0u)
251 handleReferenceCounts[tensorHandle] += numConnections;
258 for (
auto&& slot = layer->BeginInputSlots(); slot != layer->EndInputSlots(); ++slot)
261 slot->GetConnectedOutputSlot()->GetOutputHandler().GetData());
263 if (tensorHandle && !IsPreallocated(tensorHandle))
265 --handleReferenceCounts[tensorHandle];
267 if (handleReferenceCounts[tensorHandle] == 0u)
271 handleReferenceCounts.erase(tensorHandle);
282 if (!m_LayersInOrder)
285 for (
auto&& it : m_Layers)
290 auto compareLayerPriority = [](
const LayerList::value_type& layerA,
const LayerList::value_type& layerB)
292 return layerA->GetPriority() < layerB->GetPriority();
295 m_Layers.sort(compareLayerPriority);
297 m_LayersInOrder =
true;
308 auto MayNeedCompatibilityLayer = [](
const Layer& layer)
317 auto IsCompatibilityStrategy = [](
EdgeStrategy strategy)
323 ForEachLayer([
this, &backends, ®istry, MayNeedCompatibilityLayer, IsCompatibilityStrategy](
Layer* srcLayer)
327 if (!MayNeedCompatibilityLayer(*srcLayer))
333 const std::vector<OutputSlot>& srcOutputSlots = srcLayer->
GetOutputSlots();
334 for (
unsigned int srcOutputIndex = 0; srcOutputIndex < srcOutputSlots.size(); srcOutputIndex++)
337 const std::vector<InputSlot*> srcConnections = srcOutputSlot.
GetConnections();
338 const std::vector<EdgeStrategy> srcEdgeStrategies = srcOutputSlot.
GetEdgeStrategies();
339 for (
unsigned int srcConnectionIndex = 0; srcConnectionIndex < srcConnections.size(); srcConnectionIndex++)
341 InputSlot* dstInputSlot = srcConnections[srcConnectionIndex];
344 EdgeStrategy strategy = srcEdgeStrategies[srcConnectionIndex];
346 "Undefined memory strategy found while adding copy layers for compatibility");
349 if (MayNeedCompatibilityLayer(dstLayer) &&
350 IsCompatibilityStrategy(strategy))
355 const std::string compLayerName = fmt::format(
"[ {} ({}) -> {} ({}) ]",
360 Layer* compLayer =
nullptr;
363 compLayer = InsertNewLayer<MemCopyLayer>(*dstInputSlot, compLayerName.c_str());
368 compLayer = InsertNewLayer<MemImportLayer>(*dstInputSlot, compLayerName.c_str());
373 OutputSlot& compOutputSlot = compLayer->GetOutputSlot(0);
374 auto backendIt = backends.find(dstLayer.
GetBackendId());
375 if (backendIt != backends.end() &&
377 backendIt->second->SupportsTensorAllocatorAPI())
379 auto backend = backendIt->second.get();
380 auto tensorHandleFactoryIds = backend->GetHandleFactoryPreferences();
383 for (
auto preference : tensorHandleFactoryIds)
385 auto factory = registry.
GetFactory(preference);
389 auto srcFactory = registry.
GetFactory(srcPref);
393 bool canExportImport =
394 (factory->GetImportFlags() & srcFactory->GetExportFlags()) != 0;
396 if (factory->SupportsMapUnmap() || canExportImport)
420 const std::vector<InputSlot*>& newSourceConnections = srcOutputSlot.
GetConnections();
421 auto newSrcConnectionIndex = std::distance(newSourceConnections.begin(),
422 std::find(newSourceConnections.begin(),
423 newSourceConnections.end(),
424 &compLayer->GetInputSlot(0)));
427 srcOutputSlot.
SetEdgeStrategy(armnn::numeric_cast<unsigned int>(newSrcConnectionIndex),
451 if (std::find(std::begin(m_Layers),
453 iConnectableLayer) == std::end(m_Layers))
455 auto layer = PolymorphicDowncast<Layer*>(iConnectableLayer);
456 layer->Reparent(*
this, m_Layers.end());
457 m_LayersInOrder =
false;
461 ReplaceSubgraphConnections(subgraph, substituteSubgraph);
462 EraseSubgraphLayers(subgraph);
469 "New sub-graph used for substitution must not be empty");
472 std::for_each(substituteSubgraphLayers.begin(), substituteSubgraphLayers.end(), [&](
IConnectableLayer* layer)
475 layer = PolymorphicDowncast<Layer*>(layer);
476 ARMNN_ASSERT_MSG(std::find(m_Layers.begin(), m_Layers.end(), layer) != m_Layers.end(),
477 "Substitute layer is not a member of graph");
483 unsigned int subgraphNumInputSlots =
armnn::numeric_cast<
unsigned int>(subgraphInputSlots.size());
484 unsigned int subgraphNumOutputSlots =
armnn::numeric_cast<
unsigned int>(subgraphOutputSlots.size());
489 ARMNN_ASSERT(subgraphNumInputSlots == substituteSubgraphInputSlots.size());
490 ARMNN_ASSERT(subgraphNumOutputSlots == substituteSubgraphOutputSlots.size());
495 for (
unsigned int inputSlotIdx = 0; inputSlotIdx < subgraphNumInputSlots; ++inputSlotIdx)
497 IInputSlot* subgraphInputSlot = subgraphInputSlots.at(inputSlotIdx);
502 connectedOutputSlot->
Disconnect(*subgraphInputSlot);
504 IInputSlot* substituteInputSlot = substituteSubgraphInputSlots.at(inputSlotIdx);
506 connectedOutputSlot->
Connect(*substituteInputSlot);
510 for(
unsigned int outputSlotIdx = 0; outputSlotIdx < subgraphNumOutputSlots; ++outputSlotIdx)
512 auto subgraphOutputSlot =
513 PolymorphicDowncast<OutputSlot*>(subgraphOutputSlots.at(outputSlotIdx));
516 auto substituteOutputSlot =
517 PolymorphicDowncast<OutputSlot*>(substituteSubgraphOutputSlots.at(outputSlotIdx));
520 subgraphOutputSlot->MoveAllConnections(*substituteOutputSlot);
529 auto layer = PolymorphicDowncast<Layer*>(iConnectableLayer);
545 for (
auto&& output: layer->GetOutputSlots())
547 if (!output.IsTensorInfoSet())
549 std::ostringstream message;
550 message <<
"Output slot TensorInfo not set on " 566 for (
auto&& input : layer->GetInputSlots())
568 const IOutputSlot* source = input.GetConnectedOutputSlot();
573 ConstructErrorMessageForUnconnectedInputs(layer, input.GetSlotIndex());
578 std::ostringstream message;
579 message <<
"Output slot TensorInfo not set on " 582 << std::quoted(layer->GetName());
589 layer->ValidateTensorShapesFromInputs();
601 void Graph::ConstructErrorMessageForUnconnectedInputs(
Layer*
const layer,
602 unsigned int slotIndex)
604 std::ostringstream message;
605 bool noWeightsAndBias =
false;
612 message << std::endl;
618 if (biasSource == NULL)
620 message <<
"Weights and bias layers not set." << std::endl;
621 noWeightsAndBias =
true;
626 if (!noWeightsAndBias)
630 message <<
"Weights layer not set." << std::endl;
634 message <<
"Bias layer not set." << std::endl;
639 std::string slotString = noWeightsAndBias ?
"1 & 2" : std::to_string(slotIndex);
640 message <<
"Input slot(s) " 644 <<
" not connected to an output slot. " << std::endl
646 << std::quoted(layer->
GetName());
657 m_LayersInOrder =
false;
const std::vector< InputSlot * > & GetConnections() const
Interface for a layer that is connectable to other layers via InputSlots and OutputSlots.
NodeContent & AddContent(const std::string &content)
const IOutputSlots & GetIOutputSlots() const
void SetEdgeStrategy(unsigned int connectionIndex, EdgeStrategy strategy)
No strategy has been defined. Used internally to verify integrity of optimizations.
unsigned int GetNumInputSlots() const override
Returns the number of connectable input slots.
const TensorShape & GetShape() const
Status SerializeToDot(std::ostream &stream)
const IConnectableLayers & GetIConnectableLayers() const
const IInputSlots & GetIInputSlots() const
const std::vector< EdgeStrategy > & GetEdgeStrategies() const
Layer & GetOwningLayer() const
Source backends tensor data can be exported to destination backend tensor without copy...
void EraseLayer(Iterator pos)
Deletes the layer at the specified position.
DotAttributeSet & GetAttributeSet()
virtual void Allocate()=0
Indicate to the memory manager that this resource is no longer active.
virtual Layer * Clone(Graph &graph) const =0
Creates a dynamically-allocated copy of this layer.
#define ARMNN_LOG(severity)
virtual void Manage()=0
Indicate to the memory manager that this resource is active.
Copyright (c) 2021 ARM Limited and Contributors.
void IgnoreUnused(Ts &&...)
#define ARMNN_SCOPED_PROFILING_EVENT(backendId, name)
Destination backend can work directly with tensors on source backend.
The SubgraphView class represents a subgraph of a Graph.
const InputSlot & GetInputSlot(unsigned int index) const override
Get a const input slot handle by slot index.
void ForEachLayer(Func func) const
std::list< IConnectableLayer * > IConnectableLayers
#define ARMNN_ASSERT_MSG(COND, MSG)
DotAttributeSet & AddAttribute(const std::string &name, const std::stringstream &value)
NodeContent & GetContents()
An output connection slot for a layer.
std::vector< IOutputSlot * > IOutputSlots
virtual void Disconnect(IInputSlot &slot)=0
virtual ITensorHandle * GetParent() const =0
Get the parent tensor if this is a subtensor.
Validate all output shapes.
void ForEachIConnectableLayer(Func func) const
LayerType GetType() const override
Returns the armnn::LayerType of this layer.
std::vector< IInputSlot * > IInputSlots
#define ARMNN_ASSERT(COND)
void SetLayersOutOfOrder()
const BackendId & GetBackendId() const
const std::vector< OutputSlot > & GetOutputSlots() const
void VerifyConstantLayerSetTensorInfo() const
For each ConstantLayer in Graph, ensures TensorInfo is set on all output slots.
void SubstituteSubgraph(SubgraphView &subgraph, IConnectableLayer *substituteLayer)
Substitutes the given sub-graph with either a new layer or a new sub-graph.
const std::shared_ptr< IProfiler > & GetProfiler() const
void SetTensorHandleFactory(const ITensorHandleFactory::FactoryId &id)
arm::pipe::ProfilingGuid LayerGuid
Define LayerGuid type.
std::vector< OutputSlot >::iterator BeginOutputSlots()
ITensorHandleFactory * GetFactory(ITensorHandleFactory::FactoryId id) const
Find a TensorHandleFactory by Id Returns nullptr if not found.
virtual bool IsTensorInfoSet() const =0
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
const OutputSlot & GetOutputSlot(unsigned int index=0) const override
Get the const output slot handle by slot index.
DotAttributeSet & GetAttributeSet()
const char * GetName() const override
Returns the name of the layer.
ITensorHandleFactory::FactoryId GetTensorHandleFactoryId() const
Graph & TopologicalSort()
Sorts layers in topological order and return this.
virtual int Connect(IInputSlot &destination)=0
Graph(bool shapeInferenceMethod=false, bool allowExpandedDims=false)
std::function< void(const std::string &name, const std::string &value)> ParameterStringifyFunction
Status AllocateDynamicBuffers()
Allocates memory for all tensors under output tensor handers of each layer.
const TensorInfo & GetTensorInfo() const override
const char * GetLayerTypeAsCString(LayerType type)
void AddCompatibilityLayers(std::map< BackendId, std::unique_ptr< class IBackendInternal >> &backends, TensorHandleFactoryRegistry ®istry)
Modifies the graph in-place, removing edges connecting layers using different compute devices...
static const FactoryId LegacyFactoryId
LayerGuid GetGuid() const final
Returns the unique id of the layer.