19 #include <fmt/format.h> 21 #include <unordered_map> 29 : m_LayersInOrder(other.m_LayersInOrder)
31 std::unordered_map<const Layer*, Layer*> otherToClonedMap;
33 for (
auto&& otherLayer : other.m_Layers)
36 otherToClonedMap.emplace(otherLayer, layer);
40 for (
auto&& otherLayer : other.m_Layers)
42 Layer*
const thisLayer = otherToClonedMap[otherLayer];
45 for (
auto&& otherOutputSlot : otherLayer->GetOutputSlots())
47 for (
auto&& otherInputSlot : otherOutputSlot.GetConnections())
49 const Layer& otherTgtLayer = otherInputSlot->GetOwningLayer();
50 Layer*
const thisTgtLayer = otherToClonedMap[&otherTgtLayer];
53 outputSlot->Connect(inputSlot);
55 outputSlot->SetTensorInfo(otherOutputSlot.GetTensorInfo());
73 auto numInputSlots = it->GetNumInputSlots();
74 auto numOutputSlots = it->GetNumOutputSlots();
77 <<
":" << it->GetBackendId().Get()
78 <<
" has " << numInputSlots <<
" input slots" 79 <<
" and " << numOutputSlots <<
" output slots.";
81 for (
auto i : it->GetInputSlots())
83 std::ostringstream message;
84 auto inputTensorShape = i.GetConnectedOutputSlot()->GetTensorInfo().GetShape();
85 unsigned int numDims = inputTensorShape.GetNumDimensions();
87 message <<
"The input slot has shape [ ";
88 for (
unsigned int dim=0; dim < numDims; dim++)
90 message << inputTensorShape[dim] <<
",";
96 for (
unsigned int i = 0; i < it->GetNumOutputSlots(); i++)
99 std::ostringstream message;
100 auto outputTensorShape = layer->
GetOutputSlots()[i].GetTensorInfo().GetShape();
101 unsigned int numDims = outputTensorShape.GetNumDimensions();
103 message <<
"The output slot has shape [ ";
104 for (
unsigned int dim=0; dim < numDims; dim++)
106 message << outputTensorShape[dim] <<
",";
121 DotGraph graph(stream,
"Optimized");
140 for (
auto&& layer : m_Layers)
147 layer->SerializeLayerParameters(extractParams);
151 for (
auto&& layer : m_Layers)
155 for (
unsigned int i=0;i<layer->GetNumInputSlots(); i++)
159 DotEdge edge(stream, fromId, toId);
164 std::stringstream ss;
184 std::unordered_set<const ITensorHandle*> preallocatedTensors;
185 std::unordered_map<const ITensorHandle*, unsigned int> handleReferenceCounts;
189 auto TraceSubTensorHandleAncestry = [](
ITensorHandle*
const subTensorHandle)
192 while (ancestor && ancestor->
GetParent())
202 return tensorHandle && preallocatedTensors.find(tensorHandle) != preallocatedTensors.end();
207 for (
auto&& layer : m_Layers)
211 for (
auto&& slot = layer->BeginOutputSlots(); slot != layer->EndOutputSlots(); ++slot)
213 ITensorHandle *tensorHandle = TraceSubTensorHandleAncestry(slot->GetOutputHandler().GetData());
215 if (tensorHandle && !IsPreallocated(tensorHandle))
218 preallocatedTensors.insert(tensorHandle);
225 for (
auto&& layer : m_Layers)
229 for (
auto&& slot = layer->BeginOutputSlots(); slot != layer->EndOutputSlots(); ++slot)
231 ITensorHandle *tensorHandle = TraceSubTensorHandleAncestry(slot->GetOutputHandler().GetData());
233 if (tensorHandle && !IsPreallocated(tensorHandle))
235 unsigned int numConnections = slot->GetNumConnections();
236 if (handleReferenceCounts.find(tensorHandle) == handleReferenceCounts.end())
238 handleReferenceCounts[tensorHandle] = numConnections;
240 if (handleReferenceCounts[tensorHandle] == 0u)
248 handleReferenceCounts[tensorHandle] += numConnections;
255 for (
auto&& slot = layer->BeginInputSlots(); slot != layer->EndInputSlots(); ++slot)
258 slot->GetConnectedOutputSlot()->GetOutputHandler().GetData());
260 if (tensorHandle && !IsPreallocated(tensorHandle))
262 --handleReferenceCounts[tensorHandle];
264 if (handleReferenceCounts[tensorHandle] == 0u)
268 handleReferenceCounts.erase(tensorHandle);
279 if (!m_LayersInOrder)
282 for (
auto&& it : m_Layers)
287 auto compareLayerPriority = [](
const LayerList::value_type& layerA,
const LayerList::value_type& layerB)
289 return layerA->GetPriority() < layerB->GetPriority();
292 m_Layers.sort(compareLayerPriority);
294 m_LayersInOrder =
true;
305 auto MayNeedCompatibilityLayer = [](
const Layer& layer)
314 auto IsCompatibilityStrategy = [](
EdgeStrategy strategy)
320 ForEachLayer([
this, &backends, ®istry, MayNeedCompatibilityLayer, IsCompatibilityStrategy](
Layer* srcLayer)
324 if (!MayNeedCompatibilityLayer(*srcLayer))
330 const std::vector<OutputSlot>& srcOutputSlots = srcLayer->
GetOutputSlots();
331 for (
unsigned int srcOutputIndex = 0; srcOutputIndex < srcOutputSlots.size(); srcOutputIndex++)
334 const std::vector<InputSlot*> srcConnections = srcOutputSlot.
GetConnections();
335 const std::vector<EdgeStrategy> srcEdgeStrategies = srcOutputSlot.
GetEdgeStrategies();
336 for (
unsigned int srcConnectionIndex = 0; srcConnectionIndex < srcConnections.size(); srcConnectionIndex++)
338 InputSlot* dstInputSlot = srcConnections[srcConnectionIndex];
341 EdgeStrategy strategy = srcEdgeStrategies[srcConnectionIndex];
343 "Undefined memory strategy found while adding copy layers for compatibility");
346 if (MayNeedCompatibilityLayer(dstLayer) &&
347 IsCompatibilityStrategy(strategy))
352 const std::string compLayerName = fmt::format(
"[ {} ({}) -> {} ({}) ]",
357 Layer* compLayer =
nullptr;
360 compLayer = InsertNewLayer<MemCopyLayer>(*dstInputSlot, compLayerName.c_str());
365 compLayer = InsertNewLayer<MemImportLayer>(*dstInputSlot, compLayerName.c_str());
370 OutputSlot& compOutputSlot = compLayer->GetOutputSlot(0);
371 auto backendIt = backends.find(dstLayer.
GetBackendId());
372 if (backendIt != backends.end() &&
374 backendIt->second->SupportsTensorAllocatorAPI())
376 auto backend = backendIt->second.get();
377 auto tensorHandleFactoryIds = backend->GetHandleFactoryPreferences();
380 for (
auto preference : tensorHandleFactoryIds)
382 auto factory = registry.
GetFactory(preference);
386 auto srcFactory = registry.
GetFactory(srcPref);
390 bool canExportImport =
391 (factory->GetImportFlags() & srcFactory->GetExportFlags()) != 0;
393 if (factory->SupportsMapUnmap() || canExportImport)
417 const std::vector<InputSlot*>& newSourceConnections = srcOutputSlot.
GetConnections();
418 auto newSrcConnectionIndex = std::distance(newSourceConnections.begin(),
419 std::find(newSourceConnections.begin(),
420 newSourceConnections.end(),
421 &compLayer->GetInputSlot(0)));
424 srcOutputSlot.
SetEdgeStrategy(armnn::numeric_cast<unsigned int>(newSrcConnectionIndex),
436 ReplaceSubgraphConnections(subgraph, substituteLayer);
437 EraseSubgraphLayers(subgraph);
445 if (std::find(std::begin(m_Layers), std::end(m_Layers), layer) == std::end(m_Layers))
447 layer->
Reparent(*
this, m_Layers.end());
448 m_LayersInOrder =
false;
452 ReplaceSubgraphConnections(subgraph, substituteSubgraph);
453 EraseSubgraphLayers(subgraph);
464 ReplaceSubgraphConnections(subgraph, substituteSubgraph);
472 std::for_each(substituteSubgraphLayers.begin(), substituteSubgraphLayers.end(), [&](
Layer* layer)
475 ARMNN_ASSERT_MSG(std::find(m_Layers.begin(), m_Layers.end(), layer) != m_Layers.end(),
476 "Substitute layer is not a member of graph");
482 unsigned int subgraphNumInputSlots =
armnn::numeric_cast<
unsigned int>(subgraphInputSlots.size());
483 unsigned int subgraphNumOutputSlots =
armnn::numeric_cast<
unsigned int>(subgraphOutputSlots.size());
488 ARMNN_ASSERT(subgraphNumInputSlots == substituteSubgraphInputSlots.size());
489 ARMNN_ASSERT(subgraphNumOutputSlots == substituteSubgraphOutputSlots.size());
494 for (
unsigned int inputSlotIdx = 0; inputSlotIdx < subgraphNumInputSlots; ++inputSlotIdx)
496 InputSlot* subgraphInputSlot = subgraphInputSlots.at(inputSlotIdx);
501 connectedOutputSlot->
Disconnect(*subgraphInputSlot);
503 IInputSlot* substituteInputSlot = substituteSubgraphInputSlots.at(inputSlotIdx);
505 connectedOutputSlot->
Connect(*substituteInputSlot);
509 for(
unsigned int outputSlotIdx = 0; outputSlotIdx < subgraphNumOutputSlots; ++outputSlotIdx)
511 OutputSlot* subgraphOutputSlot = subgraphOutputSlots.at(outputSlotIdx);
514 OutputSlot* substituteOutputSlot = substituteSubgraphOutputSlots.at(outputSlotIdx);
533 for (
auto&& input : layer->GetInputSlots())
535 const IOutputSlot* source = input.GetConnectedOutputSlot();
538 std::ostringstream message;
539 message <<
"Input not connected on " 554 layer->ValidateTensorShapesFromInputs();
Graph(bool shapeInferenceMethod=false)
const std::vector< InputSlot * > & GetConnections() const
Interface for a layer that is connectable to other layers via InputSlots and OutputSlots.
NodeContent & AddContent(const std::string &content)
void SetEdgeStrategy(unsigned int connectionIndex, EdgeStrategy strategy)
No strategy has been defined. Used internally to verify integrity of optimizations.
const TensorShape & GetShape() const
Status SerializeToDot(std::ostream &stream)
const std::vector< EdgeStrategy > & GetEdgeStrategies() const
virtual void Reparent(Graph &dest, std::list< Layer *>::const_iterator iterator)=0
Layer & GetOwningLayer() const
Source backends tensor data can be exported to destination backend tensor without copy...
void EraseLayer(Iterator pos)
Deletes the layer at the specified position.
std::vector< OutputSlot * > OutputSlots
DotAttributeSet & GetAttributeSet()
virtual void Allocate()=0
Indicate to the memory manager that this resource is no longer active.
virtual Layer * Clone(Graph &graph) const =0
Creates a dynamically-allocated copy of this layer.
#define ARMNN_LOG(severity)
virtual void Manage()=0
Indicate to the memory manager that this resource is active.
Copyright (c) 2021 ARM Limited and Contributors.
void IgnoreUnused(Ts &&...)
Destination backend can work directly with tensors on source backend.
The SubgraphView class represents a subgraph of a Graph.
const InputSlot & GetInputSlot(unsigned int index) const override
Get a const input slot handle by slot index.
void ForEachLayer(Func func) const
#define ARMNN_ASSERT_MSG(COND, MSG)
DotAttributeSet & AddAttribute(const std::string &name, const std::stringstream &value)
NodeContent & GetContents()
An output connection slot for a layer.
virtual void Disconnect(IInputSlot &slot)=0
virtual ITensorHandle * GetParent() const =0
Get the parent tensor if this is a subtensor.
Validate all output shapes.
#define ARMNN_ASSERT(COND)
const BackendId & GetBackendId() const
const std::vector< OutputSlot > & GetOutputSlots() const
std::vector< InputSlot * > InputSlots
void SubstituteSubgraph(SubgraphView &subgraph, IConnectableLayer *substituteLayer)
Substitutes the given sub-graph with either a new layer or a new sub-graph.
void SetTensorHandleFactory(const ITensorHandleFactory::FactoryId &id)
const InputSlots & GetInputSlots() const
std::vector< OutputSlot >::iterator BeginOutputSlots()
const OutputSlots & GetOutputSlots() const
ITensorHandleFactory * GetFactory(ITensorHandleFactory::FactoryId id) const
Find a TensorHandleFactory by Id Returns nullptr if not found.
virtual bool IsTensorInfoSet() const =0
const Layers & GetLayers() const
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
const OutputSlot & GetOutputSlot(unsigned int index=0) const override
Get the const output slot handle by slot index.
DotAttributeSet & GetAttributeSet()
const char * GetName() const override
Returns the name of the layer.
ITensorHandleFactory::FactoryId GetTensorHandleFactoryId() const
Graph & TopologicalSort()
Sorts layers in topological order and return this.
virtual int Connect(IInputSlot &destination)=0
void ForEachLayer(Func func) const
std::list< Layer * > Layers
std::function< void(const std::string &name, const std::string &value)> ParameterStringifyFunction
Status AllocateDynamicBuffers()
Allocates memory for all tensors under output tensor handers of each layer.
const TensorInfo & GetTensorInfo() const override
const char * GetLayerTypeAsCString(LayerType type)
void AddCompatibilityLayers(std::map< BackendId, std::unique_ptr< class IBackendInternal >> &backends, TensorHandleFactoryRegistry ®istry)
Modifies the graph in-place, removing edges connecting layers using different compute devices...
static const FactoryId LegacyFactoryId
void MoveAllConnections(OutputSlot &destination)
Moves all connections to another OutputSlot.
LayerGuid GetGuid() const final
Returns the unique id of the layer.