From 27d830720ed706f187f2a40e2c5055e424aa8b91 Mon Sep 17 00:00:00 2001 From: Derek Lamberti Date: Tue, 5 Feb 2019 16:00:08 +0000 Subject: IVGCVSW-2606 Produce quantized InputNetwork from simple FP32 InputNetwork Change-Id: I2140a7af5961ddf8267fbb127202de3900ea79e3 Signed-off-by: Derek Lamberti --- CMakeLists.txt | 8 ++ include/armnn/INetworkQuantizer.hpp | 33 +++++++ src/armnn/Layer.cpp | 29 ++++++ src/armnn/Layer.hpp | 4 + src/armnn/LayerVisitorBase.hpp | 179 ++++++++++++++++++++++++++++++++++++ src/armnn/NetworkQuantizer.cpp | 63 +++++++++++++ src/armnn/NetworkQuantizer.hpp | 26 ++++++ src/armnn/QuantizerVisitor.cpp | 111 ++++++++++++++++++++++ src/armnn/QuantizerVisitor.hpp | 50 ++++++++++ src/armnn/StaticRangeVisitor.cpp | 38 ++++++++ src/armnn/StaticRangeVisitor.hpp | 45 +++++++++ 11 files changed, 586 insertions(+) create mode 100644 include/armnn/INetworkQuantizer.hpp create mode 100644 src/armnn/LayerVisitorBase.hpp create mode 100644 src/armnn/NetworkQuantizer.cpp create mode 100644 src/armnn/NetworkQuantizer.hpp create mode 100644 src/armnn/QuantizerVisitor.cpp create mode 100644 src/armnn/QuantizerVisitor.hpp create mode 100644 src/armnn/StaticRangeVisitor.cpp create mode 100644 src/armnn/StaticRangeVisitor.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bf885a2125..1bec6d1d70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -157,6 +157,7 @@ list(APPEND armnn_sources include/armnn/ILayerSupport.hpp include/armnn/ILayerVisitor.hpp include/armnn/INetwork.hpp + include/armnn/INetworkQuantizer.hpp include/armnn/IProfiler.hpp include/armnn/IRuntime.hpp include/armnn/LayerSupport.hpp @@ -275,10 +276,13 @@ list(APPEND armnn_sources src/armnn/LayersFwd.hpp src/armnn/LayerSupportCommon.hpp src/armnn/LayerSupport.cpp + src/armnn/LayerVisitorBase.hpp src/armnn/LoadedNetwork.cpp src/armnn/LoadedNetwork.hpp src/armnn/Network.cpp src/armnn/Network.hpp + src/armnn/NetworkQuantizer.cpp + src/armnn/NetworkQuantizer.hpp src/armnn/NetworkUtils.cpp src/armnn/NetworkUtils.hpp src/armnn/Observable.cpp @@ -289,10 +293,14 @@ list(APPEND armnn_sources src/armnn/ProfilingEvent.cpp src/armnn/ProfilingEvent.hpp src/armnn/Profiling.hpp + src/armnn/QuantizerVisitor.cpp + src/armnn/QuantizerVisitor.hpp src/armnn/Runtime.cpp src/armnn/Runtime.hpp src/armnn/SerializeLayerParameters.cpp src/armnn/SerializeLayerParameters.hpp + src/armnn/StaticRangeVisitor.cpp + src/armnn/StaticRangeVisitor.hpp src/armnn/SubGraph.cpp src/armnn/SubGraph.hpp src/armnn/SubGraphSelector.cpp diff --git a/include/armnn/INetworkQuantizer.hpp b/include/armnn/INetworkQuantizer.hpp new file mode 100644 index 0000000000..f3cc13ca35 --- /dev/null +++ b/include/armnn/INetworkQuantizer.hpp @@ -0,0 +1,33 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include + +#include + + +namespace armnn +{ + +using INetworkQuantizerPtr = std::unique_ptr; + +/// Quantizer class Quantizes a float32 InputNetwork +class INetworkQuantizer +{ +public: + static INetworkQuantizer* CreateRaw(INetwork* inputNetwork); ///< Create Quantizer object and return raw pointer + static INetworkQuantizerPtr Create(INetwork* inputNetwork); ///< Create Quantizer object wrapped in unique_ptr + static void Destroy(INetworkQuantizer* quantizer); ///< Destroy Quantizer object + + /// Extract final quantized network + virtual INetworkPtr ExportNetwork() = 0; + +protected: + virtual ~INetworkQuantizer() {}; +}; + +} //namespace armnn diff --git a/src/armnn/Layer.cpp b/src/armnn/Layer.cpp index 85e1de0b09..50b28adfc7 100644 --- a/src/armnn/Layer.cpp +++ b/src/armnn/Layer.cpp @@ -106,6 +106,35 @@ void OutputSlot::MoveAllConnections(OutputSlot& destination) } } +unsigned int OutputSlot::CalculateIndexOnOwner() const +{ + for (unsigned int i=0; i < GetOwningLayer().GetNumOutputSlots(); i++) + { + if (GetOwningLayer().GetOutputSlot(i) == (*this)) + { + return i; + } + } + BOOST_ASSERT_MSG(false, "Did not find slot on owner."); + return 0; // Error +} + +bool OutputSlot::operator==(const OutputSlot& other) const +{ + bool isSame = other.GetNumConnections() == GetNumConnections(); + if (!isSame) + { + return false; + } + + for (unsigned int i=0; i < GetNumConnections(); i++) + { + isSame &= other.GetConnection(i) == GetConnection(i); + } + return isSame; +} + + void OutputSlot::ValidateConnectionIndex(unsigned int index) const { if (boost::numeric_cast(index) >= m_Connections.size()) diff --git a/src/armnn/Layer.hpp b/src/armnn/Layer.hpp index 51c6c09563..c08c6b0631 100644 --- a/src/armnn/Layer.hpp +++ b/src/armnn/Layer.hpp @@ -141,6 +141,10 @@ public: return Disconnect(*boost::polymorphic_downcast(&slot)); } + unsigned int CalculateIndexOnOwner() const; + + bool operator==(const OutputSlot& other) const; + private: void ValidateConnectionIndex(unsigned int index) const; diff --git a/src/armnn/LayerVisitorBase.hpp b/src/armnn/LayerVisitorBase.hpp new file mode 100644 index 0000000000..037a5a75ac --- /dev/null +++ b/src/armnn/LayerVisitorBase.hpp @@ -0,0 +1,179 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include + +namespace armnn +{ + +// Visitor base class with empty implementations. +class LayerVisitorBase : public ILayerVisitor +{ +protected: + LayerVisitorBase() {} + virtual ~LayerVisitorBase() {} + +public: + virtual void VisitInputLayer(const IConnectableLayer*, + LayerBindingId, + const char*) {} + + virtual void VisitConvolution2dLayer(const IConnectableLayer*, + const Convolution2dDescriptor&, + const ConstTensor&, + const char*) {} + + virtual void VisitConvolution2dLayer(const IConnectableLayer*, + const Convolution2dDescriptor&, + const ConstTensor&, + const ConstTensor&, + const char*) {} + + virtual void VisitDepthwiseConvolution2dLayer(const IConnectableLayer*, + const DepthwiseConvolution2dDescriptor&, + const ConstTensor& , + const char*) {} + + virtual void VisitDepthwiseConvolution2dLayer(const IConnectableLayer*, + const DepthwiseConvolution2dDescriptor&, + const ConstTensor&, + const ConstTensor&, + const char*) {} + + virtual void VisitDetectionPostProcessLayer(const IConnectableLayer*, + const DetectionPostProcessDescriptor&, + const char*) {} + + virtual void VisitFullyConnectedLayer(const IConnectableLayer*, + const FullyConnectedDescriptor&, + const ConstTensor&, + const char*) {} + + virtual void VisitFullyConnectedLayer(const IConnectableLayer*, + const FullyConnectedDescriptor&, + const ConstTensor&, + const ConstTensor&, + const char*) {} + + virtual void VisitPermuteLayer(const IConnectableLayer*, + const PermuteDescriptor&, + const char*) {} + + virtual void VisitBatchToSpaceNdLayer(const IConnectableLayer*, + const BatchToSpaceNdDescriptor&, + const char*) {} + + virtual void VisitPooling2dLayer(const IConnectableLayer*, + const Pooling2dDescriptor&, + const char*) {} + + virtual void VisitActivationLayer(const IConnectableLayer*, + const ActivationDescriptor&, + const char*) {} + + virtual void VisitNormalizationLayer(const IConnectableLayer*, + const NormalizationDescriptor&, + const char*) {} + + virtual void VisitSoftmaxLayer(const IConnectableLayer*, + const SoftmaxDescriptor&, + const char*) {} + + virtual void VisitSplitterLayer(const IConnectableLayer*, + const ViewsDescriptor&, + const char*) {} + + virtual void VisitMergerLayer(const IConnectableLayer*, + const OriginsDescriptor&, + const char*) {} + + virtual void VisitAdditionLayer(const IConnectableLayer*, + const char*) {} + + virtual void VisitMultiplicationLayer(const IConnectableLayer*, + const char*) {} + + virtual void VisitBatchNormalizationLayer(const IConnectableLayer*, + const BatchNormalizationDescriptor&, + const ConstTensor&, + const ConstTensor&, + const ConstTensor&, + const ConstTensor&, + const char*) {} + + virtual void VisitResizeBilinearLayer(const IConnectableLayer*, + const ResizeBilinearDescriptor&, + const char*) {} + + virtual void VisitL2NormalizationLayer(const IConnectableLayer*, + const L2NormalizationDescriptor&, + const char*) {} + + virtual void VisitConstantLayer(const IConnectableLayer*, + const ConstTensor&, + const char*) {} + + virtual void VisitReshapeLayer(const IConnectableLayer*, + const ReshapeDescriptor&, + const char*) {} + + virtual void VisitSpaceToBatchNdLayer(const IConnectableLayer*, + const SpaceToBatchNdDescriptor&, + const char*) {} + + virtual void VisitFloorLayer(const IConnectableLayer*, + const char*) {} + + virtual void VisitOutputLayer(const IConnectableLayer*, + LayerBindingId id, + const char*) {} + + virtual void VisitLstmLayer(const IConnectableLayer*, + const LstmDescriptor&, + const LstmInputParams&, + const char*) {} + + virtual void VisitDivisionLayer(const IConnectableLayer*, + const char*) {} + + virtual void VisitSubtractionLayer(const IConnectableLayer*, + const char*) {} + + virtual void VisitMaximumLayer(const IConnectableLayer*, + const char*) {} + + virtual void VisitMeanLayer(const IConnectableLayer*, + const MeanDescriptor&, + const char*) {} + + virtual void VisitPadLayer(const IConnectableLayer*, + const PadDescriptor&, + const char*) {} + + virtual void VisitStridedSliceLayer(const IConnectableLayer*, + const StridedSliceDescriptor&, + const char*) {} + + virtual void VisitMinimumLayer(const IConnectableLayer*, + const char*) {} + + virtual void VisitGreaterLayer(const IConnectableLayer*, + const char*) {} + + virtual void VisitEqualLayer(const IConnectableLayer*, + const char*) {} + + virtual void VisitRsqrtLayer(const IConnectableLayer*, + const char*) {} + + virtual void VisitGatherLayer(const IConnectableLayer*, + const char*) {} + +}; + +} //namespace armnn + diff --git a/src/armnn/NetworkQuantizer.cpp b/src/armnn/NetworkQuantizer.cpp new file mode 100644 index 0000000000..f8e5ed23a7 --- /dev/null +++ b/src/armnn/NetworkQuantizer.cpp @@ -0,0 +1,63 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include +#include +#include +#include + +#include "Graph.hpp" +#include "Layer.hpp" +#include "Network.hpp" +#include "NetworkQuantizer.hpp" + +#include "StaticRangeVisitor.hpp" +#include "QuantizerVisitor.hpp" + +#include +#include +#include + +namespace armnn +{ + +INetworkQuantizer* INetworkQuantizer::CreateRaw(INetwork *inputNetwork) +{ + return new NetworkQuantizer(inputNetwork); +} + +INetworkQuantizerPtr INetworkQuantizer::Create(INetwork* inputNetwork) +{ + return INetworkQuantizerPtr(CreateRaw(inputNetwork), &INetworkQuantizer::Destroy); +} + +void INetworkQuantizer::Destroy(INetworkQuantizer *quantizer) +{ + delete boost::polymorphic_downcast(quantizer); +} + +INetworkPtr NetworkQuantizer::ExportNetwork() +{ + const Graph& graph = boost::polymorphic_downcast(m_InputNetwork)->GetGraph().TopologicalSort(); + auto VisitLayers = [&graph](ILayerVisitor& visitor) + { + for (auto layer : graph) + { + layer->Accept(visitor); + } + }; + + // Step 1) Walk the graph and register min/max values for intermediate tensors + StaticRangeVisitor rangeVisitor; + VisitLayers(rangeVisitor); + + // Step 2) Convert input InputNetwork to Quantized InputNetwork + QuantizerVisitor quantizerVisitor(&rangeVisitor); + VisitLayers(quantizerVisitor); + + return quantizerVisitor.RetrieveFinalNetwork(); +} + +} //namespace armn \ No newline at end of file diff --git a/src/armnn/NetworkQuantizer.hpp b/src/armnn/NetworkQuantizer.hpp new file mode 100644 index 0000000000..5543b3a444 --- /dev/null +++ b/src/armnn/NetworkQuantizer.hpp @@ -0,0 +1,26 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include +#include +#include + +namespace armnn +{ + +class NetworkQuantizer : public INetworkQuantizer +{ +public: + NetworkQuantizer(INetwork* inputNetwork) : m_InputNetwork(inputNetwork) {} + + INetworkPtr ExportNetwork() override; + +private: + INetwork* m_InputNetwork; +}; + +} //namespace armnn \ No newline at end of file diff --git a/src/armnn/QuantizerVisitor.cpp b/src/armnn/QuantizerVisitor.cpp new file mode 100644 index 0000000000..7608d0a440 --- /dev/null +++ b/src/armnn/QuantizerVisitor.cpp @@ -0,0 +1,111 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include "Network.hpp" +#include "QuantizerVisitor.hpp" +#include "StaticRangeVisitor.hpp" + +#include +#include +#include + +namespace armnn +{ + +namespace { + +std::pair ComputeQAsymmParams(int numBits, double min, double max) +{ + BOOST_ASSERT_MSG(min < max, "Min >= max will result in invalid quantization."); + double highest = (1 << numBits)-1; + + min = std::min(0.0, min); // min <= 0.0 + max = std::max(0.0, max); // max >= 0.0 + + // assumes quantization range [0-highest] + double scale = (max-min) / highest; + double offset = -min / scale; + + // clamp offset [0-highest] + offset = std::max(0.0, std::min(highest, offset)); + + return std::make_pair(static_cast(std::round(offset)), static_cast(scale)); +} + +} // namespace + +QuantizerVisitor::QuantizerVisitor(armnn::StaticRangeVisitor* ranges) +: m_Ranges(ranges) +, m_QuantizedNetwork(INetwork::Create()) +{ +} + +void QuantizerVisitor::SetQuantizedInputConnections(const IConnectableLayer *srcLayer, + IConnectableLayer *quantizedLayer) +{ + m_OldToNewGuidMap[srcLayer->GetGuid()] = quantizedLayer->GetGuid(); + + for (unsigned int i=0; i < srcLayer->GetNumInputSlots(); i++) + { + const IInputSlot& srcInputSlot = srcLayer->GetInputSlot(i); + const InputSlot* inputSlot = boost::polymorphic_downcast(&srcInputSlot); + const OutputSlot* outputSlot = inputSlot->GetConnectedOutputSlot(); + + unsigned int slotIdx = outputSlot->CalculateIndexOnOwner(); + Layer& layerToFind = outputSlot->GetOwningLayer(); + + auto found = m_OldToNewGuidMap.find(layerToFind.GetGuid()); + if (found != m_OldToNewGuidMap.end()) + { + // Connect the slots in the quantized model + IConnectableLayer* prevQuantizedLayer = m_GuidToLayerMap[found->second]; + IInputSlot& newInputSlot = quantizedLayer->GetInputSlot(i); + IOutputSlot& newOutputSlot = prevQuantizedLayer->GetOutputSlot(slotIdx); + newOutputSlot.Connect(newInputSlot); + + // Fetch the min/max ranges that were computed earlier + auto range = m_Ranges->GetRange(layerToFind.GetGuid(), i); + auto qParams = ComputeQAsymmParams(8, range.first, range.second); + + // Set the quantization params + TensorInfo info(newOutputSlot.GetTensorInfo()); + info.SetDataType(DataType::QuantisedAsymm8); + info.SetQuantizationOffset(qParams.first); + info.SetQuantizationScale(qParams.second); + } + else + { + // error in graph traversal order + BOOST_ASSERT_MSG(false, "Error in graph traversal"); + } + } +} + +void QuantizerVisitor::RecordLayer(IConnectableLayer* layer) +{ + m_GuidToLayerMap[layer->GetGuid()] = layer; +} + +void QuantizerVisitor::VisitAdditionLayer(const IConnectableLayer *layer, const char *name) +{ + IConnectableLayer* newLayer = m_QuantizedNetwork->AddAdditionLayer(name); + RecordLayer(newLayer); + SetQuantizedInputConnections(layer, newLayer); +} + +void QuantizerVisitor::VisitInputLayer(const IConnectableLayer *layer, LayerBindingId id, const char *name) +{ + IConnectableLayer* newLayer = m_QuantizedNetwork->AddInputLayer(id, name); + RecordLayer(newLayer); +} + +void QuantizerVisitor::VisitOutputLayer(const IConnectableLayer *layer, LayerBindingId id, const char *name) +{ + IConnectableLayer* newLayer = m_QuantizedNetwork->AddOutputLayer(id, name); + RecordLayer(newLayer); + SetQuantizedInputConnections(layer, newLayer); +} + +} //namespace armnn \ No newline at end of file diff --git a/src/armnn/QuantizerVisitor.hpp b/src/armnn/QuantizerVisitor.hpp new file mode 100644 index 0000000000..bf017d7205 --- /dev/null +++ b/src/armnn/QuantizerVisitor.hpp @@ -0,0 +1,50 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include "LayerVisitorBase.hpp" +#include +#include + +#include + +namespace armnn +{ + +// Forward declarations +class StaticRangeVisitor; + +/// Visitor object for quantizing layers in a network +class QuantizerVisitor : public LayerVisitorBase +{ +public: + QuantizerVisitor(StaticRangeVisitor* ranges); + ~QuantizerVisitor() = default; + + // Functions to quantize the individual layers, overridden from ILayerVisitor + void VisitInputLayer(const IConnectableLayer *layer, LayerBindingId id, const char *name = nullptr) override; + void VisitAdditionLayer(const IConnectableLayer *layer, const char *name = nullptr) override; + void VisitOutputLayer(const IConnectableLayer *layer, LayerBindingId id, const char *name = nullptr) override; + + // Extract the quantized network + INetworkPtr RetrieveFinalNetwork() { return std::move(m_QuantizedNetwork); } +private: + + /// Connects the layer to preceeding layers and sets the quantization parameters based on recorded ranges + void SetQuantizedInputConnections(const IConnectableLayer *srcLayer, IConnectableLayer *quantizedLayer); + + /// Record the guid so we can easily find it later + void RecordLayer(IConnectableLayer* layer); + + + StaticRangeVisitor* m_Ranges; ///< Previously recorded min/max ranges per intermediate tensor + INetworkPtr m_QuantizedNetwork; ///< Quantized version of the model we are building up + + std::map m_OldToNewGuidMap; ///< Mapping from input network guids to quantized network guids + std::map m_GuidToLayerMap; ///< Mapping from guid to layer in quantized network +}; + +} //namespace armnn \ No newline at end of file diff --git a/src/armnn/StaticRangeVisitor.cpp b/src/armnn/StaticRangeVisitor.cpp new file mode 100644 index 0000000000..8e90ba8b51 --- /dev/null +++ b/src/armnn/StaticRangeVisitor.cpp @@ -0,0 +1,38 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include "StaticRangeVisitor.hpp" + + +namespace armnn +{ + +void StaticRangeVisitor::SetRange(const IConnectableLayer* layer, unsigned int outputIdx, float min, float max) +{ + auto& ranges = m_GuidToRangesMap[layer->GetGuid()]; + + if (ranges.size() < layer->GetNumOutputSlots()) + { + ranges.resize(layer->GetNumOutputSlots()); + } + ranges[outputIdx] = std::make_pair(min, max); +} + +StaticRangeVisitor::MinMaxRange StaticRangeVisitor::GetRange(LayerGuid guid, unsigned int idx) const +{ + auto found = m_GuidToRangesMap.find(guid); + if (found != m_GuidToRangesMap.end()) + { + return found->second.at(idx); + } + return DefaultRange(); +} + +void StaticRangeVisitor::VisitAdditionLayer(const IConnectableLayer *layer, const char *name) +{ + SetRange(layer, 0, -20.f, 20.f); +}; + +} //namespace armnn \ No newline at end of file diff --git a/src/armnn/StaticRangeVisitor.hpp b/src/armnn/StaticRangeVisitor.hpp new file mode 100644 index 0000000000..38f0088523 --- /dev/null +++ b/src/armnn/StaticRangeVisitor.hpp @@ -0,0 +1,45 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include "LayerVisitorBase.hpp" + +#include + +#include +#include + +namespace armnn +{ + +/// Visitor class to establish min/max ranges based on the type of the layer +class StaticRangeVisitor : public LayerVisitorBase +{ +public: + StaticRangeVisitor() = default; + ~StaticRangeVisitor() = default; + + using MinMaxRange = std::pair; + using MinMaxRanges = std::vector; + + /// Functions to set the Range on a per-layer-type basis + void VisitAdditionLayer(const IConnectableLayer *layer, const char *name = nullptr) override; + + /// Retreive the default range + MinMaxRange DefaultRange() const { return std::make_pair(-15.0f, 15.0f); } + + /// Retreive the Range for a particular output slot on a particular layer + MinMaxRange GetRange(LayerGuid guid, unsigned int idx) const; + +private: + /// Set the range for an output slot on a layer + void SetRange(const IConnectableLayer* layer, unsigned int outputIdx, float min, float max); + + /// Mapping from Guid to an array of ranges for outputs + std::map m_GuidToRangesMap; +}; + +} //namespace armnn \ No newline at end of file -- cgit v1.2.1