From 4628d05455dfc179f0437913185e76888115a98a Mon Sep 17 00:00:00 2001 From: Narumol Prangnawarat Date: Mon, 25 Feb 2019 17:26:05 +0000 Subject: IVGCVSW-2560 Verify Inference test for TensorFlow Lite MobileNet SSD * Assign output shape of MobileNet SSD to ArmNN network * Add m_OverridenOutputShapes to TfLiteParser to set shape in GetNetworkOutputBindingInfo * Use input quantization instead of output quantization params * Correct data and datatype in Inference test Change-Id: I01ac2e07ed08e8928ba0df33a4847399e1dd8394 Signed-off-by: Narumol Prangnawarat Signed-off-by: Aron Virginas-Tar --- include/armnn/Descriptors.hpp | 2 +- src/armnnTfLiteParser/TfLiteParser.cpp | 32 +++++++--- src/armnnTfLiteParser/TfLiteParser.hpp | 4 ++ .../test/DetectionPostProcess.cpp | 4 -- .../reference/workloads/DetectionPostProcess.cpp | 2 +- tests/InferenceModel.hpp | 7 +++ tests/MobileNetSsdDatabase.hpp | 50 ++++++++++++---- tests/MobileNetSsdInferenceTest.hpp | 69 +++++++++++----------- tests/ObjectDetectionCommon.hpp | 12 +++- 9 files changed, 117 insertions(+), 65 deletions(-) diff --git a/include/armnn/Descriptors.hpp b/include/armnn/Descriptors.hpp index b14ed0b1ab..dc09cc60e8 100644 --- a/include/armnn/Descriptors.hpp +++ b/include/armnn/Descriptors.hpp @@ -332,7 +332,7 @@ struct DetectionPostProcessDescriptor DetectionPostProcessDescriptor() : m_MaxDetections(0) , m_MaxClassesPerDetection(1) - , m_DetectionsPerClass(100) + , m_DetectionsPerClass(1) , m_NmsScoreThreshold(0) , m_NmsIouThreshold(0) , m_NumClasses(0) diff --git a/src/armnnTfLiteParser/TfLiteParser.cpp b/src/armnnTfLiteParser/TfLiteParser.cpp index cd0e9214c2..31e808fd6e 100644 --- a/src/armnnTfLiteParser/TfLiteParser.cpp +++ b/src/armnnTfLiteParser/TfLiteParser.cpp @@ -295,7 +295,7 @@ void CalcPadding(uint32_t inputSize, } } -armnn::TensorInfo ToTensorInfo(TfLiteParser::TensorRawPtr tensorPtr) +armnn::TensorInfo ToTensorInfo(TfLiteParser::TensorRawPtr tensorPtr, const std::vector& shapes) { armnn::DataType type; CHECK_TENSOR_PTR(tensorPtr); @@ -345,17 +345,21 @@ armnn::TensorInfo ToTensorInfo(TfLiteParser::TensorRawPtr tensorPtr) } } - auto const & dimensions = AsUnsignedVector(tensorPtr->shape); - // two statements (on purpose) for easier debugging: - armnn::TensorInfo result(static_cast(tensorPtr->shape.size()), - dimensions.data(), + armnn::TensorInfo result(static_cast(shapes.size()), + shapes.data(), type, quantizationScale, quantizationOffset); return result; } +armnn::TensorInfo ToTensorInfo(TfLiteParser::TensorRawPtr tensorPtr) +{ + auto const & dimensions = AsUnsignedVector(tensorPtr->shape); + return ToTensorInfo(tensorPtr, dimensions); +} + template std::pair> CreateConstTensorImpl(TfLiteParser::BufferRawPtr bufferPtr, @@ -1796,10 +1800,17 @@ void TfLiteParser::ParseDetectionPostProcess(size_t subgraphIndex, size_t operat BOOST_ASSERT(layer != nullptr); - // Register outputs + // The model does not specify the output shapes. + // The output shapes are calculated from the max_detection and max_classes_per_detection. + unsigned int numDetectedBox = desc.m_MaxDetections * desc.m_MaxClassesPerDetection; + m_OverridenOutputShapes.push_back({ 1, numDetectedBox, 4 }); + m_OverridenOutputShapes.push_back({ 1, numDetectedBox }); + m_OverridenOutputShapes.push_back({ 1, numDetectedBox }); + m_OverridenOutputShapes.push_back({ 1 }); + for (unsigned int i = 0 ; i < outputs.size() ; ++i) { - armnn::TensorInfo detectionBoxOutputTensorInfo = ToTensorInfo(outputs[i]); + armnn::TensorInfo detectionBoxOutputTensorInfo = ToTensorInfo(outputs[i], m_OverridenOutputShapes[i]); layer->GetOutputSlot(i).SetTensorInfo(detectionBoxOutputTensorInfo); } @@ -2232,12 +2243,15 @@ BindingPointInfo TfLiteParser::GetNetworkOutputBindingInfo(size_t subgraphId, { CHECK_SUBGRAPH(m_Model, subgraphId); auto outputs = GetSubgraphOutputs(m_Model, subgraphId); - for (auto const & output : outputs) + for (unsigned int i = 0; i < outputs.size(); ++i) { + auto const output = outputs[i]; if (output.second->name == name) { auto bindingId = GenerateLayerBindingId(subgraphId, output.first); - return std::make_pair(bindingId, ToTensorInfo(output.second)); + std::vector shape = m_OverridenOutputShapes.size() > 0 ? + m_OverridenOutputShapes[i] : AsUnsignedVector(output.second->shape); + return std::make_pair(bindingId, ToTensorInfo(output.second, shape)); } } diff --git a/src/armnnTfLiteParser/TfLiteParser.hpp b/src/armnnTfLiteParser/TfLiteParser.hpp index 3fe4809aa2..2895487214 100644 --- a/src/armnnTfLiteParser/TfLiteParser.hpp +++ b/src/armnnTfLiteParser/TfLiteParser.hpp @@ -193,6 +193,10 @@ private: /// Connections for tensors in each subgraph /// The first index is the subgraph ID, the second index is the tensor ID std::vector m_SubgraphConnections; + + /// This is used in case that the model does not speciry the output. + /// The shape can be calculated from the options. + std::vector> m_OverridenOutputShapes; }; } diff --git a/src/armnnTfLiteParser/test/DetectionPostProcess.cpp b/src/armnnTfLiteParser/test/DetectionPostProcess.cpp index 3c602937ce..3002885016 100644 --- a/src/armnnTfLiteParser/test/DetectionPostProcess.cpp +++ b/src/armnnTfLiteParser/test/DetectionPostProcess.cpp @@ -76,28 +76,24 @@ struct DetectionPostProcessFixture : ParserFlatbuffersFixture } }, { - "shape": [1, 3, 4], "type": "FLOAT32", "buffer": 3, "name": "detection_boxes", "quantization": {} }, { - "shape": [1, 3], "type": "FLOAT32", "buffer": 4, "name": "detection_classes", "quantization": {} }, { - "shape": [1, 3], "type": "FLOAT32", "buffer": 5, "name": "detection_scores", "quantization": {} }, { - "shape": [1], "type": "FLOAT32", "buffer": 6, "name": "num_detections", diff --git a/src/backends/reference/workloads/DetectionPostProcess.cpp b/src/backends/reference/workloads/DetectionPostProcess.cpp index 2eb35f5ffa..6868180b0b 100644 --- a/src/backends/reference/workloads/DetectionPostProcess.cpp +++ b/src/backends/reference/workloads/DetectionPostProcess.cpp @@ -197,7 +197,7 @@ void DetectionPostProcess(const TensorInfo& boxEncodingsInfo, } std::vector selectedIndices = NonMaxSuppression(numBoxes, boxCorners, classScores, desc.m_NmsScoreThreshold, - desc.m_MaxClassesPerDetection, + desc.m_DetectionsPerClass, desc.m_NmsIouThreshold); for (unsigned int i = 0; i < selectedIndices.size(); ++i) diff --git a/tests/InferenceModel.hpp b/tests/InferenceModel.hpp index 6e73f5275f..25ccbee45a 100644 --- a/tests/InferenceModel.hpp +++ b/tests/InferenceModel.hpp @@ -577,6 +577,13 @@ public: m_OutputBindings[outputIndex].second.GetQuantizationOffset()); } + QuantizationParams GetInputQuantizationParams(unsigned int inputIndex = 0u) const + { + CheckInputIndexIsValid(inputIndex); + return std::make_pair(m_InputBindings[inputIndex].second.GetQuantizationScale(), + m_InputBindings[inputIndex].second.GetQuantizationOffset()); + } + std::vector GetAllQuantizationParams() const { std::vector quantizationParams; diff --git a/tests/MobileNetSsdDatabase.hpp b/tests/MobileNetSsdDatabase.hpp index cac558793f..7a30f221d2 100644 --- a/tests/MobileNetSsdDatabase.hpp +++ b/tests/MobileNetSsdDatabase.hpp @@ -27,14 +27,17 @@ namespace struct MobileNetSsdTestCaseData { MobileNetSsdTestCaseData( - std::vector inputData, - std::vector expectedOutput) - : m_InputData(std::move(inputData)) - , m_ExpectedOutput(std::move(expectedOutput)) + const std::vector& inputData, + const std::vector& expectedDetectedObject, + const std::vector>& expectedOutput) + : m_InputData(inputData) + , m_ExpectedDetectedObject(expectedDetectedObject) + , m_ExpectedOutput(expectedOutput) {} - std::vector m_InputData; - std::vector m_ExpectedOutput; + std::vector m_InputData; + std::vector m_ExpectedDetectedObject; + std::vector> m_ExpectedOutput; }; class MobileNetSsdDatabase @@ -59,7 +62,9 @@ const std::array g_PerTestCaseInput = ObjectDetectionInput { "Cat.jpg", - DetectedObject(16, BoundingBox(0.208961248f, 0.0852333307f, 0.92757535f, 0.940263629f), 0.79296875f) + { + DetectedObject(16.0f, BoundingBox(0.208961248f, 0.0852333307f, 0.92757535f, 0.940263629f), 0.79296875f) + } } }; @@ -100,12 +105,33 @@ std::unique_ptr MobileNetSsdDatabase::GetTestCaseData( return nullptr; } - // Prepare test case expected output - std::vector expectedOutput; - expectedOutput.reserve(1); - expectedOutput.push_back(testCaseInput.second); + std::vector numDetections = { static_cast(testCaseInput.second.size()) }; + + std::vector detectionBoxes; + std::vector detectionClasses; + std::vector detectionScores; + + for (DetectedObject expectedObject : testCaseInput.second) + { + detectionBoxes.push_back(expectedObject.m_BoundingBox.m_YMin); + detectionBoxes.push_back(expectedObject.m_BoundingBox.m_XMin); + detectionBoxes.push_back(expectedObject.m_BoundingBox.m_YMax); + detectionBoxes.push_back(expectedObject.m_BoundingBox.m_XMax); + + detectionClasses.push_back(expectedObject.m_Class); - return std::make_unique(std::move(imageData), std::move(expectedOutput)); + detectionScores.push_back(expectedObject.m_Confidence); + } + + // Prepare test case expected output + std::vector> expectedOutputs; + expectedOutputs.reserve(4); + expectedOutputs.push_back(detectionBoxes); + expectedOutputs.push_back(detectionClasses); + expectedOutputs.push_back(detectionScores); + expectedOutputs.push_back(numDetections); + + return std::make_unique(imageData, testCaseInput.second, expectedOutputs); } } // anonymous namespace diff --git a/tests/MobileNetSsdInferenceTest.hpp b/tests/MobileNetSsdInferenceTest.hpp index 10ee1dcae6..bbbf957dcf 100644 --- a/tests/MobileNetSsdInferenceTest.hpp +++ b/tests/MobileNetSsdInferenceTest.hpp @@ -29,7 +29,7 @@ public: { std::move(testCaseData.m_InputData) }, { k_OutputSize1, k_OutputSize2, k_OutputSize3, k_OutputSize4 }) , m_FloatComparer(boost::math::fpc::percent_tolerance(1.0f)) - , m_DetectedObjects(testCaseData.m_ExpectedOutput) + , m_DetectedObjects(testCaseData.m_ExpectedDetectedObject) {} TestCaseResult ProcessResult(const InferenceTestOptions& options) override @@ -46,10 +46,21 @@ public: const std::vector& output4 = boost::get>(this->GetOutputs()[3]); // valid detections BOOST_ASSERT(output4.size() == k_OutputSize4); + const size_t numDetections = boost::numeric_cast(output4[0]); + + // Check if number of valid detections matches expectations + const size_t expectedNumDetections = m_DetectedObjects.size(); + if (numDetections != expectedNumDetections) + { + BOOST_LOG_TRIVIAL(error) << "Number of detections is incorrect: Expected (" << + expectedNumDetections << ")" << " but got (" << numDetections << ")"; + return TestCaseResult::Failed; + } + // Extract detected objects from output data std::vector detectedObjects; const float* outputData = output1.data(); - for (unsigned int i = 0u; i < k_NumDetections; i++) + for (unsigned int i = 0u; i < numDetections; i++) { // NOTE: Order of coordinates in output data is yMin, xMin, yMax, xMax float yMin = *outputData++; @@ -58,61 +69,49 @@ public: float xMax = *outputData++; DetectedObject detectedObject( - static_cast(output2.at(i)), + output2.at(i), BoundingBox(xMin, yMin, xMax, yMax), output3.at(i)); detectedObjects.push_back(detectedObject); } - // Sort detected objects by confidence - std::sort(detectedObjects.begin(), detectedObjects.end(), - [](const DetectedObject& a, const DetectedObject& b) - { - return a.m_Confidence > b.m_Confidence || - (a.m_Confidence == b.m_Confidence && a.m_Class > b.m_Class); - }); - - // Check if number of valid detections matches expectations - const size_t numValidDetections = boost::numeric_cast(output4[0]); - if (numValidDetections != m_DetectedObjects.size()) - { - BOOST_LOG_TRIVIAL(error) << "Number of valid detections is incorrect: Expected (" << - m_DetectedObjects.size() << ")" << " but got (" << numValidDetections << ")"; - return TestCaseResult::Failed; - } + std::sort(detectedObjects.begin(), detectedObjects.end()); + std::sort(m_DetectedObjects.begin(), m_DetectedObjects.end()); // Compare detected objects with expected results std::vector::const_iterator it = detectedObjects.begin(); - for (const DetectedObject& expectedDetection : m_DetectedObjects) + for (unsigned int i = 0; i < numDetections; i++) { if (it == detectedObjects.end()) { - BOOST_LOG_TRIVIAL(info) << "No more detected objects to compare"; + BOOST_LOG_TRIVIAL(error) << "No more detected objects found! Index out of bounds: " << i; return TestCaseResult::Abort; } const DetectedObject& detectedObject = *it; - if (detectedObject.m_Class != expectedDetection.m_Class) + const DetectedObject& expectedObject = m_DetectedObjects[i]; + + if (detectedObject.m_Class != expectedObject.m_Class) { BOOST_LOG_TRIVIAL(error) << "Prediction for test case " << this->GetTestCaseId() << - " is incorrect: Expected (" << expectedDetection.m_Class << ")" << + " is incorrect: Expected (" << expectedObject.m_Class << ")" << " but predicted (" << detectedObject.m_Class << ")"; return TestCaseResult::Failed; } - if(!m_FloatComparer(detectedObject.m_Confidence, expectedDetection.m_Confidence)) + if(!m_FloatComparer(detectedObject.m_Confidence, expectedObject.m_Confidence)) { BOOST_LOG_TRIVIAL(error) << "Confidence of prediction for test case " << this->GetTestCaseId() << - " is incorrect: Expected (" << expectedDetection.m_Confidence << ") +- 1.0 pc" << + " is incorrect: Expected (" << expectedObject.m_Confidence << ") +- 1.0 pc" << " but predicted (" << detectedObject.m_Confidence << ")"; return TestCaseResult::Failed; } - if (!m_FloatComparer(detectedObject.m_BoundingBox.m_XMin, expectedDetection.m_BoundingBox.m_XMin) || - !m_FloatComparer(detectedObject.m_BoundingBox.m_YMin, expectedDetection.m_BoundingBox.m_YMin) || - !m_FloatComparer(detectedObject.m_BoundingBox.m_XMax, expectedDetection.m_BoundingBox.m_XMax) || - !m_FloatComparer(detectedObject.m_BoundingBox.m_YMax, expectedDetection.m_BoundingBox.m_YMax)) + if (!m_FloatComparer(detectedObject.m_BoundingBox.m_XMin, expectedObject.m_BoundingBox.m_XMin) || + !m_FloatComparer(detectedObject.m_BoundingBox.m_YMin, expectedObject.m_BoundingBox.m_YMin) || + !m_FloatComparer(detectedObject.m_BoundingBox.m_XMax, expectedObject.m_BoundingBox.m_XMax) || + !m_FloatComparer(detectedObject.m_BoundingBox.m_YMax, expectedObject.m_BoundingBox.m_YMax)) { BOOST_LOG_TRIVIAL(error) << "Detected bounding box for test case " << this->GetTestCaseId() << " is incorrect"; @@ -126,11 +125,11 @@ public: } private: - static constexpr unsigned int k_NumDetections = 1u; + static constexpr unsigned int k_Shape = 10u; - static constexpr unsigned int k_OutputSize1 = k_NumDetections * 4u; - static constexpr unsigned int k_OutputSize2 = k_NumDetections; - static constexpr unsigned int k_OutputSize3 = k_NumDetections; + static constexpr unsigned int k_OutputSize1 = k_Shape * 4u; + static constexpr unsigned int k_OutputSize2 = k_Shape; + static constexpr unsigned int k_OutputSize3 = k_Shape; static constexpr unsigned int k_OutputSize4 = 1u; boost::math::fpc::close_at_tolerance m_FloatComparer; @@ -152,7 +151,7 @@ public: options.add_options() ("data-dir,d", po::value(&m_DataDir)->required(), - "Path to directory containing test data"); + "Path to directory containing test data"); Model::AddCommandLineOptions(options, m_ModelCommandLineOptions); } @@ -169,7 +168,7 @@ public: { return false; } - std::pair qParams = m_Model->GetQuantizationParams(); + std::pair qParams = m_Model->GetInputQuantizationParams(); m_Database = std::make_unique(m_DataDir.c_str(), qParams.first, qParams.second); if (!m_Database) { diff --git a/tests/ObjectDetectionCommon.hpp b/tests/ObjectDetectionCommon.hpp index 85b54c255f..ee3afb5788 100644 --- a/tests/ObjectDetectionCommon.hpp +++ b/tests/ObjectDetectionCommon.hpp @@ -31,7 +31,7 @@ struct BoundingBox struct DetectedObject { - DetectedObject(unsigned int detectedClass, + DetectedObject(float detectedClass, const BoundingBox& boundingBox, float confidence) : m_Class(detectedClass) @@ -39,11 +39,17 @@ struct DetectedObject , m_Confidence(confidence) {} - unsigned int m_Class; + bool operator<(const DetectedObject& other) const + { + return m_Confidence < other.m_Confidence || + (m_Confidence == other.m_Confidence && m_Class < other.m_Class); + } + + float m_Class; BoundingBox m_BoundingBox; float m_Confidence; }; -using ObjectDetectionInput = std::pair; +using ObjectDetectionInput = std::pair>; } // anonymous namespace \ No newline at end of file -- cgit v1.2.1