diff options
Diffstat (limited to 'tests')
-rw-r--r-- | tests/InferenceModel.hpp | 7 | ||||
-rw-r--r-- | tests/MobileNetSsdDatabase.hpp | 50 | ||||
-rw-r--r-- | tests/MobileNetSsdInferenceTest.hpp | 69 | ||||
-rw-r--r-- | tests/ObjectDetectionCommon.hpp | 12 |
4 files changed, 88 insertions, 50 deletions
diff --git a/tests/InferenceModel.hpp b/tests/InferenceModel.hpp index 6e73f5275f..25ccbee45a 100644 --- a/tests/InferenceModel.hpp +++ b/tests/InferenceModel.hpp @@ -577,6 +577,13 @@ public: m_OutputBindings[outputIndex].second.GetQuantizationOffset()); } + QuantizationParams GetInputQuantizationParams(unsigned int inputIndex = 0u) const + { + CheckInputIndexIsValid(inputIndex); + return std::make_pair(m_InputBindings[inputIndex].second.GetQuantizationScale(), + m_InputBindings[inputIndex].second.GetQuantizationOffset()); + } + std::vector<QuantizationParams> GetAllQuantizationParams() const { std::vector<QuantizationParams> quantizationParams; diff --git a/tests/MobileNetSsdDatabase.hpp b/tests/MobileNetSsdDatabase.hpp index cac558793f..7a30f221d2 100644 --- a/tests/MobileNetSsdDatabase.hpp +++ b/tests/MobileNetSsdDatabase.hpp @@ -27,14 +27,17 @@ namespace struct MobileNetSsdTestCaseData { MobileNetSsdTestCaseData( - std::vector<uint8_t> inputData, - std::vector<DetectedObject> expectedOutput) - : m_InputData(std::move(inputData)) - , m_ExpectedOutput(std::move(expectedOutput)) + const std::vector<uint8_t>& inputData, + const std::vector<DetectedObject>& expectedDetectedObject, + const std::vector<std::vector<float>>& expectedOutput) + : m_InputData(inputData) + , m_ExpectedDetectedObject(expectedDetectedObject) + , m_ExpectedOutput(expectedOutput) {} - std::vector<uint8_t> m_InputData; - std::vector<DetectedObject> m_ExpectedOutput; + std::vector<uint8_t> m_InputData; + std::vector<DetectedObject> m_ExpectedDetectedObject; + std::vector<std::vector<float>> m_ExpectedOutput; }; class MobileNetSsdDatabase @@ -59,7 +62,9 @@ const std::array<ObjectDetectionInput, 1> g_PerTestCaseInput = ObjectDetectionInput { "Cat.jpg", - DetectedObject(16, BoundingBox(0.208961248f, 0.0852333307f, 0.92757535f, 0.940263629f), 0.79296875f) + { + DetectedObject(16.0f, BoundingBox(0.208961248f, 0.0852333307f, 0.92757535f, 0.940263629f), 0.79296875f) + } } }; @@ -100,12 +105,33 @@ std::unique_ptr<MobileNetSsdTestCaseData> MobileNetSsdDatabase::GetTestCaseData( return nullptr; } - // Prepare test case expected output - std::vector<DetectedObject> expectedOutput; - expectedOutput.reserve(1); - expectedOutput.push_back(testCaseInput.second); + std::vector<float> numDetections = { static_cast<float>(testCaseInput.second.size()) }; + + std::vector<float> detectionBoxes; + std::vector<float> detectionClasses; + std::vector<float> detectionScores; + + for (DetectedObject expectedObject : testCaseInput.second) + { + detectionBoxes.push_back(expectedObject.m_BoundingBox.m_YMin); + detectionBoxes.push_back(expectedObject.m_BoundingBox.m_XMin); + detectionBoxes.push_back(expectedObject.m_BoundingBox.m_YMax); + detectionBoxes.push_back(expectedObject.m_BoundingBox.m_XMax); + + detectionClasses.push_back(expectedObject.m_Class); - return std::make_unique<MobileNetSsdTestCaseData>(std::move(imageData), std::move(expectedOutput)); + detectionScores.push_back(expectedObject.m_Confidence); + } + + // Prepare test case expected output + std::vector<std::vector<float>> expectedOutputs; + expectedOutputs.reserve(4); + expectedOutputs.push_back(detectionBoxes); + expectedOutputs.push_back(detectionClasses); + expectedOutputs.push_back(detectionScores); + expectedOutputs.push_back(numDetections); + + return std::make_unique<MobileNetSsdTestCaseData>(imageData, testCaseInput.second, expectedOutputs); } } // anonymous namespace diff --git a/tests/MobileNetSsdInferenceTest.hpp b/tests/MobileNetSsdInferenceTest.hpp index 10ee1dcae6..bbbf957dcf 100644 --- a/tests/MobileNetSsdInferenceTest.hpp +++ b/tests/MobileNetSsdInferenceTest.hpp @@ -29,7 +29,7 @@ public: { std::move(testCaseData.m_InputData) }, { k_OutputSize1, k_OutputSize2, k_OutputSize3, k_OutputSize4 }) , m_FloatComparer(boost::math::fpc::percent_tolerance(1.0f)) - , m_DetectedObjects(testCaseData.m_ExpectedOutput) + , m_DetectedObjects(testCaseData.m_ExpectedDetectedObject) {} TestCaseResult ProcessResult(const InferenceTestOptions& options) override @@ -46,10 +46,21 @@ public: const std::vector<float>& output4 = boost::get<std::vector<float>>(this->GetOutputs()[3]); // valid detections BOOST_ASSERT(output4.size() == k_OutputSize4); + const size_t numDetections = boost::numeric_cast<size_t>(output4[0]); + + // Check if number of valid detections matches expectations + const size_t expectedNumDetections = m_DetectedObjects.size(); + if (numDetections != expectedNumDetections) + { + BOOST_LOG_TRIVIAL(error) << "Number of detections is incorrect: Expected (" << + expectedNumDetections << ")" << " but got (" << numDetections << ")"; + return TestCaseResult::Failed; + } + // Extract detected objects from output data std::vector<DetectedObject> detectedObjects; const float* outputData = output1.data(); - for (unsigned int i = 0u; i < k_NumDetections; i++) + for (unsigned int i = 0u; i < numDetections; i++) { // NOTE: Order of coordinates in output data is yMin, xMin, yMax, xMax float yMin = *outputData++; @@ -58,61 +69,49 @@ public: float xMax = *outputData++; DetectedObject detectedObject( - static_cast<unsigned int>(output2.at(i)), + output2.at(i), BoundingBox(xMin, yMin, xMax, yMax), output3.at(i)); detectedObjects.push_back(detectedObject); } - // Sort detected objects by confidence - std::sort(detectedObjects.begin(), detectedObjects.end(), - [](const DetectedObject& a, const DetectedObject& b) - { - return a.m_Confidence > b.m_Confidence || - (a.m_Confidence == b.m_Confidence && a.m_Class > b.m_Class); - }); - - // Check if number of valid detections matches expectations - const size_t numValidDetections = boost::numeric_cast<size_t>(output4[0]); - if (numValidDetections != m_DetectedObjects.size()) - { - BOOST_LOG_TRIVIAL(error) << "Number of valid detections is incorrect: Expected (" << - m_DetectedObjects.size() << ")" << " but got (" << numValidDetections << ")"; - return TestCaseResult::Failed; - } + std::sort(detectedObjects.begin(), detectedObjects.end()); + std::sort(m_DetectedObjects.begin(), m_DetectedObjects.end()); // Compare detected objects with expected results std::vector<DetectedObject>::const_iterator it = detectedObjects.begin(); - for (const DetectedObject& expectedDetection : m_DetectedObjects) + for (unsigned int i = 0; i < numDetections; i++) { if (it == detectedObjects.end()) { - BOOST_LOG_TRIVIAL(info) << "No more detected objects to compare"; + BOOST_LOG_TRIVIAL(error) << "No more detected objects found! Index out of bounds: " << i; return TestCaseResult::Abort; } const DetectedObject& detectedObject = *it; - if (detectedObject.m_Class != expectedDetection.m_Class) + const DetectedObject& expectedObject = m_DetectedObjects[i]; + + if (detectedObject.m_Class != expectedObject.m_Class) { BOOST_LOG_TRIVIAL(error) << "Prediction for test case " << this->GetTestCaseId() << - " is incorrect: Expected (" << expectedDetection.m_Class << ")" << + " is incorrect: Expected (" << expectedObject.m_Class << ")" << " but predicted (" << detectedObject.m_Class << ")"; return TestCaseResult::Failed; } - if(!m_FloatComparer(detectedObject.m_Confidence, expectedDetection.m_Confidence)) + if(!m_FloatComparer(detectedObject.m_Confidence, expectedObject.m_Confidence)) { BOOST_LOG_TRIVIAL(error) << "Confidence of prediction for test case " << this->GetTestCaseId() << - " is incorrect: Expected (" << expectedDetection.m_Confidence << ") +- 1.0 pc" << + " is incorrect: Expected (" << expectedObject.m_Confidence << ") +- 1.0 pc" << " but predicted (" << detectedObject.m_Confidence << ")"; return TestCaseResult::Failed; } - if (!m_FloatComparer(detectedObject.m_BoundingBox.m_XMin, expectedDetection.m_BoundingBox.m_XMin) || - !m_FloatComparer(detectedObject.m_BoundingBox.m_YMin, expectedDetection.m_BoundingBox.m_YMin) || - !m_FloatComparer(detectedObject.m_BoundingBox.m_XMax, expectedDetection.m_BoundingBox.m_XMax) || - !m_FloatComparer(detectedObject.m_BoundingBox.m_YMax, expectedDetection.m_BoundingBox.m_YMax)) + if (!m_FloatComparer(detectedObject.m_BoundingBox.m_XMin, expectedObject.m_BoundingBox.m_XMin) || + !m_FloatComparer(detectedObject.m_BoundingBox.m_YMin, expectedObject.m_BoundingBox.m_YMin) || + !m_FloatComparer(detectedObject.m_BoundingBox.m_XMax, expectedObject.m_BoundingBox.m_XMax) || + !m_FloatComparer(detectedObject.m_BoundingBox.m_YMax, expectedObject.m_BoundingBox.m_YMax)) { BOOST_LOG_TRIVIAL(error) << "Detected bounding box for test case " << this->GetTestCaseId() << " is incorrect"; @@ -126,11 +125,11 @@ public: } private: - static constexpr unsigned int k_NumDetections = 1u; + static constexpr unsigned int k_Shape = 10u; - static constexpr unsigned int k_OutputSize1 = k_NumDetections * 4u; - static constexpr unsigned int k_OutputSize2 = k_NumDetections; - static constexpr unsigned int k_OutputSize3 = k_NumDetections; + static constexpr unsigned int k_OutputSize1 = k_Shape * 4u; + static constexpr unsigned int k_OutputSize2 = k_Shape; + static constexpr unsigned int k_OutputSize3 = k_Shape; static constexpr unsigned int k_OutputSize4 = 1u; boost::math::fpc::close_at_tolerance<float> m_FloatComparer; @@ -152,7 +151,7 @@ public: options.add_options() ("data-dir,d", po::value<std::string>(&m_DataDir)->required(), - "Path to directory containing test data"); + "Path to directory containing test data"); Model::AddCommandLineOptions(options, m_ModelCommandLineOptions); } @@ -169,7 +168,7 @@ public: { return false; } - std::pair<float, int32_t> qParams = m_Model->GetQuantizationParams(); + std::pair<float, int32_t> qParams = m_Model->GetInputQuantizationParams(); m_Database = std::make_unique<MobileNetSsdDatabase>(m_DataDir.c_str(), qParams.first, qParams.second); if (!m_Database) { diff --git a/tests/ObjectDetectionCommon.hpp b/tests/ObjectDetectionCommon.hpp index 85b54c255f..ee3afb5788 100644 --- a/tests/ObjectDetectionCommon.hpp +++ b/tests/ObjectDetectionCommon.hpp @@ -31,7 +31,7 @@ struct BoundingBox struct DetectedObject { - DetectedObject(unsigned int detectedClass, + DetectedObject(float detectedClass, const BoundingBox& boundingBox, float confidence) : m_Class(detectedClass) @@ -39,11 +39,17 @@ struct DetectedObject , m_Confidence(confidence) {} - unsigned int m_Class; + bool operator<(const DetectedObject& other) const + { + return m_Confidence < other.m_Confidence || + (m_Confidence == other.m_Confidence && m_Class < other.m_Class); + } + + float m_Class; BoundingBox m_BoundingBox; float m_Confidence; }; -using ObjectDetectionInput = std::pair<std::string, DetectedObject>; +using ObjectDetectionInput = std::pair<std::string, std::vector<DetectedObject>>; } // anonymous namespace
\ No newline at end of file |