From 74af093826b77022a8b70ec2216a96044e5be7d4 Mon Sep 17 00:00:00 2001 From: Ryan OShea Date: Fri, 7 Aug 2020 16:27:34 +0100 Subject: IVGCVSW-5159 Add Accuracy Check for YoloV3 Big App * Add Check Accuracy Method * Add Ability to pass in comparison file paths * Add compare_detection to yolo v3 class Signed-off-by: Ryan OShea Change-Id: I914ffe4805316263dc19d76a777fff6e35f44961 --- tests/TfLiteYoloV3Big-Armnn/NMS.cpp | 13 +++ tests/TfLiteYoloV3Big-Armnn/NMS.hpp | 9 ++ .../TfLiteYoloV3Big-Armnn.cpp | 121 ++++++++++++++++++++- 3 files changed, 142 insertions(+), 1 deletion(-) diff --git a/tests/TfLiteYoloV3Big-Armnn/NMS.cpp b/tests/TfLiteYoloV3Big-Armnn/NMS.cpp index 3ef840f875..d067f1a004 100644 --- a/tests/TfLiteYoloV3Big-Armnn/NMS.cpp +++ b/tests/TfLiteYoloV3Big-Armnn/NMS.cpp @@ -6,6 +6,7 @@ #include "NMS.hpp" +#include #include #include #include @@ -80,6 +81,18 @@ std::vector convert_to_detections(const NMSConfig& config, } } // namespace +bool compare_detection(const yolov3::Detection& detection, + const std::vector& expected) +{ + float tolerance = 0.001f; + return (std::fabs(detection.classes[0] - expected[0]) < tolerance && + std::fabs(detection.box.xmin - expected[1]) < tolerance && + std::fabs(detection.box.ymin - expected[2]) < tolerance && + std::fabs(detection.box.xmax - expected[3]) < tolerance && + std::fabs(detection.box.ymax - expected[4]) < tolerance && + std::fabs(detection.confidence - expected[5]) < tolerance ); +} + void print_detection(std::ostream& os, const std::vector& detections) { diff --git a/tests/TfLiteYoloV3Big-Armnn/NMS.hpp b/tests/TfLiteYoloV3Big-Armnn/NMS.hpp index fc23994f58..f5e3cf38af 100644 --- a/tests/TfLiteYoloV3Big-Armnn/NMS.hpp +++ b/tests/TfLiteYoloV3Big-Armnn/NMS.hpp @@ -40,6 +40,15 @@ struct Detection { void print_detection(std::ostream& os, const std::vector& detections); +/** Compare a detection object with a vector of float values + * + * @param detection [in] Detection object + * @param expected [in] Vector of expected float values + * @return Boolean to represent if they match or not + */ +bool compare_detection(const yolov3::Detection& detection, + const std::vector& expected); + /** Perform Non-Maxima Supression on a list of given detections * * @param[in] config Configuration metadata for NMS diff --git a/tests/TfLiteYoloV3Big-Armnn/TfLiteYoloV3Big-Armnn.cpp b/tests/TfLiteYoloV3Big-Armnn/TfLiteYoloV3Big-Armnn.cpp index fcc21771cc..93982fdb98 100644 --- a/tests/TfLiteYoloV3Big-Armnn/TfLiteYoloV3Big-Armnn.cpp +++ b/tests/TfLiteYoloV3Big-Armnn/TfLiteYoloV3Big-Armnn.cpp @@ -198,6 +198,90 @@ bool ValidateFilePath(std::string& file) return true; } +void CheckAccuracy(std::vector* toDetector0, std::vector* toDetector1, + std::vector* toDetector2, std::vector* detectorOutput, + const std::vector& nmsOut, const std::vector& filePaths) +{ + std::ifstream pathStream; + std::vector expected; + std::vector*> outputs; + float compare = 0; + unsigned int count = 0; + + //Push back output vectors from inference for use in loop + outputs.push_back(toDetector0); + outputs.push_back(toDetector1); + outputs.push_back(toDetector2); + outputs.push_back(detectorOutput); + + for (unsigned int i = 0; i < outputs.size(); ++i) + { + // Reading expected output files and assigning them to @expected. Close and Clear to reuse stream and clean RAM + pathStream.open(filePaths[i]); + if (!pathStream.is_open()) + { + ARMNN_LOG(error) << "Expected output file can not be opened: " << filePaths[i]; + continue; + } + + expected.assign(std::istream_iterator(pathStream), {}); + pathStream.close(); + pathStream.clear(); + + // Ensure each vector is the same length + if (expected.size() != outputs[i]->size()) + { + ARMNN_LOG(error) << "Expected output size does not match actual output size: " << filePaths[i]; + } + else + { + count = 0; + + // Compare abs(difference) with tolerance to check for value by value equality + for (unsigned int j = 0; j < outputs[i]->size(); ++j) + { + compare = abs(expected[j] - outputs[i]->at(j)); + if (compare > 0.001f) + { + count++; + } + } + if (count > 0) + { + ARMNN_LOG(error) << count << " output(s) do not match expected values in: " << filePaths[i]; + } + } + } + + pathStream.open(filePaths[4]); + if (!pathStream.is_open()) + { + ARMNN_LOG(error) << "Expected output file can not be opened: " << filePaths[4]; + } + else + { + expected.assign(std::istream_iterator(pathStream), {}); + pathStream.close(); + pathStream.clear(); + unsigned int y = 0; + unsigned int numOfMember = 6; + std::vector intermediate; + + for (auto& detection: nmsOut) + { + for (unsigned int x = y * numOfMember; x < ((y * numOfMember) + numOfMember); ++x) + { + intermediate.push_back(expected[x]); + } + if (!yolov3::compare_detection(detection, intermediate)) + { + ARMNN_LOG(error) << "Expected NMS output does not match: Detection " << y + 1; + } + intermediate.clear(); + y++; + } + } +} struct ParseArgs { @@ -213,6 +297,13 @@ struct ParseArgs "can be found e.g. mydir/yoloV3big_backbone.tflite", cxxopts::value()) + ("c,comparison-files", + "Defines the expected outputs for the model " + "of yoloV3big e.g. 'mydir/file1.txt,mydir/file2.txt,mydir/file3.txt,mydir/file4.txt'->InputToDetector1" + " will be tried first then InputToDetector2 then InputToDetector3 then the Detector Output and finally" + " the NMS output. NOTE: Files are passed as comma separated list without whitespaces.", + cxxopts::value>()) + ("d,detector-path", "File path where the TfLite model for the yoloV3big " "detector can be found e.g.'mydir/yoloV3big_detector.tflite'", @@ -248,9 +339,12 @@ struct ParseArgs } backboneDir = GetPathArgument(result, "backbone-path"); + comparisonFiles = GetPathArgument(result["comparison-files"].as>()); detectorDir = GetPathArgument(result, "detector-path"); imageDir = GetPathArgument(result, "image-path"); + + prefBackendsBackbone = GetBackendIDs(result["preferred-backends-backbone"].as>()); LogBackendsInfo(prefBackendsBackbone, "Backbone"); prefBackendsDetector = GetBackendIDs(result["preferred-backends-detector"].as>()); @@ -288,6 +382,25 @@ struct ParseArgs } } + /// Assigns vector of strings to struct member variable + std::vector GetPathArgument(const std::vector& pathStrings) + { + if (pathStrings.size() < 5){ + throw cxxopts::option_syntax_exception("Comparison files requires 5 file paths."); + } + + std::vector filePaths; + for (auto& path : pathStrings) + { + filePaths.push_back(path); + if (!ValidateFilePath(filePaths.back())) + { + throw cxxopts::option_syntax_exception("Argument given to Comparison Files is not a valid file path"); + } + } + return filePaths; + } + /// Log info about assigned backends void LogBackendsInfo(std::vector& backends, std::string&& modelName) { @@ -302,6 +415,7 @@ struct ParseArgs // Member variables std::string backboneDir; + std::vector comparisonFiles; std::string detectorDir; std::string imageDir; @@ -391,6 +505,7 @@ int main(int argc, char* argv[]) static const int numIterations=2; using DurationUS = std::chrono::duration; std::vector nmsDurations(0); + std::vector filtered_boxes; nmsDurations.reserve(numIterations); for (int i=0; i < numIterations; i++) { @@ -411,7 +526,7 @@ int main(int argc, char* argv[]) config.num_classes = 80; config.confidence_threshold = 0.9f; config.iou_threshold = 0.5f; - auto filtered_boxes = yolov3::nms(config, intermediateMem3); + filtered_boxes = yolov3::nms(config, intermediateMem3); auto nmsEndTime = clock::now(); // Enable the profiling after the warm-up run @@ -457,6 +572,10 @@ int main(int argc, char* argv[]) nmsProfileStream << "}" << "\n"; nmsProfileStream.close(); + CheckAccuracy(&intermediateMem0, &intermediateMem1, + &intermediateMem2, &intermediateMem3, + filtered_boxes, progArgs.comparisonFiles); + ARMNN_LOG(info) << "Run completed"; return 0; } \ No newline at end of file -- cgit v1.2.1