From ed35a6fea4a1604db81c56fc71f7756822fcf212 Mon Sep 17 00:00:00 2001 From: Richard Burton Date: Mon, 14 Feb 2022 11:55:35 +0000 Subject: MLECO-2874: Move NMS out of the OD use_case * Add ImageUtils * Move image related code from UseCaseCommonUtils to ImageUtils * Move NMS related code to ImageUtils * Delete test specific ImageUtils and use new ImageUtils Signed-off-by: Richard Burton Change-Id: Icbf5dd9c6a941b0126ecdf69a0c9d9969f22729f --- source/application/main/ImageUtils.cc | 126 +++++++++++++++++++++ source/application/main/UseCaseCommonUtils.cc | 25 +--- source/application/main/include/ImageUtils.hpp | 116 +++++++++++++++++++ .../main/include/UseCaseCommonUtils.hpp | 32 +----- source/math/PlatformMath.cc | 7 +- source/math/include/PlatformMath.hpp | 14 ++- source/use_case/ad/src/UseCaseHandler.cc | 3 +- source/use_case/asr/src/UseCaseHandler.cc | 3 +- source/use_case/img_class/src/UseCaseHandler.cc | 5 +- source/use_case/kws/src/UseCaseHandler.cc | 3 +- source/use_case/kws_asr/src/UseCaseHandler.cc | 3 +- .../use_case/noise_reduction/src/UseCaseHandler.cc | 3 +- .../include/DetectorPostProcessing.hpp | 68 +---------- .../object_detection/src/DetectorPostProcessing.cc | 102 ++--------------- source/use_case/vww/src/UseCaseHandler.cc | 3 +- .../use_case/img_class/InferenceTestMobilenetV2.cc | 4 +- .../object_detection/InferenceTestYoloFastest.cc | 4 +- .../vww/InferenceVisualWakeWordModelTests.cc | 4 +- tests/utils/ImageUtils.cc | 37 ------ tests/utils/ImageUtils.hpp | 26 ----- 20 files changed, 298 insertions(+), 290 deletions(-) create mode 100644 source/application/main/ImageUtils.cc create mode 100644 source/application/main/include/ImageUtils.hpp delete mode 100644 tests/utils/ImageUtils.cc delete mode 100644 tests/utils/ImageUtils.hpp diff --git a/source/application/main/ImageUtils.cc b/source/application/main/ImageUtils.cc new file mode 100644 index 0000000..31b9493 --- /dev/null +++ b/source/application/main/ImageUtils.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "ImageUtils.hpp" + +#include + +namespace arm { +namespace app { +namespace image { + + float Calculate1DOverlap(float x1Center, float width1, float x2Center, float width2) + { + float left_1 = x1Center - width1/2; + float left_2 = x2Center - width2/2; + float leftest = left_1 > left_2 ? left_1 : left_2; + + float right_1 = x1Center + width1/2; + float right_2 = x2Center + width2/2; + float rightest = right_1 < right_2 ? right_1 : right_2; + + return rightest - leftest; + } + + float CalculateBoxIntersect(Box& box1, Box& box2) + { + float width = Calculate1DOverlap(box1.x, box1.w, box2.x, box2.w); + if (width < 0) { + return 0; + } + float height = Calculate1DOverlap(box1.y, box1.h, box2.y, box2.h); + if (height < 0) { + return 0; + } + + float total_area = width*height; + return total_area; + } + + float CalculateBoxUnion(Box& box1, Box& box2) + { + float boxes_intersection = CalculateBoxIntersect(box1, box2); + float boxes_union = box1.w * box1.h + box2.w * box2.h - boxes_intersection; + return boxes_union; + } + + float CalculateBoxIOU(Box& box1, Box& box2) + { + float boxes_intersection = CalculateBoxIntersect(box1, box2); + if (boxes_intersection == 0) { + return 0; + } + + float boxes_union = CalculateBoxUnion(box1, box2); + if (boxes_union == 0) { + return 0; + } + + return boxes_intersection / boxes_union; + } + + void CalculateNMS(std::forward_list& detections, int classes, float iouThreshold) + { + int idxClass{0}; + auto CompareProbs = [idxClass](Detection& prob1, Detection& prob2) { + return prob1.prob[idxClass] > prob2.prob[idxClass]; + }; + + for (idxClass = 0; idxClass < classes; ++idxClass) { + detections.sort(CompareProbs); + + for (auto it=detections.begin(); it != detections.end(); ++it) { + if (it->prob[idxClass] == 0) continue; + for (auto itc=std::next(it, 1); itc != detections.end(); ++itc) { + if (itc->prob[idxClass] == 0) { + continue; + } + if (CalculateBoxIOU(it->bbox, itc->bbox) > iouThreshold) { + itc->prob[idxClass] = 0; + } + } + } + } + } + + void ConvertImgToInt8(void* data, const size_t kMaxImageSize) + { + auto* tmp_req_data = static_cast(data); + auto* tmp_signed_req_data = static_cast(data); + + for (size_t i = 0; i < kMaxImageSize; i++) { + tmp_signed_req_data[i] = (int8_t) ( + (int32_t) (tmp_req_data[i]) - 128); + } + } + + void RgbToGrayscale(const uint8_t* srcPtr, uint8_t* dstPtr, const size_t dstImgSz) + { + const float R = 0.299; + const float G = 0.587; + const float B = 0.114; + for (size_t i = 0; i < dstImgSz; ++i, srcPtr += 3) { + uint32_t int_gray = R * (*srcPtr) + + G * (*(srcPtr + 1)) + + B * (*(srcPtr + 2)); + *dstPtr++ = int_gray <= std::numeric_limits::max() ? + int_gray : std::numeric_limits::max(); + } + } + +} /* namespace image */ +} /* namespace app */ +} /* namespace arm */ \ No newline at end of file diff --git a/source/application/main/UseCaseCommonUtils.cc b/source/application/main/UseCaseCommonUtils.cc index dd9a32d..d439446 100644 --- a/source/application/main/UseCaseCommonUtils.cc +++ b/source/application/main/UseCaseCommonUtils.cc @@ -15,6 +15,7 @@ * limitations under the License. */ #include "UseCaseCommonUtils.hpp" +#include "ImageUtils.hpp" #include "InputFiles.hpp" #include "log_macros.h" @@ -35,19 +36,10 @@ void DisplayCommonMenu() fflush(stdout); } -void image::ConvertImgToInt8(void* data, const size_t kMaxImageSize) -{ - auto* tmp_req_data = static_cast(data); - auto* tmp_signed_req_data = static_cast(data); - for (size_t i = 0; i < kMaxImageSize; i++) { - tmp_signed_req_data[i] = (int8_t) ( - (int32_t) (tmp_req_data[i]) - 128); - } -} -bool image::PresentInferenceResult( +bool PresentInferenceResult( hal_platform &platform, const std::vector &results) { @@ -93,19 +85,6 @@ bool image::PresentInferenceResult( return true; } -void image::RgbToGrayscale(const uint8_t *srcPtr, uint8_t *dstPtr, const size_t dstImgSz) -{ - float R=0.299; - float G=0.587; - float B=0.114; - for (size_t i = 0; i < dstImgSz; ++i, srcPtr += 3) { - uint32_t int_gray = R * (*srcPtr) + - G * (*(srcPtr + 1)) + - B * (*(srcPtr + 2)); - *dstPtr++ = int_gray <= std::numeric_limits::max() ? - int_gray : std::numeric_limits::max(); - } -} void IncrementAppCtxIfmIdx(arm::app::ApplicationContext& ctx, const std::string& useCase) { diff --git a/source/application/main/include/ImageUtils.hpp b/source/application/main/include/ImageUtils.hpp new file mode 100644 index 0000000..a8c7650 --- /dev/null +++ b/source/application/main/include/ImageUtils.hpp @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2022 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef IMAGE_UTILS_HPP +#define IMAGE_UTILS_HPP + +#include +#include +#include +#include + +/* Helper macro to convert RGB888 to RGB565 format. */ +#define RGB888_TO_RGB565(R8,G8,B8) ((((R8>>3) & 0x1F) << 11) | \ + (((G8>>2) & 0x3F) << 5) | \ + ((B8>>3) & 0x1F)) + +constexpr uint16_t COLOR_BLACK = 0; +constexpr uint16_t COLOR_GREEN = RGB888_TO_RGB565( 0, 255, 0); // 2016; +constexpr uint16_t COLOR_YELLOW = RGB888_TO_RGB565(255, 255, 0); // 65504; + + +namespace arm { +namespace app { +namespace image { + + /** + * Contains the x,y co-ordinates of a box centre along with the box width and height. + */ + struct Box { + float x; + float y; + float w; + float h; + }; + + struct Detection { + Box bbox; + std::vector prob; + float objectness; + }; + + /** + * @brief Calculate the 1D overlap. + * @param[in] x1Center First center point. + * @param[in] width1 First width. + * @param[in] x2Center Second center point. + * @param[in] width2 Second width. + * @return The overlap between the two lines. + **/ + float Calculate1DOverlap(float x1Center, float width1, float x2Center, float width2); + + /** + * @brief Calculate the intersection between the two given boxes. + * @param[in] box1 First box. + * @param[in] box2 Second box. + * @return The intersection value. + **/ + float CalculateBoxIntersect(Box& box1, Box& box2); + + /** + * @brief Calculate the union between the two given boxes. + * @param[in] box1 First box. + * @param[in] box2 Second box. + * @return The two given boxes union value. + **/ + float CalculateBoxUnion(Box& box1, Box& box2); + + /** + * @brief Calculate the intersection over union between the two given boxes. + * @param[in] box1 First box. + * @param[in] box2 Second box. + * @return The intersection over union value. + **/ + float CalculateBoxIOU(Box& box1, Box& box2); + + /** + * @brief Calculate the Non-Maxima suppression on the given detection boxes. + * @param[in] detections List of Detection boxes. + * @param[in] classes Number of classes. + * @param[in] iouThreshold Intersection over union threshold. + **/ + void CalculateNMS(std::forward_list& detections, int classes, float iouThreshold); + + /** + * @brief Helper function to convert a UINT8 image to INT8 format. + * @param[in,out] data Pointer to the data start. + * @param[in] kMaxImageSize Total number of pixels in the image. + **/ + void ConvertImgToInt8(void* data, size_t kMaxImageSize); + + /** + * @brief Converts RGB image to grayscale. + * @param[in] srcPtr Pointer to RGB source image. + * @param[out] dstPtr Pointer to grayscale destination image. + * @param[in] imgSz Destination image size. + **/ + void RgbToGrayscale(const uint8_t* srcPtr, uint8_t* dstPtr, size_t dstImgSz); + +} /* namespace image */ +} /* namespace app */ +} /* namespace arm */ + +#endif /* IMAGE_UTILS_HPP */ \ No newline at end of file diff --git a/source/application/main/include/UseCaseCommonUtils.hpp b/source/application/main/include/UseCaseCommonUtils.hpp index cd0cb69..7f5dde6 100644 --- a/source/application/main/include/UseCaseCommonUtils.hpp +++ b/source/application/main/include/UseCaseCommonUtils.hpp @@ -24,30 +24,10 @@ #include "UseCaseHandler.hpp" /* Handlers for different user options. */ #include "Classifier.hpp" /* Classifier. */ #include "InputFiles.hpp" -#include - - -/* Helper macro to convert RGB888 to RGB565 format. */ -#define RGB888_TO_RGB565(R8,G8,B8) ((((R8>>3) & 0x1F) << 11) | \ - (((G8>>2) & 0x3F) << 5) | \ - ((B8>>3) & 0x1F)) - -constexpr uint16_t COLOR_BLACK = 0; -constexpr uint16_t COLOR_GREEN = RGB888_TO_RGB565( 0u, 255u, 0u); // 2016; -constexpr uint16_t COLOR_YELLOW = RGB888_TO_RGB565(255u, 255u, 0u); // 65504; void DisplayCommonMenu(); -namespace image{ - - /** - * @brief Helper function to convert a UINT8 image to INT8 format. - * @param[in,out] data Pointer to the data start. - * @param[in] kMaxImageSize Total number of pixels in the image. - **/ - void ConvertImgToInt8(void * data, size_t kMaxImageSize); - /** * @brief Presents inference results using the data presentation * object. @@ -55,17 +35,9 @@ namespace image{ * @param[in] results Vector of classification results to be displayed. * @return true if successful, false otherwise. **/ - bool PresentInferenceResult(hal_platform & platform, - const std::vector < arm::app::ClassificationResult > & results); +bool PresentInferenceResult(hal_platform& platform, + const std::vector& results); - /** - * @brief Converts RGB image to grayscale. - * @param[in] srcPtr Pointer to RGB source image. - * @param[out] dstPtr Pointer to grayscale destination image. - * @param[in] imgSz Destination image size. - **/ - void RgbToGrayscale(const uint8_t *srcPtr, uint8_t *dstPtr, const size_t dstImgSz); -} /** * @brief Helper function to increment current input feature vector index. diff --git a/source/math/PlatformMath.cc b/source/math/PlatformMath.cc index cc603f3..b666d29 100644 --- a/source/math/PlatformMath.cc +++ b/source/math/PlatformMath.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -303,6 +303,11 @@ namespace math { } } + float MathUtils::SigmoidF32(float x) + { + return 1.f/(1.f + std::exp(-x)); + } + } /* namespace math */ } /* namespace app */ } /* namespace arm */ diff --git a/source/math/include/PlatformMath.hpp b/source/math/include/PlatformMath.hpp index 5ac10de..2bf7733 100644 --- a/source/math/include/PlatformMath.hpp +++ b/source/math/include/PlatformMath.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,6 @@ /* See if ARM DSP functions can be used. */ #if defined(ARM_MATH_DSP) - #include "arm_math.h" #define M_PI (PI) #else @@ -127,8 +126,8 @@ namespace math { * @param[out] output Pre-allocated buffer to be populated with * natural log values of each input element. */ - static void VecLogarithmF32(std::vector & input, - std::vector & output); + static void VecLogarithmF32(std::vector& input, + std::vector& output); /** * @brief Computes the dot product of two 1D floating point @@ -165,6 +164,13 @@ namespace math { * @param[in] vector Vector of floats modified in-place */ static void SoftmaxF32(std::vector& vec); + + /** + * @brief Calculate the Sigmoid function of the given value. + * @param[in] x Value to apply Sigmoid to. + * @return Sigmoid value of the input. + */ + static float SigmoidF32(float x); }; } /* namespace math */ diff --git a/source/use_case/ad/src/UseCaseHandler.cc b/source/use_case/ad/src/UseCaseHandler.cc index 420e6d4..853ab08 100644 --- a/source/use_case/ad/src/UseCaseHandler.cc +++ b/source/use_case/ad/src/UseCaseHandler.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,6 +22,7 @@ #include "hal.h" #include "AdMelSpectrogram.hpp" #include "AudioUtils.hpp" +#include "ImageUtils.hpp" #include "UseCaseCommonUtils.hpp" #include "AdPostProcessing.hpp" #include "log_macros.h" diff --git a/source/use_case/asr/src/UseCaseHandler.cc b/source/use_case/asr/src/UseCaseHandler.cc index afcb6e4..7bce2c6 100644 --- a/source/use_case/asr/src/UseCaseHandler.cc +++ b/source/use_case/asr/src/UseCaseHandler.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,6 +22,7 @@ #include "hal.h" #include "Wav2LetterMfcc.hpp" #include "AudioUtils.hpp" +#include "ImageUtils.hpp" #include "UseCaseCommonUtils.hpp" #include "AsrResult.hpp" #include "Wav2LetterPreprocess.hpp" diff --git a/source/use_case/img_class/src/UseCaseHandler.cc b/source/use_case/img_class/src/UseCaseHandler.cc index fafc6b9..1f1d78b 100644 --- a/source/use_case/img_class/src/UseCaseHandler.cc +++ b/source/use_case/img_class/src/UseCaseHandler.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ #include "Classifier.hpp" #include "InputFiles.hpp" #include "MobileNetModel.hpp" +#include "ImageUtils.hpp" #include "UseCaseCommonUtils.hpp" #include "hal.h" #include "log_macros.h" @@ -136,7 +137,7 @@ namespace app { arm::app::DumpTensor(outputTensor); #endif /* VERIFY_TEST_OUTPUT */ - if (!image::PresentInferenceResult(platform, results)) { + if (!PresentInferenceResult(platform, results)) { return false; } diff --git a/source/use_case/kws/src/UseCaseHandler.cc b/source/use_case/kws/src/UseCaseHandler.cc index c2d2ea4..8dd7724 100644 --- a/source/use_case/kws/src/UseCaseHandler.cc +++ b/source/use_case/kws/src/UseCaseHandler.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,6 +22,7 @@ #include "hal.h" #include "MicroNetKwsMfcc.hpp" #include "AudioUtils.hpp" +#include "ImageUtils.hpp" #include "UseCaseCommonUtils.hpp" #include "KwsResult.hpp" #include "log_macros.h" diff --git a/source/use_case/kws_asr/src/UseCaseHandler.cc b/source/use_case/kws_asr/src/UseCaseHandler.cc index bfc1d25..d598de6 100644 --- a/source/use_case/kws_asr/src/UseCaseHandler.cc +++ b/source/use_case/kws_asr/src/UseCaseHandler.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ #include "hal.h" #include "InputFiles.hpp" #include "AudioUtils.hpp" +#include "ImageUtils.hpp" #include "UseCaseCommonUtils.hpp" #include "MicroNetKwsModel.hpp" #include "MicroNetKwsMfcc.hpp" diff --git a/source/use_case/noise_reduction/src/UseCaseHandler.cc b/source/use_case/noise_reduction/src/UseCaseHandler.cc index 0c5984c..792b460 100644 --- a/source/use_case/noise_reduction/src/UseCaseHandler.cc +++ b/source/use_case/noise_reduction/src/UseCaseHandler.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,7 @@ #include "UseCaseHandler.hpp" #include "UseCaseCommonUtils.hpp" #include "AudioUtils.hpp" +#include "ImageUtils.hpp" #include "InputFiles.hpp" #include "RNNoiseModel.hpp" #include "RNNoiseProcess.hpp" diff --git a/source/use_case/object_detection/include/DetectorPostProcessing.hpp b/source/use_case/object_detection/include/DetectorPostProcessing.hpp index 3e9c819..5393f89 100644 --- a/source/use_case/object_detection/include/DetectorPostProcessing.hpp +++ b/source/use_case/object_detection/include/DetectorPostProcessing.hpp @@ -18,6 +18,7 @@ #define DETECTOR_POST_PROCESSING_HPP #include "UseCaseCommonUtils.hpp" +#include "ImageUtils.hpp" #include "DetectionResult.hpp" #include "YoloFastestModel.hpp" @@ -45,20 +46,6 @@ namespace object_detection { int topN; }; - - struct Box { - float x; - float y; - float w; - float h; - }; - - struct Detection { - Box bbox; - std::vector prob; - float objectness; - }; - /** * @brief Helper class to manage tensor post-processing for "object_detection" * output. @@ -98,19 +85,12 @@ namespace object_detection { int m_numClasses; /* Number of classes */ int m_topN; /* TopN */ - /** - * @brief Calculate the Sigmoid function of the give value. - * @param[in] x Value. - * @return Sigmoid value of the input. - **/ - float Sigmoid(float x); - /** * @brief Insert the given Detection in the list. * @param[in] detections List of detections. * @param[in] det Detection to be inserted. **/ - void InsertTopNDetections(std::forward_list& detections, Detection& det); + void InsertTopNDetections(std::forward_list& detections, image::Detection& det); /** * @brief Given a Network calculate the detection boxes. @@ -124,49 +104,7 @@ namespace object_detection { int imageWidth, int imageHeight, float threshold, - std::forward_list& detections); - - /** - * @brief Calculate the 1D overlap. - * @param[in] x1Center First center point. - * @param[in] width1 First width. - * @param[in] x2Center Second center point. - * @param[in] width2 Second width. - * @return The overlap between the two lines. - **/ - float Calculate1DOverlap(float x1Center, float width1, float x2Center, float width2); - - /** - * @brief Calculate the intersection between the two given boxes. - * @param[in] box1 First box. - * @param[in] box2 Second box. - * @return The intersection value. - **/ - float CalculateBoxIntersect(Box& box1, Box& box2); - - /** - * @brief Calculate the union between the two given boxes. - * @param[in] box1 First box. - * @param[in] box2 Second box. - * @return The two given boxes union value. - **/ - float CalculateBoxUnion(Box& box1, Box& box2); - /** - * @brief Calculate the intersection over union between the two given boxes. - * @param[in] box1 First box. - * @param[in] box2 Second box. - * @return The intersection over union value. - **/ - float CalculateBoxIOU(Box& box1, Box& box2); - - /** - * @brief Calculate the Non-Maxima suppression on the given detection boxes. - * @param[in] detections Detection boxes. - * @param[in] classes Number of classes. - * @param[in] iouThreshold Intersection over union threshold. - * @return true or false based on execution success. - **/ - void CalculateNMS(std::forward_list& detections, int classes, float iouThreshold); + std::forward_list& detections); /** * @brief Draw on the given image a bounding box starting at (boxX, boxY). diff --git a/source/use_case/object_detection/src/DetectorPostProcessing.cc b/source/use_case/object_detection/src/DetectorPostProcessing.cc index edfb137..e97e6b3 100644 --- a/source/use_case/object_detection/src/DetectorPostProcessing.cc +++ b/source/use_case/object_detection/src/DetectorPostProcessing.cc @@ -15,6 +15,7 @@ * limitations under the License. */ #include "DetectorPostProcessing.hpp" +#include "PlatformMath.hpp" #include #include @@ -75,7 +76,7 @@ void DetectorPostprocessing::RunPostProcessing( int originalImageWidth = originalImageSize; int originalImageHeight = originalImageSize; - std::forward_list detections; + std::forward_list detections; GetNetworkBoxes(net, originalImageWidth, originalImageHeight, m_threshold, detections); /* Do nms */ @@ -124,15 +125,11 @@ void DetectorPostprocessing::RunPostProcessing( } } -float DetectorPostprocessing::Sigmoid(float x) -{ - return 1.f/(1.f + exp(-x)); -} -void DetectorPostprocessing::InsertTopNDetections(std::forward_list& detections, Detection& det) +void DetectorPostprocessing::InsertTopNDetections(std::forward_list& detections, image::Detection& det) { - std::forward_list::iterator it; - std::forward_list::iterator last_it; + std::forward_list::iterator it; + std::forward_list::iterator last_it; for ( it = detections.begin(); it != detections.end(); ++it ) { if(it->objectness > det.objectness) break; @@ -144,11 +141,11 @@ void DetectorPostprocessing::InsertTopNDetections(std::forward_list& } } -void DetectorPostprocessing::GetNetworkBoxes(Network& net, int imageWidth, int imageHeight, float threshold, std::forward_list& detections) +void DetectorPostprocessing::GetNetworkBoxes(Network& net, int imageWidth, int imageHeight, float threshold, std::forward_list& detections) { int numClasses = net.numClasses; int num = 0; - auto det_objectness_comparator = [](Detection& pa, Detection& pb) { + auto det_objectness_comparator = [](image::Detection& pa, image::Detection& pb) { return pa.objectness < pb.objectness; }; for (size_t i = 0; i < net.branches.size(); ++i) { @@ -162,10 +159,10 @@ void DetectorPostprocessing::GetNetworkBoxes(Network& net, int imageWidth, int i /* Objectness score */ int bbox_obj_offset = h * width * channel + w * channel + anc * (numClasses + 5) + 4; - float objectness = Sigmoid(((float)net.branches[i].modelOutput[bbox_obj_offset] - net.branches[i].zeroPoint) * net.branches[i].scale); + float objectness = math::MathUtils::SigmoidF32(((float)net.branches[i].modelOutput[bbox_obj_offset] - net.branches[i].zeroPoint) * net.branches[i].scale); if(objectness > threshold) { - Detection det; + image::Detection det; det.objectness = objectness; /* Get bbox prediction data for each anchor, each feature point */ int bbox_x_offset = bbox_obj_offset -4; @@ -183,8 +180,8 @@ void DetectorPostprocessing::GetNetworkBoxes(Network& net, int imageWidth, int i float bbox_x, bbox_y; /* Eliminate grid sensitivity trick involved in YOLOv4 */ - bbox_x = Sigmoid(det.bbox.x); - bbox_y = Sigmoid(det.bbox.y); + bbox_x = math::MathUtils::SigmoidF32(det.bbox.x); + bbox_y = math::MathUtils::SigmoidF32(det.bbox.y); det.bbox.x = (bbox_x + w) / width; det.bbox.y = (bbox_y + h) / height; @@ -192,7 +189,7 @@ void DetectorPostprocessing::GetNetworkBoxes(Network& net, int imageWidth, int i det.bbox.h = exp(det.bbox.h) * net.branches[i].anchor[anc*2+1] / net.inputHeight; for (int s = 0; s < numClasses; s++) { - float sig = Sigmoid(((float)net.branches[i].modelOutput[bbox_scores_offset + s] - net.branches[i].zeroPoint) * net.branches[i].scale)*objectness; + float sig = math::MathUtils::SigmoidF32(((float)net.branches[i].modelOutput[bbox_scores_offset + s] - net.branches[i].zeroPoint) * net.branches[i].scale)*objectness; det.prob.emplace_back((sig > threshold) ? sig : 0); } @@ -221,81 +218,6 @@ void DetectorPostprocessing::GetNetworkBoxes(Network& net, int imageWidth, int i num -=1; } -float DetectorPostprocessing::Calculate1DOverlap(float x1Center, float width1, float x2Center, float width2) -{ - float left_1 = x1Center - width1/2; - float left_2 = x2Center - width2/2; - float leftest = left_1 > left_2 ? left_1 : left_2; - - float right_1 = x1Center + width1/2; - float right_2 = x2Center + width2/2; - float rightest = right_1 < right_2 ? right_1 : right_2; - - return rightest - leftest; -} - -float DetectorPostprocessing::CalculateBoxIntersect(Box& box1, Box& box2) -{ - float width = Calculate1DOverlap(box1.x, box1.w, box2.x, box2.w); - if (width < 0) { - return 0; - } - float height = Calculate1DOverlap(box1.y, box1.h, box2.y, box2.h); - if (height < 0) { - return 0; - } - - float total_area = width*height; - return total_area; -} - -float DetectorPostprocessing::CalculateBoxUnion(Box& box1, Box& box2) -{ - float boxes_intersection = CalculateBoxIntersect(box1, box2); - float boxes_union = box1.w * box1.h + box2.w * box2.h - boxes_intersection; - return boxes_union; -} - - -float DetectorPostprocessing::CalculateBoxIOU(Box& box1, Box& box2) -{ - float boxes_intersection = CalculateBoxIntersect(box1, box2); - if (boxes_intersection == 0) { - return 0; - } - - float boxes_union = CalculateBoxUnion(box1, box2); - if (boxes_union == 0) { - return 0; - } - - return boxes_intersection / boxes_union; -} - -void DetectorPostprocessing::CalculateNMS(std::forward_list& detections, int classes, float iouThreshold) -{ - int idxClass{0}; - auto CompareProbs = [idxClass](Detection& prob1, Detection& prob2) { - return prob1.prob[idxClass] > prob2.prob[idxClass]; - }; - - for (idxClass = 0; idxClass < classes; ++idxClass) { - detections.sort(CompareProbs); - - for (std::forward_list::iterator it=detections.begin(); it != detections.end(); ++it) { - if (it->prob[idxClass] == 0) continue; - for (std::forward_list::iterator itc=std::next(it, 1); itc != detections.end(); ++itc) { - if (itc->prob[idxClass] == 0) { - continue; - } - if (CalculateBoxIOU(it->bbox, itc->bbox) > iouThreshold) { - itc->prob[idxClass] = 0; - } - } - } - } -} - void DetectorPostprocessing::DrawBoxOnImage(uint8_t* imgIn, int imWidth, int imHeight, int boxX,int boxY, int boxWidth, int boxHeight) { auto CheckAndFixOffset = [](int im_width,int im_height,int& offset) { diff --git a/source/use_case/vww/src/UseCaseHandler.cc b/source/use_case/vww/src/UseCaseHandler.cc index 01011e2..a47f191 100644 --- a/source/use_case/vww/src/UseCaseHandler.cc +++ b/source/use_case/vww/src/UseCaseHandler.cc @@ -18,6 +18,7 @@ #include "VisualWakeWordModel.hpp" #include "Classifier.hpp" #include "InputFiles.hpp" +#include "ImageUtils.hpp" #include "UseCaseCommonUtils.hpp" #include "hal.h" #include "log_macros.h" @@ -144,7 +145,7 @@ namespace app { arm::app::DumpTensor(outputTensor); #endif /* VERIFY_TEST_OUTPUT */ - if (!image::PresentInferenceResult(platform, results)) { + if (!PresentInferenceResult(platform, results)) { return false; } diff --git a/tests/use_case/img_class/InferenceTestMobilenetV2.cc b/tests/use_case/img_class/InferenceTestMobilenetV2.cc index 294215f..7e7508b 100644 --- a/tests/use_case/img_class/InferenceTestMobilenetV2.cc +++ b/tests/use_case/img_class/InferenceTestMobilenetV2.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,7 +34,7 @@ bool RunInference(arm::app::Model& model, const int8_t imageData[]) memcpy(inputTensor->data.data, imageData, copySz); if(model.IsDataSigned()){ - convertImgIoInt8(inputTensor->data.data, copySz); + arm::app::image::ConvertImgToInt8(inputTensor->data.data, copySz); } return model.RunInference(); diff --git a/tests/use_case/object_detection/InferenceTestYoloFastest.cc b/tests/use_case/object_detection/InferenceTestYoloFastest.cc index d9cabbd..b3bd408 100644 --- a/tests/use_case/object_detection/InferenceTestYoloFastest.cc +++ b/tests/use_case/object_detection/InferenceTestYoloFastest.cc @@ -66,10 +66,10 @@ bool RunInference(arm::app::Model& model, const uint8_t imageData[]) const size_t copySz = inputTensor->bytes < IMAGE_DATA_SIZE ? inputTensor->bytes : IMAGE_DATA_SIZE; - image::RgbToGrayscale(imageData,inputTensor->data.uint8,copySz); + arm::app::image::RgbToGrayscale(imageData,inputTensor->data.uint8,copySz); if(model.IsDataSigned()){ - convertImgIoInt8(inputTensor->data.data, copySz); + arm::app::image::ConvertImgToInt8(inputTensor->data.data, copySz); } return model.RunInference(); diff --git a/tests/use_case/vww/InferenceVisualWakeWordModelTests.cc b/tests/use_case/vww/InferenceVisualWakeWordModelTests.cc index 04dce0d..194099f 100644 --- a/tests/use_case/vww/InferenceVisualWakeWordModelTests.cc +++ b/tests/use_case/vww/InferenceVisualWakeWordModelTests.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,7 +34,7 @@ bool RunInference(arm::app::Model& model, const int8_t* imageData) memcpy(inputTensor->data.data, imageData, copySz); if(model.IsDataSigned()){ - convertImgIoInt8(inputTensor->data.data, copySz); + arm::app::image::ConvertImgToInt8(inputTensor->data.data, copySz); } return model.RunInference(); diff --git a/tests/utils/ImageUtils.cc b/tests/utils/ImageUtils.cc deleted file mode 100644 index 506040f..0000000 --- a/tests/utils/ImageUtils.cc +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2021 Arm Limited. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "ImageUtils.hpp" - -void convertImgIoInt8(void * data, const size_t sz) -{ - uint8_t * tmp_req_data = static_cast(data); - int8_t * tmp_signed_req_data = static_cast(data); - - for (size_t i = 0; i < sz; ++i) { - tmp_signed_req_data[i] = static_cast( - static_cast(tmp_req_data[i]) - 128); - } -} - -void convertImgIoGreyscale(const uint8_t * srcPtr, uint8_t * dstPtr, const size_t sz) -{ - for (size_t i = 0; i < sz; ++i, srcPtr += 3) { - *dstPtr++ = 0.2989 * (*srcPtr) + - 0.587 * (*(srcPtr+1)) + - 0.114 * (*(srcPtr+2)); - } -} \ No newline at end of file diff --git a/tests/utils/ImageUtils.hpp b/tests/utils/ImageUtils.hpp deleted file mode 100644 index 838dcef..0000000 --- a/tests/utils/ImageUtils.hpp +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2021 Arm Limited. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef IMAGEUTILS_HPP -#define IMAGEUTILS_HPP - -#include - -void convertImgIoInt8(void * data, const size_t sz); - -void convertImgIoGreyscale(const uint8_t * srcPtr, uint8_t * dstPtr, const size_t sz); - -#endif /* IMAGEUTILS_HPP */ -- cgit v1.2.1