diff options
author | Michael Levit <michaell@emza-vs.com> | 2022-01-12 11:53:46 +0200 |
---|---|---|
committer | Isabella Gottardi <isabella.gottardi@arm.com> | 2022-01-27 11:36:38 +0000 |
commit | 06fcf7502b5c9b0e30ac06acfa25aa914d936500 (patch) | |
tree | f6033d9cd94e351d213b7d4fb74afda147f93b75 /source | |
parent | e958850d7672cd81a5eb48285a2df94b8f0595f0 (diff) | |
download | ml-embedded-evaluation-kit-06fcf7502b5c9b0e30ac06acfa25aa914d936500.tar.gz |
Face detection demo from Emza Visual Sense
Signed-off-by: Michael Levit michaell@emza-vs.com
Change-Id: I7958b05b5dbe9a785e0f8a241b716c17a9ca976f
Diffstat (limited to 'source')
11 files changed, 1185 insertions, 0 deletions
diff --git a/source/use_case/object_detection/include/DetectionResult.hpp b/source/use_case/object_detection/include/DetectionResult.hpp new file mode 100644 index 0000000..78895f7 --- /dev/null +++ b/source/use_case/object_detection/include/DetectionResult.hpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DETECTION_RESULT_HPP +#define DETECTION_RESULT_HPP + + +namespace arm { +namespace app { + + /** + * @brief Class representing a single detection result. + */ + class DetectionResult { + public: + double m_normalisedVal{0.0}; + int m_x0{0}; + int m_y0{0}; + int m_w{0}; + int m_h{0}; + + DetectionResult() = default; + ~DetectionResult() = default; + + DetectionResult(double normalisedVal,int x0,int y0, int w,int h) : + m_normalisedVal(normalisedVal), + m_x0(x0), + m_y0(y0), + m_w(w), + m_h(h) + { + } + }; + +} /* namespace app */ +} /* namespace arm */ + +#endif /* DETECTION_RESULT_HPP */ diff --git a/source/use_case/object_detection/include/DetectionUseCaseUtils.hpp b/source/use_case/object_detection/include/DetectionUseCaseUtils.hpp new file mode 100644 index 0000000..8ef48ac --- /dev/null +++ b/source/use_case/object_detection/include/DetectionUseCaseUtils.hpp @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DETECTION_USE_CASE_UTILS_HPP +#define DETECTION_USE_CASE_UTILS_HPP + +#include "hal.h" +#include "DetectionResult.hpp" +#include "UseCaseHandler.hpp" /* Handlers for different user options. */ +#include <inttypes.h> +#include <vector> + + +void DisplayDetectionMenu(); + +namespace image{ + + + /** + * @brief Presents inference results using the data presentation + * object. + * @param[in] platform Reference to the hal platform object. + * @param[in] results Vector of detection results to be displayed. + * @return true if successful, false otherwise. + **/ + bool PresentInferenceResult(hal_platform & platform, + const std::vector < arm::app::DetectionResult > & results); + + + /** + * @brief Presents inference results along with the inference time using the data presentation + * object. + * @param[in] platform Reference to the hal platform object. + * @param[in] results Vector of detection results to be displayed. + * @param[in] infTimeMs Inference time in ms. + * @return true if successful, false otherwise. + **/ + bool PresentInferenceResult(hal_platform & platform, + const std::vector < arm::app::DetectionResult > & results, + const time_t infTimeMs); + + /** + * @brief Presents inference results along with the inference time using the data presentation + * object. + * @param[in] platform Reference to the hal platform object. + * @param[in] results Vector of detection results to be displayed. + * @param[in] infTimeMs Inference time in ms. + * @return true if successful, false otherwise. + **/ + bool PresentInferenceResult(hal_platform & platform, + const std::vector < arm::app::DetectionResult > & results, + bool profilingEnabled, + const time_t infTimeMs = 0); + } + + + + +#endif /* DETECTION_USE_CASE_UTILS_HPP */ diff --git a/source/use_case/object_detection/include/DetectorPostProcessing.hpp b/source/use_case/object_detection/include/DetectorPostProcessing.hpp new file mode 100644 index 0000000..9a8549c --- /dev/null +++ b/source/use_case/object_detection/include/DetectorPostProcessing.hpp @@ -0,0 +1,55 @@ +/*
+ * Copyright (c) 2022 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef DETECTOR_POST_PROCESSING_HPP
+#define DETECTOR_POST_PROCESSING_HPP
+
+#include "UseCaseCommonUtils.hpp"
+#include "DetectionResult.hpp"
+
+namespace arm {
+namespace app {
+
+#if DISPLAY_RGB_IMAGE
+#define FORMAT_MULTIPLY_FACTOR 3
+#else
+#define FORMAT_MULTIPLY_FACTOR 1
+#endif /* DISPLAY_RGB_IMAGE */
+
+ /**
+ * @brief Post processing part of Yolo object detection CNN
+ * @param[in] img_in Pointer to the input image,detection bounding boxes drown on it.
+ * @param[in] model_output Output tesnsors after CNN invoked
+ * @param[out] results_out Vector of detected results.
+ * @return void
+ **/
+void RunPostProcessing(uint8_t *img_in,TfLiteTensor* model_output[2],std::vector<arm::app::DetectionResult> & results_out);
+
+
+ /**
+ * @brief Converts RGB image to grayscale
+ * @param[in] rgb Pointer to RGB input image
+ * @param[out] gray Pointer to RGB out image
+ * @param[in] im_w Input image width
+ * @param[in] im_h Input image height
+ * @return void
+ **/
+void RgbToGrayscale(const uint8_t *rgb,uint8_t *gray, int im_w,int im_h);
+
+} /* namespace app */
+} /* namespace arm */
+
+#endif /* DETECTOR_POST_PROCESSING_HPP */
diff --git a/source/use_case/object_detection/include/UseCaseHandler.hpp b/source/use_case/object_detection/include/UseCaseHandler.hpp new file mode 100644 index 0000000..56629c8 --- /dev/null +++ b/source/use_case/object_detection/include/UseCaseHandler.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef OBJ_DET_HANDLER_HPP +#define OBJ_DET_HANDLER_HPP + +#include "AppContext.hpp" + +namespace arm { +namespace app { + + /** + * @brief Handles the inference event. + * @param[in] ctx Pointer to the application context. + * @param[in] imgIndex Index to the image to run object detection. + * @param[in] runAll Flag to request classification of all the available images. + * @return true or false based on execution success. + **/ + bool ObjectDetectionHandler(ApplicationContext& ctx, uint32_t imgIndex, bool runAll); + +} /* namespace app */ +} /* namespace arm */ + +#endif /* OBJ_DET_HANDLER_HPP */ diff --git a/source/use_case/object_detection/include/YoloFastestModel.hpp b/source/use_case/object_detection/include/YoloFastestModel.hpp new file mode 100644 index 0000000..f5709ea --- /dev/null +++ b/source/use_case/object_detection/include/YoloFastestModel.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef YOLO_FASTEST_MODEL_HPP +#define YOLO_FASTEST_MODEL_HPP + +#include "Model.hpp" + +namespace arm { +namespace app { + + class YoloFastestModel : public Model { + + public: + /* Indices for the expected model - based on input tensor shape */ + static constexpr uint32_t ms_inputRowsIdx = 1; + static constexpr uint32_t ms_inputColsIdx = 2; + static constexpr uint32_t ms_inputChannelsIdx = 3; + + protected: + /** @brief Gets the reference to op resolver interface class. */ + const tflite::MicroOpResolver& GetOpResolver() override; + + /** @brief Adds operations to the op resolver instance. */ + bool EnlistOperations() override; + + const uint8_t* ModelPointer() override; + + size_t ModelSize() override; + + private: + /* Maximum number of individual operations that can be enlisted. */ + static constexpr int ms_maxOpCnt = 8; + + /* A mutable op resolver instance. */ + tflite::MicroMutableOpResolver<ms_maxOpCnt> m_opResolver; + }; + +} /* namespace app */ +} /* namespace arm */ + +#endif /* YOLO_FASTEST_MODEL_HPP */ diff --git a/source/use_case/object_detection/src/DetectionUseCaseUtils.cc b/source/use_case/object_detection/src/DetectionUseCaseUtils.cc new file mode 100644 index 0000000..1713c7e --- /dev/null +++ b/source/use_case/object_detection/src/DetectionUseCaseUtils.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2022 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "DetectionUseCaseUtils.hpp" +#include "UseCaseCommonUtils.hpp" +#include "InputFiles.hpp" +#include <inttypes.h> + + +void DisplayDetectionMenu() +{ + printf("\n\n"); + printf("User input required\n"); + printf("Enter option number from:\n\n"); + printf(" %u. Run detection on next ifm\n", common::MENU_OPT_RUN_INF_NEXT); + printf(" %u. Run detection ifm at chosen index\n", common::MENU_OPT_RUN_INF_CHOSEN); + printf(" %u. Run detection on all ifm\n", common::MENU_OPT_RUN_INF_ALL); + printf(" %u. Show NN model info\n", common::MENU_OPT_SHOW_MODEL_INFO); + printf(" %u. List ifm\n\n", common::MENU_OPT_LIST_IFM); + printf(" Choice: "); + fflush(stdout); +} + + +bool image::PresentInferenceResult(hal_platform& platform, + const std::vector<arm::app::DetectionResult>& results) +{ + return PresentInferenceResult(platform, results, false); +} + +bool image::PresentInferenceResult(hal_platform &platform, + const std::vector<arm::app::DetectionResult> &results, + const time_t infTimeMs) +{ + return PresentInferenceResult(platform, results, true, infTimeMs); +} + + +bool image::PresentInferenceResult(hal_platform &platform, + const std::vector<arm::app::DetectionResult> &results, + bool profilingEnabled, + const time_t infTimeMs) +{ + constexpr uint32_t dataPsnTxtStartX1 = 150; + constexpr uint32_t dataPsnTxtStartY1 = 30; + + + if(profilingEnabled) + { + platform.data_psn->set_text_color(COLOR_YELLOW); + + /* If profiling is enabled, and the time is valid. */ + info("Final results:\n"); + info("Total number of inferences: 1\n"); + if (infTimeMs) + { + std::string strInf = + std::string{"Inference: "} + + std::to_string(infTimeMs) + + std::string{"ms"}; + platform.data_psn->present_data_text( + strInf.c_str(), strInf.size(), + dataPsnTxtStartX1, dataPsnTxtStartY1, 0); + } + } + platform.data_psn->set_text_color(COLOR_GREEN); + + if(!profilingEnabled) { + info("Final results:\n"); + info("Total number of inferences: 1\n"); + } + + for (uint32_t i = 0; i < results.size(); ++i) { + + if(profilingEnabled) { + info("%" PRIu32 ") (%f) -> %s {x=%d,y=%d,w=%d,h=%d}\n", i, + results[i].m_normalisedVal, "Detection box:", + results[i].m_x0, results[i].m_y0, results[i].m_w, results[i].m_h ); + } + else + { + info("%" PRIu32 ") (%f) -> %s {x=%d,y=%d,w=%d,h=%d}\n", i, + results[i].m_normalisedVal, "Detection box:", + results[i].m_x0, results[i].m_y0, results[i].m_w, results[i].m_h ); + } + } + + return true; +} + + + diff --git a/source/use_case/object_detection/src/DetectorPostProcessing.cc b/source/use_case/object_detection/src/DetectorPostProcessing.cc new file mode 100755 index 0000000..e781b62 --- /dev/null +++ b/source/use_case/object_detection/src/DetectorPostProcessing.cc @@ -0,0 +1,447 @@ +/*
+ * Copyright (c) 2022 Arm Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "DetectorPostProcessing.hpp"
+#include <algorithm>
+#include <cmath>
+#include <stdint.h>
+#include <forward_list>
+
+
+typedef struct boxabs {
+ float left, right, top, bot;
+} boxabs;
+
+
+typedef struct branch {
+ int resolution;
+ int num_box;
+ float *anchor;
+ int8_t *tf_output;
+ float scale;
+ int zero_point;
+ size_t size;
+ float scale_x_y;
+} branch;
+
+typedef struct network {
+ int input_w;
+ int input_h;
+ int num_classes;
+ int num_branch;
+ branch *branchs;
+ int topN;
+} network;
+
+
+typedef struct box {
+ float x, y, w, h;
+} box;
+
+typedef struct detection{
+ box bbox;
+ float *prob;
+ float objectness;
+} detection;
+
+
+
+static int sort_class;
+
+static void free_dets(std::forward_list<detection> &dets){
+ std::forward_list<detection>::iterator it;
+ for ( it = dets.begin(); it != dets.end(); ++it ){
+ free(it->prob);
+ }
+}
+
+float sigmoid(float x)
+{
+ return 1.f/(1.f + exp(-x));
+}
+
+static bool det_objectness_comparator(detection &pa, detection &pb)
+{
+ return pa.objectness < pb.objectness;
+}
+
+static void insert_topN_det(std::forward_list<detection> &dets, detection det)
+{
+ std::forward_list<detection>::iterator it;
+ std::forward_list<detection>::iterator last_it;
+ for ( it = dets.begin(); it != dets.end(); ++it ){
+ if(it->objectness > det.objectness)
+ break;
+ last_it = it;
+ }
+ if(it != dets.begin()){
+ dets.emplace_after(last_it, det);
+ free(dets.begin()->prob);
+ dets.pop_front();
+ }
+ else{
+ free(det.prob);
+ }
+}
+
+static std::forward_list<detection> get_network_boxes(network *net, int image_w, int image_h, float thresh, int *num)
+{
+ std::forward_list<detection> dets;
+ int i;
+ int num_classes = net->num_classes;
+ *num = 0;
+
+ for (i = 0; i < net->num_branch; ++i) {
+ int height = net->branchs[i].resolution;
+ int width = net->branchs[i].resolution;
+ int channel = net->branchs[i].num_box*(5+num_classes);
+
+ for (int h = 0; h < net->branchs[i].resolution; h++) {
+ for (int w = 0; w < net->branchs[i].resolution; w++) {
+ for (int anc = 0; anc < net->branchs[i].num_box; anc++) {
+
+ // objectness score
+ int bbox_obj_offset = h * width * channel + w * channel + anc * (num_classes + 5) + 4;
+ float objectness = sigmoid(((float)net->branchs[i].tf_output[bbox_obj_offset] - net->branchs[i].zero_point) * net->branchs[i].scale);
+
+ if(objectness > thresh){
+ detection det;
+ det.prob = (float*)calloc(num_classes, sizeof(float));
+ det.objectness = objectness;
+ //get bbox prediction data for each anchor, each feature point
+ int bbox_x_offset = bbox_obj_offset -4;
+ int bbox_y_offset = bbox_x_offset + 1;
+ int bbox_w_offset = bbox_x_offset + 2;
+ int bbox_h_offset = bbox_x_offset + 3;
+ int bbox_scores_offset = bbox_x_offset + 5;
+ //int bbox_scores_step = 1;
+ det.bbox.x = ((float)net->branchs[i].tf_output[bbox_x_offset] - net->branchs[i].zero_point) * net->branchs[i].scale;
+ det.bbox.y = ((float)net->branchs[i].tf_output[bbox_y_offset] - net->branchs[i].zero_point) * net->branchs[i].scale;
+ det.bbox.w = ((float)net->branchs[i].tf_output[bbox_w_offset] - net->branchs[i].zero_point) * net->branchs[i].scale;
+ det.bbox.h = ((float)net->branchs[i].tf_output[bbox_h_offset] - net->branchs[i].zero_point) * net->branchs[i].scale;
+
+
+ float bbox_x, bbox_y;
+
+ // Eliminate grid sensitivity trick involved in YOLOv4
+ bbox_x = sigmoid(det.bbox.x); //* net->branchs[i].scale_x_y - (net->branchs[i].scale_x_y - 1) / 2;
+ bbox_y = sigmoid(det.bbox.y); //* net->branchs[i].scale_x_y - (net->branchs[i].scale_x_y - 1) / 2;
+ det.bbox.x = (bbox_x + w) / width;
+ det.bbox.y = (bbox_y + h) / height;
+
+ det.bbox.w = exp(det.bbox.w) * net->branchs[i].anchor[anc*2] / net->input_w;
+ det.bbox.h = exp(det.bbox.h) * net->branchs[i].anchor[anc*2+1] / net->input_h;
+
+ for (int s = 0; s < num_classes; s++) {
+ det.prob[s] = sigmoid(((float)net->branchs[i].tf_output[bbox_scores_offset + s] - net->branchs[i].zero_point) * net->branchs[i].scale)*objectness;
+ det.prob[s] = (det.prob[s] > thresh) ? det.prob[s] : 0;
+ }
+
+ //correct_yolo_boxes
+ det.bbox.x *= image_w;
+ det.bbox.w *= image_w;
+ det.bbox.y *= image_h;
+ det.bbox.h *= image_h;
+
+ if (*num < net->topN || net->topN <=0){
+ dets.emplace_front(det);
+ *num += 1;
+ }
+ else if(*num == net->topN){
+ dets.sort(det_objectness_comparator);
+ insert_topN_det(dets,det);
+ *num += 1;
+ }else{
+ insert_topN_det(dets,det);
+ }
+ }
+ }
+ }
+ }
+ }
+ if(*num > net->topN)
+ *num -=1;
+ return dets;
+}
+
+// init part
+
+static branch create_brach(int resolution, int num_box, float *anchor, int8_t *tf_output, size_t size, float scale, int zero_point)
+{
+ branch b;
+ b.resolution = resolution;
+ b.num_box = num_box;
+ b.anchor = anchor;
+ b.tf_output = tf_output;
+ b.size = size;
+ b.scale = scale;
+ b.zero_point = zero_point;
+ return b;
+}
+
+static network creat_network(int input_w, int input_h, int num_classes, int num_branch, branch* branchs, int topN)
+{
+ network net;
+ net.input_w = input_w;
+ net.input_h = input_h;
+ net.num_classes = num_classes;
+ net.num_branch = num_branch;
+ net.branchs = branchs;
+ net.topN = topN;
+ return net;
+}
+
+// NMS part
+
+static float Calc1DOverlap(float x1_center, float width1, float x2_center, float width2)
+{
+ float left_1 = x1_center - width1/2;
+ float left_2 = x2_center - width2/2;
+ float leftest;
+ if (left_1 > left_2) {
+ leftest = left_1;
+ } else {
+ leftest = left_2;
+ }
+
+ float right_1 = x1_center + width1/2;
+ float right_2 = x2_center + width2/2;
+ float rightest;
+ if (right_1 < right_2) {
+ rightest = right_1;
+ } else {
+ rightest = right_2;
+ }
+
+ return rightest - leftest;
+}
+
+
+static float CalcBoxIntersect(box box1, box box2)
+{
+ float width = Calc1DOverlap(box1.x, box1.w, box2.x, box2.w);
+ if (width < 0) return 0;
+ float height = Calc1DOverlap(box1.y, box1.h, box2.y, box2.h);
+ if (height < 0) return 0;
+
+ float total_area = width*height;
+ return total_area;
+}
+
+
+static float CalcBoxUnion(box box1, box box2)
+{
+ float boxes_intersection = CalcBoxIntersect(box1, box2);
+ float boxes_union = box1.w*box1.h + box2.w*box2.h - boxes_intersection;
+ return boxes_union;
+}
+
+
+static float CalcBoxIOU(box box1, box box2)
+{
+ float boxes_intersection = CalcBoxIntersect(box1, box2);
+
+ if (boxes_intersection == 0) return 0;
+
+ float boxes_union = CalcBoxUnion(box1, box2);
+
+ if (boxes_union == 0) return 0;
+
+ return boxes_intersection / boxes_union;
+}
+
+
+static bool CompareProbs(detection &prob1, detection &prob2)
+{
+ return prob1.prob[sort_class] > prob2.prob[sort_class];
+}
+
+
+static void CalcNMS(std::forward_list<detection> &detections, int classes, float iou_threshold)
+{
+ int k;
+
+ for (k = 0; k < classes; ++k) {
+ sort_class = k;
+ detections.sort(CompareProbs);
+
+ for (std::forward_list<detection>::iterator it=detections.begin(); it != detections.end(); ++it){
+ if (it->prob[k] == 0) continue;
+ for (std::forward_list<detection>::iterator itc=std::next(it, 1); itc != detections.end(); ++itc){
+ if (itc->prob[k] == 0) continue;
+ if (CalcBoxIOU(it->bbox, itc->bbox) > iou_threshold) {
+ itc->prob[k] = 0;
+ }
+ }
+ }
+ }
+}
+
+
+static void inline check_and_fix_offset(int im_w,int im_h,int *offset)
+{
+
+ if (!offset) return;
+
+ if ( (*offset) >= im_w*im_h*FORMAT_MULTIPLY_FACTOR)
+ (*offset) = im_w*im_h*FORMAT_MULTIPLY_FACTOR -1;
+ else if ( (*offset) < 0)
+ *offset =0;
+
+}
+
+
+static void DrawBoxOnImage(uint8_t *img_in,int im_w,int im_h,int bx,int by,int bw,int bh)
+{
+
+ if (!img_in) {
+ return;
+ }
+
+ int offset=0;
+ for (int i=0; i < bw; i++) {
+ /*draw two lines */
+ for (int line=0; line < 2; line++) {
+ /*top*/
+ offset =(i + (by + line)*im_w + bx)*FORMAT_MULTIPLY_FACTOR;
+ check_and_fix_offset(im_w,im_h,&offset);
+ img_in[offset] = 0xFF; /* FORMAT_MULTIPLY_FACTOR for rgb or grayscale*/
+ /*bottom*/
+ offset = (i + (by + bh - line)*im_w + bx)*FORMAT_MULTIPLY_FACTOR;
+ check_and_fix_offset(im_w,im_h,&offset);
+ img_in[offset] = 0xFF;
+ }
+ }
+
+ for (int i=0; i < bh; i++) {
+ /*draw two lines */
+ for (int line=0; line < 2; line++) {
+ /*left*/
+ offset = ((i + by)*im_w + bx + line)*FORMAT_MULTIPLY_FACTOR;
+ check_and_fix_offset(im_w,im_h,&offset);
+ img_in[offset] = 0xFF;
+ /*right*/
+ offset = ((i + by)*im_w + bx + bw - line)*FORMAT_MULTIPLY_FACTOR;
+ check_and_fix_offset(im_w,im_h,&offset);
+ img_in[offset] = 0xFF;
+ }
+ }
+
+}
+
+
+void arm::app::RunPostProcessing(uint8_t *img_in,TfLiteTensor* model_output[2],std::vector<arm::app::DetectionResult> & results_out)
+{
+
+ TfLiteTensor* output[2] = {nullptr,nullptr};
+ int input_w = INPUT_IMAGE_WIDTH;
+ int input_h = INPUT_IMAGE_HEIGHT;
+
+ for(int anchor=0;anchor<2;anchor++)
+ {
+ output[anchor] = model_output[anchor];
+ }
+
+ /* init postprocessing */
+ int num_classes = 1;
+ int num_branch = 2;
+ int topN = 0;
+
+ branch* branchs = (branch*)calloc(num_branch, sizeof(branch));
+
+ /*NOTE: anchors are different for any given input model size, estimated during training phase */
+ float anchor1[] = {38, 77, 47, 97, 61, 126};
+ float anchor2[] = {14, 26, 19, 37, 28, 55 };
+
+
+ branchs[0] = create_brach(INPUT_IMAGE_WIDTH/32, 3, anchor1, output[0]->data.int8, output[0]->bytes, ((TfLiteAffineQuantization*)(output[0]->quantization.params))->scale->data[0], ((TfLiteAffineQuantization*)(output[0]->quantization.params))->zero_point->data[0]);
+
+ branchs[1] = create_brach(INPUT_IMAGE_WIDTH/16, 3, anchor2, output[1]->data.int8, output[1]->bytes, ((TfLiteAffineQuantization*)(output[1]->quantization.params))->scale->data[0],((TfLiteAffineQuantization*)(output[1]->quantization.params))->zero_point->data[0]);
+
+ network net = creat_network(input_w, input_h, num_classes, num_branch, branchs,topN);
+ /* end init */
+
+ /* start postprocessing */
+ int nboxes=0;
+ float thresh = .5;//50%
+ float nms = .45;
+ int orig_image_width = ORIGINAL_IMAGE_WIDTH;
+ int orig_image_height = ORIGINAL_IMAGE_HEIGHT;
+ std::forward_list<detection> dets = get_network_boxes(&net, orig_image_width, orig_image_height, thresh, &nboxes);
+ /* do nms */
+ CalcNMS(dets, net.num_classes, nms);
+ uint8_t temp_unsuppressed_counter = 0;
+ int j;
+ for (std::forward_list<detection>::iterator it=dets.begin(); it != dets.end(); ++it){
+ float xmin = it->bbox.x - it->bbox.w / 2.0f;
+ float xmax = it->bbox.x + it->bbox.w / 2.0f;
+ float ymin = it->bbox.y - it->bbox.h / 2.0f;
+ float ymax = it->bbox.y + it->bbox.h / 2.0f;
+
+ if (xmin < 0) xmin = 0;
+ if (ymin < 0) ymin = 0;
+ if (xmax > orig_image_width) xmax = orig_image_width;
+ if (ymax > orig_image_height) ymax = orig_image_height;
+
+ float bx = xmin;
+ float by = ymin;
+ float bw = xmax - xmin;
+ float bh = ymax - ymin;
+
+ for (j = 0; j < net.num_classes; ++j) {
+ if (it->prob[j] > 0) {
+
+ arm::app::DetectionResult tmp_result = {};
+
+ tmp_result.m_normalisedVal = it->prob[j];
+ tmp_result.m_x0=bx;
+ tmp_result.m_y0=by;
+ tmp_result.m_w=bw;
+ tmp_result.m_h=bh;
+
+ results_out.push_back(tmp_result);
+
+ DrawBoxOnImage(img_in,orig_image_width,orig_image_height,bx,by,bw,bh);
+
+ temp_unsuppressed_counter++;
+ }
+ }
+ }
+
+ free_dets(dets);
+ free(branchs);
+
+}
+
+void arm::app::RgbToGrayscale(const uint8_t *rgb,uint8_t *gray, int im_w,int im_h)
+{
+ float R=0.299;
+ float G=0.587;
+ float B=0.114;
+ for (int i=0; i< im_w*im_h; i++ ) {
+
+ uint32_t int_gray = rgb[i*3 + 0]*R + rgb[i*3 + 1]*G+ rgb[i*3 + 2]*B;
+ /*clip if need */
+ if (int_gray <= UINT8_MAX) {
+ gray[i] = int_gray;
+ } else {
+ gray[i] = UINT8_MAX;
+ }
+
+ }
+
+}
+
diff --git a/source/use_case/object_detection/src/MainLoop.cc b/source/use_case/object_detection/src/MainLoop.cc new file mode 100644 index 0000000..b0fbf96 --- /dev/null +++ b/source/use_case/object_detection/src/MainLoop.cc @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "hal.h" /* Brings in platform definitions. */ +#include "InputFiles.hpp" /* For input images. */ +#include "YoloFastestModel.hpp" /* Model class for running inference. */ +#include "UseCaseHandler.hpp" /* Handlers for different user options. */ +#include "UseCaseCommonUtils.hpp" /* Utils functions. */ +#include "DetectionUseCaseUtils.hpp" /* Utils functions specific to object detection. */ + + +void main_loop(hal_platform& platform) +{ + arm::app::YoloFastestModel model; /* Model wrapper object. */ + + /* Load the model. */ + if (!model.Init()) { + printf_err("Failed to initialise model\n"); + return; + } + + /* Instantiate application context. */ + arm::app::ApplicationContext caseContext; + + arm::app::Profiler profiler{&platform, "object_detection"}; + caseContext.Set<arm::app::Profiler&>("profiler", profiler); + caseContext.Set<hal_platform&>("platform", platform); + caseContext.Set<arm::app::Model&>("model", model); + caseContext.Set<uint32_t>("imgIndex", 0); + + + /* Loop. */ + bool executionSuccessful = true; + constexpr bool bUseMenu = NUMBER_OF_FILES > 1 ? true : false; + + /* Loop. */ + do { + int menuOption = common::MENU_OPT_RUN_INF_NEXT; + if (bUseMenu) { + DisplayDetectionMenu(); + menuOption = arm::app::ReadUserInputAsInt(platform); + printf("\n"); + } + switch (menuOption) { + case common::MENU_OPT_RUN_INF_NEXT: + executionSuccessful = ObjectDetectionHandler(caseContext, caseContext.Get<uint32_t>("imgIndex"), false); + break; + case common::MENU_OPT_RUN_INF_CHOSEN: { + printf(" Enter the image index [0, %d]: ", NUMBER_OF_FILES-1); + fflush(stdout); + auto imgIndex = static_cast<uint32_t>(arm::app::ReadUserInputAsInt(platform)); + executionSuccessful = ObjectDetectionHandler(caseContext, imgIndex, false); + break; + } + case common::MENU_OPT_RUN_INF_ALL: + executionSuccessful = ObjectDetectionHandler(caseContext, caseContext.Get<uint32_t>("imgIndex"), true); + break; + case common::MENU_OPT_SHOW_MODEL_INFO: + executionSuccessful = model.ShowModelInfoHandler(); + break; + case common::MENU_OPT_LIST_IFM: + executionSuccessful = ListFilesHandler(caseContext); + break; + default: + printf("Incorrect choice, try again."); + break; + } + } while (executionSuccessful && bUseMenu); + info("Main loop terminated.\n"); +} diff --git a/source/use_case/object_detection/src/UseCaseHandler.cc b/source/use_case/object_detection/src/UseCaseHandler.cc new file mode 100644 index 0000000..45df4f8 --- /dev/null +++ b/source/use_case/object_detection/src/UseCaseHandler.cc @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2022 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "UseCaseHandler.hpp" +#include "InputFiles.hpp" +#include "YoloFastestModel.hpp" +#include "UseCaseCommonUtils.hpp" +#include "DetectionUseCaseUtils.hpp" +#include "DetectorPostProcessing.hpp" +#include "hal.h" + +#include <inttypes.h> + + +/* used for presentation, original images are read-only"*/ +static uint8_t g_image_buffer[INPUT_IMAGE_WIDTH*INPUT_IMAGE_HEIGHT*FORMAT_MULTIPLY_FACTOR] IFM_BUF_ATTRIBUTE = {}; + +namespace arm { +namespace app { + + + /* Object detection classification handler. */ + bool ObjectDetectionHandler(ApplicationContext& ctx, uint32_t imgIndex, bool runAll) + { + auto& platform = ctx.Get<hal_platform&>("platform"); + auto& profiler = ctx.Get<Profiler&>("profiler"); + + constexpr uint32_t dataPsnImgDownscaleFactor = 1; + constexpr uint32_t dataPsnImgStartX = 10; + constexpr uint32_t dataPsnImgStartY = 35; + + constexpr uint32_t dataPsnTxtInfStartX = 150; + constexpr uint32_t dataPsnTxtInfStartY = 40; + + platform.data_psn->clear(COLOR_BLACK); + + auto& model = ctx.Get<Model&>("model"); + + /* If the request has a valid size, set the image index. */ + if (imgIndex < NUMBER_OF_FILES) { + if (!SetAppCtxIfmIdx(ctx, imgIndex, "imgIndex")) { + return false; + } + } + if (!model.IsInited()) { + printf_err("Model is not initialised! Terminating processing.\n"); + return false; + } + + auto curImIdx = ctx.Get<uint32_t>("imgIndex"); + + TfLiteTensor* inputTensor = model.GetInputTensor(0); + + if (!inputTensor->dims) { + printf_err("Invalid input tensor dims\n"); + return false; + } else if (inputTensor->dims->size < 3) { + printf_err("Input tensor dimension should be >= 3\n"); + return false; + } + + TfLiteIntArray* inputShape = model.GetInputShape(0); + + const uint32_t nCols = inputShape->data[arm::app::YoloFastestModel::ms_inputColsIdx]; + const uint32_t nRows = inputShape->data[arm::app::YoloFastestModel::ms_inputRowsIdx]; + const uint32_t nPresentationChannels = FORMAT_MULTIPLY_FACTOR; + + std::vector<DetectionResult> results; + + do { + /* Strings for presentation/logging. */ + std::string str_inf{"Running inference... "}; + + const uint8_t* curr_image = get_img_array(ctx.Get<uint32_t>("imgIndex")); + + /* Copy over the data and convert to gryscale */ +#if DISPLAY_RGB_IMAGE + memcpy(g_image_buffer,curr_image, INPUT_IMAGE_WIDTH*INPUT_IMAGE_HEIGHT*FORMAT_MULTIPLY_FACTOR); +#else + RgbToGrayscale(curr_image,g_image_buffer,INPUT_IMAGE_WIDTH,INPUT_IMAGE_HEIGHT); +#endif /*DISPLAY_RGB_IMAGE*/ + + RgbToGrayscale(curr_image,inputTensor->data.uint8,INPUT_IMAGE_WIDTH,INPUT_IMAGE_HEIGHT); + + + /* Display this image on the LCD. */ + platform.data_psn->present_data_image( + g_image_buffer, + nCols, nRows, nPresentationChannels, + dataPsnImgStartX, dataPsnImgStartY, dataPsnImgDownscaleFactor); + + /* If the data is signed. */ + if (model.IsDataSigned()) { + image::ConvertImgToInt8(inputTensor->data.data, inputTensor->bytes); + } + + /* Display message on the LCD - inference running. */ + platform.data_psn->present_data_text(str_inf.c_str(), str_inf.size(), + dataPsnTxtInfStartX, dataPsnTxtInfStartY, 0); + + /* Run inference over this image. */ + info("Running inference on image %" PRIu32 " => %s\n", ctx.Get<uint32_t>("imgIndex"), + get_filename(ctx.Get<uint32_t>("imgIndex"))); + + if (!RunInference(model, profiler)) { + return false; + } + + /* Erase. */ + str_inf = std::string(str_inf.size(), ' '); + platform.data_psn->present_data_text(str_inf.c_str(), str_inf.size(), + dataPsnTxtInfStartX, dataPsnTxtInfStartY, 0); + + /* Detector post-processing*/ + TfLiteTensor* output_arr[2] = {nullptr,nullptr}; + output_arr[0] = model.GetOutputTensor(0); + output_arr[1] = model.GetOutputTensor(1); + RunPostProcessing(g_image_buffer,output_arr,results); + + platform.data_psn->present_data_image( + g_image_buffer, + nCols, nRows, nPresentationChannels, + dataPsnImgStartX, dataPsnImgStartY, dataPsnImgDownscaleFactor); + + /*Detector post-processing*/ + + + /* Add results to context for access outside handler. */ + ctx.Set<std::vector<DetectionResult>>("results", results); + +#if VERIFY_TEST_OUTPUT + arm::app::DumpTensor(outputTensor); +#endif /* VERIFY_TEST_OUTPUT */ + + if (!image::PresentInferenceResult(platform, results)) { + return false; + } + + profiler.PrintProfilingResult(); + + IncrementAppCtxIfmIdx(ctx,"imgIndex"); + + } while (runAll && ctx.Get<uint32_t>("imgIndex") != curImIdx); + + return true; + } + +} /* namespace app */ +} /* namespace arm */ diff --git a/source/use_case/object_detection/src/YoloFastestModel.cc b/source/use_case/object_detection/src/YoloFastestModel.cc new file mode 100644 index 0000000..a8afd59 --- /dev/null +++ b/source/use_case/object_detection/src/YoloFastestModel.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2022 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "YoloFastestModel.hpp" + +#include "hal.h" + +const tflite::MicroOpResolver& arm::app::YoloFastestModel::GetOpResolver() +{ + return this->m_opResolver; +} + +bool arm::app::YoloFastestModel::EnlistOperations() +{ + this->m_opResolver.AddDepthwiseConv2D(); + this->m_opResolver.AddConv2D(); + this->m_opResolver.AddAdd(); + this->m_opResolver.AddResizeNearestNeighbor(); + /*These are needed for UT to work, not needed on FVP */ + this->m_opResolver.AddPad(); + this->m_opResolver.AddMaxPool2D(); + this->m_opResolver.AddConcatenation(); + +#if defined(ARM_NPU) + if (kTfLiteOk == this->m_opResolver.AddEthosU()) { + info("Added %s support to op resolver\n", + tflite::GetString_ETHOSU()); + } else { + printf_err("Failed to add Arm NPU support to op resolver."); + return false; + } +#endif /* ARM_NPU */ + return true; +} + +extern uint8_t* GetModelPointer(); +const uint8_t* arm::app::YoloFastestModel::ModelPointer() +{ + return GetModelPointer(); +} + +extern size_t GetModelLen(); +size_t arm::app::YoloFastestModel::ModelSize() +{ + return GetModelLen(); +} diff --git a/source/use_case/object_detection/usecase.cmake b/source/use_case/object_detection/usecase.cmake new file mode 100644 index 0000000..15bf534 --- /dev/null +++ b/source/use_case/object_detection/usecase.cmake @@ -0,0 +1,59 @@ +#---------------------------------------------------------------------------- +# Copyright (c) 2022 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#---------------------------------------------------------------------------- + +USER_OPTION(${use_case}_FILE_PATH "Directory with custom image files to use, or path to a single image, in the evaluation application" + ${CMAKE_CURRENT_SOURCE_DIR}/resources/${use_case}/samples/ + PATH_OR_FILE) + +USER_OPTION(${use_case}_IMAGE_SIZE "Square image size in pixels. Images will be resized to this size." + 192 + STRING) + +add_compile_definitions(DISPLAY_RGB_IMAGE=1) +add_compile_definitions(INPUT_IMAGE_WIDTH=${${use_case}_IMAGE_SIZE}) +add_compile_definitions(INPUT_IMAGE_HEIGHT=${${use_case}_IMAGE_SIZE}) +add_compile_definitions(ORIGINAL_IMAGE_WIDTH=${${use_case}_IMAGE_SIZE}) +add_compile_definitions(ORIGINAL_IMAGE_HEIGHT=${${use_case}_IMAGE_SIZE}) + + +# Generate input files +generate_images_code("${${use_case}_FILE_PATH}" + ${SRC_GEN_DIR} + ${INC_GEN_DIR} + "${${use_case}_IMAGE_SIZE}") + + +USER_OPTION(${use_case}_ACTIVATION_BUF_SZ "Activation buffer size for the chosen model" + 0x00082000 + STRING) + +if (ETHOS_U_NPU_ENABLED) + set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/yolo-fastest_192_face_v4_vela_${ETHOS_U_NPU_CONFIG_ID}.tflite) +else() + set(DEFAULT_MODEL_PATH ${DEFAULT_MODEL_DIR}/yolo-fastest_192_face_v4.tflite) +endif() + +USER_OPTION(${use_case}_MODEL_TFLITE_PATH "NN models file to be used in the evaluation application. Model files must be in tflite format." + ${DEFAULT_MODEL_PATH} + FILEPATH + ) + +# Generate model file +generate_tflite_code( + MODEL_PATH ${${use_case}_MODEL_TFLITE_PATH} + DESTINATION ${SRC_GEN_DIR} + ) |