diff options
Diffstat (limited to 'source/use_case/object_detection/src')
4 files changed, 23 insertions, 353 deletions
diff --git a/source/use_case/object_detection/src/DetectorPostProcessing.cc b/source/use_case/object_detection/src/DetectorPostProcessing.cc deleted file mode 100644 index fb1606a..0000000 --- a/source/use_case/object_detection/src/DetectorPostProcessing.cc +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2022 Arm Limited. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "DetectorPostProcessing.hpp" -#include "PlatformMath.hpp" - -#include <cmath> - -namespace arm { -namespace app { - - DetectorPostProcess::DetectorPostProcess( - TfLiteTensor* modelOutput0, - TfLiteTensor* modelOutput1, - std::vector<object_detection::DetectionResult>& results, - int inputImgRows, - int inputImgCols, - const float threshold, - const float nms, - int numClasses, - int topN) - : m_outputTensor0{modelOutput0}, - m_outputTensor1{modelOutput1}, - m_results{results}, - m_inputImgRows{inputImgRows}, - m_inputImgCols{inputImgCols}, - m_threshold(threshold), - m_nms(nms), - m_numClasses(numClasses), - m_topN(topN) -{ - /* Init PostProcessing */ - this->m_net = - object_detection::Network { - .inputWidth = inputImgCols, - .inputHeight = inputImgRows, - .numClasses = numClasses, - .branches = { - object_detection::Branch { - .resolution = inputImgCols/32, - .numBox = 3, - .anchor = anchor1, - .modelOutput = this->m_outputTensor0->data.int8, - .scale = (static_cast<TfLiteAffineQuantization*>( - this->m_outputTensor0->quantization.params))->scale->data[0], - .zeroPoint = (static_cast<TfLiteAffineQuantization*>( - this->m_outputTensor0->quantization.params))->zero_point->data[0], - .size = this->m_outputTensor0->bytes - }, - object_detection::Branch { - .resolution = inputImgCols/16, - .numBox = 3, - .anchor = anchor2, - .modelOutput = this->m_outputTensor1->data.int8, - .scale = (static_cast<TfLiteAffineQuantization*>( - this->m_outputTensor1->quantization.params))->scale->data[0], - .zeroPoint = (static_cast<TfLiteAffineQuantization*>( - this->m_outputTensor1->quantization.params))->zero_point->data[0], - .size = this->m_outputTensor1->bytes - } - }, - .topN = m_topN - }; - /* End init */ -} - -bool DetectorPostProcess::DoPostProcess() -{ - /* Start postprocessing */ - int originalImageWidth = originalImageSize; - int originalImageHeight = originalImageSize; - - std::forward_list<image::Detection> detections; - GetNetworkBoxes(this->m_net, originalImageWidth, originalImageHeight, m_threshold, detections); - - /* Do nms */ - CalculateNMS(detections, this->m_net.numClasses, m_nms); - - for (auto& it: detections) { - float xMin = it.bbox.x - it.bbox.w / 2.0f; - float xMax = it.bbox.x + it.bbox.w / 2.0f; - float yMin = it.bbox.y - it.bbox.h / 2.0f; - float yMax = it.bbox.y + it.bbox.h / 2.0f; - - if (xMin < 0) { - xMin = 0; - } - if (yMin < 0) { - yMin = 0; - } - if (xMax > originalImageWidth) { - xMax = originalImageWidth; - } - if (yMax > originalImageHeight) { - yMax = originalImageHeight; - } - - float boxX = xMin; - float boxY = yMin; - float boxWidth = xMax - xMin; - float boxHeight = yMax - yMin; - - for (int j = 0; j < this->m_net.numClasses; ++j) { - if (it.prob[j] > 0) { - - object_detection::DetectionResult tmpResult = {}; - tmpResult.m_normalisedVal = it.prob[j]; - tmpResult.m_x0 = boxX; - tmpResult.m_y0 = boxY; - tmpResult.m_w = boxWidth; - tmpResult.m_h = boxHeight; - - this->m_results.push_back(tmpResult); - } - } - } - return true; -} - -void DetectorPostProcess::InsertTopNDetections(std::forward_list<image::Detection>& detections, image::Detection& det) -{ - std::forward_list<image::Detection>::iterator it; - std::forward_list<image::Detection>::iterator last_it; - for ( it = detections.begin(); it != detections.end(); ++it ) { - if(it->objectness > det.objectness) - break; - last_it = it; - } - if(it != detections.begin()) { - detections.emplace_after(last_it, det); - detections.pop_front(); - } -} - -void DetectorPostProcess::GetNetworkBoxes( - object_detection::Network& net, - int imageWidth, - int imageHeight, - float threshold, - std::forward_list<image::Detection>& detections) -{ - int numClasses = net.numClasses; - int num = 0; - auto det_objectness_comparator = [](image::Detection& pa, image::Detection& pb) { - return pa.objectness < pb.objectness; - }; - for (size_t i = 0; i < net.branches.size(); ++i) { - int height = net.branches[i].resolution; - int width = net.branches[i].resolution; - int channel = net.branches[i].numBox*(5+numClasses); - - for (int h = 0; h < net.branches[i].resolution; h++) { - for (int w = 0; w < net.branches[i].resolution; w++) { - for (int anc = 0; anc < net.branches[i].numBox; anc++) { - - /* Objectness score */ - int bbox_obj_offset = h * width * channel + w * channel + anc * (numClasses + 5) + 4; - float objectness = math::MathUtils::SigmoidF32( - (static_cast<float>(net.branches[i].modelOutput[bbox_obj_offset]) - - net.branches[i].zeroPoint - ) * net.branches[i].scale); - - if(objectness > threshold) { - image::Detection det; - det.objectness = objectness; - /* Get bbox prediction data for each anchor, each feature point */ - int bbox_x_offset = bbox_obj_offset -4; - int bbox_y_offset = bbox_x_offset + 1; - int bbox_w_offset = bbox_x_offset + 2; - int bbox_h_offset = bbox_x_offset + 3; - int bbox_scores_offset = bbox_x_offset + 5; - - det.bbox.x = (static_cast<float>(net.branches[i].modelOutput[bbox_x_offset]) - - net.branches[i].zeroPoint) * net.branches[i].scale; - det.bbox.y = (static_cast<float>(net.branches[i].modelOutput[bbox_y_offset]) - - net.branches[i].zeroPoint) * net.branches[i].scale; - det.bbox.w = (static_cast<float>(net.branches[i].modelOutput[bbox_w_offset]) - - net.branches[i].zeroPoint) * net.branches[i].scale; - det.bbox.h = (static_cast<float>(net.branches[i].modelOutput[bbox_h_offset]) - - net.branches[i].zeroPoint) * net.branches[i].scale; - - float bbox_x, bbox_y; - - /* Eliminate grid sensitivity trick involved in YOLOv4 */ - bbox_x = math::MathUtils::SigmoidF32(det.bbox.x); - bbox_y = math::MathUtils::SigmoidF32(det.bbox.y); - det.bbox.x = (bbox_x + w) / width; - det.bbox.y = (bbox_y + h) / height; - - det.bbox.w = std::exp(det.bbox.w) * net.branches[i].anchor[anc*2] / net.inputWidth; - det.bbox.h = std::exp(det.bbox.h) * net.branches[i].anchor[anc*2+1] / net.inputHeight; - - for (int s = 0; s < numClasses; s++) { - float sig = math::MathUtils::SigmoidF32( - (static_cast<float>(net.branches[i].modelOutput[bbox_scores_offset + s]) - - net.branches[i].zeroPoint) * net.branches[i].scale - ) * objectness; - det.prob.emplace_back((sig > threshold) ? sig : 0); - } - - /* Correct_YOLO_boxes */ - det.bbox.x *= imageWidth; - det.bbox.w *= imageWidth; - det.bbox.y *= imageHeight; - det.bbox.h *= imageHeight; - - if (num < net.topN || net.topN <=0) { - detections.emplace_front(det); - num += 1; - } else if (num == net.topN) { - detections.sort(det_objectness_comparator); - InsertTopNDetections(detections,det); - num += 1; - } else { - InsertTopNDetections(detections,det); - } - } - } - } - } - } - if(num > net.topN) - num -=1; -} - -} /* namespace app */ -} /* namespace arm */ diff --git a/source/use_case/object_detection/src/DetectorPreProcessing.cc b/source/use_case/object_detection/src/DetectorPreProcessing.cc deleted file mode 100644 index 7212046..0000000 --- a/source/use_case/object_detection/src/DetectorPreProcessing.cc +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2022 Arm Limited. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "DetectorPreProcessing.hpp" -#include "ImageUtils.hpp" -#include "log_macros.h" - -namespace arm { -namespace app { - - DetectorPreProcess::DetectorPreProcess(TfLiteTensor* inputTensor, bool rgb2Gray, bool convertToInt8) - : m_inputTensor{inputTensor}, - m_rgb2Gray{rgb2Gray}, - m_convertToInt8{convertToInt8} - {} - - bool DetectorPreProcess::DoPreProcess(const void* data, size_t inputSize) { - if (data == nullptr) { - printf_err("Data pointer is null"); - } - - auto input = static_cast<const uint8_t*>(data); - - if (this->m_rgb2Gray) { - image::RgbToGrayscale(input, this->m_inputTensor->data.uint8, this->m_inputTensor->bytes); - } else { - std::memcpy(this->m_inputTensor->data.data, input, inputSize); - } - debug("Input tensor populated \n"); - - if (this->m_convertToInt8) { - image::ConvertImgToInt8(this->m_inputTensor->data.data, this->m_inputTensor->bytes); - } - - return true; - } - -} /* namespace app */ -} /* namespace arm */
\ No newline at end of file diff --git a/source/use_case/object_detection/src/MainLoop.cc b/source/use_case/object_detection/src/MainLoop.cc index 4291164..d119501 100644 --- a/source/use_case/object_detection/src/MainLoop.cc +++ b/source/use_case/object_detection/src/MainLoop.cc @@ -19,7 +19,17 @@ #include "YoloFastestModel.hpp" /* Model class for running inference. */ #include "UseCaseHandler.hpp" /* Handlers for different user options. */ #include "UseCaseCommonUtils.hpp" /* Utils functions. */ -#include "log_macros.h" +#include "log_macros.h" /* Logging functions */ +#include "BufAttributes.hpp" /* Buffer attributes to be applied */ + +namespace arm { + namespace app { + static uint8_t tensorArena[ACTIVATION_BUF_SZ] ACTIVATION_BUF_ATTRIBUTE; + } /* namespace app */ +} /* namespace arm */ + +extern uint8_t* GetModelPointer(); +extern size_t GetModelLen(); static void DisplayDetectionMenu() { @@ -40,11 +50,22 @@ void main_loop() arm::app::YoloFastestModel model; /* Model wrapper object. */ /* Load the model. */ - if (!model.Init()) { + if (!model.Init(arm::app::tensorArena, + sizeof(arm::app::tensorArena), + GetModelPointer(), + GetModelLen())) { printf_err("Failed to initialise model\n"); return; } +#if !defined(ARM_NPU) + /* If it is not a NPU build check if the model contains a NPU operator */ + if (model.ContainsEthosUOperator()) { + printf_err("No driver support for Ethos-U operator found in the model.\n"); + return; + } +#endif /* ARM_NPU */ + /* Instantiate application context. */ arm::app::ApplicationContext caseContext; diff --git a/source/use_case/object_detection/src/YoloFastestModel.cc b/source/use_case/object_detection/src/YoloFastestModel.cc deleted file mode 100644 index b1fd776..0000000 --- a/source/use_case/object_detection/src/YoloFastestModel.cc +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2022 Arm Limited. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "YoloFastestModel.hpp" - -#include "log_macros.h" - -const tflite::MicroOpResolver& arm::app::YoloFastestModel::GetOpResolver() -{ - return this->m_opResolver; -} - -bool arm::app::YoloFastestModel::EnlistOperations() -{ - this->m_opResolver.AddDepthwiseConv2D(); - this->m_opResolver.AddConv2D(); - this->m_opResolver.AddAdd(); - this->m_opResolver.AddResizeNearestNeighbor(); - /*These are needed for UT to work, not needed on FVP */ - this->m_opResolver.AddPad(); - this->m_opResolver.AddMaxPool2D(); - this->m_opResolver.AddConcatenation(); - -#if defined(ARM_NPU) - if (kTfLiteOk == this->m_opResolver.AddEthosU()) { - info("Added %s support to op resolver\n", - tflite::GetString_ETHOSU()); - } else { - printf_err("Failed to add Arm NPU support to op resolver."); - return false; - } -#endif /* ARM_NPU */ - return true; -} - -extern uint8_t* GetModelPointer(); -const uint8_t* arm::app::YoloFastestModel::ModelPointer() -{ - return GetModelPointer(); -} - -extern size_t GetModelLen(); -size_t arm::app::YoloFastestModel::ModelSize() -{ - return GetModelLen(); -} |