diff options
Diffstat (limited to 'source/use_case/vww')
| -rw-r--r-- | source/use_case/vww/include/VisualWakeWordModel.hpp | 54 | ||||
| -rw-r--r-- | source/use_case/vww/include/VisualWakeWordProcessing.hpp | 93 | ||||
| -rw-r--r-- | source/use_case/vww/src/MainLoop.cc | 27 | ||||
| -rw-r--r-- | source/use_case/vww/src/VisualWakeWordModel.cc | 56 | ||||
| -rw-r--r-- | source/use_case/vww/src/VisualWakeWordProcessing.cc | 80 | ||||
| -rw-r--r-- | source/use_case/vww/usecase.cmake | 4 |
6 files changed, 28 insertions, 286 deletions
diff --git a/source/use_case/vww/include/VisualWakeWordModel.hpp b/source/use_case/vww/include/VisualWakeWordModel.hpp deleted file mode 100644 index 1ed9202..0000000 --- a/source/use_case/vww/include/VisualWakeWordModel.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2021 - 2022 Arm Limited. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef VISUAL_WAKE_WORD_MODEL_HPP -#define VISUAL_WAKE_WORD_MODEL_HPP - -#include "Model.hpp" - -namespace arm { -namespace app { - - class VisualWakeWordModel : public Model { - - public: - /* Indices for the expected model - based on input tensor shape */ - static constexpr uint32_t ms_inputRowsIdx = 1; - static constexpr uint32_t ms_inputColsIdx = 2; - static constexpr uint32_t ms_inputChannelsIdx = 3; - - protected: - /** @brief Gets the reference to op resolver interface class. */ - const tflite::MicroOpResolver& GetOpResolver() override; - - /** @brief Adds operations to the op resolver instance. */ - bool EnlistOperations() override; - - const uint8_t* ModelPointer() override; - - size_t ModelSize() override; - private: - /* Maximum number of individual operations that can be enlisted. */ - static constexpr int ms_maxOpCnt = 7; - - /* A mutable op resolver instance. */ - tflite::MicroMutableOpResolver<ms_maxOpCnt> m_opResolver; - }; - -} /* namespace app */ -} /* namespace arm */ - -#endif /* VISUAL_WAKE_WORD_MODEL_HPP */ diff --git a/source/use_case/vww/include/VisualWakeWordProcessing.hpp b/source/use_case/vww/include/VisualWakeWordProcessing.hpp deleted file mode 100644 index f9f9d72..0000000 --- a/source/use_case/vww/include/VisualWakeWordProcessing.hpp +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2022 Arm Limited. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef VWW_PROCESSING_HPP -#define VWW_PROCESSING_HPP - -#include "BaseProcessing.hpp" -#include "Model.hpp" -#include "Classifier.hpp" - -namespace arm { -namespace app { - - /** - * @brief Pre-processing class for Visual Wake Word use case. - * Implements methods declared by BasePreProcess and anything else needed - * to populate input tensors ready for inference. - */ - class VisualWakeWordPreProcess : public BasePreProcess { - - public: - /** - * @brief Constructor - * @param[in] inputTensor Pointer to the TFLite Micro input Tensor. - * @param[in] rgb2Gray Convert image from 3 channel RGB to 1 channel grayscale. - **/ - explicit VisualWakeWordPreProcess(TfLiteTensor* inputTensor, bool rgb2Gray=true); - - /** - * @brief Should perform pre-processing of 'raw' input image data and load it into - * TFLite Micro input tensors ready for inference - * @param[in] input Pointer to the data that pre-processing will work on. - * @param[in] inputSize Size of the input data. - * @return true if successful, false otherwise. - **/ - bool DoPreProcess(const void* input, size_t inputSize) override; - - private: - TfLiteTensor* m_inputTensor; - bool m_rgb2Gray; - }; - - /** - * @brief Post-processing class for Visual Wake Word use case. - * Implements methods declared by BasePostProcess and anything else needed - * to populate result vector. - */ - class VisualWakeWordPostProcess : public BasePostProcess { - - private: - TfLiteTensor* m_outputTensor; - Classifier& m_vwwClassifier; - const std::vector<std::string>& m_labels; - std::vector<ClassificationResult>& m_results; - - public: - /** - * @brief Constructor - * @param[in] outputTensor Pointer to the TFLite Micro output Tensor. - * @param[in] classifier Classifier object used to get top N results from classification. - * @param[in] model Pointer to the VWW classification Model object. - * @param[in] labels Vector of string labels to identify each output of the model. - * @param[out] results Vector of classification results to store decoded outputs. - **/ - VisualWakeWordPostProcess(TfLiteTensor* outputTensor, Classifier& classifier, - const std::vector<std::string>& labels, - std::vector<ClassificationResult>& results); - - /** - * @brief Should perform post-processing of the result of inference then - * populate classification result data for any later use. - * @return true if successful, false otherwise. - **/ - bool DoPostProcess() override; - }; - -} /* namespace app */ -} /* namespace arm */ - -#endif /* VWW_PROCESSING_HPP */
\ No newline at end of file diff --git a/source/use_case/vww/src/MainLoop.cc b/source/use_case/vww/src/MainLoop.cc index 041ea18..2161b0a 100644 --- a/source/use_case/vww/src/MainLoop.cc +++ b/source/use_case/vww/src/MainLoop.cc @@ -21,7 +21,17 @@ #include "VisualWakeWordModel.hpp" /* Model class for running inference. */ #include "UseCaseHandler.hpp" /* Handlers for different user options. */ #include "UseCaseCommonUtils.hpp" /* Utils functions. */ -#include "log_macros.h" +#include "log_macros.h" /* Logging functions */ +#include "BufAttributes.hpp" /* Buffer attributes to be applied */ + +namespace arm { + namespace app { + static uint8_t tensorArena[ACTIVATION_BUF_SZ] ACTIVATION_BUF_ATTRIBUTE; + } /* namespace app */ +} /* namespace arm */ + +extern uint8_t* GetModelPointer(); +extern size_t GetModelLen(); using ViusalWakeWordClassifier = arm::app::Classifier; @@ -30,11 +40,22 @@ void main_loop() arm::app::VisualWakeWordModel model; /* Model wrapper object. */ /* Load the model. */ - if (!model.Init()) { + if (!model.Init(arm::app::tensorArena, + sizeof(arm::app::tensorArena), + GetModelPointer(), + GetModelLen())) { printf_err("Failed to initialise model\n"); return; } +#if !defined(ARM_NPU) + /* If it is not a NPU build check if the model contains a NPU operator */ + if (model.ContainsEthosUOperator()) { + printf_err("No driver support for Ethos-U operator found in the model.\n"); + return; + } +#endif /* ARM_NPU */ + /* Instantiate application context. */ arm::app::ApplicationContext caseContext; @@ -55,7 +76,7 @@ void main_loop() constexpr bool bUseMenu = NUMBER_OF_FILES > 1 ? true : false; do { int menuOption = common::MENU_OPT_RUN_INF_NEXT; - if (bUseMenu) { + if (bUseMenu) { DisplayCommonMenu(); menuOption = arm::app::ReadUserInputAsInt(); printf("\n"); diff --git a/source/use_case/vww/src/VisualWakeWordModel.cc b/source/use_case/vww/src/VisualWakeWordModel.cc deleted file mode 100644 index 59beccc..0000000 --- a/source/use_case/vww/src/VisualWakeWordModel.cc +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2021 Arm Limited. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "VisualWakeWordModel.hpp" -#include "log_macros.h" - -const tflite::MicroOpResolver& arm::app::VisualWakeWordModel::GetOpResolver() -{ - return this->m_opResolver; -} - -bool arm::app::VisualWakeWordModel::EnlistOperations() -{ - this->m_opResolver.AddDepthwiseConv2D(); - this->m_opResolver.AddConv2D(); - this->m_opResolver.AddAveragePool2D(); - this->m_opResolver.AddReshape(); - this->m_opResolver.AddPad(); - this->m_opResolver.AddAdd(); - -#if defined(ARM_NPU) - if (kTfLiteOk == this->m_opResolver.AddEthosU()) { - info("Added %s support to op resolver\n", - tflite::GetString_ETHOSU()); - } else { - printf_err("Failed to add Arm NPU support to op resolver."); - return false; - } -#endif /* ARM_NPU */ - return true; -} - -extern uint8_t* GetModelPointer(); -const uint8_t* arm::app::VisualWakeWordModel::ModelPointer() -{ - return GetModelPointer(); -} - -extern size_t GetModelLen(); -size_t arm::app::VisualWakeWordModel::ModelSize() -{ - return GetModelLen(); -}
\ No newline at end of file diff --git a/source/use_case/vww/src/VisualWakeWordProcessing.cc b/source/use_case/vww/src/VisualWakeWordProcessing.cc deleted file mode 100644 index 4ae8a54..0000000 --- a/source/use_case/vww/src/VisualWakeWordProcessing.cc +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2022 Arm Limited. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "VisualWakeWordProcessing.hpp" - -#include "ImageUtils.hpp" -#include "VisualWakeWordModel.hpp" -#include "log_macros.h" - -namespace arm { -namespace app { - - VisualWakeWordPreProcess::VisualWakeWordPreProcess(TfLiteTensor* inputTensor, bool rgb2Gray) - :m_inputTensor{inputTensor}, - m_rgb2Gray{rgb2Gray} - {} - - bool VisualWakeWordPreProcess::DoPreProcess(const void* data, size_t inputSize) - { - if (data == nullptr) { - printf_err("Data pointer is null"); - } - - auto input = static_cast<const uint8_t*>(data); - - uint8_t* unsignedDstPtr = this->m_inputTensor->data.uint8; - - if (this->m_rgb2Gray) { - image::RgbToGrayscale(input, unsignedDstPtr, inputSize); - } else { - std::memcpy(unsignedDstPtr, input, inputSize); - } - - /* VWW model pre-processing is image conversion from uint8 to [0,1] float values, - * then quantize them with input quantization info. */ - QuantParams inQuantParams = GetTensorQuantParams(this->m_inputTensor); - - int8_t* signedDstPtr = this->m_inputTensor->data.int8; - for (size_t i = 0; i < this->m_inputTensor->bytes; i++) { - auto i_data_int8 = static_cast<int8_t>( - ((static_cast<float>(unsignedDstPtr[i]) / 255.0f) / inQuantParams.scale) + inQuantParams.offset - ); - signedDstPtr[i] = std::min<int8_t>(INT8_MAX, std::max<int8_t>(i_data_int8, INT8_MIN)); - } - - debug("Input tensor populated \n"); - - return true; - } - - VisualWakeWordPostProcess::VisualWakeWordPostProcess(TfLiteTensor* outputTensor, Classifier& classifier, - const std::vector<std::string>& labels, std::vector<ClassificationResult>& results) - :m_outputTensor{outputTensor}, - m_vwwClassifier{classifier}, - m_labels{labels}, - m_results{results} - {} - - bool VisualWakeWordPostProcess::DoPostProcess() - { - return this->m_vwwClassifier.GetClassificationResults( - this->m_outputTensor, this->m_results, - this->m_labels, 1, true); - } - -} /* namespace app */ -} /* namespace arm */
\ No newline at end of file diff --git a/source/use_case/vww/usecase.cmake b/source/use_case/vww/usecase.cmake index 8bf55fc..f6a3efe 100644 --- a/source/use_case/vww/usecase.cmake +++ b/source/use_case/vww/usecase.cmake @@ -1,3 +1,4 @@ +#---------------------------------------------------------------------------- # Copyright (c) 2021 Arm Limited. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -12,7 +13,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +#---------------------------------------------------------------------------- +# Append the API to use for this use case +list(APPEND ${use_case}_API_LIST "vww") USER_OPTION(${use_case}_FILE_PATH "Directory with custom image files, or path to a single image file, to use in the evaluation application" ${CMAKE_CURRENT_SOURCE_DIR}/resources/${use_case}/samples/ |