diff options
Diffstat (limited to 'source/use_case/kws')
| -rw-r--r-- | source/use_case/kws/include/DsCnnMfcc.hpp | 50 | ||||
| -rw-r--r-- | source/use_case/kws/include/DsCnnModel.hpp | 59 | ||||
| -rw-r--r-- | source/use_case/kws/include/KwsResult.hpp | 63 | ||||
| -rw-r--r-- | source/use_case/kws/include/UseCaseHandler.hpp | 37 | ||||
| -rw-r--r-- | source/use_case/kws/src/DsCnnModel.cc | 58 | ||||
| -rw-r--r-- | source/use_case/kws/src/MainLoop.cc | 112 | ||||
| -rw-r--r-- | source/use_case/kws/src/UseCaseHandler.cc | 452 | ||||
| -rw-r--r-- | source/use_case/kws/usecase.cmake | 159 |
8 files changed, 990 insertions, 0 deletions
diff --git a/source/use_case/kws/include/DsCnnMfcc.hpp b/source/use_case/kws/include/DsCnnMfcc.hpp new file mode 100644 index 0000000..3f681af --- /dev/null +++ b/source/use_case/kws/include/DsCnnMfcc.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef KWS_DSCNN_MFCC_HPP +#define KWS_DSCNN_MFCC_HPP + +#include "Mfcc.hpp" + +namespace arm { +namespace app { +namespace audio { + + /* Class to provide DS-CNN specific MFCC calculation requirements. */ + class DsCnnMFCC : public MFCC { + + public: + static constexpr uint32_t ms_defaultSamplingFreq = 16000; + static constexpr uint32_t ms_defaultNumFbankBins = 40; + static constexpr uint32_t ms_defaultMelLoFreq = 20; + static constexpr uint32_t ms_defaultMelHiFreq = 4000; + static constexpr bool ms_defaultUseHtkMethod = true; + + explicit DsCnnMFCC(const size_t numFeats, const size_t frameLen) + : MFCC(MfccParams( + ms_defaultSamplingFreq, ms_defaultNumFbankBins, + ms_defaultMelLoFreq, ms_defaultMelHiFreq, + numFeats, frameLen, ms_defaultUseHtkMethod)) + {} + DsCnnMFCC() = delete; + ~DsCnnMFCC() = default; + }; + +} /* namespace audio */ +} /* namespace app */ +} /* namespace arm */ + +#endif /* KWS_DSCNN_MFCC_HPP */
\ No newline at end of file diff --git a/source/use_case/kws/include/DsCnnModel.hpp b/source/use_case/kws/include/DsCnnModel.hpp new file mode 100644 index 0000000..a4e7110 --- /dev/null +++ b/source/use_case/kws/include/DsCnnModel.hpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef KWS_DSCNNMODEL_HPP +#define KWS_DSCNNMODEL_HPP + +#include "Model.hpp" + +extern const int g_FrameLength; +extern const int g_FrameStride; +extern const float g_ScoreThreshold; + +namespace arm { +namespace app { + + class DsCnnModel : public Model { + public: + /* Indices for the expected model - based on input and output tensor shapes */ + static constexpr uint32_t ms_inputRowsIdx = 2; + static constexpr uint32_t ms_inputColsIdx = 3; + static constexpr uint32_t ms_outputRowsIdx = 2; + static constexpr uint32_t ms_outputColsIdx = 3; + + protected: + /** @brief Gets the reference to op resolver interface class. */ + const tflite::MicroOpResolver& GetOpResolver() override; + + /** @brief Adds operations to the op resolver instance. */ + bool EnlistOperations() override; + + const uint8_t* ModelPointer() override; + + size_t ModelSize() override; + + private: + /* Maximum number of individual operations that can be enlisted. */ + static constexpr int _ms_maxOpCnt = 8; + + /* A mutable op resolver instance. */ + tflite::MicroMutableOpResolver<_ms_maxOpCnt> _m_opResolver; + }; + +} /* namespace app */ +} /* namespace arm */ + +#endif /* KWS_DSCNNMODEL_HPP */ diff --git a/source/use_case/kws/include/KwsResult.hpp b/source/use_case/kws/include/KwsResult.hpp new file mode 100644 index 0000000..5a26ce1 --- /dev/null +++ b/source/use_case/kws/include/KwsResult.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef KWS_RESULT_HPP +#define KWS_RESULT_HPP + +#include "ClassificationResult.hpp" + +#include <vector> + +namespace arm { +namespace app { +namespace kws { + + using ResultVec = std::vector < arm::app::ClassificationResult >; + + /* Structure for holding kws result. */ + class KwsResult { + + public: + ResultVec m_resultVec; /* Container for "thresholded" classification results. */ + float m_timeStamp; /* Audio timestamp for this result. */ + uint32_t m_inferenceNumber; /* Corresponding inference number. */ + float m_threshold; /* Threshold value for `m_resultVec`. */ + + KwsResult() = delete; + KwsResult(ResultVec& resultVec, + const float timestamp, + const uint32_t inferenceIdx, + const float scoreThreshold) { + + this->m_threshold = scoreThreshold; + this->m_timeStamp = timestamp; + this->m_inferenceNumber = inferenceIdx; + + this->m_resultVec = ResultVec(); + for (auto & i : resultVec) { + if (i.m_normalisedVal >= this->m_threshold) { + this->m_resultVec.emplace_back(i); + } + } + } + ~KwsResult() = default; + }; + +} /* namespace kws */ +} /* namespace app */ +} /* namespace arm */ + +#endif /* KWS_RESULT_HPP */
\ No newline at end of file diff --git a/source/use_case/kws/include/UseCaseHandler.hpp b/source/use_case/kws/include/UseCaseHandler.hpp new file mode 100644 index 0000000..1eb742f --- /dev/null +++ b/source/use_case/kws/include/UseCaseHandler.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef KWS_EVT_HANDLER_HPP +#define KWS_EVT_HANDLER_HPP + +#include "AppContext.hpp" + +namespace arm { +namespace app { + + /** + * @brief Handles the inference event. + * @param[in] ctx Pointer to the application context. + * @param[in] clipIndex Index to the audio clip to classify. + * @param[in] runAll Flag to request classification of all the available audio clips. + * @return true or false based on execution success. + **/ + bool ClassifyAudioHandler(ApplicationContext& ctx, uint32_t clipIndex, bool runAll); + +} /* namespace app */ +} /* namespace arm */ + +#endif /* KWS_EVT_HANDLER_HPP */
\ No newline at end of file diff --git a/source/use_case/kws/src/DsCnnModel.cc b/source/use_case/kws/src/DsCnnModel.cc new file mode 100644 index 0000000..a093eb4 --- /dev/null +++ b/source/use_case/kws/src/DsCnnModel.cc @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "DsCnnModel.hpp" + +#include "hal.h" + +const tflite::MicroOpResolver& arm::app::DsCnnModel::GetOpResolver() +{ + return this->_m_opResolver; +} + +bool arm::app::DsCnnModel::EnlistOperations() +{ + this->_m_opResolver.AddReshape(); + this->_m_opResolver.AddAveragePool2D(); + this->_m_opResolver.AddConv2D(); + this->_m_opResolver.AddDepthwiseConv2D(); + this->_m_opResolver.AddFullyConnected(); + this->_m_opResolver.AddRelu(); + this->_m_opResolver.AddSoftmax(); + +#if defined(ARM_NPU) + if (kTfLiteOk == this->_m_opResolver.AddEthosU()) { + info("Added %s support to op resolver\n", + tflite::GetString_ETHOSU()); + } else { + printf_err("Failed to add Arm NPU support to op resolver."); + return false; + } +#endif /* ARM_NPU */ + return true; +} + +extern uint8_t* GetModelPointer(); +const uint8_t* arm::app::DsCnnModel::ModelPointer() +{ + return GetModelPointer(); +} + +extern size_t GetModelLen(); +size_t arm::app::DsCnnModel::ModelSize() +{ + return GetModelLen(); +}
\ No newline at end of file diff --git a/source/use_case/kws/src/MainLoop.cc b/source/use_case/kws/src/MainLoop.cc new file mode 100644 index 0000000..24cb939 --- /dev/null +++ b/source/use_case/kws/src/MainLoop.cc @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "InputFiles.hpp" /* For input audio clips. */ +#include "Classifier.hpp" /* Classifier. */ +#include "DsCnnModel.hpp" /* Model class for running inference. */ +#include "hal.h" /* Brings in platform definitions. */ +#include "Labels.hpp" /* For label strings. */ +#include "UseCaseHandler.hpp" /* Handlers for different user options. */ +#include "UseCaseCommonUtils.hpp" /* Utils functions. */ + +using KwsClassifier = arm::app::Classifier; + +enum opcodes +{ + MENU_OPT_RUN_INF_NEXT = 1, /* Run on next vector. */ + MENU_OPT_RUN_INF_CHOSEN, /* Run on a user provided vector index. */ + MENU_OPT_RUN_INF_ALL, /* Run inference on all. */ + MENU_OPT_SHOW_MODEL_INFO, /* Show model info. */ + MENU_OPT_LIST_AUDIO_CLIPS /* List the current baked audio clips. */ +}; + +static void DisplayMenu() +{ + printf("\n\nUser input required\n"); + printf("Enter option number from:\n\n"); + printf(" %u. Classify next audio clip\n", MENU_OPT_RUN_INF_NEXT); + printf(" %u. Classify audio clip at chosen index\n", MENU_OPT_RUN_INF_CHOSEN); + printf(" %u. Run classification on all audio clips\n", MENU_OPT_RUN_INF_ALL); + printf(" %u. Show NN model info\n", MENU_OPT_SHOW_MODEL_INFO); + printf(" %u. List audio clips\n\n", MENU_OPT_LIST_AUDIO_CLIPS); + printf(" Choice: "); +} + +void main_loop(hal_platform& platform) +{ + arm::app::DsCnnModel model; /* Model wrapper object. */ + + /* Load the model. */ + if (!model.Init()) { + printf_err("Failed to initialise model\n"); + return; + } + + /* Instantiate application context. */ + arm::app::ApplicationContext caseContext; + + caseContext.Set<hal_platform&>("platform", platform); + caseContext.Set<arm::app::Model&>("model", model); + caseContext.Set<uint32_t>("clipIndex", 0); + caseContext.Set<int>("frameLength", g_FrameLength); + caseContext.Set<int>("frameStride", g_FrameStride); + caseContext.Set<float>("scoreThreshold", g_ScoreThreshold); /* Normalised score threshold. */ + + KwsClassifier classifier; /* classifier wrapper object. */ + caseContext.Set<arm::app::Classifier&>("classifier", classifier); + + std::vector <std::string> labels; + GetLabelsVector(labels); + + caseContext.Set<const std::vector <std::string>&>("labels", labels); + + bool executionSuccessful = true; + constexpr bool bUseMenu = NUMBER_OF_FILES > 1 ? true : false; + + /* Loop. */ + do { + int menuOption = MENU_OPT_RUN_INF_NEXT; + if (bUseMenu) { + DisplayMenu(); + menuOption = arm::app::ReadUserInputAsInt(platform); + printf("\n"); + } + switch (menuOption) { + case MENU_OPT_RUN_INF_NEXT: + executionSuccessful = ClassifyAudioHandler(caseContext, caseContext.Get<uint32_t>("clipIndex"), false); + break; + case MENU_OPT_RUN_INF_CHOSEN: { + printf(" Enter the audio clip index [0, %d]: ", NUMBER_OF_FILES-1); + auto clipIndex = static_cast<uint32_t>(arm::app::ReadUserInputAsInt(platform)); + executionSuccessful = ClassifyAudioHandler(caseContext, clipIndex, false); + break; + } + case MENU_OPT_RUN_INF_ALL: + executionSuccessful = ClassifyAudioHandler(caseContext,caseContext.Get<uint32_t>("clipIndex"), true); + break; + case MENU_OPT_SHOW_MODEL_INFO: + executionSuccessful = model.ShowModelInfoHandler(); + break; + case MENU_OPT_LIST_AUDIO_CLIPS: + executionSuccessful = ListFilesHandler(caseContext); + break; + default: + printf("Incorrect choice, try again."); + break; + } + } while (executionSuccessful && bUseMenu); + info("Main loop terminated.\n"); +}
\ No newline at end of file diff --git a/source/use_case/kws/src/UseCaseHandler.cc b/source/use_case/kws/src/UseCaseHandler.cc new file mode 100644 index 0000000..872d323 --- /dev/null +++ b/source/use_case/kws/src/UseCaseHandler.cc @@ -0,0 +1,452 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "UseCaseHandler.hpp" + +#include "InputFiles.hpp" +#include "Classifier.hpp" +#include "DsCnnModel.hpp" +#include "hal.h" +#include "DsCnnMfcc.hpp" +#include "AudioUtils.hpp" +#include "UseCaseCommonUtils.hpp" +#include "KwsResult.hpp" + +#include <vector> +#include <functional> + +using KwsClassifier = arm::app::Classifier; + +namespace arm { +namespace app { + + /** + * @brief Helper function to increment current audio clip index. + * @param[in,out] ctx Pointer to the application context object. + **/ + static void _IncrementAppCtxClipIdx(ApplicationContext& ctx); + + /** + * @brief Helper function to set the audio clip index. + * @param[in,out] ctx Pointer to the application context object. + * @param[in] idx Value to be set. + * @return true if index is set, false otherwise. + **/ + static bool _SetAppCtxClipIdx(ApplicationContext& ctx, uint32_t idx); + + /** + * @brief Presents inference results using the data presentation + * object. + * @param[in] platform Reference to the hal platform object. + * @param[in] results Vector of classification results to be displayed. + * @param[in] infTimeMs Inference time in milliseconds, if available, + * otherwise, this can be passed in as 0. + * @return true if successful, false otherwise. + **/ + static bool _PresentInferenceResult(hal_platform& platform, + const std::vector<arm::app::kws::KwsResult>& results); + + /** + * @brief Returns a function to perform feature calculation and populates input tensor data with + * MFCC data. + * + * Input tensor data type check is performed to choose correct MFCC feature data type. + * If tensor has an integer data type then original features are quantised. + * + * Warning: MFCC calculator provided as input must have the same life scope as returned function. + * + * @param[in] mfcc MFCC feature calculator. + * @param[in,out] inputTensor Input tensor pointer to store calculated features. + * @param[in] cacheSize Size of the feature vectors cache (number of feature vectors). + * @return Function to be called providing audio sample and sliding window index. + */ + static std::function<void (std::vector<int16_t>&, int, bool, size_t)> + GetFeatureCalculator(audio::DsCnnMFCC& mfcc, + TfLiteTensor* inputTensor, + size_t cacheSize); + + /* Audio inference handler. */ + bool ClassifyAudioHandler(ApplicationContext& ctx, uint32_t clipIndex, bool runAll) + { + auto& platform = ctx.Get<hal_platform&>("platform"); + + constexpr uint32_t dataPsnTxtInfStartX = 20; + constexpr uint32_t dataPsnTxtInfStartY = 40; + constexpr int minTensorDims = static_cast<int>( + (arm::app::DsCnnModel::ms_inputRowsIdx > arm::app::DsCnnModel::ms_inputColsIdx)? + arm::app::DsCnnModel::ms_inputRowsIdx : arm::app::DsCnnModel::ms_inputColsIdx); + + platform.data_psn->clear(COLOR_BLACK); + + auto& model = ctx.Get<Model&>("model"); + + /* If the request has a valid size, set the audio index. */ + if (clipIndex < NUMBER_OF_FILES) { + if (!_SetAppCtxClipIdx(ctx, clipIndex)) { + return false; + } + } + if (!model.IsInited()) { + printf_err("Model is not initialised! Terminating processing.\n"); + return false; + } + + const auto frameLength = ctx.Get<int>("frameLength"); + const auto frameStride = ctx.Get<int>("frameStride"); + const auto scoreThreshold = ctx.Get<float>("scoreThreshold"); + auto startClipIdx = ctx.Get<uint32_t>("clipIndex"); + + TfLiteTensor* outputTensor = model.GetOutputTensor(0); + TfLiteTensor* inputTensor = model.GetInputTensor(0); + + if (!inputTensor->dims) { + printf_err("Invalid input tensor dims\n"); + return false; + } else if (inputTensor->dims->size < minTensorDims) { + printf_err("Input tensor dimension should be >= %d\n", minTensorDims); + return false; + } + + TfLiteIntArray* inputShape = model.GetInputShape(0); + const uint32_t kNumCols = inputShape->data[arm::app::DsCnnModel::ms_inputColsIdx]; + const uint32_t kNumRows = inputShape->data[arm::app::DsCnnModel::ms_inputRowsIdx]; + + audio::DsCnnMFCC mfcc = audio::DsCnnMFCC(kNumCols, frameLength); + mfcc.Init(); + + /* Deduce the data length required for 1 inference from the network parameters. */ + auto audioDataWindowSize = kNumRows * frameStride + (frameLength - frameStride); + auto mfccWindowSize = frameLength; + auto mfccWindowStride = frameStride; + + /* We choose to move by half the window size => for a 1 second window size + * there is an overlap of 0.5 seconds. */ + auto audioDataStride = audioDataWindowSize / 2; + + /* To have the previously calculated features re-usable, stride must be multiple + * of MFCC features window stride. */ + if (0 != audioDataStride % mfccWindowStride) { + + /* Reduce the stride. */ + audioDataStride -= audioDataStride % mfccWindowStride; + } + + auto nMfccVectorsInAudioStride = audioDataStride/mfccWindowStride; + + /* We expect to be sampling 1 second worth of data at a time. + * NOTE: This is only used for time stamp calculation. */ + const float secondsPerSample = 1.0/audio::DsCnnMFCC::ms_defaultSamplingFreq; + + do { + auto currentIndex = ctx.Get<uint32_t>("clipIndex"); + + /* Creating a mfcc features sliding window for the data required for 1 inference. */ + auto audioMFCCWindowSlider = audio::SlidingWindow<const int16_t>( + get_audio_array(currentIndex), + audioDataWindowSize, mfccWindowSize, + mfccWindowStride); + + /* Creating a sliding window through the whole audio clip. */ + auto audioDataSlider = audio::SlidingWindow<const int16_t>( + get_audio_array(currentIndex), + get_audio_array_size(currentIndex), + audioDataWindowSize, audioDataStride); + + /* Calculate number of the feature vectors in the window overlap region. + * These feature vectors will be reused.*/ + auto numberOfReusedFeatureVectors = audioMFCCWindowSlider.TotalStrides() + 1 + - nMfccVectorsInAudioStride; + + /* Construct feature calculation function. */ + auto mfccFeatureCalc = GetFeatureCalculator(mfcc, inputTensor, + numberOfReusedFeatureVectors); + + if (!mfccFeatureCalc){ + return false; + } + + /* Declare a container for results. */ + std::vector<arm::app::kws::KwsResult> results; + + /* Display message on the LCD - inference running. */ + std::string str_inf{"Running inference... "}; + platform.data_psn->present_data_text( + str_inf.c_str(), str_inf.size(), + dataPsnTxtInfStartX, dataPsnTxtInfStartY, 0); + info("Running inference on audio clip %u => %s\n", currentIndex, + get_filename(currentIndex)); + + /* Start sliding through audio clip. */ + while (audioDataSlider.HasNext()) { + const int16_t *inferenceWindow = audioDataSlider.Next(); + + /* We moved to the next window - set the features sliding to the new address. */ + audioMFCCWindowSlider.Reset(inferenceWindow); + + /* The first window does not have cache ready. */ + bool useCache = audioDataSlider.Index() > 0 && numberOfReusedFeatureVectors > 0; + + /* Start calculating features inside one audio sliding window. */ + while (audioMFCCWindowSlider.HasNext()) { + const int16_t *mfccWindow = audioMFCCWindowSlider.Next(); + std::vector<int16_t> mfccAudioData = std::vector<int16_t>(mfccWindow, + mfccWindow + mfccWindowSize); + /* Compute features for this window and write them to input tensor. */ + mfccFeatureCalc(mfccAudioData, + audioMFCCWindowSlider.Index(), + useCache, + nMfccVectorsInAudioStride); + } + + info("Inference %zu/%zu\n", audioDataSlider.Index() + 1, + audioDataSlider.TotalStrides() + 1); + + /* Run inference over this audio clip sliding window. */ + arm::app::RunInference(platform, model); + + std::vector<ClassificationResult> classificationResult; + auto& classifier = ctx.Get<KwsClassifier&>("classifier"); + classifier.GetClassificationResults(outputTensor, classificationResult, + ctx.Get<std::vector<std::string>&>("labels"), 1); + + results.emplace_back(kws::KwsResult(classificationResult, + audioDataSlider.Index() * secondsPerSample * audioDataStride, + audioDataSlider.Index(), scoreThreshold)); + +#if VERIFY_TEST_OUTPUT + arm::app::DumpTensor(outputTensor); +#endif /* VERIFY_TEST_OUTPUT */ + } /* while (audioDataSlider.HasNext()) */ + + /* Erase. */ + str_inf = std::string(str_inf.size(), ' '); + platform.data_psn->present_data_text( + str_inf.c_str(), str_inf.size(), + dataPsnTxtInfStartX, dataPsnTxtInfStartY, false); + + ctx.Set<std::vector<arm::app::kws::KwsResult>>("results", results); + + if (!_PresentInferenceResult(platform, results)) { + return false; + } + + _IncrementAppCtxClipIdx(ctx); + + } while (runAll && ctx.Get<uint32_t>("clipIndex") != startClipIdx); + + return true; + } + + static void _IncrementAppCtxClipIdx(ApplicationContext& ctx) + { + auto curAudioIdx = ctx.Get<uint32_t>("clipIndex"); + + if (curAudioIdx + 1 >= NUMBER_OF_FILES) { + ctx.Set<uint32_t>("clipIndex", 0); + return; + } + ++curAudioIdx; + ctx.Set<uint32_t>("clipIndex", curAudioIdx); + } + + static bool _SetAppCtxClipIdx(ApplicationContext& ctx, const uint32_t idx) + { + if (idx >= NUMBER_OF_FILES) { + printf_err("Invalid idx %u (expected less than %u)\n", + idx, NUMBER_OF_FILES); + return false; + } + ctx.Set<uint32_t>("clipIndex", idx); + return true; + } + + static bool _PresentInferenceResult(hal_platform& platform, + const std::vector<arm::app::kws::KwsResult>& results) + { + constexpr uint32_t dataPsnTxtStartX1 = 20; + constexpr uint32_t dataPsnTxtStartY1 = 30; + constexpr uint32_t dataPsnTxtYIncr = 16; /* Row index increment. */ + + platform.data_psn->set_text_color(COLOR_GREEN); + + /* Display each result */ + uint32_t rowIdx1 = dataPsnTxtStartY1 + 2 * dataPsnTxtYIncr; + + for (uint32_t i = 0; i < results.size(); ++i) { + + std::string topKeyword{"<none>"}; + float score = 0.f; + + if (results[i].m_resultVec.size()) { + topKeyword = results[i].m_resultVec[0].m_label; + score = results[i].m_resultVec[0].m_normalisedVal; + } + + std::string resultStr = + std::string{"@"} + std::to_string(results[i].m_timeStamp) + + std::string{"s: "} + topKeyword + std::string{" ("} + + std::to_string(static_cast<int>(score * 100)) + std::string{"%)"}; + + platform.data_psn->present_data_text( + resultStr.c_str(), resultStr.size(), + dataPsnTxtStartX1, rowIdx1, false); + rowIdx1 += dataPsnTxtYIncr; + + info("For timestamp: %f (inference #: %u); threshold: %f\n", + results[i].m_timeStamp, results[i].m_inferenceNumber, + results[i].m_threshold); + for (uint32_t j = 0; j < results[i].m_resultVec.size(); ++j) { + info("\t\tlabel @ %u: %s, score: %f\n", j, + results[i].m_resultVec[j].m_label.c_str(), + results[i].m_resultVec[j].m_normalisedVal); + } + } + + return true; + } + + /** + * @brief Generic feature calculator factory. + * + * Returns lambda function to compute features using features cache. + * Real features math is done by a lambda function provided as a parameter. + * Features are written to input tensor memory. + * + * @tparam T Feature vector type. + * @param inputTensor Model input tensor pointer. + * @param cacheSize Number of feature vectors to cache. Defined by the sliding window overlap. + * @param compute Features calculator function. + * @return Lambda function to compute features. + */ + template<class T> + std::function<void (std::vector<int16_t>&, size_t, bool, size_t)> + _FeatureCalc(TfLiteTensor* inputTensor, size_t cacheSize, + std::function<std::vector<T> (std::vector<int16_t>& )> compute) + { + /* Feature cache to be captured by lambda function. */ + static std::vector<std::vector<T>> featureCache = std::vector<std::vector<T>>(cacheSize); + + return [=](std::vector<int16_t>& audioDataWindow, + size_t index, + bool useCache, + size_t featuresOverlapIndex) + { + T *tensorData = tflite::GetTensorData<T>(inputTensor); + std::vector<T> features; + + /* Reuse features from cache if cache is ready and sliding windows overlap. + * Overlap is in the beginning of sliding window with a size of a feature cache. */ + if (useCache && index < featureCache.size()) { + features = std::move(featureCache[index]); + } else { + features = std::move(compute(audioDataWindow)); + } + auto size = features.size(); + auto sizeBytes = sizeof(T) * size; + std::memcpy(tensorData + (index * size), features.data(), sizeBytes); + + /* Start renewing cache as soon iteration goes out of the windows overlap. */ + if (index >= featuresOverlapIndex) { + featureCache[index - featuresOverlapIndex] = std::move(features); + } + }; + } + + template std::function<void (std::vector<int16_t>&, size_t , bool, size_t)> + _FeatureCalc<int8_t>(TfLiteTensor* inputTensor, + size_t cacheSize, + std::function<std::vector<int8_t> (std::vector<int16_t>& )> compute); + + template std::function<void (std::vector<int16_t>&, size_t , bool, size_t)> + _FeatureCalc<uint8_t>(TfLiteTensor* inputTensor, + size_t cacheSize, + std::function<std::vector<uint8_t> (std::vector<int16_t>& )> compute); + + template std::function<void (std::vector<int16_t>&, size_t , bool, size_t)> + _FeatureCalc<int16_t>(TfLiteTensor* inputTensor, + size_t cacheSize, + std::function<std::vector<int16_t> (std::vector<int16_t>& )> compute); + + template std::function<void(std::vector<int16_t>&, size_t, bool, size_t)> + _FeatureCalc<float>(TfLiteTensor *inputTensor, + size_t cacheSize, + std::function<std::vector<float>(std::vector<int16_t>&)> compute); + + + static std::function<void (std::vector<int16_t>&, int, bool, size_t)> + GetFeatureCalculator(audio::DsCnnMFCC& mfcc, TfLiteTensor* inputTensor, size_t cacheSize) + { + std::function<void (std::vector<int16_t>&, size_t, bool, size_t)> mfccFeatureCalc; + + TfLiteQuantization quant = inputTensor->quantization; + + if (kTfLiteAffineQuantization == quant.type) { + + auto *quantParams = (TfLiteAffineQuantization *) quant.params; + const float quantScale = quantParams->scale->data[0]; + const int quantOffset = quantParams->zero_point->data[0]; + + switch (inputTensor->type) { + case kTfLiteInt8: { + mfccFeatureCalc = _FeatureCalc<int8_t>(inputTensor, + cacheSize, + [=, &mfcc](std::vector<int16_t>& audioDataWindow) { + return mfcc.MfccComputeQuant<int8_t>(audioDataWindow, + quantScale, + quantOffset); + } + ); + break; + } + case kTfLiteUInt8: { + mfccFeatureCalc = _FeatureCalc<uint8_t>(inputTensor, + cacheSize, + [=, &mfcc](std::vector<int16_t>& audioDataWindow) { + return mfcc.MfccComputeQuant<uint8_t>(audioDataWindow, + quantScale, + quantOffset); + } + ); + break; + } + case kTfLiteInt16: { + mfccFeatureCalc = _FeatureCalc<int16_t>(inputTensor, + cacheSize, + [=, &mfcc](std::vector<int16_t>& audioDataWindow) { + return mfcc.MfccComputeQuant<int16_t>(audioDataWindow, + quantScale, + quantOffset); + } + ); + break; + } + default: + printf_err("Tensor type %s not supported\n", TfLiteTypeGetName(inputTensor->type)); + } + + + } else { + mfccFeatureCalc = mfccFeatureCalc = _FeatureCalc<float>(inputTensor, + cacheSize, + [&mfcc](std::vector<int16_t>& audioDataWindow) { + return mfcc.MfccCompute(audioDataWindow); + }); + } + return mfccFeatureCalc; + } + +} /* namespace app */ +} /* namespace arm */
\ No newline at end of file diff --git a/source/use_case/kws/usecase.cmake b/source/use_case/kws/usecase.cmake new file mode 100644 index 0000000..b5ac09e --- /dev/null +++ b/source/use_case/kws/usecase.cmake @@ -0,0 +1,159 @@ +#---------------------------------------------------------------------------- +# Copyright (c) 2021 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#---------------------------------------------------------------------------- + +# If the path to a directory or source file has been defined, +# get the type here (FILEPATH or PATH): +if (DEFINED ${use_case}_FILE_PATH) + get_path_type(${${use_case}_FILE_PATH} PATH_TYPE) + + # Set the default type if path is not a dir or file path (or undefined) + if (NOT ${PATH_TYPE} STREQUAL PATH AND NOT ${PATH_TYPE} STREQUAL FILEPATH) + message(FATAL_ERROR "Invalid ${use_case}_FILE_PATH. It should be a dir or file path.") + endif() +else() + # Default is a directory path + set(PATH_TYPE PATH) +endif() + +message(STATUS "${use_case}_FILE_PATH is of type: ${PATH_TYPE}") +USER_OPTION(${use_case}_FILE_PATH "Directory with custom WAV input files, or path to a single WAV file, to use in the evaluation application." + ${CMAKE_CURRENT_SOURCE_DIR}/resources/${use_case}/samples/ + ${PATH_TYPE}) + +USER_OPTION(${use_case}_LABELS_TXT_FILE "Labels' txt file for the chosen model." + ${CMAKE_CURRENT_SOURCE_DIR}/resources/${use_case}/labels/ds_cnn_labels.txt + FILEPATH) + +USER_OPTION(${use_case}_AUDIO_RATE "Specify the target sampling rate. Default is 16000." + 16000 + STRING) + +USER_OPTION(${use_case}_AUDIO_MONO "Specify if the audio needs to be converted to mono. Default is ON." + ON + BOOL) + +USER_OPTION(${use_case}_AUDIO_OFFSET "Specify the offset to start reading after this time (in seconds). Default is 0." + 0 + STRING) + +USER_OPTION(${use_case}_AUDIO_DURATION "Specify the audio duration to load (in seconds). If set to 0 the entire audio will be processed." + 0 + STRING) + +USER_OPTION(${use_case}_AUDIO_RES_TYPE "Specify re-sampling algorithm to use. By default is 'kaiser_best'." + kaiser_best + STRING) + +USER_OPTION(${use_case}_AUDIO_MIN_SAMPLES "Specify the minimum number of samples to use. By default is 16000, if the audio is shorter will be automatically padded." + 16000 + STRING) + +USER_OPTION(${use_case}_MODEL_SCORE_THRESHOLD "Specify the score threshold [0.0, 1.0) that must be applied to the inference results for a label to be deemed valid." + 0.9 + STRING) + +# Generate input files +generate_audio_code(${${use_case}_FILE_PATH} ${SRC_GEN_DIR} ${INC_GEN_DIR} + ${${use_case}_AUDIO_RATE} + ${${use_case}_AUDIO_MONO} + ${${use_case}_AUDIO_OFFSET} + ${${use_case}_AUDIO_DURATION} + ${${use_case}_AUDIO_RES_TYPE} + ${${use_case}_AUDIO_MIN_SAMPLES}) + +# Generate labels file +set(${use_case}_LABELS_CPP_FILE Labels) +generate_labels_code( + INPUT "${${use_case}_LABELS_TXT_FILE}" + DESTINATION_SRC ${SRC_GEN_DIR} + DESTINATION_HDR ${INC_GEN_DIR} + OUTPUT_FILENAME "${${use_case}_LABELS_CPP_FILE}" +) + +USER_OPTION(${use_case}_ACTIVATION_BUF_SZ "Activation buffer size for the chosen model" + 0x00100000 + STRING) + +# If there is no tflite file pointed to +if (NOT DEFINED ${use_case}_MODEL_TFLITE_PATH) + + set(MODEL_FILENAME ds_cnn_clustered_int8.tflite) + set(MODEL_RESOURCES_DIR ${DOWNLOAD_DEP_DIR}/${use_case}) + file(MAKE_DIRECTORY ${MODEL_RESOURCES_DIR}) + set(DEFAULT_MODEL_PATH ${MODEL_RESOURCES_DIR}/${MODEL_FILENAME}) + + # Download the default model + set(ZOO_COMMON_SUBPATH "models/keyword_spotting/ds_cnn_large/tflite_clustered_int8") + set(ZOO_MODEL_SUBPATH "${ZOO_COMMON_SUBPATH}/${MODEL_FILENAME}") + + download_file_from_modelzoo(${ZOO_MODEL_SUBPATH} ${DEFAULT_MODEL_PATH}) + + if (ETHOS_U55_ENABLED) + message(STATUS + "Ethos-U55 is enabled, but the model downloaded is not optimized by vela. " + "To use Ethos-U55 acceleration, optimise the downloaded model and pass it " + "as ${use_case}_MODEL_TFLITE_PATH to the CMake configuration.") + endif() + + # If the target platform is native + if (${TARGET_PLATFORM} STREQUAL native) + + # Download test vectors + set(ZOO_TEST_IFM_SUBPATH "${ZOO_COMMON_SUBPATH}/testing_input/input_2/0.npy") + set(ZOO_TEST_OFM_SUBPATH "${ZOO_COMMON_SUBPATH}/testing_output/Identity/0.npy") + + set(${use_case}_TEST_IFM ${MODEL_RESOURCES_DIR}/ifm0.npy CACHE FILEPATH + "Input test vector for ${use_case}") + set(${use_case}_TEST_OFM ${MODEL_RESOURCES_DIR}/ofm0.npy CACHE FILEPATH + "Input test vector for ${use_case}") + + download_file_from_modelzoo(${ZOO_TEST_IFM_SUBPATH} ${${use_case}_TEST_IFM}) + download_file_from_modelzoo(${ZOO_TEST_OFM_SUBPATH} ${${use_case}_TEST_OFM}) + + set(TEST_SRC_GEN_DIR ${CMAKE_BINARY_DIR}/generated/${use_case}/tests/src) + set(TEST_INC_GEN_DIR ${CMAKE_BINARY_DIR}/generated/${use_case}/tests/include) + file(MAKE_DIRECTORY ${TEST_SRC_GEN_DIR} ${TEST_INC_GEN_DIR}) + + # Generate test data files to be included in x86 tests + generate_test_data_code( + INPUT_DIR "${DOWNLOAD_DEP_DIR}/${use_case}" + DESTINATION_SRC ${TEST_SRC_GEN_DIR} + DESTINATION_HDR ${TEST_INC_GEN_DIR} + USECASE "${use_case}") + endif() + +else() + set(DEFAULT_MODEL_PATH "N/A") +endif() + +set(EXTRA_MODEL_CODE + "/* Model parameters for ${use_case} */" + "extern const int g_FrameLength = 640" + "extern const int g_FrameStride = 320" + "extern const float g_ScoreThreshold = ${${use_case}_MODEL_SCORE_THRESHOLD}" + ) + +USER_OPTION(${use_case}_MODEL_TFLITE_PATH "NN models file to be used in the evaluation application. Model files must be in tflite format." + ${DEFAULT_MODEL_PATH} + FILEPATH) + +# Generate model file +generate_tflite_code( + MODEL_PATH ${${use_case}_MODEL_TFLITE_PATH} + DESTINATION ${SRC_GEN_DIR} + EXPRESSIONS ${EXTRA_MODEL_CODE} +) |