diff options
author | Richard Burton <richard.burton@arm.com> | 2022-05-04 09:45:02 +0100 |
---|---|---|
committer | Richard Burton <richard.burton@arm.com> | 2022-05-04 09:45:02 +0100 |
commit | 4e002791bc6781b549c6951cfe44f918289d7e82 (patch) | |
tree | b639243b5fa433657c207783a384bad1ed248536 /source/use_case/noise_reduction/include | |
parent | dd6d07b24bbf9023ebe8e8927be8aac3291d0f58 (diff) | |
download | ml-embedded-evaluation-kit-4e002791bc6781b549c6951cfe44f918289d7e82.tar.gz |
MLECO-3173: Add AD, KWS_ASR and Noise reduction use case API's
Signed-off-by: Richard Burton <richard.burton@arm.com>
Change-Id: I36f61ce74bf17f7b327cdae9704a22ca54144f37
Diffstat (limited to 'source/use_case/noise_reduction/include')
-rw-r--r-- | source/use_case/noise_reduction/include/RNNoiseFeatureProcessor.hpp (renamed from source/use_case/noise_reduction/include/RNNoiseProcess.hpp) | 16 | ||||
-rw-r--r-- | source/use_case/noise_reduction/include/RNNoiseProcessing.hpp | 113 |
2 files changed, 123 insertions, 6 deletions
diff --git a/source/use_case/noise_reduction/include/RNNoiseProcess.hpp b/source/use_case/noise_reduction/include/RNNoiseFeatureProcessor.hpp index c188e42..cbf0e4e 100644 --- a/source/use_case/noise_reduction/include/RNNoiseProcess.hpp +++ b/source/use_case/noise_reduction/include/RNNoiseFeatureProcessor.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. All rights reserved. + * Copyright (c) 2021-2022 Arm Limited. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#ifndef RNNOISE_FEATURE_PROCESSOR_HPP +#define RNNOISE_FEATURE_PROCESSOR_HPP + #include "PlatformMath.hpp" #include <cstdint> #include <vector> @@ -47,11 +50,11 @@ namespace rnn { * - https://jmvalin.ca/demo/rnnoise/ * - https://arxiv.org/abs/1709.08243 **/ - class RNNoiseProcess { + class RNNoiseFeatureProcessor { /* Public interface */ public: - RNNoiseProcess(); - ~RNNoiseProcess() = default; + RNNoiseFeatureProcessor(); + ~RNNoiseFeatureProcessor() = default; /** * @brief Calculates the features from a given audio buffer ready to be sent to RNNoise model. @@ -328,10 +331,11 @@ namespace rnn { const std::array <uint32_t, NB_BANDS> m_eband5ms { 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100}; - }; } /* namespace rnn */ -} /* namspace app */ +} /* namespace app */ } /* namespace arm */ + +#endif /* RNNOISE_FEATURE_PROCESSOR_HPP */ diff --git a/source/use_case/noise_reduction/include/RNNoiseProcessing.hpp b/source/use_case/noise_reduction/include/RNNoiseProcessing.hpp new file mode 100644 index 0000000..15e62d9 --- /dev/null +++ b/source/use_case/noise_reduction/include/RNNoiseProcessing.hpp @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2022 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RNNOISE_PROCESSING_HPP +#define RNNOISE_PROCESSING_HPP + +#include "BaseProcessing.hpp" +#include "Model.hpp" +#include "RNNoiseFeatureProcessor.hpp" + +namespace arm { +namespace app { + + /** + * @brief Pre-processing class for Noise Reduction use case. + * Implements methods declared by BasePreProcess and anything else needed + * to populate input tensors ready for inference. + */ + class RNNoisePreProcess : public BasePreProcess { + + public: + /** + * @brief Constructor + * @param[in] inputTensor Pointer to the TFLite Micro input Tensor. + * @param[in/out] featureProcessor RNNoise specific feature extractor object. + * @param[in/out] frameFeatures RNNoise specific features shared between pre & post-processing. + * + **/ + explicit RNNoisePreProcess(TfLiteTensor* inputTensor, + std::shared_ptr<rnn::RNNoiseFeatureProcessor> featureProcessor, + std::shared_ptr<rnn::FrameFeatures> frameFeatures); + + /** + * @brief Should perform pre-processing of 'raw' input audio data and load it into + * TFLite Micro input tensors ready for inference + * @param[in] input Pointer to the data that pre-processing will work on. + * @param[in] inputSize Size of the input data. + * @return true if successful, false otherwise. + **/ + bool DoPreProcess(const void* input, size_t inputSize) override; + + private: + TfLiteTensor* m_inputTensor; /* Model input tensor. */ + std::shared_ptr<rnn::RNNoiseFeatureProcessor> m_featureProcessor; /* RNNoise feature processor shared between pre & post-processing. */ + std::shared_ptr<rnn::FrameFeatures> m_frameFeatures; /* RNNoise features shared between pre & post-processing. */ + rnn::vec1D32F m_audioFrame; /* Audio frame cast to FP32 */ + + /** + * @brief Quantize the given features and populate the input Tensor. + * @param[in] inputFeatures Vector of floating point features to quantize. + * @param[in] quantScale Quantization scale for the inputTensor. + * @param[in] quantOffset Quantization offset for the inputTensor. + * @param[in,out] inputTensor TFLite micro tensor to populate. + **/ + static void QuantizeAndPopulateInput(rnn::vec1D32F& inputFeatures, + float quantScale, int quantOffset, + TfLiteTensor* inputTensor); + }; + + /** + * @brief Post-processing class for Noise Reduction use case. + * Implements methods declared by BasePostProcess and anything else needed + * to populate result vector. + */ + class RNNoisePostProcess : public BasePostProcess { + + public: + /** + * @brief Constructor + * @param[in] outputTensor Pointer to the TFLite Micro output Tensor. + * @param[out] denoisedAudioFrame Vector to store the final denoised audio frame. + * @param[in/out] featureProcessor RNNoise specific feature extractor object. + * @param[in/out] frameFeatures RNNoise specific features shared between pre & post-processing. + **/ + RNNoisePostProcess(TfLiteTensor* outputTensor, + std::vector<int16_t>& denoisedAudioFrame, + std::shared_ptr<rnn::RNNoiseFeatureProcessor> featureProcessor, + std::shared_ptr<rnn::FrameFeatures> frameFeatures); + + /** + * @brief Should perform post-processing of the result of inference then + * populate result data for any later use. + * @return true if successful, false otherwise. + **/ + bool DoPostProcess() override; + + private: + TfLiteTensor* m_outputTensor; /* Model output tensor. */ + std::vector<int16_t>& m_denoisedAudioFrame; /* Vector to store the final denoised frame. */ + rnn::vec1D32F m_denoisedAudioFrameFloat; /* Internal vector to store the final denoised frame (FP32). */ + std::shared_ptr<rnn::RNNoiseFeatureProcessor> m_featureProcessor; /* RNNoise feature processor shared between pre & post-processing. */ + std::shared_ptr<rnn::FrameFeatures> m_frameFeatures; /* RNNoise features shared between pre & post-processing. */ + std::vector<float> m_modelOutputFloat; /* Internal vector to store de-quantized model output. */ + + }; + +} /* namespace app */ +} /* namespace arm */ + +#endif /* RNNOISE_PROCESSING_HPP */
\ No newline at end of file |