diff options
author | Richard Burton <richard.burton@arm.com> | 2021-11-10 16:27:14 +0000 |
---|---|---|
committer | Richard <richard.burton@arm.com> | 2021-11-10 16:34:16 +0000 |
commit | 005534664e192cf909a11435c4bc4696b1f4c51f (patch) | |
tree | f8314bd284561e1f0ff68fc393ee22d0318ae162 /source/use_case/noise_reduction/include | |
parent | dee53bc7769d6201ec27deea4405c0df6c9b0623 (diff) | |
download | ml-embedded-evaluation-kit-005534664e192cf909a11435c4bc4696b1f4c51f.tar.gz |
MLECO-2354 MLECO-2355 MLECO-2356: Moving noise reduction to public repository
* Use RNNoise model from PMZ
* Add Noise reduction use-case
Signed-off-by: Richard burton <richard.burton@arm.com>
Change-Id: Ia8cc7ef102e22a5ff8bfbd3833594a4905a66057
Diffstat (limited to 'source/use_case/noise_reduction/include')
3 files changed, 516 insertions, 0 deletions
diff --git a/source/use_case/noise_reduction/include/RNNoiseModel.hpp b/source/use_case/noise_reduction/include/RNNoiseModel.hpp new file mode 100644 index 0000000..f6e4510 --- /dev/null +++ b/source/use_case/noise_reduction/include/RNNoiseModel.hpp @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RNNOISE_MODEL_HPP +#define RNNOISE_MODEL_HPP + +#include "Model.hpp" + +extern const uint32_t g_NumInputFeatures; +extern const uint32_t g_FrameLength; +extern const uint32_t g_FrameStride; + +namespace arm { +namespace app { + + class RNNoiseModel : public Model { + public: + /** + * @brief Runs inference for RNNoise model. + * + * Call CopyGruStates so GRU state outputs are copied to GRU state inputs before the inference run. + * Run ResetGruState() method to set states to zero before starting processing logically related data. + * @return True if inference succeeded, False - otherwise + */ + bool RunInference() override; + + /** + * @brief Sets GRU input states to zeros. + * Call this method before starting processing the new sequence of logically related data. + */ + void ResetGruState(); + + /** + * @brief Copy current GRU output states to input states. + * Call this method before starting processing the next sequence of logically related data. + */ + bool CopyGruStates(); + + /* Which index of model outputs does the main output (gains) come from. */ + const size_t m_indexForModelOutput = 1; + + protected: + /** @brief Gets the reference to op resolver interface class. */ + const tflite::MicroOpResolver& GetOpResolver() override; + + /** @brief Adds operations to the op resolver instance. */ + bool EnlistOperations() override; + + const uint8_t* ModelPointer() override; + + size_t ModelSize() override; + + /* + Each inference after the first needs to copy 3 GRU states from a output index to input index (model dependent): + 0 -> 3, 2 -> 2, 3 -> 1 + */ + const std::vector<std::pair<size_t, size_t>> m_gruStateMap = {{0,3}, {2, 2}, {3, 1}}; + private: + /* Maximum number of individual operations that can be enlisted. */ + static constexpr int ms_maxOpCnt = 15; + + /* A mutable op resolver instance. */ + tflite::MicroMutableOpResolver<ms_maxOpCnt> m_opResolver; + }; + +} /* namespace app */ +} /* namespace arm */ + +#endif /* RNNOISE_MODEL_HPP */
\ No newline at end of file diff --git a/source/use_case/noise_reduction/include/RNNoiseProcess.hpp b/source/use_case/noise_reduction/include/RNNoiseProcess.hpp new file mode 100644 index 0000000..3800019 --- /dev/null +++ b/source/use_case/noise_reduction/include/RNNoiseProcess.hpp @@ -0,0 +1,337 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "PlatformMath.hpp" +#include <cstdint> +#include <vector> +#include <array> +#include <tuple> + +namespace arm { +namespace app { +namespace rnn { + + using vec1D32F = std::vector<float>; + using vec2D32F = std::vector<vec1D32F>; + using arrHp = std::array<float, 2>; + using math::FftInstance; + using math::FftType; + + class FrameFeatures { + public: + bool m_silence{false}; /* If frame contains silence or not. */ + vec1D32F m_featuresVec{}; /* Calculated feature vector to feed to model. */ + vec1D32F m_fftX{}; /* Vector of floats arranged to represent complex numbers. */ + vec1D32F m_fftP{}; /* Vector of floats arranged to represent complex numbers. */ + vec1D32F m_Ex{}; /* Spectral band energy for audio x. */ + vec1D32F m_Ep{}; /* Spectral band energy for pitch p. */ + vec1D32F m_Exp{}; /* Correlated spectral energy between x and p. */ + }; + + /** + * @brief RNNoise pre and post processing class based on the 2018 paper from + * Jan-Marc Valin. Recommended reading: + * - https://jmvalin.ca/demo/rnnoise/ + * - https://arxiv.org/abs/1709.08243 + **/ + class RNNoiseProcess { + /* Public interface */ + public: + RNNoiseProcess(); + ~RNNoiseProcess() = default; + + /** + * @brief Calculates the features from a given audio buffer ready to be sent to RNNoise model. + * @param[in] audioData Pointer to the floating point vector + * with audio data (within the numerical + * limits of int16_t type). + * @param[in] audioLen Number of elements in the audio window. + * @param[out] features FrameFeatures object reference. + **/ + void PreprocessFrame(const float* audioData, + size_t audioLen, + FrameFeatures& features); + + /** + * @brief Use the RNNoise model output gain values with pre-processing features + * to generate audio with noise suppressed. + * @param[in] modelOutput Output gain values from model. + * @param[in] features Calculated features from pre-processing step. + * @param[out] outFrame Output frame to be populated. + **/ + void PostProcessFrame(vec1D32F& modelOutput, FrameFeatures& features, vec1D32F& outFrame); + + + /* Public constants */ + public: + static constexpr uint32_t FRAME_SIZE_SHIFT{2}; + static constexpr uint32_t FRAME_SIZE{480}; + static constexpr uint32_t WINDOW_SIZE{2 * FRAME_SIZE}; + static constexpr uint32_t FREQ_SIZE{FRAME_SIZE + 1}; + + static constexpr uint32_t PITCH_MIN_PERIOD{60}; + static constexpr uint32_t PITCH_MAX_PERIOD{768}; + static constexpr uint32_t PITCH_FRAME_SIZE{960}; + static constexpr uint32_t PITCH_BUF_SIZE{PITCH_MAX_PERIOD + PITCH_FRAME_SIZE}; + + static constexpr uint32_t NB_BANDS{22}; + static constexpr uint32_t CEPS_MEM{8}; + static constexpr uint32_t NB_DELTA_CEPS{6}; + + static constexpr uint32_t NB_FEATURES{NB_BANDS + 3*NB_DELTA_CEPS + 2}; + + /* Private functions */ + private: + + /** + * @brief Initialises the half window and DCT tables. + */ + void InitTables(); + + /** + * @brief Applies a bi-quadratic filter over the audio window. + * @param[in] bHp Constant coefficient set b (arrHp type). + * @param[in] aHp Constant coefficient set a (arrHp type). + * @param[in,out] memHpX Coefficients populated by this function. + * @param[in,out] audioWindow Floating point vector with audio data. + **/ + void BiQuad( + const arrHp& bHp, + const arrHp& aHp, + arrHp& memHpX, + vec1D32F& audioWindow); + + /** + * @brief Computes features from the "filtered" audio window. + * @param[in] audioWindow Floating point vector with audio data. + * @param[out] features FrameFeatures object reference. + **/ + void ComputeFrameFeatures(vec1D32F& audioWindow, FrameFeatures& features); + + /** + * @brief Runs analysis on the audio buffer. + * @param[in] audioWindow Floating point vector with audio data. + * @param[out] fft Floating point FFT vector containing real and + * imaginary pairs of elements. NOTE: this vector + * does not contain the mirror image (conjugates) + * part of the spectrum. + * @param[out] energy Computed energy for each band in the Bark scale. + * @param[out] analysisMem Buffer sequentially, but partially, + * populated with new audio data. + **/ + void FrameAnalysis( + const vec1D32F& audioWindow, + vec1D32F& fft, + vec1D32F& energy, + vec1D32F& analysisMem); + + /** + * @brief Applies the window function, in-place, over the given + * floating point buffer. + * @param[in,out] x Buffer the window will be applied to. + **/ + void ApplyWindow(vec1D32F& x); + + /** + * @brief Computes the FFT for a given vector. + * @param[in] x Vector to compute the FFT from. + * @param[out] fft Floating point FFT vector containing real and + * imaginary pairs of elements. NOTE: this vector + * does not contain the mirror image (conjugates) + * part of the spectrum. + **/ + void ForwardTransform( + vec1D32F& x, + vec1D32F& fft); + + /** + * @brief Computes band energy for each of the 22 Bark scale bands. + * @param[in] fft_X FFT spectrum (as computed by ForwardTransform). + * @param[out] bandE Vector with 22 elements populated with energy for + * each band. + **/ + void ComputeBandEnergy(const vec1D32F& fft_X, vec1D32F& bandE); + + /** + * @brief Computes band energy correlation. + * @param[in] X FFT vector X. + * @param[in] P FFT vector P. + * @param[out] bandC Vector with 22 elements populated with band energy + * correlation for the two input FFT vectors. + **/ + void ComputeBandCorr(const vec1D32F& X, const vec1D32F& P, vec1D32F& bandC); + + /** + * @brief Performs pitch auto-correlation for a given vector for + * given lag. + * @param[in] x Input vector. + * @param[out] ac Auto-correlation output vector. + * @param[in] lag Lag value. + * @param[in] n Number of elements to consider for correlation + * computation. + **/ + void AutoCorr(const vec1D32F &x, + vec1D32F &ac, + size_t lag, + size_t n); + + /** + * @brief Computes pitch cross-correlation. + * @param[in] x Input vector 1. + * @param[in] y Input vector 2. + * @param[out] ac Cross-correlation output vector. + * @param[in] len Number of elements to consider for correlation. + * computation. + * @param[in] maxPitch Maximum pitch. + **/ + void PitchXCorr( + const vec1D32F& x, + const vec1D32F& y, + vec1D32F& ac, + size_t len, + size_t maxPitch); + + /** + * @brief Computes "Linear Predictor Coefficients". + * @param[in] ac Correlation vector. + * @param[in] p Number of elements of input vector to consider. + * @param[out] lpc Output coefficients vector. + **/ + void LPC(const vec1D32F& ac, int32_t p, vec1D32F& lpc); + + /** + * @brief Custom FIR implementation. + * @param[in] num FIR coefficient vector. + * @param[in] N Number of elements. + * @param[out] x Vector to be be processed. + **/ + void Fir5(const vec1D32F& num, uint32_t N, vec1D32F& x); + + /** + * @brief Down-sample the pitch buffer. + * @param[in,out] pitchBuf Pitch buffer. + * @param[in] pitchBufSz Buffer size. + **/ + void PitchDownsample(vec1D32F& pitchBuf, size_t pitchBufSz); + + /** + * @brief Pitch search function. + * @param[in] xLP Shifted pitch buffer input. + * @param[in] y Pitch buffer input. + * @param[in] len Length to search for. + * @param[in] maxPitch Maximum pitch. + * @return pitch index. + **/ + int PitchSearch(vec1D32F& xLp, vec1D32F& y, uint32_t len, uint32_t maxPitch); + + /** + * @brief Finds the "best" pitch from the buffer. + * @param[in] xCorr Pitch correlation vector. + * @param[in] y Pitch buffer input. + * @param[in] len Length to search for. + * @param[in] maxPitch Maximum pitch. + * @return pitch array (2 elements). + **/ + arrHp FindBestPitch(vec1D32F& xCorr, vec1D32F& y, uint32_t len, uint32_t maxPitch); + + /** + * @brief Remove pitch period doubling errors. + * @param[in,out] pitchBuf Pitch buffer vector. + * @param[in] maxPeriod Maximum period. + * @param[in] minPeriod Minimum period. + * @param[in] frameSize Frame size. + * @param[in] pitchIdx0_ Pitch index 0. + * @return pitch index. + **/ + int RemoveDoubling( + vec1D32F& pitchBuf, + uint32_t maxPeriod, + uint32_t minPeriod, + uint32_t frameSize, + size_t pitchIdx0_); + + /** + * @brief Computes pitch gain. + * @param[in] xy Single xy cross correlation value. + * @param[in] xx Single xx auto correlation value. + * @param[in] yy Single yy auto correlation value. + * @return Calculated pitch gain. + **/ + float ComputePitchGain(float xy, float xx, float yy); + + /** + * @brief Computes DCT vector from the given input. + * @param[in] input Input vector. + * @param[out] output Output vector with DCT coefficients. + **/ + void DCT(vec1D32F& input, vec1D32F& output); + + /** + * @brief Perform inverse fourier transform on complex spectral vector. + * @param[out] out Output vector. + * @param[in] fftXIn Vector of floats arranged to represent complex numbers interleaved. + **/ + void InverseTransform(vec1D32F& out, vec1D32F& fftXIn); + + /** + * @brief Perform pitch filtering. + * @param[in] features Object with pre-processing calculated frame features. + * @param[in] g Gain values. + **/ + void PitchFilter(FrameFeatures& features, vec1D32F& g); + + /** + * @brief Interpolate the band gain values. + * @param[out] g Gain values. + * @param[in] bandE Vector with 22 elements populated with energy for + * each band. + **/ + void InterpBandGain(vec1D32F& g, vec1D32F& bandE); + + /** + * @brief Create de-noised frame. + * @param[out] outFrame Output vector for storing the created audio frame. + * @param[in] fftY Gain adjusted complex spectral vector. + */ + void FrameSynthesis(vec1D32F& outFrame, vec1D32F& fftY); + + /* Private objects */ + private: + FftInstance m_fftInstReal; /* FFT instance for real numbers */ + FftInstance m_fftInstCmplx; /* FFT instance for complex numbers */ + vec1D32F m_halfWindow; /* Window coefficients */ + vec1D32F m_dctTable; /* DCT table */ + vec1D32F m_analysisMem; /* Buffer used for frame analysis */ + vec2D32F m_cepstralMem; /* Cepstral coefficients */ + size_t m_memId; /* memory ID */ + vec1D32F m_synthesisMem; /* Synthesis mem (used by post-processing) */ + vec1D32F m_pitchBuf; /* Pitch buffer */ + float m_lastGain; /* Last gain calculated */ + int m_lastPeriod; /* Last period calculated */ + arrHp m_memHpX; /* HpX coefficients. */ + vec1D32F m_lastGVec; /* Last gain vector (used by post-processing) */ + + /* Constants */ + const std::array <uint32_t, NB_BANDS> m_eband5ms { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, + 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100}; + + }; + + +} /* namespace rnn */ +} /* namspace app */ +} /* namespace arm */ diff --git a/source/use_case/noise_reduction/include/UseCaseHandler.hpp b/source/use_case/noise_reduction/include/UseCaseHandler.hpp new file mode 100644 index 0000000..143f2ed --- /dev/null +++ b/source/use_case/noise_reduction/include/UseCaseHandler.hpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2021 Arm Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef NOISE_REDUCTION_EVT_HANDLER_HPP +#define NOISE_REDUCTION_EVT_HANDLER_HPP + +#include "AppContext.hpp" +#include "Model.hpp" + +namespace arm { +namespace app { + + /** + * @brief Handles the inference event for noise reduction. + * @param[in] ctx pointer to the application context + * @param[in] runAll flag to request classification of all the available audio clips + * @return True or false based on execution success + **/ + bool NoiseReductionHandler(ApplicationContext& ctx, bool runAll); + + /** + * @brief Dumps the output tensors to a memory address. + * This functionality is required for RNNoise use case as we want to + * save the inference output to a file. Dumping out tensors to a + * memory location will allow the Arm FVP or MPS3 to extract the + * contents of this memory location to a file. This file could then + * be used by an offline post-processing script. + * + * @param[in] model reference to a model + * @param[in] memAddress memory address at which the dump will start + * @param[in] memSize maximum size (in bytes) of the dump. + * + * @return number of bytes written to memory. + */ + size_t DumpOutputTensorsToMemory(Model& model, uint8_t* memAddress, + size_t memSize); + + /** + * @brief Dumps the audio file header. + * This functionality is required for RNNoise use case as we want to + * save the inference output to a file. Dumping out the header to a + * memory location will allow the Arm FVP or MPS3 to extract the + * contents of this memory location to a file. + * The header contains the following information + * int32_t filenameLength: filename length + * uint8_t[] filename: the string containing the file name (without trailing \0) + * int32_t dumpSizeByte: audiofile buffer size in bytes + * + * @param[in] filename the file name + * @param[in] dumpSize the size of the audio file (int elements) + * @param[in] memAddress memory address at which the dump will start + * @param[in] memSize maximum size (in bytes) of the dump. + * + * @return number of bytes written to memory. + */ + size_t DumpDenoisedAudioHeader(const char* filename, size_t dumpSize, + uint8_t* memAddress, size_t memSize); + + /** + * @brief Write a EOF marker at the end of the dump memory. + * + * @param[in] memAddress memory address at which the dump will start + * @param[in] memSize maximum size (in bytes) of the dump. + * + * @return number of bytes written to memory. + */ + size_t DumpDenoisedAudioFooter(uint8_t *memAddress, size_t memSize); + + /** + * @brief Dump the audio data to the memory + * + * @param[in] audioFrame The vector containg the audio data + * @param[in] memAddress memory address at which the dump will start + * @param[in] memSize maximum size (in bytes) of the dump. + * + * @return number of bytes written to memory. + */ + size_t DumpOutputDenoisedAudioFrame(const std::vector<int16_t> &audioFrame, + uint8_t *memAddress, size_t memSize); + +} /* namespace app */ +} /* namespace arm */ + +#endif /* NOISE_REDUCTION_EVT_HANDLER_HPP */
\ No newline at end of file |