/* * SPDX-FileCopyrightText: Copyright 2022 Arm Limited and/or its affiliates * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef AD_PROCESSING_HPP #define AD_PROCESSING_HPP #include "BaseProcessing.hpp" #include "TensorFlowLiteMicro.hpp" #include "AudioUtils.hpp" #include "AdMelSpectrogram.hpp" #include "log_macros.h" namespace arm { namespace app { /** * @brief Pre-processing class for anomaly detection use case. * Implements methods declared by BasePreProcess and anything else needed * to populate input tensors ready for inference. */ class AdPreProcess : public BasePreProcess { public: /** * @brief Constructor for AdPreProcess class objects * @param[in] inputTensor input tensor pointer from the tensor arena. * @param[in] melSpectrogramFrameLen MEL spectrogram's frame length * @param[in] melSpectrogramFrameStride MEL spectrogram's frame stride * @param[in] adModelTrainingMean Training mean for the Anomaly detection model being used. */ explicit AdPreProcess(TfLiteTensor* inputTensor, uint32_t melSpectrogramFrameLen, uint32_t melSpectrogramFrameStride, float adModelTrainingMean); ~AdPreProcess() = default; /** * @brief Function to invoke pre-processing and populate the input vector * @param input pointer to input data. For anomaly detection, this is the pointer to * the audio data. * @param inputSize Size of the data being passed in for pre-processing. * @return True if successful, false otherwise. */ bool DoPreProcess(const void* input, size_t inputSize) override; /** * @brief Getter function for audio window size computed when constructing * the class object. * @return Audio window size as 32 bit unsigned integer. */ uint32_t GetAudioWindowSize(); /** * @brief Getter function for audio window stride computed when constructing * the class object. * @return Audio window stride as 32 bit unsigned integer. */ uint32_t GetAudioDataStride(); /** * @brief Setter function for current audio index. This is only used for evaluating * if previously computed features can be re-used from cache. */ void SetAudioWindowIndex(uint32_t idx); private: bool m_validInstance{false}; /**< Indicates the current object is valid. */ uint32_t m_melSpectrogramFrameLen{}; /**< MEL spectrogram's window frame length */ uint32_t m_melSpectrogramFrameStride{}; /**< MEL spectrogram's window frame stride */ uint8_t m_inputResizeScale{}; /**< Downscaling factor for the MEL energy matrix. */ uint32_t m_numMelSpecVectorsInAudioStride{}; /**< Number of frames to move across the audio. */ uint32_t m_audioDataWindowSize{}; /**< Audio window size computed based on other parameters. */ uint32_t m_audioDataStride{}; /**< Audio window stride computed. */ uint32_t m_numReusedFeatureVectors{}; /**< Number of MEL vectors that can be re-used */ uint32_t m_audioWindowIndex{}; /**< Current audio window index (from audio's sliding window) */ audio::SlidingWindow m_melWindowSlider; /**< Internal MEL spectrogram window slider */ audio::AdMelSpectrogram m_melSpec; /**< MEL spectrogram computation object */ std::function&, int, bool, size_t, size_t)> m_featureCalc; /**< Feature calculator object */ }; class AdPostProcess : public BasePostProcess { public: /** * @brief Constructor for AdPostProcess object. * @param[in] outputTensor Output tensor pointer. */ explicit AdPostProcess(TfLiteTensor* outputTensor); ~AdPostProcess() = default; /** * @brief Function to do the post-processing on the output tensor. * @return True if successful, false otherwise. */ bool DoPostProcess() override; /** * @brief Getter function for an element from the de-quantised output vector. * @param index Index of the element to be retrieved. * @return index represented as a 32 bit floating point number. */ float GetOutputValue(uint32_t index); private: TfLiteTensor* m_outputTensor{}; /**< Output tensor pointer */ std::vector m_dequantizedOutputVec{}; /**< Internal output vector */ /** * @brief De-quantizes and flattens the output tensor into a vector. * @tparam T template parameter to indicate data type. * @return True if successful, false otherwise. */ template bool Dequantize() { TfLiteTensor* tensor = this->m_outputTensor; if (tensor == nullptr) { printf_err("Invalid output tensor.\n"); return false; } T* tensorData = tflite::GetTensorData(tensor); uint32_t totalOutputSize = 1; for (int inputDim = 0; inputDim < tensor->dims->size; inputDim++){ totalOutputSize *= tensor->dims->data[inputDim]; } /* For getting the floating point values, we need quantization parameters */ QuantParams quantParams = GetTensorQuantParams(tensor); this->m_dequantizedOutputVec = std::vector(totalOutputSize, 0); for (size_t i = 0; i < totalOutputSize; ++i) { this->m_dequantizedOutputVec[i] = quantParams.scale * (tensorData[i] - quantParams.offset); } return true; } }; /* Templated instances available: */ template bool AdPostProcess::Dequantize(); /** * @brief Generic feature calculator factory. * * Returns lambda function to compute features using features cache. * Real features math is done by a lambda function provided as a parameter. * Features are written to input tensor memory. * * @tparam T feature vector type. * @param inputTensor model input tensor pointer. * @param cacheSize number of feature vectors to cache. Defined by the sliding window overlap. * @param compute features calculator function. * @return lambda function to compute features. */ template std::function&, size_t, bool, size_t, size_t)> FeatureCalc(TfLiteTensor* inputTensor, size_t cacheSize, std::function (std::vector& )> compute) { /* Feature cache to be captured by lambda function*/ static std::vector> featureCache = std::vector>(cacheSize); return [=](std::vector& audioDataWindow, size_t index, bool useCache, size_t featuresOverlapIndex, size_t resizeScale) { T* tensorData = tflite::GetTensorData(inputTensor); std::vector features; /* Reuse features from cache if cache is ready and sliding windows overlap. * Overlap is in the beginning of sliding window with a size of a feature cache. */ if (useCache && index < featureCache.size()) { features = std::move(featureCache[index]); } else { features = std::move(compute(audioDataWindow)); } auto size = features.size() / resizeScale; auto sizeBytes = sizeof(T); /* Input should be transposed and "resized" by skipping elements. */ for (size_t outIndex = 0; outIndex < size; outIndex++) { std::memcpy(tensorData + (outIndex*size) + index, &features[outIndex*resizeScale], sizeBytes); } /* Start renewing cache as soon iteration goes out of the windows overlap. */ if (index >= featuresOverlapIndex / resizeScale) { featureCache[index - featuresOverlapIndex / resizeScale] = std::move(features); } }; } template std::function&, size_t , bool, size_t, size_t)> FeatureCalc(TfLiteTensor* inputTensor, size_t cacheSize, std::function (std::vector&)> compute); template std::function&, size_t, bool, size_t, size_t)> FeatureCalc(TfLiteTensor *inputTensor, size_t cacheSize, std::function(std::vector&)> compute); std::function&, int, bool, size_t, size_t)> GetFeatureCalculator(audio::AdMelSpectrogram& melSpec, TfLiteTensor* inputTensor, size_t cacheSize, float trainingMean); } /* namespace app */ } /* namespace arm */ #endif /* AD_PROCESSING_HPP */