diff options
Diffstat (limited to 'samples/KeywordSpotting')
| -rw-r--r-- | samples/KeywordSpotting/CMakeLists.txt | 64 | ||||
| -rw-r--r-- | samples/KeywordSpotting/Readme.md | 283 | ||||
| -rw-r--r-- | samples/KeywordSpotting/cmake/unit_tests.cmake | 65 | ||||
| -rw-r--r-- | samples/KeywordSpotting/include/Decoder.hpp | 32 | ||||
| -rw-r--r-- | samples/KeywordSpotting/include/DsCNNPreprocessor.hpp | 39 | ||||
| -rw-r--r-- | samples/KeywordSpotting/include/DsCnnMfcc.hpp | 20 | ||||
| -rw-r--r-- | samples/KeywordSpotting/include/KeywordSpottingPipeline.hpp | 91 | ||||
| -rw-r--r-- | samples/KeywordSpotting/src/Decoder.cpp | 35 | ||||
| -rw-r--r-- | samples/KeywordSpotting/src/DsCNNPreprocessor.cpp | 40 | ||||
| -rw-r--r-- | samples/KeywordSpotting/src/KeywordSpottingPipeline.cpp | 94 | ||||
| -rw-r--r-- | samples/KeywordSpotting/src/Main.cpp | 128 | ||||
| -rw-r--r-- | samples/KeywordSpotting/test/DecoderTest.cpp | 28 | ||||
| -rw-r--r-- | samples/KeywordSpotting/test/KeywordSpottingPipelineTest.cpp | 230 |
13 files changed, 1149 insertions, 0 deletions
diff --git a/samples/KeywordSpotting/CMakeLists.txt b/samples/KeywordSpotting/CMakeLists.txt new file mode 100644 index 0000000000..e8f263187f --- /dev/null +++ b/samples/KeywordSpotting/CMakeLists.txt @@ -0,0 +1,64 @@ +# Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +# SPDX-License-Identifier: MIT + +cmake_minimum_required(VERSION 3.0.2) + +set(CMAKE_C_STANDARD 99) +set(CMAKE_CXX_STANDARD 14) + +# Make the standard a requirement => prevent fallback to previous +# supported standard +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# We want to pass standard C/C++ flags, without gnu extensions +set(CMAKE_C_EXTENSIONS OFF) +set(CMAKE_CXX_EXTENSIONS OFF) + +project (keyword-spotting-example) + +set(CMAKE_C_FLAGS_DEBUG "-DDEBUG -O0 -g -fPIC -pthread") +set(CMAKE_C_FLAGS_RELEASE "-DNDEBUG -O3 -fPIC -pthread") + +set(CMAKE_CXX_FLAGS_DEBUG "-DDEBUG -O0 -g -fPIC -pthread") +set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3 -fPIC -pthread") + +include(ExternalProject) + +# Build in release mode by default +if (NOT CMAKE_BUILD_TYPE STREQUAL Debug) + set(CMAKE_BUILD_TYPE Release CACHE INTERNAL "") +endif() + +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +if (NOT DEFINED DEPENDENCIES_DIR) + set(DEPENDENCIES_DIR ${CMAKE_BINARY_DIR}/dependencies) +endif() + +include(../common/cmake/find_armnn.cmake) + +include_directories(include) +include_directories(../common/include/ArmnnUtils) +include_directories(../common/include/Utils) +include_directories(../common/include/Audio) + +file(GLOB SOURCES "src/*.cpp") +file(GLOB COMMON_UTILS_SOURCES "../common/src/Utils/*.cpp") +file(GLOB COMMON_AUDIO_SOURCES "../common/src/Audio/*.cpp") +list(REMOVE_ITEM SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/Main.cpp) +file(GLOB TEST_SOURCES "test/*.cpp") +file(GLOB APP_MAIN "src/Main.cpp") + +if(BUILD_UNIT_TESTS) + include(cmake/unit_tests.cmake) +endif() + +set(APP_TARGET_NAME "${CMAKE_PROJECT_NAME}") + +add_executable("${APP_TARGET_NAME}" ${COMMON_UTILS_SOURCES} ${COMMON_AUDIO_SOURCES} ${SOURCES} ${APP_MAIN}) + +target_link_libraries("${APP_TARGET_NAME}" PUBLIC ${ARMNN_LIBS} -lsndfile -lsamplerate) +target_include_directories("${APP_TARGET_NAME}" PUBLIC ${ARMNN_INCLUDE_DIR} ) diff --git a/samples/KeywordSpotting/Readme.md b/samples/KeywordSpotting/Readme.md new file mode 100644 index 0000000000..914d9844b0 --- /dev/null +++ b/samples/KeywordSpotting/Readme.md @@ -0,0 +1,283 @@ +# Keyword Spotting Example + +## Introduction + +This is a sample code showing keyword spotting using Arm NN public C++ API. The compiled application can take + +* an audio file + +as input and produce + +* recognised keyword in the audio file + +as output. The application works with the [fully quantised DS CNN Large model](https://github.com/ARM-software/ML-zoo/raw/68b5fbc77ed28e67b2efc915997ea4477c1d9d5b/models/keyword_spotting/ds_cnn_large/tflite_clustered_int8/) which is trained to recongize 12 keywords, including an unknown word. + +## Dependencies + +This example utilises `libsndfile`, `libasound` and `libsamplerate` libraries to capture the raw audio data from file, and to re-sample to the expected sample rate. Top level inference API is provided by Arm NN library. + +### Arm NN + +Keyword spotting example build system does not trigger Arm NN compilation. Thus, before building the application, +please ensure that Arm NN libraries and header files are available on your build platform. +The application executable binary dynamically links with the following Arm NN libraries: + +* libarmnn.so +* libarmnnTfLiteParser.so + +The build script searches for available Arm NN libraries in the following order: + +1. Inside custom user directory specified by ARMNN_LIB_DIR cmake option. +2. Inside the current Arm NN repository, assuming that Arm NN was built following [these instructions](../../BuildGuideCrossCompilation.md). +3. Inside default locations for system libraries, assuming Arm NN was installed from deb packages. + +Arm NN header files will be searched in parent directory of found libraries files under `include` directory, i.e. +libraries found in `/usr/lib` or `/usr/lib64` and header files in `/usr/include` (or `${ARMNN_LIB_DIR}/include`). + +Please see [find_armnn.cmake](./cmake/find_armnn.cmake) for implementation details. + +## Building + +There is one flow for building this application: + +* native build on a host platform + +### Build Options + +* ARMNN_LIB_DIR - point to the custom location of the Arm NN libs and headers. +* BUILD_UNIT_TESTS - set to `1` to build tests. Additionally to the main application, `keyword-spotting-example-tests` +unit tests executable will be created. + +### Native Build + +To build this application on a host platform, firstly ensure that required dependencies are installed: +For example, for raspberry PI: + +```commandline +sudo apt-get update +sudo apt-get -yq install libsndfile1-dev +sudo apt-get -yq install libasound2-dev +sudo apt-get -yq install libsamplerate-dev +``` + +To build demo application, create a build directory: + +```commandline +mkdir build +cd build +``` + +If you have already installed Arm NN and and the required libraries: + +Inside build directory, run cmake and make commands: + +```commandline +cmake .. +make +``` + +This will build the following in bin directory: + +* `keyword-spotting-example` - application executable + +If you have custom Arm NN location, use `ARMNN_LIB_DIR` options: + +```commandline +cmake -DARMNN_LIB_DIR=/path/to/armnn .. +make +``` + +## Executing + +Once the application executable is built, it can be executed with the following options: + +* --audio-file-path: Path to the audio file to run keyword spotting on **[REQUIRED]** +* --model-file-path: Path to the Keyword Spotting model to use **[REQUIRED]** + +* --preferred-backends: Takes the preferred backends in preference order, separated by comma. + For example: `CpuAcc,GpuAcc,CpuRef`. Accepted options: [`CpuAcc`, `CpuRef`, `GpuAcc`]. + Defaults to `CpuRef` **[OPTIONAL]** + +### Keyword Spotting on a supplied audio file + +A small selection of suitable wav files containing keywords can be found [here](https://git.mlplatform.org/ml/ethos-u/ml-embedded-evaluation-kit.git/plain/resources/kws/samples/). +To run keyword spotting on a supplied audio file and output the result to console: + +```commandline +./keyword-spotting-example --audio-file-path /path/to/audio/file --model-file-path /path/to/model/file +``` + +# Application Overview + +This section provides a walkthrough of the application, explaining in detail the steps: + +1. Initialisation + 1. Reading from Audio Source +2. Creating a Network + 1. Creating Parser and Importing Graph + 2. Optimizing Graph for Compute Device + 3. Creating Input and Output Binding Information +3. Keyword spotting pipeline + 1. Pre-processing the Captured Audio + 2. Making Input and Output Tensors + 3. Executing Inference + 4. Postprocessing + 5. Decoding and Processing Inference Output + +### Initialisation + +##### Reading from Audio Source + +After parsing user arguments, the chosen audio file is loaded into an AudioCapture object. +We use [`AudioCapture`](./include/AudioCapture.hpp) in our main function to capture appropriately sized audio blocks from the source using the +`Next()` function. + +The `AudioCapture` object also re-samples the audio input to a desired sample rate, and sets the number of channels used to one channel (i.e `mono`) + +### Creating a Network + +All operations with Arm NN and networks are encapsulated in [`ArmnnNetworkExecutor`](./include/ArmnnNetworkExecutor.hpp) +class. + +##### Creating Parser and Importing Graph + +The first step with Arm NN SDK is to import a graph from file by using the appropriate parser. + +The Arm NN SDK provides parsers for reading graphs from a variety of model formats. In our application we specifically +focus on `.tflite, .pb, .onnx` models. + +Based on the extension of the provided model file, the corresponding parser is created and the network file loaded with +`CreateNetworkFromBinaryFile()` method. The parser will handle the creation of the underlying Arm NN graph. + +Currently this example only supports tflite format model files and uses `ITfLiteParser`: + +```c++ +#include "armnnTfLiteParser/ITfLiteParser.hpp" + +armnnTfLiteParser::ITfLiteParserPtr parser = armnnTfLiteParser::ITfLiteParser::Create(); +armnn::INetworkPtr network = parser->CreateNetworkFromBinaryFile(modelPath.c_str()); +``` + +##### Optimizing Graph for Compute Device + +Arm NN supports optimized execution on multiple CPU and GPU devices. Prior to executing a graph, we must select the +appropriate device context. We do this by creating a runtime context with default options with `IRuntime()`. + +For example: + +```c++ +#include "armnn/ArmNN.hpp" + +auto runtime = armnn::IRuntime::Create(armnn::IRuntime::CreationOptions()); +``` + +We can optimize the imported graph by specifying a list of backends in order of preference and implement +backend-specific optimizations. The backends are identified by a string unique to the backend, +for example `CpuAcc, GpuAcc, CpuRef`. + +For example: + +```c++ +std::vector<armnn::BackendId> backends{"CpuAcc", "GpuAcc", "CpuRef"}; +``` + +Internally and transparently, Arm NN splits the graph into subgraph based on backends, it calls a optimize subgraphs +function on each of them and, if possible, substitutes the corresponding subgraph in the original graph with +its optimized version. + +Using the `Optimize()` function we optimize the graph for inference and load the optimized network onto the compute +device with `LoadNetwork()`. This function creates the backend-specific workloads +for the layers and a backend specific workload factory which is called to create the workloads. + +For example: + +```c++ +armnn::IOptimizedNetworkPtr optNet = Optimize(*network, + backends, + m_Runtime->GetDeviceSpec(), + armnn::OptimizerOptions()); +std::string errorMessage; +runtime->LoadNetwork(0, std::move(optNet), errorMessage)); +std::cerr << errorMessage << std::endl; +``` + +##### Creating Input and Output Binding Information + +Parsers can also be used to extract the input information for the network. By calling `GetSubgraphInputTensorNames` +we extract all the input names and, with `GetNetworkInputBindingInfo`, bind the input points of the graph. +For example: + +```c++ +std::vector<std::string> inputNames = parser->GetSubgraphInputTensorNames(0); +auto inputBindingInfo = parser->GetNetworkInputBindingInfo(0, inputNames[0]); +``` + +The input binding information contains all the essential information about the input. It is a tuple consisting of +integer identifiers for bindable layers (inputs, outputs) and the tensor info (data type, quantization information, +number of dimensions, total number of elements). + +Similarly, we can get the output binding information for an output layer by using the parser to retrieve output +tensor names and calling `GetNetworkOutputBindingInfo()`. + +### Keyword Spotting pipeline + +The keyword spotting pipeline has 3 steps to perform: data pre-processing, run inference and decode inference results. + +See [`KeywordSpottingPipeline`](include/KeywordSpottingPipeline.hpp) for more details. + +#### Pre-processing the Audio Input + +Each frame captured from source is read and stored by the AudioCapture object. +It's `Next()` function provides us with the correctly positioned window of data, sized appropriately for the given model, to pre-process before inference. + +```c++ +std::vector<float> audioBlock = capture.Next(); +... +std::vector<int8_t> preprocessedData = kwsPipeline->PreProcessing(audioBlock); +``` + +The `MFCC` class is then used to extract the Mel-frequency Cepstral Coefficients (MFCCs, [see Wikipedia](https://en.wikipedia.org/wiki/Mel-frequency_cepstrum)) from each stored audio frame in the provided window of audio, to be used as features for the network. MFCCs are the result of computing the dot product of the Discrete Cosine Transform (DCT) Matrix and the log of the Mel energy. + +After all the MFCCs needed for an inference have been extracted from the audio data they are concatenated to make the input tensor for the model. + +#### Executing Inference + +```c++ +common::InferenceResults results; +... +kwsPipeline->Inference(preprocessedData, results); +``` + +Inference step will call `ArmnnNetworkExecutor::Run` method that will prepare input tensors and execute inference. +A compute device performs inference for the loaded network using the `EnqueueWorkload()` function of the runtime context. +For example: + +```c++ +//const void* inputData = ...; +//outputTensors were pre-allocated before + +armnn::InputTensors inputTensors = {{ inputBindingInfo.first,armnn::ConstTensor(inputBindingInfo.second, inputData)}}; +runtime->EnqueueWorkload(0, inputTensors, outputTensors); +``` + +We allocate memory for output data once and map it to output tensor objects. After successful inference, we read data +from the pre-allocated output data buffer. See [`ArmnnNetworkExecutor::ArmnnNetworkExecutor`](./src/ArmnnNetworkExecutor.cpp) +and [`ArmnnNetworkExecutor::Run`](./src/ArmnnNetworkExecutor.cpp) for more details. + +#### Postprocessing + +##### Decoding + +The output from the inference is decoded to obtain the spotted keyword- the word with highest probability is outputted to the console. + +```c++ +kwsPipeline->PostProcessing(results, labels, + [](int index, std::string& label, float prob) -> void { + printf("Keyword \"%s\", index %d:, probability %f\n", + label.c_str(), + index, + prob); + }); +``` + +The produced string is displayed on the console. diff --git a/samples/KeywordSpotting/cmake/unit_tests.cmake b/samples/KeywordSpotting/cmake/unit_tests.cmake new file mode 100644 index 0000000000..97ba4d41b3 --- /dev/null +++ b/samples/KeywordSpotting/cmake/unit_tests.cmake @@ -0,0 +1,65 @@ +# Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +# SPDX-License-Identifier: MIT + +# Function to download a file from the Arm Model Zoo +function(download_file_from_modelzoo model_zoo_version file_sub_path download_path) + set(MODEL_ZOO_REPO "https://github.com/ARM-software/ML-zoo/raw") + string(JOIN "/" FILE_URL + ${MODEL_ZOO_REPO} ${model_zoo_version} ${file_sub_path}) + message(STATUS "Downloading ${FILE_URL} to ${download_path}...") + file(DOWNLOAD ${FILE_URL} ${download_path} + STATUS DOWNLOAD_STATE) + list(GET DOWNLOAD_STATE 0 RET_VAL) + if(${RET_VAL}) + list(GET DOWNLOAD_STATE 1 RET_MSG) + message(FATAL_ERROR "Download failed with error code: ${RET_VAL}; " + "Error message: ${RET_MSG}") + endif() +endfunction() + +set(TEST_RESOURCES_DIR ${CMAKE_SOURCE_DIR}/test/resources) +file(MAKE_DIRECTORY ${TEST_RESOURCES_DIR}) +add_definitions (-DTEST_RESOURCE_DIR="${TEST_RESOURCES_DIR}") +set(TEST_TARGET_NAME "${CMAKE_PROJECT_NAME}-tests") + +file(GLOB TEST_SOURCES "test/*") +file(GLOB TESTS_AUDIO_COMMON "../common/test/Audio/*") + +file(MAKE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test/resources) +include(../common/cmake/find_catch.cmake) + +add_executable("${TEST_TARGET_NAME}" ${COMMON_UTILS_SOURCES} ${COMMON_AUDIO_SOURCES} ${SOURCES} ${TEST_SOURCES} ${TESTS_AUDIO_COMMON}) + +ExternalProject_Add(passport + URL https://raw.githubusercontent.com/Azure-Samples/cognitive-services-speech-sdk/master/sampledata/audiofiles/myVoiceIsMyPassportVerifyMe04.wav + DOWNLOAD_NO_EXTRACT 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND ${CMAKE_COMMAND} -E copy <DOWNLOAD_DIR>/myVoiceIsMyPassportVerifyMe04.wav ${CMAKE_CURRENT_SOURCE_DIR}/test/resources + INSTALL_COMMAND "" + ) + +add_dependencies( + "${TEST_TARGET_NAME}" + "passport" + "catch2-headers" +) + + +set(MODEL_FILENAME ds_cnn_clustered_int8.tflite) +set(MODEL_RESOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/test/resources) +file(MAKE_DIRECTORY ${MODEL_RESOURCES_DIR}) +set(DEFAULT_MODEL_PATH ${CMAKE_CURRENT_SOURCE_DIR}/test/resources/${MODEL_FILENAME}) + +# Download the default model +set(ZOO_COMMON_SUBPATH "models/keyword_spotting/ds_cnn_large/tflite_clustered_int8") +set(ZOO_MODEL_SUBPATH "${ZOO_COMMON_SUBPATH}/${MODEL_FILENAME}") +set(ZOO_MODEL_VERSION "68b5fbc77ed28e67b2efc915997ea4477c1d9d5b") + +download_file_from_modelzoo(${ZOO_MODEL_VERSION} ${ZOO_MODEL_SUBPATH} ${DEFAULT_MODEL_PATH}) + + +target_include_directories("${TEST_TARGET_NAME}" PUBLIC ${TEST_TPIP_INCLUDE} + ${ARMNN_INCLUDE_DIR} + ${DEPENDENCIES_DIR} ${TEST_RESOURCES_DIR} ${COMMON_INCLUDE_DIR}) + +target_link_libraries("${TEST_TARGET_NAME}" PUBLIC ${ARMNN_LIBS} -lsndfile -lsamplerate)
\ No newline at end of file diff --git a/samples/KeywordSpotting/include/Decoder.hpp b/samples/KeywordSpotting/include/Decoder.hpp new file mode 100644 index 0000000000..aca68312bc --- /dev/null +++ b/samples/KeywordSpotting/include/Decoder.hpp @@ -0,0 +1,32 @@ +// +// Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// +# pragma once + +#include <string> +#include <map> +#include "ArmnnNetworkExecutor.hpp" + +namespace kws +{ + +/** +* @brief Decodes quantised last layer of model output +* +*/ +class Decoder +{ +private: + int quantisationOffset; + float quantisationScale; + +public: + + Decoder(int quantisationOffset, float quantisationScale) : quantisationOffset(quantisationOffset), + quantisationScale(quantisationScale) {} + + std::pair<int, float> decodeOutput(std::vector<int8_t>& modelOutput); + +}; +} // namespace kws
\ No newline at end of file diff --git a/samples/KeywordSpotting/include/DsCNNPreprocessor.hpp b/samples/KeywordSpotting/include/DsCNNPreprocessor.hpp new file mode 100644 index 0000000000..b635d1a41e --- /dev/null +++ b/samples/KeywordSpotting/include/DsCNNPreprocessor.hpp @@ -0,0 +1,39 @@ +// +// Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// +#ifndef KEYWORD_SPOTTING_EXAMPLE_DSCNNPREPROCESSOR_HPP +#define KEYWORD_SPOTTING_EXAMPLE_DSCNNPREPROCESSOR_HPP + +#include <numeric> +#include "DsCnnMfcc.hpp" + +namespace kws +{ +class DsCNNPreprocessor +{ +public: + DsCNNPreprocessor(uint32_t windowLen, uint32_t windowStride, + std::unique_ptr<DsCnnMFCC> mfccInst); + + /** + * @brief Calculates the features required from audio data. This + * includes MFCC, first and second order deltas, + * normalisation and finally, quantisation. The tensor is + * populated with feature from a given window placed along + * in a single row. + * @param[in] audioData pointer to the first element of audio data + * @param[in] output output to be populated + * @return true if successful, false in case of error. + */ + std::vector<int8_t> Invoke(const float* audioData, + size_t dataSize, + int quantOffset, + float quantScale) ; + + uint32_t m_windowLen; // Window length for MFCC + uint32_t m_windowStride; // Window stride len for MFCC + std::unique_ptr<MFCC> m_mfcc; +}; +} // namespace kws +#endif //KEYWORD_SPOTTING_EXAMPLE_DSCNNPREPROCESSOR_HPP diff --git a/samples/KeywordSpotting/include/DsCnnMfcc.hpp b/samples/KeywordSpotting/include/DsCnnMfcc.hpp new file mode 100644 index 0000000000..851e010b22 --- /dev/null +++ b/samples/KeywordSpotting/include/DsCnnMfcc.hpp @@ -0,0 +1,20 @@ +// +// Copyright © 2020 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// +#pragma once + +#include "MFCC.hpp" + +/* Class to provide DS-CNN specific MFCC calculation requirements. */ +class DsCnnMFCC : public MFCC +{ + +public: + + explicit DsCnnMFCC(MfccParams& params) + : MFCC(params) + {} + DsCnnMFCC() = delete; + ~DsCnnMFCC() = default; +}; diff --git a/samples/KeywordSpotting/include/KeywordSpottingPipeline.hpp b/samples/KeywordSpotting/include/KeywordSpottingPipeline.hpp new file mode 100644 index 0000000000..bd47987a59 --- /dev/null +++ b/samples/KeywordSpotting/include/KeywordSpottingPipeline.hpp @@ -0,0 +1,91 @@ +// +// Copyright © 2020 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include "ArmnnNetworkExecutor.hpp" +#include "Decoder.hpp" +#include "MFCC.hpp" +#include "DsCNNPreprocessor.hpp" + +namespace kws +{ +/** + * Generic Keyword Spotting pipeline with 3 steps: data pre-processing, inference execution and inference + * result post-processing. + * + */ +class KWSPipeline +{ +public: + + /** + * Creates speech recognition pipeline with given network executor and decoder. + * @param executor - unique pointer to inference runner + * @param decoder - unique pointer to inference results decoder + */ + KWSPipeline(std::unique_ptr<common::ArmnnNetworkExecutor<int8_t>> executor, + std::unique_ptr<Decoder> decoder, + std::unique_ptr<DsCNNPreprocessor> preProcessor); + + /** + * @brief Standard audio pre-processing implementation. + * + * Preprocesses and prepares the data for inference by + * extracting the MFCC features. + + * @param[in] audio - the raw audio data + */ + + std::vector<int8_t> PreProcessing(std::vector<float>& audio); + + /** + * @brief Executes inference + * + * Calls inference runner provided during instance construction. + * + * @param[in] preprocessedData - input inference data. Data type should be aligned with input tensor. + * @param[out] result - raw inference results. + */ + void Inference(const std::vector<int8_t>& preprocessedData, common::InferenceResults<int8_t>& result); + + /** + * @brief Standard inference results post-processing implementation. + * + * Decodes inference results using decoder provided during construction. + * + * @param[in] inferenceResult - inference results to be decoded. + * @param[in] labels - the words we use for the model + */ + void PostProcessing(common::InferenceResults<int8_t>& inferenceResults, + std::map<int, std::string>& labels, + const std::function<void (int, std::string&, float)>& callback); + + /** + * @brief Get the number of samples for the pipeline input + + * @return - number of samples for the pipeline + */ + int getInputSamplesSize(); + +protected: + std::unique_ptr<common::ArmnnNetworkExecutor<int8_t>> m_executor; + std::unique_ptr<Decoder> m_decoder; + std::unique_ptr<DsCNNPreprocessor> m_preProcessor; +}; + +using IPipelinePtr = std::unique_ptr<kws::KWSPipeline>; + +/** + * Constructs speech recognition pipeline based on configuration provided. + * + * @param[in] config - speech recognition pipeline configuration. + * @param[in] labels - asr labels + * + * @return unique pointer to asr pipeline. + */ +IPipelinePtr CreatePipeline(common::PipelineOptions& config); + +};// namespace kws
\ No newline at end of file diff --git a/samples/KeywordSpotting/src/Decoder.cpp b/samples/KeywordSpotting/src/Decoder.cpp new file mode 100644 index 0000000000..107e25caa9 --- /dev/null +++ b/samples/KeywordSpotting/src/Decoder.cpp @@ -0,0 +1,35 @@ +// +// Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include "Decoder.hpp" + +std::pair<int, float> kws::Decoder::decodeOutput(std::vector<int8_t>& modelOutput) +{ + + std::vector<float> dequantisedOutput; + //Normalise vector values into new vector + for (auto& value : modelOutput) + { + float normalisedModelOutput = this->quantisationScale * (static_cast<float >(value) - + static_cast<float >(this->quantisationOffset)); + dequantisedOutput.push_back(normalisedModelOutput); + } + + //Get largest value in modelOutput + const std::vector<float>::iterator& maxElementIterator = std::max_element(dequantisedOutput.begin(), + dequantisedOutput.end()); + //Find the labelMapIndex of the largest value which corresponds to a key in a label map + int labelMapIndex = static_cast<int>(std::distance(dequantisedOutput.begin(), maxElementIterator)); + + //Round to two DP + float maxModelOutputProbability = std::roundf((*maxElementIterator) * 100) / 100; + + return std::make_pair(labelMapIndex, maxModelOutputProbability); + +} + + + + diff --git a/samples/KeywordSpotting/src/DsCNNPreprocessor.cpp b/samples/KeywordSpotting/src/DsCNNPreprocessor.cpp new file mode 100644 index 0000000000..8215feeeb5 --- /dev/null +++ b/samples/KeywordSpotting/src/DsCNNPreprocessor.cpp @@ -0,0 +1,40 @@ +// +// Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// +#include <cmath> +#include <numeric> +#include <algorithm> +#include <memory> +#include "MathUtils.hpp" +#include "SlidingWindow.hpp" +#include "DsCNNPreprocessor.hpp" + +std::vector<int8_t> kws::DsCNNPreprocessor::Invoke(const float* audioData, size_t dataSize, + int quantOffset, float quantScale) +{ + auto window = SlidingWindow<const float>( + audioData, dataSize, + this->m_windowLen, this->m_windowStride); + + uint32_t mfccBufIdx = 0; + std::vector<int8_t> outputBuffer; + // While we can slide over the window + while (window.HasNext()) + { + const float* mfccWindow = window.Next(); + auto mfccAudioData = std::vector<float>(mfccWindow, mfccWindow + this->m_windowLen); + + auto mfcc = this->m_mfcc->MfccComputeQuant<int8_t>(mfccAudioData, quantScale, quantOffset); + + std::copy(mfcc.begin(), mfcc.end(), std::back_inserter(outputBuffer)); + + ++mfccBufIdx; + } + + return outputBuffer; +} + +kws::DsCNNPreprocessor::DsCNNPreprocessor(const uint32_t windowLen, const uint32_t windowStride, + std::unique_ptr<DsCnnMFCC> mfccInst) : + m_windowLen{windowLen}, m_windowStride{windowStride}, m_mfcc{std::move(mfccInst)} {} diff --git a/samples/KeywordSpotting/src/KeywordSpottingPipeline.cpp b/samples/KeywordSpotting/src/KeywordSpottingPipeline.cpp new file mode 100644 index 0000000000..e32d9476e3 --- /dev/null +++ b/samples/KeywordSpotting/src/KeywordSpottingPipeline.cpp @@ -0,0 +1,94 @@ +// +// Copyright © 2020 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include "KeywordSpottingPipeline.hpp" +#include "ArmnnNetworkExecutor.hpp" +#include "DsCNNPreprocessor.hpp" + +namespace kws +{ +KWSPipeline::KWSPipeline(std::unique_ptr<common::ArmnnNetworkExecutor<int8_t>> executor, + std::unique_ptr<Decoder> decoder, + std::unique_ptr<DsCNNPreprocessor> preProcessor + ) : + m_executor(std::move(executor)), + m_decoder(std::move(decoder)), + m_preProcessor(std::move(preProcessor)) {} + + +std::vector<int8_t> KWSPipeline::PreProcessing(std::vector<float>& audio) +{ + return m_preProcessor->Invoke(audio.data(), audio.size(), m_executor->GetQuantizationOffset(), + m_executor->GetQuantizationScale()); +} + +void KWSPipeline::Inference(const std::vector<int8_t>& preprocessedData, + common::InferenceResults<int8_t>& result) +{ + m_executor->Run(preprocessedData.data(), preprocessedData.size(), result); +} + +void KWSPipeline::PostProcessing(common::InferenceResults<int8_t>& inferenceResults, + std::map<int, std::string>& labels, + const std::function<void (int, std::string&, float)>& callback) +{ + std::pair<int,float> outputDecoder = this->m_decoder->decodeOutput(inferenceResults[0]); + int keywordIndex = std::get<0>(outputDecoder); + std::string output = labels[keywordIndex]; + callback(keywordIndex, output, std::get<1>(outputDecoder)); +} + +int KWSPipeline::getInputSamplesSize() +{ + return this->m_preProcessor->m_windowLen + + ((this->m_preProcessor->m_mfcc->m_params.m_numMfccVectors - 1) * + this->m_preProcessor->m_windowStride); +} + +IPipelinePtr CreatePipeline(common::PipelineOptions& config) +{ + if (config.m_ModelName == "DS_CNN_CLUSTERED_INT8") + { + //DS-CNN model settings + float SAMP_FREQ = 16000; + int MFCC_WINDOW_LEN = 640; + int MFCC_WINDOW_STRIDE = 320; + int NUM_MFCC_FEATS = 10; + int NUM_MFCC_VECTORS = 49; + //todo: calc in pipeline and use in main + int SAMPLES_PER_INFERENCE = NUM_MFCC_VECTORS * MFCC_WINDOW_STRIDE + + MFCC_WINDOW_LEN - MFCC_WINDOW_STRIDE; //16000 + float MEL_LO_FREQ = 20; + float MEL_HI_FREQ = 4000; + int NUM_FBANK_BIN = 40; + + MfccParams mfccParams(SAMP_FREQ, + NUM_FBANK_BIN, + MEL_LO_FREQ, + MEL_HI_FREQ, + NUM_MFCC_FEATS, + MFCC_WINDOW_LEN, false, + NUM_MFCC_VECTORS); + + std::unique_ptr<DsCnnMFCC> mfccInst = std::make_unique<DsCnnMFCC>(mfccParams); + auto preprocessor = std::make_unique<kws::DsCNNPreprocessor>( + MFCC_WINDOW_LEN, MFCC_WINDOW_STRIDE, std::move(mfccInst)); + + auto executor = std::make_unique<common::ArmnnNetworkExecutor<int8_t>>( + config.m_ModelFilePath, config.m_backends); + + auto decoder = std::make_unique<kws::Decoder>(executor->GetOutputQuantizationOffset(0), + executor->GetOutputQuantizationScale(0)); + + return std::make_unique<kws::KWSPipeline>(std::move(executor), + std::move(decoder), std::move(preprocessor)); + } + else + { + throw std::invalid_argument("Unknown Model name: " + config.m_ModelName + " ."); + } +} + +};// namespace kws
\ No newline at end of file diff --git a/samples/KeywordSpotting/src/Main.cpp b/samples/KeywordSpotting/src/Main.cpp new file mode 100644 index 0000000000..10efcd8ce7 --- /dev/null +++ b/samples/KeywordSpotting/src/Main.cpp @@ -0,0 +1,128 @@ +// +// Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// +#include <iostream> +#include <map> +#include <vector> +#include <algorithm> +#include <cmath> +#include "KeywordSpottingPipeline.hpp" +#include "CmdArgsParser.hpp" +#include "ArmnnNetworkExecutor.hpp" +#include "AudioCapture.hpp" + +const std::string AUDIO_FILE_PATH = "--audio-file-path"; +const std::string MODEL_FILE_PATH = "--model-file-path"; +const std::string LABEL_PATH = "--label-path"; +const std::string PREFERRED_BACKENDS = "--preferred-backends"; +const std::string HELP = "--help"; + +/* + * The accepted options for this Speech Recognition executable + */ +static std::map<std::string, std::string> CMD_OPTIONS = +{ + {AUDIO_FILE_PATH, "[REQUIRED] Path to the Audio file to run speech recognition on"}, + {MODEL_FILE_PATH, "[REQUIRED] Path to the Speech Recognition model to use"}, + {PREFERRED_BACKENDS, "[OPTIONAL] Takes the preferred backends in preference order, separated by comma." + " For example: CpuAcc,GpuAcc,CpuRef. Accepted options: [CpuAcc, CpuRef, GpuAcc]." + " Defaults to CpuAcc,CpuRef"} +}; + +/* + * Reads the user supplied backend preference, splits it by comma, and returns an ordered vector + */ +std::vector<armnn::BackendId> GetPreferredBackendList(const std::string& preferredBackends) +{ + std::vector<armnn::BackendId> backends; + std::stringstream ss(preferredBackends); + + while (ss.good()) + { + std::string backend; + std::getline(ss, backend, ','); + backends.emplace_back(backend); + } + return backends; +} + +//Labels for this model +std::map<int, std::string> labels = +{ + {0, "silence"}, + {1, "unknown"}, + {2, "yes"}, + {3, "no"}, + {4, "up"}, + {5, "down"}, + {6, "left"}, + {7, "right"}, + {8, "on"}, + {9, "off"}, + {10, "stop"}, + {11, "go"} +}; + + +int main(int argc, char* argv[]) +{ + printf("ArmNN major version: %d\n", ARMNN_MAJOR_VERSION); + std::map<std::string, std::string> options; + + //Read command line args + int result = ParseOptions(options, CMD_OPTIONS, argv, argc); + if (result != 0) + { + return result; + } + + // Create the ArmNN inference runner + common::PipelineOptions pipelineOptions; + pipelineOptions.m_ModelName = "DS_CNN_CLUSTERED_INT8"; + pipelineOptions.m_ModelFilePath = GetSpecifiedOption(options, MODEL_FILE_PATH); + if (CheckOptionSpecified(options, PREFERRED_BACKENDS)) + { + pipelineOptions.m_backends = GetPreferredBackendList( + (GetSpecifiedOption(options, PREFERRED_BACKENDS))); + } + else + { + pipelineOptions.m_backends = {"CpuAcc", "CpuRef"}; + } + + kws::IPipelinePtr kwsPipeline = kws::CreatePipeline(pipelineOptions); + + //Extract audio data from sound file + auto filePath = GetSpecifiedOption(options, AUDIO_FILE_PATH); + std::vector<float> audioData = audio::AudioCapture::LoadAudioFile(filePath); + + audio::AudioCapture capture; + //todo: read samples and stride from pipeline + capture.InitSlidingWindow(audioData.data(), + audioData.size(), + kwsPipeline->getInputSamplesSize(), + kwsPipeline->getInputSamplesSize()/2); + + //Loop through audio data buffer + while (capture.HasNext()) + { + std::vector<float> audioBlock = capture.Next(); + common::InferenceResults<int8_t> results; + + //Prepare input tensors + std::vector<int8_t> preprocessedData = kwsPipeline->PreProcessing(audioBlock); + //Run inference + kwsPipeline->Inference(preprocessedData, results); + //Decode output + kwsPipeline->PostProcessing(results, labels, + [](int index, std::string& label, float prob) -> void { + printf("Keyword \"%s\", index %d:, probability %f\n", + label.c_str(), + index, + prob); + }); + } + + return 0; +}
\ No newline at end of file diff --git a/samples/KeywordSpotting/test/DecoderTest.cpp b/samples/KeywordSpotting/test/DecoderTest.cpp new file mode 100644 index 0000000000..e44eb2984d --- /dev/null +++ b/samples/KeywordSpotting/test/DecoderTest.cpp @@ -0,0 +1,28 @@ +// +// Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include <catch.hpp> +#include <map> +#include "Decoder.hpp" + + +TEST_CASE("Test KWS decoder") +{ +// Actual output probability: [0.0, 0.06, 0.02, 0.03, 0.0, 0.0, 0.05, 0.0, 0.83, 0.0, 0.1, 0.0] +// int8 quantised Model output [1, 4, 2, 3, 1, 1, 3, 1, 43, 1, 6, 1] +// Reconstructed dequantised probability [0.0, 0.06, 0.02, 0.04, 0.0, 0.0, 0.04, 0.0, 0.84, 0.0, 0.1, 0.0] + + int quantisationOffset = 1; + float quantisationScale = 0.02; + + std::vector<int8_t> modelOutput = {1, 4, 2, 3, 1, 1, 3, 1, 43, 1, 6, 1}; + + kws::Decoder decoder(quantisationOffset,quantisationScale); + + std::pair<int,float> result = decoder.decodeOutput(modelOutput); + + + CHECK(result == std::pair<int,float>(8,0.84)); +} diff --git a/samples/KeywordSpotting/test/KeywordSpottingPipelineTest.cpp b/samples/KeywordSpotting/test/KeywordSpottingPipelineTest.cpp new file mode 100644 index 0000000000..9fb87fd3c9 --- /dev/null +++ b/samples/KeywordSpotting/test/KeywordSpottingPipelineTest.cpp @@ -0,0 +1,230 @@ +// +// Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include <catch.hpp> +#include <map> +#include <cinttypes> +#include "KeywordSpottingPipeline.hpp" +#include "DsCNNPreprocessor.hpp" + +static std::string GetResourceFilePath(const std::string& filename) +{ + std::string testResources = TEST_RESOURCE_DIR; + if (testResources.empty()) + { + throw std::invalid_argument("Invalid test resources directory provided"); + } + else + { + if(testResources.back() != '/') + { + return testResources + "/" + filename; + } + else + { + return testResources + filename; + } + } +} + +TEST_CASE("Test Keyword spotting pipeline") +{ + const int8_t ifm0_kws [] = + { + -0x1b, 0x4f, 0x7a, -0x55, 0x6, -0x11, 0x6e, -0x6, 0x67, -0x7e, -0xd, 0x6, 0x49, 0x79, -0x1e, 0xe, + 0x1d, 0x6e, 0x6f, 0x6f, -0x2e, -0x4b, 0x2, -0x3e, 0x40, -0x4b, -0x7, 0x31, -0x38, -0x64, -0x28, + 0xc, -0x1d, 0xf, 0x1c, 0x5a, -0x4b, 0x56, 0x7e, 0x9, -0x29, 0x13, -0x65, -0xa, 0x34, -0x59, 0x41, + -0x6f, 0x75, 0x67, -0x5f, 0x17, 0x4a, -0x76, -0x7a, 0x49, -0x19, -0x41, 0x78, 0x40, 0x44, 0xe, + -0x51, -0x5c, 0x3d, 0x24, 0x76, -0x66, -0x11, 0x5e, 0x7b, -0x4, 0x7a, 0x9, 0x13, 0x8, -0x21, -0x11, + 0x13, 0x7a, 0x25, 0x6, -0x68, 0x6a, -0x30, -0x16, -0x43, -0x27, 0x4c, 0x6b, -0x14, -0x12, -0x5f, + 0x49, -0x2a, 0x44, 0x57, -0x78, -0x72, 0x62, -0x8, -0x38, -0x73, -0x2, -0x80, 0x79, -0x3f, 0x57, + 0x9, -0x7e, -0x34, -0x59, 0x19, -0x66, 0x58, -0x3b, -0x69, -0x1a, 0x13, -0x2f, -0x2f, 0x13, 0x35, + -0x30, 0x1e, 0x3b, -0x71, 0x67, 0x7d, -0x5d, 0x1a, 0x69, -0x53, -0x38, -0xf, 0x76, 0x2, 0x7e, 0x45, + -0xa, 0x59, -0x6b, -0x28, -0x5d, -0x63, -0x7d, -0x3, 0x48, 0x74, -0x75, -0x7a, 0x1f, -0x53, 0x5b, + 0x4d, -0x18, -0x4a, 0x39, -0x52, 0x5a, -0x6b, -0x41, -0x3e, -0x61, -0x80, -0x52, 0x67, 0x71, -0x47, + 0x79, -0x41, 0x3a, -0x8, -0x1f, 0x4d, -0x7, 0x5b, 0x6b, -0x1b, -0x8, -0x20, -0x21, 0x7c, -0x74, + 0x25, -0x68, -0xe, -0x7e, -0x45, -0x28, 0x45, -0x1a, -0x39, 0x78, 0x11, 0x48, -0x6b, -0x7b, -0x43, + -0x21, 0x38, 0x46, 0x7c, -0x5d, 0x59, 0x53, -0x3f, -0x15, 0x59, -0x17, 0x75, 0x2f, 0x7c, 0x68, 0x6a, + 0x0, -0x10, 0x5b, 0x61, 0x36, -0x41, 0x33, 0x23, -0x80, -0x1d, -0xb, -0x56, 0x2d, 0x68, -0x68, + 0x2f, 0x48, -0x5d, -0x44, 0x64, -0x27, 0x68, -0x13, 0x39, -0x3f, 0x18, 0x31, 0x15, -0x78, -0x2, + 0x72, 0x60, 0x59, -0x30, -0x22, 0x73, 0x61, 0x76, -0x4, -0x62, -0x64, -0x80, -0x32, -0x16, 0x51, + -0x2, -0x70, 0x71, 0x3f, -0x5f, -0x35, -0x3c, 0x79, 0x48, 0x61, 0x5b, -0x20, -0x1e, -0x68, -0x1c, + 0x6c, 0x3a, 0x28, -0x36, -0x3e, 0x5f, -0x75, -0x73, 0x1e, 0x75, -0x66, -0x22, 0x20, -0x64, 0x67, + 0x36, 0x14, 0x37, -0xa, -0xe, 0x8, -0x37, -0x43, 0x21, -0x8, 0x54, 0x1, 0x34, -0x2c, -0x73, -0x11, + -0x48, -0x1c, -0x40, 0x14, 0x4e, -0x53, 0x25, 0x5e, 0x14, 0x4f, 0x7c, 0x6d, -0x61, -0x38, 0x35, + -0x5a, -0x44, 0x12, 0x52, -0x60, 0x22, -0x1c, -0x8, -0x4, -0x6b, -0x71, 0x43, 0xb, 0x7b, -0x7, + -0x3c, -0x3b, -0x40, -0xd, 0x44, 0x6, 0x30, 0x38, 0x57, 0x1f, -0x7, 0x2, 0x4f, 0x64, 0x7c, -0x3, + -0x13, -0x71, -0x45, -0x53, -0x52, 0x2b, -0x11, -0x1d, -0x2, -0x29, -0x37, 0x3d, 0x19, 0x76, 0x18, + 0x1d, 0x12, -0x29, -0x5e, -0x54, -0x48, 0x5d, -0x41, -0x3f, 0x7e, -0x2a, 0x41, 0x57, -0x65, -0x15, + 0x12, 0x1f, -0x57, 0x79, -0x64, 0x3a, -0x2f, 0x7f, -0x6c, 0xa, 0x52, -0x1f, -0x41, 0x6e, -0x4b, + 0x3d, -0x1b, -0x42, 0x22, -0x3c, -0x35, -0xf, 0xc, 0x32, -0x15, -0x68, -0x21, 0x0, -0x16, 0x14, + -0x10, -0x5b, 0x2f, 0x21, 0x41, -0x8, -0x12, -0xa, 0x10, 0xf, 0x7e, -0x76, -0x1d, 0x2b, -0x49, + 0x42, -0x25, -0x78, -0x69, -0x2c, 0x3f, 0xc, 0x52, 0x6d, 0x2e, -0x13, 0x76, 0x37, -0x36, -0x51, + -0x5, -0x63, -0x4f, 0x1c, 0x6b, -0x4b, 0x71, -0x12, 0x72, -0x3f,-0x4a, 0xf, 0x3a, -0xd, 0x38, 0x3b, + -0x5d, 0x75, -0x43, -0x10, -0xa, -0x7a, 0x1a, -0x44, 0x1c, 0x6a, 0x43, -0x1b, -0x35, 0x7d, -0x2c, + -0x10, 0x5b, -0x42, -0x4f, 0x69, 0x1f, 0x1b, -0x64, -0x21, 0x19, -0x5d, 0x2e, -0x2a, -0x65, -0x13, + -0x70, -0x6e + }; + + const int8_t ofm0_kws [] = + { + -0x80, 0x7f, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80 + }; + + // First 640 samples from yes.wav. + std::vector<int16_t> testWav = std::vector<int16_t> + { + 139, 143, 164, 163, 157, 156, 151, 148, 172, 171, + 165, 169, 149, 142, 145, 147, 166, 146, 112, 132, + 132, 136, 165, 176, 176, 152, 138, 158, 179, 185, + 183, 148, 121, 130, 167, 204, 163, 132, 165, 184, + 193, 205, 210, 204, 195, 178, 168, 197, 207, 201, + 197, 177, 185, 196, 191, 198, 196, 183, 193, 181, + 157, 170, 167, 159, 164, 152, 146, 167, 180, 171, + 194, 232, 204, 173, 171, 172, 184, 169, 175, 199, + 200, 195, 185, 214, 214, 193, 196, 191, 204, 191, + 172, 187, 183, 192, 203, 172, 182, 228, 232, 205, + 177, 174, 191, 210, 210, 211, 197, 177, 198, 217, + 233, 236, 203, 191, 169, 145, 149, 161, 198, 206, + 176, 137, 142, 181, 200, 215, 201, 188, 166, 162, + 184, 155, 135, 132, 126, 142, 169, 184, 172, 156, + 132, 119, 150, 147, 154, 160, 125, 130, 137, 154, + 161, 168, 195, 182, 160, 134, 138, 146, 130, 120, + 101, 122, 137, 118, 117, 131, 145, 140, 146, 148, + 148, 168, 159, 134, 114, 114, 130, 147, 147, 134, + 125, 98, 107, 127, 99, 79, 84, 107, 117, 114, + 93, 92, 127, 112, 109, 110, 96, 118, 97, 87, + 110, 95, 128, 153, 147, 165, 146, 106, 101, 137, + 139, 96, 73, 90, 91, 51, 69, 102, 100, 103, + 96, 101, 123, 107, 82, 89, 118, 127, 99, 100, + 111, 97, 111, 123, 106, 121, 133, 103, 100, 88, + 85, 111, 114, 125, 102, 91, 97, 84, 139, 157, + 109, 66, 72, 129, 111, 90, 127, 126, 101, 109, + 142, 138, 129, 159, 140, 80, 74, 78, 76, 98, + 68, 42, 106, 143, 112, 102, 115, 114, 82, 75, + 92, 80, 110, 114, 66, 86, 119, 101, 101, 103, + 118, 145, 85, 40, 62, 88, 95, 87, 73, 64, + 86, 71, 71, 105, 80, 73, 96, 92, 85, 90, + 81, 86, 105, 100, 89, 78, 102, 114, 95, 98, + 69, 70, 108, 112, 111, 90, 104, 137, 143, 160, + 145, 121, 98, 86, 91, 87, 115, 123, 109, 99, + 85, 120, 131, 116, 125, 144, 153, 111, 98, 110, + 93, 89, 101, 137, 155, 142, 108, 94, 136, 145, + 129, 129, 122, 109, 90, 76, 81, 110, 119, 96, + 95, 102, 105, 111, 90, 89, 111, 115, 86, 51, + 107, 140, 105, 105, 110, 142, 125, 76, 75, 69, + 65, 52, 61, 69, 55, 42, 47, 58, 37, 35, + 24, 20, 44, 22, 16, 26, 6, 3, 4, 23, + 60, 51, 30, 12, 24, 31, -9, -16, -13, 13, + 19, 9, 37, 55, 70, 36, 23, 57, 45, 33, + 50, 59, 18, 11, 62, 74, 52, 8, -3, 26, + 51, 48, -5, -9, 12, -7, -12, -5, 28, 41, + -2, -30, -13, 31, 33, -12, -22, -8, -15, -17, + 2, -6, -25, -27, -24, -8, 4, -9, -52, -47, + -9, -32, -45, -5, 41, 15, -32, -14, 2, -1, + -10, -30, -32, -25, -21, -17, -14, 8, -4, -13, + 34, 18, -36, -38, -18, -19, -28, -17, -14, -16, + -2, -20, -27, 12, 11, -17, -33, -12, -22, -64, + -42, -26, -23, -22, -37, -51, -53, -30, -18, -48, + -69, -38, -54, -96, -72, -49, -50, -57, -41, -22, + -43, -64, -54, -23, -49, -69, -41, -44, -42, -49, + -40, -26, -54, -50, -38, -49, -70, -94, -89, -69, + -56, -65, -71, -47, -39, -49, -79, -91, -56, -46, + -62, -86, -64, -32, -47, -50, -71, -77, -65, -68, + -52, -51, -61, -67, -61, -81, -93, -52, -59, -62, + -51, -75, -76, -50, -32, -54, -68, -70, -43, 1, + -42, -92, -80, -41, -38, -79, -69, -49, -82, -122, + -93, -21, -24, -61, -70, -73, -62, -74, -69, -43, + -25, -15, -43, -23, -26, -69, -44, -12, 1, -51, + -78, -13, 3, -53, -105, -72, -24, -62, -66, -31, + -40, -65, -86, -64, -44, -55, -63, -61, -37, -41, + }; + + // Golden audio ops mfcc output for the above wav. + const std::vector<float> testWavMfcc + { + -22.67135, -0.61615, 2.07233, 0.58137, 1.01655, 0.85816, 0.46039, 0.03393, 1.16511, 0.0072, + }; + + std::vector<float> testWavFloat(640); + constexpr float normaliser = 1.0/(1u<<15u); + std::transform(testWav.begin(), testWav.end(), testWavFloat.begin(), + std::bind1st(std::multiplies<float>(), normaliser)); + + const float DsCNNInputQuantizationScale = 1.107164; + const int DsCNNInputQuantizationOffset = 95; + + std::map<int,std::string> labels = + { + {0,"silence"}, + {1, "unknown"}, + { 2, "yes"}, + { 3,"no"}, + { 4, "up"}, + { 5, "down"}, + { 6, "left"}, + { 7, "right"}, + { 8, "on"}, + { 9, "off"}, + { 10, "stop"}, + {11, "go"} + }; + common::PipelineOptions options; + options.m_ModelFilePath = GetResourceFilePath("ds_cnn_clustered_int8.tflite"); + options.m_ModelName = "DS_CNN_CLUSTERED_INT8"; + options.m_backends = {"CpuAcc", "CpuRef"}; + kws::IPipelinePtr kwsPipeline = kws::CreatePipeline(options); + + CHECK(kwsPipeline->getInputSamplesSize() == 16000); + std::vector<int8_t> expectedWavMfcc; + for(auto& i : testWavMfcc) + { + expectedWavMfcc.push_back( + (i + DsCNNInputQuantizationScale * DsCNNInputQuantizationOffset) / DsCNNInputQuantizationScale); + } + + SECTION("Pre-processing") + { + testWavFloat.resize(16000); + expectedWavMfcc.resize(49 * 10); + std::vector<int8_t> preprocessedData = kwsPipeline->PreProcessing(testWavFloat); + CHECK(preprocessedData.size() == expectedWavMfcc.size()); + for(int i = 0; i < 10; ++i) + { + CHECK(expectedWavMfcc[i] == Approx(preprocessedData[i]).margin(1)); + } + } + + SECTION("Execute inference") + { + common::InferenceResults<int8_t> result; + std::vector<int8_t> IFM(std::begin(ifm0_kws), std::end(ifm0_kws)); + kwsPipeline->Inference(IFM, result); + std::vector<int8_t> OFM(std::begin(ofm0_kws), std::end(ofm0_kws)); + + CHECK(1 == result.size()); + CHECK(OFM.size() == result[0].size()); + + int count = 0; + for (auto& i : result) + { + for (signed char& j : i) + { + CHECK(j == OFM[count++]); + + } + } + } + + SECTION("Convert inference result to keyword") + { + std::vector< std::vector< int8_t >> modelOutput = {{1, 4, 2, 3, 1, 1, 3, 1, 43, 1, 6, 1}}; + kwsPipeline->PostProcessing(modelOutput, labels, + [](int index, std::string& label, float prob) -> void { + CHECK(index == 8); + CHECK(label == "on"); + }); + } +} |