# Copyright © 2021 Arm Ltd and Contributors. All rights reserved. # SPDX-License-Identifier: MIT import os import numpy as np import pytest import collections from context import mfcc from context import wav2letter_mfcc from context import audio_capture # Elements relevant to MFCC filter bank & feature extraction MFCC_TEST_PARAMS = collections.namedtuple('mfcc_test_params', ['algo_params', 'mfcc_constructor', 'audio_proc_constructor']) def kws_test_params(): kws_algo_params = mfcc.MFCCParams(sampling_freq=16000, num_fbank_bins=40, mel_lo_freq=20, mel_hi_freq=4000, num_mfcc_feats=10, frame_len=640, use_htk_method=True, n_fft=1024) return MFCC_TEST_PARAMS(kws_algo_params, mfcc.MFCC, mfcc.AudioPreprocessor) def asr_test_params(): asr_algo_params = mfcc.MFCCParams(sampling_freq=16000, num_fbank_bins=128, mel_lo_freq=0, mel_hi_freq=8000, num_mfcc_feats=13, frame_len=512, use_htk_method=False, n_fft=512) return MFCC_TEST_PARAMS(asr_algo_params, wav2letter_mfcc.Wav2LetterMFCC, wav2letter_mfcc.W2LAudioPreprocessor) def kws_cap_params(): return audio_capture.AudioCaptureParams(dtype=np.float32, overlap=0, min_samples=16000, sampling_freq=16000, mono=True) def asr_cap_params(): return audio_capture.AudioCaptureParams(dtype=np.float32, overlap=31712, min_samples=47712, sampling_freq=16000, mono=True) @pytest.fixture() def audio_data(test_data_folder, file, audio_cap_params): audio_file = os.path.join(test_data_folder, file) capture = audio_capture.capture_audio(audio_file, audio_cap_params) yield next(capture) @pytest.mark.parametrize("file", ["yes.wav", "myVoiceIsMyPassportVerifyMe04.wav"]) @pytest.mark.parametrize("audio_cap_params", [kws_cap_params(), asr_cap_params()]) def test_audio_file(audio_data, test_data_folder, file, audio_cap_params): assert audio_data.shape == (audio_cap_params.min_samples,) assert audio_data.dtype == audio_cap_params.dtype @pytest.mark.parametrize("mfcc_test_params, test_out", [(kws_test_params(), 25.470010570730597), (asr_test_params(), 0.24)]) def test_mel_scale_function(mfcc_test_params, test_out): mfcc_inst = mfcc_test_params.mfcc_constructor(mfcc_test_params.algo_params) mel = mfcc_inst.mel_scale(16, mfcc_test_params.algo_params.use_htk_method) assert np.isclose(mel, test_out) @pytest.mark.parametrize("mfcc_test_params, test_out", [(kws_test_params(), 10.008767240008943), (asr_test_params(), 1071.170287494467)]) def test_inverse_mel_scale_function(mfcc_test_params, test_out): mfcc_inst = mfcc_test_params.mfcc_constructor(mfcc_test_params.algo_params) mel = mfcc_inst.inv_mel_scale(16, mfcc_test_params.algo_params.use_htk_method) assert np.isclose(mel, test_out) mel_filter_test_data_kws = {0: [0.33883214, 0.80088392, 0.74663128, 0.30332531], 1: [0.25336872, 0.69667469, 0.86883317, 0.44281119, 0.02493546], 2: [0.13116683, 0.55718881, 0.97506454, 0.61490026, 0.21241678], 5: [0.32725038, 0.69579596, 0.9417706, 0.58524989, 0.23445207], -1: [0.02433275, 0.10371618, 0.1828123, 0.26162319, 0.34015089, 0.41839743, 0.49636481, 0.57405503, 0.65147004, 0.72861179, 0.8054822, 0.88208318, 0.95841659, 0.96551568, 0.88971181, 0.81416996, 0.73888833, 0.66386514, 0.58909861, 0.514587, 0.44032856, 0.3663216, 0.29256441, 0.21905531, 0.14579264, 0.07277474]} mel_filter_test_data_asr = {0: [0.02837754], 1: [0.01438901, 0.01398853], 2: [0.02877802], 5: [0.01478948, 0.01358806], -1: [4.82151203e-05, 9.48791110e-04, 1.84569875e-03, 2.73896782e-03, 3.62862771e-03, 4.51470746e-03, 5.22215439e-03, 4.34314914e-03, 3.46763895e-03, 2.59559614e-03, 1.72699334e-03, 8.61803536e-04]} @pytest.mark.parametrize("mfcc_test_params, test_out", [(kws_test_params(), mel_filter_test_data_kws), (asr_test_params(), mel_filter_test_data_asr)]) def test_create_mel_filter_bank(mfcc_test_params, test_out): mfcc_inst = mfcc_test_params.mfcc_constructor(mfcc_test_params.algo_params) mel_filter_bank = mfcc_inst.create_mel_filter_bank() assert len(mel_filter_bank) == mfcc_test_params.algo_params.num_fbank_bins for indx, data in test_out.items(): assert np.allclose(mel_filter_bank[indx], data) mfcc_test_data_kws = (-22.671347398982626, -0.6161543999707211, 2.072326974167832, 0.5813741475362223, 1.0165529747334272, 0.8581560719988703, 0.4603911069624896, 0.03392820944377398, 1.1651093266902361, 0.007200025869960908) mfcc_test_data_asr = (-735.46345398, 69.50331943, 16.39159347, 22.74874819, 24.84782893, 10.67559303, 12.82828618, -3.51084271, 4.66633677, 10.20079095, 11.34782948, 3.90499354, 9.32322384) @pytest.mark.parametrize("mfcc_test_params, test_out, file, audio_cap_params", [(kws_test_params(), mfcc_test_data_kws, "yes.wav", kws_cap_params()), (asr_test_params(), mfcc_test_data_asr, "myVoiceIsMyPassportVerifyMe04.wav", asr_cap_params())]) def test_mfcc_compute_first_frame(audio_data, mfcc_test_params, test_out, file, audio_cap_params): audio_data = np.array(audio_data)[0:mfcc_test_params.algo_params.frame_len] mfcc_inst = mfcc_test_params.mfcc_constructor(mfcc_test_params.algo_params) mfcc_feats = mfcc_inst.mfcc_compute(audio_data) assert np.allclose((mfcc_feats[0:mfcc_test_params.algo_params.num_mfcc_feats]), test_out) extract_test_data_kws = {0: [-2.2671347e+01, -6.1615437e-01, 2.0723269e+00, 5.8137417e-01, 1.0165529e+00, 8.5815609e-01, 4.6039110e-01, 3.3928208e-02, 1.1651093e+00, 7.2000260e-03], 1: [-23.488806, -1.1687667, 3.0548365, 1.5129884, 1.4142203, 0.6869772, 1.1875846, 0.5743369, 1.202258, -0.12133602], 2: [-23.909292, -1.5186096, 1.8721082, 0.7378916, 0.44974303, 0.17609395, 0.5183161, 0.37109664, 0.14186797, 0.58400506], -1: [-23.752186, -0.1796912, 1.9514247, 0.32554424, 1.8425112, 0.8763608, 0.78326845, 0.27808753, 0.73788685, 0.30338883]} extract_test_data_asr = {0: [-4.98830318e+00, 6.86444461e-01, 3.12024504e-01, 3.56840312e-01, 3.71638149e-01, 2.71728605e-01, 2.86904365e-01, 1.71718955e-01, 2.29365349e-01, 2.68381387e-01, 2.76467651e-01, 2.23998129e-01, 2.62194842e-01, -1.48247385e+01, 1.21875501e+00, 4.20235842e-01, 5.39400637e-01, 6.09882712e-01, 1.68513224e-01, 3.75330061e-01, 8.57576132e-02, 1.92831963e-01, 1.41814977e-01, 1.57615796e-01, 7.19076321e-02, 1.98729336e-01, 3.92199278e+00, -5.76856315e-01, 1.17938723e-02, -9.25096497e-02, -3.59488949e-02, 1.13284402e-03, 1.51282102e-01, 1.13404110e-01, -8.69824737e-02, -1.48449212e-01, -1.24230251e-01, -1.90728232e-01, -5.37525006e-02], 1: [-4.96694946e+00, 6.69411421e-01, 2.86189795e-01, 3.65071595e-01, 3.92671198e-01, 2.44258150e-01, 2.52177566e-01, 2.16024980e-01, 2.79812217e-01, 2.79687315e-01, 2.95228422e-01, 2.83991724e-01, 2.46358261e-01, -1.33618221e+01, 1.08920455e+00, 3.88707787e-01, 5.05674303e-01, 6.08285785e-01, 1.68113053e-01, 3.54529470e-01, 6.68609440e-02, 1.52882755e-01, 6.89579248e-02, 1.18375972e-01, 5.86742274e-02, 1.15678251e-01, 1.07892036e+01, -1.07193100e+00, -2.18140319e-01, -3.35950345e-01, -2.57241666e-01, -5.54431602e-02, -8.38544443e-02, -5.79114584e-03, -2.23973781e-01, -2.91451365e-01, -2.11069033e-01, -1.90297231e-01, -2.76504964e-01], 2: [-4.98664522e+00, 6.54802263e-01, 3.70355755e-01, 4.06837821e-01, 4.05175537e-01, 2.29149669e-01, 2.83312678e-01, 2.17573136e-01, 3.07824671e-01, 2.48388007e-01, 2.25399241e-01, 2.52003014e-01, 2.83968121e-01, -1.05043650e+01, 7.91533887e-01, 3.11546475e-01, 4.36079264e-01, 5.93271911e-01, 2.02480286e-01, 3.24254721e-01, 6.29674867e-02, 9.67641100e-02, -1.62826646e-02, 5.47595806e-02, 2.90475693e-02, 2.62522381e-02, 1.38787737e+01, -1.32597208e+00, -3.73900205e-01, -4.38065380e-01, -3.05983245e-01, 1.14390980e-02, -2.10821658e-01, -6.22789040e-02, -2.88273603e-01, -3.29794526e-01, -2.43764088e-01, -1.70954674e-01, -3.65193188e-01], -1: [-2.1894817, 1.583355, -0.45024827, 0.11657667, 0.08940444, 0.09041209, 0.2003613, 0.11800499, 0.18838657, 0.29271516, 0.22758003, 0.10634928, -0.04019014, 7.203311, -2.414309, 0.28750962, -0.24222863, 0.04680864, -0.12129474, 0.18059334, 0.06250379, 0.11363743, -0.2561094, -0.08132717, -0.08500769, 0.18916495, 1.3529671, -3.7919693, 1.937804, 0.6845761, 0.15381853, 0.41106734, -0.28207013, 0.2195526, 0.06716935, -0.02886542, -0.22860551, 0.24788341, 0.63940096]} @pytest.mark.parametrize("mfcc_test_params, model_input_size, stride, min_samples, file, audio_cap_params, test_out", [(kws_test_params(), 49, 320, 16000, "yes.wav", kws_cap_params(), extract_test_data_kws), (asr_test_params(), 296, 160, 47712, "myVoiceIsMyPassportVerifyMe04.wav", asr_cap_params(), extract_test_data_asr)]) def test_feat_extraction_full_sized_input(audio_data, mfcc_test_params, model_input_size, stride, min_samples, file, audio_cap_params, test_out): """ Test out values were gathered by printing the mfcc features collected during the first full inference on the test wav files. Note the extract_features() function simply calls the mfcc_compute() from previous test but feeds in enough samples for an inference rather than a single frame. It also computes the 1st & 2nd derivative features hence the shape (13*3 = 39). Specific model_input_size and stride parameters are also required as additional arguments. """ audio_data = np.array(audio_data) # Pad with zeros to ensure min_samples for inference audio_data.resize(min_samples) mfcc_inst = mfcc_test_params.mfcc_constructor(mfcc_test_params.algo_params) preprocessor = mfcc_test_params.audio_proc_constructor(mfcc_inst, model_input_size, stride) # extract_features passes the audio data to mfcc_compute frame by frame and concatenates results input_tensor = preprocessor.extract_features(audio_data) assert len(input_tensor) == model_input_size for indx, data in test_out.items(): assert np.allclose(input_tensor[indx], data) # Expected contents of input tensors for inference on a silent wav file extract_features_zeros_kws = {0: [-2.05949466e+02, -4.88498131e-15, 8.15428020e-15, -5.77315973e-15, 7.03142511e-15, -1.11022302e-14, 2.18015108e-14, -1.77635684e-15, 1.06581410e-14, 2.75335310e-14], -1: [-2.05949466e+02, -4.88498131e-15, 8.15428020e-15, -5.77315973e-15, 7.03142511e-15, -1.11022302e-14, 2.18015108e-14, -1.77635684e-15, 1.06581410e-14, 2.75335310e-14]} extract_features_zeros_asr = { 0: [-3.46410162e+00, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.79662980e+01, 1.75638694e-15, -9.41313626e-16, 9.66012817e-16, -1.23221521e-15, 1.75638694e-15, -1.59035349e-15, 2.41503204e-15, -1.64798493e-15, 4.39096735e-16, -4.95356004e-16, -2.19548368e-16, -3.55668355e-15, 8.19843971e+00, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02], - 1: [-3.46410162e+00, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.88675135e-01, 2.79662980e+01, 1.75638694e-15, -9.41313626e-16, 9.66012817e-16, -1.23221521e-15, 1.75638694e-15, -1.59035349e-15, 2.41503204e-15, -1.64798493e-15, 4.39096735e-16, -4.95356004e-16, -2.19548368e-16, -3.55668355e-15, 8.19843971e+00, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02, -4.28340672e-02]} @pytest.mark.parametrize("mfcc_test_params,model_input_size, stride, min_samples, test_out", [(kws_test_params(), 49, 320, 16000, extract_features_zeros_kws), (asr_test_params(), 296, 160, 47712, extract_features_zeros_asr)]) def test_feat_extraction_full_sized_input_zeros(mfcc_test_params, model_input_size, stride, min_samples, test_out): audio_data = np.zeros(min_samples).astype(np.float32) mfcc_inst = mfcc_test_params.mfcc_constructor(mfcc_test_params.algo_params) preprocessor = mfcc_test_params.audio_proc_constructor(mfcc_inst, model_input_size, stride) input_tensor = preprocessor.extract_features(audio_data) assert len(input_tensor) == model_input_size for indx, data in test_out.items(): # Element 14 of feature extraction vector differs minutely during # inference on a silent wav file compared to array of 0's # Workarounds were to skip this sample or add large tolerance argument (atol=10) assert np.allclose(input_tensor[indx][0:13], data[0:13]) assert np.allclose(input_tensor[indx][15:], data[15:])