1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
|
# Copyright © 2020 Arm Ltd and Contributors. All rights reserved.
# SPDX-License-Identifier: MIT
"""Class used to extract the Mel-frequency cepstral coefficients from a given audio frame."""
import numpy as np
class MFCCParams:
def __init__(self, sampling_freq, num_fbank_bins,
mel_lo_freq, mel_hi_freq, num_mfcc_feats, frame_len, use_htk_method, n_FFT):
self.sampling_freq = sampling_freq
self.num_fbank_bins = num_fbank_bins
self.mel_lo_freq = mel_lo_freq
self.mel_hi_freq = mel_hi_freq
self.num_mfcc_feats = num_mfcc_feats
self.frame_len = frame_len
self.use_htk_method = use_htk_method
self.n_FFT = n_FFT
class MFCC:
def __init__(self, mfcc_params):
self.mfcc_params = mfcc_params
self.FREQ_STEP = 200.0 / 3
self.MIN_LOG_HZ = 1000.0
self.MIN_LOG_MEL = self.MIN_LOG_HZ / self.FREQ_STEP
self.LOG_STEP = 1.8562979903656 / 27.0
self.__frame_len_padded = int(2 ** (np.ceil((np.log(self.mfcc_params.frame_len) / np.log(2.0)))))
self.__filter_bank_initialised = False
self.__frame = np.zeros(self.__frame_len_padded)
self.__buffer = np.zeros(self.__frame_len_padded)
self.__filter_bank_filter_first = np.zeros(self.mfcc_params.num_fbank_bins)
self.__filter_bank_filter_last = np.zeros(self.mfcc_params.num_fbank_bins)
self.__mel_energies = np.zeros(self.mfcc_params.num_fbank_bins)
self.__dct_matrix = self.create_dct_matrix(self.mfcc_params.num_fbank_bins, self.mfcc_params.num_mfcc_feats)
self.__mel_filter_bank = self.create_mel_filter_bank()
self.__np_mel_bank = np.zeros([self.mfcc_params.num_fbank_bins, int(self.mfcc_params.n_FFT / 2) + 1])
for i in range(self.mfcc_params.num_fbank_bins):
k = 0
for j in range(int(self.__filter_bank_filter_first[i]), int(self.__filter_bank_filter_last[i]) + 1):
self.__np_mel_bank[i, j] = self.__mel_filter_bank[i][k]
k += 1
def mel_scale(self, freq, use_htk_method):
"""
Gets the mel scale for a particular sample frequency.
Args:
freq: The sampling frequency.
use_htk_method: Boolean to set whether to use HTK method or not.
Returns:
the mel scale
"""
if use_htk_method:
return 1127.0 * np.log(1.0 + freq / 700.0)
else:
mel = freq / self.FREQ_STEP
if freq >= self.MIN_LOG_HZ:
mel = self.MIN_LOG_MEL + np.log(freq / self.MIN_LOG_HZ) / self.LOG_STEP
return mel
def inv_mel_scale(self, mel_freq, use_htk_method):
"""
Gets the sample frequency for a particular mel.
Args:
mel_freq: The mel frequency.
use_htk_method: Boolean to set whether to use HTK method or not.
Returns:
the sample frequency
"""
if use_htk_method:
return 700.0 * (np.exp(mel_freq / 1127.0) - 1.0)
else:
freq = self.FREQ_STEP * mel_freq
if mel_freq >= self.MIN_LOG_MEL:
freq = self.MIN_LOG_HZ * np.exp(self.LOG_STEP * (mel_freq - self.MIN_LOG_MEL))
return freq
def mfcc_compute(self, audio_data):
"""
Extracts the MFCC for a single frame.
Args:
audio_data: The audio data to process.
Returns:
the MFCC features
"""
if len(audio_data) != self.mfcc_params.frame_len:
raise ValueError(
f"audio_data buffer size {len(audio_data)} does not match the frame length {self.mfcc_params.frame_len}")
audio_data = np.array(audio_data)
spec = np.abs(np.fft.rfft(np.hanning(self.mfcc_params.n_FFT + 1)[0:self.mfcc_params.n_FFT] * audio_data,
self.mfcc_params.n_FFT)) ** 2
mel_energy = np.dot(self.__np_mel_bank.astype(np.float32),
np.transpose(spec).astype(np.float32))
mel_energy += 1e-10
log_mel_energy = 10.0 * np.log10(mel_energy)
top_db = 80.0
log_mel_energy = np.maximum(log_mel_energy, log_mel_energy.max() - top_db)
mfcc_feats = np.dot(self.__dct_matrix, log_mel_energy)
return mfcc_feats
def create_dct_matrix(self, num_fbank_bins, num_mfcc_feats):
"""
Creates the Discrete Cosine Transform matrix to be used in the compute function.
Args:
num_fbank_bins: The number of filter bank bins
num_mfcc_feats: the number of MFCC features
Returns:
the DCT matrix
"""
dct_m = np.zeros(num_fbank_bins * num_mfcc_feats)
for k in range(num_mfcc_feats):
for n in range(num_fbank_bins):
if k == 0:
dct_m[(k * num_fbank_bins) + n] = 2 * np.sqrt(1 / (4 * num_fbank_bins)) * np.cos(
(np.pi / num_fbank_bins) * (n + 0.5) * k)
else:
dct_m[(k * num_fbank_bins) + n] = 2 * np.sqrt(1 / (2 * num_fbank_bins)) * np.cos(
(np.pi / num_fbank_bins) * (n + 0.5) * k)
dct_m = np.reshape(dct_m, [self.mfcc_params.num_mfcc_feats, self.mfcc_params.num_fbank_bins])
return dct_m
def create_mel_filter_bank(self):
"""
Creates the Mel filter bank.
Returns:
the mel filter bank
"""
num_fft_bins = int(self.__frame_len_padded / 2)
fft_bin_width = self.mfcc_params.sampling_freq / self.__frame_len_padded
mel_low_freq = self.mel_scale(self.mfcc_params.mel_lo_freq, False)
mel_high_freq = self.mel_scale(self.mfcc_params.mel_hi_freq, False)
mel_freq_delta = (mel_high_freq - mel_low_freq) / (self.mfcc_params.num_fbank_bins + 1)
this_bin = np.zeros(num_fft_bins)
mel_fbank = [0] * self.mfcc_params.num_fbank_bins
for bin_num in range(self.mfcc_params.num_fbank_bins):
left_mel = mel_low_freq + bin_num * mel_freq_delta
center_mel = mel_low_freq + (bin_num + 1) * mel_freq_delta
right_mel = mel_low_freq + (bin_num + 2) * mel_freq_delta
first_index = last_index = -1
for i in range(num_fft_bins):
freq = (fft_bin_width * i)
mel = self.mel_scale(freq, False)
this_bin[i] = 0.0
if (mel > left_mel) and (mel < right_mel):
if mel <= center_mel:
weight = (mel - left_mel) / (center_mel - left_mel)
else:
weight = (right_mel - mel) / (right_mel - center_mel)
enorm = 2.0 / (self.inv_mel_scale(right_mel, False) - self.inv_mel_scale(left_mel, False))
weight *= enorm
this_bin[i] = weight
if first_index == -1:
first_index = i
last_index = i
self.__filter_bank_filter_first[bin_num] = first_index
self.__filter_bank_filter_last[bin_num] = last_index
mel_fbank[bin_num] = np.zeros(last_index - first_index + 1)
j = 0
for i in range(first_index, last_index + 1):
mel_fbank[bin_num][j] = this_bin[i]
j += 1
return mel_fbank
class Preprocessor:
def __init__(self, mfcc, model_input_size, stride):
self.model_input_size = model_input_size
self.stride = stride
# Savitzky - Golay differential filters
self.__savgol_order1_coeffs = np.array([6.66666667e-02, 5.00000000e-02, 3.33333333e-02,
1.66666667e-02, -3.46944695e-18, -1.66666667e-02,
-3.33333333e-02, -5.00000000e-02, -6.66666667e-02])
self.savgol_order2_coeffs = np.array([0.06060606, 0.01515152, -0.01731602,
-0.03679654, -0.04329004, -0.03679654,
-0.01731602, 0.01515152, 0.06060606])
self.__mfcc_calc = mfcc
def __normalize(self, values):
"""
Normalize values to mean 0 and std 1
"""
ret_val = (values - np.mean(values)) / np.std(values)
return ret_val
def __get_features(self, features, mfcc_instance, audio_data):
idx = 0
while len(features) < self.model_input_size * mfcc_instance.mfcc_params.num_mfcc_feats:
features.extend(mfcc_instance.mfcc_compute(audio_data[idx:idx + int(mfcc_instance.mfcc_params.frame_len)]))
idx += self.stride
def extract_features(self, audio_data):
"""
Extracts the MFCC features, and calculates each features first and second order derivative.
The matrix returned should be sized appropriately for input to the model, based
on the model info specified in the MFCC instance.
Args:
mfcc_instance: The instance of MFCC used for this calculation
audio_data: the audio data to be used for this calculation
Returns:
the derived MFCC feature vector, sized appropriately for inference
"""
num_samples_per_inference = ((self.model_input_size - 1)
* self.stride) + self.__mfcc_calc.mfcc_params.frame_len
if len(audio_data) < num_samples_per_inference:
raise ValueError("audio_data size for feature extraction is smaller than "
"the expected number of samples needed for inference")
features = []
self.__get_features(features, self.__mfcc_calc, np.asarray(audio_data))
features = np.reshape(np.array(features), (self.model_input_size, self.__mfcc_calc.mfcc_params.num_mfcc_feats))
mfcc_delta_np = np.zeros_like(features)
mfcc_delta2_np = np.zeros_like(features)
for i in range(features.shape[1]):
idelta = np.convolve(features[:, i], self.__savgol_order1_coeffs, 'same')
mfcc_delta_np[:, i] = (idelta)
ideltadelta = np.convolve(features[:, i], self.savgol_order2_coeffs, 'same')
mfcc_delta2_np[:, i] = (ideltadelta)
features = np.concatenate((self.__normalize(features), self.__normalize(mfcc_delta_np),
self.__normalize(mfcc_delta2_np)), axis=1)
return np.float32(features)
|
