1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
|
/*
* Copyright (c) 2017 ARM Limited.
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef ARM_COMPUTE_ASYMM_HELPER_H
#define ARM_COMPUTE_ASYMM_HELPER_H
// TODO These functions were implemented to be used in softmax-uint8 kernel and therefore process only vectors of length 16.
// But they can be managed to process arbitrary vector length using VEC_DATA_TYPE(int, size) definition to be more reusable.
// Algoriths for these functions were taken from
// https://github.com/google/gemmlowp/blob/master/fixedpoint/fixedpoint.h
// and adapted to operate on integer vectors.
/** For each element of input vector, the corresponding bits of the result item are set
* if the input item is zero.
*
* @param[in] a Input vector whose zero bits define which corresponding bits in result will be set.
*
* @returns Output vector with bits set when corresponding bit in @p a is zero.
*/
inline int16 asymm_mask_if_zero(int16 a)
{
const int16 all_zeros = 0;
const int16 all_ones = ~0;
return select(all_zeros, all_ones, a == 0);
}
/** For each element of input vector, the corresponding bits of the result item are set
* if the input item is non-zero.
*
* @param[in] a Input vector whose non-zero bits define which corresponding bits in result will be set.
*
* @returns Output vector with bits set when corresponding bit in @p a is non zero.
*/
inline int16 asymm_mask_if_non_zero(int16 a)
{
const int16 all_zeros = 0;
const int16 all_ones = ~0;
return select(all_zeros, all_ones, a != 0);
}
/** Each bit of the result is set to the corresponding bit of either then_val or
* else_val depending on whether the corresponding bit of if_mask is set.
* Equivalent to the VBSL instruction in ARM NEON.
*
* @param[in] if_mask Mask defines will bit be taken from @p then_val or @p else_val depending on corresponding bit in mask is set or not.
* @param[in] then_val Value whose bit will be used for result when corresponding bit in @p if_mask is set.
* @param[in] else_val Value whose bit will be used for result when corresponding bit in @p if_mask is not set.
*
* @returns Result contaning bits from @p then_val or from @p else_val depending on corresponding bit in @p if_mask is set or not.
*/
inline int16 asymm_select_using_mask(int16 if_mask, int16 then_val, int16 else_val)
{
return (if_mask & then_val) ^ (~if_mask & else_val);
}
/** Correctly rounded to nearest division by a power of two.
* Also known as a rounding arithmetic right shift.
*
* @param[in] x Value needed to be divided by power of two.
* @param[in] exponent Power of two, must be positive number.
*
* @return Arithmetic right shift.
*/
inline int16 asymm_rounding_divide_by_pow2(int16 x, int exponent)
{
int16 mask = (1 << exponent) - 1;
const int16 zero = 0;
const int16 one = 1;
int16 threshold = (mask >> 1) + select(zero, one, x < 0);
return (x >> exponent) + select(zero, one, (x & mask) > threshold);
}
/** Calculates the product of a integer value by a power of two, with either a positive exponent
* (equivalent to an arithmetic left shift, saturating) or a negative exponent
* (equivalent to an arithmetic right shift, rounding to nearest).
*
* @param[in] x Value needed to be multiplied or divided by power of two depending on sign of @p exponent.
* @param[in] exponent Power of two, can be positive or negative number.
*
* @return Arithmetic left or right shift.
*/
inline int16 asymm_saturating_rounding_mult_by_pow2(int16 x, int exponent)
{
if(exponent < 0)
{
return asymm_rounding_divide_by_pow2(x, -exponent);
}
const int16 min = INT_MIN;
const int16 max = INT_MAX;
int threshold = ((1 << (31 - exponent)) - 1);
int16 positive_mask = asymm_mask_if_non_zero(x > threshold);
int16 negative_mask = asymm_mask_if_non_zero(x < -threshold);
int16 result = x << exponent;
result = asymm_select_using_mask(positive_mask, max, result);
result = asymm_select_using_mask(negative_mask, min, result);
return result;
}
/** Calculates (a+b)/2, rounded to the nearest integer.
* Equivalent to VRHADD in the ARM NEON instruction set.
*
* @param[in] a First term of half-sum.
* @param[in] b Second term of half-sum.
*
* @return (a+b)/2, rounded to the nearest integer.
*/
inline int16 asymm_rounding_half_sum(int16 a, int16 b)
{
long16 a64 = convert_long16(a);
long16 b64 = convert_long16(b);
long16 sum = a64 + b64;
const long16 one = 1;
const long16 minus_one = -1;
long16 sign = select(minus_one, one, sum >= 0);
return convert_int16((sum + sign) / 2);
}
/** Product of two numbers, interpreting them as fixed-point values in the interval [-1, 1),
* rounding to the nearest value, and saturating -1 * -1 to the maximum value.
* This is equivalent to the VQRDMULH instruction in ARM NEON.
*
* @param[in] a First term of product.
* @param[in] b Second term of product.
*
* @return Product of two numbers.
*/
inline int16 asymm_saturating_rounding_doubling_high_mul(int16 a, int16 b)
{
int16 overflow = (a == b) && (a == INT_MIN);
long16 a_64 = convert_long16(a);
long16 b_64 = convert_long16(b);
long16 ab_64 = a_64 * b_64;
long16 mask1 = 1 << 30;
long16 mask2 = 1 - (1 << 30);
long16 nudge = select(mask2, mask1, ab_64 >= 0);
long16 mask = 1ll << 31;
int16 ab_x2_high32 = convert_int16((ab_64 + nudge) / mask);
return select(ab_x2_high32, INT_MAX, overflow);
}
/** Fixed-point multiplication.
*
* @param[in] a Argument 1 in fixed-point format Q(a).
* @param[in] b Argument 2 in fixed-point format Q(b).
*
* @return Result in fixed-point format Q(a+b).
*/
inline int16 asymm_mult(int16 a, int16 b)
{
return asymm_saturating_rounding_doubling_high_mul(a, b);
}
/** Calculates \f$ exp(x) \f$ for x in [-1/4, 0).
*
* @param[in] a Argument in fixed-point format Q0.
*
* @return Result in fixed-point format Q0.
*/
inline int16 asymm_exp_on_interval_between_negative_one_quarter_and_0_excl(int16 a)
{
const int16 constant_term = 1895147668;
const int16 constant_1_over_3 = 715827883;
const int k_fractional_bits = 31;
int16 x = a + (1 << (k_fractional_bits - 3));
int16 x2 = asymm_mult(x, x);
int16 x3 = asymm_mult(x2, x);
int16 x4 = asymm_mult(x2, x2);
int16 x4_over_4 = asymm_rounding_divide_by_pow2(x4, 2);
int16 x4_over_24_plus_x3_over_6_plus_x2 = asymm_mult((x4_over_4 + x3), constant_1_over_3) + x2;
int16 x4_over_24_plus_x3_over_6_plus_x2_over_2 = asymm_rounding_divide_by_pow2(x4_over_24_plus_x3_over_6_plus_x2, 1);
return constant_term + asymm_mult(constant_term, x + x4_over_24_plus_x3_over_6_plus_x2_over_2);
}
/** Calculates \f$ exp(x) \f$ for x < 0.
*
* @param[in] a Argument in fixed-point format Q(k_integer_bits).
* @param[in] k_integer_bits Number of integer bit in argument.
*
* @return Result in fixed-point format Q0.
*/
inline int16 asymm_exp_on_negative_values(int16 a, int k_integer_bits)
{
const int k_fractional_bits = 31 - k_integer_bits;
int16 k_one_quarter = 1 << (k_fractional_bits - 2);
int16 mask = k_one_quarter - 1;
int16 a_mod_quarter_minus_one_quarter = (a & mask) - k_one_quarter;
int16 a_mod_quarter_minus_one_quarter_scaled = a_mod_quarter_minus_one_quarter << k_integer_bits;
int16 result = asymm_exp_on_interval_between_negative_one_quarter_and_0_excl(a_mod_quarter_minus_one_quarter_scaled);
int16 remainder = a_mod_quarter_minus_one_quarter - a;
#define EXP_BARREL_SHIFTER(Exponent, FixedPointMultiplier) \
if(k_integer_bits > Exponent) \
{ \
const int k_shift_amount = k_integer_bits > Exponent ? k_fractional_bits + Exponent : 0; \
result = asymm_select_using_mask( \
asymm_mask_if_non_zero(remainder & (1 << k_shift_amount)), \
asymm_mult(result, FixedPointMultiplier), result); \
}
EXP_BARREL_SHIFTER(-2, 1672461947);
EXP_BARREL_SHIFTER(-1, 1302514674);
EXP_BARREL_SHIFTER(+0, 790015084);
EXP_BARREL_SHIFTER(+1, 290630308);
EXP_BARREL_SHIFTER(+2, 39332535);
EXP_BARREL_SHIFTER(+3, 720401);
EXP_BARREL_SHIFTER(+4, 242);
#undef EXP_BARREL_SHIFTER
if(k_integer_bits > 5)
{
const int16 clamp = -(1 << (k_fractional_bits + 5));
result = asymm_select_using_mask(asymm_mask_if_non_zero(a < clamp), 0, result);
}
const int16 Q0_one = INT_MAX;
return asymm_select_using_mask(asymm_mask_if_zero(a), Q0_one, result);
}
/** Calculates \f$ 1 / (1 + x) \f$ for x in (0, 1).
*
* @param[in] a Argument in fixed-point format Q0.
*
* @return Result in fixed-point format Q0.
*/
inline int16 asymm_one_over_one_plus_x_for_x_in_0_1(int16 a)
{
const int16 Q0_one = INT_MAX;
const int16 Q2_one = 1 << (31 - 2);
int16 half_denominator = asymm_rounding_half_sum(a, Q0_one);
const int16 Q2_48_over_17 = 1515870810;
const int16 Q2_neg_32_over_17 = -1010580540;
int16 x = Q2_48_over_17 + asymm_mult(half_denominator, Q2_neg_32_over_17);
for(int i = 0; i < 3; i++)
{
int16 half_denominator_times_x = asymm_mult(half_denominator, x);
int16 one_minus_half_denominator_times_x = Q2_one - half_denominator_times_x;
int16 tmp = asymm_mult(x, one_minus_half_denominator_times_x);
x = x + asymm_saturating_rounding_mult_by_pow2(tmp, 2);
}
return asymm_saturating_rounding_mult_by_pow2(x, 1);
}
/** Considering the integer value as fixed-point, change the number of integer bits and update value accordingly.
*
* @param[in] value Value to be rescaled.
* @param[in] src_integer_bits Old number of integer bits.
* @param[in] dst_integer_bits New number of integer bits.
*
* @return Rescaled value.
*/
inline int16 asymm_rescale(int16 value, int src_integer_bits, int dst_integer_bits)
{
int exponent = src_integer_bits - dst_integer_bits;
return asymm_saturating_rounding_mult_by_pow2(value, exponent);
}
#endif // ARM_COMPUTE_ASYMM_HELPER_H
|
