diff options
Diffstat (limited to 'chapters/introduction.adoc')
| -rw-r--r-- | chapters/introduction.adoc | 76 |
1 files changed, 51 insertions, 25 deletions
diff --git a/chapters/introduction.adoc b/chapters/introduction.adoc index 0765e95..d6f7bf9 100644 --- a/chapters/introduction.adoc +++ b/chapters/introduction.adoc @@ -287,64 +287,90 @@ where `E = pow(1 + pow(2, -M-1), N) - 1`. In this expression M is the number of ===== Dot product accuracy requirements -This section assumes an operation acting on two tensors named 'input' and 'weight'. -Each output tensor element can be expressed as a dot product of elements between the input and weight tensors. +This section assumes an operation acting on tensors named 'input', 'weight' and optionally 'bias'. +Each output tensor element can be expressed as a dot product of elements between the 'input' and 'weight' tensors with optional bias addition. The dot product has length KS, the kernel size. +If the operation does not specify a bias then 'bias' is taken to be zero in this section. Note: KS is defined for each relevant operator in the appendix section <<Main Inference operator test data>>. -In other words each output element `out` can be expressed as a dot product between input elements `in[k]` and weight elements `w[k]`: +In other words, each output element `out` can be expressed as a dot product between input elements `in[k]`, weight elements `w[k]`, bias `b`: -`out = in[0] * w[0] + in[1] * w[1] + ... + in[KS-1] * w[KS-1]` +`out = in[0] * w[0] + in[1] * w[1] + ... + in[KS-1] * w[KS-1] + b` -The positions of `in[k]` and `w[k]` in the input and weight tensors depends on the operation being performed (for example a convolution). +The positions of `in[k]`, `w[k]`, `b` in the input, weight and bias tensors depends on the operation being performed. +This may be, for example, a convolution. This section defines the accuracy required for these operations. -The term "fp64 arithmetic" refers to double-precision floating-point arithmetic defined by <<Other publications>>[1]. +In this section: -For an operation with given sizes and attributes to be compliant the following must hold for each data set S defined in <<Appendix A>>: +* "fp64 arithmetic" refers to double-precision floating-point arithmetic defined by IEEE 754 (<<Other publications>>[1]) +* `operation_fp64()` is an fp64 reference implementation of the operation +* `operation_imp()` is the implementation under test +* `local_bound` is defined as follows: +** For operations with a local_bound attribute it is the value of the optional attribute, with default value of false +** For operations that do not have a local_bound attribute the value is true -* Let input be the input tensor generated by <<Main Inference operator test data>> for test set S -* Let weight be the weight tensor generated by <<Main Inference operator test data>> for test set S -* Let output_ref be the output tensor calculated by the operation using fp64 arithemic -* Let output_imp be the output tensor calculated by the implementation to test -* Let input_abs be the input tensor with each element replaced with its absolute value -* Let weight_abs be the weight tensor with each element replaced with its absolute value -* Let output_bnd be the output tensor calculated using fp64 arithmetic on input_abs and weight_abs +The checks described in the following code must pass for the following data sets: -The following checks must then pass: +* Data sets defined for the operation in Appendix A <<Main Inference operator test data>>. +* Data sets that have at least MIN_DOT_PRODUCT different output values. For these data sets we take S=-1. [source,c++] ---- +output_ref = operation_fp64(input, weight, bias); +output_imp = operation_imp (input, weight, bias); +input_abs = abs(input); // Element-wise absolute +weight_abs = abs(weight); // Element-wise absolute +bias_abs = abs(bias); // Element-wise absolute +if (!local_bound) { + input_abs_max = max_value(input_abs); // maximum over all elements + for_each(index in shape(input_abs) { + input_abs[index] = input_abs_max; // set all entries to global maximum + } +} +output_bnd = operation_fp64(input_abs, weight_abs, bias_abs); + size_t T = tensor_size(output_shape) // number dot product results +size_t ksb = (max_value(bias_abs) > 0) ? (KS + 1) : KS; // kernel size and bias fp64_t out_err_sum = 0.0; fp64_t out_err_sumsq = 0.0; -fp64_t acc_prec; // 1<<(M+1) where M is the number of mantissa bits +fp64_t acc_prec; // 1<<(M+1) where M is the number of mantissa bits +fp64_t acc_min_normal; // accumulator minimum normal greater than zero +fp64_t two_m63 = -1.0/(fp64)((int64_t)-1<<63); // pow(2, -63) switch (acc_t) { - case fp32_t: acc_prec = (fp64_t)(1<<24); break; - case fp16_t: acc_prec = (fp64_t)(1<<11); break; - default: ERROR_IF(true); + case fp32_t: acc_prec = (fp64_t)(1<<24); // pow(2, 24) + acc_min_normal = two_m63 * two_m63; // pow(2, -126) + break; + case fp16_t: acc_prec = (fp64_t)(1<<11); // pow(2, 11) + acc_min_normal = 1.0/(fp64_t)(1<<14); // pow(2, -14) + break; + default: ERROR_IF(true); } for_each(index in output_shape) { fp64_t out_bnd = tensor_read<fp64_t>(output_bnd, output_shape, index); fp64_t out_ref = tensor_read<fp64_t>(output_ref, output_shape, index); acc_t out_imp = tensor_read<acc_t> (output_imp, output_shape, index); fp64_t out_err; - if (out_bnd == 0.0) { + if ((acc_t)out_bnd == infinity) { + // dot product can overflow and there is no accuracy limit + out_err = 0.0; + } else if (out_bnd == 0.0) { REQUIRE(out_ref == 0.0 && out_imp == 0.0); out_err = 0.0; - } else { // out_bnd > 0.0 + } else { // 0.0 < out_bnd < infinity + out_bnd = max(out_bnd, acc_min_normal); out_err = ((fp64_t)out_imp - out_ref)*acc_prec/out_bnd; - REQUIRE(abs(out_err) <= KS); + REQUIRE(abs(out_err) <= ksb); } out_err_sum += out_err; out_err_sumsq += out_err * out_err; } -if (S!=1 && S!=2) { +if (input and weights are data set S with 3 <= S <= 5) { // check output error bias magnitude for data sets S which are not positive biased - REQUIRE(abs(out_err_sum) <= 2*sqrt(KS*T)); + REQUIRE(abs(out_err_sum) <= 2*sqrt(ksb*T)); } // check output error variance magnitude -REQUIRE(out_err_sumsq <= 0.4*KS*T) +REQUIRE(out_err_sumsq <= 0.4*ksb*T) ---- === Tensor Definitions |