Main Compliance: REDUCE_PRODUCT support

Update and fix REDUCE_PRODUCT compliance verify lib support.
Added compliance test generation with data range to not cause infs.

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I3b3004c6caa80d97e330a6393f435f5270b56e21

10 files changed, 129 insertions, 86 deletions

diff --git a/reference_model/src/generate/generate_utils.cc b/reference_model/src/generate/generate_utils.cc
index d31048f..b94b888 100644
--- a/reference_model/src/generate/generate_utils.cc
+++ b/reference_model/src/generate/generate_utils.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2023, ARM Limited.
+// Copyright (c) 2023-2024, ARM Limited.
 //
 //    Licensed under the Apache License, Version 2.0 (the "License");
 //    you may not use this file except in compliance with the License.
@@ -69,6 +69,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(Op,
                                  { Op::Op_RSQRT, "RSQRT" },
                                  { Op::Op_REDUCE_MAX, "REDUCE_MAX" },
                                  { Op::Op_REDUCE_MIN, "REDUCE_MIN" },
+                                 { Op::Op_REDUCE_PRODUCT, "REDUCE_PRODUCT" },
                                  { Op::Op_REDUCE_SUM, "REDUCE_SUM" },
                                  { Op::Op_SCATTER, "SCATTER" },
                                  { Op::Op_SIGMOID, "SIGMOID" },
diff --git a/reference_model/src/verify/verifiers.h b/reference_model/src/verify/verifiers.h
index 152cd6a..6830115 100644
--- a/reference_model/src/verify/verifiers.h
+++ b/reference_model/src/verify/verifiers.h
@@ -45,11 +45,12 @@ bool verifyExact(const CTensor* referenceTensor, const CTensor* implementationTe
 ///
 /// \param referenceTensor    Reference tensor
 /// \param implementationTensor    Implementation resulting tensor
-/// \param m    Number of manisa bits in the floating point representation
-/// \param n    Number of elements in the product
+/// \param rpInfo Reduce-product verification meta-data
 ///
 /// \return True if compliant else false
-bool verifyReduceProduct(const CTensor* referenceTensor, const CTensor* implementationTensor, uint64_t m, uint64_t n);
+bool verifyReduceProduct(const CTensor* referenceTensor,
+                         const CTensor* implementationTensor,
+                         const ReduceProductVerifyInfo& rpInfo);
 
 /// \brief Perform ULP result verification
 ///
diff --git a/reference_model/src/verify/verify_entry.cc b/reference_model/src/verify/verify_entry.cc
index 2b318d1..afc5916 100644
--- a/reference_model/src/verify/verify_entry.cc
+++ b/reference_model/src/verify/verify_entry.cc
@@ -35,7 +35,7 @@ bool verify(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const
             return verifyExact(ref, imp);
         }
         case VerifyMode::ReduceProduct: {
-            return verifyReduceProduct(ref, imp, cfg.reduceProductInfo.m, cfg.reduceProductInfo.n);
+            return verifyReduceProduct(ref, imp, cfg.reduceProductInfo);
         }
         case VerifyMode::Ulp: {
             return verifyULP(ref, imp, cfg.ulpInfo);
diff --git a/reference_model/src/verify/verify_reduce_product.cc b/reference_model/src/verify/verify_reduce_product.cc
index 625e2cf..5306ef7 100644
--- a/reference_model/src/verify/verify_reduce_product.cc
+++ b/reference_model/src/verify/verify_reduce_product.cc
@@ -1,5 +1,5 @@
 
-// Copyright (c) 2023, ARM Limited.
+// Copyright (c) 2023-2024, ARM Limited.
 //
 //    Licensed under the Apache License, Version 2.0 (the "License");
 //    you may not use this file except in compliance with the License.
@@ -19,64 +19,60 @@
 #include "verifiers.h"
 #include "verify/verify_utils.h"
 
-namespace
+namespace TosaReference
 {
 
-auto calculateError(uint64_t M, uint64_t N)
+namespace
 {
-    return std::pow(1 + std::pow(2, -static_cast<int64_t>(M) - 1), N) - 1;
-}
-
-template <typename FP>
-auto calculateTolerance(uint64_t M, uint64_t N, FP value)
+template <typename OutDtype>
+bool validateData(const double* ref,
+                  const OutDtype* imp,
+                  const std::vector<int32_t>& shape,
+                  const ReduceProductVerifyInfo& cfg)
 {
-    return std::abs(value) * calculateError(M, N);
+    const size_t T = static_cast<size_t>(numElements(shape));
+    TOSA_REF_REQUIRE(T > 0, "[RP] Invalid shape for reference tensor");
+
+    for (size_t i = 0; i < T; ++i)
+    {
+        double errBound =
+            std::abs(ref[i]) * (std::pow(1 + std::pow(2, -AccPrecision<OutDtype>::normal_frac - 1), cfg.n) - 1);
+        bool valid = tosaCheckFloatBound(imp[i], ref[i], errBound);
+        if (!valid)
+        {
+            auto pos = indexToPosition(T, shape);
+            WARNING("[Verifier][RP] Location %s", positionToString(pos).c_str());
+            return false;
+        }
+    }
+    return true;
 }
 }    // namespace
 
-namespace TosaReference
-{
-
-bool verifyReduceProduct(const CTensor* referenceTensor, const CTensor* implementationTensor, uint64_t m, uint64_t n)
+bool verifyReduceProduct(const CTensor* referenceTensor,
+                         const CTensor* implementationTensor,
+                         const ReduceProductVerifyInfo& rpInfo)
 {
     // Validate that tensors are provided
     TOSA_REF_REQUIRE(referenceTensor != nullptr, "[RP] Reference tensor is missing");
     TOSA_REF_REQUIRE(implementationTensor != nullptr, "[RP] Implementation tensor is missing");
 
-    // Get number of elements
-    const auto elementCount =
-        numElements(std::vector<int32_t>(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims));
-    TOSA_REF_REQUIRE(elementCount > 0, "[RP] Invalid shape for reference tensor");
+    const std::vector<int32_t> refShape(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims);
+
+    const double* refData = reinterpret_cast<const double*>(referenceTensor->data);
+    TOSA_REF_REQUIRE(refData != nullptr, "[RP] Missing data for reference");
 
     switch (implementationTensor->data_type)
     {
         case tosa_datatype_fp32_t: {
-            const auto* refData = reinterpret_cast<const float*>(referenceTensor->data);
-            TOSA_REF_REQUIRE(refData != nullptr, "[RP] Missing data for reference");
-
             const auto* impData = reinterpret_cast<const float*>(implementationTensor->data);
             TOSA_REF_REQUIRE(impData != nullptr, "[RP] Missing data for implementation");
-
-            return std::equal(refData, std::next(refData, elementCount), impData, std::next(impData, elementCount),
-                              [m, n](const auto& referenceValue, const auto& implementationValue) {
-                                  // Result overflows must be set to zero of the correct sign.
-                                  if (std::isinf(implementationValue))
-                                  {
-                                      return implementationValue == referenceValue;
-                                  }
-
-                                  // Result underflows must be set to a zero of the correct sign.
-                                  if (implementationValue == 0.f || implementationValue == -0.f)
-                                  {
-                                      return implementationValue == referenceValue;
-                                  }
-
-                                  // Otherwise we are in the normal range.
-                                  const auto absoulteError = (referenceValue < implementationValue)
-                                                                 ? implementationValue - referenceValue
-                                                                 : referenceValue - implementationValue;
-                                  return absoulteError <= calculateTolerance(m, n, implementationValue);
-                              });
+            return validateData(refData, impData, refShape, rpInfo);
+        }
+        case tosa_datatype_fp16_t: {
+            const auto* impData = reinterpret_cast<const half_float::half*>(implementationTensor->data);
+            TOSA_REF_REQUIRE(impData != nullptr, "[RP] Missing data for implementation");
+            return validateData(refData, impData, refShape, rpInfo);
         }
         default:
             WARNING("[Verifier][RP] Data-type not supported.");
diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc
index 5ce646c..6f53c63 100644
--- a/reference_model/src/verify/verify_utils.cc
+++ b/reference_model/src/verify/verify_utils.cc
@@ -66,7 +66,6 @@ void from_json(const nlohmann::json& j, DotProductVerifyInfo& dotProductInfo)
 
 void from_json(const nlohmann::json& j, ReduceProductVerifyInfo& reduceProduceInfo)
 {
-    j.at("m").get_to(reduceProduceInfo.m);
     j.at("n").get_to(reduceProduceInfo.n);
 }
 
diff --git a/reference_model/test/verify_tests.cpp b/reference_model/test/verify_tests.cpp
index f92792a..ba18af1 100644
--- a/reference_model/test/verify_tests.cpp
+++ b/reference_model/test/verify_tests.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2023, ARM Limited.
+// Copyright (c) 2023-2024, ARM Limited.
 //
 //    Licensed under the Apache License, Version 2.0 (the "License");
 //    you may not use this file except in compliance with the License.
@@ -136,13 +136,13 @@ constexpr auto reduceProductError(uint64_t M, uint64_t N)
 template <typename FP>
 auto reduceProductTolerance(uint64_t M, uint64_t N, const std::vector<FP>& results)
 {
-    const auto error = reduceProductError(M, N);
-    auto tolerances  = std::vector<FP>(results.size());
+    const auto error     = reduceProductError(M, N);
+    auto tolerances_fp64 = std::vector<FP>(results.size());
     for (unsigned i = 0, end = results.size(); i < end; ++i)
     {
-        tolerances[i] = std::abs(results[i]) * error;
+        tolerances_fp64[i] = std::abs(results[i]) * error;
     }
-    return tolerances;
+    return tolerances_fp64;
 }
 
 }    // namespace
@@ -299,8 +299,7 @@ TEST_CASE("positive - reduce product")
                 "mode": "REDUCE_PRODUCT",
                 "data_type": "FP32",
                 "reduce_product_info": {
-                "m": 23,
-                "n": 8
+                    "n": 8
                 }
             }
         }
@@ -313,48 +312,49 @@ TEST_CASE("positive - reduce product")
 
     // Generate some random floats using the full range of fp32. This will be the "result" of our
     // dot product. Here we "reduced" over the z-axis of our shape.
-    auto data = generateRandomTensorData<float>(elementCount / reductionSize, false);
-    // Calculate the tolerances for each element in the result.
+    auto data_fp32 = generateRandomTensorData<float>(elementCount / reductionSize, false);
+    std::vector<double> data_fp64(data_fp32.begin(), data_fp32.end());
+    // Calculate the tolerances_fp64 for each element in the result.
     // A float has 23 bit dedicated to the fraction.
     constexpr uint64_t mantisa_count = 23;
-    const auto tolerances            = reduceProductTolerance(mantisa_count, reductionSize, data);
+    const auto tolerances_fp64       = reduceProductTolerance(mantisa_count, reductionSize, data_fp64);
 
     SUBCASE("same")
     {
-        // TODO: Generate some new floats that are as far away as possible from each result without
+        // Generate some new floats that are as far away as possible from each result without
         // exceeding the tolerance.
-        auto otherData = std::vector<float>(elementCount / reductionSize);
-        for (unsigned i = 0; i < data.size(); ++i)
+        auto otherData_fp32 = std::vector<float>(elementCount / reductionSize);
+        for (unsigned i = 0; i < data_fp32.size(); ++i)
         {
-            auto newValue     = data[i];
-            auto oldValue     = newValue;
-            const auto target = tolerances[i] + newValue;
+            auto newValue       = data_fp32[i];
+            const double target = tolerances_fp64[i] + newValue;
 
             // Here we just increment the value until we exceed the tolerance. For simplicity we go up.
+            auto previousValue = newValue;
             while (newValue < target)
             {
-                oldValue = newValue;
-                newValue = std::nextafter(newValue, std::numeric_limits<float>::infinity());
+                previousValue = newValue;
+                newValue      = std::nextafter(newValue, std::numeric_limits<float>::infinity());
             }
 
-            otherData[i] = oldValue;
+            otherData_fp32[i] = previousValue;
         }
 
         const auto referenceTensor =
-            TosaTensor("out1", tosa_datatype_fp64_t, outputShape, reinterpret_cast<uint8_t*>(data.data()));
+            TosaTensor("out1", tosa_datatype_fp64_t, outputShape, reinterpret_cast<uint8_t*>(data_fp64.data()));
         const auto implementationTensor =
-            TosaTensor("out1", tosa_datatype_fp32_t, outputShape, reinterpret_cast<uint8_t*>(otherData.data()));
+            TosaTensor("out1", tosa_datatype_fp32_t, outputShape, reinterpret_cast<uint8_t*>(otherData_fp32.data()));
         REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
     }
 
     SUBCASE("different")
     {
-        // TODO: Generate some new floats that exceed the tolerance.
-        auto otherData = std::vector<float>(elementCount / reductionSize);
-        for (unsigned i = 0; i < data.size(); ++i)
+        // Generate some new floats that exceed the tolerance.
+        auto otherData_fp32 = std::vector<float>(elementCount / reductionSize);
+        for (unsigned i = 0; i < data_fp32.size(); ++i)
         {
-            auto newValue     = data[i];
-            const auto target = tolerances[i] + newValue;
+            auto newValue       = data_fp32[i];
+            const double target = tolerances_fp64[i] + newValue;
 
             // Here we just increment the value until we exceed the tolerance. For simplicity we go up.
             while (newValue < target)
@@ -362,13 +362,13 @@ TEST_CASE("positive - reduce product")
                 newValue = std::nextafter(newValue, std::numeric_limits<float>::infinity());
             }
 
-            otherData[i] = newValue;
+            otherData_fp32[i] = newValue;
         }
 
         const auto referenceTensor =
-            TosaTensor("out1", tosa_datatype_fp64_t, outputShape, reinterpret_cast<uint8_t*>(data.data()));
+            TosaTensor("out1", tosa_datatype_fp64_t, outputShape, reinterpret_cast<uint8_t*>(data_fp64.data()));
         const auto implementationTensor =
-            TosaTensor("out1", tosa_datatype_fp32_t, outputShape, reinterpret_cast<uint8_t*>(otherData.data()));
+            TosaTensor("out1", tosa_datatype_fp32_t, outputShape, reinterpret_cast<uint8_t*>(otherData_fp32.data()));
         REQUIRE_FALSE(
             tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
     }
diff --git a/scripts/schemavalidation/compliance-config.schema.json b/scripts/schemavalidation/compliance-config.schema.json
index dd62404..c0a479d 100644
--- a/scripts/schemavalidation/compliance-config.schema.json
+++ b/scripts/schemavalidation/compliance-config.schema.json
@@ -73,6 +73,21 @@
                                 }
                             },
                             "additionalProperties": false
+                        },
+                        "reduce_product_info": {
+                            "description": "info required for the REDUCE_PRODUCT mode",
+                            "type": "object",
+                            "properties":
+                            {
+                                "n": {
+                                    "description": "number of products in the operation",
+                                    "type": "integer"
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [
+                                "n"
+                            ]
                         }
                     },
                     "additionalProperties": false,
diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json
index fb25622..2b99ed9 100644
--- a/verif/conformance/tosa_main_profile_ops_info.json
+++ b/verif/conformance/tosa_main_profile_ops_info.json
@@ -2507,6 +2507,7 @@
         "profile": [
             "tosa-mi"
         ],
+        "support_for": [ "lazy_data_gen" ],
         "generation": {
             "standard": {
                 "generator_args": [
@@ -2518,13 +2519,15 @@
                         "--target-dtype",
                         "bf16",
                         "--fp-values-range",
-                        "-2.0,2.0",
+                        "-max,max",
                         "--tensor-dim-range",
                         "1,34"
                     ],
                     [
                         "--target-dtype",
                         "fp16",
+                        "--fp-values-range",
+                        "-max,max",
                         "--target-shape",
                         "2,65527,3,1",
                         "--target-shape",
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index 1e23822..8641499 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2023, ARM Limited.
+# Copyright (c) 2021-2024, ARM Limited.
 # SPDX-License-Identifier: Apache-2.0
 import itertools
 import math
@@ -1274,6 +1274,30 @@ class TosaTensorValuesGen:
                 testGen, opName, dtypeList, shapeList, argsDict, error_name
             )
 
+    @staticmethod
+    def tvgReduceProduct(
+        testGen, opName, dtypeList, shapeList, argsDict, error_name=None
+    ):
+        dtype = dtypeList[0]
+        if error_name is None:
+            # Limit ranges for (non error) tests by using
+            # values that can be multiplied on any axis to not hit infinity
+            highval_lookup = {
+                dtype: math.pow(
+                    TosaTensorValuesGen.TVG_FLOAT_HIGH_VALUE[dtype],
+                    1 / max(shapeList[0]),
+                )
+            }
+            data_range = TosaTensorValuesGen._get_data_range(
+                testGen, dtype, highval_lookup
+            )
+            assert data_range is not None
+            argsDict["data_range"] = data_range
+
+        return TosaTensorValuesGen.tvgLazyGenDefault(
+            testGen, opName, dtypeList, shapeList, argsDict, error_name
+        )
+
     # Set the POW exponent high data range
     TVG_FLOAT_HIGH_VALUE_POW_EXP = {
         DType.FP32: 10.0,
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index 5129e24..0d072ac 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -347,6 +347,7 @@ class TosaTestGen:
             compliance_tens["ulp_info"] = {"ulp": op["compliance"]["ulp"]}
         elif op["op"] == Op.REDUCE_PRODUCT:
             mode = gtu.ComplianceMode.REDUCE_PRODUCT
+            compliance_tens["reduce_product_info"] = {"n": argsDict["n"]}
         elif op["op"] in (Op.EXP, Op.POW, Op.TANH, Op.SIGMOID):
             mode = gtu.ComplianceMode.ABS_ERROR
             if "compliance" in op and "abs_error_lower_bound" in op["compliance"]:
@@ -1251,13 +1252,13 @@ class TosaTestGen:
 
         self.ser.addOperator(op["op"], input_list, output_list, attr)
 
-        if op["op"] == Op.REDUCE_PRODUCT:
-            # TODO: Add compliance support!
-            compliance = None
-        else:
-            compliance = self.tensorComplianceMetaData(
-                op, a.dtype, args_dict, result_tensor, error_name
-            )
+        if error_name is None and op["op"] == Op.REDUCE_PRODUCT:
+            # Number of products - needed for compliance
+            args_dict["n"] = a.shape[axis]
+
+        compliance = self.tensorComplianceMetaData(
+            op, a.dtype, args_dict, result_tensor, error_name
+        )
 
         return TosaTestGen.BuildInfo(result_tensor, compliance)
 
@@ -4066,7 +4067,7 @@ class TosaTestGen:
             "build_fcn": (
                 build_reduce,
                 TosaTensorGen.tgBasic,
-                TosaTensorValuesGen.tvgLazyGenDefault,
+                TosaTensorValuesGen.tvgReduceProduct,
                 TosaArgGen.agAxis,
             ),
             "types": TYPE_FP,
@@ -4080,6 +4081,9 @@ class TosaTestGen:
                 TosaErrorValidator.evWrongInputList,
                 TosaErrorValidator.evWrongOutputList,
             ),
+            "data_gen": {
+                "fp": (gtu.DataGenType.PSEUDO_RANDOM,),
+            },
         },
         "reduce_sum": {
             "op": Op.REDUCE_SUM,