Main Compliance: Add RESIZE support

Add RELATIVE verify mode for RESIZE.

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I4fe352579507211dae7a048bf080c24426ce42a2

14 files changed, 305 insertions, 34 deletions

diff --git a/reference_model/CMakeLists.txt b/reference_model/CMakeLists.txt
index d2dce3c..0f806fc 100644
--- a/reference_model/CMakeLists.txt
+++ b/reference_model/CMakeLists.txt
@@ -81,6 +81,7 @@ set(CXX_SOURCE
     src/verify/verify_entry.cc
     src/verify/verify_exact.cc
     src/verify/verify_reduce_product.cc
+    src/verify/verify_relative.cc
     src/verify/verify_ulp.cc
     src/verify/verify_utils.cc
     src/ops/op_factory.cc
@@ -157,6 +158,7 @@ add_library(tosa_reference_verify_lib SHARED
   src/verify/verify_entry.cc
   src/verify/verify_exact.cc
   src/verify/verify_reduce_product.cc
+  src/verify/verify_relative.cc
   src/verify/verify_ulp.cc
   src/verify/verify_utils.cc
   src/verify/verify_config.cc
diff --git a/reference_model/src/generate/generate_utils.cc b/reference_model/src/generate/generate_utils.cc
index cf5308b..8b16e97 100644
--- a/reference_model/src/generate/generate_utils.cc
+++ b/reference_model/src/generate/generate_utils.cc
@@ -71,14 +71,15 @@ NLOHMANN_JSON_SERIALIZE_ENUM(Op,
                                  { Op::Op_PAD, "PAD" },
                                  { Op::Op_POW, "POW" },
                                  { Op::Op_RECIPROCAL, "RECIPROCAL" },
-                                 { Op::Op_RESHAPE, "RESHAPE" },
-                                 { Op::Op_RSQRT, "RSQRT" },
                                  { Op::Op_REDUCE_MAX, "REDUCE_MAX" },
                                  { Op::Op_REDUCE_MIN, "REDUCE_MIN" },
                                  { Op::Op_REDUCE_PRODUCT, "REDUCE_PRODUCT" },
                                  { Op::Op_REDUCE_SUM, "REDUCE_SUM" },
+                                 { Op::Op_RESHAPE, "RESHAPE" },
+                                 { Op::Op_RESIZE, "RESIZE" },
                                  { Op::Op_REVERSE, "REVERSE" },
                                  { Op::Op_RFFT2D, "RFFT2D" },
+                                 { Op::Op_RSQRT, "RSQRT" },
                                  { Op::Op_SCATTER, "SCATTER" },
                                  { Op::Op_SELECT, "SELECT" },
                                  { Op::Op_SIGMOID, "SIGMOID" },
diff --git a/reference_model/src/verify/verifiers.h b/reference_model/src/verify/verifiers.h
index 6830115..80b6e19 100644
--- a/reference_model/src/verify/verifiers.h
+++ b/reference_model/src/verify/verifiers.h
@@ -71,6 +71,17 @@ bool verifyULP(const CTensor* referenceTensor, const CTensor* implementationTens
 /// \return True if compliant else false
 bool verifyAbsError(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const AbsErrorVerifyInfo& aeInfo);
 
+/// \brief Perform relative result verification
+///
+/// \param referenceTensor    Reference tensor
+/// \param implementationTensor    Implementation resulting tensor
+/// \param rInfo Relative verification meta-data
+///
+/// \return True if compliant else false
+bool verifyRelative(const CTensor* referenceTensor,
+                    const CTensor* implementationTensor,
+                    const RelativeVerifyInfo& rInfo);
+
 };    // namespace TosaReference
 
 #endif    // VERIFIERS_H_
diff --git a/reference_model/src/verify/verify_entry.cc b/reference_model/src/verify/verify_entry.cc
index afc5916..9702c36 100644
--- a/reference_model/src/verify/verify_entry.cc
+++ b/reference_model/src/verify/verify_entry.cc
@@ -43,6 +43,9 @@ bool verify(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const
         case VerifyMode::AbsError: {
             return verifyAbsError(ref, refBnd, imp, cfg.absErrorInfo);
         }
+        case VerifyMode::Relative: {
+            return verifyRelative(ref, imp, cfg.relativeInfo);
+        }
         default: {
             WARNING("[Verifier] Unsupported verification mode.");
             break;
diff --git a/reference_model/src/verify/verify_relative.cc b/reference_model/src/verify/verify_relative.cc
new file mode 100644
index 0000000..b12daf7
--- /dev/null
+++ b/reference_model/src/verify/verify_relative.cc
@@ -0,0 +1,83 @@
+
+// Copyright (c) 2024, ARM Limited.
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//         http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+#include <cmath>
+#include <vector>
+
+#include "verifiers.h"
+#include "verify/verify_utils.h"
+
+namespace TosaReference
+{
+
+namespace
+{
+template <typename OutDtype>
+bool validateData(const double* ref,
+                  const OutDtype* imp,
+                  const std::vector<int32_t>& shape,
+                  const RelativeVerifyInfo& cfg)
+{
+    const size_t T = static_cast<size_t>(numElements(shape));
+    TOSA_REF_REQUIRE(T > 0, "[R] Invalid shape for reference tensor");
+
+    double errBound = cfg.max * cfg.scale;
+    for (size_t i = 0; i < T; ++i)
+    {
+        bool valid = tosaCheckFloatBound(imp[i], ref[i], errBound);
+        if (!valid)
+        {
+            auto pos = indexToPosition(i, shape);
+            WARNING("[Verifier][RP] Location %s", positionToString(pos).c_str());
+            return false;
+        }
+    }
+    return true;
+}
+}    // namespace
+
+bool verifyRelative(const CTensor* referenceTensor,
+                    const CTensor* implementationTensor,
+                    const RelativeVerifyInfo& rInfo)
+{
+    // Validate that tensors are provided
+    TOSA_REF_REQUIRE(referenceTensor != nullptr, "[R] Reference tensor is missing");
+    TOSA_REF_REQUIRE(implementationTensor != nullptr, "[R] Implementation tensor is missing");
+
+    const std::vector<int32_t> refShape(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims);
+
+    const double* refData = reinterpret_cast<const double*>(referenceTensor->data);
+    TOSA_REF_REQUIRE(refData != nullptr, "[R] Missing data for reference");
+
+    switch (implementationTensor->data_type)
+    {
+        case tosa_datatype_fp32_t: {
+            const auto* impData = reinterpret_cast<const float*>(implementationTensor->data);
+            TOSA_REF_REQUIRE(impData != nullptr, "[R] Missing data for implementation");
+            return validateData(refData, impData, refShape, rInfo);
+        }
+        case tosa_datatype_fp16_t: {
+            const auto* impData = reinterpret_cast<const half_float::half*>(implementationTensor->data);
+            TOSA_REF_REQUIRE(impData != nullptr, "[R] Missing data for implementation");
+            return validateData(refData, impData, refShape, rInfo);
+        }
+        default:
+            WARNING("[Verifier][R] Data-type not supported.");
+            break;
+    }
+
+    return false;
+}
+}    // namespace TosaReference
diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc
index abb55eb..14bc6f1 100644
--- a/reference_model/src/verify/verify_utils.cc
+++ b/reference_model/src/verify/verify_utils.cc
@@ -52,6 +52,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(VerifyMode,
                                  { VerifyMode::FpSpecial, "FP_SPECIAL" },
                                  { VerifyMode::ReduceProduct, "REDUCE_PRODUCT" },
                                  { VerifyMode::AbsError, "ABS_ERROR" },
+                                 { VerifyMode::Relative, "RELATIVE" },
                              })
 
 void from_json(const nlohmann::json& j, UlpVerifyInfo& ulpInfo)
@@ -78,6 +79,12 @@ void from_json(const nlohmann::json& j, AbsErrorVerifyInfo& absErrorInfo)
     }
 }
 
+void from_json(const nlohmann::json& j, RelativeVerifyInfo& rInfo)
+{
+    j.at("max").get_to(rInfo.max);
+    j.at("scale").get_to(rInfo.scale);
+}
+
 void from_json(const nlohmann::json& j, VerifyConfig& cfg)
 {
     j.at("mode").get_to(cfg.mode);
@@ -100,6 +107,10 @@ void from_json(const nlohmann::json& j, VerifyConfig& cfg)
     {
         j.at("abs_error_info").get_to(cfg.absErrorInfo);
     }
+    if (j.contains("relative_info"))
+    {
+        j.at("relative_info").get_to(cfg.relativeInfo);
+    }
 }
 
 std::optional<VerifyConfig> parseVerifyConfig(const char* tensorName, const char* json)
diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h
index 0fc68fb..341bd90 100644
--- a/reference_model/src/verify/verify_utils.h
+++ b/reference_model/src/verify/verify_utils.h
@@ -46,7 +46,8 @@ enum class VerifyMode
     DotProduct,
     FpSpecial,
     ReduceProduct,
-    AbsError
+    AbsError,
+    Relative
 };
 
 /// \brief ULP verification meta-data
@@ -83,6 +84,15 @@ struct AbsErrorVerifyInfo
     double lowerBound;
 };
 
+/// \brief relative verification meta-data
+struct RelativeVerifyInfo
+{
+    RelativeVerifyInfo() = default;
+
+    double max;
+    double scale;
+};
+
 /// \brief Verification meta-data
 struct VerifyConfig
 {
@@ -94,6 +104,7 @@ struct VerifyConfig
     DotProductVerifyInfo dotProductInfo;
     ReduceProductVerifyInfo reduceProductInfo;
     AbsErrorVerifyInfo absErrorInfo;
+    RelativeVerifyInfo relativeInfo;
 };
 
 /// \brief Parse the verification config for a tensor when given in JSON form
diff --git a/reference_model/test/verify_tests.cpp b/reference_model/test/verify_tests.cpp
index ba18af1..a85546e 100644
--- a/reference_model/test/verify_tests.cpp
+++ b/reference_model/test/verify_tests.cpp
@@ -29,6 +29,16 @@
 
 namespace
 {
+void update_json_template(std::string& str, const std::string& find, const std::string& change)
+{
+    // Update the 'str' by looking for instances of 'find' and replacing them with 'change'
+    auto pos = str.find(find);
+    while (pos != std::string::npos)
+    {
+        str.replace(pos, find.length(), change);
+        pos = str.find(find);
+    }
+}
 
 class TosaTensor
 {
@@ -472,7 +482,7 @@ TEST_CASE("positive - abs error")
 
     SUBCASE("outside")
     {
-        // Generate some data that exceeds a specified number of ULP for each value in the tensor.
+        // Generate some data that exceeds a requirements for each value in the tensor.
         auto otherData_fp32 = data_fp32;
         std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [outsideErrBound](auto& value) {
             if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value))
@@ -489,4 +499,76 @@ TEST_CASE("positive - abs error")
                                       jsonCfg.c_str()));
     }
 }
+
+TEST_CASE("positive - relative")
+{
+    std::string templateJsonCfg = R"({
+        "tensors" : {
+            "out1" : {
+                "mode": "RELATIVE",
+                "data_type": "FP32",
+                "relative_info": {
+                    "max": _MAXIMUM_,
+                    "scale": _SCALE_
+                }
+            }
+        }
+    })";
+
+    const auto shape        = std::vector<int32_t>{ 3, 3, 3 };
+    const auto elementCount = std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>());
+
+    // Generate some random floats using the full range of fp32.
+    auto data_fp32 = generateRandomTensorData<float>(elementCount, true);
+    std::vector<double> data_fp64(data_fp32.begin(), data_fp32.end());
+
+    float scale = 0.0006;
+    float max   = 0.0;
+    std::for_each(std::begin(data_fp32), std::end(data_fp32), [&max](auto& value) {
+        if (!std::isinf(value) && !std::isnan(value))
+        {
+            max = std::max(max, std::abs(value));
+        }
+    });
+    std::string jsonCfg = templateJsonCfg;
+    update_json_template(jsonCfg, "_MAXIMUM_", std::to_string(max));
+    update_json_template(jsonCfg, "_SCALE_", std::to_string(scale));
+
+    float errBound = max * scale;
+    // Use 10% error margin to test due to using v.large values in our random data
+    float insideErrBound  = errBound * 0.9;
+    float outsideErrBound = errBound * 1.1;
+
+    SUBCASE("inside")
+    {
+        // Generate some data that meets the requirements of the result.
+        auto otherData_fp32 = data_fp32;
+        std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [insideErrBound](auto& value) {
+            if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value))
+                value += insideErrBound;
+        });
+        const auto referenceTensor =
+            TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data_fp64.data()));
+        const auto implementationTensor =
+            TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData_fp32.data()));
+        REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
+    }
+
+    SUBCASE("outside")
+    {
+        // Generate some data that exceeds the requirements for each value in the tensor.
+        auto otherData_fp32 = data_fp32;
+        std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [outsideErrBound](auto& value) {
+            if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value))
+                value += outsideErrBound;
+        });
+
+        const auto referenceTensor =
+            TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data_fp64.data()));
+        const auto implementationTensor =
+            TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData_fp32.data()));
+        REQUIRE_FALSE(
+            tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
+    }
+}
 TEST_SUITE_END();    // verify
diff --git a/scripts/schemavalidation/compliance-config.schema.json b/scripts/schemavalidation/compliance-config.schema.json
index c0a479d..eb3ccde 100644
--- a/scripts/schemavalidation/compliance-config.schema.json
+++ b/scripts/schemavalidation/compliance-config.schema.json
@@ -88,7 +88,30 @@
                             "required": [
                                 "n"
                             ]
+                        },
+                        "relative_info": {
+                            "description": "info required for the RELATIVE mode",
+                            "type": "object",
+                            "properties":
+                            {
+                                "max": {
+                                    "description": "maximum absolute input value",
+                                    "type": "number",
+                                    "minimum": 0
+                                },
+                                "scale": {
+                                    "description": "relative scaling factor",
+                                    "type": "number",
+                                    "minimum": 0
+                                }
+                            },
+                            "additionalProperties": false,
+                            "required": [
+                                "max",
+                                "scale"
+                            ]
                         }
+
                     },
                     "additionalProperties": false,
                     "required": [
diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json
index b8efd35..dc28bef 100644
--- a/verif/conformance/tosa_main_profile_ops_info.json
+++ b/verif/conformance/tosa_main_profile_ops_info.json
@@ -2645,6 +2645,7 @@
         "profile": [
             "tosa-mi"
         ],
+        "support_for": [ "lazy_data_gen" ],
         "generation": {
             "standard": {
                 "negative_dim_range": "1,10",
@@ -2657,13 +2658,13 @@
                         "--target-dtype",
                         "bf16",
                         "--fp-values-range",
-                        "-2.0,2.0"
+                        "-max,max"
                     ],
                     [
                         "--target-dtype",
                         "fp32",
                         "--fp-values-range",
-                        "-2.0,2.0",
+                        "-max,max",
                         "--target-shape",
                         "1,1103,1,2",
                         "--max-resize-output-dim",
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index a4bced3..4630f35 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -1394,6 +1394,22 @@ class TosaTensorValuesGen:
             testGen, opName, dtypeList, shapeList, argsDict, error_name
         )
 
+    @staticmethod
+    def tvgResize(testGen, opName, dtypeList, shapeList, argsDict, error_name=None):
+        data_range = TosaTensorValuesGen._get_data_range(
+            testGen,
+            dtypeList[0],
+            TosaTensorValuesGen.TVG_FLOAT_HIGH_VALUE,
+        )
+        if data_range:
+            argsDict["data_range"] = data_range
+            # Needed for compliance
+            argsDict["max_abs_value"] = data_range[1]
+
+        return TosaTensorValuesGen.tvgLazyGenDefault(
+            testGen, opName, dtypeList, shapeList, argsDict, error_name
+        )
+
     # Set the POW exponent high data range
     TVG_FLOAT_HIGH_VALUE_POW_EXP = {
         DType.FP32: 10.0,
@@ -3343,14 +3359,13 @@ class TosaArgGen:
                             border[0],
                             border[1],
                         ),
-                        [
-                            mode,
-                            scale,
-                            offset,
-                            border,
-                            dtype,
-                            outputDTypeNew,
-                        ],
+                        {
+                            "mode": mode,
+                            "scale": scale,
+                            "offset": offset,
+                            "border": border,
+                            "output_dtype": outputDTypeNew,
+                        },
                     )
                     if arg_to_append in arg_list:
                         # Skip already generated test params
@@ -3359,6 +3374,16 @@ class TosaArgGen:
                     # Valid permutation
                     perm += 1
                     arg_list.append(arg_to_append)
+
+        # Now add data generator types
+        arg_list = TosaArgGen._add_data_generators(
+            testGen,
+            opName,
+            dtype,
+            arg_list,
+            error_name,
+        )
+        # Return list of tuples: (arg_str, args_dict)
         return arg_list
 
     @staticmethod
diff --git a/verif/generator/tosa_error_if.py b/verif/generator/tosa_error_if.py
index 90c3428..5fd647a 100644
--- a/verif/generator/tosa_error_if.py
+++ b/verif/generator/tosa_error_if.py
@@ -2585,9 +2585,9 @@ class TosaInvalidValidator:
     @staticmethod
     def ivWrongDataTypeOrModeResize(**kwargs):
         input_dtype = kwargs["input_dtype"]
-        args = kwargs["args"]
-        mode = args[0]
-        output_dtype = args[5]
+        args_dict = kwargs["args"]
+        mode = args_dict["mode"]
+        output_dtype = args_dict["output_dtype"]
 
         if mode == ResizeMode.BILINEAR:
             # Invalid output data type / Invalid input datatype
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index d82f919..ae689b4 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -357,6 +357,12 @@ class TosaTestGen:
         elif "compliance" in op and "ulp" in op["compliance"]:
             mode = gtu.ComplianceMode.ULP
             compliance_tens["ulp_info"] = {"ulp": op["compliance"]["ulp"]}
+        elif "compliance" in op and "relative" in op["compliance"]:
+            mode = gtu.ComplianceMode.RELATIVE
+            compliance_tens["relative_info"] = {
+                "max": argsDict["max_abs_value"],
+                "scale": op["compliance"]["relative"],
+            }
         elif op["op"] == Op.REDUCE_PRODUCT:
             mode = gtu.ComplianceMode.REDUCE_PRODUCT
             compliance_tens["reduce_product_info"] = {"n": argsDict["n"]}
@@ -1933,17 +1939,21 @@ class TosaTestGen:
     def build_resize(
         self,
         op,
-        input,
-        mode,
-        scale,
-        offset,
-        border,
-        input_dtype,
-        output_dtype,
+        inputs,
+        args_dict,
         validator_fcns,
         error_name=None,
+        qinfo=None,
     ):
-        result_tens = OutputShaper.resizeOp(
+        assert len(inputs) == 1
+        input = inputs[0]
+        mode = args_dict["mode"]
+        scale = args_dict["scale"]
+        offset = args_dict["offset"]
+        border = args_dict["border"]
+        output_dtype = args_dict["output_dtype"]
+
+        result_tensor = OutputShaper.resizeOp(
             self.ser,
             self.rng,
             input,
@@ -1951,14 +1961,14 @@ class TosaTestGen:
             scale,
             offset,
             border,
-            input_dtype,
+            input.dtype,
             output_dtype,
             error_name,
         )
 
         # Invalidate Input/Output list for error if checks.
         input_list = [input.name]
-        output_list = [result_tens.name]
+        output_list = [result_tensor.name]
         pCount, cCount = op["operands"]
         num_operands = pCount + cCount
         input_list, output_list = TosaErrorIfArgGen.eiInvalidateInputOutputList(
@@ -1972,25 +1982,28 @@ class TosaTestGen:
             op=op,
             mode=mode,
             scale=scale,
-            input_dtype=input_dtype,
+            input_dtype=input.dtype,
             output_dtype=output_dtype,
             input_shape=input.shape,
-            output_shape=result_tens.shape,
+            output_shape=result_tensor.shape,
             offset=offset,
             border=border,
             input_list=input_list,
             output_list=output_list,
-            result_tensors=[result_tens],
+            result_tensors=[result_tensor],
             num_operands=num_operands,
         ):
             return None
 
         attr = ts.TosaSerializerAttribute()
-
         attr.ResizeAttribute(scale, offset, border, mode)
-
         self.ser.addOperator(op["op"], input_list, output_list, attr)
-        return result_tens
+
+        compliance = self.tensorComplianceMetaData(
+            op, input.dtype, args_dict, result_tensor, error_name
+        )
+
+        return TosaTestGen.BuildInfo(result_tensor, compliance)
 
     def build_identityn(self, op, val, val2, validator_fcns=None, error_name=None):
         result_tens = OutputShaper.unaryOp(self.ser, self.rng, val, error_name)
@@ -4610,7 +4623,7 @@ class TosaTestGen:
             "build_fcn": (
                 build_resize,
                 TosaTensorGen.tgNHWC,
-                TosaTensorValuesGen.tvgDefault,
+                TosaTensorValuesGen.tvgResize,
                 TosaArgGen.agResize,
             ),
             "types": (DType.INT8, DType.INT16, DType.FP16, DType.BF16, DType.FP32),
@@ -4636,6 +4649,10 @@ class TosaTestGen:
                 TosaErrorValidator.evResizeOutputShapeMismatch,
                 TosaErrorValidator.evResizeOutputShapeNonInteger,
             ),
+            "data_gen": {
+                "fp": (gtu.DataGenType.PSEUDO_RANDOM,),
+            },
+            "compliance": {"relative": 0.006},
         },
         # Type conversion
         "cast": {
diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py
index 6387d06..76e7388 100644
--- a/verif/generator/tosa_utils.py
+++ b/verif/generator/tosa_utils.py
@@ -39,6 +39,7 @@ class ComplianceMode(IntEnum):
     FP_SPECIAL = 3
     REDUCE_PRODUCT = 4
     ABS_ERROR = 5
+    RELATIVE = 6
 
 
 class DataGenType(IntEnum):