From 32d0b5af61d978d9932ac5576b42203e57881168 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Thu, 1 Feb 2024 15:54:07 +0000 Subject: Main Compliance: Add RESIZE support Add RELATIVE verify mode for RESIZE. Signed-off-by: Jeremy Johnson Change-Id: I4fe352579507211dae7a048bf080c24426ce42a2 --- reference_model/CMakeLists.txt | 2 + reference_model/src/generate/generate_utils.cc | 5 +- reference_model/src/verify/verifiers.h | 11 +++ reference_model/src/verify/verify_entry.cc | 3 + reference_model/src/verify/verify_relative.cc | 83 +++++++++++++++++++++ reference_model/src/verify/verify_utils.cc | 11 +++ reference_model/src/verify/verify_utils.h | 13 +++- reference_model/test/verify_tests.cpp | 84 +++++++++++++++++++++- .../schemavalidation/compliance-config.schema.json | 23 ++++++ verif/conformance/tosa_main_profile_ops_info.json | 5 +- verif/generator/tosa_arg_gen.py | 41 ++++++++--- verif/generator/tosa_error_if.py | 6 +- verif/generator/tosa_test_gen.py | 51 ++++++++----- verif/generator/tosa_utils.py | 1 + 14 files changed, 305 insertions(+), 34 deletions(-) create mode 100644 reference_model/src/verify/verify_relative.cc diff --git a/reference_model/CMakeLists.txt b/reference_model/CMakeLists.txt index d2dce3c..0f806fc 100644 --- a/reference_model/CMakeLists.txt +++ b/reference_model/CMakeLists.txt @@ -81,6 +81,7 @@ set(CXX_SOURCE src/verify/verify_entry.cc src/verify/verify_exact.cc src/verify/verify_reduce_product.cc + src/verify/verify_relative.cc src/verify/verify_ulp.cc src/verify/verify_utils.cc src/ops/op_factory.cc @@ -157,6 +158,7 @@ add_library(tosa_reference_verify_lib SHARED src/verify/verify_entry.cc src/verify/verify_exact.cc src/verify/verify_reduce_product.cc + src/verify/verify_relative.cc src/verify/verify_ulp.cc src/verify/verify_utils.cc src/verify/verify_config.cc diff --git a/reference_model/src/generate/generate_utils.cc b/reference_model/src/generate/generate_utils.cc index cf5308b..8b16e97 100644 --- a/reference_model/src/generate/generate_utils.cc +++ b/reference_model/src/generate/generate_utils.cc @@ -71,14 +71,15 @@ NLOHMANN_JSON_SERIALIZE_ENUM(Op, { Op::Op_PAD, "PAD" }, { Op::Op_POW, "POW" }, { Op::Op_RECIPROCAL, "RECIPROCAL" }, - { Op::Op_RESHAPE, "RESHAPE" }, - { Op::Op_RSQRT, "RSQRT" }, { Op::Op_REDUCE_MAX, "REDUCE_MAX" }, { Op::Op_REDUCE_MIN, "REDUCE_MIN" }, { Op::Op_REDUCE_PRODUCT, "REDUCE_PRODUCT" }, { Op::Op_REDUCE_SUM, "REDUCE_SUM" }, + { Op::Op_RESHAPE, "RESHAPE" }, + { Op::Op_RESIZE, "RESIZE" }, { Op::Op_REVERSE, "REVERSE" }, { Op::Op_RFFT2D, "RFFT2D" }, + { Op::Op_RSQRT, "RSQRT" }, { Op::Op_SCATTER, "SCATTER" }, { Op::Op_SELECT, "SELECT" }, { Op::Op_SIGMOID, "SIGMOID" }, diff --git a/reference_model/src/verify/verifiers.h b/reference_model/src/verify/verifiers.h index 6830115..80b6e19 100644 --- a/reference_model/src/verify/verifiers.h +++ b/reference_model/src/verify/verifiers.h @@ -71,6 +71,17 @@ bool verifyULP(const CTensor* referenceTensor, const CTensor* implementationTens /// \return True if compliant else false bool verifyAbsError(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const AbsErrorVerifyInfo& aeInfo); +/// \brief Perform relative result verification +/// +/// \param referenceTensor Reference tensor +/// \param implementationTensor Implementation resulting tensor +/// \param rInfo Relative verification meta-data +/// +/// \return True if compliant else false +bool verifyRelative(const CTensor* referenceTensor, + const CTensor* implementationTensor, + const RelativeVerifyInfo& rInfo); + }; // namespace TosaReference #endif // VERIFIERS_H_ diff --git a/reference_model/src/verify/verify_entry.cc b/reference_model/src/verify/verify_entry.cc index afc5916..9702c36 100644 --- a/reference_model/src/verify/verify_entry.cc +++ b/reference_model/src/verify/verify_entry.cc @@ -43,6 +43,9 @@ bool verify(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const case VerifyMode::AbsError: { return verifyAbsError(ref, refBnd, imp, cfg.absErrorInfo); } + case VerifyMode::Relative: { + return verifyRelative(ref, imp, cfg.relativeInfo); + } default: { WARNING("[Verifier] Unsupported verification mode."); break; diff --git a/reference_model/src/verify/verify_relative.cc b/reference_model/src/verify/verify_relative.cc new file mode 100644 index 0000000..b12daf7 --- /dev/null +++ b/reference_model/src/verify/verify_relative.cc @@ -0,0 +1,83 @@ + +// Copyright (c) 2024, ARM Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "verifiers.h" +#include "verify/verify_utils.h" + +namespace TosaReference +{ + +namespace +{ +template +bool validateData(const double* ref, + const OutDtype* imp, + const std::vector& shape, + const RelativeVerifyInfo& cfg) +{ + const size_t T = static_cast(numElements(shape)); + TOSA_REF_REQUIRE(T > 0, "[R] Invalid shape for reference tensor"); + + double errBound = cfg.max * cfg.scale; + for (size_t i = 0; i < T; ++i) + { + bool valid = tosaCheckFloatBound(imp[i], ref[i], errBound); + if (!valid) + { + auto pos = indexToPosition(i, shape); + WARNING("[Verifier][RP] Location %s", positionToString(pos).c_str()); + return false; + } + } + return true; +} +} // namespace + +bool verifyRelative(const CTensor* referenceTensor, + const CTensor* implementationTensor, + const RelativeVerifyInfo& rInfo) +{ + // Validate that tensors are provided + TOSA_REF_REQUIRE(referenceTensor != nullptr, "[R] Reference tensor is missing"); + TOSA_REF_REQUIRE(implementationTensor != nullptr, "[R] Implementation tensor is missing"); + + const std::vector refShape(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims); + + const double* refData = reinterpret_cast(referenceTensor->data); + TOSA_REF_REQUIRE(refData != nullptr, "[R] Missing data for reference"); + + switch (implementationTensor->data_type) + { + case tosa_datatype_fp32_t: { + const auto* impData = reinterpret_cast(implementationTensor->data); + TOSA_REF_REQUIRE(impData != nullptr, "[R] Missing data for implementation"); + return validateData(refData, impData, refShape, rInfo); + } + case tosa_datatype_fp16_t: { + const auto* impData = reinterpret_cast(implementationTensor->data); + TOSA_REF_REQUIRE(impData != nullptr, "[R] Missing data for implementation"); + return validateData(refData, impData, refShape, rInfo); + } + default: + WARNING("[Verifier][R] Data-type not supported."); + break; + } + + return false; +} +} // namespace TosaReference diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc index abb55eb..14bc6f1 100644 --- a/reference_model/src/verify/verify_utils.cc +++ b/reference_model/src/verify/verify_utils.cc @@ -52,6 +52,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(VerifyMode, { VerifyMode::FpSpecial, "FP_SPECIAL" }, { VerifyMode::ReduceProduct, "REDUCE_PRODUCT" }, { VerifyMode::AbsError, "ABS_ERROR" }, + { VerifyMode::Relative, "RELATIVE" }, }) void from_json(const nlohmann::json& j, UlpVerifyInfo& ulpInfo) @@ -78,6 +79,12 @@ void from_json(const nlohmann::json& j, AbsErrorVerifyInfo& absErrorInfo) } } +void from_json(const nlohmann::json& j, RelativeVerifyInfo& rInfo) +{ + j.at("max").get_to(rInfo.max); + j.at("scale").get_to(rInfo.scale); +} + void from_json(const nlohmann::json& j, VerifyConfig& cfg) { j.at("mode").get_to(cfg.mode); @@ -100,6 +107,10 @@ void from_json(const nlohmann::json& j, VerifyConfig& cfg) { j.at("abs_error_info").get_to(cfg.absErrorInfo); } + if (j.contains("relative_info")) + { + j.at("relative_info").get_to(cfg.relativeInfo); + } } std::optional parseVerifyConfig(const char* tensorName, const char* json) diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h index 0fc68fb..341bd90 100644 --- a/reference_model/src/verify/verify_utils.h +++ b/reference_model/src/verify/verify_utils.h @@ -46,7 +46,8 @@ enum class VerifyMode DotProduct, FpSpecial, ReduceProduct, - AbsError + AbsError, + Relative }; /// \brief ULP verification meta-data @@ -83,6 +84,15 @@ struct AbsErrorVerifyInfo double lowerBound; }; +/// \brief relative verification meta-data +struct RelativeVerifyInfo +{ + RelativeVerifyInfo() = default; + + double max; + double scale; +}; + /// \brief Verification meta-data struct VerifyConfig { @@ -94,6 +104,7 @@ struct VerifyConfig DotProductVerifyInfo dotProductInfo; ReduceProductVerifyInfo reduceProductInfo; AbsErrorVerifyInfo absErrorInfo; + RelativeVerifyInfo relativeInfo; }; /// \brief Parse the verification config for a tensor when given in JSON form diff --git a/reference_model/test/verify_tests.cpp b/reference_model/test/verify_tests.cpp index ba18af1..a85546e 100644 --- a/reference_model/test/verify_tests.cpp +++ b/reference_model/test/verify_tests.cpp @@ -29,6 +29,16 @@ namespace { +void update_json_template(std::string& str, const std::string& find, const std::string& change) +{ + // Update the 'str' by looking for instances of 'find' and replacing them with 'change' + auto pos = str.find(find); + while (pos != std::string::npos) + { + str.replace(pos, find.length(), change); + pos = str.find(find); + } +} class TosaTensor { @@ -472,7 +482,7 @@ TEST_CASE("positive - abs error") SUBCASE("outside") { - // Generate some data that exceeds a specified number of ULP for each value in the tensor. + // Generate some data that exceeds a requirements for each value in the tensor. auto otherData_fp32 = data_fp32; std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [outsideErrBound](auto& value) { if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value)) @@ -489,4 +499,76 @@ TEST_CASE("positive - abs error") jsonCfg.c_str())); } } + +TEST_CASE("positive - relative") +{ + std::string templateJsonCfg = R"({ + "tensors" : { + "out1" : { + "mode": "RELATIVE", + "data_type": "FP32", + "relative_info": { + "max": _MAXIMUM_, + "scale": _SCALE_ + } + } + } + })"; + + const auto shape = std::vector{ 3, 3, 3 }; + const auto elementCount = std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>()); + + // Generate some random floats using the full range of fp32. + auto data_fp32 = generateRandomTensorData(elementCount, true); + std::vector data_fp64(data_fp32.begin(), data_fp32.end()); + + float scale = 0.0006; + float max = 0.0; + std::for_each(std::begin(data_fp32), std::end(data_fp32), [&max](auto& value) { + if (!std::isinf(value) && !std::isnan(value)) + { + max = std::max(max, std::abs(value)); + } + }); + std::string jsonCfg = templateJsonCfg; + update_json_template(jsonCfg, "_MAXIMUM_", std::to_string(max)); + update_json_template(jsonCfg, "_SCALE_", std::to_string(scale)); + + float errBound = max * scale; + // Use 10% error margin to test due to using v.large values in our random data + float insideErrBound = errBound * 0.9; + float outsideErrBound = errBound * 1.1; + + SUBCASE("inside") + { + // Generate some data that meets the requirements of the result. + auto otherData_fp32 = data_fp32; + std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [insideErrBound](auto& value) { + if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value)) + value += insideErrBound; + }); + const auto referenceTensor = + TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast(data_fp64.data())); + const auto implementationTensor = + TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast(otherData_fp32.data())); + REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str())); + } + + SUBCASE("outside") + { + // Generate some data that exceeds the requirements for each value in the tensor. + auto otherData_fp32 = data_fp32; + std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [outsideErrBound](auto& value) { + if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value)) + value += outsideErrBound; + }); + + const auto referenceTensor = + TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast(data_fp64.data())); + const auto implementationTensor = + TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast(otherData_fp32.data())); + REQUIRE_FALSE( + tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str())); + } +} TEST_SUITE_END(); // verify diff --git a/scripts/schemavalidation/compliance-config.schema.json b/scripts/schemavalidation/compliance-config.schema.json index c0a479d..eb3ccde 100644 --- a/scripts/schemavalidation/compliance-config.schema.json +++ b/scripts/schemavalidation/compliance-config.schema.json @@ -88,7 +88,30 @@ "required": [ "n" ] + }, + "relative_info": { + "description": "info required for the RELATIVE mode", + "type": "object", + "properties": + { + "max": { + "description": "maximum absolute input value", + "type": "number", + "minimum": 0 + }, + "scale": { + "description": "relative scaling factor", + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "max", + "scale" + ] } + }, "additionalProperties": false, "required": [ diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json index b8efd35..dc28bef 100644 --- a/verif/conformance/tosa_main_profile_ops_info.json +++ b/verif/conformance/tosa_main_profile_ops_info.json @@ -2645,6 +2645,7 @@ "profile": [ "tosa-mi" ], + "support_for": [ "lazy_data_gen" ], "generation": { "standard": { "negative_dim_range": "1,10", @@ -2657,13 +2658,13 @@ "--target-dtype", "bf16", "--fp-values-range", - "-2.0,2.0" + "-max,max" ], [ "--target-dtype", "fp32", "--fp-values-range", - "-2.0,2.0", + "-max,max", "--target-shape", "1,1103,1,2", "--max-resize-output-dim", diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py index a4bced3..4630f35 100644 --- a/verif/generator/tosa_arg_gen.py +++ b/verif/generator/tosa_arg_gen.py @@ -1394,6 +1394,22 @@ class TosaTensorValuesGen: testGen, opName, dtypeList, shapeList, argsDict, error_name ) + @staticmethod + def tvgResize(testGen, opName, dtypeList, shapeList, argsDict, error_name=None): + data_range = TosaTensorValuesGen._get_data_range( + testGen, + dtypeList[0], + TosaTensorValuesGen.TVG_FLOAT_HIGH_VALUE, + ) + if data_range: + argsDict["data_range"] = data_range + # Needed for compliance + argsDict["max_abs_value"] = data_range[1] + + return TosaTensorValuesGen.tvgLazyGenDefault( + testGen, opName, dtypeList, shapeList, argsDict, error_name + ) + # Set the POW exponent high data range TVG_FLOAT_HIGH_VALUE_POW_EXP = { DType.FP32: 10.0, @@ -3343,14 +3359,13 @@ class TosaArgGen: border[0], border[1], ), - [ - mode, - scale, - offset, - border, - dtype, - outputDTypeNew, - ], + { + "mode": mode, + "scale": scale, + "offset": offset, + "border": border, + "output_dtype": outputDTypeNew, + }, ) if arg_to_append in arg_list: # Skip already generated test params @@ -3359,6 +3374,16 @@ class TosaArgGen: # Valid permutation perm += 1 arg_list.append(arg_to_append) + + # Now add data generator types + arg_list = TosaArgGen._add_data_generators( + testGen, + opName, + dtype, + arg_list, + error_name, + ) + # Return list of tuples: (arg_str, args_dict) return arg_list @staticmethod diff --git a/verif/generator/tosa_error_if.py b/verif/generator/tosa_error_if.py index 90c3428..5fd647a 100644 --- a/verif/generator/tosa_error_if.py +++ b/verif/generator/tosa_error_if.py @@ -2585,9 +2585,9 @@ class TosaInvalidValidator: @staticmethod def ivWrongDataTypeOrModeResize(**kwargs): input_dtype = kwargs["input_dtype"] - args = kwargs["args"] - mode = args[0] - output_dtype = args[5] + args_dict = kwargs["args"] + mode = args_dict["mode"] + output_dtype = args_dict["output_dtype"] if mode == ResizeMode.BILINEAR: # Invalid output data type / Invalid input datatype diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index d82f919..ae689b4 100644 --- a/verif/generator/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -357,6 +357,12 @@ class TosaTestGen: elif "compliance" in op and "ulp" in op["compliance"]: mode = gtu.ComplianceMode.ULP compliance_tens["ulp_info"] = {"ulp": op["compliance"]["ulp"]} + elif "compliance" in op and "relative" in op["compliance"]: + mode = gtu.ComplianceMode.RELATIVE + compliance_tens["relative_info"] = { + "max": argsDict["max_abs_value"], + "scale": op["compliance"]["relative"], + } elif op["op"] == Op.REDUCE_PRODUCT: mode = gtu.ComplianceMode.REDUCE_PRODUCT compliance_tens["reduce_product_info"] = {"n": argsDict["n"]} @@ -1933,17 +1939,21 @@ class TosaTestGen: def build_resize( self, op, - input, - mode, - scale, - offset, - border, - input_dtype, - output_dtype, + inputs, + args_dict, validator_fcns, error_name=None, + qinfo=None, ): - result_tens = OutputShaper.resizeOp( + assert len(inputs) == 1 + input = inputs[0] + mode = args_dict["mode"] + scale = args_dict["scale"] + offset = args_dict["offset"] + border = args_dict["border"] + output_dtype = args_dict["output_dtype"] + + result_tensor = OutputShaper.resizeOp( self.ser, self.rng, input, @@ -1951,14 +1961,14 @@ class TosaTestGen: scale, offset, border, - input_dtype, + input.dtype, output_dtype, error_name, ) # Invalidate Input/Output list for error if checks. input_list = [input.name] - output_list = [result_tens.name] + output_list = [result_tensor.name] pCount, cCount = op["operands"] num_operands = pCount + cCount input_list, output_list = TosaErrorIfArgGen.eiInvalidateInputOutputList( @@ -1972,25 +1982,28 @@ class TosaTestGen: op=op, mode=mode, scale=scale, - input_dtype=input_dtype, + input_dtype=input.dtype, output_dtype=output_dtype, input_shape=input.shape, - output_shape=result_tens.shape, + output_shape=result_tensor.shape, offset=offset, border=border, input_list=input_list, output_list=output_list, - result_tensors=[result_tens], + result_tensors=[result_tensor], num_operands=num_operands, ): return None attr = ts.TosaSerializerAttribute() - attr.ResizeAttribute(scale, offset, border, mode) - self.ser.addOperator(op["op"], input_list, output_list, attr) - return result_tens + + compliance = self.tensorComplianceMetaData( + op, input.dtype, args_dict, result_tensor, error_name + ) + + return TosaTestGen.BuildInfo(result_tensor, compliance) def build_identityn(self, op, val, val2, validator_fcns=None, error_name=None): result_tens = OutputShaper.unaryOp(self.ser, self.rng, val, error_name) @@ -4610,7 +4623,7 @@ class TosaTestGen: "build_fcn": ( build_resize, TosaTensorGen.tgNHWC, - TosaTensorValuesGen.tvgDefault, + TosaTensorValuesGen.tvgResize, TosaArgGen.agResize, ), "types": (DType.INT8, DType.INT16, DType.FP16, DType.BF16, DType.FP32), @@ -4636,6 +4649,10 @@ class TosaTestGen: TosaErrorValidator.evResizeOutputShapeMismatch, TosaErrorValidator.evResizeOutputShapeNonInteger, ), + "data_gen": { + "fp": (gtu.DataGenType.PSEUDO_RANDOM,), + }, + "compliance": {"relative": 0.006}, }, # Type conversion "cast": { diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py index 6387d06..76e7388 100644 --- a/verif/generator/tosa_utils.py +++ b/verif/generator/tosa_utils.py @@ -39,6 +39,7 @@ class ComplianceMode(IntEnum): FP_SPECIAL = 3 REDUCE_PRODUCT = 4 ABS_ERROR = 5 + RELATIVE = 6 class DataGenType(IntEnum): -- cgit v1.2.1