Improve verfiy and generate library validation

Improved libraries validation to catch unknown values.
Improved verify output to match generate library.
Refactored generate tests to use less code duplication.

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I9c38745fbc8e70f46c19ddae6c62ee248d33b5f1

15 files changed, 213 insertions, 166 deletions

diff --git a/reference_model/src/generate/generate_dot_product.cc b/reference_model/src/generate/generate_dot_product.cc
index 90710ba..1d2325f 100644
--- a/reference_model/src/generate/generate_dot_product.cc
+++ b/reference_model/src/generate/generate_dot_product.cc
@@ -91,7 +91,7 @@ bool generateDotProduct(const GenerateConfig& cfg, void* data, size_t size)
         case tosa::Op_MATMUL:
             return generateMatMul(cfg, *generator, data, size);
         default:
-            WARNING("[Generator][DP] Unsupported operator");
+            WARNING("[Generator][DP] Unsupported operator.");
             return false;
     }
 
diff --git a/reference_model/src/generate/generate_dot_product_states.cc b/reference_model/src/generate/generate_dot_product_states.cc
index d3eeb6d..649e55e 100644
--- a/reference_model/src/generate/generate_dot_product_states.cc
+++ b/reference_model/src/generate/generate_dot_product_states.cc
@@ -309,9 +309,11 @@ std::unique_ptr<IDotProductGenerator> pickDotProductGenerator(const GenerateConf
             case 5:
                 return std::make_unique<GeneratorS5>(cfg.inputPos, dpinfo.ks, B);
             default:
+                WARNING("[Generator][DP] Unsupported dot product test series for generator.");
                 return nullptr;
         }
     }
+    WARNING("[Generator][DP] Unsupported data types for generator.");
     return nullptr;
 }
 
diff --git a/reference_model/src/generate/generate_entry.cc b/reference_model/src/generate/generate_entry.cc
index 95dbe8f..e7a0044 100644
--- a/reference_model/src/generate/generate_entry.cc
+++ b/reference_model/src/generate/generate_entry.cc
@@ -56,7 +56,6 @@ extern "C"
         auto cfg = TosaReference::parseGenerateConfig(config_json, tensor_name);
         if (!cfg)
         {
-            WARNING("[Generator] Invalid json config.");
             return false;
         }
 
diff --git a/reference_model/src/generate/generate_utils.cc b/reference_model/src/generate/generate_utils.cc
index c32d0fb..da16632 100644
--- a/reference_model/src/generate/generate_utils.cc
+++ b/reference_model/src/generate/generate_utils.cc
@@ -23,6 +23,7 @@ namespace tosa
 
 NLOHMANN_JSON_SERIALIZE_ENUM(DType,
                              {
+                                 { DType::DType_UNKNOWN, "UNKNOWN" },
                                  { DType::DType_BOOL, "BOOL" },
                                  { DType::DType_INT4, "INT4" },
                                  { DType::DType_INT8, "INT8" },
@@ -36,6 +37,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(DType,
 
 NLOHMANN_JSON_SERIALIZE_ENUM(Op,
                              {
+                                 { Op::Op_UNKNOWN, "UNKNOWN" },
                                  { Op::Op_MATMUL, "MATMUL" },
                              })
 
@@ -46,6 +48,7 @@ namespace TosaReference
 
 NLOHMANN_JSON_SERIALIZE_ENUM(GeneratorType,
                              {
+                                 { GeneratorType::Unknown, "UNKNOWN" },
                                  { GeneratorType::PseudoRandom, "PSEUDO_RANDOM" },
                                  { GeneratorType::DotProduct, "DOT_PRODUCT" },
                                  { GeneratorType::OpFullRange, "OP_FULL_RANGE" },
@@ -53,6 +56,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(GeneratorType,
                                  { GeneratorType::OpSpecial, "OP_SPECIAL" },
                              })
 
+// NOTE: This assumes it's VARIABLE if the InputType is not recognized
 NLOHMANN_JSON_SERIALIZE_ENUM(InputType,
                              {
                                  { InputType::Variable, "VARIABLE" },
@@ -96,14 +100,21 @@ std::optional<GenerateConfig> parseGenerateConfig(const char* json, const char*
     auto jsonCfg = nlohmann::json::parse(json, nullptr, /* allow exceptions */ false);
 
     if (jsonCfg.is_discarded())
+    {
+        WARNING("[Generator] Invalid json config.");
         return std::nullopt;
+    }
     if (!jsonCfg.contains("tensors"))
+    {
+        WARNING("[Generator] Missing tensors in json config.");
         return std::nullopt;
-
+    }
     const auto& tensors = jsonCfg["tensors"];
     if (!tensors.contains(tensorName))
+    {
+        WARNING("[Generator] Missing tensor %s in json config.", tensorName);
         return std::nullopt;
-
+    }
     const auto& namedTensor = tensors[tensorName];
     return namedTensor.get<GenerateConfig>();
 }
diff --git a/reference_model/src/generate/generate_utils.h b/reference_model/src/generate/generate_utils.h
index 2d5b7f8..e8e67bb 100644
--- a/reference_model/src/generate/generate_utils.h
+++ b/reference_model/src/generate/generate_utils.h
@@ -28,6 +28,7 @@ namespace TosaReference
 /// \brief Supported generator types
 enum class GeneratorType
 {
+    Unknown,
     PseudoRandom,
     DotProduct,
     OpFullRange,
diff --git a/reference_model/src/verify/verify_dot_product.cc b/reference_model/src/verify/verify_dot_product.cc
index 39895b1..2a1d273 100644
--- a/reference_model/src/verify/verify_dot_product.cc
+++ b/reference_model/src/verify/verify_dot_product.cc
@@ -82,7 +82,7 @@ bool validateData(const double* ref, const double* bnd, const AccType* imp, size
     for (size_t i = 0; i < T; ++i)
     {
         auto out_err = validateElement<AccType>(ref[i], bnd[i], imp[i], KS);
-        TOSA_REF_REQUIRE(out_err, "data required to be zero or error within range");
+        TOSA_REF_REQUIRE(out_err, "[DP] Data required to be zero or error within range");
         out_err_sum += out_err.value();
         out_err_sumsq += out_err.value() * out_err.value();
     }
@@ -90,10 +90,10 @@ bool validateData(const double* ref, const double* bnd, const AccType* imp, size
     if (S >= 3 && S <= 5)
     {
         // Check error bias magnitude for data sets S which are not positive biased
-        TOSA_REF_REQUIRE(std::abs(out_err_sum) <= 2 * sqrt(KS * T), "bias magnitude is out of range");
+        TOSA_REF_REQUIRE(std::abs(out_err_sum) <= 2 * sqrt(KS * T), "[DP] Bias magnitude is out of range");
     }
     // Check error variance magnitude
-    TOSA_REF_REQUIRE(out_err_sumsq <= 0.4 * KS * T, "error variance magnitude is out of range");
+    TOSA_REF_REQUIRE(out_err_sumsq <= 0.4 * KS * T, "[DP] Error variance magnitude is out of range");
     return true;
 }
 }    // namespace
@@ -101,9 +101,9 @@ bool validateData(const double* ref, const double* bnd, const AccType* imp, size
 bool verifyDotProduct(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const DotProductVerifyInfo& dpInfo)
 {
     // Validate that tensors are provided
-    TOSA_REF_REQUIRE(ref != nullptr, "reference tensor is missing");
-    TOSA_REF_REQUIRE(refBnd != nullptr, "reference bounds tensor is missing");
-    TOSA_REF_REQUIRE(imp != nullptr, "implementation tensor is missing");
+    TOSA_REF_REQUIRE(ref != nullptr, "[DP] Reference tensor is missing");
+    TOSA_REF_REQUIRE(refBnd != nullptr, "[DP] Reference bounds tensor is missing");
+    TOSA_REF_REQUIRE(imp != nullptr, "[DP] Implementation tensor is missing");
 
     // Get number of dot-product elements
     const int64_t T = numElements(std::vector<int32_t>(ref->shape, ref->shape + ref->num_dims));
@@ -111,18 +111,18 @@ bool verifyDotProduct(const CTensor* ref, const CTensor* refBnd, const CTensor*
 
     const double* refData    = reinterpret_cast<const double*>(ref->data);
     const double* refBndData = reinterpret_cast<const double*>(refBnd->data);
-    TOSA_REF_REQUIRE(refData != nullptr && refBndData != nullptr, "missing data for reference or bounds tensors");
+    TOSA_REF_REQUIRE(refData != nullptr && refBndData != nullptr, "[DP] Missing data for reference or bounds tensors");
 
     switch (imp->data_type)
     {
         case tosa_datatype_fp32_t: {
             const float* impData = reinterpret_cast<const float*>(imp->data);
-            TOSA_REF_REQUIRE(impData != nullptr, "missing data for implementation");
+            TOSA_REF_REQUIRE(impData != nullptr, "[DP] Missing data for implementation");
             return validateData(refData, refBndData, impData, static_cast<size_t>(T), dpInfo);
             break;
         }
         default: {
-            WARNING("tosa verifier: data-type not supported.");
+            WARNING("[Verifier][DP] Data-type not supported.");
             break;
         }
     }
diff --git a/reference_model/src/verify/verify_entry.cc b/reference_model/src/verify/verify_entry.cc
index 1ddd52b..67eb7df 100644
--- a/reference_model/src/verify/verify_entry.cc
+++ b/reference_model/src/verify/verify_entry.cc
@@ -41,7 +41,7 @@ bool verify(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const
             return verifyULP(ref, imp, cfg.ulpInfo.ulp);
         }
         default: {
-            WARNING("tosa verifier: unsupported verification mode.");
+            WARNING("[Verifier] Unsupported verification mode.");
             break;
         }
     }
@@ -60,37 +60,36 @@ extern "C"
         // Check inputs for nullptr
         if (!ref || !imp || !config_json)
         {
-            WARNING("tosa verifier: one of the inputs is missing.");
+            WARNING("[Verifier] One of the inputs is missing.");
             return false;
         }
 
         // Extract verification config
         if (!ref->name)
         {
-            WARNING("tosa verifier: tensor name is not specified.");
+            WARNING("[Verifier] Tensor name is not specified.");
             return false;
         }
         auto cfg = TosaReference::parseVerifyConfig(ref->name, config_json);
         if (!cfg)
         {
-            WARNING("tosa verifier: invalid json config.");
             return false;
         }
 
         // Validate shape
         if (ref->num_dims != imp->num_dims)
         {
-            WARNING("tosa verifier: tensors have different number of dimensions.");
+            WARNING("[Verifier] Tensors have different number of dimensions.");
             return false;
         }
         if (!ref->shape || !imp->shape)
         {
-            WARNING("tosa verifier: one of tensors' shape is missing.");
+            WARNING("[Verifier] One of tensors' shape is missing.");
             return false;
         }
         if (std::vector(ref->shape, ref->shape + ref->num_dims) != std::vector(imp->shape, imp->shape + imp->num_dims))
         {
-            WARNING("tosa verifier: tensors have different shapes.");
+            WARNING("[Verifier] Tensors have different shapes.");
             return false;
         }
 
@@ -99,7 +98,7 @@ extern "C"
         {
             if (cfg->dataType != TosaReference::mapToDType(imp->data_type))
             {
-                WARNING("tosa verifier: incorrect tensor data type.");
+                WARNING("[Verifier] Incorrect tensor data type.");
                 return false;
             }
         }
@@ -107,7 +106,7 @@ extern "C"
         {
             if (ref->data_type != imp->data_type)
             {
-                WARNING("tosa verifier: tensors have different data types.");
+                WARNING("[Verifier] Tensors have different data types.");
                 return false;
             }
         }
diff --git a/reference_model/src/verify/verify_exact.cc b/reference_model/src/verify/verify_exact.cc
index 1a0c44c..4d6c72f 100644
--- a/reference_model/src/verify/verify_exact.cc
+++ b/reference_model/src/verify/verify_exact.cc
@@ -22,21 +22,21 @@ namespace TosaReference
 bool verifyExact(const CTensor* referenceTensor, const CTensor* implementationTensor)
 {
     // Validate that tensors are provided
-    TOSA_REF_REQUIRE(referenceTensor != nullptr, "reference tensor is missing");
-    TOSA_REF_REQUIRE(implementationTensor != nullptr, "implementation tensor is missing");
+    TOSA_REF_REQUIRE(referenceTensor != nullptr, "[E] Reference tensor is missing");
+    TOSA_REF_REQUIRE(implementationTensor != nullptr, "[E] Implementation tensor is missing");
 
     // Get number of elements
     const auto elementCount =
         numElements(std::vector<int32_t>(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims));
-    TOSA_REF_REQUIRE(elementCount > 0, "invalid shape for reference tensor");
+    TOSA_REF_REQUIRE(elementCount > 0, "[E] Invalid shape for reference tensor");
 
     switch (implementationTensor->data_type)
     {
         case tosa_datatype_fp32_t: {
             const auto* refData = reinterpret_cast<const float*>(referenceTensor->data);
-            TOSA_REF_REQUIRE(refData != nullptr, "missing data for reference");
+            TOSA_REF_REQUIRE(refData != nullptr, "[E] Missing data for reference");
             const auto* impData = reinterpret_cast<const float*>(implementationTensor->data);
-            TOSA_REF_REQUIRE(impData != nullptr, "missing data for implementation");
+            TOSA_REF_REQUIRE(impData != nullptr, "[E] Missing data for implementation");
             return std::equal(refData, std::next(refData, elementCount), impData, std::next(impData, elementCount),
                               [](const auto& referenceValue, const auto& implementationValue) {
                                   return std::isnan(referenceValue) ? std::isnan(implementationValue)
@@ -44,7 +44,7 @@ bool verifyExact(const CTensor* referenceTensor, const CTensor* implementationTe
                               });
         }
         default:
-            WARNING("tosa verifier: data-type not supported.");
+            WARNING("[Verifier][E] Data-type not supported.");
             break;
     }
 
diff --git a/reference_model/src/verify/verify_reduce_product.cc b/reference_model/src/verify/verify_reduce_product.cc
index d233b0f..625e2cf 100644
--- a/reference_model/src/verify/verify_reduce_product.cc
+++ b/reference_model/src/verify/verify_reduce_product.cc
@@ -40,22 +40,22 @@ namespace TosaReference
 bool verifyReduceProduct(const CTensor* referenceTensor, const CTensor* implementationTensor, uint64_t m, uint64_t n)
 {
     // Validate that tensors are provided
-    TOSA_REF_REQUIRE(referenceTensor != nullptr, "reference tensor is missing");
-    TOSA_REF_REQUIRE(implementationTensor != nullptr, "implementation tensor is missing");
+    TOSA_REF_REQUIRE(referenceTensor != nullptr, "[RP] Reference tensor is missing");
+    TOSA_REF_REQUIRE(implementationTensor != nullptr, "[RP] Implementation tensor is missing");
 
     // Get number of elements
     const auto elementCount =
         numElements(std::vector<int32_t>(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims));
-    TOSA_REF_REQUIRE(elementCount > 0, "invalid shape for reference tensor");
+    TOSA_REF_REQUIRE(elementCount > 0, "[RP] Invalid shape for reference tensor");
 
     switch (implementationTensor->data_type)
     {
         case tosa_datatype_fp32_t: {
             const auto* refData = reinterpret_cast<const float*>(referenceTensor->data);
-            TOSA_REF_REQUIRE(refData != nullptr, "missing data for reference");
+            TOSA_REF_REQUIRE(refData != nullptr, "[RP] Missing data for reference");
 
             const auto* impData = reinterpret_cast<const float*>(implementationTensor->data);
-            TOSA_REF_REQUIRE(impData != nullptr, "missing data for implementation");
+            TOSA_REF_REQUIRE(impData != nullptr, "[RP] Missing data for implementation");
 
             return std::equal(refData, std::next(refData, elementCount), impData, std::next(impData, elementCount),
                               [m, n](const auto& referenceValue, const auto& implementationValue) {
@@ -79,7 +79,7 @@ bool verifyReduceProduct(const CTensor* referenceTensor, const CTensor* implemen
                               });
         }
         default:
-            WARNING("tosa verifier: data-type not supported.");
+            WARNING("[Verifier][RP] Data-type not supported.");
             break;
     }
 
diff --git a/reference_model/src/verify/verify_ulp.cc b/reference_model/src/verify/verify_ulp.cc
index 223bc48..486c0ff 100644
--- a/reference_model/src/verify/verify_ulp.cc
+++ b/reference_model/src/verify/verify_ulp.cc
@@ -124,28 +124,28 @@ bool tosaCheckULP(float testValue, double referenceValue, int64_t ulpCount)
 bool verifyULP(const CTensor* referenceTensor, const CTensor* implementationTensor, uint64_t ulp)
 {
     // Validate that tensors are provided
-    TOSA_REF_REQUIRE(referenceTensor != nullptr, "reference tensor is missing");
-    TOSA_REF_REQUIRE(implementationTensor != nullptr, "implementation tensor is missing");
+    TOSA_REF_REQUIRE(referenceTensor != nullptr, "[ULP] Reference tensor is missing");
+    TOSA_REF_REQUIRE(implementationTensor != nullptr, "[ULP] Implementation tensor is missing");
 
     // Get number of elements
     const auto elementCount =
         numElements(std::vector<int32_t>(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims));
-    TOSA_REF_REQUIRE(elementCount > 0, "invalid shape for reference tensor");
+    TOSA_REF_REQUIRE(elementCount > 0, "[ULP] Invalid shape for reference tensor");
 
     switch (implementationTensor->data_type)
     {
         case tosa_datatype_fp32_t: {
             const auto* refData = reinterpret_cast<const float*>(referenceTensor->data);
-            TOSA_REF_REQUIRE(refData != nullptr, "missing data for reference");
+            TOSA_REF_REQUIRE(refData != nullptr, "[ULP] Missing data for reference");
             const auto* impData = reinterpret_cast<const float*>(implementationTensor->data);
-            TOSA_REF_REQUIRE(impData != nullptr, "missing data for implementation");
+            TOSA_REF_REQUIRE(impData != nullptr, "[ULP] Missing data for implementation");
             return std::equal(refData, std::next(refData, elementCount), impData, std::next(impData, elementCount),
                               [ulp](const auto& referenceValue, const auto& implementationValue) {
                                   return tosaCheckULP(referenceValue, implementationValue, ulp);
                               });
         }
         default:
-            WARNING("tosa verifier: data-type not supported.");
+            WARNING("[Verifier][ULP] Data-type not supported.");
             break;
     }
 
diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc
index 366238b..ee11c41 100644
--- a/reference_model/src/verify/verify_utils.cc
+++ b/reference_model/src/verify/verify_utils.cc
@@ -25,6 +25,7 @@ namespace tosa
 
 NLOHMANN_JSON_SERIALIZE_ENUM(DType,
                              {
+                                 { DType::DType_UNKNOWN, "UNKNOWN" },
                                  { DType::DType_BOOL, "BOOL" },
                                  { DType::DType_INT4, "INT4" },
                                  { DType::DType_INT8, "INT8" },
@@ -43,6 +44,7 @@ namespace TosaReference
 
 NLOHMANN_JSON_SERIALIZE_ENUM(VerifyMode,
                              {
+                                 { VerifyMode::Unknown, "UNKNOWN" },
                                  { VerifyMode::Exact, "EXACT" },
                                  { VerifyMode::Ulp, "ULP" },
                                  { VerifyMode::DotProduct, "DOT_PRODUCT" },
@@ -94,14 +96,23 @@ std::optional<VerifyConfig> parseVerifyConfig(const char* tensorName, const char
     auto jsonCfg = nlohmann::json::parse(json, nullptr, /* allow exceptions */ false);
 
     if (jsonCfg.is_discarded())
+    {
+        WARNING("[Verifier] Invalid json config.");
         return std::nullopt;
+    }
     if (!jsonCfg.contains("tensors"))
+    {
+        WARNING("[Verifier] Missing tensors in json config.");
         return std::nullopt;
+    }
 
     const auto& tensors = jsonCfg["tensors"];
     if (!tensors.contains(tensorName))
-        return std::nullopt;
-
+        if (!tensors.contains(tensorName))
+        {
+            WARNING("[Verifier] Missing tensor %s in json config.", tensorName);
+            return std::nullopt;
+        }
     const auto& namedTensor = tensors[tensorName];
     return namedTensor.get<VerifyConfig>();
 }
diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h
index 0afd804..bbe4b4e 100644
--- a/reference_model/src/verify/verify_utils.h
+++ b/reference_model/src/verify/verify_utils.h
@@ -26,7 +26,7 @@
 #define TOSA_REF_REQUIRE(COND, MESSAGE)                                                                                \
     if (!(COND))                                                                                                       \
     {                                                                                                                  \
-        WARNING("tosa verifier: " MESSAGE ".");                                                                        \
+        WARNING("[Verifier]" MESSAGE ".");                                                                             \
         return false;                                                                                                  \
     }
 
@@ -39,6 +39,7 @@ using CTensor = tosa_tensor_t;
 /// \brief Supported verification modes
 enum class VerifyMode
 {
+    Unknown,
     Exact,
     Ulp,
     DotProduct,
diff --git a/reference_model/test/generate_tests.cpp b/reference_model/test/generate_tests.cpp
index bce7092..503ecfe 100644
--- a/reference_model/test/generate_tests.cpp
+++ b/reference_model/test/generate_tests.cpp
@@ -16,32 +16,34 @@
 #include <doctest.h>
 
 #include <array>
-#include <iostream>
+#include <sstream>
 #include <string>
 #include <vector>
 
 namespace
 {
-template <typename T>
-void debug_vec_print(const std::vector<T>& vec)
+void update_json_template(std::string& str, const std::string& find, const std::string& change)
 {
-    std::cout << "vector: ";
-    for (auto v = vec.begin(); v != vec.end(); ++v)
+    // Update the 'str' by looking for instances of 'find' and replacing them with 'change'
+    auto pos = str.find(find);
+    while (pos != std::string::npos)
     {
-        T f = *v;
-        std::cout << std::dec << f << " [" << std::hex << *(uint32_t*)&f << "] ";
+        str.replace(pos, find.length(), change);
+        pos = str.find(find);
     }
-    std::cout << std::dec << '\n';
 }
 
-void update_json_template(std::string& str, const std::string& set)
+void check_value(bool match, uint32_t result, uint32_t expected, uint32_t idx)
 {
-    std::string find = "_SET_";
-    auto pos         = str.find(find);
-    while (pos != std::string::npos)
+    std::stringstream msg;
+    msg << "index: " << idx << " expected: " << std::hex << expected << " got: " << result;
+    if (match)
     {
-        str.replace(pos, find.length(), set);
-        pos = str.find(find);
+        REQUIRE_MESSAGE(expected == result, msg.str());
+    }
+    else
+    {
+        REQUIRE_MESSAGE(expected != result, msg.str());
     }
 }
 
@@ -50,7 +52,7 @@ void check_output(const std::vector<T>& results, const std::vector<uint32_t>& ex
 {
     for (size_t idx = 0; idx < expected.size(); ++idx)
     {
-        REQUIRE_MESSAGE(expected[idx] == *(uint32_t*)&results[idx], "index: ", idx);
+        check_value(true, *(uint32_t*)&results[idx], expected[idx], idx);
     }
 }
 
@@ -60,19 +62,19 @@ TEST_SUITE_BEGIN("generate");
 
 TEST_CASE("negative - api")
 {
-    std::string json_cfg = R"({
+    std::string templateJsonCfg = R"({
         "tensors" : {
             "in1" : {
-                "generator": "DOT_PRODUCT",
-                "data_type": "FP32",
+                "generator": "_GENERATOR_",
+                "data_type": "_TYPE_",
                 "input_type": "VARIABLE",
                 "shape" : [ 4, 8, 8 ],
                 "input_pos": 0,
-                "op" : "MATMUL",
+                "op" : "_OP_",
                 "dot_product_info": {
                     "s": 0,
                     "ks": 8,
-                    "acc_type": "FP32"
+                    "acc_type": "_TYPE_"
                 }
             }
         }
@@ -88,7 +90,7 @@ TEST_CASE("negative - api")
     }
     SUBCASE("invalid json")
     {
-        std::string invalid_json_cfg = R"({
+        std::string invalidJsonCfg = R"({
             "tensors" : {
                 "in1" : {
                     "generator": DOT_PRODUCT,
@@ -97,28 +99,79 @@ TEST_CASE("negative - api")
         })";
 
         std::vector<float> buffer(tosaElements);
-        REQUIRE_FALSE(tgd_generate_data(invalid_json_cfg.c_str(), tosaName.c_str(), (void*)buffer.data(), tosaSize));
+        REQUIRE_FALSE(tgd_generate_data(invalidJsonCfg.c_str(), tosaName.c_str(), (void*)buffer.data(), tosaSize));
     }
-    SUBCASE("invalid json - mismatching name")
+    SUBCASE("unknown generator")
     {
+        std::string jsonCfg = templateJsonCfg;
+        update_json_template(jsonCfg, "_GENERATOR_", "SOLAR");
+        update_json_template(jsonCfg, "_TYPE_", "FP32");
+        update_json_template(jsonCfg, "_OP_", "MATMUL");
+        std::vector<float> buffer(tosaElements);
+        REQUIRE_FALSE(tgd_generate_data(jsonCfg.c_str(), tosaName.c_str(), (void*)buffer.data(), tosaSize));
+    }
+    SUBCASE("unknown op")
+    {
+        std::string jsonCfg = templateJsonCfg;
+        update_json_template(jsonCfg, "_GENERATOR_", "DOT_PRODUCT");
+        update_json_template(jsonCfg, "_TYPE_", "FP32");
+        update_json_template(jsonCfg, "_OP_", "GREEN");
+
+        std::vector<float> buffer(tosaElements);
+        REQUIRE_FALSE(tgd_generate_data(jsonCfg.c_str(), tosaName.c_str(), (void*)buffer.data(), tosaSize));
+    }
+    SUBCASE("unknown type")
+    {
+        std::string jsonCfg = templateJsonCfg;
+        update_json_template(jsonCfg, "_GENERATOR_", "DOT_PRODUCT");
+        update_json_template(jsonCfg, "_TYPE_", "WATT");
+        update_json_template(jsonCfg, "_OP_", "MATMUL");
+
+        std::vector<float> buffer(tosaElements);
+        REQUIRE_FALSE(tgd_generate_data(jsonCfg.c_str(), tosaName.c_str(), (void*)buffer.data(), tosaSize));
+    }
+    SUBCASE("mismatching name")
+    {
+        std::string jsonCfg = templateJsonCfg;
+        update_json_template(jsonCfg, "_GENERATOR_", "DOT_PRODUCT");
+        update_json_template(jsonCfg, "_TYPE_", "FP32");
+        update_json_template(jsonCfg, "_OP_", "MATMUL");
         std::string invalidName = "notFound1";
 
         std::vector<float> buffer(tosaElements);
-        REQUIRE_FALSE(tgd_generate_data(json_cfg.c_str(), invalidName.c_str(), (void*)buffer.data(), tosaSize));
+        REQUIRE_FALSE(tgd_generate_data(jsonCfg.c_str(), invalidName.c_str(), (void*)buffer.data(), tosaSize));
     }
     SUBCASE("mismatching size")
     {
+        std::string jsonCfg = templateJsonCfg;
+        update_json_template(jsonCfg, "_GENERATOR_", "DOT_PRODUCT");
+        update_json_template(jsonCfg, "_TYPE_", "FP32");
+        update_json_template(jsonCfg, "_OP_", "MATMUL");
         size_t smallElements = 4 * 8 * 7;
         size_t smallSize     = smallElements * 4;
 
         std::vector<float> buffer(smallElements);
-        REQUIRE_FALSE(tgd_generate_data(json_cfg.c_str(), tosaName.c_str(), (void*)buffer.data(), smallSize));
+        REQUIRE_FALSE(tgd_generate_data(jsonCfg.c_str(), tosaName.c_str(), (void*)buffer.data(), smallSize));
     }
 }
 
-TEST_CASE("positive - dot product")
+void matmul_test_FP32(const std::string tosaName[2],
+                      const size_t tosaElements[2],
+                      const std::string templateJsonCfg,
+                      const std::string setStr,
+                      int32_t param,
+                      const std::vector<uint32_t> expected)
 {
-    std::string template_json_cfg = R"({
+    std::string jsonCfg = templateJsonCfg;
+    update_json_template(jsonCfg, "_SET_", setStr);
+    std::vector<float> buffer(tosaElements[param]);
+    REQUIRE(tgd_generate_data(jsonCfg.c_str(), tosaName[param].c_str(), (void*)buffer.data(), tosaElements[param] * 4));
+    check_output<float>(buffer, expected);
+}
+
+TEST_CASE("positive - FP32 matmul dot product (first 3 values)")
+{
+    std::string templateJsonCfg = R"({
         "tensors" : {
             "in1" : {
                 "generator": "DOT_PRODUCT",
@@ -150,131 +203,69 @@ TEST_CASE("positive - dot product")
         }
     })";
 
-    const std::string tosaNameP0 = "in1";
-    const size_t tosaElementsP0  = 4 * 8 * 2;
-    const std::string tosaNameP1 = "in2";
-    const size_t tosaElementsP1  = 4 * 2 * 5;
+    const std::string tosaName[2] = { "in1", "in2" };
+    const size_t tosaElements[2]  = { (4 * 8 * 2), (4 * 2 * 5) };
 
     SUBCASE("matmul, set 0, param 0")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "0");
-
         std::vector<uint32_t> expected = { 0xbf665aa4, 0xbf736bd3, 0x0 };
-        std::vector<float> buffer(tosaElementsP0);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP0.c_str(), (void*)buffer.data(), tosaElementsP0 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "0", 0, expected);
     }
     SUBCASE("matmul, set 0, param 1")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "0");
-
         std::vector<uint32_t> expected = { 0x0, 0x0, 0x3f34f2dd };
-        std::vector<float> buffer(tosaElementsP1);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP1.c_str(), (void*)buffer.data(), tosaElementsP1 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "0", 1, expected);
     }
     SUBCASE("matmul, set 1, param 0")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "1");
-
         std::vector<uint32_t> expected = { 0x5e97f1b0, 0x5ea6a18e, 0x5eb811af };
-        std::vector<float> buffer(tosaElementsP0);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP0.c_str(), (void*)buffer.data(), tosaElementsP0 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "1", 0, expected);
     }
     SUBCASE("matmul, set 1, param 1")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "1");
-
         std::vector<uint32_t> expected = { 0x5f128bb1, 0x5ef54579, 0x5ebd65b8 };
-        std::vector<float> buffer(tosaElementsP1);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP1.c_str(), (void*)buffer.data(), tosaElementsP1 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "1", 1, expected);
     }
     SUBCASE("matmul, set 2, param 0")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "2");
-
         std::vector<uint32_t> expected = { 0x3f800000, 0x3e66ed53, 0x3f800000 };
-        std::vector<float> buffer(tosaElementsP0);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP0.c_str(), (void*)buffer.data(), tosaElementsP0 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "2", 0, expected);
     }
     SUBCASE("matmul, set 2, param 1")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "2");
-
         std::vector<uint32_t> expected = { 0x3f800000, 0x3f800000, 0x3f800000 };
-        std::vector<float> buffer(tosaElementsP1);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP1.c_str(), (void*)buffer.data(), tosaElementsP1 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "2", 1, expected);
     }
     SUBCASE("matmul, set 3, param 0")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "3");
-
-        // NOTE: Python test script produced  0xbf256686 - so off by 1
+        // NOTE: Python test script produced 0xbf256686 - so off by 1
         std::vector<uint32_t> expected = { 0x41800000, 0xbf256685, 0x41800000 };
-        std::vector<float> buffer(tosaElementsP0);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP0.c_str(), (void*)buffer.data(), tosaElementsP0 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "3", 0, expected);
     }
     SUBCASE("matmul, set 3, param 1")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "3");
-
         std::vector<uint32_t> expected = { 0x41800000, 0x41800000, 0x41800000 };
-        std::vector<float> buffer(tosaElementsP1);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP1.c_str(), (void*)buffer.data(), tosaElementsP1 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "3", 1, expected);
     }
     SUBCASE("matmul, set 4, param 0")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "4");
-
         std::vector<uint32_t> expected = { 0x0, 0x3f000000, 0x5f14e80c };
-        std::vector<float> buffer(tosaElementsP0);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP0.c_str(), (void*)buffer.data(), tosaElementsP0 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "4", 0, expected);
     }
     SUBCASE("matmul, set 4, param 1")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "4");
-
         std::vector<uint32_t> expected = { 0x5d5d0db2, 0xdf2c82a8, 0x0 };
-        std::vector<float> buffer(tosaElementsP1);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP1.c_str(), (void*)buffer.data(), tosaElementsP1 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "4", 1, expected);
     }
     SUBCASE("matmul, set 5, param 0")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "5");
-
         std::vector<uint32_t> expected = { 0x5df6c4b3, 0x5e6b4088, 0x5ed0fe71 };
-        std::vector<float> buffer(tosaElementsP0);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP0.c_str(), (void*)buffer.data(), tosaElementsP0 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "5", 0, expected);
     }
     SUBCASE("matmul, set 5, param 1")
     {
-        std::string json_cfg = template_json_cfg;
-        update_json_template(json_cfg, "5");
-
         std::vector<uint32_t> expected = { 0xde086d85, 0x5e630878, 0x5eba5c7b };
-        std::vector<float> buffer(tosaElementsP1);
-        REQUIRE(tgd_generate_data(json_cfg.c_str(), tosaNameP1.c_str(), (void*)buffer.data(), tosaElementsP1 * 4));
-        check_output<float>(buffer, expected);
+        matmul_test_FP32(tosaName, tosaElements, templateJsonCfg, "5", 1, expected);
     }
 }
 TEST_SUITE_END();    // generate
diff --git a/reference_model/test/verify_tests.cpp b/reference_model/test/verify_tests.cpp
index f36efbf..3aa477f 100644
--- a/reference_model/test/verify_tests.cpp
+++ b/reference_model/test/verify_tests.cpp
@@ -151,7 +151,7 @@ TEST_SUITE_BEGIN("verify");
 
 TEST_CASE("negative - api")
 {
-    std::string json_cfg = R"({
+    std::string jsonCfg = R"({
         "tensors" : {
             "out1" : {
                 "mode": "DOT_PRODUCT",
@@ -166,7 +166,7 @@ TEST_CASE("negative - api")
 
     SUBCASE("invalid json")
     {
-        std::string invalid_json_cfg = R"({
+        std::string invalidJsonCfg = R"({
             "tensors" : {
                 "out1" : {
                     "mode": DOT_PRODUCT,
@@ -178,7 +178,39 @@ TEST_CASE("negative - api")
         const TosaTensor refAbs("out1", tosa_datatype_fp64_t, { 8, 8, 8 });
         const TosaTensor imp("out1", tosa_datatype_fp32_t, { 8, 8, 8 });
 
-        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), refAbs.cTensor(), imp.cTensor(), invalid_json_cfg.c_str()));
+        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), refAbs.cTensor(), imp.cTensor(), invalidJsonCfg.c_str()));
+    }
+    SUBCASE("unknown mode")
+    {
+        std::string unknownJsonCfg = R"({
+            "tensors" : {
+                "out1" : {
+                    "mode": "WIND",
+                    "data_type": "FP32"
+                }
+            }
+        })";
+
+        const TosaTensor ref("out1", tosa_datatype_fp64_t, { 8 });
+        const TosaTensor imp("out1", tosa_datatype_fp32_t, { 8 });
+
+        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), nullptr, imp.cTensor(), unknownJsonCfg.c_str()));
+    }
+    SUBCASE("unknown type")
+    {
+        std::string unknownJsonCfg = R"({
+            "tensors" : {
+                "out1" : {
+                    "mode": "DOT_PRODUCT",
+                    "data_type": "JOULES"
+                }
+            }
+        })";
+
+        const TosaTensor ref("out1", tosa_datatype_fp64_t, { 8 });
+        const TosaTensor imp("out1", tosa_datatype_fp32_t, { 8 });
+
+        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), nullptr, imp.cTensor(), unknownJsonCfg.c_str()));
     }
     SUBCASE("mismatching dimensions")
     {
@@ -186,7 +218,7 @@ TEST_CASE("negative - api")
         const TosaTensor refAbs("out1", tosa_datatype_fp64_t, { 4, 4 });
         const TosaTensor imp("out1", tosa_datatype_fp32_t, { 8, 8, 8 });
 
-        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), refAbs.cTensor(), imp.cTensor(), json_cfg.c_str()));
+        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), refAbs.cTensor(), imp.cTensor(), jsonCfg.c_str()));
     }
     SUBCASE("mismatching shapes")
     {
@@ -194,7 +226,7 @@ TEST_CASE("negative - api")
         const TosaTensor refAbs("out1", tosa_datatype_fp64_t, { 8, 8, 8 });
         const TosaTensor imp("out1", tosa_datatype_fp32_t, { 4, 4, 4 });
 
-        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), refAbs.cTensor(), imp.cTensor(), json_cfg.c_str()));
+        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), refAbs.cTensor(), imp.cTensor(), jsonCfg.c_str()));
     }
     SUBCASE("mismatching data types")
     {
@@ -202,7 +234,7 @@ TEST_CASE("negative - api")
         const TosaTensor refAbs("out1", tosa_datatype_fp64_t, { 8, 8, 8 });
         const TosaTensor imp("out1", tosa_datatype_fp16_t, { 8, 8, 8 });
 
-        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), refAbs.cTensor(), imp.cTensor(), json_cfg.c_str()));
+        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), refAbs.cTensor(), imp.cTensor(), jsonCfg.c_str()));
     }
     SUBCASE("missing tensor data")
     {
@@ -210,13 +242,13 @@ TEST_CASE("negative - api")
         const TosaTensor refAbs("out1", tosa_datatype_fp64_t, { 8, 8, 8 });
         const TosaTensor imp("out1", tosa_datatype_fp32_t, { 8, 8, 8 });
 
-        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), refAbs.cTensor(), imp.cTensor(), json_cfg.c_str()));
+        REQUIRE_FALSE(tvf_verify_data(ref.cTensor(), refAbs.cTensor(), imp.cTensor(), jsonCfg.c_str()));
     }
 }
 
 TEST_CASE("positive - exact")
 {
-    std::string json_cfg = R"({
+    std::string jsonCfg = R"({
         "tensors" : {
             "out1" : {
                 "mode": "EXACT",
@@ -236,7 +268,7 @@ TEST_CASE("positive - exact")
             TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data.data()));
         const auto implementationTensor =
             TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(data.data()));
-        REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str()));
+        REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
     }
 
     SUBCASE("different")
@@ -254,13 +286,13 @@ TEST_CASE("positive - exact")
         const auto implementationTensor =
             TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData.data()));
         REQUIRE_FALSE(
-            tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str()));
+            tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
     }
 }
 
 TEST_CASE("positive - reduce product")
 {
-    std::string json_cfg = R"({
+    std::string jsonCfg = R"({
         "tensors" : {
             "out1" : {
                 "mode": "REDUCE_PRODUCT",
@@ -311,7 +343,7 @@ TEST_CASE("positive - reduce product")
             TosaTensor("out1", tosa_datatype_fp64_t, outputShape, reinterpret_cast<uint8_t*>(data.data()));
         const auto implementationTensor =
             TosaTensor("out1", tosa_datatype_fp32_t, outputShape, reinterpret_cast<uint8_t*>(otherData.data()));
-        REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str()));
+        REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
     }
 
     SUBCASE("different")
@@ -337,13 +369,13 @@ TEST_CASE("positive - reduce product")
         const auto implementationTensor =
             TosaTensor("out1", tosa_datatype_fp32_t, outputShape, reinterpret_cast<uint8_t*>(otherData.data()));
         REQUIRE_FALSE(
-            tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str()));
+            tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
     }
 }
 
 TEST_CASE("positive - ulp")
 {
-    std::string json_cfg = R"({
+    std::string jsonCfg = R"({
         "tensors" : {
             "out1" : {
                 "mode": "ULP",
@@ -369,7 +401,7 @@ TEST_CASE("positive - ulp")
             TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data.data()));
         const auto implementationTensor =
             TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData.data()));
-        REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str()));
+        REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
     }
 
     SUBCASE("different")
@@ -383,7 +415,7 @@ TEST_CASE("positive - ulp")
         const auto implementationTensor =
             TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData.data()));
         REQUIRE_FALSE(
-            tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str()));
+            tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str()));
     }
 }
 
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index 3b5d458..475f062 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -665,13 +665,13 @@ class TosaTensorValuesGen:
             tens_meta["data_type"] = gtu.DTYPE_ATTRIBUTES[dtypeList[idx]]["json"]
             tens_meta["shape"] = [int(i) for i in shape]
             tens_meta["input_pos"] = idx
-            tens_meta["op"] = opName
+            tens_meta["op"] = opName.upper()
 
             if idx < pCount:
-                tens_meta["input_type"] = "variable"
+                tens_meta["input_type"] = "VARIABLE"
                 tens = testGen.ser.addPlaceholder(shape, dtypeList[idx], None)
             else:
-                tens_meta["input_type"] = "constant"
+                tens_meta["input_type"] = "CONSTANT"
                 tens = testGen.ser.addConst(shape, dtypeList[idx], None)
             tens_ser_list.append(tens)