From e2b5e87804e158cb3e5d06a131c317b3890b87b3 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Thu, 14 Sep 2023 17:02:09 +0100 Subject: Support for compliance checking testing Updated to conformance generator to not generate tests with results for compliance tests. Updated test runner to run compliance mode version (precise & abs mode) of reference model to create test results to use against SUT results. Updated reference model to enable abs_mode on correct desc.json flags. Updated test checker to support compliance checking using verifier lib. Seperated color printing from test checker. Change-Id: I7e2fbfc6883916caa5d94d4ece122c48bf45f530 Signed-off-by: Jeremy Johnson --- reference_model/src/main.cpp | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'reference_model/src/main.cpp') diff --git a/reference_model/src/main.cpp b/reference_model/src/main.cpp index 62b8f6f..6d50f9e 100644 --- a/reference_model/src/main.cpp +++ b/reference_model/src/main.cpp @@ -90,6 +90,12 @@ int main(int argc, char** argv) GraphStatus status = GraphStatus::TOSA_VALID; + if (isComplianceModeDotProduct(test_desc) && !g_func_config.precise_mode) + { + // Warn about precise mode for dot product compliance + DEBUG_INFO(CONFIG, "DOT_PRODUCT compliance: NOTE - enable precise mode for compliance results") + } + // max of 2 runs, second run only happens when precise_mode is set, to do an abs_mode run for (int run = 0; run < 2; run++) { @@ -213,10 +219,11 @@ int main(int argc, char** argv) fprintf(stderr, "Unknown graph status code=%d.\n", (int)main_gt.getGraphStatus()); } - if (status == GraphStatus::TOSA_VALID && g_func_config.eval && g_func_config.precise_mode && + if (run == 0 && status == GraphStatus::TOSA_VALID && g_func_config.precise_mode && g_func_config.eval && isComplianceModeDotProduct(test_desc)) { - // first run result is valid, in precise mode and eval is true: turn on abs_mode for second run + // first run result is valid and precise mode and eval is true: turn on abs_mode for second run + DEBUG_INFO(CONFIG, "DOT_PRODUCT compliance: Evaluating the graph again to produce bounds results") g_func_config.abs_mode = true; continue; } @@ -354,14 +361,21 @@ const std::string getResultFilenamePrefix() return g_func_config.abs_mode ? "bounds_" : ""; } -// returns true iff test_desc contains a dictionay, "compliance", -// which contains entry "mode" whose value is "dot product" +// returns true iff test_desc contains a "meta" object containing a "compliance" +// object which contains "tensors" and one of those has a "mode" whose value is +// "DOT_PRODUCT" bool isComplianceModeDotProduct(json& test_desc) { - if (test_desc.contains("compliance") && test_desc["compliance"].contains("mode") && - test_desc["compliance"]["mode"] == "dot product") + if (test_desc.contains("meta") && test_desc["meta"].contains("compliance") && + test_desc["meta"]["compliance"].contains("tensors")) { - return true; + for (auto t : test_desc["meta"]["compliance"]["tensors"]) + { + if (t.contains("mode") && t["mode"] == "DOT_PRODUCT") + { + return true; + } + } } return false; } -- cgit v1.2.1