Support for compliance checking testing

Updated to conformance generator to not generate tests with results for
compliance tests.
Updated test runner to run compliance mode version (precise & abs mode)
of reference model to create test results to use against SUT results.
Updated reference model to enable abs_mode on correct desc.json flags.
Updated test checker to support compliance checking using verifier lib.
Seperated color printing from test checker.

Change-Id: I7e2fbfc6883916caa5d94d4ece122c48bf45f530
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>

1 files changed, 10 insertions, 4 deletions

diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py
index 3654f9a..171ec3e 100755
--- a/scripts/convert2conformance/convert2conformance.py
+++ b/scripts/convert2conformance/convert2conformance.py
@@ -162,11 +162,11 @@ def update_desc_json(
     test_dir: Path,
     test_desc,
     output_dir: Optional[Path] = None,
-    create_result=True,
+    record_result=True,
     profiles=None,
     tags=None,
 ):
-    """Update the desc.json format for conformance and optionally create result."""
+    """Update the desc.json format for conformance and optionally record result."""
     ofm_files = []
     cfm_files = []
     if not output_dir:
@@ -175,7 +175,7 @@ def update_desc_json(
         ofm_path = test_dir / ofm
         if not test_desc["expected_failure"]:
             cfm = NAME_CONFORMANCE_RESULT_PREFIX + test_desc["ofm_name"][index]
-            if create_result:
+            if record_result:
                 if ofm_path.is_file():
                     # Use the desc.json name
                     ofm_refmodel = ofm_path
@@ -338,12 +338,18 @@ def main(argv=None):
     for cpp in cpp_files:
         shutil.copy(str(cpp), str(args.output_dir))
 
+    # Work out if we have a result to record
+    record_result = not args.lazy_data_generation
+    if "meta" in test_desc and "compliance" in test_desc["meta"]:
+        # We don't have pre-generated results for compliance tests
+        record_result = False
+
     # Update desc.json and convert result files to JSON
     test_desc = update_desc_json(
         desc_filename.parent,
         test_desc,
         output_dir=args.output_dir,
-        create_result=(not args.lazy_data_generation),
+        record_result=record_result,
         profiles=args.profile,
         tags=args.tags,
     )