Fix conformance regressions due to type and region changes.

Compress flatbuffer JSON files for conformance to allow while_loop
tests to fit in the 30MB file size limit.
Do not include new SHAPE type in usable DTypes so that selection of
ERROR_IF tests are not changed.

Also enable strict json mode for flatbuffers.

Change-Id: Id89a9963244c34769a4662f9951adde0aa3334d9
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>

2 files changed, 67 insertions, 5 deletions

diff --git a/scripts/json2fbbin/json2fbbin.py b/scripts/json2fbbin/json2fbbin.py
index 8f9f274..08b13be 100644
--- a/scripts/json2fbbin/json2fbbin.py
+++ b/scripts/json2fbbin/json2fbbin.py
@@ -1,14 +1,69 @@
 """Conversion utility from flatbuffer JSON files to binary and the reverse."""
-# Copyright (c) 2021-2022, ARM Limited.
+# Copyright (c) 2021-2023, ARM Limited.
 # SPDX-License-Identifier: Apache-2.0
+import re
 from pathlib import Path
 from typing import Optional
 
 from runner.run_command import run_sh_command
 from runner.run_command import RunShCommandError
 
-
-def fbbin_to_json(flatc: Path, fbs: Path, t_path: Path, o_path: Optional[Path] = None):
+MAX_LINE_LEN = 120
+MAX_INDENT_LEN = 20
+
+
+def json_squeeze(json_path: Path):
+    """File compression for JSONs, reducing spaces used for number lists."""
+    # Move existing file to a new name
+    temp_path = json_path.with_suffix(".json_unsqueezed")
+    json_path.rename(temp_path)
+    # Now read the original file and write a smaller output with less new lines/spaces
+    with temp_path.open("r") as tfd:
+        with json_path.open("w") as jfd:
+            found = False
+
+            for line in tfd:
+                # Find lines that are part of number lists
+                match = re.match(r"(\s+)(-?[0-9]+),?", line)
+                if match:
+                    # Found a line with just a number on it (and optional comma)
+                    if not found:
+                        # New list of numbers
+                        numbers = []
+                        # Save indent (upto maximum)
+                        indent = match.group(1)[0:MAX_INDENT_LEN]
+                        found = True
+                    numbers.append(match.group(2))
+                else:
+                    # Found a line without just a number
+                    if found:
+                        # Format the list of numbers recorded into a concise output
+                        # with multiple numbers on a single line, rather than one per line
+                        numbers_str = indent
+                        for num in numbers:
+                            nums = f"{num},"
+                            if len(numbers_str) + len(nums) > MAX_LINE_LEN:
+                                print(numbers_str, file=jfd)
+                                numbers_str = indent
+                            numbers_str += nums
+                        # print all but the last comma
+                        print(numbers_str[:-1], file=jfd)
+
+                        found = False
+                    # print the line we just read (that wasn't just a number)
+                    print(line, file=jfd, end="")
+
+    # Remove the uncompressed version
+    temp_path.unlink()
+
+
+def fbbin_to_json(
+    flatc: Path,
+    fbs: Path,
+    t_path: Path,
+    o_path: Optional[Path] = None,
+    squeeze: Optional[bool] = True,
+):
     """Convert the binary flatbuffer to JSON.
 
     flatc: the Path to the flatc compiler program
@@ -23,6 +78,7 @@ def fbbin_to_json(flatc: Path, fbs: Path, t_path: Path, o_path: Optional[Path] =
         "-o",
         str(o_path.absolute()),
         "--json",
+        "--strict-json",
         "--defaults-json",
         "--raw-binary",
         str(fbs.absolute()),
@@ -30,6 +86,9 @@ def fbbin_to_json(flatc: Path, fbs: Path, t_path: Path, o_path: Optional[Path] =
         str(t_path.absolute()),
     ]
     run_sh_command(verbose=False, full_cmd=cmd)
+    if squeeze:
+        json_path = (o_path / t_path.name).with_suffix(".json")
+        json_squeeze(json_path)
 
 
 def json_to_fbbin(flatc: Path, fbs: Path, j_path: Path, o_path: Optional[Path] = None):
@@ -62,6 +121,9 @@ def main(argv=None):
 
     parser = argparse.ArgumentParser()
     parser.add_argument(
+        "--no-squeeze", action="store_true", help="no compression of json output"
+    )
+    parser.add_argument(
         "--flatc",
         type=Path,
         default=(
@@ -97,7 +159,7 @@ def main(argv=None):
             json_to_fbbin(args.flatc, args.fbs, path)
         else:
             # Have to assume this is a binary flatbuffer file as could have any suffix
-            fbbin_to_json(args.flatc, args.fbs, path)
+            fbbin_to_json(args.flatc, args.fbs, path, squeeze=(not args.no_squeeze))
     except RunShCommandError as e:
         print(e)
         return 1
diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py
index f9df8d5..3cd0370 100644
--- a/verif/generator/tosa_utils.py
+++ b/verif/generator/tosa_utils.py
@@ -93,7 +93,7 @@ def usableDTypes(*, excludes=None):
     Returns:
         A set of DType values
     """
-    omit = {DType.UNKNOWN, DType.UINT8, DType.UINT16}
+    omit = {DType.UNKNOWN, DType.UINT8, DType.UINT16, DType.SHAPE}
     omit.update(excludes if excludes else ())
     return allDTypes(excludes=omit)